1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve -tail-predication=enabled --verify-machineinstrs %s -o - | FileCheck %s
3
4define dso_local i32 @vpsel_mul_reduce_add(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture readonly %c, i32 %N) {
5; CHECK-LABEL: vpsel_mul_reduce_add:
6; CHECK:       @ %bb.0: @ %entry
7; CHECK-NEXT:    cmp r3, #0
8; CHECK-NEXT:    itt eq
9; CHECK-NEXT:    moveq r0, #0
10; CHECK-NEXT:    bxeq lr
11; CHECK-NEXT:    push {r4, lr}
12; CHECK-NEXT:    sub sp, #4
13; CHECK-NEXT:    add.w r12, r3, #3
14; CHECK-NEXT:    mov.w lr, #1
15; CHECK-NEXT:    bic r12, r12, #3
16; CHECK-NEXT:    vmov.i32 q1, #0x0
17; CHECK-NEXT:    sub.w r12, r12, #4
18; CHECK-NEXT:    add.w lr, lr, r12, lsr #2
19; CHECK-NEXT:    mov.w r12, #0
20; CHECK-NEXT:    dls lr, lr
21; CHECK-NEXT:  .LBB0_1: @ %vector.body
22; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
23; CHECK-NEXT:    vctp.32 r3
24; CHECK-NEXT:    and r4, r12, #15
25; CHECK-NEXT:    vstr p0, [sp] @ 4-byte Spill
26; CHECK-NEXT:    vdup.32 q3, r4
27; CHECK-NEXT:    vmov q0, q1
28; CHECK-NEXT:    vpstt
29; CHECK-NEXT:    vldrwt.u32 q1, [r2], #16
30; CHECK-NEXT:    vldrwt.u32 q2, [r1], #16
31; CHECK-NEXT:    vcmp.i32 eq, q3, zr
32; CHECK-NEXT:    add.w r12, r12, #4
33; CHECK-NEXT:    vpsel q1, q2, q1
34; CHECK-NEXT:    vldr p0, [sp] @ 4-byte Reload
35; CHECK-NEXT:    vpst
36; CHECK-NEXT:    vldrwt.u32 q2, [r0], #16
37; CHECK-NEXT:    vmul.i32 q1, q1, q2
38; CHECK-NEXT:    subs r3, #4
39; CHECK-NEXT:    vadd.i32 q1, q1, q0
40; CHECK-NEXT:    le lr, .LBB0_1
41; CHECK-NEXT:  @ %bb.2: @ %middle.block
42; CHECK-NEXT:    vpsel q0, q1, q0
43; CHECK-NEXT:    vaddv.u32 r0, q0
44; CHECK-NEXT:    add sp, #4
45; CHECK-NEXT:    pop {r4, pc}
46entry:
47  %cmp8 = icmp eq i32 %N, 0
48  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
49
50vector.ph:                                        ; preds = %entry
51  %n.rnd.up = add i32 %N, 3
52  %n.vec = and i32 %n.rnd.up, -4
53  %trip.count.minus.1 = add i32 %N, -1
54  %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
55  %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
56  br label %vector.body
57
58vector.body:                                      ; preds = %vector.body, %vector.ph
59  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
60  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %add, %vector.body ]
61  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
62  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
63  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
64  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
65
66;  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat12
67  %tmp1 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
68
69  %tmp2 = bitcast i32* %tmp to <4 x i32>*
70  %wide.masked.load.a = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
71  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
72  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
73  %wide.masked.load.b = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
74  %tmp5 = getelementptr inbounds i32, i32* %c, i32 %index
75  %tmp6 = bitcast i32* %tmp5 to <4 x i32>*
76  %wide.masked.load.c = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp6, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
77  %rem = urem i32 %index, 16
78  %rem.broadcast.splatinsert = insertelement <4 x i32> undef, i32 %rem, i32 0
79  %rem.broadcast.splat = shufflevector <4 x i32> %rem.broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
80  %cmp = icmp eq <4 x i32> %rem.broadcast.splat, <i32 0, i32 0, i32 0, i32 0>
81  %wide.masked.load = select <4 x i1> %cmp, <4 x i32> %wide.masked.load.b, <4 x i32> %wide.masked.load.c
82  %mul = mul nsw <4 x i32> %wide.masked.load, %wide.masked.load.a
83  %add = add nsw <4 x i32> %mul, %vec.phi
84  %index.next = add i32 %index, 4
85  %tmp7 = icmp eq i32 %index.next, %n.vec
86  br i1 %tmp7, label %middle.block, label %vector.body
87
88middle.block:                                     ; preds = %vector.body
89  %tmp8 = select <4 x i1> %tmp1, <4 x i32> %add, <4 x i32> %vec.phi
90  %tmp9 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %tmp8)
91  br label %for.cond.cleanup
92
93for.cond.cleanup:                                 ; preds = %middle.block, %entry
94  %res.0.lcssa = phi i32 [ 0, %entry ], [ %tmp9, %middle.block ]
95  ret i32 %res.0.lcssa
96}
97
98define dso_local i32 @vpsel_mul_reduce_add_2(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b,
99; CHECK-LABEL: vpsel_mul_reduce_add_2:
100; CHECK:       @ %bb.0: @ %entry
101; CHECK-NEXT:    push {r4, r5, r7, lr}
102; CHECK-NEXT:    sub sp, #4
103; CHECK-NEXT:    ldr.w r12, [sp, #20]
104; CHECK-NEXT:    cmp.w r12, #0
105; CHECK-NEXT:    beq .LBB1_4
106; CHECK-NEXT:  @ %bb.1: @ %vector.ph
107; CHECK-NEXT:    add.w r4, r12, #3
108; CHECK-NEXT:    vmov.i32 q1, #0x0
109; CHECK-NEXT:    bic r4, r4, #3
110; CHECK-NEXT:    sub.w lr, r4, #4
111; CHECK-NEXT:    movs r4, #1
112; CHECK-NEXT:    add.w lr, r4, lr, lsr #2
113; CHECK-NEXT:    movs r4, #0
114; CHECK-NEXT:    dls lr, lr
115; CHECK-NEXT:  .LBB1_2: @ %vector.body
116; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
117; CHECK-NEXT:    vctp.32 r12
118; CHECK-NEXT:    and r5, r4, #15
119; CHECK-NEXT:    vstr p0, [sp] @ 4-byte Spill
120; CHECK-NEXT:    vmov q0, q1
121; CHECK-NEXT:    vpstt
122; CHECK-NEXT:    vldrwt.u32 q1, [r3], #16
123; CHECK-NEXT:    vldrwt.u32 q2, [r2], #16
124; CHECK-NEXT:    vdup.32 q3, r5
125; CHECK-NEXT:    vsub.i32 q1, q2, q1
126; CHECK-NEXT:    vpst
127; CHECK-NEXT:    vldrwt.u32 q2, [r1], #16
128; CHECK-NEXT:    vcmp.i32 eq, q3, zr
129; CHECK-NEXT:    adds r4, #4
130; CHECK-NEXT:    vpsel q1, q1, q2
131; CHECK-NEXT:    vldr p0, [sp] @ 4-byte Reload
132; CHECK-NEXT:    vpst
133; CHECK-NEXT:    vldrwt.u32 q2, [r0], #16
134; CHECK-NEXT:    vmul.i32 q1, q1, q2
135; CHECK-NEXT:    sub.w r12, r12, #4
136; CHECK-NEXT:    vadd.i32 q1, q1, q0
137; CHECK-NEXT:    le lr, .LBB1_2
138; CHECK-NEXT:  @ %bb.3: @ %middle.block
139; CHECK-NEXT:    vpsel q0, q1, q0
140; CHECK-NEXT:    vaddv.u32 r0, q0
141; CHECK-NEXT:    add sp, #4
142; CHECK-NEXT:    pop {r4, r5, r7, pc}
143; CHECK-NEXT:  .LBB1_4:
144; CHECK-NEXT:    movs r0, #0
145; CHECK-NEXT:    add sp, #4
146; CHECK-NEXT:    pop {r4, r5, r7, pc}
147                                         i32* noalias nocapture readonly %c, i32* noalias nocapture readonly %d, i32 %N) {
148entry:
149  %cmp8 = icmp eq i32 %N, 0
150  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
151
152vector.ph:                                        ; preds = %entry
153  %n.rnd.up = add i32 %N, 3
154  %n.vec = and i32 %n.rnd.up, -4
155  %trip.count.minus.1 = add i32 %N, -1
156  %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
157  %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
158  br label %vector.body
159
160vector.body:                                      ; preds = %vector.body, %vector.ph
161  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
162  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %add, %vector.body ]
163  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
164  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
165  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
166  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
167
168;  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat12
169  %tmp1 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
170
171  %tmp2 = bitcast i32* %tmp to <4 x i32>*
172  %wide.masked.load.a = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
173  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
174  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
175  %wide.masked.load.b = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
176  %tmp5 = getelementptr inbounds i32, i32* %c, i32 %index
177  %tmp6 = bitcast i32* %tmp5 to <4 x i32>*
178  %wide.masked.load.c = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp6, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
179  %tmp7 = getelementptr inbounds i32, i32* %d, i32 %index
180  %tmp8 = bitcast i32* %tmp7 to <4 x i32>*
181  %wide.masked.load.d = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp8, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
182  %sub = sub <4 x i32> %wide.masked.load.c, %wide.masked.load.d
183  %rem = urem i32 %index, 16
184  %rem.broadcast.splatinsert = insertelement <4 x i32> undef, i32 %rem, i32 0
185  %rem.broadcast.splat = shufflevector <4 x i32> %rem.broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
186  %cmp = icmp eq <4 x i32> %rem.broadcast.splat, <i32 0, i32 0, i32 0, i32 0>
187  %sel = select <4 x i1> %cmp, <4 x i32> %sub, <4 x i32> %wide.masked.load.b
188  %mul = mul  <4 x i32> %sel, %wide.masked.load.a
189  %add = add  <4 x i32> %mul, %vec.phi
190  %index.next = add i32 %index, 4
191  %cmp.exit = icmp eq i32 %index.next, %n.vec
192  br i1 %cmp.exit, label %middle.block, label %vector.body
193
194middle.block:                                     ; preds = %vector.body
195  %acc = select <4 x i1> %tmp1, <4 x i32> %add, <4 x i32> %vec.phi
196  %reduce = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %acc)
197  br label %for.cond.cleanup
198
199for.cond.cleanup:                                 ; preds = %middle.block, %entry
200  %res.0.lcssa = phi i32 [ 0, %entry ], [ %reduce, %middle.block ]
201  ret i32 %res.0.lcssa
202}
203
204define dso_local i32 @and_mul_reduce_add(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b,
205; CHECK-LABEL: and_mul_reduce_add:
206; CHECK:       @ %bb.0: @ %entry
207; CHECK-NEXT:    push {r4, lr}
208; CHECK-NEXT:    sub sp, #4
209; CHECK-NEXT:    ldr.w r12, [sp, #12]
210; CHECK-NEXT:    cmp.w r12, #0
211; CHECK-NEXT:    beq .LBB2_4
212; CHECK-NEXT:  @ %bb.1: @ %vector.ph
213; CHECK-NEXT:    add.w r4, r12, #3
214; CHECK-NEXT:    vmov.i32 q1, #0x0
215; CHECK-NEXT:    bic r4, r4, #3
216; CHECK-NEXT:    sub.w lr, r4, #4
217; CHECK-NEXT:    movs r4, #1
218; CHECK-NEXT:    add.w lr, r4, lr, lsr #2
219; CHECK-NEXT:    movs r4, #0
220; CHECK-NEXT:    dls lr, lr
221; CHECK-NEXT:  .LBB2_2: @ %vector.body
222; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
223; CHECK-NEXT:    vctp.32 r12
224; CHECK-NEXT:    vmov q0, q1
225; CHECK-NEXT:    vpstt
226; CHECK-NEXT:    vldrwt.u32 q1, [r1], #16
227; CHECK-NEXT:    vldrwt.u32 q2, [r0], #16
228; CHECK-NEXT:    vstr p0, [sp] @ 4-byte Spill
229; CHECK-NEXT:    vsub.i32 q1, q2, q1
230; CHECK-NEXT:    adds r4, #4
231; CHECK-NEXT:    vpsttt
232; CHECK-NEXT:    vcmpt.i32 eq, q1, zr
233; CHECK-NEXT:    vldrwt.u32 q1, [r3], #16
234; CHECK-NEXT:    vldrwt.u32 q2, [r2], #16
235; CHECK-NEXT:    sub.w r12, r12, #4
236; CHECK-NEXT:    vmul.i32 q1, q2, q1
237; CHECK-NEXT:    vadd.i32 q1, q1, q0
238; CHECK-NEXT:    le lr, .LBB2_2
239; CHECK-NEXT:  @ %bb.3: @ %middle.block
240; CHECK-NEXT:    vldr p0, [sp] @ 4-byte Reload
241; CHECK-NEXT:    vpsel q0, q1, q0
242; CHECK-NEXT:    vaddv.u32 r0, q0
243; CHECK-NEXT:    add sp, #4
244; CHECK-NEXT:    pop {r4, pc}
245; CHECK-NEXT:  .LBB2_4:
246; CHECK-NEXT:    movs r0, #0
247; CHECK-NEXT:    add sp, #4
248; CHECK-NEXT:    pop {r4, pc}
249                                         i32* noalias nocapture readonly %c, i32* noalias nocapture readonly %d, i32 %N) {
250entry:
251  %cmp8 = icmp eq i32 %N, 0
252  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
253
254vector.ph:                                        ; preds = %entry
255  %n.rnd.up = add i32 %N, 3
256  %n.vec = and i32 %n.rnd.up, -4
257  %trip.count.minus.1 = add i32 %N, -1
258  %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
259  %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
260  br label %vector.body
261
262vector.body:                                      ; preds = %vector.body, %vector.ph
263  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
264  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %add, %vector.body ]
265  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
266  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
267  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
268  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
269
270;  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat12
271  %tmp1 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
272
273  %tmp2 = bitcast i32* %tmp to <4 x i32>*
274  %wide.masked.load.a = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
275  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
276  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
277  %wide.masked.load.b = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
278  %sub = sub <4 x i32> %wide.masked.load.a, %wide.masked.load.b
279  %cmp = icmp eq <4 x i32> %sub, <i32 0, i32 0, i32 0, i32 0>
280  %mask = and <4 x i1> %cmp, %tmp1
281  %tmp5 = getelementptr inbounds i32, i32* %c, i32 %index
282  %tmp6 = bitcast i32* %tmp5 to <4 x i32>*
283  %wide.masked.load.c = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp6, i32 4, <4 x i1> %mask, <4 x i32> undef)
284  %tmp7 = getelementptr inbounds i32, i32* %d, i32 %index
285  %tmp8 = bitcast i32* %tmp7 to <4 x i32>*
286  %wide.masked.load.d = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp8, i32 4, <4 x i1> %mask, <4 x i32> undef)
287  %mul = mul  <4 x i32> %wide.masked.load.c, %wide.masked.load.d
288  %add = add  <4 x i32> %mul, %vec.phi
289  %index.next = add i32 %index, 4
290  %cmp.exit = icmp eq i32 %index.next, %n.vec
291  br i1 %cmp.exit, label %middle.block, label %vector.body
292
293middle.block:                                     ; preds = %vector.body
294  %acc = select <4 x i1> %tmp1, <4 x i32> %add, <4 x i32> %vec.phi
295  %reduce = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %acc)
296  br label %for.cond.cleanup
297
298for.cond.cleanup:                                 ; preds = %middle.block, %entry
299  %res.0.lcssa = phi i32 [ 0, %entry ], [ %reduce, %middle.block ]
300  ret i32 %res.0.lcssa
301}
302
303define dso_local i32 @or_mul_reduce_add(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture readonly %c, i32* noalias nocapture readonly %d, i32 %N) {
304; CHECK-LABEL: or_mul_reduce_add:
305; CHECK:       @ %bb.0: @ %entry
306; CHECK-NEXT:    push {r4, lr}
307; CHECK-NEXT:    sub sp, #4
308; CHECK-NEXT:    ldr.w r12, [sp, #12]
309; CHECK-NEXT:    cmp.w r12, #0
310; CHECK-NEXT:    beq .LBB3_4
311; CHECK-NEXT:  @ %bb.1: @ %vector.ph
312; CHECK-NEXT:    add.w r4, r12, #3
313; CHECK-NEXT:    vmov.i32 q1, #0x0
314; CHECK-NEXT:    bic r4, r4, #3
315; CHECK-NEXT:    sub.w lr, r4, #4
316; CHECK-NEXT:    movs r4, #1
317; CHECK-NEXT:    add.w lr, r4, lr, lsr #2
318; CHECK-NEXT:    movs r4, #0
319; CHECK-NEXT:    dls lr, lr
320; CHECK-NEXT:  .LBB3_2: @ %vector.body
321; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
322; CHECK-NEXT:    vctp.32 r12
323; CHECK-NEXT:    vmov q0, q1
324; CHECK-NEXT:    vpstt
325; CHECK-NEXT:    vldrwt.u32 q1, [r1], #16
326; CHECK-NEXT:    vldrwt.u32 q2, [r0], #16
327; CHECK-NEXT:    vstr p0, [sp] @ 4-byte Spill
328; CHECK-NEXT:    vsub.i32 q1, q2, q1
329; CHECK-NEXT:    vpnot
330; CHECK-NEXT:    vpstee
331; CHECK-NEXT:    vcmpt.i32 ne, q1, zr
332; CHECK-NEXT:    vldrwe.u32 q1, [r3], #16
333; CHECK-NEXT:    vldrwe.u32 q2, [r2], #16
334; CHECK-NEXT:    adds r4, #4
335; CHECK-NEXT:    vmul.i32 q1, q2, q1
336; CHECK-NEXT:    sub.w r12, r12, #4
337; CHECK-NEXT:    vadd.i32 q1, q1, q0
338; CHECK-NEXT:    le lr, .LBB3_2
339; CHECK-NEXT:  @ %bb.3: @ %middle.block
340; CHECK-NEXT:    vldr p0, [sp] @ 4-byte Reload
341; CHECK-NEXT:    vpsel q0, q1, q0
342; CHECK-NEXT:    vaddv.u32 r0, q0
343; CHECK-NEXT:    add sp, #4
344; CHECK-NEXT:    pop {r4, pc}
345; CHECK-NEXT:  .LBB3_4:
346; CHECK-NEXT:    movs r0, #0
347; CHECK-NEXT:    add sp, #4
348; CHECK-NEXT:    pop {r4, pc}
349entry:
350  %cmp8 = icmp eq i32 %N, 0
351  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
352
353vector.ph:                                        ; preds = %entry
354  %n.rnd.up = add i32 %N, 3
355  %n.vec = and i32 %n.rnd.up, -4
356  %trip.count.minus.1 = add i32 %N, -1
357  %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
358  %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
359  br label %vector.body
360
361vector.body:                                      ; preds = %vector.body, %vector.ph
362  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
363  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %add, %vector.body ]
364  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
365  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
366  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
367  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
368
369;  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat12
370  %tmp1 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
371
372  %tmp2 = bitcast i32* %tmp to <4 x i32>*
373  %wide.masked.load.a = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
374  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
375  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
376  %wide.masked.load.b = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
377  %sub = sub <4 x i32> %wide.masked.load.a, %wide.masked.load.b
378  %cmp = icmp eq <4 x i32> %sub, <i32 0, i32 0, i32 0, i32 0>
379  %mask = or <4 x i1> %cmp, %tmp1
380  %tmp5 = getelementptr inbounds i32, i32* %c, i32 %index
381  %tmp6 = bitcast i32* %tmp5 to <4 x i32>*
382  %wide.masked.load.c = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp6, i32 4, <4 x i1> %mask, <4 x i32> undef)
383  %tmp7 = getelementptr inbounds i32, i32* %d, i32 %index
384  %tmp8 = bitcast i32* %tmp7 to <4 x i32>*
385  %wide.masked.load.d = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp8, i32 4, <4 x i1> %mask, <4 x i32> undef)
386  %mul = mul  <4 x i32> %wide.masked.load.c, %wide.masked.load.d
387  %add = add  <4 x i32> %mul, %vec.phi
388  %index.next = add i32 %index, 4
389  %cmp.exit = icmp eq i32 %index.next, %n.vec
390  br i1 %cmp.exit, label %middle.block, label %vector.body
391
392middle.block:                                     ; preds = %vector.body
393  %acc = select <4 x i1> %tmp1, <4 x i32> %add, <4 x i32> %vec.phi
394  %reduce = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %acc)
395  br label %for.cond.cleanup
396
397for.cond.cleanup:                                 ; preds = %middle.block, %entry
398  %res.0.lcssa = phi i32 [ 0, %entry ], [ %reduce, %middle.block ]
399  ret i32 %res.0.lcssa
400}
401
402define dso_local void @continue_on_zero(i32* noalias nocapture %arg, i32* noalias nocapture readonly %arg1, i32 %arg2) {
403; CHECK-LABEL: continue_on_zero:
404; CHECK:       @ %bb.0: @ %bb
405; CHECK-NEXT:    push {r7, lr}
406; CHECK-NEXT:    cmp r2, #0
407; CHECK-NEXT:    it eq
408; CHECK-NEXT:    popeq {r7, pc}
409; CHECK-NEXT:    movs r3, #0
410; CHECK-NEXT:    dlstp.32 lr, r2
411; CHECK-NEXT:  .LBB4_1: @ %bb9
412; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
413; CHECK-NEXT:    adds r3, #4
414; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
415; CHECK-NEXT:    vcmp.i32 ne, q0, zr
416; CHECK-NEXT:    vpst
417; CHECK-NEXT:    vldrwt.u32 q1, [r0]
418; CHECK-NEXT:    vmul.i32 q0, q1, q0
419; CHECK-NEXT:    vpst
420; CHECK-NEXT:    vstrwt.32 q0, [r0], #16
421; CHECK-NEXT:    letp lr, .LBB4_1
422; CHECK-NEXT:  @ %bb.2: @ %bb27
423; CHECK-NEXT:    pop {r7, pc}
424bb:
425  %tmp = icmp eq i32 %arg2, 0
426  br i1 %tmp, label %bb27, label %bb3
427
428bb3:                                              ; preds = %bb
429  %tmp4 = add i32 %arg2, 3
430  %tmp5 = and i32 %tmp4, -4
431  %tmp6 = add i32 %arg2, -1
432  %tmp7 = insertelement <4 x i32> undef, i32 %tmp6, i32 0
433  %tmp8 = shufflevector <4 x i32> %tmp7, <4 x i32> undef, <4 x i32> zeroinitializer
434  br label %bb9
435
436bb9:                                              ; preds = %bb9, %bb3
437  %tmp10 = phi i32 [ 0, %bb3 ], [ %tmp25, %bb9 ]
438  %tmp11 = insertelement <4 x i32> undef, i32 %tmp10, i32 0
439  %tmp12 = shufflevector <4 x i32> %tmp11, <4 x i32> undef, <4 x i32> zeroinitializer
440  %tmp13 = add <4 x i32> %tmp12, <i32 0, i32 1, i32 2, i32 3>
441  %tmp14 = getelementptr inbounds i32, i32* %arg1, i32 %tmp10
442
443  ; %tmp15 = icmp ule <4 x i32> %tmp13, %tmp8
444  %tmp15 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %tmp10, i32 %tmp6)
445
446  %tmp16 = bitcast i32* %tmp14 to <4 x i32>*
447  %tmp17 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp16, i32 4, <4 x i1> %tmp15, <4 x i32> undef)
448  %tmp18 = icmp ne <4 x i32> %tmp17, zeroinitializer
449  %tmp19 = getelementptr inbounds i32, i32* %arg, i32 %tmp10
450  %tmp20 = and <4 x i1> %tmp18, %tmp15
451  %tmp21 = bitcast i32* %tmp19 to <4 x i32>*
452  %tmp22 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp21, i32 4, <4 x i1> %tmp20, <4 x i32> undef)
453  %tmp23 = mul nsw <4 x i32> %tmp22, %tmp17
454  %tmp24 = bitcast i32* %tmp19 to <4 x i32>*
455  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp23, <4 x i32>* %tmp24, i32 4, <4 x i1> %tmp20)
456  %tmp25 = add i32 %tmp10, 4
457  %tmp26 = icmp eq i32 %tmp25, %tmp5
458  br i1 %tmp26, label %bb27, label %bb9
459
460bb27:                                             ; preds = %bb9, %bb
461  ret void
462}
463
464define dso_local arm_aapcs_vfpcc void @range_test(i32* noalias nocapture %arg, i32* noalias nocapture readonly %arg1, i32 %arg2, i32 %arg3) {
465; CHECK-LABEL: range_test:
466; CHECK:       @ %bb.0: @ %bb
467; CHECK-NEXT:    push {r7, lr}
468; CHECK-NEXT:    cmp r3, #0
469; CHECK-NEXT:    it eq
470; CHECK-NEXT:    popeq {r7, pc}
471; CHECK-NEXT:    mov.w r12, #0
472; CHECK-NEXT:    dlstp.32 lr, r3
473; CHECK-NEXT:  .LBB5_1: @ %bb12
474; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
475; CHECK-NEXT:    vldrw.u32 q0, [r0]
476; CHECK-NEXT:    vptt.i32 ne, q0, zr
477; CHECK-NEXT:    vcmpt.s32 le, q0, r2
478; CHECK-NEXT:    vldrwt.u32 q1, [r1], #16
479; CHECK-NEXT:    add.w r12, r12, #4
480; CHECK-NEXT:    vmul.i32 q0, q1, q0
481; CHECK-NEXT:    vpst
482; CHECK-NEXT:    vstrwt.32 q0, [r0], #16
483; CHECK-NEXT:    letp lr, .LBB5_1
484; CHECK-NEXT:  @ %bb.2: @ %bb32
485; CHECK-NEXT:    pop {r7, pc}
486bb:
487  %tmp = icmp eq i32 %arg3, 0
488  br i1 %tmp, label %bb32, label %bb4
489
490bb4:                                              ; preds = %bb
491  %tmp5 = add i32 %arg3, 3
492  %tmp6 = and i32 %tmp5, -4
493  %tmp7 = add i32 %arg3, -1
494  %tmp8 = insertelement <4 x i32> undef, i32 %tmp7, i32 0
495  %tmp9 = shufflevector <4 x i32> %tmp8, <4 x i32> undef, <4 x i32> zeroinitializer
496  %tmp10 = insertelement <4 x i32> undef, i32 %arg2, i32 0
497  %tmp11 = shufflevector <4 x i32> %tmp10, <4 x i32> undef, <4 x i32> zeroinitializer
498  br label %bb12
499
500bb12:                                             ; preds = %bb12, %bb4
501  %tmp13 = phi i32 [ 0, %bb4 ], [ %tmp30, %bb12 ]
502  %tmp14 = insertelement <4 x i32> undef, i32 %tmp13, i32 0
503  %tmp15 = shufflevector <4 x i32> %tmp14, <4 x i32> undef, <4 x i32> zeroinitializer
504  %tmp16 = add <4 x i32> %tmp15, <i32 0, i32 1, i32 2, i32 3>
505  %tmp17 = getelementptr inbounds i32, i32* %arg, i32 %tmp13
506
507  ; %tmp18 = icmp ule <4 x i32> %tmp16, %tmp9
508  %tmp18= call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %tmp13, i32 %tmp7)
509
510  %tmp19 = bitcast i32* %tmp17 to <4 x i32>*
511  %tmp20 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp19, i32 4, <4 x i1> %tmp18, <4 x i32> undef)
512  %tmp21 = icmp ne <4 x i32> %tmp20, zeroinitializer
513  %tmp22 = icmp sle <4 x i32> %tmp20, %tmp11
514  %tmp23 = getelementptr inbounds i32, i32* %arg1, i32 %tmp13
515  %tmp24 = and <4 x i1> %tmp22, %tmp21
516  %tmp25 = and <4 x i1> %tmp24, %tmp18
517  %tmp26 = bitcast i32* %tmp23 to <4 x i32>*
518  %tmp27 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp26, i32 4, <4 x i1> %tmp25, <4 x i32> undef)
519  %tmp28 = mul nsw <4 x i32> %tmp27, %tmp20
520  %tmp29 = bitcast i32* %tmp17 to <4 x i32>*
521  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp28, <4 x i32>* %tmp29, i32 4, <4 x i1> %tmp25)
522  %tmp30 = add i32 %tmp13, 4
523  %tmp31 = icmp eq i32 %tmp30, %tmp6
524  br i1 %tmp31, label %bb32, label %bb12
525
526bb32:                                             ; preds = %bb12, %bb
527  ret void
528}
529
530; Function Attrs: argmemonly nounwind readonly willreturn
531declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>)
532declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32, <4 x i1>)
533
534; Function Attrs: nounwind readnone willreturn
535declare i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32>)
536
537declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32)
538