1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=armv8.1m.main -mattr=+mve -tail-predication=enabled --verify-machineinstrs %s -o - | FileCheck %s
3
4define dso_local i32 @mul_reduce_add(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32 %N) {
5; CHECK-LABEL: mul_reduce_add:
6; CHECK:       @ %bb.0: @ %entry
7; CHECK-NEXT:    cmp r2, #0
8; CHECK-NEXT:    itt eq
9; CHECK-NEXT:    moveq r0, #0
10; CHECK-NEXT:    bxeq lr
11; CHECK-NEXT:    push {r7, lr}
12; CHECK-NEXT:    vmov.i32 q1, #0x0
13; CHECK-NEXT:    movs r3, #0
14; CHECK-NEXT:    dlstp.32 lr, r2
15; CHECK-NEXT:  .LBB0_1: @ %vector.body
16; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
17; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
18; CHECK-NEXT:    vldrw.u32 q2, [r1], #16
19; CHECK-NEXT:    adds r3, #4
20; CHECK-NEXT:    vmul.i32 q0, q2, q0
21; CHECK-NEXT:    vadd.i32 q1, q0, q1
22; CHECK-NEXT:    letp lr, .LBB0_1
23; CHECK-NEXT:  @ %bb.2: @ %middle.block
24; CHECK-NEXT:    vmov q0, q1
25; CHECK-NEXT:    vaddv.u32 r0, q0
26; CHECK-NEXT:    pop {r7, pc}
27entry:
28  %cmp8 = icmp eq i32 %N, 0
29  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
30
31vector.ph:                                        ; preds = %entry
32  %n.rnd.up = add i32 %N, 3
33  %n.vec = and i32 %n.rnd.up, -4
34  %trip.count.minus.1 = add i32 %N, -1
35  %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
36  %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
37  br label %vector.body
38
39vector.body:                                      ; preds = %vector.body, %vector.ph
40  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
41  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %6, %vector.body ]
42  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
43  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
44  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
45  %0 = getelementptr inbounds i32, i32* %a, i32 %index
46
47  ; %1 = icmp ule <4 x i32> %induction, %broadcast.splat12
48  %1 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
49
50  %2 = bitcast i32* %0 to <4 x i32>*
51  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
52  %3 = getelementptr inbounds i32, i32* %b, i32 %index
53  %4 = bitcast i32* %3 to <4 x i32>*
54  %wide.masked.load13 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %4, i32 4, <4 x i1> %1, <4 x i32> undef)
55  %5 = mul nsw <4 x i32> %wide.masked.load13, %wide.masked.load
56  %6 = add nsw <4 x i32> %5, %vec.phi
57  %index.next = add i32 %index, 4
58  %7 = icmp eq i32 %index.next, %n.vec
59  br i1 %7, label %middle.block, label %vector.body
60
61middle.block:                                     ; preds = %vector.body
62  %8 = select <4 x i1> %1, <4 x i32> %6, <4 x i32> %vec.phi
63  %9 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %8)
64  br label %for.cond.cleanup
65
66for.cond.cleanup:                                 ; preds = %middle.block, %entry
67  %res.0.lcssa = phi i32 [ 0, %entry ], [ %9, %middle.block ]
68  ret i32 %res.0.lcssa
69}
70
71define dso_local i32 @mul_reduce_add_const(i32* noalias nocapture readonly %a, i32 %b, i32 %N) {
72; CHECK-LABEL: mul_reduce_add_const:
73; CHECK:       @ %bb.0: @ %entry
74; CHECK-NEXT:    cmp r2, #0
75; CHECK-NEXT:    itt eq
76; CHECK-NEXT:    moveq r0, #0
77; CHECK-NEXT:    bxeq lr
78; CHECK-NEXT:    push {r7, lr}
79; CHECK-NEXT:    vmov.i32 q1, #0x0
80; CHECK-NEXT:    movs r1, #0
81; CHECK-NEXT:    dlstp.32 lr, r2
82; CHECK-NEXT:  .LBB1_1: @ %vector.body
83; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
84; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
85; CHECK-NEXT:    adds r1, #4
86; CHECK-NEXT:    vadd.i32 q1, q0, q1
87; CHECK-NEXT:    letp lr, .LBB1_1
88; CHECK-NEXT:  @ %bb.2: @ %middle.block
89; CHECK-NEXT:    vmov q0, q1
90; CHECK-NEXT:    vaddv.u32 r0, q0
91; CHECK-NEXT:    pop {r7, pc}
92entry:
93  %cmp6 = icmp eq i32 %N, 0
94  br i1 %cmp6, label %for.cond.cleanup, label %vector.ph
95
96vector.ph:                                        ; preds = %entry
97  %n.rnd.up = add i32 %N, 3
98  %n.vec = and i32 %n.rnd.up, -4
99  %trip.count.minus.1 = add i32 %N, -1
100  %broadcast.splatinsert9 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
101  %broadcast.splat10 = shufflevector <4 x i32> %broadcast.splatinsert9, <4 x i32> undef, <4 x i32> zeroinitializer
102  br label %vector.body
103
104vector.body:                                      ; preds = %vector.body, %vector.ph
105  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
106  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %3, %vector.body ]
107  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
108  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
109  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
110  %0 = getelementptr inbounds i32, i32* %a, i32 %index
111
112  ; %1 = icmp ule <4 x i32> %induction, %broadcast.splat10
113  %1 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
114
115  %2 = bitcast i32* %0 to <4 x i32>*
116  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
117  %3 = add nsw <4 x i32> %wide.masked.load, %vec.phi
118  %index.next = add i32 %index, 4
119  %4 = icmp eq i32 %index.next, %n.vec
120  br i1 %4, label %middle.block, label %vector.body
121
122middle.block:                                     ; preds = %vector.body
123  %5 = select <4 x i1> %1, <4 x i32> %3, <4 x i32> %vec.phi
124  %6 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %5)
125  br label %for.cond.cleanup
126
127for.cond.cleanup:                                 ; preds = %middle.block, %entry
128  %res.0.lcssa = phi i32 [ 0, %entry ], [ %6, %middle.block ]
129  ret i32 %res.0.lcssa
130}
131
132define dso_local i32 @add_reduce_add_const(i32* noalias nocapture readonly %a, i32 %b, i32 %N) {
133; CHECK-LABEL: add_reduce_add_const:
134; CHECK:       @ %bb.0: @ %entry
135; CHECK-NEXT:    cmp r2, #0
136; CHECK-NEXT:    itt eq
137; CHECK-NEXT:    moveq r0, #0
138; CHECK-NEXT:    bxeq lr
139; CHECK-NEXT:    push {r7, lr}
140; CHECK-NEXT:    vmov.i32 q1, #0x0
141; CHECK-NEXT:    movs r1, #0
142; CHECK-NEXT:    dlstp.32 lr, r2
143; CHECK-NEXT:  .LBB2_1: @ %vector.body
144; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
145; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
146; CHECK-NEXT:    adds r1, #4
147; CHECK-NEXT:    vadd.i32 q1, q0, q1
148; CHECK-NEXT:    letp lr, .LBB2_1
149; CHECK-NEXT:  @ %bb.2: @ %middle.block
150; CHECK-NEXT:    vmov q0, q1
151; CHECK-NEXT:    vaddv.u32 r0, q0
152; CHECK-NEXT:    pop {r7, pc}
153entry:
154  %cmp6 = icmp eq i32 %N, 0
155  br i1 %cmp6, label %for.cond.cleanup, label %vector.ph
156
157vector.ph:                                        ; preds = %entry
158  %n.rnd.up = add i32 %N, 3
159  %n.vec = and i32 %n.rnd.up, -4
160  %trip.count.minus.1 = add i32 %N, -1
161  %broadcast.splatinsert9 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
162  %broadcast.splat10 = shufflevector <4 x i32> %broadcast.splatinsert9, <4 x i32> undef, <4 x i32> zeroinitializer
163  br label %vector.body
164
165vector.body:                                      ; preds = %vector.body, %vector.ph
166  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
167  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %3, %vector.body ]
168  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
169  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
170  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
171  %0 = getelementptr inbounds i32, i32* %a, i32 %index
172
173  ; %1 = icmp ule <4 x i32> %induction, %broadcast.splat10
174  %1 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
175
176  %2 = bitcast i32* %0 to <4 x i32>*
177  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
178  %3 = add nsw <4 x i32> %wide.masked.load, %vec.phi
179  %index.next = add i32 %index, 4
180  %4 = icmp eq i32 %index.next, %n.vec
181  br i1 %4, label %middle.block, label %vector.body
182
183middle.block:                                     ; preds = %vector.body
184  %5 = select <4 x i1> %1, <4 x i32> %3, <4 x i32> %vec.phi
185  %6 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %5)
186  br label %for.cond.cleanup
187
188for.cond.cleanup:                                 ; preds = %middle.block, %entry
189  %res.0.lcssa = phi i32 [ 0, %entry ], [ %6, %middle.block ]
190  ret i32 %res.0.lcssa
191}
192
193define dso_local void @vector_mul_const(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32 %c, i32 %N) {
194; CHECK-LABEL: vector_mul_const:
195; CHECK:       @ %bb.0: @ %entry
196; CHECK-NEXT:    push {r7, lr}
197; CHECK-NEXT:    cmp r3, #0
198; CHECK-NEXT:    it eq
199; CHECK-NEXT:    popeq {r7, pc}
200; CHECK-NEXT:    mov.w r12, #0
201; CHECK-NEXT:    dlstp.32 lr, r3
202; CHECK-NEXT:  .LBB3_1: @ %vector.body
203; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
204; CHECK-NEXT:    add.w r12, r12, #4
205; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
206; CHECK-NEXT:    vmul.i32 q0, q0, r2
207; CHECK-NEXT:    vstrw.32 q0, [r0], #16
208; CHECK-NEXT:    letp lr, .LBB3_1
209; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
210; CHECK-NEXT:    pop {r7, pc}
211entry:
212  %cmp6 = icmp eq i32 %N, 0
213  br i1 %cmp6, label %for.cond.cleanup, label %vector.ph
214
215vector.ph:                                        ; preds = %entry
216  %n.rnd.up = add i32 %N, 3
217  %n.vec = and i32 %n.rnd.up, -4
218  %trip.count.minus.1 = add i32 %N, -1
219  %broadcast.splatinsert8 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
220  %broadcast.splat9 = shufflevector <4 x i32> %broadcast.splatinsert8, <4 x i32> undef, <4 x i32> zeroinitializer
221  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %c, i32 0
222  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
223  br label %vector.body
224
225vector.body:                                      ; preds = %vector.body, %vector.ph
226  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
227  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
228  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
229  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
230  %0 = getelementptr inbounds i32, i32* %b, i32 %index
231
232  ; %1 = icmp ule <4 x i32> %induction, %broadcast.splat9
233  %1 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
234
235  %2 = bitcast i32* %0 to <4 x i32>*
236  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
237  %3 = mul nsw <4 x i32> %wide.masked.load, %broadcast.splat11
238  %4 = getelementptr inbounds i32, i32* %a, i32 %index
239  %5 = bitcast i32* %4 to <4 x i32>*
240  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %3, <4 x i32>* %5, i32 4, <4 x i1> %1)
241  %index.next = add i32 %index, 4
242  %6 = icmp eq i32 %index.next, %n.vec
243  br i1 %6, label %for.cond.cleanup, label %vector.body
244
245for.cond.cleanup:                                 ; preds = %vector.body, %entry
246  ret void
247}
248
249define dso_local void @vector_add_const(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32 %c, i32 %N) {
250; CHECK-LABEL: vector_add_const:
251; CHECK:       @ %bb.0: @ %entry
252; CHECK-NEXT:    push {r7, lr}
253; CHECK-NEXT:    cmp r3, #0
254; CHECK-NEXT:    it eq
255; CHECK-NEXT:    popeq {r7, pc}
256; CHECK-NEXT:    mov.w r12, #0
257; CHECK-NEXT:    dlstp.32 lr, r3
258; CHECK-NEXT:  .LBB4_1: @ %vector.body
259; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
260; CHECK-NEXT:    add.w r12, r12, #4
261; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
262; CHECK-NEXT:    vadd.i32 q0, q0, r2
263; CHECK-NEXT:    vstrw.32 q0, [r0], #16
264; CHECK-NEXT:    letp lr, .LBB4_1
265; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
266; CHECK-NEXT:    pop {r7, pc}
267entry:
268  %cmp6 = icmp eq i32 %N, 0
269  br i1 %cmp6, label %for.cond.cleanup, label %vector.ph
270
271vector.ph:                                        ; preds = %entry
272  %n.rnd.up = add i32 %N, 3
273  %n.vec = and i32 %n.rnd.up, -4
274  %trip.count.minus.1 = add i32 %N, -1
275  %broadcast.splatinsert8 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
276  %broadcast.splat9 = shufflevector <4 x i32> %broadcast.splatinsert8, <4 x i32> undef, <4 x i32> zeroinitializer
277  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %c, i32 0
278  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
279  br label %vector.body
280
281vector.body:                                      ; preds = %vector.body, %vector.ph
282  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
283  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
284  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
285  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
286  %0 = getelementptr inbounds i32, i32* %b, i32 %index
287
288  ; %1 = icmp ule <4 x i32> %induction, %broadcast.splat9
289  %1 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
290
291  %2 = bitcast i32* %0 to <4 x i32>*
292  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
293  %3 = add nsw <4 x i32> %wide.masked.load, %broadcast.splat11
294  %4 = getelementptr inbounds i32, i32* %a, i32 %index
295  %5 = bitcast i32* %4 to <4 x i32>*
296  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %3, <4 x i32>* %5, i32 4, <4 x i1> %1)
297  %index.next = add i32 %index, 4
298  %6 = icmp eq i32 %index.next, %n.vec
299  br i1 %6, label %for.cond.cleanup, label %vector.body
300
301for.cond.cleanup:                                 ; preds = %vector.body, %entry
302  ret void
303}
304
305define dso_local arm_aapcs_vfpcc void @vector_mul_vector_i8(i8* noalias nocapture %a, i8* noalias nocapture readonly %b, i8* noalias nocapture readonly %c, i32 %N) {
306; CHECK-LABEL: vector_mul_vector_i8:
307; CHECK:       @ %bb.0: @ %entry
308; CHECK-NEXT:    push {r7, lr}
309; CHECK-NEXT:    cmp r3, #0
310; CHECK-NEXT:    it eq
311; CHECK-NEXT:    popeq {r7, pc}
312; CHECK-NEXT:    mov.w r12, #0
313; CHECK-NEXT:    dlstp.8 lr, r3
314; CHECK-NEXT:  .LBB5_1: @ %vector.body
315; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
316; CHECK-NEXT:    add.w r12, r12, #16
317; CHECK-NEXT:    vldrb.u8 q0, [r1], #16
318; CHECK-NEXT:    vldrb.u8 q1, [r2], #16
319; CHECK-NEXT:    vmul.i8 q0, q1, q0
320; CHECK-NEXT:    vstrb.8 q0, [r0], #16
321; CHECK-NEXT:    letp lr, .LBB5_1
322; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
323; CHECK-NEXT:    pop {r7, pc}
324entry:
325  %cmp10 = icmp eq i32 %N, 0
326  br i1 %cmp10, label %for.cond.cleanup, label %vector.ph
327
328vector.ph:                                        ; preds = %entry
329  %n.rnd.up = add i32 %N, 15
330  %n.vec = and i32 %n.rnd.up, -16
331  %trip.count.minus.1 = add i32 %N, -1
332  %broadcast.splatinsert12 = insertelement <16 x i32> undef, i32 %trip.count.minus.1, i32 0
333  %broadcast.splat13 = shufflevector <16 x i32> %broadcast.splatinsert12, <16 x i32> undef, <16 x i32> zeroinitializer
334  br label %vector.body
335
336vector.body:                                      ; preds = %vector.body, %vector.ph
337  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
338  %broadcast.splatinsert = insertelement <16 x i32> undef, i32 %index, i32 0
339  %broadcast.splat = shufflevector <16 x i32> %broadcast.splatinsert, <16 x i32> undef, <16 x i32> zeroinitializer
340  %induction = add <16 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
341  %0 = getelementptr inbounds i8, i8* %b, i32 %index
342
343  ; %1 = icmp ule <16 x i32> %induction, %broadcast.splat13
344  %1 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %trip.count.minus.1)
345
346  %2 = bitcast i8* %0 to <16 x i8>*
347  %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %2, i32 1, <16 x i1> %1, <16 x i8> undef)
348  %3 = getelementptr inbounds i8, i8* %c, i32 %index
349  %4 = bitcast i8* %3 to <16 x i8>*
350  %wide.masked.load14 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %4, i32 1, <16 x i1> %1, <16 x i8> undef)
351  %5 = mul <16 x i8> %wide.masked.load14, %wide.masked.load
352  %6 = getelementptr inbounds i8, i8* %a, i32 %index
353  %7 = bitcast i8* %6 to <16 x i8>*
354  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %5, <16 x i8>* %7, i32 1, <16 x i1> %1)
355  %index.next = add i32 %index, 16
356  %8 = icmp eq i32 %index.next, %n.vec
357  br i1 %8, label %for.cond.cleanup, label %vector.body
358
359for.cond.cleanup:                                 ; preds = %vector.body, %entry
360  ret void
361}
362
363; Function Attrs: nofree norecurse nounwind
364define dso_local arm_aapcs_vfpcc void @vector_mul_vector_i16(i16* noalias nocapture %a, i16* noalias nocapture readonly %b, i16* noalias nocapture readonly %c, i32 %N) local_unnamed_addr #0 {
365; CHECK-LABEL: vector_mul_vector_i16:
366; CHECK:       @ %bb.0: @ %entry
367; CHECK-NEXT:    push {r7, lr}
368; CHECK-NEXT:    cmp r3, #0
369; CHECK-NEXT:    it eq
370; CHECK-NEXT:    popeq {r7, pc}
371; CHECK-NEXT:    mov.w r12, #0
372; CHECK-NEXT:    dlstp.16 lr, r3
373; CHECK-NEXT:  .LBB6_1: @ %vector.body
374; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
375; CHECK-NEXT:    add.w r12, r12, #8
376; CHECK-NEXT:    vldrh.u16 q0, [r1], #16
377; CHECK-NEXT:    vldrh.u16 q1, [r2], #16
378; CHECK-NEXT:    vmul.i16 q0, q1, q0
379; CHECK-NEXT:    vstrh.16 q0, [r0], #16
380; CHECK-NEXT:    letp lr, .LBB6_1
381; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
382; CHECK-NEXT:    pop {r7, pc}
383entry:
384  %cmp10 = icmp eq i32 %N, 0
385  br i1 %cmp10, label %for.cond.cleanup, label %vector.ph
386
387vector.ph:                                        ; preds = %entry
388  %n.rnd.up = add i32 %N, 7
389  %n.vec = and i32 %n.rnd.up, -8
390  %trip.count.minus.1 = add i32 %N, -1
391  %broadcast.splatinsert12 = insertelement <8 x i32> undef, i32 %trip.count.minus.1, i32 0
392  %broadcast.splat13 = shufflevector <8 x i32> %broadcast.splatinsert12, <8 x i32> undef, <8 x i32> zeroinitializer
393  br label %vector.body
394
395vector.body:                                      ; preds = %vector.body, %vector.ph
396  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
397  %broadcast.splatinsert = insertelement <8 x i32> undef, i32 %index, i32 0
398  %broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> undef, <8 x i32> zeroinitializer
399  %induction = add <8 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
400  %0 = getelementptr inbounds i16, i16* %b, i32 %index
401
402  ; %1 = icmp ule <8 x i32> %induction, %broadcast.splat13
403  %1 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %trip.count.minus.1)
404
405  %2 = bitcast i16* %0 to <8 x i16>*
406  %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %2, i32 2, <8 x i1> %1, <8 x i16> undef)
407  %3 = getelementptr inbounds i16, i16* %c, i32 %index
408  %4 = bitcast i16* %3 to <8 x i16>*
409  %wide.masked.load14 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %4, i32 2, <8 x i1> %1, <8 x i16> undef)
410  %5 = mul <8 x i16> %wide.masked.load14, %wide.masked.load
411  %6 = getelementptr inbounds i16, i16* %a, i32 %index
412  %7 = bitcast i16* %6 to <8 x i16>*
413  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %5, <8 x i16>* %7, i32 2, <8 x i1> %1)
414  %index.next = add i32 %index, 8
415  %8 = icmp eq i32 %index.next, %n.vec
416  br i1 %8, label %for.cond.cleanup, label %vector.body
417
418for.cond.cleanup:                                 ; preds = %vector.body, %entry
419  ret void
420}
421
422declare <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>*, i32 immarg, <16 x i1>, <16 x i8>)
423declare <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>*, i32 immarg, <8 x i1>, <8 x i16>)
424declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>)
425declare void @llvm.masked.store.v16i8.p0v16i8(<16 x i8>, <16 x i8>*, i32 immarg, <16 x i1>)
426declare void @llvm.masked.store.v8i16.p0v8i16(<8 x i16>, <8 x i16>*, i32 immarg, <8 x i1>)
427declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32 immarg, <4 x i1>)
428declare i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32>)
429declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32)
430declare <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32, i32)
431declare <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32, i32)
432