1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=armv8.1m.main -mattr=+mve -disable-mve-tail-predication=false --verify-machineinstrs %s -o - | FileCheck %s
3
4define dso_local i32 @mul_reduce_add(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32 %N) {
5; CHECK-LABEL: mul_reduce_add:
6; CHECK:       @ %bb.0: @ %entry
7; CHECK-NEXT:    cmp r2, #0
8; CHECK-NEXT:    itt eq
9; CHECK-NEXT:    moveq r0, #0
10; CHECK-NEXT:    bxeq lr
11; CHECK-NEXT:    push {r7, lr}
12; CHECK-NEXT:    adds r3, r2, #3
13; CHECK-NEXT:    vmov.i32 q0, #0x0
14; CHECK-NEXT:    bic r3, r3, #3
15; CHECK-NEXT:    sub.w r12, r3, #4
16; CHECK-NEXT:    movs r3, #1
17; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
18; CHECK-NEXT:    movs r3, #0
19; CHECK-NEXT:    dls lr, lr
20; CHECK-NEXT:  .LBB0_1: @ %vector.body
21; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
22; CHECK-NEXT:    vctp.32 r2
23; CHECK-NEXT:    vmov q1, q0
24; CHECK-NEXT:    vpstt
25; CHECK-NEXT:    vldrwt.u32 q0, [r0], #16
26; CHECK-NEXT:    vldrwt.u32 q2, [r1], #16
27; CHECK-NEXT:    adds r3, #4
28; CHECK-NEXT:    vmul.i32 q0, q2, q0
29; CHECK-NEXT:    subs r2, #4
30; CHECK-NEXT:    vadd.i32 q0, q0, q1
31; CHECK-NEXT:    le lr, .LBB0_1
32; CHECK-NEXT:  @ %bb.2: @ %middle.block
33; CHECK-NEXT:    vpsel q0, q0, q1
34; CHECK-NEXT:    vaddv.u32 r0, q0
35; CHECK-NEXT:    pop {r7, pc}
36entry:
37  %cmp8 = icmp eq i32 %N, 0
38  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
39
40vector.ph:                                        ; preds = %entry
41  %n.rnd.up = add i32 %N, 3
42  %n.vec = and i32 %n.rnd.up, -4
43  %trip.count.minus.1 = add i32 %N, -1
44  %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
45  %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
46  br label %vector.body
47
48vector.body:                                      ; preds = %vector.body, %vector.ph
49  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
50  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %6, %vector.body ]
51  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
52  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
53  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
54  %0 = getelementptr inbounds i32, i32* %a, i32 %index
55
56  ; %1 = icmp ule <4 x i32> %induction, %broadcast.splat12
57  %1 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
58
59  %2 = bitcast i32* %0 to <4 x i32>*
60  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
61  %3 = getelementptr inbounds i32, i32* %b, i32 %index
62  %4 = bitcast i32* %3 to <4 x i32>*
63  %wide.masked.load13 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %4, i32 4, <4 x i1> %1, <4 x i32> undef)
64  %5 = mul nsw <4 x i32> %wide.masked.load13, %wide.masked.load
65  %6 = add nsw <4 x i32> %5, %vec.phi
66  %index.next = add i32 %index, 4
67  %7 = icmp eq i32 %index.next, %n.vec
68  br i1 %7, label %middle.block, label %vector.body
69
70middle.block:                                     ; preds = %vector.body
71  %8 = select <4 x i1> %1, <4 x i32> %6, <4 x i32> %vec.phi
72  %9 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %8)
73  br label %for.cond.cleanup
74
75for.cond.cleanup:                                 ; preds = %middle.block, %entry
76  %res.0.lcssa = phi i32 [ 0, %entry ], [ %9, %middle.block ]
77  ret i32 %res.0.lcssa
78}
79
80define dso_local i32 @mul_reduce_add_const(i32* noalias nocapture readonly %a, i32 %b, i32 %N) {
81; CHECK-LABEL: mul_reduce_add_const:
82; CHECK:       @ %bb.0: @ %entry
83; CHECK-NEXT:    cmp r2, #0
84; CHECK-NEXT:    itt eq
85; CHECK-NEXT:    moveq r0, #0
86; CHECK-NEXT:    bxeq lr
87; CHECK-NEXT:    push {r7, lr}
88; CHECK-NEXT:    adds r1, r2, #3
89; CHECK-NEXT:    movs r3, #1
90; CHECK-NEXT:    bic r1, r1, #3
91; CHECK-NEXT:    vmov.i32 q0, #0x0
92; CHECK-NEXT:    subs r1, #4
93; CHECK-NEXT:    add.w lr, r3, r1, lsr #2
94; CHECK-NEXT:    movs r1, #0
95; CHECK-NEXT:    dls lr, lr
96; CHECK-NEXT:  .LBB1_1: @ %vector.body
97; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
98; CHECK-NEXT:    vctp.32 r2
99; CHECK-NEXT:    vmov q1, q0
100; CHECK-NEXT:    vpst
101; CHECK-NEXT:    vldrwt.u32 q0, [r0], #16
102; CHECK-NEXT:    adds r1, #4
103; CHECK-NEXT:    subs r2, #4
104; CHECK-NEXT:    vadd.i32 q0, q0, q1
105; CHECK-NEXT:    le lr, .LBB1_1
106; CHECK-NEXT:  @ %bb.2: @ %middle.block
107; CHECK-NEXT:    vpsel q0, q0, q1
108; CHECK-NEXT:    vaddv.u32 r0, q0
109; CHECK-NEXT:    pop {r7, pc}
110entry:
111  %cmp6 = icmp eq i32 %N, 0
112  br i1 %cmp6, label %for.cond.cleanup, label %vector.ph
113
114vector.ph:                                        ; preds = %entry
115  %n.rnd.up = add i32 %N, 3
116  %n.vec = and i32 %n.rnd.up, -4
117  %trip.count.minus.1 = add i32 %N, -1
118  %broadcast.splatinsert9 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
119  %broadcast.splat10 = shufflevector <4 x i32> %broadcast.splatinsert9, <4 x i32> undef, <4 x i32> zeroinitializer
120  br label %vector.body
121
122vector.body:                                      ; preds = %vector.body, %vector.ph
123  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
124  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %3, %vector.body ]
125  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
126  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
127  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
128  %0 = getelementptr inbounds i32, i32* %a, i32 %index
129
130  ; %1 = icmp ule <4 x i32> %induction, %broadcast.splat10
131  %1 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
132
133  %2 = bitcast i32* %0 to <4 x i32>*
134  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
135  %3 = add nsw <4 x i32> %wide.masked.load, %vec.phi
136  %index.next = add i32 %index, 4
137  %4 = icmp eq i32 %index.next, %n.vec
138  br i1 %4, label %middle.block, label %vector.body
139
140middle.block:                                     ; preds = %vector.body
141  %5 = select <4 x i1> %1, <4 x i32> %3, <4 x i32> %vec.phi
142  %6 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %5)
143  br label %for.cond.cleanup
144
145for.cond.cleanup:                                 ; preds = %middle.block, %entry
146  %res.0.lcssa = phi i32 [ 0, %entry ], [ %6, %middle.block ]
147  ret i32 %res.0.lcssa
148}
149
150define dso_local i32 @add_reduce_add_const(i32* noalias nocapture readonly %a, i32 %b, i32 %N) {
151; CHECK-LABEL: add_reduce_add_const:
152; CHECK:       @ %bb.0: @ %entry
153; CHECK-NEXT:    cmp r2, #0
154; CHECK-NEXT:    itt eq
155; CHECK-NEXT:    moveq r0, #0
156; CHECK-NEXT:    bxeq lr
157; CHECK-NEXT:    push {r7, lr}
158; CHECK-NEXT:    adds r1, r2, #3
159; CHECK-NEXT:    movs r3, #1
160; CHECK-NEXT:    bic r1, r1, #3
161; CHECK-NEXT:    vmov.i32 q0, #0x0
162; CHECK-NEXT:    subs r1, #4
163; CHECK-NEXT:    add.w lr, r3, r1, lsr #2
164; CHECK-NEXT:    movs r1, #0
165; CHECK-NEXT:    dls lr, lr
166; CHECK-NEXT:  .LBB2_1: @ %vector.body
167; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
168; CHECK-NEXT:    vctp.32 r2
169; CHECK-NEXT:    vmov q1, q0
170; CHECK-NEXT:    vpst
171; CHECK-NEXT:    vldrwt.u32 q0, [r0], #16
172; CHECK-NEXT:    adds r1, #4
173; CHECK-NEXT:    subs r2, #4
174; CHECK-NEXT:    vadd.i32 q0, q0, q1
175; CHECK-NEXT:    le lr, .LBB2_1
176; CHECK-NEXT:  @ %bb.2: @ %middle.block
177; CHECK-NEXT:    vpsel q0, q0, q1
178; CHECK-NEXT:    vaddv.u32 r0, q0
179; CHECK-NEXT:    pop {r7, pc}
180entry:
181  %cmp6 = icmp eq i32 %N, 0
182  br i1 %cmp6, label %for.cond.cleanup, label %vector.ph
183
184vector.ph:                                        ; preds = %entry
185  %n.rnd.up = add i32 %N, 3
186  %n.vec = and i32 %n.rnd.up, -4
187  %trip.count.minus.1 = add i32 %N, -1
188  %broadcast.splatinsert9 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
189  %broadcast.splat10 = shufflevector <4 x i32> %broadcast.splatinsert9, <4 x i32> undef, <4 x i32> zeroinitializer
190  br label %vector.body
191
192vector.body:                                      ; preds = %vector.body, %vector.ph
193  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
194  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %3, %vector.body ]
195  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
196  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
197  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
198  %0 = getelementptr inbounds i32, i32* %a, i32 %index
199
200  ; %1 = icmp ule <4 x i32> %induction, %broadcast.splat10
201  %1 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
202
203  %2 = bitcast i32* %0 to <4 x i32>*
204  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
205  %3 = add nsw <4 x i32> %wide.masked.load, %vec.phi
206  %index.next = add i32 %index, 4
207  %4 = icmp eq i32 %index.next, %n.vec
208  br i1 %4, label %middle.block, label %vector.body
209
210middle.block:                                     ; preds = %vector.body
211  %5 = select <4 x i1> %1, <4 x i32> %3, <4 x i32> %vec.phi
212  %6 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %5)
213  br label %for.cond.cleanup
214
215for.cond.cleanup:                                 ; preds = %middle.block, %entry
216  %res.0.lcssa = phi i32 [ 0, %entry ], [ %6, %middle.block ]
217  ret i32 %res.0.lcssa
218}
219
220define dso_local void @vector_mul_const(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32 %c, i32 %N) {
221; CHECK-LABEL: vector_mul_const:
222; CHECK:       @ %bb.0: @ %entry
223; CHECK-NEXT:    push {r7, lr}
224; CHECK-NEXT:    cmp r3, #0
225; CHECK-NEXT:    it eq
226; CHECK-NEXT:    popeq {r7, pc}
227; CHECK-NEXT:    mov.w r12, #0
228; CHECK-NEXT:    dlstp.32 lr, r3
229; CHECK-NEXT:  .LBB3_1: @ %vector.body
230; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
231; CHECK-NEXT:    add.w r12, r12, #4
232; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
233; CHECK-NEXT:    vmul.i32 q0, q0, r2
234; CHECK-NEXT:    vstrw.32 q0, [r0], #16
235; CHECK-NEXT:    letp lr, .LBB3_1
236; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
237; CHECK-NEXT:    pop {r7, pc}
238entry:
239  %cmp6 = icmp eq i32 %N, 0
240  br i1 %cmp6, label %for.cond.cleanup, label %vector.ph
241
242vector.ph:                                        ; preds = %entry
243  %n.rnd.up = add i32 %N, 3
244  %n.vec = and i32 %n.rnd.up, -4
245  %trip.count.minus.1 = add i32 %N, -1
246  %broadcast.splatinsert8 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
247  %broadcast.splat9 = shufflevector <4 x i32> %broadcast.splatinsert8, <4 x i32> undef, <4 x i32> zeroinitializer
248  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %c, i32 0
249  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
250  br label %vector.body
251
252vector.body:                                      ; preds = %vector.body, %vector.ph
253  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
254  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
255  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
256  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
257  %0 = getelementptr inbounds i32, i32* %b, i32 %index
258
259  ; %1 = icmp ule <4 x i32> %induction, %broadcast.splat9
260  %1 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
261
262  %2 = bitcast i32* %0 to <4 x i32>*
263  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
264  %3 = mul nsw <4 x i32> %wide.masked.load, %broadcast.splat11
265  %4 = getelementptr inbounds i32, i32* %a, i32 %index
266  %5 = bitcast i32* %4 to <4 x i32>*
267  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %3, <4 x i32>* %5, i32 4, <4 x i1> %1)
268  %index.next = add i32 %index, 4
269  %6 = icmp eq i32 %index.next, %n.vec
270  br i1 %6, label %for.cond.cleanup, label %vector.body
271
272for.cond.cleanup:                                 ; preds = %vector.body, %entry
273  ret void
274}
275
276define dso_local void @vector_add_const(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32 %c, i32 %N) {
277; CHECK-LABEL: vector_add_const:
278; CHECK:       @ %bb.0: @ %entry
279; CHECK-NEXT:    push {r7, lr}
280; CHECK-NEXT:    cmp r3, #0
281; CHECK-NEXT:    it eq
282; CHECK-NEXT:    popeq {r7, pc}
283; CHECK-NEXT:    mov.w r12, #0
284; CHECK-NEXT:    dlstp.32 lr, r3
285; CHECK-NEXT:  .LBB4_1: @ %vector.body
286; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
287; CHECK-NEXT:    add.w r12, r12, #4
288; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
289; CHECK-NEXT:    vadd.i32 q0, q0, r2
290; CHECK-NEXT:    vstrw.32 q0, [r0], #16
291; CHECK-NEXT:    letp lr, .LBB4_1
292; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
293; CHECK-NEXT:    pop {r7, pc}
294entry:
295  %cmp6 = icmp eq i32 %N, 0
296  br i1 %cmp6, label %for.cond.cleanup, label %vector.ph
297
298vector.ph:                                        ; preds = %entry
299  %n.rnd.up = add i32 %N, 3
300  %n.vec = and i32 %n.rnd.up, -4
301  %trip.count.minus.1 = add i32 %N, -1
302  %broadcast.splatinsert8 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
303  %broadcast.splat9 = shufflevector <4 x i32> %broadcast.splatinsert8, <4 x i32> undef, <4 x i32> zeroinitializer
304  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %c, i32 0
305  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
306  br label %vector.body
307
308vector.body:                                      ; preds = %vector.body, %vector.ph
309  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
310  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
311  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
312  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
313  %0 = getelementptr inbounds i32, i32* %b, i32 %index
314
315  ; %1 = icmp ule <4 x i32> %induction, %broadcast.splat9
316  %1 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
317
318  %2 = bitcast i32* %0 to <4 x i32>*
319  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
320  %3 = add nsw <4 x i32> %wide.masked.load, %broadcast.splat11
321  %4 = getelementptr inbounds i32, i32* %a, i32 %index
322  %5 = bitcast i32* %4 to <4 x i32>*
323  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %3, <4 x i32>* %5, i32 4, <4 x i1> %1)
324  %index.next = add i32 %index, 4
325  %6 = icmp eq i32 %index.next, %n.vec
326  br i1 %6, label %for.cond.cleanup, label %vector.body
327
328for.cond.cleanup:                                 ; preds = %vector.body, %entry
329  ret void
330}
331
332define dso_local arm_aapcs_vfpcc void @vector_mul_vector_i8(i8* noalias nocapture %a, i8* noalias nocapture readonly %b, i8* noalias nocapture readonly %c, i32 %N) {
333; CHECK-LABEL: vector_mul_vector_i8:
334; CHECK:       @ %bb.0: @ %entry
335; CHECK-NEXT:    push {r7, lr}
336; CHECK-NEXT:    cmp r3, #0
337; CHECK-NEXT:    it eq
338; CHECK-NEXT:    popeq {r7, pc}
339; CHECK-NEXT:    mov.w r12, #0
340; CHECK-NEXT:    dlstp.8 lr, r3
341; CHECK-NEXT:  .LBB5_1: @ %vector.body
342; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
343; CHECK-NEXT:    add.w r12, r12, #16
344; CHECK-NEXT:    vldrb.u8 q0, [r1], #16
345; CHECK-NEXT:    vldrb.u8 q1, [r2], #16
346; CHECK-NEXT:    vmul.i8 q0, q1, q0
347; CHECK-NEXT:    vstrb.8 q0, [r0], #16
348; CHECK-NEXT:    letp lr, .LBB5_1
349; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
350; CHECK-NEXT:    pop {r7, pc}
351entry:
352  %cmp10 = icmp eq i32 %N, 0
353  br i1 %cmp10, label %for.cond.cleanup, label %vector.ph
354
355vector.ph:                                        ; preds = %entry
356  %n.rnd.up = add i32 %N, 15
357  %n.vec = and i32 %n.rnd.up, -16
358  %trip.count.minus.1 = add i32 %N, -1
359  %broadcast.splatinsert12 = insertelement <16 x i32> undef, i32 %trip.count.minus.1, i32 0
360  %broadcast.splat13 = shufflevector <16 x i32> %broadcast.splatinsert12, <16 x i32> undef, <16 x i32> zeroinitializer
361  br label %vector.body
362
363vector.body:                                      ; preds = %vector.body, %vector.ph
364  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
365  %broadcast.splatinsert = insertelement <16 x i32> undef, i32 %index, i32 0
366  %broadcast.splat = shufflevector <16 x i32> %broadcast.splatinsert, <16 x i32> undef, <16 x i32> zeroinitializer
367  %induction = add <16 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
368  %0 = getelementptr inbounds i8, i8* %b, i32 %index
369
370  ; %1 = icmp ule <16 x i32> %induction, %broadcast.splat13
371  %1 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %trip.count.minus.1)
372
373  %2 = bitcast i8* %0 to <16 x i8>*
374  %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %2, i32 1, <16 x i1> %1, <16 x i8> undef)
375  %3 = getelementptr inbounds i8, i8* %c, i32 %index
376  %4 = bitcast i8* %3 to <16 x i8>*
377  %wide.masked.load14 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %4, i32 1, <16 x i1> %1, <16 x i8> undef)
378  %5 = mul <16 x i8> %wide.masked.load14, %wide.masked.load
379  %6 = getelementptr inbounds i8, i8* %a, i32 %index
380  %7 = bitcast i8* %6 to <16 x i8>*
381  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %5, <16 x i8>* %7, i32 1, <16 x i1> %1)
382  %index.next = add i32 %index, 16
383  %8 = icmp eq i32 %index.next, %n.vec
384  br i1 %8, label %for.cond.cleanup, label %vector.body
385
386for.cond.cleanup:                                 ; preds = %vector.body, %entry
387  ret void
388}
389
390; Function Attrs: nofree norecurse nounwind
391define dso_local arm_aapcs_vfpcc void @vector_mul_vector_i16(i16* noalias nocapture %a, i16* noalias nocapture readonly %b, i16* noalias nocapture readonly %c, i32 %N) local_unnamed_addr #0 {
392; CHECK-LABEL: vector_mul_vector_i16:
393; CHECK:       @ %bb.0: @ %entry
394; CHECK-NEXT:    push {r7, lr}
395; CHECK-NEXT:    cmp r3, #0
396; CHECK-NEXT:    it eq
397; CHECK-NEXT:    popeq {r7, pc}
398; CHECK-NEXT:    mov.w r12, #0
399; CHECK-NEXT:    dlstp.16 lr, r3
400; CHECK-NEXT:  .LBB6_1: @ %vector.body
401; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
402; CHECK-NEXT:    add.w r12, r12, #8
403; CHECK-NEXT:    vldrh.u16 q0, [r1], #16
404; CHECK-NEXT:    vldrh.u16 q1, [r2], #16
405; CHECK-NEXT:    vmul.i16 q0, q1, q0
406; CHECK-NEXT:    vstrh.16 q0, [r0], #16
407; CHECK-NEXT:    letp lr, .LBB6_1
408; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
409; CHECK-NEXT:    pop {r7, pc}
410entry:
411  %cmp10 = icmp eq i32 %N, 0
412  br i1 %cmp10, label %for.cond.cleanup, label %vector.ph
413
414vector.ph:                                        ; preds = %entry
415  %n.rnd.up = add i32 %N, 7
416  %n.vec = and i32 %n.rnd.up, -8
417  %trip.count.minus.1 = add i32 %N, -1
418  %broadcast.splatinsert12 = insertelement <8 x i32> undef, i32 %trip.count.minus.1, i32 0
419  %broadcast.splat13 = shufflevector <8 x i32> %broadcast.splatinsert12, <8 x i32> undef, <8 x i32> zeroinitializer
420  br label %vector.body
421
422vector.body:                                      ; preds = %vector.body, %vector.ph
423  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
424  %broadcast.splatinsert = insertelement <8 x i32> undef, i32 %index, i32 0
425  %broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> undef, <8 x i32> zeroinitializer
426  %induction = add <8 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
427  %0 = getelementptr inbounds i16, i16* %b, i32 %index
428
429  ; %1 = icmp ule <8 x i32> %induction, %broadcast.splat13
430  %1 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %trip.count.minus.1)
431
432  %2 = bitcast i16* %0 to <8 x i16>*
433  %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %2, i32 2, <8 x i1> %1, <8 x i16> undef)
434  %3 = getelementptr inbounds i16, i16* %c, i32 %index
435  %4 = bitcast i16* %3 to <8 x i16>*
436  %wide.masked.load14 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %4, i32 2, <8 x i1> %1, <8 x i16> undef)
437  %5 = mul <8 x i16> %wide.masked.load14, %wide.masked.load
438  %6 = getelementptr inbounds i16, i16* %a, i32 %index
439  %7 = bitcast i16* %6 to <8 x i16>*
440  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %5, <8 x i16>* %7, i32 2, <8 x i1> %1)
441  %index.next = add i32 %index, 8
442  %8 = icmp eq i32 %index.next, %n.vec
443  br i1 %8, label %for.cond.cleanup, label %vector.body
444
445for.cond.cleanup:                                 ; preds = %vector.body, %entry
446  ret void
447}
448
449declare <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>*, i32 immarg, <16 x i1>, <16 x i8>)
450declare <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>*, i32 immarg, <8 x i1>, <8 x i16>)
451declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>)
452declare void @llvm.masked.store.v16i8.p0v16i8(<16 x i8>, <16 x i8>*, i32 immarg, <16 x i1>)
453declare void @llvm.masked.store.v8i16.p0v8i16(<8 x i16>, <8 x i16>*, i32 immarg, <8 x i1>)
454declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32 immarg, <4 x i1>)
455declare i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32>)
456declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32)
457declare <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32, i32)
458declare <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32, i32)
459