1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=armv8.1m.main -mattr=+mve -disable-mve-tail-predication=false --verify-machineinstrs %s -o - | FileCheck %s
3
4define dso_local i32 @mul_reduce_add(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32 %N) {
5; CHECK-LABEL: mul_reduce_add:
6; CHECK:       @ %bb.0: @ %entry
7; CHECK-NEXT:    cmp r2, #0
8; CHECK-NEXT:    itt eq
9; CHECK-NEXT:    moveq r0, #0
10; CHECK-NEXT:    bxeq lr
11; CHECK-NEXT:    push {r7, lr}
12; CHECK-NEXT:    vmov.i32 q1, #0x0
13; CHECK-NEXT:    dlstp.32 lr, r2
14; CHECK-NEXT:  .LBB0_1: @ %vector.body
15; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
16; CHECK-NEXT:    vmov q0, q1
17; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
18; CHECK-NEXT:    vldrw.u32 q2, [r1], #16
19; CHECK-NEXT:    mov r3, r2
20; CHECK-NEXT:    vmul.i32 q1, q2, q1
21; CHECK-NEXT:    subs r2, #4
22; CHECK-NEXT:    vadd.i32 q1, q1, q0
23; CHECK-NEXT:    letp lr, .LBB0_1
24; CHECK-NEXT:  @ %bb.2: @ %middle.block
25; CHECK-NEXT:    vctp.32 r3
26; CHECK-NEXT:    vpsel q0, q1, q0
27; CHECK-NEXT:    vaddv.u32 r0, q0
28; CHECK-NEXT:    pop {r7, pc}
29entry:
30  %cmp8 = icmp eq i32 %N, 0
31  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
32
33vector.ph:                                        ; preds = %entry
34  %n.rnd.up = add i32 %N, 3
35  %n.vec = and i32 %n.rnd.up, -4
36  %trip.count.minus.1 = add i32 %N, -1
37  %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
38  %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
39  br label %vector.body
40
41vector.body:                                      ; preds = %vector.body, %vector.ph
42  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
43  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %6, %vector.body ]
44  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
45  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
46  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
47  %0 = getelementptr inbounds i32, i32* %a, i32 %index
48  %1 = icmp ule <4 x i32> %induction, %broadcast.splat12
49  %2 = bitcast i32* %0 to <4 x i32>*
50  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
51  %3 = getelementptr inbounds i32, i32* %b, i32 %index
52  %4 = bitcast i32* %3 to <4 x i32>*
53  %wide.masked.load13 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %4, i32 4, <4 x i1> %1, <4 x i32> undef)
54  %5 = mul nsw <4 x i32> %wide.masked.load13, %wide.masked.load
55  %6 = add nsw <4 x i32> %5, %vec.phi
56  %index.next = add i32 %index, 4
57  %7 = icmp eq i32 %index.next, %n.vec
58  br i1 %7, label %middle.block, label %vector.body
59
60middle.block:                                     ; preds = %vector.body
61  %8 = select <4 x i1> %1, <4 x i32> %6, <4 x i32> %vec.phi
62  %9 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %8)
63  br label %for.cond.cleanup
64
65for.cond.cleanup:                                 ; preds = %middle.block, %entry
66  %res.0.lcssa = phi i32 [ 0, %entry ], [ %9, %middle.block ]
67  ret i32 %res.0.lcssa
68}
69
70define dso_local i32 @mul_reduce_add_const(i32* noalias nocapture readonly %a, i32 %b, i32 %N) {
71; CHECK-LABEL: mul_reduce_add_const:
72; CHECK:       @ %bb.0: @ %entry
73; CHECK-NEXT:    cmp r2, #0
74; CHECK-NEXT:    itt eq
75; CHECK-NEXT:    moveq r0, #0
76; CHECK-NEXT:    bxeq lr
77; CHECK-NEXT:    push {r7, lr}
78; CHECK-NEXT:    vmov.i32 q0, #0x0
79; CHECK-NEXT:    dlstp.32 lr, r2
80; CHECK-NEXT:  .LBB1_1: @ %vector.body
81; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
82; CHECK-NEXT:    mov r1, r2
83; CHECK-NEXT:    vmov q1, q0
84; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
85; CHECK-NEXT:    subs r2, #4
86; CHECK-NEXT:    vadd.i32 q0, q0, q1
87; CHECK-NEXT:    letp lr, .LBB1_1
88; CHECK-NEXT:  @ %bb.2: @ %middle.block
89; CHECK-NEXT:    vctp.32 r1
90; CHECK-NEXT:    vpsel q0, q0, q1
91; CHECK-NEXT:    vaddv.u32 r0, q0
92; CHECK-NEXT:    pop {r7, pc}
93entry:
94  %cmp6 = icmp eq i32 %N, 0
95  br i1 %cmp6, label %for.cond.cleanup, label %vector.ph
96
97vector.ph:                                        ; preds = %entry
98  %n.rnd.up = add i32 %N, 3
99  %n.vec = and i32 %n.rnd.up, -4
100  %trip.count.minus.1 = add i32 %N, -1
101  %broadcast.splatinsert9 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
102  %broadcast.splat10 = shufflevector <4 x i32> %broadcast.splatinsert9, <4 x i32> undef, <4 x i32> zeroinitializer
103  br label %vector.body
104
105vector.body:                                      ; preds = %vector.body, %vector.ph
106  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
107  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %3, %vector.body ]
108  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
109  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
110  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
111  %0 = getelementptr inbounds i32, i32* %a, i32 %index
112  %1 = icmp ule <4 x i32> %induction, %broadcast.splat10
113  %2 = bitcast i32* %0 to <4 x i32>*
114  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
115  %3 = add nsw <4 x i32> %wide.masked.load, %vec.phi
116  %index.next = add i32 %index, 4
117  %4 = icmp eq i32 %index.next, %n.vec
118  br i1 %4, label %middle.block, label %vector.body
119
120middle.block:                                     ; preds = %vector.body
121  %5 = select <4 x i1> %1, <4 x i32> %3, <4 x i32> %vec.phi
122  %6 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %5)
123  br label %for.cond.cleanup
124
125for.cond.cleanup:                                 ; preds = %middle.block, %entry
126  %res.0.lcssa = phi i32 [ 0, %entry ], [ %6, %middle.block ]
127  ret i32 %res.0.lcssa
128}
129
130define dso_local i32 @add_reduce_add_const(i32* noalias nocapture readonly %a, i32 %b, i32 %N) {
131; CHECK-LABEL: add_reduce_add_const:
132; CHECK:       @ %bb.0: @ %entry
133; CHECK-NEXT:    cmp r2, #0
134; CHECK-NEXT:    itt eq
135; CHECK-NEXT:    moveq r0, #0
136; CHECK-NEXT:    bxeq lr
137; CHECK-NEXT:    push {r7, lr}
138; CHECK-NEXT:    vmov.i32 q0, #0x0
139; CHECK-NEXT:    dlstp.32 lr, r2
140; CHECK-NEXT:  .LBB2_1: @ %vector.body
141; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
142; CHECK-NEXT:    mov r1, r2
143; CHECK-NEXT:    vmov q1, q0
144; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
145; CHECK-NEXT:    subs r2, #4
146; CHECK-NEXT:    vadd.i32 q0, q0, q1
147; CHECK-NEXT:    letp lr, .LBB2_1
148; CHECK-NEXT:  @ %bb.2: @ %middle.block
149; CHECK-NEXT:    vctp.32 r1
150; CHECK-NEXT:    vpsel q0, q0, q1
151; CHECK-NEXT:    vaddv.u32 r0, q0
152; CHECK-NEXT:    pop {r7, pc}
153entry:
154  %cmp6 = icmp eq i32 %N, 0
155  br i1 %cmp6, label %for.cond.cleanup, label %vector.ph
156
157vector.ph:                                        ; preds = %entry
158  %n.rnd.up = add i32 %N, 3
159  %n.vec = and i32 %n.rnd.up, -4
160  %trip.count.minus.1 = add i32 %N, -1
161  %broadcast.splatinsert9 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
162  %broadcast.splat10 = shufflevector <4 x i32> %broadcast.splatinsert9, <4 x i32> undef, <4 x i32> zeroinitializer
163  br label %vector.body
164
165vector.body:                                      ; preds = %vector.body, %vector.ph
166  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
167  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %3, %vector.body ]
168  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
169  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
170  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
171  %0 = getelementptr inbounds i32, i32* %a, i32 %index
172  %1 = icmp ule <4 x i32> %induction, %broadcast.splat10
173  %2 = bitcast i32* %0 to <4 x i32>*
174  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
175  %3 = add nsw <4 x i32> %wide.masked.load, %vec.phi
176  %index.next = add i32 %index, 4
177  %4 = icmp eq i32 %index.next, %n.vec
178  br i1 %4, label %middle.block, label %vector.body
179
180middle.block:                                     ; preds = %vector.body
181  %5 = select <4 x i1> %1, <4 x i32> %3, <4 x i32> %vec.phi
182  %6 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %5)
183  br label %for.cond.cleanup
184
185for.cond.cleanup:                                 ; preds = %middle.block, %entry
186  %res.0.lcssa = phi i32 [ 0, %entry ], [ %6, %middle.block ]
187  ret i32 %res.0.lcssa
188}
189
190define dso_local void @vector_mul_const(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32 %c, i32 %N) {
191; CHECK-LABEL: vector_mul_const:
192; CHECK:       @ %bb.0: @ %entry
193; CHECK-NEXT:    push {r7, lr}
194; CHECK-NEXT:    cmp r3, #0
195; CHECK-NEXT:    it eq
196; CHECK-NEXT:    popeq {r7, pc}
197; CHECK-NEXT:    dlstp.32 lr, r3
198; CHECK-NEXT:  .LBB3_1: @ %vector.body
199; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
200; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
201; CHECK-NEXT:    vmul.i32 q0, q0, r2
202; CHECK-NEXT:    vstrw.32 q0, [r0], #16
203; CHECK-NEXT:    letp lr, .LBB3_1
204; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
205; CHECK-NEXT:    pop {r7, pc}
206entry:
207  %cmp6 = icmp eq i32 %N, 0
208  br i1 %cmp6, label %for.cond.cleanup, label %vector.ph
209
210vector.ph:                                        ; preds = %entry
211  %n.rnd.up = add i32 %N, 3
212  %n.vec = and i32 %n.rnd.up, -4
213  %trip.count.minus.1 = add i32 %N, -1
214  %broadcast.splatinsert8 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
215  %broadcast.splat9 = shufflevector <4 x i32> %broadcast.splatinsert8, <4 x i32> undef, <4 x i32> zeroinitializer
216  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %c, i32 0
217  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
218  br label %vector.body
219
220vector.body:                                      ; preds = %vector.body, %vector.ph
221  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
222  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
223  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
224  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
225  %0 = getelementptr inbounds i32, i32* %b, i32 %index
226  %1 = icmp ule <4 x i32> %induction, %broadcast.splat9
227  %2 = bitcast i32* %0 to <4 x i32>*
228  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
229  %3 = mul nsw <4 x i32> %wide.masked.load, %broadcast.splat11
230  %4 = getelementptr inbounds i32, i32* %a, i32 %index
231  %5 = bitcast i32* %4 to <4 x i32>*
232  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %3, <4 x i32>* %5, i32 4, <4 x i1> %1)
233  %index.next = add i32 %index, 4
234  %6 = icmp eq i32 %index.next, %n.vec
235  br i1 %6, label %for.cond.cleanup, label %vector.body
236
237for.cond.cleanup:                                 ; preds = %vector.body, %entry
238  ret void
239}
240
241define dso_local void @vector_add_const(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32 %c, i32 %N) {
242; CHECK-LABEL: vector_add_const:
243; CHECK:       @ %bb.0: @ %entry
244; CHECK-NEXT:    push {r7, lr}
245; CHECK-NEXT:    cmp r3, #0
246; CHECK-NEXT:    it eq
247; CHECK-NEXT:    popeq {r7, pc}
248; CHECK-NEXT:    dlstp.32 lr, r3
249; CHECK-NEXT:  .LBB4_1: @ %vector.body
250; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
251; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
252; CHECK-NEXT:    vadd.i32 q0, q0, r2
253; CHECK-NEXT:    vstrw.32 q0, [r0], #16
254; CHECK-NEXT:    letp lr, .LBB4_1
255; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
256; CHECK-NEXT:    pop {r7, pc}
257entry:
258  %cmp6 = icmp eq i32 %N, 0
259  br i1 %cmp6, label %for.cond.cleanup, label %vector.ph
260
261vector.ph:                                        ; preds = %entry
262  %n.rnd.up = add i32 %N, 3
263  %n.vec = and i32 %n.rnd.up, -4
264  %trip.count.minus.1 = add i32 %N, -1
265  %broadcast.splatinsert8 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
266  %broadcast.splat9 = shufflevector <4 x i32> %broadcast.splatinsert8, <4 x i32> undef, <4 x i32> zeroinitializer
267  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %c, i32 0
268  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
269  br label %vector.body
270
271vector.body:                                      ; preds = %vector.body, %vector.ph
272  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
273  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
274  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
275  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
276  %0 = getelementptr inbounds i32, i32* %b, i32 %index
277  %1 = icmp ule <4 x i32> %induction, %broadcast.splat9
278  %2 = bitcast i32* %0 to <4 x i32>*
279  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
280  %3 = add nsw <4 x i32> %wide.masked.load, %broadcast.splat11
281  %4 = getelementptr inbounds i32, i32* %a, i32 %index
282  %5 = bitcast i32* %4 to <4 x i32>*
283  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %3, <4 x i32>* %5, i32 4, <4 x i1> %1)
284  %index.next = add i32 %index, 4
285  %6 = icmp eq i32 %index.next, %n.vec
286  br i1 %6, label %for.cond.cleanup, label %vector.body
287
288for.cond.cleanup:                                 ; preds = %vector.body, %entry
289  ret void
290}
291
292define dso_local arm_aapcs_vfpcc void @vector_mul_vector_i8(i8* noalias nocapture %a, i8* noalias nocapture readonly %b, i8* noalias nocapture readonly %c, i32 %N) {
293; CHECK-LABEL: vector_mul_vector_i8:
294; CHECK:       @ %bb.0: @ %entry
295; CHECK-NEXT:    push {r4, lr}
296; CHECK-NEXT:    cmp r3, #0
297; CHECK-NEXT:    it eq
298; CHECK-NEXT:    popeq {r4, pc}
299; CHECK-NEXT:    mov.w r12, #0
300; CHECK-NEXT:    dlstp.8 lr, r3
301; CHECK-NEXT:  .LBB5_1: @ %vector.body
302; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
303; CHECK-NEXT:    add.w r4, r1, r12
304; CHECK-NEXT:    vldrb.u8 q0, [r4]
305; CHECK-NEXT:    add.w r4, r2, r12
306; CHECK-NEXT:    vldrb.u8 q1, [r4]
307; CHECK-NEXT:    add.w r4, r0, r12
308; CHECK-NEXT:    add.w r12, r12, #16
309; CHECK-NEXT:    vmul.i8 q0, q1, q0
310; CHECK-NEXT:    vstrb.8 q0, [r4]
311; CHECK-NEXT:    letp lr, .LBB5_1
312; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
313; CHECK-NEXT:    pop {r4, pc}
314entry:
315  %cmp10 = icmp eq i32 %N, 0
316  br i1 %cmp10, label %for.cond.cleanup, label %vector.ph
317
318vector.ph:                                        ; preds = %entry
319  %n.rnd.up = add i32 %N, 15
320  %n.vec = and i32 %n.rnd.up, -16
321  %trip.count.minus.1 = add i32 %N, -1
322  %broadcast.splatinsert12 = insertelement <16 x i32> undef, i32 %trip.count.minus.1, i32 0
323  %broadcast.splat13 = shufflevector <16 x i32> %broadcast.splatinsert12, <16 x i32> undef, <16 x i32> zeroinitializer
324  br label %vector.body
325
326vector.body:                                      ; preds = %vector.body, %vector.ph
327  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
328  %broadcast.splatinsert = insertelement <16 x i32> undef, i32 %index, i32 0
329  %broadcast.splat = shufflevector <16 x i32> %broadcast.splatinsert, <16 x i32> undef, <16 x i32> zeroinitializer
330  %induction = add <16 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
331  %0 = getelementptr inbounds i8, i8* %b, i32 %index
332  %1 = icmp ule <16 x i32> %induction, %broadcast.splat13
333  %2 = bitcast i8* %0 to <16 x i8>*
334  %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %2, i32 1, <16 x i1> %1, <16 x i8> undef)
335  %3 = getelementptr inbounds i8, i8* %c, i32 %index
336  %4 = bitcast i8* %3 to <16 x i8>*
337  %wide.masked.load14 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %4, i32 1, <16 x i1> %1, <16 x i8> undef)
338  %5 = mul <16 x i8> %wide.masked.load14, %wide.masked.load
339  %6 = getelementptr inbounds i8, i8* %a, i32 %index
340  %7 = bitcast i8* %6 to <16 x i8>*
341  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %5, <16 x i8>* %7, i32 1, <16 x i1> %1)
342  %index.next = add i32 %index, 16
343  %8 = icmp eq i32 %index.next, %n.vec
344  br i1 %8, label %for.cond.cleanup, label %vector.body
345
346for.cond.cleanup:                                 ; preds = %vector.body, %entry
347  ret void
348}
349
350; Function Attrs: nofree norecurse nounwind
351define dso_local arm_aapcs_vfpcc void @vector_mul_vector_i16(i16* noalias nocapture %a, i16* noalias nocapture readonly %b, i16* noalias nocapture readonly %c, i32 %N) local_unnamed_addr #0 {
352; CHECK-LABEL: vector_mul_vector_i16:
353; CHECK:       @ %bb.0: @ %entry
354; CHECK-NEXT:    push {r7, lr}
355; CHECK-NEXT:    cmp r3, #0
356; CHECK-NEXT:    it eq
357; CHECK-NEXT:    popeq {r7, pc}
358; CHECK-NEXT:    dlstp.16 lr, r3
359; CHECK-NEXT:  .LBB6_1: @ %vector.body
360; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
361; CHECK-NEXT:    vldrh.u16 q0, [r1], #16
362; CHECK-NEXT:    vldrh.u16 q1, [r2], #16
363; CHECK-NEXT:    vmul.i16 q0, q1, q0
364; CHECK-NEXT:    vstrh.16 q0, [r0], #16
365; CHECK-NEXT:    letp lr, .LBB6_1
366; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
367; CHECK-NEXT:    pop {r7, pc}
368entry:
369  %cmp10 = icmp eq i32 %N, 0
370  br i1 %cmp10, label %for.cond.cleanup, label %vector.ph
371
372vector.ph:                                        ; preds = %entry
373  %n.rnd.up = add i32 %N, 7
374  %n.vec = and i32 %n.rnd.up, -8
375  %trip.count.minus.1 = add i32 %N, -1
376  %broadcast.splatinsert12 = insertelement <8 x i32> undef, i32 %trip.count.minus.1, i32 0
377  %broadcast.splat13 = shufflevector <8 x i32> %broadcast.splatinsert12, <8 x i32> undef, <8 x i32> zeroinitializer
378  br label %vector.body
379
380vector.body:                                      ; preds = %vector.body, %vector.ph
381  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
382  %broadcast.splatinsert = insertelement <8 x i32> undef, i32 %index, i32 0
383  %broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> undef, <8 x i32> zeroinitializer
384  %induction = add <8 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
385  %0 = getelementptr inbounds i16, i16* %b, i32 %index
386  %1 = icmp ule <8 x i32> %induction, %broadcast.splat13
387  %2 = bitcast i16* %0 to <8 x i16>*
388  %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %2, i32 2, <8 x i1> %1, <8 x i16> undef)
389  %3 = getelementptr inbounds i16, i16* %c, i32 %index
390  %4 = bitcast i16* %3 to <8 x i16>*
391  %wide.masked.load14 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %4, i32 2, <8 x i1> %1, <8 x i16> undef)
392  %5 = mul <8 x i16> %wide.masked.load14, %wide.masked.load
393  %6 = getelementptr inbounds i16, i16* %a, i32 %index
394  %7 = bitcast i16* %6 to <8 x i16>*
395  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %5, <8 x i16>* %7, i32 2, <8 x i1> %1)
396  %index.next = add i32 %index, 8
397  %8 = icmp eq i32 %index.next, %n.vec
398  br i1 %8, label %for.cond.cleanup, label %vector.body
399
400for.cond.cleanup:                                 ; preds = %vector.body, %entry
401  ret void
402}
403
404declare <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>*, i32 immarg, <16 x i1>, <16 x i8>)
405declare <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>*, i32 immarg, <8 x i1>, <8 x i16>)
406declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>)
407declare void @llvm.masked.store.v16i8.p0v16i8(<16 x i8>, <16 x i8>*, i32 immarg, <16 x i1>)
408declare void @llvm.masked.store.v8i16.p0v8i16(<8 x i16>, <8 x i16>*, i32 immarg, <8 x i1>)
409declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32 immarg, <4 x i1>)
410declare i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32>)
411
412