1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=armv8.1m.main -mattr=+mve -disable-mve-tail-predication=false --verify-machineinstrs %s -o - | FileCheck %s
3
4define dso_local i32 @mul_reduce_add(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32 %N) {
5; CHECK-LABEL: mul_reduce_add:
6; CHECK:       @ %bb.0: @ %entry
7; CHECK-NEXT:    cmp r2, #0
8; CHECK-NEXT:    itt eq
9; CHECK-NEXT:    moveq r0, #0
10; CHECK-NEXT:    bxeq lr
11; CHECK-NEXT:    push {r7, lr}
12; CHECK-NEXT:    adds r3, r2, #3
13; CHECK-NEXT:    vmov.i32 q0, #0x0
14; CHECK-NEXT:    bic r3, r3, #3
15; CHECK-NEXT:    sub.w r12, r3, #4
16; CHECK-NEXT:    movs r3, #1
17; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
18; CHECK-NEXT:    lsr.w r3, r12, #2
19; CHECK-NEXT:    sub.w r3, r2, r3, lsl #2
20; CHECK-NEXT:    dls lr, lr
21; CHECK-NEXT:  .LBB0_1: @ %vector.body
22; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
23; CHECK-NEXT:    vctp.32 r2
24; CHECK-NEXT:    vmov q1, q0
25; CHECK-NEXT:    vpstt
26; CHECK-NEXT:    vldrwt.u32 q0, [r0], #16
27; CHECK-NEXT:    vldrwt.u32 q2, [r1], #16
28; CHECK-NEXT:    subs r2, #4
29; CHECK-NEXT:    vmul.i32 q0, q2, q0
30; CHECK-NEXT:    vadd.i32 q0, q0, q1
31; CHECK-NEXT:    le lr, .LBB0_1
32; CHECK-NEXT:  @ %bb.2: @ %middle.block
33; CHECK-NEXT:    vctp.32 r3
34; CHECK-NEXT:    vpsel q0, q0, q1
35; CHECK-NEXT:    vaddv.u32 r0, q0
36; CHECK-NEXT:    pop {r7, pc}
37entry:
38  %cmp8 = icmp eq i32 %N, 0
39  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
40
41vector.ph:                                        ; preds = %entry
42  %n.rnd.up = add i32 %N, 3
43  %n.vec = and i32 %n.rnd.up, -4
44  %trip.count.minus.1 = add i32 %N, -1
45  %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
46  %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
47  br label %vector.body
48
49vector.body:                                      ; preds = %vector.body, %vector.ph
50  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
51  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %6, %vector.body ]
52  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
53  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
54  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
55  %0 = getelementptr inbounds i32, i32* %a, i32 %index
56  %1 = icmp ule <4 x i32> %induction, %broadcast.splat12
57  %2 = bitcast i32* %0 to <4 x i32>*
58  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
59  %3 = getelementptr inbounds i32, i32* %b, i32 %index
60  %4 = bitcast i32* %3 to <4 x i32>*
61  %wide.masked.load13 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %4, i32 4, <4 x i1> %1, <4 x i32> undef)
62  %5 = mul nsw <4 x i32> %wide.masked.load13, %wide.masked.load
63  %6 = add nsw <4 x i32> %5, %vec.phi
64  %index.next = add i32 %index, 4
65  %7 = icmp eq i32 %index.next, %n.vec
66  br i1 %7, label %middle.block, label %vector.body
67
68middle.block:                                     ; preds = %vector.body
69  %8 = select <4 x i1> %1, <4 x i32> %6, <4 x i32> %vec.phi
70  %9 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %8)
71  br label %for.cond.cleanup
72
73for.cond.cleanup:                                 ; preds = %middle.block, %entry
74  %res.0.lcssa = phi i32 [ 0, %entry ], [ %9, %middle.block ]
75  ret i32 %res.0.lcssa
76}
77
78define dso_local i32 @mul_reduce_add_const(i32* noalias nocapture readonly %a, i32 %b, i32 %N) {
79; CHECK-LABEL: mul_reduce_add_const:
80; CHECK:       @ %bb.0: @ %entry
81; CHECK-NEXT:    cmp r2, #0
82; CHECK-NEXT:    itt eq
83; CHECK-NEXT:    moveq r0, #0
84; CHECK-NEXT:    bxeq lr
85; CHECK-NEXT:    push {r7, lr}
86; CHECK-NEXT:    adds r1, r2, #3
87; CHECK-NEXT:    movs r3, #1
88; CHECK-NEXT:    bic r1, r1, #3
89; CHECK-NEXT:    vmov.i32 q0, #0x0
90; CHECK-NEXT:    subs r1, #4
91; CHECK-NEXT:    add.w lr, r3, r1, lsr #2
92; CHECK-NEXT:    lsrs r1, r1, #2
93; CHECK-NEXT:    sub.w r1, r2, r1, lsl #2
94; CHECK-NEXT:    dls lr, lr
95; CHECK-NEXT:  .LBB1_1: @ %vector.body
96; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
97; CHECK-NEXT:    vctp.32 r2
98; CHECK-NEXT:    vmov q1, q0
99; CHECK-NEXT:    vpst
100; CHECK-NEXT:    vldrwt.u32 q0, [r0], #16
101; CHECK-NEXT:    subs r2, #4
102; CHECK-NEXT:    vadd.i32 q0, q0, q1
103; CHECK-NEXT:    le lr, .LBB1_1
104; CHECK-NEXT:  @ %bb.2: @ %middle.block
105; CHECK-NEXT:    vctp.32 r1
106; CHECK-NEXT:    vpsel q0, q0, q1
107; CHECK-NEXT:    vaddv.u32 r0, q0
108; CHECK-NEXT:    pop {r7, pc}
109entry:
110  %cmp6 = icmp eq i32 %N, 0
111  br i1 %cmp6, label %for.cond.cleanup, label %vector.ph
112
113vector.ph:                                        ; preds = %entry
114  %n.rnd.up = add i32 %N, 3
115  %n.vec = and i32 %n.rnd.up, -4
116  %trip.count.minus.1 = add i32 %N, -1
117  %broadcast.splatinsert9 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
118  %broadcast.splat10 = shufflevector <4 x i32> %broadcast.splatinsert9, <4 x i32> undef, <4 x i32> zeroinitializer
119  br label %vector.body
120
121vector.body:                                      ; preds = %vector.body, %vector.ph
122  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
123  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %3, %vector.body ]
124  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
125  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
126  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
127  %0 = getelementptr inbounds i32, i32* %a, i32 %index
128  %1 = icmp ule <4 x i32> %induction, %broadcast.splat10
129  %2 = bitcast i32* %0 to <4 x i32>*
130  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
131  %3 = add nsw <4 x i32> %wide.masked.load, %vec.phi
132  %index.next = add i32 %index, 4
133  %4 = icmp eq i32 %index.next, %n.vec
134  br i1 %4, label %middle.block, label %vector.body
135
136middle.block:                                     ; preds = %vector.body
137  %5 = select <4 x i1> %1, <4 x i32> %3, <4 x i32> %vec.phi
138  %6 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %5)
139  br label %for.cond.cleanup
140
141for.cond.cleanup:                                 ; preds = %middle.block, %entry
142  %res.0.lcssa = phi i32 [ 0, %entry ], [ %6, %middle.block ]
143  ret i32 %res.0.lcssa
144}
145
146define dso_local i32 @add_reduce_add_const(i32* noalias nocapture readonly %a, i32 %b, i32 %N) {
147; CHECK-LABEL: add_reduce_add_const:
148; CHECK:       @ %bb.0: @ %entry
149; CHECK-NEXT:    cmp r2, #0
150; CHECK-NEXT:    itt eq
151; CHECK-NEXT:    moveq r0, #0
152; CHECK-NEXT:    bxeq lr
153; CHECK-NEXT:    push {r7, lr}
154; CHECK-NEXT:    adds r1, r2, #3
155; CHECK-NEXT:    movs r3, #1
156; CHECK-NEXT:    bic r1, r1, #3
157; CHECK-NEXT:    vmov.i32 q0, #0x0
158; CHECK-NEXT:    subs r1, #4
159; CHECK-NEXT:    add.w lr, r3, r1, lsr #2
160; CHECK-NEXT:    lsrs r1, r1, #2
161; CHECK-NEXT:    sub.w r1, r2, r1, lsl #2
162; CHECK-NEXT:    dls lr, lr
163; CHECK-NEXT:  .LBB2_1: @ %vector.body
164; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
165; CHECK-NEXT:    vctp.32 r2
166; CHECK-NEXT:    vmov q1, q0
167; CHECK-NEXT:    vpst
168; CHECK-NEXT:    vldrwt.u32 q0, [r0], #16
169; CHECK-NEXT:    subs r2, #4
170; CHECK-NEXT:    vadd.i32 q0, q0, q1
171; CHECK-NEXT:    le lr, .LBB2_1
172; CHECK-NEXT:  @ %bb.2: @ %middle.block
173; CHECK-NEXT:    vctp.32 r1
174; CHECK-NEXT:    vpsel q0, q0, q1
175; CHECK-NEXT:    vaddv.u32 r0, q0
176; CHECK-NEXT:    pop {r7, pc}
177entry:
178  %cmp6 = icmp eq i32 %N, 0
179  br i1 %cmp6, label %for.cond.cleanup, label %vector.ph
180
181vector.ph:                                        ; preds = %entry
182  %n.rnd.up = add i32 %N, 3
183  %n.vec = and i32 %n.rnd.up, -4
184  %trip.count.minus.1 = add i32 %N, -1
185  %broadcast.splatinsert9 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
186  %broadcast.splat10 = shufflevector <4 x i32> %broadcast.splatinsert9, <4 x i32> undef, <4 x i32> zeroinitializer
187  br label %vector.body
188
189vector.body:                                      ; preds = %vector.body, %vector.ph
190  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
191  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %3, %vector.body ]
192  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
193  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
194  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
195  %0 = getelementptr inbounds i32, i32* %a, i32 %index
196  %1 = icmp ule <4 x i32> %induction, %broadcast.splat10
197  %2 = bitcast i32* %0 to <4 x i32>*
198  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
199  %3 = add nsw <4 x i32> %wide.masked.load, %vec.phi
200  %index.next = add i32 %index, 4
201  %4 = icmp eq i32 %index.next, %n.vec
202  br i1 %4, label %middle.block, label %vector.body
203
204middle.block:                                     ; preds = %vector.body
205  %5 = select <4 x i1> %1, <4 x i32> %3, <4 x i32> %vec.phi
206  %6 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %5)
207  br label %for.cond.cleanup
208
209for.cond.cleanup:                                 ; preds = %middle.block, %entry
210  %res.0.lcssa = phi i32 [ 0, %entry ], [ %6, %middle.block ]
211  ret i32 %res.0.lcssa
212}
213
214define dso_local void @vector_mul_const(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32 %c, i32 %N) {
215; CHECK-LABEL: vector_mul_const:
216; CHECK:       @ %bb.0: @ %entry
217; CHECK-NEXT:    push {r7, lr}
218; CHECK-NEXT:    cmp r3, #0
219; CHECK-NEXT:    it eq
220; CHECK-NEXT:    popeq {r7, pc}
221; CHECK-NEXT:    dlstp.32 lr, r3
222; CHECK-NEXT:  .LBB3_1: @ %vector.body
223; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
224; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
225; CHECK-NEXT:    vmul.i32 q0, q0, r2
226; CHECK-NEXT:    vstrw.32 q0, [r0], #16
227; CHECK-NEXT:    letp lr, .LBB3_1
228; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
229; CHECK-NEXT:    pop {r7, pc}
230entry:
231  %cmp6 = icmp eq i32 %N, 0
232  br i1 %cmp6, label %for.cond.cleanup, label %vector.ph
233
234vector.ph:                                        ; preds = %entry
235  %n.rnd.up = add i32 %N, 3
236  %n.vec = and i32 %n.rnd.up, -4
237  %trip.count.minus.1 = add i32 %N, -1
238  %broadcast.splatinsert8 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
239  %broadcast.splat9 = shufflevector <4 x i32> %broadcast.splatinsert8, <4 x i32> undef, <4 x i32> zeroinitializer
240  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %c, i32 0
241  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
242  br label %vector.body
243
244vector.body:                                      ; preds = %vector.body, %vector.ph
245  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
246  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
247  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
248  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
249  %0 = getelementptr inbounds i32, i32* %b, i32 %index
250  %1 = icmp ule <4 x i32> %induction, %broadcast.splat9
251  %2 = bitcast i32* %0 to <4 x i32>*
252  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
253  %3 = mul nsw <4 x i32> %wide.masked.load, %broadcast.splat11
254  %4 = getelementptr inbounds i32, i32* %a, i32 %index
255  %5 = bitcast i32* %4 to <4 x i32>*
256  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %3, <4 x i32>* %5, i32 4, <4 x i1> %1)
257  %index.next = add i32 %index, 4
258  %6 = icmp eq i32 %index.next, %n.vec
259  br i1 %6, label %for.cond.cleanup, label %vector.body
260
261for.cond.cleanup:                                 ; preds = %vector.body, %entry
262  ret void
263}
264
265define dso_local void @vector_add_const(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32 %c, i32 %N) {
266; CHECK-LABEL: vector_add_const:
267; CHECK:       @ %bb.0: @ %entry
268; CHECK-NEXT:    push {r7, lr}
269; CHECK-NEXT:    cmp r3, #0
270; CHECK-NEXT:    it eq
271; CHECK-NEXT:    popeq {r7, pc}
272; CHECK-NEXT:    dlstp.32 lr, r3
273; CHECK-NEXT:  .LBB4_1: @ %vector.body
274; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
275; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
276; CHECK-NEXT:    vadd.i32 q0, q0, r2
277; CHECK-NEXT:    vstrw.32 q0, [r0], #16
278; CHECK-NEXT:    letp lr, .LBB4_1
279; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
280; CHECK-NEXT:    pop {r7, pc}
281entry:
282  %cmp6 = icmp eq i32 %N, 0
283  br i1 %cmp6, label %for.cond.cleanup, label %vector.ph
284
285vector.ph:                                        ; preds = %entry
286  %n.rnd.up = add i32 %N, 3
287  %n.vec = and i32 %n.rnd.up, -4
288  %trip.count.minus.1 = add i32 %N, -1
289  %broadcast.splatinsert8 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
290  %broadcast.splat9 = shufflevector <4 x i32> %broadcast.splatinsert8, <4 x i32> undef, <4 x i32> zeroinitializer
291  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %c, i32 0
292  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
293  br label %vector.body
294
295vector.body:                                      ; preds = %vector.body, %vector.ph
296  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
297  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
298  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
299  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
300  %0 = getelementptr inbounds i32, i32* %b, i32 %index
301  %1 = icmp ule <4 x i32> %induction, %broadcast.splat9
302  %2 = bitcast i32* %0 to <4 x i32>*
303  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
304  %3 = add nsw <4 x i32> %wide.masked.load, %broadcast.splat11
305  %4 = getelementptr inbounds i32, i32* %a, i32 %index
306  %5 = bitcast i32* %4 to <4 x i32>*
307  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %3, <4 x i32>* %5, i32 4, <4 x i1> %1)
308  %index.next = add i32 %index, 4
309  %6 = icmp eq i32 %index.next, %n.vec
310  br i1 %6, label %for.cond.cleanup, label %vector.body
311
312for.cond.cleanup:                                 ; preds = %vector.body, %entry
313  ret void
314}
315
316define dso_local arm_aapcs_vfpcc void @vector_mul_vector_i8(i8* noalias nocapture %a, i8* noalias nocapture readonly %b, i8* noalias nocapture readonly %c, i32 %N) {
317; CHECK-LABEL: vector_mul_vector_i8:
318; CHECK:       @ %bb.0: @ %entry
319; CHECK-NEXT:    push {r7, lr}
320; CHECK-NEXT:    cmp r3, #0
321; CHECK-NEXT:    it eq
322; CHECK-NEXT:    popeq {r7, pc}
323; CHECK-NEXT:    dlstp.8 lr, r3
324; CHECK-NEXT:  .LBB5_1: @ %vector.body
325; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
326; CHECK-NEXT:    vldrb.u8 q0, [r1], #16
327; CHECK-NEXT:    vldrb.u8 q1, [r2], #16
328; CHECK-NEXT:    vmul.i8 q0, q1, q0
329; CHECK-NEXT:    vstrb.8 q0, [r0], #16
330; CHECK-NEXT:    letp lr, .LBB5_1
331; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
332; CHECK-NEXT:    pop {r7, pc}
333entry:
334  %cmp10 = icmp eq i32 %N, 0
335  br i1 %cmp10, label %for.cond.cleanup, label %vector.ph
336
337vector.ph:                                        ; preds = %entry
338  %n.rnd.up = add i32 %N, 15
339  %n.vec = and i32 %n.rnd.up, -16
340  %trip.count.minus.1 = add i32 %N, -1
341  %broadcast.splatinsert12 = insertelement <16 x i32> undef, i32 %trip.count.minus.1, i32 0
342  %broadcast.splat13 = shufflevector <16 x i32> %broadcast.splatinsert12, <16 x i32> undef, <16 x i32> zeroinitializer
343  br label %vector.body
344
345vector.body:                                      ; preds = %vector.body, %vector.ph
346  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
347  %broadcast.splatinsert = insertelement <16 x i32> undef, i32 %index, i32 0
348  %broadcast.splat = shufflevector <16 x i32> %broadcast.splatinsert, <16 x i32> undef, <16 x i32> zeroinitializer
349  %induction = add <16 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
350  %0 = getelementptr inbounds i8, i8* %b, i32 %index
351  %1 = icmp ule <16 x i32> %induction, %broadcast.splat13
352  %2 = bitcast i8* %0 to <16 x i8>*
353  %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %2, i32 1, <16 x i1> %1, <16 x i8> undef)
354  %3 = getelementptr inbounds i8, i8* %c, i32 %index
355  %4 = bitcast i8* %3 to <16 x i8>*
356  %wide.masked.load14 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %4, i32 1, <16 x i1> %1, <16 x i8> undef)
357  %5 = mul <16 x i8> %wide.masked.load14, %wide.masked.load
358  %6 = getelementptr inbounds i8, i8* %a, i32 %index
359  %7 = bitcast i8* %6 to <16 x i8>*
360  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %5, <16 x i8>* %7, i32 1, <16 x i1> %1)
361  %index.next = add i32 %index, 16
362  %8 = icmp eq i32 %index.next, %n.vec
363  br i1 %8, label %for.cond.cleanup, label %vector.body
364
365for.cond.cleanup:                                 ; preds = %vector.body, %entry
366  ret void
367}
368
369; Function Attrs: nofree norecurse nounwind
370define dso_local arm_aapcs_vfpcc void @vector_mul_vector_i16(i16* noalias nocapture %a, i16* noalias nocapture readonly %b, i16* noalias nocapture readonly %c, i32 %N) local_unnamed_addr #0 {
371; CHECK-LABEL: vector_mul_vector_i16:
372; CHECK:       @ %bb.0: @ %entry
373; CHECK-NEXT:    push {r7, lr}
374; CHECK-NEXT:    cmp r3, #0
375; CHECK-NEXT:    it eq
376; CHECK-NEXT:    popeq {r7, pc}
377; CHECK-NEXT:    dlstp.16 lr, r3
378; CHECK-NEXT:  .LBB6_1: @ %vector.body
379; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
380; CHECK-NEXT:    vldrh.u16 q0, [r1], #16
381; CHECK-NEXT:    vldrh.u16 q1, [r2], #16
382; CHECK-NEXT:    vmul.i16 q0, q1, q0
383; CHECK-NEXT:    vstrh.16 q0, [r0], #16
384; CHECK-NEXT:    letp lr, .LBB6_1
385; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
386; CHECK-NEXT:    pop {r7, pc}
387entry:
388  %cmp10 = icmp eq i32 %N, 0
389  br i1 %cmp10, label %for.cond.cleanup, label %vector.ph
390
391vector.ph:                                        ; preds = %entry
392  %n.rnd.up = add i32 %N, 7
393  %n.vec = and i32 %n.rnd.up, -8
394  %trip.count.minus.1 = add i32 %N, -1
395  %broadcast.splatinsert12 = insertelement <8 x i32> undef, i32 %trip.count.minus.1, i32 0
396  %broadcast.splat13 = shufflevector <8 x i32> %broadcast.splatinsert12, <8 x i32> undef, <8 x i32> zeroinitializer
397  br label %vector.body
398
399vector.body:                                      ; preds = %vector.body, %vector.ph
400  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
401  %broadcast.splatinsert = insertelement <8 x i32> undef, i32 %index, i32 0
402  %broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> undef, <8 x i32> zeroinitializer
403  %induction = add <8 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
404  %0 = getelementptr inbounds i16, i16* %b, i32 %index
405  %1 = icmp ule <8 x i32> %induction, %broadcast.splat13
406  %2 = bitcast i16* %0 to <8 x i16>*
407  %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %2, i32 2, <8 x i1> %1, <8 x i16> undef)
408  %3 = getelementptr inbounds i16, i16* %c, i32 %index
409  %4 = bitcast i16* %3 to <8 x i16>*
410  %wide.masked.load14 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %4, i32 2, <8 x i1> %1, <8 x i16> undef)
411  %5 = mul <8 x i16> %wide.masked.load14, %wide.masked.load
412  %6 = getelementptr inbounds i16, i16* %a, i32 %index
413  %7 = bitcast i16* %6 to <8 x i16>*
414  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %5, <8 x i16>* %7, i32 2, <8 x i1> %1)
415  %index.next = add i32 %index, 8
416  %8 = icmp eq i32 %index.next, %n.vec
417  br i1 %8, label %for.cond.cleanup, label %vector.body
418
419for.cond.cleanup:                                 ; preds = %vector.body, %entry
420  ret void
421}
422
423declare <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>*, i32 immarg, <16 x i1>, <16 x i8>)
424declare <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>*, i32 immarg, <8 x i1>, <8 x i16>)
425declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>)
426declare void @llvm.masked.store.v16i8.p0v16i8(<16 x i8>, <16 x i8>*, i32 immarg, <16 x i1>)
427declare void @llvm.masked.store.v8i16.p0v8i16(<8 x i16>, <8 x i16>*, i32 immarg, <8 x i1>)
428declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32 immarg, <4 x i1>)
429declare i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32>)
430
431