1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
2; RUN: opt -loop-vectorize -instcombine -simplifycfg -simplifycfg-require-and-preserve-domtree=1 -tail-predication=enabled < %s -S -o - | FileCheck %s
3
4target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
5target triple = "thumbv8.1m.main-arm-none-eabi"
6
7; Should not be vectorized
8define i64 @add_i64_i64(i64* nocapture readonly %x, i32 %n) #0 {
9; CHECK-LABEL: @add_i64_i64(
10; CHECK-NEXT:  entry:
11; CHECK-NEXT:    [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0
12; CHECK-NEXT:    br i1 [[CMP6]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
13; CHECK:       for.body:
14; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
15; CHECK-NEXT:    [[R_07:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
16; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, i64* [[X:%.*]], i32 [[I_08]]
17; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[ARRAYIDX]], align 8
18; CHECK-NEXT:    [[ADD]] = add nsw i64 [[TMP0]], [[R_07]]
19; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
20; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
21; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]]
22; CHECK:       for.cond.cleanup:
23; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[ADD]], [[FOR_BODY]] ]
24; CHECK-NEXT:    ret i64 [[R_0_LCSSA]]
25;
26entry:
27  %cmp6 = icmp sgt i32 %n, 0
28  br i1 %cmp6, label %for.body, label %for.cond.cleanup
29
30for.body:                                         ; preds = %entry, %for.body
31  %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
32  %r.07 = phi i64 [ %add, %for.body ], [ 0, %entry ]
33  %arrayidx = getelementptr inbounds i64, i64* %x, i32 %i.08
34  %0 = load i64, i64* %arrayidx, align 8
35  %add = add nsw i64 %0, %r.07
36  %inc = add nuw nsw i32 %i.08, 1
37  %exitcond = icmp eq i32 %inc, %n
38  br i1 %exitcond, label %for.cond.cleanup, label %for.body
39
40for.cond.cleanup:                                 ; preds = %for.body, %entry
41  %r.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
42  ret i64 %r.0.lcssa
43}
44
45; 4x to use VADDLV
46; FIXME: TailPredicate
47define i64 @add_i32_i64(i32* nocapture readonly %x, i32 %n) #0 {
48; CHECK-LABEL: @add_i32_i64(
49; CHECK-NEXT:  entry:
50; CHECK-NEXT:    [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0
51; CHECK-NEXT:    br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
52; CHECK:       for.body.preheader:
53; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
54; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
55; CHECK:       vector.ph:
56; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[N]], -4
57; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
58; CHECK:       vector.body:
59; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
60; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
61; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i32 [[INDEX]]
62; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[TMP0]] to <4 x i32>*
63; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
64; CHECK-NEXT:    [[TMP2:%.*]] = sext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
65; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP2]])
66; CHECK-NEXT:    [[TMP4]] = add i64 [[TMP3]], [[VEC_PHI]]
67; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
68; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
69; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
70; CHECK:       middle.block:
71; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N_VEC]], [[N]]
72; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]
73; CHECK:       scalar.ph:
74; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
75; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP4]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
76; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
77; CHECK:       for.body:
78; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
79; CHECK-NEXT:    [[R_07:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
80; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[I_08]]
81; CHECK-NEXT:    [[TMP6:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
82; CHECK-NEXT:    [[CONV:%.*]] = sext i32 [[TMP6]] to i64
83; CHECK-NEXT:    [[ADD]] = add nsw i64 [[R_07]], [[CONV]]
84; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
85; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
86; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
87; CHECK:       for.cond.cleanup:
88; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD]], [[FOR_BODY]] ], [ [[TMP4]], [[MIDDLE_BLOCK]] ]
89; CHECK-NEXT:    ret i64 [[R_0_LCSSA]]
90;
91entry:
92  %cmp6 = icmp sgt i32 %n, 0
93  br i1 %cmp6, label %for.body, label %for.cond.cleanup
94
95for.body:                                         ; preds = %entry, %for.body
96  %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
97  %r.07 = phi i64 [ %add, %for.body ], [ 0, %entry ]
98  %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
99  %0 = load i32, i32* %arrayidx, align 4
100  %conv = sext i32 %0 to i64
101  %add = add nsw i64 %r.07, %conv
102  %inc = add nuw nsw i32 %i.08, 1
103  %exitcond = icmp eq i32 %inc, %n
104  br i1 %exitcond, label %for.cond.cleanup, label %for.body
105
106for.cond.cleanup:                                 ; preds = %for.body, %entry
107  %r.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
108  ret i64 %r.0.lcssa
109}
110
111; 4x to use VADDLV
112; FIXME: TailPredicate
113define i64 @add_i16_i64(i16* nocapture readonly %x, i32 %n) #0 {
114; CHECK-LABEL: @add_i16_i64(
115; CHECK-NEXT:  entry:
116; CHECK-NEXT:    [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0
117; CHECK-NEXT:    br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
118; CHECK:       for.body.preheader:
119; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
120; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
121; CHECK:       vector.ph:
122; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[N]], -4
123; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
124; CHECK:       vector.body:
125; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
126; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
127; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i16, i16* [[X:%.*]], i32 [[INDEX]]
128; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16* [[TMP0]] to <4 x i16>*
129; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 2
130; CHECK-NEXT:    [[TMP2:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i64>
131; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP2]])
132; CHECK-NEXT:    [[TMP4]] = add i64 [[TMP3]], [[VEC_PHI]]
133; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
134; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
135; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
136; CHECK:       middle.block:
137; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N_VEC]], [[N]]
138; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]
139; CHECK:       scalar.ph:
140; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
141; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP4]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
142; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
143; CHECK:       for.body:
144; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
145; CHECK-NEXT:    [[R_07:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
146; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, i16* [[X]], i32 [[I_08]]
147; CHECK-NEXT:    [[TMP6:%.*]] = load i16, i16* [[ARRAYIDX]], align 2
148; CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[TMP6]] to i64
149; CHECK-NEXT:    [[ADD]] = add nsw i64 [[R_07]], [[CONV]]
150; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
151; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
152; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
153; CHECK:       for.cond.cleanup:
154; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD]], [[FOR_BODY]] ], [ [[TMP4]], [[MIDDLE_BLOCK]] ]
155; CHECK-NEXT:    ret i64 [[R_0_LCSSA]]
156;
157entry:
158  %cmp6 = icmp sgt i32 %n, 0
159  br i1 %cmp6, label %for.body, label %for.cond.cleanup
160
161for.body:                                         ; preds = %entry, %for.body
162  %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
163  %r.07 = phi i64 [ %add, %for.body ], [ 0, %entry ]
164  %arrayidx = getelementptr inbounds i16, i16* %x, i32 %i.08
165  %0 = load i16, i16* %arrayidx, align 2
166  %conv = sext i16 %0 to i64
167  %add = add nsw i64 %r.07, %conv
168  %inc = add nuw nsw i32 %i.08, 1
169  %exitcond = icmp eq i32 %inc, %n
170  br i1 %exitcond, label %for.cond.cleanup, label %for.body
171
172for.cond.cleanup:                                 ; preds = %for.body, %entry
173  %r.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
174  ret i64 %r.0.lcssa
175}
176
177; 4x to use VADDLV
178; FIXME: TailPredicate
179define i64 @add_i8_i64(i8* nocapture readonly %x, i32 %n) #0 {
180; CHECK-LABEL: @add_i8_i64(
181; CHECK-NEXT:  entry:
182; CHECK-NEXT:    [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0
183; CHECK-NEXT:    br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
184; CHECK:       for.body.preheader:
185; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
186; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
187; CHECK:       vector.ph:
188; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[N]], -4
189; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
190; CHECK:       vector.body:
191; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
192; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
193; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[X:%.*]], i32 [[INDEX]]
194; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[TMP0]] to <4 x i8>*
195; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
196; CHECK-NEXT:    [[TMP2:%.*]] = zext <4 x i8> [[WIDE_LOAD]] to <4 x i64>
197; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP2]])
198; CHECK-NEXT:    [[TMP4]] = add i64 [[TMP3]], [[VEC_PHI]]
199; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
200; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
201; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
202; CHECK:       middle.block:
203; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N_VEC]], [[N]]
204; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]
205; CHECK:       scalar.ph:
206; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
207; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP4]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
208; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
209; CHECK:       for.body:
210; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
211; CHECK-NEXT:    [[R_07:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
212; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, i8* [[X]], i32 [[I_08]]
213; CHECK-NEXT:    [[TMP6:%.*]] = load i8, i8* [[ARRAYIDX]], align 1
214; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP6]] to i64
215; CHECK-NEXT:    [[ADD]] = add nuw nsw i64 [[R_07]], [[CONV]]
216; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
217; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
218; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
219; CHECK:       for.cond.cleanup:
220; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD]], [[FOR_BODY]] ], [ [[TMP4]], [[MIDDLE_BLOCK]] ]
221; CHECK-NEXT:    ret i64 [[R_0_LCSSA]]
222;
223entry:
224  %cmp6 = icmp sgt i32 %n, 0
225  br i1 %cmp6, label %for.body, label %for.cond.cleanup
226
227for.body:                                         ; preds = %entry, %for.body
228  %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
229  %r.07 = phi i64 [ %add, %for.body ], [ 0, %entry ]
230  %arrayidx = getelementptr inbounds i8, i8* %x, i32 %i.08
231  %0 = load i8, i8* %arrayidx, align 1
232  %conv = zext i8 %0 to i64
233  %add = add nuw nsw i64 %r.07, %conv
234  %inc = add nuw nsw i32 %i.08, 1
235  %exitcond = icmp eq i32 %inc, %n
236  br i1 %exitcond, label %for.cond.cleanup, label %for.body
237
238for.cond.cleanup:                                 ; preds = %for.body, %entry
239  %r.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
240  ret i64 %r.0.lcssa
241}
242
243; 4x to use VADDV.u32
244define i32 @add_i32_i32(i32* nocapture readonly %x, i32 %n) #0 {
245; CHECK-LABEL: @add_i32_i32(
246; CHECK-NEXT:  entry:
247; CHECK-NEXT:    [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0
248; CHECK-NEXT:    br i1 [[CMP6]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
249; CHECK:       vector.ph:
250; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 3
251; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -4
252; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
253; CHECK:       vector.body:
254; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
255; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
256; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 [[N]])
257; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i32 [[INDEX]]
258; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[TMP0]] to <4 x i32>*
259; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP1]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)
260; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> [[WIDE_MASKED_LOAD]], <4 x i32> zeroinitializer
261; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP2]])
262; CHECK-NEXT:    [[TMP4]] = add i32 [[TMP3]], [[VEC_PHI]]
263; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
264; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
265; CHECK-NEXT:    br i1 [[TMP5]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
266; CHECK:       for.cond.cleanup:
267; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP4]], [[VECTOR_BODY]] ]
268; CHECK-NEXT:    ret i32 [[R_0_LCSSA]]
269;
270entry:
271  %cmp6 = icmp sgt i32 %n, 0
272  br i1 %cmp6, label %for.body, label %for.cond.cleanup
273
274for.body:                                         ; preds = %entry, %for.body
275  %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
276  %r.07 = phi i32 [ %add, %for.body ], [ 0, %entry ]
277  %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
278  %0 = load i32, i32* %arrayidx, align 4
279  %add = add nsw i32 %0, %r.07
280  %inc = add nuw nsw i32 %i.08, 1
281  %exitcond = icmp eq i32 %inc, %n
282  br i1 %exitcond, label %for.cond.cleanup, label %for.body
283
284for.cond.cleanup:                                 ; preds = %for.body, %entry
285  %r.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
286  ret i32 %r.0.lcssa
287}
288
289; 8x to use VADDV.u16
290define i32 @add_i16_i32(i16* nocapture readonly %x, i32 %n) #0 {
291; CHECK-LABEL: @add_i16_i32(
292; CHECK-NEXT:  entry:
293; CHECK-NEXT:    [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0
294; CHECK-NEXT:    br i1 [[CMP6]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
295; CHECK:       vector.ph:
296; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 7
297; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8
298; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
299; CHECK:       vector.body:
300; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
301; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
302; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX]], i32 [[N]])
303; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i16, i16* [[X:%.*]], i32 [[INDEX]]
304; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16* [[TMP0]] to <8 x i16>*
305; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* [[TMP1]], i32 2, <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> poison)
306; CHECK-NEXT:    [[TMP2:%.*]] = sext <8 x i16> [[WIDE_MASKED_LOAD]] to <8 x i32>
307; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i32> [[TMP2]], <8 x i32> zeroinitializer
308; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP3]])
309; CHECK-NEXT:    [[TMP5]] = add i32 [[TMP4]], [[VEC_PHI]]
310; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
311; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
312; CHECK-NEXT:    br i1 [[TMP6]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
313; CHECK:       for.cond.cleanup:
314; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP5]], [[VECTOR_BODY]] ]
315; CHECK-NEXT:    ret i32 [[R_0_LCSSA]]
316;
317entry:
318  %cmp6 = icmp sgt i32 %n, 0
319  br i1 %cmp6, label %for.body, label %for.cond.cleanup
320
321for.body:                                         ; preds = %entry, %for.body
322  %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
323  %r.07 = phi i32 [ %add, %for.body ], [ 0, %entry ]
324  %arrayidx = getelementptr inbounds i16, i16* %x, i32 %i.08
325  %0 = load i16, i16* %arrayidx, align 2
326  %conv = sext i16 %0 to i32
327  %add = add nsw i32 %r.07, %conv
328  %inc = add nuw nsw i32 %i.08, 1
329  %exitcond = icmp eq i32 %inc, %n
330  br i1 %exitcond, label %for.cond.cleanup, label %for.body
331
332for.cond.cleanup:                                 ; preds = %for.body, %entry
333  %r.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
334  ret i32 %r.0.lcssa
335}
336
337; 16x to use VADDV.u16
338define i32 @add_i8_i32(i8* nocapture readonly %x, i32 %n) #0 {
339; CHECK-LABEL: @add_i8_i32(
340; CHECK-NEXT:  entry:
341; CHECK-NEXT:    [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0
342; CHECK-NEXT:    br i1 [[CMP6]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
343; CHECK:       vector.ph:
344; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 15
345; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -16
346; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
347; CHECK:       vector.body:
348; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
349; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
350; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])
351; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[X:%.*]], i32 [[INDEX]]
352; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[TMP0]] to <16 x i8>*
353; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* [[TMP1]], i32 1, <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
354; CHECK-NEXT:    [[TMP2:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD]] to <16 x i32>
355; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i32> [[TMP2]], <16 x i32> zeroinitializer
356; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP3]])
357; CHECK-NEXT:    [[TMP5]] = add i32 [[TMP4]], [[VEC_PHI]]
358; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 16
359; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
360; CHECK-NEXT:    br i1 [[TMP6]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
361; CHECK:       for.cond.cleanup:
362; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP5]], [[VECTOR_BODY]] ]
363; CHECK-NEXT:    ret i32 [[R_0_LCSSA]]
364;
365entry:
366  %cmp6 = icmp sgt i32 %n, 0
367  br i1 %cmp6, label %for.body, label %for.cond.cleanup
368
369for.body:                                         ; preds = %entry, %for.body
370  %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
371  %r.07 = phi i32 [ %add, %for.body ], [ 0, %entry ]
372  %arrayidx = getelementptr inbounds i8, i8* %x, i32 %i.08
373  %0 = load i8, i8* %arrayidx, align 1
374  %conv = zext i8 %0 to i32
375  %add = add nuw nsw i32 %r.07, %conv
376  %inc = add nuw nsw i32 %i.08, 1
377  %exitcond = icmp eq i32 %inc, %n
378  br i1 %exitcond, label %for.cond.cleanup, label %for.body
379
380for.cond.cleanup:                                 ; preds = %for.body, %entry
381  %r.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
382  ret i32 %r.0.lcssa
383}
384
385; 8x to use VADDV.u16
386define signext i16 @add_i16_i16(i16* nocapture readonly %x, i32 %n) #0 {
387; CHECK-LABEL: @add_i16_i16(
388; CHECK-NEXT:  entry:
389; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[N:%.*]], 0
390; CHECK-NEXT:    br i1 [[CMP8]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
391; CHECK:       vector.ph:
392; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 7
393; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8
394; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
395; CHECK:       vector.body:
396; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
397; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i16 [ 0, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
398; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX]], i32 [[N]])
399; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i16, i16* [[X:%.*]], i32 [[INDEX]]
400; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16* [[TMP0]] to <8 x i16>*
401; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* [[TMP1]], i32 2, <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> poison)
402; CHECK-NEXT:    [[TMP2:%.*]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> [[WIDE_MASKED_LOAD]], <8 x i16> zeroinitializer
403; CHECK-NEXT:    [[TMP3:%.*]] = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> [[TMP2]])
404; CHECK-NEXT:    [[TMP4]] = add i16 [[TMP3]], [[VEC_PHI]]
405; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
406; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
407; CHECK-NEXT:    br i1 [[TMP5]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
408; CHECK:       for.cond.cleanup:
409; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[TMP4]], [[VECTOR_BODY]] ]
410; CHECK-NEXT:    ret i16 [[R_0_LCSSA]]
411;
412entry:
413  %cmp8 = icmp sgt i32 %n, 0
414  br i1 %cmp8, label %for.body, label %for.cond.cleanup
415
416for.body:                                         ; preds = %entry, %for.body
417  %i.010 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
418  %r.09 = phi i16 [ %add, %for.body ], [ 0, %entry ]
419  %arrayidx = getelementptr inbounds i16, i16* %x, i32 %i.010
420  %0 = load i16, i16* %arrayidx, align 2
421  %add = add i16 %0, %r.09
422  %inc = add nuw nsw i32 %i.010, 1
423  %exitcond = icmp eq i32 %inc, %n
424  br i1 %exitcond, label %for.cond.cleanup, label %for.body
425
426for.cond.cleanup:                                 ; preds = %for.body, %entry
427  %r.0.lcssa = phi i16 [ 0, %entry ], [ %add, %for.body ]
428  ret i16 %r.0.lcssa
429}
430
431; 16x to use VADDV.u8
432define signext i16 @add_i8_i16(i8* nocapture readonly %x, i32 %n) #0 {
433; CHECK-LABEL: @add_i8_i16(
434; CHECK-NEXT:  entry:
435; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[N:%.*]], 0
436; CHECK-NEXT:    br i1 [[CMP8]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
437; CHECK:       vector.ph:
438; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 15
439; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -16
440; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
441; CHECK:       vector.body:
442; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
443; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i16 [ 0, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
444; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])
445; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[X:%.*]], i32 [[INDEX]]
446; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[TMP0]] to <16 x i8>*
447; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* [[TMP1]], i32 1, <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
448; CHECK-NEXT:    [[TMP2:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD]] to <16 x i16>
449; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i16> [[TMP2]], <16 x i16> zeroinitializer
450; CHECK-NEXT:    [[TMP4:%.*]] = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> [[TMP3]])
451; CHECK-NEXT:    [[TMP5]] = add i16 [[TMP4]], [[VEC_PHI]]
452; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 16
453; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
454; CHECK-NEXT:    br i1 [[TMP6]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
455; CHECK:       for.cond.cleanup:
456; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[TMP5]], [[VECTOR_BODY]] ]
457; CHECK-NEXT:    ret i16 [[R_0_LCSSA]]
458;
459entry:
460  %cmp8 = icmp sgt i32 %n, 0
461  br i1 %cmp8, label %for.body, label %for.cond.cleanup
462
463for.body:                                         ; preds = %entry, %for.body
464  %i.010 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
465  %r.09 = phi i16 [ %add, %for.body ], [ 0, %entry ]
466  %arrayidx = getelementptr inbounds i8, i8* %x, i32 %i.010
467  %0 = load i8, i8* %arrayidx, align 1
468  %conv = zext i8 %0 to i16
469  %add = add i16 %r.09, %conv
470  %inc = add nuw nsw i32 %i.010, 1
471  %exitcond = icmp eq i32 %inc, %n
472  br i1 %exitcond, label %for.cond.cleanup, label %for.body
473
474for.cond.cleanup:                                 ; preds = %for.body, %entry
475  %r.0.lcssa = phi i16 [ 0, %entry ], [ %add, %for.body ]
476  ret i16 %r.0.lcssa
477}
478
479; 16x to use VADDV.u8
480define zeroext i8 @add_i8_i8(i8* nocapture readonly %x, i32 %n) #0 {
481; CHECK-LABEL: @add_i8_i8(
482; CHECK-NEXT:  entry:
483; CHECK-NEXT:    [[CMP7:%.*]] = icmp sgt i32 [[N:%.*]], 0
484; CHECK-NEXT:    br i1 [[CMP7]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
485; CHECK:       vector.ph:
486; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 15
487; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -16
488; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
489; CHECK:       vector.body:
490; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
491; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i8 [ 0, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
492; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])
493; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[X:%.*]], i32 [[INDEX]]
494; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[TMP0]] to <16 x i8>*
495; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* [[TMP1]], i32 1, <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
496; CHECK-NEXT:    [[TMP2:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> [[WIDE_MASKED_LOAD]], <16 x i8> zeroinitializer
497; CHECK-NEXT:    [[TMP3:%.*]] = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> [[TMP2]])
498; CHECK-NEXT:    [[TMP4]] = add i8 [[TMP3]], [[VEC_PHI]]
499; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 16
500; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
501; CHECK-NEXT:    br i1 [[TMP5]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
502; CHECK:       for.cond.cleanup:
503; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i8 [ 0, [[ENTRY:%.*]] ], [ [[TMP4]], [[VECTOR_BODY]] ]
504; CHECK-NEXT:    ret i8 [[R_0_LCSSA]]
505;
506entry:
507  %cmp7 = icmp sgt i32 %n, 0
508  br i1 %cmp7, label %for.body, label %for.cond.cleanup
509
510for.body:                                         ; preds = %entry, %for.body
511  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
512  %r.08 = phi i8 [ %add, %for.body ], [ 0, %entry ]
513  %arrayidx = getelementptr inbounds i8, i8* %x, i32 %i.09
514  %0 = load i8, i8* %arrayidx, align 1
515  %add = add i8 %0, %r.08
516  %inc = add nuw nsw i32 %i.09, 1
517  %exitcond = icmp eq i32 %inc, %n
518  br i1 %exitcond, label %for.cond.cleanup, label %for.body
519
520for.cond.cleanup:                                 ; preds = %for.body, %entry
521  %r.0.lcssa = phi i8 [ 0, %entry ], [ %add, %for.body ]
522  ret i8 %r.0.lcssa
523}
524
525; Not vectorized
526define i64 @mla_i64_i64(i64* nocapture readonly %x, i64* nocapture readonly %y, i32 %n) #0 {
527; CHECK-LABEL: @mla_i64_i64(
528; CHECK-NEXT:  entry:
529; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[N:%.*]], 0
530; CHECK-NEXT:    br i1 [[CMP8]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
531; CHECK:       for.body:
532; CHECK-NEXT:    [[I_010:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
533; CHECK-NEXT:    [[R_09:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
534; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, i64* [[X:%.*]], i32 [[I_010]]
535; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[ARRAYIDX]], align 8
536; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, i64* [[Y:%.*]], i32 [[I_010]]
537; CHECK-NEXT:    [[TMP1:%.*]] = load i64, i64* [[ARRAYIDX1]], align 8
538; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP1]], [[TMP0]]
539; CHECK-NEXT:    [[ADD]] = add nsw i64 [[MUL]], [[R_09]]
540; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_010]], 1
541; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
542; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]]
543; CHECK:       for.cond.cleanup:
544; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[ADD]], [[FOR_BODY]] ]
545; CHECK-NEXT:    ret i64 [[R_0_LCSSA]]
546;
547entry:
548  %cmp8 = icmp sgt i32 %n, 0
549  br i1 %cmp8, label %for.body, label %for.cond.cleanup
550
551for.body:                                         ; preds = %entry, %for.body
552  %i.010 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
553  %r.09 = phi i64 [ %add, %for.body ], [ 0, %entry ]
554  %arrayidx = getelementptr inbounds i64, i64* %x, i32 %i.010
555  %0 = load i64, i64* %arrayidx, align 8
556  %arrayidx1 = getelementptr inbounds i64, i64* %y, i32 %i.010
557  %1 = load i64, i64* %arrayidx1, align 8
558  %mul = mul nsw i64 %1, %0
559  %add = add nsw i64 %mul, %r.09
560  %inc = add nuw nsw i32 %i.010, 1
561  %exitcond = icmp eq i32 %inc, %n
562  br i1 %exitcond, label %for.cond.cleanup, label %for.body
563
564for.cond.cleanup:                                 ; preds = %for.body, %entry
565  %r.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
566  ret i64 %r.0.lcssa
567}
568
569; 4x to use VMLAL.u32
570; FIXME: TailPredicate
571define i64 @mla_i32_i64(i32* nocapture readonly %x, i32* nocapture readonly %y, i32 %n) #0 {
572; CHECK-LABEL: @mla_i32_i64(
573; CHECK-NEXT:  entry:
574; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[N:%.*]], 0
575; CHECK-NEXT:    br i1 [[CMP8]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
576; CHECK:       for.body.preheader:
577; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
578; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
579; CHECK:       vector.ph:
580; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[N]], -4
581; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
582; CHECK:       vector.body:
583; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
584; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
585; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i32 [[INDEX]]
586; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[TMP0]] to <4 x i32>*
587; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
588; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[Y:%.*]], i32 [[INDEX]]
589; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <4 x i32>*
590; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP3]], align 4
591; CHECK-NEXT:    [[TMP4:%.*]] = mul nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]
592; CHECK-NEXT:    [[TMP5:%.*]] = sext <4 x i32> [[TMP4]] to <4 x i64>
593; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP5]])
594; CHECK-NEXT:    [[TMP7]] = add i64 [[TMP6]], [[VEC_PHI]]
595; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
596; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
597; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
598; CHECK:       middle.block:
599; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N_VEC]], [[N]]
600; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]
601; CHECK:       scalar.ph:
602; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
603; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
604; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
605; CHECK:       for.body:
606; CHECK-NEXT:    [[I_010:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
607; CHECK-NEXT:    [[R_09:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
608; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[I_010]]
609; CHECK-NEXT:    [[TMP9:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
610; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[Y]], i32 [[I_010]]
611; CHECK-NEXT:    [[TMP10:%.*]] = load i32, i32* [[ARRAYIDX1]], align 4
612; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP10]], [[TMP9]]
613; CHECK-NEXT:    [[CONV:%.*]] = sext i32 [[MUL]] to i64
614; CHECK-NEXT:    [[ADD]] = add nsw i64 [[R_09]], [[CONV]]
615; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_010]], 1
616; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
617; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
618; CHECK:       for.cond.cleanup:
619; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD]], [[FOR_BODY]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
620; CHECK-NEXT:    ret i64 [[R_0_LCSSA]]
621;
622entry:
623  %cmp8 = icmp sgt i32 %n, 0
624  br i1 %cmp8, label %for.body, label %for.cond.cleanup
625
626for.body:                                         ; preds = %entry, %for.body
627  %i.010 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
628  %r.09 = phi i64 [ %add, %for.body ], [ 0, %entry ]
629  %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.010
630  %0 = load i32, i32* %arrayidx, align 4
631  %arrayidx1 = getelementptr inbounds i32, i32* %y, i32 %i.010
632  %1 = load i32, i32* %arrayidx1, align 4
633  %mul = mul nsw i32 %1, %0
634  %conv = sext i32 %mul to i64
635  %add = add nsw i64 %r.09, %conv
636  %inc = add nuw nsw i32 %i.010, 1
637  %exitcond = icmp eq i32 %inc, %n
638  br i1 %exitcond, label %for.cond.cleanup, label %for.body
639
640for.cond.cleanup:                                 ; preds = %for.body, %entry
641  %r.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
642  ret i64 %r.0.lcssa
643}
644
645; 8x to use VMLAL.u16
646; FIXME: TailPredicate
647define i64 @mla_i16_i64(i16* nocapture readonly %x, i16* nocapture readonly %y, i32 %n) #0 {
648; CHECK-LABEL: @mla_i16_i64(
649; CHECK-NEXT:  entry:
650; CHECK-NEXT:    [[CMP10:%.*]] = icmp sgt i32 [[N:%.*]], 0
651; CHECK-NEXT:    br i1 [[CMP10]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
652; CHECK:       for.body.preheader:
653; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 8
654; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
655; CHECK:       vector.ph:
656; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[N]], -8
657; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
658; CHECK:       vector.body:
659; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
660; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
661; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i16, i16* [[X:%.*]], i32 [[INDEX]]
662; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16* [[TMP0]] to <8 x i16>*
663; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i16>, <8 x i16>* [[TMP1]], align 2
664; CHECK-NEXT:    [[TMP2:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i32>
665; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i16, i16* [[Y:%.*]], i32 [[INDEX]]
666; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16* [[TMP3]] to <8 x i16>*
667; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <8 x i16>, <8 x i16>* [[TMP4]], align 2
668; CHECK-NEXT:    [[TMP5:%.*]] = sext <8 x i16> [[WIDE_LOAD1]] to <8 x i32>
669; CHECK-NEXT:    [[TMP6:%.*]] = mul nsw <8 x i32> [[TMP5]], [[TMP2]]
670; CHECK-NEXT:    [[TMP7:%.*]] = sext <8 x i32> [[TMP6]] to <8 x i64>
671; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP7]])
672; CHECK-NEXT:    [[TMP9]] = add i64 [[TMP8]], [[VEC_PHI]]
673; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
674; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
675; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
676; CHECK:       middle.block:
677; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N_VEC]], [[N]]
678; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]
679; CHECK:       scalar.ph:
680; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
681; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
682; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
683; CHECK:       for.body:
684; CHECK-NEXT:    [[I_012:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
685; CHECK-NEXT:    [[R_011:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
686; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, i16* [[X]], i32 [[I_012]]
687; CHECK-NEXT:    [[TMP11:%.*]] = load i16, i16* [[ARRAYIDX]], align 2
688; CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[TMP11]] to i32
689; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i16, i16* [[Y]], i32 [[I_012]]
690; CHECK-NEXT:    [[TMP12:%.*]] = load i16, i16* [[ARRAYIDX1]], align 2
691; CHECK-NEXT:    [[CONV2:%.*]] = sext i16 [[TMP12]] to i32
692; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]]
693; CHECK-NEXT:    [[CONV3:%.*]] = sext i32 [[MUL]] to i64
694; CHECK-NEXT:    [[ADD]] = add nsw i64 [[R_011]], [[CONV3]]
695; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_012]], 1
696; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
697; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
698; CHECK:       for.cond.cleanup:
699; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD]], [[FOR_BODY]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]
700; CHECK-NEXT:    ret i64 [[R_0_LCSSA]]
701;
702entry:
703  %cmp10 = icmp sgt i32 %n, 0
704  br i1 %cmp10, label %for.body, label %for.cond.cleanup
705
706for.body:                                         ; preds = %entry, %for.body
707  %i.012 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
708  %r.011 = phi i64 [ %add, %for.body ], [ 0, %entry ]
709  %arrayidx = getelementptr inbounds i16, i16* %x, i32 %i.012
710  %0 = load i16, i16* %arrayidx, align 2
711  %conv = sext i16 %0 to i32
712  %arrayidx1 = getelementptr inbounds i16, i16* %y, i32 %i.012
713  %1 = load i16, i16* %arrayidx1, align 2
714  %conv2 = sext i16 %1 to i32
715  %mul = mul nsw i32 %conv2, %conv
716  %conv3 = sext i32 %mul to i64
717  %add = add nsw i64 %r.011, %conv3
718  %inc = add nuw nsw i32 %i.012, 1
719  %exitcond = icmp eq i32 %inc, %n
720  br i1 %exitcond, label %for.cond.cleanup, label %for.body
721
722for.cond.cleanup:                                 ; preds = %for.body, %entry
723  %r.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
724  ret i64 %r.0.lcssa
725}
726
727; 8x to use VMLAL.u16
728; FIXME: 8x, TailPredicate, double-extended
729define i64 @mla_i8_i64(i8* nocapture readonly %x, i8* nocapture readonly %y, i32 %n) #0 {
730; CHECK-LABEL: @mla_i8_i64(
731; CHECK-NEXT:  entry:
732; CHECK-NEXT:    [[CMP10:%.*]] = icmp sgt i32 [[N:%.*]], 0
733; CHECK-NEXT:    br i1 [[CMP10]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
734; CHECK:       for.body.preheader:
735; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 16
736; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
737; CHECK:       vector.ph:
738; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[N]], -16
739; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
740; CHECK:       vector.body:
741; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
742; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
743; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[X:%.*]], i32 [[INDEX]]
744; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[TMP0]] to <16 x i8>*
745; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, <16 x i8>* [[TMP1]], align 1
746; CHECK-NEXT:    [[TMP2:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
747; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, i8* [[Y:%.*]], i32 [[INDEX]]
748; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8* [[TMP3]] to <16 x i8>*
749; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, <16 x i8>* [[TMP4]], align 1
750; CHECK-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
751; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw nsw <16 x i32> [[TMP5]], [[TMP2]]
752; CHECK-NEXT:    [[TMP7:%.*]] = zext <16 x i32> [[TMP6]] to <16 x i64>
753; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> [[TMP7]])
754; CHECK-NEXT:    [[TMP9]] = add i64 [[TMP8]], [[VEC_PHI]]
755; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
756; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
757; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
758; CHECK:       middle.block:
759; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N_VEC]], [[N]]
760; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]
761; CHECK:       scalar.ph:
762; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
763; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
764; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
765; CHECK:       for.body:
766; CHECK-NEXT:    [[I_012:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
767; CHECK-NEXT:    [[R_011:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
768; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, i8* [[X]], i32 [[I_012]]
769; CHECK-NEXT:    [[TMP11:%.*]] = load i8, i8* [[ARRAYIDX]], align 1
770; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP11]] to i32
771; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, i8* [[Y]], i32 [[I_012]]
772; CHECK-NEXT:    [[TMP12:%.*]] = load i8, i8* [[ARRAYIDX1]], align 1
773; CHECK-NEXT:    [[CONV2:%.*]] = zext i8 [[TMP12]] to i32
774; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[CONV2]], [[CONV]]
775; CHECK-NEXT:    [[CONV3:%.*]] = zext i32 [[MUL]] to i64
776; CHECK-NEXT:    [[ADD]] = add nuw nsw i64 [[R_011]], [[CONV3]]
777; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_012]], 1
778; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
779; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
780; CHECK:       for.cond.cleanup:
781; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD]], [[FOR_BODY]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]
782; CHECK-NEXT:    ret i64 [[R_0_LCSSA]]
783;
784entry:
785  %cmp10 = icmp sgt i32 %n, 0
786  br i1 %cmp10, label %for.body, label %for.cond.cleanup
787
788for.body:                                         ; preds = %entry, %for.body
789  %i.012 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
790  %r.011 = phi i64 [ %add, %for.body ], [ 0, %entry ]
791  %arrayidx = getelementptr inbounds i8, i8* %x, i32 %i.012
792  %0 = load i8, i8* %arrayidx, align 1
793  %conv = zext i8 %0 to i32
794  %arrayidx1 = getelementptr inbounds i8, i8* %y, i32 %i.012
795  %1 = load i8, i8* %arrayidx1, align 1
796  %conv2 = zext i8 %1 to i32
797  %mul = mul nuw nsw i32 %conv2, %conv
798  %conv3 = zext i32 %mul to i64
799  %add = add nuw nsw i64 %r.011, %conv3
800  %inc = add nuw nsw i32 %i.012, 1
801  %exitcond = icmp eq i32 %inc, %n
802  br i1 %exitcond, label %for.cond.cleanup, label %for.body
803
804for.cond.cleanup:                                 ; preds = %for.body, %entry
805  %r.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
806  ret i64 %r.0.lcssa
807}
808
809; 4x to use VMLA.u32
810define i32 @mla_i32_i32(i32* nocapture readonly %x, i32* nocapture readonly %y, i32 %n) #0 {
811; CHECK-LABEL: @mla_i32_i32(
812; CHECK-NEXT:  entry:
813; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[N:%.*]], 0
814; CHECK-NEXT:    br i1 [[CMP8]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
815; CHECK:       vector.ph:
816; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 3
817; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -4
818; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
819; CHECK:       vector.body:
820; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
821; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
822; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 [[N]])
823; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i32 [[INDEX]]
824; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[TMP0]] to <4 x i32>*
825; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP1]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)
826; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[Y:%.*]], i32 [[INDEX]]
827; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <4 x i32>*
828; CHECK-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP3]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)
829; CHECK-NEXT:    [[TMP4:%.*]] = mul nsw <4 x i32> [[WIDE_MASKED_LOAD1]], [[WIDE_MASKED_LOAD]]
830; CHECK-NEXT:    [[TMP5:%.*]] = select <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> [[TMP4]], <4 x i32> zeroinitializer
831; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP5]])
832; CHECK-NEXT:    [[TMP7]] = add i32 [[TMP6]], [[VEC_PHI]]
833; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
834; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
835; CHECK-NEXT:    br i1 [[TMP8]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
836; CHECK:       for.cond.cleanup:
837; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP7]], [[VECTOR_BODY]] ]
838; CHECK-NEXT:    ret i32 [[R_0_LCSSA]]
839;
840entry:
841  %cmp8 = icmp sgt i32 %n, 0
842  br i1 %cmp8, label %for.body, label %for.cond.cleanup
843
844for.body:                                         ; preds = %entry, %for.body
845  %i.010 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
846  %r.09 = phi i32 [ %add, %for.body ], [ 0, %entry ]
847  %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.010
848  %0 = load i32, i32* %arrayidx, align 4
849  %arrayidx1 = getelementptr inbounds i32, i32* %y, i32 %i.010
850  %1 = load i32, i32* %arrayidx1, align 4
851  %mul = mul nsw i32 %1, %0
852  %add = add nsw i32 %mul, %r.09
853  %inc = add nuw nsw i32 %i.010, 1
854  %exitcond = icmp eq i32 %inc, %n
855  br i1 %exitcond, label %for.cond.cleanup, label %for.body
856
857for.cond.cleanup:                                 ; preds = %for.body, %entry
858  %r.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
859  ret i32 %r.0.lcssa
860}
861
862; 8x to use VMLA.u16
863define i32 @mla_i16_i32(i16* nocapture readonly %x, i16* nocapture readonly %y, i32 %n) #0 {
864; CHECK-LABEL: @mla_i16_i32(
865; CHECK-NEXT:  entry:
866; CHECK-NEXT:    [[CMP9:%.*]] = icmp sgt i32 [[N:%.*]], 0
867; CHECK-NEXT:    br i1 [[CMP9]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
868; CHECK:       vector.ph:
869; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 7
870; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8
871; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
872; CHECK:       vector.body:
873; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
874; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
875; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX]], i32 [[N]])
876; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i16, i16* [[X:%.*]], i32 [[INDEX]]
877; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16* [[TMP0]] to <8 x i16>*
878; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* [[TMP1]], i32 2, <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> poison)
879; CHECK-NEXT:    [[TMP2:%.*]] = sext <8 x i16> [[WIDE_MASKED_LOAD]] to <8 x i32>
880; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i16, i16* [[Y:%.*]], i32 [[INDEX]]
881; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16* [[TMP3]] to <8 x i16>*
882; CHECK-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* [[TMP4]], i32 2, <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> poison)
883; CHECK-NEXT:    [[TMP5:%.*]] = sext <8 x i16> [[WIDE_MASKED_LOAD1]] to <8 x i32>
884; CHECK-NEXT:    [[TMP6:%.*]] = mul nsw <8 x i32> [[TMP5]], [[TMP2]]
885; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i32> [[TMP6]], <8 x i32> zeroinitializer
886; CHECK-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP7]])
887; CHECK-NEXT:    [[TMP9]] = add i32 [[TMP8]], [[VEC_PHI]]
888; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
889; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
890; CHECK-NEXT:    br i1 [[TMP10]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
891; CHECK:       for.cond.cleanup:
892; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP9]], [[VECTOR_BODY]] ]
893; CHECK-NEXT:    ret i32 [[R_0_LCSSA]]
894;
895entry:
896  %cmp9 = icmp sgt i32 %n, 0
897  br i1 %cmp9, label %for.body, label %for.cond.cleanup
898
899for.body:                                         ; preds = %entry, %for.body
900  %i.011 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
901  %r.010 = phi i32 [ %add, %for.body ], [ 0, %entry ]
902  %arrayidx = getelementptr inbounds i16, i16* %x, i32 %i.011
903  %0 = load i16, i16* %arrayidx, align 2
904  %conv = sext i16 %0 to i32
905  %arrayidx1 = getelementptr inbounds i16, i16* %y, i32 %i.011
906  %1 = load i16, i16* %arrayidx1, align 2
907  %conv2 = sext i16 %1 to i32
908  %mul = mul nsw i32 %conv2, %conv
909  %add = add nsw i32 %mul, %r.010
910  %inc = add nuw nsw i32 %i.011, 1
911  %exitcond = icmp eq i32 %inc, %n
912  br i1 %exitcond, label %for.cond.cleanup, label %for.body
913
914for.cond.cleanup:                                 ; preds = %for.body, %entry
915  %r.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
916  ret i32 %r.0.lcssa
917}
918
919; 16x to use VMLA.u8
920define i32 @mla_i8_i32(i8* nocapture readonly %x, i8* nocapture readonly %y, i32 %n) #0 {
921; CHECK-LABEL: @mla_i8_i32(
922; CHECK-NEXT:  entry:
923; CHECK-NEXT:    [[CMP9:%.*]] = icmp sgt i32 [[N:%.*]], 0
924; CHECK-NEXT:    br i1 [[CMP9]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
925; CHECK:       vector.ph:
926; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 15
927; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -16
928; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
929; CHECK:       vector.body:
930; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
931; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
932; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])
933; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[X:%.*]], i32 [[INDEX]]
934; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[TMP0]] to <16 x i8>*
935; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* [[TMP1]], i32 1, <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
936; CHECK-NEXT:    [[TMP2:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD]] to <16 x i32>
937; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, i8* [[Y:%.*]], i32 [[INDEX]]
938; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8* [[TMP3]] to <16 x i8>*
939; CHECK-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* [[TMP4]], i32 1, <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
940; CHECK-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD1]] to <16 x i32>
941; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw nsw <16 x i32> [[TMP5]], [[TMP2]]
942; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i32> [[TMP6]], <16 x i32> zeroinitializer
943; CHECK-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP7]])
944; CHECK-NEXT:    [[TMP9]] = add i32 [[TMP8]], [[VEC_PHI]]
945; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 16
946; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
947; CHECK-NEXT:    br i1 [[TMP10]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
948; CHECK:       for.cond.cleanup:
949; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP9]], [[VECTOR_BODY]] ]
950; CHECK-NEXT:    ret i32 [[R_0_LCSSA]]
951;
952entry:
953  %cmp9 = icmp sgt i32 %n, 0
954  br i1 %cmp9, label %for.body, label %for.cond.cleanup
955
956for.body:                                         ; preds = %entry, %for.body
957  %i.011 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
958  %r.010 = phi i32 [ %add, %for.body ], [ 0, %entry ]
959  %arrayidx = getelementptr inbounds i8, i8* %x, i32 %i.011
960  %0 = load i8, i8* %arrayidx, align 1
961  %conv = zext i8 %0 to i32
962  %arrayidx1 = getelementptr inbounds i8, i8* %y, i32 %i.011
963  %1 = load i8, i8* %arrayidx1, align 1
964  %conv2 = zext i8 %1 to i32
965  %mul = mul nuw nsw i32 %conv2, %conv
966  %add = add nuw nsw i32 %mul, %r.010
967  %inc = add nuw nsw i32 %i.011, 1
968  %exitcond = icmp eq i32 %inc, %n
969  br i1 %exitcond, label %for.cond.cleanup, label %for.body
970
971for.cond.cleanup:                                 ; preds = %for.body, %entry
972  %r.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
973  ret i32 %r.0.lcssa
974}
975
976; 8x to use VMLA.u16
977define signext i16 @mla_i16_i16(i16* nocapture readonly %x, i16* nocapture readonly %y, i32 %n) #0 {
978; CHECK-LABEL: @mla_i16_i16(
979; CHECK-NEXT:  entry:
980; CHECK-NEXT:    [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 0
981; CHECK-NEXT:    br i1 [[CMP11]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
982; CHECK:       vector.ph:
983; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 7
984; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8
985; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
986; CHECK:       vector.body:
987; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
988; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i16 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
989; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX]], i32 [[N]])
990; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i16, i16* [[X:%.*]], i32 [[INDEX]]
991; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16* [[TMP0]] to <8 x i16>*
992; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* [[TMP1]], i32 2, <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> poison)
993; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, i16* [[Y:%.*]], i32 [[INDEX]]
994; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16* [[TMP2]] to <8 x i16>*
995; CHECK-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* [[TMP3]], i32 2, <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> poison)
996; CHECK-NEXT:    [[TMP4:%.*]] = mul <8 x i16> [[WIDE_MASKED_LOAD1]], [[WIDE_MASKED_LOAD]]
997; CHECK-NEXT:    [[TMP5:%.*]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> [[TMP4]], <8 x i16> zeroinitializer
998; CHECK-NEXT:    [[TMP6:%.*]] = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> [[TMP5]])
999; CHECK-NEXT:    [[TMP7]] = add i16 [[TMP6]], [[VEC_PHI]]
1000; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
1001; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
1002; CHECK-NEXT:    br i1 [[TMP8]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
1003; CHECK:       for.cond.cleanup:
1004; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[TMP7]], [[VECTOR_BODY]] ]
1005; CHECK-NEXT:    ret i16 [[R_0_LCSSA]]
1006;
1007entry:
1008  %cmp11 = icmp sgt i32 %n, 0
1009  br i1 %cmp11, label %for.body, label %for.cond.cleanup
1010
1011for.body:                                         ; preds = %entry, %for.body
1012  %i.013 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
1013  %r.012 = phi i16 [ %add, %for.body ], [ 0, %entry ]
1014  %arrayidx = getelementptr inbounds i16, i16* %x, i32 %i.013
1015  %0 = load i16, i16* %arrayidx, align 2
1016  %arrayidx1 = getelementptr inbounds i16, i16* %y, i32 %i.013
1017  %1 = load i16, i16* %arrayidx1, align 2
1018  %mul = mul i16 %1, %0
1019  %add = add i16 %mul, %r.012
1020  %inc = add nuw nsw i32 %i.013, 1
1021  %exitcond = icmp eq i32 %inc, %n
1022  br i1 %exitcond, label %for.cond.cleanup, label %for.body
1023
1024for.cond.cleanup:                                 ; preds = %for.body, %entry
1025  %r.0.lcssa = phi i16 [ 0, %entry ], [ %add, %for.body ]
1026  ret i16 %r.0.lcssa
1027}
1028
1029; 16x to use VMLA.u8
1030define signext i16 @mla_i8_i16(i8* nocapture readonly %x, i8* nocapture readonly %y, i32 %n) #0 {
1031; CHECK-LABEL: @mla_i8_i16(
1032; CHECK-NEXT:  entry:
1033; CHECK-NEXT:    [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 0
1034; CHECK-NEXT:    br i1 [[CMP11]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
1035; CHECK:       vector.ph:
1036; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 15
1037; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -16
1038; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
1039; CHECK:       vector.body:
1040; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
1041; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i16 [ 0, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
1042; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])
1043; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[X:%.*]], i32 [[INDEX]]
1044; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[TMP0]] to <16 x i8>*
1045; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* [[TMP1]], i32 1, <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
1046; CHECK-NEXT:    [[TMP2:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD]] to <16 x i16>
1047; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, i8* [[Y:%.*]], i32 [[INDEX]]
1048; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8* [[TMP3]] to <16 x i8>*
1049; CHECK-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* [[TMP4]], i32 1, <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
1050; CHECK-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD1]] to <16 x i16>
1051; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw <16 x i16> [[TMP5]], [[TMP2]]
1052; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i16> [[TMP6]], <16 x i16> zeroinitializer
1053; CHECK-NEXT:    [[TMP8:%.*]] = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> [[TMP7]])
1054; CHECK-NEXT:    [[TMP9]] = add i16 [[TMP8]], [[VEC_PHI]]
1055; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 16
1056; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
1057; CHECK-NEXT:    br i1 [[TMP10]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
1058; CHECK:       for.cond.cleanup:
1059; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[TMP9]], [[VECTOR_BODY]] ]
1060; CHECK-NEXT:    ret i16 [[R_0_LCSSA]]
1061;
1062entry:
1063  %cmp11 = icmp sgt i32 %n, 0
1064  br i1 %cmp11, label %for.body, label %for.cond.cleanup
1065
1066for.body:                                         ; preds = %entry, %for.body
1067  %i.013 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
1068  %r.012 = phi i16 [ %add, %for.body ], [ 0, %entry ]
1069  %arrayidx = getelementptr inbounds i8, i8* %x, i32 %i.013
1070  %0 = load i8, i8* %arrayidx, align 1
1071  %conv = zext i8 %0 to i16
1072  %arrayidx1 = getelementptr inbounds i8, i8* %y, i32 %i.013
1073  %1 = load i8, i8* %arrayidx1, align 1
1074  %conv2 = zext i8 %1 to i16
1075  %mul = mul nuw i16 %conv2, %conv
1076  %add = add i16 %mul, %r.012
1077  %inc = add nuw nsw i32 %i.013, 1
1078  %exitcond = icmp eq i32 %inc, %n
1079  br i1 %exitcond, label %for.cond.cleanup, label %for.body
1080
1081for.cond.cleanup:                                 ; preds = %for.body, %entry
1082  %r.0.lcssa = phi i16 [ 0, %entry ], [ %add, %for.body ]
1083  ret i16 %r.0.lcssa
1084}
1085
1086; 16x to use VMLA.u8
1087define zeroext i8 @mla_i8_i8(i8* nocapture readonly %x, i8* nocapture readonly %y, i32 %n) #0 {
1088; CHECK-LABEL: @mla_i8_i8(
1089; CHECK-NEXT:  entry:
1090; CHECK-NEXT:    [[CMP10:%.*]] = icmp sgt i32 [[N:%.*]], 0
1091; CHECK-NEXT:    br i1 [[CMP10]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
1092; CHECK:       vector.ph:
1093; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 15
1094; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -16
1095; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
1096; CHECK:       vector.body:
1097; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
1098; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i8 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
1099; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])
1100; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[X:%.*]], i32 [[INDEX]]
1101; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[TMP0]] to <16 x i8>*
1102; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* [[TMP1]], i32 1, <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
1103; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[Y:%.*]], i32 [[INDEX]]
1104; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[TMP2]] to <16 x i8>*
1105; CHECK-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* [[TMP3]], i32 1, <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
1106; CHECK-NEXT:    [[TMP4:%.*]] = mul <16 x i8> [[WIDE_MASKED_LOAD1]], [[WIDE_MASKED_LOAD]]
1107; CHECK-NEXT:    [[TMP5:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> [[TMP4]], <16 x i8> zeroinitializer
1108; CHECK-NEXT:    [[TMP6:%.*]] = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> [[TMP5]])
1109; CHECK-NEXT:    [[TMP7]] = add i8 [[TMP6]], [[VEC_PHI]]
1110; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 16
1111; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
1112; CHECK-NEXT:    br i1 [[TMP8]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
1113; CHECK:       for.cond.cleanup:
1114; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i8 [ 0, [[ENTRY:%.*]] ], [ [[TMP7]], [[VECTOR_BODY]] ]
1115; CHECK-NEXT:    ret i8 [[R_0_LCSSA]]
1116;
1117entry:
1118  %cmp10 = icmp sgt i32 %n, 0
1119  br i1 %cmp10, label %for.body, label %for.cond.cleanup
1120
1121for.body:                                         ; preds = %entry, %for.body
1122  %i.012 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
1123  %r.011 = phi i8 [ %add, %for.body ], [ 0, %entry ]
1124  %arrayidx = getelementptr inbounds i8, i8* %x, i32 %i.012
1125  %0 = load i8, i8* %arrayidx, align 1
1126  %arrayidx1 = getelementptr inbounds i8, i8* %y, i32 %i.012
1127  %1 = load i8, i8* %arrayidx1, align 1
1128  %mul = mul i8 %1, %0
1129  %add = add i8 %mul, %r.011
1130  %inc = add nuw nsw i32 %i.012, 1
1131  %exitcond = icmp eq i32 %inc, %n
1132  br i1 %exitcond, label %for.cond.cleanup, label %for.body
1133
1134for.cond.cleanup:                                 ; preds = %for.body, %entry
1135  %r.0.lcssa = phi i8 [ 0, %entry ], [ %add, %for.body ]
1136  ret i8 %r.0.lcssa
1137}
1138
1139; Make sure interleave group members feeding in-loop reductions can be handled.
1140define i32 @reduction_interleave_group(i32 %n, i32* %arr) #0 {
1141; CHECK-LABEL: @reduction_interleave_group(
1142; CHECK-NEXT:  entry:
1143; CHECK-NEXT:    [[GUARD:%.*]] = icmp sgt i32 [[N:%.*]], 0
1144; CHECK-NEXT:    br i1 [[GUARD]], label [[FOR_BODY_PREHEADER:%.*]], label [[EXIT:%.*]]
1145; CHECK:       for.body.preheader:
1146; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N]], -1
1147; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[TMP0]], 1
1148; CHECK-NEXT:    [[TMP2:%.*]] = add nuw i32 [[TMP1]], 1
1149; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 6
1150; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
1151; CHECK:       vector.ph:
1152; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[TMP2]], -4
1153; CHECK-NEXT:    [[IND_END:%.*]] = shl i32 [[N_VEC]], 1
1154; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
1155; CHECK:       vector.body:
1156; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
1157; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
1158; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 1
1159; CHECK-NEXT:    [[TMP3:%.*]] = or i32 [[OFFSET_IDX]], 1
1160; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[ARR:%.*]], i32 -1
1161; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 [[TMP3]]
1162; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[TMP5]] to <8 x i32>*
1163; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i32>, <8 x i32>* [[TMP6]], align 4
1164; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
1165; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
1166; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[STRIDED_VEC1]])
1167; CHECK-NEXT:    [[TMP8:%.*]] = add i32 [[TMP7]], [[VEC_PHI]]
1168; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[STRIDED_VEC]])
1169; CHECK-NEXT:    [[TMP10]] = add i32 [[TMP9]], [[TMP8]]
1170; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
1171; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
1172; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
1173; CHECK:       middle.block:
1174; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
1175; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT]], label [[SCALAR_PH]]
1176; CHECK:       scalar.ph:
1177; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
1178; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
1179; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
1180; CHECK:       for.body:
1181; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
1182; CHECK-NEXT:    [[RED_PHI:%.*]] = phi i32 [ [[RED_2:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
1183; CHECK-NEXT:    [[ADD:%.*]] = or i32 [[IV]], 1
1184; CHECK-NEXT:    [[GEP_0:%.*]] = getelementptr inbounds i32, i32* [[ARR]], i32 [[ADD]]
1185; CHECK-NEXT:    [[L_0:%.*]] = load i32, i32* [[GEP_0]], align 4
1186; CHECK-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds i32, i32* [[ARR]], i32 [[IV]]
1187; CHECK-NEXT:    [[L_1:%.*]] = load i32, i32* [[GEP_1]], align 4
1188; CHECK-NEXT:    [[RED_1:%.*]] = add i32 [[L_0]], [[RED_PHI]]
1189; CHECK-NEXT:    [[RED_2]] = add i32 [[RED_1]], [[L_1]]
1190; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 2
1191; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV_NEXT]], [[N]]
1192; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[EXIT]], !llvm.loop [[LOOP27:![0-9]+]]
1193; CHECK:       exit:
1194; CHECK-NEXT:    [[RET_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[RED_2]], [[FOR_BODY]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ]
1195; CHECK-NEXT:    ret i32 [[RET_LCSSA]]
1196;
1197entry:
1198  %guard = icmp sgt i32 %n, 0
1199  br i1 %guard , label %for.body, label %exit
1200
1201for.body:                                         ; preds = %for.body.preheader, %for.body
1202  %iv = phi i32 [ %iv.next, %for.body ], [ 0, %entry ]
1203  %red.phi = phi i32 [ %red.2, %for.body ], [ 0, %entry ]
1204  %add = or i32 %iv, 1
1205  %gep.0 = getelementptr inbounds i32, i32* %arr, i32 %add
1206  %l.0 = load i32, i32* %gep.0, align 4
1207  %gep.1 = getelementptr inbounds i32, i32* %arr, i32 %iv
1208  %l.1 = load i32, i32* %gep.1, align 4
1209  %red.1 = add i32 %l.0, %red.phi
1210  %red.2 = add i32 %red.1, %l.1
1211  %iv.next = add nuw nsw i32 %iv, 2
1212  %cmp = icmp slt i32 %iv.next, %n
1213  br i1 %cmp, label %for.body, label %exit
1214
1215exit:
1216  %ret.lcssa = phi i32 [ 0, %entry ], [ %red.2, %for.body ]
1217  ret i32 %ret.lcssa
1218}
1219
1220attributes #0 = { "target-features"="+mve" }
1221