1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
2; RUN: opt -loop-vectorize < %s -S -o - | FileCheck %s
3
4target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
5target triple = "thumbv8.1m.main-arm-none-eabi"
6
7define i64 @add_i64_i64(i64* nocapture readonly %x, i32 %n) #0 {
8; CHECK-LABEL: @add_i64_i64(
9; CHECK-NEXT:  entry:
10; CHECK-NEXT:    [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0
11; CHECK-NEXT:    br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
12; CHECK:       for.body.preheader:
13; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
14; CHECK:       for.body:
15; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
16; CHECK-NEXT:    [[R_07:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
17; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, i64* [[X:%.*]], i32 [[I_08]]
18; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[ARRAYIDX]], align 8
19; CHECK-NEXT:    [[ADD]] = add nsw i64 [[TMP0]], [[R_07]]
20; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
21; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
22; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[FOR_BODY]]
23; CHECK:       for.cond.cleanup.loopexit:
24; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ]
25; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
26; CHECK:       for.cond.cleanup:
27; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ]
28; CHECK-NEXT:    ret i64 [[R_0_LCSSA]]
29;
30entry:
31  %cmp6 = icmp sgt i32 %n, 0
32  br i1 %cmp6, label %for.body, label %for.cond.cleanup
33
34for.body:                                         ; preds = %entry, %for.body
35  %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
36  %r.07 = phi i64 [ %add, %for.body ], [ 0, %entry ]
37  %arrayidx = getelementptr inbounds i64, i64* %x, i32 %i.08
38  %0 = load i64, i64* %arrayidx, align 8
39  %add = add nsw i64 %0, %r.07
40  %inc = add nuw nsw i32 %i.08, 1
41  %exitcond = icmp eq i32 %inc, %n
42  br i1 %exitcond, label %for.cond.cleanup, label %for.body
43
44for.cond.cleanup:                                 ; preds = %for.body, %entry
45  %r.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
46  ret i64 %r.0.lcssa
47}
48
49; FIXME: 4x
50define i64 @add_i32_i64(i32* nocapture readonly %x, i32 %n) #0 {
51; CHECK-LABEL: @add_i32_i64(
52; CHECK-NEXT:  entry:
53; CHECK-NEXT:    [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0
54; CHECK-NEXT:    br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
55; CHECK:       for.body.preheader:
56; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
57; CHECK:       for.body:
58; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
59; CHECK-NEXT:    [[R_07:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
60; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i32 [[I_08]]
61; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
62; CHECK-NEXT:    [[CONV:%.*]] = sext i32 [[TMP0]] to i64
63; CHECK-NEXT:    [[ADD]] = add nsw i64 [[R_07]], [[CONV]]
64; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
65; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
66; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[FOR_BODY]]
67; CHECK:       for.cond.cleanup.loopexit:
68; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ]
69; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
70; CHECK:       for.cond.cleanup:
71; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ]
72; CHECK-NEXT:    ret i64 [[R_0_LCSSA]]
73;
74entry:
75  %cmp6 = icmp sgt i32 %n, 0
76  br i1 %cmp6, label %for.body, label %for.cond.cleanup
77
78for.body:                                         ; preds = %entry, %for.body
79  %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
80  %r.07 = phi i64 [ %add, %for.body ], [ 0, %entry ]
81  %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
82  %0 = load i32, i32* %arrayidx, align 4
83  %conv = sext i32 %0 to i64
84  %add = add nsw i64 %r.07, %conv
85  %inc = add nuw nsw i32 %i.08, 1
86  %exitcond = icmp eq i32 %inc, %n
87  br i1 %exitcond, label %for.cond.cleanup, label %for.body
88
89for.cond.cleanup:                                 ; preds = %for.body, %entry
90  %r.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
91  ret i64 %r.0.lcssa
92}
93
94; FIXME: 4x ?
95define i64 @add_i16_i64(i16* nocapture readonly %x, i32 %n) #0 {
96; CHECK-LABEL: @add_i16_i64(
97; CHECK-NEXT:  entry:
98; CHECK-NEXT:    [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0
99; CHECK-NEXT:    br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
100; CHECK:       for.body.preheader:
101; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
102; CHECK:       for.body:
103; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
104; CHECK-NEXT:    [[R_07:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
105; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, i16* [[X:%.*]], i32 [[I_08]]
106; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[ARRAYIDX]], align 2
107; CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[TMP0]] to i64
108; CHECK-NEXT:    [[ADD]] = add nsw i64 [[R_07]], [[CONV]]
109; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
110; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
111; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[FOR_BODY]]
112; CHECK:       for.cond.cleanup.loopexit:
113; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ]
114; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
115; CHECK:       for.cond.cleanup:
116; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ]
117; CHECK-NEXT:    ret i64 [[R_0_LCSSA]]
118;
119entry:
120  %cmp6 = icmp sgt i32 %n, 0
121  br i1 %cmp6, label %for.body, label %for.cond.cleanup
122
123for.body:                                         ; preds = %entry, %for.body
124  %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
125  %r.07 = phi i64 [ %add, %for.body ], [ 0, %entry ]
126  %arrayidx = getelementptr inbounds i16, i16* %x, i32 %i.08
127  %0 = load i16, i16* %arrayidx, align 2
128  %conv = sext i16 %0 to i64
129  %add = add nsw i64 %r.07, %conv
130  %inc = add nuw nsw i32 %i.08, 1
131  %exitcond = icmp eq i32 %inc, %n
132  br i1 %exitcond, label %for.cond.cleanup, label %for.body
133
134for.cond.cleanup:                                 ; preds = %for.body, %entry
135  %r.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
136  ret i64 %r.0.lcssa
137}
138
139; FIXME: 4x ?
140define i64 @add_i8_i64(i8* nocapture readonly %x, i32 %n) #0 {
141; CHECK-LABEL: @add_i8_i64(
142; CHECK-NEXT:  entry:
143; CHECK-NEXT:    [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0
144; CHECK-NEXT:    br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
145; CHECK:       for.body.preheader:
146; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
147; CHECK:       for.body:
148; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
149; CHECK-NEXT:    [[R_07:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
150; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, i8* [[X:%.*]], i32 [[I_08]]
151; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[ARRAYIDX]], align 1
152; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP0]] to i64
153; CHECK-NEXT:    [[ADD]] = add nuw nsw i64 [[R_07]], [[CONV]]
154; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
155; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
156; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[FOR_BODY]]
157; CHECK:       for.cond.cleanup.loopexit:
158; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ]
159; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
160; CHECK:       for.cond.cleanup:
161; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ]
162; CHECK-NEXT:    ret i64 [[R_0_LCSSA]]
163;
164entry:
165  %cmp6 = icmp sgt i32 %n, 0
166  br i1 %cmp6, label %for.body, label %for.cond.cleanup
167
168for.body:                                         ; preds = %entry, %for.body
169  %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
170  %r.07 = phi i64 [ %add, %for.body ], [ 0, %entry ]
171  %arrayidx = getelementptr inbounds i8, i8* %x, i32 %i.08
172  %0 = load i8, i8* %arrayidx, align 1
173  %conv = zext i8 %0 to i64
174  %add = add nuw nsw i64 %r.07, %conv
175  %inc = add nuw nsw i32 %i.08, 1
176  %exitcond = icmp eq i32 %inc, %n
177  br i1 %exitcond, label %for.cond.cleanup, label %for.body
178
179for.cond.cleanup:                                 ; preds = %for.body, %entry
180  %r.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
181  ret i64 %r.0.lcssa
182}
183
184define i32 @add_i32_i32(i32* nocapture readonly %x, i32 %n) #0 {
185; CHECK-LABEL: @add_i32_i32(
186; CHECK-NEXT:  entry:
187; CHECK-NEXT:    [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0
188; CHECK-NEXT:    br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
189; CHECK:       for.body.preheader:
190; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
191; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
192; CHECK:       vector.ph:
193; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N]], 4
194; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
195; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
196; CHECK:       vector.body:
197; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
198; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
199; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
200; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i32 [[TMP0]]
201; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i32 0
202; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <4 x i32>*
203; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP3]], align 4
204; CHECK-NEXT:    [[TMP4]] = add <4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
205; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
206; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
207; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
208; CHECK:       middle.block:
209; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[TMP4]])
210; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
211; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
212; CHECK:       scalar.ph:
213; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
214; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
215; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
216; CHECK:       for.body:
217; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
218; CHECK-NEXT:    [[R_07:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
219; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[I_08]]
220; CHECK-NEXT:    [[TMP7:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
221; CHECK-NEXT:    [[ADD]] = add nsw i32 [[TMP7]], [[R_07]]
222; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
223; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
224; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop !2
225; CHECK:       for.cond.cleanup.loopexit:
226; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
227; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
228; CHECK:       for.cond.cleanup:
229; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ]
230; CHECK-NEXT:    ret i32 [[R_0_LCSSA]]
231;
232entry:
233  %cmp6 = icmp sgt i32 %n, 0
234  br i1 %cmp6, label %for.body, label %for.cond.cleanup
235
236for.body:                                         ; preds = %entry, %for.body
237  %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
238  %r.07 = phi i32 [ %add, %for.body ], [ 0, %entry ]
239  %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
240  %0 = load i32, i32* %arrayidx, align 4
241  %add = add nsw i32 %0, %r.07
242  %inc = add nuw nsw i32 %i.08, 1
243  %exitcond = icmp eq i32 %inc, %n
244  br i1 %exitcond, label %for.cond.cleanup, label %for.body
245
246for.cond.cleanup:                                 ; preds = %for.body, %entry
247  %r.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
248  ret i32 %r.0.lcssa
249}
250
251; FIXME: 8x
252define i32 @add_i16_i32(i16* nocapture readonly %x, i32 %n) #0 {
253; CHECK-LABEL: @add_i16_i32(
254; CHECK-NEXT:  entry:
255; CHECK-NEXT:    [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0
256; CHECK-NEXT:    br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
257; CHECK:       for.body.preheader:
258; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
259; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
260; CHECK:       vector.ph:
261; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N]], 4
262; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
263; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
264; CHECK:       vector.body:
265; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
266; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
267; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
268; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, i16* [[X:%.*]], i32 [[TMP0]]
269; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, i16* [[TMP1]], i32 0
270; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16* [[TMP2]] to <4 x i16>*
271; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i16>, <4 x i16>* [[TMP3]], align 2
272; CHECK-NEXT:    [[TMP4:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i32>
273; CHECK-NEXT:    [[TMP5]] = add <4 x i32> [[VEC_PHI]], [[TMP4]]
274; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
275; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
276; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !4
277; CHECK:       middle.block:
278; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[TMP5]])
279; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
280; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
281; CHECK:       scalar.ph:
282; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
283; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
284; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
285; CHECK:       for.body:
286; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
287; CHECK-NEXT:    [[R_07:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
288; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, i16* [[X]], i32 [[I_08]]
289; CHECK-NEXT:    [[TMP8:%.*]] = load i16, i16* [[ARRAYIDX]], align 2
290; CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[TMP8]] to i32
291; CHECK-NEXT:    [[ADD]] = add nsw i32 [[R_07]], [[CONV]]
292; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
293; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
294; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop !5
295; CHECK:       for.cond.cleanup.loopexit:
296; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
297; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
298; CHECK:       for.cond.cleanup:
299; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ]
300; CHECK-NEXT:    ret i32 [[R_0_LCSSA]]
301;
302entry:
303  %cmp6 = icmp sgt i32 %n, 0
304  br i1 %cmp6, label %for.body, label %for.cond.cleanup
305
306for.body:                                         ; preds = %entry, %for.body
307  %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
308  %r.07 = phi i32 [ %add, %for.body ], [ 0, %entry ]
309  %arrayidx = getelementptr inbounds i16, i16* %x, i32 %i.08
310  %0 = load i16, i16* %arrayidx, align 2
311  %conv = sext i16 %0 to i32
312  %add = add nsw i32 %r.07, %conv
313  %inc = add nuw nsw i32 %i.08, 1
314  %exitcond = icmp eq i32 %inc, %n
315  br i1 %exitcond, label %for.cond.cleanup, label %for.body
316
317for.cond.cleanup:                                 ; preds = %for.body, %entry
318  %r.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
319  ret i32 %r.0.lcssa
320}
321
322; FIXME: 16x
323define i32 @add_i8_i32(i8* nocapture readonly %x, i32 %n) #0 {
324; CHECK-LABEL: @add_i8_i32(
325; CHECK-NEXT:  entry:
326; CHECK-NEXT:    [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0
327; CHECK-NEXT:    br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
328; CHECK:       for.body.preheader:
329; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
330; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
331; CHECK:       vector.ph:
332; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N]], 4
333; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
334; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
335; CHECK:       vector.body:
336; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
337; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
338; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
339; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, i8* [[X:%.*]], i32 [[TMP0]]
340; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[TMP1]], i32 0
341; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[TMP2]] to <4 x i8>*
342; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, <4 x i8>* [[TMP3]], align 1
343; CHECK-NEXT:    [[TMP4:%.*]] = zext <4 x i8> [[WIDE_LOAD]] to <4 x i32>
344; CHECK-NEXT:    [[TMP5]] = add <4 x i32> [[VEC_PHI]], [[TMP4]]
345; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
346; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
347; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !6
348; CHECK:       middle.block:
349; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[TMP5]])
350; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
351; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
352; CHECK:       scalar.ph:
353; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
354; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
355; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
356; CHECK:       for.body:
357; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
358; CHECK-NEXT:    [[R_07:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
359; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, i8* [[X]], i32 [[I_08]]
360; CHECK-NEXT:    [[TMP8:%.*]] = load i8, i8* [[ARRAYIDX]], align 1
361; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP8]] to i32
362; CHECK-NEXT:    [[ADD]] = add nuw nsw i32 [[R_07]], [[CONV]]
363; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
364; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
365; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop !7
366; CHECK:       for.cond.cleanup.loopexit:
367; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
368; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
369; CHECK:       for.cond.cleanup:
370; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ]
371; CHECK-NEXT:    ret i32 [[R_0_LCSSA]]
372;
373entry:
374  %cmp6 = icmp sgt i32 %n, 0
375  br i1 %cmp6, label %for.body, label %for.cond.cleanup
376
377for.body:                                         ; preds = %entry, %for.body
378  %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
379  %r.07 = phi i32 [ %add, %for.body ], [ 0, %entry ]
380  %arrayidx = getelementptr inbounds i8, i8* %x, i32 %i.08
381  %0 = load i8, i8* %arrayidx, align 1
382  %conv = zext i8 %0 to i32
383  %add = add nuw nsw i32 %r.07, %conv
384  %inc = add nuw nsw i32 %i.08, 1
385  %exitcond = icmp eq i32 %inc, %n
386  br i1 %exitcond, label %for.cond.cleanup, label %for.body
387
388for.cond.cleanup:                                 ; preds = %for.body, %entry
389  %r.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
390  ret i32 %r.0.lcssa
391}
392
393define signext i16 @add_i16_i16(i16* nocapture readonly %x, i32 %n) #0 {
394; CHECK-LABEL: @add_i16_i16(
395; CHECK-NEXT:  entry:
396; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[N:%.*]], 0
397; CHECK-NEXT:    br i1 [[CMP8]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
398; CHECK:       for.body.preheader:
399; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 8
400; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
401; CHECK:       vector.ph:
402; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N]], 8
403; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
404; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
405; CHECK:       vector.body:
406; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
407; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i16> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
408; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
409; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, i16* [[X:%.*]], i32 [[TMP0]]
410; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, i16* [[TMP1]], i32 0
411; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16* [[TMP2]] to <8 x i16>*
412; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i16>, <8 x i16>* [[TMP3]], align 2
413; CHECK-NEXT:    [[TMP4]] = add <8 x i16> [[WIDE_LOAD]], [[VEC_PHI]]
414; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
415; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
416; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !8
417; CHECK:       middle.block:
418; CHECK-NEXT:    [[TMP6:%.*]] = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> [[TMP4]])
419; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
420; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
421; CHECK:       scalar.ph:
422; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
423; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
424; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
425; CHECK:       for.body:
426; CHECK-NEXT:    [[I_010:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
427; CHECK-NEXT:    [[R_09:%.*]] = phi i16 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
428; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, i16* [[X]], i32 [[I_010]]
429; CHECK-NEXT:    [[TMP7:%.*]] = load i16, i16* [[ARRAYIDX]], align 2
430; CHECK-NEXT:    [[ADD]] = add i16 [[TMP7]], [[R_09]]
431; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_010]], 1
432; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
433; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop !9
434; CHECK:       for.cond.cleanup.loopexit:
435; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i16 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
436; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
437; CHECK:       for.cond.cleanup:
438; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ]
439; CHECK-NEXT:    ret i16 [[R_0_LCSSA]]
440;
441entry:
442  %cmp8 = icmp sgt i32 %n, 0
443  br i1 %cmp8, label %for.body, label %for.cond.cleanup
444
445for.body:                                         ; preds = %entry, %for.body
446  %i.010 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
447  %r.09 = phi i16 [ %add, %for.body ], [ 0, %entry ]
448  %arrayidx = getelementptr inbounds i16, i16* %x, i32 %i.010
449  %0 = load i16, i16* %arrayidx, align 2
450  %add = add i16 %0, %r.09
451  %inc = add nuw nsw i32 %i.010, 1
452  %exitcond = icmp eq i32 %inc, %n
453  br i1 %exitcond, label %for.cond.cleanup, label %for.body
454
455for.cond.cleanup:                                 ; preds = %for.body, %entry
456  %r.0.lcssa = phi i16 [ 0, %entry ], [ %add, %for.body ]
457  ret i16 %r.0.lcssa
458}
459
460; FIXME: 16x ?
461define signext i16 @add_i8_i16(i8* nocapture readonly %x, i32 %n) #0 {
462; CHECK-LABEL: @add_i8_i16(
463; CHECK-NEXT:  entry:
464; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[N:%.*]], 0
465; CHECK-NEXT:    br i1 [[CMP8]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
466; CHECK:       for.body.preheader:
467; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 8
468; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
469; CHECK:       vector.ph:
470; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N]], 8
471; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
472; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
473; CHECK:       vector.body:
474; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
475; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i16> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
476; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
477; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, i8* [[X:%.*]], i32 [[TMP0]]
478; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[TMP1]], i32 0
479; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[TMP2]] to <8 x i8>*
480; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i8>, <8 x i8>* [[TMP3]], align 1
481; CHECK-NEXT:    [[TMP4:%.*]] = zext <8 x i8> [[WIDE_LOAD]] to <8 x i16>
482; CHECK-NEXT:    [[TMP5]] = add <8 x i16> [[VEC_PHI]], [[TMP4]]
483; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
484; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
485; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !10
486; CHECK:       middle.block:
487; CHECK-NEXT:    [[TMP7:%.*]] = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> [[TMP5]])
488; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
489; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
490; CHECK:       scalar.ph:
491; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
492; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
493; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
494; CHECK:       for.body:
495; CHECK-NEXT:    [[I_010:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
496; CHECK-NEXT:    [[R_09:%.*]] = phi i16 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
497; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, i8* [[X]], i32 [[I_010]]
498; CHECK-NEXT:    [[TMP8:%.*]] = load i8, i8* [[ARRAYIDX]], align 1
499; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP8]] to i16
500; CHECK-NEXT:    [[ADD]] = add i16 [[R_09]], [[CONV]]
501; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_010]], 1
502; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
503; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop !11
504; CHECK:       for.cond.cleanup.loopexit:
505; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i16 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
506; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
507; CHECK:       for.cond.cleanup:
508; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ]
509; CHECK-NEXT:    ret i16 [[R_0_LCSSA]]
510;
511entry:
512  %cmp8 = icmp sgt i32 %n, 0
513  br i1 %cmp8, label %for.body, label %for.cond.cleanup
514
515for.body:                                         ; preds = %entry, %for.body
516  %i.010 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
517  %r.09 = phi i16 [ %add, %for.body ], [ 0, %entry ]
518  %arrayidx = getelementptr inbounds i8, i8* %x, i32 %i.010
519  %0 = load i8, i8* %arrayidx, align 1
520  %conv = zext i8 %0 to i16
521  %add = add i16 %r.09, %conv
522  %inc = add nuw nsw i32 %i.010, 1
523  %exitcond = icmp eq i32 %inc, %n
524  br i1 %exitcond, label %for.cond.cleanup, label %for.body
525
526for.cond.cleanup:                                 ; preds = %for.body, %entry
527  %r.0.lcssa = phi i16 [ 0, %entry ], [ %add, %for.body ]
528  ret i16 %r.0.lcssa
529}
530
531define zeroext i8 @add_i8_i8(i8* nocapture readonly %x, i32 %n) #0 {
532; CHECK-LABEL: @add_i8_i8(
533; CHECK-NEXT:  entry:
534; CHECK-NEXT:    [[CMP7:%.*]] = icmp sgt i32 [[N:%.*]], 0
535; CHECK-NEXT:    br i1 [[CMP7]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
536; CHECK:       for.body.preheader:
537; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 16
538; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
539; CHECK:       vector.ph:
540; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N]], 16
541; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
542; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
543; CHECK:       vector.body:
544; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
545; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i8> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
546; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
547; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, i8* [[X:%.*]], i32 [[TMP0]]
548; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[TMP1]], i32 0
549; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[TMP2]] to <16 x i8>*
550; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, <16 x i8>* [[TMP3]], align 1
551; CHECK-NEXT:    [[TMP4]] = add <16 x i8> [[WIDE_LOAD]], [[VEC_PHI]]
552; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 16
553; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
554; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !12
555; CHECK:       middle.block:
556; CHECK-NEXT:    [[TMP6:%.*]] = call i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8> [[TMP4]])
557; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
558; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
559; CHECK:       scalar.ph:
560; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
561; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i8 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
562; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
563; CHECK:       for.body:
564; CHECK-NEXT:    [[I_09:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
565; CHECK-NEXT:    [[R_08:%.*]] = phi i8 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
566; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, i8* [[X]], i32 [[I_09]]
567; CHECK-NEXT:    [[TMP7:%.*]] = load i8, i8* [[ARRAYIDX]], align 1
568; CHECK-NEXT:    [[ADD]] = add i8 [[TMP7]], [[R_08]]
569; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_09]], 1
570; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
571; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop !13
572; CHECK:       for.cond.cleanup.loopexit:
573; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i8 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
574; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
575; CHECK:       for.cond.cleanup:
576; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i8 [ 0, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ]
577; CHECK-NEXT:    ret i8 [[R_0_LCSSA]]
578;
579entry:
580  %cmp7 = icmp sgt i32 %n, 0
581  br i1 %cmp7, label %for.body, label %for.cond.cleanup
582
583for.body:                                         ; preds = %entry, %for.body
584  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
585  %r.08 = phi i8 [ %add, %for.body ], [ 0, %entry ]
586  %arrayidx = getelementptr inbounds i8, i8* %x, i32 %i.09
587  %0 = load i8, i8* %arrayidx, align 1
588  %add = add i8 %0, %r.08
589  %inc = add nuw nsw i32 %i.09, 1
590  %exitcond = icmp eq i32 %inc, %n
591  br i1 %exitcond, label %for.cond.cleanup, label %for.body
592
593for.cond.cleanup:                                 ; preds = %for.body, %entry
594  %r.0.lcssa = phi i8 [ 0, %entry ], [ %add, %for.body ]
595  ret i8 %r.0.lcssa
596}
597
598define i64 @mla_i64_i64(i64* nocapture readonly %x, i64* nocapture readonly %y, i32 %n) #0 {
599; CHECK-LABEL: @mla_i64_i64(
600; CHECK-NEXT:  entry:
601; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[N:%.*]], 0
602; CHECK-NEXT:    br i1 [[CMP8]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
603; CHECK:       for.body.preheader:
604; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
605; CHECK:       for.body:
606; CHECK-NEXT:    [[I_010:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
607; CHECK-NEXT:    [[R_09:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
608; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, i64* [[X:%.*]], i32 [[I_010]]
609; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[ARRAYIDX]], align 8
610; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, i64* [[Y:%.*]], i32 [[I_010]]
611; CHECK-NEXT:    [[TMP1:%.*]] = load i64, i64* [[ARRAYIDX1]], align 8
612; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP1]], [[TMP0]]
613; CHECK-NEXT:    [[ADD]] = add nsw i64 [[MUL]], [[R_09]]
614; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_010]], 1
615; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
616; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[FOR_BODY]]
617; CHECK:       for.cond.cleanup.loopexit:
618; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ]
619; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
620; CHECK:       for.cond.cleanup:
621; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ]
622; CHECK-NEXT:    ret i64 [[R_0_LCSSA]]
623;
624entry:
625  %cmp8 = icmp sgt i32 %n, 0
626  br i1 %cmp8, label %for.body, label %for.cond.cleanup
627
628for.body:                                         ; preds = %entry, %for.body
629  %i.010 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
630  %r.09 = phi i64 [ %add, %for.body ], [ 0, %entry ]
631  %arrayidx = getelementptr inbounds i64, i64* %x, i32 %i.010
632  %0 = load i64, i64* %arrayidx, align 8
633  %arrayidx1 = getelementptr inbounds i64, i64* %y, i32 %i.010
634  %1 = load i64, i64* %arrayidx1, align 8
635  %mul = mul nsw i64 %1, %0
636  %add = add nsw i64 %mul, %r.09
637  %inc = add nuw nsw i32 %i.010, 1
638  %exitcond = icmp eq i32 %inc, %n
639  br i1 %exitcond, label %for.cond.cleanup, label %for.body
640
641for.cond.cleanup:                                 ; preds = %for.body, %entry
642  %r.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
643  ret i64 %r.0.lcssa
644}
645
646define i64 @mla_i32_i64(i32* nocapture readonly %x, i32* nocapture readonly %y, i32 %n) #0 {
647; CHECK-LABEL: @mla_i32_i64(
648; CHECK-NEXT:  entry:
649; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[N:%.*]], 0
650; CHECK-NEXT:    br i1 [[CMP8]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
651; CHECK:       for.body.preheader:
652; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
653; CHECK:       for.body:
654; CHECK-NEXT:    [[I_010:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
655; CHECK-NEXT:    [[R_09:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
656; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i32 [[I_010]]
657; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
658; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[Y:%.*]], i32 [[I_010]]
659; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[ARRAYIDX1]], align 4
660; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP1]], [[TMP0]]
661; CHECK-NEXT:    [[CONV:%.*]] = sext i32 [[MUL]] to i64
662; CHECK-NEXT:    [[ADD]] = add nsw i64 [[R_09]], [[CONV]]
663; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_010]], 1
664; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
665; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[FOR_BODY]]
666; CHECK:       for.cond.cleanup.loopexit:
667; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ]
668; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
669; CHECK:       for.cond.cleanup:
670; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ]
671; CHECK-NEXT:    ret i64 [[R_0_LCSSA]]
672;
673entry:
674  %cmp8 = icmp sgt i32 %n, 0
675  br i1 %cmp8, label %for.body, label %for.cond.cleanup
676
677for.body:                                         ; preds = %entry, %for.body
678  %i.010 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
679  %r.09 = phi i64 [ %add, %for.body ], [ 0, %entry ]
680  %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.010
681  %0 = load i32, i32* %arrayidx, align 4
682  %arrayidx1 = getelementptr inbounds i32, i32* %y, i32 %i.010
683  %1 = load i32, i32* %arrayidx1, align 4
684  %mul = mul nsw i32 %1, %0
685  %conv = sext i32 %mul to i64
686  %add = add nsw i64 %r.09, %conv
687  %inc = add nuw nsw i32 %i.010, 1
688  %exitcond = icmp eq i32 %inc, %n
689  br i1 %exitcond, label %for.cond.cleanup, label %for.body
690
691for.cond.cleanup:                                 ; preds = %for.body, %entry
692  %r.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
693  ret i64 %r.0.lcssa
694}
695
696define i64 @mla_i16_i64(i16* nocapture readonly %x, i16* nocapture readonly %y, i32 %n) #0 {
697; CHECK-LABEL: @mla_i16_i64(
698; CHECK-NEXT:  entry:
699; CHECK-NEXT:    [[CMP10:%.*]] = icmp sgt i32 [[N:%.*]], 0
700; CHECK-NEXT:    br i1 [[CMP10]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
701; CHECK:       for.body.preheader:
702; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
703; CHECK:       for.body:
704; CHECK-NEXT:    [[I_012:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
705; CHECK-NEXT:    [[R_011:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
706; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, i16* [[X:%.*]], i32 [[I_012]]
707; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[ARRAYIDX]], align 2
708; CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[TMP0]] to i32
709; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i16, i16* [[Y:%.*]], i32 [[I_012]]
710; CHECK-NEXT:    [[TMP1:%.*]] = load i16, i16* [[ARRAYIDX1]], align 2
711; CHECK-NEXT:    [[CONV2:%.*]] = sext i16 [[TMP1]] to i32
712; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]]
713; CHECK-NEXT:    [[CONV3:%.*]] = sext i32 [[MUL]] to i64
714; CHECK-NEXT:    [[ADD]] = add nsw i64 [[R_011]], [[CONV3]]
715; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_012]], 1
716; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
717; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[FOR_BODY]]
718; CHECK:       for.cond.cleanup.loopexit:
719; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ]
720; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
721; CHECK:       for.cond.cleanup:
722; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ]
723; CHECK-NEXT:    ret i64 [[R_0_LCSSA]]
724;
725entry:
726  %cmp10 = icmp sgt i32 %n, 0
727  br i1 %cmp10, label %for.body, label %for.cond.cleanup
728
729for.body:                                         ; preds = %entry, %for.body
730  %i.012 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
731  %r.011 = phi i64 [ %add, %for.body ], [ 0, %entry ]
732  %arrayidx = getelementptr inbounds i16, i16* %x, i32 %i.012
733  %0 = load i16, i16* %arrayidx, align 2
734  %conv = sext i16 %0 to i32
735  %arrayidx1 = getelementptr inbounds i16, i16* %y, i32 %i.012
736  %1 = load i16, i16* %arrayidx1, align 2
737  %conv2 = sext i16 %1 to i32
738  %mul = mul nsw i32 %conv2, %conv
739  %conv3 = sext i32 %mul to i64
740  %add = add nsw i64 %r.011, %conv3
741  %inc = add nuw nsw i32 %i.012, 1
742  %exitcond = icmp eq i32 %inc, %n
743  br i1 %exitcond, label %for.cond.cleanup, label %for.body
744
745for.cond.cleanup:                                 ; preds = %for.body, %entry
746  %r.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
747  ret i64 %r.0.lcssa
748}
749
750define i64 @mla_i8_i64(i8* nocapture readonly %x, i8* nocapture readonly %y, i32 %n) #0 {
751; CHECK-LABEL: @mla_i8_i64(
752; CHECK-NEXT:  entry:
753; CHECK-NEXT:    [[CMP10:%.*]] = icmp sgt i32 [[N:%.*]], 0
754; CHECK-NEXT:    br i1 [[CMP10]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
755; CHECK:       for.body.preheader:
756; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
757; CHECK:       for.body:
758; CHECK-NEXT:    [[I_012:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
759; CHECK-NEXT:    [[R_011:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
760; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, i8* [[X:%.*]], i32 [[I_012]]
761; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[ARRAYIDX]], align 1
762; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP0]] to i32
763; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, i8* [[Y:%.*]], i32 [[I_012]]
764; CHECK-NEXT:    [[TMP1:%.*]] = load i8, i8* [[ARRAYIDX1]], align 1
765; CHECK-NEXT:    [[CONV2:%.*]] = zext i8 [[TMP1]] to i32
766; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[CONV2]], [[CONV]]
767; CHECK-NEXT:    [[CONV3:%.*]] = zext i32 [[MUL]] to i64
768; CHECK-NEXT:    [[ADD]] = add nuw nsw i64 [[R_011]], [[CONV3]]
769; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_012]], 1
770; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
771; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[FOR_BODY]]
772; CHECK:       for.cond.cleanup.loopexit:
773; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ]
774; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
775; CHECK:       for.cond.cleanup:
776; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ]
777; CHECK-NEXT:    ret i64 [[R_0_LCSSA]]
778;
779entry:
780  %cmp10 = icmp sgt i32 %n, 0
781  br i1 %cmp10, label %for.body, label %for.cond.cleanup
782
783for.body:                                         ; preds = %entry, %for.body
784  %i.012 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
785  %r.011 = phi i64 [ %add, %for.body ], [ 0, %entry ]
786  %arrayidx = getelementptr inbounds i8, i8* %x, i32 %i.012
787  %0 = load i8, i8* %arrayidx, align 1
788  %conv = zext i8 %0 to i32
789  %arrayidx1 = getelementptr inbounds i8, i8* %y, i32 %i.012
790  %1 = load i8, i8* %arrayidx1, align 1
791  %conv2 = zext i8 %1 to i32
792  %mul = mul nuw nsw i32 %conv2, %conv
793  %conv3 = zext i32 %mul to i64
794  %add = add nuw nsw i64 %r.011, %conv3
795  %inc = add nuw nsw i32 %i.012, 1
796  %exitcond = icmp eq i32 %inc, %n
797  br i1 %exitcond, label %for.cond.cleanup, label %for.body
798
799for.cond.cleanup:                                 ; preds = %for.body, %entry
800  %r.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
801  ret i64 %r.0.lcssa
802}
803
804define i32 @mla_i32_i32(i32* nocapture readonly %x, i32* nocapture readonly %y, i32 %n) #0 {
805; CHECK-LABEL: @mla_i32_i32(
806; CHECK-NEXT:  entry:
807; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[N:%.*]], 0
808; CHECK-NEXT:    br i1 [[CMP8]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
809; CHECK:       for.body.preheader:
810; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
811; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
812; CHECK:       vector.ph:
813; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N]], 4
814; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
815; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
816; CHECK:       vector.body:
817; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
818; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[VECTOR_BODY]] ]
819; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
820; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i32 [[TMP0]]
821; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i32 0
822; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <4 x i32>*
823; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP3]], align 4
824; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[Y:%.*]], i32 [[TMP0]]
825; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 0
826; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[TMP5]] to <4 x i32>*
827; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP6]], align 4
828; CHECK-NEXT:    [[TMP7:%.*]] = mul nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]
829; CHECK-NEXT:    [[TMP8]] = add <4 x i32> [[TMP7]], [[VEC_PHI]]
830; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
831; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
832; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !14
833; CHECK:       middle.block:
834; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[TMP8]])
835; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
836; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
837; CHECK:       scalar.ph:
838; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
839; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ]
840; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
841; CHECK:       for.body:
842; CHECK-NEXT:    [[I_010:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
843; CHECK-NEXT:    [[R_09:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
844; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[I_010]]
845; CHECK-NEXT:    [[TMP11:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
846; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[Y]], i32 [[I_010]]
847; CHECK-NEXT:    [[TMP12:%.*]] = load i32, i32* [[ARRAYIDX1]], align 4
848; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP12]], [[TMP11]]
849; CHECK-NEXT:    [[ADD]] = add nsw i32 [[MUL]], [[R_09]]
850; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_010]], 1
851; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
852; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop !15
853; CHECK:       for.cond.cleanup.loopexit:
854; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ]
855; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
856; CHECK:       for.cond.cleanup:
857; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ]
858; CHECK-NEXT:    ret i32 [[R_0_LCSSA]]
859;
860entry:
861  %cmp8 = icmp sgt i32 %n, 0
862  br i1 %cmp8, label %for.body, label %for.cond.cleanup
863
864for.body:                                         ; preds = %entry, %for.body
865  %i.010 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
866  %r.09 = phi i32 [ %add, %for.body ], [ 0, %entry ]
867  %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.010
868  %0 = load i32, i32* %arrayidx, align 4
869  %arrayidx1 = getelementptr inbounds i32, i32* %y, i32 %i.010
870  %1 = load i32, i32* %arrayidx1, align 4
871  %mul = mul nsw i32 %1, %0
872  %add = add nsw i32 %mul, %r.09
873  %inc = add nuw nsw i32 %i.010, 1
874  %exitcond = icmp eq i32 %inc, %n
875  br i1 %exitcond, label %for.cond.cleanup, label %for.body
876
877for.cond.cleanup:                                 ; preds = %for.body, %entry
878  %r.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
879  ret i32 %r.0.lcssa
880}
881
882define i32 @mla_i16_i32(i16* nocapture readonly %x, i16* nocapture readonly %y, i32 %n) #0 {
883; CHECK-LABEL: @mla_i16_i32(
884; CHECK-NEXT:  entry:
885; CHECK-NEXT:    [[CMP9:%.*]] = icmp sgt i32 [[N:%.*]], 0
886; CHECK-NEXT:    br i1 [[CMP9]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
887; CHECK:       for.body.preheader:
888; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
889; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
890; CHECK:       vector.ph:
891; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N]], 4
892; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
893; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
894; CHECK:       vector.body:
895; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
896; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
897; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
898; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, i16* [[X:%.*]], i32 [[TMP0]]
899; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, i16* [[TMP1]], i32 0
900; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16* [[TMP2]] to <4 x i16>*
901; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i16>, <4 x i16>* [[TMP3]], align 2
902; CHECK-NEXT:    [[TMP4:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i32>
903; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i16, i16* [[Y:%.*]], i32 [[TMP0]]
904; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i16, i16* [[TMP5]], i32 0
905; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16* [[TMP6]] to <4 x i16>*
906; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i16>, <4 x i16>* [[TMP7]], align 2
907; CHECK-NEXT:    [[TMP8:%.*]] = sext <4 x i16> [[WIDE_LOAD1]] to <4 x i32>
908; CHECK-NEXT:    [[TMP9:%.*]] = mul nsw <4 x i32> [[TMP8]], [[TMP4]]
909; CHECK-NEXT:    [[TMP10]] = add <4 x i32> [[TMP9]], [[VEC_PHI]]
910; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
911; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
912; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !16
913; CHECK:       middle.block:
914; CHECK-NEXT:    [[TMP12:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[TMP10]])
915; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
916; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
917; CHECK:       scalar.ph:
918; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
919; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ]
920; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
921; CHECK:       for.body:
922; CHECK-NEXT:    [[I_011:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
923; CHECK-NEXT:    [[R_010:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
924; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, i16* [[X]], i32 [[I_011]]
925; CHECK-NEXT:    [[TMP13:%.*]] = load i16, i16* [[ARRAYIDX]], align 2
926; CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[TMP13]] to i32
927; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i16, i16* [[Y]], i32 [[I_011]]
928; CHECK-NEXT:    [[TMP14:%.*]] = load i16, i16* [[ARRAYIDX1]], align 2
929; CHECK-NEXT:    [[CONV2:%.*]] = sext i16 [[TMP14]] to i32
930; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]]
931; CHECK-NEXT:    [[ADD]] = add nsw i32 [[MUL]], [[R_010]]
932; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_011]], 1
933; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
934; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop !17
935; CHECK:       for.cond.cleanup.loopexit:
936; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ]
937; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
938; CHECK:       for.cond.cleanup:
939; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ]
940; CHECK-NEXT:    ret i32 [[R_0_LCSSA]]
941;
942entry:
943  %cmp9 = icmp sgt i32 %n, 0
944  br i1 %cmp9, label %for.body, label %for.cond.cleanup
945
946for.body:                                         ; preds = %entry, %for.body
947  %i.011 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
948  %r.010 = phi i32 [ %add, %for.body ], [ 0, %entry ]
949  %arrayidx = getelementptr inbounds i16, i16* %x, i32 %i.011
950  %0 = load i16, i16* %arrayidx, align 2
951  %conv = sext i16 %0 to i32
952  %arrayidx1 = getelementptr inbounds i16, i16* %y, i32 %i.011
953  %1 = load i16, i16* %arrayidx1, align 2
954  %conv2 = sext i16 %1 to i32
955  %mul = mul nsw i32 %conv2, %conv
956  %add = add nsw i32 %mul, %r.010
957  %inc = add nuw nsw i32 %i.011, 1
958  %exitcond = icmp eq i32 %inc, %n
959  br i1 %exitcond, label %for.cond.cleanup, label %for.body
960
961for.cond.cleanup:                                 ; preds = %for.body, %entry
962  %r.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
963  ret i32 %r.0.lcssa
964}
965
966define i32 @mla_i8_i32(i8* nocapture readonly %x, i8* nocapture readonly %y, i32 %n) #0 {
967; CHECK-LABEL: @mla_i8_i32(
968; CHECK-NEXT:  entry:
969; CHECK-NEXT:    [[CMP9:%.*]] = icmp sgt i32 [[N:%.*]], 0
970; CHECK-NEXT:    br i1 [[CMP9]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
971; CHECK:       for.body.preheader:
972; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
973; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
974; CHECK:       vector.ph:
975; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N]], 4
976; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
977; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
978; CHECK:       vector.body:
979; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
980; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
981; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
982; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, i8* [[X:%.*]], i32 [[TMP0]]
983; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[TMP1]], i32 0
984; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[TMP2]] to <4 x i8>*
985; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, <4 x i8>* [[TMP3]], align 1
986; CHECK-NEXT:    [[TMP4:%.*]] = zext <4 x i8> [[WIDE_LOAD]] to <4 x i32>
987; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, i8* [[Y:%.*]], i32 [[TMP0]]
988; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, i8* [[TMP5]], i32 0
989; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to <4 x i8>*
990; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i8>, <4 x i8>* [[TMP7]], align 1
991; CHECK-NEXT:    [[TMP8:%.*]] = zext <4 x i8> [[WIDE_LOAD1]] to <4 x i32>
992; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw nsw <4 x i32> [[TMP8]], [[TMP4]]
993; CHECK-NEXT:    [[TMP10]] = add <4 x i32> [[TMP9]], [[VEC_PHI]]
994; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
995; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
996; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !18
997; CHECK:       middle.block:
998; CHECK-NEXT:    [[TMP12:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[TMP10]])
999; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
1000; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
1001; CHECK:       scalar.ph:
1002; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
1003; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ]
1004; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
1005; CHECK:       for.body:
1006; CHECK-NEXT:    [[I_011:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
1007; CHECK-NEXT:    [[R_010:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
1008; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, i8* [[X]], i32 [[I_011]]
1009; CHECK-NEXT:    [[TMP13:%.*]] = load i8, i8* [[ARRAYIDX]], align 1
1010; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP13]] to i32
1011; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, i8* [[Y]], i32 [[I_011]]
1012; CHECK-NEXT:    [[TMP14:%.*]] = load i8, i8* [[ARRAYIDX1]], align 1
1013; CHECK-NEXT:    [[CONV2:%.*]] = zext i8 [[TMP14]] to i32
1014; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[CONV2]], [[CONV]]
1015; CHECK-NEXT:    [[ADD]] = add nuw nsw i32 [[MUL]], [[R_010]]
1016; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_011]], 1
1017; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
1018; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop !19
1019; CHECK:       for.cond.cleanup.loopexit:
1020; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ]
1021; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
1022; CHECK:       for.cond.cleanup:
1023; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ]
1024; CHECK-NEXT:    ret i32 [[R_0_LCSSA]]
1025;
1026entry:
1027  %cmp9 = icmp sgt i32 %n, 0
1028  br i1 %cmp9, label %for.body, label %for.cond.cleanup
1029
1030for.body:                                         ; preds = %entry, %for.body
1031  %i.011 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
1032  %r.010 = phi i32 [ %add, %for.body ], [ 0, %entry ]
1033  %arrayidx = getelementptr inbounds i8, i8* %x, i32 %i.011
1034  %0 = load i8, i8* %arrayidx, align 1
1035  %conv = zext i8 %0 to i32
1036  %arrayidx1 = getelementptr inbounds i8, i8* %y, i32 %i.011
1037  %1 = load i8, i8* %arrayidx1, align 1
1038  %conv2 = zext i8 %1 to i32
1039  %mul = mul nuw nsw i32 %conv2, %conv
1040  %add = add nuw nsw i32 %mul, %r.010
1041  %inc = add nuw nsw i32 %i.011, 1
1042  %exitcond = icmp eq i32 %inc, %n
1043  br i1 %exitcond, label %for.cond.cleanup, label %for.body
1044
1045for.cond.cleanup:                                 ; preds = %for.body, %entry
1046  %r.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
1047  ret i32 %r.0.lcssa
1048}
1049
1050define signext i16 @mla_i16_i16(i16* nocapture readonly %x, i16* nocapture readonly %y, i32 %n) #0 {
1051; CHECK-LABEL: @mla_i16_i16(
1052; CHECK-NEXT:  entry:
1053; CHECK-NEXT:    [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 0
1054; CHECK-NEXT:    br i1 [[CMP11]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
1055; CHECK:       for.body.preheader:
1056; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 8
1057; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
1058; CHECK:       vector.ph:
1059; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N]], 8
1060; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
1061; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
1062; CHECK:       vector.body:
1063; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
1064; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i16> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[VECTOR_BODY]] ]
1065; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
1066; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, i16* [[X:%.*]], i32 [[TMP0]]
1067; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, i16* [[TMP1]], i32 0
1068; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16* [[TMP2]] to <8 x i16>*
1069; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i16>, <8 x i16>* [[TMP3]], align 2
1070; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i16, i16* [[Y:%.*]], i32 [[TMP0]]
1071; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i16, i16* [[TMP4]], i32 0
1072; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16* [[TMP5]] to <8 x i16>*
1073; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <8 x i16>, <8 x i16>* [[TMP6]], align 2
1074; CHECK-NEXT:    [[TMP7:%.*]] = mul <8 x i16> [[WIDE_LOAD1]], [[WIDE_LOAD]]
1075; CHECK-NEXT:    [[TMP8]] = add <8 x i16> [[TMP7]], [[VEC_PHI]]
1076; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
1077; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
1078; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !20
1079; CHECK:       middle.block:
1080; CHECK-NEXT:    [[TMP10:%.*]] = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> [[TMP8]])
1081; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
1082; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
1083; CHECK:       scalar.ph:
1084; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
1085; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ]
1086; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
1087; CHECK:       for.body:
1088; CHECK-NEXT:    [[I_013:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
1089; CHECK-NEXT:    [[R_012:%.*]] = phi i16 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
1090; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, i16* [[X]], i32 [[I_013]]
1091; CHECK-NEXT:    [[TMP11:%.*]] = load i16, i16* [[ARRAYIDX]], align 2
1092; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i16, i16* [[Y]], i32 [[I_013]]
1093; CHECK-NEXT:    [[TMP12:%.*]] = load i16, i16* [[ARRAYIDX1]], align 2
1094; CHECK-NEXT:    [[MUL:%.*]] = mul i16 [[TMP12]], [[TMP11]]
1095; CHECK-NEXT:    [[ADD]] = add i16 [[MUL]], [[R_012]]
1096; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_013]], 1
1097; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
1098; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop !21
1099; CHECK:       for.cond.cleanup.loopexit:
1100; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i16 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ]
1101; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
1102; CHECK:       for.cond.cleanup:
1103; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ]
1104; CHECK-NEXT:    ret i16 [[R_0_LCSSA]]
1105;
1106entry:
1107  %cmp11 = icmp sgt i32 %n, 0
1108  br i1 %cmp11, label %for.body, label %for.cond.cleanup
1109
1110for.body:                                         ; preds = %entry, %for.body
1111  %i.013 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
1112  %r.012 = phi i16 [ %add, %for.body ], [ 0, %entry ]
1113  %arrayidx = getelementptr inbounds i16, i16* %x, i32 %i.013
1114  %0 = load i16, i16* %arrayidx, align 2
1115  %arrayidx1 = getelementptr inbounds i16, i16* %y, i32 %i.013
1116  %1 = load i16, i16* %arrayidx1, align 2
1117  %mul = mul i16 %1, %0
1118  %add = add i16 %mul, %r.012
1119  %inc = add nuw nsw i32 %i.013, 1
1120  %exitcond = icmp eq i32 %inc, %n
1121  br i1 %exitcond, label %for.cond.cleanup, label %for.body
1122
1123for.cond.cleanup:                                 ; preds = %for.body, %entry
1124  %r.0.lcssa = phi i16 [ 0, %entry ], [ %add, %for.body ]
1125  ret i16 %r.0.lcssa
1126}
1127
1128define signext i16 @mla_i8_i16(i8* nocapture readonly %x, i8* nocapture readonly %y, i32 %n) #0 {
1129; CHECK-LABEL: @mla_i8_i16(
1130; CHECK-NEXT:  entry:
1131; CHECK-NEXT:    [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 0
1132; CHECK-NEXT:    br i1 [[CMP11]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
1133; CHECK:       for.body.preheader:
1134; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 8
1135; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
1136; CHECK:       vector.ph:
1137; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N]], 8
1138; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
1139; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
1140; CHECK:       vector.body:
1141; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
1142; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i16> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
1143; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
1144; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, i8* [[X:%.*]], i32 [[TMP0]]
1145; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[TMP1]], i32 0
1146; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[TMP2]] to <8 x i8>*
1147; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i8>, <8 x i8>* [[TMP3]], align 1
1148; CHECK-NEXT:    [[TMP4:%.*]] = zext <8 x i8> [[WIDE_LOAD]] to <8 x i16>
1149; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, i8* [[Y:%.*]], i32 [[TMP0]]
1150; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, i8* [[TMP5]], i32 0
1151; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to <8 x i8>*
1152; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <8 x i8>, <8 x i8>* [[TMP7]], align 1
1153; CHECK-NEXT:    [[TMP8:%.*]] = zext <8 x i8> [[WIDE_LOAD1]] to <8 x i16>
1154; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw <8 x i16> [[TMP8]], [[TMP4]]
1155; CHECK-NEXT:    [[TMP10]] = add <8 x i16> [[TMP9]], [[VEC_PHI]]
1156; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
1157; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
1158; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !22
1159; CHECK:       middle.block:
1160; CHECK-NEXT:    [[TMP12:%.*]] = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> [[TMP10]])
1161; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
1162; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
1163; CHECK:       scalar.ph:
1164; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
1165; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ]
1166; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
1167; CHECK:       for.body:
1168; CHECK-NEXT:    [[I_013:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
1169; CHECK-NEXT:    [[R_012:%.*]] = phi i16 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
1170; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, i8* [[X]], i32 [[I_013]]
1171; CHECK-NEXT:    [[TMP13:%.*]] = load i8, i8* [[ARRAYIDX]], align 1
1172; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP13]] to i16
1173; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, i8* [[Y]], i32 [[I_013]]
1174; CHECK-NEXT:    [[TMP14:%.*]] = load i8, i8* [[ARRAYIDX1]], align 1
1175; CHECK-NEXT:    [[CONV2:%.*]] = zext i8 [[TMP14]] to i16
1176; CHECK-NEXT:    [[MUL:%.*]] = mul nuw i16 [[CONV2]], [[CONV]]
1177; CHECK-NEXT:    [[ADD]] = add i16 [[MUL]], [[R_012]]
1178; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_013]], 1
1179; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
1180; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop !23
1181; CHECK:       for.cond.cleanup.loopexit:
1182; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i16 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ]
1183; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
1184; CHECK:       for.cond.cleanup:
1185; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ]
1186; CHECK-NEXT:    ret i16 [[R_0_LCSSA]]
1187;
1188entry:
1189  %cmp11 = icmp sgt i32 %n, 0
1190  br i1 %cmp11, label %for.body, label %for.cond.cleanup
1191
1192for.body:                                         ; preds = %entry, %for.body
1193  %i.013 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
1194  %r.012 = phi i16 [ %add, %for.body ], [ 0, %entry ]
1195  %arrayidx = getelementptr inbounds i8, i8* %x, i32 %i.013
1196  %0 = load i8, i8* %arrayidx, align 1
1197  %conv = zext i8 %0 to i16
1198  %arrayidx1 = getelementptr inbounds i8, i8* %y, i32 %i.013
1199  %1 = load i8, i8* %arrayidx1, align 1
1200  %conv2 = zext i8 %1 to i16
1201  %mul = mul nuw i16 %conv2, %conv
1202  %add = add i16 %mul, %r.012
1203  %inc = add nuw nsw i32 %i.013, 1
1204  %exitcond = icmp eq i32 %inc, %n
1205  br i1 %exitcond, label %for.cond.cleanup, label %for.body
1206
1207for.cond.cleanup:                                 ; preds = %for.body, %entry
1208  %r.0.lcssa = phi i16 [ 0, %entry ], [ %add, %for.body ]
1209  ret i16 %r.0.lcssa
1210}
1211
1212define zeroext i8 @mla_i8_i8(i8* nocapture readonly %x, i8* nocapture readonly %y, i32 %n) #0 {
1213; CHECK-LABEL: @mla_i8_i8(
1214; CHECK-NEXT:  entry:
1215; CHECK-NEXT:    [[CMP10:%.*]] = icmp sgt i32 [[N:%.*]], 0
1216; CHECK-NEXT:    br i1 [[CMP10]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
1217; CHECK:       for.body.preheader:
1218; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 16
1219; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
1220; CHECK:       vector.ph:
1221; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N]], 16
1222; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
1223; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
1224; CHECK:       vector.body:
1225; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
1226; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i8> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[VECTOR_BODY]] ]
1227; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
1228; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, i8* [[X:%.*]], i32 [[TMP0]]
1229; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[TMP1]], i32 0
1230; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[TMP2]] to <16 x i8>*
1231; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, <16 x i8>* [[TMP3]], align 1
1232; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, i8* [[Y:%.*]], i32 [[TMP0]]
1233; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, i8* [[TMP4]], i32 0
1234; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8* [[TMP5]] to <16 x i8>*
1235; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, <16 x i8>* [[TMP6]], align 1
1236; CHECK-NEXT:    [[TMP7:%.*]] = mul <16 x i8> [[WIDE_LOAD1]], [[WIDE_LOAD]]
1237; CHECK-NEXT:    [[TMP8]] = add <16 x i8> [[TMP7]], [[VEC_PHI]]
1238; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 16
1239; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
1240; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !24
1241; CHECK:       middle.block:
1242; CHECK-NEXT:    [[TMP10:%.*]] = call i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8> [[TMP8]])
1243; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
1244; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
1245; CHECK:       scalar.ph:
1246; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
1247; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i8 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ]
1248; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
1249; CHECK:       for.body:
1250; CHECK-NEXT:    [[I_012:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
1251; CHECK-NEXT:    [[R_011:%.*]] = phi i8 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
1252; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, i8* [[X]], i32 [[I_012]]
1253; CHECK-NEXT:    [[TMP11:%.*]] = load i8, i8* [[ARRAYIDX]], align 1
1254; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, i8* [[Y]], i32 [[I_012]]
1255; CHECK-NEXT:    [[TMP12:%.*]] = load i8, i8* [[ARRAYIDX1]], align 1
1256; CHECK-NEXT:    [[MUL:%.*]] = mul i8 [[TMP12]], [[TMP11]]
1257; CHECK-NEXT:    [[ADD]] = add i8 [[MUL]], [[R_011]]
1258; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_012]], 1
1259; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
1260; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop !25
1261; CHECK:       for.cond.cleanup.loopexit:
1262; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i8 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ]
1263; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
1264; CHECK:       for.cond.cleanup:
1265; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i8 [ 0, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ]
1266; CHECK-NEXT:    ret i8 [[R_0_LCSSA]]
1267;
1268entry:
1269  %cmp10 = icmp sgt i32 %n, 0
1270  br i1 %cmp10, label %for.body, label %for.cond.cleanup
1271
1272for.body:                                         ; preds = %entry, %for.body
1273  %i.012 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
1274  %r.011 = phi i8 [ %add, %for.body ], [ 0, %entry ]
1275  %arrayidx = getelementptr inbounds i8, i8* %x, i32 %i.012
1276  %0 = load i8, i8* %arrayidx, align 1
1277  %arrayidx1 = getelementptr inbounds i8, i8* %y, i32 %i.012
1278  %1 = load i8, i8* %arrayidx1, align 1
1279  %mul = mul i8 %1, %0
1280  %add = add i8 %mul, %r.011
1281  %inc = add nuw nsw i32 %i.012, 1
1282  %exitcond = icmp eq i32 %inc, %n
1283  br i1 %exitcond, label %for.cond.cleanup, label %for.body
1284
1285for.cond.cleanup:                                 ; preds = %for.body, %entry
1286  %r.0.lcssa = phi i8 [ 0, %entry ], [ %add, %for.body ]
1287  ret i8 %r.0.lcssa
1288}
1289
1290attributes #0 = { "target-features"="+mve" }
1291