1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
2; RUN: opt -loop-vectorize -force-vector-width=4 -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp -tail-predication=force-enabled -S %s -o - | FileCheck %s
3
4target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
5
6define void @test_stride1_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n) {
7; CHECK-LABEL: @test_stride1_4i32(
8; CHECK-NEXT:  entry:
9; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
10; CHECK:       vector.ph:
11; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N:%.*]], 3
12; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 4
13; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
14; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
15; CHECK:       vector.body:
16; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
17; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
18; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[TMP0]], i32 [[N]])
19; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw nsw i32 [[TMP0]], 1
20; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i32 [[TMP1]], 2
21; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], i32 [[TMP2]]
22; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[TMP3]], i32 0
23; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32* [[TMP4]] to <4 x i32>*
24; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP5]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)
25; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> <i32 5, i32 5, i32 5, i32 5>, [[WIDE_MASKED_LOAD]]
26; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[TMP0]]
27; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TMP7]], i32 0
28; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>*
29; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> [[TMP6]], <4 x i32>* [[TMP9]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]])
30; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
31; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
32; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
33; CHECK:       middle.block:
34; CHECK-NEXT:    br i1 true, label [[END:%.*]], label [[SCALAR_PH]]
35; CHECK:       scalar.ph:
36; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
37; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
38; CHECK:       for.body:
39; CHECK-NEXT:    [[I_023:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
40; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[I_023]], 1
41; CHECK-NEXT:    [[ADD5:%.*]] = add nuw nsw i32 [[MUL]], 2
42; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, i32* [[DATA]], i32 [[ADD5]]
43; CHECK-NEXT:    [[TMP11:%.*]] = load i32, i32* [[ARRAYIDX6]], align 4
44; CHECK-NEXT:    [[ADD7:%.*]] = add nsw i32 5, [[TMP11]]
45; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[DST]], i32 [[I_023]]
46; CHECK-NEXT:    store i32 [[ADD7]], i32* [[ARRAYIDX9]], align 4
47; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_023]], 1
48; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
49; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[END]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
50; CHECK:       end:
51; CHECK-NEXT:    ret void
52;
53entry:
54  br label %for.body
55for.body:                                         ; preds = %for.body.preheader, %for.body
56  %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
57  %mul = mul nuw nsw i32 %i.023, 1
58  %add5 = add nuw nsw i32 %mul, 2
59  %arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5
60  %0 = load i32, i32* %arrayidx6, align 4
61  %add7 = add nsw i32 5, %0
62  %arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023
63  store i32 %add7, i32* %arrayidx9, align 4
64  %inc = add nuw nsw i32 %i.023, 1
65  %exitcond.not = icmp eq i32 %inc, %n
66  br i1 %exitcond.not, label %end, label %for.body
67end:                                 ; preds = %end, %entry
68  ret void
69}
70
71define void @test_stride-1_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n) {
72; CHECK-LABEL: @test_stride-1_4i32(
73; CHECK-NEXT:  entry:
74; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N:%.*]], 4
75; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
76; CHECK:       vector.ph:
77; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N]], 4
78; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
79; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
80; CHECK:       vector.body:
81; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
82; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
83; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw nsw i32 [[TMP0]], -1
84; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i32 [[TMP1]], 2
85; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], i32 [[TMP2]]
86; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[TMP3]], i32 0
87; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 -3
88; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[TMP5]] to <4 x i32>*
89; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP6]], align 4
90; CHECK-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
91; CHECK-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> <i32 5, i32 5, i32 5, i32 5>, [[REVERSE]]
92; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[TMP0]]
93; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP8]], i32 0
94; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32* [[TMP9]] to <4 x i32>*
95; CHECK-NEXT:    store <4 x i32> [[TMP7]], <4 x i32>* [[TMP10]], align 4
96; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
97; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
98; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
99; CHECK:       middle.block:
100; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
101; CHECK-NEXT:    br i1 [[CMP_N]], label [[END:%.*]], label [[SCALAR_PH]]
102; CHECK:       scalar.ph:
103; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
104; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
105; CHECK:       for.body:
106; CHECK-NEXT:    [[I_023:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
107; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[I_023]], -1
108; CHECK-NEXT:    [[ADD5:%.*]] = add nuw nsw i32 [[MUL]], 2
109; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, i32* [[DATA]], i32 [[ADD5]]
110; CHECK-NEXT:    [[TMP12:%.*]] = load i32, i32* [[ARRAYIDX6]], align 4
111; CHECK-NEXT:    [[ADD7:%.*]] = add nsw i32 5, [[TMP12]]
112; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[DST]], i32 [[I_023]]
113; CHECK-NEXT:    store i32 [[ADD7]], i32* [[ARRAYIDX9]], align 4
114; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_023]], 1
115; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
116; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[END]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
117; CHECK:       end:
118; CHECK-NEXT:    ret void
119;
120entry:
121  br label %for.body
122for.body:                                         ; preds = %for.body.preheader, %for.body
123  %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
124  %mul = mul nuw nsw i32 %i.023, -1
125  %add5 = add nuw nsw i32 %mul, 2
126  %arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5
127  %0 = load i32, i32* %arrayidx6, align 4
128  %add7 = add nsw i32 5, %0
129  %arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023
130  store i32 %add7, i32* %arrayidx9, align 4
131  %inc = add nuw nsw i32 %i.023, 1
132  %exitcond.not = icmp eq i32 %inc, %n
133  br i1 %exitcond.not, label %end, label %for.body
134end:                                 ; preds = %end, %entry
135  ret void
136}
137
138define void @test_stride2_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n) {
139;
140; CHECK-LABEL: @test_stride2_4i32(
141; CHECK-NEXT:  entry:
142; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[N:%.*]], 4
143; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
144; CHECK:       vector.ph:
145; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N]], 4
146; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
147; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[TMP0]], i32 4, i32 [[N_MOD_VF]]
148; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[TMP1]]
149; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
150; CHECK:       vector.body:
151; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
152; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[INDEX]], 0
153; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw nsw i32 [[TMP2]], 2
154; CHECK-NEXT:    [[TMP4:%.*]] = add nuw nsw i32 [[TMP3]], 2
155; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], i32 [[TMP4]]
156; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP5]], i32 0
157; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <8 x i32>*
158; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i32>, <8 x i32>* [[TMP7]], align 4
159; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
160; CHECK-NEXT:    [[TMP8:%.*]] = add nsw <4 x i32> <i32 5, i32 5, i32 5, i32 5>, [[STRIDED_VEC]]
161; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[TMP2]]
162; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP9]], i32 0
163; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <4 x i32>*
164; CHECK-NEXT:    store <4 x i32> [[TMP8]], <4 x i32>* [[TMP11]], align 4
165; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
166; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
167; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
168; CHECK:       middle.block:
169; CHECK-NEXT:    br label [[SCALAR_PH]]
170; CHECK:       scalar.ph:
171; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
172; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
173; CHECK:       for.body:
174; CHECK-NEXT:    [[I_023:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
175; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[I_023]], 2
176; CHECK-NEXT:    [[ADD5:%.*]] = add nuw nsw i32 [[MUL]], 2
177; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, i32* [[DATA]], i32 [[ADD5]]
178; CHECK-NEXT:    [[TMP13:%.*]] = load i32, i32* [[ARRAYIDX6]], align 4
179; CHECK-NEXT:    [[ADD7:%.*]] = add nsw i32 5, [[TMP13]]
180; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[DST]], i32 [[I_023]]
181; CHECK-NEXT:    store i32 [[ADD7]], i32* [[ARRAYIDX9]], align 4
182; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_023]], 1
183; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
184; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
185; CHECK:       end:
186; CHECK-NEXT:    ret void
187;
188entry:
189  br label %for.body
190for.body:                                         ; preds = %for.body.preheader, %for.body
191  %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
192  %mul = mul nuw nsw i32 %i.023, 2
193  %add5 = add nuw nsw i32 %mul, 2
194  %arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5
195  %0 = load i32, i32* %arrayidx6, align 4
196  %add7 = add nsw i32 5, %0
197  %arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023
198  store i32 %add7, i32* %arrayidx9, align 4
199  %inc = add nuw nsw i32 %i.023, 1
200  %exitcond.not = icmp eq i32 %inc, %n
201  br i1 %exitcond.not, label %end, label %for.body
202end:                                 ; preds = %end, %entry
203  ret void
204}
205
206define void @test_stride3_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n) {
207; CHECK-LABEL: @test_stride3_4i32(
208; CHECK-NEXT:  entry:
209; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
210; CHECK:       vector.ph:
211; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N:%.*]], 3
212; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 4
213; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
214; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
215; CHECK:       vector.body:
216; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
217; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
218; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
219; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[TMP0]], i32 [[N]])
220; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw nsw <4 x i32> [[VEC_IND]], <i32 3, i32 3, i32 3, i32 3>
221; CHECK-NEXT:    [[TMP5:%.*]] = add nuw nsw <4 x i32> [[TMP4]], <i32 2, i32 2, i32 2, i32 2>
222; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], <4 x i32> [[TMP5]]
223; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP6]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> undef)
224; CHECK-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> <i32 5, i32 5, i32 5, i32 5>, [[WIDE_MASKED_GATHER]]
225; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[TMP0]]
226; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP8]], i32 0
227; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32* [[TMP9]] to <4 x i32>*
228; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> [[TMP7]], <4 x i32>* [[TMP10]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]])
229; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
230; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
231; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
232; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
233; CHECK:       middle.block:
234; CHECK-NEXT:    br i1 true, label [[END:%.*]], label [[SCALAR_PH]]
235; CHECK:       scalar.ph:
236; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
237; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
238; CHECK:       for.body:
239; CHECK-NEXT:    [[I_023:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
240; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[I_023]], 3
241; CHECK-NEXT:    [[ADD5:%.*]] = add nuw nsw i32 [[MUL]], 2
242; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, i32* [[DATA]], i32 [[ADD5]]
243; CHECK-NEXT:    [[TMP12:%.*]] = load i32, i32* [[ARRAYIDX6]], align 4
244; CHECK-NEXT:    [[ADD7:%.*]] = add nsw i32 5, [[TMP12]]
245; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[DST]], i32 [[I_023]]
246; CHECK-NEXT:    store i32 [[ADD7]], i32* [[ARRAYIDX9]], align 4
247; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_023]], 1
248; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
249; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[END]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
250; CHECK:       end:
251; CHECK-NEXT:    ret void
252;
253entry:
254  br label %for.body
255for.body:                                         ; preds = %for.body.preheader, %for.body
256  %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
257  %mul = mul nuw nsw i32 %i.023, 3
258  %add5 = add nuw nsw i32 %mul, 2
259  %arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5
260  %0 = load i32, i32* %arrayidx6, align 4
261  %add7 = add nsw i32 5, %0
262  %arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023
263  store i32 %add7, i32* %arrayidx9, align 4
264  %inc = add nuw nsw i32 %i.023, 1
265  %exitcond.not = icmp eq i32 %inc, %n
266  br i1 %exitcond.not, label %end, label %for.body
267end:                                 ; preds = %end, %entry
268  ret void
269}
270
271define void @test_stride4_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n) {
272; CHECK-LABEL: @test_stride4_4i32(
273; CHECK-NEXT:  entry:
274; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
275; CHECK:       vector.ph:
276; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N:%.*]], 3
277; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 4
278; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
279; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
280; CHECK:       vector.body:
281; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
282; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
283; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
284; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[TMP0]], i32 [[N]])
285; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw nsw <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
286; CHECK-NEXT:    [[TMP5:%.*]] = add nuw nsw <4 x i32> [[TMP4]], <i32 2, i32 2, i32 2, i32 2>
287; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], <4 x i32> [[TMP5]]
288; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP6]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> undef)
289; CHECK-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> <i32 5, i32 5, i32 5, i32 5>, [[WIDE_MASKED_GATHER]]
290; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[TMP0]]
291; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP8]], i32 0
292; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32* [[TMP9]] to <4 x i32>*
293; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> [[TMP7]], <4 x i32>* [[TMP10]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]])
294; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
295; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
296; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
297; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
298; CHECK:       middle.block:
299; CHECK-NEXT:    br i1 true, label [[END:%.*]], label [[SCALAR_PH]]
300; CHECK:       scalar.ph:
301; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
302; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
303; CHECK:       for.body:
304; CHECK-NEXT:    [[I_023:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
305; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[I_023]], 4
306; CHECK-NEXT:    [[ADD5:%.*]] = add nuw nsw i32 [[MUL]], 2
307; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, i32* [[DATA]], i32 [[ADD5]]
308; CHECK-NEXT:    [[TMP12:%.*]] = load i32, i32* [[ARRAYIDX6]], align 4
309; CHECK-NEXT:    [[ADD7:%.*]] = add nsw i32 5, [[TMP12]]
310; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[DST]], i32 [[I_023]]
311; CHECK-NEXT:    store i32 [[ADD7]], i32* [[ARRAYIDX9]], align 4
312; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_023]], 1
313; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
314; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[END]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
315; CHECK:       end:
316; CHECK-NEXT:    ret void
317;
318entry:
319  br label %for.body
320for.body:                                         ; preds = %for.body.preheader, %for.body
321  %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
322  %mul = mul nuw nsw i32 %i.023, 4
323  %add5 = add nuw nsw i32 %mul, 2
324  %arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5
325  %0 = load i32, i32* %arrayidx6, align 4
326  %add7 = add nsw i32 5, %0
327  %arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023
328  store i32 %add7, i32* %arrayidx9, align 4
329  %inc = add nuw nsw i32 %i.023, 1
330  %exitcond.not = icmp eq i32 %inc, %n
331  br i1 %exitcond.not, label %end, label %for.body
332end:                                 ; preds = %end, %entry
333  ret void
334}
335
336define void @test_stride_loopinvar_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n, i32 %stride) {
337; CHECK-LABEL: @test_stride_loopinvar_4i32(
338; CHECK-NEXT:  entry:
339; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
340; CHECK:       vector.scevcheck:
341; CHECK-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i32 [[STRIDE:%.*]], 1
342; CHECK-NEXT:    br i1 [[IDENT_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
343; CHECK:       vector.ph:
344; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N:%.*]], 3
345; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 4
346; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
347; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
348; CHECK:       vector.body:
349; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
350; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[INDEX]], 0
351; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[TMP1]], i32 [[N]])
352; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw nsw i32 [[TMP1]], [[STRIDE]]
353; CHECK-NEXT:    [[TMP3:%.*]] = add nuw nsw i32 [[TMP2]], 2
354; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], i32 [[TMP3]]
355; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 0
356; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[TMP5]] to <4 x i32>*
357; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP6]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)
358; CHECK-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> <i32 5, i32 5, i32 5, i32 5>, [[WIDE_MASKED_LOAD]]
359; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[TMP1]]
360; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP8]], i32 0
361; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32* [[TMP9]] to <4 x i32>*
362; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> [[TMP7]], <4 x i32>* [[TMP10]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]])
363; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
364; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
365; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
366; CHECK:       middle.block:
367; CHECK-NEXT:    br i1 true, label [[END:%.*]], label [[SCALAR_PH]]
368; CHECK:       scalar.ph:
369; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
370; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
371; CHECK:       for.body:
372; CHECK-NEXT:    [[I_023:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
373; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[I_023]], [[STRIDE]]
374; CHECK-NEXT:    [[ADD5:%.*]] = add nuw nsw i32 [[MUL]], 2
375; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, i32* [[DATA]], i32 [[ADD5]]
376; CHECK-NEXT:    [[TMP12:%.*]] = load i32, i32* [[ARRAYIDX6]], align 4
377; CHECK-NEXT:    [[ADD7:%.*]] = add nsw i32 5, [[TMP12]]
378; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[DST]], i32 [[I_023]]
379; CHECK-NEXT:    store i32 [[ADD7]], i32* [[ARRAYIDX9]], align 4
380; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_023]], 1
381; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
382; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[END]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
383; CHECK:       end:
384; CHECK-NEXT:    ret void
385;
386entry:
387  br label %for.body
388for.body:                                         ; preds = %for.body.preheader, %for.body
389  %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
390  %mul = mul nuw nsw i32 %i.023, %stride
391  %add5 = add nuw nsw i32 %mul, 2
392  %arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5
393  %0 = load i32, i32* %arrayidx6, align 4
394  %add7 = add nsw i32 5, %0
395  %arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023
396  store i32 %add7, i32* %arrayidx9, align 4
397  %inc = add nuw nsw i32 %i.023, 1
398  %exitcond.not = icmp eq i32 %inc, %n
399  br i1 %exitcond.not, label %end, label %for.body
400end:                                 ; preds = %end, %entry
401  ret void
402}
403
404define void @test_stride_noninvar_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n) {
405; CHECK-LABEL: @test_stride_noninvar_4i32(
406; CHECK-NEXT:  entry:
407; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N:%.*]], 4
408; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
409; CHECK:       vector.ph:
410; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N]], 4
411; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
412; CHECK-NEXT:    [[TMP0:%.*]] = mul i32 [[N_VEC]], 8
413; CHECK-NEXT:    [[IND_END:%.*]] = add i32 3, [[TMP0]]
414; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
415; CHECK:       vector.body:
416; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
417; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
418; CHECK-NEXT:    [[VEC_IND2:%.*]] = phi <4 x i32> [ <i32 3, i32 11, i32 19, i32 27>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
419; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[INDEX]], 0
420; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw nsw <4 x i32> [[VEC_IND]], [[VEC_IND2]]
421; CHECK-NEXT:    [[TMP6:%.*]] = add nuw nsw <4 x i32> [[TMP5]], <i32 2, i32 2, i32 2, i32 2>
422; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], <4 x i32> [[TMP6]]
423; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP7]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
424; CHECK-NEXT:    [[TMP8:%.*]] = add nsw <4 x i32> <i32 5, i32 5, i32 5, i32 5>, [[WIDE_MASKED_GATHER]]
425; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[TMP1]]
426; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP9]], i32 0
427; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <4 x i32>*
428; CHECK-NEXT:    store <4 x i32> [[TMP8]], <4 x i32>* [[TMP11]], align 4
429; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
430; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
431; CHECK-NEXT:    [[VEC_IND_NEXT3]] = add <4 x i32> [[VEC_IND2]], <i32 32, i32 32, i32 32, i32 32>
432; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
433; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
434; CHECK:       middle.block:
435; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
436; CHECK-NEXT:    br i1 [[CMP_N]], label [[END:%.*]], label [[SCALAR_PH]]
437; CHECK:       scalar.ph:
438; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
439; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY]] ]
440; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
441; CHECK:       for.body:
442; CHECK-NEXT:    [[I_023:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
443; CHECK-NEXT:    [[STRIDE:%.*]] = phi i32 [ [[NEXT_STRIDE:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
444; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[I_023]], [[STRIDE]]
445; CHECK-NEXT:    [[ADD5:%.*]] = add nuw nsw i32 [[MUL]], 2
446; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, i32* [[DATA]], i32 [[ADD5]]
447; CHECK-NEXT:    [[TMP13:%.*]] = load i32, i32* [[ARRAYIDX6]], align 4
448; CHECK-NEXT:    [[ADD7:%.*]] = add nsw i32 5, [[TMP13]]
449; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[DST]], i32 [[I_023]]
450; CHECK-NEXT:    store i32 [[ADD7]], i32* [[ARRAYIDX9]], align 4
451; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_023]], 1
452; CHECK-NEXT:    [[NEXT_STRIDE]] = add nuw nsw i32 [[STRIDE]], 8
453; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
454; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[END]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
455; CHECK:       end:
456; CHECK-NEXT:    ret void
457;
458entry:
459  br label %for.body
460for.body:                                         ; preds = %for.body.preheader, %for.body
461  %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
462  %stride = phi i32 [ %next.stride, %for.body ], [ 3, %entry ]
463  %mul = mul nuw nsw i32 %i.023, %stride
464  %add5 = add nuw nsw i32 %mul, 2
465  %arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5
466  %0 = load i32, i32* %arrayidx6, align 4
467  %add7 = add nsw i32 5, %0
468  %arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023
469  store i32 %add7, i32* %arrayidx9, align 4
470  %inc = add nuw nsw i32 %i.023, 1
471  %next.stride = add nuw nsw i32 %stride, 8
472  %exitcond.not = icmp eq i32 %inc, %n
473  br i1 %exitcond.not, label %end, label %for.body
474end:                                 ; preds = %end, %entry
475  ret void
476}
477
478define void @test_stride_noninvar2_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n) {
479; CHECK-LABEL: @test_stride_noninvar2_4i32(
480; CHECK-NEXT:  entry:
481; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
482; CHECK:       for.body:
483; CHECK-NEXT:    [[I_023:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
484; CHECK-NEXT:    [[STRIDE:%.*]] = phi i32 [ [[NEXT_STRIDE:%.*]], [[FOR_BODY]] ], [ 3, [[ENTRY]] ]
485; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[I_023]], [[STRIDE]]
486; CHECK-NEXT:    [[ADD5:%.*]] = add nuw nsw i32 [[MUL]], 2
487; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], i32 [[ADD5]]
488; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX6]], align 4
489; CHECK-NEXT:    [[ADD7:%.*]] = add nsw i32 5, [[TMP0]]
490; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[I_023]]
491; CHECK-NEXT:    store i32 [[ADD7]], i32* [[ARRAYIDX9]], align 4
492; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_023]], 1
493; CHECK-NEXT:    [[NEXT_STRIDE]] = mul nuw nsw i32 [[STRIDE]], 8
494; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N:%.*]]
495; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[END:%.*]], label [[FOR_BODY]]
496; CHECK:       end:
497; CHECK-NEXT:    ret void
498;
499entry:
500  br label %for.body
501for.body:                                         ; preds = %for.body.preheader, %for.body
502  %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
503  %stride = phi i32 [ %next.stride, %for.body ], [ 3, %entry ]
504  %mul = mul nuw nsw i32 %i.023, %stride
505  %add5 = add nuw nsw i32 %mul, 2
506  %arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5
507  %0 = load i32, i32* %arrayidx6, align 4
508  %add7 = add nsw i32 5, %0
509  %arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023
510  store i32 %add7, i32* %arrayidx9, align 4
511  %inc = add nuw nsw i32 %i.023, 1
512  %next.stride = mul nuw nsw i32 %stride, 8
513  %exitcond.not = icmp eq i32 %inc, %n
514  br i1 %exitcond.not, label %end, label %for.body
515end:                                 ; preds = %end, %entry
516  ret void
517}
518
519define void @test_stride_noninvar3_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n, i32 %x) {
520; CHECK-LABEL: @test_stride_noninvar3_4i32(
521; CHECK-NEXT:  entry:
522; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N:%.*]], 4
523; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
524; CHECK:       vector.ph:
525; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N]], 4
526; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
527; CHECK-NEXT:    [[TMP0:%.*]] = mul i32 [[N_VEC]], [[X:%.*]]
528; CHECK-NEXT:    [[IND_END:%.*]] = add i32 3, [[TMP0]]
529; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i32 0
530; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
531; CHECK-NEXT:    [[TMP1:%.*]] = mul <4 x i32> <i32 0, i32 1, i32 2, i32 3>, [[DOTSPLAT]]
532; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i32> <i32 3, i32 3, i32 3, i32 3>, [[TMP1]]
533; CHECK-NEXT:    [[TMP2:%.*]] = mul i32 [[X]], 4
534; CHECK-NEXT:    [[DOTSPLATINSERT2:%.*]] = insertelement <4 x i32> poison, i32 [[TMP2]], i32 0
535; CHECK-NEXT:    [[DOTSPLAT3:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT2]], <4 x i32> poison, <4 x i32> zeroinitializer
536; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
537; CHECK:       vector.body:
538; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
539; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
540; CHECK-NEXT:    [[VEC_IND4:%.*]] = phi <4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT5:%.*]], [[VECTOR_BODY]] ]
541; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[INDEX]], 0
542; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw nsw <4 x i32> [[VEC_IND]], [[VEC_IND4]]
543; CHECK-NEXT:    [[TMP8:%.*]] = add nuw nsw <4 x i32> [[TMP7]], <i32 2, i32 2, i32 2, i32 2>
544; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], <4 x i32> [[TMP8]]
545; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP9]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
546; CHECK-NEXT:    [[TMP10:%.*]] = add nsw <4 x i32> <i32 5, i32 5, i32 5, i32 5>, [[WIDE_MASKED_GATHER]]
547; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[TMP3]]
548; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TMP11]], i32 0
549; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i32* [[TMP12]] to <4 x i32>*
550; CHECK-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* [[TMP13]], align 4
551; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
552; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
553; CHECK-NEXT:    [[VEC_IND_NEXT5]] = add <4 x i32> [[VEC_IND4]], [[DOTSPLAT3]]
554; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
555; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
556; CHECK:       middle.block:
557; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
558; CHECK-NEXT:    br i1 [[CMP_N]], label [[END:%.*]], label [[SCALAR_PH]]
559; CHECK:       scalar.ph:
560; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
561; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY]] ]
562; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
563; CHECK:       for.body:
564; CHECK-NEXT:    [[I_023:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
565; CHECK-NEXT:    [[STRIDE:%.*]] = phi i32 [ [[NEXT_STRIDE:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
566; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[I_023]], [[STRIDE]]
567; CHECK-NEXT:    [[ADD5:%.*]] = add nuw nsw i32 [[MUL]], 2
568; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, i32* [[DATA]], i32 [[ADD5]]
569; CHECK-NEXT:    [[TMP15:%.*]] = load i32, i32* [[ARRAYIDX6]], align 4
570; CHECK-NEXT:    [[ADD7:%.*]] = add nsw i32 5, [[TMP15]]
571; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[DST]], i32 [[I_023]]
572; CHECK-NEXT:    store i32 [[ADD7]], i32* [[ARRAYIDX9]], align 4
573; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_023]], 1
574; CHECK-NEXT:    [[NEXT_STRIDE]] = add nuw nsw i32 [[STRIDE]], [[X]]
575; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
576; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[END]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
577; CHECK:       end:
578; CHECK-NEXT:    ret void
579;
580entry:
581  br label %for.body
582for.body:                                         ; preds = %for.body.preheader, %for.body
583  %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
584  %stride = phi i32 [ %next.stride, %for.body ], [ 3, %entry ]
585  %mul = mul nuw nsw i32 %i.023, %stride
586  %add5 = add nuw nsw i32 %mul, 2
587  %arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5
588  %0 = load i32, i32* %arrayidx6, align 4
589  %add7 = add nsw i32 5, %0
590  %arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023
591  store i32 %add7, i32* %arrayidx9, align 4
592  %inc = add nuw nsw i32 %i.023, 1
593  %next.stride = add nuw nsw i32 %stride, %x
594  %exitcond.not = icmp eq i32 %inc, %n
595  br i1 %exitcond.not, label %end, label %for.body
596end:                                 ; preds = %end, %entry
597  ret void
598}
599
600declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
601declare <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*>, i32, <4 x i1>, <4 x i32>)
602declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32)
603declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32, <4 x i1>, <4 x i32>)
604declare void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32>, <4 x i32*>, i32, <4 x i1>)
605