1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py 2; RUN: opt -loop-vectorize -force-vector-width=4 -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp -enable-arm-maskedgatscat -tail-predication=force-enabled -S %s -o - | FileCheck %s 3 4define void @test_stride1_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n) { 5; CHECK-LABEL: @test_stride1_4i32( 6; CHECK: vector.body: 7; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY:%.*]] ] 8; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> undef, i32 [[INDEX]], i32 0 9; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer 10; CHECK-NEXT: [[INDUCTION:%.*]] = add <4 x i32> [[BROADCAST_SPLAT2]], <i32 0, i32 1, i32 2, i32 3> 11; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0 12; CHECK-NEXT: [[TMP1:%.*]] = mul nuw nsw i32 [[TMP0]], 1 13; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i32 [[TMP1]], 2 14; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], i32 [[TMP2]] 15; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[BROADCAST_SPLAT:%.*]], i32 0 16; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[TMP0]], i32 [[TMP4]]) 17; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP3]], i32 0 18; CHECK-NEXT: [[TMP6:%.*]] = bitcast i32* [[TMP5]] to <4 x i32>* 19; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP6]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> undef) 20; CHECK-NEXT: [[TMP7:%.*]] = add nsw <4 x i32> <i32 5, i32 5, i32 5, i32 5>, [[WIDE_MASKED_LOAD]] 21; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[TMP0]] 22; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP8]], i32 0 23; CHECK-NEXT: [[TMP10:%.*]] = bitcast i32* [[TMP9]] to <4 x i32>* 24; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> [[TMP7]], <4 x i32>* [[TMP10]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]]) 25; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4 26; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC:%.*]] 27; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0 28; 29entry: 30 br label %for.body 31for.body: ; preds = %for.body.preheader, %for.body 32 %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ] 33 %mul = mul nuw nsw i32 %i.023, 1 34 %add5 = add nuw nsw i32 %mul, 2 35 %arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5 36 %0 = load i32, i32* %arrayidx6, align 4 37 %add7 = add nsw i32 5, %0 38 %arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023 39 store i32 %add7, i32* %arrayidx9, align 4 40 %inc = add nuw nsw i32 %i.023, 1 41 %exitcond.not = icmp eq i32 %inc, %n 42 br i1 %exitcond.not, label %end, label %for.body 43end: ; preds = %end, %entry 44 ret void 45} 46 47define void @test_stride-1_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n) { 48; CHECK-LABEL: @test_stride-1_4i32( 49; CHECK: vector.body: 50; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY:%.*]] ] 51; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[INDEX]], 0 52; CHECK-NEXT: [[TMP9:%.*]] = mul nuw nsw i32 [[TMP8]], -1 53; CHECK-NEXT: [[TMP10:%.*]] = add nuw nsw i32 [[TMP9]], 2 54; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], i32 [[TMP10]] 55; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TMP11]], i32 0 56; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[TMP12]], i32 -3 57; CHECK-NEXT: [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <4 x i32>* 58; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP14]], align 4 59; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0> 60; CHECK-NEXT: [[TMP15:%.*]] = add nsw <4 x i32> <i32 5, i32 5, i32 5, i32 5>, [[REVERSE]] 61; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[TMP8]] 62; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[TMP16]], i32 0 63; CHECK-NEXT: [[TMP18:%.*]] = bitcast i32* [[TMP17]] to <4 x i32>* 64; CHECK-NEXT: store <4 x i32> [[TMP15]], <4 x i32>* [[TMP18]], align 4 65; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4 66; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC:%.*]] 67; CHECK-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !4 68; 69entry: 70 br label %for.body 71for.body: ; preds = %for.body.preheader, %for.body 72 %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ] 73 %mul = mul nuw nsw i32 %i.023, -1 74 %add5 = add nuw nsw i32 %mul, 2 75 %arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5 76 %0 = load i32, i32* %arrayidx6, align 4 77 %add7 = add nsw i32 5, %0 78 %arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023 79 store i32 %add7, i32* %arrayidx9, align 4 80 %inc = add nuw nsw i32 %i.023, 1 81 %exitcond.not = icmp eq i32 %inc, %n 82 br i1 %exitcond.not, label %end, label %for.body 83end: ; preds = %end, %entry 84 ret void 85} 86 87define void @test_stride2_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n) { 88; 89; CHECK-LABEL: @test_stride2_4i32( 90; CHECK: vector.body: 91; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY:%.*]] ] 92; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 0 93; CHECK-NEXT: [[TMP3:%.*]] = mul nuw nsw i32 [[TMP2]], 2 94; CHECK-NEXT: [[TMP4:%.*]] = add nuw nsw i32 [[TMP3]], 2 95; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], i32 [[TMP4]] 96; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP5]], i32 0 97; CHECK-NEXT: [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <8 x i32>* 98; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, <8 x i32>* [[TMP7]], align 4 99; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6> 100; CHECK-NEXT: [[TMP8:%.*]] = add nsw <4 x i32> <i32 5, i32 5, i32 5, i32 5>, [[STRIDED_VEC]] 101; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[TMP2]] 102; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP9]], i32 0 103; CHECK-NEXT: [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <4 x i32>* 104; CHECK-NEXT: store <4 x i32> [[TMP8]], <4 x i32>* [[TMP11]], align 4 105; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4 106; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC:%.*]] 107; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !6 108; 109entry: 110 br label %for.body 111for.body: ; preds = %for.body.preheader, %for.body 112 %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ] 113 %mul = mul nuw nsw i32 %i.023, 2 114 %add5 = add nuw nsw i32 %mul, 2 115 %arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5 116 %0 = load i32, i32* %arrayidx6, align 4 117 %add7 = add nsw i32 5, %0 118 %arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023 119 store i32 %add7, i32* %arrayidx9, align 4 120 %inc = add nuw nsw i32 %i.023, 1 121 %exitcond.not = icmp eq i32 %inc, %n 122 br i1 %exitcond.not, label %end, label %for.body 123end: ; preds = %end, %entry 124 ret void 125} 126 127define void @test_stride3_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n) { 128; CHECK-LABEL: @test_stride3_4i32( 129; CHECK: vector.body: 130; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY:%.*]] ] 131; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 132; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0 133; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1 134; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 2 135; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 3 136; CHECK-NEXT: [[TMP4:%.*]] = mul nuw nsw <4 x i32> [[VEC_IND]], <i32 3, i32 3, i32 3, i32 3> 137; CHECK-NEXT: [[TMP5:%.*]] = add nuw nsw <4 x i32> [[TMP4]], <i32 2, i32 2, i32 2, i32 2> 138; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], <4 x i32> [[TMP5]] 139; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[BROADCAST_SPLAT:%.*]], i32 0 140; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[TMP0]], i32 [[TMP7]]) 141; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP6]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> undef) 142; CHECK-NEXT: [[TMP8:%.*]] = add nsw <4 x i32> <i32 5, i32 5, i32 5, i32 5>, [[WIDE_MASKED_GATHER]] 143; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[TMP0]] 144; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP9]], i32 0 145; CHECK-NEXT: [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <4 x i32>* 146; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> [[TMP8]], <4 x i32>* [[TMP11]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]]) 147; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4 148; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4> 149; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC:%.*]] 150; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !8 151; 152entry: 153 br label %for.body 154for.body: ; preds = %for.body.preheader, %for.body 155 %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ] 156 %mul = mul nuw nsw i32 %i.023, 3 157 %add5 = add nuw nsw i32 %mul, 2 158 %arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5 159 %0 = load i32, i32* %arrayidx6, align 4 160 %add7 = add nsw i32 5, %0 161 %arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023 162 store i32 %add7, i32* %arrayidx9, align 4 163 %inc = add nuw nsw i32 %i.023, 1 164 %exitcond.not = icmp eq i32 %inc, %n 165 br i1 %exitcond.not, label %end, label %for.body 166end: ; preds = %end, %entry 167 ret void 168} 169 170define void @test_stride4_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n) { 171; CHECK-LABEL: @test_stride4_4i32( 172; CHECK: vector.body: 173; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY:%.*]] ] 174; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 175; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0 176; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1 177; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 2 178; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 3 179; CHECK-NEXT: [[TMP4:%.*]] = mul nuw nsw <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4> 180; CHECK-NEXT: [[TMP5:%.*]] = add nuw nsw <4 x i32> [[TMP4]], <i32 2, i32 2, i32 2, i32 2> 181; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], <4 x i32> [[TMP5]] 182; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[BROADCAST_SPLAT:%.*]], i32 0 183; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[TMP0]], i32 [[TMP7]]) 184; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP6]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> undef) 185; CHECK-NEXT: [[TMP8:%.*]] = add nsw <4 x i32> <i32 5, i32 5, i32 5, i32 5>, [[WIDE_MASKED_GATHER]] 186; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[TMP0]] 187; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP9]], i32 0 188; CHECK-NEXT: [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <4 x i32>* 189; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> [[TMP8]], <4 x i32>* [[TMP11]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]]) 190; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4 191; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4> 192; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC:%.*]] 193; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP10:!llvm.loop !.*]] 194; 195entry: 196 br label %for.body 197for.body: ; preds = %for.body.preheader, %for.body 198 %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ] 199 %mul = mul nuw nsw i32 %i.023, 4 200 %add5 = add nuw nsw i32 %mul, 2 201 %arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5 202 %0 = load i32, i32* %arrayidx6, align 4 203 %add7 = add nsw i32 5, %0 204 %arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023 205 store i32 %add7, i32* %arrayidx9, align 4 206 %inc = add nuw nsw i32 %i.023, 1 207 %exitcond.not = icmp eq i32 %inc, %n 208 br i1 %exitcond.not, label %end, label %for.body 209end: ; preds = %end, %entry 210 ret void 211} 212 213define void @test_stride_loopinvar_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n, i32 %stride) { 214; CHECK-LABEL: @test_stride_loopinvar_4i32( 215; CHECK: vector.body: 216; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY:%.*]] ] 217; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> undef, i32 [[INDEX]], i32 0 218; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> undef, <4 x i32> zeroinitializer 219; CHECK-NEXT: [[INDUCTION:%.*]] = add <4 x i32> [[BROADCAST_SPLAT2]], <i32 0, i32 1, i32 2, i32 3> 220; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 0 221; CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i32 [[TMP1]], [[STRIDE:%.*]] 222; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i32 [[TMP2]], 2 223; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], i32 [[TMP3]] 224; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[BROADCAST_SPLAT]], i32 0 225; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[TMP1]], i32 [[TMP5]]) 226; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 0 227; CHECK-NEXT: [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>* 228; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP7]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> undef) 229; CHECK-NEXT: [[TMP8:%.*]] = add nsw <4 x i32> <i32 5, i32 5, i32 5, i32 5>, [[WIDE_MASKED_LOAD]] 230; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[TMP1]] 231; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP9]], i32 0 232; CHECK-NEXT: [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <4 x i32>* 233; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> [[TMP8]], <4 x i32>* [[TMP11]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]]) 234; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4 235; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC:%.*]] 236; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP12:!llvm.loop !.*]] 237; 238entry: 239 br label %for.body 240for.body: ; preds = %for.body.preheader, %for.body 241 %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ] 242 %mul = mul nuw nsw i32 %i.023, %stride 243 %add5 = add nuw nsw i32 %mul, 2 244 %arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5 245 %0 = load i32, i32* %arrayidx6, align 4 246 %add7 = add nsw i32 5, %0 247 %arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023 248 store i32 %add7, i32* %arrayidx9, align 4 249 %inc = add nuw nsw i32 %i.023, 1 250 %exitcond.not = icmp eq i32 %inc, %n 251 br i1 %exitcond.not, label %end, label %for.body 252end: ; preds = %end, %entry 253 ret void 254} 255 256define void @test_stride_noninvar_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n) { 257; CHECK-LABEL: @test_stride_noninvar_4i32( 258; CHECK: vector.body: 259; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY:%.*]] ] 260; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 261; CHECK-NEXT: [[VEC_IND2:%.*]] = phi <4 x i32> [ <i32 3, i32 11, i32 19, i32 27>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ] 262; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 0 263; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 1 264; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 2 265; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 3 266; CHECK-NEXT: [[TMP5:%.*]] = mul nuw nsw <4 x i32> [[VEC_IND]], [[VEC_IND2]] 267; CHECK-NEXT: [[TMP6:%.*]] = add nuw nsw <4 x i32> [[TMP5]], <i32 2, i32 2, i32 2, i32 2> 268; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], <4 x i32> [[TMP6]] 269; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP7]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef) 270; CHECK-NEXT: [[TMP8:%.*]] = add nsw <4 x i32> <i32 5, i32 5, i32 5, i32 5>, [[WIDE_MASKED_GATHER]] 271; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[TMP1]] 272; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP9]], i32 0 273; CHECK-NEXT: [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <4 x i32>* 274; CHECK-NEXT: store <4 x i32> [[TMP8]], <4 x i32>* [[TMP11]], align 4 275; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4 276; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4> 277; CHECK-NEXT: [[VEC_IND_NEXT3]] = add <4 x i32> [[VEC_IND2]], <i32 32, i32 32, i32 32, i32 32> 278; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC:%.*]] 279; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP14:!llvm.loop !.*]] 280; 281entry: 282 br label %for.body 283for.body: ; preds = %for.body.preheader, %for.body 284 %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ] 285 %stride = phi i32 [ %next.stride, %for.body ], [ 3, %entry ] 286 %mul = mul nuw nsw i32 %i.023, %stride 287 %add5 = add nuw nsw i32 %mul, 2 288 %arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5 289 %0 = load i32, i32* %arrayidx6, align 4 290 %add7 = add nsw i32 5, %0 291 %arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023 292 store i32 %add7, i32* %arrayidx9, align 4 293 %inc = add nuw nsw i32 %i.023, 1 294 %next.stride = add nuw nsw i32 %stride, 8 295 %exitcond.not = icmp eq i32 %inc, %n 296 br i1 %exitcond.not, label %end, label %for.body 297end: ; preds = %end, %entry 298 ret void 299} 300 301define void @test_stride_noninvar2_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n) { 302; CHECK-LABEL: @test_stride_noninvar2_4i32( 303; CHECK-NEXT: entry: 304; CHECK-NEXT: br label [[FOR_BODY:%.*]] 305; CHECK: for.body: 306; CHECK-NEXT: [[I_023:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ] 307; CHECK-NEXT: [[STRIDE:%.*]] = phi i32 [ [[NEXT_STRIDE:%.*]], [[FOR_BODY]] ], [ 3, [[ENTRY]] ] 308; CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[I_023]], [[STRIDE]] 309; CHECK-NEXT: [[ADD5:%.*]] = add nuw nsw i32 [[MUL]], 2 310; CHECK-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], i32 [[ADD5]] 311; CHECK-NEXT: [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX6]], align 4 312; CHECK-NEXT: [[ADD7:%.*]] = add nsw i32 5, [[TMP0]] 313; CHECK-NEXT: [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[I_023]] 314; CHECK-NEXT: store i32 [[ADD7]], i32* [[ARRAYIDX9]], align 4 315; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_023]], 1 316; CHECK-NEXT: [[NEXT_STRIDE]] = mul nuw nsw i32 [[STRIDE]], 8 317; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N:%.*]] 318; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[END:%.*]], label [[FOR_BODY]] 319; CHECK: end: 320; CHECK-NEXT: ret void 321; 322entry: 323 br label %for.body 324for.body: ; preds = %for.body.preheader, %for.body 325 %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ] 326 %stride = phi i32 [ %next.stride, %for.body ], [ 3, %entry ] 327 %mul = mul nuw nsw i32 %i.023, %stride 328 %add5 = add nuw nsw i32 %mul, 2 329 %arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5 330 %0 = load i32, i32* %arrayidx6, align 4 331 %add7 = add nsw i32 5, %0 332 %arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023 333 store i32 %add7, i32* %arrayidx9, align 4 334 %inc = add nuw nsw i32 %i.023, 1 335 %next.stride = mul nuw nsw i32 %stride, 8 336 %exitcond.not = icmp eq i32 %inc, %n 337 br i1 %exitcond.not, label %end, label %for.body 338end: ; preds = %end, %entry 339 ret void 340} 341 342define void @test_stride_noninvar3_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n, i32 %x) { 343; CHECK-LABEL: @test_stride_noninvar3_4i32( 344; CHECK: vector.body: 345; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY:%.*]] ] 346; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 347; CHECK-NEXT: [[VEC_IND4:%.*]] = phi <4 x i32> [ [[INDUCTION:%.*]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT5:%.*]], [[VECTOR_BODY]] ] 348; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 0 349; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 1 350; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[INDEX]], 2 351; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[INDEX]], 3 352; CHECK-NEXT: [[TMP7:%.*]] = mul nuw nsw <4 x i32> [[VEC_IND]], [[VEC_IND4]] 353; CHECK-NEXT: [[TMP8:%.*]] = add nuw nsw <4 x i32> [[TMP7]], <i32 2, i32 2, i32 2, i32 2> 354; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], <4 x i32> [[TMP8]] 355; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP9]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef) 356; CHECK-NEXT: [[TMP10:%.*]] = add nsw <4 x i32> <i32 5, i32 5, i32 5, i32 5>, [[WIDE_MASKED_GATHER]] 357; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[TMP3]] 358; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TMP11]], i32 0 359; CHECK-NEXT: [[TMP13:%.*]] = bitcast i32* [[TMP12]] to <4 x i32>* 360; CHECK-NEXT: store <4 x i32> [[TMP10]], <4 x i32>* [[TMP13]], align 4 361; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4 362; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4> 363; CHECK-NEXT: [[VEC_IND_NEXT5]] = add <4 x i32> [[VEC_IND4]], [[DOTSPLAT3:%.*]] 364; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC:%.*]] 365; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP16:!llvm.loop !.*]] 366; 367entry: 368 br label %for.body 369for.body: ; preds = %for.body.preheader, %for.body 370 %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ] 371 %stride = phi i32 [ %next.stride, %for.body ], [ 3, %entry ] 372 %mul = mul nuw nsw i32 %i.023, %stride 373 %add5 = add nuw nsw i32 %mul, 2 374 %arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5 375 %0 = load i32, i32* %arrayidx6, align 4 376 %add7 = add nsw i32 5, %0 377 %arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023 378 store i32 %add7, i32* %arrayidx9, align 4 379 %inc = add nuw nsw i32 %i.023, 1 380 %next.stride = add nuw nsw i32 %stride, %x 381 %exitcond.not = icmp eq i32 %inc, %n 382 br i1 %exitcond.not, label %end, label %for.body 383end: ; preds = %end, %entry 384 ret void 385} 386 387declare i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32>) 388declare <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*>, i32, <4 x i1>, <4 x i32>) 389declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32) 390declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32, <4 x i1>, <4 x i32>) 391declare void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32>, <4 x i32*>, i32, <4 x i1>) 392