1; RUN: opt -S -hints-allow-reordering=false -loop-vectorize -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue < %s | FileCheck %s 2 3target triple = "aarch64-unknown-linux-gnu" 4 5 6define void @simple_memset(i32 %val, i32* %ptr, i64 %n) #0 { 7; CHECK-LABEL: @simple_memset( 8; CHECK-NEXT: entry: 9; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1) 10; CHECK-NEXT: [[TMP2:%.*]] = sub i64 -1, [[UMAX]] 11; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 12; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 13; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]] 14; CHECK-NEXT: br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 15; CHECK: vector.ph: 16; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() 17; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 18; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() 19; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 20; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[TMP7]], 1 21; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP8]] 22; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]] 23; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 24; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]]) 25; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL:%.*]], i32 0 26; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer 27; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 28; CHECK: vector.body: 29; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ] 30; CHECK-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK4:%.*]], [[VECTOR_BODY]] ] 31; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX1]], 0 32; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, i32* [[PTR:%.*]], i64 [[TMP9]] 33; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, i32* [[TMP10]], i32 0 34; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <vscale x 4 x i32>* 35; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> [[BROADCAST_SPLAT]], <vscale x 4 x i32>* [[TMP12]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]]) 36; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() 37; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 4 38; CHECK-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX1]], [[TMP14]] 39; CHECK-NEXT: [[ACTIVE_LANE_MASK4]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT3]], i64 [[UMAX]]) 40; CHECK-NEXT: [[TMP15:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK4]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i32 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer) 41; CHECK-NEXT: [[TMP16:%.*]] = extractelement <vscale x 4 x i1> [[TMP15]], i32 0 42; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] 43; CHECK: middle.block: 44; CHECK-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] 45; 46entry: 47 br label %while.body 48 49while.body: ; preds = %while.body, %entry 50 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ] 51 %gep = getelementptr i32, i32* %ptr, i64 %index 52 store i32 %val, i32* %gep 53 %index.next = add nsw i64 %index, 1 54 %cmp10 = icmp ult i64 %index.next, %n 55 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0 56 57while.end.loopexit: ; preds = %while.body 58 ret void 59} 60 61 62define void @simple_memset_v4i32(i32 %val, i32* %ptr, i64 %n) #0 { 63; CHECK-LABEL: @simple_memset_v4i32( 64; CHECK-NEXT: entry: 65; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1) 66; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 67; CHECK: vector.ph: 68; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], 3 69; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4 70; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 71; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[UMAX]]) 72; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[VAL:%.*]], i32 0 73; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 74; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 75; CHECK: vector.body: 76; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ] 77; CHECK-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK4:%.*]], [[VECTOR_BODY]] ] 78; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX1]], 0 79; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, i32* [[PTR:%.*]], i64 [[TMP0]] 80; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, i32* [[TMP1]], i32 0 81; CHECK-NEXT: [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <4 x i32>* 82; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> [[BROADCAST_SPLAT]], <4 x i32>* [[TMP3]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK2]]) 83; CHECK-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX1]], 4 84; CHECK-NEXT: [[ACTIVE_LANE_MASK4]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX_NEXT3]], i64 [[UMAX]]) 85; CHECK-NEXT: [[TMP4:%.*]] = xor <4 x i1> [[ACTIVE_LANE_MASK4]], <i1 true, i1 true, i1 true, i1 true> 86; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0 87; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] 88; CHECK: middle.block: 89; CHECK-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] 90; 91entry: 92 br label %while.body 93 94while.body: ; preds = %while.body, %entry 95 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ] 96 %gep = getelementptr i32, i32* %ptr, i64 %index 97 store i32 %val, i32* %gep 98 %index.next = add nsw i64 %index, 1 99 %cmp10 = icmp ult i64 %index.next, %n 100 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !3 101 102while.end.loopexit: ; preds = %while.body 103 ret void 104} 105 106 107define void @simple_memcpy(i32* noalias %dst, i32* noalias %src, i64 %n) #0 { 108; CHECK-LABEL: @simple_memcpy( 109; CHECK-NEXT: entry: 110; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1) 111; CHECK-NEXT: [[TMP2:%.*]] = sub i64 -1, [[UMAX]] 112; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 113; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 114; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]] 115; CHECK-NEXT: br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 116; CHECK: vector.ph: 117; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() 118; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 119; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() 120; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 121; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[TMP7]], 1 122; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP8]] 123; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]] 124; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 125; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]]) 126; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 127; CHECK: vector.body: 128; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ] 129; CHECK-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK4:%.*]], [[VECTOR_BODY]] ] 130; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX1]], 0 131; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, i32* [[SRC:%.*]], i64 [[TMP9]] 132; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, i32* [[TMP10]], i32 0 133; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <vscale x 4 x i32>* 134; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP12]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 4 x i32> poison) 135; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i32, i32* [[DST:%.*]], i64 [[TMP9]] 136; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i32, i32* [[TMP13]], i32 0 137; CHECK-NEXT: [[TMP15:%.*]] = bitcast i32* [[TMP14]] to <vscale x 4 x i32>* 138; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> [[WIDE_MASKED_LOAD]], <vscale x 4 x i32>* [[TMP15]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]]) 139; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() 140; CHECK-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 4 141; CHECK-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX1]], [[TMP17]] 142; CHECK-NEXT: [[ACTIVE_LANE_MASK4]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT3]], i64 [[UMAX]]) 143; CHECK-NEXT: [[TMP18:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK4]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i32 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer) 144; CHECK-NEXT: [[TMP19:%.*]] = extractelement <vscale x 4 x i1> [[TMP18]], i32 0 145; CHECK-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] 146; CHECK: middle.block: 147; CHECK-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] 148; 149entry: 150 br label %while.body 151 152while.body: ; preds = %while.body, %entry 153 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ] 154 %gep1 = getelementptr i32, i32* %src, i64 %index 155 %val = load i32, i32* %gep1 156 %gep2 = getelementptr i32, i32* %dst, i64 %index 157 store i32 %val, i32* %gep2 158 %index.next = add nsw i64 %index, 1 159 %cmp10 = icmp ult i64 %index.next, %n 160 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0 161 162while.end.loopexit: ; preds = %while.body 163 ret void 164} 165 166 167define void @copy_stride4(i32* noalias %dst, i32* noalias %src, i64 %n) #0 { 168; CHECK-LABEL: @copy_stride4( 169; CHECK-NEXT: entry: 170; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 4) 171; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[UMAX]], -1 172; CHECK-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 2 173; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 174; CHECK-NEXT: [[TMP5:%.*]] = sub i64 -1, [[TMP2]] 175; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() 176; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], 4 177; CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[TMP5]], [[TMP4]] 178; CHECK-NEXT: br i1 [[TMP6]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 179; CHECK: vector.ph: 180; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() 181; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 4 182; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() 183; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 4 184; CHECK-NEXT: [[TMP11:%.*]] = sub i64 [[TMP10]], 1 185; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP2]], [[TMP11]] 186; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP8]] 187; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 188; CHECK-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 4 189; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP2]]) 190; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64() 191; CHECK-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i64> [[TMP12]], zeroinitializer 192; CHECK-NEXT: [[TMP14:%.*]] = mul <vscale x 4 x i64> [[TMP13]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 4, i32 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer) 193; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP14]] 194; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() 195; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 196; CHECK-NEXT: [[TMP17:%.*]] = mul i64 4, [[TMP16]] 197; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP17]], i32 0 198; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer 199; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 200; CHECK: vector.body: 201; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ] 202; CHECK-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK4:%.*]], [[VECTOR_BODY]] ] 203; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 204; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i32, i32* [[SRC:%.*]], <vscale x 4 x i64> [[VEC_IND]] 205; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> [[TMP18]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 4 x i32> undef) 206; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i32, i32* [[DST:%.*]], <vscale x 4 x i64> [[VEC_IND]] 207; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0i32(<vscale x 4 x i32> [[WIDE_MASKED_GATHER]], <vscale x 4 x i32*> [[TMP19]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]]) 208; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() 209; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 4 210; CHECK-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX1]], [[TMP21]] 211; CHECK-NEXT: [[ACTIVE_LANE_MASK4]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT3]], i64 [[TMP2]]) 212; CHECK-NEXT: [[TMP22:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK4]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i32 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer) 213; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]] 214; CHECK-NEXT: [[TMP23:%.*]] = extractelement <vscale x 4 x i1> [[TMP22]], i32 0 215; CHECK-NEXT: br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] 216; CHECK: middle.block: 217; CHECK-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] 218; 219entry: 220 br label %while.body 221 222while.body: ; preds = %while.body, %entry 223 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ] 224 %gep1 = getelementptr i32, i32* %src, i64 %index 225 %val = load i32, i32* %gep1 226 %gep2 = getelementptr i32, i32* %dst, i64 %index 227 store i32 %val, i32* %gep2 228 %index.next = add nsw i64 %index, 4 229 %cmp10 = icmp ult i64 %index.next, %n 230 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0 231 232while.end.loopexit: ; preds = %while.body 233 ret void 234} 235 236 237define void @simple_gather_scatter(i32* noalias %dst, i32* noalias %src, i32* noalias %ind, i64 %n) #0 { 238; CHECK-LABEL: @simple_gather_scatter( 239; CHECK-NEXT: entry: 240; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1) 241; CHECK-NEXT: [[TMP2:%.*]] = sub i64 -1, [[UMAX]] 242; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 243; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 244; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]] 245; CHECK-NEXT: br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 246; CHECK: vector.ph: 247; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() 248; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 249; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() 250; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 251; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[TMP7]], 1 252; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP8]] 253; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]] 254; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 255; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]]) 256; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 257; CHECK: vector.body: 258; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ] 259; CHECK-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK4:%.*]], [[VECTOR_BODY]] ] 260; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX1]], 0 261; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, i32* [[IND:%.*]], i64 [[TMP9]] 262; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, i32* [[TMP10]], i32 0 263; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <vscale x 4 x i32>* 264; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP12]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 4 x i32> poison) 265; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i32, i32* [[SRC:%.*]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]] 266; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> [[TMP13]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 4 x i32> undef) 267; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i32, i32* [[DST:%.*]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]] 268; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0i32(<vscale x 4 x i32> [[WIDE_MASKED_GATHER]], <vscale x 4 x i32*> [[TMP14]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]]) 269; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() 270; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 271; CHECK-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX1]], [[TMP16]] 272; CHECK-NEXT: [[ACTIVE_LANE_MASK4]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT3]], i64 [[UMAX]]) 273; CHECK-NEXT: [[TMP17:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK4]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i32 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer) 274; CHECK-NEXT: [[TMP18:%.*]] = extractelement <vscale x 4 x i1> [[TMP17]], i32 0 275; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] 276; CHECK: middle.block: 277; CHECK-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] 278; 279entry: 280 br label %while.body 281 282while.body: ; preds = %while.body, %entry 283 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ] 284 %gep1 = getelementptr i32, i32* %ind, i64 %index 285 %ind_val = load i32, i32* %gep1 286 %gep2 = getelementptr i32, i32* %src, i32 %ind_val 287 %val = load i32, i32* %gep2 288 %gep3 = getelementptr i32, i32* %dst, i32 %ind_val 289 store i32 %val, i32* %gep3 290 %index.next = add nsw i64 %index, 1 291 %cmp10 = icmp ult i64 %index.next, %n 292 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0 293 294while.end.loopexit: ; preds = %while.body 295 ret void 296} 297 298 299; The original loop had an unconditional uniform load. Let's make sure 300; we don't artificially create new predicated blocks for the load. 301define void @uniform_load(i32* noalias %dst, i32* noalias readonly %src, i64 %n) #0 { 302; CHECK-LABEL: @uniform_load( 303; CHECK-NEXT: entry: 304; CHECK-NEXT: [[TMP2:%.*]] = sub i64 -1, [[N:%.*]] 305; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 306; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 307; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]] 308; CHECK-NEXT: br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 309; CHECK: vector.ph: 310; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() 311; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 312; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() 313; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 314; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[TMP7]], 1 315; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP8]] 316; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]] 317; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 318; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]]) 319; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 320; CHECK: vector.body: 321; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 322; CHECK-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK2:%.*]], [[VECTOR_BODY]] ] 323; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 0 324; CHECK-NEXT: [[TMP10:%.*]] = load i32, i32* [[SRC:%.*]], align 4 325; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP10]], i32 0 326; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer 327; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 [[TMP9]] 328; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TMP11]], i32 0 329; CHECK-NEXT: [[TMP13:%.*]] = bitcast i32* [[TMP12]] to <vscale x 4 x i32>* 330; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> [[BROADCAST_SPLAT]], <vscale x 4 x i32>* [[TMP13]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK1]]) 331; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() 332; CHECK-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 4 333; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP15]] 334; CHECK-NEXT: [[ACTIVE_LANE_MASK2]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]]) 335; CHECK-NEXT: [[TMP16:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i32 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer) 336; CHECK-NEXT: [[TMP17:%.*]] = extractelement <vscale x 4 x i1> [[TMP16]], i32 0 337; CHECK-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] 338; CHECK: middle.block: 339; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]] 340; 341 342entry: 343 br label %for.body 344 345for.body: ; preds = %entry, %for.body 346 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 347 %val = load i32, i32* %src, align 4 348 %arrayidx = getelementptr inbounds i32, i32* %dst, i64 %indvars.iv 349 store i32 %val, i32* %arrayidx, align 4 350 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 351 %exitcond.not = icmp eq i64 %indvars.iv.next, %n 352 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0 353 354for.end: ; preds = %for.body, %entry 355 ret void 356} 357 358 359; The original loop had a conditional uniform load. In this case we actually 360; do need to perform conditional loads and so we end up using a gather instead. 361; However, we at least ensure the mask is the overlap of the loop predicate 362; and the original condition. 363define void @cond_uniform_load(i32* noalias %dst, i32* noalias readonly %src, i32* noalias readonly %cond, i64 %n) #0 { 364; CHECK-LABEL: @cond_uniform_load( 365; CHECK-NEXT: entry: 366; CHECK-NEXT: [[TMP2:%.*]] = sub i64 -1, [[N:%.*]] 367; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 368; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 369; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]] 370; CHECK-NEXT: br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 371; CHECK: vector.ph: 372; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() 373; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 374; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() 375; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 376; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[TMP7]], 1 377; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP8]] 378; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]] 379; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 380; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]]) 381; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32*> poison, i32* [[SRC:%.*]], i32 0 382; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32*> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32*> poison, <vscale x 4 x i32> zeroinitializer 383; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 384; CHECK: vector.body: 385; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ] 386; CHECK-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK4:%.*]], [[VECTOR_BODY]] ] 387; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX1]], 0 388; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[COND:%.*]], i64 [[TMP9]] 389; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP10]], i32 0 390; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <vscale x 4 x i32>* 391; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP12]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 4 x i32> poison) 392; CHECK-NEXT: [[TMP13:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], zeroinitializer 393; CHECK-NEXT: [[TMP14:%.*]] = xor <vscale x 4 x i1> [[TMP13]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i32 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer) 394; CHECK-NEXT: [[TMP15:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 4 x i1> [[TMP14]], <vscale x 4 x i1> zeroinitializer 395; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> [[BROADCAST_SPLAT]], i32 4, <vscale x 4 x i1> [[TMP15]], <vscale x 4 x i32> undef) 396; CHECK-NEXT: [[TMP16:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i1> zeroinitializer 397; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP16]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[WIDE_MASKED_GATHER]] 398; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 [[TMP9]] 399; CHECK-NEXT: [[TMP18:%.*]] = or <vscale x 4 x i1> [[TMP15]], [[TMP16]] 400; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, i32* [[TMP17]], i32 0 401; CHECK-NEXT: [[TMP20:%.*]] = bitcast i32* [[TMP19]] to <vscale x 4 x i32>* 402; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x i32>* [[TMP20]], i32 4, <vscale x 4 x i1> [[TMP18]]) 403; CHECK-NEXT: [[TMP21:%.*]] = call i64 @llvm.vscale.i64() 404; CHECK-NEXT: [[TMP22:%.*]] = mul i64 [[TMP21]], 4 405; CHECK-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX1]], [[TMP22]] 406; CHECK-NEXT: [[ACTIVE_LANE_MASK4]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT3]], i64 [[N]]) 407; CHECK-NEXT: [[TMP23:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK4]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i32 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer) 408; CHECK-NEXT: [[TMP24:%.*]] = extractelement <vscale x 4 x i1> [[TMP23]], i32 0 409; CHECK-NEXT: br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] 410; CHECK: middle.block: 411; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]] 412; 413 414entry: 415 br label %for.body 416 417for.body: ; preds = %entry, %if.end 418 %index = phi i64 [ %index.next, %if.end ], [ 0, %entry ] 419 %arrayidx = getelementptr inbounds i32, i32* %cond, i64 %index 420 %0 = load i32, i32* %arrayidx, align 4 421 %tobool.not = icmp eq i32 %0, 0 422 br i1 %tobool.not, label %if.end, label %if.then 423 424if.then: ; preds = %for.body 425 %1 = load i32, i32* %src, align 4 426 br label %if.end 427 428if.end: ; preds = %if.then, %for.body 429 %val.0 = phi i32 [ %1, %if.then ], [ 0, %for.body ] 430 %arrayidx1 = getelementptr inbounds i32, i32* %dst, i64 %index 431 store i32 %val.0, i32* %arrayidx1, align 4 432 %index.next = add nuw i64 %index, 1 433 %exitcond.not = icmp eq i64 %index.next, %n 434 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0 435 436for.end: ; preds = %for.inc, %entry 437 ret void 438} 439 440 441; The original loop had an unconditional uniform store. Let's make sure 442; we don't artificially create new predicated blocks for the load. 443define void @uniform_store(i32* noalias %dst, i32* noalias readonly %src, i64 %n) #0 { 444; CHECK-LABEL: @uniform_store( 445; CHECK-NEXT: entry: 446; CHECK-NEXT: [[TMP2:%.*]] = sub i64 -1, [[N:%.*]] 447; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 448; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 449; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]] 450; CHECK-NEXT: br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 451; CHECK: vector.ph: 452; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() 453; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 454; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() 455; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 456; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[TMP7]], 1 457; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP8]] 458; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]] 459; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 460; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]]) 461; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32*> poison, i32* [[DST:%.*]], i32 0 462; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32*> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32*> poison, <vscale x 4 x i32> zeroinitializer 463; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 464; CHECK: vector.body: 465; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 466; CHECK-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK2:%.*]], [[VECTOR_BODY]] ] 467; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 0 468; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 [[TMP9]] 469; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP10]], i32 0 470; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <vscale x 4 x i32>* 471; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP12]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK1]], <vscale x 4 x i32> poison) 472; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0i32(<vscale x 4 x i32> [[WIDE_MASKED_LOAD]], <vscale x 4 x i32*> [[BROADCAST_SPLAT]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK1]]) 473; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() 474; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 4 475; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP14]] 476; CHECK-NEXT: [[ACTIVE_LANE_MASK2]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]]) 477; CHECK-NEXT: [[TMP15:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i32 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer) 478; CHECK-NEXT: [[TMP16:%.*]] = extractelement <vscale x 4 x i1> [[TMP15]], i32 0 479; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] 480; CHECK: middle.block: 481; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]] 482; 483 484entry: 485 br label %for.body 486 487for.body: ; preds = %entry, %for.body 488 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 489 %arrayidx = getelementptr inbounds i32, i32* %src, i64 %indvars.iv 490 %val = load i32, i32* %arrayidx, align 4 491 store i32 %val, i32* %dst, align 4 492 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 493 %exitcond.not = icmp eq i64 %indvars.iv.next, %n 494 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0 495 496for.end: ; preds = %for.body, %entry 497 ret void 498} 499 500 501define void @simple_fdiv(float* noalias %dst, float* noalias %src, i64 %n) #0 { 502; CHECK-LABEL: @simple_fdiv( 503; CHECK-NEXT: entry: 504; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1) 505; CHECK-NEXT: [[TMP2:%.*]] = sub i64 -1, [[UMAX]] 506; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 507; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 508; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]] 509; CHECK-NEXT: br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 510; CHECK: vector.ph: 511; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() 512; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 513; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() 514; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 515; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[TMP7]], 1 516; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP8]] 517; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]] 518; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 519; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]]) 520; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 521; CHECK: vector.body: 522; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT4:%.*]], [[VECTOR_BODY]] ] 523; CHECK-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK5:%.*]], [[VECTOR_BODY]] ] 524; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX1]], 0 525; CHECK-NEXT: [[TMP10:%.*]] = getelementptr float, float* [[SRC:%.*]], i64 [[TMP9]] 526; CHECK-NEXT: [[TMP11:%.*]] = getelementptr float, float* [[DST:%.*]], i64 [[TMP9]] 527; CHECK-NEXT: [[TMP12:%.*]] = getelementptr float, float* [[TMP10]], i32 0 528; CHECK-NEXT: [[TMP13:%.*]] = bitcast float* [[TMP12]] to <vscale x 4 x float>* 529; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0nxv4f32(<vscale x 4 x float>* [[TMP13]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 4 x float> poison) 530; CHECK-NEXT: [[TMP14:%.*]] = getelementptr float, float* [[TMP11]], i32 0 531; CHECK-NEXT: [[TMP15:%.*]] = bitcast float* [[TMP14]] to <vscale x 4 x float>* 532; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0nxv4f32(<vscale x 4 x float>* [[TMP15]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 4 x float> poison) 533; CHECK-NEXT: [[TMP16:%.*]] = fdiv <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD3]] 534; CHECK-NEXT: [[TMP17:%.*]] = bitcast float* [[TMP14]] to <vscale x 4 x float>* 535; CHECK-NEXT: call void @llvm.masked.store.nxv4f32.p0nxv4f32(<vscale x 4 x float> [[TMP16]], <vscale x 4 x float>* [[TMP17]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]]) 536; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() 537; CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 4 538; CHECK-NEXT: [[INDEX_NEXT4]] = add i64 [[INDEX1]], [[TMP19]] 539; CHECK-NEXT: [[ACTIVE_LANE_MASK5]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT4]], i64 [[UMAX]]) 540; CHECK-NEXT: [[TMP20:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK5]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i32 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer) 541; CHECK-NEXT: [[TMP21:%.*]] = extractelement <vscale x 4 x i1> [[TMP20]], i32 0 542; CHECK-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] 543; CHECK: middle.block: 544; CHECK-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] 545; 546entry: 547 br label %while.body 548 549while.body: ; preds = %while.body, %entry 550 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ] 551 %gep1 = getelementptr float, float* %src, i64 %index 552 %gep2 = getelementptr float, float* %dst, i64 %index 553 %val1 = load float, float* %gep1 554 %val2 = load float, float* %gep2 555 %res = fdiv float %val1, %val2 556 store float %res, float* %gep2 557 %index.next = add nsw i64 %index, 1 558 %cmp10 = icmp ult i64 %index.next, %n 559 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0 560 561while.end.loopexit: ; preds = %while.body 562 ret void 563} 564 565; Negative tests where we don't expect tail-folding 566 567; Integer divides can throw exceptions and since we can't scalarize conditional 568; divides for scalable vectors we just don't bother vectorizing. 569define void @simple_idiv(i32* noalias %dst, i32* noalias %src, i64 %n) #0 { 570; CHECK-LABEL: @simple_idiv( 571; CHECK-NOT: vector.body 572; 573entry: 574 br label %while.body 575 576while.body: ; preds = %while.body, %entry 577 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ] 578 %gep1 = getelementptr i32, i32* %src, i64 %index 579 %gep2 = getelementptr i32, i32* %dst, i64 %index 580 %val1 = load i32, i32* %gep1 581 %val2 = load i32, i32* %gep2 582 %res = udiv i32 %val1, %val2 583 store i32 %res, i32* %gep2 584 %index.next = add nsw i64 %index, 1 585 %cmp10 = icmp ult i64 %index.next, %n 586 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0 587 588while.end.loopexit: ; preds = %while.body 589 ret void 590} 591 592!0 = distinct !{!0, !1, !2} 593!1 = !{!"llvm.loop.vectorize.width", i32 4} 594!2 = !{!"llvm.loop.vectorize.scalable.enable", i1 true} 595!3 = distinct !{!3, !4} 596!4 = !{!"llvm.loop.vectorize.width", i32 4} 597 598attributes #0 = { "target-features"="+sve" } 599