1; RUN: opt -S -hints-allow-reordering=false -loop-vectorize -prefer-predicate-over-epilogue=predicate-dont-vectorize -prefer-inloop-reductions < %s | FileCheck %s 2; RUN: opt -S -hints-allow-reordering=false -loop-vectorize -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -prefer-inloop-reductions < %s | FileCheck %s 3 4target triple = "aarch64-unknown-linux-gnu" 5 6 7define void @simple_memset(i32 %val, i32* %ptr, i64 %n) #0 { 8; CHECK-LABEL: @simple_memset( 9; CHECK-NEXT: entry: 10; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1) 11; CHECK-NEXT: [[TMP2:%.*]] = sub i64 -1, [[UMAX]] 12; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 13; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 14; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]] 15; CHECK-NEXT: br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 16; CHECK: vector.ph: 17; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() 18; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 19; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() 20; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 21; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[TMP7]], 1 22; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP8]] 23; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]] 24; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 25; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]]) 26; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL:%.*]], i32 0 27; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer 28; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 29; CHECK: vector.body: 30; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ] 31; CHECK-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK4:%.*]], [[VECTOR_BODY]] ] 32; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX1]], 0 33; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, i32* [[PTR:%.*]], i64 [[TMP9]] 34; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, i32* [[TMP10]], i32 0 35; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <vscale x 4 x i32>* 36; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> [[BROADCAST_SPLAT]], <vscale x 4 x i32>* [[TMP12]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]]) 37; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() 38; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 4 39; CHECK-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX1]], [[TMP14]] 40; CHECK-NEXT: [[ACTIVE_LANE_MASK4]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT3]], i64 [[UMAX]]) 41; CHECK-NEXT: [[TMP15:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK4]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i32 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer) 42; CHECK-NEXT: [[TMP16:%.*]] = extractelement <vscale x 4 x i1> [[TMP15]], i32 0 43; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] 44; CHECK: middle.block: 45; CHECK-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] 46; 47entry: 48 br label %while.body 49 50while.body: ; preds = %while.body, %entry 51 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ] 52 %gep = getelementptr i32, i32* %ptr, i64 %index 53 store i32 %val, i32* %gep 54 %index.next = add nsw i64 %index, 1 55 %cmp10 = icmp ult i64 %index.next, %n 56 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0 57 58while.end.loopexit: ; preds = %while.body 59 ret void 60} 61 62 63define void @simple_memset_v4i32(i32 %val, i32* %ptr, i64 %n) #0 { 64; CHECK-LABEL: @simple_memset_v4i32( 65; CHECK-NEXT: entry: 66; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1) 67; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 68; CHECK: vector.ph: 69; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], 3 70; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4 71; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 72; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[UMAX]]) 73; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[VAL:%.*]], i32 0 74; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 75; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 76; CHECK: vector.body: 77; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ] 78; CHECK-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK4:%.*]], [[VECTOR_BODY]] ] 79; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX1]], 0 80; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, i32* [[PTR:%.*]], i64 [[TMP0]] 81; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, i32* [[TMP1]], i32 0 82; CHECK-NEXT: [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <4 x i32>* 83; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> [[BROADCAST_SPLAT]], <4 x i32>* [[TMP3]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK2]]) 84; CHECK-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX1]], 4 85; CHECK-NEXT: [[ACTIVE_LANE_MASK4]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX_NEXT3]], i64 [[UMAX]]) 86; CHECK-NEXT: [[TMP4:%.*]] = xor <4 x i1> [[ACTIVE_LANE_MASK4]], <i1 true, i1 true, i1 true, i1 true> 87; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0 88; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] 89; CHECK: middle.block: 90; CHECK-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] 91; 92entry: 93 br label %while.body 94 95while.body: ; preds = %while.body, %entry 96 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ] 97 %gep = getelementptr i32, i32* %ptr, i64 %index 98 store i32 %val, i32* %gep 99 %index.next = add nsw i64 %index, 1 100 %cmp10 = icmp ult i64 %index.next, %n 101 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !3 102 103while.end.loopexit: ; preds = %while.body 104 ret void 105} 106 107 108define void @simple_memcpy(i32* noalias %dst, i32* noalias %src, i64 %n) #0 { 109; CHECK-LABEL: @simple_memcpy( 110; CHECK-NEXT: entry: 111; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1) 112; CHECK-NEXT: [[TMP2:%.*]] = sub i64 -1, [[UMAX]] 113; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 114; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 115; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]] 116; CHECK-NEXT: br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 117; CHECK: vector.ph: 118; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() 119; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 120; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() 121; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 122; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[TMP7]], 1 123; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP8]] 124; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]] 125; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 126; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]]) 127; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 128; CHECK: vector.body: 129; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ] 130; CHECK-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK4:%.*]], [[VECTOR_BODY]] ] 131; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX1]], 0 132; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, i32* [[SRC:%.*]], i64 [[TMP9]] 133; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, i32* [[TMP10]], i32 0 134; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <vscale x 4 x i32>* 135; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP12]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 4 x i32> poison) 136; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i32, i32* [[DST:%.*]], i64 [[TMP9]] 137; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i32, i32* [[TMP13]], i32 0 138; CHECK-NEXT: [[TMP15:%.*]] = bitcast i32* [[TMP14]] to <vscale x 4 x i32>* 139; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> [[WIDE_MASKED_LOAD]], <vscale x 4 x i32>* [[TMP15]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]]) 140; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() 141; CHECK-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 4 142; CHECK-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX1]], [[TMP17]] 143; CHECK-NEXT: [[ACTIVE_LANE_MASK4]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT3]], i64 [[UMAX]]) 144; CHECK-NEXT: [[TMP18:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK4]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i32 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer) 145; CHECK-NEXT: [[TMP19:%.*]] = extractelement <vscale x 4 x i1> [[TMP18]], i32 0 146; CHECK-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] 147; CHECK: middle.block: 148; CHECK-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] 149; 150entry: 151 br label %while.body 152 153while.body: ; preds = %while.body, %entry 154 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ] 155 %gep1 = getelementptr i32, i32* %src, i64 %index 156 %val = load i32, i32* %gep1 157 %gep2 = getelementptr i32, i32* %dst, i64 %index 158 store i32 %val, i32* %gep2 159 %index.next = add nsw i64 %index, 1 160 %cmp10 = icmp ult i64 %index.next, %n 161 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0 162 163while.end.loopexit: ; preds = %while.body 164 ret void 165} 166 167 168define void @copy_stride4(i32* noalias %dst, i32* noalias %src, i64 %n) #0 { 169; CHECK-LABEL: @copy_stride4( 170; CHECK-NEXT: entry: 171; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 4) 172; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[UMAX]], -1 173; CHECK-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 2 174; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 175; CHECK-NEXT: [[TMP5:%.*]] = sub i64 -1, [[TMP2]] 176; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() 177; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], 4 178; CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[TMP5]], [[TMP4]] 179; CHECK-NEXT: br i1 [[TMP6]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 180; CHECK: vector.ph: 181; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() 182; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 4 183; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() 184; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 4 185; CHECK-NEXT: [[TMP11:%.*]] = sub i64 [[TMP10]], 1 186; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP2]], [[TMP11]] 187; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP8]] 188; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 189; CHECK-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 4 190; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP2]]) 191; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64() 192; CHECK-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i64> [[TMP12]], zeroinitializer 193; CHECK-NEXT: [[TMP14:%.*]] = mul <vscale x 4 x i64> [[TMP13]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 4, i32 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer) 194; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP14]] 195; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() 196; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 197; CHECK-NEXT: [[TMP17:%.*]] = mul i64 4, [[TMP16]] 198; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP17]], i32 0 199; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer 200; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 201; CHECK: vector.body: 202; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ] 203; CHECK-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK4:%.*]], [[VECTOR_BODY]] ] 204; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 205; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i32, i32* [[SRC:%.*]], <vscale x 4 x i64> [[VEC_IND]] 206; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> [[TMP18]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 4 x i32> undef) 207; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i32, i32* [[DST:%.*]], <vscale x 4 x i64> [[VEC_IND]] 208; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0i32(<vscale x 4 x i32> [[WIDE_MASKED_GATHER]], <vscale x 4 x i32*> [[TMP19]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]]) 209; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() 210; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 4 211; CHECK-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX1]], [[TMP21]] 212; CHECK-NEXT: [[ACTIVE_LANE_MASK4]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT3]], i64 [[TMP2]]) 213; CHECK-NEXT: [[TMP22:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK4]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i32 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer) 214; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]] 215; CHECK-NEXT: [[TMP23:%.*]] = extractelement <vscale x 4 x i1> [[TMP22]], i32 0 216; CHECK-NEXT: br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] 217; CHECK: middle.block: 218; CHECK-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] 219; 220entry: 221 br label %while.body 222 223while.body: ; preds = %while.body, %entry 224 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ] 225 %gep1 = getelementptr i32, i32* %src, i64 %index 226 %val = load i32, i32* %gep1 227 %gep2 = getelementptr i32, i32* %dst, i64 %index 228 store i32 %val, i32* %gep2 229 %index.next = add nsw i64 %index, 4 230 %cmp10 = icmp ult i64 %index.next, %n 231 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0 232 233while.end.loopexit: ; preds = %while.body 234 ret void 235} 236 237 238define void @simple_gather_scatter(i32* noalias %dst, i32* noalias %src, i32* noalias %ind, i64 %n) #0 { 239; CHECK-LABEL: @simple_gather_scatter( 240; CHECK-NEXT: entry: 241; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1) 242; CHECK-NEXT: [[TMP2:%.*]] = sub i64 -1, [[UMAX]] 243; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 244; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 245; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]] 246; CHECK-NEXT: br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 247; CHECK: vector.ph: 248; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() 249; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 250; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() 251; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 252; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[TMP7]], 1 253; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP8]] 254; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]] 255; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 256; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]]) 257; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 258; CHECK: vector.body: 259; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ] 260; CHECK-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK4:%.*]], [[VECTOR_BODY]] ] 261; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX1]], 0 262; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, i32* [[IND:%.*]], i64 [[TMP9]] 263; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, i32* [[TMP10]], i32 0 264; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <vscale x 4 x i32>* 265; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP12]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 4 x i32> poison) 266; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i32, i32* [[SRC:%.*]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]] 267; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> [[TMP13]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 4 x i32> undef) 268; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i32, i32* [[DST:%.*]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]] 269; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0i32(<vscale x 4 x i32> [[WIDE_MASKED_GATHER]], <vscale x 4 x i32*> [[TMP14]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]]) 270; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() 271; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 272; CHECK-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX1]], [[TMP16]] 273; CHECK-NEXT: [[ACTIVE_LANE_MASK4]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT3]], i64 [[UMAX]]) 274; CHECK-NEXT: [[TMP17:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK4]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i32 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer) 275; CHECK-NEXT: [[TMP18:%.*]] = extractelement <vscale x 4 x i1> [[TMP17]], i32 0 276; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] 277; CHECK: middle.block: 278; CHECK-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] 279; 280entry: 281 br label %while.body 282 283while.body: ; preds = %while.body, %entry 284 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ] 285 %gep1 = getelementptr i32, i32* %ind, i64 %index 286 %ind_val = load i32, i32* %gep1 287 %gep2 = getelementptr i32, i32* %src, i32 %ind_val 288 %val = load i32, i32* %gep2 289 %gep3 = getelementptr i32, i32* %dst, i32 %ind_val 290 store i32 %val, i32* %gep3 291 %index.next = add nsw i64 %index, 1 292 %cmp10 = icmp ult i64 %index.next, %n 293 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0 294 295while.end.loopexit: ; preds = %while.body 296 ret void 297} 298 299 300; The original loop had an unconditional uniform load. Let's make sure 301; we don't artificially create new predicated blocks for the load. 302define void @uniform_load(i32* noalias %dst, i32* noalias readonly %src, i64 %n) #0 { 303; CHECK-LABEL: @uniform_load( 304; CHECK-NEXT: entry: 305; CHECK-NEXT: [[TMP2:%.*]] = sub i64 -1, [[N:%.*]] 306; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 307; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 308; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]] 309; CHECK-NEXT: br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 310; CHECK: vector.ph: 311; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() 312; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 313; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() 314; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 315; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[TMP7]], 1 316; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP8]] 317; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]] 318; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 319; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]]) 320; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 321; CHECK: vector.body: 322; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 323; CHECK-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK2:%.*]], [[VECTOR_BODY]] ] 324; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 0 325; CHECK-NEXT: [[TMP10:%.*]] = load i32, i32* [[SRC:%.*]], align 4 326; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP10]], i32 0 327; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer 328; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 [[TMP9]] 329; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TMP11]], i32 0 330; CHECK-NEXT: [[TMP13:%.*]] = bitcast i32* [[TMP12]] to <vscale x 4 x i32>* 331; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> [[BROADCAST_SPLAT]], <vscale x 4 x i32>* [[TMP13]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK1]]) 332; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() 333; CHECK-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 4 334; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP15]] 335; CHECK-NEXT: [[ACTIVE_LANE_MASK2]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]]) 336; CHECK-NEXT: [[TMP16:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i32 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer) 337; CHECK-NEXT: [[TMP17:%.*]] = extractelement <vscale x 4 x i1> [[TMP16]], i32 0 338; CHECK-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] 339; CHECK: middle.block: 340; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]] 341; 342 343entry: 344 br label %for.body 345 346for.body: ; preds = %entry, %for.body 347 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 348 %val = load i32, i32* %src, align 4 349 %arrayidx = getelementptr inbounds i32, i32* %dst, i64 %indvars.iv 350 store i32 %val, i32* %arrayidx, align 4 351 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 352 %exitcond.not = icmp eq i64 %indvars.iv.next, %n 353 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0 354 355for.end: ; preds = %for.body, %entry 356 ret void 357} 358 359 360; The original loop had a conditional uniform load. In this case we actually 361; do need to perform conditional loads and so we end up using a gather instead. 362; However, we at least ensure the mask is the overlap of the loop predicate 363; and the original condition. 364define void @cond_uniform_load(i32* noalias %dst, i32* noalias readonly %src, i32* noalias readonly %cond, i64 %n) #0 { 365; CHECK-LABEL: @cond_uniform_load( 366; CHECK-NEXT: entry: 367; CHECK-NEXT: [[TMP2:%.*]] = sub i64 -1, [[N:%.*]] 368; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 369; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 370; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]] 371; CHECK-NEXT: br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 372; CHECK: vector.ph: 373; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() 374; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 375; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() 376; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 377; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[TMP7]], 1 378; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP8]] 379; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]] 380; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 381; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]]) 382; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32*> poison, i32* [[SRC:%.*]], i32 0 383; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32*> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32*> poison, <vscale x 4 x i32> zeroinitializer 384; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 385; CHECK: vector.body: 386; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ] 387; CHECK-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK4:%.*]], [[VECTOR_BODY]] ] 388; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX1]], 0 389; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[COND:%.*]], i64 [[TMP9]] 390; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP10]], i32 0 391; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <vscale x 4 x i32>* 392; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP12]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 4 x i32> poison) 393; CHECK-NEXT: [[TMP13:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], zeroinitializer 394; CHECK-NEXT: [[TMP14:%.*]] = xor <vscale x 4 x i1> [[TMP13]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i32 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer) 395; CHECK-NEXT: [[TMP15:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 4 x i1> [[TMP14]], <vscale x 4 x i1> zeroinitializer 396; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> [[BROADCAST_SPLAT]], i32 4, <vscale x 4 x i1> [[TMP15]], <vscale x 4 x i32> undef) 397; CHECK-NEXT: [[TMP16:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i1> zeroinitializer 398; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP16]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[WIDE_MASKED_GATHER]] 399; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 [[TMP9]] 400; CHECK-NEXT: [[TMP18:%.*]] = or <vscale x 4 x i1> [[TMP15]], [[TMP16]] 401; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, i32* [[TMP17]], i32 0 402; CHECK-NEXT: [[TMP20:%.*]] = bitcast i32* [[TMP19]] to <vscale x 4 x i32>* 403; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x i32>* [[TMP20]], i32 4, <vscale x 4 x i1> [[TMP18]]) 404; CHECK-NEXT: [[TMP21:%.*]] = call i64 @llvm.vscale.i64() 405; CHECK-NEXT: [[TMP22:%.*]] = mul i64 [[TMP21]], 4 406; CHECK-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX1]], [[TMP22]] 407; CHECK-NEXT: [[ACTIVE_LANE_MASK4]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT3]], i64 [[N]]) 408; CHECK-NEXT: [[TMP23:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK4]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i32 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer) 409; CHECK-NEXT: [[TMP24:%.*]] = extractelement <vscale x 4 x i1> [[TMP23]], i32 0 410; CHECK-NEXT: br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] 411; CHECK: middle.block: 412; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]] 413; 414 415entry: 416 br label %for.body 417 418for.body: ; preds = %entry, %if.end 419 %index = phi i64 [ %index.next, %if.end ], [ 0, %entry ] 420 %arrayidx = getelementptr inbounds i32, i32* %cond, i64 %index 421 %0 = load i32, i32* %arrayidx, align 4 422 %tobool.not = icmp eq i32 %0, 0 423 br i1 %tobool.not, label %if.end, label %if.then 424 425if.then: ; preds = %for.body 426 %1 = load i32, i32* %src, align 4 427 br label %if.end 428 429if.end: ; preds = %if.then, %for.body 430 %val.0 = phi i32 [ %1, %if.then ], [ 0, %for.body ] 431 %arrayidx1 = getelementptr inbounds i32, i32* %dst, i64 %index 432 store i32 %val.0, i32* %arrayidx1, align 4 433 %index.next = add nuw i64 %index, 1 434 %exitcond.not = icmp eq i64 %index.next, %n 435 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0 436 437for.end: ; preds = %for.inc, %entry 438 ret void 439} 440 441 442; The original loop had an unconditional uniform store. Let's make sure 443; we don't artificially create new predicated blocks for the load. 444define void @uniform_store(i32* noalias %dst, i32* noalias readonly %src, i64 %n) #0 { 445; CHECK-LABEL: @uniform_store( 446; CHECK-NEXT: entry: 447; CHECK-NEXT: [[TMP2:%.*]] = sub i64 -1, [[N:%.*]] 448; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 449; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 450; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]] 451; CHECK-NEXT: br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 452; CHECK: vector.ph: 453; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() 454; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 455; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() 456; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 457; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[TMP7]], 1 458; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP8]] 459; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]] 460; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 461; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]]) 462; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32*> poison, i32* [[DST:%.*]], i32 0 463; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32*> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32*> poison, <vscale x 4 x i32> zeroinitializer 464; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 465; CHECK: vector.body: 466; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 467; CHECK-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK2:%.*]], [[VECTOR_BODY]] ] 468; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 0 469; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 [[TMP9]] 470; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP10]], i32 0 471; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <vscale x 4 x i32>* 472; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP12]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK1]], <vscale x 4 x i32> poison) 473; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0i32(<vscale x 4 x i32> [[WIDE_MASKED_LOAD]], <vscale x 4 x i32*> [[BROADCAST_SPLAT]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK1]]) 474; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() 475; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 4 476; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP14]] 477; CHECK-NEXT: [[ACTIVE_LANE_MASK2]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]]) 478; CHECK-NEXT: [[TMP15:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i32 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer) 479; CHECK-NEXT: [[TMP16:%.*]] = extractelement <vscale x 4 x i1> [[TMP15]], i32 0 480; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] 481; CHECK: middle.block: 482; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]] 483; 484 485entry: 486 br label %for.body 487 488for.body: ; preds = %entry, %for.body 489 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 490 %arrayidx = getelementptr inbounds i32, i32* %src, i64 %indvars.iv 491 %val = load i32, i32* %arrayidx, align 4 492 store i32 %val, i32* %dst, align 4 493 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 494 %exitcond.not = icmp eq i64 %indvars.iv.next, %n 495 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0 496 497for.end: ; preds = %for.body, %entry 498 ret void 499} 500 501 502define void @simple_fdiv(float* noalias %dst, float* noalias %src, i64 %n) #0 { 503; CHECK-LABEL: @simple_fdiv( 504; CHECK-NEXT: entry: 505; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1) 506; CHECK-NEXT: [[TMP2:%.*]] = sub i64 -1, [[UMAX]] 507; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 508; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 509; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]] 510; CHECK-NEXT: br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 511; CHECK: vector.ph: 512; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() 513; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 514; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() 515; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 516; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[TMP7]], 1 517; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP8]] 518; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]] 519; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 520; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]]) 521; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 522; CHECK: vector.body: 523; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT4:%.*]], [[VECTOR_BODY]] ] 524; CHECK-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK5:%.*]], [[VECTOR_BODY]] ] 525; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX1]], 0 526; CHECK-NEXT: [[TMP10:%.*]] = getelementptr float, float* [[SRC:%.*]], i64 [[TMP9]] 527; CHECK-NEXT: [[TMP11:%.*]] = getelementptr float, float* [[DST:%.*]], i64 [[TMP9]] 528; CHECK-NEXT: [[TMP12:%.*]] = getelementptr float, float* [[TMP10]], i32 0 529; CHECK-NEXT: [[TMP13:%.*]] = bitcast float* [[TMP12]] to <vscale x 4 x float>* 530; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0nxv4f32(<vscale x 4 x float>* [[TMP13]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 4 x float> poison) 531; CHECK-NEXT: [[TMP14:%.*]] = getelementptr float, float* [[TMP11]], i32 0 532; CHECK-NEXT: [[TMP15:%.*]] = bitcast float* [[TMP14]] to <vscale x 4 x float>* 533; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0nxv4f32(<vscale x 4 x float>* [[TMP15]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 4 x float> poison) 534; CHECK-NEXT: [[TMP16:%.*]] = fdiv <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD3]] 535; CHECK-NEXT: [[TMP17:%.*]] = bitcast float* [[TMP14]] to <vscale x 4 x float>* 536; CHECK-NEXT: call void @llvm.masked.store.nxv4f32.p0nxv4f32(<vscale x 4 x float> [[TMP16]], <vscale x 4 x float>* [[TMP17]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]]) 537; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() 538; CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 4 539; CHECK-NEXT: [[INDEX_NEXT4]] = add i64 [[INDEX1]], [[TMP19]] 540; CHECK-NEXT: [[ACTIVE_LANE_MASK5]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT4]], i64 [[UMAX]]) 541; CHECK-NEXT: [[TMP20:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK5]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i32 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer) 542; CHECK-NEXT: [[TMP21:%.*]] = extractelement <vscale x 4 x i1> [[TMP20]], i32 0 543; CHECK-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] 544; CHECK: middle.block: 545; CHECK-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] 546; 547entry: 548 br label %while.body 549 550while.body: ; preds = %while.body, %entry 551 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ] 552 %gep1 = getelementptr float, float* %src, i64 %index 553 %gep2 = getelementptr float, float* %dst, i64 %index 554 %val1 = load float, float* %gep1 555 %val2 = load float, float* %gep2 556 %res = fdiv float %val1, %val2 557 store float %res, float* %gep2 558 %index.next = add nsw i64 %index, 1 559 %cmp10 = icmp ult i64 %index.next, %n 560 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0 561 562while.end.loopexit: ; preds = %while.body 563 ret void 564} 565 566 567define i32 @add_reduction_i32(i32* %ptr, i64 %n) #0 { 568; CHECK-LABEL: @add_reduction_i32( 569; CHECK-NEXT: entry: 570; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1) 571; CHECK-NEXT: [[TMP2:%.*]] = sub i64 -1, [[UMAX]] 572; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 573; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 574; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]] 575; CHECK-NEXT: br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 576; CHECK: vector.ph: 577; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() 578; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 579; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() 580; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 581; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[TMP7]], 1 582; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP8]] 583; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]] 584; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 585; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]]) 586; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 587; CHECK: vector.body: 588; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ] 589; CHECK-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK4:%.*]], [[VECTOR_BODY]] ] 590; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ] 591; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX1]], 0 592; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, i32* [[PTR:%.*]], i64 [[TMP9]] 593; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, i32* [[TMP10]], i32 0 594; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <vscale x 4 x i32>* 595; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP12]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 4 x i32> poison) 596; CHECK-NEXT: [[TMP13:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], <vscale x 4 x i32> zeroinitializer 597; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP13]]) 598; CHECK-NEXT: [[TMP15]] = add i32 [[TMP14]], [[VEC_PHI]] 599; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() 600; CHECK-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 4 601; CHECK-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX1]], [[TMP17]] 602; CHECK-NEXT: [[ACTIVE_LANE_MASK4]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT3]], i64 [[UMAX]]) 603; CHECK-NEXT: [[TMP18:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK4]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i32 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer) 604; CHECK-NEXT: [[TMP19:%.*]] = extractelement <vscale x 4 x i1> [[TMP18]], i32 0 605; CHECK-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] 606; CHECK: middle.block: 607; CHECK-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] 608; 609entry: 610 br label %while.body 611 612while.body: ; preds = %while.body, %entry 613 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ] 614 %red = phi i32 [ %red.next, %while.body ], [ 0, %entry ] 615 %gep = getelementptr i32, i32* %ptr, i64 %index 616 %val = load i32, i32* %gep 617 %red.next = add i32 %red, %val 618 %index.next = add nsw i64 %index, 1 619 %cmp10 = icmp ult i64 %index.next, %n 620 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0 621 622while.end.loopexit: ; preds = %while.body 623 ret i32 %red.next 624} 625 626define float @add_reduction_f32(float* %ptr, i64 %n) #0 { 627; CHECK-LABEL: @add_reduction_f32( 628; CHECK-NEXT: entry: 629; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1) 630; CHECK-NEXT: [[TMP2:%.*]] = sub i64 -1, [[UMAX]] 631; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 632; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 633; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]] 634; CHECK-NEXT: br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 635; CHECK: vector.ph: 636; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() 637; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 638; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() 639; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 640; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[TMP7]], 1 641; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP8]] 642; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]] 643; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 644; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]]) 645; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 646; CHECK: vector.body: 647; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ] 648; CHECK-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK4:%.*]], [[VECTOR_BODY]] ] 649; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ] 650; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX1]], 0 651; CHECK-NEXT: [[TMP10:%.*]] = getelementptr float, float* [[PTR:%.*]], i64 [[TMP9]] 652; CHECK-NEXT: [[TMP11:%.*]] = getelementptr float, float* [[TMP10]], i32 0 653; CHECK-NEXT: [[TMP12:%.*]] = bitcast float* [[TMP11]] to <vscale x 4 x float>* 654; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0nxv4f32(<vscale x 4 x float>* [[TMP12]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 4 x float> poison) 655; CHECK-NEXT: [[TMP13:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float> shufflevector (<vscale x 4 x float> insertelement (<vscale x 4 x float> poison, float -0.000000e+00, i32 0), <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer) 656; CHECK-NEXT: [[TMP14]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[TMP13]]) 657; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() 658; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 659; CHECK-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX1]], [[TMP16]] 660; CHECK-NEXT: [[ACTIVE_LANE_MASK4]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT3]], i64 [[UMAX]]) 661; CHECK-NEXT: [[TMP17:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK4]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i32 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer) 662; CHECK-NEXT: [[TMP18:%.*]] = extractelement <vscale x 4 x i1> [[TMP17]], i32 0 663; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] 664; CHECK: middle.block: 665; CHECK-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] 666; 667entry: 668 br label %while.body 669 670while.body: ; preds = %while.body, %entry 671 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ] 672 %red = phi float [ %red.next, %while.body ], [ 0.000000, %entry ] 673 %gep = getelementptr float, float* %ptr, i64 %index 674 %val = load float, float* %gep 675 %red.next = fadd float %red, %val 676 %index.next = add nsw i64 %index, 1 677 %cmp10 = icmp ult i64 %index.next, %n 678 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0 679 680while.end.loopexit: ; preds = %while.body 681 ret float %red.next 682} 683 684define i32 @cond_xor_reduction(i32* noalias %a, i32* noalias %cond, i64 %N) #0 { 685; CHECK-LABEL: @cond_xor_reduction( 686; CHECK-NEXT: entry: 687; CHECK-NEXT: [[TMP2:%.*]] = sub i64 -1, [[N:%.*]] 688; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 689; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 690; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]] 691; CHECK-NEXT: br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 692; CHECK: vector.ph: 693; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() 694; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 695; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() 696; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 697; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[TMP7]], 1 698; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP8]] 699; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]] 700; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 701; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]]) 702; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 703; CHECK: vector.body: 704; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 705; CHECK-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK3:%.*]], [[VECTOR_BODY]] ] 706; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 7, [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ] 707; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 0 708; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[COND:%.*]], i64 [[TMP9]] 709; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP10]], i32 0 710; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <vscale x 4 x i32>* 711; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP12]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK1]], <vscale x 4 x i32> poison) 712; CHECK-NEXT: [[TMP13:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 5, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer) 713; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i32, i32* [[A:%.*]], i64 [[TMP9]] 714; CHECK-NEXT: [[TMP15:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK1]], <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i1> zeroinitializer 715; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i32, i32* [[TMP14]], i32 0 716; CHECK-NEXT: [[TMP17:%.*]] = bitcast i32* [[TMP16]] to <vscale x 4 x i32>* 717; CHECK-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP17]], i32 4, <vscale x 4 x i1> [[TMP15]], <vscale x 4 x i32> poison) 718; CHECK-NEXT: [[TMP18:%.*]] = select <vscale x 4 x i1> [[TMP15]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD2]], <vscale x 4 x i32> zeroinitializer 719; CHECK-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.xor.nxv4i32(<vscale x 4 x i32> [[TMP18]]) 720; CHECK-NEXT: [[TMP20]] = xor i32 [[TMP19]], [[VEC_PHI]] 721; CHECK-NEXT: [[TMP21:%.*]] = call i64 @llvm.vscale.i64() 722; CHECK-NEXT: [[TMP22:%.*]] = mul i64 [[TMP21]], 4 723; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP22]] 724; CHECK-NEXT: [[ACTIVE_LANE_MASK3]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]]) 725; CHECK-NEXT: [[TMP23:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK3]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i32 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer) 726; CHECK-NEXT: [[TMP24:%.*]] = extractelement <vscale x 4 x i1> [[TMP23]], i32 0 727; CHECK-NEXT: br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]] 728; CHECK: middle.block: 729; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]] 730; 731entry: 732 br label %for.body 733 734for.body: 735 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] 736 %rdx = phi i32 [ 7, %entry ], [ %res, %for.inc ] 737 %arrayidx = getelementptr inbounds i32, i32* %cond, i64 %iv 738 %0 = load i32, i32* %arrayidx 739 %tobool = icmp eq i32 %0, 5 740 br i1 %tobool, label %if.then, label %for.inc 741 742if.then: 743 %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %iv 744 %1 = load i32, i32* %arrayidx2 745 %xor = xor i32 %rdx, %1 746 br label %for.inc 747 748for.inc: 749 %res = phi i32 [ %rdx, %for.body ], [ %xor, %if.then ] 750 %iv.next = add i64 %iv, 1 751 %exitcond.not = icmp eq i64 %iv.next, %N 752 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0 753 754for.end: 755 ret i32 %res 756} 757 758; Negative tests where we don't expect tail-folding 759 760; Integer divides can throw exceptions and since we can't scalarize conditional 761; divides for scalable vectors we just don't bother vectorizing. 762define void @simple_idiv(i32* noalias %dst, i32* noalias %src, i64 %n) #0 { 763; CHECK-LABEL: @simple_idiv( 764; CHECK-NOT: vector.body 765; 766entry: 767 br label %while.body 768 769while.body: ; preds = %while.body, %entry 770 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ] 771 %gep1 = getelementptr i32, i32* %src, i64 %index 772 %gep2 = getelementptr i32, i32* %dst, i64 %index 773 %val1 = load i32, i32* %gep1 774 %val2 = load i32, i32* %gep2 775 %res = udiv i32 %val1, %val2 776 store i32 %res, i32* %gep2 777 %index.next = add nsw i64 %index, 1 778 %cmp10 = icmp ult i64 %index.next, %n 779 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0 780 781while.end.loopexit: ; preds = %while.body 782 ret void 783} 784 785!0 = distinct !{!0, !1, !2} 786!1 = !{!"llvm.loop.vectorize.width", i32 4} 787!2 = !{!"llvm.loop.vectorize.scalable.enable", i1 true} 788!3 = distinct !{!3, !4} 789!4 = !{!"llvm.loop.vectorize.width", i32 4} 790 791attributes #0 = { "target-features"="+sve" } 792