1; RUN: opt -S -hints-allow-reordering=false -loop-vectorize -prefer-predicate-over-epilogue=predicate-dont-vectorize -prefer-inloop-reductions < %s | FileCheck %s 2; RUN: opt -S -hints-allow-reordering=false -loop-vectorize -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -prefer-inloop-reductions < %s | FileCheck %s 3 4; CHECK-NOT: vector.body: 5 6target triple = "aarch64-unknown-linux-gnu" 7 8 9define void @simple_memset(i32 %val, i32* %ptr, i64 %n) #0 { 10; CHECK-LABEL: @simple_memset( 11; CHECK-NEXT: entry: 12; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1) 13; CHECK-NEXT: br i1 false, label %scalar.ph, label %vector.ph 14; CHECK: vector.ph: 15; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 16; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 17; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() 18; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 19; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[TMP3]], 1 20; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] 21; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] 22; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 23; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL:%.*]], i32 0 24; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT5]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer 25; CHECK-NEXT: br label %vector.body 26; CHECK: vector.body: 27; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT2:%.*]], %vector.body ] 28; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[INDEX1]], i32 0 29; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer 30; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64() 31; CHECK-NEXT: [[TMP6:%.*]] = add <vscale x 4 x i64> [[TMP5]], zeroinitializer 32; CHECK-NEXT: [[TMP7:%.*]] = mul <vscale x 4 x i64> [[TMP6]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i32 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer) 33; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 4 x i64> [[BROADCAST_SPLAT4]], [[TMP7]] 34; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX1]], 0 35; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP8]], i64 [[UMAX]]) 36; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, i32* [[PTR:%.*]], i64 [[TMP8]] 37; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, i32* [[TMP10]], i32 0 38; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <vscale x 4 x i32>* 39; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> [[BROADCAST_SPLAT6]], <vscale x 4 x i32>* [[TMP12]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]]) 40; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() 41; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 4 42; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP14]] 43; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]] 44; CHECK-NEXT: br i1 [[TMP15]], label %middle.block, label %vector.body 45; CHECK: middle.block: 46; CHECK-NEXT: br i1 true, label %while.end.loopexit, label %scalar.ph 47; 48entry: 49 br label %while.body 50 51while.body: ; preds = %while.body, %entry 52 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ] 53 %gep = getelementptr i32, i32* %ptr, i64 %index 54 store i32 %val, i32* %gep 55 %index.next = add nsw i64 %index, 1 56 %cmp10 = icmp ult i64 %index.next, %n 57 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0 58 59while.end.loopexit: ; preds = %while.body 60 ret void 61} 62 63 64define void @simple_memcpy(i32* noalias %dst, i32* noalias %src, i64 %n) #0 { 65; CHECK-LABEL: @simple_memcpy( 66; CHECK-NEXT: entry: 67; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1) 68; CHECK-NEXT: br i1 false, label %scalar.ph, label %vector.ph 69; CHECK: vector.ph: 70; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 71; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 72; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() 73; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 74; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[TMP3]], 1 75; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] 76; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] 77; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 78; CHECK-NEXT: br label %vector.body 79; CHECK: vector.body: 80; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT2:%.*]], %vector.body ] 81; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[INDEX1]], i32 0 82; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer 83; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64() 84; CHECK-NEXT: [[TMP6:%.*]] = add <vscale x 4 x i64> [[TMP5]], zeroinitializer 85; CHECK-NEXT: [[TMP7:%.*]] = mul <vscale x 4 x i64> [[TMP6]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i32 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer) 86; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 4 x i64> [[BROADCAST_SPLAT4]], [[TMP7]] 87; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX1]], 0 88; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP8]], i64 [[UMAX]]) 89; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, i32* [[SRC:%.*]], i64 [[TMP8]] 90; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, i32* [[TMP10]], i32 0 91; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <vscale x 4 x i32>* 92; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP12]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison) 93; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i32, i32* [[DST:%.*]], i64 [[TMP8]] 94; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i32, i32* [[TMP13]], i32 0 95; CHECK-NEXT: [[TMP15:%.*]] = bitcast i32* [[TMP14]] to <vscale x 4 x i32>* 96; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> [[WIDE_MASKED_LOAD]], <vscale x 4 x i32>* [[TMP15]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]]) 97; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() 98; CHECK-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 4 99; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP17]] 100; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]] 101; CHECK-NEXT: br i1 [[TMP18]], label %middle.block, label %vector.body, !llvm.loop [[LOOP4:![0-9]+]] 102; CHECK: middle.block: 103; CHECK-NEXT: br i1 true, label %while.end.loopexit, label %scalar.ph 104; 105entry: 106 br label %while.body 107 108while.body: ; preds = %while.body, %entry 109 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ] 110 %gep1 = getelementptr i32, i32* %src, i64 %index 111 %val = load i32, i32* %gep1 112 %gep2 = getelementptr i32, i32* %dst, i64 %index 113 store i32 %val, i32* %gep2 114 %index.next = add nsw i64 %index, 1 115 %cmp10 = icmp ult i64 %index.next, %n 116 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0 117 118while.end.loopexit: ; preds = %while.body 119 ret void 120} 121 122 123define void @copy_stride4(i32* noalias %dst, i32* noalias %src, i64 %n) #0 { 124; CHECK-LABEL: @copy_stride4( 125; CHECK-NEXT: entry: 126; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 4) 127; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[UMAX]], -1 128; CHECK-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 2 129; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 130; CHECK-NEXT: br i1 false, label %scalar.ph, label %vector.ph 131; CHECK: vector.ph: 132; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() 133; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], 4 134; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() 135; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 136; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[TMP6]], 1 137; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP2]], [[TMP7]] 138; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP4]] 139; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 140; CHECK-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 4 141; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64() 142; CHECK-NEXT: [[TMP9:%.*]] = add <vscale x 4 x i64> [[TMP8]], zeroinitializer 143; CHECK-NEXT: [[TMP10:%.*]] = mul <vscale x 4 x i64> [[TMP9]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 4, i32 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer) 144; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP10]] 145; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64() 146; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 4 147; CHECK-NEXT: [[TMP13:%.*]] = mul i64 4, [[TMP12]] 148; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP13]], i32 0 149; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer 150; CHECK-NEXT: br label %vector.body 151; CHECK: vector.body: 152; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT2:%.*]], %vector.body ] 153; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], %vector.ph ], [ [[VEC_IND_NEXT:%.*]], %vector.body ] 154; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[INDEX1]], i32 0 155; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer 156; CHECK-NEXT: [[TMP14:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64() 157; CHECK-NEXT: [[TMP15:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP14]] 158; CHECK-NEXT: [[VEC_IV:%.*]] = add <vscale x 4 x i64> [[BROADCAST_SPLAT4]], [[TMP15]] 159; CHECK-NEXT: [[TMP16:%.*]] = extractelement <vscale x 4 x i64> [[VEC_IV]], i32 0 160; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP16]], i64 [[TMP2]]) 161; CHECK-NEXT: [[TMP17:%.*]] = getelementptr i32, i32* [[SRC:%.*]], <vscale x 4 x i64> [[VEC_IND]] 162; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> [[TMP17]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> undef) 163; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i32, i32* [[DST:%.*]], <vscale x 4 x i64> [[VEC_IND]] 164; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0i32(<vscale x 4 x i32> [[WIDE_MASKED_GATHER]], <vscale x 4 x i32*> [[TMP18]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]]) 165; CHECK-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64() 166; CHECK-NEXT: [[TMP20:%.*]] = mul i64 [[TMP19]], 4 167; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP20]] 168; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]] 169; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]] 170; CHECK-NEXT: br i1 [[TMP21]], label %middle.block, label %vector.body 171; CHECK: middle.block: 172; CHECK-NEXT: br i1 true, label %while.end.loopexit, label %scalar.ph 173; 174entry: 175 br label %while.body 176 177while.body: ; preds = %while.body, %entry 178 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ] 179 %gep1 = getelementptr i32, i32* %src, i64 %index 180 %val = load i32, i32* %gep1 181 %gep2 = getelementptr i32, i32* %dst, i64 %index 182 store i32 %val, i32* %gep2 183 %index.next = add nsw i64 %index, 4 184 %cmp10 = icmp ult i64 %index.next, %n 185 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0 186 187while.end.loopexit: ; preds = %while.body 188 ret void 189} 190 191 192define void @simple_gather_scatter(i32* noalias %dst, i32* noalias %src, i32* noalias %ind, i64 %n) #0 { 193; CHECK-LABEL: @simple_gather_scatter( 194; CHECK-NEXT: entry: 195; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1) 196; CHECK-NEXT: br i1 false, label %scalar.ph, label %vector.ph 197; CHECK: vector.ph: 198; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 199; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 200; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() 201; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 202; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[TMP3]], 1 203; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] 204; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] 205; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 206; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 207; CHECK: vector.body: 208; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT2:%.*]], %vector.body ] 209; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[INDEX1]], i32 0 210; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer 211; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64() 212; CHECK-NEXT: [[TMP6:%.*]] = add <vscale x 4 x i64> [[TMP5]], zeroinitializer 213; CHECK-NEXT: [[TMP7:%.*]] = mul <vscale x 4 x i64> [[TMP6]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i32 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer) 214; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 4 x i64> [[BROADCAST_SPLAT4]], [[TMP7]] 215; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX1]], 0 216; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP8]], i64 [[UMAX]]) 217; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, i32* [[IND:%.*]], i64 [[TMP8]] 218; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, i32* [[TMP10]], i32 0 219; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <vscale x 4 x i32>* 220; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP12]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison) 221; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i32, i32* [[SRC:%.*]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]] 222; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> [[TMP13]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> undef) 223; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i32, i32* [[DST:%.*]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]] 224; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0i32(<vscale x 4 x i32> [[WIDE_MASKED_GATHER]], <vscale x 4 x i32*> [[TMP14]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]]) 225; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() 226; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 227; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP16]] 228; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]] 229; CHECK-NEXT: br i1 [[TMP17]], label %middle.block, label %vector.body 230; CHECK: middle.block: 231; CHECK-NEXT: br i1 true, label %while.end.loopexit, label %scalar.ph 232; 233entry: 234 br label %while.body 235 236while.body: ; preds = %while.body, %entry 237 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ] 238 %gep1 = getelementptr i32, i32* %ind, i64 %index 239 %ind_val = load i32, i32* %gep1 240 %gep2 = getelementptr i32, i32* %src, i32 %ind_val 241 %val = load i32, i32* %gep2 242 %gep3 = getelementptr i32, i32* %dst, i32 %ind_val 243 store i32 %val, i32* %gep3 244 %index.next = add nsw i64 %index, 1 245 %cmp10 = icmp ult i64 %index.next, %n 246 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0 247 248while.end.loopexit: ; preds = %while.body 249 ret void 250} 251 252 253; The original loop had an unconditional uniform load. Let's make sure 254; we don't artificially create new predicated blocks for the load. 255define void @uniform_load(i32* noalias %dst, i32* noalias readonly %src, i64 %n) #0 { 256; CHECK-LABEL: @uniform_load( 257; CHECK-NEXT: entry: 258; CHECK-NEXT: br i1 false, label %scalar.ph, label %vector.ph 259; CHECK: vector.ph: 260; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 261; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 262; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() 263; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 264; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[TMP3]], 1 265; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP4]] 266; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] 267; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 268; CHECK-NEXT: br label %vector.body 269; CHECK: vector.body: 270; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ] 271; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[INDEX]], i32 0 272; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer 273; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64() 274; CHECK-NEXT: [[TMP6:%.*]] = add <vscale x 4 x i64> [[TMP5]], zeroinitializer 275; CHECK-NEXT: [[TMP7:%.*]] = mul <vscale x 4 x i64> [[TMP6]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i32 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer) 276; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 4 x i64> [[BROADCAST_SPLAT2]], [[TMP7]] 277; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 0 278; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP8]], i64 %n) 279; CHECK-NEXT: [[TMP10:%.*]] = load i32, i32* [[SRC:%.*]], align 4 280; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP10]], i32 0 281; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer 282; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 [[TMP8]] 283; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TMP11]], i32 0 284; CHECK-NEXT: [[TMP13:%.*]] = bitcast i32* [[TMP12]] to <vscale x 4 x i32>* 285; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> [[BROADCAST_SPLAT4]], <vscale x 4 x i32>* [[TMP13]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]]) 286; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() 287; CHECK-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 4 288; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP15]] 289; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 290; CHECK-NEXT: br i1 [[TMP16]], label %middle.block, label %vector.body, !llvm.loop [[LOOP6:![0-9]+]] 291; CHECK: middle.block: 292; CHECK-NEXT: br i1 true, label %for.end, label %scalar.ph 293; 294 295entry: 296 br label %for.body 297 298for.body: ; preds = %entry, %for.body 299 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 300 %val = load i32, i32* %src, align 4 301 %arrayidx = getelementptr inbounds i32, i32* %dst, i64 %indvars.iv 302 store i32 %val, i32* %arrayidx, align 4 303 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 304 %exitcond.not = icmp eq i64 %indvars.iv.next, %n 305 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0 306 307for.end: ; preds = %for.body, %entry 308 ret void 309} 310 311 312; The original loop had a conditional uniform load. In this case we actually 313; do need to perform conditional loads and so we end up using a gather instead. 314; However, we at least ensure the mask is the overlap of the loop predicate 315; and the original condition. 316define void @cond_uniform_load(i32* noalias %dst, i32* noalias readonly %src, i32* noalias readonly %cond, i64 %n) #0 { 317; CHECK-LABEL: @cond_uniform_load( 318; CHECK-NEXT: entry: 319; CHECK-NEXT: br i1 false, label %scalar.ph, label %vector.ph 320; CHECK: vector.ph: 321; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 322; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 323; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() 324; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 325; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[TMP3]], 1 326; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP4]] 327; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] 328; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 329; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 4 x i32*> poison, i32* [[SRC:%.*]], i32 0 330; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 4 x i32*> [[BROADCAST_SPLATINSERT5]], <vscale x 4 x i32*> poison, <vscale x 4 x i32> zeroinitializer 331; CHECK-NEXT: br label %vector.body 332; CHECK: vector.body: 333; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT2:%.*]], %vector.body ] 334; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[INDEX1]], i32 0 335; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer 336; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64() 337; CHECK-NEXT: [[TMP6:%.*]] = add <vscale x 4 x i64> [[TMP5]], zeroinitializer 338; CHECK-NEXT: [[TMP7:%.*]] = mul <vscale x 4 x i64> [[TMP6]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i32 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer) 339; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 4 x i64> [[BROADCAST_SPLAT4]], [[TMP7]] 340; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX1]], 0 341; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP8]], i64 %n) 342; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[COND:%.*]], i64 [[TMP8]] 343; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP10]], i32 0 344; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <vscale x 4 x i32>* 345; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP12]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison) 346; CHECK-NEXT: [[TMP13:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], zeroinitializer 347; CHECK-NEXT: [[TMP14:%.*]] = xor <vscale x 4 x i1> [[TMP13]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i32 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer) 348; CHECK-NEXT: [[TMP15:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i1> [[TMP14]], <vscale x 4 x i1> zeroinitializer 349; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> [[BROADCAST_SPLAT6]], i32 4, <vscale x 4 x i1> [[TMP15]], <vscale x 4 x i32> undef) 350; CHECK-NEXT: [[TMP16:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i1> zeroinitializer 351; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP16]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[WIDE_MASKED_GATHER]] 352; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 [[TMP8]] 353; CHECK-NEXT: [[TMP18:%.*]] = or <vscale x 4 x i1> [[TMP15]], [[TMP16]] 354; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, i32* [[TMP17]], i32 0 355; CHECK-NEXT: [[TMP20:%.*]] = bitcast i32* [[TMP19]] to <vscale x 4 x i32>* 356; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x i32>* [[TMP20]], i32 4, <vscale x 4 x i1> [[TMP18]]) 357; CHECK-NEXT: [[TMP21:%.*]] = call i64 @llvm.vscale.i64() 358; CHECK-NEXT: [[TMP22:%.*]] = mul i64 [[TMP21]], 4 359; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP22]] 360; CHECK-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]] 361; CHECK-NEXT: br i1 [[TMP23]], label %middle.block, label %vector.body 362; CHECK: middle.block: 363; CHECK-NEXT: br i1 true, label %for.end, label %scalar.ph 364; 365 366entry: 367 br label %for.body 368 369for.body: ; preds = %entry, %if.end 370 %index = phi i64 [ %index.next, %if.end ], [ 0, %entry ] 371 %arrayidx = getelementptr inbounds i32, i32* %cond, i64 %index 372 %0 = load i32, i32* %arrayidx, align 4 373 %tobool.not = icmp eq i32 %0, 0 374 br i1 %tobool.not, label %if.end, label %if.then 375 376if.then: ; preds = %for.body 377 %1 = load i32, i32* %src, align 4 378 br label %if.end 379 380if.end: ; preds = %if.then, %for.body 381 %val.0 = phi i32 [ %1, %if.then ], [ 0, %for.body ] 382 %arrayidx1 = getelementptr inbounds i32, i32* %dst, i64 %index 383 store i32 %val.0, i32* %arrayidx1, align 4 384 %index.next = add nuw i64 %index, 1 385 %exitcond.not = icmp eq i64 %index.next, %n 386 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0 387 388for.end: ; preds = %for.inc, %entry 389 ret void 390} 391 392 393; The original loop had an unconditional uniform store. Let's make sure 394; we don't artificially create new predicated blocks for the load. 395define void @uniform_store(i32* noalias %dst, i32* noalias readonly %src, i64 %n) #0 { 396; CHECK-LABEL: @uniform_store( 397; CHECK-NEXT: entry: 398; CHECK-NEXT: br i1 false, label %scalar.ph, label %vector.ph 399; CHECK: vector.ph: 400; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 401; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 402; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() 403; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 404; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[TMP3]], 1 405; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP4]] 406; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] 407; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 408; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i32*> poison, i32* [[DST:%.*]], i32 0 409; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i32*> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i32*> poison, <vscale x 4 x i32> zeroinitializer 410; CHECK-NEXT: br label %vector.body 411; CHECK: vector.body: 412; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ] 413; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[INDEX]], i32 0 414; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer 415; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64() 416; CHECK-NEXT: [[TMP6:%.*]] = add <vscale x 4 x i64> [[TMP5]], zeroinitializer 417; CHECK-NEXT: [[TMP7:%.*]] = mul <vscale x 4 x i64> [[TMP6]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i32 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer) 418; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 4 x i64> [[BROADCAST_SPLAT2]], [[TMP7]] 419; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 0 420; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP8]], i64 %n) 421; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 [[TMP8]] 422; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP10]], i32 0 423; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <vscale x 4 x i32>* 424; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP12]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison) 425; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0i32(<vscale x 4 x i32> [[WIDE_MASKED_LOAD]], <vscale x 4 x i32*> [[BROADCAST_SPLAT4]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]]) 426; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() 427; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 4 428; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP14]] 429; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 430; CHECK-NEXT: br i1 [[TMP15]], label %middle.block, label %vector.body 431; CHECK: middle.block: 432; CHECK-NEXT: br i1 true, label %for.end, label %scalar.ph 433; 434 435entry: 436 br label %for.body 437 438for.body: ; preds = %entry, %for.body 439 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 440 %arrayidx = getelementptr inbounds i32, i32* %src, i64 %indvars.iv 441 %val = load i32, i32* %arrayidx, align 4 442 store i32 %val, i32* %dst, align 4 443 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 444 %exitcond.not = icmp eq i64 %indvars.iv.next, %n 445 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0 446 447for.end: ; preds = %for.body, %entry 448 ret void 449} 450 451 452define void @simple_fdiv(float* noalias %dst, float* noalias %src, i64 %n) #0 { 453; CHECK-LABEL: @simple_fdiv( 454; CHECK-NEXT: entry: 455; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1) 456; CHECK-NEXT: br i1 false, label %scalar.ph, label %vector.ph 457; CHECK: vector.ph: 458; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 459; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 460; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() 461; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 462; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[TMP3]], 1 463; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] 464; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] 465; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 466; CHECK-NEXT: br label %vector.body 467; CHECK: vector.body: 468; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT2:%.*]], %vector.body ] 469; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[INDEX1]], i32 0 470; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer 471; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64() 472; CHECK-NEXT: [[TMP6:%.*]] = add <vscale x 4 x i64> [[TMP5]], zeroinitializer 473; CHECK-NEXT: [[TMP7:%.*]] = mul <vscale x 4 x i64> [[TMP6]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i32 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer) 474; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 4 x i64> [[BROADCAST_SPLAT4]], [[TMP7]] 475; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX1]], 0 476; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP8]], i64 [[UMAX]]) 477; CHECK-NEXT: [[TMP10:%.*]] = getelementptr float, float* [[SRC:%.*]], i64 [[TMP8]] 478; CHECK-NEXT: [[TMP11:%.*]] = getelementptr float, float* [[DST:%.*]], i64 [[TMP8]] 479; CHECK-NEXT: [[TMP12:%.*]] = getelementptr float, float* [[TMP10]], i32 0 480; CHECK-NEXT: [[TMP13:%.*]] = bitcast float* [[TMP12]] to <vscale x 4 x float>* 481; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0nxv4f32(<vscale x 4 x float>* [[TMP13]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison) 482; CHECK-NEXT: [[TMP14:%.*]] = getelementptr float, float* [[TMP11]], i32 0 483; CHECK-NEXT: [[TMP15:%.*]] = bitcast float* [[TMP14]] to <vscale x 4 x float>* 484; CHECK-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0nxv4f32(<vscale x 4 x float>* [[TMP15]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison) 485; CHECK-NEXT: [[TMP16:%.*]] = fdiv <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD5]] 486; CHECK-NEXT: [[TMP17:%.*]] = bitcast float* [[TMP14]] to <vscale x 4 x float>* 487; CHECK-NEXT: call void @llvm.masked.store.nxv4f32.p0nxv4f32(<vscale x 4 x float> [[TMP16]], <vscale x 4 x float>* [[TMP17]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]]) 488; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() 489; CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 4 490; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP19]] 491; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]] 492; CHECK-NEXT: br i1 [[TMP20]], label %middle.block, label %vector.body, !llvm.loop [[LOOP18:![0-9]+]] 493; CHECK: middle.block: 494; CHECK-NEXT: br i1 true, label %while.end.loopexit, label %scalar.ph 495; 496entry: 497 br label %while.body 498 499while.body: ; preds = %while.body, %entry 500 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ] 501 %gep1 = getelementptr float, float* %src, i64 %index 502 %gep2 = getelementptr float, float* %dst, i64 %index 503 %val1 = load float, float* %gep1 504 %val2 = load float, float* %gep2 505 %res = fdiv float %val1, %val2 506 store float %res, float* %gep2 507 %index.next = add nsw i64 %index, 1 508 %cmp10 = icmp ult i64 %index.next, %n 509 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0 510 511while.end.loopexit: ; preds = %while.body 512 ret void 513} 514 515 516define i32 @add_reduction_i32(i32* %ptr, i64 %n) #0 { 517; CHECK-LABEL: @add_reduction_i32( 518; CHECK-NEXT: entry: 519; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1) 520; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label %vector.ph 521; CHECK: vector.ph: 522; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 523; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 524; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() 525; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 526; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[TMP3]], 1 527; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] 528; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] 529; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 530; CHECK-NEXT: br label %vector.body 531; CHECK: vector.body: 532; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT2:%.*]], %vector.body ] 533; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %vector.ph ], [ [[TMP14:%.*]], %vector.body ] 534; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[INDEX1]], i32 0 535; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer 536; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64() 537; CHECK-NEXT: [[TMP6:%.*]] = add <vscale x 4 x i64> [[TMP5]], zeroinitializer 538; CHECK-NEXT: [[TMP7:%.*]] = mul <vscale x 4 x i64> [[TMP6]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i32 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer) 539; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 4 x i64> [[BROADCAST_SPLAT]], [[TMP7]] 540; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX1]], 0 541; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP8]], i64 [[UMAX]]) 542; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i32, i32* [[PTR:%.*]], i64 [[TMP8]] 543; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, i32* [[TMP9]], i32 0 544; CHECK-NEXT: [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <vscale x 4 x i32>* 545; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP11]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison) 546; CHECK-NEXT: [[TMP12:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], <vscale x 4 x i32> zeroinitializer 547; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP12]]) 548; CHECK-NEXT: [[TMP14]] = add i32 [[TMP13]], [[VEC_PHI]] 549; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() 550; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 551; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP16]] 552; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]] 553; CHECK-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label %vector.body, !llvm.loop [[LOOP18:![0-9]+]] 554; CHECK: middle.block: 555; CHECK-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label %scalar.ph 556; 557entry: 558 br label %while.body 559 560while.body: ; preds = %while.body, %entry 561 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ] 562 %red = phi i32 [ %red.next, %while.body ], [ 0, %entry ] 563 %gep = getelementptr i32, i32* %ptr, i64 %index 564 %val = load i32, i32* %gep 565 %red.next = add i32 %red, %val 566 %index.next = add nsw i64 %index, 1 567 %cmp10 = icmp ult i64 %index.next, %n 568 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0 569 570while.end.loopexit: ; preds = %while.body 571 ret i32 %red.next 572} 573 574define float @add_reduction_f32(float* %ptr, i64 %n) #0 { 575; CHECK-LABEL: @add_reduction_f32( 576; CHECK-NEXT: entry: 577; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1) 578; CHECK-NEXT: br i1 false, label %scalar.ph, label %vector.ph 579; CHECK: vector.ph: 580; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 581; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 582; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() 583; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 584; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[TMP3]], 1 585; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] 586; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] 587; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 588; CHECK-NEXT: br label %vector.body 589; CHECK: vector.body: 590; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT2:%.*]], %vector.body ] 591; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %vector.ph ], [ [[TMP14:%.*]], %vector.body ] 592; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[INDEX1]], i32 0 593; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer 594; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64() 595; CHECK-NEXT: [[TMP6:%.*]] = add <vscale x 4 x i64> [[TMP5]], zeroinitializer 596; CHECK-NEXT: [[TMP7:%.*]] = mul <vscale x 4 x i64> [[TMP6]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i32 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer) 597; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 4 x i64> [[BROADCAST_SPLAT4]], [[TMP7]] 598; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX1]], 0 599; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP8]], i64 [[UMAX]]) 600; CHECK-NEXT: [[TMP10:%.*]] = getelementptr float, float* [[PTR:%.*]], i64 [[TMP8]] 601; CHECK-NEXT: [[TMP11:%.*]] = getelementptr float, float* [[TMP10]], i32 0 602; CHECK-NEXT: [[TMP12:%.*]] = bitcast float* [[TMP11]] to <vscale x 4 x float>* 603; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0nxv4f32(<vscale x 4 x float>* [[TMP12]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison) 604; CHECK-NEXT: [[TMP13:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float> shufflevector (<vscale x 4 x float> insertelement (<vscale x 4 x float> poison, float -0.000000e+00, i32 0), <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer) 605; CHECK-NEXT: [[TMP14]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[TMP13]]) 606; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() 607; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 608; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP16]] 609; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]] 610; CHECK-NEXT: br i1 [[TMP17]], label %middle.block, label %vector.body, !llvm.loop [[LOOP22:![0-9]+]] 611; CHECK: middle.block: 612; CHECK-NEXT: br i1 true, label %while.end.loopexit, label %scalar.ph 613; 614entry: 615 br label %while.body 616 617while.body: ; preds = %while.body, %entry 618 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ] 619 %red = phi float [ %red.next, %while.body ], [ 0.000000, %entry ] 620 %gep = getelementptr float, float* %ptr, i64 %index 621 %val = load float, float* %gep 622 %red.next = fadd float %red, %val 623 %index.next = add nsw i64 %index, 1 624 %cmp10 = icmp ult i64 %index.next, %n 625 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0 626 627while.end.loopexit: ; preds = %while.body 628 ret float %red.next 629} 630 631define i32 @cond_xor_reduction(i32* noalias %a, i32* noalias %cond, i64 %N) #0 { 632; CHECK-LABEL: @cond_xor_reduction( 633; CHECK-NEXT: entry: 634; CHECK-NEXT: br i1 false, label %scalar.ph, label %vector.ph 635; CHECK: vector.ph: 636; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 637; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 638; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() 639; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 640; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[TMP3]], 1 641; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP4]] 642; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] 643; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] 644; CHECK-NEXT: br label %vector.body 645; CHECK: vector.body: 646; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ] 647; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ insertelement (<vscale x 4 x i32> zeroinitializer, i32 7, i32 0), %vector.ph ], [ [[PREDPHI:%.*]], %vector.body ] 648; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[INDEX]], i32 0 649; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer 650; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64() 651; CHECK-NEXT: [[TMP6:%.*]] = add <vscale x 4 x i64> [[TMP5]], zeroinitializer 652; CHECK-NEXT: [[TMP7:%.*]] = mul <vscale x 4 x i64> [[TMP6]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i32 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer) 653; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 4 x i64> [[BROADCAST_SPLAT]], [[TMP7]] 654; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 0 655; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP8]], i64 [[N]]) 656; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[COND:%.*]], i64 [[TMP8]] 657; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP9]], i32 0 658; CHECK-NEXT: [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <vscale x 4 x i32>* 659; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP11]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison) 660; CHECK-NEXT: [[TMP12:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 5, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer) 661; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i32, i32* [[A:%.*]], i64 [[TMP8]] 662; CHECK-NEXT: [[TMP14:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i1> [[TMP12]], <vscale x 4 x i1> zeroinitializer 663; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i32, i32* [[TMP13]], i32 0 664; CHECK-NEXT: [[TMP16:%.*]] = bitcast i32* [[TMP15]] to <vscale x 4 x i32>* 665; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP16]], i32 4, <vscale x 4 x i1> [[TMP14]], <vscale x 4 x i32> poison) 666; CHECK-NEXT: [[TMP17:%.*]] = xor <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_LOAD1]] 667; CHECK-NEXT: [[TMP18:%.*]] = xor <vscale x 4 x i1> [[TMP12]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i32 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer) 668; CHECK-NEXT: [[TMP19:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i1> [[TMP18]], <vscale x 4 x i1> zeroinitializer 669; CHECK-NEXT: [[PREDPHI]] = select <vscale x 4 x i1> [[TMP14]], <vscale x 4 x i32> [[TMP17]], <vscale x 4 x i32> [[VEC_PHI]] 670; CHECK-NEXT: [[TMP20:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x i32> [[VEC_PHI]] 671; CHECK-NEXT: [[TMP21:%.*]] = call i64 @llvm.vscale.i64() 672; CHECK-NEXT: [[TMP22:%.*]] = mul i64 [[TMP21]], 4 673; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP22]] 674; CHECK-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 675; CHECK-NEXT: br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label %vector.body, !llvm.loop [[LOOP22:![0-9]+]] 676; CHECK: middle.block: 677; CHECK-NEXT: [[TMP24:%.*]] = call i32 @llvm.vector.reduce.xor.nxv4i32(<vscale x 4 x i32> [[TMP20]]) 678; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label %scalar.ph 679; 680entry: 681 br label %for.body 682 683for.body: 684 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] 685 %rdx = phi i32 [ 7, %entry ], [ %res, %for.inc ] 686 %arrayidx = getelementptr inbounds i32, i32* %cond, i64 %iv 687 %0 = load i32, i32* %arrayidx 688 %tobool = icmp eq i32 %0, 5 689 br i1 %tobool, label %if.then, label %for.inc 690 691if.then: 692 %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %iv 693 %1 = load i32, i32* %arrayidx2 694 %xor = xor i32 %rdx, %1 695 br label %for.inc 696 697for.inc: 698 %res = phi i32 [ %rdx, %for.body ], [ %xor, %if.then ] 699 %iv.next = add i64 %iv, 1 700 %exitcond.not = icmp eq i64 %iv.next, %N 701 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0 702 703for.end: 704 ret i32 %res 705} 706 707; Negative tests where we don't expect tail-folding 708 709; Integer divides can throw exceptions and since we can't scalarize conditional 710; divides for scalable vectors we just don't bother vectorizing. 711define void @simple_idiv(i32* noalias %dst, i32* noalias %src, i64 %n) #0 { 712; CHECK-LABEL: @simple_idiv( 713; CHECK-NOT: vector.body 714; 715entry: 716 br label %while.body 717 718while.body: ; preds = %while.body, %entry 719 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ] 720 %gep1 = getelementptr i32, i32* %src, i64 %index 721 %gep2 = getelementptr i32, i32* %dst, i64 %index 722 %val1 = load i32, i32* %gep1 723 %val2 = load i32, i32* %gep2 724 %res = udiv i32 %val1, %val2 725 store i32 %res, i32* %gep2 726 %index.next = add nsw i64 %index, 1 727 %cmp10 = icmp ult i64 %index.next, %n 728 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0 729 730while.end.loopexit: ; preds = %while.body 731 ret void 732} 733 734!0 = distinct !{!0, !1, !2} 735!1 = !{!"llvm.loop.vectorize.width", i32 4} 736!2 = !{!"llvm.loop.vectorize.scalable.enable", i1 true} 737 738attributes #0 = { "target-features"="+sve" } 739