1; RUN: opt < %s -licm -loop-vectorize -force-vector-width=4 -dce -instcombine -licm -S | FileCheck %s 2 3; First licm pass is to hoist/sink invariant stores if possible. Today LICM does 4; not hoist/sink the invariant stores. Even if that changes, we should still 5; vectorize this loop in case licm is not run. 6 7; The next licm pass after vectorization is to hoist/sink loop invariant 8; instructions. 9target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" 10 11; all tests check that it is legal to vectorize the stores to invariant 12; address. 13 14 15; CHECK-LABEL: inv_val_store_to_inv_address_with_reduction( 16; memory check is found.conflict = b[max(n-1,1)] > a && (i8* a)+1 > (i8* b) 17; CHECK: vector.memcheck: 18; CHECK: found.conflict 19 20; CHECK-LABEL: vector.body: 21; CHECK: %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ [[ADD:%[a-zA-Z0-9.]+]], %vector.body ] 22; CHECK: %wide.load = load <4 x i32> 23; CHECK: [[ADD]] = add <4 x i32> %vec.phi, %wide.load 24; CHECK-NEXT: store i32 %ntrunc, i32* %a 25; CHECK-NEXT: %index.next = add i64 %index, 4 26; CHECK-NEXT: icmp eq i64 %index.next, %n.vec 27; CHECK-NEXT: br i1 28 29; CHECK-LABEL: middle.block: 30; CHECK: call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> 31define i32 @inv_val_store_to_inv_address_with_reduction(i32* %a, i64 %n, i32* %b) { 32entry: 33 %ntrunc = trunc i64 %n to i32 34 br label %for.body 35 36for.body: ; preds = %for.body, %entry 37 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ] 38 %tmp0 = phi i32 [ %tmp3, %for.body ], [ 0, %entry ] 39 %tmp1 = getelementptr inbounds i32, i32* %b, i64 %i 40 %tmp2 = load i32, i32* %tmp1, align 8 41 %tmp3 = add i32 %tmp0, %tmp2 42 store i32 %ntrunc, i32* %a 43 %i.next = add nuw nsw i64 %i, 1 44 %cond = icmp slt i64 %i.next, %n 45 br i1 %cond, label %for.body, label %for.end 46 47for.end: ; preds = %for.body 48 %tmp4 = phi i32 [ %tmp3, %for.body ] 49 ret i32 %tmp4 50} 51 52; CHECK-LABEL: inv_val_store_to_inv_address( 53; CHECK-LABEL: vector.body: 54; CHECK: store i32 %ntrunc, i32* %a 55; CHECK: store <4 x i32> 56; CHECK-NEXT: %index.next = add i64 %index, 4 57; CHECK-NEXT: icmp eq i64 %index.next, %n.vec 58; CHECK-NEXT: br i1 59define void @inv_val_store_to_inv_address(i32* %a, i64 %n, i32* %b) { 60entry: 61 %ntrunc = trunc i64 %n to i32 62 br label %for.body 63 64for.body: ; preds = %for.body, %entry 65 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ] 66 %tmp1 = getelementptr inbounds i32, i32* %b, i64 %i 67 %tmp2 = load i32, i32* %tmp1, align 8 68 store i32 %ntrunc, i32* %a 69 store i32 %ntrunc, i32* %tmp1 70 %i.next = add nuw nsw i64 %i, 1 71 %cond = icmp slt i64 %i.next, %n 72 br i1 %cond, label %for.body, label %for.end 73 74for.end: ; preds = %for.body 75 ret void 76} 77 78 79; Both of these tests below are handled as predicated stores. 80 81; Conditional store 82; if (b[i] == k) a = ntrunc 83; TODO: We can be better with the code gen for the first test and we can have 84; just one scalar store if vector.or.reduce(vector_cmp(b[i] == k)) is 1. 85 86; CHECK-LABEL:inv_val_store_to_inv_address_conditional( 87; CHECK-LABEL: vector.body: 88; CHECK: %wide.load = load <4 x i32>, <4 x i32>* 89; CHECK: [[CMP:%[a-zA-Z0-9.]+]] = icmp eq <4 x i32> %wide.load, %{{.*}} 90; CHECK: store <4 x i32> 91; CHECK-NEXT: [[EE:%[a-zA-Z0-9.]+]] = extractelement <4 x i1> [[CMP]], i32 0 92; CHECK-NEXT: br i1 [[EE]], label %pred.store.if, label %pred.store.continue 93 94; CHECK-LABEL: pred.store.if: 95; CHECK-NEXT: store i32 %ntrunc, i32* %a 96; CHECK-NEXT: br label %pred.store.continue 97 98; CHECK-LABEL: pred.store.continue: 99; CHECK-NEXT: [[EE1:%[a-zA-Z0-9.]+]] = extractelement <4 x i1> [[CMP]], i32 1 100define void @inv_val_store_to_inv_address_conditional(i32* %a, i64 %n, i32* %b, i32 %k) { 101entry: 102 %ntrunc = trunc i64 %n to i32 103 br label %for.body 104 105for.body: ; preds = %for.body, %entry 106 %i = phi i64 [ %i.next, %latch ], [ 0, %entry ] 107 %tmp1 = getelementptr inbounds i32, i32* %b, i64 %i 108 %tmp2 = load i32, i32* %tmp1, align 8 109 %cmp = icmp eq i32 %tmp2, %k 110 store i32 %ntrunc, i32* %tmp1 111 br i1 %cmp, label %cond_store, label %latch 112 113cond_store: 114 store i32 %ntrunc, i32* %a 115 br label %latch 116 117latch: 118 %i.next = add nuw nsw i64 %i, 1 119 %cond = icmp slt i64 %i.next, %n 120 br i1 %cond, label %for.body, label %for.end 121 122for.end: ; preds = %for.body 123 ret void 124} 125 126; if (b[i] == k) 127; a = ntrunc 128; else a = k; 129; TODO: We could vectorize this once we support multiple uniform stores to the 130; same address. 131; CHECK-LABEL:inv_val_store_to_inv_address_conditional_diff_values( 132; CHECK-NOT: load <4 x i32> 133define void @inv_val_store_to_inv_address_conditional_diff_values(i32* %a, i64 %n, i32* %b, i32 %k) { 134entry: 135 %ntrunc = trunc i64 %n to i32 136 br label %for.body 137 138for.body: ; preds = %for.body, %entry 139 %i = phi i64 [ %i.next, %latch ], [ 0, %entry ] 140 %tmp1 = getelementptr inbounds i32, i32* %b, i64 %i 141 %tmp2 = load i32, i32* %tmp1, align 8 142 %cmp = icmp eq i32 %tmp2, %k 143 store i32 %ntrunc, i32* %tmp1 144 br i1 %cmp, label %cond_store, label %cond_store_k 145 146cond_store: 147 store i32 %ntrunc, i32* %a 148 br label %latch 149 150cond_store_k: 151 store i32 %k, i32 * %a 152 br label %latch 153 154latch: 155 %i.next = add nuw nsw i64 %i, 1 156 %cond = icmp slt i64 %i.next, %n 157 br i1 %cond, label %for.body, label %for.end 158 159for.end: ; preds = %for.body 160 ret void 161} 162 163; Instcombine'd version of above test. Now the store is no longer of invariant 164; value. 165; scalar store the value extracted from the last element of the vector value. 166; CHECK-LABEL: inv_val_store_to_inv_address_conditional_diff_values_ic 167; CHECK-NEXT: entry: 168; CHECK-NEXT: [[NTRUNC:%.*]] = trunc i64 [[N:%.*]] to i32 169; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) 170; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4 171; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 172; CHECK: vector.memcheck: 173; CHECK-NEXT: [[A4:%.*]] = bitcast i32* [[A:%.*]] to i8* 174; CHECK-NEXT: [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8* 175; CHECK-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) 176; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[SMAX2]] 177; CHECK-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, i8* [[A4]], i64 1 178; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt i8* [[UGLYGEP]], [[B1]] 179; CHECK-NEXT: [[BOUND1:%.*]] = icmp ugt i32* [[SCEVGEP]], [[A]] 180; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 181; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 182; CHECK: vector.ph: 183; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775804 184; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i32> poison, i32 [[K:%.*]], i32 0 185; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT5]], <4 x i32> poison, <4 x i32> zeroinitializer 186; CHECK-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <4 x i32> poison, i32 [[NTRUNC]], i32 0 187; CHECK-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT7]], <4 x i32> poison, <4 x i32> zeroinitializer 188; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 189; CHECK: vector.body: 190; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 191; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]] 192; CHECK-NEXT: [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <4 x i32>* 193; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP3]], align 8 194; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT6]] 195; CHECK-NEXT: [[TMP5:%.*]] = bitcast i32* [[TMP2]] to <4 x i32>* 196; CHECK-NEXT: store <4 x i32> [[BROADCAST_SPLAT8]], <4 x i32>* [[TMP5]], align 4 197; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP4]], <4 x i32> [[BROADCAST_SPLAT8]], <4 x i32> [[BROADCAST_SPLAT6]] 198; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[PREDPHI]], i32 3 199; CHECK-NEXT: store i32 [[TMP6]], i32* [[A]], align 4 200; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 201; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 202; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]] 203; CHECK: middle.block: 204; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]] 205; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 206; CHECK: scalar.ph: 207; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ] 208; CHECK-NEXT: br label [[FOR_BODY:%.*]] 209; CHECK: for.body: 210; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 211; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[I]] 212; CHECK-NEXT: [[TMP2:%.*]] = load i32, i32* [[TMP1]], align 8 213; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP2]], [[K]] 214; CHECK-NEXT: store i32 [[NTRUNC]], i32* [[TMP1]], align 4 215; CHECK-NEXT: br i1 [[CMP]], label [[COND_STORE:%.*]], label [[COND_STORE_K:%.*]] 216; CHECK: cond_store: 217; CHECK-NEXT: br label [[LATCH]] 218; CHECK: cond_store_k: 219; CHECK-NEXT: br label [[LATCH]] 220; CHECK: latch: 221; CHECK-NEXT: [[STOREVAL:%.*]] = phi i32 [ [[NTRUNC]], [[COND_STORE]] ], [ [[K]], [[COND_STORE_K]] ] 222; CHECK-NEXT: store i32 [[STOREVAL]], i32* [[A]], align 4 223; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 224; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 225; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT:%.*]] 226; CHECK: for.end.loopexit: 227; CHECK-NEXT: br label [[FOR_END]] 228; CHECK: for.end: 229; CHECK-NEXT: ret void 230; 231define void @inv_val_store_to_inv_address_conditional_diff_values_ic(i32* %a, i64 %n, i32* %b, i32 %k) { 232entry: 233 %ntrunc = trunc i64 %n to i32 234 br label %for.body 235 236for.body: ; preds = %for.body, %entry 237 %i = phi i64 [ %i.next, %latch ], [ 0, %entry ] 238 %tmp1 = getelementptr inbounds i32, i32* %b, i64 %i 239 %tmp2 = load i32, i32* %tmp1, align 8 240 %cmp = icmp eq i32 %tmp2, %k 241 store i32 %ntrunc, i32* %tmp1 242 br i1 %cmp, label %cond_store, label %cond_store_k 243 244cond_store: 245 br label %latch 246 247cond_store_k: 248 br label %latch 249 250latch: 251 %storeval = phi i32 [ %ntrunc, %cond_store ], [ %k, %cond_store_k ] 252 store i32 %storeval, i32* %a 253 %i.next = add nuw nsw i64 %i, 1 254 %cond = icmp slt i64 %i.next, %n 255 br i1 %cond, label %for.body, label %for.end 256 257for.end: ; preds = %for.body 258 ret void 259} 260 261; invariant val stored to invariant address predicated on invariant condition 262; This is not treated as a predicated store since the block the store belongs to 263; is the latch block (which doesn't need to be predicated). 264; variant/invariant values being stored to invariant address. 265; test checks that the last element of the phi is extracted and scalar stored 266; into the uniform address within the loop. 267; Since the condition and the phi is loop invariant, they are LICM'ed after 268; vectorization. 269; CHECK-LABEL: inv_val_store_to_inv_address_conditional_inv 270; CHECK-NEXT: entry: 271; CHECK-NEXT: [[NTRUNC:%.*]] = trunc i64 [[N:%.*]] to i32 272; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[NTRUNC]], [[K:%.*]] 273; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) 274; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4 275; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 276; CHECK: vector.memcheck: 277; CHECK-NEXT: [[A4:%.*]] = bitcast i32* [[A:%.*]] to i8* 278; CHECK-NEXT: [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8* 279; CHECK-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) 280; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[SMAX2]] 281; CHECK-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, i8* [[A4]], i64 1 282; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt i8* [[UGLYGEP]], [[B1]] 283; CHECK-NEXT: [[BOUND1:%.*]] = icmp ugt i32* [[SCEVGEP]], [[A]] 284; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 285; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 286; CHECK: vector.ph: 287; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775804 288; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i32> poison, i32 [[NTRUNC]], i32 0 289; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT5]], <4 x i32> poison, <4 x i32> zeroinitializer 290; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i1> undef, i1 [[CMP]], i32 3 291; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> poison, i32 [[K]], i32 3 292; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[BROADCAST_SPLAT6]], <4 x i32> [[TMP3]] 293; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[PREDPHI]], i32 3 294; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 295; CHECK: vector.body: 296; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 297; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]] 298; CHECK-NEXT: [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>* 299; CHECK-NEXT: store <4 x i32> [[BROADCAST_SPLAT6]], <4 x i32>* [[TMP7]], align 4 300; CHECK-NEXT: store i32 [[TMP5]], i32* [[A]], align 4 301; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 302; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 303; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]] 304; CHECK: middle.block: 305; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]] 306; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 307; CHECK: scalar.ph: 308; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ] 309; CHECK-NEXT: br label [[FOR_BODY:%.*]] 310; CHECK: for.body: 311; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 312; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[I]] 313; CHECK-NEXT: store i32 [[NTRUNC]], i32* [[TMP1]], align 4 314; CHECK-NEXT: br i1 [[CMP]], label [[COND_STORE:%.*]], label [[COND_STORE_K:%.*]] 315; CHECK: cond_store: 316; CHECK-NEXT: br label [[LATCH]] 317; CHECK: cond_store_k: 318; CHECK-NEXT: br label [[LATCH]] 319; CHECK: latch: 320; CHECK-NEXT: [[STOREVAL:%.*]] = phi i32 [ [[NTRUNC]], [[COND_STORE]] ], [ [[K]], [[COND_STORE_K]] ] 321; CHECK-NEXT: store i32 [[STOREVAL]], i32* [[A]], align 4 322; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 323; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 324; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT:%.*]] 325; CHECK: for.end.loopexit: 326; CHECK-NEXT: br label [[FOR_END]] 327; CHECK: for.end: 328; CHECK-NEXT: ret void 329; 330define void @inv_val_store_to_inv_address_conditional_inv(i32* %a, i64 %n, i32* %b, i32 %k) { 331entry: 332 %ntrunc = trunc i64 %n to i32 333 %cmp = icmp eq i32 %ntrunc, %k 334 br label %for.body 335 336for.body: ; preds = %for.body, %entry 337 %i = phi i64 [ %i.next, %latch ], [ 0, %entry ] 338 %tmp1 = getelementptr inbounds i32, i32* %b, i64 %i 339 %tmp2 = load i32, i32* %tmp1, align 8 340 store i32 %ntrunc, i32* %tmp1 341 br i1 %cmp, label %cond_store, label %cond_store_k 342 343cond_store: 344 br label %latch 345 346cond_store_k: 347 br label %latch 348 349latch: 350 %storeval = phi i32 [ %ntrunc, %cond_store ], [ %k, %cond_store_k ] 351 store i32 %storeval, i32* %a 352 %i.next = add nuw nsw i64 %i, 1 353 %cond = icmp slt i64 %i.next, %n 354 br i1 %cond, label %for.body, label %for.end 355 356for.end: ; preds = %for.body 357 ret void 358} 359 360; variant value stored to uniform address tests that the code gen extracts the 361; last element from the variant vector and scalar stores it into the uniform 362; address. 363define i32 @variant_val_store_to_inv_address(i32* %a, i64 %n, i32* %b, i32 %k) { 364; CHECK-LABEL: @variant_val_store_to_inv_address( 365; CHECK-NEXT: entry: 366; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) 367; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4 368; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 369; CHECK: vector.memcheck: 370; CHECK-NEXT: [[B2:%.*]] = bitcast i32* [[B:%.*]] to i8* 371; CHECK-NEXT: [[A1:%.*]] = bitcast i32* [[A:%.*]] to i8* 372; CHECK-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, i8* [[A1]], i64 1 373; CHECK-NEXT: [[SMAX3:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) 374; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[SMAX3]] 375; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt i32* [[SCEVGEP]], [[A]] 376; CHECK-NEXT: [[BOUND1:%.*]] = icmp ugt i8* [[UGLYGEP]], [[B2]] 377; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 378; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 379; CHECK: vector.ph: 380; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775804 381; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 382; CHECK: vector.body: 383; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 384; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ] 385; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]] 386; CHECK-NEXT: [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <4 x i32>* 387; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP3]], align 8, !alias.scope !36 388; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i32 3 389; CHECK-NEXT: store i32 [[TMP4]], i32* [[A]], align 4, !alias.scope !39, !noalias !36 390; CHECK-NEXT: [[TMP5]] = add <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]] 391; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 392; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 393; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP41:!llvm.loop !.*]] 394; CHECK: middle.block: 395; CHECK-NEXT: [[DOTLCSSA:%.*]] = phi <4 x i32> [ [[TMP5]], [[VECTOR_BODY]] ] 396; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[DOTLCSSA]]) 397; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]] 398; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 399; CHECK: scalar.ph: 400; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ] 401; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ] 402; CHECK-NEXT: br label [[FOR_BODY:%.*]] 403; CHECK: for.body: 404; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 405; CHECK-NEXT: [[TMP0:%.*]] = phi i32 [ [[TMP3:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 406; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[I]] 407; CHECK-NEXT: [[TMP2:%.*]] = load i32, i32* [[TMP1]], align 8 408; CHECK-NEXT: store i32 [[TMP2]], i32* [[A]], align 4 409; CHECK-NEXT: [[TMP3]] = add i32 [[TMP0]], [[TMP2]] 410; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 411; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 412; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT:%.*]], [[LOOP42:!llvm.loop !.*]] 413; CHECK: for.end.loopexit: 414; CHECK-NEXT: [[TMP3_LCSSA:%.*]] = phi i32 [ [[TMP3]], [[FOR_BODY]] ] 415; CHECK-NEXT: br label [[FOR_END]] 416; CHECK: for.end: 417; CHECK-NEXT: [[RDX_LCSSA:%.*]] = phi i32 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ [[TMP3_LCSSA]], [[FOR_END_LOOPEXIT]] ] 418; CHECK-NEXT: ret i32 [[RDX_LCSSA]] 419; 420entry: 421 %ntrunc = trunc i64 %n to i32 422 %cmp = icmp eq i32 %ntrunc, %k 423 br label %for.body 424 425for.body: ; preds = %for.body, %entry 426 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ] 427 %tmp0 = phi i32 [ %tmp3, %for.body ], [ 0, %entry ] 428 %tmp1 = getelementptr inbounds i32, i32* %b, i64 %i 429 %tmp2 = load i32, i32* %tmp1, align 8 430 store i32 %tmp2, i32* %a 431 %tmp3 = add i32 %tmp0, %tmp2 432 %i.next = add nuw nsw i64 %i, 1 433 %cond = icmp slt i64 %i.next, %n 434 br i1 %cond, label %for.body, label %for.end 435 436for.end: ; preds = %for.body 437 %rdx.lcssa = phi i32 [ %tmp3, %for.body ] 438 ret i32 %rdx.lcssa 439} 440 441; Multiple variant stores to the same uniform address 442; We do not vectorize such loops currently. 443; for(; i < itr; i++) { 444; for(; j < itr; j++) { 445; var1[i] = var2[j] + var1[i]; 446; var1[i]++; 447; } 448; } 449 450; CHECK-LABEL: multiple_uniform_stores 451; CHECK-NOT: <4 x i32> 452define i32 @multiple_uniform_stores(i32* nocapture %var1, i32* nocapture readonly %var2, i32 %itr) #0 { 453entry: 454 %cmp20 = icmp eq i32 %itr, 0 455 br i1 %cmp20, label %for.end10, label %for.cond1.preheader 456 457for.cond1.preheader: ; preds = %entry, %for.inc8 458 %indvars.iv23 = phi i64 [ %indvars.iv.next24, %for.inc8 ], [ 0, %entry ] 459 %j.022 = phi i32 [ %j.1.lcssa, %for.inc8 ], [ 0, %entry ] 460 %cmp218 = icmp ult i32 %j.022, %itr 461 br i1 %cmp218, label %for.body3.lr.ph, label %for.inc8 462 463for.body3.lr.ph: ; preds = %for.cond1.preheader 464 %arrayidx5 = getelementptr inbounds i32, i32* %var1, i64 %indvars.iv23 465 %0 = zext i32 %j.022 to i64 466 br label %for.body3 467 468for.body3: ; preds = %for.body3, %for.body3.lr.ph 469 %indvars.iv = phi i64 [ %0, %for.body3.lr.ph ], [ %indvars.iv.next, %for.body3 ] 470 %arrayidx = getelementptr inbounds i32, i32* %var2, i64 %indvars.iv 471 %1 = load i32, i32* %arrayidx, align 4 472 %2 = load i32, i32* %arrayidx5, align 4 473 %add = add nsw i32 %2, %1 474 store i32 %add, i32* %arrayidx5, align 4 475 %3 = load i32, i32* %arrayidx5, align 4 476 %4 = add nsw i32 %3, 1 477 store i32 %4, i32* %arrayidx5, align 4 478 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 479 %lftr.wideiv = trunc i64 %indvars.iv.next to i32 480 %exitcond = icmp eq i32 %lftr.wideiv, %itr 481 br i1 %exitcond, label %for.inc8, label %for.body3 482 483for.inc8: ; preds = %for.body3, %for.cond1.preheader 484 %j.1.lcssa = phi i32 [ %j.022, %for.cond1.preheader ], [ %itr, %for.body3 ] 485 %indvars.iv.next24 = add nuw nsw i64 %indvars.iv23, 1 486 %lftr.wideiv25 = trunc i64 %indvars.iv.next24 to i32 487 %exitcond26 = icmp eq i32 %lftr.wideiv25, %itr 488 br i1 %exitcond26, label %for.end10, label %for.cond1.preheader 489 490for.end10: ; preds = %for.inc8, %entry 491 ret i32 undef 492} 493 494; second uniform store to the same address is conditional. 495; we do not vectorize this. 496; CHECK-LABEL: multiple_uniform_stores_conditional 497; CHECK-NOT: <4 x i32> 498define i32 @multiple_uniform_stores_conditional(i32* nocapture %var1, i32* nocapture readonly %var2, i32 %itr) #0 { 499entry: 500 %cmp20 = icmp eq i32 %itr, 0 501 br i1 %cmp20, label %for.end10, label %for.cond1.preheader 502 503for.cond1.preheader: ; preds = %entry, %for.inc8 504 %indvars.iv23 = phi i64 [ %indvars.iv.next24, %for.inc8 ], [ 0, %entry ] 505 %j.022 = phi i32 [ %j.1.lcssa, %for.inc8 ], [ 0, %entry ] 506 %cmp218 = icmp ult i32 %j.022, %itr 507 br i1 %cmp218, label %for.body3.lr.ph, label %for.inc8 508 509for.body3.lr.ph: ; preds = %for.cond1.preheader 510 %arrayidx5 = getelementptr inbounds i32, i32* %var1, i64 %indvars.iv23 511 %0 = zext i32 %j.022 to i64 512 br label %for.body3 513 514for.body3: ; preds = %for.body3, %for.body3.lr.ph 515 %indvars.iv = phi i64 [ %0, %for.body3.lr.ph ], [ %indvars.iv.next, %latch ] 516 %arrayidx = getelementptr inbounds i32, i32* %var2, i64 %indvars.iv 517 %1 = load i32, i32* %arrayidx, align 4 518 %2 = load i32, i32* %arrayidx5, align 4 519 %add = add nsw i32 %2, %1 520 store i32 %add, i32* %arrayidx5, align 4 521 %3 = load i32, i32* %arrayidx5, align 4 522 %4 = add nsw i32 %3, 1 523 %5 = icmp ugt i32 %3, 42 524 br i1 %5, label %cond_store, label %latch 525 526cond_store: 527 store i32 %4, i32* %arrayidx5, align 4 528 br label %latch 529 530latch: 531 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 532 %lftr.wideiv = trunc i64 %indvars.iv.next to i32 533 %exitcond = icmp eq i32 %lftr.wideiv, %itr 534 br i1 %exitcond, label %for.inc8, label %for.body3 535 536for.inc8: ; preds = %for.body3, %for.cond1.preheader 537 %j.1.lcssa = phi i32 [ %j.022, %for.cond1.preheader ], [ %itr, %latch ] 538 %indvars.iv.next24 = add nuw nsw i64 %indvars.iv23, 1 539 %lftr.wideiv25 = trunc i64 %indvars.iv.next24 to i32 540 %exitcond26 = icmp eq i32 %lftr.wideiv25, %itr 541 br i1 %exitcond26, label %for.end10, label %for.cond1.preheader 542 543for.end10: ; preds = %for.inc8, %entry 544 ret i32 undef 545} 546 547; cannot vectorize loop with unsafe dependency between uniform load (%tmp10) and store 548; (%tmp12) to the same address 549; PR39653 550; Note: %tmp10 could be replaced by phi(%arg4, %tmp12), a potentially vectorizable 551; 1st-order-recurrence 552define void @unsafe_dep_uniform_load_store(i32 %arg, i32 %arg1, i64 %arg2, i16* %arg3, i32 %arg4, i64 %arg5) { 553; CHECK-LABEL: unsafe_dep_uniform_load_store 554; CHECK-NOT: <4 x i32> 555bb: 556 %tmp = alloca i32 557 store i32 %arg4, i32* %tmp 558 %tmp6 = getelementptr inbounds i16, i16* %arg3, i64 %arg5 559 br label %bb7 560 561bb7: 562 %tmp8 = phi i64 [ 0, %bb ], [ %tmp24, %bb7 ] 563 %tmp9 = phi i32 [ %arg1, %bb ], [ %tmp23, %bb7 ] 564 %tmp10 = load i32, i32* %tmp 565 %tmp11 = mul nsw i32 %tmp9, %tmp10 566 %tmp12 = srem i32 %tmp11, 65536 567 %tmp13 = add nsw i32 %tmp12, %tmp9 568 %tmp14 = trunc i32 %tmp13 to i16 569 %tmp15 = trunc i64 %tmp8 to i32 570 %tmp16 = add i32 %arg, %tmp15 571 %tmp17 = zext i32 %tmp16 to i64 572 %tmp18 = getelementptr inbounds i16, i16* %tmp6, i64 %tmp17 573 store i16 %tmp14, i16* %tmp18, align 2 574 %tmp19 = add i32 %tmp13, %tmp9 575 %tmp20 = trunc i32 %tmp19 to i16 576 %tmp21 = and i16 %tmp20, 255 577 %tmp22 = getelementptr inbounds i16, i16* %arg3, i64 %tmp17 578 store i16 %tmp21, i16* %tmp22, align 2 579 %tmp23 = add nsw i32 %tmp9, 1 580 %tmp24 = add nuw nsw i64 %tmp8, 1 581 %tmp25 = icmp eq i64 %tmp24, %arg2 582 store i32 %tmp12, i32* %tmp 583 br i1 %tmp25, label %bb26, label %bb7 584 585bb26: 586 ret void 587} 588 589; Make sure any check-not directives are not triggered by function declarations. 590; CHECK: declare 591