1; RUN: opt < %s -loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -S | FileCheck %s 2; RUN: opt < %s -loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -instcombine -S | FileCheck %s --check-prefix=IND 3; RUN: opt < %s -loop-vectorize -force-vector-interleave=2 -force-vector-width=2 -instcombine -S | FileCheck %s --check-prefix=UNROLL 4; RUN: opt < %s -loop-vectorize -force-vector-interleave=2 -force-vector-width=2 -S | FileCheck %s --check-prefix=UNROLL-NO-IC 5; RUN: opt < %s -loop-vectorize -force-vector-interleave=2 -force-vector-width=4 -enable-interleaved-mem-accesses -instcombine -S | FileCheck %s --check-prefix=INTERLEAVE 6 7target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" 8 9; Make sure that we can handle multiple integer induction variables. 10; CHECK-LABEL: @multi_int_induction( 11; CHECK: vector.body: 12; CHECK: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] 13; CHECK: %[[VAR:.*]] = trunc i64 %index to i32 14; CHECK: %offset.idx = add i32 190, %[[VAR]] 15define void @multi_int_induction(i32* %A, i32 %N) { 16for.body.lr.ph: 17 br label %for.body 18 19for.body: 20 %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ] 21 %count.09 = phi i32 [ 190, %for.body.lr.ph ], [ %inc, %for.body ] 22 %arrayidx2 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv 23 store i32 %count.09, i32* %arrayidx2, align 4 24 %inc = add nsw i32 %count.09, 1 25 %indvars.iv.next = add i64 %indvars.iv, 1 26 %lftr.wideiv = trunc i64 %indvars.iv.next to i32 27 %exitcond = icmp ne i32 %lftr.wideiv, %N 28 br i1 %exitcond, label %for.body, label %for.end 29 30for.end: 31 ret void 32} 33 34; Make sure we remove unneeded vectorization of induction variables. 35; In order for instcombine to cleanup the vectorized induction variables that we 36; create in the loop vectorizer we need to perform some form of redundancy 37; elimination to get rid of multiple uses. 38 39; IND-LABEL: scalar_use 40 41; IND: br label %vector.body 42; IND: vector.body: 43; Vectorized induction variable. 44; IND-NOT: insertelement <2 x i64> 45; IND-NOT: shufflevector <2 x i64> 46; IND: br {{.*}}, label %vector.body 47 48define void @scalar_use(float* %a, float %b, i64 %offset, i64 %offset2, i64 %n) { 49entry: 50 br label %for.body 51 52for.body: 53 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] 54 %ind.sum = add i64 %iv, %offset 55 %arr.idx = getelementptr inbounds float, float* %a, i64 %ind.sum 56 %l1 = load float, float* %arr.idx, align 4 57 %ind.sum2 = add i64 %iv, %offset2 58 %arr.idx2 = getelementptr inbounds float, float* %a, i64 %ind.sum2 59 %l2 = load float, float* %arr.idx2, align 4 60 %m = fmul fast float %b, %l2 61 %ad = fadd fast float %l1, %m 62 store float %ad, float* %arr.idx, align 4 63 %iv.next = add nuw nsw i64 %iv, 1 64 %exitcond = icmp eq i64 %iv.next, %n 65 br i1 %exitcond, label %loopexit, label %for.body 66 67loopexit: 68 ret void 69} 70 71; Make sure we don't create a vector induction phi node that is unused. 72; Scalarize the step vectors instead. 73; 74; for (int i = 0; i < n; ++i) 75; sum += a[i]; 76; 77; CHECK-LABEL: @scalarize_induction_variable_01( 78; CHECK: vector.body: 79; CHECK: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] 80; CHECK: %[[i0:.+]] = add i64 %index, 0 81; CHECK: getelementptr inbounds i64, i64* %a, i64 %[[i0]] 82; 83; UNROLL-NO-IC-LABEL: @scalarize_induction_variable_01( 84; UNROLL-NO-IC: vector.body: 85; UNROLL-NO-IC: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] 86; UNROLL-NO-IC: %[[i0:.+]] = add i64 %index, 0 87; UNROLL-NO-IC: %[[i2:.+]] = add i64 %index, 2 88; UNROLL-NO-IC: getelementptr inbounds i64, i64* %a, i64 %[[i0]] 89; UNROLL-NO-IC: getelementptr inbounds i64, i64* %a, i64 %[[i2]] 90; 91; IND-LABEL: @scalarize_induction_variable_01( 92; IND: vector.body: 93; IND: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] 94; IND-NOT: add i64 {{.*}}, 2 95; IND: getelementptr inbounds i64, i64* %a, i64 %index 96; 97; UNROLL-LABEL: @scalarize_induction_variable_01( 98; UNROLL: vector.body: 99; UNROLL: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] 100; UNROLL-NOT: add i64 {{.*}}, 4 101; UNROLL: %[[g1:.+]] = getelementptr inbounds i64, i64* %a, i64 %index 102; UNROLL: getelementptr i64, i64* %[[g1]], i64 2 103 104define i64 @scalarize_induction_variable_01(i64 *%a, i64 %n) { 105entry: 106 br label %for.body 107 108for.body: 109 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ] 110 %sum = phi i64 [ %2, %for.body ], [ 0, %entry ] 111 %0 = getelementptr inbounds i64, i64* %a, i64 %i 112 %1 = load i64, i64* %0, align 8 113 %2 = add i64 %1, %sum 114 %i.next = add nuw nsw i64 %i, 1 115 %cond = icmp slt i64 %i.next, %n 116 br i1 %cond, label %for.body, label %for.end 117 118for.end: 119 %3 = phi i64 [ %2, %for.body ] 120 ret i64 %3 121} 122 123; Make sure we scalarize the step vectors used for the pointer arithmetic. We 124; can't easily simplify vectorized step vectors. 125; 126; float s = 0; 127; for (int i ; 0; i < n; i += 8) 128; s += (a[i] + b[i] + 1.0f); 129; 130; CHECK-LABEL: @scalarize_induction_variable_02( 131; CHECK: vector.body: 132; CHECK: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] 133; CHECK: %offset.idx = shl i64 %index, 3 134; CHECK: %[[i0:.+]] = add i64 %offset.idx, 0 135; CHECK: %[[i1:.+]] = add i64 %offset.idx, 8 136; CHECK: getelementptr inbounds float, float* %a, i64 %[[i0]] 137; CHECK: getelementptr inbounds float, float* %a, i64 %[[i1]] 138; CHECK: getelementptr inbounds float, float* %b, i64 %[[i0]] 139; CHECK: getelementptr inbounds float, float* %b, i64 %[[i1]] 140; 141; UNROLL-NO-IC-LABEL: @scalarize_induction_variable_02( 142; UNROLL-NO-IC: vector.body: 143; UNROLL-NO-IC: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] 144; UNROLL-NO-IC: %offset.idx = shl i64 %index, 3 145; UNROLL-NO-IC: %[[i0:.+]] = add i64 %offset.idx, 0 146; UNROLL-NO-IC: %[[i1:.+]] = add i64 %offset.idx, 8 147; UNROLL-NO-IC: %[[i2:.+]] = add i64 %offset.idx, 16 148; UNROLL-NO-IC: %[[i3:.+]] = add i64 %offset.idx, 24 149; UNROLL-NO-IC: getelementptr inbounds float, float* %a, i64 %[[i0]] 150; UNROLL-NO-IC: getelementptr inbounds float, float* %a, i64 %[[i1]] 151; UNROLL-NO-IC: getelementptr inbounds float, float* %a, i64 %[[i2]] 152; UNROLL-NO-IC: getelementptr inbounds float, float* %a, i64 %[[i3]] 153; UNROLL-NO-IC: getelementptr inbounds float, float* %b, i64 %[[i0]] 154; UNROLL-NO-IC: getelementptr inbounds float, float* %b, i64 %[[i1]] 155; UNROLL-NO-IC: getelementptr inbounds float, float* %b, i64 %[[i2]] 156; UNROLL-NO-IC: getelementptr inbounds float, float* %b, i64 %[[i3]] 157; 158; IND-LABEL: @scalarize_induction_variable_02( 159; IND: vector.body: 160; IND: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] 161; IND: %[[i0:.+]] = shl i64 %index, 3 162; IND: %[[i1:.+]] = or i64 %[[i0]], 8 163; IND: getelementptr inbounds float, float* %a, i64 %[[i0]] 164; IND: getelementptr inbounds float, float* %a, i64 %[[i1]] 165; 166; UNROLL-LABEL: @scalarize_induction_variable_02( 167; UNROLL: vector.body: 168; UNROLL: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] 169; UNROLL: %[[i0:.+]] = shl i64 %index, 3 170; UNROLL: %[[i1:.+]] = or i64 %[[i0]], 8 171; UNROLL: %[[i2:.+]] = or i64 %[[i0]], 16 172; UNROLL: %[[i3:.+]] = or i64 %[[i0]], 24 173; UNROLL: getelementptr inbounds float, float* %a, i64 %[[i0]] 174; UNROLL: getelementptr inbounds float, float* %a, i64 %[[i1]] 175; UNROLL: getelementptr inbounds float, float* %a, i64 %[[i2]] 176; UNROLL: getelementptr inbounds float, float* %a, i64 %[[i3]] 177 178define float @scalarize_induction_variable_02(float* %a, float* %b, i64 %n) { 179entry: 180 br label %for.body 181 182for.body: 183 %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ] 184 %s = phi float [ 0.0, %entry ], [ %6, %for.body ] 185 %0 = getelementptr inbounds float, float* %a, i64 %i 186 %1 = load float, float* %0, align 4 187 %2 = getelementptr inbounds float, float* %b, i64 %i 188 %3 = load float, float* %2, align 4 189 %4 = fadd fast float %s, 1.0 190 %5 = fadd fast float %4, %1 191 %6 = fadd fast float %5, %3 192 %i.next = add nuw nsw i64 %i, 8 193 %cond = icmp slt i64 %i.next, %n 194 br i1 %cond, label %for.body, label %for.end 195 196for.end: 197 %s.lcssa = phi float [ %6, %for.body ] 198 ret float %s.lcssa 199} 200 201; Make sure we scalarize the step vectors used for the pointer arithmetic. We 202; can't easily simplify vectorized step vectors. (Interleaved accesses.) 203; 204; for (int i = 0; i < n; ++i) 205; a[i].f ^= y; 206; 207; INTERLEAVE-LABEL: @scalarize_induction_variable_03( 208; INTERLEAVE: vector.body: 209; INTERLEAVE: %[[i0:.+]] = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] 210; INTERLEAVE: %[[i1:.+]] = or i64 %[[i0]], 1 211; INTERLEAVE: %[[i2:.+]] = or i64 %[[i0]], 2 212; INTERLEAVE: %[[i3:.+]] = or i64 %[[i0]], 3 213; INTERLEAVE: %[[i4:.+]] = or i64 %[[i0]], 4 214; INTERLEAVE: %[[i5:.+]] = or i64 %[[i0]], 5 215; INTERLEAVE: %[[i6:.+]] = or i64 %[[i0]], 6 216; INTERLEAVE: %[[i7:.+]] = or i64 %[[i0]], 7 217; INTERLEAVE: getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i0]], i32 1 218; INTERLEAVE: getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i1]], i32 1 219; INTERLEAVE: getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i2]], i32 1 220; INTERLEAVE: getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i3]], i32 1 221; INTERLEAVE: getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i4]], i32 1 222; INTERLEAVE: getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i5]], i32 1 223; INTERLEAVE: getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i6]], i32 1 224; INTERLEAVE: getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i7]], i32 1 225 226%pair.i32 = type { i32, i32 } 227define void @scalarize_induction_variable_03(%pair.i32 *%p, i32 %y, i64 %n) { 228entry: 229 br label %for.body 230 231for.body: 232 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ] 233 %f = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 1 234 %0 = load i32, i32* %f, align 8 235 %1 = xor i32 %0, %y 236 store i32 %1, i32* %f, align 8 237 %i.next = add nuw nsw i64 %i, 1 238 %cond = icmp slt i64 %i.next, %n 239 br i1 %cond, label %for.body, label %for.end 240 241for.end: 242 ret void 243} 244 245; Make sure we scalarize the step vectors used for the pointer arithmetic. We 246; can't easily simplify vectorized step vectors. (Interleaved accesses.) 247; 248; for (int i = 0; i < n; ++i) 249; p[i].f = a[i * 4] 250; 251; INTERLEAVE-LABEL: @scalarize_induction_variable_04( 252; INTERLEAVE: vector.body: 253; INTERLEAVE: %[[i0:.+]] = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] 254; INTERLEAVE: %[[i1:.+]] = or i64 %[[i0]], 1 255; INTERLEAVE: %[[i2:.+]] = or i64 %[[i0]], 2 256; INTERLEAVE: %[[i3:.+]] = or i64 %[[i0]], 3 257; INTERLEAVE: %[[i4:.+]] = or i64 %[[i0]], 4 258; INTERLEAVE: %[[i5:.+]] = or i64 %[[i0]], 5 259; INTERLEAVE: %[[i6:.+]] = or i64 %[[i0]], 6 260; INTERLEAVE: %[[i7:.+]] = or i64 %[[i0]], 7 261; INTERLEAVE: getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i0]], i32 1 262; INTERLEAVE: getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i1]], i32 1 263; INTERLEAVE: getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i2]], i32 1 264; INTERLEAVE: getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i3]], i32 1 265; INTERLEAVE: getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i4]], i32 1 266; INTERLEAVE: getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i5]], i32 1 267; INTERLEAVE: getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i6]], i32 1 268; INTERLEAVE: getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i7]], i32 1 269 270define void @scalarize_induction_variable_04(i32* %a, %pair.i32* %p, i32 %n) { 271entry: 272 br label %for.body 273 274for.body: 275 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry] 276 %0 = shl nsw i64 %i, 2 277 %1 = getelementptr inbounds i32, i32* %a, i64 %0 278 %2 = load i32, i32* %1, align 1 279 %3 = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 1 280 store i32 %2, i32* %3, align 1 281 %i.next = add nuw nsw i64 %i, 1 282 %4 = trunc i64 %i.next to i32 283 %cond = icmp eq i32 %4, %n 284 br i1 %cond, label %for.end, label %for.body 285 286for.end: 287 ret void 288} 289 290; Ensure we generate both a vector and a scalar induction variable. In this 291; test, the induction variable is used by an instruction that will be 292; vectorized (trunc) as well as an instruction that will remain in scalar form 293; (gepelementptr). 294; 295; CHECK-LABEL: @iv_vector_and_scalar_users( 296; CHECK: vector.body: 297; CHECK: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] 298; CHECK: %vec.ind = phi <2 x i64> [ <i64 0, i64 1>, %vector.ph ], [ %vec.ind.next, %vector.body ] 299; CHECK: %vec.ind1 = phi <2 x i32> [ <i32 0, i32 1>, %vector.ph ], [ %vec.ind.next2, %vector.body ] 300; CHECK: %[[i0:.+]] = add i64 %index, 0 301; CHECK: %[[i1:.+]] = add i64 %index, 1 302; CHECK: getelementptr inbounds %pair.i16, %pair.i16* %p, i64 %[[i0]], i32 1 303; CHECK: getelementptr inbounds %pair.i16, %pair.i16* %p, i64 %[[i1]], i32 1 304; CHECK: %index.next = add i64 %index, 2 305; CHECK: %vec.ind.next = add <2 x i64> %vec.ind, <i64 2, i64 2> 306; CHECK: %vec.ind.next2 = add <2 x i32> %vec.ind1, <i32 2, i32 2> 307; 308; IND-LABEL: @iv_vector_and_scalar_users( 309; IND: vector.body: 310; IND: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] 311; IND: %vec.ind1 = phi <2 x i32> [ <i32 0, i32 1>, %vector.ph ], [ %vec.ind.next2, %vector.body ] 312; IND: %[[i1:.+]] = or i64 %index, 1 313; IND: getelementptr inbounds %pair.i16, %pair.i16* %p, i64 %index, i32 1 314; IND: getelementptr inbounds %pair.i16, %pair.i16* %p, i64 %[[i1]], i32 1 315; IND: %index.next = add i64 %index, 2 316; IND: %vec.ind.next2 = add <2 x i32> %vec.ind1, <i32 2, i32 2> 317; 318; UNROLL-LABEL: @iv_vector_and_scalar_users( 319; UNROLL: vector.body: 320; UNROLL: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] 321; UNROLL: %vec.ind2 = phi <2 x i32> [ <i32 0, i32 1>, %vector.ph ], [ %vec.ind.next5, %vector.body ] 322; UNROLL: %[[i1:.+]] = or i64 %index, 1 323; UNROLL: %[[i2:.+]] = or i64 %index, 2 324; UNROLL: %[[i3:.+]] = or i64 %index, 3 325; UNROLL: %step.add3 = add <2 x i32> %vec.ind2, <i32 2, i32 2> 326; UNROLL: getelementptr inbounds %pair.i16, %pair.i16* %p, i64 %index, i32 1 327; UNROLL: getelementptr inbounds %pair.i16, %pair.i16* %p, i64 %[[i1]], i32 1 328; UNROLL: getelementptr inbounds %pair.i16, %pair.i16* %p, i64 %[[i2]], i32 1 329; UNROLL: getelementptr inbounds %pair.i16, %pair.i16* %p, i64 %[[i3]], i32 1 330; UNROLL: %index.next = add i64 %index, 4 331; UNROLL: %vec.ind.next5 = add <2 x i32> %vec.ind2, <i32 4, i32 4> 332 333%pair.i16 = type { i16, i16 } 334define void @iv_vector_and_scalar_users(%pair.i16* %p, i32 %a, i32 %n) { 335entry: 336 br label %for.body 337 338for.body: 339 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ] 340 %0 = trunc i64 %i to i32 341 %1 = add i32 %a, %0 342 %2 = trunc i32 %1 to i16 343 %3 = getelementptr inbounds %pair.i16, %pair.i16* %p, i64 %i, i32 1 344 store i16 %2, i16* %3, align 2 345 %i.next = add nuw nsw i64 %i, 1 346 %4 = trunc i64 %i.next to i32 347 %cond = icmp eq i32 %4, %n 348 br i1 %cond, label %for.end, label %for.body 349 350for.end: 351 ret void 352} 353 354; Make sure that the loop exit count computation does not overflow for i8 and 355; i16. The exit count of these loops is i8/i16 max + 1. If we don't cast the 356; induction variable to a bigger type the exit count computation will overflow 357; to 0. 358; PR17532 359 360; CHECK-LABEL: i8_loop 361; CHECK: icmp eq i32 {{.*}}, 256 362define i32 @i8_loop() nounwind readnone ssp uwtable { 363 br label %1 364 365; <label>:1 ; preds = %1, %0 366 %a.0 = phi i32 [ 1, %0 ], [ %2, %1 ] 367 %b.0 = phi i8 [ 0, %0 ], [ %3, %1 ] 368 %2 = and i32 %a.0, 4 369 %3 = add i8 %b.0, -1 370 %4 = icmp eq i8 %3, 0 371 br i1 %4, label %5, label %1 372 373; <label>:5 ; preds = %1 374 ret i32 %2 375} 376 377; CHECK-LABEL: i16_loop 378; CHECK: icmp eq i32 {{.*}}, 65536 379 380define i32 @i16_loop() nounwind readnone ssp uwtable { 381 br label %1 382 383; <label>:1 ; preds = %1, %0 384 %a.0 = phi i32 [ 1, %0 ], [ %2, %1 ] 385 %b.0 = phi i16 [ 0, %0 ], [ %3, %1 ] 386 %2 = and i32 %a.0, 4 387 %3 = add i16 %b.0, -1 388 %4 = icmp eq i16 %3, 0 389 br i1 %4, label %5, label %1 390 391; <label>:5 ; preds = %1 392 ret i32 %2 393} 394 395; This loop has a backedge taken count of i32_max. We need to check for this 396; condition and branch directly to the scalar loop. 397 398; CHECK-LABEL: max_i32_backedgetaken 399; CHECK: br i1 true, label %scalar.ph, label %min.iters.checked 400 401; CHECK: middle.block: 402; CHECK: %[[v9:.+]] = extractelement <2 x i32> %bin.rdx, i32 0 403; CHECK: scalar.ph: 404; CHECK: %bc.resume.val = phi i32 [ 0, %middle.block ], [ 0, %[[v0:.+]] ] 405; CHECK: %bc.merge.rdx = phi i32 [ 1, %[[v0:.+]] ], [ 1, %min.iters.checked ], [ %[[v9]], %middle.block ] 406 407define i32 @max_i32_backedgetaken() nounwind readnone ssp uwtable { 408 409 br label %1 410 411; <label>:1 ; preds = %1, %0 412 %a.0 = phi i32 [ 1, %0 ], [ %2, %1 ] 413 %b.0 = phi i32 [ 0, %0 ], [ %3, %1 ] 414 %2 = and i32 %a.0, 4 415 %3 = add i32 %b.0, -1 416 %4 = icmp eq i32 %3, 0 417 br i1 %4, label %5, label %1 418 419; <label>:5 ; preds = %1 420 ret i32 %2 421} 422 423; When generating the overflow check we must sure that the induction start value 424; is defined before the branch to the scalar preheader. 425 426; CHECK-LABEL: testoverflowcheck 427; CHECK: entry 428; CHECK: %[[LOAD:.*]] = load i8 429; CHECK: br 430 431; CHECK: scalar.ph 432; CHECK: phi i8 [ %{{.*}}, %middle.block ], [ %[[LOAD]], %entry ] 433 434@e = global i8 1, align 1 435@d = common global i32 0, align 4 436@c = common global i32 0, align 4 437define i32 @testoverflowcheck() { 438entry: 439 %.pr.i = load i8, i8* @e, align 1 440 %0 = load i32, i32* @d, align 4 441 %c.promoted.i = load i32, i32* @c, align 4 442 br label %cond.end.i 443 444cond.end.i: 445 %inc4.i = phi i8 [ %.pr.i, %entry ], [ %inc.i, %cond.end.i ] 446 %and3.i = phi i32 [ %c.promoted.i, %entry ], [ %and.i, %cond.end.i ] 447 %and.i = and i32 %0, %and3.i 448 %inc.i = add i8 %inc4.i, 1 449 %tobool.i = icmp eq i8 %inc.i, 0 450 br i1 %tobool.i, label %loopexit, label %cond.end.i 451 452loopexit: 453 ret i32 %and.i 454} 455 456; The SCEV expression of %sphi is (zext i8 {%t,+,1}<%loop> to i32) 457; In order to recognize %sphi as an induction PHI and vectorize this loop, 458; we need to convert the SCEV expression into an AddRecExpr. 459; The expression gets converted to {zext i8 %t to i32,+,1}. 460 461; CHECK-LABEL: wrappingindvars1 462; CHECK-LABEL: vector.scevcheck 463; CHECK-LABEL: vector.ph 464; CHECK: %[[START:.*]] = add <2 x i32> %{{.*}}, <i32 0, i32 1> 465; CHECK-LABEL: vector.body 466; CHECK: %[[PHI:.*]] = phi <2 x i32> [ %[[START]], %vector.ph ], [ %[[STEP:.*]], %vector.body ] 467; CHECK: %[[STEP]] = add <2 x i32> %[[PHI]], <i32 2, i32 2> 468define void @wrappingindvars1(i8 %t, i32 %len, i32 *%A) { 469 entry: 470 %st = zext i8 %t to i16 471 %ext = zext i8 %t to i32 472 %ecmp = icmp ult i16 %st, 42 473 br i1 %ecmp, label %loop, label %exit 474 475 loop: 476 477 %idx = phi i8 [ %t, %entry ], [ %idx.inc, %loop ] 478 %idx.b = phi i32 [ 0, %entry ], [ %idx.b.inc, %loop ] 479 %sphi = phi i32 [ %ext, %entry ], [%idx.inc.ext, %loop] 480 481 %ptr = getelementptr inbounds i32, i32* %A, i8 %idx 482 store i32 %sphi, i32* %ptr 483 484 %idx.inc = add i8 %idx, 1 485 %idx.inc.ext = zext i8 %idx.inc to i32 486 %idx.b.inc = add nuw nsw i32 %idx.b, 1 487 488 %c = icmp ult i32 %idx.b, %len 489 br i1 %c, label %loop, label %exit 490 491 exit: 492 ret void 493} 494 495; The SCEV expression of %sphi is (4 * (zext i8 {%t,+,1}<%loop> to i32)) 496; In order to recognize %sphi as an induction PHI and vectorize this loop, 497; we need to convert the SCEV expression into an AddRecExpr. 498; The expression gets converted to ({4 * (zext %t to i32),+,4}). 499; CHECK-LABEL: wrappingindvars2 500; CHECK-LABEL: vector.scevcheck 501; CHECK-LABEL: vector.ph 502; CHECK: %[[START:.*]] = add <2 x i32> %{{.*}}, <i32 0, i32 4> 503; CHECK-LABEL: vector.body 504; CHECK: %[[PHI:.*]] = phi <2 x i32> [ %[[START]], %vector.ph ], [ %[[STEP:.*]], %vector.body ] 505; CHECK: %[[STEP]] = add <2 x i32> %[[PHI]], <i32 8, i32 8> 506define void @wrappingindvars2(i8 %t, i32 %len, i32 *%A) { 507 508entry: 509 %st = zext i8 %t to i16 510 %ext = zext i8 %t to i32 511 %ext.mul = mul i32 %ext, 4 512 513 %ecmp = icmp ult i16 %st, 42 514 br i1 %ecmp, label %loop, label %exit 515 516 loop: 517 518 %idx = phi i8 [ %t, %entry ], [ %idx.inc, %loop ] 519 %sphi = phi i32 [ %ext.mul, %entry ], [%mul, %loop] 520 %idx.b = phi i32 [ 0, %entry ], [ %idx.b.inc, %loop ] 521 522 %ptr = getelementptr inbounds i32, i32* %A, i8 %idx 523 store i32 %sphi, i32* %ptr 524 525 %idx.inc = add i8 %idx, 1 526 %idx.inc.ext = zext i8 %idx.inc to i32 527 %mul = mul i32 %idx.inc.ext, 4 528 %idx.b.inc = add nuw nsw i32 %idx.b, 1 529 530 %c = icmp ult i32 %idx.b, %len 531 br i1 %c, label %loop, label %exit 532 533 exit: 534 ret void 535} 536 537; Check that we generate vectorized IVs in the pre-header 538; instead of widening the scalar IV inside the loop, when 539; we know how to do that. 540; IND-LABEL: veciv 541; IND: vector.body: 542; IND: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] 543; IND: %vec.ind = phi <2 x i32> [ <i32 0, i32 1>, %vector.ph ], [ %vec.ind.next, %vector.body ] 544; IND: %index.next = add i32 %index, 2 545; IND: %vec.ind.next = add <2 x i32> %vec.ind, <i32 2, i32 2> 546; IND: %[[CMP:.*]] = icmp eq i32 %index.next 547; IND: br i1 %[[CMP]] 548; UNROLL-LABEL: veciv 549; UNROLL: vector.body: 550; UNROLL: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] 551; UNROLL: %vec.ind = phi <2 x i32> [ <i32 0, i32 1>, %vector.ph ], [ %vec.ind.next, %vector.body ] 552; UNROLL: %step.add = add <2 x i32> %vec.ind, <i32 2, i32 2> 553; UNROLL: %index.next = add i32 %index, 4 554; UNROLL: %vec.ind.next = add <2 x i32> %vec.ind, <i32 4, i32 4> 555; UNROLL: %[[CMP:.*]] = icmp eq i32 %index.next 556; UNROLL: br i1 %[[CMP]] 557define void @veciv(i32* nocapture %a, i32 %start, i32 %k) { 558for.body.preheader: 559 br label %for.body 560 561for.body: 562 %indvars.iv = phi i32 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ] 563 %arrayidx = getelementptr inbounds i32, i32* %a, i32 %indvars.iv 564 store i32 %indvars.iv, i32* %arrayidx, align 4 565 %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1 566 %exitcond = icmp eq i32 %indvars.iv.next, %k 567 br i1 %exitcond, label %exit, label %for.body 568 569exit: 570 ret void 571} 572 573; IND-LABEL: trunciv 574; IND: vector.body: 575; IND: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] 576; IND: %[[VECIND:.*]] = phi <2 x i32> [ <i32 0, i32 1>, %vector.ph ], [ %[[STEPADD:.*]], %vector.body ] 577; IND: %index.next = add i64 %index, 2 578; IND: %[[STEPADD]] = add <2 x i32> %[[VECIND]], <i32 2, i32 2> 579; IND: %[[CMP:.*]] = icmp eq i64 %index.next 580; IND: br i1 %[[CMP]] 581define void @trunciv(i32* nocapture %a, i32 %start, i64 %k) { 582for.body.preheader: 583 br label %for.body 584 585for.body: 586 %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ] 587 %trunc.iv = trunc i64 %indvars.iv to i32 588 %arrayidx = getelementptr inbounds i32, i32* %a, i32 %trunc.iv 589 store i32 %trunc.iv, i32* %arrayidx, align 4 590 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 591 %exitcond = icmp eq i64 %indvars.iv.next, %k 592 br i1 %exitcond, label %exit, label %for.body 593 594exit: 595 ret void 596} 597 598; CHECK-LABEL: @nonprimary( 599; CHECK: vector.ph: 600; CHECK: %[[INSERT:.*]] = insertelement <2 x i32> undef, i32 %i, i32 0 601; CHECK: %[[SPLAT:.*]] = shufflevector <2 x i32> %[[INSERT]], <2 x i32> undef, <2 x i32> zeroinitializer 602; CHECK: %[[START:.*]] = add <2 x i32> %[[SPLAT]], <i32 0, i32 1> 603; CHECK: vector.body: 604; CHECK: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] 605; CHECK: %vec.ind = phi <2 x i32> [ %[[START]], %vector.ph ], [ %vec.ind.next, %vector.body ] 606; CHECK: %offset.idx = add i32 %i, %index 607; CHECK: %[[A1:.*]] = add i32 %offset.idx, 0 608; CHECK: %[[G1:.*]] = getelementptr inbounds i32, i32* %a, i32 %[[A1]] 609; CHECK: %[[G3:.*]] = getelementptr i32, i32* %[[G1]], i32 0 610; CHECK: %[[B1:.*]] = bitcast i32* %[[G3]] to <2 x i32>* 611; CHECK: store <2 x i32> %vec.ind, <2 x i32>* %[[B1]] 612; CHECK: %index.next = add i32 %index, 2 613; CHECK: %vec.ind.next = add <2 x i32> %vec.ind, <i32 2, i32 2> 614; CHECK: %[[CMP:.*]] = icmp eq i32 %index.next, %n.vec 615; CHECK: br i1 %[[CMP]] 616; 617; IND-LABEL: @nonprimary( 618; IND: vector.ph: 619; IND: %[[INSERT:.*]] = insertelement <2 x i32> undef, i32 %i, i32 0 620; IND: %[[SPLAT:.*]] = shufflevector <2 x i32> %[[INSERT]], <2 x i32> undef, <2 x i32> zeroinitializer 621; IND: %[[START:.*]] = add <2 x i32> %[[SPLAT]], <i32 0, i32 1> 622; IND: vector.body: 623; IND: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] 624; IND: %vec.ind = phi <2 x i32> [ %[[START]], %vector.ph ], [ %vec.ind.next, %vector.body ] 625; IND: %[[A1:.*]] = add i32 %index, %i 626; IND: %[[S1:.*]] = sext i32 %[[A1]] to i64 627; IND: %[[G1:.*]] = getelementptr inbounds i32, i32* %a, i64 %[[S1]] 628; IND: %[[B1:.*]] = bitcast i32* %[[G1]] to <2 x i32>* 629; IND: store <2 x i32> %vec.ind, <2 x i32>* %[[B1]] 630; IND: %index.next = add i32 %index, 2 631; IND: %vec.ind.next = add <2 x i32> %vec.ind, <i32 2, i32 2> 632; IND: %[[CMP:.*]] = icmp eq i32 %index.next, %n.vec 633; IND: br i1 %[[CMP]] 634; 635; UNROLL-LABEL: @nonprimary( 636; UNROLL: vector.ph: 637; UNROLL: %[[INSERT:.*]] = insertelement <2 x i32> undef, i32 %i, i32 0 638; UNROLL: %[[SPLAT:.*]] = shufflevector <2 x i32> %[[INSERT]], <2 x i32> undef, <2 x i32> zeroinitializer 639; UNROLL: %[[START:.*]] = add <2 x i32> %[[SPLAT]], <i32 0, i32 1> 640; UNROLL: vector.body: 641; UNROLL: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] 642; UNROLL: %vec.ind = phi <2 x i32> [ %[[START]], %vector.ph ], [ %vec.ind.next, %vector.body ] 643; UNROLL: %step.add = add <2 x i32> %vec.ind, <i32 2, i32 2> 644; UNROLL: %[[A1:.*]] = add i32 %index, %i 645; UNROLL: %[[S1:.*]] = sext i32 %[[A1]] to i64 646; UNROLL: %[[G1:.*]] = getelementptr inbounds i32, i32* %a, i64 %[[S1]] 647; UNROLL: %[[B1:.*]] = bitcast i32* %[[G1]] to <2 x i32>* 648; UNROLL: store <2 x i32> %vec.ind, <2 x i32>* %[[B1]] 649; UNROLL: %[[G2:.*]] = getelementptr i32, i32* %[[G1]], i64 2 650; UNROLL: %[[B2:.*]] = bitcast i32* %[[G2]] to <2 x i32>* 651; UNROLL: store <2 x i32> %step.add, <2 x i32>* %[[B2]] 652; UNROLL: %index.next = add i32 %index, 4 653; UNROLL: %vec.ind.next = add <2 x i32> %vec.ind, <i32 4, i32 4> 654; UNROLL: %[[CMP:.*]] = icmp eq i32 %index.next, %n.vec 655; UNROLL: br i1 %[[CMP]] 656define void @nonprimary(i32* nocapture %a, i32 %start, i32 %i, i32 %k) { 657for.body.preheader: 658 br label %for.body 659 660for.body: 661 %indvars.iv = phi i32 [ %indvars.iv.next, %for.body ], [ %i, %for.body.preheader ] 662 %arrayidx = getelementptr inbounds i32, i32* %a, i32 %indvars.iv 663 store i32 %indvars.iv, i32* %arrayidx, align 4 664 %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1 665 %exitcond = icmp eq i32 %indvars.iv.next, %k 666 br i1 %exitcond, label %exit, label %for.body 667 668exit: 669 ret void 670} 671