1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py 2; RUN: opt -S -loop-vectorize -instcombine -force-vector-width=4 -force-vector-interleave=1 -enable-interleaved-mem-accesses=true -runtime-memory-check-threshold=24 < %s | FileCheck %s 3 4target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" 5 6; Check vectorization on an interleaved load group of factor 2 and an interleaved 7; store group of factor 2. 8 9; int AB[1024]; 10; int CD[1024]; 11; void test_array_load2_store2(int C, int D) { 12; for (int i = 0; i < 1024; i+=2) { 13; int A = AB[i]; 14; int B = AB[i+1]; 15; CD[i] = A + C; 16; CD[i+1] = B * D; 17; } 18; } 19 20 21@AB = common global [1024 x i32] zeroinitializer, align 4 22@CD = common global [1024 x i32] zeroinitializer, align 4 23 24define void @test_array_load2_store2(i32 %C, i32 %D) { 25; CHECK-LABEL: @test_array_load2_store2( 26; CHECK-NEXT: entry: 27; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 28; CHECK: vector.ph: 29; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C:%.*]], i32 0 30; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 31; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x i32> poison, i32 [[D:%.*]], i32 0 32; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT2]], <4 x i32> poison, <4 x i32> zeroinitializer 33; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 34; CHECK: vector.body: 35; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 36; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1 37; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [1024 x i32], [1024 x i32]* @AB, i64 0, i64 [[OFFSET_IDX]] 38; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32* [[TMP0]] to <8 x i32>* 39; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, <8 x i32>* [[TMP1]], align 4 40; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6> 41; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7> 42; CHECK-NEXT: [[TMP2:%.*]] = or i64 [[OFFSET_IDX]], 1 43; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i32> [[STRIDED_VEC]], [[BROADCAST_SPLAT]] 44; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <4 x i32> [[STRIDED_VEC1]], [[BROADCAST_SPLAT3]] 45; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [1024 x i32], [1024 x i32]* @CD, i64 0, i64 [[TMP2]] 46; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP5]], i64 -1 47; CHECK-NEXT: [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <8 x i32>* 48; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7> 49; CHECK-NEXT: store <8 x i32> [[INTERLEAVED_VEC]], <8 x i32>* [[TMP7]], align 4 50; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 51; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512 52; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]] 53; CHECK: middle.block: 54; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]] 55; CHECK: scalar.ph: 56; CHECK-NEXT: br label [[FOR_BODY:%.*]] 57; CHECK: for.body: 58; CHECK-NEXT: br i1 undef, label [[FOR_BODY]], label [[FOR_END]], [[LOOP2:!llvm.loop !.*]] 59; CHECK: for.end: 60; CHECK-NEXT: ret void 61; 62entry: 63 br label %for.body 64 65for.body: ; preds = %for.body, %entry 66 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 67 %arrayidx0 = getelementptr inbounds [1024 x i32], [1024 x i32]* @AB, i64 0, i64 %indvars.iv 68 %tmp = load i32, i32* %arrayidx0, align 4 69 %tmp1 = or i64 %indvars.iv, 1 70 %arrayidx1 = getelementptr inbounds [1024 x i32], [1024 x i32]* @AB, i64 0, i64 %tmp1 71 %tmp2 = load i32, i32* %arrayidx1, align 4 72 %add = add nsw i32 %tmp, %C 73 %mul = mul nsw i32 %tmp2, %D 74 %arrayidx2 = getelementptr inbounds [1024 x i32], [1024 x i32]* @CD, i64 0, i64 %indvars.iv 75 store i32 %add, i32* %arrayidx2, align 4 76 %arrayidx3 = getelementptr inbounds [1024 x i32], [1024 x i32]* @CD, i64 0, i64 %tmp1 77 store i32 %mul, i32* %arrayidx3, align 4 78 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2 79 %cmp = icmp slt i64 %indvars.iv.next, 1024 80 br i1 %cmp, label %for.body, label %for.end 81 82for.end: ; preds = %for.body 83 ret void 84} 85 86; int A[3072]; 87; struct ST S[1024]; 88; void test_struct_st3() { 89; int *ptr = A; 90; for (int i = 0; i < 1024; i++) { 91; int X1 = *ptr++; 92; int X2 = *ptr++; 93; int X3 = *ptr++; 94; T[i].x = X1 + 1; 95; T[i].y = X2 + 2; 96; T[i].z = X3 + 3; 97; } 98; } 99 100 101%struct.ST3 = type { i32, i32, i32 } 102@A = common global [3072 x i32] zeroinitializer, align 4 103@S = common global [1024 x %struct.ST3] zeroinitializer, align 4 104 105define void @test_struct_array_load3_store3() { 106; CHECK-LABEL: @test_struct_array_load3_store3( 107; CHECK-NEXT: entry: 108; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 109; CHECK: vector.ph: 110; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 111; CHECK: vector.body: 112; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 113; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[INDEX]], 3 114; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr [3072 x i32], [3072 x i32]* @A, i64 0, i64 [[TMP0]] 115; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32* [[NEXT_GEP]] to <12 x i32>* 116; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <12 x i32>, <12 x i32>* [[TMP1]], align 4 117; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <12 x i32> [[WIDE_VEC]], <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9> 118; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <12 x i32> [[WIDE_VEC]], <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10> 119; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <12 x i32> [[WIDE_VEC]], <12 x i32> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11> 120; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[STRIDED_VEC]], <i32 1, i32 1, i32 1, i32 1> 121; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i32> [[STRIDED_VEC2]], <i32 2, i32 2, i32 2, i32 2> 122; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i32> [[STRIDED_VEC3]], <i32 3, i32 3, i32 3, i32 3> 123; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [1024 x %struct.ST3], [1024 x %struct.ST3]* @S, i64 0, i64 [[INDEX]], i32 2 124; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP5]], i64 -2 125; CHECK-NEXT: [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <12 x i32>* 126; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7> 127; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef> 128; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP9]], <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11> 129; CHECK-NEXT: store <12 x i32> [[INTERLEAVED_VEC]], <12 x i32>* [[TMP7]], align 4 130; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 131; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 132; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP4:!llvm.loop !.*]] 133; CHECK: middle.block: 134; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]] 135; CHECK: scalar.ph: 136; CHECK-NEXT: br label [[FOR_BODY:%.*]] 137; CHECK: for.body: 138; CHECK-NEXT: br i1 undef, label [[FOR_END]], label [[FOR_BODY]], [[LOOP5:!llvm.loop !.*]] 139; CHECK: for.end: 140; CHECK-NEXT: ret void 141; 142entry: 143 br label %for.body 144 145for.body: ; preds = %for.body, %entry 146 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 147 %ptr.016 = phi i32* [ getelementptr inbounds ([3072 x i32], [3072 x i32]* @A, i64 0, i64 0), %entry ], [ %incdec.ptr2, %for.body ] 148 %incdec.ptr = getelementptr inbounds i32, i32* %ptr.016, i64 1 149 %tmp = load i32, i32* %ptr.016, align 4 150 %incdec.ptr1 = getelementptr inbounds i32, i32* %ptr.016, i64 2 151 %tmp1 = load i32, i32* %incdec.ptr, align 4 152 %incdec.ptr2 = getelementptr inbounds i32, i32* %ptr.016, i64 3 153 %tmp2 = load i32, i32* %incdec.ptr1, align 4 154 %add = add nsw i32 %tmp, 1 155 %x = getelementptr inbounds [1024 x %struct.ST3], [1024 x %struct.ST3]* @S, i64 0, i64 %indvars.iv, i32 0 156 store i32 %add, i32* %x, align 4 157 %add3 = add nsw i32 %tmp1, 2 158 %y = getelementptr inbounds [1024 x %struct.ST3], [1024 x %struct.ST3]* @S, i64 0, i64 %indvars.iv, i32 1 159 store i32 %add3, i32* %y, align 4 160 %add6 = add nsw i32 %tmp2, 3 161 %z = getelementptr inbounds [1024 x %struct.ST3], [1024 x %struct.ST3]* @S, i64 0, i64 %indvars.iv, i32 2 162 store i32 %add6, i32* %z, align 4 163 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 164 %exitcond = icmp eq i64 %indvars.iv.next, 1024 165 br i1 %exitcond, label %for.end, label %for.body 166 167for.end: ; preds = %for.body 168 ret void 169} 170 171; Check vectorization on an interleaved load group of factor 4. 172 173; struct ST4{ 174; int x; 175; int y; 176; int z; 177; int w; 178; }; 179; int test_struct_load4(struct ST4 *S) { 180; int r = 0; 181; for (int i = 0; i < 1024; i++) { 182; r += S[i].x; 183; r -= S[i].y; 184; r += S[i].z; 185; r -= S[i].w; 186; } 187; return r; 188; } 189 190%struct.ST4 = type { i32, i32, i32, i32 } 191 192define i32 @test_struct_load4(%struct.ST4* nocapture readonly %S) { 193; 194; CHECK-LABEL: @test_struct_load4( 195; CHECK-NEXT: entry: 196; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 197; CHECK: vector.ph: 198; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 199; CHECK: vector.body: 200; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 201; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ] 202; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[STRUCT_ST4:%.*]], %struct.ST4* [[S:%.*]], i64 [[INDEX]], i32 0 203; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32* [[TMP0]] to <16 x i32>* 204; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x i32>, <16 x i32>* [[TMP1]], align 4 205; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12> 206; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13> 207; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14> 208; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15> 209; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[STRIDED_VEC]], [[VEC_PHI]] 210; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[TMP2]], [[STRIDED_VEC2]] 211; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[STRIDED_VEC1]], [[STRIDED_VEC3]] 212; CHECK-NEXT: [[TMP5]] = sub <4 x i32> [[TMP3]], [[TMP4]] 213; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 214; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 215; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP6:!llvm.loop !.*]] 216; CHECK: middle.block: 217; CHECK-NEXT: [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef> 218; CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP5]], [[RDX_SHUF]] 219; CHECK-NEXT: [[RDX_SHUF4:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef> 220; CHECK-NEXT: [[BIN_RDX5:%.*]] = add <4 x i32> [[BIN_RDX]], [[RDX_SHUF4]] 221; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[BIN_RDX5]], i32 0 222; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]] 223; CHECK: scalar.ph: 224; CHECK-NEXT: br label [[FOR_BODY:%.*]] 225; CHECK: for.body: 226; CHECK-NEXT: br i1 undef, label [[FOR_END]], label [[FOR_BODY]], [[LOOP7:!llvm.loop !.*]] 227; CHECK: for.end: 228; CHECK-NEXT: [[SUB8_LCSSA:%.*]] = phi i32 [ undef, [[FOR_BODY]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ] 229; CHECK-NEXT: ret i32 [[SUB8_LCSSA]] 230; 231entry: 232 br label %for.body 233 234for.body: ; preds = %for.body, %entry 235 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 236 %r.022 = phi i32 [ 0, %entry ], [ %sub8, %for.body ] 237 %x = getelementptr inbounds %struct.ST4, %struct.ST4* %S, i64 %indvars.iv, i32 0 238 %tmp = load i32, i32* %x, align 4 239 %add = add nsw i32 %tmp, %r.022 240 %y = getelementptr inbounds %struct.ST4, %struct.ST4* %S, i64 %indvars.iv, i32 1 241 %tmp1 = load i32, i32* %y, align 4 242 %sub = sub i32 %add, %tmp1 243 %z = getelementptr inbounds %struct.ST4, %struct.ST4* %S, i64 %indvars.iv, i32 2 244 %tmp2 = load i32, i32* %z, align 4 245 %add5 = add nsw i32 %sub, %tmp2 246 %w = getelementptr inbounds %struct.ST4, %struct.ST4* %S, i64 %indvars.iv, i32 3 247 %tmp3 = load i32, i32* %w, align 4 248 %sub8 = sub i32 %add5, %tmp3 249 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 250 %exitcond = icmp eq i64 %indvars.iv.next, 1024 251 br i1 %exitcond, label %for.end, label %for.body 252 253for.end: ; preds = %for.body 254 ret i32 %sub8 255} 256 257; Check vectorization on an interleaved store group of factor 4. 258 259; void test_struct_store4(int *A, struct ST4 *B) { 260; int *ptr = A; 261; for (int i = 0; i < 1024; i++) { 262; int X = *ptr++; 263; B[i].x = X + 1; 264; B[i].y = X * 2; 265; B[i].z = X + 3; 266; B[i].w = X + 4; 267; } 268; } 269 270 271define void @test_struct_store4(i32* noalias nocapture readonly %A, %struct.ST4* noalias nocapture %B) { 272; CHECK-LABEL: @test_struct_store4( 273; CHECK-NEXT: entry: 274; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 275; CHECK: vector.ph: 276; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 277; CHECK: vector.body: 278; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 279; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i32, i32* [[A:%.*]], i64 [[INDEX]] 280; CHECK-NEXT: [[TMP0:%.*]] = bitcast i32* [[NEXT_GEP]] to <4 x i32>* 281; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4 282; CHECK-NEXT: [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], <i32 1, i32 1, i32 1, i32 1> 283; CHECK-NEXT: [[TMP2:%.*]] = shl nsw <4 x i32> [[WIDE_LOAD]], <i32 1, i32 1, i32 1, i32 1> 284; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], <i32 3, i32 3, i32 3, i32 3> 285; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], <i32 4, i32 4, i32 4, i32 4> 286; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_ST4:%.*]], %struct.ST4* [[B:%.*]], i64 [[INDEX]], i32 3 287; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP5]], i64 -3 288; CHECK-NEXT: [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <16 x i32>* 289; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7> 290; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7> 291; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP9]], <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15> 292; CHECK-NEXT: store <16 x i32> [[INTERLEAVED_VEC]], <16 x i32>* [[TMP7]], align 4 293; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 294; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 295; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP8:!llvm.loop !.*]] 296; CHECK: middle.block: 297; CHECK-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]] 298; CHECK: scalar.ph: 299; CHECK-NEXT: br label [[FOR_BODY:%.*]] 300; CHECK: for.cond.cleanup: 301; CHECK-NEXT: ret void 302; CHECK: for.body: 303; CHECK-NEXT: br i1 undef, label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], [[LOOP9:!llvm.loop !.*]] 304; 305entry: 306 br label %for.body 307 308for.cond.cleanup: ; preds = %for.body 309 ret void 310 311for.body: ; preds = %for.body, %entry 312 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 313 %ptr.024 = phi i32* [ %A, %entry ], [ %incdec.ptr, %for.body ] 314 %incdec.ptr = getelementptr inbounds i32, i32* %ptr.024, i64 1 315 %tmp = load i32, i32* %ptr.024, align 4 316 %add = add nsw i32 %tmp, 1 317 %x = getelementptr inbounds %struct.ST4, %struct.ST4* %B, i64 %indvars.iv, i32 0 318 store i32 %add, i32* %x, align 4 319 %mul = shl nsw i32 %tmp, 1 320 %y = getelementptr inbounds %struct.ST4, %struct.ST4* %B, i64 %indvars.iv, i32 1 321 store i32 %mul, i32* %y, align 4 322 %add3 = add nsw i32 %tmp, 3 323 %z = getelementptr inbounds %struct.ST4, %struct.ST4* %B, i64 %indvars.iv, i32 2 324 store i32 %add3, i32* %z, align 4 325 %add6 = add nsw i32 %tmp, 4 326 %w = getelementptr inbounds %struct.ST4, %struct.ST4* %B, i64 %indvars.iv, i32 3 327 store i32 %add6, i32* %w, align 4 328 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 329 %exitcond = icmp eq i64 %indvars.iv.next, 1024 330 br i1 %exitcond, label %for.cond.cleanup, label %for.body 331} 332 333; Check vectorization on a reverse interleaved load group of factor 2 and 334; a reverse interleaved store group of factor 2. 335 336; struct ST2 { 337; int x; 338; int y; 339; }; 340; 341; void test_reversed_load2_store2(struct ST2 *A, struct ST2 *B) { 342; for (int i = 1023; i >= 0; i--) { 343; int a = A[i].x + i; // interleaved load of index 0 344; int b = A[i].y - i; // interleaved load of index 1 345; B[i].x = a; // interleaved store of index 0 346; B[i].y = b; // interleaved store of index 1 347; } 348; } 349 350 351%struct.ST2 = type { i32, i32 } 352 353define void @test_reversed_load2_store2(%struct.ST2* noalias nocapture readonly %A, %struct.ST2* noalias nocapture %B) { 354; CHECK-LABEL: @test_reversed_load2_store2( 355; CHECK-NEXT: entry: 356; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 357; CHECK: vector.ph: 358; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 359; CHECK: vector.body: 360; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 361; CHECK-NEXT: [[VEC_IND3:%.*]] = phi <4 x i32> [ <i32 1023, i32 1022, i32 1021, i32 1020>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT4:%.*]], [[VECTOR_BODY]] ] 362; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 1023, [[INDEX]] 363; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[STRUCT_ST2:%.*]], %struct.ST2* [[A:%.*]], i64 [[OFFSET_IDX]], i32 0 364; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 -6 365; CHECK-NEXT: [[TMP2:%.*]] = bitcast i32* [[TMP1]] to <8 x i32>* 366; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, <8 x i32>* [[TMP2]], align 4 367; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6> 368; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0> 369; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7> 370; CHECK-NEXT: [[REVERSE2:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC1]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0> 371; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i32> [[REVERSE]], [[VEC_IND3]] 372; CHECK-NEXT: [[TMP4:%.*]] = sub nsw <4 x i32> [[REVERSE2]], [[VEC_IND3]] 373; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_ST2]], %struct.ST2* [[B:%.*]], i64 [[OFFSET_IDX]], i32 1 374; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP5]], i64 -7 375; CHECK-NEXT: [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <8 x i32>* 376; CHECK-NEXT: [[REVERSE5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0> 377; CHECK-NEXT: [[REVERSE6:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0> 378; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i32> [[REVERSE5]], <4 x i32> [[REVERSE6]], <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7> 379; CHECK-NEXT: store <8 x i32> [[INTERLEAVED_VEC]], <8 x i32>* [[TMP7]], align 4 380; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 381; CHECK-NEXT: [[VEC_IND_NEXT4]] = add <4 x i32> [[VEC_IND3]], <i32 -4, i32 -4, i32 -4, i32 -4> 382; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 383; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP10:!llvm.loop !.*]] 384; CHECK: middle.block: 385; CHECK-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]] 386; CHECK: scalar.ph: 387; CHECK-NEXT: br label [[FOR_BODY:%.*]] 388; CHECK: for.cond.cleanup: 389; CHECK-NEXT: ret void 390; CHECK: for.body: 391; CHECK-NEXT: br i1 undef, label [[FOR_BODY]], label [[FOR_COND_CLEANUP]], [[LOOP11:!llvm.loop !.*]] 392; 393entry: 394 br label %for.body 395 396for.cond.cleanup: ; preds = %for.body 397 ret void 398 399for.body: ; preds = %for.body, %entry 400 %indvars.iv = phi i64 [ 1023, %entry ], [ %indvars.iv.next, %for.body ] 401 %x = getelementptr inbounds %struct.ST2, %struct.ST2* %A, i64 %indvars.iv, i32 0 402 %tmp = load i32, i32* %x, align 4 403 %tmp1 = trunc i64 %indvars.iv to i32 404 %add = add nsw i32 %tmp, %tmp1 405 %y = getelementptr inbounds %struct.ST2, %struct.ST2* %A, i64 %indvars.iv, i32 1 406 %tmp2 = load i32, i32* %y, align 4 407 %sub = sub nsw i32 %tmp2, %tmp1 408 %x5 = getelementptr inbounds %struct.ST2, %struct.ST2* %B, i64 %indvars.iv, i32 0 409 store i32 %add, i32* %x5, align 4 410 %y8 = getelementptr inbounds %struct.ST2, %struct.ST2* %B, i64 %indvars.iv, i32 1 411 store i32 %sub, i32* %y8, align 4 412 %indvars.iv.next = add nsw i64 %indvars.iv, -1 413 %cmp = icmp sgt i64 %indvars.iv, 0 414 br i1 %cmp, label %for.body, label %for.cond.cleanup 415} 416 417; Check vectorization on an interleaved load group of factor 2 with 1 gap 418; (missing the load of odd elements). Because the vectorized loop would 419; speculatively access memory out-of-bounds, we must execute at least one 420; iteration of the scalar loop. 421 422; void even_load_static_tc(int *A, int *B) { 423; for (unsigned i = 0; i < 1024; i+=2) 424; B[i/2] = A[i] * 2; 425; } 426 427 428define void @even_load_static_tc(i32* noalias nocapture readonly %A, i32* noalias nocapture %B) { 429; CHECK-LABEL: @even_load_static_tc( 430; CHECK-NEXT: entry: 431; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 432; CHECK: vector.ph: 433; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 434; CHECK: vector.body: 435; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 436; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1 437; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[OFFSET_IDX]] 438; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32* [[TMP0]] to <8 x i32>* 439; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, <8 x i32>* [[TMP1]], align 4 440; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6> 441; CHECK-NEXT: [[TMP2:%.*]] = shl nsw <4 x i32> [[STRIDED_VEC]], <i32 1, i32 1, i32 1, i32 1> 442; CHECK-NEXT: [[TMP3:%.*]] = and i64 [[INDEX]], 9223372036854775804 443; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[TMP3]] 444; CHECK-NEXT: [[TMP5:%.*]] = bitcast i32* [[TMP4]] to <4 x i32>* 445; CHECK-NEXT: store <4 x i32> [[TMP2]], <4 x i32>* [[TMP5]], align 4 446; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 447; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 508 448; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP12:!llvm.loop !.*]] 449; CHECK: middle.block: 450; CHECK-NEXT: br i1 false, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]] 451; CHECK: scalar.ph: 452; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1016, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 453; CHECK-NEXT: br label [[FOR_BODY:%.*]] 454; CHECK: for.cond.cleanup: 455; CHECK-NEXT: ret void 456; CHECK: for.body: 457; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 458; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]] 459; CHECK-NEXT: [[TMP:%.*]] = load i32, i32* [[ARRAYIDX]], align 4 460; CHECK-NEXT: [[MUL:%.*]] = shl nsw i32 [[TMP]], 1 461; CHECK-NEXT: [[TMP1:%.*]] = lshr exact i64 [[INDVARS_IV]], 1 462; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP1]] 463; CHECK-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX2]], align 4 464; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 2 465; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[INDVARS_IV]], 1022 466; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP]], [[LOOP13:!llvm.loop !.*]] 467; 468entry: 469 br label %for.body 470 471for.cond.cleanup: ; preds = %for.body 472 ret void 473 474for.body: ; preds = %for.body, %entry 475 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 476 %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv 477 %tmp = load i32, i32* %arrayidx, align 4 478 %mul = shl nsw i32 %tmp, 1 479 %tmp1 = lshr exact i64 %indvars.iv, 1 480 %arrayidx2 = getelementptr inbounds i32, i32* %B, i64 %tmp1 481 store i32 %mul, i32* %arrayidx2, align 4 482 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2 483 %cmp = icmp ult i64 %indvars.iv.next, 1024 484 br i1 %cmp, label %for.body, label %for.cond.cleanup 485} 486 487; Check vectorization on an interleaved load group of factor 2 with 1 gap 488; (missing the load of odd elements). Because the vectorized loop would 489; speculatively access memory out-of-bounds, we must execute at least one 490; iteration of the scalar loop. 491 492; void even_load_dynamic_tc(int *A, int *B, unsigned N) { 493; for (unsigned i = 0; i < N; i+=2) 494; B[i/2] = A[i] * 2; 495; } 496 497 498define void @even_load_dynamic_tc(i32* noalias nocapture readonly %A, i32* noalias nocapture %B, i64 %N) { 499; CHECK-LABEL: @even_load_dynamic_tc( 500; CHECK-NEXT: entry: 501; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt i64 [[N:%.*]], 2 502; CHECK-NEXT: [[UMAX:%.*]] = select i1 [[TMP0]], i64 [[N]], i64 2 503; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[UMAX]], -1 504; CHECK-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 1 505; CHECK-NEXT: [[TMP3:%.*]] = add nuw i64 [[TMP2]], 1 506; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 8 507; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 508; CHECK: vector.ph: 509; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP3]], 3 510; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 511; CHECK-NEXT: [[TMP5:%.*]] = select i1 [[TMP4]], i64 4, i64 [[N_MOD_VF]] 512; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[TMP5]] 513; CHECK-NEXT: [[IND_END:%.*]] = shl i64 [[N_VEC]], 1 514; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 515; CHECK: vector.body: 516; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 517; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1 518; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[OFFSET_IDX]] 519; CHECK-NEXT: [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <8 x i32>* 520; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, <8 x i32>* [[TMP7]], align 4 521; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6> 522; CHECK-NEXT: [[TMP8:%.*]] = shl nsw <4 x i32> [[STRIDED_VEC]], <i32 1, i32 1, i32 1, i32 1> 523; CHECK-NEXT: [[TMP9:%.*]] = and i64 [[INDEX]], 9223372036854775804 524; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[TMP9]] 525; CHECK-NEXT: [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <4 x i32>* 526; CHECK-NEXT: store <4 x i32> [[TMP8]], <4 x i32>* [[TMP11]], align 4 527; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 528; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 529; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP14:!llvm.loop !.*]] 530; CHECK: middle.block: 531; CHECK-NEXT: br i1 false, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]] 532; CHECK: scalar.ph: 533; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 534; CHECK-NEXT: br label [[FOR_BODY:%.*]] 535; CHECK: for.cond.cleanup: 536; CHECK-NEXT: ret void 537; CHECK: for.body: 538; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 539; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]] 540; CHECK-NEXT: [[TMP:%.*]] = load i32, i32* [[ARRAYIDX]], align 4 541; CHECK-NEXT: [[MUL:%.*]] = shl nsw i32 [[TMP]], 1 542; CHECK-NEXT: [[TMP1:%.*]] = lshr exact i64 [[INDVARS_IV]], 1 543; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP1]] 544; CHECK-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX2]], align 4 545; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 2 546; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[INDVARS_IV_NEXT]], [[N]] 547; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP]], [[LOOP15:!llvm.loop !.*]] 548; 549entry: 550 br label %for.body 551 552for.cond.cleanup: ; preds = %for.body 553 ret void 554 555for.body: ; preds = %for.body, %entry 556 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 557 %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv 558 %tmp = load i32, i32* %arrayidx, align 4 559 %mul = shl nsw i32 %tmp, 1 560 %tmp1 = lshr exact i64 %indvars.iv, 1 561 %arrayidx2 = getelementptr inbounds i32, i32* %B, i64 %tmp1 562 store i32 %mul, i32* %arrayidx2, align 4 563 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2 564 %cmp = icmp ult i64 %indvars.iv.next, %N 565 br i1 %cmp, label %for.body, label %for.cond.cleanup 566} 567 568; Check vectorization on a reverse interleaved load group of factor 2 with 1 569; gap and a reverse interleaved store group of factor 2. The interleaved load 570; group should be removed since it has a gap and is reverse. 571 572; struct pair { 573; int x; 574; int y; 575; }; 576; 577; void load_gap_reverse(struct pair *P1, struct pair *P2, int X) { 578; for (int i = 1023; i >= 0; i--) { 579; int a = X + i; 580; int b = A[i].y - i; 581; B[i].x = a; 582; B[i].y = b; 583; } 584; } 585 586 587%pair = type { i64, i64 } 588define void @load_gap_reverse(%pair* noalias nocapture readonly %P1, %pair* noalias nocapture readonly %P2, i64 %X) { 589; CHECK-LABEL: @load_gap_reverse( 590; CHECK-NEXT: entry: 591; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 592; CHECK: vector.ph: 593; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[X:%.*]], i32 0 594; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer 595; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 596; CHECK: vector.body: 597; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 598; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1023, i64 1022, i64 1021, i64 1020>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 599; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 1023, [[INDEX]] 600; CHECK-NEXT: [[TMP0:%.*]] = sub i64 1022, [[INDEX]] 601; CHECK-NEXT: [[TMP1:%.*]] = sub i64 1021, [[INDEX]] 602; CHECK-NEXT: [[TMP2:%.*]] = sub i64 1020, [[INDEX]] 603; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i64> [[BROADCAST_SPLAT]], [[VEC_IND]] 604; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[PAIR:%.*]], %pair* [[P1:%.*]], i64 [[OFFSET_IDX]], i32 0 605; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[PAIR]], %pair* [[P1]], i64 [[TMP0]], i32 0 606; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[PAIR]], %pair* [[P1]], i64 [[TMP1]], i32 0 607; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[PAIR]], %pair* [[P1]], i64 [[TMP2]], i32 0 608; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[PAIR]], %pair* [[P2:%.*]], i64 [[OFFSET_IDX]], i32 1 609; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[PAIR]], %pair* [[P2]], i64 [[TMP0]], i32 1 610; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [[PAIR]], %pair* [[P2]], i64 [[TMP1]], i32 1 611; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[PAIR]], %pair* [[P2]], i64 [[TMP2]], i32 1 612; CHECK-NEXT: [[TMP12:%.*]] = load i64, i64* [[TMP8]], align 8 613; CHECK-NEXT: [[TMP13:%.*]] = load i64, i64* [[TMP9]], align 8 614; CHECK-NEXT: [[TMP14:%.*]] = load i64, i64* [[TMP10]], align 8 615; CHECK-NEXT: [[TMP15:%.*]] = load i64, i64* [[TMP11]], align 8 616; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> poison, i64 [[TMP12]], i32 0 617; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 1 618; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i32 2 619; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i64> [[TMP18]], i64 [[TMP15]], i32 3 620; CHECK-NEXT: [[TMP20:%.*]] = sub nsw <4 x i64> [[TMP19]], [[VEC_IND]] 621; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0 622; CHECK-NEXT: store i64 [[TMP21]], i64* [[TMP4]], align 8 623; CHECK-NEXT: [[TMP22:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1 624; CHECK-NEXT: store i64 [[TMP22]], i64* [[TMP5]], align 8 625; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2 626; CHECK-NEXT: store i64 [[TMP23]], i64* [[TMP6]], align 8 627; CHECK-NEXT: [[TMP24:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3 628; CHECK-NEXT: store i64 [[TMP24]], i64* [[TMP7]], align 8 629; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x i64> [[TMP20]], i32 0 630; CHECK-NEXT: store i64 [[TMP25]], i64* [[TMP8]], align 8 631; CHECK-NEXT: [[TMP26:%.*]] = extractelement <4 x i64> [[TMP20]], i32 1 632; CHECK-NEXT: store i64 [[TMP26]], i64* [[TMP9]], align 8 633; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i64> [[TMP20]], i32 2 634; CHECK-NEXT: store i64 [[TMP27]], i64* [[TMP10]], align 8 635; CHECK-NEXT: [[TMP28:%.*]] = extractelement <4 x i64> [[TMP20]], i32 3 636; CHECK-NEXT: store i64 [[TMP28]], i64* [[TMP11]], align 8 637; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 638; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 -4, i64 -4, i64 -4, i64 -4> 639; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 640; CHECK-NEXT: br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP16:!llvm.loop !.*]] 641; CHECK: middle.block: 642; CHECK-NEXT: br i1 true, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]] 643; CHECK: scalar.ph: 644; CHECK-NEXT: br label [[FOR_BODY:%.*]] 645; CHECK: for.body: 646; CHECK-NEXT: br i1 undef, label [[FOR_BODY]], label [[FOR_EXIT]], [[LOOP17:!llvm.loop !.*]] 647; CHECK: for.exit: 648; CHECK-NEXT: ret void 649; 650entry: 651 br label %for.body 652 653for.body: 654 %i = phi i64 [ 1023, %entry ], [ %i.next, %for.body ] 655 %0 = add nsw i64 %X, %i 656 %1 = getelementptr inbounds %pair, %pair* %P1, i64 %i, i32 0 657 %2 = getelementptr inbounds %pair, %pair* %P2, i64 %i, i32 1 658 %3 = load i64, i64* %2, align 8 659 %4 = sub nsw i64 %3, %i 660 store i64 %0, i64* %1, align 8 661 store i64 %4, i64* %2, align 8 662 %i.next = add nsw i64 %i, -1 663 %cond = icmp sgt i64 %i, 0 664 br i1 %cond, label %for.body, label %for.exit 665 666for.exit: 667 ret void 668} 669 670; Check vectorization on interleaved access groups identified from mixed 671; loads/stores. 672; void mixed_load2_store2(int *A, int *B) { 673; for (unsigned i = 0; i < 1024; i+=2) { 674; B[i] = A[i] * A[i+1]; 675; B[i+1] = A[i] + A[i+1]; 676; } 677; } 678 679 680define void @mixed_load2_store2(i32* noalias nocapture readonly %A, i32* noalias nocapture %B) { 681; CHECK-LABEL: @mixed_load2_store2( 682; CHECK-NEXT: entry: 683; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 684; CHECK: vector.ph: 685; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 686; CHECK: vector.body: 687; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 688; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1 689; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[OFFSET_IDX]] 690; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32* [[TMP0]] to <8 x i32>* 691; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, <8 x i32>* [[TMP1]], align 4 692; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6> 693; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7> 694; CHECK-NEXT: [[TMP2:%.*]] = or i64 [[OFFSET_IDX]], 1 695; CHECK-NEXT: [[TMP3:%.*]] = mul nsw <4 x i32> [[STRIDED_VEC1]], [[STRIDED_VEC]] 696; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6> 697; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7> 698; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i32> [[STRIDED_VEC4]], [[STRIDED_VEC3]] 699; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 -1 700; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP5]], i64 [[TMP2]] 701; CHECK-NEXT: [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <8 x i32>* 702; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7> 703; CHECK-NEXT: store <8 x i32> [[INTERLEAVED_VEC]], <8 x i32>* [[TMP7]], align 4 704; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 705; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512 706; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP18:!llvm.loop !.*]] 707; CHECK: middle.block: 708; CHECK-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]] 709; CHECK: scalar.ph: 710; CHECK-NEXT: br label [[FOR_BODY:%.*]] 711; CHECK: for.cond.cleanup: 712; CHECK-NEXT: ret void 713; CHECK: for.body: 714; CHECK-NEXT: br i1 undef, label [[FOR_BODY]], label [[FOR_COND_CLEANUP]], [[LOOP19:!llvm.loop !.*]] 715; 716entry: 717 br label %for.body 718 719for.cond.cleanup: ; preds = %for.body 720 ret void 721 722for.body: ; preds = %for.body, %entry 723 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 724 %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv 725 %tmp = load i32, i32* %arrayidx, align 4 726 %tmp1 = or i64 %indvars.iv, 1 727 %arrayidx2 = getelementptr inbounds i32, i32* %A, i64 %tmp1 728 %tmp2 = load i32, i32* %arrayidx2, align 4 729 %mul = mul nsw i32 %tmp2, %tmp 730 %arrayidx4 = getelementptr inbounds i32, i32* %B, i64 %indvars.iv 731 store i32 %mul, i32* %arrayidx4, align 4 732 %tmp3 = load i32, i32* %arrayidx, align 4 733 %tmp4 = load i32, i32* %arrayidx2, align 4 734 %add10 = add nsw i32 %tmp4, %tmp3 735 %arrayidx13 = getelementptr inbounds i32, i32* %B, i64 %tmp1 736 store i32 %add10, i32* %arrayidx13, align 4 737 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2 738 %cmp = icmp ult i64 %indvars.iv.next, 1024 739 br i1 %cmp, label %for.body, label %for.cond.cleanup 740} 741 742; Check vectorization on interleaved access groups identified from mixed 743; loads/stores. 744; void mixed_load3_store3(int *A) { 745; for (unsigned i = 0; i < 1024; i++) { 746; *A++ += i; 747; *A++ += i; 748; *A++ += i; 749; } 750; } 751 752 753define void @mixed_load3_store3(i32* nocapture %A) { 754; CHECK-LABEL: @mixed_load3_store3( 755; CHECK-NEXT: entry: 756; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 757; CHECK: vector.ph: 758; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 759; CHECK: vector.body: 760; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 761; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 762; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[INDEX]], 3 763; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i32, i32* [[A:%.*]], i64 [[TMP0]] 764; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32* [[NEXT_GEP]] to <12 x i32>* 765; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <12 x i32>, <12 x i32>* [[TMP1]], align 4 766; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <12 x i32> [[WIDE_VEC]], <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9> 767; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <12 x i32> [[WIDE_VEC]], <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10> 768; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <12 x i32> [[WIDE_VEC]], <12 x i32> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11> 769; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[STRIDED_VEC]], [[VEC_IND]] 770; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[NEXT_GEP]], i64 2 771; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[STRIDED_VEC2]], [[VEC_IND]] 772; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i32> [[STRIDED_VEC3]], [[VEC_IND]] 773; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP3]], i64 -2 774; CHECK-NEXT: [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <12 x i32>* 775; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7> 776; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef> 777; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP9]], <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11> 778; CHECK-NEXT: store <12 x i32> [[INTERLEAVED_VEC]], <12 x i32>* [[TMP7]], align 4 779; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 780; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4> 781; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 782; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP20:!llvm.loop !.*]] 783; CHECK: middle.block: 784; CHECK-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]] 785; CHECK: scalar.ph: 786; CHECK-NEXT: br label [[FOR_BODY:%.*]] 787; CHECK: for.cond.cleanup: 788; CHECK-NEXT: ret void 789; CHECK: for.body: 790; CHECK-NEXT: br i1 undef, label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], [[LOOP21:!llvm.loop !.*]] 791; 792entry: 793 br label %for.body 794 795for.cond.cleanup: ; preds = %for.body 796 ret void 797 798for.body: ; preds = %for.body, %entry 799 %i.013 = phi i32 [ 0, %entry ], [ %inc, %for.body ] 800 %A.addr.012 = phi i32* [ %A, %entry ], [ %incdec.ptr3, %for.body ] 801 %incdec.ptr = getelementptr inbounds i32, i32* %A.addr.012, i64 1 802 %tmp = load i32, i32* %A.addr.012, align 4 803 %add = add i32 %tmp, %i.013 804 store i32 %add, i32* %A.addr.012, align 4 805 %incdec.ptr1 = getelementptr inbounds i32, i32* %A.addr.012, i64 2 806 %tmp1 = load i32, i32* %incdec.ptr, align 4 807 %add2 = add i32 %tmp1, %i.013 808 store i32 %add2, i32* %incdec.ptr, align 4 809 %incdec.ptr3 = getelementptr inbounds i32, i32* %A.addr.012, i64 3 810 %tmp2 = load i32, i32* %incdec.ptr1, align 4 811 %add4 = add i32 %tmp2, %i.013 812 store i32 %add4, i32* %incdec.ptr1, align 4 813 %inc = add nuw nsw i32 %i.013, 1 814 %exitcond = icmp eq i32 %inc, 1024 815 br i1 %exitcond, label %for.cond.cleanup, label %for.body 816} 817 818; Check vectorization on interleaved access groups with members having different 819; kinds of type. 820 821; struct IntFloat { 822; int a; 823; float b; 824; }; 825; 826; int SA; 827; float SB; 828; 829; void int_float_struct(struct IntFloat *A) { 830; int SumA; 831; float SumB; 832; for (unsigned i = 0; i < 1024; i++) { 833; SumA += A[i].a; 834; SumB += A[i].b; 835; } 836; SA = SumA; 837; SB = SumB; 838; } 839 840 841%struct.IntFloat = type { i32, float } 842 843@SA = common global i32 0, align 4 844@SB = common global float 0.000000e+00, align 4 845 846define void @int_float_struct(%struct.IntFloat* nocapture readonly %A) #0 { 847; CHECK-LABEL: @int_float_struct( 848; CHECK-NEXT: entry: 849; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 850; CHECK: vector.ph: 851; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 852; CHECK: vector.body: 853; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 854; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ <float undef, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ] 855; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ <i32 undef, i32 0, i32 0, i32 0>, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ] 856; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[STRUCT_INTFLOAT:%.*]], %struct.IntFloat* [[A:%.*]], i64 [[INDEX]], i32 0 857; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32* [[TMP0]] to <8 x i32>* 858; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, <8 x i32>* [[TMP1]], align 4 859; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6> 860; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7> 861; CHECK-NEXT: [[TMP2:%.*]] = bitcast <4 x i32> [[STRIDED_VEC2]] to <4 x float> 862; CHECK-NEXT: [[TMP3]] = add <4 x i32> [[STRIDED_VEC]], [[VEC_PHI1]] 863; CHECK-NEXT: [[TMP4]] = fadd fast <4 x float> [[VEC_PHI]], [[TMP2]] 864; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 865; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 866; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP22:!llvm.loop !.*]] 867; CHECK: middle.block: 868; CHECK-NEXT: [[RDX_SHUF5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef> 869; CHECK-NEXT: [[BIN_RDX6:%.*]] = add <4 x i32> [[TMP3]], [[RDX_SHUF5]] 870; CHECK-NEXT: [[RDX_SHUF7:%.*]] = shufflevector <4 x i32> [[BIN_RDX6]], <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef> 871; CHECK-NEXT: [[BIN_RDX8:%.*]] = add <4 x i32> [[BIN_RDX6]], [[RDX_SHUF7]] 872; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[BIN_RDX8]], i32 0 873; CHECK-NEXT: [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef> 874; CHECK-NEXT: [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP4]], [[RDX_SHUF]] 875; CHECK-NEXT: [[RDX_SHUF3:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef> 876; CHECK-NEXT: [[BIN_RDX4:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF3]] 877; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x float> [[BIN_RDX4]], i32 0 878; CHECK-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]] 879; CHECK: scalar.ph: 880; CHECK-NEXT: br label [[FOR_BODY:%.*]] 881; CHECK: for.cond.cleanup: 882; CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ undef, [[FOR_BODY]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ] 883; CHECK-NEXT: [[ADD3_LCSSA:%.*]] = phi float [ undef, [[FOR_BODY]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ] 884; CHECK-NEXT: store i32 [[ADD_LCSSA]], i32* @SA, align 4 885; CHECK-NEXT: store float [[ADD3_LCSSA]], float* @SB, align 4 886; CHECK-NEXT: ret void 887; CHECK: for.body: 888; CHECK-NEXT: br i1 undef, label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], [[LOOP23:!llvm.loop !.*]] 889; 890entry: 891 br label %for.body 892 893for.cond.cleanup: ; preds = %for.body 894 store i32 %add, i32* @SA, align 4 895 store float %add3, float* @SB, align 4 896 ret void 897 898for.body: ; preds = %for.body, %entry 899 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 900 %SumB.014 = phi float [ undef, %entry ], [ %add3, %for.body ] 901 %SumA.013 = phi i32 [ undef, %entry ], [ %add, %for.body ] 902 %a = getelementptr inbounds %struct.IntFloat, %struct.IntFloat* %A, i64 %indvars.iv, i32 0 903 %tmp = load i32, i32* %a, align 4 904 %add = add nsw i32 %tmp, %SumA.013 905 %b = getelementptr inbounds %struct.IntFloat, %struct.IntFloat* %A, i64 %indvars.iv, i32 1 906 %tmp1 = load float, float* %b, align 4 907 %add3 = fadd fast float %SumB.014, %tmp1 908 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 909 %exitcond = icmp eq i64 %indvars.iv.next, 1024 910 br i1 %exitcond, label %for.cond.cleanup, label %for.body 911} 912 913; Check vectorization of interleaved access groups in the presence of 914; dependences (PR27626). The following tests check that we don't reorder 915; dependent loads and stores when generating code for interleaved access 916; groups. Stores should be scalarized because the required code motion would 917; break dependences, and the remaining interleaved load groups should have 918; gaps. 919 920; PR27626_0: Ensure a strided store is not moved after a dependent (zero 921; distance) strided load. 922 923; void PR27626_0(struct pair *p, int z, int n) { 924; for (int i = 0; i < n; i++) { 925; p[i].x = z; 926; p[i].y = p[i].x; 927; } 928; } 929 930 931%pair.i32 = type { i32, i32 } 932define void @PR27626_0(%pair.i32 *%p, i32 %z, i64 %n) { 933; CHECK-LABEL: @PR27626_0( 934; CHECK-NEXT: entry: 935; CHECK-NEXT: [[TMP0:%.*]] = icmp sgt i64 [[N:%.*]], 1 936; CHECK-NEXT: [[SMAX:%.*]] = select i1 [[TMP0]], i64 [[N]], i64 1 937; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 5 938; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 939; CHECK: vector.ph: 940; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3 941; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 942; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i64 4, i64 [[N_MOD_VF]] 943; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP2]] 944; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 945; CHECK: vector.body: 946; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 947; CHECK-NEXT: [[TMP3:%.*]] = or i64 [[INDEX]], 1 948; CHECK-NEXT: [[TMP4:%.*]] = or i64 [[INDEX]], 2 949; CHECK-NEXT: [[TMP5:%.*]] = or i64 [[INDEX]], 3 950; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], %pair.i32* [[P:%.*]], i64 [[INDEX]], i32 0 951; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[TMP3]], i32 0 952; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[TMP4]], i32 0 953; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[TMP5]], i32 0 954; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[INDEX]], i32 1 955; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[TMP3]], i32 1 956; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[TMP4]], i32 1 957; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[TMP5]], i32 1 958; CHECK-NEXT: store i32 [[Z:%.*]], i32* [[TMP6]], align 4 959; CHECK-NEXT: store i32 [[Z]], i32* [[TMP7]], align 4 960; CHECK-NEXT: store i32 [[Z]], i32* [[TMP8]], align 4 961; CHECK-NEXT: store i32 [[Z]], i32* [[TMP9]], align 4 962; CHECK-NEXT: [[TMP14:%.*]] = bitcast i32* [[TMP6]] to <8 x i32>* 963; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, <8 x i32>* [[TMP14]], align 4 964; CHECK-NEXT: [[TMP15:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i32 0 965; CHECK-NEXT: store i32 [[TMP15]], i32* [[TMP10]], align 4 966; CHECK-NEXT: [[TMP16:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i32 2 967; CHECK-NEXT: store i32 [[TMP16]], i32* [[TMP11]], align 4 968; CHECK-NEXT: [[TMP17:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i32 4 969; CHECK-NEXT: store i32 [[TMP17]], i32* [[TMP12]], align 4 970; CHECK-NEXT: [[TMP18:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i32 6 971; CHECK-NEXT: store i32 [[TMP18]], i32* [[TMP13]], align 4 972; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 973; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 974; CHECK-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP24:!llvm.loop !.*]] 975; CHECK: middle.block: 976; CHECK-NEXT: br i1 false, label [[FOR_END:%.*]], label [[SCALAR_PH]] 977; CHECK: scalar.ph: 978; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 979; CHECK-NEXT: br label [[FOR_BODY:%.*]] 980; CHECK: for.body: 981; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 982; CHECK-NEXT: [[P_I_X:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[I]], i32 0 983; CHECK-NEXT: [[P_I_Y:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[I]], i32 1 984; CHECK-NEXT: store i32 [[Z]], i32* [[P_I_X]], align 4 985; CHECK-NEXT: store i32 [[Z]], i32* [[P_I_Y]], align 4 986; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 987; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 988; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], [[LOOP25:!llvm.loop !.*]] 989; CHECK: for.end: 990; CHECK-NEXT: ret void 991; 992entry: 993 br label %for.body 994 995for.body: 996 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ] 997 %p_i.x = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 0 998 %p_i.y = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 1 999 store i32 %z, i32* %p_i.x, align 4 1000 %0 = load i32, i32* %p_i.x, align 4 1001 store i32 %0, i32 *%p_i.y, align 4 1002 %i.next = add nuw nsw i64 %i, 1 1003 %cond = icmp slt i64 %i.next, %n 1004 br i1 %cond, label %for.body, label %for.end 1005 1006for.end: 1007 ret void 1008} 1009 1010; PR27626_1: Ensure a strided load is not moved before a dependent (zero 1011; distance) strided store. 1012 1013; void PR27626_1(struct pair *p, int n) { 1014; int s = 0; 1015; for (int i = 0; i < n; i++) { 1016; p[i].y = p[i].x; 1017; s += p[i].y 1018; } 1019; } 1020 1021 1022define i32 @PR27626_1(%pair.i32 *%p, i64 %n) { 1023; CHECK-LABEL: @PR27626_1( 1024; CHECK-NEXT: entry: 1025; CHECK-NEXT: [[TMP0:%.*]] = icmp sgt i64 [[N:%.*]], 1 1026; CHECK-NEXT: [[SMAX:%.*]] = select i1 [[TMP0]], i64 [[N]], i64 1 1027; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 5 1028; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 1029; CHECK: vector.ph: 1030; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3 1031; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 1032; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i64 4, i64 [[N_MOD_VF]] 1033; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP2]] 1034; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 1035; CHECK: vector.body: 1036; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1037; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[VECTOR_BODY]] ] 1038; CHECK-NEXT: [[TMP3:%.*]] = or i64 [[INDEX]], 1 1039; CHECK-NEXT: [[TMP4:%.*]] = or i64 [[INDEX]], 2 1040; CHECK-NEXT: [[TMP5:%.*]] = or i64 [[INDEX]], 3 1041; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], %pair.i32* [[P:%.*]], i64 [[INDEX]], i32 0 1042; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[INDEX]], i32 1 1043; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[TMP3]], i32 1 1044; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[TMP4]], i32 1 1045; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[TMP5]], i32 1 1046; CHECK-NEXT: [[TMP11:%.*]] = bitcast i32* [[TMP6]] to <8 x i32>* 1047; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, <8 x i32>* [[TMP11]], align 4 1048; CHECK-NEXT: [[TMP12:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i32 0 1049; CHECK-NEXT: store i32 [[TMP12]], i32* [[TMP7]], align 4 1050; CHECK-NEXT: [[TMP13:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i32 2 1051; CHECK-NEXT: store i32 [[TMP13]], i32* [[TMP8]], align 4 1052; CHECK-NEXT: [[TMP14:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i32 4 1053; CHECK-NEXT: store i32 [[TMP14]], i32* [[TMP9]], align 4 1054; CHECK-NEXT: [[TMP15:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i32 6 1055; CHECK-NEXT: store i32 [[TMP15]], i32* [[TMP10]], align 4 1056; CHECK-NEXT: [[TMP16:%.*]] = bitcast i32* [[TMP7]] to <8 x i32>* 1057; CHECK-NEXT: [[WIDE_VEC1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP16]], align 4 1058; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <8 x i32> [[WIDE_VEC1]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6> 1059; CHECK-NEXT: [[TMP17]] = add <4 x i32> [[STRIDED_VEC2]], [[VEC_PHI]] 1060; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 1061; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1062; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP26:!llvm.loop !.*]] 1063; CHECK: middle.block: 1064; CHECK-NEXT: [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP17]], <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef> 1065; CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP17]], [[RDX_SHUF]] 1066; CHECK-NEXT: [[RDX_SHUF3:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef> 1067; CHECK-NEXT: [[BIN_RDX4:%.*]] = add <4 x i32> [[BIN_RDX]], [[RDX_SHUF3]] 1068; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i32> [[BIN_RDX4]], i32 0 1069; CHECK-NEXT: br i1 false, label [[FOR_END:%.*]], label [[SCALAR_PH]] 1070; CHECK: scalar.ph: 1071; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 1072; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP19]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 1073; CHECK-NEXT: br label [[FOR_BODY:%.*]] 1074; CHECK: for.body: 1075; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 1076; CHECK-NEXT: [[S:%.*]] = phi i32 [ [[TMP21:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 1077; CHECK-NEXT: [[P_I_X:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[I]], i32 0 1078; CHECK-NEXT: [[P_I_Y:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[I]], i32 1 1079; CHECK-NEXT: [[TMP20:%.*]] = load i32, i32* [[P_I_X]], align 4 1080; CHECK-NEXT: store i32 [[TMP20]], i32* [[P_I_Y]], align 4 1081; CHECK-NEXT: [[TMP21]] = add nsw i32 [[TMP20]], [[S]] 1082; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 1083; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 1084; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], [[LOOP27:!llvm.loop !.*]] 1085; CHECK: for.end: 1086; CHECK-NEXT: [[TMP22:%.*]] = phi i32 [ [[TMP21]], [[FOR_BODY]] ], [ [[TMP19]], [[MIDDLE_BLOCK]] ] 1087; CHECK-NEXT: ret i32 [[TMP22]] 1088; 1089entry: 1090 br label %for.body 1091 1092for.body: 1093 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ] 1094 %s = phi i32 [ %2, %for.body ], [ 0, %entry ] 1095 %p_i.x = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 0 1096 %p_i.y = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 1 1097 %0 = load i32, i32* %p_i.x, align 4 1098 store i32 %0, i32* %p_i.y, align 4 1099 %1 = load i32, i32* %p_i.y, align 4 1100 %2 = add nsw i32 %1, %s 1101 %i.next = add nuw nsw i64 %i, 1 1102 %cond = icmp slt i64 %i.next, %n 1103 br i1 %cond, label %for.body, label %for.end 1104 1105for.end: 1106 %3 = phi i32 [ %2, %for.body ] 1107 ret i32 %3 1108} 1109 1110; PR27626_2: Ensure a strided store is not moved after a dependent (negative 1111; distance) strided load. 1112 1113; void PR27626_2(struct pair *p, int z, int n) { 1114; for (int i = 0; i < n; i++) { 1115; p[i].x = z; 1116; p[i].y = p[i - 1].x; 1117; } 1118; } 1119 1120 1121define void @PR27626_2(%pair.i32 *%p, i64 %n, i32 %z) { 1122; CHECK-LABEL: @PR27626_2( 1123; CHECK-NEXT: entry: 1124; CHECK-NEXT: [[TMP0:%.*]] = icmp sgt i64 [[N:%.*]], 1 1125; CHECK-NEXT: [[SMAX:%.*]] = select i1 [[TMP0]], i64 [[N]], i64 1 1126; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 5 1127; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 1128; CHECK: vector.ph: 1129; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3 1130; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 1131; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i64 4, i64 [[N_MOD_VF]] 1132; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP2]] 1133; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 1134; CHECK: vector.body: 1135; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1136; CHECK-NEXT: [[TMP3:%.*]] = or i64 [[INDEX]], 1 1137; CHECK-NEXT: [[TMP4:%.*]] = or i64 [[INDEX]], 2 1138; CHECK-NEXT: [[TMP5:%.*]] = or i64 [[INDEX]], 3 1139; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], %pair.i32* [[P:%.*]], i64 [[INDEX]], i32 0 1140; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[TMP3]], i32 0 1141; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[TMP4]], i32 0 1142; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[TMP5]], i32 0 1143; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 -1, i32 0 1144; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[INDEX]], i32 1 1145; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[TMP3]], i32 1 1146; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[TMP4]], i32 1 1147; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[TMP5]], i32 1 1148; CHECK-NEXT: store i32 [[Z:%.*]], i32* [[TMP6]], align 4 1149; CHECK-NEXT: store i32 [[Z]], i32* [[TMP7]], align 4 1150; CHECK-NEXT: store i32 [[Z]], i32* [[TMP8]], align 4 1151; CHECK-NEXT: store i32 [[Z]], i32* [[TMP9]], align 4 1152; CHECK-NEXT: [[TMP15:%.*]] = bitcast i32* [[TMP10]] to <8 x i32>* 1153; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, <8 x i32>* [[TMP15]], align 4 1154; CHECK-NEXT: [[TMP16:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i32 0 1155; CHECK-NEXT: store i32 [[TMP16]], i32* [[TMP11]], align 4 1156; CHECK-NEXT: [[TMP17:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i32 2 1157; CHECK-NEXT: store i32 [[TMP17]], i32* [[TMP12]], align 4 1158; CHECK-NEXT: [[TMP18:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i32 4 1159; CHECK-NEXT: store i32 [[TMP18]], i32* [[TMP13]], align 4 1160; CHECK-NEXT: [[TMP19:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i32 6 1161; CHECK-NEXT: store i32 [[TMP19]], i32* [[TMP14]], align 4 1162; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 1163; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1164; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP28:!llvm.loop !.*]] 1165; CHECK: middle.block: 1166; CHECK-NEXT: br i1 false, label [[FOR_END:%.*]], label [[SCALAR_PH]] 1167; CHECK: scalar.ph: 1168; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 1169; CHECK-NEXT: br label [[FOR_BODY:%.*]] 1170; CHECK: for.body: 1171; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 1172; CHECK-NEXT: [[P_I_X:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[I]], i32 0 1173; CHECK-NEXT: [[P_I_MINUS_1_X:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 -1, i32 0 1174; CHECK-NEXT: [[P_I_Y:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[I]], i32 1 1175; CHECK-NEXT: store i32 [[Z]], i32* [[P_I_X]], align 4 1176; CHECK-NEXT: [[TMP21:%.*]] = load i32, i32* [[P_I_MINUS_1_X]], align 4 1177; CHECK-NEXT: store i32 [[TMP21]], i32* [[P_I_Y]], align 4 1178; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 1179; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 1180; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], [[LOOP29:!llvm.loop !.*]] 1181; CHECK: for.end: 1182; CHECK-NEXT: ret void 1183; 1184entry: 1185 br label %for.body 1186 1187for.body: 1188 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ] 1189 %i_minus_1 = add nuw nsw i64 %i, -1 1190 %p_i.x = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 0 1191 %p_i_minus_1.x = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i_minus_1, i32 0 1192 %p_i.y = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 1 1193 store i32 %z, i32* %p_i.x, align 4 1194 %0 = load i32, i32* %p_i_minus_1.x, align 4 1195 store i32 %0, i32 *%p_i.y, align 4 1196 %i.next = add nuw nsw i64 %i, 1 1197 %cond = icmp slt i64 %i.next, %n 1198 br i1 %cond, label %for.body, label %for.end 1199 1200for.end: 1201 ret void 1202} 1203 1204; PR27626_3: Ensure a strided load is not moved before a dependent (negative 1205; distance) strided store. 1206 1207; void PR27626_3(struct pair *p, int z, int n) { 1208; for (int i = 0; i < n; i++) { 1209; p[i + 1].y = p[i].x; 1210; s += p[i].y; 1211; } 1212; } 1213 1214 1215define i32 @PR27626_3(%pair.i32 *%p, i64 %n, i32 %z) { 1216; CHECK-LABEL: @PR27626_3( 1217; CHECK-NEXT: entry: 1218; CHECK-NEXT: [[TMP0:%.*]] = icmp sgt i64 [[N:%.*]], 1 1219; CHECK-NEXT: [[SMAX:%.*]] = select i1 [[TMP0]], i64 [[N]], i64 1 1220; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 5 1221; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 1222; CHECK: vector.ph: 1223; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3 1224; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 1225; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i64 4, i64 [[N_MOD_VF]] 1226; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP2]] 1227; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 1228; CHECK: vector.body: 1229; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1230; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 1231; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ] 1232; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw <4 x i64> [[VEC_IND]], <i64 1, i64 1, i64 1, i64 1> 1233; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], %pair.i32* [[P:%.*]], i64 [[INDEX]], i32 0 1234; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[INDEX]], i32 1 1235; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0 1236; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[TMP6]], i32 1 1237; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1 1238; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[TMP8]], i32 1 1239; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2 1240; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[TMP10]], i32 1 1241; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3 1242; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[TMP12]], i32 1 1243; CHECK-NEXT: [[TMP14:%.*]] = bitcast i32* [[TMP4]] to <8 x i32>* 1244; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, <8 x i32>* [[TMP14]], align 4 1245; CHECK-NEXT: [[TMP15:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i32 0 1246; CHECK-NEXT: store i32 [[TMP15]], i32* [[TMP7]], align 4 1247; CHECK-NEXT: [[TMP16:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i32 2 1248; CHECK-NEXT: store i32 [[TMP16]], i32* [[TMP9]], align 4 1249; CHECK-NEXT: [[TMP17:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i32 4 1250; CHECK-NEXT: store i32 [[TMP17]], i32* [[TMP11]], align 4 1251; CHECK-NEXT: [[TMP18:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i32 6 1252; CHECK-NEXT: store i32 [[TMP18]], i32* [[TMP13]], align 4 1253; CHECK-NEXT: [[TMP19:%.*]] = bitcast i32* [[TMP5]] to <8 x i32>* 1254; CHECK-NEXT: [[WIDE_VEC1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP19]], align 4 1255; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <8 x i32> [[WIDE_VEC1]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6> 1256; CHECK-NEXT: [[TMP20]] = add <4 x i32> [[STRIDED_VEC2]], [[VEC_PHI]] 1257; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 1258; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4> 1259; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1260; CHECK-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP30:!llvm.loop !.*]] 1261; CHECK: middle.block: 1262; CHECK-NEXT: [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP20]], <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef> 1263; CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP20]], [[RDX_SHUF]] 1264; CHECK-NEXT: [[RDX_SHUF3:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef> 1265; CHECK-NEXT: [[BIN_RDX4:%.*]] = add <4 x i32> [[BIN_RDX]], [[RDX_SHUF3]] 1266; CHECK-NEXT: [[TMP22:%.*]] = extractelement <4 x i32> [[BIN_RDX4]], i32 0 1267; CHECK-NEXT: br i1 false, label [[FOR_END:%.*]], label [[SCALAR_PH]] 1268; CHECK: scalar.ph: 1269; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 1270; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP22]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 1271; CHECK-NEXT: br label [[FOR_BODY:%.*]] 1272; CHECK: for.body: 1273; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 1274; CHECK-NEXT: [[S:%.*]] = phi i32 [ [[TMP25:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 1275; CHECK-NEXT: [[I_PLUS_1:%.*]] = add nuw nsw i64 [[I]], 1 1276; CHECK-NEXT: [[P_I_X:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[I]], i32 0 1277; CHECK-NEXT: [[P_I_Y:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[I]], i32 1 1278; CHECK-NEXT: [[P_I_PLUS_1_Y:%.*]] = getelementptr inbounds [[PAIR_I32]], %pair.i32* [[P]], i64 [[I_PLUS_1]], i32 1 1279; CHECK-NEXT: [[TMP23:%.*]] = load i32, i32* [[P_I_X]], align 4 1280; CHECK-NEXT: store i32 [[TMP23]], i32* [[P_I_PLUS_1_Y]], align 4 1281; CHECK-NEXT: [[TMP24:%.*]] = load i32, i32* [[P_I_Y]], align 4 1282; CHECK-NEXT: [[TMP25]] = add nsw i32 [[TMP24]], [[S]] 1283; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 1284; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 1285; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], [[LOOP31:!llvm.loop !.*]] 1286; CHECK: for.end: 1287; CHECK-NEXT: [[TMP26:%.*]] = phi i32 [ [[TMP25]], [[FOR_BODY]] ], [ [[TMP22]], [[MIDDLE_BLOCK]] ] 1288; CHECK-NEXT: ret i32 [[TMP26]] 1289; 1290entry: 1291 br label %for.body 1292 1293for.body: 1294 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ] 1295 %s = phi i32 [ %2, %for.body ], [ 0, %entry ] 1296 %i_plus_1 = add nuw nsw i64 %i, 1 1297 %p_i.x = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 0 1298 %p_i.y = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 1 1299 %p_i_plus_1.y = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i_plus_1, i32 1 1300 %0 = load i32, i32* %p_i.x, align 4 1301 store i32 %0, i32* %p_i_plus_1.y, align 4 1302 %1 = load i32, i32* %p_i.y, align 4 1303 %2 = add nsw i32 %1, %s 1304 %i.next = add nuw nsw i64 %i, 1 1305 %cond = icmp slt i64 %i.next, %n 1306 br i1 %cond, label %for.body, label %for.end 1307 1308for.end: 1309 %3 = phi i32 [ %2, %for.body ] 1310 ret i32 %3 1311} 1312 1313; PR27626_4: Ensure we form an interleaved group for strided stores in the 1314; presence of a write-after-write dependence. We create a group for 1315; (2) and (3) while excluding (1). 1316 1317; void PR27626_4(int *a, int x, int y, int z, int n) { 1318; for (int i = 0; i < n; i += 2) { 1319; a[i] = x; // (1) 1320; a[i] = y; // (2) 1321; a[i + 1] = z; // (3) 1322; } 1323; } 1324 1325 1326define void @PR27626_4(i32 *%a, i32 %x, i32 %y, i32 %z, i64 %n) { 1327; CHECK-LABEL: @PR27626_4( 1328; CHECK-NEXT: entry: 1329; CHECK-NEXT: [[TMP0:%.*]] = icmp sgt i64 [[N:%.*]], 2 1330; CHECK-NEXT: [[SMAX:%.*]] = select i1 [[TMP0]], i64 [[N]], i64 2 1331; CHECK-NEXT: [[TMP1:%.*]] = add nsw i64 [[SMAX]], -1 1332; CHECK-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 1 1333; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1 1334; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 6 1335; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 1336; CHECK: vector.ph: 1337; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP3]], 9223372036854775804 1338; CHECK-NEXT: [[IND_END:%.*]] = shl nuw i64 [[N_VEC]], 1 1339; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y:%.*]], i32 0 1340; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 1341; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[Z:%.*]], i32 0 1342; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer 1343; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 1344; CHECK: vector.body: 1345; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1346; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1 1347; CHECK-NEXT: [[TMP4:%.*]] = or i64 [[OFFSET_IDX]], 2 1348; CHECK-NEXT: [[TMP5:%.*]] = or i64 [[OFFSET_IDX]], 4 1349; CHECK-NEXT: [[TMP6:%.*]] = or i64 [[OFFSET_IDX]], 6 1350; CHECK-NEXT: [[TMP7:%.*]] = or i64 [[OFFSET_IDX]], 1 1351; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[OFFSET_IDX]] 1352; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP4]] 1353; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP5]] 1354; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP6]] 1355; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 -1 1356; CHECK-NEXT: store i32 [[X:%.*]], i32* [[TMP8]], align 4 1357; CHECK-NEXT: store i32 [[X]], i32* [[TMP9]], align 4 1358; CHECK-NEXT: store i32 [[X]], i32* [[TMP10]], align 4 1359; CHECK-NEXT: store i32 [[X]], i32* [[TMP11]], align 4 1360; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[TMP12]], i64 [[TMP7]] 1361; CHECK-NEXT: [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <8 x i32>* 1362; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> [[BROADCAST_SPLAT2]], <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7> 1363; CHECK-NEXT: store <8 x i32> [[INTERLEAVED_VEC]], <8 x i32>* [[TMP14]], align 4 1364; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 1365; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1366; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP32:!llvm.loop !.*]] 1367; CHECK: middle.block: 1368; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]] 1369; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 1370; CHECK: scalar.ph: 1371; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 1372; CHECK-NEXT: br label [[FOR_BODY:%.*]] 1373; CHECK: for.body: 1374; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 1375; CHECK-NEXT: [[I_PLUS_1:%.*]] = or i64 [[I]], 1 1376; CHECK-NEXT: [[A_I:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[I]] 1377; CHECK-NEXT: [[A_I_PLUS_1:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[I_PLUS_1]] 1378; CHECK-NEXT: store i32 [[Y]], i32* [[A_I]], align 4 1379; CHECK-NEXT: store i32 [[Z]], i32* [[A_I_PLUS_1]], align 4 1380; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 2 1381; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 1382; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], [[LOOP33:!llvm.loop !.*]] 1383; CHECK: for.end: 1384; CHECK-NEXT: ret void 1385; 1386entry: 1387 br label %for.body 1388 1389for.body: 1390 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ] 1391 %i_plus_1 = add i64 %i, 1 1392 %a_i = getelementptr inbounds i32, i32* %a, i64 %i 1393 %a_i_plus_1 = getelementptr inbounds i32, i32* %a, i64 %i_plus_1 1394 store i32 %x, i32* %a_i, align 4 1395 store i32 %y, i32* %a_i, align 4 1396 store i32 %z, i32* %a_i_plus_1, align 4 1397 %i.next = add nuw nsw i64 %i, 2 1398 %cond = icmp slt i64 %i.next, %n 1399 br i1 %cond, label %for.body, label %for.end 1400 1401for.end: 1402 ret void 1403} 1404 1405; PR27626_5: Ensure we do not form an interleaved group for strided stores in 1406; the presence of a write-after-write dependence. 1407 1408; void PR27626_5(int *a, int x, int y, int z, int n) { 1409; for (int i = 3; i < n; i += 2) { 1410; a[i - 1] = x; 1411; a[i - 3] = y; 1412; a[i] = z; 1413; } 1414; } 1415 1416 1417define void @PR27626_5(i32 *%a, i32 %x, i32 %y, i32 %z, i64 %n) { 1418; CHECK-LABEL: @PR27626_5( 1419; CHECK-NEXT: entry: 1420; CHECK-NEXT: [[TMP0:%.*]] = icmp sgt i64 [[N:%.*]], 5 1421; CHECK-NEXT: [[SMAX:%.*]] = select i1 [[TMP0]], i64 [[N]], i64 5 1422; CHECK-NEXT: [[TMP1:%.*]] = add nsw i64 [[SMAX]], -4 1423; CHECK-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 1 1424; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1 1425; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 6 1426; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 1427; CHECK: vector.ph: 1428; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP3]], 9223372036854775804 1429; CHECK-NEXT: [[TMP4:%.*]] = shl nuw i64 [[N_VEC]], 1 1430; CHECK-NEXT: [[IND_END:%.*]] = or i64 [[TMP4]], 3 1431; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 1432; CHECK: vector.body: 1433; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1434; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 5, i64 7, i64 9>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 1435; CHECK-NEXT: [[TMP5:%.*]] = shl i64 [[INDEX]], 1 1436; CHECK-NEXT: [[OFFSET_IDX:%.*]] = or i64 [[TMP5]], 3 1437; CHECK-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[OFFSET_IDX]], 2 1438; CHECK-NEXT: [[TMP7:%.*]] = or i64 [[TMP5]], 7 1439; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[OFFSET_IDX]], 6 1440; CHECK-NEXT: [[TMP9:%.*]] = add <4 x i64> [[VEC_IND]], <i64 -1, i64 -1, i64 -1, i64 -1> 1441; CHECK-NEXT: [[TMP10:%.*]] = add <4 x i64> [[VEC_IND]], <i64 -3, i64 -3, i64 -3, i64 -3> 1442; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[OFFSET_IDX]] 1443; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP6]] 1444; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP7]] 1445; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP8]] 1446; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i64> [[TMP9]], i32 0 1447; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP15]] 1448; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i64> [[TMP9]], i32 1 1449; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP17]] 1450; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i64> [[TMP9]], i32 2 1451; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP19]] 1452; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i64> [[TMP9]], i32 3 1453; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP21]] 1454; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x i64> [[TMP10]], i32 0 1455; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP23]] 1456; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x i64> [[TMP10]], i32 1 1457; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP25]] 1458; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i64> [[TMP10]], i32 2 1459; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP27]] 1460; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i64> [[TMP10]], i32 3 1461; CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP29]] 1462; CHECK-NEXT: store i32 [[X:%.*]], i32* [[TMP16]], align 4 1463; CHECK-NEXT: store i32 [[X]], i32* [[TMP18]], align 4 1464; CHECK-NEXT: store i32 [[X]], i32* [[TMP20]], align 4 1465; CHECK-NEXT: store i32 [[X]], i32* [[TMP22]], align 4 1466; CHECK-NEXT: store i32 [[Y:%.*]], i32* [[TMP24]], align 4 1467; CHECK-NEXT: store i32 [[Y]], i32* [[TMP26]], align 4 1468; CHECK-NEXT: store i32 [[Y]], i32* [[TMP28]], align 4 1469; CHECK-NEXT: store i32 [[Y]], i32* [[TMP30]], align 4 1470; CHECK-NEXT: store i32 [[Z:%.*]], i32* [[TMP11]], align 4 1471; CHECK-NEXT: store i32 [[Z]], i32* [[TMP12]], align 4 1472; CHECK-NEXT: store i32 [[Z]], i32* [[TMP13]], align 4 1473; CHECK-NEXT: store i32 [[Z]], i32* [[TMP14]], align 4 1474; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 1475; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 8, i64 8, i64 8, i64 8> 1476; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1477; CHECK-NEXT: br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP34:!llvm.loop !.*]] 1478; CHECK: middle.block: 1479; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]] 1480; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 1481; CHECK: scalar.ph: 1482; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ] 1483; CHECK-NEXT: br label [[FOR_BODY:%.*]] 1484; CHECK: for.body: 1485; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 1486; CHECK-NEXT: [[I_MINUS_1:%.*]] = add i64 [[I]], -1 1487; CHECK-NEXT: [[I_MINUS_3:%.*]] = add i64 [[I]], -3 1488; CHECK-NEXT: [[A_I:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[I]] 1489; CHECK-NEXT: [[A_I_MINUS_1:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[I_MINUS_1]] 1490; CHECK-NEXT: [[A_I_MINUS_3:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[I_MINUS_3]] 1491; CHECK-NEXT: store i32 [[X]], i32* [[A_I_MINUS_1]], align 4 1492; CHECK-NEXT: store i32 [[Y]], i32* [[A_I_MINUS_3]], align 4 1493; CHECK-NEXT: store i32 [[Z]], i32* [[A_I]], align 4 1494; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 2 1495; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 1496; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], [[LOOP35:!llvm.loop !.*]] 1497; CHECK: for.end: 1498; CHECK-NEXT: ret void 1499; 1500entry: 1501 br label %for.body 1502 1503for.body: 1504 %i = phi i64 [ %i.next, %for.body ], [ 3, %entry ] 1505 %i_minus_1 = sub i64 %i, 1 1506 %i_minus_3 = sub i64 %i_minus_1, 2 1507 %a_i = getelementptr inbounds i32, i32* %a, i64 %i 1508 %a_i_minus_1 = getelementptr inbounds i32, i32* %a, i64 %i_minus_1 1509 %a_i_minus_3 = getelementptr inbounds i32, i32* %a, i64 %i_minus_3 1510 store i32 %x, i32* %a_i_minus_1, align 4 1511 store i32 %y, i32* %a_i_minus_3, align 4 1512 store i32 %z, i32* %a_i, align 4 1513 %i.next = add nuw nsw i64 %i, 2 1514 %cond = icmp slt i64 %i.next, %n 1515 br i1 %cond, label %for.body, label %for.end 1516 1517for.end: 1518 ret void 1519} 1520 1521; PR34743: Ensure that a cast which needs to sink after a load that belongs to 1522; an interleaved group, indeeded gets sunk. 1523 1524; void PR34743(short *a, int *b, int n) { 1525; for (int i = 0, iv = 0; iv < n; i++, iv += 2) { 1526; b[i] = a[iv] * a[iv+1] * a[iv+2]; 1527; } 1528; } 1529 1530 1531define void @PR34743(i16* %a, i32* %b, i64 %n) { 1532; CHECK-LABEL: @PR34743( 1533; CHECK-NEXT: entry: 1534; CHECK-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[A:%.*]], align 2 1535; CHECK-NEXT: [[TMP0:%.*]] = lshr i64 [[N:%.*]], 1 1536; CHECK-NEXT: [[TMP1:%.*]] = add nuw i64 [[TMP0]], 1 1537; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 6 1538; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 1539; CHECK: vector.memcheck: 1540; CHECK-NEXT: [[TMP2:%.*]] = lshr i64 [[N]], 1 1541; CHECK-NEXT: [[TMP3:%.*]] = add nuw i64 [[TMP2]], 1 1542; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B:%.*]], i64 [[TMP3]] 1543; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i16, i16* [[A]], i64 1 1544; CHECK-NEXT: [[TMP4:%.*]] = and i64 [[N]], -2 1545; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[TMP4]], 3 1546; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP5]] 1547; CHECK-NEXT: [[TMP6:%.*]] = bitcast i16* [[SCEVGEP5]] to i32* 1548; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt i32* [[TMP6]], [[B]] 1549; CHECK-NEXT: [[TMP7:%.*]] = bitcast i32* [[SCEVGEP]] to i16* 1550; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult i16* [[SCEVGEP3]], [[TMP7]] 1551; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 1552; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 1553; CHECK: vector.ph: 1554; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP1]], -4 1555; CHECK-NEXT: [[IND_END:%.*]] = shl i64 [[N_VEC]], 1 1556; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i32 3 1557; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 1558; CHECK: vector.body: 1559; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1560; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[STRIDED_VEC8:%.*]], [[VECTOR_BODY]] ] 1561; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1 1562; CHECK-NEXT: [[TMP8:%.*]] = or i64 [[OFFSET_IDX]], 1 1563; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP8]] 1564; CHECK-NEXT: [[TMP10:%.*]] = bitcast i16* [[TMP9]] to <8 x i16>* 1565; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i16>, <8 x i16>* [[TMP10]], align 4 1566; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i16> [[WIDE_VEC]], <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6> 1567; CHECK-NEXT: [[STRIDED_VEC8]] = shufflevector <8 x i16> [[WIDE_VEC]], <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7> 1568; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[STRIDED_VEC8]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 1569; CHECK-NEXT: [[TMP12:%.*]] = sext <4 x i16> [[STRIDED_VEC]] to <4 x i32> 1570; CHECK-NEXT: [[TMP13:%.*]] = sext <4 x i16> [[TMP11]] to <4 x i32> 1571; CHECK-NEXT: [[TMP14:%.*]] = sext <4 x i16> [[STRIDED_VEC8]] to <4 x i32> 1572; CHECK-NEXT: [[TMP15:%.*]] = mul nsw <4 x i32> [[TMP13]], [[TMP12]] 1573; CHECK-NEXT: [[TMP16:%.*]] = mul nsw <4 x i32> [[TMP15]], [[TMP14]] 1574; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]] 1575; CHECK-NEXT: [[TMP18:%.*]] = bitcast i32* [[TMP17]] to <4 x i32>* 1576; CHECK-NEXT: store <4 x i32> [[TMP16]], <4 x i32>* [[TMP18]], align 4, !alias.scope !36, !noalias !39 1577; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 1578; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1579; CHECK-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP41:!llvm.loop !.*]] 1580; CHECK: middle.block: 1581; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]] 1582; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <8 x i16> [[WIDE_VEC]], i32 7 1583; CHECK-NEXT: br i1 [[CMP_N]], label [[END:%.*]], label [[SCALAR_PH]] 1584; CHECK: scalar.ph: 1585; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 1586; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY]] ], [ [[IND_END]], [[MIDDLE_BLOCK]] ] 1587; CHECK-NEXT: [[BC_RESUME_VAL7:%.*]] = phi i64 [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ] 1588; CHECK-NEXT: br label [[LOOP:%.*]] 1589; CHECK: loop: 1590; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[LOAD2:%.*]], [[LOOP]] ] 1591; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV2:%.*]], [[LOOP]] ] 1592; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL7]], [[SCALAR_PH]] ], [ [[I1:%.*]], [[LOOP]] ] 1593; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 1594; CHECK-NEXT: [[I1]] = add nuw nsw i64 [[I]], 1 1595; CHECK-NEXT: [[IV1:%.*]] = or i64 [[IV]], 1 1596; CHECK-NEXT: [[IV2]] = add nuw nsw i64 [[IV]], 2 1597; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[IV1]] 1598; CHECK-NEXT: [[LOAD1:%.*]] = load i16, i16* [[GEP1]], align 4 1599; CHECK-NEXT: [[CONV1:%.*]] = sext i16 [[LOAD1]] to i32 1600; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[IV2]] 1601; CHECK-NEXT: [[LOAD2]] = load i16, i16* [[GEP2]], align 4 1602; CHECK-NEXT: [[CONV2:%.*]] = sext i16 [[LOAD2]] to i32 1603; CHECK-NEXT: [[MUL01:%.*]] = mul nsw i32 [[CONV]], [[CONV1]] 1604; CHECK-NEXT: [[MUL012:%.*]] = mul nsw i32 [[MUL01]], [[CONV2]] 1605; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[I]] 1606; CHECK-NEXT: store i32 [[MUL012]], i32* [[ARRAYIDX5]], align 4 1607; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV]], [[N]] 1608; CHECK-NEXT: br i1 [[EXITCOND]], label [[END]], label [[LOOP]], [[LOOP42:!llvm.loop !.*]] 1609; CHECK: end: 1610; CHECK-NEXT: ret void 1611; 1612entry: 1613 %.pre = load i16, i16* %a 1614 br label %loop 1615 1616loop: 1617 %0 = phi i16 [ %.pre, %entry ], [ %load2, %loop ] 1618 %iv = phi i64 [ 0, %entry ], [ %iv2, %loop ] 1619 %i = phi i64 [ 0, %entry ], [ %i1, %loop ] 1620 %conv = sext i16 %0 to i32 1621 %i1 = add nuw nsw i64 %i, 1 1622 %iv1 = add nuw nsw i64 %iv, 1 1623 %iv2 = add nuw nsw i64 %iv, 2 1624 %gep1 = getelementptr inbounds i16, i16* %a, i64 %iv1 1625 %load1 = load i16, i16* %gep1, align 4 1626 %conv1 = sext i16 %load1 to i32 1627 %gep2 = getelementptr inbounds i16, i16* %a, i64 %iv2 1628 %load2 = load i16, i16* %gep2, align 4 1629 %conv2 = sext i16 %load2 to i32 1630 %mul01 = mul nsw i32 %conv, %conv1 1631 %mul012 = mul nsw i32 %mul01, %conv2 1632 %arrayidx5 = getelementptr inbounds i32, i32* %b, i64 %i 1633 store i32 %mul012, i32* %arrayidx5 1634 %exitcond = icmp eq i64 %iv, %n 1635 br i1 %exitcond, label %end, label %loop 1636 1637end: 1638 ret void 1639} 1640 1641attributes #0 = { "unsafe-fp-math"="true" } 1642