1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py 2; RUN: opt -loop-vectorize -force-vector-width=4 -enable-vplan-native-path -S %s | FileCheck %s 3 4; Vectorize explict marked outer loop using vplan native path. Inner loop 5; contains simple double add reduction. IR is compiled and modified by hand 6; from following C code: 7; void inner_loop_reduction(const double* restrict in_a, const double* restrict in_b, double* restrict out) 8; { 9; #pragma clang loop vectorize(enable) 10; for (int i = 0; i < 1000; ++i) { 11; double a = in_a[i]; 12; double b = in_b[i]; 13; for (int j = 0; j < 10000; ++j) { 14; a = a + b; 15; } 16; out[i] = a; 17; } 18; } 19define void @inner_loop_reduction(double* noalias nocapture readonly %a.in, double* noalias nocapture readonly %b.in, double* noalias nocapture %c.out) { 20; CHECK-LABEL: @inner_loop_reduction( 21; CHECK-NEXT: entry: 22; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 23; CHECK: vector.ph: 24; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 25; CHECK: vector.body: 26; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR1_LATCH4:%.*]] ] 27; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR1_LATCH4]] ] 28; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, double* [[A_IN:%.*]], <4 x i64> [[VEC_IND]] 29; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0f64(<4 x double*> [[TMP0]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x double> undef) 30; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, double* [[B_IN:%.*]], <4 x i64> [[VEC_IND]] 31; CHECK-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0f64(<4 x double*> [[TMP1]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x double> undef) 32; CHECK-NEXT: br label [[FOR2_HEADER2:%.*]] 33; CHECK: for2.header2: 34; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP3:%.*]], [[FOR2_HEADER2]] ] 35; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x double> [ [[WIDE_MASKED_GATHER]], [[VECTOR_BODY]] ], [ [[TMP2:%.*]], [[FOR2_HEADER2]] ] 36; CHECK-NEXT: [[TMP2]] = fadd <4 x double> [[WIDE_MASKED_GATHER1]], [[VEC_PHI3]] 37; CHECK-NEXT: [[TMP3]] = add nuw nsw <4 x i32> [[VEC_PHI]], <i32 1, i32 1, i32 1, i32 1> 38; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i32> [[TMP3]], <i32 10000, i32 10000, i32 10000, i32 10000> 39; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0 40; CHECK-NEXT: br i1 [[TMP5]], label [[FOR1_LATCH4]], label [[FOR2_HEADER2]] 41; CHECK: for1.latch4: 42; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x double> [ [[TMP2]], [[FOR2_HEADER2]] ] 43; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds double, double* [[C_OUT:%.*]], <4 x i64> [[VEC_IND]] 44; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0f64(<4 x double> [[VEC_PHI5]], <4 x double*> [[TMP6]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>) 45; CHECK-NEXT: [[TMP7:%.*]] = add nuw nsw <4 x i64> [[VEC_IND]], <i64 1, i64 1, i64 1, i64 1> 46; CHECK-NEXT: [[TMP8:%.*]] = icmp eq <4 x i64> [[TMP7]], <i64 1000, i64 1000, i64 1000, i64 1000> 47; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 48; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4> 49; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000 50; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] 51; CHECK: middle.block: 52; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1000, 1000 53; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] 54; CHECK: scalar.ph: 55; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 56; CHECK-NEXT: br label [[FOR1_HEADER:%.*]] 57; CHECK: for1.header: 58; CHECK-NEXT: [[INDVAR1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVAR11:%.*]], [[FOR1_LATCH:%.*]] ] 59; CHECK-NEXT: [[A_PTR:%.*]] = getelementptr inbounds double, double* [[A_IN]], i64 [[INDVAR1]] 60; CHECK-NEXT: [[A:%.*]] = load double, double* [[A_PTR]], align 8 61; CHECK-NEXT: [[B_PTR:%.*]] = getelementptr inbounds double, double* [[B_IN]], i64 [[INDVAR1]] 62; CHECK-NEXT: [[B:%.*]] = load double, double* [[B_PTR]], align 8 63; CHECK-NEXT: br label [[FOR2_HEADER:%.*]] 64; CHECK: for2.header: 65; CHECK-NEXT: [[INDVAR2:%.*]] = phi i32 [ 0, [[FOR1_HEADER]] ], [ [[INDVAR21:%.*]], [[FOR2_HEADER]] ] 66; CHECK-NEXT: [[A_REDUCTION:%.*]] = phi double [ [[A]], [[FOR1_HEADER]] ], [ [[A_REDUCTION1:%.*]], [[FOR2_HEADER]] ] 67; CHECK-NEXT: [[A_REDUCTION1]] = fadd double [[B]], [[A_REDUCTION]] 68; CHECK-NEXT: [[INDVAR21]] = add nuw nsw i32 [[INDVAR2]], 1 69; CHECK-NEXT: [[FOR2_COND:%.*]] = icmp eq i32 [[INDVAR21]], 10000 70; CHECK-NEXT: br i1 [[FOR2_COND]], label [[FOR1_LATCH]], label [[FOR2_HEADER]] 71; CHECK: for1.latch: 72; CHECK-NEXT: [[A_REDUCTION1_LCSSA:%.*]] = phi double [ [[A_REDUCTION1]], [[FOR2_HEADER]] ] 73; CHECK-NEXT: [[C_PTR:%.*]] = getelementptr inbounds double, double* [[C_OUT]], i64 [[INDVAR1]] 74; CHECK-NEXT: store double [[A_REDUCTION1_LCSSA]], double* [[C_PTR]], align 8 75; CHECK-NEXT: [[INDVAR11]] = add nuw nsw i64 [[INDVAR1]], 1 76; CHECK-NEXT: [[FOR1_COND:%.*]] = icmp eq i64 [[INDVAR11]], 1000 77; CHECK-NEXT: br i1 [[FOR1_COND]], label [[EXIT]], label [[FOR1_HEADER]], !llvm.loop [[LOOP2:![0-9]+]] 78; CHECK: exit: 79; CHECK-NEXT: ret void 80; 81 82 83 84 85entry: 86 br label %for1.header 87 88for1.header: ; preds = %entry 89 %indvar1 = phi i64 [ 0, %entry ], [ %indvar11, %for1.latch ] 90 %a.ptr = getelementptr inbounds double, double* %a.in, i64 %indvar1 91 %a = load double, double* %a.ptr, align 8 92 %b.ptr = getelementptr inbounds double, double* %b.in, i64 %indvar1 93 %b = load double, double* %b.ptr, align 8 94 br label %for2.header 95 96for2.header: ; preds = %for1.header, %for2.header 97 %indvar2 = phi i32 [ 0, %for1.header ], [ %indvar21, %for2.header ] 98 %a.reduction = phi double [ %a, %for1.header ], [ %a.reduction1, %for2.header ] 99 %a.reduction1 = fadd double %b, %a.reduction 100 %indvar21 = add nuw nsw i32 %indvar2, 1 101 %for2.cond = icmp eq i32 %indvar21, 10000 102 br i1 %for2.cond, label %for1.latch, label %for2.header 103 104for1.latch: ; preds = %for2.header 105 %c.ptr = getelementptr inbounds double, double* %c.out, i64 %indvar1 106 store double %a.reduction1, double* %c.ptr, align 8 107 %indvar11 = add nuw nsw i64 %indvar1, 1 108 %for1.cond = icmp eq i64 %indvar11, 1000 109 br i1 %for1.cond, label %exit, label %for1.header, !llvm.loop !0 110 111exit: ; preds = %for1.latch 112 ret void 113} 114 115!0 = distinct !{!0, !1} 116!1 = !{!"llvm.loop.vectorize.enable", i1 true} 117