1; RUN: opt < %s -loop-vectorize -mtriple aarch64-unknown-linux-gnu -enable-strict-reductions -S | FileCheck %s -check-prefix=CHECK 2 3define float @fadd_strict(float* noalias nocapture readonly %a, i64 %n) { 4; CHECK-LABEL: @fadd_strict 5; CHECK: vector.body: 6; CHECK: %[[VEC_PHI:.*]] = phi float [ 0.000000e+00, %vector.ph ], [ %[[RDX:.*]], %vector.body ] 7; CHECK: %[[LOAD:.*]] = load <8 x float>, <8 x float>* 8; CHECK: %[[RDX]] = call float @llvm.vector.reduce.fadd.v8f32(float %[[VEC_PHI]], <8 x float> %[[LOAD]]) 9; CHECK: for.end 10; CHECK: %[[PHI:.*]] = phi float [ %[[SCALAR:.*]], %for.body ], [ %[[RDX]], %middle.block ] 11; CHECK: ret float %[[PHI]] 12entry: 13 br label %for.body 14 15for.body: 16 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] 17 %sum.07 = phi float [ 0.000000e+00, %entry ], [ %add, %for.body ] 18 %arrayidx = getelementptr inbounds float, float* %a, i64 %iv 19 %0 = load float, float* %arrayidx, align 4 20 %add = fadd float %0, %sum.07 21 %iv.next = add nuw nsw i64 %iv, 1 22 %exitcond.not = icmp eq i64 %iv.next, %n 23 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0 24 25for.end: 26 ret float %add 27} 28 29define float @fadd_strict_unroll(float* noalias nocapture readonly %a, i64 %n) { 30; CHECK-LABEL: @fadd_strict_unroll 31; CHECK: vector.body: 32; CHECK: %[[VEC_PHI1:.*]] = phi float [ 0.000000e+00, %vector.ph ], [ %[[RDX4:.*]], %vector.body ] 33; CHECK-NOT: phi float [ 0.000000e+00, %vector.ph ], [ %[[RDX4]], %vector.body ] 34; CHECK: %[[LOAD1:.*]] = load <8 x float>, <8 x float>* 35; CHECK: %[[LOAD2:.*]] = load <8 x float>, <8 x float>* 36; CHECK: %[[LOAD3:.*]] = load <8 x float>, <8 x float>* 37; CHECK: %[[LOAD4:.*]] = load <8 x float>, <8 x float>* 38; CHECK: %[[RDX1:.*]] = call float @llvm.vector.reduce.fadd.v8f32(float %[[VEC_PHI1]], <8 x float> %[[LOAD1]]) 39; CHECK: %[[RDX2:.*]] = call float @llvm.vector.reduce.fadd.v8f32(float %[[RDX1]], <8 x float> %[[LOAD2]]) 40; CHECK: %[[RDX3:.*]] = call float @llvm.vector.reduce.fadd.v8f32(float %[[RDX2]], <8 x float> %[[LOAD3]]) 41; CHECK: %[[RDX4]] = call float @llvm.vector.reduce.fadd.v8f32(float %[[RDX3]], <8 x float> %[[LOAD4]]) 42; CHECK: for.end 43; CHECK: %[[PHI:.*]] = phi float [ %[[SCALAR:.*]], %for.body ], [ %[[RDX4]], %middle.block ] 44; CHECK: ret float %[[PHI]] 45entry: 46 br label %for.body 47 48for.body: 49 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] 50 %sum.07 = phi float [ 0.000000e+00, %entry ], [ %add, %for.body ] 51 %arrayidx = getelementptr inbounds float, float* %a, i64 %iv 52 %0 = load float, float* %arrayidx, align 4 53 %add = fadd float %0, %sum.07 54 %iv.next = add nuw nsw i64 %iv, 1 55 %exitcond.not = icmp eq i64 %iv.next, %n 56 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !1 57 58for.end: 59 ret float %add 60} 61 62; An additional test for unrolling where we need the last value of the reduction, i.e: 63; float sum = 0, sum2; 64; for(int i=0; i<N; ++i) { 65; sum += ptr[i]; 66; *ptr2 = sum + 42; 67; } 68; return sum; 69 70define float @fadd_strict_unroll_last_val(float* noalias nocapture readonly %a, float* noalias nocapture readonly %b, i64 %n) { 71; CHECK-LABEL: @fadd_strict_unroll_last_val 72; CHECK: vector.body 73; CHECK: %[[VEC_PHI1:.*]] = phi float [ 0.000000e+00, %vector.ph ], [ %[[RDX4:.*]], %vector.body ] 74; CHECK-NOT: phi float [ 0.000000e+00, %vector.ph ], [ %[[RDX4]], %vector.body ] 75; CHECK: %[[LOAD1:.*]] = load <8 x float>, <8 x float>* 76; CHECK: %[[LOAD2:.*]] = load <8 x float>, <8 x float>* 77; CHECK: %[[LOAD3:.*]] = load <8 x float>, <8 x float>* 78; CHECK: %[[LOAD4:.*]] = load <8 x float>, <8 x float>* 79; CHECK: %[[RDX1:.*]] = call float @llvm.vector.reduce.fadd.v8f32(float %[[VEC_PHI1]], <8 x float> %[[LOAD1]]) 80; CHECK: %[[RDX2:.*]] = call float @llvm.vector.reduce.fadd.v8f32(float %[[RDX1]], <8 x float> %[[LOAD2]]) 81; CHECK: %[[RDX3:.*]] = call float @llvm.vector.reduce.fadd.v8f32(float %[[RDX2]], <8 x float> %[[LOAD3]]) 82; CHECK: %[[RDX4]] = call float @llvm.vector.reduce.fadd.v8f32(float %[[RDX3]], <8 x float> %[[LOAD4]]) 83; CHECK: for.body 84; CHECK: %[[SUM_PHI:.*]] = phi float [ %[[FADD:.*]], %for.body ], [ {{.*}}, %scalar.ph ] 85; CHECK: %[[LOAD5:.*]] = load float, float* 86; CHECK: %[[FADD]] = fadd float %[[SUM_PHI]], %[[LOAD5]] 87; CHECK: for.cond.cleanup 88; CHECK: %[[FADD_LCSSA:.*]] = phi float [ %[[FADD]], %for.body ], [ %[[RDX4]], %middle.block ] 89; CHECK: %[[FADD_42:.*]] = fadd float %[[FADD_LCSSA]], 4.200000e+01 90; CHECK: store float %[[FADD_42]], float* %b 91; CHECK: for.end 92; CHECK: %[[SUM_LCSSA:.*]] = phi float [ %[[FADD_LCSSA]], %for.cond.cleanup ], [ 0.000000e+00, %entry ] 93; CHECK: ret float %[[SUM_LCSSA]] 94entry: 95 %cmp = icmp sgt i64 %n, 0 96 br i1 %cmp, label %for.body, label %for.end 97 98for.body: 99 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] 100 %sum = phi float [ 0.000000e+00, %entry ], [ %fadd, %for.body ] 101 %arrayidx = getelementptr inbounds float, float* %a, i64 %iv 102 %0 = load float, float* %arrayidx, align 4 103 %fadd = fadd float %sum, %0 104 %iv.next = add nuw nsw i64 %iv, 1 105 %exitcond.not = icmp eq i64 %iv.next, %n 106 br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !1 107 108for.cond.cleanup: 109 %fadd.lcssa = phi float [ %fadd, %for.body ] 110 %fadd2 = fadd float %fadd.lcssa, 4.200000e+01 111 store float %fadd2, float* %b, align 4 112 br label %for.end 113 114for.end: 115 %sum.lcssa = phi float [ %fadd.lcssa, %for.cond.cleanup ], [ 0.000000e+00, %entry ] 116 ret float %sum.lcssa 117} 118 119define void @fadd_strict_interleave(float* noalias nocapture readonly %a, float* noalias nocapture readonly %b, i64 %n) { 120; CHECK-LABEL: @fadd_strict_interleave 121; CHECK: entry 122; CHECK: %[[ARRAYIDX:.*]] = getelementptr inbounds float, float* %a, i64 1 123; CHECK: %[[LOAD1:.*]] = load float, float* %a 124; CHECK: %[[LOAD2:.*]] = load float, float* %[[ARRAYIDX]] 125; CHECK: vector.body 126; CHECK: %[[VEC_PHI1:.*]] = phi float [ %[[LOAD2]], %vector.ph ], [ %[[RDX2:.*]], %vector.body ] 127; CHECK: %[[VEC_PHI2:.*]] = phi float [ %[[LOAD1]], %vector.ph ], [ %[[RDX1:.*]], %vector.body ] 128; CHECK: %[[WIDE_LOAD:.*]] = load <8 x float>, <8 x float>* 129; CHECK: %[[STRIDED1:.*]] = shufflevector <8 x float> %[[WIDE_LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6> 130; CHECK: %[[STRIDED2:.*]] = shufflevector <8 x float> %[[WIDE_LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7> 131; CHECK: %[[RDX1]] = call float @llvm.vector.reduce.fadd.v4f32(float %[[VEC_PHI2]], <4 x float> %[[STRIDED1]]) 132; CHECK: %[[RDX2]] = call float @llvm.vector.reduce.fadd.v4f32(float %[[VEC_PHI1]], <4 x float> %[[STRIDED2]]) 133; CHECK: for.end 134; CHECK ret void 135entry: 136 %arrayidxa = getelementptr inbounds float, float* %a, i64 1 137 %a1 = load float, float* %a, align 4 138 %a2 = load float, float* %arrayidxa, align 4 139 br label %for.body 140 141for.body: 142 %add.phi1 = phi float [ %a2, %entry ], [ %add2, %for.body ] 143 %add.phi2 = phi float [ %a1, %entry ], [ %add1, %for.body ] 144 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] 145 %arrayidxb1 = getelementptr inbounds float, float* %b, i64 %iv 146 %0 = load float, float* %arrayidxb1, align 4 147 %add1 = fadd float %0, %add.phi2 148 %or = or i64 %iv, 1 149 %arrayidxb2 = getelementptr inbounds float, float* %b, i64 %or 150 %1 = load float, float* %arrayidxb2, align 4 151 %add2 = fadd float %1, %add.phi1 152 %iv.next = add nuw nsw i64 %iv, 2 153 %exitcond.not = icmp eq i64 %iv.next, %n 154 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !2 155 156for.end: 157 store float %add1, float* %a, align 4 158 store float %add2, float* %arrayidxa, align 4 159 ret void 160} 161 162define float @fadd_invariant(float* noalias nocapture readonly %a, float* noalias nocapture readonly %b, i64 %n) { 163; CHECK-LABEL: @fadd_invariant 164; CHECK: vector.body 165; CHECK: %[[VEC_PHI1:.*]] = phi float [ 0.000000e+00, %vector.ph ], [ %[[RDX:.*]], %vector.body ] 166; CHECK: %[[LOAD1:.*]] = load <4 x float>, <4 x float>* 167; CHECK: %[[LOAD2:.*]] = load <4 x float>, <4 x float>* 168; CHECK: %[[ADD:.*]] = fadd <4 x float> %[[LOAD1]], %[[LOAD2]] 169; CHECK: %[[RDX]] = call float @llvm.vector.reduce.fadd.v4f32(float %[[VEC_PHI1]], <4 x float> %[[ADD]]) 170; CHECK: for.end.loopexit 171; CHECK: %[[EXIT_PHI:.*]] = phi float [ %[[SCALAR:.*]], %for.body ], [ %[[RDX]], %middle.block ] 172; CHECK: for.end 173; CHECK: %[[PHI:.*]] = phi float [ 0.000000e+00, %entry ], [ %[[EXIT_PHI]], %for.end.loopexit ] 174; CHECK: ret float %[[PHI]] 175entry: 176 %arrayidx = getelementptr inbounds float, float* %a, i64 1 177 %0 = load float, float* %arrayidx, align 4 178 %cmp1 = fcmp ogt float %0, 5.000000e-01 179 br i1 %cmp1, label %for.body, label %for.end 180 181for.body: ; preds = %for.body 182 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] 183 %res.014 = phi float [ 0.000000e+00, %entry ], [ %rdx, %for.body ] 184 %arrayidx2 = getelementptr inbounds float, float* %a, i64 %iv 185 %1 = load float, float* %arrayidx2, align 4 186 %arrayidx4 = getelementptr inbounds float, float* %b, i64 %iv 187 %2 = load float, float* %arrayidx4, align 4 188 %add = fadd float %1, %2 189 %rdx = fadd float %res.014, %add 190 %iv.next = add nuw nsw i64 %iv, 1 191 %exitcond.not = icmp eq i64 %iv.next, %n 192 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !2 193 194for.end: ; preds = %for.body, %entry 195 %res = phi float [ 0.000000e+00, %entry ], [ %rdx, %for.body ] 196 ret float %res 197} 198 199define float @fadd_conditional(float* noalias nocapture readonly %a, float* noalias nocapture readonly %b, i64 %n) { 200; CHECK-LABEL: @fadd_conditional 201; CHECK: vector.body: 202; CHECK: %[[PHI:.*]] = phi float [ 1.000000e+00, %vector.ph ], [ %[[RDX:.*]], %pred.load.continue6 ] 203; CHECK: %[[LOAD1:.*]] = load <4 x float>, <4 x float>* 204; CHECK: %[[FCMP1:.*]] = fcmp une <4 x float> %[[LOAD1]], zeroinitializer 205; CHECK: %[[EXTRACT:.*]] = extractelement <4 x i1> %[[FCMP1]], i32 0 206; CHECK: br i1 %[[EXTRACT]], label %pred.load.if, label %pred.load.continue 207; CHECK: pred.load.continue6 208; CHECK: %[[PHI1:.*]] = phi <4 x float> [ %[[PHI0:.*]], %pred.load.continue4 ], [ %[[INS_ELT:.*]], %pred.load.if5 ] 209; CHECK: %[[XOR:.*]] = xor <4 x i1> %[[FCMP1]], <i1 true, i1 true, i1 true, i1 true> 210; CHECK: %[[PRED:.*]] = select <4 x i1> %[[XOR]], <4 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>, <4 x float> %[[PHI1]] 211; CHECK: %[[RDX]] = call float @llvm.vector.reduce.fadd.v4f32(float %[[PHI]], <4 x float> %[[PRED]]) 212; CHECK: for.body 213; CHECK: %[[RES_PHI:.*]] = phi float [ %[[MERGE_RDX:.*]], %scalar.ph ], [ %[[FADD:.*]], %for.inc ] 214; CHECK: %[[LOAD2:.*]] = load float, float* 215; CHECK: %[[FCMP2:.*]] = fcmp une float %[[LOAD2]], 0.000000e+00 216; CHECK: br i1 %[[FCMP2]], label %if.then, label %for.inc 217; CHECK: if.then 218; CHECK: %[[LOAD3:.*]] = load float, float* 219; CHECK: br label %for.inc 220; CHECK: for.inc 221; CHECK: %[[PHI2:.*]] = phi float [ %[[LOAD3]], %if.then ], [ 3.000000e+00, %for.body ] 222; CHECK: %[[FADD]] = fadd float %[[RES_PHI]], %[[PHI2]] 223; CHECK: for.end 224; CHECK: %[[RDX_PHI:.*]] = phi float [ %[[FADD]], %for.inc ], [ %[[RDX]], %middle.block ] 225; CHECK: ret float %[[RDX_PHI]] 226entry: 227 br label %for.body 228 229for.body: ; preds = %for.body 230 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] 231 %res = phi float [ 1.000000e+00, %entry ], [ %fadd, %for.inc ] 232 %arrayidx = getelementptr inbounds float, float* %b, i64 %iv 233 %0 = load float, float* %arrayidx, align 4 234 %tobool = fcmp une float %0, 0.000000e+00 235 br i1 %tobool, label %if.then, label %for.inc 236 237if.then: ; preds = %for.body 238 %arrayidx2 = getelementptr inbounds float, float* %a, i64 %iv 239 %1 = load float, float* %arrayidx2, align 4 240 br label %for.inc 241 242for.inc: 243 %phi = phi float [ %1, %if.then ], [ 3.000000e+00, %for.body ] 244 %fadd = fadd float %res, %phi 245 %iv.next = add nuw nsw i64 %iv, 1 246 %exitcond.not = icmp eq i64 %iv.next, %n 247 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !2 248 249for.end: 250 %rdx = phi float [ %fadd, %for.inc ] 251 ret float %rdx 252} 253 254; Test to check masking correct, using the "llvm.loop.vectorize.predicate.enable" attribute 255define float @fadd_predicated(float* noalias nocapture %a, i64 %n) { 256; CHECK-LABEL: @fadd_predicated 257; CHECK: vector.ph 258; CHECK: %[[TRIP_MINUS_ONE:.*]] = sub i64 %n, 1 259; CHECK: %[[BROADCAST_INS:.*]] = insertelement <2 x i64> poison, i64 %[[TRIP_MINUS_ONE]], i32 0 260; CHECK: %[[SPLAT:.*]] = shufflevector <2 x i64> %[[BROADCAST_INS]], <2 x i64> poison, <2 x i32> zeroinitializer 261; CHECK: vector.body 262; CHECK: %[[RDX_PHI:.*]] = phi float [ 0.000000e+00, %vector.ph ], [ %[[RDX:.*]], %pred.load.continue2 ] 263; CHECK: pred.load.continue2 264; CHECK: %[[PHI:.*]] = phi <2 x float> [ %[[PHI0:.*]], %pred.load.continue ], [ %[[INS_ELT:.*]], %pred.load.if1 ] 265; CHECK: %[[MASK:.*]] = select <2 x i1> %0, <2 x float> %[[PHI]], <2 x float> <float -0.000000e+00, float -0.000000e+00> 266; CHECK: %[[RDX]] = call float @llvm.vector.reduce.fadd.v2f32(float %[[RDX_PHI]], <2 x float> %[[MASK]]) 267; CHECK: for.end: 268; CHECK: %[[RES_PHI:.*]] = phi float [ %[[FADD:.*]], %for.body ], [ %[[RDX]], %middle.block ] 269; CHECK: ret float %[[RES_PHI]] 270entry: 271 br label %for.body 272 273for.body: ; preds = %entry, %for.body 274 %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ] 275 %sum.02 = phi float [ %l7, %for.body ], [ 0.000000e+00, %entry ] 276 %l2 = getelementptr inbounds float, float* %a, i64 %iv 277 %l3 = load float, float* %l2, align 4 278 %l7 = fadd float %sum.02, %l3 279 %iv.next = add i64 %iv, 1 280 %exitcond = icmp eq i64 %iv.next, %n 281 br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !3 282 283for.end: ; preds = %for.body 284 %sum.0.lcssa = phi float [ %l7, %for.body ] 285 ret float %sum.0.lcssa 286} 287 288; Negative test - loop contains multiple fadds which we cannot safely reorder 289define float @fadd_multiple(float* noalias nocapture %a, float* noalias nocapture %b, i64 %n) { 290; CHECK-LABEL: @fadd_multiple 291; CHECK: vector.body 292; CHECK: %[[PHI:.*]] = phi <8 x float> [ <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %vector.ph ], [ %[[VEC_FADD2:.*]], %vector.body ] 293; CHECK: %[[VEC_LOAD1:.*]] = load <8 x float>, <8 x float> 294; CHECK: %[[VEC_FADD1:.*]] = fadd <8 x float> %[[PHI]], %[[VEC_LOAD1]] 295; CHECK: %[[VEC_LOAD2:.*]] = load <8 x float>, <8 x float> 296; CHECK: %[[VEC_FADD2]] = fadd <8 x float> %[[VEC_FADD1]], %[[VEC_LOAD2]] 297; CHECK: middle.block 298; CHECK: %[[RDX:.*]] = call float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> %[[VEC_FADD2]]) 299; CHECK: for.body 300; CHECK: %[[SUM:.*]] = phi float [ %bc.merge.rdx, %scalar.ph ], [ %[[FADD2:.*]], %for.body ] 301; CHECK: %[[LOAD1:.*]] = load float, float* 302; CHECK: %[[FADD1:.*]] = fadd float %sum, %[[LOAD1]] 303; CHECK: %[[LOAD2:.*]] = load float, float* 304; CHECK: %[[FADD2]] = fadd float %[[FADD1]], %[[LOAD2]] 305; CHECK: for.end 306; CHECK: %[[RET:.*]] = phi float [ %[[FADD2]], %for.body ], [ %[[RDX]], %middle.block ] 307; CHECK: ret float %[[RET]] 308entry: 309 br label %for.body 310 311for.body: ; preds = %entry, %for.body 312 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] 313 %sum = phi float [ -0.000000e+00, %entry ], [ %add3, %for.body ] 314 %arrayidx = getelementptr inbounds float, float* %a, i64 %iv 315 %0 = load float, float* %arrayidx, align 4 316 %add = fadd float %sum, %0 317 %arrayidx2 = getelementptr inbounds float, float* %b, i64 %iv 318 %1 = load float, float* %arrayidx2, align 4 319 %add3 = fadd float %add, %1 320 %iv.next = add nuw nsw i64 %iv, 1 321 %exitcond.not = icmp eq i64 %iv.next, %n 322 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0 323 324for.end: ; preds = %for.body 325 %rdx = phi float [ %add3, %for.body ] 326 ret float %rdx 327} 328 329!0 = distinct !{!0, !4, !7, !9} 330!1 = distinct !{!1, !4, !8, !9} 331!2 = distinct !{!2, !5, !7, !9} 332!3 = distinct !{!3, !6, !7, !9, !10} 333!4 = !{!"llvm.loop.vectorize.width", i32 8} 334!5 = !{!"llvm.loop.vectorize.width", i32 4} 335!6 = !{!"llvm.loop.vectorize.width", i32 2} 336!7 = !{!"llvm.loop.interleave.count", i32 1} 337!8 = !{!"llvm.loop.interleave.count", i32 4} 338!9 = !{!"llvm.loop.vectorize.enable", i1 true} 339!10 = !{!"llvm.loop.vectorize.predicate.enable", i1 true} 340