1; REQUIRES: asserts 2 3; RUN: opt -passes=loop-vectorize -force-vector-width=1 -force-vector-interleave=2 -debug -disable-output %s 2>&1 | FileCheck --check-prefix=DBG %s 4; RUN: opt -passes=loop-vectorize -force-vector-width=1 -force-vector-interleave=2 -S %s | FileCheck %s 5 6; DBG-LABEL: 'test_scalarize_call' 7; DBG: VPlan 'Initial VPlan for VF={1},UF>=1' { 8; DBG-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count 9; DBG-EMPTY: 10; DBG-NEXT: vector.ph: 11; DBG-NEXT: Successor(s): vector loop 12; DBG-EMPTY: 13; DBG-NEXT: <x1> vector loop: { 14; DBG-NEXT: vector.body: 15; DBG-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION 16; DBG-NEXT: vp<[[IV_STEPS:%.]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<%start>, ir<1> 17; DBG-NEXT: CLONE ir<%min> = call @llvm.smin.i32(vp<[[IV_STEPS]]>, ir<65535>) 18; DBG-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%dst>, vp<[[IV_STEPS]]> 19; DBG-NEXT: CLONE store ir<%min>, ir<%arrayidx> 20; DBG-NEXT: EMIT vp<[[INC:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> 21; DBG-NEXT: EMIT branch-on-count vp<[[INC]]> vp<[[VEC_TC]]> 22; DBG-NEXT: No successors 23; DBG-NEXT: } 24; 25define void @test_scalarize_call(i32 %start, ptr %dst) { 26; CHECK-LABEL: @test_scalarize_call( 27; CHECK: vector.body: 28; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ] 29; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i32 %start, [[INDEX]] 30; CHECK-NEXT: [[INDUCTION:%.*]] = add i32 [[OFFSET_IDX]], 0 31; CHECK-NEXT: [[INDUCTION1:%.*]] = add i32 [[OFFSET_IDX]], 1 32; CHECK-NEXT: [[TMP1:%.*]] = tail call i32 @llvm.smin.i32(i32 [[INDUCTION]], i32 65535) 33; CHECK-NEXT: [[TMP2:%.*]] = tail call i32 @llvm.smin.i32(i32 [[INDUCTION1]], i32 65535) 34; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i32 [[INDUCTION]] 35; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[INDUCTION1]] 36; CHECK-NEXT: store i32 [[TMP1]], ptr [[TMP3]], align 8 37; CHECK-NEXT: store i32 [[TMP2]], ptr [[TMP4]], align 8 38; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 39; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], %n.vec 40; CHECK-NEXT: br i1 [[TMP5]], label %middle.block, label %vector.body 41; CHECK: middle.block: 42; 43entry: 44 br label %loop 45 46loop: 47 %iv = phi i32 [ %start, %entry ], [ %iv.next, %loop ] 48 %min = tail call i32 @llvm.smin.i32(i32 %iv, i32 65535) 49 %arrayidx = getelementptr inbounds i32 , ptr %dst, i32 %iv 50 store i32 %min, ptr %arrayidx, align 8 51 %iv.next = add nsw i32 %iv, 1 52 %tobool.not = icmp eq i32 %iv.next, 1000 53 br i1 %tobool.not, label %exit, label %loop 54 55exit: 56 ret void 57} 58 59declare i32 @llvm.smin.i32(i32, i32) 60 61 62; DBG-LABEL: 'test_scalarize_with_branch_cond' 63 64; DBG: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count 65; DBG-EMPTY: 66; DBG-NEXT: vector.ph: 67; DBG-NEXT: Successor(s): vector loop 68; DBG-EMPTY: 69; DBG-NEXT: <x1> vector loop: { 70; DBG-NEXT: vector.body: 71; DBG-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION 72; DBG-NEXT: vp<[[STEPS1:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<false>, ir<true> 73; DBG-NEXT: vp<[[STEPS2:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1> 74; DBG-NEXT: Successor(s): cond.false 75; DBG-EMPTY: 76; DBG-NEXT: cond.false: 77; DBG-NEXT: CLONE ir<%gep.src> = getelementptr ir<%src>, vp<[[STEPS2]]> 78; DBG-NEXT: CLONE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS2]]> 79; DBG-NEXT: Successor(s): cond.false.0 80; DBG-EMPTY: 81; DBG-NEXT: cond.false.0: 82; DBG-NEXT: Successor(s): pred.store 83; DBG-EMPTY: 84; DBG-NEXT: <xVFxUF> pred.store: { 85; DBG-NEXT: pred.store.entry: 86; DBG-NEXT: BRANCH-ON-MASK vp<[[STEPS1]]> 87; DBG-NEXT: Successor(s): pred.store.if, pred.store.continue 88; DBG-EMPTY: 89; DBG-NEXT: pred.store.if: 90; DBG-NEXT: CLONE ir<%l> = load ir<%gep.src> 91; DBG-NEXT: CLONE store ir<%l>, ir<%gep.dst> 92; DBG-NEXT: Successor(s): pred.store.continue 93; DBG-EMPTY: 94; DBG-NEXT: pred.store.continue: 95; DBG-NEXT: PHI-PREDICATED-INSTRUCTION vp<{{.+}}> = ir<%l> 96; DBG-NEXT: No successors 97; DBG-NEXT: } 98; DBG-NEXT: Successor(s): cond.false.1 99; DBG-EMPTY: 100; DBG-NEXT: cond.false.1: 101; DBG-NEXT: Successor(s): loop.latch 102; DBG-EMPTY: 103; DBG-NEXT: loop.latch: 104; DBG-NEXT: EMIT vp<[[CAN_IV_INC:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> 105; DBG-NEXT: EMIT branch-on-count vp<[[CAN_IV_INC]]> vp<[[VEC_TC]]> 106; DBG-NEXT: No successors 107; DBG-NEXT: } 108; DBG-NEXT: Successor(s): middle.block 109; DBG-EMPTY: 110; DBG-NEXT: middle.block: 111; DBG-NEXT: No successors 112; DBG-NEXT: } 113 114define void @test_scalarize_with_branch_cond(ptr %src, ptr %dst) { 115; CHECK-LABEL: @test_scalarize_with_branch_cond( 116; CHECK: vector.body: 117; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %pred.store.continue8 ] 118; CHECK-NEXT: [[TMP0:%.*]] = trunc i64 [[INDEX]] to i1 119; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i1 false, [[TMP0]] 120; CHECK-NEXT: [[INDUCTION:%.*]] = add i1 [[OFFSET_IDX]], false 121; CHECK-NEXT: [[INDUCTION3:%.*]] = add i1 [[OFFSET_IDX]], true 122; CHECK-NEXT: br i1 [[INDUCTION]], label %pred.store.if, label %pred.store.continue 123; CHECK: pred.store.if: 124; CHECK-NEXT: [[INDUCTION4:%.*]] = add i64 [[INDEX]], 0 125; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr %dst, i64 [[INDUCTION4]] 126; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr %src, i64 [[INDUCTION4]] 127; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4 128; CHECK-NEXT: store i32 [[TMP4]], ptr [[TMP1]], align 4 129; CHECK-NEXT: br label %pred.store.continue 130; CHECK: pred.store.continue: 131; CHECK-NEXT: [[TMP5:%.*]] = phi i32 [ poison, %vector.body ], [ [[TMP4]], %pred.store.if ] 132; CHECK-NEXT: br i1 [[INDUCTION3]], label %pred.store.if7, label %pred.store.continue8 133; CHECK: pred.store.if7: 134; CHECK-NEXT: [[INDUCTION5:%.*]] = add i64 [[INDEX]], 1 135; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr %dst, i64 [[INDUCTION5]] 136; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr %src, i64 [[INDUCTION5]] 137; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4 138; CHECK-NEXT: store i32 [[TMP7]], ptr [[TMP2]], align 4 139; CHECK-NEXT: br label %pred.store.continue8 140; CHECK: pred.store.continue8: 141; CHECK-NEXT: [[TMP8:%.*]] = phi i32 [ poison, %pred.store.continue ], [ [[TMP7]], %pred.store.if7 ] 142; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 143; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000 144; CHECK-NEXT: br i1 [[TMP9]], label %middle.block, label %vector.body 145; CHECK: middle.block: 146; 147entry: 148 br label %loop.header 149 150loop.header: 151 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] 152 %d = phi i1 [ false, %entry ], [ %d.next, %loop.latch ] 153 %d.next = xor i1 %d, true 154 br i1 %d, label %cond.false, label %loop.latch 155 156cond.false: 157 %gep.src = getelementptr inbounds i32, ptr %src, i64 %iv 158 %gep.dst = getelementptr inbounds i32, ptr %dst, i64 %iv 159 %l = load i32, ptr %gep.src, align 4 160 store i32 %l, ptr %gep.dst 161 br label %loop.latch 162 163loop.latch: 164 %iv.next = add nsw i64 %iv, 1 165 %ec = icmp eq i64 %iv.next, 1000 166 br i1 %ec, label %exit, label %loop.header 167 168exit: 169 ret void 170} 171 172; Make sure the widened induction gets replaced by scalar-steps for plans 173; including the scalar VF, if it is used in first-order recurrences. 174 175; DBG-LABEL: 'first_order_recurrence_using_induction' 176; DBG: VPlan 'Initial VPlan for VF={1},UF>=1' { 177; DBG-NEXT: Live-in vp<%1> = vector-trip-count 178; DBG-EMPTY: 179; DBG-NEXT: vector.ph: 180; DBG-NEXT: Successor(s): vector loop 181; DBG-EMPTY: 182; DBG-NEXT: <x1> vector loop: { 183; DBG-NEXT: vector.body: 184; DBG-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION 185; DBG-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for> = phi ir<0>, vp<%4> 186; DBG-NEXT: vp<%4> = SCALAR-STEPS vp<%2>, ir<0>, ir<1> 187; DBG-NEXT: EMIT vp<%5> = first-order splice ir<%for> vp<%4> 188; DBG-NEXT: CLONE store vp<%5>, ir<%dst> 189; DBG-NEXT: EMIT vp<%7> = VF * UF +(nuw) vp<%2> 190; DBG-NEXT: EMIT branch-on-count vp<%7> vp<%1> 191; DBG-NEXT: No successors 192; DBG-NEXT: } 193; DBG-NEXT: Successor(s): middle.block 194; DBG-EMPTY: 195; DBG-NEXT: middle.block: 196; DBG-NEXT: No successors 197; DBG-NEXT: } 198 199define void @first_order_recurrence_using_induction(i32 %n, ptr %dst) { 200; CHECK-LABEL: @first_order_recurrence_using_induction( 201; CHECK: vector.body: 202; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ] 203; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi i32 [ 0, %vector.ph ], [ [[INDUCTION1:%.*]], %vector.body ] 204; CHECK-NEXT: [[TMP3:%.*]] = trunc i64 [[INDEX]] to i32 205; CHECK-NEXT: [[INDUCTION:%.*]] = add i32 [[TMP3]], 0 206; CHECK-NEXT: [[INDUCTION1]] = add i32 [[TMP3]], 1 207; CHECK-NEXT: store i32 [[VECTOR_RECUR]], ptr [[DST:%.*]], align 4 208; CHECK-NEXT: store i32 [[INDUCTION]], ptr [[DST]], align 4 209; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 210; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], %n.vec 211; CHECK-NEXT: br i1 [[TMP4]], label %middle.block, label %vector.body 212; CHECK: middle.block: 213; 214entry: 215 br label %loop 216 217loop: 218 %iv = phi i64 [ 0, %entry ],[ %iv.next, %loop ] 219 %for = phi i32 [ 0, %entry ], [ %iv.trunc, %loop ] 220 %iv.trunc = trunc i64 %iv to i32 221 store i32 %for, ptr %dst 222 %iv.next = add nuw nsw i64 %iv, 1 223 %iv.next.trunc = trunc i64 %iv.next to i32 224 %ec = icmp slt i32 %iv.next.trunc, %n 225 br i1 %ec, label %loop, label %exit 226 227exit: 228 ret void 229} 230