1; REQUIRES: asserts 2; RUN: opt < %s -loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -debug-only=loop-vectorize 2>&1 | FileCheck %s 3 4target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" 5 6; Test cases for PR50009, which require sinking a replicate-region due to a 7; first-order recurrence. 8 9define void @sink_replicate_region_1(i32 %x, i8* %ptr) optsize { 10; CHECK-LABEL: sink_replicate_region_1 11; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { 12; CHECK-NEXT: loop: 13; CHECK-NEXT: WIDEN-PHI %0 = phi 0, %conv 14; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next 15; CHECK-NEXT: EMIT vp<%3> = icmp ule ir<%iv> vp<%0> 16; CHECK-NEXT: Successor(s): loop.0 17 18; CHECK: loop.0: 19; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, ir<%iv> 20; CHECK-NEXT: Successor(s): pred.load 21 22; CHECK: <xVFxUF> pred.load: { 23; CHECK-NEXT: pred.load.entry: 24; CHECK-NEXT: BRANCH-ON-MASK vp<%3> 25; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue 26; CHECK-NEXT: CondBit: vp<%3> (loop) 27 28; CHECK: pred.load.if: 29; CHECK-NEXT: REPLICATE ir<%lv> = load ir<%gep> (S->V) 30; CHECK-NEXT: Successor(s): pred.load.continue 31 32; CHECK: pred.load.continue: 33; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%6> = ir<%lv> 34; CHECK-NEXT: No successors 35; CHECK-NEXT: } 36 37; CHECK: loop.1: 38; CHECK-NEXT: WIDEN ir<%conv> = sext vp<%6> 39; CHECK-NEXT: Successor(s): pred.srem 40 41; CHECK: <xVFxUF> pred.srem: { 42; CHECK-NEXT: pred.srem.entry: 43; CHECK-NEXT: BRANCH-ON-MASK vp<%3> 44; CHECK-NEXT: Successor(s): pred.srem.if, pred.srem.continue 45; CHECK-NEXT: CondBit: vp<%3> (loop) 46 47; CHECK: pred.srem.if: 48; CHECK-NEXT: REPLICATE ir<%rem> = srem ir<%0>, ir<%x> (S->V) 49; CHECK-NEXT: Successor(s): pred.srem.continue 50 51; CHECK: pred.srem.continue: 52; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%9> = ir<%rem> 53; CHECK-NEXT: No successors 54; CHECK-NEXT: } 55 56; CHECK: loop.1.split: 57; CHECK-NEXT: WIDEN ir<%add> = add ir<%conv>, vp<%9> 58; CHECK-NEXT: No successors 59; CHECK-NEXT: } 60; 61entry: 62 br label %loop 63 64loop: 65 %0 = phi i32 [ 0, %entry ], [ %conv, %loop ] 66 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] 67 %rem = srem i32 %0, %x 68 %gep = getelementptr i8, i8* %ptr, i32 %iv 69 %lv = load i8, i8* %gep 70 %conv = sext i8 %lv to i32 71 %add = add i32 %conv, %rem 72 %iv.next = add nsw i32 %iv, 1 73 %ec = icmp eq i32 %iv.next, 20001 74 br i1 %ec, label %exit, label %loop 75 76exit: 77 ret void 78} 79 80define void @sink_replicate_region_2(i32 %x, i8 %y, i32* %ptr) optsize { 81; CHECK-LABEL: sink_replicate_region_2 82; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { 83; CHECK-NEXT: loop: 84; CHECK-NEXT: WIDEN-PHI %recur = phi 0, %recur.next 85; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next 86; CHECK-NEXT: EMIT vp<%3> = icmp ule ir<%iv> vp<%0> 87; CHECK-NEXT: Successor(s): loop.0 88 89; CHECK: loop.0: 90; CHECK-NEXT: WIDEN ir<%recur.next> = sext ir<%y> 91; CHECK-NEXT: Successor(s): pred.srem 92 93; CHECK: <xVFxUF> pred.srem: { 94; CHECK-NEXT: pred.srem.entry: 95; CHECK-NEXT: BRANCH-ON-MASK vp<%3> 96; CHECK-NEXT: Successor(s): pred.srem.if, pred.srem.continue 97; CHECK-NEXT: CondBit: vp<%3> (loop) 98 99; CHECK: pred.srem.if: 100; CHECK-NEXT: REPLICATE ir<%rem> = srem ir<%recur>, ir<%x> 101; CHECK-NEXT: Successor(s): pred.srem.continue 102 103; CHECK: pred.srem.continue: 104; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%6> = ir<%rem> 105; CHECK-NEXT: No successors 106; CHECK-NEXT: } 107 108; CHECK: loop.0.split: 109; CHECK-NEXT: REPLICATE ir<%add> = add vp<%6>, ir<%recur.next> 110; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, ir<%iv> 111; CHECK-NEXT: Successor(s): pred.store 112 113; CHECK: <xVFxUF> pred.store: { 114; CHECK-NEXT: pred.store.entry: 115; CHECK-NEXT: BRANCH-ON-MASK vp<%3> 116; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue 117; CHECK-NEXT: CondBit: vp<%3> (loop) 118 119; CHECK: pred.store.if: 120; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep> 121; CHECK-NEXT: Successor(s): pred.store.continue 122 123; CHECK: pred.store.continue: 124; CHECK-NEXT: No successors 125; CHECK-NEXT: } 126 127; CHECK: loop.1: 128; CHECK-NEXT: No successors 129; CHECK-NEXT: } 130; 131entry: 132 br label %loop 133 134loop: 135 %recur = phi i32 [ 0, %entry ], [ %recur.next, %loop ] 136 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] 137 %rem = srem i32 %recur, %x 138 %recur.next = sext i8 %y to i32 139 %add = add i32 %rem, %recur.next 140 %gep = getelementptr i32, i32* %ptr, i32 %iv 141 store i32 %add, i32* %gep 142 %iv.next = add nsw i32 %iv, 1 143 %ec = icmp eq i32 %iv.next, 20001 144 br i1 %ec, label %exit, label %loop 145 146exit: 147 ret void 148} 149 150define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, i32* %ptr) optsize { 151; CHECK-LABEL: sink_replicate_region_3_reduction 152; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { 153; CHECK-NEXT: loop: 154; CHECK-NEXT: WIDEN-PHI %recur = phi 0, %recur.next 155; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next 156; CHECK-NEXT: WIDEN-PHI ir<%and.red> = phi ir<1234>, ir<%and.red.next> 157; CHECK-NEXT: EMIT vp<%4> = icmp ule ir<%iv> vp<%0> 158; CHECK-NEXT: Successor(s): loop.0 159 160; CHECK: loop.0: 161; CHECK-NEXT: WIDEN ir<%recur.next> = sext ir<%y> 162; CHECK-NEXT: Successor(s): pred.srem 163 164; CHECK: <xVFxUF> pred.srem: { 165; CHECK-NEXT: pred.srem.entry: 166; CHECK-NEXT: BRANCH-ON-MASK vp<%4> 167; CHECK-NEXT: Successor(s): pred.srem.if, pred.srem.continue 168; CHECK-NEXT: CondBit: vp<%4> (loop) 169 170; CHECK: pred.srem.if: 171; CHECK-NEXT: REPLICATE ir<%rem> = srem ir<%recur>, ir<%x> (S->V) 172; CHECK-NEXT: Successor(s): pred.srem.continue 173 174; CHECK: pred.srem.continue: 175; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%7> = ir<%rem> 176; CHECK-NEXT: No successors 177; CHECK-NEXT: } 178 179; CHECK: loop.0.split: 180; CHECK-NEXT: WIDEN ir<%add> = add vp<%7>, ir<%recur.next> 181; CHECK-NEXT: WIDEN ir<%and.red.next> = and ir<%and.red>, ir<%add> 182; CHECK-NEXT: EMIT vp<%10> = select vp<%4> ir<%and.red.next> ir<%and.red> 183; CHECK-NEXT: No successors 184; CHECK-NEXT: } 185; 186entry: 187 br label %loop 188 189loop: 190 %recur = phi i32 [ 0, %entry ], [ %recur.next, %loop ] 191 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] 192 %and.red = phi i32 [ 1234, %entry ], [ %and.red.next, %loop ] 193 %rem = srem i32 %recur, %x 194 %recur.next = sext i8 %y to i32 195 %add = add i32 %rem, %recur.next 196 %and.red.next = and i32 %and.red, %add 197 %iv.next = add nsw i32 %iv, 1 198 %ec = icmp eq i32 %iv.next, 20001 199 br i1 %ec, label %exit, label %loop 200 201exit: 202 %res = phi i32 [ %and.red.next, %loop ] 203 ret i32 %res 204} 205 206; To sink the replicate region containing %rem, we need to split the block 207; containing %conv at the end, because %conv is the last recipe in the block. 208define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, i8* %ptr) optsize { 209; CHECK-LABEL: sink_replicate_region_4_requires_split_at_end_of_block 210; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { 211; CHECK-NEXT: loop: 212; CHECK-NEXT: WIDEN-PHI %0 = phi 0, %conv 213; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next 214; CHECK-NEXT: EMIT vp<%3> = icmp ule ir<%iv> vp<%0> 215; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, ir<%iv> 216; CHECK-NEXT: Successor(s): loop.0 217 218; CHECK: loop.0: 219; CHECK-NEXT: Successor(s): pred.load 220 221; CHECK: <xVFxUF> pred.load: { 222; CHECK-NEXT: pred.load.entry: 223; CHECK-NEXT: BRANCH-ON-MASK vp<%3> 224; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue 225; CHECK-NEXT: CondBit: vp<%3> (loop) 226 227; CHECK: pred.load.if: 228; CHECK-NEXT: REPLICATE ir<%lv> = load ir<%gep> (S->V) 229; CHECK-NEXT: Successor(s): pred.load.continue 230 231; CHECK: pred.load.continue: 232; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%6> = ir<%lv> 233; CHECK-NEXT: No successors 234; CHECK-NEXT: } 235 236; CHECK: loop.1: 237; CHECK-NEXT: WIDEN ir<%conv> = sext vp<%6> 238; CHECK-NEXT: Successor(s): pred.srem 239 240; CHECK: <xVFxUF> pred.srem: { 241; CHECK-NEXT: pred.srem.entry: 242; CHECK-NEXT: BRANCH-ON-MASK vp<%3> 243; CHECK-NEXT: Successor(s): pred.srem.if, pred.srem.continue 244; CHECK-NEXT: CondBit: vp<%3> (loop) 245 246; CHECK: pred.srem.if: 247; CHECK-NEXT: REPLICATE ir<%rem> = srem ir<%0>, ir<%x> (S->V) 248; CHECK-NEXT: Successor(s): pred.srem.continue 249 250; CHECK: pred.srem.continue: 251; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%9> = ir<%rem> 252; CHECK-NEXT: No successors 253; CHECK-NEXT: } 254 255; CHECK: loop.1.split: 256; CHECK-NEXT: Successor(s): pred.load 257 258; CHECK: <xVFxUF> pred.load: { 259; CHECK-NEXT: pred.load.entry: 260; CHECK-NEXT: BRANCH-ON-MASK vp<%3> 261; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue 262; CHECK-NEXT: CondBit: vp<%3> (loop) 263 264; CHECK: pred.load.if: 265; CHECK-NEXT: REPLICATE ir<%lv.2> = load ir<%gep> (S->V) 266; CHECK-NEXT: Successor(s): pred.load.continue 267 268; CHECK: pred.load.continue: 269; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%11> = ir<%lv.2> 270; CHECK-NEXT: No successors 271; CHECK-NEXT: } 272 273; CHECK: loop.2: 274; CHECK-NEXT: WIDEN ir<%add.1> = add ir<%conv>, vp<%9> 275; CHECK-NEXT: WIDEN ir<%conv.lv.2> = sext vp<%11> 276; CHECK-NEXT: WIDEN ir<%add> = add ir<%add.1>, ir<%conv.lv.2> 277; CHECK-NEXT: No successors 278; CHECK-NEXT: } 279; 280entry: 281 br label %loop 282 283loop: 284 %0 = phi i32 [ 0, %entry ], [ %conv, %loop ] 285 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] 286 %gep = getelementptr i8, i8* %ptr, i32 %iv 287 %rem = srem i32 %0, %x 288 %lv = load i8, i8* %gep 289 %conv = sext i8 %lv to i32 290 %lv.2 = load i8, i8* %gep 291 %add.1 = add i32 %conv, %rem 292 %conv.lv.2 = sext i8 %lv.2 to i32 293 %add = add i32 %add.1, %conv.lv.2 294 %iv.next = add nsw i32 %iv, 1 295 %ec = icmp eq i32 %iv.next, 20001 296 br i1 %ec, label %exit, label %loop 297 298exit: 299 ret void 300} 301