1; REQUIRES: asserts 2; RUN: opt < %s -loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -debug-only=loop-vectorize 2>&1 | FileCheck %s 3 4target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" 5 6; Test cases for PR50009, which require sinking a replicate-region due to a 7; first-order recurrence. 8 9define void @sink_replicate_region_1(i32 %x, i8* %ptr) optsize { 10; CHECK-LABEL: sink_replicate_region_1 11; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { 12; CHECK-NEXT: loop: 13; CHECK-NEXT: WIDEN-PHI %0 = phi 0, %conv 14; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next 15; CHECK-NEXT: EMIT vp<%3> = icmp ule ir<%iv> vp<%0> 16; CHECK-NEXT: Successor(s): loop.0 17; CHECK-EMPTY: 18; CHECK-NEXT: loop.0: 19; CHECK-NEXT: Successor(s): pred.load 20; CHECK-EMPTY: 21; CHECK-NEXT: <xVFxUF> pred.load: { 22; CHECK-NEXT: pred.load.entry: 23; CHECK-NEXT: BRANCH-ON-MASK vp<%3> 24; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue 25; CHECK-NEXT: CondBit: vp<%3> (loop) 26; CHECK-EMPTY: 27; CHECK-NEXT: pred.load.if: 28; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, ir<%iv> 29; CHECK-NEXT: REPLICATE ir<%lv> = load ir<%gep> (S->V) 30; CHECK-NEXT: Successor(s): pred.load.continue 31; CHECK-EMPTY: 32; CHECK-NEXT: pred.load.continue: 33; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%6> = ir<%lv> 34; CHECK-NEXT: No successors 35; CHECK-NEXT: } 36; CHECK-NEXT: Successor(s): loop.1 37; CHECK-EMPTY: 38; CHECK-NEXT: loop.1: 39; CHECK-NEXT: WIDEN ir<%conv> = sext vp<%6> 40; CHECK-NEXT: Successor(s): pred.srem 41; CHECK-EMPTY: 42; CHECK-NEXT: <xVFxUF> pred.srem: { 43; CHECK-NEXT: pred.srem.entry: 44; CHECK-NEXT: BRANCH-ON-MASK vp<%3> 45; CHECK-NEXT: Successor(s): pred.srem.if, pred.srem.continue 46; CHECK-NEXT: CondBit: vp<%3> (loop) 47; CHECK-EMPTY: 48; CHECK-NEXT: pred.srem.if: 49; CHECK-NEXT: REPLICATE ir<%rem> = srem ir<%0>, ir<%x> (S->V) 50; CHECK-NEXT: Successor(s): pred.srem.continue 51; CHECK-EMPTY: 52; CHECK-NEXT: pred.srem.continue: 53; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%9> = ir<%rem> 54; CHECK-NEXT: No successors 55; CHECK-NEXT: } 56; CHECK-NEXT: Successor(s): loop.1.split 57; CHECK-EMPTY: 58; CHECK-NEXT: loop.1.split: 59; CHECK-NEXT: WIDEN ir<%add> = add ir<%conv>, vp<%9> 60; CHECK-NEXT: No successors 61; CHECK-NEXT: } 62; 63entry: 64 br label %loop 65 66loop: 67 %0 = phi i32 [ 0, %entry ], [ %conv, %loop ] 68 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] 69 %rem = srem i32 %0, %x 70 %gep = getelementptr i8, i8* %ptr, i32 %iv 71 %lv = load i8, i8* %gep 72 %conv = sext i8 %lv to i32 73 %add = add i32 %conv, %rem 74 %iv.next = add nsw i32 %iv, 1 75 %ec = icmp eq i32 %iv.next, 20001 76 br i1 %ec, label %exit, label %loop 77 78exit: 79 ret void 80} 81 82define void @sink_replicate_region_2(i32 %x, i8 %y, i32* %ptr) optsize { 83; CHECK-LABEL: sink_replicate_region_2 84; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { 85; CHECK-NEXT: loop: 86; CHECK-NEXT: WIDEN-PHI %recur = phi 0, %recur.next 87; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next 88; CHECK-NEXT: EMIT vp<%3> = icmp ule ir<%iv> vp<%0> 89; CHECK-NEXT: Successor(s): loop.0 90; CHECK-EMPTY: 91; CHECK-NEXT: loop.0: 92; CHECK-NEXT: WIDEN ir<%recur.next> = sext ir<%y> 93; CHECK-NEXT: Successor(s): pred.srem 94; CHECK-EMPTY: 95; CHECK-NEXT: <xVFxUF> pred.srem: { 96; CHECK-NEXT: pred.srem.entry: 97; CHECK-NEXT: BRANCH-ON-MASK vp<%3> 98; CHECK-NEXT: Successor(s): pred.srem.if, pred.srem.continue 99; CHECK-NEXT: CondBit: vp<%3> (loop) 100; CHECK-EMPTY: 101; CHECK-NEXT: pred.srem.if: 102; CHECK-NEXT: REPLICATE ir<%rem> = srem ir<%recur>, ir<%x> 103; CHECK-NEXT: Successor(s): pred.srem.continue 104; CHECK-EMPTY: 105; CHECK-NEXT: pred.srem.continue: 106; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%6> = ir<%rem> 107; CHECK-NEXT: No successors 108; CHECK-NEXT: } 109; CHECK-NEXT: Successor(s): loop.0.split 110; CHECK-EMPTY: 111; CHECK-NEXT: loop.0.split: 112; CHECK-NEXT: Successor(s): pred.store 113; CHECK-EMPTY: 114; CHECK-NEXT: <xVFxUF> pred.store: { 115; CHECK-NEXT: pred.store.entry: 116; CHECK-NEXT: BRANCH-ON-MASK vp<%3> 117; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue 118; CHECK-NEXT: CondBit: vp<%3> (loop) 119; CHECK-EMPTY: 120; CHECK-NEXT: pred.store.if: 121; CHECK-NEXT: REPLICATE ir<%add> = add vp<%6>, ir<%recur.next> 122; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, ir<%iv> 123; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep> 124; CHECK-NEXT: Successor(s): pred.store.continue 125; CHECK-EMPTY: 126; CHECK-NEXT: pred.store.continue: 127; CHECK-NEXT: No successors 128; CHECK-NEXT: } 129; CHECK-NEXT: Successor(s): loop.1 130; CHECK-EMPTY: 131; CHECK-NEXT: loop.1: 132; CHECK-NEXT: No successors 133; CHECK-NEXT: } 134; 135entry: 136 br label %loop 137 138loop: 139 %recur = phi i32 [ 0, %entry ], [ %recur.next, %loop ] 140 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] 141 %rem = srem i32 %recur, %x 142 %recur.next = sext i8 %y to i32 143 %add = add i32 %rem, %recur.next 144 %gep = getelementptr i32, i32* %ptr, i32 %iv 145 store i32 %add, i32* %gep 146 %iv.next = add nsw i32 %iv, 1 147 %ec = icmp eq i32 %iv.next, 20001 148 br i1 %ec, label %exit, label %loop 149 150exit: 151 ret void 152} 153 154define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, i32* %ptr) optsize { 155; CHECK-LABEL: sink_replicate_region_3_reduction 156; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { 157; CHECK-NEXT: loop: 158; CHECK-NEXT: WIDEN-PHI %recur = phi 0, %recur.next 159; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next 160; CHECK-NEXT: WIDEN-PHI ir<%and.red> = phi ir<1234>, ir<%and.red.next> 161; CHECK-NEXT: EMIT vp<%4> = icmp ule ir<%iv> vp<%0> 162; CHECK-NEXT: Successor(s): loop.0 163; CHECK-EMPTY: 164; CHECK-NEXT: loop.0: 165; CHECK-NEXT: WIDEN ir<%recur.next> = sext ir<%y> 166; CHECK-NEXT: Successor(s): pred.srem 167; CHECK-EMPTY: 168; CHECK-NEXT: <xVFxUF> pred.srem: { 169; CHECK-NEXT: pred.srem.entry: 170; CHECK-NEXT: BRANCH-ON-MASK vp<%4> 171; CHECK-NEXT: Successor(s): pred.srem.if, pred.srem.continue 172; CHECK-NEXT: CondBit: vp<%4> (loop) 173; CHECK-EMPTY: 174; CHECK-NEXT: pred.srem.if: 175; CHECK-NEXT: REPLICATE ir<%rem> = srem ir<%recur>, ir<%x> (S->V) 176; CHECK-NEXT: Successor(s): pred.srem.continue 177; CHECK-EMPTY: 178; CHECK-NEXT: pred.srem.continue: 179; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%7> = ir<%rem> 180; CHECK-NEXT: No successors 181; CHECK-NEXT: } 182; CHECK-NEXT: Successor(s): loop.0.split 183; CHECK-EMPTY: 184; CHECK-NEXT: loop.0.split: 185; CHECK-NEXT: WIDEN ir<%add> = add vp<%7>, ir<%recur.next> 186; CHECK-NEXT: WIDEN ir<%and.red.next> = and ir<%and.red>, ir<%add> 187; CHECK-NEXT: EMIT vp<%10> = select vp<%4> ir<%and.red.next> ir<%and.red> 188; CHECK-NEXT: No successors 189; CHECK-NEXT: } 190; 191entry: 192 br label %loop 193 194loop: 195 %recur = phi i32 [ 0, %entry ], [ %recur.next, %loop ] 196 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] 197 %and.red = phi i32 [ 1234, %entry ], [ %and.red.next, %loop ] 198 %rem = srem i32 %recur, %x 199 %recur.next = sext i8 %y to i32 200 %add = add i32 %rem, %recur.next 201 %and.red.next = and i32 %and.red, %add 202 %iv.next = add nsw i32 %iv, 1 203 %ec = icmp eq i32 %iv.next, 20001 204 br i1 %ec, label %exit, label %loop 205 206exit: 207 %res = phi i32 [ %and.red.next, %loop ] 208 ret i32 %res 209} 210 211; To sink the replicate region containing %rem, we need to split the block 212; containing %conv at the end, because %conv is the last recipe in the block. 213define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, i8* %ptr) optsize { 214; CHECK-LABEL: sink_replicate_region_4_requires_split_at_end_of_block 215; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { 216; CHECK-NEXT: loop: 217; CHECK-NEXT: WIDEN-PHI %0 = phi 0, %conv 218; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next 219; CHECK-NEXT: EMIT vp<%3> = icmp ule ir<%iv> vp<%0> 220; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, ir<%iv> 221; CHECK-NEXT: Successor(s): loop.0 222; CHECK-EMPTY: 223; CHECK-NEXT: loop.0: 224; CHECK-NEXT: Successor(s): pred.load 225; CHECK-EMPTY: 226; CHECK-NEXT: <xVFxUF> pred.load: { 227; CHECK-NEXT: pred.load.entry: 228; CHECK-NEXT: BRANCH-ON-MASK vp<%3> 229; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue 230; CHECK-NEXT: CondBit: vp<%3> (loop) 231; CHECK-EMPTY: 232; CHECK-NEXT: pred.load.if: 233; CHECK-NEXT: REPLICATE ir<%lv> = load ir<%gep> (S->V) 234; CHECK-NEXT: Successor(s): pred.load.continue 235; CHECK-EMPTY: 236; CHECK-NEXT: pred.load.continue: 237; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%6> = ir<%lv> 238; CHECK-NEXT: No successors 239; CHECK-NEXT: } 240; CHECK-NEXT: Successor(s): loop.1 241; CHECK-EMPTY: 242; CHECK-NEXT: loop.1: 243; CHECK-NEXT: WIDEN ir<%conv> = sext vp<%6> 244; CHECK-NEXT: Successor(s): pred.srem 245; CHECK-EMPTY: 246; CHECK-NEXT: <xVFxUF> pred.srem: { 247; CHECK-NEXT: pred.srem.entry: 248; CHECK-NEXT: BRANCH-ON-MASK vp<%3> 249; CHECK-NEXT: Successor(s): pred.srem.if, pred.srem.continue 250; CHECK-NEXT: CondBit: vp<%3> (loop) 251; CHECK-EMPTY: 252; CHECK-NEXT: pred.srem.if: 253; CHECK-NEXT: REPLICATE ir<%rem> = srem ir<%0>, ir<%x> (S->V) 254; CHECK-NEXT: Successor(s): pred.srem.continue 255; CHECK-EMPTY: 256; CHECK-NEXT: pred.srem.continue: 257; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%9> = ir<%rem> 258; CHECK-NEXT: No successors 259; CHECK-NEXT: } 260; CHECK-NEXT: Successor(s): loop.1.split 261; CHECK-EMPTY: 262; CHECK-NEXT: loop.1.split: 263; CHECK-NEXT: Successor(s): pred.load 264; CHECK-EMPTY: 265; CHECK-NEXT: <xVFxUF> pred.load: { 266; CHECK-NEXT: pred.load.entry: 267; CHECK-NEXT: BRANCH-ON-MASK vp<%3> 268; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue 269; CHECK-NEXT: CondBit: vp<%3> (loop) 270; CHECK-EMPTY: 271; CHECK-NEXT: pred.load.if: 272; CHECK-NEXT: REPLICATE ir<%lv.2> = load ir<%gep> (S->V) 273; CHECK-NEXT: Successor(s): pred.load.continue 274; CHECK-EMPTY: 275; CHECK-NEXT: pred.load.continue: 276; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%11> = ir<%lv.2> 277; CHECK-NEXT: No successors 278; CHECK-NEXT: } 279; CHECK-NEXT: Successor(s): loop.2 280; CHECK-EMPTY: 281; CHECK-NEXT: loop.2: 282; CHECK-NEXT: WIDEN ir<%add.1> = add ir<%conv>, vp<%9> 283; CHECK-NEXT: WIDEN ir<%conv.lv.2> = sext vp<%11> 284; CHECK-NEXT: WIDEN ir<%add> = add ir<%add.1>, ir<%conv.lv.2> 285; CHECK-NEXT: No successors 286; CHECK-NEXT: } 287; 288entry: 289 br label %loop 290 291loop: 292 %0 = phi i32 [ 0, %entry ], [ %conv, %loop ] 293 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] 294 %gep = getelementptr i8, i8* %ptr, i32 %iv 295 %rem = srem i32 %0, %x 296 %lv = load i8, i8* %gep 297 %conv = sext i8 %lv to i32 298 %lv.2 = load i8, i8* %gep 299 %add.1 = add i32 %conv, %rem 300 %conv.lv.2 = sext i8 %lv.2 to i32 301 %add = add i32 %add.1, %conv.lv.2 302 %iv.next = add nsw i32 %iv, 1 303 %ec = icmp eq i32 %iv.next, 20001 304 br i1 %ec, label %exit, label %loop 305 306exit: 307 ret void 308} 309