1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py 2; RUN: opt < %s -basic-aa -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -loop-vectorize-with-block-frequency -dce -instcombine -S | FileCheck %s 3 4target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" 5target triple = "x86_64-apple-macosx10.8.0" 6 7@b = common global [2048 x i32] zeroinitializer, align 16 8@c = common global [2048 x i32] zeroinitializer, align 16 9@a = common global [2048 x i32] zeroinitializer, align 16 10@G = common global [32 x [1024 x i32]] zeroinitializer, align 16 11@ub = common global [1024 x i32] zeroinitializer, align 16 12@uc = common global [1024 x i32] zeroinitializer, align 16 13@d = common global [2048 x i32] zeroinitializer, align 16 14@fa = common global [1024 x float] zeroinitializer, align 16 15@fb = common global [1024 x float] zeroinitializer, align 16 16@ic = common global [1024 x i32] zeroinitializer, align 16 17@da = common global [1024 x float] zeroinitializer, align 16 18@db = common global [1024 x float] zeroinitializer, align 16 19@dc = common global [1024 x float] zeroinitializer, align 16 20@dd = common global [1024 x float] zeroinitializer, align 16 21@dj = common global [1024 x i32] zeroinitializer, align 16 22 23; We can optimize this test without a tail. 24define void @example1() optsize { 25; CHECK-LABEL: @example1( 26; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 27; CHECK: vector.ph: 28; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 29; CHECK: vector.body: 30; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 31; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[INDEX]] 32; CHECK-NEXT: [[TMP2:%.*]] = bitcast i32* [[TMP1]] to <4 x i32>* 33; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP2]], align 16 34; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 [[INDEX]] 35; CHECK-NEXT: [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>* 36; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 16 37; CHECK-NEXT: [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]] 38; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 [[INDEX]] 39; CHECK-NEXT: [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>* 40; CHECK-NEXT: store <4 x i32> [[TMP5]], <4 x i32>* [[TMP7]], align 16 41; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 42; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256 43; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]] 44; CHECK: middle.block: 45; CHECK-NEXT: br i1 true, label [[TMP10:%.*]], label [[SCALAR_PH]] 46; CHECK: scalar.ph: 47; CHECK-NEXT: br label [[TMP9:%.*]] 48; CHECK: 9: 49; CHECK-NEXT: br i1 undef, label [[TMP10]], label [[TMP9]], [[LOOP2:!llvm.loop !.*]] 50; CHECK: 10: 51; CHECK-NEXT: ret void 52; 53 br label %1 54 55; <label>:1 ; preds = %1, %0 56 %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ] 57 %2 = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 %indvars.iv 58 %3 = load i32, i32* %2, align 4 59 %4 = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 %indvars.iv 60 %5 = load i32, i32* %4, align 4 61 %6 = add nsw i32 %5, %3 62 %7 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv 63 store i32 %6, i32* %7, align 4 64 %indvars.iv.next = add i64 %indvars.iv, 1 65 %lftr.wideiv = trunc i64 %indvars.iv.next to i32 66 %exitcond = icmp eq i32 %lftr.wideiv, 256 67 br i1 %exitcond, label %8, label %1 68 69; <label>:8 ; preds = %1 70 ret void 71} 72 73; Can vectorize in 'optsize' mode by masking the needed tail. 74define void @example2(i32 %n, i32 %x) optsize { 75; CHECK-LABEL: @example2( 76; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt i32 [[N:%.*]], 0 77; CHECK-NEXT: br i1 [[TMP1]], label [[DOTLR_PH5_PREHEADER:%.*]], label [[DOTPREHEADER:%.*]] 78; CHECK: .lr.ph5.preheader: 79; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[N]], -1 80; CHECK-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64 81; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 82; CHECK: vector.ph: 83; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw nsw i64 [[TMP3]], 4 84; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N_RND_UP]], 8589934588 85; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP3]], i32 0 86; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer 87; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 88; CHECK: vector.body: 89; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE6:%.*]] ] 90; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE6]] ] 91; CHECK-NEXT: [[TMP4:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]] 92; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0 93; CHECK-NEXT: br i1 [[TMP5]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] 94; CHECK: pred.store.if: 95; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[INDEX]] 96; CHECK-NEXT: store i32 [[X:%.*]], i32* [[TMP6]], align 16 97; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]] 98; CHECK: pred.store.continue: 99; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1 100; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]] 101; CHECK: pred.store.if1: 102; CHECK-NEXT: [[TMP8:%.*]] = or i64 [[INDEX]], 1 103; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[TMP8]] 104; CHECK-NEXT: store i32 [[X]], i32* [[TMP9]], align 4 105; CHECK-NEXT: br label [[PRED_STORE_CONTINUE2]] 106; CHECK: pred.store.continue2: 107; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2 108; CHECK-NEXT: br i1 [[TMP10]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]] 109; CHECK: pred.store.if3: 110; CHECK-NEXT: [[TMP11:%.*]] = or i64 [[INDEX]], 2 111; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[TMP11]] 112; CHECK-NEXT: store i32 [[X]], i32* [[TMP12]], align 8 113; CHECK-NEXT: br label [[PRED_STORE_CONTINUE4]] 114; CHECK: pred.store.continue4: 115; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3 116; CHECK-NEXT: br i1 [[TMP13]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]] 117; CHECK: pred.store.if5: 118; CHECK-NEXT: [[TMP14:%.*]] = or i64 [[INDEX]], 3 119; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[TMP14]] 120; CHECK-NEXT: store i32 [[X]], i32* [[TMP15]], align 4 121; CHECK-NEXT: br label [[PRED_STORE_CONTINUE6]] 122; CHECK: pred.store.continue6: 123; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 124; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4> 125; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 126; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP4:!llvm.loop !.*]] 127; CHECK: middle.block: 128; CHECK-NEXT: br i1 true, label [[DOT_PREHEADER_CRIT_EDGE:%.*]], label [[SCALAR_PH]] 129; CHECK: scalar.ph: 130; CHECK-NEXT: br label [[DOTLR_PH5:%.*]] 131; CHECK: ..preheader_crit_edge: 132; CHECK-NEXT: [[PHITMP:%.*]] = sext i32 [[N]] to i64 133; CHECK-NEXT: br label [[DOTPREHEADER]] 134; CHECK: .preheader: 135; CHECK-NEXT: [[I_0_LCSSA:%.*]] = phi i64 [ [[PHITMP]], [[DOT_PREHEADER_CRIT_EDGE]] ], [ 0, [[TMP0:%.*]] ] 136; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i32 [[N]], 0 137; CHECK-NEXT: br i1 [[TMP17]], label [[DOT_CRIT_EDGE:%.*]], label [[DOTLR_PH_PREHEADER:%.*]] 138; CHECK: .lr.ph.preheader: 139; CHECK-NEXT: [[TMP18:%.*]] = add i32 [[N]], -1 140; CHECK-NEXT: [[TMP19:%.*]] = zext i32 [[TMP18]] to i64 141; CHECK-NEXT: br i1 false, label [[SCALAR_PH8:%.*]], label [[VECTOR_PH10:%.*]] 142; CHECK: vector.ph10: 143; CHECK-NEXT: [[N_RND_UP11:%.*]] = add nuw nsw i64 [[TMP19]], 4 144; CHECK-NEXT: [[N_VEC13:%.*]] = and i64 [[N_RND_UP11]], 8589934588 145; CHECK-NEXT: [[BROADCAST_SPLATINSERT20:%.*]] = insertelement <4 x i64> poison, i64 [[TMP19]], i32 0 146; CHECK-NEXT: [[BROADCAST_SPLAT21:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT20]], <4 x i64> poison, <4 x i32> zeroinitializer 147; CHECK-NEXT: br label [[VECTOR_BODY9:%.*]] 148; CHECK: vector.body9: 149; CHECK-NEXT: [[INDEX14:%.*]] = phi i64 [ 0, [[VECTOR_PH10]] ], [ [[INDEX_NEXT15:%.*]], [[PRED_STORE_CONTINUE51:%.*]] ] 150; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 [[I_0_LCSSA]], [[INDEX14]] 151; CHECK-NEXT: [[TMP20:%.*]] = add i64 [[OFFSET_IDX]], 1 152; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[OFFSET_IDX]], 2 153; CHECK-NEXT: [[TMP22:%.*]] = add i64 [[OFFSET_IDX]], 3 154; CHECK-NEXT: [[BROADCAST_SPLATINSERT28:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX14]], i32 0 155; CHECK-NEXT: [[BROADCAST_SPLAT29:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT28]], <4 x i64> poison, <4 x i32> zeroinitializer 156; CHECK-NEXT: [[VEC_IV:%.*]] = or <4 x i64> [[BROADCAST_SPLAT29]], <i64 0, i64 1, i64 2, i64 3> 157; CHECK-NEXT: [[TMP23:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT21]] 158; CHECK-NEXT: [[TMP24:%.*]] = extractelement <4 x i1> [[TMP23]], i32 0 159; CHECK-NEXT: br i1 [[TMP24]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] 160; CHECK: pred.load.if: 161; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[OFFSET_IDX]] 162; CHECK-NEXT: [[TMP26:%.*]] = load i32, i32* [[TMP25]], align 4 163; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]] 164; CHECK: pred.load.continue: 165; CHECK-NEXT: [[TMP27:%.*]] = phi i32 [ poison, [[VECTOR_BODY9]] ], [ [[TMP26]], [[PRED_LOAD_IF]] ] 166; CHECK-NEXT: [[TMP28:%.*]] = extractelement <4 x i1> [[TMP23]], i32 1 167; CHECK-NEXT: br i1 [[TMP28]], label [[PRED_LOAD_IF30:%.*]], label [[PRED_LOAD_CONTINUE31:%.*]] 168; CHECK: pred.load.if30: 169; CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[TMP20]] 170; CHECK-NEXT: [[TMP30:%.*]] = load i32, i32* [[TMP29]], align 4 171; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE31]] 172; CHECK: pred.load.continue31: 173; CHECK-NEXT: [[TMP31:%.*]] = phi i32 [ poison, [[PRED_LOAD_CONTINUE]] ], [ [[TMP30]], [[PRED_LOAD_IF30]] ] 174; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i1> [[TMP23]], i32 2 175; CHECK-NEXT: br i1 [[TMP32]], label [[PRED_LOAD_IF32:%.*]], label [[PRED_LOAD_CONTINUE33:%.*]] 176; CHECK: pred.load.if32: 177; CHECK-NEXT: [[TMP33:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[TMP21]] 178; CHECK-NEXT: [[TMP34:%.*]] = load i32, i32* [[TMP33]], align 4 179; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE33]] 180; CHECK: pred.load.continue33: 181; CHECK-NEXT: [[TMP35:%.*]] = phi i32 [ poison, [[PRED_LOAD_CONTINUE31]] ], [ [[TMP34]], [[PRED_LOAD_IF32]] ] 182; CHECK-NEXT: [[TMP36:%.*]] = extractelement <4 x i1> [[TMP23]], i32 3 183; CHECK-NEXT: br i1 [[TMP36]], label [[PRED_LOAD_IF34:%.*]], label [[PRED_LOAD_CONTINUE35:%.*]] 184; CHECK: pred.load.if34: 185; CHECK-NEXT: [[TMP37:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[TMP22]] 186; CHECK-NEXT: [[TMP38:%.*]] = load i32, i32* [[TMP37]], align 4 187; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE35]] 188; CHECK: pred.load.continue35: 189; CHECK-NEXT: [[TMP39:%.*]] = phi i32 [ poison, [[PRED_LOAD_CONTINUE33]] ], [ [[TMP38]], [[PRED_LOAD_IF34]] ] 190; CHECK-NEXT: [[TMP40:%.*]] = extractelement <4 x i1> [[TMP23]], i32 0 191; CHECK-NEXT: br i1 [[TMP40]], label [[PRED_LOAD_IF36:%.*]], label [[PRED_LOAD_CONTINUE37:%.*]] 192; CHECK: pred.load.if36: 193; CHECK-NEXT: [[TMP41:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 [[OFFSET_IDX]] 194; CHECK-NEXT: [[TMP42:%.*]] = load i32, i32* [[TMP41]], align 4 195; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE37]] 196; CHECK: pred.load.continue37: 197; CHECK-NEXT: [[TMP43:%.*]] = phi i32 [ poison, [[PRED_LOAD_CONTINUE35]] ], [ [[TMP42]], [[PRED_LOAD_IF36]] ] 198; CHECK-NEXT: [[TMP44:%.*]] = extractelement <4 x i1> [[TMP23]], i32 1 199; CHECK-NEXT: br i1 [[TMP44]], label [[PRED_LOAD_IF38:%.*]], label [[PRED_LOAD_CONTINUE39:%.*]] 200; CHECK: pred.load.if38: 201; CHECK-NEXT: [[TMP45:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 [[TMP20]] 202; CHECK-NEXT: [[TMP46:%.*]] = load i32, i32* [[TMP45]], align 4 203; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE39]] 204; CHECK: pred.load.continue39: 205; CHECK-NEXT: [[TMP47:%.*]] = phi i32 [ poison, [[PRED_LOAD_CONTINUE37]] ], [ [[TMP46]], [[PRED_LOAD_IF38]] ] 206; CHECK-NEXT: [[TMP48:%.*]] = extractelement <4 x i1> [[TMP23]], i32 2 207; CHECK-NEXT: br i1 [[TMP48]], label [[PRED_LOAD_IF40:%.*]], label [[PRED_LOAD_CONTINUE41:%.*]] 208; CHECK: pred.load.if40: 209; CHECK-NEXT: [[TMP49:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 [[TMP21]] 210; CHECK-NEXT: [[TMP50:%.*]] = load i32, i32* [[TMP49]], align 4 211; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE41]] 212; CHECK: pred.load.continue41: 213; CHECK-NEXT: [[TMP51:%.*]] = phi i32 [ poison, [[PRED_LOAD_CONTINUE39]] ], [ [[TMP50]], [[PRED_LOAD_IF40]] ] 214; CHECK-NEXT: [[TMP52:%.*]] = extractelement <4 x i1> [[TMP23]], i32 3 215; CHECK-NEXT: br i1 [[TMP52]], label [[PRED_LOAD_IF42:%.*]], label [[PRED_LOAD_CONTINUE43:%.*]] 216; CHECK: pred.load.if42: 217; CHECK-NEXT: [[TMP53:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 [[TMP22]] 218; CHECK-NEXT: [[TMP54:%.*]] = load i32, i32* [[TMP53]], align 4 219; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE43]] 220; CHECK: pred.load.continue43: 221; CHECK-NEXT: [[TMP55:%.*]] = phi i32 [ poison, [[PRED_LOAD_CONTINUE41]] ], [ [[TMP54]], [[PRED_LOAD_IF42]] ] 222; CHECK-NEXT: [[TMP56:%.*]] = extractelement <4 x i1> [[TMP23]], i32 0 223; CHECK-NEXT: br i1 [[TMP56]], label [[PRED_STORE_IF44:%.*]], label [[PRED_STORE_CONTINUE45:%.*]] 224; CHECK: pred.store.if44: 225; CHECK-NEXT: [[TMP57:%.*]] = and i32 [[TMP43]], [[TMP27]] 226; CHECK-NEXT: [[TMP58:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 [[OFFSET_IDX]] 227; CHECK-NEXT: store i32 [[TMP57]], i32* [[TMP58]], align 4 228; CHECK-NEXT: br label [[PRED_STORE_CONTINUE45]] 229; CHECK: pred.store.continue45: 230; CHECK-NEXT: [[TMP59:%.*]] = extractelement <4 x i1> [[TMP23]], i32 1 231; CHECK-NEXT: br i1 [[TMP59]], label [[PRED_STORE_IF46:%.*]], label [[PRED_STORE_CONTINUE47:%.*]] 232; CHECK: pred.store.if46: 233; CHECK-NEXT: [[TMP60:%.*]] = and i32 [[TMP47]], [[TMP31]] 234; CHECK-NEXT: [[TMP61:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 [[TMP20]] 235; CHECK-NEXT: store i32 [[TMP60]], i32* [[TMP61]], align 4 236; CHECK-NEXT: br label [[PRED_STORE_CONTINUE47]] 237; CHECK: pred.store.continue47: 238; CHECK-NEXT: [[TMP62:%.*]] = extractelement <4 x i1> [[TMP23]], i32 2 239; CHECK-NEXT: br i1 [[TMP62]], label [[PRED_STORE_IF48:%.*]], label [[PRED_STORE_CONTINUE49:%.*]] 240; CHECK: pred.store.if48: 241; CHECK-NEXT: [[TMP63:%.*]] = and i32 [[TMP51]], [[TMP35]] 242; CHECK-NEXT: [[TMP64:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 [[TMP21]] 243; CHECK-NEXT: store i32 [[TMP63]], i32* [[TMP64]], align 4 244; CHECK-NEXT: br label [[PRED_STORE_CONTINUE49]] 245; CHECK: pred.store.continue49: 246; CHECK-NEXT: [[TMP65:%.*]] = extractelement <4 x i1> [[TMP23]], i32 3 247; CHECK-NEXT: br i1 [[TMP65]], label [[PRED_STORE_IF50:%.*]], label [[PRED_STORE_CONTINUE51]] 248; CHECK: pred.store.if50: 249; CHECK-NEXT: [[TMP66:%.*]] = and i32 [[TMP55]], [[TMP39]] 250; CHECK-NEXT: [[TMP67:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 [[TMP22]] 251; CHECK-NEXT: store i32 [[TMP66]], i32* [[TMP67]], align 4 252; CHECK-NEXT: br label [[PRED_STORE_CONTINUE51]] 253; CHECK: pred.store.continue51: 254; CHECK-NEXT: [[INDEX_NEXT15]] = add i64 [[INDEX14]], 4 255; CHECK-NEXT: [[TMP68:%.*]] = icmp eq i64 [[INDEX_NEXT15]], [[N_VEC13]] 256; CHECK-NEXT: br i1 [[TMP68]], label [[MIDDLE_BLOCK7:%.*]], label [[VECTOR_BODY9]], [[LOOP5:!llvm.loop !.*]] 257; CHECK: middle.block7: 258; CHECK-NEXT: br i1 true, label [[DOT_CRIT_EDGE_LOOPEXIT:%.*]], label [[SCALAR_PH8]] 259; CHECK: scalar.ph8: 260; CHECK-NEXT: br label [[DOTLR_PH:%.*]] 261; CHECK: .lr.ph5: 262; CHECK-NEXT: br i1 undef, label [[DOT_PREHEADER_CRIT_EDGE]], label [[DOTLR_PH5]], [[LOOP6:!llvm.loop !.*]] 263; CHECK: .lr.ph: 264; CHECK-NEXT: br i1 undef, label [[DOT_CRIT_EDGE_LOOPEXIT]], label [[DOTLR_PH]], [[LOOP7:!llvm.loop !.*]] 265; CHECK: ._crit_edge.loopexit: 266; CHECK-NEXT: br label [[DOT_CRIT_EDGE]] 267; CHECK: ._crit_edge: 268; CHECK-NEXT: ret void 269; 270 %1 = icmp sgt i32 %n, 0 271 br i1 %1, label %.lr.ph5, label %.preheader 272 273..preheader_crit_edge: ; preds = %.lr.ph5 274 %phitmp = sext i32 %n to i64 275 br label %.preheader 276 277.preheader: ; preds = %..preheader_crit_edge, %0 278 %i.0.lcssa = phi i64 [ %phitmp, %..preheader_crit_edge ], [ 0, %0 ] 279 %2 = icmp eq i32 %n, 0 280 br i1 %2, label %._crit_edge, label %.lr.ph 281 282.lr.ph5: ; preds = %0, %.lr.ph5 283 %indvars.iv6 = phi i64 [ %indvars.iv.next7, %.lr.ph5 ], [ 0, %0 ] 284 %3 = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 %indvars.iv6 285 store i32 %x, i32* %3, align 4 286 %indvars.iv.next7 = add i64 %indvars.iv6, 1 287 %lftr.wideiv = trunc i64 %indvars.iv.next7 to i32 288 %exitcond = icmp eq i32 %lftr.wideiv, %n 289 br i1 %exitcond, label %..preheader_crit_edge, label %.lr.ph5 290 291.lr.ph: ; preds = %.preheader, %.lr.ph 292 %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ %i.0.lcssa, %.preheader ] 293 %.02 = phi i32 [ %4, %.lr.ph ], [ %n, %.preheader ] 294 %4 = add nsw i32 %.02, -1 295 %5 = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 %indvars.iv 296 %6 = load i32, i32* %5, align 4 297 %7 = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 %indvars.iv 298 %8 = load i32, i32* %7, align 4 299 %9 = and i32 %8, %6 300 %10 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv 301 store i32 %9, i32* %10, align 4 302 %indvars.iv.next = add i64 %indvars.iv, 1 303 %11 = icmp eq i32 %4, 0 304 br i1 %11, label %._crit_edge, label %.lr.ph 305 306._crit_edge: ; preds = %.lr.ph, %.preheader 307 ret void 308} 309 310; Loop has no primary induction as its integer IV has step -1 starting at 311; unknown N, but can still be vectorized. 312define void @example3(i32 %n, i32* noalias nocapture %p, i32* noalias nocapture %q) optsize { 313; CHECK-LABEL: @example3( 314; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[N:%.*]], 0 315; CHECK-NEXT: br i1 [[TMP1]], label [[DOT_CRIT_EDGE:%.*]], label [[DOTLR_PH_PREHEADER:%.*]] 316; CHECK: .lr.ph.preheader: 317; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[N]], -1 318; CHECK-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64 319; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 320; CHECK: vector.ph: 321; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw nsw i64 [[TMP3]], 4 322; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N_RND_UP]], 8589934588 323; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP3]], i32 0 324; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer 325; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 326; CHECK: vector.body: 327; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE27:%.*]] ] 328; CHECK-NEXT: [[BROADCAST_SPLATINSERT14:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i32 0 329; CHECK-NEXT: [[BROADCAST_SPLAT15:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT14]], <4 x i64> poison, <4 x i32> zeroinitializer 330; CHECK-NEXT: [[VEC_IV:%.*]] = or <4 x i64> [[BROADCAST_SPLAT15]], <i64 0, i64 1, i64 2, i64 3> 331; CHECK-NEXT: [[TMP4:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]] 332; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0 333; CHECK-NEXT: br i1 [[TMP5]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] 334; CHECK: pred.load.if: 335; CHECK-NEXT: [[NEXT_GEP10:%.*]] = getelementptr i32, i32* [[Q:%.*]], i64 [[INDEX]] 336; CHECK-NEXT: [[TMP6:%.*]] = load i32, i32* [[NEXT_GEP10]], align 16 337; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]] 338; CHECK: pred.load.continue: 339; CHECK-NEXT: [[TMP7:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ] 340; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1 341; CHECK-NEXT: br i1 [[TMP8]], label [[PRED_LOAD_IF16:%.*]], label [[PRED_LOAD_CONTINUE17:%.*]] 342; CHECK: pred.load.if16: 343; CHECK-NEXT: [[TMP9:%.*]] = or i64 [[INDEX]], 1 344; CHECK-NEXT: [[NEXT_GEP11:%.*]] = getelementptr i32, i32* [[Q]], i64 [[TMP9]] 345; CHECK-NEXT: [[TMP10:%.*]] = load i32, i32* [[NEXT_GEP11]], align 16 346; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE17]] 347; CHECK: pred.load.continue17: 348; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ poison, [[PRED_LOAD_CONTINUE]] ], [ [[TMP10]], [[PRED_LOAD_IF16]] ] 349; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2 350; CHECK-NEXT: br i1 [[TMP12]], label [[PRED_LOAD_IF18:%.*]], label [[PRED_LOAD_CONTINUE19:%.*]] 351; CHECK: pred.load.if18: 352; CHECK-NEXT: [[TMP13:%.*]] = or i64 [[INDEX]], 2 353; CHECK-NEXT: [[NEXT_GEP12:%.*]] = getelementptr i32, i32* [[Q]], i64 [[TMP13]] 354; CHECK-NEXT: [[TMP14:%.*]] = load i32, i32* [[NEXT_GEP12]], align 16 355; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE19]] 356; CHECK: pred.load.continue19: 357; CHECK-NEXT: [[TMP15:%.*]] = phi i32 [ poison, [[PRED_LOAD_CONTINUE17]] ], [ [[TMP14]], [[PRED_LOAD_IF18]] ] 358; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3 359; CHECK-NEXT: br i1 [[TMP16]], label [[PRED_LOAD_IF20:%.*]], label [[PRED_LOAD_CONTINUE21:%.*]] 360; CHECK: pred.load.if20: 361; CHECK-NEXT: [[TMP17:%.*]] = or i64 [[INDEX]], 3 362; CHECK-NEXT: [[NEXT_GEP13:%.*]] = getelementptr i32, i32* [[Q]], i64 [[TMP17]] 363; CHECK-NEXT: [[TMP18:%.*]] = load i32, i32* [[NEXT_GEP13]], align 16 364; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE21]] 365; CHECK: pred.load.continue21: 366; CHECK-NEXT: [[TMP19:%.*]] = phi i32 [ poison, [[PRED_LOAD_CONTINUE19]] ], [ [[TMP18]], [[PRED_LOAD_IF20]] ] 367; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0 368; CHECK-NEXT: br i1 [[TMP20]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] 369; CHECK: pred.store.if: 370; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i32, i32* [[P:%.*]], i64 [[INDEX]] 371; CHECK-NEXT: store i32 [[TMP7]], i32* [[NEXT_GEP]], align 16 372; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]] 373; CHECK: pred.store.continue: 374; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1 375; CHECK-NEXT: br i1 [[TMP21]], label [[PRED_STORE_IF22:%.*]], label [[PRED_STORE_CONTINUE23:%.*]] 376; CHECK: pred.store.if22: 377; CHECK-NEXT: [[TMP22:%.*]] = or i64 [[INDEX]], 1 378; CHECK-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i32, i32* [[P]], i64 [[TMP22]] 379; CHECK-NEXT: store i32 [[TMP11]], i32* [[NEXT_GEP7]], align 16 380; CHECK-NEXT: br label [[PRED_STORE_CONTINUE23]] 381; CHECK: pred.store.continue23: 382; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2 383; CHECK-NEXT: br i1 [[TMP23]], label [[PRED_STORE_IF24:%.*]], label [[PRED_STORE_CONTINUE25:%.*]] 384; CHECK: pred.store.if24: 385; CHECK-NEXT: [[TMP24:%.*]] = or i64 [[INDEX]], 2 386; CHECK-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i32, i32* [[P]], i64 [[TMP24]] 387; CHECK-NEXT: store i32 [[TMP15]], i32* [[NEXT_GEP8]], align 16 388; CHECK-NEXT: br label [[PRED_STORE_CONTINUE25]] 389; CHECK: pred.store.continue25: 390; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3 391; CHECK-NEXT: br i1 [[TMP25]], label [[PRED_STORE_IF26:%.*]], label [[PRED_STORE_CONTINUE27]] 392; CHECK: pred.store.if26: 393; CHECK-NEXT: [[TMP26:%.*]] = or i64 [[INDEX]], 3 394; CHECK-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i32, i32* [[P]], i64 [[TMP26]] 395; CHECK-NEXT: store i32 [[TMP19]], i32* [[NEXT_GEP9]], align 16 396; CHECK-NEXT: br label [[PRED_STORE_CONTINUE27]] 397; CHECK: pred.store.continue27: 398; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 399; CHECK-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 400; CHECK-NEXT: br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP8:!llvm.loop !.*]] 401; CHECK: middle.block: 402; CHECK-NEXT: br i1 true, label [[DOT_CRIT_EDGE_LOOPEXIT:%.*]], label [[SCALAR_PH]] 403; CHECK: scalar.ph: 404; CHECK-NEXT: br label [[DOTLR_PH:%.*]] 405; CHECK: .lr.ph: 406; CHECK-NEXT: br i1 undef, label [[DOT_CRIT_EDGE_LOOPEXIT]], label [[DOTLR_PH]], [[LOOP9:!llvm.loop !.*]] 407; CHECK: ._crit_edge.loopexit: 408; CHECK-NEXT: br label [[DOT_CRIT_EDGE]] 409; CHECK: ._crit_edge: 410; CHECK-NEXT: ret void 411; 412 %1 = icmp eq i32 %n, 0 413 br i1 %1, label %._crit_edge, label %.lr.ph 414 415.lr.ph: ; preds = %0, %.lr.ph 416 %.05 = phi i32 [ %2, %.lr.ph ], [ %n, %0 ] 417 %.014 = phi i32* [ %5, %.lr.ph ], [ %p, %0 ] 418 %.023 = phi i32* [ %3, %.lr.ph ], [ %q, %0 ] 419 %2 = add nsw i32 %.05, -1 420 %3 = getelementptr inbounds i32, i32* %.023, i64 1 421 %4 = load i32, i32* %.023, align 16 422 %5 = getelementptr inbounds i32, i32* %.014, i64 1 423 store i32 %4, i32* %.014, align 16 424 %6 = icmp eq i32 %2, 0 425 br i1 %6, label %._crit_edge, label %.lr.ph 426 427._crit_edge: ; preds = %.lr.ph, %0 428 ret void 429} 430 431; We can't vectorize this one because we need a runtime ptr check. 432define void @example23(i16* nocapture %src, i32* nocapture %dst) optsize { 433; CHECK-LABEL: @example23( 434; CHECK-NEXT: br label [[TMP1:%.*]] 435; CHECK: 1: 436; CHECK-NEXT: [[DOT04:%.*]] = phi i16* [ [[SRC:%.*]], [[TMP0:%.*]] ], [ [[TMP2:%.*]], [[TMP1]] ] 437; CHECK-NEXT: [[DOT013:%.*]] = phi i32* [ [[DST:%.*]], [[TMP0]] ], [ [[TMP6:%.*]], [[TMP1]] ] 438; CHECK-NEXT: [[I_02:%.*]] = phi i32 [ 0, [[TMP0]] ], [ [[TMP7:%.*]], [[TMP1]] ] 439; CHECK-NEXT: [[TMP2]] = getelementptr inbounds i16, i16* [[DOT04]], i64 1 440; CHECK-NEXT: [[TMP3:%.*]] = load i16, i16* [[DOT04]], align 2 441; CHECK-NEXT: [[TMP4:%.*]] = zext i16 [[TMP3]] to i32 442; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i32 [[TMP4]], 7 443; CHECK-NEXT: [[TMP6]] = getelementptr inbounds i32, i32* [[DOT013]], i64 1 444; CHECK-NEXT: store i32 [[TMP5]], i32* [[DOT013]], align 4 445; CHECK-NEXT: [[TMP7]] = add nuw nsw i32 [[I_02]], 1 446; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[TMP7]], 256 447; CHECK-NEXT: br i1 [[EXITCOND]], label [[TMP8:%.*]], label [[TMP1]] 448; CHECK: 8: 449; CHECK-NEXT: ret void 450; 451 br label %1 452 453; <label>:1 ; preds = %1, %0 454 %.04 = phi i16* [ %src, %0 ], [ %2, %1 ] 455 %.013 = phi i32* [ %dst, %0 ], [ %6, %1 ] 456 %i.02 = phi i32 [ 0, %0 ], [ %7, %1 ] 457 %2 = getelementptr inbounds i16, i16* %.04, i64 1 458 %3 = load i16, i16* %.04, align 2 459 %4 = zext i16 %3 to i32 460 %5 = shl nuw nsw i32 %4, 7 461 %6 = getelementptr inbounds i32, i32* %.013, i64 1 462 store i32 %5, i32* %.013, align 4 463 %7 = add nsw i32 %i.02, 1 464 %exitcond = icmp eq i32 %7, 256 465 br i1 %exitcond, label %8, label %1 466 467; <label>:8 ; preds = %1 468 ret void 469} 470 471 472; We CAN vectorize this example because the pointers are marked as noalias. 473define void @example23b(i16* noalias nocapture %src, i32* noalias nocapture %dst) optsize { 474; CHECK-LABEL: @example23b( 475; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 476; CHECK: vector.ph: 477; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 478; CHECK: vector.body: 479; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 480; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i16, i16* [[SRC:%.*]], i64 [[INDEX]] 481; CHECK-NEXT: [[NEXT_GEP4:%.*]] = getelementptr i32, i32* [[DST:%.*]], i64 [[INDEX]] 482; CHECK-NEXT: [[TMP1:%.*]] = bitcast i16* [[NEXT_GEP]] to <4 x i16>* 483; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 2 484; CHECK-NEXT: [[TMP2:%.*]] = zext <4 x i16> [[WIDE_LOAD]] to <4 x i32> 485; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw <4 x i32> [[TMP2]], <i32 7, i32 7, i32 7, i32 7> 486; CHECK-NEXT: [[TMP4:%.*]] = bitcast i32* [[NEXT_GEP4]] to <4 x i32>* 487; CHECK-NEXT: store <4 x i32> [[TMP3]], <4 x i32>* [[TMP4]], align 4 488; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 489; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256 490; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP10:!llvm.loop !.*]] 491; CHECK: middle.block: 492; CHECK-NEXT: br i1 true, label [[TMP7:%.*]], label [[SCALAR_PH]] 493; CHECK: scalar.ph: 494; CHECK-NEXT: br label [[TMP6:%.*]] 495; CHECK: 6: 496; CHECK-NEXT: br i1 undef, label [[TMP7]], label [[TMP6]], [[LOOP11:!llvm.loop !.*]] 497; CHECK: 7: 498; CHECK-NEXT: ret void 499; 500 br label %1 501 502; <label>:1 ; preds = %1, %0 503 %.04 = phi i16* [ %src, %0 ], [ %2, %1 ] 504 %.013 = phi i32* [ %dst, %0 ], [ %6, %1 ] 505 %i.02 = phi i32 [ 0, %0 ], [ %7, %1 ] 506 %2 = getelementptr inbounds i16, i16* %.04, i64 1 507 %3 = load i16, i16* %.04, align 2 508 %4 = zext i16 %3 to i32 509 %5 = shl nuw nsw i32 %4, 7 510 %6 = getelementptr inbounds i32, i32* %.013, i64 1 511 store i32 %5, i32* %.013, align 4 512 %7 = add nsw i32 %i.02, 1 513 %exitcond = icmp eq i32 %7, 256 514 br i1 %exitcond, label %8, label %1 515 516; <label>:8 ; preds = %1 517 ret void 518} 519 520; We CAN vectorize this example by folding the tail it entails. 521define void @example23c(i16* noalias nocapture %src, i32* noalias nocapture %dst) optsize { 522; CHECK-LABEL: @example23c( 523; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 524; CHECK: vector.ph: 525; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 526; CHECK: vector.body: 527; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE22:%.*]] ] 528; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i32 0 529; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer 530; CHECK-NEXT: [[INDUCTION:%.*]] = or <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3> 531; CHECK-NEXT: [[TMP1:%.*]] = icmp ult <4 x i64> [[INDUCTION]], <i64 257, i64 257, i64 257, i64 257> 532; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i32 0 533; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] 534; CHECK: pred.load.if: 535; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i16, i16* [[SRC:%.*]], i64 [[INDEX]] 536; CHECK-NEXT: [[TMP3:%.*]] = load i16, i16* [[NEXT_GEP]], align 2 537; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]] 538; CHECK: pred.load.continue: 539; CHECK-NEXT: [[TMP4:%.*]] = phi i16 [ poison, [[VECTOR_BODY]] ], [ [[TMP3]], [[PRED_LOAD_IF]] ] 540; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP1]], i32 1 541; CHECK-NEXT: br i1 [[TMP5]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]] 542; CHECK: pred.load.if11: 543; CHECK-NEXT: [[TMP6:%.*]] = or i64 [[INDEX]], 1 544; CHECK-NEXT: [[NEXT_GEP4:%.*]] = getelementptr i16, i16* [[SRC]], i64 [[TMP6]] 545; CHECK-NEXT: [[TMP7:%.*]] = load i16, i16* [[NEXT_GEP4]], align 2 546; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE12]] 547; CHECK: pred.load.continue12: 548; CHECK-NEXT: [[TMP8:%.*]] = phi i16 [ poison, [[PRED_LOAD_CONTINUE]] ], [ [[TMP7]], [[PRED_LOAD_IF11]] ] 549; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP1]], i32 2 550; CHECK-NEXT: br i1 [[TMP9]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14:%.*]] 551; CHECK: pred.load.if13: 552; CHECK-NEXT: [[TMP10:%.*]] = or i64 [[INDEX]], 2 553; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i16, i16* [[SRC]], i64 [[TMP10]] 554; CHECK-NEXT: [[TMP11:%.*]] = load i16, i16* [[NEXT_GEP5]], align 2 555; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE14]] 556; CHECK: pred.load.continue14: 557; CHECK-NEXT: [[TMP12:%.*]] = phi i16 [ poison, [[PRED_LOAD_CONTINUE12]] ], [ [[TMP11]], [[PRED_LOAD_IF13]] ] 558; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP1]], i32 3 559; CHECK-NEXT: br i1 [[TMP13]], label [[PRED_LOAD_IF15:%.*]], label [[PRED_LOAD_CONTINUE16:%.*]] 560; CHECK: pred.load.if15: 561; CHECK-NEXT: [[TMP14:%.*]] = or i64 [[INDEX]], 3 562; CHECK-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i16, i16* [[SRC]], i64 [[TMP14]] 563; CHECK-NEXT: [[TMP15:%.*]] = load i16, i16* [[NEXT_GEP6]], align 2 564; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE16]] 565; CHECK: pred.load.continue16: 566; CHECK-NEXT: [[TMP16:%.*]] = phi i16 [ poison, [[PRED_LOAD_CONTINUE14]] ], [ [[TMP15]], [[PRED_LOAD_IF15]] ] 567; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP1]], i32 0 568; CHECK-NEXT: br i1 [[TMP17]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] 569; CHECK: pred.store.if: 570; CHECK-NEXT: [[TMP18:%.*]] = zext i16 [[TMP4]] to i32 571; CHECK-NEXT: [[TMP19:%.*]] = shl nuw nsw i32 [[TMP18]], 7 572; CHECK-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i32, i32* [[DST:%.*]], i64 [[INDEX]] 573; CHECK-NEXT: store i32 [[TMP19]], i32* [[NEXT_GEP7]], align 4 574; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]] 575; CHECK: pred.store.continue: 576; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP1]], i32 1 577; CHECK-NEXT: br i1 [[TMP20]], label [[PRED_STORE_IF17:%.*]], label [[PRED_STORE_CONTINUE18:%.*]] 578; CHECK: pred.store.if17: 579; CHECK-NEXT: [[TMP21:%.*]] = zext i16 [[TMP8]] to i32 580; CHECK-NEXT: [[TMP22:%.*]] = shl nuw nsw i32 [[TMP21]], 7 581; CHECK-NEXT: [[TMP23:%.*]] = or i64 [[INDEX]], 1 582; CHECK-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i32, i32* [[DST]], i64 [[TMP23]] 583; CHECK-NEXT: store i32 [[TMP22]], i32* [[NEXT_GEP8]], align 4 584; CHECK-NEXT: br label [[PRED_STORE_CONTINUE18]] 585; CHECK: pred.store.continue18: 586; CHECK-NEXT: [[TMP24:%.*]] = extractelement <4 x i1> [[TMP1]], i32 2 587; CHECK-NEXT: br i1 [[TMP24]], label [[PRED_STORE_IF19:%.*]], label [[PRED_STORE_CONTINUE20:%.*]] 588; CHECK: pred.store.if19: 589; CHECK-NEXT: [[TMP25:%.*]] = zext i16 [[TMP12]] to i32 590; CHECK-NEXT: [[TMP26:%.*]] = shl nuw nsw i32 [[TMP25]], 7 591; CHECK-NEXT: [[TMP27:%.*]] = or i64 [[INDEX]], 2 592; CHECK-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i32, i32* [[DST]], i64 [[TMP27]] 593; CHECK-NEXT: store i32 [[TMP26]], i32* [[NEXT_GEP9]], align 4 594; CHECK-NEXT: br label [[PRED_STORE_CONTINUE20]] 595; CHECK: pred.store.continue20: 596; CHECK-NEXT: [[TMP28:%.*]] = extractelement <4 x i1> [[TMP1]], i32 3 597; CHECK-NEXT: br i1 [[TMP28]], label [[PRED_STORE_IF21:%.*]], label [[PRED_STORE_CONTINUE22]] 598; CHECK: pred.store.if21: 599; CHECK-NEXT: [[TMP29:%.*]] = zext i16 [[TMP16]] to i32 600; CHECK-NEXT: [[TMP30:%.*]] = shl nuw nsw i32 [[TMP29]], 7 601; CHECK-NEXT: [[TMP31:%.*]] = or i64 [[INDEX]], 3 602; CHECK-NEXT: [[NEXT_GEP10:%.*]] = getelementptr i32, i32* [[DST]], i64 [[TMP31]] 603; CHECK-NEXT: store i32 [[TMP30]], i32* [[NEXT_GEP10]], align 4 604; CHECK-NEXT: br label [[PRED_STORE_CONTINUE22]] 605; CHECK: pred.store.continue22: 606; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 607; CHECK-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260 608; CHECK-NEXT: br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP12:!llvm.loop !.*]] 609; CHECK: middle.block: 610; CHECK-NEXT: br i1 true, label [[TMP34:%.*]], label [[SCALAR_PH]] 611; CHECK: scalar.ph: 612; CHECK-NEXT: br label [[TMP33:%.*]] 613; CHECK: 33: 614; CHECK-NEXT: br i1 undef, label [[TMP34]], label [[TMP33]], [[LOOP13:!llvm.loop !.*]] 615; CHECK: 34: 616; CHECK-NEXT: ret void 617; 618 br label %1 619 620; <label>:1 ; preds = %1, %0 621 %.04 = phi i16* [ %src, %0 ], [ %2, %1 ] 622 %.013 = phi i32* [ %dst, %0 ], [ %6, %1 ] 623 %i.02 = phi i64 [ 0, %0 ], [ %7, %1 ] 624 %2 = getelementptr inbounds i16, i16* %.04, i64 1 625 %3 = load i16, i16* %.04, align 2 626 %4 = zext i16 %3 to i32 627 %5 = shl nuw nsw i32 %4, 7 628 %6 = getelementptr inbounds i32, i32* %.013, i64 1 629 store i32 %5, i32* %.013, align 4 630 %7 = add nsw i64 %i.02, 1 631 %exitcond = icmp eq i64 %7, 257 632 br i1 %exitcond, label %8, label %1 633 634; <label>:8 ; preds = %1 635 ret void 636} 637 638; We CAN'T vectorize this example because it would entail a tail and an 639; induction is used outside the loop. 640define i64 @example23d(i16* noalias nocapture %src, i32* noalias nocapture %dst) optsize { 641; CHECK-LABEL: @example23d( 642; CHECK-NEXT: br label [[TMP1:%.*]] 643; CHECK: 1: 644; CHECK-NEXT: [[DOT04:%.*]] = phi i16* [ [[SRC:%.*]], [[TMP0:%.*]] ], [ [[TMP2:%.*]], [[TMP1]] ] 645; CHECK-NEXT: [[DOT013:%.*]] = phi i32* [ [[DST:%.*]], [[TMP0]] ], [ [[TMP6:%.*]], [[TMP1]] ] 646; CHECK-NEXT: [[I_02:%.*]] = phi i64 [ 0, [[TMP0]] ], [ [[TMP7:%.*]], [[TMP1]] ] 647; CHECK-NEXT: [[TMP2]] = getelementptr inbounds i16, i16* [[DOT04]], i64 1 648; CHECK-NEXT: [[TMP3:%.*]] = load i16, i16* [[DOT04]], align 2 649; CHECK-NEXT: [[TMP4:%.*]] = zext i16 [[TMP3]] to i32 650; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i32 [[TMP4]], 7 651; CHECK-NEXT: [[TMP6]] = getelementptr inbounds i32, i32* [[DOT013]], i64 1 652; CHECK-NEXT: store i32 [[TMP5]], i32* [[DOT013]], align 4 653; CHECK-NEXT: [[TMP7]] = add nuw nsw i64 [[I_02]], 1 654; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[TMP7]], 257 655; CHECK-NEXT: br i1 [[EXITCOND]], label [[TMP8:%.*]], label [[TMP1]] 656; CHECK: 8: 657; CHECK-NEXT: ret i64 [[TMP7]] 658; 659 br label %1 660 661; <label>:1 ; preds = %1, %0 662 %.04 = phi i16* [ %src, %0 ], [ %2, %1 ] 663 %.013 = phi i32* [ %dst, %0 ], [ %6, %1 ] 664 %i.02 = phi i64 [ 0, %0 ], [ %7, %1 ] 665 %2 = getelementptr inbounds i16, i16* %.04, i64 1 666 %3 = load i16, i16* %.04, align 2 667 %4 = zext i16 %3 to i32 668 %5 = shl nuw nsw i32 %4, 7 669 %6 = getelementptr inbounds i32, i32* %.013, i64 1 670 store i32 %5, i32* %.013, align 4 671 %7 = add nsw i64 %i.02, 1 672 %exitcond = icmp eq i64 %7, 257 673 br i1 %exitcond, label %8, label %1 674 675; <label>:8 ; preds = %1 676 ret i64 %7 677} 678