1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py 2; RUN: opt < %s -basic-aa -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -loop-vectorize-with-block-frequency -dce -instcombine -S | FileCheck %s 3 4target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" 5target triple = "x86_64-apple-macosx10.8.0" 6 7@b = common global [2048 x i32] zeroinitializer, align 16 8@c = common global [2048 x i32] zeroinitializer, align 16 9@a = common global [2048 x i32] zeroinitializer, align 16 10@G = common global [32 x [1024 x i32]] zeroinitializer, align 16 11@ub = common global [1024 x i32] zeroinitializer, align 16 12@uc = common global [1024 x i32] zeroinitializer, align 16 13@d = common global [2048 x i32] zeroinitializer, align 16 14@fa = common global [1024 x float] zeroinitializer, align 16 15@fb = common global [1024 x float] zeroinitializer, align 16 16@ic = common global [1024 x i32] zeroinitializer, align 16 17@da = common global [1024 x float] zeroinitializer, align 16 18@db = common global [1024 x float] zeroinitializer, align 16 19@dc = common global [1024 x float] zeroinitializer, align 16 20@dd = common global [1024 x float] zeroinitializer, align 16 21@dj = common global [1024 x i32] zeroinitializer, align 16 22 23; We can optimize this test without a tail. 24define void @example1() optsize { 25; CHECK-LABEL: @example1( 26; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 27; CHECK: vector.ph: 28; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 29; CHECK: vector.body: 30; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 31; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[INDEX]] 32; CHECK-NEXT: [[TMP2:%.*]] = bitcast i32* [[TMP1]] to <4 x i32>* 33; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP2]], align 16 34; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 [[INDEX]] 35; CHECK-NEXT: [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>* 36; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 16 37; CHECK-NEXT: [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]] 38; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 [[INDEX]] 39; CHECK-NEXT: [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>* 40; CHECK-NEXT: store <4 x i32> [[TMP5]], <4 x i32>* [[TMP7]], align 16 41; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 42; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256 43; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] 44; CHECK: middle.block: 45; CHECK-NEXT: br i1 true, label [[TMP10:%.*]], label [[SCALAR_PH]] 46; CHECK: scalar.ph: 47; CHECK-NEXT: br label [[TMP9:%.*]] 48; CHECK: 9: 49; CHECK-NEXT: br i1 poison, label [[TMP10]], label [[TMP9]], !llvm.loop [[LOOP2:![0-9]+]] 50; CHECK: 10: 51; CHECK-NEXT: ret void 52; 53 br label %1 54 55; <label>:1 ; preds = %1, %0 56 %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ] 57 %2 = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 %indvars.iv 58 %3 = load i32, i32* %2, align 4 59 %4 = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 %indvars.iv 60 %5 = load i32, i32* %4, align 4 61 %6 = add nsw i32 %5, %3 62 %7 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv 63 store i32 %6, i32* %7, align 4 64 %indvars.iv.next = add i64 %indvars.iv, 1 65 %lftr.wideiv = trunc i64 %indvars.iv.next to i32 66 %exitcond = icmp eq i32 %lftr.wideiv, 256 67 br i1 %exitcond, label %8, label %1 68 69; <label>:8 ; preds = %1 70 ret void 71} 72 73; Can vectorize in 'optsize' mode by masking the needed tail. 74define void @example2(i32 %n, i32 %x) optsize { 75; CHECK-LABEL: @example2( 76; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt i32 [[N:%.*]], 0 77; CHECK-NEXT: br i1 [[TMP1]], label [[DOTLR_PH5_PREHEADER:%.*]], label [[DOTPREHEADER:%.*]] 78; CHECK: .lr.ph5.preheader: 79; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 80; CHECK: vector.ph: 81; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[N]], -1 82; CHECK-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64 83; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw nsw i64 [[TMP3]], 4 84; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N_RND_UP]], 8589934588 85; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP3]], i64 0 86; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer 87; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 88; CHECK: vector.body: 89; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE6:%.*]] ] 90; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE6]] ] 91; CHECK-NEXT: [[TMP4:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]] 92; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i64 0 93; CHECK-NEXT: br i1 [[TMP5]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] 94; CHECK: pred.store.if: 95; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[INDEX]] 96; CHECK-NEXT: store i32 [[X:%.*]], i32* [[TMP6]], align 16 97; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]] 98; CHECK: pred.store.continue: 99; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP4]], i64 1 100; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]] 101; CHECK: pred.store.if1: 102; CHECK-NEXT: [[TMP8:%.*]] = or i64 [[INDEX]], 1 103; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[TMP8]] 104; CHECK-NEXT: store i32 [[X]], i32* [[TMP9]], align 4 105; CHECK-NEXT: br label [[PRED_STORE_CONTINUE2]] 106; CHECK: pred.store.continue2: 107; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP4]], i64 2 108; CHECK-NEXT: br i1 [[TMP10]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]] 109; CHECK: pred.store.if3: 110; CHECK-NEXT: [[TMP11:%.*]] = or i64 [[INDEX]], 2 111; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[TMP11]] 112; CHECK-NEXT: store i32 [[X]], i32* [[TMP12]], align 8 113; CHECK-NEXT: br label [[PRED_STORE_CONTINUE4]] 114; CHECK: pred.store.continue4: 115; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP4]], i64 3 116; CHECK-NEXT: br i1 [[TMP13]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]] 117; CHECK: pred.store.if5: 118; CHECK-NEXT: [[TMP14:%.*]] = or i64 [[INDEX]], 3 119; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[TMP14]] 120; CHECK-NEXT: store i32 [[X]], i32* [[TMP15]], align 4 121; CHECK-NEXT: br label [[PRED_STORE_CONTINUE6]] 122; CHECK: pred.store.continue6: 123; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 124; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4> 125; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 126; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] 127; CHECK: middle.block: 128; CHECK-NEXT: br i1 true, label [[DOT_PREHEADER_CRIT_EDGE:%.*]], label [[SCALAR_PH]] 129; CHECK: scalar.ph: 130; CHECK-NEXT: br label [[DOTLR_PH5:%.*]] 131; CHECK: ..preheader_crit_edge: 132; CHECK-NEXT: [[PHITMP:%.*]] = sext i32 [[N]] to i64 133; CHECK-NEXT: br label [[DOTPREHEADER]] 134; CHECK: .preheader: 135; CHECK-NEXT: [[I_0_LCSSA:%.*]] = phi i64 [ [[PHITMP]], [[DOT_PREHEADER_CRIT_EDGE]] ], [ 0, [[TMP0:%.*]] ] 136; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i32 [[N]], 0 137; CHECK-NEXT: br i1 [[TMP17]], label [[DOT_CRIT_EDGE:%.*]], label [[DOTLR_PH_PREHEADER:%.*]] 138; CHECK: .lr.ph.preheader: 139; CHECK-NEXT: br i1 false, label [[SCALAR_PH8:%.*]], label [[VECTOR_PH9:%.*]] 140; CHECK: vector.ph9: 141; CHECK-NEXT: [[TMP18:%.*]] = add i32 [[N]], -1 142; CHECK-NEXT: [[TMP19:%.*]] = zext i32 [[TMP18]] to i64 143; CHECK-NEXT: [[N_RND_UP10:%.*]] = add nuw nsw i64 [[TMP19]], 4 144; CHECK-NEXT: [[N_VEC12:%.*]] = and i64 [[N_RND_UP10]], 8589934588 145; CHECK-NEXT: [[BROADCAST_SPLATINSERT17:%.*]] = insertelement <4 x i64> poison, i64 [[TMP19]], i64 0 146; CHECK-NEXT: [[BROADCAST_SPLAT18:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT17]], <4 x i64> poison, <4 x i32> zeroinitializer 147; CHECK-NEXT: br label [[VECTOR_BODY19:%.*]] 148; CHECK: vector.body19: 149; CHECK-NEXT: [[INDEX20:%.*]] = phi i64 [ 0, [[VECTOR_PH9]] ], [ [[INDEX_NEXT31:%.*]], [[PRED_STORE_CONTINUE30:%.*]] ] 150; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 [[I_0_LCSSA]], [[INDEX20]] 151; CHECK-NEXT: [[BROADCAST_SPLATINSERT21:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX20]], i64 0 152; CHECK-NEXT: [[BROADCAST_SPLAT22:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT21]], <4 x i64> poison, <4 x i32> zeroinitializer 153; CHECK-NEXT: [[VEC_IV:%.*]] = or <4 x i64> [[BROADCAST_SPLAT22]], <i64 0, i64 1, i64 2, i64 3> 154; CHECK-NEXT: [[TMP20:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT18]] 155; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP20]], i64 0 156; CHECK-NEXT: br i1 [[TMP21]], label [[PRED_STORE_IF23:%.*]], label [[PRED_STORE_CONTINUE24:%.*]] 157; CHECK: pred.store.if23: 158; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[OFFSET_IDX]] 159; CHECK-NEXT: [[TMP23:%.*]] = load i32, i32* [[TMP22]], align 4 160; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 [[OFFSET_IDX]] 161; CHECK-NEXT: [[TMP25:%.*]] = load i32, i32* [[TMP24]], align 4 162; CHECK-NEXT: [[TMP26:%.*]] = and i32 [[TMP25]], [[TMP23]] 163; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 [[OFFSET_IDX]] 164; CHECK-NEXT: store i32 [[TMP26]], i32* [[TMP27]], align 4 165; CHECK-NEXT: br label [[PRED_STORE_CONTINUE24]] 166; CHECK: pred.store.continue24: 167; CHECK-NEXT: [[TMP28:%.*]] = extractelement <4 x i1> [[TMP20]], i64 1 168; CHECK-NEXT: br i1 [[TMP28]], label [[PRED_STORE_IF25:%.*]], label [[PRED_STORE_CONTINUE26:%.*]] 169; CHECK: pred.store.if25: 170; CHECK-NEXT: [[TMP29:%.*]] = add i64 [[OFFSET_IDX]], 1 171; CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[TMP29]] 172; CHECK-NEXT: [[TMP31:%.*]] = load i32, i32* [[TMP30]], align 4 173; CHECK-NEXT: [[TMP32:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 [[TMP29]] 174; CHECK-NEXT: [[TMP33:%.*]] = load i32, i32* [[TMP32]], align 4 175; CHECK-NEXT: [[TMP34:%.*]] = and i32 [[TMP33]], [[TMP31]] 176; CHECK-NEXT: [[TMP35:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 [[TMP29]] 177; CHECK-NEXT: store i32 [[TMP34]], i32* [[TMP35]], align 4 178; CHECK-NEXT: br label [[PRED_STORE_CONTINUE26]] 179; CHECK: pred.store.continue26: 180; CHECK-NEXT: [[TMP36:%.*]] = extractelement <4 x i1> [[TMP20]], i64 2 181; CHECK-NEXT: br i1 [[TMP36]], label [[PRED_STORE_IF27:%.*]], label [[PRED_STORE_CONTINUE28:%.*]] 182; CHECK: pred.store.if27: 183; CHECK-NEXT: [[TMP37:%.*]] = add i64 [[OFFSET_IDX]], 2 184; CHECK-NEXT: [[TMP38:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[TMP37]] 185; CHECK-NEXT: [[TMP39:%.*]] = load i32, i32* [[TMP38]], align 4 186; CHECK-NEXT: [[TMP40:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 [[TMP37]] 187; CHECK-NEXT: [[TMP41:%.*]] = load i32, i32* [[TMP40]], align 4 188; CHECK-NEXT: [[TMP42:%.*]] = and i32 [[TMP41]], [[TMP39]] 189; CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 [[TMP37]] 190; CHECK-NEXT: store i32 [[TMP42]], i32* [[TMP43]], align 4 191; CHECK-NEXT: br label [[PRED_STORE_CONTINUE28]] 192; CHECK: pred.store.continue28: 193; CHECK-NEXT: [[TMP44:%.*]] = extractelement <4 x i1> [[TMP20]], i64 3 194; CHECK-NEXT: br i1 [[TMP44]], label [[PRED_STORE_IF29:%.*]], label [[PRED_STORE_CONTINUE30]] 195; CHECK: pred.store.if29: 196; CHECK-NEXT: [[TMP45:%.*]] = add i64 [[OFFSET_IDX]], 3 197; CHECK-NEXT: [[TMP46:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[TMP45]] 198; CHECK-NEXT: [[TMP47:%.*]] = load i32, i32* [[TMP46]], align 4 199; CHECK-NEXT: [[TMP48:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 [[TMP45]] 200; CHECK-NEXT: [[TMP49:%.*]] = load i32, i32* [[TMP48]], align 4 201; CHECK-NEXT: [[TMP50:%.*]] = and i32 [[TMP49]], [[TMP47]] 202; CHECK-NEXT: [[TMP51:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 [[TMP45]] 203; CHECK-NEXT: store i32 [[TMP50]], i32* [[TMP51]], align 4 204; CHECK-NEXT: br label [[PRED_STORE_CONTINUE30]] 205; CHECK: pred.store.continue30: 206; CHECK-NEXT: [[INDEX_NEXT31]] = add i64 [[INDEX20]], 4 207; CHECK-NEXT: [[TMP52:%.*]] = icmp eq i64 [[INDEX_NEXT31]], [[N_VEC12]] 208; CHECK-NEXT: br i1 [[TMP52]], label [[MIDDLE_BLOCK7:%.*]], label [[VECTOR_BODY19]], !llvm.loop [[LOOP5:![0-9]+]] 209; CHECK: middle.block7: 210; CHECK-NEXT: br i1 true, label [[DOT_CRIT_EDGE_LOOPEXIT:%.*]], label [[SCALAR_PH8]] 211; CHECK: scalar.ph8: 212; CHECK-NEXT: br label [[DOTLR_PH:%.*]] 213; CHECK: .lr.ph5: 214; CHECK-NEXT: br i1 poison, label [[DOT_PREHEADER_CRIT_EDGE]], label [[DOTLR_PH5]], !llvm.loop [[LOOP6:![0-9]+]] 215; CHECK: .lr.ph: 216; CHECK-NEXT: br i1 poison, label [[DOT_CRIT_EDGE_LOOPEXIT]], label [[DOTLR_PH]], !llvm.loop [[LOOP7:![0-9]+]] 217; CHECK: ._crit_edge.loopexit: 218; CHECK-NEXT: br label [[DOT_CRIT_EDGE]] 219; CHECK: ._crit_edge: 220; CHECK-NEXT: ret void 221; 222 %1 = icmp sgt i32 %n, 0 223 br i1 %1, label %.lr.ph5, label %.preheader 224 225..preheader_crit_edge: ; preds = %.lr.ph5 226 %phitmp = sext i32 %n to i64 227 br label %.preheader 228 229.preheader: ; preds = %..preheader_crit_edge, %0 230 %i.0.lcssa = phi i64 [ %phitmp, %..preheader_crit_edge ], [ 0, %0 ] 231 %2 = icmp eq i32 %n, 0 232 br i1 %2, label %._crit_edge, label %.lr.ph 233 234.lr.ph5: ; preds = %0, %.lr.ph5 235 %indvars.iv6 = phi i64 [ %indvars.iv.next7, %.lr.ph5 ], [ 0, %0 ] 236 %3 = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 %indvars.iv6 237 store i32 %x, i32* %3, align 4 238 %indvars.iv.next7 = add i64 %indvars.iv6, 1 239 %lftr.wideiv = trunc i64 %indvars.iv.next7 to i32 240 %exitcond = icmp eq i32 %lftr.wideiv, %n 241 br i1 %exitcond, label %..preheader_crit_edge, label %.lr.ph5 242 243.lr.ph: ; preds = %.preheader, %.lr.ph 244 %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ %i.0.lcssa, %.preheader ] 245 %.02 = phi i32 [ %4, %.lr.ph ], [ %n, %.preheader ] 246 %4 = add nsw i32 %.02, -1 247 %5 = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 %indvars.iv 248 %6 = load i32, i32* %5, align 4 249 %7 = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 %indvars.iv 250 %8 = load i32, i32* %7, align 4 251 %9 = and i32 %8, %6 252 %10 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv 253 store i32 %9, i32* %10, align 4 254 %indvars.iv.next = add i64 %indvars.iv, 1 255 %11 = icmp eq i32 %4, 0 256 br i1 %11, label %._crit_edge, label %.lr.ph 257 258._crit_edge: ; preds = %.lr.ph, %.preheader 259 ret void 260} 261 262; Loop has no primary induction as its integer IV has step -1 starting at 263; unknown N, but can still be vectorized. 264define void @example3(i32 %n, i32* noalias nocapture %p, i32* noalias nocapture %q) optsize { 265; CHECK-LABEL: @example3( 266; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[N:%.*]], 0 267; CHECK-NEXT: br i1 [[TMP1]], label [[DOT_CRIT_EDGE:%.*]], label [[DOTLR_PH_PREHEADER:%.*]] 268; CHECK: .lr.ph.preheader: 269; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 270; CHECK: vector.ph: 271; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[N]], -1 272; CHECK-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64 273; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw nsw i64 [[TMP3]], 4 274; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N_RND_UP]], 8589934588 275; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP3]], i64 0 276; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer 277; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 278; CHECK: vector.body: 279; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE19:%.*]] ] 280; CHECK-NEXT: [[BROADCAST_SPLATINSERT12:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0 281; CHECK-NEXT: [[BROADCAST_SPLAT13:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT12]], <4 x i64> poison, <4 x i32> zeroinitializer 282; CHECK-NEXT: [[VEC_IV:%.*]] = or <4 x i64> [[BROADCAST_SPLAT13]], <i64 0, i64 1, i64 2, i64 3> 283; CHECK-NEXT: [[TMP4:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]] 284; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i64 0 285; CHECK-NEXT: br i1 [[TMP5]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] 286; CHECK: pred.store.if: 287; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i32, i32* [[P:%.*]], i64 [[INDEX]] 288; CHECK-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i32, i32* [[Q:%.*]], i64 [[INDEX]] 289; CHECK-NEXT: [[TMP6:%.*]] = load i32, i32* [[NEXT_GEP8]], align 16 290; CHECK-NEXT: store i32 [[TMP6]], i32* [[NEXT_GEP]], align 16 291; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]] 292; CHECK: pred.store.continue: 293; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP4]], i64 1 294; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_STORE_IF14:%.*]], label [[PRED_STORE_CONTINUE15:%.*]] 295; CHECK: pred.store.if14: 296; CHECK-NEXT: [[TMP8:%.*]] = or i64 [[INDEX]], 1 297; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i32, i32* [[P]], i64 [[TMP8]] 298; CHECK-NEXT: [[TMP9:%.*]] = or i64 [[INDEX]], 1 299; CHECK-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i32, i32* [[Q]], i64 [[TMP9]] 300; CHECK-NEXT: [[TMP10:%.*]] = load i32, i32* [[NEXT_GEP9]], align 16 301; CHECK-NEXT: store i32 [[TMP10]], i32* [[NEXT_GEP5]], align 16 302; CHECK-NEXT: br label [[PRED_STORE_CONTINUE15]] 303; CHECK: pred.store.continue15: 304; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP4]], i64 2 305; CHECK-NEXT: br i1 [[TMP11]], label [[PRED_STORE_IF16:%.*]], label [[PRED_STORE_CONTINUE17:%.*]] 306; CHECK: pred.store.if16: 307; CHECK-NEXT: [[TMP12:%.*]] = or i64 [[INDEX]], 2 308; CHECK-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i32, i32* [[P]], i64 [[TMP12]] 309; CHECK-NEXT: [[TMP13:%.*]] = or i64 [[INDEX]], 2 310; CHECK-NEXT: [[NEXT_GEP10:%.*]] = getelementptr i32, i32* [[Q]], i64 [[TMP13]] 311; CHECK-NEXT: [[TMP14:%.*]] = load i32, i32* [[NEXT_GEP10]], align 16 312; CHECK-NEXT: store i32 [[TMP14]], i32* [[NEXT_GEP6]], align 16 313; CHECK-NEXT: br label [[PRED_STORE_CONTINUE17]] 314; CHECK: pred.store.continue17: 315; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP4]], i64 3 316; CHECK-NEXT: br i1 [[TMP15]], label [[PRED_STORE_IF18:%.*]], label [[PRED_STORE_CONTINUE19]] 317; CHECK: pred.store.if18: 318; CHECK-NEXT: [[TMP16:%.*]] = or i64 [[INDEX]], 3 319; CHECK-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i32, i32* [[P]], i64 [[TMP16]] 320; CHECK-NEXT: [[TMP17:%.*]] = or i64 [[INDEX]], 3 321; CHECK-NEXT: [[NEXT_GEP11:%.*]] = getelementptr i32, i32* [[Q]], i64 [[TMP17]] 322; CHECK-NEXT: [[TMP18:%.*]] = load i32, i32* [[NEXT_GEP11]], align 16 323; CHECK-NEXT: store i32 [[TMP18]], i32* [[NEXT_GEP7]], align 16 324; CHECK-NEXT: br label [[PRED_STORE_CONTINUE19]] 325; CHECK: pred.store.continue19: 326; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 327; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 328; CHECK-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] 329; CHECK: middle.block: 330; CHECK-NEXT: br i1 true, label [[DOT_CRIT_EDGE_LOOPEXIT:%.*]], label [[SCALAR_PH]] 331; CHECK: scalar.ph: 332; CHECK-NEXT: br label [[DOTLR_PH:%.*]] 333; CHECK: .lr.ph: 334; CHECK-NEXT: br i1 poison, label [[DOT_CRIT_EDGE_LOOPEXIT]], label [[DOTLR_PH]], !llvm.loop [[LOOP9:![0-9]+]] 335; CHECK: ._crit_edge.loopexit: 336; CHECK-NEXT: br label [[DOT_CRIT_EDGE]] 337; CHECK: ._crit_edge: 338; CHECK-NEXT: ret void 339; 340 %1 = icmp eq i32 %n, 0 341 br i1 %1, label %._crit_edge, label %.lr.ph 342 343.lr.ph: ; preds = %0, %.lr.ph 344 %.05 = phi i32 [ %2, %.lr.ph ], [ %n, %0 ] 345 %.014 = phi i32* [ %5, %.lr.ph ], [ %p, %0 ] 346 %.023 = phi i32* [ %3, %.lr.ph ], [ %q, %0 ] 347 %2 = add nsw i32 %.05, -1 348 %3 = getelementptr inbounds i32, i32* %.023, i64 1 349 %4 = load i32, i32* %.023, align 16 350 %5 = getelementptr inbounds i32, i32* %.014, i64 1 351 store i32 %4, i32* %.014, align 16 352 %6 = icmp eq i32 %2, 0 353 br i1 %6, label %._crit_edge, label %.lr.ph 354 355._crit_edge: ; preds = %.lr.ph, %0 356 ret void 357} 358 359; We can't vectorize this one because we need a runtime ptr check. 360define void @example23(i16* nocapture %src, i32* nocapture %dst) optsize { 361; CHECK-LABEL: @example23( 362; CHECK-NEXT: br label [[TMP1:%.*]] 363; CHECK: 1: 364; CHECK-NEXT: [[DOT04:%.*]] = phi i16* [ [[SRC:%.*]], [[TMP0:%.*]] ], [ [[TMP2:%.*]], [[TMP1]] ] 365; CHECK-NEXT: [[DOT013:%.*]] = phi i32* [ [[DST:%.*]], [[TMP0]] ], [ [[TMP6:%.*]], [[TMP1]] ] 366; CHECK-NEXT: [[I_02:%.*]] = phi i32 [ 0, [[TMP0]] ], [ [[TMP7:%.*]], [[TMP1]] ] 367; CHECK-NEXT: [[TMP2]] = getelementptr inbounds i16, i16* [[DOT04]], i64 1 368; CHECK-NEXT: [[TMP3:%.*]] = load i16, i16* [[DOT04]], align 2 369; CHECK-NEXT: [[TMP4:%.*]] = zext i16 [[TMP3]] to i32 370; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i32 [[TMP4]], 7 371; CHECK-NEXT: [[TMP6]] = getelementptr inbounds i32, i32* [[DOT013]], i64 1 372; CHECK-NEXT: store i32 [[TMP5]], i32* [[DOT013]], align 4 373; CHECK-NEXT: [[TMP7]] = add nuw nsw i32 [[I_02]], 1 374; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[TMP7]], 256 375; CHECK-NEXT: br i1 [[EXITCOND]], label [[TMP8:%.*]], label [[TMP1]] 376; CHECK: 8: 377; CHECK-NEXT: ret void 378; 379 br label %1 380 381; <label>:1 ; preds = %1, %0 382 %.04 = phi i16* [ %src, %0 ], [ %2, %1 ] 383 %.013 = phi i32* [ %dst, %0 ], [ %6, %1 ] 384 %i.02 = phi i32 [ 0, %0 ], [ %7, %1 ] 385 %2 = getelementptr inbounds i16, i16* %.04, i64 1 386 %3 = load i16, i16* %.04, align 2 387 %4 = zext i16 %3 to i32 388 %5 = shl nuw nsw i32 %4, 7 389 %6 = getelementptr inbounds i32, i32* %.013, i64 1 390 store i32 %5, i32* %.013, align 4 391 %7 = add nsw i32 %i.02, 1 392 %exitcond = icmp eq i32 %7, 256 393 br i1 %exitcond, label %8, label %1 394 395; <label>:8 ; preds = %1 396 ret void 397} 398 399 400; We CAN vectorize this example because the pointers are marked as noalias. 401define void @example23b(i16* noalias nocapture %src, i32* noalias nocapture %dst) optsize { 402; CHECK-LABEL: @example23b( 403; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 404; CHECK: vector.ph: 405; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 406; CHECK: vector.body: 407; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 408; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i16, i16* [[SRC:%.*]], i64 [[INDEX]] 409; CHECK-NEXT: [[NEXT_GEP4:%.*]] = getelementptr i32, i32* [[DST:%.*]], i64 [[INDEX]] 410; CHECK-NEXT: [[TMP1:%.*]] = bitcast i16* [[NEXT_GEP]] to <4 x i16>* 411; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 2 412; CHECK-NEXT: [[TMP2:%.*]] = zext <4 x i16> [[WIDE_LOAD]] to <4 x i32> 413; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw <4 x i32> [[TMP2]], <i32 7, i32 7, i32 7, i32 7> 414; CHECK-NEXT: [[TMP4:%.*]] = bitcast i32* [[NEXT_GEP4]] to <4 x i32>* 415; CHECK-NEXT: store <4 x i32> [[TMP3]], <4 x i32>* [[TMP4]], align 4 416; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 417; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256 418; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] 419; CHECK: middle.block: 420; CHECK-NEXT: br i1 true, label [[TMP7:%.*]], label [[SCALAR_PH]] 421; CHECK: scalar.ph: 422; CHECK-NEXT: br label [[TMP6:%.*]] 423; CHECK: 6: 424; CHECK-NEXT: br i1 poison, label [[TMP7]], label [[TMP6]], !llvm.loop [[LOOP11:![0-9]+]] 425; CHECK: 7: 426; CHECK-NEXT: ret void 427; 428 br label %1 429 430; <label>:1 ; preds = %1, %0 431 %.04 = phi i16* [ %src, %0 ], [ %2, %1 ] 432 %.013 = phi i32* [ %dst, %0 ], [ %6, %1 ] 433 %i.02 = phi i32 [ 0, %0 ], [ %7, %1 ] 434 %2 = getelementptr inbounds i16, i16* %.04, i64 1 435 %3 = load i16, i16* %.04, align 2 436 %4 = zext i16 %3 to i32 437 %5 = shl nuw nsw i32 %4, 7 438 %6 = getelementptr inbounds i32, i32* %.013, i64 1 439 store i32 %5, i32* %.013, align 4 440 %7 = add nsw i32 %i.02, 1 441 %exitcond = icmp eq i32 %7, 256 442 br i1 %exitcond, label %8, label %1 443 444; <label>:8 ; preds = %1 445 ret void 446} 447 448; We CAN vectorize this example by folding the tail it entails. 449define void @example23c(i16* noalias nocapture %src, i32* noalias nocapture %dst) optsize { 450; CHECK-LABEL: @example23c( 451; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 452; CHECK: vector.ph: 453; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 454; CHECK: vector.body: 455; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE16:%.*]] ] 456; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0 457; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer 458; CHECK-NEXT: [[VEC_IV:%.*]] = or <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3> 459; CHECK-NEXT: [[TMP1:%.*]] = icmp ult <4 x i64> [[VEC_IV]], <i64 257, i64 257, i64 257, i64 257> 460; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0 461; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] 462; CHECK: pred.store.if: 463; CHECK-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i32, i32* [[DST:%.*]], i64 [[INDEX]] 464; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i16, i16* [[SRC:%.*]], i64 [[INDEX]] 465; CHECK-NEXT: [[TMP3:%.*]] = load i16, i16* [[NEXT_GEP]], align 2 466; CHECK-NEXT: [[TMP4:%.*]] = zext i16 [[TMP3]] to i32 467; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i32 [[TMP4]], 7 468; CHECK-NEXT: store i32 [[TMP5]], i32* [[NEXT_GEP7]], align 4 469; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]] 470; CHECK: pred.store.continue: 471; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1 472; CHECK-NEXT: br i1 [[TMP6]], label [[PRED_STORE_IF11:%.*]], label [[PRED_STORE_CONTINUE12:%.*]] 473; CHECK: pred.store.if11: 474; CHECK-NEXT: [[TMP7:%.*]] = or i64 [[INDEX]], 1 475; CHECK-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i32, i32* [[DST]], i64 [[TMP7]] 476; CHECK-NEXT: [[TMP8:%.*]] = or i64 [[INDEX]], 1 477; CHECK-NEXT: [[NEXT_GEP4:%.*]] = getelementptr i16, i16* [[SRC]], i64 [[TMP8]] 478; CHECK-NEXT: [[TMP9:%.*]] = load i16, i16* [[NEXT_GEP4]], align 2 479; CHECK-NEXT: [[TMP10:%.*]] = zext i16 [[TMP9]] to i32 480; CHECK-NEXT: [[TMP11:%.*]] = shl nuw nsw i32 [[TMP10]], 7 481; CHECK-NEXT: store i32 [[TMP11]], i32* [[NEXT_GEP8]], align 4 482; CHECK-NEXT: br label [[PRED_STORE_CONTINUE12]] 483; CHECK: pred.store.continue12: 484; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2 485; CHECK-NEXT: br i1 [[TMP12]], label [[PRED_STORE_IF13:%.*]], label [[PRED_STORE_CONTINUE14:%.*]] 486; CHECK: pred.store.if13: 487; CHECK-NEXT: [[TMP13:%.*]] = or i64 [[INDEX]], 2 488; CHECK-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i32, i32* [[DST]], i64 [[TMP13]] 489; CHECK-NEXT: [[TMP14:%.*]] = or i64 [[INDEX]], 2 490; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i16, i16* [[SRC]], i64 [[TMP14]] 491; CHECK-NEXT: [[TMP15:%.*]] = load i16, i16* [[NEXT_GEP5]], align 2 492; CHECK-NEXT: [[TMP16:%.*]] = zext i16 [[TMP15]] to i32 493; CHECK-NEXT: [[TMP17:%.*]] = shl nuw nsw i32 [[TMP16]], 7 494; CHECK-NEXT: store i32 [[TMP17]], i32* [[NEXT_GEP9]], align 4 495; CHECK-NEXT: br label [[PRED_STORE_CONTINUE14]] 496; CHECK: pred.store.continue14: 497; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3 498; CHECK-NEXT: br i1 [[TMP18]], label [[PRED_STORE_IF15:%.*]], label [[PRED_STORE_CONTINUE16]] 499; CHECK: pred.store.if15: 500; CHECK-NEXT: [[TMP19:%.*]] = or i64 [[INDEX]], 3 501; CHECK-NEXT: [[NEXT_GEP10:%.*]] = getelementptr i32, i32* [[DST]], i64 [[TMP19]] 502; CHECK-NEXT: [[TMP20:%.*]] = or i64 [[INDEX]], 3 503; CHECK-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i16, i16* [[SRC]], i64 [[TMP20]] 504; CHECK-NEXT: [[TMP21:%.*]] = load i16, i16* [[NEXT_GEP6]], align 2 505; CHECK-NEXT: [[TMP22:%.*]] = zext i16 [[TMP21]] to i32 506; CHECK-NEXT: [[TMP23:%.*]] = shl nuw nsw i32 [[TMP22]], 7 507; CHECK-NEXT: store i32 [[TMP23]], i32* [[NEXT_GEP10]], align 4 508; CHECK-NEXT: br label [[PRED_STORE_CONTINUE16]] 509; CHECK: pred.store.continue16: 510; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 511; CHECK-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260 512; CHECK-NEXT: br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] 513; CHECK: middle.block: 514; CHECK-NEXT: br i1 true, label [[TMP26:%.*]], label [[SCALAR_PH]] 515; CHECK: scalar.ph: 516; CHECK-NEXT: br label [[TMP25:%.*]] 517; CHECK: 25: 518; CHECK-NEXT: br i1 poison, label [[TMP26]], label [[TMP25]], !llvm.loop [[LOOP13:![0-9]+]] 519; CHECK: 26: 520; CHECK-NEXT: ret void 521; 522 br label %1 523 524; <label>:1 ; preds = %1, %0 525 %.04 = phi i16* [ %src, %0 ], [ %2, %1 ] 526 %.013 = phi i32* [ %dst, %0 ], [ %6, %1 ] 527 %i.02 = phi i64 [ 0, %0 ], [ %7, %1 ] 528 %2 = getelementptr inbounds i16, i16* %.04, i64 1 529 %3 = load i16, i16* %.04, align 2 530 %4 = zext i16 %3 to i32 531 %5 = shl nuw nsw i32 %4, 7 532 %6 = getelementptr inbounds i32, i32* %.013, i64 1 533 store i32 %5, i32* %.013, align 4 534 %7 = add nsw i64 %i.02, 1 535 %exitcond = icmp eq i64 %7, 257 536 br i1 %exitcond, label %8, label %1 537 538; <label>:8 ; preds = %1 539 ret void 540} 541 542; We CAN'T vectorize this example because it would entail a tail and an 543; induction is used outside the loop. 544define i64 @example23d(i16* noalias nocapture %src, i32* noalias nocapture %dst) optsize { 545; CHECK-LABEL: @example23d( 546; CHECK-NEXT: br label [[TMP1:%.*]] 547; CHECK: 1: 548; CHECK-NEXT: [[DOT04:%.*]] = phi i16* [ [[SRC:%.*]], [[TMP0:%.*]] ], [ [[TMP2:%.*]], [[TMP1]] ] 549; CHECK-NEXT: [[DOT013:%.*]] = phi i32* [ [[DST:%.*]], [[TMP0]] ], [ [[TMP6:%.*]], [[TMP1]] ] 550; CHECK-NEXT: [[I_02:%.*]] = phi i64 [ 0, [[TMP0]] ], [ [[TMP7:%.*]], [[TMP1]] ] 551; CHECK-NEXT: [[TMP2]] = getelementptr inbounds i16, i16* [[DOT04]], i64 1 552; CHECK-NEXT: [[TMP3:%.*]] = load i16, i16* [[DOT04]], align 2 553; CHECK-NEXT: [[TMP4:%.*]] = zext i16 [[TMP3]] to i32 554; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i32 [[TMP4]], 7 555; CHECK-NEXT: [[TMP6]] = getelementptr inbounds i32, i32* [[DOT013]], i64 1 556; CHECK-NEXT: store i32 [[TMP5]], i32* [[DOT013]], align 4 557; CHECK-NEXT: [[TMP7]] = add nuw nsw i64 [[I_02]], 1 558; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[TMP7]], 257 559; CHECK-NEXT: br i1 [[EXITCOND]], label [[TMP8:%.*]], label [[TMP1]] 560; CHECK: 8: 561; CHECK-NEXT: ret i64 [[TMP7]] 562; 563 br label %1 564 565; <label>:1 ; preds = %1, %0 566 %.04 = phi i16* [ %src, %0 ], [ %2, %1 ] 567 %.013 = phi i32* [ %dst, %0 ], [ %6, %1 ] 568 %i.02 = phi i64 [ 0, %0 ], [ %7, %1 ] 569 %2 = getelementptr inbounds i16, i16* %.04, i64 1 570 %3 = load i16, i16* %.04, align 2 571 %4 = zext i16 %3 to i32 572 %5 = shl nuw nsw i32 %4, 7 573 %6 = getelementptr inbounds i32, i32* %.013, i64 1 574 store i32 %5, i32* %.013, align 4 575 %7 = add nsw i64 %i.02, 1 576 %exitcond = icmp eq i64 %7, 257 577 br i1 %exitcond, label %8, label %1 578 579; <label>:8 ; preds = %1 580 ret i64 %7 581} 582