1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py 2; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -dce -instcombine -S | FileCheck %s 3; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -dce -instcombine -S | FileCheck %s --check-prefix=UNROLL 4; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S | FileCheck %s --check-prefix=UNROLL-NO-IC 5; RUN: opt < %s -loop-vectorize -force-vector-width=1 -force-vector-interleave=2 -S | FileCheck %s --check-prefix=UNROLL-NO-VF 6; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s --check-prefix=SINK-AFTER 7; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s --check-prefix=NO-SINK-AFTER 8 9target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" 10 11; void recurrence_1(int *a, int *b, int n) { 12; for(int i = 0; i < n; i++) 13; b[i] = a[i] + a[i - 1] 14; } 15; 16; 17; 18define void @recurrence_1(i32* nocapture readonly %a, i32* nocapture %b, i32 %n) { 19; CHECK-LABEL: @recurrence_1( 20; CHECK-NEXT: entry: 21; CHECK-NEXT: br label [[FOR_PREHEADER:%.*]] 22; CHECK: for.preheader: 23; CHECK-NEXT: [[PRE_LOAD:%.*]] = load i32, i32* [[A:%.*]], align 4 24; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 25; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 26; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 27; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 3 28; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 29; CHECK: vector.memcheck: 30; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[N]], -1 31; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP3]] to i64 32; CHECK-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[TMP4]], 1 33; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B:%.*]], i64 [[TMP5]] 34; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i32, i32* [[A]], i64 1 35; CHECK-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[TMP4]], 2 36; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i32, i32* [[A]], i64 [[TMP6]] 37; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt i32* [[SCEVGEP5]], [[B]] 38; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult i32* [[SCEVGEP3]], [[SCEVGEP]] 39; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 40; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 41; CHECK: vector.ph: 42; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934588 43; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[PRE_LOAD]], i64 3 44; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 45; CHECK: vector.body: 46; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 47; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ] 48; CHECK-NEXT: [[TMP7:%.*]] = or i64 [[INDEX]], 1 49; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP7]] 50; CHECK-NEXT: [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>* 51; CHECK-NEXT: [[WIDE_LOAD]] = load <4 x i32>, <4 x i32>* [[TMP9]], align 4 52; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 53; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]] 54; CHECK-NEXT: [[TMP12:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[TMP10]] 55; CHECK-NEXT: [[TMP13:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>* 56; CHECK-NEXT: store <4 x i32> [[TMP12]], <4 x i32>* [[TMP13]], align 4 57; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 58; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 59; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] 60; CHECK: middle.block: 61; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 62; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 3 63; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]] 64; CHECK: scalar.ph: 65; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[PRE_LOAD]], [[VECTOR_MEMCHECK]] ], [ [[PRE_LOAD]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 66; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[FOR_PREHEADER]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ] 67; CHECK-NEXT: br label [[SCALAR_BODY:%.*]] 68; CHECK: scalar.body: 69; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP15:%.*]], [[SCALAR_BODY]] ] 70; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ] 71; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 72; CHECK-NEXT: [[ARRAYIDX32:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT]] 73; CHECK-NEXT: [[TMP15]] = load i32, i32* [[ARRAYIDX32]], align 4 74; CHECK-NEXT: [[ARRAYIDX34:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 75; CHECK-NEXT: [[ADD35:%.*]] = add i32 [[TMP15]], [[SCALAR_RECUR]] 76; CHECK-NEXT: store i32 [[ADD35]], i32* [[ARRAYIDX34]], align 4 77; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 78; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 79; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] 80; CHECK: for.exit: 81; CHECK-NEXT: ret void 82; 83; UNROLL-LABEL: @recurrence_1( 84; UNROLL-NEXT: entry: 85; UNROLL-NEXT: br label [[FOR_PREHEADER:%.*]] 86; UNROLL: for.preheader: 87; UNROLL-NEXT: [[PRE_LOAD:%.*]] = load i32, i32* [[A:%.*]], align 4 88; UNROLL-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 89; UNROLL-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 90; UNROLL-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 91; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 7 92; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 93; UNROLL: vector.memcheck: 94; UNROLL-NEXT: [[TMP3:%.*]] = add i32 [[N]], -1 95; UNROLL-NEXT: [[TMP4:%.*]] = zext i32 [[TMP3]] to i64 96; UNROLL-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[TMP4]], 1 97; UNROLL-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B:%.*]], i64 [[TMP5]] 98; UNROLL-NEXT: [[SCEVGEP3:%.*]] = getelementptr i32, i32* [[A]], i64 1 99; UNROLL-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[TMP4]], 2 100; UNROLL-NEXT: [[SCEVGEP5:%.*]] = getelementptr i32, i32* [[A]], i64 [[TMP6]] 101; UNROLL-NEXT: [[BOUND0:%.*]] = icmp ugt i32* [[SCEVGEP5]], [[B]] 102; UNROLL-NEXT: [[BOUND1:%.*]] = icmp ult i32* [[SCEVGEP3]], [[SCEVGEP]] 103; UNROLL-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 104; UNROLL-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 105; UNROLL: vector.ph: 106; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934584 107; UNROLL-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[PRE_LOAD]], i64 3 108; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 109; UNROLL: vector.body: 110; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 111; UNROLL-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD7:%.*]], [[VECTOR_BODY]] ] 112; UNROLL-NEXT: [[TMP7:%.*]] = or i64 [[INDEX]], 1 113; UNROLL-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP7]] 114; UNROLL-NEXT: [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>* 115; UNROLL-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP9]], align 4 116; UNROLL-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP8]], i64 4 117; UNROLL-NEXT: [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <4 x i32>* 118; UNROLL-NEXT: [[WIDE_LOAD7]] = load <4 x i32>, <4 x i32>* [[TMP11]], align 4 119; UNROLL-NEXT: [[TMP12:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 120; UNROLL-NEXT: [[TMP13:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> [[WIDE_LOAD7]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 121; UNROLL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]] 122; UNROLL-NEXT: [[TMP15:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[TMP12]] 123; UNROLL-NEXT: [[TMP16:%.*]] = add <4 x i32> [[WIDE_LOAD7]], [[TMP13]] 124; UNROLL-NEXT: [[TMP17:%.*]] = bitcast i32* [[TMP14]] to <4 x i32>* 125; UNROLL-NEXT: store <4 x i32> [[TMP15]], <4 x i32>* [[TMP17]], align 4 126; UNROLL-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[TMP14]], i64 4 127; UNROLL-NEXT: [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <4 x i32>* 128; UNROLL-NEXT: store <4 x i32> [[TMP16]], <4 x i32>* [[TMP19]], align 4 129; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 130; UNROLL-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 131; UNROLL-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] 132; UNROLL: middle.block: 133; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 134; UNROLL-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[WIDE_LOAD7]], i64 3 135; UNROLL-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]] 136; UNROLL: scalar.ph: 137; UNROLL-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[PRE_LOAD]], [[VECTOR_MEMCHECK]] ], [ [[PRE_LOAD]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 138; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[FOR_PREHEADER]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ] 139; UNROLL-NEXT: br label [[SCALAR_BODY:%.*]] 140; UNROLL: scalar.body: 141; UNROLL-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP21:%.*]], [[SCALAR_BODY]] ] 142; UNROLL-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ] 143; UNROLL-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 144; UNROLL-NEXT: [[ARRAYIDX32:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT]] 145; UNROLL-NEXT: [[TMP21]] = load i32, i32* [[ARRAYIDX32]], align 4 146; UNROLL-NEXT: [[ARRAYIDX34:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 147; UNROLL-NEXT: [[ADD35:%.*]] = add i32 [[TMP21]], [[SCALAR_RECUR]] 148; UNROLL-NEXT: store i32 [[ADD35]], i32* [[ARRAYIDX34]], align 4 149; UNROLL-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 150; UNROLL-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 151; UNROLL-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] 152; UNROLL: for.exit: 153; UNROLL-NEXT: ret void 154; 155; UNROLL-NO-IC-LABEL: @recurrence_1( 156; UNROLL-NO-IC-NEXT: entry: 157; UNROLL-NO-IC-NEXT: [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8* 158; UNROLL-NO-IC-NEXT: br label [[FOR_PREHEADER:%.*]] 159; UNROLL-NO-IC: for.preheader: 160; UNROLL-NO-IC-NEXT: [[ARRAYIDX_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 0 161; UNROLL-NO-IC-NEXT: [[PRE_LOAD:%.*]] = load i32, i32* [[ARRAYIDX_PHI_TRANS_INSERT]], align 4 162; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 163; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 164; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 165; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 8 166; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 167; UNROLL-NO-IC: vector.memcheck: 168; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add i32 [[N]], -1 169; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = zext i32 [[TMP3]] to i64 170; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[TMP4]], 1 171; UNROLL-NO-IC-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[TMP5]] 172; UNROLL-NO-IC-NEXT: [[SCEVGEP2:%.*]] = bitcast i32* [[SCEVGEP]] to i8* 173; UNROLL-NO-IC-NEXT: [[SCEVGEP3:%.*]] = getelementptr i32, i32* [[A]], i64 1 174; UNROLL-NO-IC-NEXT: [[SCEVGEP34:%.*]] = bitcast i32* [[SCEVGEP3]] to i8* 175; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[TMP4]], 2 176; UNROLL-NO-IC-NEXT: [[SCEVGEP5:%.*]] = getelementptr i32, i32* [[A]], i64 [[TMP6]] 177; UNROLL-NO-IC-NEXT: [[SCEVGEP56:%.*]] = bitcast i32* [[SCEVGEP5]] to i8* 178; UNROLL-NO-IC-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[B1]], [[SCEVGEP56]] 179; UNROLL-NO-IC-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP34]], [[SCEVGEP2]] 180; UNROLL-NO-IC-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 181; UNROLL-NO-IC-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 182; UNROLL-NO-IC: vector.ph: 183; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 8 184; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 185; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[PRE_LOAD]], i32 3 186; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 187; UNROLL-NO-IC: vector.body: 188; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 189; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD7:%.*]], [[VECTOR_BODY]] ] 190; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 0 191; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 4 192; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = add nuw nsw i64 [[TMP7]], 1 193; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = add nuw nsw i64 [[TMP8]], 1 194; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP9]] 195; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP10]] 196; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[TMP11]], i32 0 197; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <4 x i32>* 198; UNROLL-NO-IC-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP14]], align 4 199; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, i32* [[TMP11]], i32 4 200; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = bitcast i32* [[TMP15]] to <4 x i32>* 201; UNROLL-NO-IC-NEXT: [[WIDE_LOAD7]] = load <4 x i32>, <4 x i32>* [[TMP16]], align 4 202; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 203; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> [[WIDE_LOAD7]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 204; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP7]] 205; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP8]] 206; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[TMP17]] 207; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = add <4 x i32> [[WIDE_LOAD7]], [[TMP18]] 208; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[TMP19]], i32 0 209; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <4 x i32>* 210; UNROLL-NO-IC-NEXT: store <4 x i32> [[TMP21]], <4 x i32>* [[TMP24]], align 4 211; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, i32* [[TMP19]], i32 4 212; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = bitcast i32* [[TMP25]] to <4 x i32>* 213; UNROLL-NO-IC-NEXT: store <4 x i32> [[TMP22]], <4 x i32>* [[TMP26]], align 4 214; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 215; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 216; UNROLL-NO-IC-NEXT: br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] 217; UNROLL-NO-IC: middle.block: 218; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 219; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[WIDE_LOAD7]], i32 3 220; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[WIDE_LOAD7]], i32 2 221; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]] 222; UNROLL-NO-IC: scalar.ph: 223; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[PRE_LOAD]], [[VECTOR_MEMCHECK]] ], [ [[PRE_LOAD]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 224; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ] 225; UNROLL-NO-IC-NEXT: br label [[SCALAR_BODY:%.*]] 226; UNROLL-NO-IC: scalar.body: 227; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP28:%.*]], [[SCALAR_BODY]] ] 228; UNROLL-NO-IC-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ] 229; UNROLL-NO-IC-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 230; UNROLL-NO-IC-NEXT: [[ARRAYIDX32:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT]] 231; UNROLL-NO-IC-NEXT: [[TMP28]] = load i32, i32* [[ARRAYIDX32]], align 4 232; UNROLL-NO-IC-NEXT: [[ARRAYIDX34:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 233; UNROLL-NO-IC-NEXT: [[ADD35:%.*]] = add i32 [[TMP28]], [[SCALAR_RECUR]] 234; UNROLL-NO-IC-NEXT: store i32 [[ADD35]], i32* [[ARRAYIDX34]], align 4 235; UNROLL-NO-IC-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 236; UNROLL-NO-IC-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 237; UNROLL-NO-IC-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] 238; UNROLL-NO-IC: for.exit: 239; UNROLL-NO-IC-NEXT: ret void 240; 241; UNROLL-NO-VF-LABEL: @recurrence_1( 242; UNROLL-NO-VF-NEXT: entry: 243; UNROLL-NO-VF-NEXT: [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8* 244; UNROLL-NO-VF-NEXT: br label [[FOR_PREHEADER:%.*]] 245; UNROLL-NO-VF: for.preheader: 246; UNROLL-NO-VF-NEXT: [[ARRAYIDX_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 0 247; UNROLL-NO-VF-NEXT: [[PRE_LOAD:%.*]] = load i32, i32* [[ARRAYIDX_PHI_TRANS_INSERT]], align 4 248; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 249; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 250; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 251; UNROLL-NO-VF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 2 252; UNROLL-NO-VF-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 253; UNROLL-NO-VF: vector.memcheck: 254; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = add i32 [[N]], -1 255; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = zext i32 [[TMP3]] to i64 256; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[TMP4]], 1 257; UNROLL-NO-VF-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[TMP5]] 258; UNROLL-NO-VF-NEXT: [[SCEVGEP2:%.*]] = bitcast i32* [[SCEVGEP]] to i8* 259; UNROLL-NO-VF-NEXT: [[SCEVGEP3:%.*]] = getelementptr i32, i32* [[A]], i64 1 260; UNROLL-NO-VF-NEXT: [[SCEVGEP34:%.*]] = bitcast i32* [[SCEVGEP3]] to i8* 261; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[TMP4]], 2 262; UNROLL-NO-VF-NEXT: [[SCEVGEP5:%.*]] = getelementptr i32, i32* [[A]], i64 [[TMP6]] 263; UNROLL-NO-VF-NEXT: [[SCEVGEP56:%.*]] = bitcast i32* [[SCEVGEP5]] to i8* 264; UNROLL-NO-VF-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[B1]], [[SCEVGEP56]] 265; UNROLL-NO-VF-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP34]], [[SCEVGEP2]] 266; UNROLL-NO-VF-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 267; UNROLL-NO-VF-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 268; UNROLL-NO-VF: vector.ph: 269; UNROLL-NO-VF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 2 270; UNROLL-NO-VF-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 271; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 272; UNROLL-NO-VF: vector.body: 273; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 274; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i32 [ [[PRE_LOAD]], [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ] 275; UNROLL-NO-VF-NEXT: [[INDUCTION:%.*]] = add i64 [[INDEX]], 0 276; UNROLL-NO-VF-NEXT: [[INDUCTION7:%.*]] = add i64 [[INDEX]], 1 277; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = add nuw nsw i64 [[INDUCTION]], 1 278; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = add nuw nsw i64 [[INDUCTION7]], 1 279; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP7]] 280; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP8]] 281; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = load i32, i32* [[TMP9]], align 4 282; UNROLL-NO-VF-NEXT: [[TMP12]] = load i32, i32* [[TMP10]], align 4 283; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDUCTION]] 284; UNROLL-NO-VF-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDUCTION7]] 285; UNROLL-NO-VF-NEXT: [[TMP15:%.*]] = add i32 [[TMP11]], [[VECTOR_RECUR]] 286; UNROLL-NO-VF-NEXT: [[TMP16:%.*]] = add i32 [[TMP12]], [[TMP11]] 287; UNROLL-NO-VF-NEXT: store i32 [[TMP15]], i32* [[TMP13]], align 4 288; UNROLL-NO-VF-NEXT: store i32 [[TMP16]], i32* [[TMP14]], align 4 289; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 290; UNROLL-NO-VF-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 291; UNROLL-NO-VF-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] 292; UNROLL-NO-VF: middle.block: 293; UNROLL-NO-VF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 294; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]] 295; UNROLL-NO-VF: scalar.ph: 296; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[PRE_LOAD]], [[VECTOR_MEMCHECK]] ], [ [[PRE_LOAD]], [[FOR_PREHEADER]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ] 297; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ] 298; UNROLL-NO-VF-NEXT: br label [[SCALAR_BODY:%.*]] 299; UNROLL-NO-VF: scalar.body: 300; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP18:%.*]], [[SCALAR_BODY]] ] 301; UNROLL-NO-VF-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ] 302; UNROLL-NO-VF-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 303; UNROLL-NO-VF-NEXT: [[ARRAYIDX32:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT]] 304; UNROLL-NO-VF-NEXT: [[TMP18]] = load i32, i32* [[ARRAYIDX32]], align 4 305; UNROLL-NO-VF-NEXT: [[ARRAYIDX34:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 306; UNROLL-NO-VF-NEXT: [[ADD35:%.*]] = add i32 [[TMP18]], [[SCALAR_RECUR]] 307; UNROLL-NO-VF-NEXT: store i32 [[ADD35]], i32* [[ARRAYIDX34]], align 4 308; UNROLL-NO-VF-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 309; UNROLL-NO-VF-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 310; UNROLL-NO-VF-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] 311; UNROLL-NO-VF: for.exit: 312; UNROLL-NO-VF-NEXT: ret void 313; 314; SINK-AFTER-LABEL: @recurrence_1( 315; SINK-AFTER-NEXT: entry: 316; SINK-AFTER-NEXT: [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8* 317; SINK-AFTER-NEXT: br label [[FOR_PREHEADER:%.*]] 318; SINK-AFTER: for.preheader: 319; SINK-AFTER-NEXT: [[ARRAYIDX_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 0 320; SINK-AFTER-NEXT: [[PRE_LOAD:%.*]] = load i32, i32* [[ARRAYIDX_PHI_TRANS_INSERT]], align 4 321; SINK-AFTER-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 322; SINK-AFTER-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 323; SINK-AFTER-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 324; SINK-AFTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4 325; SINK-AFTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 326; SINK-AFTER: vector.memcheck: 327; SINK-AFTER-NEXT: [[TMP3:%.*]] = add i32 [[N]], -1 328; SINK-AFTER-NEXT: [[TMP4:%.*]] = zext i32 [[TMP3]] to i64 329; SINK-AFTER-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[TMP4]], 1 330; SINK-AFTER-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[TMP5]] 331; SINK-AFTER-NEXT: [[SCEVGEP2:%.*]] = bitcast i32* [[SCEVGEP]] to i8* 332; SINK-AFTER-NEXT: [[SCEVGEP3:%.*]] = getelementptr i32, i32* [[A]], i64 1 333; SINK-AFTER-NEXT: [[SCEVGEP34:%.*]] = bitcast i32* [[SCEVGEP3]] to i8* 334; SINK-AFTER-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[TMP4]], 2 335; SINK-AFTER-NEXT: [[SCEVGEP5:%.*]] = getelementptr i32, i32* [[A]], i64 [[TMP6]] 336; SINK-AFTER-NEXT: [[SCEVGEP56:%.*]] = bitcast i32* [[SCEVGEP5]] to i8* 337; SINK-AFTER-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[B1]], [[SCEVGEP56]] 338; SINK-AFTER-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP34]], [[SCEVGEP2]] 339; SINK-AFTER-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 340; SINK-AFTER-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 341; SINK-AFTER: vector.ph: 342; SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4 343; SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 344; SINK-AFTER-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[PRE_LOAD]], i32 3 345; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 346; SINK-AFTER: vector.body: 347; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 348; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ] 349; SINK-AFTER-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 0 350; SINK-AFTER-NEXT: [[TMP8:%.*]] = add nuw nsw i64 [[TMP7]], 1 351; SINK-AFTER-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP8]] 352; SINK-AFTER-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP9]], i32 0 353; SINK-AFTER-NEXT: [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <4 x i32>* 354; SINK-AFTER-NEXT: [[WIDE_LOAD]] = load <4 x i32>, <4 x i32>* [[TMP11]], align 4 355; SINK-AFTER-NEXT: [[TMP12:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 356; SINK-AFTER-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP7]] 357; SINK-AFTER-NEXT: [[TMP14:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[TMP12]] 358; SINK-AFTER-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, i32* [[TMP13]], i32 0 359; SINK-AFTER-NEXT: [[TMP16:%.*]] = bitcast i32* [[TMP15]] to <4 x i32>* 360; SINK-AFTER-NEXT: store <4 x i32> [[TMP14]], <4 x i32>* [[TMP16]], align 4 361; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 362; SINK-AFTER-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 363; SINK-AFTER-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] 364; SINK-AFTER: middle.block: 365; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 366; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i32 3 367; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i32 2 368; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]] 369; SINK-AFTER: scalar.ph: 370; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[PRE_LOAD]], [[VECTOR_MEMCHECK]] ], [ [[PRE_LOAD]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 371; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ] 372; SINK-AFTER-NEXT: br label [[SCALAR_BODY:%.*]] 373; SINK-AFTER: scalar.body: 374; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP18:%.*]], [[SCALAR_BODY]] ] 375; SINK-AFTER-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ] 376; SINK-AFTER-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 377; SINK-AFTER-NEXT: [[ARRAYIDX32:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT]] 378; SINK-AFTER-NEXT: [[TMP18]] = load i32, i32* [[ARRAYIDX32]], align 4 379; SINK-AFTER-NEXT: [[ARRAYIDX34:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 380; SINK-AFTER-NEXT: [[ADD35:%.*]] = add i32 [[TMP18]], [[SCALAR_RECUR]] 381; SINK-AFTER-NEXT: store i32 [[ADD35]], i32* [[ARRAYIDX34]], align 4 382; SINK-AFTER-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 383; SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 384; SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] 385; SINK-AFTER: for.exit: 386; SINK-AFTER-NEXT: ret void 387; 388; NO-SINK-AFTER-LABEL: @recurrence_1( 389; NO-SINK-AFTER-NEXT: entry: 390; NO-SINK-AFTER-NEXT: [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8* 391; NO-SINK-AFTER-NEXT: br label [[FOR_PREHEADER:%.*]] 392; NO-SINK-AFTER: for.preheader: 393; NO-SINK-AFTER-NEXT: [[ARRAYIDX_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 0 394; NO-SINK-AFTER-NEXT: [[PRE_LOAD:%.*]] = load i32, i32* [[ARRAYIDX_PHI_TRANS_INSERT]], align 4 395; NO-SINK-AFTER-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 396; NO-SINK-AFTER-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 397; NO-SINK-AFTER-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 398; NO-SINK-AFTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4 399; NO-SINK-AFTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 400; NO-SINK-AFTER: vector.memcheck: 401; NO-SINK-AFTER-NEXT: [[TMP3:%.*]] = add i32 [[N]], -1 402; NO-SINK-AFTER-NEXT: [[TMP4:%.*]] = zext i32 [[TMP3]] to i64 403; NO-SINK-AFTER-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[TMP4]], 1 404; NO-SINK-AFTER-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[TMP5]] 405; NO-SINK-AFTER-NEXT: [[SCEVGEP2:%.*]] = bitcast i32* [[SCEVGEP]] to i8* 406; NO-SINK-AFTER-NEXT: [[SCEVGEP3:%.*]] = getelementptr i32, i32* [[A]], i64 1 407; NO-SINK-AFTER-NEXT: [[SCEVGEP34:%.*]] = bitcast i32* [[SCEVGEP3]] to i8* 408; NO-SINK-AFTER-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[TMP4]], 2 409; NO-SINK-AFTER-NEXT: [[SCEVGEP5:%.*]] = getelementptr i32, i32* [[A]], i64 [[TMP6]] 410; NO-SINK-AFTER-NEXT: [[SCEVGEP56:%.*]] = bitcast i32* [[SCEVGEP5]] to i8* 411; NO-SINK-AFTER-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[B1]], [[SCEVGEP56]] 412; NO-SINK-AFTER-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP34]], [[SCEVGEP2]] 413; NO-SINK-AFTER-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 414; NO-SINK-AFTER-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 415; NO-SINK-AFTER: vector.ph: 416; NO-SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4 417; NO-SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 418; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[PRE_LOAD]], i32 3 419; NO-SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 420; NO-SINK-AFTER: vector.body: 421; NO-SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 422; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ] 423; NO-SINK-AFTER-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 0 424; NO-SINK-AFTER-NEXT: [[TMP8:%.*]] = add nuw nsw i64 [[TMP7]], 1 425; NO-SINK-AFTER-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP8]] 426; NO-SINK-AFTER-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP9]], i32 0 427; NO-SINK-AFTER-NEXT: [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <4 x i32>* 428; NO-SINK-AFTER-NEXT: [[WIDE_LOAD]] = load <4 x i32>, <4 x i32>* [[TMP11]], align 4 429; NO-SINK-AFTER-NEXT: [[TMP12:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 430; NO-SINK-AFTER-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP7]] 431; NO-SINK-AFTER-NEXT: [[TMP14:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[TMP12]] 432; NO-SINK-AFTER-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, i32* [[TMP13]], i32 0 433; NO-SINK-AFTER-NEXT: [[TMP16:%.*]] = bitcast i32* [[TMP15]] to <4 x i32>* 434; NO-SINK-AFTER-NEXT: store <4 x i32> [[TMP14]], <4 x i32>* [[TMP16]], align 4 435; NO-SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 436; NO-SINK-AFTER-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 437; NO-SINK-AFTER-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] 438; NO-SINK-AFTER: middle.block: 439; NO-SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 440; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i32 3 441; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i32 2 442; NO-SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]] 443; NO-SINK-AFTER: scalar.ph: 444; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[PRE_LOAD]], [[VECTOR_MEMCHECK]] ], [ [[PRE_LOAD]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 445; NO-SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ] 446; NO-SINK-AFTER-NEXT: br label [[SCALAR_BODY:%.*]] 447; NO-SINK-AFTER: scalar.body: 448; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP18:%.*]], [[SCALAR_BODY]] ] 449; NO-SINK-AFTER-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ] 450; NO-SINK-AFTER-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 451; NO-SINK-AFTER-NEXT: [[ARRAYIDX32:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT]] 452; NO-SINK-AFTER-NEXT: [[TMP18]] = load i32, i32* [[ARRAYIDX32]], align 4 453; NO-SINK-AFTER-NEXT: [[ARRAYIDX34:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 454; NO-SINK-AFTER-NEXT: [[ADD35:%.*]] = add i32 [[TMP18]], [[SCALAR_RECUR]] 455; NO-SINK-AFTER-NEXT: store i32 [[ADD35]], i32* [[ARRAYIDX34]], align 4 456; NO-SINK-AFTER-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 457; NO-SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 458; NO-SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] 459; NO-SINK-AFTER: for.exit: 460; NO-SINK-AFTER-NEXT: ret void 461; 462entry: 463 br label %for.preheader 464 465for.preheader: 466 %arrayidx.phi.trans.insert = getelementptr inbounds i32, i32* %a, i64 0 467 %pre_load = load i32, i32* %arrayidx.phi.trans.insert 468 br label %scalar.body 469 470scalar.body: 471 %0 = phi i32 [ %pre_load, %for.preheader ], [ %1, %scalar.body ] 472 %indvars.iv = phi i64 [ 0, %for.preheader ], [ %indvars.iv.next, %scalar.body ] 473 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 474 %arrayidx32 = getelementptr inbounds i32, i32* %a, i64 %indvars.iv.next 475 %1 = load i32, i32* %arrayidx32 476 %arrayidx34 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv 477 %add35 = add i32 %1, %0 478 store i32 %add35, i32* %arrayidx34 479 %lftr.wideiv = trunc i64 %indvars.iv.next to i32 480 %exitcond = icmp eq i32 %lftr.wideiv, %n 481 br i1 %exitcond, label %for.exit, label %scalar.body 482 483for.exit: 484 ret void 485} 486 487; int recurrence_2(int *a, int n) { 488; int minmax; 489; for (int i = 0; i < n; ++i) 490; minmax = min(minmax, max(a[i] - a[i-1], 0)); 491; return minmax; 492; } 493; 494; 495; 496define i32 @recurrence_2(i32* nocapture readonly %a, i32 %n) { 497; CHECK-LABEL: @recurrence_2( 498; CHECK-NEXT: entry: 499; CHECK-NEXT: [[CMP27:%.*]] = icmp sgt i32 [[N:%.*]], 0 500; CHECK-NEXT: br i1 [[CMP27]], label [[FOR_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]] 501; CHECK: for.preheader: 502; CHECK-NEXT: [[ARRAYIDX2_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 -1 503; CHECK-NEXT: [[DOTPRE:%.*]] = load i32, i32* [[ARRAYIDX2_PHI_TRANS_INSERT]], align 4 504; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1 505; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 506; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 507; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 3 508; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 509; CHECK: vector.ph: 510; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934588 511; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[DOTPRE]], i64 3 512; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 513; CHECK: vector.body: 514; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 515; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ] 516; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ poison, [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[VECTOR_BODY]] ] 517; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDEX]] 518; CHECK-NEXT: [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>* 519; CHECK-NEXT: [[WIDE_LOAD]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4 520; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 521; CHECK-NEXT: [[TMP6:%.*]] = sub nsw <4 x i32> [[WIDE_LOAD]], [[TMP5]] 522; CHECK-NEXT: [[TMP7:%.*]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[TMP6]], <4 x i32> zeroinitializer) 523; CHECK-NEXT: [[TMP8]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[TMP7]]) 524; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 525; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 526; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] 527; CHECK: middle.block: 528; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[TMP8]]) 529; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 530; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 3 531; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]] 532; CHECK: scalar.ph: 533; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[DOTPRE]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 534; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[FOR_PREHEADER]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ] 535; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ poison, [[FOR_PREHEADER]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ] 536; CHECK-NEXT: br label [[SCALAR_BODY:%.*]] 537; CHECK: for.cond.cleanup.loopexit: 538; CHECK-NEXT: [[MINMAX_0_COND_LCSSA:%.*]] = phi i32 [ [[TMP13:%.*]], [[SCALAR_BODY]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ] 539; CHECK-NEXT: br label [[FOR_COND_CLEANUP]] 540; CHECK: for.cond.cleanup: 541; CHECK-NEXT: [[MINMAX_0_LCSSA:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[MINMAX_0_COND_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ] 542; CHECK-NEXT: ret i32 [[MINMAX_0_LCSSA]] 543; CHECK: scalar.body: 544; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP11:%.*]], [[SCALAR_BODY]] ] 545; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ] 546; CHECK-NEXT: [[MINMAX_028:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[TMP13]], [[SCALAR_BODY]] ] 547; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]] 548; CHECK-NEXT: [[TMP11]] = load i32, i32* [[ARRAYIDX]], align 4 549; CHECK-NEXT: [[SUB3:%.*]] = sub nsw i32 [[TMP11]], [[SCALAR_RECUR]] 550; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.smax.i32(i32 [[SUB3]], i32 0) 551; CHECK-NEXT: [[TMP13]] = call i32 @llvm.smin.i32(i32 [[MINMAX_028]], i32 [[TMP12]]) 552; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 553; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 554; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 555; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] 556; 557; UNROLL-LABEL: @recurrence_2( 558; UNROLL-NEXT: entry: 559; UNROLL-NEXT: [[CMP27:%.*]] = icmp sgt i32 [[N:%.*]], 0 560; UNROLL-NEXT: br i1 [[CMP27]], label [[FOR_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]] 561; UNROLL: for.preheader: 562; UNROLL-NEXT: [[ARRAYIDX2_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 -1 563; UNROLL-NEXT: [[DOTPRE:%.*]] = load i32, i32* [[ARRAYIDX2_PHI_TRANS_INSERT]], align 4 564; UNROLL-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1 565; UNROLL-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 566; UNROLL-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 567; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 7 568; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 569; UNROLL: vector.ph: 570; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934584 571; UNROLL-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[DOTPRE]], i64 3 572; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 573; UNROLL: vector.body: 574; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 575; UNROLL-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD2:%.*]], [[VECTOR_BODY]] ] 576; UNROLL-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ poison, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ] 577; UNROLL-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ poison, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ] 578; UNROLL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDEX]] 579; UNROLL-NEXT: [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>* 580; UNROLL-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4 581; UNROLL-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP3]], i64 4 582; UNROLL-NEXT: [[TMP6:%.*]] = bitcast i32* [[TMP5]] to <4 x i32>* 583; UNROLL-NEXT: [[WIDE_LOAD2]] = load <4 x i32>, <4 x i32>* [[TMP6]], align 4 584; UNROLL-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 585; UNROLL-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> [[WIDE_LOAD2]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 586; UNROLL-NEXT: [[TMP9:%.*]] = sub nsw <4 x i32> [[WIDE_LOAD]], [[TMP7]] 587; UNROLL-NEXT: [[TMP10:%.*]] = sub nsw <4 x i32> [[WIDE_LOAD2]], [[TMP8]] 588; UNROLL-NEXT: [[TMP11:%.*]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[TMP9]], <4 x i32> zeroinitializer) 589; UNROLL-NEXT: [[TMP12:%.*]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[TMP10]], <4 x i32> zeroinitializer) 590; UNROLL-NEXT: [[TMP13]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[TMP11]]) 591; UNROLL-NEXT: [[TMP14]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[VEC_PHI1]], <4 x i32> [[TMP12]]) 592; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 593; UNROLL-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 594; UNROLL-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] 595; UNROLL: middle.block: 596; UNROLL-NEXT: [[TMP16:%.*]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[TMP13]], <4 x i32> [[TMP14]]) 597; UNROLL-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[TMP16]]) 598; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 599; UNROLL-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i64 3 600; UNROLL-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]] 601; UNROLL: scalar.ph: 602; UNROLL-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[DOTPRE]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 603; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[FOR_PREHEADER]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ] 604; UNROLL-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ poison, [[FOR_PREHEADER]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ] 605; UNROLL-NEXT: br label [[SCALAR_BODY:%.*]] 606; UNROLL: for.cond.cleanup.loopexit: 607; UNROLL-NEXT: [[MINMAX_0_COND_LCSSA:%.*]] = phi i32 [ [[TMP20:%.*]], [[SCALAR_BODY]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ] 608; UNROLL-NEXT: br label [[FOR_COND_CLEANUP]] 609; UNROLL: for.cond.cleanup: 610; UNROLL-NEXT: [[MINMAX_0_LCSSA:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[MINMAX_0_COND_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ] 611; UNROLL-NEXT: ret i32 [[MINMAX_0_LCSSA]] 612; UNROLL: scalar.body: 613; UNROLL-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP18:%.*]], [[SCALAR_BODY]] ] 614; UNROLL-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ] 615; UNROLL-NEXT: [[MINMAX_028:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[TMP20]], [[SCALAR_BODY]] ] 616; UNROLL-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]] 617; UNROLL-NEXT: [[TMP18]] = load i32, i32* [[ARRAYIDX]], align 4 618; UNROLL-NEXT: [[SUB3:%.*]] = sub nsw i32 [[TMP18]], [[SCALAR_RECUR]] 619; UNROLL-NEXT: [[TMP19:%.*]] = call i32 @llvm.smax.i32(i32 [[SUB3]], i32 0) 620; UNROLL-NEXT: [[TMP20]] = call i32 @llvm.smin.i32(i32 [[MINMAX_028]], i32 [[TMP19]]) 621; UNROLL-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 622; UNROLL-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 623; UNROLL-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 624; UNROLL-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] 625; 626; UNROLL-NO-IC-LABEL: @recurrence_2( 627; UNROLL-NO-IC-NEXT: entry: 628; UNROLL-NO-IC-NEXT: [[CMP27:%.*]] = icmp sgt i32 [[N:%.*]], 0 629; UNROLL-NO-IC-NEXT: br i1 [[CMP27]], label [[FOR_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]] 630; UNROLL-NO-IC: for.preheader: 631; UNROLL-NO-IC-NEXT: [[ARRAYIDX2_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 -1 632; UNROLL-NO-IC-NEXT: [[DOTPRE:%.*]] = load i32, i32* [[ARRAYIDX2_PHI_TRANS_INSERT]], align 4 633; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1 634; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 635; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 636; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 8 637; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 638; UNROLL-NO-IC: vector.ph: 639; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 8 640; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 641; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[DOTPRE]], i32 3 642; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 643; UNROLL-NO-IC: vector.body: 644; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 645; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD2:%.*]], [[VECTOR_BODY]] ] 646; UNROLL-NO-IC-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ poison, [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ] 647; UNROLL-NO-IC-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ poison, [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ] 648; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 0 649; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 4 650; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP3]] 651; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP4]] 652; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[TMP5]], i32 0 653; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = bitcast i32* [[TMP7]] to <4 x i32>* 654; UNROLL-NO-IC-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP8]], align 4 655; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP5]], i32 4 656; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = bitcast i32* [[TMP9]] to <4 x i32>* 657; UNROLL-NO-IC-NEXT: [[WIDE_LOAD2]] = load <4 x i32>, <4 x i32>* [[TMP10]], align 4 658; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 659; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> [[WIDE_LOAD2]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 660; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = sub nsw <4 x i32> [[WIDE_LOAD]], [[TMP11]] 661; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = sub nsw <4 x i32> [[WIDE_LOAD2]], [[TMP12]] 662; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = icmp sgt <4 x i32> [[TMP13]], zeroinitializer 663; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = icmp sgt <4 x i32> [[TMP14]], zeroinitializer 664; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = select <4 x i1> [[TMP15]], <4 x i32> [[TMP13]], <4 x i32> zeroinitializer 665; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = select <4 x i1> [[TMP16]], <4 x i32> [[TMP14]], <4 x i32> zeroinitializer 666; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = icmp slt <4 x i32> [[VEC_PHI]], [[TMP17]] 667; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = icmp slt <4 x i32> [[VEC_PHI1]], [[TMP18]] 668; UNROLL-NO-IC-NEXT: [[TMP21]] = select <4 x i1> [[TMP19]], <4 x i32> [[VEC_PHI]], <4 x i32> [[TMP17]] 669; UNROLL-NO-IC-NEXT: [[TMP22]] = select <4 x i1> [[TMP20]], <4 x i32> [[VEC_PHI1]], <4 x i32> [[TMP18]] 670; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 671; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 672; UNROLL-NO-IC-NEXT: br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] 673; UNROLL-NO-IC: middle.block: 674; UNROLL-NO-IC-NEXT: [[RDX_MINMAX_CMP:%.*]] = icmp slt <4 x i32> [[TMP21]], [[TMP22]] 675; UNROLL-NO-IC-NEXT: [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP21]], <4 x i32> [[TMP22]] 676; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[RDX_MINMAX_SELECT]]) 677; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 678; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i32 3 679; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i32 2 680; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]] 681; UNROLL-NO-IC: scalar.ph: 682; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[DOTPRE]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 683; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_PREHEADER]] ] 684; UNROLL-NO-IC-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ poison, [[FOR_PREHEADER]] ], [ [[TMP24]], [[MIDDLE_BLOCK]] ] 685; UNROLL-NO-IC-NEXT: br label [[SCALAR_BODY:%.*]] 686; UNROLL-NO-IC: for.cond.cleanup.loopexit: 687; UNROLL-NO-IC-NEXT: [[MINMAX_0_COND_LCSSA:%.*]] = phi i32 [ [[MINMAX_0_COND:%.*]], [[SCALAR_BODY]] ], [ [[TMP24]], [[MIDDLE_BLOCK]] ] 688; UNROLL-NO-IC-NEXT: br label [[FOR_COND_CLEANUP]] 689; UNROLL-NO-IC: for.cond.cleanup: 690; UNROLL-NO-IC-NEXT: [[MINMAX_0_LCSSA:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[MINMAX_0_COND_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ] 691; UNROLL-NO-IC-NEXT: ret i32 [[MINMAX_0_LCSSA]] 692; UNROLL-NO-IC: scalar.body: 693; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP25:%.*]], [[SCALAR_BODY]] ] 694; UNROLL-NO-IC-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ] 695; UNROLL-NO-IC-NEXT: [[MINMAX_028:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MINMAX_0_COND]], [[SCALAR_BODY]] ] 696; UNROLL-NO-IC-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]] 697; UNROLL-NO-IC-NEXT: [[TMP25]] = load i32, i32* [[ARRAYIDX]], align 4 698; UNROLL-NO-IC-NEXT: [[SUB3:%.*]] = sub nsw i32 [[TMP25]], [[SCALAR_RECUR]] 699; UNROLL-NO-IC-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[SUB3]], 0 700; UNROLL-NO-IC-NEXT: [[COND:%.*]] = select i1 [[CMP4]], i32 [[SUB3]], i32 0 701; UNROLL-NO-IC-NEXT: [[CMP5:%.*]] = icmp slt i32 [[MINMAX_028]], [[COND]] 702; UNROLL-NO-IC-NEXT: [[MINMAX_0_COND]] = select i1 [[CMP5]], i32 [[MINMAX_028]], i32 [[COND]] 703; UNROLL-NO-IC-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 704; UNROLL-NO-IC-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 705; UNROLL-NO-IC-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 706; UNROLL-NO-IC-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] 707; 708; UNROLL-NO-VF-LABEL: @recurrence_2( 709; UNROLL-NO-VF-NEXT: entry: 710; UNROLL-NO-VF-NEXT: [[CMP27:%.*]] = icmp sgt i32 [[N:%.*]], 0 711; UNROLL-NO-VF-NEXT: br i1 [[CMP27]], label [[FOR_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]] 712; UNROLL-NO-VF: for.preheader: 713; UNROLL-NO-VF-NEXT: [[ARRAYIDX2_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 -1 714; UNROLL-NO-VF-NEXT: [[DOTPRE:%.*]] = load i32, i32* [[ARRAYIDX2_PHI_TRANS_INSERT]], align 4 715; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1 716; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 717; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 718; UNROLL-NO-VF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 2 719; UNROLL-NO-VF-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 720; UNROLL-NO-VF: vector.ph: 721; UNROLL-NO-VF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 2 722; UNROLL-NO-VF-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 723; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 724; UNROLL-NO-VF: vector.body: 725; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 726; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i32 [ [[DOTPRE]], [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ] 727; UNROLL-NO-VF-NEXT: [[VEC_PHI:%.*]] = phi i32 [ poison, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ] 728; UNROLL-NO-VF-NEXT: [[VEC_PHI2:%.*]] = phi i32 [ poison, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ] 729; UNROLL-NO-VF-NEXT: [[INDUCTION:%.*]] = add i64 [[INDEX]], 0 730; UNROLL-NO-VF-NEXT: [[INDUCTION1:%.*]] = add i64 [[INDEX]], 1 731; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDUCTION]] 732; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDUCTION1]] 733; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = load i32, i32* [[TMP3]], align 4 734; UNROLL-NO-VF-NEXT: [[TMP6]] = load i32, i32* [[TMP4]], align 4 735; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = sub nsw i32 [[TMP5]], [[VECTOR_RECUR]] 736; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = sub nsw i32 [[TMP6]], [[TMP5]] 737; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = icmp sgt i32 [[TMP7]], 0 738; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = icmp sgt i32 [[TMP8]], 0 739; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = select i1 [[TMP9]], i32 [[TMP7]], i32 0 740; UNROLL-NO-VF-NEXT: [[TMP12:%.*]] = select i1 [[TMP10]], i32 [[TMP8]], i32 0 741; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = icmp slt i32 [[VEC_PHI]], [[TMP11]] 742; UNROLL-NO-VF-NEXT: [[TMP14:%.*]] = icmp slt i32 [[VEC_PHI2]], [[TMP12]] 743; UNROLL-NO-VF-NEXT: [[TMP15]] = select i1 [[TMP13]], i32 [[VEC_PHI]], i32 [[TMP11]] 744; UNROLL-NO-VF-NEXT: [[TMP16]] = select i1 [[TMP14]], i32 [[VEC_PHI2]], i32 [[TMP12]] 745; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 746; UNROLL-NO-VF-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 747; UNROLL-NO-VF-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] 748; UNROLL-NO-VF: middle.block: 749; UNROLL-NO-VF-NEXT: [[RDX_MINMAX_CMP:%.*]] = icmp slt i32 [[TMP15]], [[TMP16]] 750; UNROLL-NO-VF-NEXT: [[RDX_MINMAX_SELECT:%.*]] = select i1 [[RDX_MINMAX_CMP]], i32 [[TMP15]], i32 [[TMP16]] 751; UNROLL-NO-VF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 752; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]] 753; UNROLL-NO-VF: scalar.ph: 754; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[DOTPRE]], [[FOR_PREHEADER]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ] 755; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_PREHEADER]] ] 756; UNROLL-NO-VF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ poison, [[FOR_PREHEADER]] ], [ [[RDX_MINMAX_SELECT]], [[MIDDLE_BLOCK]] ] 757; UNROLL-NO-VF-NEXT: br label [[SCALAR_BODY:%.*]] 758; UNROLL-NO-VF: for.cond.cleanup.loopexit: 759; UNROLL-NO-VF-NEXT: [[MINMAX_0_COND_LCSSA:%.*]] = phi i32 [ [[MINMAX_0_COND:%.*]], [[SCALAR_BODY]] ], [ [[RDX_MINMAX_SELECT]], [[MIDDLE_BLOCK]] ] 760; UNROLL-NO-VF-NEXT: br label [[FOR_COND_CLEANUP]] 761; UNROLL-NO-VF: for.cond.cleanup: 762; UNROLL-NO-VF-NEXT: [[MINMAX_0_LCSSA:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[MINMAX_0_COND_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ] 763; UNROLL-NO-VF-NEXT: ret i32 [[MINMAX_0_LCSSA]] 764; UNROLL-NO-VF: scalar.body: 765; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP18:%.*]], [[SCALAR_BODY]] ] 766; UNROLL-NO-VF-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ] 767; UNROLL-NO-VF-NEXT: [[MINMAX_028:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MINMAX_0_COND]], [[SCALAR_BODY]] ] 768; UNROLL-NO-VF-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]] 769; UNROLL-NO-VF-NEXT: [[TMP18]] = load i32, i32* [[ARRAYIDX]], align 4 770; UNROLL-NO-VF-NEXT: [[SUB3:%.*]] = sub nsw i32 [[TMP18]], [[SCALAR_RECUR]] 771; UNROLL-NO-VF-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[SUB3]], 0 772; UNROLL-NO-VF-NEXT: [[COND:%.*]] = select i1 [[CMP4]], i32 [[SUB3]], i32 0 773; UNROLL-NO-VF-NEXT: [[CMP5:%.*]] = icmp slt i32 [[MINMAX_028]], [[COND]] 774; UNROLL-NO-VF-NEXT: [[MINMAX_0_COND]] = select i1 [[CMP5]], i32 [[MINMAX_028]], i32 [[COND]] 775; UNROLL-NO-VF-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 776; UNROLL-NO-VF-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 777; UNROLL-NO-VF-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 778; UNROLL-NO-VF-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] 779; 780; SINK-AFTER-LABEL: @recurrence_2( 781; SINK-AFTER-NEXT: entry: 782; SINK-AFTER-NEXT: [[CMP27:%.*]] = icmp sgt i32 [[N:%.*]], 0 783; SINK-AFTER-NEXT: br i1 [[CMP27]], label [[FOR_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]] 784; SINK-AFTER: for.preheader: 785; SINK-AFTER-NEXT: [[ARRAYIDX2_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 -1 786; SINK-AFTER-NEXT: [[DOTPRE:%.*]] = load i32, i32* [[ARRAYIDX2_PHI_TRANS_INSERT]], align 4 787; SINK-AFTER-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1 788; SINK-AFTER-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 789; SINK-AFTER-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 790; SINK-AFTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4 791; SINK-AFTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 792; SINK-AFTER: vector.ph: 793; SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4 794; SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 795; SINK-AFTER-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[DOTPRE]], i32 3 796; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 797; SINK-AFTER: vector.body: 798; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 799; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ] 800; SINK-AFTER-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ poison, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ] 801; SINK-AFTER-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 0 802; SINK-AFTER-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP3]] 803; SINK-AFTER-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 0 804; SINK-AFTER-NEXT: [[TMP6:%.*]] = bitcast i32* [[TMP5]] to <4 x i32>* 805; SINK-AFTER-NEXT: [[WIDE_LOAD]] = load <4 x i32>, <4 x i32>* [[TMP6]], align 4 806; SINK-AFTER-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 807; SINK-AFTER-NEXT: [[TMP8:%.*]] = sub nsw <4 x i32> [[WIDE_LOAD]], [[TMP7]] 808; SINK-AFTER-NEXT: [[TMP9:%.*]] = icmp sgt <4 x i32> [[TMP8]], zeroinitializer 809; SINK-AFTER-NEXT: [[TMP10:%.*]] = select <4 x i1> [[TMP9]], <4 x i32> [[TMP8]], <4 x i32> zeroinitializer 810; SINK-AFTER-NEXT: [[TMP11:%.*]] = icmp slt <4 x i32> [[VEC_PHI]], [[TMP10]] 811; SINK-AFTER-NEXT: [[TMP12]] = select <4 x i1> [[TMP11]], <4 x i32> [[VEC_PHI]], <4 x i32> [[TMP10]] 812; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 813; SINK-AFTER-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 814; SINK-AFTER-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] 815; SINK-AFTER: middle.block: 816; SINK-AFTER-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[TMP12]]) 817; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 818; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i32 3 819; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i32 2 820; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]] 821; SINK-AFTER: scalar.ph: 822; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[DOTPRE]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 823; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_PREHEADER]] ] 824; SINK-AFTER-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ poison, [[FOR_PREHEADER]] ], [ [[TMP14]], [[MIDDLE_BLOCK]] ] 825; SINK-AFTER-NEXT: br label [[SCALAR_BODY:%.*]] 826; SINK-AFTER: for.cond.cleanup.loopexit: 827; SINK-AFTER-NEXT: [[MINMAX_0_COND_LCSSA:%.*]] = phi i32 [ [[MINMAX_0_COND:%.*]], [[SCALAR_BODY]] ], [ [[TMP14]], [[MIDDLE_BLOCK]] ] 828; SINK-AFTER-NEXT: br label [[FOR_COND_CLEANUP]] 829; SINK-AFTER: for.cond.cleanup: 830; SINK-AFTER-NEXT: [[MINMAX_0_LCSSA:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[MINMAX_0_COND_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ] 831; SINK-AFTER-NEXT: ret i32 [[MINMAX_0_LCSSA]] 832; SINK-AFTER: scalar.body: 833; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP15:%.*]], [[SCALAR_BODY]] ] 834; SINK-AFTER-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ] 835; SINK-AFTER-NEXT: [[MINMAX_028:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MINMAX_0_COND]], [[SCALAR_BODY]] ] 836; SINK-AFTER-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]] 837; SINK-AFTER-NEXT: [[TMP15]] = load i32, i32* [[ARRAYIDX]], align 4 838; SINK-AFTER-NEXT: [[SUB3:%.*]] = sub nsw i32 [[TMP15]], [[SCALAR_RECUR]] 839; SINK-AFTER-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[SUB3]], 0 840; SINK-AFTER-NEXT: [[COND:%.*]] = select i1 [[CMP4]], i32 [[SUB3]], i32 0 841; SINK-AFTER-NEXT: [[CMP5:%.*]] = icmp slt i32 [[MINMAX_028]], [[COND]] 842; SINK-AFTER-NEXT: [[MINMAX_0_COND]] = select i1 [[CMP5]], i32 [[MINMAX_028]], i32 [[COND]] 843; SINK-AFTER-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 844; SINK-AFTER-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 845; SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 846; SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] 847; 848; NO-SINK-AFTER-LABEL: @recurrence_2( 849; NO-SINK-AFTER-NEXT: entry: 850; NO-SINK-AFTER-NEXT: [[CMP27:%.*]] = icmp sgt i32 [[N:%.*]], 0 851; NO-SINK-AFTER-NEXT: br i1 [[CMP27]], label [[FOR_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]] 852; NO-SINK-AFTER: for.preheader: 853; NO-SINK-AFTER-NEXT: [[ARRAYIDX2_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 -1 854; NO-SINK-AFTER-NEXT: [[DOTPRE:%.*]] = load i32, i32* [[ARRAYIDX2_PHI_TRANS_INSERT]], align 4 855; NO-SINK-AFTER-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1 856; NO-SINK-AFTER-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 857; NO-SINK-AFTER-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 858; NO-SINK-AFTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4 859; NO-SINK-AFTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 860; NO-SINK-AFTER: vector.ph: 861; NO-SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4 862; NO-SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 863; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[DOTPRE]], i32 3 864; NO-SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 865; NO-SINK-AFTER: vector.body: 866; NO-SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 867; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ] 868; NO-SINK-AFTER-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ poison, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ] 869; NO-SINK-AFTER-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 0 870; NO-SINK-AFTER-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP3]] 871; NO-SINK-AFTER-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 0 872; NO-SINK-AFTER-NEXT: [[TMP6:%.*]] = bitcast i32* [[TMP5]] to <4 x i32>* 873; NO-SINK-AFTER-NEXT: [[WIDE_LOAD]] = load <4 x i32>, <4 x i32>* [[TMP6]], align 4 874; NO-SINK-AFTER-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 875; NO-SINK-AFTER-NEXT: [[TMP8:%.*]] = sub nsw <4 x i32> [[WIDE_LOAD]], [[TMP7]] 876; NO-SINK-AFTER-NEXT: [[TMP9:%.*]] = icmp sgt <4 x i32> [[TMP8]], zeroinitializer 877; NO-SINK-AFTER-NEXT: [[TMP10:%.*]] = select <4 x i1> [[TMP9]], <4 x i32> [[TMP8]], <4 x i32> zeroinitializer 878; NO-SINK-AFTER-NEXT: [[TMP11:%.*]] = icmp slt <4 x i32> [[VEC_PHI]], [[TMP10]] 879; NO-SINK-AFTER-NEXT: [[TMP12]] = select <4 x i1> [[TMP11]], <4 x i32> [[VEC_PHI]], <4 x i32> [[TMP10]] 880; NO-SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 881; NO-SINK-AFTER-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 882; NO-SINK-AFTER-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] 883; NO-SINK-AFTER: middle.block: 884; NO-SINK-AFTER-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[TMP12]]) 885; NO-SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 886; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i32 3 887; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i32 2 888; NO-SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]] 889; NO-SINK-AFTER: scalar.ph: 890; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[DOTPRE]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 891; NO-SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_PREHEADER]] ] 892; NO-SINK-AFTER-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ poison, [[FOR_PREHEADER]] ], [ [[TMP14]], [[MIDDLE_BLOCK]] ] 893; NO-SINK-AFTER-NEXT: br label [[SCALAR_BODY:%.*]] 894; NO-SINK-AFTER: for.cond.cleanup.loopexit: 895; NO-SINK-AFTER-NEXT: [[MINMAX_0_COND_LCSSA:%.*]] = phi i32 [ [[MINMAX_0_COND:%.*]], [[SCALAR_BODY]] ], [ [[TMP14]], [[MIDDLE_BLOCK]] ] 896; NO-SINK-AFTER-NEXT: br label [[FOR_COND_CLEANUP]] 897; NO-SINK-AFTER: for.cond.cleanup: 898; NO-SINK-AFTER-NEXT: [[MINMAX_0_LCSSA:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[MINMAX_0_COND_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ] 899; NO-SINK-AFTER-NEXT: ret i32 [[MINMAX_0_LCSSA]] 900; NO-SINK-AFTER: scalar.body: 901; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP15:%.*]], [[SCALAR_BODY]] ] 902; NO-SINK-AFTER-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ] 903; NO-SINK-AFTER-NEXT: [[MINMAX_028:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MINMAX_0_COND]], [[SCALAR_BODY]] ] 904; NO-SINK-AFTER-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]] 905; NO-SINK-AFTER-NEXT: [[TMP15]] = load i32, i32* [[ARRAYIDX]], align 4 906; NO-SINK-AFTER-NEXT: [[SUB3:%.*]] = sub nsw i32 [[TMP15]], [[SCALAR_RECUR]] 907; NO-SINK-AFTER-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[SUB3]], 0 908; NO-SINK-AFTER-NEXT: [[COND:%.*]] = select i1 [[CMP4]], i32 [[SUB3]], i32 0 909; NO-SINK-AFTER-NEXT: [[CMP5:%.*]] = icmp slt i32 [[MINMAX_028]], [[COND]] 910; NO-SINK-AFTER-NEXT: [[MINMAX_0_COND]] = select i1 [[CMP5]], i32 [[MINMAX_028]], i32 [[COND]] 911; NO-SINK-AFTER-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 912; NO-SINK-AFTER-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 913; NO-SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 914; NO-SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] 915; 916entry: 917 %cmp27 = icmp sgt i32 %n, 0 918 br i1 %cmp27, label %for.preheader, label %for.cond.cleanup 919 920for.preheader: 921 %arrayidx2.phi.trans.insert = getelementptr inbounds i32, i32* %a, i64 -1 922 %.pre = load i32, i32* %arrayidx2.phi.trans.insert, align 4 923 br label %scalar.body 924 925for.cond.cleanup.loopexit: 926 %minmax.0.cond.lcssa = phi i32 [ %minmax.0.cond, %scalar.body ] 927 br label %for.cond.cleanup 928 929for.cond.cleanup: 930 %minmax.0.lcssa = phi i32 [ poison, %entry ], [ %minmax.0.cond.lcssa, %for.cond.cleanup.loopexit ] 931 ret i32 %minmax.0.lcssa 932 933scalar.body: 934 %0 = phi i32 [ %.pre, %for.preheader ], [ %1, %scalar.body ] 935 %indvars.iv = phi i64 [ 0, %for.preheader ], [ %indvars.iv.next, %scalar.body ] 936 %minmax.028 = phi i32 [ poison, %for.preheader ], [ %minmax.0.cond, %scalar.body ] 937 %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv 938 %1 = load i32, i32* %arrayidx, align 4 939 %sub3 = sub nsw i32 %1, %0 940 %cmp4 = icmp sgt i32 %sub3, 0 941 %cond = select i1 %cmp4, i32 %sub3, i32 0 942 %cmp5 = icmp slt i32 %minmax.028, %cond 943 %minmax.0.cond = select i1 %cmp5, i32 %minmax.028, i32 %cond 944 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 945 %lftr.wideiv = trunc i64 %indvars.iv.next to i32 946 %exitcond = icmp eq i32 %lftr.wideiv, %n 947 br i1 %exitcond, label %for.cond.cleanup.loopexit, label %scalar.body 948} 949 950; void recurrence_3(short *a, double *b, int n, float f, short p) { 951; b[0] = (double)a[0] - f * (double)p; 952; for (int i = 1; i < n; i++) 953; b[i] = (double)a[i] - f * (double)a[i - 1]; 954; } 955; 956; Check also that the casts were not moved needlessly. 957; 958; 959define void @recurrence_3(i16* nocapture readonly %a, double* nocapture %b, i32 %n, float %f, i16 %p) { 960; CHECK-LABEL: @recurrence_3( 961; CHECK-NEXT: entry: 962; CHECK-NEXT: [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2 963; CHECK-NEXT: [[CONV:%.*]] = sitofp i16 [[TMP0]] to double 964; CHECK-NEXT: [[CONV1:%.*]] = fpext float [[F:%.*]] to double 965; CHECK-NEXT: [[CONV2:%.*]] = sitofp i16 [[P:%.*]] to double 966; CHECK-NEXT: [[MUL:%.*]] = fmul fast double [[CONV2]], [[CONV1]] 967; CHECK-NEXT: [[SUB:%.*]] = fsub fast double [[CONV]], [[MUL]] 968; CHECK-NEXT: store double [[SUB]], double* [[B:%.*]], align 8 969; CHECK-NEXT: [[CMP25:%.*]] = icmp sgt i32 [[N:%.*]], 1 970; CHECK-NEXT: br i1 [[CMP25]], label [[FOR_PREHEADER:%.*]], label [[FOR_END:%.*]] 971; CHECK: for.preheader: 972; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[N]], -2 973; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64 974; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1 975; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], 3 976; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 977; CHECK: vector.memcheck: 978; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr double, double* [[B]], i64 1 979; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[N]], -2 980; CHECK-NEXT: [[TMP5:%.*]] = zext i32 [[TMP4]] to i64 981; CHECK-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[TMP5]], 2 982; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr double, double* [[B]], i64 [[TMP6]] 983; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i16, i16* [[A]], i64 1 984; CHECK-NEXT: [[SCEVGEP6:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP6]] 985; CHECK-NEXT: [[TMP7:%.*]] = bitcast i16* [[SCEVGEP6]] to double* 986; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult double* [[SCEVGEP]], [[TMP7]] 987; CHECK-NEXT: [[TMP8:%.*]] = bitcast double* [[SCEVGEP2]] to i16* 988; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult i16* [[SCEVGEP4]], [[TMP8]] 989; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 990; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 991; CHECK: vector.ph: 992; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP3]], 8589934588 993; CHECK-NEXT: [[IND_END:%.*]] = or i64 [[N_VEC]], 1 994; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[TMP0]], i64 3 995; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[CONV1]], i64 0 996; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer 997; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 998; CHECK: vector.body: 999; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1000; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ] 1001; CHECK-NEXT: [[OFFSET_IDX:%.*]] = or i64 [[INDEX]], 1 1002; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[OFFSET_IDX]] 1003; CHECK-NEXT: [[TMP10:%.*]] = bitcast i16* [[TMP9]] to <4 x i16>* 1004; CHECK-NEXT: [[WIDE_LOAD]] = load <4 x i16>, <4 x i16>* [[TMP10]], align 2 1005; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 1006; CHECK-NEXT: [[TMP12:%.*]] = sitofp <4 x i16> [[WIDE_LOAD]] to <4 x double> 1007; CHECK-NEXT: [[TMP13:%.*]] = sitofp <4 x i16> [[TMP11]] to <4 x double> 1008; CHECK-NEXT: [[TMP14:%.*]] = fmul fast <4 x double> [[BROADCAST_SPLAT]], [[TMP13]] 1009; CHECK-NEXT: [[TMP15:%.*]] = fsub fast <4 x double> [[TMP12]], [[TMP14]] 1010; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[OFFSET_IDX]] 1011; CHECK-NEXT: [[TMP17:%.*]] = bitcast double* [[TMP16]] to <4 x double>* 1012; CHECK-NEXT: store <4 x double> [[TMP15]], <4 x double>* [[TMP17]], align 8 1013; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 1014; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1015; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] 1016; CHECK: middle.block: 1017; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]] 1018; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 3 1019; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] 1020; CHECK: scalar.ph: 1021; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[TMP0]], [[VECTOR_MEMCHECK]] ], [ [[TMP0]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 1022; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1, [[VECTOR_MEMCHECK]] ], [ 1, [[FOR_PREHEADER]] ], [ [[IND_END]], [[MIDDLE_BLOCK]] ] 1023; CHECK-NEXT: br label [[SCALAR_BODY:%.*]] 1024; CHECK: scalar.body: 1025; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP19:%.*]], [[SCALAR_BODY]] ] 1026; CHECK-NEXT: [[ADVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ] 1027; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[ADVARS_IV]] 1028; CHECK-NEXT: [[TMP19]] = load i16, i16* [[ARRAYIDX5]], align 2 1029; CHECK-NEXT: [[CONV6:%.*]] = sitofp i16 [[TMP19]] to double 1030; CHECK-NEXT: [[CONV11:%.*]] = sitofp i16 [[SCALAR_RECUR]] to double 1031; CHECK-NEXT: [[MUL12:%.*]] = fmul fast double [[CONV11]], [[CONV1]] 1032; CHECK-NEXT: [[SUB13:%.*]] = fsub fast double [[CONV6]], [[MUL12]] 1033; CHECK-NEXT: [[ARRAYIDX15:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[ADVARS_IV]] 1034; CHECK-NEXT: store double [[SUB13]], double* [[ARRAYIDX15]], align 8 1035; CHECK-NEXT: [[ADVARS_IV_NEXT]] = add nuw nsw i64 [[ADVARS_IV]], 1 1036; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[ADVARS_IV_NEXT]] to i32 1037; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 1038; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP17:![0-9]+]] 1039; CHECK: for.end.loopexit: 1040; CHECK-NEXT: br label [[FOR_END]] 1041; CHECK: for.end: 1042; CHECK-NEXT: ret void 1043; 1044; UNROLL-LABEL: @recurrence_3( 1045; UNROLL-NEXT: entry: 1046; UNROLL-NEXT: [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2 1047; UNROLL-NEXT: [[CONV:%.*]] = sitofp i16 [[TMP0]] to double 1048; UNROLL-NEXT: [[CONV1:%.*]] = fpext float [[F:%.*]] to double 1049; UNROLL-NEXT: [[CONV2:%.*]] = sitofp i16 [[P:%.*]] to double 1050; UNROLL-NEXT: [[MUL:%.*]] = fmul fast double [[CONV2]], [[CONV1]] 1051; UNROLL-NEXT: [[SUB:%.*]] = fsub fast double [[CONV]], [[MUL]] 1052; UNROLL-NEXT: store double [[SUB]], double* [[B:%.*]], align 8 1053; UNROLL-NEXT: [[CMP25:%.*]] = icmp sgt i32 [[N:%.*]], 1 1054; UNROLL-NEXT: br i1 [[CMP25]], label [[FOR_PREHEADER:%.*]], label [[FOR_END:%.*]] 1055; UNROLL: for.preheader: 1056; UNROLL-NEXT: [[TMP1:%.*]] = add i32 [[N]], -2 1057; UNROLL-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64 1058; UNROLL-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1 1059; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], 7 1060; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 1061; UNROLL: vector.memcheck: 1062; UNROLL-NEXT: [[SCEVGEP:%.*]] = getelementptr double, double* [[B]], i64 1 1063; UNROLL-NEXT: [[TMP4:%.*]] = add i32 [[N]], -2 1064; UNROLL-NEXT: [[TMP5:%.*]] = zext i32 [[TMP4]] to i64 1065; UNROLL-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[TMP5]], 2 1066; UNROLL-NEXT: [[SCEVGEP2:%.*]] = getelementptr double, double* [[B]], i64 [[TMP6]] 1067; UNROLL-NEXT: [[SCEVGEP4:%.*]] = getelementptr i16, i16* [[A]], i64 1 1068; UNROLL-NEXT: [[SCEVGEP6:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP6]] 1069; UNROLL-NEXT: [[TMP7:%.*]] = bitcast i16* [[SCEVGEP6]] to double* 1070; UNROLL-NEXT: [[BOUND0:%.*]] = icmp ult double* [[SCEVGEP]], [[TMP7]] 1071; UNROLL-NEXT: [[TMP8:%.*]] = bitcast double* [[SCEVGEP2]] to i16* 1072; UNROLL-NEXT: [[BOUND1:%.*]] = icmp ult i16* [[SCEVGEP4]], [[TMP8]] 1073; UNROLL-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 1074; UNROLL-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 1075; UNROLL: vector.ph: 1076; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[TMP3]], 8589934584 1077; UNROLL-NEXT: [[IND_END:%.*]] = or i64 [[N_VEC]], 1 1078; UNROLL-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[TMP0]], i64 3 1079; UNROLL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[CONV1]], i64 0 1080; UNROLL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer 1081; UNROLL-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <4 x double> poison, double [[CONV1]], i64 0 1082; UNROLL-NEXT: [[BROADCAST_SPLAT10:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT9]], <4 x double> poison, <4 x i32> zeroinitializer 1083; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 1084; UNROLL: vector.body: 1085; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1086; UNROLL-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD8:%.*]], [[VECTOR_BODY]] ] 1087; UNROLL-NEXT: [[OFFSET_IDX:%.*]] = or i64 [[INDEX]], 1 1088; UNROLL-NEXT: [[TMP9:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[OFFSET_IDX]] 1089; UNROLL-NEXT: [[TMP10:%.*]] = bitcast i16* [[TMP9]] to <4 x i16>* 1090; UNROLL-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, <4 x i16>* [[TMP10]], align 2 1091; UNROLL-NEXT: [[TMP11:%.*]] = getelementptr inbounds i16, i16* [[TMP9]], i64 4 1092; UNROLL-NEXT: [[TMP12:%.*]] = bitcast i16* [[TMP11]] to <4 x i16>* 1093; UNROLL-NEXT: [[WIDE_LOAD8]] = load <4 x i16>, <4 x i16>* [[TMP12]], align 2 1094; UNROLL-NEXT: [[TMP13:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 1095; UNROLL-NEXT: [[TMP14:%.*]] = shufflevector <4 x i16> [[WIDE_LOAD]], <4 x i16> [[WIDE_LOAD8]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 1096; UNROLL-NEXT: [[TMP15:%.*]] = sitofp <4 x i16> [[WIDE_LOAD]] to <4 x double> 1097; UNROLL-NEXT: [[TMP16:%.*]] = sitofp <4 x i16> [[WIDE_LOAD8]] to <4 x double> 1098; UNROLL-NEXT: [[TMP17:%.*]] = sitofp <4 x i16> [[TMP13]] to <4 x double> 1099; UNROLL-NEXT: [[TMP18:%.*]] = sitofp <4 x i16> [[TMP14]] to <4 x double> 1100; UNROLL-NEXT: [[TMP19:%.*]] = fmul fast <4 x double> [[BROADCAST_SPLAT]], [[TMP17]] 1101; UNROLL-NEXT: [[TMP20:%.*]] = fmul fast <4 x double> [[BROADCAST_SPLAT10]], [[TMP18]] 1102; UNROLL-NEXT: [[TMP21:%.*]] = fsub fast <4 x double> [[TMP15]], [[TMP19]] 1103; UNROLL-NEXT: [[TMP22:%.*]] = fsub fast <4 x double> [[TMP16]], [[TMP20]] 1104; UNROLL-NEXT: [[TMP23:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[OFFSET_IDX]] 1105; UNROLL-NEXT: [[TMP24:%.*]] = bitcast double* [[TMP23]] to <4 x double>* 1106; UNROLL-NEXT: store <4 x double> [[TMP21]], <4 x double>* [[TMP24]], align 8 1107; UNROLL-NEXT: [[TMP25:%.*]] = getelementptr inbounds double, double* [[TMP23]], i64 4 1108; UNROLL-NEXT: [[TMP26:%.*]] = bitcast double* [[TMP25]] to <4 x double>* 1109; UNROLL-NEXT: store <4 x double> [[TMP22]], <4 x double>* [[TMP26]], align 8 1110; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 1111; UNROLL-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1112; UNROLL-NEXT: br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] 1113; UNROLL: middle.block: 1114; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]] 1115; UNROLL-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD8]], i64 3 1116; UNROLL-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] 1117; UNROLL: scalar.ph: 1118; UNROLL-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[TMP0]], [[VECTOR_MEMCHECK]] ], [ [[TMP0]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 1119; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1, [[VECTOR_MEMCHECK]] ], [ 1, [[FOR_PREHEADER]] ], [ [[IND_END]], [[MIDDLE_BLOCK]] ] 1120; UNROLL-NEXT: br label [[SCALAR_BODY:%.*]] 1121; UNROLL: scalar.body: 1122; UNROLL-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP28:%.*]], [[SCALAR_BODY]] ] 1123; UNROLL-NEXT: [[ADVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ] 1124; UNROLL-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[ADVARS_IV]] 1125; UNROLL-NEXT: [[TMP28]] = load i16, i16* [[ARRAYIDX5]], align 2 1126; UNROLL-NEXT: [[CONV6:%.*]] = sitofp i16 [[TMP28]] to double 1127; UNROLL-NEXT: [[CONV11:%.*]] = sitofp i16 [[SCALAR_RECUR]] to double 1128; UNROLL-NEXT: [[MUL12:%.*]] = fmul fast double [[CONV11]], [[CONV1]] 1129; UNROLL-NEXT: [[SUB13:%.*]] = fsub fast double [[CONV6]], [[MUL12]] 1130; UNROLL-NEXT: [[ARRAYIDX15:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[ADVARS_IV]] 1131; UNROLL-NEXT: store double [[SUB13]], double* [[ARRAYIDX15]], align 8 1132; UNROLL-NEXT: [[ADVARS_IV_NEXT]] = add nuw nsw i64 [[ADVARS_IV]], 1 1133; UNROLL-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[ADVARS_IV_NEXT]] to i32 1134; UNROLL-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 1135; UNROLL-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP17:![0-9]+]] 1136; UNROLL: for.end.loopexit: 1137; UNROLL-NEXT: br label [[FOR_END]] 1138; UNROLL: for.end: 1139; UNROLL-NEXT: ret void 1140; 1141; UNROLL-NO-IC-LABEL: @recurrence_3( 1142; UNROLL-NO-IC-NEXT: entry: 1143; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2 1144; UNROLL-NO-IC-NEXT: [[CONV:%.*]] = sitofp i16 [[TMP0]] to double 1145; UNROLL-NO-IC-NEXT: [[CONV1:%.*]] = fpext float [[F:%.*]] to double 1146; UNROLL-NO-IC-NEXT: [[CONV2:%.*]] = sitofp i16 [[P:%.*]] to double 1147; UNROLL-NO-IC-NEXT: [[MUL:%.*]] = fmul fast double [[CONV2]], [[CONV1]] 1148; UNROLL-NO-IC-NEXT: [[SUB:%.*]] = fsub fast double [[CONV]], [[MUL]] 1149; UNROLL-NO-IC-NEXT: store double [[SUB]], double* [[B:%.*]], align 8 1150; UNROLL-NO-IC-NEXT: [[CMP25:%.*]] = icmp sgt i32 [[N:%.*]], 1 1151; UNROLL-NO-IC-NEXT: br i1 [[CMP25]], label [[FOR_PREHEADER:%.*]], label [[FOR_END:%.*]] 1152; UNROLL-NO-IC: for.preheader: 1153; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = add i32 [[N]], -2 1154; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64 1155; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1 1156; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 8 1157; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 1158; UNROLL-NO-IC: vector.memcheck: 1159; UNROLL-NO-IC-NEXT: [[SCEVGEP:%.*]] = getelementptr double, double* [[B]], i64 1 1160; UNROLL-NO-IC-NEXT: [[SCEVGEP1:%.*]] = bitcast double* [[SCEVGEP]] to i8* 1161; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = add i32 [[N]], -2 1162; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = zext i32 [[TMP4]] to i64 1163; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[TMP5]], 2 1164; UNROLL-NO-IC-NEXT: [[SCEVGEP2:%.*]] = getelementptr double, double* [[B]], i64 [[TMP6]] 1165; UNROLL-NO-IC-NEXT: [[SCEVGEP23:%.*]] = bitcast double* [[SCEVGEP2]] to i8* 1166; UNROLL-NO-IC-NEXT: [[SCEVGEP4:%.*]] = getelementptr i16, i16* [[A]], i64 1 1167; UNROLL-NO-IC-NEXT: [[SCEVGEP45:%.*]] = bitcast i16* [[SCEVGEP4]] to i8* 1168; UNROLL-NO-IC-NEXT: [[SCEVGEP6:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP6]] 1169; UNROLL-NO-IC-NEXT: [[SCEVGEP67:%.*]] = bitcast i16* [[SCEVGEP6]] to i8* 1170; UNROLL-NO-IC-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[SCEVGEP1]], [[SCEVGEP67]] 1171; UNROLL-NO-IC-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP45]], [[SCEVGEP23]] 1172; UNROLL-NO-IC-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 1173; UNROLL-NO-IC-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 1174; UNROLL-NO-IC: vector.ph: 1175; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 8 1176; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]] 1177; UNROLL-NO-IC-NEXT: [[IND_END:%.*]] = add i64 1, [[N_VEC]] 1178; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[TMP0]], i32 3 1179; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[CONV1]], i32 0 1180; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer 1181; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <4 x double> poison, double [[CONV1]], i32 0 1182; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLAT10:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT9]], <4 x double> poison, <4 x i32> zeroinitializer 1183; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 1184; UNROLL-NO-IC: vector.body: 1185; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1186; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD8:%.*]], [[VECTOR_BODY]] ] 1187; UNROLL-NO-IC-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]] 1188; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 0 1189; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = add i64 [[OFFSET_IDX]], 4 1190; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP7]] 1191; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP8]] 1192; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = getelementptr inbounds i16, i16* [[TMP9]], i32 0 1193; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = bitcast i16* [[TMP11]] to <4 x i16>* 1194; UNROLL-NO-IC-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, <4 x i16>* [[TMP12]], align 2 1195; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = getelementptr inbounds i16, i16* [[TMP9]], i32 4 1196; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = bitcast i16* [[TMP13]] to <4 x i16>* 1197; UNROLL-NO-IC-NEXT: [[WIDE_LOAD8]] = load <4 x i16>, <4 x i16>* [[TMP14]], align 2 1198; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 1199; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = shufflevector <4 x i16> [[WIDE_LOAD]], <4 x i16> [[WIDE_LOAD8]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 1200; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = sitofp <4 x i16> [[WIDE_LOAD]] to <4 x double> 1201; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = sitofp <4 x i16> [[WIDE_LOAD8]] to <4 x double> 1202; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = sitofp <4 x i16> [[TMP15]] to <4 x double> 1203; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = sitofp <4 x i16> [[TMP16]] to <4 x double> 1204; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = fmul fast <4 x double> [[TMP19]], [[BROADCAST_SPLAT]] 1205; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = fmul fast <4 x double> [[TMP20]], [[BROADCAST_SPLAT10]] 1206; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = fsub fast <4 x double> [[TMP17]], [[TMP21]] 1207; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = fsub fast <4 x double> [[TMP18]], [[TMP22]] 1208; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[TMP7]] 1209; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[TMP8]] 1210; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = getelementptr inbounds double, double* [[TMP25]], i32 0 1211; UNROLL-NO-IC-NEXT: [[TMP28:%.*]] = bitcast double* [[TMP27]] to <4 x double>* 1212; UNROLL-NO-IC-NEXT: store <4 x double> [[TMP23]], <4 x double>* [[TMP28]], align 8 1213; UNROLL-NO-IC-NEXT: [[TMP29:%.*]] = getelementptr inbounds double, double* [[TMP25]], i32 4 1214; UNROLL-NO-IC-NEXT: [[TMP30:%.*]] = bitcast double* [[TMP29]] to <4 x double>* 1215; UNROLL-NO-IC-NEXT: store <4 x double> [[TMP24]], <4 x double>* [[TMP30]], align 8 1216; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 1217; UNROLL-NO-IC-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1218; UNROLL-NO-IC-NEXT: br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] 1219; UNROLL-NO-IC: middle.block: 1220; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]] 1221; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD8]], i32 3 1222; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD8]], i32 2 1223; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] 1224; UNROLL-NO-IC: scalar.ph: 1225; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[TMP0]], [[VECTOR_MEMCHECK]] ], [ [[TMP0]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 1226; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[FOR_PREHEADER]] ], [ 1, [[VECTOR_MEMCHECK]] ] 1227; UNROLL-NO-IC-NEXT: br label [[SCALAR_BODY:%.*]] 1228; UNROLL-NO-IC: scalar.body: 1229; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP32:%.*]], [[SCALAR_BODY]] ] 1230; UNROLL-NO-IC-NEXT: [[ADVARS_IV:%.*]] = phi i64 [ [[ADVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 1231; UNROLL-NO-IC-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[ADVARS_IV]] 1232; UNROLL-NO-IC-NEXT: [[TMP32]] = load i16, i16* [[ARRAYIDX5]], align 2 1233; UNROLL-NO-IC-NEXT: [[CONV6:%.*]] = sitofp i16 [[TMP32]] to double 1234; UNROLL-NO-IC-NEXT: [[CONV11:%.*]] = sitofp i16 [[SCALAR_RECUR]] to double 1235; UNROLL-NO-IC-NEXT: [[MUL12:%.*]] = fmul fast double [[CONV11]], [[CONV1]] 1236; UNROLL-NO-IC-NEXT: [[SUB13:%.*]] = fsub fast double [[CONV6]], [[MUL12]] 1237; UNROLL-NO-IC-NEXT: [[ARRAYIDX15:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[ADVARS_IV]] 1238; UNROLL-NO-IC-NEXT: store double [[SUB13]], double* [[ARRAYIDX15]], align 8 1239; UNROLL-NO-IC-NEXT: [[ADVARS_IV_NEXT]] = add nuw nsw i64 [[ADVARS_IV]], 1 1240; UNROLL-NO-IC-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[ADVARS_IV_NEXT]] to i32 1241; UNROLL-NO-IC-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 1242; UNROLL-NO-IC-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP17:![0-9]+]] 1243; UNROLL-NO-IC: for.end.loopexit: 1244; UNROLL-NO-IC-NEXT: br label [[FOR_END]] 1245; UNROLL-NO-IC: for.end: 1246; UNROLL-NO-IC-NEXT: ret void 1247; 1248; UNROLL-NO-VF-LABEL: @recurrence_3( 1249; UNROLL-NO-VF-NEXT: entry: 1250; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2 1251; UNROLL-NO-VF-NEXT: [[CONV:%.*]] = sitofp i16 [[TMP0]] to double 1252; UNROLL-NO-VF-NEXT: [[CONV1:%.*]] = fpext float [[F:%.*]] to double 1253; UNROLL-NO-VF-NEXT: [[CONV2:%.*]] = sitofp i16 [[P:%.*]] to double 1254; UNROLL-NO-VF-NEXT: [[MUL:%.*]] = fmul fast double [[CONV2]], [[CONV1]] 1255; UNROLL-NO-VF-NEXT: [[SUB:%.*]] = fsub fast double [[CONV]], [[MUL]] 1256; UNROLL-NO-VF-NEXT: store double [[SUB]], double* [[B:%.*]], align 8 1257; UNROLL-NO-VF-NEXT: [[CMP25:%.*]] = icmp sgt i32 [[N:%.*]], 1 1258; UNROLL-NO-VF-NEXT: br i1 [[CMP25]], label [[FOR_PREHEADER:%.*]], label [[FOR_END:%.*]] 1259; UNROLL-NO-VF: for.preheader: 1260; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = add i32 [[N]], -2 1261; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64 1262; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1 1263; UNROLL-NO-VF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 2 1264; UNROLL-NO-VF-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 1265; UNROLL-NO-VF: vector.memcheck: 1266; UNROLL-NO-VF-NEXT: [[SCEVGEP:%.*]] = getelementptr double, double* [[B]], i64 1 1267; UNROLL-NO-VF-NEXT: [[SCEVGEP1:%.*]] = bitcast double* [[SCEVGEP]] to i8* 1268; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = add i32 [[N]], -2 1269; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = zext i32 [[TMP4]] to i64 1270; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[TMP5]], 2 1271; UNROLL-NO-VF-NEXT: [[SCEVGEP2:%.*]] = getelementptr double, double* [[B]], i64 [[TMP6]] 1272; UNROLL-NO-VF-NEXT: [[SCEVGEP23:%.*]] = bitcast double* [[SCEVGEP2]] to i8* 1273; UNROLL-NO-VF-NEXT: [[SCEVGEP4:%.*]] = getelementptr i16, i16* [[A]], i64 1 1274; UNROLL-NO-VF-NEXT: [[SCEVGEP45:%.*]] = bitcast i16* [[SCEVGEP4]] to i8* 1275; UNROLL-NO-VF-NEXT: [[SCEVGEP6:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP6]] 1276; UNROLL-NO-VF-NEXT: [[SCEVGEP67:%.*]] = bitcast i16* [[SCEVGEP6]] to i8* 1277; UNROLL-NO-VF-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[SCEVGEP1]], [[SCEVGEP67]] 1278; UNROLL-NO-VF-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP45]], [[SCEVGEP23]] 1279; UNROLL-NO-VF-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 1280; UNROLL-NO-VF-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 1281; UNROLL-NO-VF: vector.ph: 1282; UNROLL-NO-VF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 2 1283; UNROLL-NO-VF-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]] 1284; UNROLL-NO-VF-NEXT: [[IND_END:%.*]] = add i64 1, [[N_VEC]] 1285; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 1286; UNROLL-NO-VF: vector.body: 1287; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1288; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i16 [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ] 1289; UNROLL-NO-VF-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]] 1290; UNROLL-NO-VF-NEXT: [[INDUCTION:%.*]] = add i64 [[OFFSET_IDX]], 0 1291; UNROLL-NO-VF-NEXT: [[INDUCTION8:%.*]] = add i64 [[OFFSET_IDX]], 1 1292; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[INDUCTION]] 1293; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[INDUCTION8]] 1294; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = load i16, i16* [[TMP7]], align 2 1295; UNROLL-NO-VF-NEXT: [[TMP10]] = load i16, i16* [[TMP8]], align 2 1296; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = sitofp i16 [[TMP9]] to double 1297; UNROLL-NO-VF-NEXT: [[TMP12:%.*]] = sitofp i16 [[TMP10]] to double 1298; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = sitofp i16 [[VECTOR_RECUR]] to double 1299; UNROLL-NO-VF-NEXT: [[TMP14:%.*]] = sitofp i16 [[TMP9]] to double 1300; UNROLL-NO-VF-NEXT: [[TMP15:%.*]] = fmul fast double [[TMP13]], [[CONV1]] 1301; UNROLL-NO-VF-NEXT: [[TMP16:%.*]] = fmul fast double [[TMP14]], [[CONV1]] 1302; UNROLL-NO-VF-NEXT: [[TMP17:%.*]] = fsub fast double [[TMP11]], [[TMP15]] 1303; UNROLL-NO-VF-NEXT: [[TMP18:%.*]] = fsub fast double [[TMP12]], [[TMP16]] 1304; UNROLL-NO-VF-NEXT: [[TMP19:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[INDUCTION]] 1305; UNROLL-NO-VF-NEXT: [[TMP20:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[INDUCTION8]] 1306; UNROLL-NO-VF-NEXT: store double [[TMP17]], double* [[TMP19]], align 8 1307; UNROLL-NO-VF-NEXT: store double [[TMP18]], double* [[TMP20]], align 8 1308; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 1309; UNROLL-NO-VF-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1310; UNROLL-NO-VF-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]] 1311; UNROLL-NO-VF: middle.block: 1312; UNROLL-NO-VF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]] 1313; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] 1314; UNROLL-NO-VF: scalar.ph: 1315; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[TMP0]], [[VECTOR_MEMCHECK]] ], [ [[TMP0]], [[FOR_PREHEADER]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ] 1316; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[FOR_PREHEADER]] ], [ 1, [[VECTOR_MEMCHECK]] ] 1317; UNROLL-NO-VF-NEXT: br label [[SCALAR_BODY:%.*]] 1318; UNROLL-NO-VF: scalar.body: 1319; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP22:%.*]], [[SCALAR_BODY]] ] 1320; UNROLL-NO-VF-NEXT: [[ADVARS_IV:%.*]] = phi i64 [ [[ADVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 1321; UNROLL-NO-VF-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[ADVARS_IV]] 1322; UNROLL-NO-VF-NEXT: [[TMP22]] = load i16, i16* [[ARRAYIDX5]], align 2 1323; UNROLL-NO-VF-NEXT: [[CONV6:%.*]] = sitofp i16 [[TMP22]] to double 1324; UNROLL-NO-VF-NEXT: [[CONV11:%.*]] = sitofp i16 [[SCALAR_RECUR]] to double 1325; UNROLL-NO-VF-NEXT: [[MUL12:%.*]] = fmul fast double [[CONV11]], [[CONV1]] 1326; UNROLL-NO-VF-NEXT: [[SUB13:%.*]] = fsub fast double [[CONV6]], [[MUL12]] 1327; UNROLL-NO-VF-NEXT: [[ARRAYIDX15:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[ADVARS_IV]] 1328; UNROLL-NO-VF-NEXT: store double [[SUB13]], double* [[ARRAYIDX15]], align 8 1329; UNROLL-NO-VF-NEXT: [[ADVARS_IV_NEXT]] = add nuw nsw i64 [[ADVARS_IV]], 1 1330; UNROLL-NO-VF-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[ADVARS_IV_NEXT]] to i32 1331; UNROLL-NO-VF-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 1332; UNROLL-NO-VF-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] 1333; UNROLL-NO-VF: for.end.loopexit: 1334; UNROLL-NO-VF-NEXT: br label [[FOR_END]] 1335; UNROLL-NO-VF: for.end: 1336; UNROLL-NO-VF-NEXT: ret void 1337; 1338; SINK-AFTER-LABEL: @recurrence_3( 1339; SINK-AFTER-NEXT: entry: 1340; SINK-AFTER-NEXT: [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2 1341; SINK-AFTER-NEXT: [[CONV:%.*]] = sitofp i16 [[TMP0]] to double 1342; SINK-AFTER-NEXT: [[CONV1:%.*]] = fpext float [[F:%.*]] to double 1343; SINK-AFTER-NEXT: [[CONV2:%.*]] = sitofp i16 [[P:%.*]] to double 1344; SINK-AFTER-NEXT: [[MUL:%.*]] = fmul fast double [[CONV2]], [[CONV1]] 1345; SINK-AFTER-NEXT: [[SUB:%.*]] = fsub fast double [[CONV]], [[MUL]] 1346; SINK-AFTER-NEXT: store double [[SUB]], double* [[B:%.*]], align 8 1347; SINK-AFTER-NEXT: [[CMP25:%.*]] = icmp sgt i32 [[N:%.*]], 1 1348; SINK-AFTER-NEXT: br i1 [[CMP25]], label [[FOR_PREHEADER:%.*]], label [[FOR_END:%.*]] 1349; SINK-AFTER: for.preheader: 1350; SINK-AFTER-NEXT: [[TMP1:%.*]] = add i32 [[N]], -2 1351; SINK-AFTER-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64 1352; SINK-AFTER-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1 1353; SINK-AFTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4 1354; SINK-AFTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 1355; SINK-AFTER: vector.memcheck: 1356; SINK-AFTER-NEXT: [[SCEVGEP:%.*]] = getelementptr double, double* [[B]], i64 1 1357; SINK-AFTER-NEXT: [[SCEVGEP1:%.*]] = bitcast double* [[SCEVGEP]] to i8* 1358; SINK-AFTER-NEXT: [[TMP4:%.*]] = add i32 [[N]], -2 1359; SINK-AFTER-NEXT: [[TMP5:%.*]] = zext i32 [[TMP4]] to i64 1360; SINK-AFTER-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[TMP5]], 2 1361; SINK-AFTER-NEXT: [[SCEVGEP2:%.*]] = getelementptr double, double* [[B]], i64 [[TMP6]] 1362; SINK-AFTER-NEXT: [[SCEVGEP23:%.*]] = bitcast double* [[SCEVGEP2]] to i8* 1363; SINK-AFTER-NEXT: [[SCEVGEP4:%.*]] = getelementptr i16, i16* [[A]], i64 1 1364; SINK-AFTER-NEXT: [[SCEVGEP45:%.*]] = bitcast i16* [[SCEVGEP4]] to i8* 1365; SINK-AFTER-NEXT: [[SCEVGEP6:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP6]] 1366; SINK-AFTER-NEXT: [[SCEVGEP67:%.*]] = bitcast i16* [[SCEVGEP6]] to i8* 1367; SINK-AFTER-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[SCEVGEP1]], [[SCEVGEP67]] 1368; SINK-AFTER-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP45]], [[SCEVGEP23]] 1369; SINK-AFTER-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 1370; SINK-AFTER-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 1371; SINK-AFTER: vector.ph: 1372; SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4 1373; SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]] 1374; SINK-AFTER-NEXT: [[IND_END:%.*]] = add i64 1, [[N_VEC]] 1375; SINK-AFTER-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[TMP0]], i32 3 1376; SINK-AFTER-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[CONV1]], i32 0 1377; SINK-AFTER-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer 1378; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 1379; SINK-AFTER: vector.body: 1380; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1381; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ] 1382; SINK-AFTER-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]] 1383; SINK-AFTER-NEXT: [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 0 1384; SINK-AFTER-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP7]] 1385; SINK-AFTER-NEXT: [[TMP9:%.*]] = getelementptr inbounds i16, i16* [[TMP8]], i32 0 1386; SINK-AFTER-NEXT: [[TMP10:%.*]] = bitcast i16* [[TMP9]] to <4 x i16>* 1387; SINK-AFTER-NEXT: [[WIDE_LOAD]] = load <4 x i16>, <4 x i16>* [[TMP10]], align 2 1388; SINK-AFTER-NEXT: [[TMP11:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 1389; SINK-AFTER-NEXT: [[TMP12:%.*]] = sitofp <4 x i16> [[WIDE_LOAD]] to <4 x double> 1390; SINK-AFTER-NEXT: [[TMP13:%.*]] = sitofp <4 x i16> [[TMP11]] to <4 x double> 1391; SINK-AFTER-NEXT: [[TMP14:%.*]] = fmul fast <4 x double> [[TMP13]], [[BROADCAST_SPLAT]] 1392; SINK-AFTER-NEXT: [[TMP15:%.*]] = fsub fast <4 x double> [[TMP12]], [[TMP14]] 1393; SINK-AFTER-NEXT: [[TMP16:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[TMP7]] 1394; SINK-AFTER-NEXT: [[TMP17:%.*]] = getelementptr inbounds double, double* [[TMP16]], i32 0 1395; SINK-AFTER-NEXT: [[TMP18:%.*]] = bitcast double* [[TMP17]] to <4 x double>* 1396; SINK-AFTER-NEXT: store <4 x double> [[TMP15]], <4 x double>* [[TMP18]], align 8 1397; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 1398; SINK-AFTER-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1399; SINK-AFTER-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] 1400; SINK-AFTER: middle.block: 1401; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]] 1402; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3 1403; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2 1404; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] 1405; SINK-AFTER: scalar.ph: 1406; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[TMP0]], [[VECTOR_MEMCHECK]] ], [ [[TMP0]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 1407; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[FOR_PREHEADER]] ], [ 1, [[VECTOR_MEMCHECK]] ] 1408; SINK-AFTER-NEXT: br label [[SCALAR_BODY:%.*]] 1409; SINK-AFTER: scalar.body: 1410; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP20:%.*]], [[SCALAR_BODY]] ] 1411; SINK-AFTER-NEXT: [[ADVARS_IV:%.*]] = phi i64 [ [[ADVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 1412; SINK-AFTER-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[ADVARS_IV]] 1413; SINK-AFTER-NEXT: [[TMP20]] = load i16, i16* [[ARRAYIDX5]], align 2 1414; SINK-AFTER-NEXT: [[CONV6:%.*]] = sitofp i16 [[TMP20]] to double 1415; SINK-AFTER-NEXT: [[CONV11:%.*]] = sitofp i16 [[SCALAR_RECUR]] to double 1416; SINK-AFTER-NEXT: [[MUL12:%.*]] = fmul fast double [[CONV11]], [[CONV1]] 1417; SINK-AFTER-NEXT: [[SUB13:%.*]] = fsub fast double [[CONV6]], [[MUL12]] 1418; SINK-AFTER-NEXT: [[ARRAYIDX15:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[ADVARS_IV]] 1419; SINK-AFTER-NEXT: store double [[SUB13]], double* [[ARRAYIDX15]], align 8 1420; SINK-AFTER-NEXT: [[ADVARS_IV_NEXT]] = add nuw nsw i64 [[ADVARS_IV]], 1 1421; SINK-AFTER-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[ADVARS_IV_NEXT]] to i32 1422; SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 1423; SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP17:![0-9]+]] 1424; SINK-AFTER: for.end.loopexit: 1425; SINK-AFTER-NEXT: br label [[FOR_END]] 1426; SINK-AFTER: for.end: 1427; SINK-AFTER-NEXT: ret void 1428; 1429; NO-SINK-AFTER-LABEL: @recurrence_3( 1430; NO-SINK-AFTER-NEXT: entry: 1431; NO-SINK-AFTER-NEXT: [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2 1432; NO-SINK-AFTER-NEXT: [[CONV:%.*]] = sitofp i16 [[TMP0]] to double 1433; NO-SINK-AFTER-NEXT: [[CONV1:%.*]] = fpext float [[F:%.*]] to double 1434; NO-SINK-AFTER-NEXT: [[CONV2:%.*]] = sitofp i16 [[P:%.*]] to double 1435; NO-SINK-AFTER-NEXT: [[MUL:%.*]] = fmul fast double [[CONV2]], [[CONV1]] 1436; NO-SINK-AFTER-NEXT: [[SUB:%.*]] = fsub fast double [[CONV]], [[MUL]] 1437; NO-SINK-AFTER-NEXT: store double [[SUB]], double* [[B:%.*]], align 8 1438; NO-SINK-AFTER-NEXT: [[CMP25:%.*]] = icmp sgt i32 [[N:%.*]], 1 1439; NO-SINK-AFTER-NEXT: br i1 [[CMP25]], label [[FOR_PREHEADER:%.*]], label [[FOR_END:%.*]] 1440; NO-SINK-AFTER: for.preheader: 1441; NO-SINK-AFTER-NEXT: [[TMP1:%.*]] = add i32 [[N]], -2 1442; NO-SINK-AFTER-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64 1443; NO-SINK-AFTER-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1 1444; NO-SINK-AFTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4 1445; NO-SINK-AFTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 1446; NO-SINK-AFTER: vector.memcheck: 1447; NO-SINK-AFTER-NEXT: [[SCEVGEP:%.*]] = getelementptr double, double* [[B]], i64 1 1448; NO-SINK-AFTER-NEXT: [[SCEVGEP1:%.*]] = bitcast double* [[SCEVGEP]] to i8* 1449; NO-SINK-AFTER-NEXT: [[TMP4:%.*]] = add i32 [[N]], -2 1450; NO-SINK-AFTER-NEXT: [[TMP5:%.*]] = zext i32 [[TMP4]] to i64 1451; NO-SINK-AFTER-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[TMP5]], 2 1452; NO-SINK-AFTER-NEXT: [[SCEVGEP2:%.*]] = getelementptr double, double* [[B]], i64 [[TMP6]] 1453; NO-SINK-AFTER-NEXT: [[SCEVGEP23:%.*]] = bitcast double* [[SCEVGEP2]] to i8* 1454; NO-SINK-AFTER-NEXT: [[SCEVGEP4:%.*]] = getelementptr i16, i16* [[A]], i64 1 1455; NO-SINK-AFTER-NEXT: [[SCEVGEP45:%.*]] = bitcast i16* [[SCEVGEP4]] to i8* 1456; NO-SINK-AFTER-NEXT: [[SCEVGEP6:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP6]] 1457; NO-SINK-AFTER-NEXT: [[SCEVGEP67:%.*]] = bitcast i16* [[SCEVGEP6]] to i8* 1458; NO-SINK-AFTER-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[SCEVGEP1]], [[SCEVGEP67]] 1459; NO-SINK-AFTER-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP45]], [[SCEVGEP23]] 1460; NO-SINK-AFTER-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 1461; NO-SINK-AFTER-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 1462; NO-SINK-AFTER: vector.ph: 1463; NO-SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4 1464; NO-SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]] 1465; NO-SINK-AFTER-NEXT: [[IND_END:%.*]] = add i64 1, [[N_VEC]] 1466; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[TMP0]], i32 3 1467; NO-SINK-AFTER-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[CONV1]], i32 0 1468; NO-SINK-AFTER-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer 1469; NO-SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 1470; NO-SINK-AFTER: vector.body: 1471; NO-SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1472; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ] 1473; NO-SINK-AFTER-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]] 1474; NO-SINK-AFTER-NEXT: [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 0 1475; NO-SINK-AFTER-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP7]] 1476; NO-SINK-AFTER-NEXT: [[TMP9:%.*]] = getelementptr inbounds i16, i16* [[TMP8]], i32 0 1477; NO-SINK-AFTER-NEXT: [[TMP10:%.*]] = bitcast i16* [[TMP9]] to <4 x i16>* 1478; NO-SINK-AFTER-NEXT: [[WIDE_LOAD]] = load <4 x i16>, <4 x i16>* [[TMP10]], align 2 1479; NO-SINK-AFTER-NEXT: [[TMP11:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 1480; NO-SINK-AFTER-NEXT: [[TMP12:%.*]] = sitofp <4 x i16> [[WIDE_LOAD]] to <4 x double> 1481; NO-SINK-AFTER-NEXT: [[TMP13:%.*]] = sitofp <4 x i16> [[TMP11]] to <4 x double> 1482; NO-SINK-AFTER-NEXT: [[TMP14:%.*]] = fmul fast <4 x double> [[TMP13]], [[BROADCAST_SPLAT]] 1483; NO-SINK-AFTER-NEXT: [[TMP15:%.*]] = fsub fast <4 x double> [[TMP12]], [[TMP14]] 1484; NO-SINK-AFTER-NEXT: [[TMP16:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[TMP7]] 1485; NO-SINK-AFTER-NEXT: [[TMP17:%.*]] = getelementptr inbounds double, double* [[TMP16]], i32 0 1486; NO-SINK-AFTER-NEXT: [[TMP18:%.*]] = bitcast double* [[TMP17]] to <4 x double>* 1487; NO-SINK-AFTER-NEXT: store <4 x double> [[TMP15]], <4 x double>* [[TMP18]], align 8 1488; NO-SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 1489; NO-SINK-AFTER-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1490; NO-SINK-AFTER-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] 1491; NO-SINK-AFTER: middle.block: 1492; NO-SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]] 1493; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3 1494; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2 1495; NO-SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] 1496; NO-SINK-AFTER: scalar.ph: 1497; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[TMP0]], [[VECTOR_MEMCHECK]] ], [ [[TMP0]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 1498; NO-SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[FOR_PREHEADER]] ], [ 1, [[VECTOR_MEMCHECK]] ] 1499; NO-SINK-AFTER-NEXT: br label [[SCALAR_BODY:%.*]] 1500; NO-SINK-AFTER: scalar.body: 1501; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP20:%.*]], [[SCALAR_BODY]] ] 1502; NO-SINK-AFTER-NEXT: [[ADVARS_IV:%.*]] = phi i64 [ [[ADVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 1503; NO-SINK-AFTER-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[ADVARS_IV]] 1504; NO-SINK-AFTER-NEXT: [[TMP20]] = load i16, i16* [[ARRAYIDX5]], align 2 1505; NO-SINK-AFTER-NEXT: [[CONV6:%.*]] = sitofp i16 [[TMP20]] to double 1506; NO-SINK-AFTER-NEXT: [[CONV11:%.*]] = sitofp i16 [[SCALAR_RECUR]] to double 1507; NO-SINK-AFTER-NEXT: [[MUL12:%.*]] = fmul fast double [[CONV11]], [[CONV1]] 1508; NO-SINK-AFTER-NEXT: [[SUB13:%.*]] = fsub fast double [[CONV6]], [[MUL12]] 1509; NO-SINK-AFTER-NEXT: [[ARRAYIDX15:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[ADVARS_IV]] 1510; NO-SINK-AFTER-NEXT: store double [[SUB13]], double* [[ARRAYIDX15]], align 8 1511; NO-SINK-AFTER-NEXT: [[ADVARS_IV_NEXT]] = add nuw nsw i64 [[ADVARS_IV]], 1 1512; NO-SINK-AFTER-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[ADVARS_IV_NEXT]] to i32 1513; NO-SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 1514; NO-SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP17:![0-9]+]] 1515; NO-SINK-AFTER: for.end.loopexit: 1516; NO-SINK-AFTER-NEXT: br label [[FOR_END]] 1517; NO-SINK-AFTER: for.end: 1518; NO-SINK-AFTER-NEXT: ret void 1519; 1520entry: 1521 %0 = load i16, i16* %a, align 2 1522 %conv = sitofp i16 %0 to double 1523 %conv1 = fpext float %f to double 1524 %conv2 = sitofp i16 %p to double 1525 %mul = fmul fast double %conv2, %conv1 1526 %sub = fsub fast double %conv, %mul 1527 store double %sub, double* %b, align 8 1528 %cmp25 = icmp sgt i32 %n, 1 1529 br i1 %cmp25, label %for.preheader, label %for.end 1530 1531for.preheader: 1532 br label %scalar.body 1533 1534scalar.body: 1535 %1 = phi i16 [ %0, %for.preheader ], [ %2, %scalar.body ] 1536 %advars.iv = phi i64 [ %advars.iv.next, %scalar.body ], [ 1, %for.preheader ] 1537 %arrayidx5 = getelementptr inbounds i16, i16* %a, i64 %advars.iv 1538 %2 = load i16, i16* %arrayidx5, align 2 1539 %conv6 = sitofp i16 %2 to double 1540 %conv11 = sitofp i16 %1 to double 1541 %mul12 = fmul fast double %conv11, %conv1 1542 %sub13 = fsub fast double %conv6, %mul12 1543 %arrayidx15 = getelementptr inbounds double, double* %b, i64 %advars.iv 1544 store double %sub13, double* %arrayidx15, align 8 1545 %advars.iv.next = add nuw nsw i64 %advars.iv, 1 1546 %lftr.wideiv = trunc i64 %advars.iv.next to i32 1547 %exitcond = icmp eq i32 %lftr.wideiv, %n 1548 br i1 %exitcond, label %for.end.loopexit, label %scalar.body 1549 1550for.end.loopexit: 1551 br label %for.end 1552 1553for.end: 1554 ret void 1555} 1556 1557; void PR26734(short *a, int *b, int *c, int d, short *e) { 1558; for (; d != 21; d++) { 1559; *b &= *c; 1560; *e = *a - 6; 1561; *c = *e; 1562; } 1563; } 1564; 1565; 1566define void @PR26734(i16* %a, i32* %b, i32* %c, i32 %d, i16* %e) { 1567; CHECK-LABEL: @PR26734( 1568; CHECK-NEXT: entry: 1569; CHECK-NEXT: [[CMP4:%.*]] = icmp eq i32 [[D:%.*]], 21 1570; CHECK-NEXT: br i1 [[CMP4]], label [[ENTRY_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY_LR_PH:%.*]] 1571; CHECK: entry.for.end_crit_edge: 1572; CHECK-NEXT: br label [[FOR_END:%.*]] 1573; CHECK: for.body.lr.ph: 1574; CHECK-NEXT: [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2 1575; CHECK-NEXT: [[SUB:%.*]] = add i16 [[TMP0]], -6 1576; CHECK-NEXT: [[CONV2:%.*]] = sext i16 [[SUB]] to i32 1577; CHECK-NEXT: [[C_PROMOTED:%.*]] = load i32, i32* [[C:%.*]], align 4 1578; CHECK-NEXT: [[B_PROMOTED:%.*]] = load i32, i32* [[B:%.*]], align 4 1579; CHECK-NEXT: br label [[FOR_BODY:%.*]] 1580; CHECK: for.body: 1581; CHECK-NEXT: [[INC7:%.*]] = phi i32 [ [[D]], [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ] 1582; CHECK-NEXT: [[AND6:%.*]] = phi i32 [ [[B_PROMOTED]], [[FOR_BODY_LR_PH]] ], [ [[AND:%.*]], [[FOR_BODY]] ] 1583; CHECK-NEXT: [[CONV25:%.*]] = phi i32 [ [[C_PROMOTED]], [[FOR_BODY_LR_PH]] ], [ [[CONV2]], [[FOR_BODY]] ] 1584; CHECK-NEXT: [[AND]] = and i32 [[AND6]], [[CONV25]] 1585; CHECK-NEXT: [[INC]] = add nsw i32 [[INC7]], 1 1586; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], 21 1587; CHECK-NEXT: br i1 [[CMP]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY]] 1588; CHECK: for.cond.for.end_crit_edge: 1589; CHECK-NEXT: store i32 [[CONV2]], i32* [[C]], align 4 1590; CHECK-NEXT: store i32 [[AND]], i32* [[B]], align 4 1591; CHECK-NEXT: store i16 [[SUB]], i16* [[E:%.*]], align 2 1592; CHECK-NEXT: br label [[FOR_END]] 1593; CHECK: for.end: 1594; CHECK-NEXT: ret void 1595; 1596; UNROLL-LABEL: @PR26734( 1597; UNROLL-NEXT: entry: 1598; UNROLL-NEXT: [[CMP4:%.*]] = icmp eq i32 [[D:%.*]], 21 1599; UNROLL-NEXT: br i1 [[CMP4]], label [[ENTRY_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY_LR_PH:%.*]] 1600; UNROLL: entry.for.end_crit_edge: 1601; UNROLL-NEXT: br label [[FOR_END:%.*]] 1602; UNROLL: for.body.lr.ph: 1603; UNROLL-NEXT: [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2 1604; UNROLL-NEXT: [[SUB:%.*]] = add i16 [[TMP0]], -6 1605; UNROLL-NEXT: [[CONV2:%.*]] = sext i16 [[SUB]] to i32 1606; UNROLL-NEXT: [[C_PROMOTED:%.*]] = load i32, i32* [[C:%.*]], align 4 1607; UNROLL-NEXT: [[B_PROMOTED:%.*]] = load i32, i32* [[B:%.*]], align 4 1608; UNROLL-NEXT: br label [[FOR_BODY:%.*]] 1609; UNROLL: for.body: 1610; UNROLL-NEXT: [[INC7:%.*]] = phi i32 [ [[D]], [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ] 1611; UNROLL-NEXT: [[AND6:%.*]] = phi i32 [ [[B_PROMOTED]], [[FOR_BODY_LR_PH]] ], [ [[AND:%.*]], [[FOR_BODY]] ] 1612; UNROLL-NEXT: [[CONV25:%.*]] = phi i32 [ [[C_PROMOTED]], [[FOR_BODY_LR_PH]] ], [ [[CONV2]], [[FOR_BODY]] ] 1613; UNROLL-NEXT: [[AND]] = and i32 [[AND6]], [[CONV25]] 1614; UNROLL-NEXT: [[INC]] = add nsw i32 [[INC7]], 1 1615; UNROLL-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], 21 1616; UNROLL-NEXT: br i1 [[CMP]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY]] 1617; UNROLL: for.cond.for.end_crit_edge: 1618; UNROLL-NEXT: store i32 [[CONV2]], i32* [[C]], align 4 1619; UNROLL-NEXT: store i32 [[AND]], i32* [[B]], align 4 1620; UNROLL-NEXT: store i16 [[SUB]], i16* [[E:%.*]], align 2 1621; UNROLL-NEXT: br label [[FOR_END]] 1622; UNROLL: for.end: 1623; UNROLL-NEXT: ret void 1624; 1625; UNROLL-NO-IC-LABEL: @PR26734( 1626; UNROLL-NO-IC-NEXT: entry: 1627; UNROLL-NO-IC-NEXT: [[CMP4:%.*]] = icmp eq i32 [[D:%.*]], 21 1628; UNROLL-NO-IC-NEXT: br i1 [[CMP4]], label [[ENTRY_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY_LR_PH:%.*]] 1629; UNROLL-NO-IC: entry.for.end_crit_edge: 1630; UNROLL-NO-IC-NEXT: [[DOTPRE:%.*]] = load i32, i32* [[B:%.*]], align 4 1631; UNROLL-NO-IC-NEXT: br label [[FOR_END:%.*]] 1632; UNROLL-NO-IC: for.body.lr.ph: 1633; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2 1634; UNROLL-NO-IC-NEXT: [[SUB:%.*]] = add i16 [[TMP0]], -6 1635; UNROLL-NO-IC-NEXT: [[CONV2:%.*]] = sext i16 [[SUB]] to i32 1636; UNROLL-NO-IC-NEXT: [[C_PROMOTED:%.*]] = load i32, i32* [[C:%.*]], align 4 1637; UNROLL-NO-IC-NEXT: [[B_PROMOTED:%.*]] = load i32, i32* [[B]], align 4 1638; UNROLL-NO-IC-NEXT: br label [[FOR_BODY:%.*]] 1639; UNROLL-NO-IC: for.body: 1640; UNROLL-NO-IC-NEXT: [[INC7:%.*]] = phi i32 [ [[D]], [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ] 1641; UNROLL-NO-IC-NEXT: [[AND6:%.*]] = phi i32 [ [[B_PROMOTED]], [[FOR_BODY_LR_PH]] ], [ [[AND:%.*]], [[FOR_BODY]] ] 1642; UNROLL-NO-IC-NEXT: [[CONV25:%.*]] = phi i32 [ [[C_PROMOTED]], [[FOR_BODY_LR_PH]] ], [ [[CONV2]], [[FOR_BODY]] ] 1643; UNROLL-NO-IC-NEXT: [[AND]] = and i32 [[AND6]], [[CONV25]] 1644; UNROLL-NO-IC-NEXT: [[INC]] = add nsw i32 [[INC7]], 1 1645; UNROLL-NO-IC-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], 21 1646; UNROLL-NO-IC-NEXT: br i1 [[CMP]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY]] 1647; UNROLL-NO-IC: for.cond.for.end_crit_edge: 1648; UNROLL-NO-IC-NEXT: [[AND_LCSSA:%.*]] = phi i32 [ [[AND]], [[FOR_BODY]] ] 1649; UNROLL-NO-IC-NEXT: store i32 [[CONV2]], i32* [[C]], align 4 1650; UNROLL-NO-IC-NEXT: store i32 [[AND_LCSSA]], i32* [[B]], align 4 1651; UNROLL-NO-IC-NEXT: store i16 [[SUB]], i16* [[E:%.*]], align 2 1652; UNROLL-NO-IC-NEXT: br label [[FOR_END]] 1653; UNROLL-NO-IC: for.end: 1654; UNROLL-NO-IC-NEXT: ret void 1655; 1656; UNROLL-NO-VF-LABEL: @PR26734( 1657; UNROLL-NO-VF-NEXT: entry: 1658; UNROLL-NO-VF-NEXT: [[CMP4:%.*]] = icmp eq i32 [[D:%.*]], 21 1659; UNROLL-NO-VF-NEXT: br i1 [[CMP4]], label [[ENTRY_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY_LR_PH:%.*]] 1660; UNROLL-NO-VF: entry.for.end_crit_edge: 1661; UNROLL-NO-VF-NEXT: [[DOTPRE:%.*]] = load i32, i32* [[B:%.*]], align 4 1662; UNROLL-NO-VF-NEXT: br label [[FOR_END:%.*]] 1663; UNROLL-NO-VF: for.body.lr.ph: 1664; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2 1665; UNROLL-NO-VF-NEXT: [[SUB:%.*]] = add i16 [[TMP0]], -6 1666; UNROLL-NO-VF-NEXT: [[CONV2:%.*]] = sext i16 [[SUB]] to i32 1667; UNROLL-NO-VF-NEXT: [[C_PROMOTED:%.*]] = load i32, i32* [[C:%.*]], align 4 1668; UNROLL-NO-VF-NEXT: [[B_PROMOTED:%.*]] = load i32, i32* [[B]], align 4 1669; UNROLL-NO-VF-NEXT: br label [[FOR_BODY:%.*]] 1670; UNROLL-NO-VF: for.body: 1671; UNROLL-NO-VF-NEXT: [[INC7:%.*]] = phi i32 [ [[D]], [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ] 1672; UNROLL-NO-VF-NEXT: [[AND6:%.*]] = phi i32 [ [[B_PROMOTED]], [[FOR_BODY_LR_PH]] ], [ [[AND:%.*]], [[FOR_BODY]] ] 1673; UNROLL-NO-VF-NEXT: [[CONV25:%.*]] = phi i32 [ [[C_PROMOTED]], [[FOR_BODY_LR_PH]] ], [ [[CONV2]], [[FOR_BODY]] ] 1674; UNROLL-NO-VF-NEXT: [[AND]] = and i32 [[AND6]], [[CONV25]] 1675; UNROLL-NO-VF-NEXT: [[INC]] = add nsw i32 [[INC7]], 1 1676; UNROLL-NO-VF-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], 21 1677; UNROLL-NO-VF-NEXT: br i1 [[CMP]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY]] 1678; UNROLL-NO-VF: for.cond.for.end_crit_edge: 1679; UNROLL-NO-VF-NEXT: [[AND_LCSSA:%.*]] = phi i32 [ [[AND]], [[FOR_BODY]] ] 1680; UNROLL-NO-VF-NEXT: store i32 [[CONV2]], i32* [[C]], align 4 1681; UNROLL-NO-VF-NEXT: store i32 [[AND_LCSSA]], i32* [[B]], align 4 1682; UNROLL-NO-VF-NEXT: store i16 [[SUB]], i16* [[E:%.*]], align 2 1683; UNROLL-NO-VF-NEXT: br label [[FOR_END]] 1684; UNROLL-NO-VF: for.end: 1685; UNROLL-NO-VF-NEXT: ret void 1686; 1687; SINK-AFTER-LABEL: @PR26734( 1688; SINK-AFTER-NEXT: entry: 1689; SINK-AFTER-NEXT: [[CMP4:%.*]] = icmp eq i32 [[D:%.*]], 21 1690; SINK-AFTER-NEXT: br i1 [[CMP4]], label [[ENTRY_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY_LR_PH:%.*]] 1691; SINK-AFTER: entry.for.end_crit_edge: 1692; SINK-AFTER-NEXT: [[DOTPRE:%.*]] = load i32, i32* [[B:%.*]], align 4 1693; SINK-AFTER-NEXT: br label [[FOR_END:%.*]] 1694; SINK-AFTER: for.body.lr.ph: 1695; SINK-AFTER-NEXT: [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2 1696; SINK-AFTER-NEXT: [[SUB:%.*]] = add i16 [[TMP0]], -6 1697; SINK-AFTER-NEXT: [[CONV2:%.*]] = sext i16 [[SUB]] to i32 1698; SINK-AFTER-NEXT: [[C_PROMOTED:%.*]] = load i32, i32* [[C:%.*]], align 4 1699; SINK-AFTER-NEXT: [[B_PROMOTED:%.*]] = load i32, i32* [[B]], align 4 1700; SINK-AFTER-NEXT: br label [[FOR_BODY:%.*]] 1701; SINK-AFTER: for.body: 1702; SINK-AFTER-NEXT: [[INC7:%.*]] = phi i32 [ [[D]], [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ] 1703; SINK-AFTER-NEXT: [[AND6:%.*]] = phi i32 [ [[B_PROMOTED]], [[FOR_BODY_LR_PH]] ], [ [[AND:%.*]], [[FOR_BODY]] ] 1704; SINK-AFTER-NEXT: [[CONV25:%.*]] = phi i32 [ [[C_PROMOTED]], [[FOR_BODY_LR_PH]] ], [ [[CONV2]], [[FOR_BODY]] ] 1705; SINK-AFTER-NEXT: [[AND]] = and i32 [[AND6]], [[CONV25]] 1706; SINK-AFTER-NEXT: [[INC]] = add nsw i32 [[INC7]], 1 1707; SINK-AFTER-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], 21 1708; SINK-AFTER-NEXT: br i1 [[CMP]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY]] 1709; SINK-AFTER: for.cond.for.end_crit_edge: 1710; SINK-AFTER-NEXT: [[AND_LCSSA:%.*]] = phi i32 [ [[AND]], [[FOR_BODY]] ] 1711; SINK-AFTER-NEXT: store i32 [[CONV2]], i32* [[C]], align 4 1712; SINK-AFTER-NEXT: store i32 [[AND_LCSSA]], i32* [[B]], align 4 1713; SINK-AFTER-NEXT: store i16 [[SUB]], i16* [[E:%.*]], align 2 1714; SINK-AFTER-NEXT: br label [[FOR_END]] 1715; SINK-AFTER: for.end: 1716; SINK-AFTER-NEXT: ret void 1717; 1718; NO-SINK-AFTER-LABEL: @PR26734( 1719; NO-SINK-AFTER-NEXT: entry: 1720; NO-SINK-AFTER-NEXT: [[CMP4:%.*]] = icmp eq i32 [[D:%.*]], 21 1721; NO-SINK-AFTER-NEXT: br i1 [[CMP4]], label [[ENTRY_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY_LR_PH:%.*]] 1722; NO-SINK-AFTER: entry.for.end_crit_edge: 1723; NO-SINK-AFTER-NEXT: [[DOTPRE:%.*]] = load i32, i32* [[B:%.*]], align 4 1724; NO-SINK-AFTER-NEXT: br label [[FOR_END:%.*]] 1725; NO-SINK-AFTER: for.body.lr.ph: 1726; NO-SINK-AFTER-NEXT: [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2 1727; NO-SINK-AFTER-NEXT: [[SUB:%.*]] = add i16 [[TMP0]], -6 1728; NO-SINK-AFTER-NEXT: [[CONV2:%.*]] = sext i16 [[SUB]] to i32 1729; NO-SINK-AFTER-NEXT: [[C_PROMOTED:%.*]] = load i32, i32* [[C:%.*]], align 4 1730; NO-SINK-AFTER-NEXT: [[B_PROMOTED:%.*]] = load i32, i32* [[B]], align 4 1731; NO-SINK-AFTER-NEXT: br label [[FOR_BODY:%.*]] 1732; NO-SINK-AFTER: for.body: 1733; NO-SINK-AFTER-NEXT: [[INC7:%.*]] = phi i32 [ [[D]], [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ] 1734; NO-SINK-AFTER-NEXT: [[AND6:%.*]] = phi i32 [ [[B_PROMOTED]], [[FOR_BODY_LR_PH]] ], [ [[AND:%.*]], [[FOR_BODY]] ] 1735; NO-SINK-AFTER-NEXT: [[CONV25:%.*]] = phi i32 [ [[C_PROMOTED]], [[FOR_BODY_LR_PH]] ], [ [[CONV2]], [[FOR_BODY]] ] 1736; NO-SINK-AFTER-NEXT: [[AND]] = and i32 [[AND6]], [[CONV25]] 1737; NO-SINK-AFTER-NEXT: [[INC]] = add nsw i32 [[INC7]], 1 1738; NO-SINK-AFTER-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], 21 1739; NO-SINK-AFTER-NEXT: br i1 [[CMP]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY]] 1740; NO-SINK-AFTER: for.cond.for.end_crit_edge: 1741; NO-SINK-AFTER-NEXT: [[AND_LCSSA:%.*]] = phi i32 [ [[AND]], [[FOR_BODY]] ] 1742; NO-SINK-AFTER-NEXT: store i32 [[CONV2]], i32* [[C]], align 4 1743; NO-SINK-AFTER-NEXT: store i32 [[AND_LCSSA]], i32* [[B]], align 4 1744; NO-SINK-AFTER-NEXT: store i16 [[SUB]], i16* [[E:%.*]], align 2 1745; NO-SINK-AFTER-NEXT: br label [[FOR_END]] 1746; NO-SINK-AFTER: for.end: 1747; NO-SINK-AFTER-NEXT: ret void 1748; 1749entry: 1750 %cmp4 = icmp eq i32 %d, 21 1751 br i1 %cmp4, label %entry.for.end_crit_edge, label %for.body.lr.ph 1752 1753entry.for.end_crit_edge: 1754 %.pre = load i32, i32* %b, align 4 1755 br label %for.end 1756 1757for.body.lr.ph: 1758 %0 = load i16, i16* %a, align 2 1759 %sub = add i16 %0, -6 1760 %conv2 = sext i16 %sub to i32 1761 %c.promoted = load i32, i32* %c, align 4 1762 %b.promoted = load i32, i32* %b, align 4 1763 br label %for.body 1764 1765for.body: 1766 %inc7 = phi i32 [ %d, %for.body.lr.ph ], [ %inc, %for.body ] 1767 %and6 = phi i32 [ %b.promoted, %for.body.lr.ph ], [ %and, %for.body ] 1768 %conv25 = phi i32 [ %c.promoted, %for.body.lr.ph ], [ %conv2, %for.body ] 1769 %and = and i32 %and6, %conv25 1770 %inc = add nsw i32 %inc7, 1 1771 %cmp = icmp eq i32 %inc, 21 1772 br i1 %cmp, label %for.cond.for.end_crit_edge, label %for.body 1773 1774for.cond.for.end_crit_edge: 1775 %and.lcssa = phi i32 [ %and, %for.body ] 1776 store i32 %conv2, i32* %c, align 4 1777 store i32 %and.lcssa, i32* %b, align 4 1778 store i16 %sub, i16* %e, align 2 1779 br label %for.end 1780 1781for.end: 1782 ret void 1783} 1784 1785; int PR27246() { 1786; unsigned int e, n; 1787; for (int i = 1; i < 49; ++i) { 1788; for (int k = i; k > 1; --k) 1789; e = k; 1790; n = e; 1791; } 1792; return n; 1793; } 1794; 1795; 1796define i32 @PR27246() { 1797; CHECK-LABEL: @PR27246( 1798; CHECK-NEXT: entry: 1799; CHECK-NEXT: br label [[FOR_COND1_PREHEADER:%.*]] 1800; CHECK: for.cond1.preheader: 1801; CHECK-NEXT: [[I_016:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_COND_CLEANUP3:%.*]] ] 1802; CHECK-NEXT: [[E_015:%.*]] = phi i32 [ poison, [[ENTRY]] ], [ [[E_1:%.*]], [[FOR_COND_CLEANUP3]] ] 1803; CHECK-NEXT: br label [[FOR_COND1:%.*]] 1804; CHECK: for.cond.cleanup: 1805; CHECK-NEXT: ret i32 [[E_1]] 1806; CHECK: for.cond1: 1807; CHECK-NEXT: [[E_1]] = phi i32 [ [[K_0:%.*]], [[FOR_COND1]] ], [ [[E_015]], [[FOR_COND1_PREHEADER]] ] 1808; CHECK-NEXT: [[K_0]] = phi i32 [ [[DEC:%.*]], [[FOR_COND1]] ], [ [[I_016]], [[FOR_COND1_PREHEADER]] ] 1809; CHECK-NEXT: [[CMP2:%.*]] = icmp sgt i32 [[K_0]], 1 1810; CHECK-NEXT: [[DEC]] = add nsw i32 [[K_0]], -1 1811; CHECK-NEXT: br i1 [[CMP2]], label [[FOR_COND1]], label [[FOR_COND_CLEANUP3]] 1812; CHECK: for.cond.cleanup3: 1813; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_016]], 1 1814; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 49 1815; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER]] 1816; 1817; UNROLL-LABEL: @PR27246( 1818; UNROLL-NEXT: entry: 1819; UNROLL-NEXT: br label [[FOR_COND1_PREHEADER:%.*]] 1820; UNROLL: for.cond1.preheader: 1821; UNROLL-NEXT: [[I_016:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_COND_CLEANUP3:%.*]] ] 1822; UNROLL-NEXT: [[E_015:%.*]] = phi i32 [ poison, [[ENTRY]] ], [ [[E_1:%.*]], [[FOR_COND_CLEANUP3]] ] 1823; UNROLL-NEXT: br label [[FOR_COND1:%.*]] 1824; UNROLL: for.cond.cleanup: 1825; UNROLL-NEXT: ret i32 [[E_1]] 1826; UNROLL: for.cond1: 1827; UNROLL-NEXT: [[E_1]] = phi i32 [ [[K_0:%.*]], [[FOR_COND1]] ], [ [[E_015]], [[FOR_COND1_PREHEADER]] ] 1828; UNROLL-NEXT: [[K_0]] = phi i32 [ [[DEC:%.*]], [[FOR_COND1]] ], [ [[I_016]], [[FOR_COND1_PREHEADER]] ] 1829; UNROLL-NEXT: [[CMP2:%.*]] = icmp sgt i32 [[K_0]], 1 1830; UNROLL-NEXT: [[DEC]] = add nsw i32 [[K_0]], -1 1831; UNROLL-NEXT: br i1 [[CMP2]], label [[FOR_COND1]], label [[FOR_COND_CLEANUP3]] 1832; UNROLL: for.cond.cleanup3: 1833; UNROLL-NEXT: [[INC]] = add nuw nsw i32 [[I_016]], 1 1834; UNROLL-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 49 1835; UNROLL-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER]] 1836; 1837; UNROLL-NO-IC-LABEL: @PR27246( 1838; UNROLL-NO-IC-NEXT: entry: 1839; UNROLL-NO-IC-NEXT: br label [[FOR_COND1_PREHEADER:%.*]] 1840; UNROLL-NO-IC: for.cond1.preheader: 1841; UNROLL-NO-IC-NEXT: [[I_016:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_COND_CLEANUP3:%.*]] ] 1842; UNROLL-NO-IC-NEXT: [[E_015:%.*]] = phi i32 [ poison, [[ENTRY]] ], [ [[E_1_LCSSA:%.*]], [[FOR_COND_CLEANUP3]] ] 1843; UNROLL-NO-IC-NEXT: br label [[FOR_COND1:%.*]] 1844; UNROLL-NO-IC: for.cond.cleanup: 1845; UNROLL-NO-IC-NEXT: [[E_1_LCSSA_LCSSA:%.*]] = phi i32 [ [[E_1_LCSSA]], [[FOR_COND_CLEANUP3]] ] 1846; UNROLL-NO-IC-NEXT: ret i32 [[E_1_LCSSA_LCSSA]] 1847; UNROLL-NO-IC: for.cond1: 1848; UNROLL-NO-IC-NEXT: [[E_1:%.*]] = phi i32 [ [[K_0:%.*]], [[FOR_COND1]] ], [ [[E_015]], [[FOR_COND1_PREHEADER]] ] 1849; UNROLL-NO-IC-NEXT: [[K_0]] = phi i32 [ [[DEC:%.*]], [[FOR_COND1]] ], [ [[I_016]], [[FOR_COND1_PREHEADER]] ] 1850; UNROLL-NO-IC-NEXT: [[CMP2:%.*]] = icmp sgt i32 [[K_0]], 1 1851; UNROLL-NO-IC-NEXT: [[DEC]] = add nsw i32 [[K_0]], -1 1852; UNROLL-NO-IC-NEXT: br i1 [[CMP2]], label [[FOR_COND1]], label [[FOR_COND_CLEANUP3]] 1853; UNROLL-NO-IC: for.cond.cleanup3: 1854; UNROLL-NO-IC-NEXT: [[E_1_LCSSA]] = phi i32 [ [[E_1]], [[FOR_COND1]] ] 1855; UNROLL-NO-IC-NEXT: [[INC]] = add nuw nsw i32 [[I_016]], 1 1856; UNROLL-NO-IC-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 49 1857; UNROLL-NO-IC-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER]] 1858; 1859; UNROLL-NO-VF-LABEL: @PR27246( 1860; UNROLL-NO-VF-NEXT: entry: 1861; UNROLL-NO-VF-NEXT: br label [[FOR_COND1_PREHEADER:%.*]] 1862; UNROLL-NO-VF: for.cond1.preheader: 1863; UNROLL-NO-VF-NEXT: [[I_016:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_COND_CLEANUP3:%.*]] ] 1864; UNROLL-NO-VF-NEXT: [[E_015:%.*]] = phi i32 [ poison, [[ENTRY]] ], [ [[E_1_LCSSA:%.*]], [[FOR_COND_CLEANUP3]] ] 1865; UNROLL-NO-VF-NEXT: br label [[FOR_COND1:%.*]] 1866; UNROLL-NO-VF: for.cond.cleanup: 1867; UNROLL-NO-VF-NEXT: [[E_1_LCSSA_LCSSA:%.*]] = phi i32 [ [[E_1_LCSSA]], [[FOR_COND_CLEANUP3]] ] 1868; UNROLL-NO-VF-NEXT: ret i32 [[E_1_LCSSA_LCSSA]] 1869; UNROLL-NO-VF: for.cond1: 1870; UNROLL-NO-VF-NEXT: [[E_1:%.*]] = phi i32 [ [[K_0:%.*]], [[FOR_COND1]] ], [ [[E_015]], [[FOR_COND1_PREHEADER]] ] 1871; UNROLL-NO-VF-NEXT: [[K_0]] = phi i32 [ [[DEC:%.*]], [[FOR_COND1]] ], [ [[I_016]], [[FOR_COND1_PREHEADER]] ] 1872; UNROLL-NO-VF-NEXT: [[CMP2:%.*]] = icmp sgt i32 [[K_0]], 1 1873; UNROLL-NO-VF-NEXT: [[DEC]] = add nsw i32 [[K_0]], -1 1874; UNROLL-NO-VF-NEXT: br i1 [[CMP2]], label [[FOR_COND1]], label [[FOR_COND_CLEANUP3]] 1875; UNROLL-NO-VF: for.cond.cleanup3: 1876; UNROLL-NO-VF-NEXT: [[E_1_LCSSA]] = phi i32 [ [[E_1]], [[FOR_COND1]] ] 1877; UNROLL-NO-VF-NEXT: [[INC]] = add nuw nsw i32 [[I_016]], 1 1878; UNROLL-NO-VF-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 49 1879; UNROLL-NO-VF-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER]] 1880; 1881; SINK-AFTER-LABEL: @PR27246( 1882; SINK-AFTER-NEXT: entry: 1883; SINK-AFTER-NEXT: br label [[FOR_COND1_PREHEADER:%.*]] 1884; SINK-AFTER: for.cond1.preheader: 1885; SINK-AFTER-NEXT: [[I_016:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_COND_CLEANUP3:%.*]] ] 1886; SINK-AFTER-NEXT: [[E_015:%.*]] = phi i32 [ poison, [[ENTRY]] ], [ [[E_1_LCSSA:%.*]], [[FOR_COND_CLEANUP3]] ] 1887; SINK-AFTER-NEXT: br label [[FOR_COND1:%.*]] 1888; SINK-AFTER: for.cond.cleanup: 1889; SINK-AFTER-NEXT: [[E_1_LCSSA_LCSSA:%.*]] = phi i32 [ [[E_1_LCSSA]], [[FOR_COND_CLEANUP3]] ] 1890; SINK-AFTER-NEXT: ret i32 [[E_1_LCSSA_LCSSA]] 1891; SINK-AFTER: for.cond1: 1892; SINK-AFTER-NEXT: [[E_1:%.*]] = phi i32 [ [[K_0:%.*]], [[FOR_COND1]] ], [ [[E_015]], [[FOR_COND1_PREHEADER]] ] 1893; SINK-AFTER-NEXT: [[K_0]] = phi i32 [ [[DEC:%.*]], [[FOR_COND1]] ], [ [[I_016]], [[FOR_COND1_PREHEADER]] ] 1894; SINK-AFTER-NEXT: [[CMP2:%.*]] = icmp sgt i32 [[K_0]], 1 1895; SINK-AFTER-NEXT: [[DEC]] = add nsw i32 [[K_0]], -1 1896; SINK-AFTER-NEXT: br i1 [[CMP2]], label [[FOR_COND1]], label [[FOR_COND_CLEANUP3]] 1897; SINK-AFTER: for.cond.cleanup3: 1898; SINK-AFTER-NEXT: [[E_1_LCSSA]] = phi i32 [ [[E_1]], [[FOR_COND1]] ] 1899; SINK-AFTER-NEXT: [[INC]] = add nuw nsw i32 [[I_016]], 1 1900; SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 49 1901; SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER]] 1902; 1903; NO-SINK-AFTER-LABEL: @PR27246( 1904; NO-SINK-AFTER-NEXT: entry: 1905; NO-SINK-AFTER-NEXT: br label [[FOR_COND1_PREHEADER:%.*]] 1906; NO-SINK-AFTER: for.cond1.preheader: 1907; NO-SINK-AFTER-NEXT: [[I_016:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_COND_CLEANUP3:%.*]] ] 1908; NO-SINK-AFTER-NEXT: [[E_015:%.*]] = phi i32 [ poison, [[ENTRY]] ], [ [[E_1_LCSSA:%.*]], [[FOR_COND_CLEANUP3]] ] 1909; NO-SINK-AFTER-NEXT: br label [[FOR_COND1:%.*]] 1910; NO-SINK-AFTER: for.cond.cleanup: 1911; NO-SINK-AFTER-NEXT: [[E_1_LCSSA_LCSSA:%.*]] = phi i32 [ [[E_1_LCSSA]], [[FOR_COND_CLEANUP3]] ] 1912; NO-SINK-AFTER-NEXT: ret i32 [[E_1_LCSSA_LCSSA]] 1913; NO-SINK-AFTER: for.cond1: 1914; NO-SINK-AFTER-NEXT: [[E_1:%.*]] = phi i32 [ [[K_0:%.*]], [[FOR_COND1]] ], [ [[E_015]], [[FOR_COND1_PREHEADER]] ] 1915; NO-SINK-AFTER-NEXT: [[K_0]] = phi i32 [ [[DEC:%.*]], [[FOR_COND1]] ], [ [[I_016]], [[FOR_COND1_PREHEADER]] ] 1916; NO-SINK-AFTER-NEXT: [[CMP2:%.*]] = icmp sgt i32 [[K_0]], 1 1917; NO-SINK-AFTER-NEXT: [[DEC]] = add nsw i32 [[K_0]], -1 1918; NO-SINK-AFTER-NEXT: br i1 [[CMP2]], label [[FOR_COND1]], label [[FOR_COND_CLEANUP3]] 1919; NO-SINK-AFTER: for.cond.cleanup3: 1920; NO-SINK-AFTER-NEXT: [[E_1_LCSSA]] = phi i32 [ [[E_1]], [[FOR_COND1]] ] 1921; NO-SINK-AFTER-NEXT: [[INC]] = add nuw nsw i32 [[I_016]], 1 1922; NO-SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 49 1923; NO-SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER]] 1924; 1925entry: 1926 br label %for.cond1.preheader 1927 1928for.cond1.preheader: 1929 %i.016 = phi i32 [ 1, %entry ], [ %inc, %for.cond.cleanup3 ] 1930 %e.015 = phi i32 [ poison, %entry ], [ %e.1.lcssa, %for.cond.cleanup3 ] 1931 br label %for.cond1 1932 1933for.cond.cleanup: 1934 %e.1.lcssa.lcssa = phi i32 [ %e.1.lcssa, %for.cond.cleanup3 ] 1935 ret i32 %e.1.lcssa.lcssa 1936 1937for.cond1: 1938 %e.1 = phi i32 [ %k.0, %for.cond1 ], [ %e.015, %for.cond1.preheader ] 1939 %k.0 = phi i32 [ %dec, %for.cond1 ], [ %i.016, %for.cond1.preheader ] 1940 %cmp2 = icmp sgt i32 %k.0, 1 1941 %dec = add nsw i32 %k.0, -1 1942 br i1 %cmp2, label %for.cond1, label %for.cond.cleanup3 1943 1944for.cond.cleanup3: 1945 %e.1.lcssa = phi i32 [ %e.1, %for.cond1 ] 1946 %inc = add nuw nsw i32 %i.016, 1 1947 %exitcond = icmp eq i32 %inc, 49 1948 br i1 %exitcond, label %for.cond.cleanup, label %for.cond1.preheader 1949} 1950 1951; 1952define void @PR30183(i32 %pre_load, i32* %a, i32* %b, i64 %n) { 1953; CHECK-LABEL: @PR30183( 1954; CHECK-NEXT: entry: 1955; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], -2 1956; CHECK-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 1 1957; CHECK-NEXT: [[TMP2:%.*]] = add nuw i64 [[TMP1]], 1 1958; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 6 1959; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 1960; CHECK: vector.ph: 1961; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], -4 1962; CHECK-NEXT: [[IND_END:%.*]] = shl i64 [[N_VEC]], 1 1963; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 1964; CHECK: vector.body: 1965; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1966; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 1967; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1968; CHECK-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] 1969; CHECK: middle.block: 1970; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 1971; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 1972; CHECK: scalar.ph: 1973; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 1974; CHECK-NEXT: br label [[SCALAR_BODY:%.*]] 1975; CHECK: scalar.body: 1976; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ] 1977; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 2 1978; CHECK-NEXT: [[COND:%.*]] = icmp eq i64 [[I_NEXT]], [[N]] 1979; CHECK-NEXT: br i1 [[COND]], label [[FOR_END]], label [[SCALAR_BODY]], !llvm.loop [[LOOP19:![0-9]+]] 1980; CHECK: for.end: 1981; CHECK-NEXT: ret void 1982; 1983; UNROLL-LABEL: @PR30183( 1984; UNROLL-NEXT: entry: 1985; UNROLL-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], -2 1986; UNROLL-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 1 1987; UNROLL-NEXT: [[TMP2:%.*]] = add nuw i64 [[TMP1]], 1 1988; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 14 1989; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 1990; UNROLL: vector.ph: 1991; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], -8 1992; UNROLL-NEXT: [[IND_END:%.*]] = shl i64 [[N_VEC]], 1 1993; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 1994; UNROLL: vector.body: 1995; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1996; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 1997; UNROLL-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1998; UNROLL-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] 1999; UNROLL: middle.block: 2000; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 2001; UNROLL-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2002; UNROLL: scalar.ph: 2003; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 2004; UNROLL-NEXT: br label [[SCALAR_BODY:%.*]] 2005; UNROLL: scalar.body: 2006; UNROLL-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ] 2007; UNROLL-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 2 2008; UNROLL-NEXT: [[COND:%.*]] = icmp eq i64 [[I_NEXT]], [[N]] 2009; UNROLL-NEXT: br i1 [[COND]], label [[FOR_END]], label [[SCALAR_BODY]], !llvm.loop [[LOOP19:![0-9]+]] 2010; UNROLL: for.end: 2011; UNROLL-NEXT: ret void 2012; 2013; UNROLL-NO-IC-LABEL: @PR30183( 2014; UNROLL-NO-IC-NEXT: entry: 2015; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], -2 2016; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 1 2017; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add nuw i64 [[TMP1]], 1 2018; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 8 2019; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2020; UNROLL-NO-IC: vector.ph: 2021; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 8 2022; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 2023; UNROLL-NO-IC-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 2 2024; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[PRE_LOAD:%.*]], i32 3 2025; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 2026; UNROLL-NO-IC: vector.body: 2027; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2028; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP42:%.*]], [[VECTOR_BODY]] ] 2029; UNROLL-NO-IC-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2 2030; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 0 2031; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 2 2032; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 4 2033; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 6 2034; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 8 2035; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = add i64 [[OFFSET_IDX]], 10 2036; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = add i64 [[OFFSET_IDX]], 12 2037; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = add i64 [[OFFSET_IDX]], 14 2038; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = add nuw nsw i64 [[TMP3]], 2 2039; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = add nuw nsw i64 [[TMP4]], 2 2040; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = add nuw nsw i64 [[TMP5]], 2 2041; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = add nuw nsw i64 [[TMP6]], 2 2042; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = add nuw nsw i64 [[TMP7]], 2 2043; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = add nuw nsw i64 [[TMP8]], 2 2044; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = add nuw nsw i64 [[TMP9]], 2 2045; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = add nuw nsw i64 [[TMP10]], 2 2046; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP11]] 2047; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP12]] 2048; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP13]] 2049; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP14]] 2050; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP15]] 2051; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP16]] 2052; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP17]] 2053; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP18]] 2054; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = load i32, i32* [[TMP19]], align 4 2055; UNROLL-NO-IC-NEXT: [[TMP28:%.*]] = load i32, i32* [[TMP20]], align 4 2056; UNROLL-NO-IC-NEXT: [[TMP29:%.*]] = load i32, i32* [[TMP21]], align 4 2057; UNROLL-NO-IC-NEXT: [[TMP30:%.*]] = load i32, i32* [[TMP22]], align 4 2058; UNROLL-NO-IC-NEXT: [[TMP31:%.*]] = insertelement <4 x i32> poison, i32 [[TMP27]], i32 0 2059; UNROLL-NO-IC-NEXT: [[TMP32:%.*]] = insertelement <4 x i32> [[TMP31]], i32 [[TMP28]], i32 1 2060; UNROLL-NO-IC-NEXT: [[TMP33:%.*]] = insertelement <4 x i32> [[TMP32]], i32 [[TMP29]], i32 2 2061; UNROLL-NO-IC-NEXT: [[TMP34:%.*]] = insertelement <4 x i32> [[TMP33]], i32 [[TMP30]], i32 3 2062; UNROLL-NO-IC-NEXT: [[TMP35:%.*]] = load i32, i32* [[TMP23]], align 4 2063; UNROLL-NO-IC-NEXT: [[TMP36:%.*]] = load i32, i32* [[TMP24]], align 4 2064; UNROLL-NO-IC-NEXT: [[TMP37:%.*]] = load i32, i32* [[TMP25]], align 4 2065; UNROLL-NO-IC-NEXT: [[TMP38:%.*]] = load i32, i32* [[TMP26]], align 4 2066; UNROLL-NO-IC-NEXT: [[TMP39:%.*]] = insertelement <4 x i32> poison, i32 [[TMP35]], i32 0 2067; UNROLL-NO-IC-NEXT: [[TMP40:%.*]] = insertelement <4 x i32> [[TMP39]], i32 [[TMP36]], i32 1 2068; UNROLL-NO-IC-NEXT: [[TMP41:%.*]] = insertelement <4 x i32> [[TMP40]], i32 [[TMP37]], i32 2 2069; UNROLL-NO-IC-NEXT: [[TMP42]] = insertelement <4 x i32> [[TMP41]], i32 [[TMP38]], i32 3 2070; UNROLL-NO-IC-NEXT: [[TMP43:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP34]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2071; UNROLL-NO-IC-NEXT: [[TMP44:%.*]] = shufflevector <4 x i32> [[TMP34]], <4 x i32> [[TMP42]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2072; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 2073; UNROLL-NO-IC-NEXT: [[TMP45:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 2074; UNROLL-NO-IC-NEXT: br i1 [[TMP45]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] 2075; UNROLL-NO-IC: middle.block: 2076; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 2077; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP42]], i32 3 2078; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP42]], i32 2 2079; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2080; UNROLL-NO-IC: scalar.ph: 2081; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[PRE_LOAD]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 2082; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2083; UNROLL-NO-IC-NEXT: br label [[SCALAR_BODY:%.*]] 2084; UNROLL-NO-IC: scalar.body: 2085; UNROLL-NO-IC-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ] 2086; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[VAR2:%.*]], [[SCALAR_BODY]] ] 2087; UNROLL-NO-IC-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 2 2088; UNROLL-NO-IC-NEXT: [[VAR1:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[I_NEXT]] 2089; UNROLL-NO-IC-NEXT: [[VAR2]] = load i32, i32* [[VAR1]], align 4 2090; UNROLL-NO-IC-NEXT: [[COND:%.*]] = icmp eq i64 [[I_NEXT]], [[N]] 2091; UNROLL-NO-IC-NEXT: br i1 [[COND]], label [[FOR_END]], label [[SCALAR_BODY]], !llvm.loop [[LOOP19:![0-9]+]] 2092; UNROLL-NO-IC: for.end: 2093; UNROLL-NO-IC-NEXT: ret void 2094; 2095; UNROLL-NO-VF-LABEL: @PR30183( 2096; UNROLL-NO-VF-NEXT: entry: 2097; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], -2 2098; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 1 2099; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = add nuw i64 [[TMP1]], 1 2100; UNROLL-NO-VF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 2 2101; UNROLL-NO-VF-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2102; UNROLL-NO-VF: vector.ph: 2103; UNROLL-NO-VF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 2 2104; UNROLL-NO-VF-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 2105; UNROLL-NO-VF-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 2 2106; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 2107; UNROLL-NO-VF: vector.body: 2108; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2109; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i32 [ [[PRE_LOAD:%.*]], [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[VECTOR_BODY]] ] 2110; UNROLL-NO-VF-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2 2111; UNROLL-NO-VF-NEXT: [[INDUCTION:%.*]] = add i64 [[OFFSET_IDX]], 0 2112; UNROLL-NO-VF-NEXT: [[INDUCTION1:%.*]] = add i64 [[OFFSET_IDX]], 2 2113; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[INDUCTION]], 2 2114; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = add nuw nsw i64 [[INDUCTION1]], 2 2115; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP3]] 2116; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP4]] 2117; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = load i32, i32* [[TMP5]], align 4 2118; UNROLL-NO-VF-NEXT: [[TMP8]] = load i32, i32* [[TMP6]], align 4 2119; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 2120; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 2121; UNROLL-NO-VF-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]] 2122; UNROLL-NO-VF: middle.block: 2123; UNROLL-NO-VF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 2124; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2125; UNROLL-NO-VF: scalar.ph: 2126; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[PRE_LOAD]], [[ENTRY:%.*]] ], [ [[TMP8]], [[MIDDLE_BLOCK]] ] 2127; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2128; UNROLL-NO-VF-NEXT: br label [[SCALAR_BODY:%.*]] 2129; UNROLL-NO-VF: scalar.body: 2130; UNROLL-NO-VF-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ] 2131; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[VAR2:%.*]], [[SCALAR_BODY]] ] 2132; UNROLL-NO-VF-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 2 2133; UNROLL-NO-VF-NEXT: [[VAR1:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[I_NEXT]] 2134; UNROLL-NO-VF-NEXT: [[VAR2]] = load i32, i32* [[VAR1]], align 4 2135; UNROLL-NO-VF-NEXT: [[COND:%.*]] = icmp eq i64 [[I_NEXT]], [[N]] 2136; UNROLL-NO-VF-NEXT: br i1 [[COND]], label [[FOR_END]], label [[SCALAR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] 2137; UNROLL-NO-VF: for.end: 2138; UNROLL-NO-VF-NEXT: ret void 2139; 2140; SINK-AFTER-LABEL: @PR30183( 2141; SINK-AFTER-NEXT: entry: 2142; SINK-AFTER-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], -2 2143; SINK-AFTER-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 1 2144; SINK-AFTER-NEXT: [[TMP2:%.*]] = add nuw i64 [[TMP1]], 1 2145; SINK-AFTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4 2146; SINK-AFTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2147; SINK-AFTER: vector.ph: 2148; SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4 2149; SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 2150; SINK-AFTER-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 2 2151; SINK-AFTER-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[PRE_LOAD:%.*]], i32 3 2152; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 2153; SINK-AFTER: vector.body: 2154; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2155; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ] 2156; SINK-AFTER-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2 2157; SINK-AFTER-NEXT: [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 0 2158; SINK-AFTER-NEXT: [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 2 2159; SINK-AFTER-NEXT: [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 4 2160; SINK-AFTER-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 6 2161; SINK-AFTER-NEXT: [[TMP7:%.*]] = add nuw nsw i64 [[TMP3]], 2 2162; SINK-AFTER-NEXT: [[TMP8:%.*]] = add nuw nsw i64 [[TMP4]], 2 2163; SINK-AFTER-NEXT: [[TMP9:%.*]] = add nuw nsw i64 [[TMP5]], 2 2164; SINK-AFTER-NEXT: [[TMP10:%.*]] = add nuw nsw i64 [[TMP6]], 2 2165; SINK-AFTER-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP7]] 2166; SINK-AFTER-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP8]] 2167; SINK-AFTER-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP9]] 2168; SINK-AFTER-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP10]] 2169; SINK-AFTER-NEXT: [[TMP15:%.*]] = load i32, i32* [[TMP11]], align 4 2170; SINK-AFTER-NEXT: [[TMP16:%.*]] = load i32, i32* [[TMP12]], align 4 2171; SINK-AFTER-NEXT: [[TMP17:%.*]] = load i32, i32* [[TMP13]], align 4 2172; SINK-AFTER-NEXT: [[TMP18:%.*]] = load i32, i32* [[TMP14]], align 4 2173; SINK-AFTER-NEXT: [[TMP19:%.*]] = insertelement <4 x i32> poison, i32 [[TMP15]], i32 0 2174; SINK-AFTER-NEXT: [[TMP20:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP16]], i32 1 2175; SINK-AFTER-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP17]], i32 2 2176; SINK-AFTER-NEXT: [[TMP22]] = insertelement <4 x i32> [[TMP21]], i32 [[TMP18]], i32 3 2177; SINK-AFTER-NEXT: [[TMP23:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP22]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2178; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 2179; SINK-AFTER-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 2180; SINK-AFTER-NEXT: br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] 2181; SINK-AFTER: middle.block: 2182; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 2183; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP22]], i32 3 2184; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP22]], i32 2 2185; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2186; SINK-AFTER: scalar.ph: 2187; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[PRE_LOAD]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 2188; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2189; SINK-AFTER-NEXT: br label [[SCALAR_BODY:%.*]] 2190; SINK-AFTER: scalar.body: 2191; SINK-AFTER-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ] 2192; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[VAR2:%.*]], [[SCALAR_BODY]] ] 2193; SINK-AFTER-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 2 2194; SINK-AFTER-NEXT: [[VAR1:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[I_NEXT]] 2195; SINK-AFTER-NEXT: [[VAR2]] = load i32, i32* [[VAR1]], align 4 2196; SINK-AFTER-NEXT: [[COND:%.*]] = icmp eq i64 [[I_NEXT]], [[N]] 2197; SINK-AFTER-NEXT: br i1 [[COND]], label [[FOR_END]], label [[SCALAR_BODY]], !llvm.loop [[LOOP19:![0-9]+]] 2198; SINK-AFTER: for.end: 2199; SINK-AFTER-NEXT: ret void 2200; 2201; NO-SINK-AFTER-LABEL: @PR30183( 2202; NO-SINK-AFTER-NEXT: entry: 2203; NO-SINK-AFTER-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], -2 2204; NO-SINK-AFTER-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 1 2205; NO-SINK-AFTER-NEXT: [[TMP2:%.*]] = add nuw i64 [[TMP1]], 1 2206; NO-SINK-AFTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4 2207; NO-SINK-AFTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2208; NO-SINK-AFTER: vector.ph: 2209; NO-SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4 2210; NO-SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 2211; NO-SINK-AFTER-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 2 2212; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[PRE_LOAD:%.*]], i32 3 2213; NO-SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 2214; NO-SINK-AFTER: vector.body: 2215; NO-SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2216; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ] 2217; NO-SINK-AFTER-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2 2218; NO-SINK-AFTER-NEXT: [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 0 2219; NO-SINK-AFTER-NEXT: [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 2 2220; NO-SINK-AFTER-NEXT: [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 4 2221; NO-SINK-AFTER-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 6 2222; NO-SINK-AFTER-NEXT: [[TMP7:%.*]] = add nuw nsw i64 [[TMP3]], 2 2223; NO-SINK-AFTER-NEXT: [[TMP8:%.*]] = add nuw nsw i64 [[TMP4]], 2 2224; NO-SINK-AFTER-NEXT: [[TMP9:%.*]] = add nuw nsw i64 [[TMP5]], 2 2225; NO-SINK-AFTER-NEXT: [[TMP10:%.*]] = add nuw nsw i64 [[TMP6]], 2 2226; NO-SINK-AFTER-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP7]] 2227; NO-SINK-AFTER-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP8]] 2228; NO-SINK-AFTER-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP9]] 2229; NO-SINK-AFTER-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP10]] 2230; NO-SINK-AFTER-NEXT: [[TMP15:%.*]] = load i32, i32* [[TMP11]], align 4 2231; NO-SINK-AFTER-NEXT: [[TMP16:%.*]] = load i32, i32* [[TMP12]], align 4 2232; NO-SINK-AFTER-NEXT: [[TMP17:%.*]] = load i32, i32* [[TMP13]], align 4 2233; NO-SINK-AFTER-NEXT: [[TMP18:%.*]] = load i32, i32* [[TMP14]], align 4 2234; NO-SINK-AFTER-NEXT: [[TMP19:%.*]] = insertelement <4 x i32> poison, i32 [[TMP15]], i32 0 2235; NO-SINK-AFTER-NEXT: [[TMP20:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP16]], i32 1 2236; NO-SINK-AFTER-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP17]], i32 2 2237; NO-SINK-AFTER-NEXT: [[TMP22]] = insertelement <4 x i32> [[TMP21]], i32 [[TMP18]], i32 3 2238; NO-SINK-AFTER-NEXT: [[TMP23:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP22]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2239; NO-SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 2240; NO-SINK-AFTER-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 2241; NO-SINK-AFTER-NEXT: br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] 2242; NO-SINK-AFTER: middle.block: 2243; NO-SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 2244; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP22]], i32 3 2245; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP22]], i32 2 2246; NO-SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2247; NO-SINK-AFTER: scalar.ph: 2248; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[PRE_LOAD]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 2249; NO-SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2250; NO-SINK-AFTER-NEXT: br label [[SCALAR_BODY:%.*]] 2251; NO-SINK-AFTER: scalar.body: 2252; NO-SINK-AFTER-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ] 2253; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[VAR2:%.*]], [[SCALAR_BODY]] ] 2254; NO-SINK-AFTER-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 2 2255; NO-SINK-AFTER-NEXT: [[VAR1:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[I_NEXT]] 2256; NO-SINK-AFTER-NEXT: [[VAR2]] = load i32, i32* [[VAR1]], align 4 2257; NO-SINK-AFTER-NEXT: [[COND:%.*]] = icmp eq i64 [[I_NEXT]], [[N]] 2258; NO-SINK-AFTER-NEXT: br i1 [[COND]], label [[FOR_END]], label [[SCALAR_BODY]], !llvm.loop [[LOOP19:![0-9]+]] 2259; NO-SINK-AFTER: for.end: 2260; NO-SINK-AFTER-NEXT: ret void 2261; 2262entry: 2263 br label %scalar.body 2264 2265scalar.body: 2266 %i = phi i64 [ 0, %entry ], [ %i.next, %scalar.body ] 2267 %var0 = phi i32 [ %pre_load, %entry ], [ %var2, %scalar.body ] 2268 %i.next = add nuw nsw i64 %i, 2 2269 %var1 = getelementptr inbounds i32, i32* %a, i64 %i.next 2270 %var2 = load i32, i32* %var1 2271 %cond = icmp eq i64 %i.next,%n 2272 br i1 %cond, label %for.end, label %scalar.body 2273 2274for.end: 2275 ret void 2276} 2277 2278; 2279define void @constant_folded_previous_value() { 2280; CHECK-LABEL: @constant_folded_previous_value( 2281; CHECK-NEXT: entry: 2282; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2283; CHECK: vector.ph: 2284; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 2285; CHECK: vector.body: 2286; CHECK-NEXT: br i1 undef, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] 2287; CHECK: middle.block: 2288; CHECK-NEXT: br i1 undef, label [[FOR_END:%.*]], label [[SCALAR_PH]] 2289; CHECK: scalar.ph: 2290; CHECK-NEXT: br label [[SCALAR_BODY:%.*]] 2291; CHECK: scalar.body: 2292; CHECK-NEXT: br i1 undef, label [[FOR_END]], label [[SCALAR_BODY]], !llvm.loop [[LOOP21:![0-9]+]] 2293; CHECK: for.end: 2294; CHECK-NEXT: ret void 2295; 2296; UNROLL-LABEL: @constant_folded_previous_value( 2297; UNROLL-NEXT: entry: 2298; UNROLL-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2299; UNROLL: vector.ph: 2300; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 2301; UNROLL: vector.body: 2302; UNROLL-NEXT: br i1 undef, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] 2303; UNROLL: middle.block: 2304; UNROLL-NEXT: br i1 undef, label [[FOR_END:%.*]], label [[SCALAR_PH]] 2305; UNROLL: scalar.ph: 2306; UNROLL-NEXT: br label [[SCALAR_BODY:%.*]] 2307; UNROLL: scalar.body: 2308; UNROLL-NEXT: br i1 undef, label [[FOR_END]], label [[SCALAR_BODY]], !llvm.loop [[LOOP21:![0-9]+]] 2309; UNROLL: for.end: 2310; UNROLL-NEXT: ret void 2311; 2312; UNROLL-NO-IC-LABEL: @constant_folded_previous_value( 2313; UNROLL-NO-IC-NEXT: entry: 2314; UNROLL-NO-IC-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2315; UNROLL-NO-IC: vector.ph: 2316; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 2317; UNROLL-NO-IC: vector.body: 2318; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2319; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 0>, [[VECTOR_PH]] ], [ <i64 1, i64 1, i64 1, i64 1>, [[VECTOR_BODY]] ] 2320; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 2321; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 4 2322; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> <i64 1, i64 1, i64 1, i64 1>, <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2323; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 2324; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], undef 2325; UNROLL-NO-IC-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] 2326; UNROLL-NO-IC: middle.block: 2327; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 undef, undef 2328; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2329; UNROLL-NO-IC: scalar.ph: 2330; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ 1, [[MIDDLE_BLOCK]] ] 2331; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ undef, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2332; UNROLL-NO-IC-NEXT: br label [[SCALAR_BODY:%.*]] 2333; UNROLL-NO-IC: scalar.body: 2334; UNROLL-NO-IC-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ] 2335; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[VAR3:%.*]], [[SCALAR_BODY]] ] 2336; UNROLL-NO-IC-NEXT: [[VAR3]] = add i64 0, 1 2337; UNROLL-NO-IC-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 2338; UNROLL-NO-IC-NEXT: [[COND:%.*]] = icmp eq i64 [[I_NEXT]], undef 2339; UNROLL-NO-IC-NEXT: br i1 [[COND]], label [[FOR_END]], label [[SCALAR_BODY]], !llvm.loop [[LOOP21:![0-9]+]] 2340; UNROLL-NO-IC: for.end: 2341; UNROLL-NO-IC-NEXT: ret void 2342; 2343; UNROLL-NO-VF-LABEL: @constant_folded_previous_value( 2344; UNROLL-NO-VF-NEXT: entry: 2345; UNROLL-NO-VF-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2346; UNROLL-NO-VF: vector.ph: 2347; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 2348; UNROLL-NO-VF: vector.body: 2349; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2350; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ] 2351; UNROLL-NO-VF-NEXT: [[INDUCTION:%.*]] = add i64 [[INDEX]], 0 2352; UNROLL-NO-VF-NEXT: [[INDUCTION1:%.*]] = add i64 [[INDEX]], 1 2353; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i64 0, 1 2354; UNROLL-NO-VF-NEXT: [[TMP1]] = add i64 0, 1 2355; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 2356; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], undef 2357; UNROLL-NO-VF-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]] 2358; UNROLL-NO-VF: middle.block: 2359; UNROLL-NO-VF-NEXT: [[CMP_N:%.*]] = icmp eq i64 undef, undef 2360; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2361; UNROLL-NO-VF: scalar.ph: 2362; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[TMP1]], [[MIDDLE_BLOCK]] ] 2363; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ undef, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2364; UNROLL-NO-VF-NEXT: br label [[SCALAR_BODY:%.*]] 2365; UNROLL-NO-VF: scalar.body: 2366; UNROLL-NO-VF-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ] 2367; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[VAR3:%.*]], [[SCALAR_BODY]] ] 2368; UNROLL-NO-VF-NEXT: [[VAR3]] = add i64 0, 1 2369; UNROLL-NO-VF-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 2370; UNROLL-NO-VF-NEXT: [[COND:%.*]] = icmp eq i64 [[I_NEXT]], undef 2371; UNROLL-NO-VF-NEXT: br i1 [[COND]], label [[FOR_END]], label [[SCALAR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] 2372; UNROLL-NO-VF: for.end: 2373; UNROLL-NO-VF-NEXT: ret void 2374; 2375; SINK-AFTER-LABEL: @constant_folded_previous_value( 2376; SINK-AFTER-NEXT: entry: 2377; SINK-AFTER-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2378; SINK-AFTER: vector.ph: 2379; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 2380; SINK-AFTER: vector.body: 2381; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2382; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 0>, [[VECTOR_PH]] ], [ <i64 1, i64 1, i64 1, i64 1>, [[VECTOR_BODY]] ] 2383; SINK-AFTER-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 2384; SINK-AFTER-NEXT: [[TMP1:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> <i64 1, i64 1, i64 1, i64 1>, <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2385; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 2386; SINK-AFTER-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], undef 2387; SINK-AFTER-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] 2388; SINK-AFTER: middle.block: 2389; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 undef, undef 2390; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2391; SINK-AFTER: scalar.ph: 2392; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ 1, [[MIDDLE_BLOCK]] ] 2393; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ undef, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2394; SINK-AFTER-NEXT: br label [[SCALAR_BODY:%.*]] 2395; SINK-AFTER: scalar.body: 2396; SINK-AFTER-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ] 2397; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[VAR3:%.*]], [[SCALAR_BODY]] ] 2398; SINK-AFTER-NEXT: [[VAR3]] = add i64 0, 1 2399; SINK-AFTER-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 2400; SINK-AFTER-NEXT: [[COND:%.*]] = icmp eq i64 [[I_NEXT]], undef 2401; SINK-AFTER-NEXT: br i1 [[COND]], label [[FOR_END]], label [[SCALAR_BODY]], !llvm.loop [[LOOP21:![0-9]+]] 2402; SINK-AFTER: for.end: 2403; SINK-AFTER-NEXT: ret void 2404; 2405; NO-SINK-AFTER-LABEL: @constant_folded_previous_value( 2406; NO-SINK-AFTER-NEXT: entry: 2407; NO-SINK-AFTER-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2408; NO-SINK-AFTER: vector.ph: 2409; NO-SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 2410; NO-SINK-AFTER: vector.body: 2411; NO-SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2412; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 0>, [[VECTOR_PH]] ], [ <i64 1, i64 1, i64 1, i64 1>, [[VECTOR_BODY]] ] 2413; NO-SINK-AFTER-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 2414; NO-SINK-AFTER-NEXT: [[TMP1:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> <i64 1, i64 1, i64 1, i64 1>, <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2415; NO-SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 2416; NO-SINK-AFTER-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], undef 2417; NO-SINK-AFTER-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] 2418; NO-SINK-AFTER: middle.block: 2419; NO-SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 undef, undef 2420; NO-SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2421; NO-SINK-AFTER: scalar.ph: 2422; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ 1, [[MIDDLE_BLOCK]] ] 2423; NO-SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ undef, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2424; NO-SINK-AFTER-NEXT: br label [[SCALAR_BODY:%.*]] 2425; NO-SINK-AFTER: scalar.body: 2426; NO-SINK-AFTER-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ] 2427; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[VAR3:%.*]], [[SCALAR_BODY]] ] 2428; NO-SINK-AFTER-NEXT: [[VAR3]] = add i64 0, 1 2429; NO-SINK-AFTER-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 2430; NO-SINK-AFTER-NEXT: [[COND:%.*]] = icmp eq i64 [[I_NEXT]], undef 2431; NO-SINK-AFTER-NEXT: br i1 [[COND]], label [[FOR_END]], label [[SCALAR_BODY]], !llvm.loop [[LOOP21:![0-9]+]] 2432; NO-SINK-AFTER: for.end: 2433; NO-SINK-AFTER-NEXT: ret void 2434; 2435entry: 2436 br label %scalar.body 2437 2438scalar.body: 2439 %i = phi i64 [ 0, %entry ], [ %i.next, %scalar.body ] 2440 %var2 = phi i64 [ 0, %entry ], [ %var3, %scalar.body ] 2441 %var3 = add i64 0, 1 2442 %i.next = add nuw nsw i64 %i, 1 2443 %cond = icmp eq i64 %i.next, undef 2444 br i1 %cond, label %for.end, label %scalar.body 2445 2446for.end: 2447 ret void 2448} 2449 2450; We vectorize this first order recurrence, by generating two 2451; extracts for the phi `val.phi` - one at the last index and 2452; another at the second last index. We need these 2 extracts because 2453; the first order recurrence phi is used outside the loop, so we require the phi 2454; itself and not its update (addx). 2455; Check the case when unrolled but not vectorized. 2456define i32 @extract_second_last_iteration(i32* %cval, i32 %x) { 2457; CHECK-LABEL: @extract_second_last_iteration( 2458; CHECK-NEXT: entry: 2459; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2460; CHECK: vector.ph: 2461; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 2462; CHECK: vector.body: 2463; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2464; CHECK-NEXT: [[TMP0:%.*]] = phi i32 [ 2, [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ] 2465; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 2466; CHECK-NEXT: [[TMP1]] = add i32 [[TMP0]], 4 2467; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96 2468; CHECK-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] 2469; CHECK: middle.block: 2470; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = add i32 [[TMP0]], [[X:%.*]] 2471; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]] 2472; CHECK: scalar.ph: 2473; CHECK-NEXT: br label [[FOR_BODY:%.*]] 2474; CHECK: for.body: 2475; CHECK-NEXT: br i1 undef, label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]] 2476; CHECK: for.end: 2477; CHECK-NEXT: [[VAL_PHI_LCSSA:%.*]] = phi i32 [ undef, [[FOR_BODY]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], [[MIDDLE_BLOCK]] ] 2478; CHECK-NEXT: ret i32 [[VAL_PHI_LCSSA]] 2479; 2480; UNROLL-LABEL: @extract_second_last_iteration( 2481; UNROLL-NEXT: entry: 2482; UNROLL-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2483; UNROLL: vector.ph: 2484; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 2485; UNROLL: vector.body: 2486; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2487; UNROLL-NEXT: [[TMP0:%.*]] = phi i32 [ 2, [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ] 2488; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 2489; UNROLL-NEXT: [[TMP1]] = add i32 [[TMP0]], 8 2490; UNROLL-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96 2491; UNROLL-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] 2492; UNROLL: middle.block: 2493; UNROLL-NEXT: [[TMP3:%.*]] = add i32 [[TMP0]], 4 2494; UNROLL-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = add i32 [[TMP3]], [[X:%.*]] 2495; UNROLL-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]] 2496; UNROLL: scalar.ph: 2497; UNROLL-NEXT: br label [[FOR_BODY:%.*]] 2498; UNROLL: for.body: 2499; UNROLL-NEXT: br i1 undef, label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]] 2500; UNROLL: for.end: 2501; UNROLL-NEXT: [[VAL_PHI_LCSSA:%.*]] = phi i32 [ undef, [[FOR_BODY]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], [[MIDDLE_BLOCK]] ] 2502; UNROLL-NEXT: ret i32 [[VAL_PHI_LCSSA]] 2503; 2504; UNROLL-NO-IC-LABEL: @extract_second_last_iteration( 2505; UNROLL-NO-IC-NEXT: entry: 2506; UNROLL-NO-IC-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2507; UNROLL-NO-IC: vector.ph: 2508; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i32 0 2509; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 2510; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i32 0 2511; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT2]], <4 x i32> poison, <4 x i32> zeroinitializer 2512; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 2513; UNROLL-NO-IC: vector.body: 2514; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2515; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 2516; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ] 2517; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4> 2518; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0 2519; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1 2520; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 2 2521; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 3 2522; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 4 2523; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = add i32 [[INDEX]], 5 2524; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = add i32 [[INDEX]], 6 2525; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = add i32 [[INDEX]], 7 2526; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = zext <4 x i32> [[VEC_IND]] to <4 x i64> 2527; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = zext <4 x i32> [[STEP_ADD]] to <4 x i64> 2528; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = add <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]] 2529; UNROLL-NO-IC-NEXT: [[TMP11]] = add <4 x i32> [[STEP_ADD]], [[BROADCAST_SPLAT3]] 2530; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP10]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2531; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> [[TMP11]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2532; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 2533; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], <i32 4, i32 4, i32 4, i32 4> 2534; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96 2535; UNROLL-NO-IC-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] 2536; UNROLL-NO-IC: middle.block: 2537; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i32 96, 96 2538; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP11]], i32 3 2539; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP11]], i32 2 2540; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2541; UNROLL-NO-IC: scalar.ph: 2542; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 2543; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 96, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2544; UNROLL-NO-IC-NEXT: br label [[FOR_BODY:%.*]] 2545; UNROLL-NO-IC: for.body: 2546; UNROLL-NO-IC-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ] 2547; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[ADDX:%.*]], [[FOR_BODY]] ] 2548; UNROLL-NO-IC-NEXT: [[INC]] = add i32 [[INC_PHI]], 1 2549; UNROLL-NO-IC-NEXT: [[BC:%.*]] = zext i32 [[INC_PHI]] to i64 2550; UNROLL-NO-IC-NEXT: [[ADDX]] = add i32 [[INC_PHI]], [[X]] 2551; UNROLL-NO-IC-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC_PHI]], 95 2552; UNROLL-NO-IC-NEXT: br i1 [[CMP]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]] 2553; UNROLL-NO-IC: for.end: 2554; UNROLL-NO-IC-NEXT: [[VAL_PHI_LCSSA:%.*]] = phi i32 [ [[SCALAR_RECUR]], [[FOR_BODY]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], [[MIDDLE_BLOCK]] ] 2555; UNROLL-NO-IC-NEXT: ret i32 [[VAL_PHI_LCSSA]] 2556; 2557; UNROLL-NO-VF-LABEL: @extract_second_last_iteration( 2558; UNROLL-NO-VF-NEXT: entry: 2559; UNROLL-NO-VF-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2560; UNROLL-NO-VF: vector.ph: 2561; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 2562; UNROLL-NO-VF: vector.body: 2563; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2564; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ] 2565; UNROLL-NO-VF-NEXT: [[INDUCTION:%.*]] = add i32 [[INDEX]], 0 2566; UNROLL-NO-VF-NEXT: [[INDUCTION1:%.*]] = add i32 [[INDEX]], 1 2567; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = zext i32 [[INDUCTION]] to i64 2568; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = zext i32 [[INDUCTION1]] to i64 2569; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = add i32 [[INDUCTION]], [[X:%.*]] 2570; UNROLL-NO-VF-NEXT: [[TMP3]] = add i32 [[INDUCTION1]], [[X]] 2571; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 2572; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96 2573; UNROLL-NO-VF-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]] 2574; UNROLL-NO-VF: middle.block: 2575; UNROLL-NO-VF-NEXT: [[CMP_N:%.*]] = icmp eq i32 96, 96 2576; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2577; UNROLL-NO-VF: scalar.ph: 2578; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP3]], [[MIDDLE_BLOCK]] ] 2579; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 96, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2580; UNROLL-NO-VF-NEXT: br label [[FOR_BODY:%.*]] 2581; UNROLL-NO-VF: for.body: 2582; UNROLL-NO-VF-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ] 2583; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[ADDX:%.*]], [[FOR_BODY]] ] 2584; UNROLL-NO-VF-NEXT: [[INC]] = add i32 [[INC_PHI]], 1 2585; UNROLL-NO-VF-NEXT: [[BC:%.*]] = zext i32 [[INC_PHI]] to i64 2586; UNROLL-NO-VF-NEXT: [[ADDX]] = add i32 [[INC_PHI]], [[X]] 2587; UNROLL-NO-VF-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC_PHI]], 95 2588; UNROLL-NO-VF-NEXT: br i1 [[CMP]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] 2589; UNROLL-NO-VF: for.end: 2590; UNROLL-NO-VF-NEXT: [[VAL_PHI_LCSSA:%.*]] = phi i32 [ [[SCALAR_RECUR]], [[FOR_BODY]] ], [ [[TMP2]], [[MIDDLE_BLOCK]] ] 2591; UNROLL-NO-VF-NEXT: ret i32 [[VAL_PHI_LCSSA]] 2592; 2593; SINK-AFTER-LABEL: @extract_second_last_iteration( 2594; SINK-AFTER-NEXT: entry: 2595; SINK-AFTER-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2596; SINK-AFTER: vector.ph: 2597; SINK-AFTER-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i32 0 2598; SINK-AFTER-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 2599; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 2600; SINK-AFTER: vector.body: 2601; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2602; SINK-AFTER-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 2603; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ] 2604; SINK-AFTER-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0 2605; SINK-AFTER-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1 2606; SINK-AFTER-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 2 2607; SINK-AFTER-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 3 2608; SINK-AFTER-NEXT: [[TMP4:%.*]] = zext <4 x i32> [[VEC_IND]] to <4 x i64> 2609; SINK-AFTER-NEXT: [[TMP5]] = add <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]] 2610; SINK-AFTER-NEXT: [[TMP6:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP5]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2611; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 2612; SINK-AFTER-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4> 2613; SINK-AFTER-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96 2614; SINK-AFTER-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] 2615; SINK-AFTER: middle.block: 2616; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i32 96, 96 2617; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP5]], i32 3 2618; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP5]], i32 2 2619; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2620; SINK-AFTER: scalar.ph: 2621; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 2622; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 96, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2623; SINK-AFTER-NEXT: br label [[FOR_BODY:%.*]] 2624; SINK-AFTER: for.body: 2625; SINK-AFTER-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ] 2626; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[ADDX:%.*]], [[FOR_BODY]] ] 2627; SINK-AFTER-NEXT: [[INC]] = add i32 [[INC_PHI]], 1 2628; SINK-AFTER-NEXT: [[BC:%.*]] = zext i32 [[INC_PHI]] to i64 2629; SINK-AFTER-NEXT: [[ADDX]] = add i32 [[INC_PHI]], [[X]] 2630; SINK-AFTER-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC_PHI]], 95 2631; SINK-AFTER-NEXT: br i1 [[CMP]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]] 2632; SINK-AFTER: for.end: 2633; SINK-AFTER-NEXT: [[VAL_PHI_LCSSA:%.*]] = phi i32 [ [[SCALAR_RECUR]], [[FOR_BODY]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], [[MIDDLE_BLOCK]] ] 2634; SINK-AFTER-NEXT: ret i32 [[VAL_PHI_LCSSA]] 2635; 2636; NO-SINK-AFTER-LABEL: @extract_second_last_iteration( 2637; NO-SINK-AFTER-NEXT: entry: 2638; NO-SINK-AFTER-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2639; NO-SINK-AFTER: vector.ph: 2640; NO-SINK-AFTER-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i32 0 2641; NO-SINK-AFTER-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 2642; NO-SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 2643; NO-SINK-AFTER: vector.body: 2644; NO-SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2645; NO-SINK-AFTER-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 2646; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ] 2647; NO-SINK-AFTER-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0 2648; NO-SINK-AFTER-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1 2649; NO-SINK-AFTER-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 2 2650; NO-SINK-AFTER-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 3 2651; NO-SINK-AFTER-NEXT: [[TMP4:%.*]] = zext <4 x i32> [[VEC_IND]] to <4 x i64> 2652; NO-SINK-AFTER-NEXT: [[TMP5]] = add <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]] 2653; NO-SINK-AFTER-NEXT: [[TMP6:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP5]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2654; NO-SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 2655; NO-SINK-AFTER-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4> 2656; NO-SINK-AFTER-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96 2657; NO-SINK-AFTER-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] 2658; NO-SINK-AFTER: middle.block: 2659; NO-SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i32 96, 96 2660; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP5]], i32 3 2661; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP5]], i32 2 2662; NO-SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2663; NO-SINK-AFTER: scalar.ph: 2664; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 2665; NO-SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 96, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2666; NO-SINK-AFTER-NEXT: br label [[FOR_BODY:%.*]] 2667; NO-SINK-AFTER: for.body: 2668; NO-SINK-AFTER-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ] 2669; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[ADDX:%.*]], [[FOR_BODY]] ] 2670; NO-SINK-AFTER-NEXT: [[INC]] = add i32 [[INC_PHI]], 1 2671; NO-SINK-AFTER-NEXT: [[BC:%.*]] = zext i32 [[INC_PHI]] to i64 2672; NO-SINK-AFTER-NEXT: [[ADDX]] = add i32 [[INC_PHI]], [[X]] 2673; NO-SINK-AFTER-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC_PHI]], 95 2674; NO-SINK-AFTER-NEXT: br i1 [[CMP]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]] 2675; NO-SINK-AFTER: for.end: 2676; NO-SINK-AFTER-NEXT: [[VAL_PHI_LCSSA:%.*]] = phi i32 [ [[SCALAR_RECUR]], [[FOR_BODY]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], [[MIDDLE_BLOCK]] ] 2677; NO-SINK-AFTER-NEXT: ret i32 [[VAL_PHI_LCSSA]] 2678; 2679entry: 2680 br label %for.body 2681 2682for.body: 2683 %inc.phi = phi i32 [ 0, %entry ], [ %inc, %for.body ] 2684 %val.phi = phi i32 [ 0, %entry ], [ %addx, %for.body ] 2685 %inc = add i32 %inc.phi, 1 2686 %bc = zext i32 %inc.phi to i64 2687 %addx = add i32 %inc.phi, %x 2688 %cmp = icmp eq i32 %inc.phi, 95 2689 br i1 %cmp, label %for.end, label %for.body 2690 2691for.end: 2692 ret i32 %val.phi 2693} 2694 2695; We vectorize this first order recurrence, with a set of insertelements for 2696; each unrolled part. Make sure these insertelements are generated in-order, 2697; because the shuffle of the first order recurrence will be added after the 2698; insertelement of the last part UF - 1, assuming the latter appears after the 2699; insertelements of all other parts. 2700; 2701; int PR33613(double *b, double j, int d) { 2702; int a = 0; 2703; for(int i = 0; i < 10240; i++, b+=25) { 2704; double f = b[d]; // Scalarize to form insertelements 2705; if (j * f) 2706; a++; 2707; j = f; 2708; } 2709; return a; 2710; } 2711; 2712; 2713define i32 @PR33613(double* %b, double %j, i32 %d) { 2714; CHECK-LABEL: @PR33613( 2715; CHECK-NEXT: entry: 2716; CHECK-NEXT: [[IDXPROM:%.*]] = sext i32 [[D:%.*]] to i64 2717; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2718; CHECK: vector.ph: 2719; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x double> poison, double [[J:%.*]], i64 3 2720; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 2721; CHECK: vector.body: 2722; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2723; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[VECTOR_BODY]] ] 2724; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x double> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[VECTOR_BODY]] ] 2725; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[INDEX]], 25 2726; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr double, double* [[B:%.*]], i64 [[IDXPROM]] 2727; CHECK-NEXT: [[TMP1:%.*]] = or i64 [[INDEX]], 1 2728; CHECK-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 25 2729; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr double, double* [[B]], i64 [[IDXPROM]] 2730; CHECK-NEXT: [[TMP3:%.*]] = or i64 [[INDEX]], 2 2731; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], 25 2732; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr double, double* [[B]], i64 [[IDXPROM]] 2733; CHECK-NEXT: [[TMP5:%.*]] = or i64 [[INDEX]], 3 2734; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 25 2735; CHECK-NEXT: [[NEXT_GEP4:%.*]] = getelementptr double, double* [[B]], i64 [[IDXPROM]] 2736; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP]], i64 [[TMP0]] 2737; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP2]], i64 [[TMP2]] 2738; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP3]], i64 [[TMP4]] 2739; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP4]], i64 [[TMP6]] 2740; CHECK-NEXT: [[TMP11:%.*]] = load double, double* [[TMP7]], align 8 2741; CHECK-NEXT: [[TMP12:%.*]] = load double, double* [[TMP8]], align 8 2742; CHECK-NEXT: [[TMP13:%.*]] = load double, double* [[TMP9]], align 8 2743; CHECK-NEXT: [[TMP14:%.*]] = load double, double* [[TMP10]], align 8 2744; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x double> poison, double [[TMP11]], i64 0 2745; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x double> [[TMP15]], double [[TMP12]], i64 1 2746; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x double> [[TMP16]], double [[TMP13]], i64 2 2747; CHECK-NEXT: [[TMP18]] = insertelement <4 x double> [[TMP17]], double [[TMP14]], i64 3 2748; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR]], <4 x double> [[TMP17]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2749; CHECK-NEXT: [[TMP20:%.*]] = fmul <4 x double> [[TMP19]], [[TMP18]] 2750; CHECK-NEXT: [[TMP21:%.*]] = fcmp une <4 x double> [[TMP20]], zeroinitializer 2751; CHECK-NEXT: [[TMP22:%.*]] = zext <4 x i1> [[TMP21]] to <4 x i32> 2752; CHECK-NEXT: [[TMP23]] = add <4 x i32> [[VEC_PHI]], [[TMP22]] 2753; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 2754; CHECK-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10240 2755; CHECK-NEXT: br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]] 2756; CHECK: middle.block: 2757; CHECK-NEXT: [[TMP25:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP23]]) 2758; CHECK-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]] 2759; CHECK: scalar.ph: 2760; CHECK-NEXT: br label [[FOR_BODY:%.*]] 2761; CHECK: for.cond.cleanup: 2762; CHECK-NEXT: [[A_1_LCSSA:%.*]] = phi i32 [ undef, [[FOR_BODY]] ], [ [[TMP25]], [[MIDDLE_BLOCK]] ] 2763; CHECK-NEXT: ret i32 [[A_1_LCSSA]] 2764; CHECK: for.body: 2765; CHECK-NEXT: br i1 undef, label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]] 2766; 2767; UNROLL-LABEL: @PR33613( 2768; UNROLL-NEXT: entry: 2769; UNROLL-NEXT: [[IDXPROM:%.*]] = sext i32 [[D:%.*]] to i64 2770; UNROLL-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2771; UNROLL: vector.ph: 2772; UNROLL-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x double> poison, double [[J:%.*]], i64 3 2773; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 2774; UNROLL: vector.body: 2775; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2776; UNROLL-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP47:%.*]], [[VECTOR_BODY]] ] 2777; UNROLL-NEXT: [[VEC_PHI9:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP48:%.*]], [[VECTOR_BODY]] ] 2778; UNROLL-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x double> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP38:%.*]], [[VECTOR_BODY]] ] 2779; UNROLL-NEXT: [[TMP0:%.*]] = mul i64 [[INDEX]], 25 2780; UNROLL-NEXT: [[NEXT_GEP:%.*]] = getelementptr double, double* [[B:%.*]], i64 [[IDXPROM]] 2781; UNROLL-NEXT: [[TMP1:%.*]] = or i64 [[INDEX]], 1 2782; UNROLL-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 25 2783; UNROLL-NEXT: [[NEXT_GEP2:%.*]] = getelementptr double, double* [[B]], i64 [[IDXPROM]] 2784; UNROLL-NEXT: [[TMP3:%.*]] = or i64 [[INDEX]], 2 2785; UNROLL-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], 25 2786; UNROLL-NEXT: [[NEXT_GEP3:%.*]] = getelementptr double, double* [[B]], i64 [[IDXPROM]] 2787; UNROLL-NEXT: [[TMP5:%.*]] = or i64 [[INDEX]], 3 2788; UNROLL-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 25 2789; UNROLL-NEXT: [[NEXT_GEP4:%.*]] = getelementptr double, double* [[B]], i64 [[IDXPROM]] 2790; UNROLL-NEXT: [[TMP7:%.*]] = or i64 [[INDEX]], 4 2791; UNROLL-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 25 2792; UNROLL-NEXT: [[NEXT_GEP5:%.*]] = getelementptr double, double* [[B]], i64 [[IDXPROM]] 2793; UNROLL-NEXT: [[TMP9:%.*]] = or i64 [[INDEX]], 5 2794; UNROLL-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 25 2795; UNROLL-NEXT: [[NEXT_GEP6:%.*]] = getelementptr double, double* [[B]], i64 [[IDXPROM]] 2796; UNROLL-NEXT: [[TMP11:%.*]] = or i64 [[INDEX]], 6 2797; UNROLL-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 25 2798; UNROLL-NEXT: [[NEXT_GEP7:%.*]] = getelementptr double, double* [[B]], i64 [[IDXPROM]] 2799; UNROLL-NEXT: [[TMP13:%.*]] = or i64 [[INDEX]], 7 2800; UNROLL-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 25 2801; UNROLL-NEXT: [[NEXT_GEP8:%.*]] = getelementptr double, double* [[B]], i64 [[IDXPROM]] 2802; UNROLL-NEXT: [[TMP15:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP]], i64 [[TMP0]] 2803; UNROLL-NEXT: [[TMP16:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP2]], i64 [[TMP2]] 2804; UNROLL-NEXT: [[TMP17:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP3]], i64 [[TMP4]] 2805; UNROLL-NEXT: [[TMP18:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP4]], i64 [[TMP6]] 2806; UNROLL-NEXT: [[TMP19:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP5]], i64 [[TMP8]] 2807; UNROLL-NEXT: [[TMP20:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP6]], i64 [[TMP10]] 2808; UNROLL-NEXT: [[TMP21:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP7]], i64 [[TMP12]] 2809; UNROLL-NEXT: [[TMP22:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP8]], i64 [[TMP14]] 2810; UNROLL-NEXT: [[TMP23:%.*]] = load double, double* [[TMP15]], align 8 2811; UNROLL-NEXT: [[TMP24:%.*]] = load double, double* [[TMP16]], align 8 2812; UNROLL-NEXT: [[TMP25:%.*]] = load double, double* [[TMP17]], align 8 2813; UNROLL-NEXT: [[TMP26:%.*]] = load double, double* [[TMP18]], align 8 2814; UNROLL-NEXT: [[TMP27:%.*]] = insertelement <4 x double> poison, double [[TMP23]], i64 0 2815; UNROLL-NEXT: [[TMP28:%.*]] = insertelement <4 x double> [[TMP27]], double [[TMP24]], i64 1 2816; UNROLL-NEXT: [[TMP29:%.*]] = insertelement <4 x double> [[TMP28]], double [[TMP25]], i64 2 2817; UNROLL-NEXT: [[TMP30:%.*]] = insertelement <4 x double> [[TMP29]], double [[TMP26]], i64 3 2818; UNROLL-NEXT: [[TMP31:%.*]] = load double, double* [[TMP19]], align 8 2819; UNROLL-NEXT: [[TMP32:%.*]] = load double, double* [[TMP20]], align 8 2820; UNROLL-NEXT: [[TMP33:%.*]] = load double, double* [[TMP21]], align 8 2821; UNROLL-NEXT: [[TMP34:%.*]] = load double, double* [[TMP22]], align 8 2822; UNROLL-NEXT: [[TMP35:%.*]] = insertelement <4 x double> poison, double [[TMP31]], i64 0 2823; UNROLL-NEXT: [[TMP36:%.*]] = insertelement <4 x double> [[TMP35]], double [[TMP32]], i64 1 2824; UNROLL-NEXT: [[TMP37:%.*]] = insertelement <4 x double> [[TMP36]], double [[TMP33]], i64 2 2825; UNROLL-NEXT: [[TMP38]] = insertelement <4 x double> [[TMP37]], double [[TMP34]], i64 3 2826; UNROLL-NEXT: [[TMP39:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR]], <4 x double> [[TMP29]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2827; UNROLL-NEXT: [[TMP40:%.*]] = shufflevector <4 x double> [[TMP30]], <4 x double> [[TMP37]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2828; UNROLL-NEXT: [[TMP41:%.*]] = fmul <4 x double> [[TMP39]], [[TMP30]] 2829; UNROLL-NEXT: [[TMP42:%.*]] = fmul <4 x double> [[TMP40]], [[TMP38]] 2830; UNROLL-NEXT: [[TMP43:%.*]] = fcmp une <4 x double> [[TMP41]], zeroinitializer 2831; UNROLL-NEXT: [[TMP44:%.*]] = fcmp une <4 x double> [[TMP42]], zeroinitializer 2832; UNROLL-NEXT: [[TMP45:%.*]] = zext <4 x i1> [[TMP43]] to <4 x i32> 2833; UNROLL-NEXT: [[TMP46:%.*]] = zext <4 x i1> [[TMP44]] to <4 x i32> 2834; UNROLL-NEXT: [[TMP47]] = add <4 x i32> [[VEC_PHI]], [[TMP45]] 2835; UNROLL-NEXT: [[TMP48]] = add <4 x i32> [[VEC_PHI9]], [[TMP46]] 2836; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 2837; UNROLL-NEXT: [[TMP49:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10240 2838; UNROLL-NEXT: br i1 [[TMP49]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]] 2839; UNROLL: middle.block: 2840; UNROLL-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP48]], [[TMP47]] 2841; UNROLL-NEXT: [[TMP50:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]]) 2842; UNROLL-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]] 2843; UNROLL: scalar.ph: 2844; UNROLL-NEXT: br label [[FOR_BODY:%.*]] 2845; UNROLL: for.cond.cleanup: 2846; UNROLL-NEXT: [[A_1_LCSSA:%.*]] = phi i32 [ undef, [[FOR_BODY]] ], [ [[TMP50]], [[MIDDLE_BLOCK]] ] 2847; UNROLL-NEXT: ret i32 [[A_1_LCSSA]] 2848; UNROLL: for.body: 2849; UNROLL-NEXT: br i1 undef, label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]] 2850; 2851; UNROLL-NO-IC-LABEL: @PR33613( 2852; UNROLL-NO-IC-NEXT: entry: 2853; UNROLL-NO-IC-NEXT: [[IDXPROM:%.*]] = sext i32 [[D:%.*]] to i64 2854; UNROLL-NO-IC-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2855; UNROLL-NO-IC: vector.ph: 2856; UNROLL-NO-IC-NEXT: [[IND_END:%.*]] = getelementptr double, double* [[B:%.*]], i64 256000 2857; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x double> poison, double [[J:%.*]], i32 3 2858; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 2859; UNROLL-NO-IC: vector.body: 2860; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2861; UNROLL-NO-IC-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP50:%.*]], [[VECTOR_BODY]] ] 2862; UNROLL-NO-IC-NEXT: [[VEC_PHI9:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP51:%.*]], [[VECTOR_BODY]] ] 2863; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x double> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP41:%.*]], [[VECTOR_BODY]] ] 2864; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 2865; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 25 2866; UNROLL-NO-IC-NEXT: [[NEXT_GEP:%.*]] = getelementptr double, double* [[B]], i64 [[TMP1]] 2867; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1 2868; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 25 2869; UNROLL-NO-IC-NEXT: [[NEXT_GEP2:%.*]] = getelementptr double, double* [[B]], i64 [[TMP3]] 2870; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 2 2871; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 25 2872; UNROLL-NO-IC-NEXT: [[NEXT_GEP3:%.*]] = getelementptr double, double* [[B]], i64 [[TMP5]] 2873; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 3 2874; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 25 2875; UNROLL-NO-IC-NEXT: [[NEXT_GEP4:%.*]] = getelementptr double, double* [[B]], i64 [[TMP7]] 2876; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 4 2877; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 25 2878; UNROLL-NO-IC-NEXT: [[NEXT_GEP5:%.*]] = getelementptr double, double* [[B]], i64 [[TMP9]] 2879; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 5 2880; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 25 2881; UNROLL-NO-IC-NEXT: [[NEXT_GEP6:%.*]] = getelementptr double, double* [[B]], i64 [[TMP11]] 2882; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 6 2883; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 25 2884; UNROLL-NO-IC-NEXT: [[NEXT_GEP7:%.*]] = getelementptr double, double* [[B]], i64 [[TMP13]] 2885; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = add i64 [[INDEX]], 7 2886; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 25 2887; UNROLL-NO-IC-NEXT: [[NEXT_GEP8:%.*]] = getelementptr double, double* [[B]], i64 [[TMP15]] 2888; UNROLL-NO-IC-NEXT: [[OFFSET_IDX:%.*]] = trunc i64 [[INDEX]] to i32 2889; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = add i32 [[OFFSET_IDX]], 0 2890; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = add i32 [[OFFSET_IDX]], 4 2891; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP]], i64 [[IDXPROM]] 2892; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP2]], i64 [[IDXPROM]] 2893; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP3]], i64 [[IDXPROM]] 2894; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP4]], i64 [[IDXPROM]] 2895; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP5]], i64 [[IDXPROM]] 2896; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP6]], i64 [[IDXPROM]] 2897; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP7]], i64 [[IDXPROM]] 2898; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP8]], i64 [[IDXPROM]] 2899; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = load double, double* [[TMP18]], align 8 2900; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = load double, double* [[TMP19]], align 8 2901; UNROLL-NO-IC-NEXT: [[TMP28:%.*]] = load double, double* [[TMP20]], align 8 2902; UNROLL-NO-IC-NEXT: [[TMP29:%.*]] = load double, double* [[TMP21]], align 8 2903; UNROLL-NO-IC-NEXT: [[TMP30:%.*]] = insertelement <4 x double> poison, double [[TMP26]], i32 0 2904; UNROLL-NO-IC-NEXT: [[TMP31:%.*]] = insertelement <4 x double> [[TMP30]], double [[TMP27]], i32 1 2905; UNROLL-NO-IC-NEXT: [[TMP32:%.*]] = insertelement <4 x double> [[TMP31]], double [[TMP28]], i32 2 2906; UNROLL-NO-IC-NEXT: [[TMP33:%.*]] = insertelement <4 x double> [[TMP32]], double [[TMP29]], i32 3 2907; UNROLL-NO-IC-NEXT: [[TMP34:%.*]] = load double, double* [[TMP22]], align 8 2908; UNROLL-NO-IC-NEXT: [[TMP35:%.*]] = load double, double* [[TMP23]], align 8 2909; UNROLL-NO-IC-NEXT: [[TMP36:%.*]] = load double, double* [[TMP24]], align 8 2910; UNROLL-NO-IC-NEXT: [[TMP37:%.*]] = load double, double* [[TMP25]], align 8 2911; UNROLL-NO-IC-NEXT: [[TMP38:%.*]] = insertelement <4 x double> poison, double [[TMP34]], i32 0 2912; UNROLL-NO-IC-NEXT: [[TMP39:%.*]] = insertelement <4 x double> [[TMP38]], double [[TMP35]], i32 1 2913; UNROLL-NO-IC-NEXT: [[TMP40:%.*]] = insertelement <4 x double> [[TMP39]], double [[TMP36]], i32 2 2914; UNROLL-NO-IC-NEXT: [[TMP41]] = insertelement <4 x double> [[TMP40]], double [[TMP37]], i32 3 2915; UNROLL-NO-IC-NEXT: [[TMP42:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR]], <4 x double> [[TMP33]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2916; UNROLL-NO-IC-NEXT: [[TMP43:%.*]] = shufflevector <4 x double> [[TMP33]], <4 x double> [[TMP41]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2917; UNROLL-NO-IC-NEXT: [[TMP44:%.*]] = fmul <4 x double> [[TMP42]], [[TMP33]] 2918; UNROLL-NO-IC-NEXT: [[TMP45:%.*]] = fmul <4 x double> [[TMP43]], [[TMP41]] 2919; UNROLL-NO-IC-NEXT: [[TMP46:%.*]] = fcmp une <4 x double> [[TMP44]], zeroinitializer 2920; UNROLL-NO-IC-NEXT: [[TMP47:%.*]] = fcmp une <4 x double> [[TMP45]], zeroinitializer 2921; UNROLL-NO-IC-NEXT: [[TMP48:%.*]] = zext <4 x i1> [[TMP46]] to <4 x i32> 2922; UNROLL-NO-IC-NEXT: [[TMP49:%.*]] = zext <4 x i1> [[TMP47]] to <4 x i32> 2923; UNROLL-NO-IC-NEXT: [[TMP50]] = add <4 x i32> [[VEC_PHI]], [[TMP48]] 2924; UNROLL-NO-IC-NEXT: [[TMP51]] = add <4 x i32> [[VEC_PHI9]], [[TMP49]] 2925; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 2926; UNROLL-NO-IC-NEXT: [[TMP52:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10240 2927; UNROLL-NO-IC-NEXT: br i1 [[TMP52]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]] 2928; UNROLL-NO-IC: middle.block: 2929; UNROLL-NO-IC-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP51]], [[TMP50]] 2930; UNROLL-NO-IC-NEXT: [[TMP53:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]]) 2931; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 10240, 10240 2932; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x double> [[TMP41]], i32 3 2933; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x double> [[TMP41]], i32 2 2934; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]] 2935; UNROLL-NO-IC: scalar.ph: 2936; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi double [ [[J]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 2937; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi double* [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[B]], [[ENTRY]] ] 2938; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ 10240, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2939; UNROLL-NO-IC-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP53]], [[MIDDLE_BLOCK]] ] 2940; UNROLL-NO-IC-NEXT: br label [[FOR_BODY:%.*]] 2941; UNROLL-NO-IC: for.cond.cleanup: 2942; UNROLL-NO-IC-NEXT: [[A_1_LCSSA:%.*]] = phi i32 [ [[A_1:%.*]], [[FOR_BODY]] ], [ [[TMP53]], [[MIDDLE_BLOCK]] ] 2943; UNROLL-NO-IC-NEXT: ret i32 [[A_1_LCSSA]] 2944; UNROLL-NO-IC: for.body: 2945; UNROLL-NO-IC-NEXT: [[B_ADDR_012:%.*]] = phi double* [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADD_PTR:%.*]], [[FOR_BODY]] ] 2946; UNROLL-NO-IC-NEXT: [[I_011:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[INC1:%.*]], [[FOR_BODY]] ] 2947; UNROLL-NO-IC-NEXT: [[A_010:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[A_1]], [[FOR_BODY]] ] 2948; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi double [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP54:%.*]], [[FOR_BODY]] ] 2949; UNROLL-NO-IC-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[B_ADDR_012]], i64 [[IDXPROM]] 2950; UNROLL-NO-IC-NEXT: [[TMP54]] = load double, double* [[ARRAYIDX]], align 8 2951; UNROLL-NO-IC-NEXT: [[MUL:%.*]] = fmul double [[SCALAR_RECUR]], [[TMP54]] 2952; UNROLL-NO-IC-NEXT: [[TOBOOL:%.*]] = fcmp une double [[MUL]], 0.000000e+00 2953; UNROLL-NO-IC-NEXT: [[INC:%.*]] = zext i1 [[TOBOOL]] to i32 2954; UNROLL-NO-IC-NEXT: [[A_1]] = add nsw i32 [[A_010]], [[INC]] 2955; UNROLL-NO-IC-NEXT: [[INC1]] = add nuw nsw i32 [[I_011]], 1 2956; UNROLL-NO-IC-NEXT: [[ADD_PTR]] = getelementptr inbounds double, double* [[B_ADDR_012]], i64 25 2957; UNROLL-NO-IC-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC1]], 10240 2958; UNROLL-NO-IC-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]] 2959; 2960; UNROLL-NO-VF-LABEL: @PR33613( 2961; UNROLL-NO-VF-NEXT: entry: 2962; UNROLL-NO-VF-NEXT: [[IDXPROM:%.*]] = sext i32 [[D:%.*]] to i64 2963; UNROLL-NO-VF-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2964; UNROLL-NO-VF: vector.ph: 2965; UNROLL-NO-VF-NEXT: [[IND_END:%.*]] = getelementptr double, double* [[B:%.*]], i64 256000 2966; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 2967; UNROLL-NO-VF: vector.body: 2968; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2969; UNROLL-NO-VF-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ] 2970; UNROLL-NO-VF-NEXT: [[VEC_PHI4:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ] 2971; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi double [ [[J:%.*]], [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ] 2972; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 2973; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 25 2974; UNROLL-NO-VF-NEXT: [[NEXT_GEP:%.*]] = getelementptr double, double* [[B]], i64 [[TMP1]] 2975; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1 2976; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 25 2977; UNROLL-NO-VF-NEXT: [[NEXT_GEP2:%.*]] = getelementptr double, double* [[B]], i64 [[TMP3]] 2978; UNROLL-NO-VF-NEXT: [[OFFSET_IDX:%.*]] = trunc i64 [[INDEX]] to i32 2979; UNROLL-NO-VF-NEXT: [[INDUCTION:%.*]] = add i32 [[OFFSET_IDX]], 0 2980; UNROLL-NO-VF-NEXT: [[INDUCTION3:%.*]] = add i32 [[OFFSET_IDX]], 1 2981; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP]], i64 [[IDXPROM]] 2982; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP2]], i64 [[IDXPROM]] 2983; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = load double, double* [[TMP4]], align 8 2984; UNROLL-NO-VF-NEXT: [[TMP7]] = load double, double* [[TMP5]], align 8 2985; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = fmul double [[VECTOR_RECUR]], [[TMP6]] 2986; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = fmul double [[TMP6]], [[TMP7]] 2987; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = fcmp une double [[TMP8]], 0.000000e+00 2988; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = fcmp une double [[TMP9]], 0.000000e+00 2989; UNROLL-NO-VF-NEXT: [[TMP12:%.*]] = zext i1 [[TMP10]] to i32 2990; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = zext i1 [[TMP11]] to i32 2991; UNROLL-NO-VF-NEXT: [[TMP14]] = add i32 [[VEC_PHI]], [[TMP12]] 2992; UNROLL-NO-VF-NEXT: [[TMP15]] = add i32 [[VEC_PHI4]], [[TMP13]] 2993; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 2994; UNROLL-NO-VF-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10240 2995; UNROLL-NO-VF-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]] 2996; UNROLL-NO-VF: middle.block: 2997; UNROLL-NO-VF-NEXT: [[BIN_RDX:%.*]] = add i32 [[TMP15]], [[TMP14]] 2998; UNROLL-NO-VF-NEXT: [[CMP_N:%.*]] = icmp eq i64 10240, 10240 2999; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]] 3000; UNROLL-NO-VF: scalar.ph: 3001; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi double [ [[J]], [[ENTRY:%.*]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ] 3002; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi double* [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[B]], [[ENTRY]] ] 3003; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ 10240, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 3004; UNROLL-NO-VF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ] 3005; UNROLL-NO-VF-NEXT: br label [[FOR_BODY:%.*]] 3006; UNROLL-NO-VF: for.cond.cleanup: 3007; UNROLL-NO-VF-NEXT: [[A_1_LCSSA:%.*]] = phi i32 [ [[A_1:%.*]], [[FOR_BODY]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ] 3008; UNROLL-NO-VF-NEXT: ret i32 [[A_1_LCSSA]] 3009; UNROLL-NO-VF: for.body: 3010; UNROLL-NO-VF-NEXT: [[B_ADDR_012:%.*]] = phi double* [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADD_PTR:%.*]], [[FOR_BODY]] ] 3011; UNROLL-NO-VF-NEXT: [[I_011:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[INC1:%.*]], [[FOR_BODY]] ] 3012; UNROLL-NO-VF-NEXT: [[A_010:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[A_1]], [[FOR_BODY]] ] 3013; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi double [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP17:%.*]], [[FOR_BODY]] ] 3014; UNROLL-NO-VF-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[B_ADDR_012]], i64 [[IDXPROM]] 3015; UNROLL-NO-VF-NEXT: [[TMP17]] = load double, double* [[ARRAYIDX]], align 8 3016; UNROLL-NO-VF-NEXT: [[MUL:%.*]] = fmul double [[SCALAR_RECUR]], [[TMP17]] 3017; UNROLL-NO-VF-NEXT: [[TOBOOL:%.*]] = fcmp une double [[MUL]], 0.000000e+00 3018; UNROLL-NO-VF-NEXT: [[INC:%.*]] = zext i1 [[TOBOOL]] to i32 3019; UNROLL-NO-VF-NEXT: [[A_1]] = add nsw i32 [[A_010]], [[INC]] 3020; UNROLL-NO-VF-NEXT: [[INC1]] = add nuw nsw i32 [[I_011]], 1 3021; UNROLL-NO-VF-NEXT: [[ADD_PTR]] = getelementptr inbounds double, double* [[B_ADDR_012]], i64 25 3022; UNROLL-NO-VF-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC1]], 10240 3023; UNROLL-NO-VF-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]] 3024; 3025; SINK-AFTER-LABEL: @PR33613( 3026; SINK-AFTER-NEXT: entry: 3027; SINK-AFTER-NEXT: [[IDXPROM:%.*]] = sext i32 [[D:%.*]] to i64 3028; SINK-AFTER-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 3029; SINK-AFTER: vector.ph: 3030; SINK-AFTER-NEXT: [[IND_END:%.*]] = getelementptr double, double* [[B:%.*]], i64 256000 3031; SINK-AFTER-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x double> poison, double [[J:%.*]], i32 3 3032; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 3033; SINK-AFTER: vector.body: 3034; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3035; SINK-AFTER-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP25:%.*]], [[VECTOR_BODY]] ] 3036; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x double> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ] 3037; SINK-AFTER-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 3038; SINK-AFTER-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 25 3039; SINK-AFTER-NEXT: [[NEXT_GEP:%.*]] = getelementptr double, double* [[B]], i64 [[TMP1]] 3040; SINK-AFTER-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1 3041; SINK-AFTER-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 25 3042; SINK-AFTER-NEXT: [[NEXT_GEP2:%.*]] = getelementptr double, double* [[B]], i64 [[TMP3]] 3043; SINK-AFTER-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 2 3044; SINK-AFTER-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 25 3045; SINK-AFTER-NEXT: [[NEXT_GEP3:%.*]] = getelementptr double, double* [[B]], i64 [[TMP5]] 3046; SINK-AFTER-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 3 3047; SINK-AFTER-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 25 3048; SINK-AFTER-NEXT: [[NEXT_GEP4:%.*]] = getelementptr double, double* [[B]], i64 [[TMP7]] 3049; SINK-AFTER-NEXT: [[OFFSET_IDX:%.*]] = trunc i64 [[INDEX]] to i32 3050; SINK-AFTER-NEXT: [[TMP8:%.*]] = add i32 [[OFFSET_IDX]], 0 3051; SINK-AFTER-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP]], i64 [[IDXPROM]] 3052; SINK-AFTER-NEXT: [[TMP10:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP2]], i64 [[IDXPROM]] 3053; SINK-AFTER-NEXT: [[TMP11:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP3]], i64 [[IDXPROM]] 3054; SINK-AFTER-NEXT: [[TMP12:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP4]], i64 [[IDXPROM]] 3055; SINK-AFTER-NEXT: [[TMP13:%.*]] = load double, double* [[TMP9]], align 8 3056; SINK-AFTER-NEXT: [[TMP14:%.*]] = load double, double* [[TMP10]], align 8 3057; SINK-AFTER-NEXT: [[TMP15:%.*]] = load double, double* [[TMP11]], align 8 3058; SINK-AFTER-NEXT: [[TMP16:%.*]] = load double, double* [[TMP12]], align 8 3059; SINK-AFTER-NEXT: [[TMP17:%.*]] = insertelement <4 x double> poison, double [[TMP13]], i32 0 3060; SINK-AFTER-NEXT: [[TMP18:%.*]] = insertelement <4 x double> [[TMP17]], double [[TMP14]], i32 1 3061; SINK-AFTER-NEXT: [[TMP19:%.*]] = insertelement <4 x double> [[TMP18]], double [[TMP15]], i32 2 3062; SINK-AFTER-NEXT: [[TMP20]] = insertelement <4 x double> [[TMP19]], double [[TMP16]], i32 3 3063; SINK-AFTER-NEXT: [[TMP21:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR]], <4 x double> [[TMP20]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 3064; SINK-AFTER-NEXT: [[TMP22:%.*]] = fmul <4 x double> [[TMP21]], [[TMP20]] 3065; SINK-AFTER-NEXT: [[TMP23:%.*]] = fcmp une <4 x double> [[TMP22]], zeroinitializer 3066; SINK-AFTER-NEXT: [[TMP24:%.*]] = zext <4 x i1> [[TMP23]] to <4 x i32> 3067; SINK-AFTER-NEXT: [[TMP25]] = add <4 x i32> [[VEC_PHI]], [[TMP24]] 3068; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 3069; SINK-AFTER-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10240 3070; SINK-AFTER-NEXT: br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]] 3071; SINK-AFTER: middle.block: 3072; SINK-AFTER-NEXT: [[TMP27:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP25]]) 3073; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 10240, 10240 3074; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x double> [[TMP20]], i32 3 3075; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x double> [[TMP20]], i32 2 3076; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]] 3077; SINK-AFTER: scalar.ph: 3078; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi double [ [[J]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 3079; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi double* [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[B]], [[ENTRY]] ] 3080; SINK-AFTER-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ 10240, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 3081; SINK-AFTER-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP27]], [[MIDDLE_BLOCK]] ] 3082; SINK-AFTER-NEXT: br label [[FOR_BODY:%.*]] 3083; SINK-AFTER: for.cond.cleanup: 3084; SINK-AFTER-NEXT: [[A_1_LCSSA:%.*]] = phi i32 [ [[A_1:%.*]], [[FOR_BODY]] ], [ [[TMP27]], [[MIDDLE_BLOCK]] ] 3085; SINK-AFTER-NEXT: ret i32 [[A_1_LCSSA]] 3086; SINK-AFTER: for.body: 3087; SINK-AFTER-NEXT: [[B_ADDR_012:%.*]] = phi double* [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADD_PTR:%.*]], [[FOR_BODY]] ] 3088; SINK-AFTER-NEXT: [[I_011:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[INC1:%.*]], [[FOR_BODY]] ] 3089; SINK-AFTER-NEXT: [[A_010:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[A_1]], [[FOR_BODY]] ] 3090; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi double [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP28:%.*]], [[FOR_BODY]] ] 3091; SINK-AFTER-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[B_ADDR_012]], i64 [[IDXPROM]] 3092; SINK-AFTER-NEXT: [[TMP28]] = load double, double* [[ARRAYIDX]], align 8 3093; SINK-AFTER-NEXT: [[MUL:%.*]] = fmul double [[SCALAR_RECUR]], [[TMP28]] 3094; SINK-AFTER-NEXT: [[TOBOOL:%.*]] = fcmp une double [[MUL]], 0.000000e+00 3095; SINK-AFTER-NEXT: [[INC:%.*]] = zext i1 [[TOBOOL]] to i32 3096; SINK-AFTER-NEXT: [[A_1]] = add nsw i32 [[A_010]], [[INC]] 3097; SINK-AFTER-NEXT: [[INC1]] = add nuw nsw i32 [[I_011]], 1 3098; SINK-AFTER-NEXT: [[ADD_PTR]] = getelementptr inbounds double, double* [[B_ADDR_012]], i64 25 3099; SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC1]], 10240 3100; SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]] 3101; 3102; NO-SINK-AFTER-LABEL: @PR33613( 3103; NO-SINK-AFTER-NEXT: entry: 3104; NO-SINK-AFTER-NEXT: [[IDXPROM:%.*]] = sext i32 [[D:%.*]] to i64 3105; NO-SINK-AFTER-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 3106; NO-SINK-AFTER: vector.ph: 3107; NO-SINK-AFTER-NEXT: [[IND_END:%.*]] = getelementptr double, double* [[B:%.*]], i64 256000 3108; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x double> poison, double [[J:%.*]], i32 3 3109; NO-SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 3110; NO-SINK-AFTER: vector.body: 3111; NO-SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3112; NO-SINK-AFTER-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP25:%.*]], [[VECTOR_BODY]] ] 3113; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x double> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ] 3114; NO-SINK-AFTER-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 3115; NO-SINK-AFTER-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 25 3116; NO-SINK-AFTER-NEXT: [[NEXT_GEP:%.*]] = getelementptr double, double* [[B]], i64 [[TMP1]] 3117; NO-SINK-AFTER-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1 3118; NO-SINK-AFTER-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 25 3119; NO-SINK-AFTER-NEXT: [[NEXT_GEP2:%.*]] = getelementptr double, double* [[B]], i64 [[TMP3]] 3120; NO-SINK-AFTER-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 2 3121; NO-SINK-AFTER-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 25 3122; NO-SINK-AFTER-NEXT: [[NEXT_GEP3:%.*]] = getelementptr double, double* [[B]], i64 [[TMP5]] 3123; NO-SINK-AFTER-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 3 3124; NO-SINK-AFTER-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 25 3125; NO-SINK-AFTER-NEXT: [[NEXT_GEP4:%.*]] = getelementptr double, double* [[B]], i64 [[TMP7]] 3126; NO-SINK-AFTER-NEXT: [[OFFSET_IDX:%.*]] = trunc i64 [[INDEX]] to i32 3127; NO-SINK-AFTER-NEXT: [[TMP8:%.*]] = add i32 [[OFFSET_IDX]], 0 3128; NO-SINK-AFTER-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP]], i64 [[IDXPROM]] 3129; NO-SINK-AFTER-NEXT: [[TMP10:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP2]], i64 [[IDXPROM]] 3130; NO-SINK-AFTER-NEXT: [[TMP11:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP3]], i64 [[IDXPROM]] 3131; NO-SINK-AFTER-NEXT: [[TMP12:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP4]], i64 [[IDXPROM]] 3132; NO-SINK-AFTER-NEXT: [[TMP13:%.*]] = load double, double* [[TMP9]], align 8 3133; NO-SINK-AFTER-NEXT: [[TMP14:%.*]] = load double, double* [[TMP10]], align 8 3134; NO-SINK-AFTER-NEXT: [[TMP15:%.*]] = load double, double* [[TMP11]], align 8 3135; NO-SINK-AFTER-NEXT: [[TMP16:%.*]] = load double, double* [[TMP12]], align 8 3136; NO-SINK-AFTER-NEXT: [[TMP17:%.*]] = insertelement <4 x double> poison, double [[TMP13]], i32 0 3137; NO-SINK-AFTER-NEXT: [[TMP18:%.*]] = insertelement <4 x double> [[TMP17]], double [[TMP14]], i32 1 3138; NO-SINK-AFTER-NEXT: [[TMP19:%.*]] = insertelement <4 x double> [[TMP18]], double [[TMP15]], i32 2 3139; NO-SINK-AFTER-NEXT: [[TMP20]] = insertelement <4 x double> [[TMP19]], double [[TMP16]], i32 3 3140; NO-SINK-AFTER-NEXT: [[TMP21:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR]], <4 x double> [[TMP20]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 3141; NO-SINK-AFTER-NEXT: [[TMP22:%.*]] = fmul <4 x double> [[TMP21]], [[TMP20]] 3142; NO-SINK-AFTER-NEXT: [[TMP23:%.*]] = fcmp une <4 x double> [[TMP22]], zeroinitializer 3143; NO-SINK-AFTER-NEXT: [[TMP24:%.*]] = zext <4 x i1> [[TMP23]] to <4 x i32> 3144; NO-SINK-AFTER-NEXT: [[TMP25]] = add <4 x i32> [[VEC_PHI]], [[TMP24]] 3145; NO-SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 3146; NO-SINK-AFTER-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10240 3147; NO-SINK-AFTER-NEXT: br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]] 3148; NO-SINK-AFTER: middle.block: 3149; NO-SINK-AFTER-NEXT: [[TMP27:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP25]]) 3150; NO-SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 10240, 10240 3151; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x double> [[TMP20]], i32 3 3152; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x double> [[TMP20]], i32 2 3153; NO-SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]] 3154; NO-SINK-AFTER: scalar.ph: 3155; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi double [ [[J]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 3156; NO-SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi double* [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[B]], [[ENTRY]] ] 3157; NO-SINK-AFTER-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ 10240, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 3158; NO-SINK-AFTER-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP27]], [[MIDDLE_BLOCK]] ] 3159; NO-SINK-AFTER-NEXT: br label [[FOR_BODY:%.*]] 3160; NO-SINK-AFTER: for.cond.cleanup: 3161; NO-SINK-AFTER-NEXT: [[A_1_LCSSA:%.*]] = phi i32 [ [[A_1:%.*]], [[FOR_BODY]] ], [ [[TMP27]], [[MIDDLE_BLOCK]] ] 3162; NO-SINK-AFTER-NEXT: ret i32 [[A_1_LCSSA]] 3163; NO-SINK-AFTER: for.body: 3164; NO-SINK-AFTER-NEXT: [[B_ADDR_012:%.*]] = phi double* [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADD_PTR:%.*]], [[FOR_BODY]] ] 3165; NO-SINK-AFTER-NEXT: [[I_011:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[INC1:%.*]], [[FOR_BODY]] ] 3166; NO-SINK-AFTER-NEXT: [[A_010:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[A_1]], [[FOR_BODY]] ] 3167; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi double [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP28:%.*]], [[FOR_BODY]] ] 3168; NO-SINK-AFTER-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[B_ADDR_012]], i64 [[IDXPROM]] 3169; NO-SINK-AFTER-NEXT: [[TMP28]] = load double, double* [[ARRAYIDX]], align 8 3170; NO-SINK-AFTER-NEXT: [[MUL:%.*]] = fmul double [[SCALAR_RECUR]], [[TMP28]] 3171; NO-SINK-AFTER-NEXT: [[TOBOOL:%.*]] = fcmp une double [[MUL]], 0.000000e+00 3172; NO-SINK-AFTER-NEXT: [[INC:%.*]] = zext i1 [[TOBOOL]] to i32 3173; NO-SINK-AFTER-NEXT: [[A_1]] = add nsw i32 [[A_010]], [[INC]] 3174; NO-SINK-AFTER-NEXT: [[INC1]] = add nuw nsw i32 [[I_011]], 1 3175; NO-SINK-AFTER-NEXT: [[ADD_PTR]] = getelementptr inbounds double, double* [[B_ADDR_012]], i64 25 3176; NO-SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC1]], 10240 3177; NO-SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]] 3178; 3179entry: 3180 %idxprom = sext i32 %d to i64 3181 br label %for.body 3182 3183for.cond.cleanup: 3184 %a.1.lcssa = phi i32 [ %a.1, %for.body ] 3185 ret i32 %a.1.lcssa 3186 3187for.body: 3188 %b.addr.012 = phi double* [ %b, %entry ], [ %add.ptr, %for.body ] 3189 %i.011 = phi i32 [ 0, %entry ], [ %inc1, %for.body ] 3190 %a.010 = phi i32 [ 0, %entry ], [ %a.1, %for.body ] 3191 %j.addr.09 = phi double [ %j, %entry ], [ %0, %for.body ] 3192 %arrayidx = getelementptr inbounds double, double* %b.addr.012, i64 %idxprom 3193 %0 = load double, double* %arrayidx, align 8 3194 %mul = fmul double %j.addr.09, %0 3195 %tobool = fcmp une double %mul, 0.000000e+00 3196 %inc = zext i1 %tobool to i32 3197 %a.1 = add nsw i32 %a.010, %inc 3198 %inc1 = add nuw nsw i32 %i.011, 1 3199 %add.ptr = getelementptr inbounds double, double* %b.addr.012, i64 25 3200 %exitcond = icmp eq i32 %inc1, 10240 3201 br i1 %exitcond, label %for.cond.cleanup, label %for.body 3202} 3203 3204; void sink_after(short *a, int n, int *b) { 3205; for(int i = 0; i < n; i++) 3206; b[i] = (a[i] * a[i + 1]); 3207; } 3208; 3209; Check that the sext sank after the load in the vector loop. 3210; 3211define void @sink_after(i16* %a, i32* %b, i64 %n) { 3212; CHECK-LABEL: @sink_after( 3213; CHECK-NEXT: entry: 3214; CHECK-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[A:%.*]], align 2 3215; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 3216; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 3217; CHECK: vector.memcheck: 3218; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B:%.*]], i64 [[N]] 3219; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i16, i16* [[A]], i64 1 3220; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 3221; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP0]] 3222; CHECK-NEXT: [[TMP1:%.*]] = bitcast i16* [[SCEVGEP5]] to i32* 3223; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt i32* [[TMP1]], [[B]] 3224; CHECK-NEXT: [[TMP2:%.*]] = bitcast i32* [[SCEVGEP]] to i16* 3225; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult i16* [[SCEVGEP3]], [[TMP2]] 3226; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 3227; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3228; CHECK: vector.ph: 3229; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4 3230; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i64 3 3231; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 3232; CHECK: vector.body: 3233; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3234; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ] 3235; CHECK-NEXT: [[TMP3:%.*]] = or i64 [[INDEX]], 1 3236; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP3]] 3237; CHECK-NEXT: [[TMP5:%.*]] = bitcast i16* [[TMP4]] to <4 x i16>* 3238; CHECK-NEXT: [[WIDE_LOAD]] = load <4 x i16>, <4 x i16>* [[TMP5]], align 2 3239; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 3240; CHECK-NEXT: [[TMP7:%.*]] = sext <4 x i16> [[TMP6]] to <4 x i32> 3241; CHECK-NEXT: [[TMP8:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i32> 3242; CHECK-NEXT: [[TMP9:%.*]] = mul nsw <4 x i32> [[TMP8]], [[TMP7]] 3243; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]] 3244; CHECK-NEXT: [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <4 x i32>* 3245; CHECK-NEXT: store <4 x i32> [[TMP9]], <4 x i32>* [[TMP11]], align 4 3246; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 3247; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 3248; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]] 3249; CHECK: middle.block: 3250; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] 3251; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 3 3252; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 3253; CHECK: scalar.ph: 3254; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 3255; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ] 3256; CHECK-NEXT: br label [[FOR_BODY:%.*]] 3257; CHECK: for.body: 3258; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP13:%.*]], [[FOR_BODY]] ] 3259; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 3260; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 3261; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 3262; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[INDVARS_IV_NEXT]] 3263; CHECK-NEXT: [[TMP13]] = load i16, i16* [[ARRAYIDX2]], align 2 3264; CHECK-NEXT: [[CONV3:%.*]] = sext i16 [[TMP13]] to i32 3265; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV3]], [[CONV]] 3266; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 3267; CHECK-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 3268; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 3269; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]] 3270; CHECK: for.end: 3271; CHECK-NEXT: ret void 3272; 3273; UNROLL-LABEL: @sink_after( 3274; UNROLL-NEXT: entry: 3275; UNROLL-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[A:%.*]], align 2 3276; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 8 3277; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 3278; UNROLL: vector.memcheck: 3279; UNROLL-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B:%.*]], i64 [[N]] 3280; UNROLL-NEXT: [[SCEVGEP3:%.*]] = getelementptr i16, i16* [[A]], i64 1 3281; UNROLL-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 3282; UNROLL-NEXT: [[SCEVGEP5:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP0]] 3283; UNROLL-NEXT: [[TMP1:%.*]] = bitcast i16* [[SCEVGEP5]] to i32* 3284; UNROLL-NEXT: [[BOUND0:%.*]] = icmp ugt i32* [[TMP1]], [[B]] 3285; UNROLL-NEXT: [[TMP2:%.*]] = bitcast i32* [[SCEVGEP]] to i16* 3286; UNROLL-NEXT: [[BOUND1:%.*]] = icmp ult i16* [[SCEVGEP3]], [[TMP2]] 3287; UNROLL-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 3288; UNROLL-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3289; UNROLL: vector.ph: 3290; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -8 3291; UNROLL-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i64 3 3292; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 3293; UNROLL: vector.body: 3294; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3295; UNROLL-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD7:%.*]], [[VECTOR_BODY]] ] 3296; UNROLL-NEXT: [[TMP3:%.*]] = or i64 [[INDEX]], 1 3297; UNROLL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP3]] 3298; UNROLL-NEXT: [[TMP5:%.*]] = bitcast i16* [[TMP4]] to <4 x i16>* 3299; UNROLL-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, <4 x i16>* [[TMP5]], align 2 3300; UNROLL-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, i16* [[TMP4]], i64 4 3301; UNROLL-NEXT: [[TMP7:%.*]] = bitcast i16* [[TMP6]] to <4 x i16>* 3302; UNROLL-NEXT: [[WIDE_LOAD7]] = load <4 x i16>, <4 x i16>* [[TMP7]], align 2 3303; UNROLL-NEXT: [[TMP8:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 3304; UNROLL-NEXT: [[TMP9:%.*]] = shufflevector <4 x i16> [[WIDE_LOAD]], <4 x i16> [[WIDE_LOAD7]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 3305; UNROLL-NEXT: [[TMP10:%.*]] = sext <4 x i16> [[TMP8]] to <4 x i32> 3306; UNROLL-NEXT: [[TMP11:%.*]] = sext <4 x i16> [[TMP9]] to <4 x i32> 3307; UNROLL-NEXT: [[TMP12:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i32> 3308; UNROLL-NEXT: [[TMP13:%.*]] = sext <4 x i16> [[WIDE_LOAD7]] to <4 x i32> 3309; UNROLL-NEXT: [[TMP14:%.*]] = mul nsw <4 x i32> [[TMP12]], [[TMP10]] 3310; UNROLL-NEXT: [[TMP15:%.*]] = mul nsw <4 x i32> [[TMP13]], [[TMP11]] 3311; UNROLL-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]] 3312; UNROLL-NEXT: [[TMP17:%.*]] = bitcast i32* [[TMP16]] to <4 x i32>* 3313; UNROLL-NEXT: store <4 x i32> [[TMP14]], <4 x i32>* [[TMP17]], align 4 3314; UNROLL-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[TMP16]], i64 4 3315; UNROLL-NEXT: [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <4 x i32>* 3316; UNROLL-NEXT: store <4 x i32> [[TMP15]], <4 x i32>* [[TMP19]], align 4 3317; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 3318; UNROLL-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 3319; UNROLL-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]] 3320; UNROLL: middle.block: 3321; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] 3322; UNROLL-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD7]], i64 3 3323; UNROLL-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 3324; UNROLL: scalar.ph: 3325; UNROLL-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 3326; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ] 3327; UNROLL-NEXT: br label [[FOR_BODY:%.*]] 3328; UNROLL: for.body: 3329; UNROLL-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP21:%.*]], [[FOR_BODY]] ] 3330; UNROLL-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 3331; UNROLL-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 3332; UNROLL-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 3333; UNROLL-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[INDVARS_IV_NEXT]] 3334; UNROLL-NEXT: [[TMP21]] = load i16, i16* [[ARRAYIDX2]], align 2 3335; UNROLL-NEXT: [[CONV3:%.*]] = sext i16 [[TMP21]] to i32 3336; UNROLL-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV3]], [[CONV]] 3337; UNROLL-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 3338; UNROLL-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 3339; UNROLL-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 3340; UNROLL-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]] 3341; UNROLL: for.end: 3342; UNROLL-NEXT: ret void 3343; 3344; UNROLL-NO-IC-LABEL: @sink_after( 3345; UNROLL-NO-IC-NEXT: entry: 3346; UNROLL-NO-IC-NEXT: [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8* 3347; UNROLL-NO-IC-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[A:%.*]], align 2 3348; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 8 3349; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 3350; UNROLL-NO-IC: vector.memcheck: 3351; UNROLL-NO-IC-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[N]] 3352; UNROLL-NO-IC-NEXT: [[SCEVGEP2:%.*]] = bitcast i32* [[SCEVGEP]] to i8* 3353; UNROLL-NO-IC-NEXT: [[SCEVGEP3:%.*]] = getelementptr i16, i16* [[A]], i64 1 3354; UNROLL-NO-IC-NEXT: [[SCEVGEP34:%.*]] = bitcast i16* [[SCEVGEP3]] to i8* 3355; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 3356; UNROLL-NO-IC-NEXT: [[SCEVGEP5:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP0]] 3357; UNROLL-NO-IC-NEXT: [[SCEVGEP56:%.*]] = bitcast i16* [[SCEVGEP5]] to i8* 3358; UNROLL-NO-IC-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[B1]], [[SCEVGEP56]] 3359; UNROLL-NO-IC-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP34]], [[SCEVGEP2]] 3360; UNROLL-NO-IC-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 3361; UNROLL-NO-IC-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3362; UNROLL-NO-IC: vector.ph: 3363; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8 3364; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 3365; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i32 3 3366; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 3367; UNROLL-NO-IC: vector.body: 3368; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3369; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD7:%.*]], [[VECTOR_BODY]] ] 3370; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 0 3371; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 4 3372; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP1]], 1 3373; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = add nuw nsw i64 [[TMP2]], 1 3374; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP3]] 3375; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP4]] 3376; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, i16* [[TMP5]], i32 0 3377; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = bitcast i16* [[TMP7]] to <4 x i16>* 3378; UNROLL-NO-IC-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, <4 x i16>* [[TMP8]], align 2 3379; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = getelementptr inbounds i16, i16* [[TMP5]], i32 4 3380; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = bitcast i16* [[TMP9]] to <4 x i16>* 3381; UNROLL-NO-IC-NEXT: [[WIDE_LOAD7]] = load <4 x i16>, <4 x i16>* [[TMP10]], align 2 3382; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 3383; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = shufflevector <4 x i16> [[WIDE_LOAD]], <4 x i16> [[WIDE_LOAD7]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 3384; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = sext <4 x i16> [[TMP11]] to <4 x i32> 3385; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = sext <4 x i16> [[TMP12]] to <4 x i32> 3386; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i32> 3387; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = sext <4 x i16> [[WIDE_LOAD7]] to <4 x i32> 3388; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = mul nsw <4 x i32> [[TMP15]], [[TMP13]] 3389; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = mul nsw <4 x i32> [[TMP16]], [[TMP14]] 3390; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP1]] 3391; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP2]] 3392; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[TMP19]], i32 0 3393; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = bitcast i32* [[TMP21]] to <4 x i32>* 3394; UNROLL-NO-IC-NEXT: store <4 x i32> [[TMP17]], <4 x i32>* [[TMP22]], align 4 3395; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[TMP19]], i32 4 3396; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <4 x i32>* 3397; UNROLL-NO-IC-NEXT: store <4 x i32> [[TMP18]], <4 x i32>* [[TMP24]], align 4 3398; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 3399; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 3400; UNROLL-NO-IC-NEXT: br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]] 3401; UNROLL-NO-IC: middle.block: 3402; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 3403; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD7]], i32 3 3404; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD7]], i32 2 3405; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 3406; UNROLL-NO-IC: scalar.ph: 3407; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 3408; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ] 3409; UNROLL-NO-IC-NEXT: br label [[FOR_BODY:%.*]] 3410; UNROLL-NO-IC: for.body: 3411; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP26:%.*]], [[FOR_BODY]] ] 3412; UNROLL-NO-IC-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 3413; UNROLL-NO-IC-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 3414; UNROLL-NO-IC-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 3415; UNROLL-NO-IC-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[INDVARS_IV_NEXT]] 3416; UNROLL-NO-IC-NEXT: [[TMP26]] = load i16, i16* [[ARRAYIDX2]], align 2 3417; UNROLL-NO-IC-NEXT: [[CONV3:%.*]] = sext i16 [[TMP26]] to i32 3418; UNROLL-NO-IC-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV3]], [[CONV]] 3419; UNROLL-NO-IC-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 3420; UNROLL-NO-IC-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 3421; UNROLL-NO-IC-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 3422; UNROLL-NO-IC-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]] 3423; UNROLL-NO-IC: for.end: 3424; UNROLL-NO-IC-NEXT: ret void 3425; 3426; UNROLL-NO-VF-LABEL: @sink_after( 3427; UNROLL-NO-VF-NEXT: entry: 3428; UNROLL-NO-VF-NEXT: [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8* 3429; UNROLL-NO-VF-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[A:%.*]], align 2 3430; UNROLL-NO-VF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 2 3431; UNROLL-NO-VF-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 3432; UNROLL-NO-VF: vector.memcheck: 3433; UNROLL-NO-VF-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[N]] 3434; UNROLL-NO-VF-NEXT: [[SCEVGEP2:%.*]] = bitcast i32* [[SCEVGEP]] to i8* 3435; UNROLL-NO-VF-NEXT: [[SCEVGEP3:%.*]] = getelementptr i16, i16* [[A]], i64 1 3436; UNROLL-NO-VF-NEXT: [[SCEVGEP34:%.*]] = bitcast i16* [[SCEVGEP3]] to i8* 3437; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 3438; UNROLL-NO-VF-NEXT: [[SCEVGEP5:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP0]] 3439; UNROLL-NO-VF-NEXT: [[SCEVGEP56:%.*]] = bitcast i16* [[SCEVGEP5]] to i8* 3440; UNROLL-NO-VF-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[B1]], [[SCEVGEP56]] 3441; UNROLL-NO-VF-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP34]], [[SCEVGEP2]] 3442; UNROLL-NO-VF-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 3443; UNROLL-NO-VF-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3444; UNROLL-NO-VF: vector.ph: 3445; UNROLL-NO-VF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2 3446; UNROLL-NO-VF-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 3447; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 3448; UNROLL-NO-VF: vector.body: 3449; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3450; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ] 3451; UNROLL-NO-VF-NEXT: [[INDUCTION:%.*]] = add i64 [[INDEX]], 0 3452; UNROLL-NO-VF-NEXT: [[INDUCTION7:%.*]] = add i64 [[INDEX]], 1 3453; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[INDUCTION]], 1 3454; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[INDUCTION7]], 1 3455; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP1]] 3456; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP2]] 3457; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = load i16, i16* [[TMP3]], align 2 3458; UNROLL-NO-VF-NEXT: [[TMP6]] = load i16, i16* [[TMP4]], align 2 3459; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = sext i16 [[VECTOR_RECUR]] to i32 3460; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = sext i16 [[TMP5]] to i32 3461; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = sext i16 [[TMP5]] to i32 3462; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = sext i16 [[TMP6]] to i32 3463; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = mul nsw i32 [[TMP9]], [[TMP7]] 3464; UNROLL-NO-VF-NEXT: [[TMP12:%.*]] = mul nsw i32 [[TMP10]], [[TMP8]] 3465; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDUCTION]] 3466; UNROLL-NO-VF-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDUCTION7]] 3467; UNROLL-NO-VF-NEXT: store i32 [[TMP11]], i32* [[TMP13]], align 4 3468; UNROLL-NO-VF-NEXT: store i32 [[TMP12]], i32* [[TMP14]], align 4 3469; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 3470; UNROLL-NO-VF-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 3471; UNROLL-NO-VF-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]] 3472; UNROLL-NO-VF: middle.block: 3473; UNROLL-NO-VF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 3474; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 3475; UNROLL-NO-VF: scalar.ph: 3476; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ] 3477; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ] 3478; UNROLL-NO-VF-NEXT: br label [[FOR_BODY:%.*]] 3479; UNROLL-NO-VF: for.body: 3480; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP16:%.*]], [[FOR_BODY]] ] 3481; UNROLL-NO-VF-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 3482; UNROLL-NO-VF-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 3483; UNROLL-NO-VF-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 3484; UNROLL-NO-VF-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[INDVARS_IV_NEXT]] 3485; UNROLL-NO-VF-NEXT: [[TMP16]] = load i16, i16* [[ARRAYIDX2]], align 2 3486; UNROLL-NO-VF-NEXT: [[CONV3:%.*]] = sext i16 [[TMP16]] to i32 3487; UNROLL-NO-VF-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV3]], [[CONV]] 3488; UNROLL-NO-VF-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 3489; UNROLL-NO-VF-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 3490; UNROLL-NO-VF-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 3491; UNROLL-NO-VF-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]] 3492; UNROLL-NO-VF: for.end: 3493; UNROLL-NO-VF-NEXT: ret void 3494; 3495; SINK-AFTER-LABEL: @sink_after( 3496; SINK-AFTER-NEXT: entry: 3497; SINK-AFTER-NEXT: [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8* 3498; SINK-AFTER-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[A:%.*]], align 2 3499; SINK-AFTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 3500; SINK-AFTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 3501; SINK-AFTER: vector.memcheck: 3502; SINK-AFTER-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[N]] 3503; SINK-AFTER-NEXT: [[SCEVGEP2:%.*]] = bitcast i32* [[SCEVGEP]] to i8* 3504; SINK-AFTER-NEXT: [[SCEVGEP3:%.*]] = getelementptr i16, i16* [[A]], i64 1 3505; SINK-AFTER-NEXT: [[SCEVGEP34:%.*]] = bitcast i16* [[SCEVGEP3]] to i8* 3506; SINK-AFTER-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 3507; SINK-AFTER-NEXT: [[SCEVGEP5:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP0]] 3508; SINK-AFTER-NEXT: [[SCEVGEP56:%.*]] = bitcast i16* [[SCEVGEP5]] to i8* 3509; SINK-AFTER-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[B1]], [[SCEVGEP56]] 3510; SINK-AFTER-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP34]], [[SCEVGEP2]] 3511; SINK-AFTER-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 3512; SINK-AFTER-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3513; SINK-AFTER: vector.ph: 3514; SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4 3515; SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 3516; SINK-AFTER-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i32 3 3517; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 3518; SINK-AFTER: vector.body: 3519; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3520; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ] 3521; SINK-AFTER-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 0 3522; SINK-AFTER-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 3523; SINK-AFTER-NEXT: [[TMP3:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP2]] 3524; SINK-AFTER-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, i16* [[TMP3]], i32 0 3525; SINK-AFTER-NEXT: [[TMP5:%.*]] = bitcast i16* [[TMP4]] to <4 x i16>* 3526; SINK-AFTER-NEXT: [[WIDE_LOAD]] = load <4 x i16>, <4 x i16>* [[TMP5]], align 2 3527; SINK-AFTER-NEXT: [[TMP6:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 3528; SINK-AFTER-NEXT: [[TMP7:%.*]] = sext <4 x i16> [[TMP6]] to <4 x i32> 3529; SINK-AFTER-NEXT: [[TMP8:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i32> 3530; SINK-AFTER-NEXT: [[TMP9:%.*]] = mul nsw <4 x i32> [[TMP8]], [[TMP7]] 3531; SINK-AFTER-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP1]] 3532; SINK-AFTER-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP10]], i32 0 3533; SINK-AFTER-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>* 3534; SINK-AFTER-NEXT: store <4 x i32> [[TMP9]], <4 x i32>* [[TMP12]], align 4 3535; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 3536; SINK-AFTER-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 3537; SINK-AFTER-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]] 3538; SINK-AFTER: middle.block: 3539; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 3540; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3 3541; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2 3542; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 3543; SINK-AFTER: scalar.ph: 3544; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 3545; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ] 3546; SINK-AFTER-NEXT: br label [[FOR_BODY:%.*]] 3547; SINK-AFTER: for.body: 3548; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP14:%.*]], [[FOR_BODY]] ] 3549; SINK-AFTER-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 3550; SINK-AFTER-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 3551; SINK-AFTER-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 3552; SINK-AFTER-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[INDVARS_IV_NEXT]] 3553; SINK-AFTER-NEXT: [[TMP14]] = load i16, i16* [[ARRAYIDX2]], align 2 3554; SINK-AFTER-NEXT: [[CONV3:%.*]] = sext i16 [[TMP14]] to i32 3555; SINK-AFTER-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV3]], [[CONV]] 3556; SINK-AFTER-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 3557; SINK-AFTER-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 3558; SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 3559; SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]] 3560; SINK-AFTER: for.end: 3561; SINK-AFTER-NEXT: ret void 3562; 3563; NO-SINK-AFTER-LABEL: @sink_after( 3564; NO-SINK-AFTER-NEXT: entry: 3565; NO-SINK-AFTER-NEXT: [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8* 3566; NO-SINK-AFTER-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[A:%.*]], align 2 3567; NO-SINK-AFTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 3568; NO-SINK-AFTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 3569; NO-SINK-AFTER: vector.memcheck: 3570; NO-SINK-AFTER-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[N]] 3571; NO-SINK-AFTER-NEXT: [[SCEVGEP2:%.*]] = bitcast i32* [[SCEVGEP]] to i8* 3572; NO-SINK-AFTER-NEXT: [[SCEVGEP3:%.*]] = getelementptr i16, i16* [[A]], i64 1 3573; NO-SINK-AFTER-NEXT: [[SCEVGEP34:%.*]] = bitcast i16* [[SCEVGEP3]] to i8* 3574; NO-SINK-AFTER-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 3575; NO-SINK-AFTER-NEXT: [[SCEVGEP5:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP0]] 3576; NO-SINK-AFTER-NEXT: [[SCEVGEP56:%.*]] = bitcast i16* [[SCEVGEP5]] to i8* 3577; NO-SINK-AFTER-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[B1]], [[SCEVGEP56]] 3578; NO-SINK-AFTER-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP34]], [[SCEVGEP2]] 3579; NO-SINK-AFTER-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 3580; NO-SINK-AFTER-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3581; NO-SINK-AFTER: vector.ph: 3582; NO-SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4 3583; NO-SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 3584; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i32 3 3585; NO-SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 3586; NO-SINK-AFTER: vector.body: 3587; NO-SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3588; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ] 3589; NO-SINK-AFTER-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 0 3590; NO-SINK-AFTER-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 3591; NO-SINK-AFTER-NEXT: [[TMP3:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP2]] 3592; NO-SINK-AFTER-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, i16* [[TMP3]], i32 0 3593; NO-SINK-AFTER-NEXT: [[TMP5:%.*]] = bitcast i16* [[TMP4]] to <4 x i16>* 3594; NO-SINK-AFTER-NEXT: [[WIDE_LOAD]] = load <4 x i16>, <4 x i16>* [[TMP5]], align 2 3595; NO-SINK-AFTER-NEXT: [[TMP6:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 3596; NO-SINK-AFTER-NEXT: [[TMP7:%.*]] = sext <4 x i16> [[TMP6]] to <4 x i32> 3597; NO-SINK-AFTER-NEXT: [[TMP8:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i32> 3598; NO-SINK-AFTER-NEXT: [[TMP9:%.*]] = mul nsw <4 x i32> [[TMP8]], [[TMP7]] 3599; NO-SINK-AFTER-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP1]] 3600; NO-SINK-AFTER-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP10]], i32 0 3601; NO-SINK-AFTER-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>* 3602; NO-SINK-AFTER-NEXT: store <4 x i32> [[TMP9]], <4 x i32>* [[TMP12]], align 4 3603; NO-SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 3604; NO-SINK-AFTER-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 3605; NO-SINK-AFTER-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]] 3606; NO-SINK-AFTER: middle.block: 3607; NO-SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 3608; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3 3609; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2 3610; NO-SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 3611; NO-SINK-AFTER: scalar.ph: 3612; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 3613; NO-SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ] 3614; NO-SINK-AFTER-NEXT: br label [[FOR_BODY:%.*]] 3615; NO-SINK-AFTER: for.body: 3616; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP14:%.*]], [[FOR_BODY]] ] 3617; NO-SINK-AFTER-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 3618; NO-SINK-AFTER-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 3619; NO-SINK-AFTER-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 3620; NO-SINK-AFTER-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[INDVARS_IV_NEXT]] 3621; NO-SINK-AFTER-NEXT: [[TMP14]] = load i16, i16* [[ARRAYIDX2]], align 2 3622; NO-SINK-AFTER-NEXT: [[CONV3:%.*]] = sext i16 [[TMP14]] to i32 3623; NO-SINK-AFTER-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV3]], [[CONV]] 3624; NO-SINK-AFTER-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 3625; NO-SINK-AFTER-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 3626; NO-SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 3627; NO-SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]] 3628; NO-SINK-AFTER: for.end: 3629; NO-SINK-AFTER-NEXT: ret void 3630; 3631entry: 3632 %.pre = load i16, i16* %a 3633 br label %for.body 3634 3635for.body: 3636 %0 = phi i16 [ %.pre, %entry ], [ %1, %for.body ] 3637 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 3638 %conv = sext i16 %0 to i32 3639 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 3640 %arrayidx2 = getelementptr inbounds i16, i16* %a, i64 %indvars.iv.next 3641 %1 = load i16, i16* %arrayidx2 3642 %conv3 = sext i16 %1 to i32 3643 %mul = mul nsw i32 %conv3, %conv 3644 %arrayidx5 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv 3645 store i32 %mul, i32* %arrayidx5 3646 %exitcond = icmp eq i64 %indvars.iv.next, %n 3647 br i1 %exitcond, label %for.end, label %for.body 3648 3649for.end: 3650 ret void 3651} 3652 3653; PR34711: given three consecutive instructions such that the first will be 3654; widened, the second is a cast that will be widened and needs to sink after the 3655; third, and the third is a first-order-recurring load that will be replicated 3656; instead of widened. Although the cast and the first instruction will both be 3657; widened, and are originally adjacent to each other, make sure the replicated 3658; load ends up appearing between them. 3659; 3660; void PR34711(short[2] *a, int *b, int *c, int n) { 3661; for(int i = 0; i < n; i++) { 3662; c[i] = 7; 3663; b[i] = (a[i][0] * a[i][1]); 3664; } 3665; } 3666; 3667; Check that the sext sank after the load in the vector loop. 3668; 3669define void @PR34711([2 x i16]* %a, i32* %b, i32* %c, i64 %n) { 3670; CHECK-LABEL: @PR34711( 3671; CHECK-NEXT: entry: 3672; CHECK-NEXT: [[PRE_INDEX:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A:%.*]], i64 0, i64 0 3673; CHECK-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[PRE_INDEX]], align 2 3674; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 3675; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 3676; CHECK: vector.memcheck: 3677; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[C:%.*]], i64 [[N]] 3678; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i32, i32* [[B:%.*]], i64 [[N]] 3679; CHECK-NEXT: [[SCEVGEP6:%.*]] = getelementptr [2 x i16], [2 x i16]* [[A]], i64 0, i64 1 3680; CHECK-NEXT: [[SCEVGEP8:%.*]] = getelementptr [2 x i16], [2 x i16]* [[A]], i64 [[N]], i64 0 3681; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt i32* [[SCEVGEP4]], [[C]] 3682; CHECK-NEXT: [[BOUND1:%.*]] = icmp ugt i32* [[SCEVGEP]], [[B]] 3683; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 3684; CHECK-NEXT: [[TMP0:%.*]] = bitcast i16* [[SCEVGEP8]] to i32* 3685; CHECK-NEXT: [[BOUND010:%.*]] = icmp ugt i32* [[TMP0]], [[C]] 3686; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32* [[SCEVGEP]] to i16* 3687; CHECK-NEXT: [[BOUND111:%.*]] = icmp ult i16* [[SCEVGEP6]], [[TMP1]] 3688; CHECK-NEXT: [[FOUND_CONFLICT12:%.*]] = and i1 [[BOUND010]], [[BOUND111]] 3689; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT12]] 3690; CHECK-NEXT: [[TMP2:%.*]] = bitcast i16* [[SCEVGEP8]] to i32* 3691; CHECK-NEXT: [[BOUND013:%.*]] = icmp ugt i32* [[TMP2]], [[B]] 3692; CHECK-NEXT: [[TMP3:%.*]] = bitcast i32* [[SCEVGEP4]] to i16* 3693; CHECK-NEXT: [[BOUND114:%.*]] = icmp ult i16* [[SCEVGEP6]], [[TMP3]] 3694; CHECK-NEXT: [[FOUND_CONFLICT15:%.*]] = and i1 [[BOUND013]], [[BOUND114]] 3695; CHECK-NEXT: [[CONFLICT_RDX16:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT15]] 3696; CHECK-NEXT: br i1 [[CONFLICT_RDX16]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3697; CHECK: vector.ph: 3698; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4 3699; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i64 3 3700; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 3701; CHECK: vector.body: 3702; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3703; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ] 3704; CHECK-NEXT: [[TMP4:%.*]] = or i64 [[INDEX]], 1 3705; CHECK-NEXT: [[TMP5:%.*]] = or i64 [[INDEX]], 2 3706; CHECK-NEXT: [[TMP6:%.*]] = or i64 [[INDEX]], 3 3707; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDEX]] 3708; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[INDEX]], i64 1 3709; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP4]], i64 1 3710; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP5]], i64 1 3711; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP6]], i64 1 3712; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP7]] to <4 x i32>* 3713; CHECK-NEXT: store <4 x i32> <i32 7, i32 7, i32 7, i32 7>, <4 x i32>* [[TMP12]], align 4 3714; CHECK-NEXT: [[TMP13:%.*]] = load i16, i16* [[TMP8]], align 2 3715; CHECK-NEXT: [[TMP14:%.*]] = load i16, i16* [[TMP9]], align 2 3716; CHECK-NEXT: [[TMP15:%.*]] = load i16, i16* [[TMP10]], align 2 3717; CHECK-NEXT: [[TMP16:%.*]] = load i16, i16* [[TMP11]], align 2 3718; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i16> poison, i16 [[TMP13]], i64 0 3719; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i16> [[TMP17]], i16 [[TMP14]], i64 1 3720; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i16> [[TMP18]], i16 [[TMP15]], i64 2 3721; CHECK-NEXT: [[TMP20]] = insertelement <4 x i16> [[TMP19]], i16 [[TMP16]], i64 3 3722; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP19]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 3723; CHECK-NEXT: [[TMP22:%.*]] = sext <4 x i16> [[TMP21]] to <4 x i32> 3724; CHECK-NEXT: [[TMP23:%.*]] = sext <4 x i16> [[TMP20]] to <4 x i32> 3725; CHECK-NEXT: [[TMP24:%.*]] = mul nsw <4 x i32> [[TMP23]], [[TMP22]] 3726; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]] 3727; CHECK-NEXT: [[TMP26:%.*]] = bitcast i32* [[TMP25]] to <4 x i32>* 3728; CHECK-NEXT: store <4 x i32> [[TMP24]], <4 x i32>* [[TMP26]], align 4 3729; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 3730; CHECK-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 3731; CHECK-NEXT: br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]] 3732; CHECK: middle.block: 3733; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] 3734; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 3735; CHECK: scalar.ph: 3736; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[TMP16]], [[MIDDLE_BLOCK]] ] 3737; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ] 3738; CHECK-NEXT: br label [[FOR_BODY:%.*]] 3739; CHECK: for.body: 3740; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP28:%.*]], [[FOR_BODY]] ] 3741; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 3742; CHECK-NEXT: [[ARRAYCIDX:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDVARS_IV]] 3743; CHECK-NEXT: [[CUR_INDEX:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[INDVARS_IV]], i64 1 3744; CHECK-NEXT: store i32 7, i32* [[ARRAYCIDX]], align 4 3745; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 3746; CHECK-NEXT: [[TMP28]] = load i16, i16* [[CUR_INDEX]], align 2 3747; CHECK-NEXT: [[CONV3:%.*]] = sext i16 [[TMP28]] to i32 3748; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV3]], [[CONV]] 3749; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 3750; CHECK-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 3751; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 3752; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 3753; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]] 3754; CHECK: for.end: 3755; CHECK-NEXT: ret void 3756; 3757; UNROLL-LABEL: @PR34711( 3758; UNROLL-NEXT: entry: 3759; UNROLL-NEXT: [[PRE_INDEX:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A:%.*]], i64 0, i64 0 3760; UNROLL-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[PRE_INDEX]], align 2 3761; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 8 3762; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 3763; UNROLL: vector.memcheck: 3764; UNROLL-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[C:%.*]], i64 [[N]] 3765; UNROLL-NEXT: [[SCEVGEP4:%.*]] = getelementptr i32, i32* [[B:%.*]], i64 [[N]] 3766; UNROLL-NEXT: [[SCEVGEP6:%.*]] = getelementptr [2 x i16], [2 x i16]* [[A]], i64 0, i64 1 3767; UNROLL-NEXT: [[SCEVGEP8:%.*]] = getelementptr [2 x i16], [2 x i16]* [[A]], i64 [[N]], i64 0 3768; UNROLL-NEXT: [[BOUND0:%.*]] = icmp ugt i32* [[SCEVGEP4]], [[C]] 3769; UNROLL-NEXT: [[BOUND1:%.*]] = icmp ugt i32* [[SCEVGEP]], [[B]] 3770; UNROLL-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 3771; UNROLL-NEXT: [[TMP0:%.*]] = bitcast i16* [[SCEVGEP8]] to i32* 3772; UNROLL-NEXT: [[BOUND010:%.*]] = icmp ugt i32* [[TMP0]], [[C]] 3773; UNROLL-NEXT: [[TMP1:%.*]] = bitcast i32* [[SCEVGEP]] to i16* 3774; UNROLL-NEXT: [[BOUND111:%.*]] = icmp ult i16* [[SCEVGEP6]], [[TMP1]] 3775; UNROLL-NEXT: [[FOUND_CONFLICT12:%.*]] = and i1 [[BOUND010]], [[BOUND111]] 3776; UNROLL-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT12]] 3777; UNROLL-NEXT: [[TMP2:%.*]] = bitcast i16* [[SCEVGEP8]] to i32* 3778; UNROLL-NEXT: [[BOUND013:%.*]] = icmp ugt i32* [[TMP2]], [[B]] 3779; UNROLL-NEXT: [[TMP3:%.*]] = bitcast i32* [[SCEVGEP4]] to i16* 3780; UNROLL-NEXT: [[BOUND114:%.*]] = icmp ult i16* [[SCEVGEP6]], [[TMP3]] 3781; UNROLL-NEXT: [[FOUND_CONFLICT15:%.*]] = and i1 [[BOUND013]], [[BOUND114]] 3782; UNROLL-NEXT: [[CONFLICT_RDX16:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT15]] 3783; UNROLL-NEXT: br i1 [[CONFLICT_RDX16]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3784; UNROLL: vector.ph: 3785; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -8 3786; UNROLL-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i64 3 3787; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 3788; UNROLL: vector.body: 3789; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3790; UNROLL-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP38:%.*]], [[VECTOR_BODY]] ] 3791; UNROLL-NEXT: [[TMP4:%.*]] = or i64 [[INDEX]], 1 3792; UNROLL-NEXT: [[TMP5:%.*]] = or i64 [[INDEX]], 2 3793; UNROLL-NEXT: [[TMP6:%.*]] = or i64 [[INDEX]], 3 3794; UNROLL-NEXT: [[TMP7:%.*]] = or i64 [[INDEX]], 4 3795; UNROLL-NEXT: [[TMP8:%.*]] = or i64 [[INDEX]], 5 3796; UNROLL-NEXT: [[TMP9:%.*]] = or i64 [[INDEX]], 6 3797; UNROLL-NEXT: [[TMP10:%.*]] = or i64 [[INDEX]], 7 3798; UNROLL-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDEX]] 3799; UNROLL-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[INDEX]], i64 1 3800; UNROLL-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP4]], i64 1 3801; UNROLL-NEXT: [[TMP14:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP5]], i64 1 3802; UNROLL-NEXT: [[TMP15:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP6]], i64 1 3803; UNROLL-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP7]], i64 1 3804; UNROLL-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP8]], i64 1 3805; UNROLL-NEXT: [[TMP18:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP9]], i64 1 3806; UNROLL-NEXT: [[TMP19:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP10]], i64 1 3807; UNROLL-NEXT: [[TMP20:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>* 3808; UNROLL-NEXT: store <4 x i32> <i32 7, i32 7, i32 7, i32 7>, <4 x i32>* [[TMP20]], align 4 3809; UNROLL-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[TMP11]], i64 4 3810; UNROLL-NEXT: [[TMP22:%.*]] = bitcast i32* [[TMP21]] to <4 x i32>* 3811; UNROLL-NEXT: store <4 x i32> <i32 7, i32 7, i32 7, i32 7>, <4 x i32>* [[TMP22]], align 4 3812; UNROLL-NEXT: [[TMP23:%.*]] = load i16, i16* [[TMP12]], align 2 3813; UNROLL-NEXT: [[TMP24:%.*]] = load i16, i16* [[TMP13]], align 2 3814; UNROLL-NEXT: [[TMP25:%.*]] = load i16, i16* [[TMP14]], align 2 3815; UNROLL-NEXT: [[TMP26:%.*]] = load i16, i16* [[TMP15]], align 2 3816; UNROLL-NEXT: [[TMP27:%.*]] = insertelement <4 x i16> poison, i16 [[TMP23]], i64 0 3817; UNROLL-NEXT: [[TMP28:%.*]] = insertelement <4 x i16> [[TMP27]], i16 [[TMP24]], i64 1 3818; UNROLL-NEXT: [[TMP29:%.*]] = insertelement <4 x i16> [[TMP28]], i16 [[TMP25]], i64 2 3819; UNROLL-NEXT: [[TMP30:%.*]] = insertelement <4 x i16> [[TMP29]], i16 [[TMP26]], i64 3 3820; UNROLL-NEXT: [[TMP31:%.*]] = load i16, i16* [[TMP16]], align 2 3821; UNROLL-NEXT: [[TMP32:%.*]] = load i16, i16* [[TMP17]], align 2 3822; UNROLL-NEXT: [[TMP33:%.*]] = load i16, i16* [[TMP18]], align 2 3823; UNROLL-NEXT: [[TMP34:%.*]] = load i16, i16* [[TMP19]], align 2 3824; UNROLL-NEXT: [[TMP35:%.*]] = insertelement <4 x i16> poison, i16 [[TMP31]], i64 0 3825; UNROLL-NEXT: [[TMP36:%.*]] = insertelement <4 x i16> [[TMP35]], i16 [[TMP32]], i64 1 3826; UNROLL-NEXT: [[TMP37:%.*]] = insertelement <4 x i16> [[TMP36]], i16 [[TMP33]], i64 2 3827; UNROLL-NEXT: [[TMP38]] = insertelement <4 x i16> [[TMP37]], i16 [[TMP34]], i64 3 3828; UNROLL-NEXT: [[TMP39:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP29]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 3829; UNROLL-NEXT: [[TMP40:%.*]] = shufflevector <4 x i16> [[TMP30]], <4 x i16> [[TMP37]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 3830; UNROLL-NEXT: [[TMP41:%.*]] = sext <4 x i16> [[TMP39]] to <4 x i32> 3831; UNROLL-NEXT: [[TMP42:%.*]] = sext <4 x i16> [[TMP40]] to <4 x i32> 3832; UNROLL-NEXT: [[TMP43:%.*]] = sext <4 x i16> [[TMP30]] to <4 x i32> 3833; UNROLL-NEXT: [[TMP44:%.*]] = sext <4 x i16> [[TMP38]] to <4 x i32> 3834; UNROLL-NEXT: [[TMP45:%.*]] = mul nsw <4 x i32> [[TMP43]], [[TMP41]] 3835; UNROLL-NEXT: [[TMP46:%.*]] = mul nsw <4 x i32> [[TMP44]], [[TMP42]] 3836; UNROLL-NEXT: [[TMP47:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]] 3837; UNROLL-NEXT: [[TMP48:%.*]] = bitcast i32* [[TMP47]] to <4 x i32>* 3838; UNROLL-NEXT: store <4 x i32> [[TMP45]], <4 x i32>* [[TMP48]], align 4 3839; UNROLL-NEXT: [[TMP49:%.*]] = getelementptr inbounds i32, i32* [[TMP47]], i64 4 3840; UNROLL-NEXT: [[TMP50:%.*]] = bitcast i32* [[TMP49]] to <4 x i32>* 3841; UNROLL-NEXT: store <4 x i32> [[TMP46]], <4 x i32>* [[TMP50]], align 4 3842; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 3843; UNROLL-NEXT: [[TMP51:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 3844; UNROLL-NEXT: br i1 [[TMP51]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]] 3845; UNROLL: middle.block: 3846; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] 3847; UNROLL-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 3848; UNROLL: scalar.ph: 3849; UNROLL-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[TMP34]], [[MIDDLE_BLOCK]] ] 3850; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ] 3851; UNROLL-NEXT: br label [[FOR_BODY:%.*]] 3852; UNROLL: for.body: 3853; UNROLL-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP52:%.*]], [[FOR_BODY]] ] 3854; UNROLL-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 3855; UNROLL-NEXT: [[ARRAYCIDX:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDVARS_IV]] 3856; UNROLL-NEXT: [[CUR_INDEX:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[INDVARS_IV]], i64 1 3857; UNROLL-NEXT: store i32 7, i32* [[ARRAYCIDX]], align 4 3858; UNROLL-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 3859; UNROLL-NEXT: [[TMP52]] = load i16, i16* [[CUR_INDEX]], align 2 3860; UNROLL-NEXT: [[CONV3:%.*]] = sext i16 [[TMP52]] to i32 3861; UNROLL-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV3]], [[CONV]] 3862; UNROLL-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 3863; UNROLL-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 3864; UNROLL-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 3865; UNROLL-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 3866; UNROLL-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]] 3867; UNROLL: for.end: 3868; UNROLL-NEXT: ret void 3869; 3870; UNROLL-NO-IC-LABEL: @PR34711( 3871; UNROLL-NO-IC-NEXT: entry: 3872; UNROLL-NO-IC-NEXT: [[C1:%.*]] = bitcast i32* [[C:%.*]] to i8* 3873; UNROLL-NO-IC-NEXT: [[B3:%.*]] = bitcast i32* [[B:%.*]] to i8* 3874; UNROLL-NO-IC-NEXT: [[PRE_INDEX:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A:%.*]], i64 0, i64 0 3875; UNROLL-NO-IC-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[PRE_INDEX]], align 2 3876; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 8 3877; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 3878; UNROLL-NO-IC: vector.memcheck: 3879; UNROLL-NO-IC-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[C]], i64 [[N]] 3880; UNROLL-NO-IC-NEXT: [[SCEVGEP2:%.*]] = bitcast i32* [[SCEVGEP]] to i8* 3881; UNROLL-NO-IC-NEXT: [[SCEVGEP4:%.*]] = getelementptr i32, i32* [[B]], i64 [[N]] 3882; UNROLL-NO-IC-NEXT: [[SCEVGEP45:%.*]] = bitcast i32* [[SCEVGEP4]] to i8* 3883; UNROLL-NO-IC-NEXT: [[SCEVGEP6:%.*]] = getelementptr [2 x i16], [2 x i16]* [[A]], i64 0, i64 1 3884; UNROLL-NO-IC-NEXT: [[SCEVGEP67:%.*]] = bitcast i16* [[SCEVGEP6]] to i8* 3885; UNROLL-NO-IC-NEXT: [[SCEVGEP8:%.*]] = getelementptr [2 x i16], [2 x i16]* [[A]], i64 [[N]], i64 0 3886; UNROLL-NO-IC-NEXT: [[SCEVGEP89:%.*]] = bitcast i16* [[SCEVGEP8]] to i8* 3887; UNROLL-NO-IC-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[C1]], [[SCEVGEP45]] 3888; UNROLL-NO-IC-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[B3]], [[SCEVGEP2]] 3889; UNROLL-NO-IC-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 3890; UNROLL-NO-IC-NEXT: [[BOUND010:%.*]] = icmp ult i8* [[C1]], [[SCEVGEP89]] 3891; UNROLL-NO-IC-NEXT: [[BOUND111:%.*]] = icmp ult i8* [[SCEVGEP67]], [[SCEVGEP2]] 3892; UNROLL-NO-IC-NEXT: [[FOUND_CONFLICT12:%.*]] = and i1 [[BOUND010]], [[BOUND111]] 3893; UNROLL-NO-IC-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT12]] 3894; UNROLL-NO-IC-NEXT: [[BOUND013:%.*]] = icmp ult i8* [[B3]], [[SCEVGEP89]] 3895; UNROLL-NO-IC-NEXT: [[BOUND114:%.*]] = icmp ult i8* [[SCEVGEP67]], [[SCEVGEP45]] 3896; UNROLL-NO-IC-NEXT: [[FOUND_CONFLICT15:%.*]] = and i1 [[BOUND013]], [[BOUND114]] 3897; UNROLL-NO-IC-NEXT: [[CONFLICT_RDX16:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT15]] 3898; UNROLL-NO-IC-NEXT: br i1 [[CONFLICT_RDX16]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3899; UNROLL-NO-IC: vector.ph: 3900; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8 3901; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 3902; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i32 3 3903; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 3904; UNROLL-NO-IC: vector.body: 3905; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3906; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP37:%.*]], [[VECTOR_BODY]] ] 3907; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 3908; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1 3909; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2 3910; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3 3911; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 4 3912; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 5 3913; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 6 3914; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 7 3915; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[TMP0]] 3916; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[TMP4]] 3917; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP0]], i64 1 3918; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP1]], i64 1 3919; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP2]], i64 1 3920; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP3]], i64 1 3921; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP4]], i64 1 3922; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP5]], i64 1 3923; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP6]], i64 1 3924; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP7]], i64 1 3925; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[TMP8]], i32 0 3926; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <4 x i32>* 3927; UNROLL-NO-IC-NEXT: store <4 x i32> <i32 7, i32 7, i32 7, i32 7>, <4 x i32>* [[TMP19]], align 4 3928; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[TMP8]], i32 4 3929; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = bitcast i32* [[TMP20]] to <4 x i32>* 3930; UNROLL-NO-IC-NEXT: store <4 x i32> <i32 7, i32 7, i32 7, i32 7>, <4 x i32>* [[TMP21]], align 4 3931; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = load i16, i16* [[TMP10]], align 2 3932; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = load i16, i16* [[TMP11]], align 2 3933; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = load i16, i16* [[TMP12]], align 2 3934; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = load i16, i16* [[TMP13]], align 2 3935; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = insertelement <4 x i16> poison, i16 [[TMP22]], i32 0 3936; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = insertelement <4 x i16> [[TMP26]], i16 [[TMP23]], i32 1 3937; UNROLL-NO-IC-NEXT: [[TMP28:%.*]] = insertelement <4 x i16> [[TMP27]], i16 [[TMP24]], i32 2 3938; UNROLL-NO-IC-NEXT: [[TMP29:%.*]] = insertelement <4 x i16> [[TMP28]], i16 [[TMP25]], i32 3 3939; UNROLL-NO-IC-NEXT: [[TMP30:%.*]] = load i16, i16* [[TMP14]], align 2 3940; UNROLL-NO-IC-NEXT: [[TMP31:%.*]] = load i16, i16* [[TMP15]], align 2 3941; UNROLL-NO-IC-NEXT: [[TMP32:%.*]] = load i16, i16* [[TMP16]], align 2 3942; UNROLL-NO-IC-NEXT: [[TMP33:%.*]] = load i16, i16* [[TMP17]], align 2 3943; UNROLL-NO-IC-NEXT: [[TMP34:%.*]] = insertelement <4 x i16> poison, i16 [[TMP30]], i32 0 3944; UNROLL-NO-IC-NEXT: [[TMP35:%.*]] = insertelement <4 x i16> [[TMP34]], i16 [[TMP31]], i32 1 3945; UNROLL-NO-IC-NEXT: [[TMP36:%.*]] = insertelement <4 x i16> [[TMP35]], i16 [[TMP32]], i32 2 3946; UNROLL-NO-IC-NEXT: [[TMP37]] = insertelement <4 x i16> [[TMP36]], i16 [[TMP33]], i32 3 3947; UNROLL-NO-IC-NEXT: [[TMP38:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP29]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 3948; UNROLL-NO-IC-NEXT: [[TMP39:%.*]] = shufflevector <4 x i16> [[TMP29]], <4 x i16> [[TMP37]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 3949; UNROLL-NO-IC-NEXT: [[TMP40:%.*]] = sext <4 x i16> [[TMP38]] to <4 x i32> 3950; UNROLL-NO-IC-NEXT: [[TMP41:%.*]] = sext <4 x i16> [[TMP39]] to <4 x i32> 3951; UNROLL-NO-IC-NEXT: [[TMP42:%.*]] = sext <4 x i16> [[TMP29]] to <4 x i32> 3952; UNROLL-NO-IC-NEXT: [[TMP43:%.*]] = sext <4 x i16> [[TMP37]] to <4 x i32> 3953; UNROLL-NO-IC-NEXT: [[TMP44:%.*]] = mul nsw <4 x i32> [[TMP42]], [[TMP40]] 3954; UNROLL-NO-IC-NEXT: [[TMP45:%.*]] = mul nsw <4 x i32> [[TMP43]], [[TMP41]] 3955; UNROLL-NO-IC-NEXT: [[TMP46:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP0]] 3956; UNROLL-NO-IC-NEXT: [[TMP47:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP4]] 3957; UNROLL-NO-IC-NEXT: [[TMP48:%.*]] = getelementptr inbounds i32, i32* [[TMP46]], i32 0 3958; UNROLL-NO-IC-NEXT: [[TMP49:%.*]] = bitcast i32* [[TMP48]] to <4 x i32>* 3959; UNROLL-NO-IC-NEXT: store <4 x i32> [[TMP44]], <4 x i32>* [[TMP49]], align 4 3960; UNROLL-NO-IC-NEXT: [[TMP50:%.*]] = getelementptr inbounds i32, i32* [[TMP46]], i32 4 3961; UNROLL-NO-IC-NEXT: [[TMP51:%.*]] = bitcast i32* [[TMP50]] to <4 x i32>* 3962; UNROLL-NO-IC-NEXT: store <4 x i32> [[TMP45]], <4 x i32>* [[TMP51]], align 4 3963; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 3964; UNROLL-NO-IC-NEXT: [[TMP52:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 3965; UNROLL-NO-IC-NEXT: br i1 [[TMP52]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]] 3966; UNROLL-NO-IC: middle.block: 3967; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 3968; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[TMP37]], i32 3 3969; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[TMP37]], i32 2 3970; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 3971; UNROLL-NO-IC: scalar.ph: 3972; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 3973; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ] 3974; UNROLL-NO-IC-NEXT: br label [[FOR_BODY:%.*]] 3975; UNROLL-NO-IC: for.body: 3976; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP53:%.*]], [[FOR_BODY]] ] 3977; UNROLL-NO-IC-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 3978; UNROLL-NO-IC-NEXT: [[ARRAYCIDX:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDVARS_IV]] 3979; UNROLL-NO-IC-NEXT: [[CUR_INDEX:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[INDVARS_IV]], i64 1 3980; UNROLL-NO-IC-NEXT: store i32 7, i32* [[ARRAYCIDX]], align 4 3981; UNROLL-NO-IC-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 3982; UNROLL-NO-IC-NEXT: [[TMP53]] = load i16, i16* [[CUR_INDEX]], align 2 3983; UNROLL-NO-IC-NEXT: [[CONV3:%.*]] = sext i16 [[TMP53]] to i32 3984; UNROLL-NO-IC-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV3]], [[CONV]] 3985; UNROLL-NO-IC-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 3986; UNROLL-NO-IC-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 3987; UNROLL-NO-IC-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 3988; UNROLL-NO-IC-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 3989; UNROLL-NO-IC-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]] 3990; UNROLL-NO-IC: for.end: 3991; UNROLL-NO-IC-NEXT: ret void 3992; 3993; UNROLL-NO-VF-LABEL: @PR34711( 3994; UNROLL-NO-VF-NEXT: entry: 3995; UNROLL-NO-VF-NEXT: [[C1:%.*]] = bitcast i32* [[C:%.*]] to i8* 3996; UNROLL-NO-VF-NEXT: [[B3:%.*]] = bitcast i32* [[B:%.*]] to i8* 3997; UNROLL-NO-VF-NEXT: [[PRE_INDEX:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A:%.*]], i64 0, i64 0 3998; UNROLL-NO-VF-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[PRE_INDEX]], align 2 3999; UNROLL-NO-VF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 2 4000; UNROLL-NO-VF-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 4001; UNROLL-NO-VF: vector.memcheck: 4002; UNROLL-NO-VF-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[C]], i64 [[N]] 4003; UNROLL-NO-VF-NEXT: [[SCEVGEP2:%.*]] = bitcast i32* [[SCEVGEP]] to i8* 4004; UNROLL-NO-VF-NEXT: [[SCEVGEP4:%.*]] = getelementptr i32, i32* [[B]], i64 [[N]] 4005; UNROLL-NO-VF-NEXT: [[SCEVGEP45:%.*]] = bitcast i32* [[SCEVGEP4]] to i8* 4006; UNROLL-NO-VF-NEXT: [[SCEVGEP6:%.*]] = getelementptr [2 x i16], [2 x i16]* [[A]], i64 0, i64 1 4007; UNROLL-NO-VF-NEXT: [[SCEVGEP67:%.*]] = bitcast i16* [[SCEVGEP6]] to i8* 4008; UNROLL-NO-VF-NEXT: [[SCEVGEP8:%.*]] = getelementptr [2 x i16], [2 x i16]* [[A]], i64 [[N]], i64 0 4009; UNROLL-NO-VF-NEXT: [[SCEVGEP89:%.*]] = bitcast i16* [[SCEVGEP8]] to i8* 4010; UNROLL-NO-VF-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[C1]], [[SCEVGEP45]] 4011; UNROLL-NO-VF-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[B3]], [[SCEVGEP2]] 4012; UNROLL-NO-VF-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 4013; UNROLL-NO-VF-NEXT: [[BOUND010:%.*]] = icmp ult i8* [[C1]], [[SCEVGEP89]] 4014; UNROLL-NO-VF-NEXT: [[BOUND111:%.*]] = icmp ult i8* [[SCEVGEP67]], [[SCEVGEP2]] 4015; UNROLL-NO-VF-NEXT: [[FOUND_CONFLICT12:%.*]] = and i1 [[BOUND010]], [[BOUND111]] 4016; UNROLL-NO-VF-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT12]] 4017; UNROLL-NO-VF-NEXT: [[BOUND013:%.*]] = icmp ult i8* [[B3]], [[SCEVGEP89]] 4018; UNROLL-NO-VF-NEXT: [[BOUND114:%.*]] = icmp ult i8* [[SCEVGEP67]], [[SCEVGEP45]] 4019; UNROLL-NO-VF-NEXT: [[FOUND_CONFLICT15:%.*]] = and i1 [[BOUND013]], [[BOUND114]] 4020; UNROLL-NO-VF-NEXT: [[CONFLICT_RDX16:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT15]] 4021; UNROLL-NO-VF-NEXT: br i1 [[CONFLICT_RDX16]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 4022; UNROLL-NO-VF: vector.ph: 4023; UNROLL-NO-VF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2 4024; UNROLL-NO-VF-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 4025; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 4026; UNROLL-NO-VF: vector.body: 4027; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4028; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ] 4029; UNROLL-NO-VF-NEXT: [[INDUCTION:%.*]] = add i64 [[INDEX]], 0 4030; UNROLL-NO-VF-NEXT: [[INDUCTION17:%.*]] = add i64 [[INDEX]], 1 4031; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDUCTION]] 4032; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDUCTION17]] 4033; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[INDUCTION]], i64 1 4034; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[INDUCTION17]], i64 1 4035; UNROLL-NO-VF-NEXT: store i32 7, i32* [[TMP0]], align 4 4036; UNROLL-NO-VF-NEXT: store i32 7, i32* [[TMP1]], align 4 4037; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = load i16, i16* [[TMP2]], align 2 4038; UNROLL-NO-VF-NEXT: [[TMP5]] = load i16, i16* [[TMP3]], align 2 4039; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = sext i16 [[VECTOR_RECUR]] to i32 4040; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = sext i16 [[TMP4]] to i32 4041; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = sext i16 [[TMP4]] to i32 4042; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = sext i16 [[TMP5]] to i32 4043; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = mul nsw i32 [[TMP8]], [[TMP6]] 4044; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = mul nsw i32 [[TMP9]], [[TMP7]] 4045; UNROLL-NO-VF-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDUCTION]] 4046; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDUCTION17]] 4047; UNROLL-NO-VF-NEXT: store i32 [[TMP10]], i32* [[TMP12]], align 4 4048; UNROLL-NO-VF-NEXT: store i32 [[TMP11]], i32* [[TMP13]], align 4 4049; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 4050; UNROLL-NO-VF-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 4051; UNROLL-NO-VF-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]] 4052; UNROLL-NO-VF: middle.block: 4053; UNROLL-NO-VF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 4054; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 4055; UNROLL-NO-VF: scalar.ph: 4056; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ] 4057; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ] 4058; UNROLL-NO-VF-NEXT: br label [[FOR_BODY:%.*]] 4059; UNROLL-NO-VF: for.body: 4060; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP15:%.*]], [[FOR_BODY]] ] 4061; UNROLL-NO-VF-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 4062; UNROLL-NO-VF-NEXT: [[ARRAYCIDX:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDVARS_IV]] 4063; UNROLL-NO-VF-NEXT: [[CUR_INDEX:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[INDVARS_IV]], i64 1 4064; UNROLL-NO-VF-NEXT: store i32 7, i32* [[ARRAYCIDX]], align 4 4065; UNROLL-NO-VF-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 4066; UNROLL-NO-VF-NEXT: [[TMP15]] = load i16, i16* [[CUR_INDEX]], align 2 4067; UNROLL-NO-VF-NEXT: [[CONV3:%.*]] = sext i16 [[TMP15]] to i32 4068; UNROLL-NO-VF-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV3]], [[CONV]] 4069; UNROLL-NO-VF-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 4070; UNROLL-NO-VF-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 4071; UNROLL-NO-VF-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 4072; UNROLL-NO-VF-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 4073; UNROLL-NO-VF-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]] 4074; UNROLL-NO-VF: for.end: 4075; UNROLL-NO-VF-NEXT: ret void 4076; 4077; SINK-AFTER-LABEL: @PR34711( 4078; SINK-AFTER-NEXT: entry: 4079; SINK-AFTER-NEXT: [[C1:%.*]] = bitcast i32* [[C:%.*]] to i8* 4080; SINK-AFTER-NEXT: [[B3:%.*]] = bitcast i32* [[B:%.*]] to i8* 4081; SINK-AFTER-NEXT: [[PRE_INDEX:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A:%.*]], i64 0, i64 0 4082; SINK-AFTER-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[PRE_INDEX]], align 2 4083; SINK-AFTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 4084; SINK-AFTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 4085; SINK-AFTER: vector.memcheck: 4086; SINK-AFTER-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[C]], i64 [[N]] 4087; SINK-AFTER-NEXT: [[SCEVGEP2:%.*]] = bitcast i32* [[SCEVGEP]] to i8* 4088; SINK-AFTER-NEXT: [[SCEVGEP4:%.*]] = getelementptr i32, i32* [[B]], i64 [[N]] 4089; SINK-AFTER-NEXT: [[SCEVGEP45:%.*]] = bitcast i32* [[SCEVGEP4]] to i8* 4090; SINK-AFTER-NEXT: [[SCEVGEP6:%.*]] = getelementptr [2 x i16], [2 x i16]* [[A]], i64 0, i64 1 4091; SINK-AFTER-NEXT: [[SCEVGEP67:%.*]] = bitcast i16* [[SCEVGEP6]] to i8* 4092; SINK-AFTER-NEXT: [[SCEVGEP8:%.*]] = getelementptr [2 x i16], [2 x i16]* [[A]], i64 [[N]], i64 0 4093; SINK-AFTER-NEXT: [[SCEVGEP89:%.*]] = bitcast i16* [[SCEVGEP8]] to i8* 4094; SINK-AFTER-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[C1]], [[SCEVGEP45]] 4095; SINK-AFTER-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[B3]], [[SCEVGEP2]] 4096; SINK-AFTER-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 4097; SINK-AFTER-NEXT: [[BOUND010:%.*]] = icmp ult i8* [[C1]], [[SCEVGEP89]] 4098; SINK-AFTER-NEXT: [[BOUND111:%.*]] = icmp ult i8* [[SCEVGEP67]], [[SCEVGEP2]] 4099; SINK-AFTER-NEXT: [[FOUND_CONFLICT12:%.*]] = and i1 [[BOUND010]], [[BOUND111]] 4100; SINK-AFTER-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT12]] 4101; SINK-AFTER-NEXT: [[BOUND013:%.*]] = icmp ult i8* [[B3]], [[SCEVGEP89]] 4102; SINK-AFTER-NEXT: [[BOUND114:%.*]] = icmp ult i8* [[SCEVGEP67]], [[SCEVGEP45]] 4103; SINK-AFTER-NEXT: [[FOUND_CONFLICT15:%.*]] = and i1 [[BOUND013]], [[BOUND114]] 4104; SINK-AFTER-NEXT: [[CONFLICT_RDX16:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT15]] 4105; SINK-AFTER-NEXT: br i1 [[CONFLICT_RDX16]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 4106; SINK-AFTER: vector.ph: 4107; SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4 4108; SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 4109; SINK-AFTER-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i32 3 4110; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 4111; SINK-AFTER: vector.body: 4112; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4113; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[VECTOR_BODY]] ] 4114; SINK-AFTER-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 4115; SINK-AFTER-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1 4116; SINK-AFTER-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2 4117; SINK-AFTER-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3 4118; SINK-AFTER-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[TMP0]] 4119; SINK-AFTER-NEXT: [[TMP5:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP0]], i64 1 4120; SINK-AFTER-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP1]], i64 1 4121; SINK-AFTER-NEXT: [[TMP7:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP2]], i64 1 4122; SINK-AFTER-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP3]], i64 1 4123; SINK-AFTER-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 0 4124; SINK-AFTER-NEXT: [[TMP10:%.*]] = bitcast i32* [[TMP9]] to <4 x i32>* 4125; SINK-AFTER-NEXT: store <4 x i32> <i32 7, i32 7, i32 7, i32 7>, <4 x i32>* [[TMP10]], align 4 4126; SINK-AFTER-NEXT: [[TMP11:%.*]] = load i16, i16* [[TMP5]], align 2 4127; SINK-AFTER-NEXT: [[TMP12:%.*]] = load i16, i16* [[TMP6]], align 2 4128; SINK-AFTER-NEXT: [[TMP13:%.*]] = load i16, i16* [[TMP7]], align 2 4129; SINK-AFTER-NEXT: [[TMP14:%.*]] = load i16, i16* [[TMP8]], align 2 4130; SINK-AFTER-NEXT: [[TMP15:%.*]] = insertelement <4 x i16> poison, i16 [[TMP11]], i32 0 4131; SINK-AFTER-NEXT: [[TMP16:%.*]] = insertelement <4 x i16> [[TMP15]], i16 [[TMP12]], i32 1 4132; SINK-AFTER-NEXT: [[TMP17:%.*]] = insertelement <4 x i16> [[TMP16]], i16 [[TMP13]], i32 2 4133; SINK-AFTER-NEXT: [[TMP18]] = insertelement <4 x i16> [[TMP17]], i16 [[TMP14]], i32 3 4134; SINK-AFTER-NEXT: [[TMP19:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP18]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 4135; SINK-AFTER-NEXT: [[TMP20:%.*]] = sext <4 x i16> [[TMP19]] to <4 x i32> 4136; SINK-AFTER-NEXT: [[TMP21:%.*]] = sext <4 x i16> [[TMP18]] to <4 x i32> 4137; SINK-AFTER-NEXT: [[TMP22:%.*]] = mul nsw <4 x i32> [[TMP21]], [[TMP20]] 4138; SINK-AFTER-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP0]] 4139; SINK-AFTER-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, i32* [[TMP23]], i32 0 4140; SINK-AFTER-NEXT: [[TMP25:%.*]] = bitcast i32* [[TMP24]] to <4 x i32>* 4141; SINK-AFTER-NEXT: store <4 x i32> [[TMP22]], <4 x i32>* [[TMP25]], align 4 4142; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 4143; SINK-AFTER-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 4144; SINK-AFTER-NEXT: br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]] 4145; SINK-AFTER: middle.block: 4146; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 4147; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[TMP18]], i32 3 4148; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[TMP18]], i32 2 4149; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 4150; SINK-AFTER: scalar.ph: 4151; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 4152; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ] 4153; SINK-AFTER-NEXT: br label [[FOR_BODY:%.*]] 4154; SINK-AFTER: for.body: 4155; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP27:%.*]], [[FOR_BODY]] ] 4156; SINK-AFTER-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 4157; SINK-AFTER-NEXT: [[ARRAYCIDX:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDVARS_IV]] 4158; SINK-AFTER-NEXT: [[CUR_INDEX:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[INDVARS_IV]], i64 1 4159; SINK-AFTER-NEXT: store i32 7, i32* [[ARRAYCIDX]], align 4 4160; SINK-AFTER-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 4161; SINK-AFTER-NEXT: [[TMP27]] = load i16, i16* [[CUR_INDEX]], align 2 4162; SINK-AFTER-NEXT: [[CONV3:%.*]] = sext i16 [[TMP27]] to i32 4163; SINK-AFTER-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV3]], [[CONV]] 4164; SINK-AFTER-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 4165; SINK-AFTER-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 4166; SINK-AFTER-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 4167; SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 4168; SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]] 4169; SINK-AFTER: for.end: 4170; SINK-AFTER-NEXT: ret void 4171; 4172; NO-SINK-AFTER-LABEL: @PR34711( 4173; NO-SINK-AFTER-NEXT: entry: 4174; NO-SINK-AFTER-NEXT: [[C1:%.*]] = bitcast i32* [[C:%.*]] to i8* 4175; NO-SINK-AFTER-NEXT: [[B3:%.*]] = bitcast i32* [[B:%.*]] to i8* 4176; NO-SINK-AFTER-NEXT: [[PRE_INDEX:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A:%.*]], i64 0, i64 0 4177; NO-SINK-AFTER-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[PRE_INDEX]], align 2 4178; NO-SINK-AFTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 4179; NO-SINK-AFTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 4180; NO-SINK-AFTER: vector.memcheck: 4181; NO-SINK-AFTER-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[C]], i64 [[N]] 4182; NO-SINK-AFTER-NEXT: [[SCEVGEP2:%.*]] = bitcast i32* [[SCEVGEP]] to i8* 4183; NO-SINK-AFTER-NEXT: [[SCEVGEP4:%.*]] = getelementptr i32, i32* [[B]], i64 [[N]] 4184; NO-SINK-AFTER-NEXT: [[SCEVGEP45:%.*]] = bitcast i32* [[SCEVGEP4]] to i8* 4185; NO-SINK-AFTER-NEXT: [[SCEVGEP6:%.*]] = getelementptr [2 x i16], [2 x i16]* [[A]], i64 0, i64 1 4186; NO-SINK-AFTER-NEXT: [[SCEVGEP67:%.*]] = bitcast i16* [[SCEVGEP6]] to i8* 4187; NO-SINK-AFTER-NEXT: [[SCEVGEP8:%.*]] = getelementptr [2 x i16], [2 x i16]* [[A]], i64 [[N]], i64 0 4188; NO-SINK-AFTER-NEXT: [[SCEVGEP89:%.*]] = bitcast i16* [[SCEVGEP8]] to i8* 4189; NO-SINK-AFTER-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[C1]], [[SCEVGEP45]] 4190; NO-SINK-AFTER-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[B3]], [[SCEVGEP2]] 4191; NO-SINK-AFTER-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 4192; NO-SINK-AFTER-NEXT: [[BOUND010:%.*]] = icmp ult i8* [[C1]], [[SCEVGEP89]] 4193; NO-SINK-AFTER-NEXT: [[BOUND111:%.*]] = icmp ult i8* [[SCEVGEP67]], [[SCEVGEP2]] 4194; NO-SINK-AFTER-NEXT: [[FOUND_CONFLICT12:%.*]] = and i1 [[BOUND010]], [[BOUND111]] 4195; NO-SINK-AFTER-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT12]] 4196; NO-SINK-AFTER-NEXT: [[BOUND013:%.*]] = icmp ult i8* [[B3]], [[SCEVGEP89]] 4197; NO-SINK-AFTER-NEXT: [[BOUND114:%.*]] = icmp ult i8* [[SCEVGEP67]], [[SCEVGEP45]] 4198; NO-SINK-AFTER-NEXT: [[FOUND_CONFLICT15:%.*]] = and i1 [[BOUND013]], [[BOUND114]] 4199; NO-SINK-AFTER-NEXT: [[CONFLICT_RDX16:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT15]] 4200; NO-SINK-AFTER-NEXT: br i1 [[CONFLICT_RDX16]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 4201; NO-SINK-AFTER: vector.ph: 4202; NO-SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4 4203; NO-SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 4204; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i32 3 4205; NO-SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 4206; NO-SINK-AFTER: vector.body: 4207; NO-SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4208; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[VECTOR_BODY]] ] 4209; NO-SINK-AFTER-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 4210; NO-SINK-AFTER-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1 4211; NO-SINK-AFTER-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2 4212; NO-SINK-AFTER-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3 4213; NO-SINK-AFTER-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[TMP0]] 4214; NO-SINK-AFTER-NEXT: [[TMP5:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP0]], i64 1 4215; NO-SINK-AFTER-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP1]], i64 1 4216; NO-SINK-AFTER-NEXT: [[TMP7:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP2]], i64 1 4217; NO-SINK-AFTER-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP3]], i64 1 4218; NO-SINK-AFTER-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 0 4219; NO-SINK-AFTER-NEXT: [[TMP10:%.*]] = bitcast i32* [[TMP9]] to <4 x i32>* 4220; NO-SINK-AFTER-NEXT: store <4 x i32> <i32 7, i32 7, i32 7, i32 7>, <4 x i32>* [[TMP10]], align 4 4221; NO-SINK-AFTER-NEXT: [[TMP11:%.*]] = load i16, i16* [[TMP5]], align 2 4222; NO-SINK-AFTER-NEXT: [[TMP12:%.*]] = load i16, i16* [[TMP6]], align 2 4223; NO-SINK-AFTER-NEXT: [[TMP13:%.*]] = load i16, i16* [[TMP7]], align 2 4224; NO-SINK-AFTER-NEXT: [[TMP14:%.*]] = load i16, i16* [[TMP8]], align 2 4225; NO-SINK-AFTER-NEXT: [[TMP15:%.*]] = insertelement <4 x i16> poison, i16 [[TMP11]], i32 0 4226; NO-SINK-AFTER-NEXT: [[TMP16:%.*]] = insertelement <4 x i16> [[TMP15]], i16 [[TMP12]], i32 1 4227; NO-SINK-AFTER-NEXT: [[TMP17:%.*]] = insertelement <4 x i16> [[TMP16]], i16 [[TMP13]], i32 2 4228; NO-SINK-AFTER-NEXT: [[TMP18]] = insertelement <4 x i16> [[TMP17]], i16 [[TMP14]], i32 3 4229; NO-SINK-AFTER-NEXT: [[TMP19:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP18]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 4230; NO-SINK-AFTER-NEXT: [[TMP20:%.*]] = sext <4 x i16> [[TMP19]] to <4 x i32> 4231; NO-SINK-AFTER-NEXT: [[TMP21:%.*]] = sext <4 x i16> [[TMP18]] to <4 x i32> 4232; NO-SINK-AFTER-NEXT: [[TMP22:%.*]] = mul nsw <4 x i32> [[TMP21]], [[TMP20]] 4233; NO-SINK-AFTER-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP0]] 4234; NO-SINK-AFTER-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, i32* [[TMP23]], i32 0 4235; NO-SINK-AFTER-NEXT: [[TMP25:%.*]] = bitcast i32* [[TMP24]] to <4 x i32>* 4236; NO-SINK-AFTER-NEXT: store <4 x i32> [[TMP22]], <4 x i32>* [[TMP25]], align 4 4237; NO-SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 4238; NO-SINK-AFTER-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 4239; NO-SINK-AFTER-NEXT: br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]] 4240; NO-SINK-AFTER: middle.block: 4241; NO-SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 4242; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[TMP18]], i32 3 4243; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[TMP18]], i32 2 4244; NO-SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 4245; NO-SINK-AFTER: scalar.ph: 4246; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 4247; NO-SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ] 4248; NO-SINK-AFTER-NEXT: br label [[FOR_BODY:%.*]] 4249; NO-SINK-AFTER: for.body: 4250; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP27:%.*]], [[FOR_BODY]] ] 4251; NO-SINK-AFTER-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 4252; NO-SINK-AFTER-NEXT: [[ARRAYCIDX:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDVARS_IV]] 4253; NO-SINK-AFTER-NEXT: [[CUR_INDEX:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[INDVARS_IV]], i64 1 4254; NO-SINK-AFTER-NEXT: store i32 7, i32* [[ARRAYCIDX]], align 4 4255; NO-SINK-AFTER-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 4256; NO-SINK-AFTER-NEXT: [[TMP27]] = load i16, i16* [[CUR_INDEX]], align 2 4257; NO-SINK-AFTER-NEXT: [[CONV3:%.*]] = sext i16 [[TMP27]] to i32 4258; NO-SINK-AFTER-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV3]], [[CONV]] 4259; NO-SINK-AFTER-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 4260; NO-SINK-AFTER-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 4261; NO-SINK-AFTER-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 4262; NO-SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 4263; NO-SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]] 4264; NO-SINK-AFTER: for.end: 4265; NO-SINK-AFTER-NEXT: ret void 4266; 4267entry: 4268 %pre.index = getelementptr inbounds [2 x i16], [2 x i16]* %a, i64 0, i64 0 4269 %.pre = load i16, i16* %pre.index 4270 br label %for.body 4271 4272for.body: 4273 %0 = phi i16 [ %.pre, %entry ], [ %1, %for.body ] 4274 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 4275 %arraycidx = getelementptr inbounds i32, i32* %c, i64 %indvars.iv 4276 %cur.index = getelementptr inbounds [2 x i16], [2 x i16]* %a, i64 %indvars.iv, i64 1 4277 store i32 7, i32* %arraycidx ; 1st instruction, to be widened. 4278 %conv = sext i16 %0 to i32 ; 2nd, cast to sink after third. 4279 %1 = load i16, i16* %cur.index ; 3rd, first-order-recurring load not widened. 4280 %conv3 = sext i16 %1 to i32 4281 %mul = mul nsw i32 %conv3, %conv 4282 %arrayidx5 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv 4283 store i32 %mul, i32* %arrayidx5 4284 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 4285 %exitcond = icmp eq i64 %indvars.iv.next, %n 4286 br i1 %exitcond, label %for.end, label %for.body 4287 4288for.end: 4289 ret void 4290} 4291 4292; void no_sink_after(short *a, int n, int *b) { 4293; for(int i = 0; i < n; i++) 4294; b[i] = ((a[i] + 2) * a[i + 1]); 4295; } 4296; 4297 4298; 4299define void @sink_after_with_multiple_users(i16* %a, i32* %b, i64 %n) { 4300; CHECK-LABEL: @sink_after_with_multiple_users( 4301; CHECK-NEXT: entry: 4302; CHECK-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[A:%.*]], align 2 4303; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 4304; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 4305; CHECK: vector.memcheck: 4306; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B:%.*]], i64 [[N]] 4307; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i16, i16* [[A]], i64 1 4308; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 4309; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP0]] 4310; CHECK-NEXT: [[TMP1:%.*]] = bitcast i16* [[SCEVGEP5]] to i32* 4311; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt i32* [[TMP1]], [[B]] 4312; CHECK-NEXT: [[TMP2:%.*]] = bitcast i32* [[SCEVGEP]] to i16* 4313; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult i16* [[SCEVGEP3]], [[TMP2]] 4314; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 4315; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 4316; CHECK: vector.ph: 4317; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4 4318; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i64 3 4319; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 4320; CHECK: vector.body: 4321; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4322; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ] 4323; CHECK-NEXT: [[TMP3:%.*]] = or i64 [[INDEX]], 1 4324; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP3]] 4325; CHECK-NEXT: [[TMP5:%.*]] = bitcast i16* [[TMP4]] to <4 x i16>* 4326; CHECK-NEXT: [[WIDE_LOAD]] = load <4 x i16>, <4 x i16>* [[TMP5]], align 2 4327; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 4328; CHECK-NEXT: [[TMP7:%.*]] = sext <4 x i16> [[TMP6]] to <4 x i32> 4329; CHECK-NEXT: [[TMP8:%.*]] = add nsw <4 x i32> [[TMP7]], <i32 2, i32 2, i32 2, i32 2> 4330; CHECK-NEXT: [[TMP9:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i32> 4331; CHECK-NEXT: [[TMP10:%.*]] = mul nsw <4 x i32> [[TMP8]], [[TMP9]] 4332; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]] 4333; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>* 4334; CHECK-NEXT: store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4 4335; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 4336; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 4337; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]] 4338; CHECK: middle.block: 4339; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] 4340; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 3 4341; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 4342; CHECK: scalar.ph: 4343; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 4344; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ] 4345; CHECK-NEXT: br label [[FOR_BODY:%.*]] 4346; CHECK: for.body: 4347; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP14:%.*]], [[FOR_BODY]] ] 4348; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 4349; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 4350; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[CONV]], 2 4351; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 4352; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[INDVARS_IV_NEXT]] 4353; CHECK-NEXT: [[TMP14]] = load i16, i16* [[ARRAYIDX2]], align 2 4354; CHECK-NEXT: [[CONV3:%.*]] = sext i16 [[TMP14]] to i32 4355; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[ADD]], [[CONV3]] 4356; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 4357; CHECK-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 4358; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 4359; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]] 4360; CHECK: for.end: 4361; CHECK-NEXT: ret void 4362; 4363; UNROLL-LABEL: @sink_after_with_multiple_users( 4364; UNROLL-NEXT: entry: 4365; UNROLL-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[A:%.*]], align 2 4366; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 8 4367; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 4368; UNROLL: vector.memcheck: 4369; UNROLL-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B:%.*]], i64 [[N]] 4370; UNROLL-NEXT: [[SCEVGEP3:%.*]] = getelementptr i16, i16* [[A]], i64 1 4371; UNROLL-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 4372; UNROLL-NEXT: [[SCEVGEP5:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP0]] 4373; UNROLL-NEXT: [[TMP1:%.*]] = bitcast i16* [[SCEVGEP5]] to i32* 4374; UNROLL-NEXT: [[BOUND0:%.*]] = icmp ugt i32* [[TMP1]], [[B]] 4375; UNROLL-NEXT: [[TMP2:%.*]] = bitcast i32* [[SCEVGEP]] to i16* 4376; UNROLL-NEXT: [[BOUND1:%.*]] = icmp ult i16* [[SCEVGEP3]], [[TMP2]] 4377; UNROLL-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 4378; UNROLL-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 4379; UNROLL: vector.ph: 4380; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -8 4381; UNROLL-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i64 3 4382; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 4383; UNROLL: vector.body: 4384; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4385; UNROLL-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD7:%.*]], [[VECTOR_BODY]] ] 4386; UNROLL-NEXT: [[TMP3:%.*]] = or i64 [[INDEX]], 1 4387; UNROLL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP3]] 4388; UNROLL-NEXT: [[TMP5:%.*]] = bitcast i16* [[TMP4]] to <4 x i16>* 4389; UNROLL-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, <4 x i16>* [[TMP5]], align 2 4390; UNROLL-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, i16* [[TMP4]], i64 4 4391; UNROLL-NEXT: [[TMP7:%.*]] = bitcast i16* [[TMP6]] to <4 x i16>* 4392; UNROLL-NEXT: [[WIDE_LOAD7]] = load <4 x i16>, <4 x i16>* [[TMP7]], align 2 4393; UNROLL-NEXT: [[TMP8:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 4394; UNROLL-NEXT: [[TMP9:%.*]] = shufflevector <4 x i16> [[WIDE_LOAD]], <4 x i16> [[WIDE_LOAD7]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 4395; UNROLL-NEXT: [[TMP10:%.*]] = sext <4 x i16> [[TMP8]] to <4 x i32> 4396; UNROLL-NEXT: [[TMP11:%.*]] = sext <4 x i16> [[TMP9]] to <4 x i32> 4397; UNROLL-NEXT: [[TMP12:%.*]] = add nsw <4 x i32> [[TMP10]], <i32 2, i32 2, i32 2, i32 2> 4398; UNROLL-NEXT: [[TMP13:%.*]] = add nsw <4 x i32> [[TMP11]], <i32 2, i32 2, i32 2, i32 2> 4399; UNROLL-NEXT: [[TMP14:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i32> 4400; UNROLL-NEXT: [[TMP15:%.*]] = sext <4 x i16> [[WIDE_LOAD7]] to <4 x i32> 4401; UNROLL-NEXT: [[TMP16:%.*]] = mul nsw <4 x i32> [[TMP12]], [[TMP14]] 4402; UNROLL-NEXT: [[TMP17:%.*]] = mul nsw <4 x i32> [[TMP13]], [[TMP15]] 4403; UNROLL-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]] 4404; UNROLL-NEXT: [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <4 x i32>* 4405; UNROLL-NEXT: store <4 x i32> [[TMP16]], <4 x i32>* [[TMP19]], align 4 4406; UNROLL-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[TMP18]], i64 4 4407; UNROLL-NEXT: [[TMP21:%.*]] = bitcast i32* [[TMP20]] to <4 x i32>* 4408; UNROLL-NEXT: store <4 x i32> [[TMP17]], <4 x i32>* [[TMP21]], align 4 4409; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 4410; UNROLL-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 4411; UNROLL-NEXT: br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]] 4412; UNROLL: middle.block: 4413; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] 4414; UNROLL-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD7]], i64 3 4415; UNROLL-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 4416; UNROLL: scalar.ph: 4417; UNROLL-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 4418; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ] 4419; UNROLL-NEXT: br label [[FOR_BODY:%.*]] 4420; UNROLL: for.body: 4421; UNROLL-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP23:%.*]], [[FOR_BODY]] ] 4422; UNROLL-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 4423; UNROLL-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 4424; UNROLL-NEXT: [[ADD:%.*]] = add nsw i32 [[CONV]], 2 4425; UNROLL-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 4426; UNROLL-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[INDVARS_IV_NEXT]] 4427; UNROLL-NEXT: [[TMP23]] = load i16, i16* [[ARRAYIDX2]], align 2 4428; UNROLL-NEXT: [[CONV3:%.*]] = sext i16 [[TMP23]] to i32 4429; UNROLL-NEXT: [[MUL:%.*]] = mul nsw i32 [[ADD]], [[CONV3]] 4430; UNROLL-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 4431; UNROLL-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 4432; UNROLL-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 4433; UNROLL-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]] 4434; UNROLL: for.end: 4435; UNROLL-NEXT: ret void 4436; 4437; UNROLL-NO-IC-LABEL: @sink_after_with_multiple_users( 4438; UNROLL-NO-IC-NEXT: entry: 4439; UNROLL-NO-IC-NEXT: [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8* 4440; UNROLL-NO-IC-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[A:%.*]], align 2 4441; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 8 4442; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 4443; UNROLL-NO-IC: vector.memcheck: 4444; UNROLL-NO-IC-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[N]] 4445; UNROLL-NO-IC-NEXT: [[SCEVGEP2:%.*]] = bitcast i32* [[SCEVGEP]] to i8* 4446; UNROLL-NO-IC-NEXT: [[SCEVGEP3:%.*]] = getelementptr i16, i16* [[A]], i64 1 4447; UNROLL-NO-IC-NEXT: [[SCEVGEP34:%.*]] = bitcast i16* [[SCEVGEP3]] to i8* 4448; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 4449; UNROLL-NO-IC-NEXT: [[SCEVGEP5:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP0]] 4450; UNROLL-NO-IC-NEXT: [[SCEVGEP56:%.*]] = bitcast i16* [[SCEVGEP5]] to i8* 4451; UNROLL-NO-IC-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[B1]], [[SCEVGEP56]] 4452; UNROLL-NO-IC-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP34]], [[SCEVGEP2]] 4453; UNROLL-NO-IC-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 4454; UNROLL-NO-IC-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 4455; UNROLL-NO-IC: vector.ph: 4456; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8 4457; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 4458; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i32 3 4459; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 4460; UNROLL-NO-IC: vector.body: 4461; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4462; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD7:%.*]], [[VECTOR_BODY]] ] 4463; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 0 4464; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 4 4465; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP1]], 1 4466; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = add nuw nsw i64 [[TMP2]], 1 4467; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP3]] 4468; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP4]] 4469; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, i16* [[TMP5]], i32 0 4470; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = bitcast i16* [[TMP7]] to <4 x i16>* 4471; UNROLL-NO-IC-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, <4 x i16>* [[TMP8]], align 2 4472; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = getelementptr inbounds i16, i16* [[TMP5]], i32 4 4473; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = bitcast i16* [[TMP9]] to <4 x i16>* 4474; UNROLL-NO-IC-NEXT: [[WIDE_LOAD7]] = load <4 x i16>, <4 x i16>* [[TMP10]], align 2 4475; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 4476; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = shufflevector <4 x i16> [[WIDE_LOAD]], <4 x i16> [[WIDE_LOAD7]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 4477; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = sext <4 x i16> [[TMP11]] to <4 x i32> 4478; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = sext <4 x i16> [[TMP12]] to <4 x i32> 4479; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = add nsw <4 x i32> [[TMP13]], <i32 2, i32 2, i32 2, i32 2> 4480; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = add nsw <4 x i32> [[TMP14]], <i32 2, i32 2, i32 2, i32 2> 4481; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i32> 4482; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = sext <4 x i16> [[WIDE_LOAD7]] to <4 x i32> 4483; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = mul nsw <4 x i32> [[TMP15]], [[TMP17]] 4484; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = mul nsw <4 x i32> [[TMP16]], [[TMP18]] 4485; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP1]] 4486; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP2]] 4487; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[TMP21]], i32 0 4488; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <4 x i32>* 4489; UNROLL-NO-IC-NEXT: store <4 x i32> [[TMP19]], <4 x i32>* [[TMP24]], align 4 4490; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, i32* [[TMP21]], i32 4 4491; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = bitcast i32* [[TMP25]] to <4 x i32>* 4492; UNROLL-NO-IC-NEXT: store <4 x i32> [[TMP20]], <4 x i32>* [[TMP26]], align 4 4493; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 4494; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 4495; UNROLL-NO-IC-NEXT: br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]] 4496; UNROLL-NO-IC: middle.block: 4497; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 4498; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD7]], i32 3 4499; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD7]], i32 2 4500; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 4501; UNROLL-NO-IC: scalar.ph: 4502; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 4503; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ] 4504; UNROLL-NO-IC-NEXT: br label [[FOR_BODY:%.*]] 4505; UNROLL-NO-IC: for.body: 4506; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP28:%.*]], [[FOR_BODY]] ] 4507; UNROLL-NO-IC-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 4508; UNROLL-NO-IC-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 4509; UNROLL-NO-IC-NEXT: [[ADD:%.*]] = add nsw i32 [[CONV]], 2 4510; UNROLL-NO-IC-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 4511; UNROLL-NO-IC-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[INDVARS_IV_NEXT]] 4512; UNROLL-NO-IC-NEXT: [[TMP28]] = load i16, i16* [[ARRAYIDX2]], align 2 4513; UNROLL-NO-IC-NEXT: [[CONV3:%.*]] = sext i16 [[TMP28]] to i32 4514; UNROLL-NO-IC-NEXT: [[MUL:%.*]] = mul nsw i32 [[ADD]], [[CONV3]] 4515; UNROLL-NO-IC-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 4516; UNROLL-NO-IC-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 4517; UNROLL-NO-IC-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 4518; UNROLL-NO-IC-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]] 4519; UNROLL-NO-IC: for.end: 4520; UNROLL-NO-IC-NEXT: ret void 4521; 4522; UNROLL-NO-VF-LABEL: @sink_after_with_multiple_users( 4523; UNROLL-NO-VF-NEXT: entry: 4524; UNROLL-NO-VF-NEXT: [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8* 4525; UNROLL-NO-VF-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[A:%.*]], align 2 4526; UNROLL-NO-VF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 2 4527; UNROLL-NO-VF-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 4528; UNROLL-NO-VF: vector.memcheck: 4529; UNROLL-NO-VF-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[N]] 4530; UNROLL-NO-VF-NEXT: [[SCEVGEP2:%.*]] = bitcast i32* [[SCEVGEP]] to i8* 4531; UNROLL-NO-VF-NEXT: [[SCEVGEP3:%.*]] = getelementptr i16, i16* [[A]], i64 1 4532; UNROLL-NO-VF-NEXT: [[SCEVGEP34:%.*]] = bitcast i16* [[SCEVGEP3]] to i8* 4533; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 4534; UNROLL-NO-VF-NEXT: [[SCEVGEP5:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP0]] 4535; UNROLL-NO-VF-NEXT: [[SCEVGEP56:%.*]] = bitcast i16* [[SCEVGEP5]] to i8* 4536; UNROLL-NO-VF-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[B1]], [[SCEVGEP56]] 4537; UNROLL-NO-VF-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP34]], [[SCEVGEP2]] 4538; UNROLL-NO-VF-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 4539; UNROLL-NO-VF-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 4540; UNROLL-NO-VF: vector.ph: 4541; UNROLL-NO-VF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2 4542; UNROLL-NO-VF-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 4543; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 4544; UNROLL-NO-VF: vector.body: 4545; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4546; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ] 4547; UNROLL-NO-VF-NEXT: [[INDUCTION:%.*]] = add i64 [[INDEX]], 0 4548; UNROLL-NO-VF-NEXT: [[INDUCTION7:%.*]] = add i64 [[INDEX]], 1 4549; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[INDUCTION]], 1 4550; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[INDUCTION7]], 1 4551; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP1]] 4552; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP2]] 4553; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = load i16, i16* [[TMP3]], align 2 4554; UNROLL-NO-VF-NEXT: [[TMP6]] = load i16, i16* [[TMP4]], align 2 4555; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = sext i16 [[VECTOR_RECUR]] to i32 4556; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = sext i16 [[TMP5]] to i32 4557; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = add nsw i32 [[TMP7]], 2 4558; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = add nsw i32 [[TMP8]], 2 4559; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = sext i16 [[TMP5]] to i32 4560; UNROLL-NO-VF-NEXT: [[TMP12:%.*]] = sext i16 [[TMP6]] to i32 4561; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = mul nsw i32 [[TMP9]], [[TMP11]] 4562; UNROLL-NO-VF-NEXT: [[TMP14:%.*]] = mul nsw i32 [[TMP10]], [[TMP12]] 4563; UNROLL-NO-VF-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDUCTION]] 4564; UNROLL-NO-VF-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDUCTION7]] 4565; UNROLL-NO-VF-NEXT: store i32 [[TMP13]], i32* [[TMP15]], align 4 4566; UNROLL-NO-VF-NEXT: store i32 [[TMP14]], i32* [[TMP16]], align 4 4567; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 4568; UNROLL-NO-VF-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 4569; UNROLL-NO-VF-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP47:![0-9]+]] 4570; UNROLL-NO-VF: middle.block: 4571; UNROLL-NO-VF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 4572; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 4573; UNROLL-NO-VF: scalar.ph: 4574; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ] 4575; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ] 4576; UNROLL-NO-VF-NEXT: br label [[FOR_BODY:%.*]] 4577; UNROLL-NO-VF: for.body: 4578; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP18:%.*]], [[FOR_BODY]] ] 4579; UNROLL-NO-VF-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 4580; UNROLL-NO-VF-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 4581; UNROLL-NO-VF-NEXT: [[ADD:%.*]] = add nsw i32 [[CONV]], 2 4582; UNROLL-NO-VF-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 4583; UNROLL-NO-VF-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[INDVARS_IV_NEXT]] 4584; UNROLL-NO-VF-NEXT: [[TMP18]] = load i16, i16* [[ARRAYIDX2]], align 2 4585; UNROLL-NO-VF-NEXT: [[CONV3:%.*]] = sext i16 [[TMP18]] to i32 4586; UNROLL-NO-VF-NEXT: [[MUL:%.*]] = mul nsw i32 [[ADD]], [[CONV3]] 4587; UNROLL-NO-VF-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 4588; UNROLL-NO-VF-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 4589; UNROLL-NO-VF-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 4590; UNROLL-NO-VF-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]] 4591; UNROLL-NO-VF: for.end: 4592; UNROLL-NO-VF-NEXT: ret void 4593; 4594; SINK-AFTER-LABEL: @sink_after_with_multiple_users( 4595; SINK-AFTER-NEXT: entry: 4596; SINK-AFTER-NEXT: [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8* 4597; SINK-AFTER-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[A:%.*]], align 2 4598; SINK-AFTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 4599; SINK-AFTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 4600; SINK-AFTER: vector.memcheck: 4601; SINK-AFTER-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[N]] 4602; SINK-AFTER-NEXT: [[SCEVGEP2:%.*]] = bitcast i32* [[SCEVGEP]] to i8* 4603; SINK-AFTER-NEXT: [[SCEVGEP3:%.*]] = getelementptr i16, i16* [[A]], i64 1 4604; SINK-AFTER-NEXT: [[SCEVGEP34:%.*]] = bitcast i16* [[SCEVGEP3]] to i8* 4605; SINK-AFTER-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 4606; SINK-AFTER-NEXT: [[SCEVGEP5:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP0]] 4607; SINK-AFTER-NEXT: [[SCEVGEP56:%.*]] = bitcast i16* [[SCEVGEP5]] to i8* 4608; SINK-AFTER-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[B1]], [[SCEVGEP56]] 4609; SINK-AFTER-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP34]], [[SCEVGEP2]] 4610; SINK-AFTER-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 4611; SINK-AFTER-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 4612; SINK-AFTER: vector.ph: 4613; SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4 4614; SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 4615; SINK-AFTER-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i32 3 4616; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 4617; SINK-AFTER: vector.body: 4618; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4619; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ] 4620; SINK-AFTER-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 0 4621; SINK-AFTER-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 4622; SINK-AFTER-NEXT: [[TMP3:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP2]] 4623; SINK-AFTER-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, i16* [[TMP3]], i32 0 4624; SINK-AFTER-NEXT: [[TMP5:%.*]] = bitcast i16* [[TMP4]] to <4 x i16>* 4625; SINK-AFTER-NEXT: [[WIDE_LOAD]] = load <4 x i16>, <4 x i16>* [[TMP5]], align 2 4626; SINK-AFTER-NEXT: [[TMP6:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 4627; SINK-AFTER-NEXT: [[TMP7:%.*]] = sext <4 x i16> [[TMP6]] to <4 x i32> 4628; SINK-AFTER-NEXT: [[TMP8:%.*]] = add nsw <4 x i32> [[TMP7]], <i32 2, i32 2, i32 2, i32 2> 4629; SINK-AFTER-NEXT: [[TMP9:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i32> 4630; SINK-AFTER-NEXT: [[TMP10:%.*]] = mul nsw <4 x i32> [[TMP8]], [[TMP9]] 4631; SINK-AFTER-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP1]] 4632; SINK-AFTER-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TMP11]], i32 0 4633; SINK-AFTER-NEXT: [[TMP13:%.*]] = bitcast i32* [[TMP12]] to <4 x i32>* 4634; SINK-AFTER-NEXT: store <4 x i32> [[TMP10]], <4 x i32>* [[TMP13]], align 4 4635; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 4636; SINK-AFTER-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 4637; SINK-AFTER-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]] 4638; SINK-AFTER: middle.block: 4639; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 4640; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3 4641; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2 4642; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 4643; SINK-AFTER: scalar.ph: 4644; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 4645; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ] 4646; SINK-AFTER-NEXT: br label [[FOR_BODY:%.*]] 4647; SINK-AFTER: for.body: 4648; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP15:%.*]], [[FOR_BODY]] ] 4649; SINK-AFTER-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 4650; SINK-AFTER-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 4651; SINK-AFTER-NEXT: [[ADD:%.*]] = add nsw i32 [[CONV]], 2 4652; SINK-AFTER-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 4653; SINK-AFTER-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[INDVARS_IV_NEXT]] 4654; SINK-AFTER-NEXT: [[TMP15]] = load i16, i16* [[ARRAYIDX2]], align 2 4655; SINK-AFTER-NEXT: [[CONV3:%.*]] = sext i16 [[TMP15]] to i32 4656; SINK-AFTER-NEXT: [[MUL:%.*]] = mul nsw i32 [[ADD]], [[CONV3]] 4657; SINK-AFTER-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 4658; SINK-AFTER-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 4659; SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 4660; SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]] 4661; SINK-AFTER: for.end: 4662; SINK-AFTER-NEXT: ret void 4663; 4664; NO-SINK-AFTER-LABEL: @sink_after_with_multiple_users( 4665; NO-SINK-AFTER-NEXT: entry: 4666; NO-SINK-AFTER-NEXT: [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8* 4667; NO-SINK-AFTER-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[A:%.*]], align 2 4668; NO-SINK-AFTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 4669; NO-SINK-AFTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 4670; NO-SINK-AFTER: vector.memcheck: 4671; NO-SINK-AFTER-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[N]] 4672; NO-SINK-AFTER-NEXT: [[SCEVGEP2:%.*]] = bitcast i32* [[SCEVGEP]] to i8* 4673; NO-SINK-AFTER-NEXT: [[SCEVGEP3:%.*]] = getelementptr i16, i16* [[A]], i64 1 4674; NO-SINK-AFTER-NEXT: [[SCEVGEP34:%.*]] = bitcast i16* [[SCEVGEP3]] to i8* 4675; NO-SINK-AFTER-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 4676; NO-SINK-AFTER-NEXT: [[SCEVGEP5:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP0]] 4677; NO-SINK-AFTER-NEXT: [[SCEVGEP56:%.*]] = bitcast i16* [[SCEVGEP5]] to i8* 4678; NO-SINK-AFTER-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[B1]], [[SCEVGEP56]] 4679; NO-SINK-AFTER-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP34]], [[SCEVGEP2]] 4680; NO-SINK-AFTER-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 4681; NO-SINK-AFTER-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 4682; NO-SINK-AFTER: vector.ph: 4683; NO-SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4 4684; NO-SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 4685; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i32 3 4686; NO-SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 4687; NO-SINK-AFTER: vector.body: 4688; NO-SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4689; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ] 4690; NO-SINK-AFTER-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 0 4691; NO-SINK-AFTER-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 4692; NO-SINK-AFTER-NEXT: [[TMP3:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP2]] 4693; NO-SINK-AFTER-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, i16* [[TMP3]], i32 0 4694; NO-SINK-AFTER-NEXT: [[TMP5:%.*]] = bitcast i16* [[TMP4]] to <4 x i16>* 4695; NO-SINK-AFTER-NEXT: [[WIDE_LOAD]] = load <4 x i16>, <4 x i16>* [[TMP5]], align 2 4696; NO-SINK-AFTER-NEXT: [[TMP6:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 4697; NO-SINK-AFTER-NEXT: [[TMP7:%.*]] = sext <4 x i16> [[TMP6]] to <4 x i32> 4698; NO-SINK-AFTER-NEXT: [[TMP8:%.*]] = add nsw <4 x i32> [[TMP7]], <i32 2, i32 2, i32 2, i32 2> 4699; NO-SINK-AFTER-NEXT: [[TMP9:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i32> 4700; NO-SINK-AFTER-NEXT: [[TMP10:%.*]] = mul nsw <4 x i32> [[TMP8]], [[TMP9]] 4701; NO-SINK-AFTER-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP1]] 4702; NO-SINK-AFTER-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TMP11]], i32 0 4703; NO-SINK-AFTER-NEXT: [[TMP13:%.*]] = bitcast i32* [[TMP12]] to <4 x i32>* 4704; NO-SINK-AFTER-NEXT: store <4 x i32> [[TMP10]], <4 x i32>* [[TMP13]], align 4 4705; NO-SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 4706; NO-SINK-AFTER-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 4707; NO-SINK-AFTER-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]] 4708; NO-SINK-AFTER: middle.block: 4709; NO-SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 4710; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3 4711; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2 4712; NO-SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 4713; NO-SINK-AFTER: scalar.ph: 4714; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 4715; NO-SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ] 4716; NO-SINK-AFTER-NEXT: br label [[FOR_BODY:%.*]] 4717; NO-SINK-AFTER: for.body: 4718; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP15:%.*]], [[FOR_BODY]] ] 4719; NO-SINK-AFTER-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 4720; NO-SINK-AFTER-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 4721; NO-SINK-AFTER-NEXT: [[ADD:%.*]] = add nsw i32 [[CONV]], 2 4722; NO-SINK-AFTER-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 4723; NO-SINK-AFTER-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[INDVARS_IV_NEXT]] 4724; NO-SINK-AFTER-NEXT: [[TMP15]] = load i16, i16* [[ARRAYIDX2]], align 2 4725; NO-SINK-AFTER-NEXT: [[CONV3:%.*]] = sext i16 [[TMP15]] to i32 4726; NO-SINK-AFTER-NEXT: [[MUL:%.*]] = mul nsw i32 [[ADD]], [[CONV3]] 4727; NO-SINK-AFTER-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 4728; NO-SINK-AFTER-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 4729; NO-SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 4730; NO-SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]] 4731; NO-SINK-AFTER: for.end: 4732; NO-SINK-AFTER-NEXT: ret void 4733; 4734entry: 4735 %.pre = load i16, i16* %a 4736 br label %for.body 4737 4738for.body: 4739 %0 = phi i16 [ %.pre, %entry ], [ %1, %for.body ] 4740 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 4741 %conv = sext i16 %0 to i32 4742 %add = add nsw i32 %conv, 2 4743 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 4744 %arrayidx2 = getelementptr inbounds i16, i16* %a, i64 %indvars.iv.next 4745 %1 = load i16, i16* %arrayidx2 4746 %conv3 = sext i16 %1 to i32 4747 %mul = mul nsw i32 %add, %conv3 4748 %arrayidx5 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv 4749 store i32 %mul, i32* %arrayidx5 4750 %exitcond = icmp eq i64 %indvars.iv.next, %n 4751 br i1 %exitcond, label %for.end, label %for.body 4752 4753for.end: 4754 ret void 4755} 4756 4757; Do not sink branches: While branches are if-converted and do not require 4758; sinking, instructions with side effects (e.g. loads) conditioned by those 4759; branches will become users of the condition bit after vectorization and would 4760; need to be sunk if the loop is vectorized. 4761define void @do_not_sink_branch(i32 %x, i32* %in, i32* %out, i32 %tc) local_unnamed_addr #0 { 4762; CHECK-LABEL: @do_not_sink_branch( 4763; CHECK-NEXT: entry: 4764; CHECK-NEXT: [[CMP530:%.*]] = icmp sgt i32 [[TC:%.*]], 0 4765; CHECK-NEXT: br label [[FOR_BODY4:%.*]] 4766; CHECK: for.body4: 4767; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[COND_END:%.*]] ] 4768; CHECK-NEXT: [[CMP534:%.*]] = phi i1 [ [[CMP530]], [[ENTRY]] ], [ [[CMP5:%.*]], [[COND_END]] ] 4769; CHECK-NEXT: br i1 [[CMP534]], label [[COND_TRUE:%.*]], label [[COND_END]] 4770; CHECK: cond.true: 4771; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[INDVARS_IV]] to i64 4772; CHECK-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[IN:%.*]], i64 [[TMP0]] 4773; CHECK-NEXT: [[IN_VAL:%.*]] = load i32, i32* [[ARRAYIDX7]], align 4 4774; CHECK-NEXT: br label [[COND_END]] 4775; CHECK: cond.end: 4776; CHECK-NEXT: [[COND:%.*]] = phi i32 [ [[IN_VAL]], [[COND_TRUE]] ], [ 0, [[FOR_BODY4]] ] 4777; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[INDVARS_IV]] to i64 4778; CHECK-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, i32* [[OUT:%.*]], i64 [[TMP1]] 4779; CHECK-NEXT: store i32 [[COND]], i32* [[ARRAYIDX8]], align 4 4780; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[INDVARS_IV]], 1 4781; CHECK-NEXT: [[CMP5]] = icmp slt i32 [[INDVARS_IV_NEXT]], [[TC]] 4782; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], [[X:%.*]] 4783; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END12_LOOPEXIT:%.*]], label [[FOR_BODY4]] 4784; CHECK: for.end12.loopexit: 4785; CHECK-NEXT: ret void 4786; 4787; UNROLL-LABEL: @do_not_sink_branch( 4788; UNROLL-NEXT: entry: 4789; UNROLL-NEXT: [[CMP530:%.*]] = icmp sgt i32 [[TC:%.*]], 0 4790; UNROLL-NEXT: br label [[FOR_BODY4:%.*]] 4791; UNROLL: for.body4: 4792; UNROLL-NEXT: [[INDVARS_IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[COND_END:%.*]] ] 4793; UNROLL-NEXT: [[CMP534:%.*]] = phi i1 [ [[CMP530]], [[ENTRY]] ], [ [[CMP5:%.*]], [[COND_END]] ] 4794; UNROLL-NEXT: br i1 [[CMP534]], label [[COND_TRUE:%.*]], label [[COND_END]] 4795; UNROLL: cond.true: 4796; UNROLL-NEXT: [[TMP0:%.*]] = zext i32 [[INDVARS_IV]] to i64 4797; UNROLL-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[IN:%.*]], i64 [[TMP0]] 4798; UNROLL-NEXT: [[IN_VAL:%.*]] = load i32, i32* [[ARRAYIDX7]], align 4 4799; UNROLL-NEXT: br label [[COND_END]] 4800; UNROLL: cond.end: 4801; UNROLL-NEXT: [[COND:%.*]] = phi i32 [ [[IN_VAL]], [[COND_TRUE]] ], [ 0, [[FOR_BODY4]] ] 4802; UNROLL-NEXT: [[TMP1:%.*]] = zext i32 [[INDVARS_IV]] to i64 4803; UNROLL-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, i32* [[OUT:%.*]], i64 [[TMP1]] 4804; UNROLL-NEXT: store i32 [[COND]], i32* [[ARRAYIDX8]], align 4 4805; UNROLL-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[INDVARS_IV]], 1 4806; UNROLL-NEXT: [[CMP5]] = icmp slt i32 [[INDVARS_IV_NEXT]], [[TC]] 4807; UNROLL-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], [[X:%.*]] 4808; UNROLL-NEXT: br i1 [[EXITCOND]], label [[FOR_END12_LOOPEXIT:%.*]], label [[FOR_BODY4]] 4809; UNROLL: for.end12.loopexit: 4810; UNROLL-NEXT: ret void 4811; 4812; UNROLL-NO-IC-LABEL: @do_not_sink_branch( 4813; UNROLL-NO-IC-NEXT: entry: 4814; UNROLL-NO-IC-NEXT: [[CMP530:%.*]] = icmp slt i32 0, [[TC:%.*]] 4815; UNROLL-NO-IC-NEXT: br label [[FOR_BODY4:%.*]] 4816; UNROLL-NO-IC: for.body4: 4817; UNROLL-NO-IC-NEXT: [[INDVARS_IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[COND_END:%.*]] ] 4818; UNROLL-NO-IC-NEXT: [[CMP534:%.*]] = phi i1 [ [[CMP530]], [[ENTRY]] ], [ [[CMP5:%.*]], [[COND_END]] ] 4819; UNROLL-NO-IC-NEXT: br i1 [[CMP534]], label [[COND_TRUE:%.*]], label [[COND_END]] 4820; UNROLL-NO-IC: cond.true: 4821; UNROLL-NO-IC-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[IN:%.*]], i32 [[INDVARS_IV]] 4822; UNROLL-NO-IC-NEXT: [[IN_VAL:%.*]] = load i32, i32* [[ARRAYIDX7]], align 4 4823; UNROLL-NO-IC-NEXT: br label [[COND_END]] 4824; UNROLL-NO-IC: cond.end: 4825; UNROLL-NO-IC-NEXT: [[COND:%.*]] = phi i32 [ [[IN_VAL]], [[COND_TRUE]] ], [ 0, [[FOR_BODY4]] ] 4826; UNROLL-NO-IC-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, i32* [[OUT:%.*]], i32 [[INDVARS_IV]] 4827; UNROLL-NO-IC-NEXT: store i32 [[COND]], i32* [[ARRAYIDX8]], align 4 4828; UNROLL-NO-IC-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[INDVARS_IV]], 1 4829; UNROLL-NO-IC-NEXT: [[CMP5]] = icmp slt i32 [[INDVARS_IV_NEXT]], [[TC]] 4830; UNROLL-NO-IC-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], [[X:%.*]] 4831; UNROLL-NO-IC-NEXT: br i1 [[EXITCOND]], label [[FOR_END12_LOOPEXIT:%.*]], label [[FOR_BODY4]] 4832; UNROLL-NO-IC: for.end12.loopexit: 4833; UNROLL-NO-IC-NEXT: ret void 4834; 4835; UNROLL-NO-VF-LABEL: @do_not_sink_branch( 4836; UNROLL-NO-VF-NEXT: entry: 4837; UNROLL-NO-VF-NEXT: [[CMP530:%.*]] = icmp slt i32 0, [[TC:%.*]] 4838; UNROLL-NO-VF-NEXT: br label [[FOR_BODY4:%.*]] 4839; UNROLL-NO-VF: for.body4: 4840; UNROLL-NO-VF-NEXT: [[INDVARS_IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[COND_END:%.*]] ] 4841; UNROLL-NO-VF-NEXT: [[CMP534:%.*]] = phi i1 [ [[CMP530]], [[ENTRY]] ], [ [[CMP5:%.*]], [[COND_END]] ] 4842; UNROLL-NO-VF-NEXT: br i1 [[CMP534]], label [[COND_TRUE:%.*]], label [[COND_END]] 4843; UNROLL-NO-VF: cond.true: 4844; UNROLL-NO-VF-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[IN:%.*]], i32 [[INDVARS_IV]] 4845; UNROLL-NO-VF-NEXT: [[IN_VAL:%.*]] = load i32, i32* [[ARRAYIDX7]], align 4 4846; UNROLL-NO-VF-NEXT: br label [[COND_END]] 4847; UNROLL-NO-VF: cond.end: 4848; UNROLL-NO-VF-NEXT: [[COND:%.*]] = phi i32 [ [[IN_VAL]], [[COND_TRUE]] ], [ 0, [[FOR_BODY4]] ] 4849; UNROLL-NO-VF-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, i32* [[OUT:%.*]], i32 [[INDVARS_IV]] 4850; UNROLL-NO-VF-NEXT: store i32 [[COND]], i32* [[ARRAYIDX8]], align 4 4851; UNROLL-NO-VF-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[INDVARS_IV]], 1 4852; UNROLL-NO-VF-NEXT: [[CMP5]] = icmp slt i32 [[INDVARS_IV_NEXT]], [[TC]] 4853; UNROLL-NO-VF-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], [[X:%.*]] 4854; UNROLL-NO-VF-NEXT: br i1 [[EXITCOND]], label [[FOR_END12_LOOPEXIT:%.*]], label [[FOR_BODY4]] 4855; UNROLL-NO-VF: for.end12.loopexit: 4856; UNROLL-NO-VF-NEXT: ret void 4857; 4858; SINK-AFTER-LABEL: @do_not_sink_branch( 4859; SINK-AFTER-NEXT: entry: 4860; SINK-AFTER-NEXT: [[CMP530:%.*]] = icmp slt i32 0, [[TC:%.*]] 4861; SINK-AFTER-NEXT: br label [[FOR_BODY4:%.*]] 4862; SINK-AFTER: for.body4: 4863; SINK-AFTER-NEXT: [[INDVARS_IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[COND_END:%.*]] ] 4864; SINK-AFTER-NEXT: [[CMP534:%.*]] = phi i1 [ [[CMP530]], [[ENTRY]] ], [ [[CMP5:%.*]], [[COND_END]] ] 4865; SINK-AFTER-NEXT: br i1 [[CMP534]], label [[COND_TRUE:%.*]], label [[COND_END]] 4866; SINK-AFTER: cond.true: 4867; SINK-AFTER-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[IN:%.*]], i32 [[INDVARS_IV]] 4868; SINK-AFTER-NEXT: [[IN_VAL:%.*]] = load i32, i32* [[ARRAYIDX7]], align 4 4869; SINK-AFTER-NEXT: br label [[COND_END]] 4870; SINK-AFTER: cond.end: 4871; SINK-AFTER-NEXT: [[COND:%.*]] = phi i32 [ [[IN_VAL]], [[COND_TRUE]] ], [ 0, [[FOR_BODY4]] ] 4872; SINK-AFTER-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, i32* [[OUT:%.*]], i32 [[INDVARS_IV]] 4873; SINK-AFTER-NEXT: store i32 [[COND]], i32* [[ARRAYIDX8]], align 4 4874; SINK-AFTER-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[INDVARS_IV]], 1 4875; SINK-AFTER-NEXT: [[CMP5]] = icmp slt i32 [[INDVARS_IV_NEXT]], [[TC]] 4876; SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], [[X:%.*]] 4877; SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_END12_LOOPEXIT:%.*]], label [[FOR_BODY4]] 4878; SINK-AFTER: for.end12.loopexit: 4879; SINK-AFTER-NEXT: ret void 4880; 4881; NO-SINK-AFTER-LABEL: @do_not_sink_branch( 4882; NO-SINK-AFTER-NEXT: entry: 4883; NO-SINK-AFTER-NEXT: [[CMP530:%.*]] = icmp slt i32 0, [[TC:%.*]] 4884; NO-SINK-AFTER-NEXT: br label [[FOR_BODY4:%.*]] 4885; NO-SINK-AFTER: for.body4: 4886; NO-SINK-AFTER-NEXT: [[INDVARS_IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[COND_END:%.*]] ] 4887; NO-SINK-AFTER-NEXT: [[CMP534:%.*]] = phi i1 [ [[CMP530]], [[ENTRY]] ], [ [[CMP5:%.*]], [[COND_END]] ] 4888; NO-SINK-AFTER-NEXT: br i1 [[CMP534]], label [[COND_TRUE:%.*]], label [[COND_END]] 4889; NO-SINK-AFTER: cond.true: 4890; NO-SINK-AFTER-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[IN:%.*]], i32 [[INDVARS_IV]] 4891; NO-SINK-AFTER-NEXT: [[IN_VAL:%.*]] = load i32, i32* [[ARRAYIDX7]], align 4 4892; NO-SINK-AFTER-NEXT: br label [[COND_END]] 4893; NO-SINK-AFTER: cond.end: 4894; NO-SINK-AFTER-NEXT: [[COND:%.*]] = phi i32 [ [[IN_VAL]], [[COND_TRUE]] ], [ 0, [[FOR_BODY4]] ] 4895; NO-SINK-AFTER-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, i32* [[OUT:%.*]], i32 [[INDVARS_IV]] 4896; NO-SINK-AFTER-NEXT: store i32 [[COND]], i32* [[ARRAYIDX8]], align 4 4897; NO-SINK-AFTER-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[INDVARS_IV]], 1 4898; NO-SINK-AFTER-NEXT: [[CMP5]] = icmp slt i32 [[INDVARS_IV_NEXT]], [[TC]] 4899; NO-SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], [[X:%.*]] 4900; NO-SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_END12_LOOPEXIT:%.*]], label [[FOR_BODY4]] 4901; NO-SINK-AFTER: for.end12.loopexit: 4902; NO-SINK-AFTER-NEXT: ret void 4903; 4904entry: 4905 %cmp530 = icmp slt i32 0, %tc 4906 br label %for.body4 4907 4908for.body4: ; preds = %cond.end, %entry 4909 %indvars.iv = phi i32 [ 0, %entry ], [ %indvars.iv.next, %cond.end ] 4910 %cmp534 = phi i1 [ %cmp530, %entry ], [ %cmp5, %cond.end ] 4911 br i1 %cmp534, label %cond.true, label %cond.end 4912 4913cond.true: ; preds = %for.body4 4914 %arrayidx7 = getelementptr inbounds i32, i32* %in, i32 %indvars.iv 4915 %in.val = load i32, i32* %arrayidx7, align 4 4916 br label %cond.end 4917 4918cond.end: ; preds = %for.body4, %cond.true 4919 %cond = phi i32 [ %in.val, %cond.true ], [ 0, %for.body4 ] 4920 %arrayidx8 = getelementptr inbounds i32, i32* %out, i32 %indvars.iv 4921 store i32 %cond, i32* %arrayidx8, align 4 4922 %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1 4923 %cmp5 = icmp slt i32 %indvars.iv.next, %tc 4924 %exitcond = icmp eq i32 %indvars.iv.next, %x 4925 br i1 %exitcond, label %for.end12.loopexit, label %for.body4 4926 4927for.end12.loopexit: ; preds = %cond.end 4928 ret void 4929} 4930 4931; Dead instructions, like the exit condition are not part of the actual VPlan 4932; and do not need to be sunk. PR44634. 4933define void @sink_dead_inst() { 4934; 4935; CHECK-LABEL: @sink_dead_inst( 4936; CHECK-NEXT: entry: 4937; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4938; CHECK: vector.ph: 4939; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 4940; CHECK: vector.body: 4941; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4942; CHECK-NEXT: [[TMP0:%.*]] = phi i16 [ -24, [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ] 4943; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 4944; CHECK-NEXT: [[TMP1]] = add i16 [[TMP0]], 4 4945; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 40 4946; CHECK-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP50:![0-9]+]] 4947; CHECK: middle.block: 4948; CHECK-NEXT: [[TMP3:%.*]] = or i16 [[TMP0]], 1 4949; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT2:%.*]] = zext i16 [[TMP3]] to i32 4950; CHECK-NEXT: br i1 false, label [[FOR_END:%.*]], label [[SCALAR_PH]] 4951; CHECK: scalar.ph: 4952; CHECK-NEXT: [[SCALAR_RECUR_INIT4:%.*]] = phi i32 [ -27, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT2]], [[MIDDLE_BLOCK]] ] 4953; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ -27, [[ENTRY]] ], [ 13, [[MIDDLE_BLOCK]] ] 4954; CHECK-NEXT: br label [[FOR_COND:%.*]] 4955; CHECK: for.cond: 4956; CHECK-NEXT: [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_COND]] ] 4957; CHECK-NEXT: [[SCALAR_RECUR5:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT4]], [[SCALAR_PH]] ], [ [[REC_2_PREV:%.*]], [[FOR_COND]] ] 4958; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[SCALAR_RECUR5]], 15 4959; CHECK-NEXT: [[IV_NEXT]] = add i16 [[IV]], 1 4960; CHECK-NEXT: [[REC_2_PREV]] = zext i16 [[IV_NEXT]] to i32 4961; CHECK-NEXT: br i1 [[CMP]], label [[FOR_END]], label [[FOR_COND]], !llvm.loop [[LOOP51:![0-9]+]] 4962; CHECK: for.end: 4963; CHECK-NEXT: ret void 4964; 4965; UNROLL-LABEL: @sink_dead_inst( 4966; UNROLL-NEXT: entry: 4967; UNROLL-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4968; UNROLL: vector.ph: 4969; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 4970; UNROLL: vector.body: 4971; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4972; UNROLL-NEXT: [[TMP0:%.*]] = phi i16 [ -24, [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ] 4973; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 4974; UNROLL-NEXT: [[TMP1]] = add i16 [[TMP0]], 8 4975; UNROLL-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 40 4976; UNROLL-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP50:![0-9]+]] 4977; UNROLL: middle.block: 4978; UNROLL-NEXT: [[TMP3:%.*]] = or i16 [[TMP0]], 5 4979; UNROLL-NEXT: [[VECTOR_RECUR_EXTRACT3:%.*]] = zext i16 [[TMP3]] to i32 4980; UNROLL-NEXT: br i1 false, label [[FOR_END:%.*]], label [[SCALAR_PH]] 4981; UNROLL: scalar.ph: 4982; UNROLL-NEXT: [[SCALAR_RECUR_INIT5:%.*]] = phi i32 [ -27, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT3]], [[MIDDLE_BLOCK]] ] 4983; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ -27, [[ENTRY]] ], [ 13, [[MIDDLE_BLOCK]] ] 4984; UNROLL-NEXT: br label [[FOR_COND:%.*]] 4985; UNROLL: for.cond: 4986; UNROLL-NEXT: [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_COND]] ] 4987; UNROLL-NEXT: [[SCALAR_RECUR6:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT5]], [[SCALAR_PH]] ], [ [[REC_2_PREV:%.*]], [[FOR_COND]] ] 4988; UNROLL-NEXT: [[CMP:%.*]] = icmp eq i32 [[SCALAR_RECUR6]], 15 4989; UNROLL-NEXT: [[IV_NEXT]] = add i16 [[IV]], 1 4990; UNROLL-NEXT: [[REC_2_PREV]] = zext i16 [[IV_NEXT]] to i32 4991; UNROLL-NEXT: br i1 [[CMP]], label [[FOR_END]], label [[FOR_COND]], !llvm.loop [[LOOP51:![0-9]+]] 4992; UNROLL: for.end: 4993; UNROLL-NEXT: ret void 4994; 4995; UNROLL-NO-IC-LABEL: @sink_dead_inst( 4996; UNROLL-NO-IC-NEXT: entry: 4997; UNROLL-NO-IC-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4998; UNROLL-NO-IC: vector.ph: 4999; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 5000; UNROLL-NO-IC: vector.body: 5001; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 5002; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 -27, i16 -26, i16 -25, i16 -24>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 5003; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 0>, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ] 5004; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR2:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 -27>, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ] 5005; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add <4 x i16> [[VEC_IND]], <i16 4, i16 4, i16 4, i16 4> 5006; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add <4 x i16> [[VEC_IND]], <i16 1, i16 1, i16 1, i16 1> 5007; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = add <4 x i16> [[STEP_ADD]], <i16 1, i16 1, i16 1, i16 1> 5008; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = zext <4 x i16> [[TMP0]] to <4 x i32> 5009; UNROLL-NO-IC-NEXT: [[TMP3]] = zext <4 x i16> [[TMP1]] to <4 x i32> 5010; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR2]], <4 x i32> [[TMP2]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 5011; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 5012; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = add <4 x i16> [[TMP0]], <i16 5, i16 5, i16 5, i16 5> 5013; UNROLL-NO-IC-NEXT: [[TMP7]] = add <4 x i16> [[TMP1]], <i16 5, i16 5, i16 5, i16 5> 5014; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP6]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 5015; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = shufflevector <4 x i16> [[TMP6]], <4 x i16> [[TMP7]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 5016; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = sub <4 x i16> [[TMP8]], <i16 10, i16 10, i16 10, i16 10> 5017; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = sub <4 x i16> [[TMP9]], <i16 10, i16 10, i16 10, i16 10> 5018; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 5019; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT]] = add <4 x i16> [[STEP_ADD]], <i16 4, i16 4, i16 4, i16 4> 5020; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], 40 5021; UNROLL-NO-IC-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP50:![0-9]+]] 5022; UNROLL-NO-IC: middle.block: 5023; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i32 43, 40 5024; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[TMP7]], i32 3 5025; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[TMP7]], i32 2 5026; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT3:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3 5027; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2 5028; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 5029; UNROLL-NO-IC: scalar.ph: 5030; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT5:%.*]] = phi i32 [ -27, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT3]], [[MIDDLE_BLOCK]] ] 5031; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ 0, [[ENTRY]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 5032; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ 13, [[MIDDLE_BLOCK]] ], [ -27, [[ENTRY]] ] 5033; UNROLL-NO-IC-NEXT: br label [[FOR_COND:%.*]] 5034; UNROLL-NO-IC: for.cond: 5035; UNROLL-NO-IC-NEXT: [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_COND]] ] 5036; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[REC_1_PREV:%.*]], [[FOR_COND]] ] 5037; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR6:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT5]], [[SCALAR_PH]] ], [ [[REC_2_PREV:%.*]], [[FOR_COND]] ] 5038; UNROLL-NO-IC-NEXT: [[USE_REC_1:%.*]] = sub i16 [[SCALAR_RECUR]], 10 5039; UNROLL-NO-IC-NEXT: [[CMP:%.*]] = icmp eq i32 [[SCALAR_RECUR6]], 15 5040; UNROLL-NO-IC-NEXT: [[IV_NEXT]] = add i16 [[IV]], 1 5041; UNROLL-NO-IC-NEXT: [[REC_2_PREV]] = zext i16 [[IV_NEXT]] to i32 5042; UNROLL-NO-IC-NEXT: [[REC_1_PREV]] = add i16 [[IV_NEXT]], 5 5043; UNROLL-NO-IC-NEXT: br i1 [[CMP]], label [[FOR_END]], label [[FOR_COND]], !llvm.loop [[LOOP51:![0-9]+]] 5044; UNROLL-NO-IC: for.end: 5045; UNROLL-NO-IC-NEXT: ret void 5046; 5047; UNROLL-NO-VF-LABEL: @sink_dead_inst( 5048; UNROLL-NO-VF-NEXT: entry: 5049; UNROLL-NO-VF-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5050; UNROLL-NO-VF: vector.ph: 5051; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 5052; UNROLL-NO-VF: vector.body: 5053; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 5054; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i16 [ 0, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ] 5055; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR2:%.*]] = phi i32 [ -27, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ] 5056; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = trunc i32 [[INDEX]] to i16 5057; UNROLL-NO-VF-NEXT: [[OFFSET_IDX:%.*]] = add i16 -27, [[TMP0]] 5058; UNROLL-NO-VF-NEXT: [[INDUCTION:%.*]] = add i16 [[OFFSET_IDX]], 0 5059; UNROLL-NO-VF-NEXT: [[INDUCTION1:%.*]] = add i16 [[OFFSET_IDX]], 1 5060; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = add i16 [[INDUCTION]], 1 5061; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = add i16 [[INDUCTION1]], 1 5062; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = zext i16 [[TMP1]] to i32 5063; UNROLL-NO-VF-NEXT: [[TMP4]] = zext i16 [[TMP2]] to i32 5064; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = add i16 [[TMP1]], 5 5065; UNROLL-NO-VF-NEXT: [[TMP6]] = add i16 [[TMP2]], 5 5066; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = sub i16 [[VECTOR_RECUR]], 10 5067; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = sub i16 [[TMP5]], 10 5068; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 5069; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], 42 5070; UNROLL-NO-VF-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]] 5071; UNROLL-NO-VF: middle.block: 5072; UNROLL-NO-VF-NEXT: [[CMP_N:%.*]] = icmp eq i32 43, 42 5073; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 5074; UNROLL-NO-VF: scalar.ph: 5075; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT3:%.*]] = phi i32 [ -27, [[ENTRY:%.*]] ], [ [[TMP4]], [[MIDDLE_BLOCK]] ] 5076; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ 0, [[ENTRY]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ] 5077; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ 15, [[MIDDLE_BLOCK]] ], [ -27, [[ENTRY]] ] 5078; UNROLL-NO-VF-NEXT: br label [[FOR_COND:%.*]] 5079; UNROLL-NO-VF: for.cond: 5080; UNROLL-NO-VF-NEXT: [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_COND]] ] 5081; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[REC_1_PREV:%.*]], [[FOR_COND]] ] 5082; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR4:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT3]], [[SCALAR_PH]] ], [ [[REC_2_PREV:%.*]], [[FOR_COND]] ] 5083; UNROLL-NO-VF-NEXT: [[USE_REC_1:%.*]] = sub i16 [[SCALAR_RECUR]], 10 5084; UNROLL-NO-VF-NEXT: [[CMP:%.*]] = icmp eq i32 [[SCALAR_RECUR4]], 15 5085; UNROLL-NO-VF-NEXT: [[IV_NEXT]] = add i16 [[IV]], 1 5086; UNROLL-NO-VF-NEXT: [[REC_2_PREV]] = zext i16 [[IV_NEXT]] to i32 5087; UNROLL-NO-VF-NEXT: [[REC_1_PREV]] = add i16 [[IV_NEXT]], 5 5088; UNROLL-NO-VF-NEXT: br i1 [[CMP]], label [[FOR_END]], label [[FOR_COND]], !llvm.loop [[LOOP50:![0-9]+]] 5089; UNROLL-NO-VF: for.end: 5090; UNROLL-NO-VF-NEXT: ret void 5091; 5092; SINK-AFTER-LABEL: @sink_dead_inst( 5093; SINK-AFTER-NEXT: entry: 5094; SINK-AFTER-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5095; SINK-AFTER: vector.ph: 5096; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 5097; SINK-AFTER: vector.body: 5098; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 5099; SINK-AFTER-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 -27, i16 -26, i16 -25, i16 -24>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 5100; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 0>, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ] 5101; SINK-AFTER-NEXT: [[VECTOR_RECUR1:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 -27>, [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ] 5102; SINK-AFTER-NEXT: [[TMP0:%.*]] = add <4 x i16> [[VEC_IND]], <i16 1, i16 1, i16 1, i16 1> 5103; SINK-AFTER-NEXT: [[TMP1]] = zext <4 x i16> [[TMP0]] to <4 x i32> 5104; SINK-AFTER-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR1]], <4 x i32> [[TMP1]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 5105; SINK-AFTER-NEXT: [[TMP3]] = add <4 x i16> [[TMP0]], <i16 5, i16 5, i16 5, i16 5> 5106; SINK-AFTER-NEXT: [[TMP4:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP3]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 5107; SINK-AFTER-NEXT: [[TMP5:%.*]] = sub <4 x i16> [[TMP4]], <i16 10, i16 10, i16 10, i16 10> 5108; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 5109; SINK-AFTER-NEXT: [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], <i16 4, i16 4, i16 4, i16 4> 5110; SINK-AFTER-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 40 5111; SINK-AFTER-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP50:![0-9]+]] 5112; SINK-AFTER: middle.block: 5113; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i32 43, 40 5114; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[TMP3]], i32 3 5115; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[TMP3]], i32 2 5116; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT2:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3 5117; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI3:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2 5118; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 5119; SINK-AFTER: scalar.ph: 5120; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT4:%.*]] = phi i32 [ -27, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT2]], [[MIDDLE_BLOCK]] ] 5121; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ 0, [[ENTRY]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 5122; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ 13, [[MIDDLE_BLOCK]] ], [ -27, [[ENTRY]] ] 5123; SINK-AFTER-NEXT: br label [[FOR_COND:%.*]] 5124; SINK-AFTER: for.cond: 5125; SINK-AFTER-NEXT: [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_COND]] ] 5126; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[REC_1_PREV:%.*]], [[FOR_COND]] ] 5127; SINK-AFTER-NEXT: [[SCALAR_RECUR5:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT4]], [[SCALAR_PH]] ], [ [[REC_2_PREV:%.*]], [[FOR_COND]] ] 5128; SINK-AFTER-NEXT: [[USE_REC_1:%.*]] = sub i16 [[SCALAR_RECUR]], 10 5129; SINK-AFTER-NEXT: [[CMP:%.*]] = icmp eq i32 [[SCALAR_RECUR5]], 15 5130; SINK-AFTER-NEXT: [[IV_NEXT]] = add i16 [[IV]], 1 5131; SINK-AFTER-NEXT: [[REC_2_PREV]] = zext i16 [[IV_NEXT]] to i32 5132; SINK-AFTER-NEXT: [[REC_1_PREV]] = add i16 [[IV_NEXT]], 5 5133; SINK-AFTER-NEXT: br i1 [[CMP]], label [[FOR_END]], label [[FOR_COND]], !llvm.loop [[LOOP51:![0-9]+]] 5134; SINK-AFTER: for.end: 5135; SINK-AFTER-NEXT: ret void 5136; 5137; NO-SINK-AFTER-LABEL: @sink_dead_inst( 5138; NO-SINK-AFTER-NEXT: entry: 5139; NO-SINK-AFTER-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5140; NO-SINK-AFTER: vector.ph: 5141; NO-SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 5142; NO-SINK-AFTER: vector.body: 5143; NO-SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 5144; NO-SINK-AFTER-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 -27, i16 -26, i16 -25, i16 -24>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 5145; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 0>, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ] 5146; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR1:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 -27>, [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ] 5147; NO-SINK-AFTER-NEXT: [[TMP0:%.*]] = add <4 x i16> [[VEC_IND]], <i16 1, i16 1, i16 1, i16 1> 5148; NO-SINK-AFTER-NEXT: [[TMP1]] = zext <4 x i16> [[TMP0]] to <4 x i32> 5149; NO-SINK-AFTER-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR1]], <4 x i32> [[TMP1]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 5150; NO-SINK-AFTER-NEXT: [[TMP3]] = add <4 x i16> [[TMP0]], <i16 5, i16 5, i16 5, i16 5> 5151; NO-SINK-AFTER-NEXT: [[TMP4:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP3]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 5152; NO-SINK-AFTER-NEXT: [[TMP5:%.*]] = sub <4 x i16> [[TMP4]], <i16 10, i16 10, i16 10, i16 10> 5153; NO-SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 5154; NO-SINK-AFTER-NEXT: [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], <i16 4, i16 4, i16 4, i16 4> 5155; NO-SINK-AFTER-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 40 5156; NO-SINK-AFTER-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP50:![0-9]+]] 5157; NO-SINK-AFTER: middle.block: 5158; NO-SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i32 43, 40 5159; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[TMP3]], i32 3 5160; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[TMP3]], i32 2 5161; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT2:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3 5162; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI3:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2 5163; NO-SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 5164; NO-SINK-AFTER: scalar.ph: 5165; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT4:%.*]] = phi i32 [ -27, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT2]], [[MIDDLE_BLOCK]] ] 5166; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ 0, [[ENTRY]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 5167; NO-SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ 13, [[MIDDLE_BLOCK]] ], [ -27, [[ENTRY]] ] 5168; NO-SINK-AFTER-NEXT: br label [[FOR_COND:%.*]] 5169; NO-SINK-AFTER: for.cond: 5170; NO-SINK-AFTER-NEXT: [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_COND]] ] 5171; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[REC_1_PREV:%.*]], [[FOR_COND]] ] 5172; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR5:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT4]], [[SCALAR_PH]] ], [ [[REC_2_PREV:%.*]], [[FOR_COND]] ] 5173; NO-SINK-AFTER-NEXT: [[USE_REC_1:%.*]] = sub i16 [[SCALAR_RECUR]], 10 5174; NO-SINK-AFTER-NEXT: [[CMP:%.*]] = icmp eq i32 [[SCALAR_RECUR5]], 15 5175; NO-SINK-AFTER-NEXT: [[IV_NEXT]] = add i16 [[IV]], 1 5176; NO-SINK-AFTER-NEXT: [[REC_2_PREV]] = zext i16 [[IV_NEXT]] to i32 5177; NO-SINK-AFTER-NEXT: [[REC_1_PREV]] = add i16 [[IV_NEXT]], 5 5178; NO-SINK-AFTER-NEXT: br i1 [[CMP]], label [[FOR_END]], label [[FOR_COND]], !llvm.loop [[LOOP51:![0-9]+]] 5179; NO-SINK-AFTER: for.end: 5180; NO-SINK-AFTER-NEXT: ret void 5181; 5182entry: 5183 br label %for.cond 5184 5185for.cond: 5186 %iv = phi i16 [ -27, %entry ], [ %iv.next, %for.cond ] 5187 %rec.1 = phi i16 [ 0, %entry ], [ %rec.1.prev, %for.cond ] 5188 %rec.2 = phi i32 [ -27, %entry ], [ %rec.2.prev, %for.cond ] 5189 %use.rec.1 = sub i16 %rec.1, 10 5190 %cmp = icmp eq i32 %rec.2, 15 5191 %iv.next = add i16 %iv, 1 5192 %rec.2.prev = zext i16 %iv.next to i32 5193 %rec.1.prev = add i16 %iv.next, 5 5194 br i1 %cmp, label %for.end, label %for.cond 5195 5196for.end: 5197 ret void 5198} 5199 5200define i32 @sink_into_replication_region(i32 %y) { 5201; 5202; CHECK-LABEL: @sink_into_replication_region( 5203; CHECK-NEXT: bb: 5204; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[Y:%.*]], 1 5205; CHECK-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[Y]], i32 1) 5206; CHECK-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]] 5207; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5208; CHECK: vector.ph: 5209; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 3 5210; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -4 5211; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = add i32 [[TMP1]], -1 5212; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0 5213; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 5214; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 5215; CHECK: vector.body: 5216; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE8:%.*]] ] 5217; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[PRED_UDIV_CONTINUE8]] ] 5218; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[PRED_UDIV_CONTINUE8]] ] 5219; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]] 5220; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[INDEX]], i64 0 5221; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer 5222; CHECK-NEXT: [[VEC_IV:%.*]] = or <4 x i32> [[BROADCAST_SPLAT2]], <i32 0, i32 1, i32 2, i32 3> 5223; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <4 x i32> [[VEC_IV]], [[BROADCAST_SPLAT]] 5224; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0 5225; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 5226; CHECK: pred.udiv.if: 5227; CHECK-NEXT: [[TMP4:%.*]] = udiv i32 219220132, [[OFFSET_IDX]] 5228; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0 5229; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE]] 5230; CHECK: pred.udiv.continue: 5231; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_UDIV_IF]] ] 5232; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1 5233; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]] 5234; CHECK: pred.udiv.if3: 5235; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[OFFSET_IDX]], -1 5236; CHECK-NEXT: [[TMP9:%.*]] = udiv i32 219220132, [[TMP8]] 5237; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[TMP9]], i64 1 5238; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE4]] 5239; CHECK: pred.udiv.continue4: 5240; CHECK-NEXT: [[TMP11:%.*]] = phi <4 x i32> [ [[TMP6]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP10]], [[PRED_UDIV_IF3]] ] 5241; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2 5242; CHECK-NEXT: br i1 [[TMP12]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]] 5243; CHECK: pred.udiv.if5: 5244; CHECK-NEXT: [[TMP13:%.*]] = add i32 [[OFFSET_IDX]], -2 5245; CHECK-NEXT: [[TMP14:%.*]] = udiv i32 219220132, [[TMP13]] 5246; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP14]], i64 2 5247; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE6]] 5248; CHECK: pred.udiv.continue6: 5249; CHECK-NEXT: [[TMP16:%.*]] = phi <4 x i32> [ [[TMP11]], [[PRED_UDIV_CONTINUE4]] ], [ [[TMP15]], [[PRED_UDIV_IF5]] ] 5250; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3 5251; CHECK-NEXT: br i1 [[TMP17]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8]] 5252; CHECK: pred.udiv.if7: 5253; CHECK-NEXT: [[TMP18:%.*]] = add i32 [[OFFSET_IDX]], -3 5254; CHECK-NEXT: [[TMP19:%.*]] = udiv i32 219220132, [[TMP18]] 5255; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP19]], i64 3 5256; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE8]] 5257; CHECK: pred.udiv.continue8: 5258; CHECK-NEXT: [[TMP21]] = phi <4 x i32> [ [[TMP16]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP20]], [[PRED_UDIV_IF7]] ] 5259; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP21]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 5260; CHECK-NEXT: [[TMP23]] = add <4 x i32> [[VEC_PHI]], [[TMP22]] 5261; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4 5262; CHECK-NEXT: [[TMP24:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 5263; CHECK-NEXT: br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF52:![0-9]+]], !llvm.loop [[LOOP53:![0-9]+]] 5264; CHECK: middle.block: 5265; CHECK-NEXT: [[TMP25:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP23]], <4 x i32> [[VEC_PHI]] 5266; CHECK-NEXT: [[TMP26:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP25]]) 5267; CHECK-NEXT: br i1 true, label [[BB1:%.*]], label [[SCALAR_PH]] 5268; CHECK: scalar.ph: 5269; CHECK-NEXT: br label [[BB2:%.*]] 5270; CHECK: bb1: 5271; CHECK-NEXT: [[VAR:%.*]] = phi i32 [ undef, [[BB2]] ], [ [[TMP26]], [[MIDDLE_BLOCK]] ] 5272; CHECK-NEXT: ret i32 [[VAR]] 5273; CHECK: bb2: 5274; CHECK-NEXT: br i1 undef, label [[BB1]], label [[BB2]], !prof [[PROF54:![0-9]+]], !llvm.loop [[LOOP55:![0-9]+]] 5275; 5276; UNROLL-LABEL: @sink_into_replication_region( 5277; UNROLL-NEXT: bb: 5278; UNROLL-NEXT: [[TMP0:%.*]] = add i32 [[Y:%.*]], 1 5279; UNROLL-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[Y]], i32 1) 5280; UNROLL-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]] 5281; UNROLL-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5282; UNROLL: vector.ph: 5283; UNROLL-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 7 5284; UNROLL-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8 5285; UNROLL-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = add i32 [[TMP1]], -1 5286; UNROLL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0 5287; UNROLL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 5288; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 5289; UNROLL: vector.body: 5290; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE19:%.*]] ] 5291; UNROLL-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP42:%.*]], [[PRED_UDIV_CONTINUE19]] ] 5292; UNROLL-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP45:%.*]], [[PRED_UDIV_CONTINUE19]] ] 5293; UNROLL-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP46:%.*]], [[PRED_UDIV_CONTINUE19]] ] 5294; UNROLL-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]] 5295; UNROLL-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i32> poison, i32 [[INDEX]], i64 0 5296; UNROLL-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT3]], <4 x i32> poison, <4 x i32> zeroinitializer 5297; UNROLL-NEXT: [[VEC_IV:%.*]] = or <4 x i32> [[BROADCAST_SPLAT4]], <i32 0, i32 1, i32 2, i32 3> 5298; UNROLL-NEXT: [[VEC_IV5:%.*]] = or <4 x i32> [[BROADCAST_SPLAT4]], <i32 4, i32 5, i32 6, i32 7> 5299; UNROLL-NEXT: [[TMP2:%.*]] = icmp ule <4 x i32> [[VEC_IV]], [[BROADCAST_SPLAT]] 5300; UNROLL-NEXT: [[TMP3:%.*]] = icmp ule <4 x i32> [[VEC_IV5]], [[BROADCAST_SPLAT]] 5301; UNROLL-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0 5302; UNROLL-NEXT: br i1 [[TMP4]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 5303; UNROLL: pred.udiv.if: 5304; UNROLL-NEXT: [[TMP5:%.*]] = udiv i32 219220132, [[OFFSET_IDX]] 5305; UNROLL-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0 5306; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE]] 5307; UNROLL: pred.udiv.continue: 5308; UNROLL-NEXT: [[TMP7:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_UDIV_IF]] ] 5309; UNROLL-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1 5310; UNROLL-NEXT: br i1 [[TMP8]], label [[PRED_UDIV_IF6:%.*]], label [[PRED_UDIV_CONTINUE7:%.*]] 5311; UNROLL: pred.udiv.if6: 5312; UNROLL-NEXT: [[TMP9:%.*]] = add i32 [[OFFSET_IDX]], -1 5313; UNROLL-NEXT: [[TMP10:%.*]] = udiv i32 219220132, [[TMP9]] 5314; UNROLL-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[TMP10]], i64 1 5315; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE7]] 5316; UNROLL: pred.udiv.continue7: 5317; UNROLL-NEXT: [[TMP12:%.*]] = phi <4 x i32> [ [[TMP7]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP11]], [[PRED_UDIV_IF6]] ] 5318; UNROLL-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2 5319; UNROLL-NEXT: br i1 [[TMP13]], label [[PRED_UDIV_IF8:%.*]], label [[PRED_UDIV_CONTINUE9:%.*]] 5320; UNROLL: pred.udiv.if8: 5321; UNROLL-NEXT: [[TMP14:%.*]] = add i32 [[OFFSET_IDX]], -2 5322; UNROLL-NEXT: [[TMP15:%.*]] = udiv i32 219220132, [[TMP14]] 5323; UNROLL-NEXT: [[TMP16:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP15]], i64 2 5324; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE9]] 5325; UNROLL: pred.udiv.continue9: 5326; UNROLL-NEXT: [[TMP17:%.*]] = phi <4 x i32> [ [[TMP12]], [[PRED_UDIV_CONTINUE7]] ], [ [[TMP16]], [[PRED_UDIV_IF8]] ] 5327; UNROLL-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3 5328; UNROLL-NEXT: br i1 [[TMP18]], label [[PRED_UDIV_IF10:%.*]], label [[PRED_UDIV_CONTINUE11:%.*]] 5329; UNROLL: pred.udiv.if10: 5330; UNROLL-NEXT: [[TMP19:%.*]] = add i32 [[OFFSET_IDX]], -3 5331; UNROLL-NEXT: [[TMP20:%.*]] = udiv i32 219220132, [[TMP19]] 5332; UNROLL-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP20]], i64 3 5333; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE11]] 5334; UNROLL: pred.udiv.continue11: 5335; UNROLL-NEXT: [[TMP22:%.*]] = phi <4 x i32> [ [[TMP17]], [[PRED_UDIV_CONTINUE9]] ], [ [[TMP21]], [[PRED_UDIV_IF10]] ] 5336; UNROLL-NEXT: [[TMP23:%.*]] = extractelement <4 x i1> [[TMP3]], i64 0 5337; UNROLL-NEXT: br i1 [[TMP23]], label [[PRED_UDIV_IF12:%.*]], label [[PRED_UDIV_CONTINUE13:%.*]] 5338; UNROLL: pred.udiv.if12: 5339; UNROLL-NEXT: [[TMP24:%.*]] = add i32 [[OFFSET_IDX]], -4 5340; UNROLL-NEXT: [[TMP25:%.*]] = udiv i32 219220132, [[TMP24]] 5341; UNROLL-NEXT: [[TMP26:%.*]] = insertelement <4 x i32> poison, i32 [[TMP25]], i64 0 5342; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE13]] 5343; UNROLL: pred.udiv.continue13: 5344; UNROLL-NEXT: [[TMP27:%.*]] = phi <4 x i32> [ poison, [[PRED_UDIV_CONTINUE11]] ], [ [[TMP26]], [[PRED_UDIV_IF12]] ] 5345; UNROLL-NEXT: [[TMP28:%.*]] = extractelement <4 x i1> [[TMP3]], i64 1 5346; UNROLL-NEXT: br i1 [[TMP28]], label [[PRED_UDIV_IF14:%.*]], label [[PRED_UDIV_CONTINUE15:%.*]] 5347; UNROLL: pred.udiv.if14: 5348; UNROLL-NEXT: [[TMP29:%.*]] = add i32 [[OFFSET_IDX]], -5 5349; UNROLL-NEXT: [[TMP30:%.*]] = udiv i32 219220132, [[TMP29]] 5350; UNROLL-NEXT: [[TMP31:%.*]] = insertelement <4 x i32> [[TMP27]], i32 [[TMP30]], i64 1 5351; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE15]] 5352; UNROLL: pred.udiv.continue15: 5353; UNROLL-NEXT: [[TMP32:%.*]] = phi <4 x i32> [ [[TMP27]], [[PRED_UDIV_CONTINUE13]] ], [ [[TMP31]], [[PRED_UDIV_IF14]] ] 5354; UNROLL-NEXT: [[TMP33:%.*]] = extractelement <4 x i1> [[TMP3]], i64 2 5355; UNROLL-NEXT: br i1 [[TMP33]], label [[PRED_UDIV_IF16:%.*]], label [[PRED_UDIV_CONTINUE17:%.*]] 5356; UNROLL: pred.udiv.if16: 5357; UNROLL-NEXT: [[TMP34:%.*]] = add i32 [[OFFSET_IDX]], -6 5358; UNROLL-NEXT: [[TMP35:%.*]] = udiv i32 219220132, [[TMP34]] 5359; UNROLL-NEXT: [[TMP36:%.*]] = insertelement <4 x i32> [[TMP32]], i32 [[TMP35]], i64 2 5360; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE17]] 5361; UNROLL: pred.udiv.continue17: 5362; UNROLL-NEXT: [[TMP37:%.*]] = phi <4 x i32> [ [[TMP32]], [[PRED_UDIV_CONTINUE15]] ], [ [[TMP36]], [[PRED_UDIV_IF16]] ] 5363; UNROLL-NEXT: [[TMP38:%.*]] = extractelement <4 x i1> [[TMP3]], i64 3 5364; UNROLL-NEXT: br i1 [[TMP38]], label [[PRED_UDIV_IF18:%.*]], label [[PRED_UDIV_CONTINUE19]] 5365; UNROLL: pred.udiv.if18: 5366; UNROLL-NEXT: [[TMP39:%.*]] = add i32 [[OFFSET_IDX]], -7 5367; UNROLL-NEXT: [[TMP40:%.*]] = udiv i32 219220132, [[TMP39]] 5368; UNROLL-NEXT: [[TMP41:%.*]] = insertelement <4 x i32> [[TMP37]], i32 [[TMP40]], i64 3 5369; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE19]] 5370; UNROLL: pred.udiv.continue19: 5371; UNROLL-NEXT: [[TMP42]] = phi <4 x i32> [ [[TMP37]], [[PRED_UDIV_CONTINUE17]] ], [ [[TMP41]], [[PRED_UDIV_IF18]] ] 5372; UNROLL-NEXT: [[TMP43:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP22]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 5373; UNROLL-NEXT: [[TMP44:%.*]] = shufflevector <4 x i32> [[TMP22]], <4 x i32> [[TMP42]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 5374; UNROLL-NEXT: [[TMP45]] = add <4 x i32> [[VEC_PHI]], [[TMP43]] 5375; UNROLL-NEXT: [[TMP46]] = add <4 x i32> [[VEC_PHI2]], [[TMP44]] 5376; UNROLL-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8 5377; UNROLL-NEXT: [[TMP47:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 5378; UNROLL-NEXT: br i1 [[TMP47]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF52:![0-9]+]], !llvm.loop [[LOOP53:![0-9]+]] 5379; UNROLL: middle.block: 5380; UNROLL-NEXT: [[TMP48:%.*]] = select <4 x i1> [[TMP3]], <4 x i32> [[TMP46]], <4 x i32> [[VEC_PHI2]] 5381; UNROLL-NEXT: [[TMP49:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP45]], <4 x i32> [[VEC_PHI]] 5382; UNROLL-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP48]], [[TMP49]] 5383; UNROLL-NEXT: [[TMP50:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]]) 5384; UNROLL-NEXT: br i1 true, label [[BB1:%.*]], label [[SCALAR_PH]] 5385; UNROLL: scalar.ph: 5386; UNROLL-NEXT: br label [[BB2:%.*]] 5387; UNROLL: bb1: 5388; UNROLL-NEXT: [[VAR:%.*]] = phi i32 [ undef, [[BB2]] ], [ [[TMP50]], [[MIDDLE_BLOCK]] ] 5389; UNROLL-NEXT: ret i32 [[VAR]] 5390; UNROLL: bb2: 5391; UNROLL-NEXT: br i1 undef, label [[BB1]], label [[BB2]], !prof [[PROF54:![0-9]+]], !llvm.loop [[LOOP55:![0-9]+]] 5392; 5393; UNROLL-NO-IC-LABEL: @sink_into_replication_region( 5394; UNROLL-NO-IC-NEXT: bb: 5395; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i32 [[Y:%.*]], 1 5396; UNROLL-NO-IC-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[Y]], i32 1) 5397; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]] 5398; UNROLL-NO-IC-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5399; UNROLL-NO-IC: vector.ph: 5400; UNROLL-NO-IC-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 7 5401; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 8 5402; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]] 5403; UNROLL-NO-IC-NEXT: [[IND_END:%.*]] = sub i32 [[Y]], [[N_VEC]] 5404; UNROLL-NO-IC-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[TMP1]], 1 5405; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i32 0 5406; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 5407; UNROLL-NO-IC-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i32 0 5408; UNROLL-NO-IC-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 5409; UNROLL-NO-IC-NEXT: [[INDUCTION:%.*]] = add <4 x i32> [[DOTSPLAT]], <i32 0, i32 -1, i32 -2, i32 -3> 5410; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 5411; UNROLL-NO-IC: vector.body: 5412; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE19:%.*]] ] 5413; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UDIV_CONTINUE19]] ] 5414; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP43:%.*]], [[PRED_UDIV_CONTINUE19]] ] 5415; UNROLL-NO-IC-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP46:%.*]], [[PRED_UDIV_CONTINUE19]] ] 5416; UNROLL-NO-IC-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP47:%.*]], [[PRED_UDIV_CONTINUE19]] ] 5417; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], <i32 -4, i32 -4, i32 -4, i32 -4> 5418; UNROLL-NO-IC-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]] 5419; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i32> poison, i32 [[INDEX]], i32 0 5420; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT3]], <4 x i32> poison, <4 x i32> zeroinitializer 5421; UNROLL-NO-IC-NEXT: [[VEC_IV:%.*]] = add <4 x i32> [[BROADCAST_SPLAT4]], <i32 0, i32 1, i32 2, i32 3> 5422; UNROLL-NO-IC-NEXT: [[VEC_IV5:%.*]] = add <4 x i32> [[BROADCAST_SPLAT4]], <i32 4, i32 5, i32 6, i32 7> 5423; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = icmp ule <4 x i32> [[VEC_IV]], [[BROADCAST_SPLAT]] 5424; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = icmp ule <4 x i32> [[VEC_IV5]], [[BROADCAST_SPLAT]] 5425; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP2]], i32 0 5426; UNROLL-NO-IC-NEXT: br i1 [[TMP4]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 5427; UNROLL-NO-IC: pred.udiv.if: 5428; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = add i32 [[OFFSET_IDX]], 0 5429; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = udiv i32 219220132, [[TMP5]] 5430; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[TMP6]], i32 0 5431; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE]] 5432; UNROLL-NO-IC: pred.udiv.continue: 5433; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP7]], [[PRED_UDIV_IF]] ] 5434; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP2]], i32 1 5435; UNROLL-NO-IC-NEXT: br i1 [[TMP9]], label [[PRED_UDIV_IF6:%.*]], label [[PRED_UDIV_CONTINUE7:%.*]] 5436; UNROLL-NO-IC: pred.udiv.if6: 5437; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = add i32 [[OFFSET_IDX]], -1 5438; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = udiv i32 219220132, [[TMP10]] 5439; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP11]], i32 1 5440; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE7]] 5441; UNROLL-NO-IC: pred.udiv.continue7: 5442; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = phi <4 x i32> [ [[TMP8]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP12]], [[PRED_UDIV_IF6]] ] 5443; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP2]], i32 2 5444; UNROLL-NO-IC-NEXT: br i1 [[TMP14]], label [[PRED_UDIV_IF8:%.*]], label [[PRED_UDIV_CONTINUE9:%.*]] 5445; UNROLL-NO-IC: pred.udiv.if8: 5446; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = add i32 [[OFFSET_IDX]], -2 5447; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = udiv i32 219220132, [[TMP15]] 5448; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP16]], i32 2 5449; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE9]] 5450; UNROLL-NO-IC: pred.udiv.continue9: 5451; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = phi <4 x i32> [ [[TMP13]], [[PRED_UDIV_CONTINUE7]] ], [ [[TMP17]], [[PRED_UDIV_IF8]] ] 5452; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP2]], i32 3 5453; UNROLL-NO-IC-NEXT: br i1 [[TMP19]], label [[PRED_UDIV_IF10:%.*]], label [[PRED_UDIV_CONTINUE11:%.*]] 5454; UNROLL-NO-IC: pred.udiv.if10: 5455; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = add i32 [[OFFSET_IDX]], -3 5456; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = udiv i32 219220132, [[TMP20]] 5457; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = insertelement <4 x i32> [[TMP18]], i32 [[TMP21]], i32 3 5458; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE11]] 5459; UNROLL-NO-IC: pred.udiv.continue11: 5460; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = phi <4 x i32> [ [[TMP18]], [[PRED_UDIV_CONTINUE9]] ], [ [[TMP22]], [[PRED_UDIV_IF10]] ] 5461; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = extractelement <4 x i1> [[TMP3]], i32 0 5462; UNROLL-NO-IC-NEXT: br i1 [[TMP24]], label [[PRED_UDIV_IF12:%.*]], label [[PRED_UDIV_CONTINUE13:%.*]] 5463; UNROLL-NO-IC: pred.udiv.if12: 5464; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = add i32 [[OFFSET_IDX]], -4 5465; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = udiv i32 219220132, [[TMP25]] 5466; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = insertelement <4 x i32> poison, i32 [[TMP26]], i32 0 5467; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE13]] 5468; UNROLL-NO-IC: pred.udiv.continue13: 5469; UNROLL-NO-IC-NEXT: [[TMP28:%.*]] = phi <4 x i32> [ poison, [[PRED_UDIV_CONTINUE11]] ], [ [[TMP27]], [[PRED_UDIV_IF12]] ] 5470; UNROLL-NO-IC-NEXT: [[TMP29:%.*]] = extractelement <4 x i1> [[TMP3]], i32 1 5471; UNROLL-NO-IC-NEXT: br i1 [[TMP29]], label [[PRED_UDIV_IF14:%.*]], label [[PRED_UDIV_CONTINUE15:%.*]] 5472; UNROLL-NO-IC: pred.udiv.if14: 5473; UNROLL-NO-IC-NEXT: [[TMP30:%.*]] = add i32 [[OFFSET_IDX]], -5 5474; UNROLL-NO-IC-NEXT: [[TMP31:%.*]] = udiv i32 219220132, [[TMP30]] 5475; UNROLL-NO-IC-NEXT: [[TMP32:%.*]] = insertelement <4 x i32> [[TMP28]], i32 [[TMP31]], i32 1 5476; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE15]] 5477; UNROLL-NO-IC: pred.udiv.continue15: 5478; UNROLL-NO-IC-NEXT: [[TMP33:%.*]] = phi <4 x i32> [ [[TMP28]], [[PRED_UDIV_CONTINUE13]] ], [ [[TMP32]], [[PRED_UDIV_IF14]] ] 5479; UNROLL-NO-IC-NEXT: [[TMP34:%.*]] = extractelement <4 x i1> [[TMP3]], i32 2 5480; UNROLL-NO-IC-NEXT: br i1 [[TMP34]], label [[PRED_UDIV_IF16:%.*]], label [[PRED_UDIV_CONTINUE17:%.*]] 5481; UNROLL-NO-IC: pred.udiv.if16: 5482; UNROLL-NO-IC-NEXT: [[TMP35:%.*]] = add i32 [[OFFSET_IDX]], -6 5483; UNROLL-NO-IC-NEXT: [[TMP36:%.*]] = udiv i32 219220132, [[TMP35]] 5484; UNROLL-NO-IC-NEXT: [[TMP37:%.*]] = insertelement <4 x i32> [[TMP33]], i32 [[TMP36]], i32 2 5485; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE17]] 5486; UNROLL-NO-IC: pred.udiv.continue17: 5487; UNROLL-NO-IC-NEXT: [[TMP38:%.*]] = phi <4 x i32> [ [[TMP33]], [[PRED_UDIV_CONTINUE15]] ], [ [[TMP37]], [[PRED_UDIV_IF16]] ] 5488; UNROLL-NO-IC-NEXT: [[TMP39:%.*]] = extractelement <4 x i1> [[TMP3]], i32 3 5489; UNROLL-NO-IC-NEXT: br i1 [[TMP39]], label [[PRED_UDIV_IF18:%.*]], label [[PRED_UDIV_CONTINUE19]] 5490; UNROLL-NO-IC: pred.udiv.if18: 5491; UNROLL-NO-IC-NEXT: [[TMP40:%.*]] = add i32 [[OFFSET_IDX]], -7 5492; UNROLL-NO-IC-NEXT: [[TMP41:%.*]] = udiv i32 219220132, [[TMP40]] 5493; UNROLL-NO-IC-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> [[TMP38]], i32 [[TMP41]], i32 3 5494; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE19]] 5495; UNROLL-NO-IC: pred.udiv.continue19: 5496; UNROLL-NO-IC-NEXT: [[TMP43]] = phi <4 x i32> [ [[TMP38]], [[PRED_UDIV_CONTINUE17]] ], [ [[TMP42]], [[PRED_UDIV_IF18]] ] 5497; UNROLL-NO-IC-NEXT: [[TMP44:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP23]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 5498; UNROLL-NO-IC-NEXT: [[TMP45:%.*]] = shufflevector <4 x i32> [[TMP23]], <4 x i32> [[TMP43]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 5499; UNROLL-NO-IC-NEXT: [[TMP46]] = add <4 x i32> [[VEC_PHI]], [[TMP44]] 5500; UNROLL-NO-IC-NEXT: [[TMP47]] = add <4 x i32> [[VEC_PHI2]], [[TMP45]] 5501; UNROLL-NO-IC-NEXT: [[TMP48:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP46]], <4 x i32> [[VEC_PHI]] 5502; UNROLL-NO-IC-NEXT: [[TMP49:%.*]] = select <4 x i1> [[TMP3]], <4 x i32> [[TMP47]], <4 x i32> [[VEC_PHI2]] 5503; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8 5504; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], <i32 -4, i32 -4, i32 -4, i32 -4> 5505; UNROLL-NO-IC-NEXT: [[TMP50:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 5506; UNROLL-NO-IC-NEXT: br i1 [[TMP50]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF52:![0-9]+]], !llvm.loop [[LOOP53:![0-9]+]] 5507; UNROLL-NO-IC: middle.block: 5508; UNROLL-NO-IC-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP49]], [[TMP48]] 5509; UNROLL-NO-IC-NEXT: [[TMP51:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]]) 5510; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP43]], i32 3 5511; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP43]], i32 2 5512; UNROLL-NO-IC-NEXT: br i1 true, label [[BB1:%.*]], label [[SCALAR_PH]] 5513; UNROLL-NO-IC: scalar.ph: 5514; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[BB:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 5515; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[Y]], [[BB]] ] 5516; UNROLL-NO-IC-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[BB]] ], [ [[TMP51]], [[MIDDLE_BLOCK]] ] 5517; UNROLL-NO-IC-NEXT: br label [[BB2:%.*]] 5518; UNROLL-NO-IC: bb1: 5519; UNROLL-NO-IC-NEXT: [[VAR:%.*]] = phi i32 [ [[VAR6:%.*]], [[BB2]] ], [ [[TMP51]], [[MIDDLE_BLOCK]] ] 5520; UNROLL-NO-IC-NEXT: ret i32 [[VAR]] 5521; UNROLL-NO-IC: bb2: 5522; UNROLL-NO-IC-NEXT: [[VAR3:%.*]] = phi i32 [ [[VAR8:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 5523; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[VAR7:%.*]], [[BB2]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ] 5524; UNROLL-NO-IC-NEXT: [[VAR5:%.*]] = phi i32 [ [[VAR6]], [[BB2]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 5525; UNROLL-NO-IC-NEXT: [[VAR6]] = add i32 [[VAR5]], [[SCALAR_RECUR]] 5526; UNROLL-NO-IC-NEXT: [[VAR7]] = udiv i32 219220132, [[VAR3]] 5527; UNROLL-NO-IC-NEXT: [[VAR8]] = add nsw i32 [[VAR3]], -1 5528; UNROLL-NO-IC-NEXT: [[VAR9:%.*]] = icmp slt i32 [[VAR3]], 2 5529; UNROLL-NO-IC-NEXT: br i1 [[VAR9]], label [[BB1]], label [[BB2]], !prof [[PROF54:![0-9]+]], !llvm.loop [[LOOP55:![0-9]+]] 5530; 5531; UNROLL-NO-VF-LABEL: @sink_into_replication_region( 5532; UNROLL-NO-VF-NEXT: bb: 5533; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i32 [[Y:%.*]], 1 5534; UNROLL-NO-VF-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[Y]], i32 1) 5535; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]] 5536; UNROLL-NO-VF-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5537; UNROLL-NO-VF: vector.ph: 5538; UNROLL-NO-VF-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 1 5539; UNROLL-NO-VF-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2 5540; UNROLL-NO-VF-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]] 5541; UNROLL-NO-VF-NEXT: [[IND_END:%.*]] = sub i32 [[Y]], [[N_VEC]] 5542; UNROLL-NO-VF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[TMP1]], 1 5543; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 5544; UNROLL-NO-VF: vector.body: 5545; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE5:%.*]] ] 5546; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[PRED_UDIV_CONTINUE5]] ] 5547; UNROLL-NO-VF-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[PRED_UDIV_CONTINUE5]] ] 5548; UNROLL-NO-VF-NEXT: [[VEC_PHI2:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[PRED_UDIV_CONTINUE5]] ] 5549; UNROLL-NO-VF-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]] 5550; UNROLL-NO-VF-NEXT: [[VEC_IV:%.*]] = add i32 [[INDEX]], 0 5551; UNROLL-NO-VF-NEXT: [[VEC_IV3:%.*]] = add i32 [[INDEX]], 1 5552; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = icmp ule i32 [[VEC_IV]], [[TRIP_COUNT_MINUS_1]] 5553; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = icmp ule i32 [[VEC_IV3]], [[TRIP_COUNT_MINUS_1]] 5554; UNROLL-NO-VF-NEXT: br i1 [[TMP2]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 5555; UNROLL-NO-VF: pred.udiv.if: 5556; UNROLL-NO-VF-NEXT: [[INDUCTION:%.*]] = add i32 [[OFFSET_IDX]], 0 5557; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = udiv i32 219220132, [[INDUCTION]] 5558; UNROLL-NO-VF-NEXT: br label [[PRED_UDIV_CONTINUE]] 5559; UNROLL-NO-VF: pred.udiv.continue: 5560; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP4]], [[PRED_UDIV_IF]] ] 5561; UNROLL-NO-VF-NEXT: br i1 [[TMP3]], label [[PRED_UDIV_IF4:%.*]], label [[PRED_UDIV_CONTINUE5]] 5562; UNROLL-NO-VF: pred.udiv.if4: 5563; UNROLL-NO-VF-NEXT: [[INDUCTION1:%.*]] = add i32 [[OFFSET_IDX]], -1 5564; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = udiv i32 219220132, [[INDUCTION1]] 5565; UNROLL-NO-VF-NEXT: br label [[PRED_UDIV_CONTINUE5]] 5566; UNROLL-NO-VF: pred.udiv.continue5: 5567; UNROLL-NO-VF-NEXT: [[TMP7]] = phi i32 [ poison, [[PRED_UDIV_CONTINUE]] ], [ [[TMP6]], [[PRED_UDIV_IF4]] ] 5568; UNROLL-NO-VF-NEXT: [[TMP8]] = add i32 [[VEC_PHI]], [[VECTOR_RECUR]] 5569; UNROLL-NO-VF-NEXT: [[TMP9]] = add i32 [[VEC_PHI2]], [[TMP5]] 5570; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = select i1 [[TMP2]], i32 [[TMP8]], i32 [[VEC_PHI]] 5571; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = select i1 [[TMP3]], i32 [[TMP9]], i32 [[VEC_PHI2]] 5572; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2 5573; UNROLL-NO-VF-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 5574; UNROLL-NO-VF-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF51:![0-9]+]], !llvm.loop [[LOOP52:![0-9]+]] 5575; UNROLL-NO-VF: middle.block: 5576; UNROLL-NO-VF-NEXT: [[BIN_RDX:%.*]] = add i32 [[TMP11]], [[TMP10]] 5577; UNROLL-NO-VF-NEXT: br i1 true, label [[BB1:%.*]], label [[SCALAR_PH]] 5578; UNROLL-NO-VF: scalar.ph: 5579; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[BB:%.*]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ] 5580; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[Y]], [[BB]] ] 5581; UNROLL-NO-VF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[BB]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ] 5582; UNROLL-NO-VF-NEXT: br label [[BB2:%.*]] 5583; UNROLL-NO-VF: bb1: 5584; UNROLL-NO-VF-NEXT: [[VAR:%.*]] = phi i32 [ [[VAR6:%.*]], [[BB2]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ] 5585; UNROLL-NO-VF-NEXT: ret i32 [[VAR]] 5586; UNROLL-NO-VF: bb2: 5587; UNROLL-NO-VF-NEXT: [[VAR3:%.*]] = phi i32 [ [[VAR8:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 5588; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[VAR7:%.*]], [[BB2]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ] 5589; UNROLL-NO-VF-NEXT: [[VAR5:%.*]] = phi i32 [ [[VAR6]], [[BB2]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 5590; UNROLL-NO-VF-NEXT: [[VAR6]] = add i32 [[VAR5]], [[SCALAR_RECUR]] 5591; UNROLL-NO-VF-NEXT: [[VAR7]] = udiv i32 219220132, [[VAR3]] 5592; UNROLL-NO-VF-NEXT: [[VAR8]] = add nsw i32 [[VAR3]], -1 5593; UNROLL-NO-VF-NEXT: [[VAR9:%.*]] = icmp slt i32 [[VAR3]], 2 5594; UNROLL-NO-VF-NEXT: br i1 [[VAR9]], label [[BB1]], label [[BB2]], !prof [[PROF53:![0-9]+]], !llvm.loop [[LOOP54:![0-9]+]] 5595; 5596; SINK-AFTER-LABEL: @sink_into_replication_region( 5597; SINK-AFTER-NEXT: bb: 5598; SINK-AFTER-NEXT: [[TMP0:%.*]] = add i32 [[Y:%.*]], 1 5599; SINK-AFTER-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[Y]], i32 1) 5600; SINK-AFTER-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]] 5601; SINK-AFTER-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5602; SINK-AFTER: vector.ph: 5603; SINK-AFTER-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 3 5604; SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 4 5605; SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]] 5606; SINK-AFTER-NEXT: [[IND_END:%.*]] = sub i32 [[Y]], [[N_VEC]] 5607; SINK-AFTER-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[TMP1]], 1 5608; SINK-AFTER-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i32 0 5609; SINK-AFTER-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 5610; SINK-AFTER-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i32 0 5611; SINK-AFTER-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 5612; SINK-AFTER-NEXT: [[INDUCTION:%.*]] = add <4 x i32> [[DOTSPLAT]], <i32 0, i32 -1, i32 -2, i32 -3> 5613; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 5614; SINK-AFTER: vector.body: 5615; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE8:%.*]] ] 5616; SINK-AFTER-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UDIV_CONTINUE8]] ] 5617; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[PRED_UDIV_CONTINUE8]] ] 5618; SINK-AFTER-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[PRED_UDIV_CONTINUE8]] ] 5619; SINK-AFTER-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]] 5620; SINK-AFTER-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[INDEX]], i32 0 5621; SINK-AFTER-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer 5622; SINK-AFTER-NEXT: [[VEC_IV:%.*]] = add <4 x i32> [[BROADCAST_SPLAT2]], <i32 0, i32 1, i32 2, i32 3> 5623; SINK-AFTER-NEXT: [[TMP2:%.*]] = icmp ule <4 x i32> [[VEC_IV]], [[BROADCAST_SPLAT]] 5624; SINK-AFTER-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i32 0 5625; SINK-AFTER-NEXT: br i1 [[TMP3]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 5626; SINK-AFTER: pred.udiv.if: 5627; SINK-AFTER-NEXT: [[TMP4:%.*]] = add i32 [[OFFSET_IDX]], 0 5628; SINK-AFTER-NEXT: [[TMP5:%.*]] = udiv i32 219220132, [[TMP4]] 5629; SINK-AFTER-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i32 0 5630; SINK-AFTER-NEXT: br label [[PRED_UDIV_CONTINUE]] 5631; SINK-AFTER: pred.udiv.continue: 5632; SINK-AFTER-NEXT: [[TMP7:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_UDIV_IF]] ] 5633; SINK-AFTER-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP2]], i32 1 5634; SINK-AFTER-NEXT: br i1 [[TMP8]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]] 5635; SINK-AFTER: pred.udiv.if3: 5636; SINK-AFTER-NEXT: [[TMP9:%.*]] = add i32 [[OFFSET_IDX]], -1 5637; SINK-AFTER-NEXT: [[TMP10:%.*]] = udiv i32 219220132, [[TMP9]] 5638; SINK-AFTER-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[TMP10]], i32 1 5639; SINK-AFTER-NEXT: br label [[PRED_UDIV_CONTINUE4]] 5640; SINK-AFTER: pred.udiv.continue4: 5641; SINK-AFTER-NEXT: [[TMP12:%.*]] = phi <4 x i32> [ [[TMP7]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP11]], [[PRED_UDIV_IF3]] ] 5642; SINK-AFTER-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP2]], i32 2 5643; SINK-AFTER-NEXT: br i1 [[TMP13]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]] 5644; SINK-AFTER: pred.udiv.if5: 5645; SINK-AFTER-NEXT: [[TMP14:%.*]] = add i32 [[OFFSET_IDX]], -2 5646; SINK-AFTER-NEXT: [[TMP15:%.*]] = udiv i32 219220132, [[TMP14]] 5647; SINK-AFTER-NEXT: [[TMP16:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP15]], i32 2 5648; SINK-AFTER-NEXT: br label [[PRED_UDIV_CONTINUE6]] 5649; SINK-AFTER: pred.udiv.continue6: 5650; SINK-AFTER-NEXT: [[TMP17:%.*]] = phi <4 x i32> [ [[TMP12]], [[PRED_UDIV_CONTINUE4]] ], [ [[TMP16]], [[PRED_UDIV_IF5]] ] 5651; SINK-AFTER-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[TMP2]], i32 3 5652; SINK-AFTER-NEXT: br i1 [[TMP18]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8]] 5653; SINK-AFTER: pred.udiv.if7: 5654; SINK-AFTER-NEXT: [[TMP19:%.*]] = add i32 [[OFFSET_IDX]], -3 5655; SINK-AFTER-NEXT: [[TMP20:%.*]] = udiv i32 219220132, [[TMP19]] 5656; SINK-AFTER-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP20]], i32 3 5657; SINK-AFTER-NEXT: br label [[PRED_UDIV_CONTINUE8]] 5658; SINK-AFTER: pred.udiv.continue8: 5659; SINK-AFTER-NEXT: [[TMP22]] = phi <4 x i32> [ [[TMP17]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP21]], [[PRED_UDIV_IF7]] ] 5660; SINK-AFTER-NEXT: [[TMP23:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP22]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 5661; SINK-AFTER-NEXT: [[TMP24]] = add <4 x i32> [[VEC_PHI]], [[TMP23]] 5662; SINK-AFTER-NEXT: [[TMP25:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP24]], <4 x i32> [[VEC_PHI]] 5663; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4 5664; SINK-AFTER-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 -4, i32 -4, i32 -4, i32 -4> 5665; SINK-AFTER-NEXT: [[TMP26:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 5666; SINK-AFTER-NEXT: br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF52:![0-9]+]], !llvm.loop [[LOOP53:![0-9]+]] 5667; SINK-AFTER: middle.block: 5668; SINK-AFTER-NEXT: [[TMP27:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP25]]) 5669; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP22]], i32 3 5670; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP22]], i32 2 5671; SINK-AFTER-NEXT: br i1 true, label [[BB1:%.*]], label [[SCALAR_PH]] 5672; SINK-AFTER: scalar.ph: 5673; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[BB:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 5674; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[Y]], [[BB]] ] 5675; SINK-AFTER-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[BB]] ], [ [[TMP27]], [[MIDDLE_BLOCK]] ] 5676; SINK-AFTER-NEXT: br label [[BB2:%.*]] 5677; SINK-AFTER: bb1: 5678; SINK-AFTER-NEXT: [[VAR:%.*]] = phi i32 [ [[VAR6:%.*]], [[BB2]] ], [ [[TMP27]], [[MIDDLE_BLOCK]] ] 5679; SINK-AFTER-NEXT: ret i32 [[VAR]] 5680; SINK-AFTER: bb2: 5681; SINK-AFTER-NEXT: [[VAR3:%.*]] = phi i32 [ [[VAR8:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 5682; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[VAR7:%.*]], [[BB2]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ] 5683; SINK-AFTER-NEXT: [[VAR5:%.*]] = phi i32 [ [[VAR6]], [[BB2]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 5684; SINK-AFTER-NEXT: [[VAR6]] = add i32 [[VAR5]], [[SCALAR_RECUR]] 5685; SINK-AFTER-NEXT: [[VAR7]] = udiv i32 219220132, [[VAR3]] 5686; SINK-AFTER-NEXT: [[VAR8]] = add nsw i32 [[VAR3]], -1 5687; SINK-AFTER-NEXT: [[VAR9:%.*]] = icmp slt i32 [[VAR3]], 2 5688; SINK-AFTER-NEXT: br i1 [[VAR9]], label [[BB1]], label [[BB2]], !prof [[PROF54:![0-9]+]], !llvm.loop [[LOOP55:![0-9]+]] 5689; 5690; NO-SINK-AFTER-LABEL: @sink_into_replication_region( 5691; NO-SINK-AFTER-NEXT: bb: 5692; NO-SINK-AFTER-NEXT: [[TMP0:%.*]] = add i32 [[Y:%.*]], 1 5693; NO-SINK-AFTER-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[Y]], i32 1) 5694; NO-SINK-AFTER-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]] 5695; NO-SINK-AFTER-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5696; NO-SINK-AFTER: vector.ph: 5697; NO-SINK-AFTER-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 3 5698; NO-SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 4 5699; NO-SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]] 5700; NO-SINK-AFTER-NEXT: [[IND_END:%.*]] = sub i32 [[Y]], [[N_VEC]] 5701; NO-SINK-AFTER-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[TMP1]], 1 5702; NO-SINK-AFTER-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i32 0 5703; NO-SINK-AFTER-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 5704; NO-SINK-AFTER-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i32 0 5705; NO-SINK-AFTER-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 5706; NO-SINK-AFTER-NEXT: [[INDUCTION:%.*]] = add <4 x i32> [[DOTSPLAT]], <i32 0, i32 -1, i32 -2, i32 -3> 5707; NO-SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 5708; NO-SINK-AFTER: vector.body: 5709; NO-SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE8:%.*]] ] 5710; NO-SINK-AFTER-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UDIV_CONTINUE8]] ] 5711; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[PRED_UDIV_CONTINUE8]] ] 5712; NO-SINK-AFTER-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[PRED_UDIV_CONTINUE8]] ] 5713; NO-SINK-AFTER-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]] 5714; NO-SINK-AFTER-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[INDEX]], i32 0 5715; NO-SINK-AFTER-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer 5716; NO-SINK-AFTER-NEXT: [[VEC_IV:%.*]] = add <4 x i32> [[BROADCAST_SPLAT2]], <i32 0, i32 1, i32 2, i32 3> 5717; NO-SINK-AFTER-NEXT: [[TMP2:%.*]] = icmp ule <4 x i32> [[VEC_IV]], [[BROADCAST_SPLAT]] 5718; NO-SINK-AFTER-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i32 0 5719; NO-SINK-AFTER-NEXT: br i1 [[TMP3]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 5720; NO-SINK-AFTER: pred.udiv.if: 5721; NO-SINK-AFTER-NEXT: [[TMP4:%.*]] = add i32 [[OFFSET_IDX]], 0 5722; NO-SINK-AFTER-NEXT: [[TMP5:%.*]] = udiv i32 219220132, [[TMP4]] 5723; NO-SINK-AFTER-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i32 0 5724; NO-SINK-AFTER-NEXT: br label [[PRED_UDIV_CONTINUE]] 5725; NO-SINK-AFTER: pred.udiv.continue: 5726; NO-SINK-AFTER-NEXT: [[TMP7:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_UDIV_IF]] ] 5727; NO-SINK-AFTER-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP2]], i32 1 5728; NO-SINK-AFTER-NEXT: br i1 [[TMP8]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]] 5729; NO-SINK-AFTER: pred.udiv.if3: 5730; NO-SINK-AFTER-NEXT: [[TMP9:%.*]] = add i32 [[OFFSET_IDX]], -1 5731; NO-SINK-AFTER-NEXT: [[TMP10:%.*]] = udiv i32 219220132, [[TMP9]] 5732; NO-SINK-AFTER-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[TMP10]], i32 1 5733; NO-SINK-AFTER-NEXT: br label [[PRED_UDIV_CONTINUE4]] 5734; NO-SINK-AFTER: pred.udiv.continue4: 5735; NO-SINK-AFTER-NEXT: [[TMP12:%.*]] = phi <4 x i32> [ [[TMP7]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP11]], [[PRED_UDIV_IF3]] ] 5736; NO-SINK-AFTER-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP2]], i32 2 5737; NO-SINK-AFTER-NEXT: br i1 [[TMP13]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]] 5738; NO-SINK-AFTER: pred.udiv.if5: 5739; NO-SINK-AFTER-NEXT: [[TMP14:%.*]] = add i32 [[OFFSET_IDX]], -2 5740; NO-SINK-AFTER-NEXT: [[TMP15:%.*]] = udiv i32 219220132, [[TMP14]] 5741; NO-SINK-AFTER-NEXT: [[TMP16:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP15]], i32 2 5742; NO-SINK-AFTER-NEXT: br label [[PRED_UDIV_CONTINUE6]] 5743; NO-SINK-AFTER: pred.udiv.continue6: 5744; NO-SINK-AFTER-NEXT: [[TMP17:%.*]] = phi <4 x i32> [ [[TMP12]], [[PRED_UDIV_CONTINUE4]] ], [ [[TMP16]], [[PRED_UDIV_IF5]] ] 5745; NO-SINK-AFTER-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[TMP2]], i32 3 5746; NO-SINK-AFTER-NEXT: br i1 [[TMP18]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8]] 5747; NO-SINK-AFTER: pred.udiv.if7: 5748; NO-SINK-AFTER-NEXT: [[TMP19:%.*]] = add i32 [[OFFSET_IDX]], -3 5749; NO-SINK-AFTER-NEXT: [[TMP20:%.*]] = udiv i32 219220132, [[TMP19]] 5750; NO-SINK-AFTER-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP20]], i32 3 5751; NO-SINK-AFTER-NEXT: br label [[PRED_UDIV_CONTINUE8]] 5752; NO-SINK-AFTER: pred.udiv.continue8: 5753; NO-SINK-AFTER-NEXT: [[TMP22]] = phi <4 x i32> [ [[TMP17]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP21]], [[PRED_UDIV_IF7]] ] 5754; NO-SINK-AFTER-NEXT: [[TMP23:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP22]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 5755; NO-SINK-AFTER-NEXT: [[TMP24]] = add <4 x i32> [[VEC_PHI]], [[TMP23]] 5756; NO-SINK-AFTER-NEXT: [[TMP25:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP24]], <4 x i32> [[VEC_PHI]] 5757; NO-SINK-AFTER-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4 5758; NO-SINK-AFTER-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 -4, i32 -4, i32 -4, i32 -4> 5759; NO-SINK-AFTER-NEXT: [[TMP26:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 5760; NO-SINK-AFTER-NEXT: br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF52:![0-9]+]], !llvm.loop [[LOOP53:![0-9]+]] 5761; NO-SINK-AFTER: middle.block: 5762; NO-SINK-AFTER-NEXT: [[TMP27:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP25]]) 5763; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP22]], i32 3 5764; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP22]], i32 2 5765; NO-SINK-AFTER-NEXT: br i1 true, label [[BB1:%.*]], label [[SCALAR_PH]] 5766; NO-SINK-AFTER: scalar.ph: 5767; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[BB:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 5768; NO-SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[Y]], [[BB]] ] 5769; NO-SINK-AFTER-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[BB]] ], [ [[TMP27]], [[MIDDLE_BLOCK]] ] 5770; NO-SINK-AFTER-NEXT: br label [[BB2:%.*]] 5771; NO-SINK-AFTER: bb1: 5772; NO-SINK-AFTER-NEXT: [[VAR:%.*]] = phi i32 [ [[VAR6:%.*]], [[BB2]] ], [ [[TMP27]], [[MIDDLE_BLOCK]] ] 5773; NO-SINK-AFTER-NEXT: ret i32 [[VAR]] 5774; NO-SINK-AFTER: bb2: 5775; NO-SINK-AFTER-NEXT: [[VAR3:%.*]] = phi i32 [ [[VAR8:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 5776; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[VAR7:%.*]], [[BB2]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ] 5777; NO-SINK-AFTER-NEXT: [[VAR5:%.*]] = phi i32 [ [[VAR6]], [[BB2]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 5778; NO-SINK-AFTER-NEXT: [[VAR6]] = add i32 [[VAR5]], [[SCALAR_RECUR]] 5779; NO-SINK-AFTER-NEXT: [[VAR7]] = udiv i32 219220132, [[VAR3]] 5780; NO-SINK-AFTER-NEXT: [[VAR8]] = add nsw i32 [[VAR3]], -1 5781; NO-SINK-AFTER-NEXT: [[VAR9:%.*]] = icmp slt i32 [[VAR3]], 2 5782; NO-SINK-AFTER-NEXT: br i1 [[VAR9]], label [[BB1]], label [[BB2]], !prof [[PROF54:![0-9]+]], !llvm.loop [[LOOP55:![0-9]+]] 5783; 5784bb: 5785 br label %bb2 5786 5787 bb1: ; preds = %bb2 5788 %var = phi i32 [ %var6, %bb2 ] 5789 ret i32 %var 5790 5791 bb2: ; preds = %bb2, %bb 5792 %var3 = phi i32 [ %var8, %bb2 ], [ %y, %bb ] 5793 %var4 = phi i32 [ %var7, %bb2 ], [ 0, %bb ] 5794 %var5 = phi i32 [ %var6, %bb2 ], [ 0, %bb ] 5795 %var6 = add i32 %var5, %var4 5796 %var7 = udiv i32 219220132, %var3 5797 %var8 = add nsw i32 %var3, -1 5798 %var9 = icmp slt i32 %var3, 2 5799 br i1 %var9, label %bb1, label %bb2, !prof !2 5800} 5801 5802define i32 @sink_into_replication_region_multiple(i32 *%x, i32 %y) { 5803; 5804; CHECK-LABEL: @sink_into_replication_region_multiple( 5805; CHECK-NEXT: bb: 5806; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[Y:%.*]], 1 5807; CHECK-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[Y]], i32 1) 5808; CHECK-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]] 5809; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5810; CHECK: vector.ph: 5811; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 3 5812; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -4 5813; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = add i32 [[TMP1]], -1 5814; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0 5815; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 5816; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 5817; CHECK: vector.body: 5818; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE15:%.*]] ] 5819; CHECK-NEXT: [[VEC_IND2:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[PRED_STORE_CONTINUE15]] ] 5820; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[PRED_STORE_CONTINUE15]] ] 5821; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[PRED_STORE_CONTINUE15]] ] 5822; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]] 5823; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[OFFSET_IDX]], -1 5824; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[OFFSET_IDX]], -2 5825; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[OFFSET_IDX]], -3 5826; CHECK-NEXT: [[TMP5:%.*]] = icmp ule <4 x i32> [[VEC_IND2]], [[BROADCAST_SPLAT]] 5827; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i64 0 5828; CHECK-NEXT: br i1 [[TMP6]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 5829; CHECK: pred.udiv.if: 5830; CHECK-NEXT: [[TMP7:%.*]] = udiv i32 219220132, [[OFFSET_IDX]] 5831; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x i32> poison, i32 [[TMP7]], i64 0 5832; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE]] 5833; CHECK: pred.udiv.continue: 5834; CHECK-NEXT: [[TMP9:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_UDIV_IF]] ] 5835; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP5]], i64 1 5836; CHECK-NEXT: br i1 [[TMP10]], label [[PRED_UDIV_IF4:%.*]], label [[PRED_UDIV_CONTINUE5:%.*]] 5837; CHECK: pred.udiv.if4: 5838; CHECK-NEXT: [[TMP11:%.*]] = udiv i32 219220132, [[TMP2]] 5839; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP9]], i32 [[TMP11]], i64 1 5840; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE5]] 5841; CHECK: pred.udiv.continue5: 5842; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x i32> [ [[TMP9]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP12]], [[PRED_UDIV_IF4]] ] 5843; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP5]], i64 2 5844; CHECK-NEXT: br i1 [[TMP14]], label [[PRED_UDIV_IF6:%.*]], label [[PRED_UDIV_CONTINUE7:%.*]] 5845; CHECK: pred.udiv.if6: 5846; CHECK-NEXT: [[TMP15:%.*]] = udiv i32 219220132, [[TMP3]] 5847; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP15]], i64 2 5848; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE7]] 5849; CHECK: pred.udiv.continue7: 5850; CHECK-NEXT: [[TMP17:%.*]] = phi <4 x i32> [ [[TMP13]], [[PRED_UDIV_CONTINUE5]] ], [ [[TMP16]], [[PRED_UDIV_IF6]] ] 5851; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[TMP5]], i64 3 5852; CHECK-NEXT: br i1 [[TMP18]], label [[PRED_UDIV_IF8:%.*]], label [[PRED_UDIV_CONTINUE9:%.*]] 5853; CHECK: pred.udiv.if8: 5854; CHECK-NEXT: [[TMP19:%.*]] = udiv i32 219220132, [[TMP4]] 5855; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP19]], i64 3 5856; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE9]] 5857; CHECK: pred.udiv.continue9: 5858; CHECK-NEXT: [[TMP21]] = phi <4 x i32> [ [[TMP17]], [[PRED_UDIV_CONTINUE7]] ], [ [[TMP20]], [[PRED_UDIV_IF8]] ] 5859; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP21]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 5860; CHECK-NEXT: [[TMP23]] = add <4 x i32> [[VEC_PHI]], [[TMP22]] 5861; CHECK-NEXT: [[TMP24:%.*]] = extractelement <4 x i1> [[TMP5]], i64 0 5862; CHECK-NEXT: br i1 [[TMP24]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] 5863; CHECK: pred.store.if: 5864; CHECK-NEXT: [[TMP25:%.*]] = sext i32 [[INDEX]] to i64 5865; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 [[TMP25]] 5866; CHECK-NEXT: store i32 [[OFFSET_IDX]], i32* [[TMP26]], align 4 5867; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]] 5868; CHECK: pred.store.continue: 5869; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i1> [[TMP5]], i64 1 5870; CHECK-NEXT: br i1 [[TMP27]], label [[PRED_STORE_IF10:%.*]], label [[PRED_STORE_CONTINUE11:%.*]] 5871; CHECK: pred.store.if10: 5872; CHECK-NEXT: [[TMP28:%.*]] = or i32 [[INDEX]], 1 5873; CHECK-NEXT: [[TMP29:%.*]] = sext i32 [[TMP28]] to i64 5874; CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 [[TMP29]] 5875; CHECK-NEXT: store i32 [[TMP2]], i32* [[TMP30]], align 4 5876; CHECK-NEXT: br label [[PRED_STORE_CONTINUE11]] 5877; CHECK: pred.store.continue11: 5878; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x i1> [[TMP5]], i64 2 5879; CHECK-NEXT: br i1 [[TMP31]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE13:%.*]] 5880; CHECK: pred.store.if12: 5881; CHECK-NEXT: [[TMP32:%.*]] = or i32 [[INDEX]], 2 5882; CHECK-NEXT: [[TMP33:%.*]] = sext i32 [[TMP32]] to i64 5883; CHECK-NEXT: [[TMP34:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 [[TMP33]] 5884; CHECK-NEXT: store i32 [[TMP3]], i32* [[TMP34]], align 4 5885; CHECK-NEXT: br label [[PRED_STORE_CONTINUE13]] 5886; CHECK: pred.store.continue13: 5887; CHECK-NEXT: [[TMP35:%.*]] = extractelement <4 x i1> [[TMP5]], i64 3 5888; CHECK-NEXT: br i1 [[TMP35]], label [[PRED_STORE_IF14:%.*]], label [[PRED_STORE_CONTINUE15]] 5889; CHECK: pred.store.if14: 5890; CHECK-NEXT: [[TMP36:%.*]] = or i32 [[INDEX]], 3 5891; CHECK-NEXT: [[TMP37:%.*]] = sext i32 [[TMP36]] to i64 5892; CHECK-NEXT: [[TMP38:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 [[TMP37]] 5893; CHECK-NEXT: store i32 [[TMP4]], i32* [[TMP38]], align 4 5894; CHECK-NEXT: br label [[PRED_STORE_CONTINUE15]] 5895; CHECK: pred.store.continue15: 5896; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4 5897; CHECK-NEXT: [[VEC_IND_NEXT3]] = add <4 x i32> [[VEC_IND2]], <i32 4, i32 4, i32 4, i32 4> 5898; CHECK-NEXT: [[TMP39:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 5899; CHECK-NEXT: br i1 [[TMP39]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF52]], !llvm.loop [[LOOP56:![0-9]+]] 5900; CHECK: middle.block: 5901; CHECK-NEXT: [[TMP40:%.*]] = select <4 x i1> [[TMP5]], <4 x i32> [[TMP23]], <4 x i32> [[VEC_PHI]] 5902; CHECK-NEXT: [[TMP41:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP40]]) 5903; CHECK-NEXT: br i1 true, label [[BB1:%.*]], label [[SCALAR_PH]] 5904; CHECK: scalar.ph: 5905; CHECK-NEXT: br label [[BB2:%.*]] 5906; CHECK: bb1: 5907; CHECK-NEXT: [[VAR:%.*]] = phi i32 [ undef, [[BB2]] ], [ [[TMP41]], [[MIDDLE_BLOCK]] ] 5908; CHECK-NEXT: ret i32 [[VAR]] 5909; CHECK: bb2: 5910; CHECK-NEXT: br i1 undef, label [[BB1]], label [[BB2]], !prof [[PROF54]], !llvm.loop [[LOOP57:![0-9]+]] 5911; 5912; UNROLL-LABEL: @sink_into_replication_region_multiple( 5913; UNROLL-NEXT: bb: 5914; UNROLL-NEXT: [[TMP0:%.*]] = add i32 [[Y:%.*]], 1 5915; UNROLL-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[Y]], i32 1) 5916; UNROLL-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]] 5917; UNROLL-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5918; UNROLL: vector.ph: 5919; UNROLL-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 7 5920; UNROLL-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8 5921; UNROLL-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = add i32 [[TMP1]], -1 5922; UNROLL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0 5923; UNROLL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 5924; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 5925; UNROLL: vector.body: 5926; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE35:%.*]] ] 5927; UNROLL-NEXT: [[VEC_IND3:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT6:%.*]], [[PRED_STORE_CONTINUE35]] ] 5928; UNROLL-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP42:%.*]], [[PRED_STORE_CONTINUE35]] ] 5929; UNROLL-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP45:%.*]], [[PRED_STORE_CONTINUE35]] ] 5930; UNROLL-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP46:%.*]], [[PRED_STORE_CONTINUE35]] ] 5931; UNROLL-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]] 5932; UNROLL-NEXT: [[TMP2:%.*]] = add i32 [[OFFSET_IDX]], -1 5933; UNROLL-NEXT: [[TMP3:%.*]] = add i32 [[OFFSET_IDX]], -2 5934; UNROLL-NEXT: [[TMP4:%.*]] = add i32 [[OFFSET_IDX]], -3 5935; UNROLL-NEXT: [[TMP5:%.*]] = add i32 [[OFFSET_IDX]], -4 5936; UNROLL-NEXT: [[TMP6:%.*]] = add i32 [[OFFSET_IDX]], -5 5937; UNROLL-NEXT: [[TMP7:%.*]] = add i32 [[OFFSET_IDX]], -6 5938; UNROLL-NEXT: [[TMP8:%.*]] = add i32 [[OFFSET_IDX]], -7 5939; UNROLL-NEXT: [[STEP_ADD4:%.*]] = add <4 x i32> [[VEC_IND3]], <i32 4, i32 4, i32 4, i32 4> 5940; UNROLL-NEXT: [[TMP9:%.*]] = icmp ule <4 x i32> [[VEC_IND3]], [[BROADCAST_SPLAT]] 5941; UNROLL-NEXT: [[TMP10:%.*]] = icmp ule <4 x i32> [[STEP_ADD4]], [[BROADCAST_SPLAT]] 5942; UNROLL-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP9]], i64 0 5943; UNROLL-NEXT: br i1 [[TMP11]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 5944; UNROLL: pred.udiv.if: 5945; UNROLL-NEXT: [[TMP12:%.*]] = udiv i32 219220132, [[OFFSET_IDX]] 5946; UNROLL-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> poison, i32 [[TMP12]], i64 0 5947; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE]] 5948; UNROLL: pred.udiv.continue: 5949; UNROLL-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP13]], [[PRED_UDIV_IF]] ] 5950; UNROLL-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP9]], i64 1 5951; UNROLL-NEXT: br i1 [[TMP15]], label [[PRED_UDIV_IF8:%.*]], label [[PRED_UDIV_CONTINUE9:%.*]] 5952; UNROLL: pred.udiv.if8: 5953; UNROLL-NEXT: [[TMP16:%.*]] = udiv i32 219220132, [[TMP2]] 5954; UNROLL-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP16]], i64 1 5955; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE9]] 5956; UNROLL: pred.udiv.continue9: 5957; UNROLL-NEXT: [[TMP18:%.*]] = phi <4 x i32> [ [[TMP14]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP17]], [[PRED_UDIV_IF8]] ] 5958; UNROLL-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP9]], i64 2 5959; UNROLL-NEXT: br i1 [[TMP19]], label [[PRED_UDIV_IF10:%.*]], label [[PRED_UDIV_CONTINUE11:%.*]] 5960; UNROLL: pred.udiv.if10: 5961; UNROLL-NEXT: [[TMP20:%.*]] = udiv i32 219220132, [[TMP3]] 5962; UNROLL-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP18]], i32 [[TMP20]], i64 2 5963; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE11]] 5964; UNROLL: pred.udiv.continue11: 5965; UNROLL-NEXT: [[TMP22:%.*]] = phi <4 x i32> [ [[TMP18]], [[PRED_UDIV_CONTINUE9]] ], [ [[TMP21]], [[PRED_UDIV_IF10]] ] 5966; UNROLL-NEXT: [[TMP23:%.*]] = extractelement <4 x i1> [[TMP9]], i64 3 5967; UNROLL-NEXT: br i1 [[TMP23]], label [[PRED_UDIV_IF12:%.*]], label [[PRED_UDIV_CONTINUE13:%.*]] 5968; UNROLL: pred.udiv.if12: 5969; UNROLL-NEXT: [[TMP24:%.*]] = udiv i32 219220132, [[TMP4]] 5970; UNROLL-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP22]], i32 [[TMP24]], i64 3 5971; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE13]] 5972; UNROLL: pred.udiv.continue13: 5973; UNROLL-NEXT: [[TMP26:%.*]] = phi <4 x i32> [ [[TMP22]], [[PRED_UDIV_CONTINUE11]] ], [ [[TMP25]], [[PRED_UDIV_IF12]] ] 5974; UNROLL-NEXT: [[TMP27:%.*]] = extractelement <4 x i1> [[TMP10]], i64 0 5975; UNROLL-NEXT: br i1 [[TMP27]], label [[PRED_UDIV_IF14:%.*]], label [[PRED_UDIV_CONTINUE15:%.*]] 5976; UNROLL: pred.udiv.if14: 5977; UNROLL-NEXT: [[TMP28:%.*]] = udiv i32 219220132, [[TMP5]] 5978; UNROLL-NEXT: [[TMP29:%.*]] = insertelement <4 x i32> poison, i32 [[TMP28]], i64 0 5979; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE15]] 5980; UNROLL: pred.udiv.continue15: 5981; UNROLL-NEXT: [[TMP30:%.*]] = phi <4 x i32> [ poison, [[PRED_UDIV_CONTINUE13]] ], [ [[TMP29]], [[PRED_UDIV_IF14]] ] 5982; UNROLL-NEXT: [[TMP31:%.*]] = extractelement <4 x i1> [[TMP10]], i64 1 5983; UNROLL-NEXT: br i1 [[TMP31]], label [[PRED_UDIV_IF16:%.*]], label [[PRED_UDIV_CONTINUE17:%.*]] 5984; UNROLL: pred.udiv.if16: 5985; UNROLL-NEXT: [[TMP32:%.*]] = udiv i32 219220132, [[TMP6]] 5986; UNROLL-NEXT: [[TMP33:%.*]] = insertelement <4 x i32> [[TMP30]], i32 [[TMP32]], i64 1 5987; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE17]] 5988; UNROLL: pred.udiv.continue17: 5989; UNROLL-NEXT: [[TMP34:%.*]] = phi <4 x i32> [ [[TMP30]], [[PRED_UDIV_CONTINUE15]] ], [ [[TMP33]], [[PRED_UDIV_IF16]] ] 5990; UNROLL-NEXT: [[TMP35:%.*]] = extractelement <4 x i1> [[TMP10]], i64 2 5991; UNROLL-NEXT: br i1 [[TMP35]], label [[PRED_UDIV_IF18:%.*]], label [[PRED_UDIV_CONTINUE19:%.*]] 5992; UNROLL: pred.udiv.if18: 5993; UNROLL-NEXT: [[TMP36:%.*]] = udiv i32 219220132, [[TMP7]] 5994; UNROLL-NEXT: [[TMP37:%.*]] = insertelement <4 x i32> [[TMP34]], i32 [[TMP36]], i64 2 5995; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE19]] 5996; UNROLL: pred.udiv.continue19: 5997; UNROLL-NEXT: [[TMP38:%.*]] = phi <4 x i32> [ [[TMP34]], [[PRED_UDIV_CONTINUE17]] ], [ [[TMP37]], [[PRED_UDIV_IF18]] ] 5998; UNROLL-NEXT: [[TMP39:%.*]] = extractelement <4 x i1> [[TMP10]], i64 3 5999; UNROLL-NEXT: br i1 [[TMP39]], label [[PRED_UDIV_IF20:%.*]], label [[PRED_UDIV_CONTINUE21:%.*]] 6000; UNROLL: pred.udiv.if20: 6001; UNROLL-NEXT: [[TMP40:%.*]] = udiv i32 219220132, [[TMP8]] 6002; UNROLL-NEXT: [[TMP41:%.*]] = insertelement <4 x i32> [[TMP38]], i32 [[TMP40]], i64 3 6003; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE21]] 6004; UNROLL: pred.udiv.continue21: 6005; UNROLL-NEXT: [[TMP42]] = phi <4 x i32> [ [[TMP38]], [[PRED_UDIV_CONTINUE19]] ], [ [[TMP41]], [[PRED_UDIV_IF20]] ] 6006; UNROLL-NEXT: [[TMP43:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP26]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 6007; UNROLL-NEXT: [[TMP44:%.*]] = shufflevector <4 x i32> [[TMP26]], <4 x i32> [[TMP42]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 6008; UNROLL-NEXT: [[TMP45]] = add <4 x i32> [[VEC_PHI]], [[TMP43]] 6009; UNROLL-NEXT: [[TMP46]] = add <4 x i32> [[VEC_PHI7]], [[TMP44]] 6010; UNROLL-NEXT: [[TMP47:%.*]] = extractelement <4 x i1> [[TMP9]], i64 0 6011; UNROLL-NEXT: br i1 [[TMP47]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] 6012; UNROLL: pred.store.if: 6013; UNROLL-NEXT: [[TMP48:%.*]] = sext i32 [[INDEX]] to i64 6014; UNROLL-NEXT: [[TMP49:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 [[TMP48]] 6015; UNROLL-NEXT: store i32 [[OFFSET_IDX]], i32* [[TMP49]], align 4 6016; UNROLL-NEXT: br label [[PRED_STORE_CONTINUE]] 6017; UNROLL: pred.store.continue: 6018; UNROLL-NEXT: [[TMP50:%.*]] = extractelement <4 x i1> [[TMP9]], i64 1 6019; UNROLL-NEXT: br i1 [[TMP50]], label [[PRED_STORE_IF22:%.*]], label [[PRED_STORE_CONTINUE23:%.*]] 6020; UNROLL: pred.store.if22: 6021; UNROLL-NEXT: [[TMP51:%.*]] = or i32 [[INDEX]], 1 6022; UNROLL-NEXT: [[TMP52:%.*]] = sext i32 [[TMP51]] to i64 6023; UNROLL-NEXT: [[TMP53:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 [[TMP52]] 6024; UNROLL-NEXT: store i32 [[TMP2]], i32* [[TMP53]], align 4 6025; UNROLL-NEXT: br label [[PRED_STORE_CONTINUE23]] 6026; UNROLL: pred.store.continue23: 6027; UNROLL-NEXT: [[TMP54:%.*]] = extractelement <4 x i1> [[TMP9]], i64 2 6028; UNROLL-NEXT: br i1 [[TMP54]], label [[PRED_STORE_IF24:%.*]], label [[PRED_STORE_CONTINUE25:%.*]] 6029; UNROLL: pred.store.if24: 6030; UNROLL-NEXT: [[TMP55:%.*]] = or i32 [[INDEX]], 2 6031; UNROLL-NEXT: [[TMP56:%.*]] = sext i32 [[TMP55]] to i64 6032; UNROLL-NEXT: [[TMP57:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 [[TMP56]] 6033; UNROLL-NEXT: store i32 [[TMP3]], i32* [[TMP57]], align 4 6034; UNROLL-NEXT: br label [[PRED_STORE_CONTINUE25]] 6035; UNROLL: pred.store.continue25: 6036; UNROLL-NEXT: [[TMP58:%.*]] = extractelement <4 x i1> [[TMP9]], i64 3 6037; UNROLL-NEXT: br i1 [[TMP58]], label [[PRED_STORE_IF26:%.*]], label [[PRED_STORE_CONTINUE27:%.*]] 6038; UNROLL: pred.store.if26: 6039; UNROLL-NEXT: [[TMP59:%.*]] = or i32 [[INDEX]], 3 6040; UNROLL-NEXT: [[TMP60:%.*]] = sext i32 [[TMP59]] to i64 6041; UNROLL-NEXT: [[TMP61:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 [[TMP60]] 6042; UNROLL-NEXT: store i32 [[TMP4]], i32* [[TMP61]], align 4 6043; UNROLL-NEXT: br label [[PRED_STORE_CONTINUE27]] 6044; UNROLL: pred.store.continue27: 6045; UNROLL-NEXT: [[TMP62:%.*]] = extractelement <4 x i1> [[TMP10]], i64 0 6046; UNROLL-NEXT: br i1 [[TMP62]], label [[PRED_STORE_IF28:%.*]], label [[PRED_STORE_CONTINUE29:%.*]] 6047; UNROLL: pred.store.if28: 6048; UNROLL-NEXT: [[TMP63:%.*]] = or i32 [[INDEX]], 4 6049; UNROLL-NEXT: [[TMP64:%.*]] = sext i32 [[TMP63]] to i64 6050; UNROLL-NEXT: [[TMP65:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 [[TMP64]] 6051; UNROLL-NEXT: store i32 [[TMP5]], i32* [[TMP65]], align 4 6052; UNROLL-NEXT: br label [[PRED_STORE_CONTINUE29]] 6053; UNROLL: pred.store.continue29: 6054; UNROLL-NEXT: [[TMP66:%.*]] = extractelement <4 x i1> [[TMP10]], i64 1 6055; UNROLL-NEXT: br i1 [[TMP66]], label [[PRED_STORE_IF30:%.*]], label [[PRED_STORE_CONTINUE31:%.*]] 6056; UNROLL: pred.store.if30: 6057; UNROLL-NEXT: [[TMP67:%.*]] = or i32 [[INDEX]], 5 6058; UNROLL-NEXT: [[TMP68:%.*]] = sext i32 [[TMP67]] to i64 6059; UNROLL-NEXT: [[TMP69:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 [[TMP68]] 6060; UNROLL-NEXT: store i32 [[TMP6]], i32* [[TMP69]], align 4 6061; UNROLL-NEXT: br label [[PRED_STORE_CONTINUE31]] 6062; UNROLL: pred.store.continue31: 6063; UNROLL-NEXT: [[TMP70:%.*]] = extractelement <4 x i1> [[TMP10]], i64 2 6064; UNROLL-NEXT: br i1 [[TMP70]], label [[PRED_STORE_IF32:%.*]], label [[PRED_STORE_CONTINUE33:%.*]] 6065; UNROLL: pred.store.if32: 6066; UNROLL-NEXT: [[TMP71:%.*]] = or i32 [[INDEX]], 6 6067; UNROLL-NEXT: [[TMP72:%.*]] = sext i32 [[TMP71]] to i64 6068; UNROLL-NEXT: [[TMP73:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 [[TMP72]] 6069; UNROLL-NEXT: store i32 [[TMP7]], i32* [[TMP73]], align 4 6070; UNROLL-NEXT: br label [[PRED_STORE_CONTINUE33]] 6071; UNROLL: pred.store.continue33: 6072; UNROLL-NEXT: [[TMP74:%.*]] = extractelement <4 x i1> [[TMP10]], i64 3 6073; UNROLL-NEXT: br i1 [[TMP74]], label [[PRED_STORE_IF34:%.*]], label [[PRED_STORE_CONTINUE35]] 6074; UNROLL: pred.store.if34: 6075; UNROLL-NEXT: [[TMP75:%.*]] = or i32 [[INDEX]], 7 6076; UNROLL-NEXT: [[TMP76:%.*]] = sext i32 [[TMP75]] to i64 6077; UNROLL-NEXT: [[TMP77:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 [[TMP76]] 6078; UNROLL-NEXT: store i32 [[TMP8]], i32* [[TMP77]], align 4 6079; UNROLL-NEXT: br label [[PRED_STORE_CONTINUE35]] 6080; UNROLL: pred.store.continue35: 6081; UNROLL-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8 6082; UNROLL-NEXT: [[VEC_IND_NEXT6]] = add <4 x i32> [[VEC_IND3]], <i32 8, i32 8, i32 8, i32 8> 6083; UNROLL-NEXT: [[TMP78:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 6084; UNROLL-NEXT: br i1 [[TMP78]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF52]], !llvm.loop [[LOOP56:![0-9]+]] 6085; UNROLL: middle.block: 6086; UNROLL-NEXT: [[TMP79:%.*]] = select <4 x i1> [[TMP10]], <4 x i32> [[TMP46]], <4 x i32> [[VEC_PHI7]] 6087; UNROLL-NEXT: [[TMP80:%.*]] = select <4 x i1> [[TMP9]], <4 x i32> [[TMP45]], <4 x i32> [[VEC_PHI]] 6088; UNROLL-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP79]], [[TMP80]] 6089; UNROLL-NEXT: [[TMP81:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]]) 6090; UNROLL-NEXT: br i1 true, label [[BB1:%.*]], label [[SCALAR_PH]] 6091; UNROLL: scalar.ph: 6092; UNROLL-NEXT: br label [[BB2:%.*]] 6093; UNROLL: bb1: 6094; UNROLL-NEXT: [[VAR:%.*]] = phi i32 [ undef, [[BB2]] ], [ [[TMP81]], [[MIDDLE_BLOCK]] ] 6095; UNROLL-NEXT: ret i32 [[VAR]] 6096; UNROLL: bb2: 6097; UNROLL-NEXT: br i1 undef, label [[BB1]], label [[BB2]], !prof [[PROF54]], !llvm.loop [[LOOP57:![0-9]+]] 6098; 6099; UNROLL-NO-IC-LABEL: @sink_into_replication_region_multiple( 6100; UNROLL-NO-IC-NEXT: bb: 6101; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i32 [[Y:%.*]], 1 6102; UNROLL-NO-IC-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[Y]], i32 1) 6103; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]] 6104; UNROLL-NO-IC-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 6105; UNROLL-NO-IC: vector.ph: 6106; UNROLL-NO-IC-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 7 6107; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 8 6108; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]] 6109; UNROLL-NO-IC-NEXT: [[IND_END:%.*]] = sub i32 [[Y]], [[N_VEC]] 6110; UNROLL-NO-IC-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[TMP1]], 1 6111; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i32 0 6112; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 6113; UNROLL-NO-IC-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i32 0 6114; UNROLL-NO-IC-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 6115; UNROLL-NO-IC-NEXT: [[INDUCTION:%.*]] = add <4 x i32> [[DOTSPLAT]], <i32 0, i32 -1, i32 -2, i32 -3> 6116; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 6117; UNROLL-NO-IC: vector.body: 6118; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE35:%.*]] ] 6119; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE35]] ] 6120; UNROLL-NO-IC-NEXT: [[VEC_IND3:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT6:%.*]], [[PRED_STORE_CONTINUE35]] ] 6121; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP43:%.*]], [[PRED_STORE_CONTINUE35]] ] 6122; UNROLL-NO-IC-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP46:%.*]], [[PRED_STORE_CONTINUE35]] ] 6123; UNROLL-NO-IC-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP47:%.*]], [[PRED_STORE_CONTINUE35]] ] 6124; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], <i32 -4, i32 -4, i32 -4, i32 -4> 6125; UNROLL-NO-IC-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]] 6126; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add i32 [[OFFSET_IDX]], 0 6127; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add i32 [[OFFSET_IDX]], -1 6128; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = add i32 [[OFFSET_IDX]], -2 6129; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = add i32 [[OFFSET_IDX]], -3 6130; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = add i32 [[OFFSET_IDX]], -4 6131; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = add i32 [[OFFSET_IDX]], -5 6132; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = add i32 [[OFFSET_IDX]], -6 6133; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = add i32 [[OFFSET_IDX]], -7 6134; UNROLL-NO-IC-NEXT: [[STEP_ADD4:%.*]] = add <4 x i32> [[VEC_IND3]], <i32 4, i32 4, i32 4, i32 4> 6135; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = icmp ule <4 x i32> [[VEC_IND3]], [[BROADCAST_SPLAT]] 6136; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = icmp ule <4 x i32> [[STEP_ADD4]], [[BROADCAST_SPLAT]] 6137; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP10]], i32 0 6138; UNROLL-NO-IC-NEXT: br i1 [[TMP12]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 6139; UNROLL-NO-IC: pred.udiv.if: 6140; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = udiv i32 219220132, [[TMP2]] 6141; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = insertelement <4 x i32> poison, i32 [[TMP13]], i32 0 6142; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE]] 6143; UNROLL-NO-IC: pred.udiv.continue: 6144; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP14]], [[PRED_UDIV_IF]] ] 6145; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP10]], i32 1 6146; UNROLL-NO-IC-NEXT: br i1 [[TMP16]], label [[PRED_UDIV_IF8:%.*]], label [[PRED_UDIV_CONTINUE9:%.*]] 6147; UNROLL-NO-IC: pred.udiv.if8: 6148; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = udiv i32 219220132, [[TMP3]] 6149; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP17]], i32 1 6150; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE9]] 6151; UNROLL-NO-IC: pred.udiv.continue9: 6152; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = phi <4 x i32> [ [[TMP15]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP18]], [[PRED_UDIV_IF8]] ] 6153; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP10]], i32 2 6154; UNROLL-NO-IC-NEXT: br i1 [[TMP20]], label [[PRED_UDIV_IF10:%.*]], label [[PRED_UDIV_CONTINUE11:%.*]] 6155; UNROLL-NO-IC: pred.udiv.if10: 6156; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = udiv i32 219220132, [[TMP4]] 6157; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP21]], i32 2 6158; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE11]] 6159; UNROLL-NO-IC: pred.udiv.continue11: 6160; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = phi <4 x i32> [ [[TMP19]], [[PRED_UDIV_CONTINUE9]] ], [ [[TMP22]], [[PRED_UDIV_IF10]] ] 6161; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = extractelement <4 x i1> [[TMP10]], i32 3 6162; UNROLL-NO-IC-NEXT: br i1 [[TMP24]], label [[PRED_UDIV_IF12:%.*]], label [[PRED_UDIV_CONTINUE13:%.*]] 6163; UNROLL-NO-IC: pred.udiv.if12: 6164; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = udiv i32 219220132, [[TMP5]] 6165; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = insertelement <4 x i32> [[TMP23]], i32 [[TMP25]], i32 3 6166; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE13]] 6167; UNROLL-NO-IC: pred.udiv.continue13: 6168; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = phi <4 x i32> [ [[TMP23]], [[PRED_UDIV_CONTINUE11]] ], [ [[TMP26]], [[PRED_UDIV_IF12]] ] 6169; UNROLL-NO-IC-NEXT: [[TMP28:%.*]] = extractelement <4 x i1> [[TMP11]], i32 0 6170; UNROLL-NO-IC-NEXT: br i1 [[TMP28]], label [[PRED_UDIV_IF14:%.*]], label [[PRED_UDIV_CONTINUE15:%.*]] 6171; UNROLL-NO-IC: pred.udiv.if14: 6172; UNROLL-NO-IC-NEXT: [[TMP29:%.*]] = udiv i32 219220132, [[TMP6]] 6173; UNROLL-NO-IC-NEXT: [[TMP30:%.*]] = insertelement <4 x i32> poison, i32 [[TMP29]], i32 0 6174; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE15]] 6175; UNROLL-NO-IC: pred.udiv.continue15: 6176; UNROLL-NO-IC-NEXT: [[TMP31:%.*]] = phi <4 x i32> [ poison, [[PRED_UDIV_CONTINUE13]] ], [ [[TMP30]], [[PRED_UDIV_IF14]] ] 6177; UNROLL-NO-IC-NEXT: [[TMP32:%.*]] = extractelement <4 x i1> [[TMP11]], i32 1 6178; UNROLL-NO-IC-NEXT: br i1 [[TMP32]], label [[PRED_UDIV_IF16:%.*]], label [[PRED_UDIV_CONTINUE17:%.*]] 6179; UNROLL-NO-IC: pred.udiv.if16: 6180; UNROLL-NO-IC-NEXT: [[TMP33:%.*]] = udiv i32 219220132, [[TMP7]] 6181; UNROLL-NO-IC-NEXT: [[TMP34:%.*]] = insertelement <4 x i32> [[TMP31]], i32 [[TMP33]], i32 1 6182; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE17]] 6183; UNROLL-NO-IC: pred.udiv.continue17: 6184; UNROLL-NO-IC-NEXT: [[TMP35:%.*]] = phi <4 x i32> [ [[TMP31]], [[PRED_UDIV_CONTINUE15]] ], [ [[TMP34]], [[PRED_UDIV_IF16]] ] 6185; UNROLL-NO-IC-NEXT: [[TMP36:%.*]] = extractelement <4 x i1> [[TMP11]], i32 2 6186; UNROLL-NO-IC-NEXT: br i1 [[TMP36]], label [[PRED_UDIV_IF18:%.*]], label [[PRED_UDIV_CONTINUE19:%.*]] 6187; UNROLL-NO-IC: pred.udiv.if18: 6188; UNROLL-NO-IC-NEXT: [[TMP37:%.*]] = udiv i32 219220132, [[TMP8]] 6189; UNROLL-NO-IC-NEXT: [[TMP38:%.*]] = insertelement <4 x i32> [[TMP35]], i32 [[TMP37]], i32 2 6190; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE19]] 6191; UNROLL-NO-IC: pred.udiv.continue19: 6192; UNROLL-NO-IC-NEXT: [[TMP39:%.*]] = phi <4 x i32> [ [[TMP35]], [[PRED_UDIV_CONTINUE17]] ], [ [[TMP38]], [[PRED_UDIV_IF18]] ] 6193; UNROLL-NO-IC-NEXT: [[TMP40:%.*]] = extractelement <4 x i1> [[TMP11]], i32 3 6194; UNROLL-NO-IC-NEXT: br i1 [[TMP40]], label [[PRED_UDIV_IF20:%.*]], label [[PRED_UDIV_CONTINUE21:%.*]] 6195; UNROLL-NO-IC: pred.udiv.if20: 6196; UNROLL-NO-IC-NEXT: [[TMP41:%.*]] = udiv i32 219220132, [[TMP9]] 6197; UNROLL-NO-IC-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> [[TMP39]], i32 [[TMP41]], i32 3 6198; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE21]] 6199; UNROLL-NO-IC: pred.udiv.continue21: 6200; UNROLL-NO-IC-NEXT: [[TMP43]] = phi <4 x i32> [ [[TMP39]], [[PRED_UDIV_CONTINUE19]] ], [ [[TMP42]], [[PRED_UDIV_IF20]] ] 6201; UNROLL-NO-IC-NEXT: [[TMP44:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP27]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 6202; UNROLL-NO-IC-NEXT: [[TMP45:%.*]] = shufflevector <4 x i32> [[TMP27]], <4 x i32> [[TMP43]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 6203; UNROLL-NO-IC-NEXT: [[TMP46]] = add <4 x i32> [[VEC_PHI]], [[TMP44]] 6204; UNROLL-NO-IC-NEXT: [[TMP47]] = add <4 x i32> [[VEC_PHI7]], [[TMP45]] 6205; UNROLL-NO-IC-NEXT: [[TMP48:%.*]] = extractelement <4 x i1> [[TMP10]], i32 0 6206; UNROLL-NO-IC-NEXT: br i1 [[TMP48]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] 6207; UNROLL-NO-IC: pred.store.if: 6208; UNROLL-NO-IC-NEXT: [[TMP49:%.*]] = add i32 [[INDEX]], 0 6209; UNROLL-NO-IC-NEXT: [[TMP50:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i32 [[TMP49]] 6210; UNROLL-NO-IC-NEXT: store i32 [[TMP2]], i32* [[TMP50]], align 4 6211; UNROLL-NO-IC-NEXT: br label [[PRED_STORE_CONTINUE]] 6212; UNROLL-NO-IC: pred.store.continue: 6213; UNROLL-NO-IC-NEXT: [[TMP51:%.*]] = extractelement <4 x i1> [[TMP10]], i32 1 6214; UNROLL-NO-IC-NEXT: br i1 [[TMP51]], label [[PRED_STORE_IF22:%.*]], label [[PRED_STORE_CONTINUE23:%.*]] 6215; UNROLL-NO-IC: pred.store.if22: 6216; UNROLL-NO-IC-NEXT: [[TMP52:%.*]] = add i32 [[INDEX]], 1 6217; UNROLL-NO-IC-NEXT: [[TMP53:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[TMP52]] 6218; UNROLL-NO-IC-NEXT: store i32 [[TMP3]], i32* [[TMP53]], align 4 6219; UNROLL-NO-IC-NEXT: br label [[PRED_STORE_CONTINUE23]] 6220; UNROLL-NO-IC: pred.store.continue23: 6221; UNROLL-NO-IC-NEXT: [[TMP54:%.*]] = extractelement <4 x i1> [[TMP10]], i32 2 6222; UNROLL-NO-IC-NEXT: br i1 [[TMP54]], label [[PRED_STORE_IF24:%.*]], label [[PRED_STORE_CONTINUE25:%.*]] 6223; UNROLL-NO-IC: pred.store.if24: 6224; UNROLL-NO-IC-NEXT: [[TMP55:%.*]] = add i32 [[INDEX]], 2 6225; UNROLL-NO-IC-NEXT: [[TMP56:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[TMP55]] 6226; UNROLL-NO-IC-NEXT: store i32 [[TMP4]], i32* [[TMP56]], align 4 6227; UNROLL-NO-IC-NEXT: br label [[PRED_STORE_CONTINUE25]] 6228; UNROLL-NO-IC: pred.store.continue25: 6229; UNROLL-NO-IC-NEXT: [[TMP57:%.*]] = extractelement <4 x i1> [[TMP10]], i32 3 6230; UNROLL-NO-IC-NEXT: br i1 [[TMP57]], label [[PRED_STORE_IF26:%.*]], label [[PRED_STORE_CONTINUE27:%.*]] 6231; UNROLL-NO-IC: pred.store.if26: 6232; UNROLL-NO-IC-NEXT: [[TMP58:%.*]] = add i32 [[INDEX]], 3 6233; UNROLL-NO-IC-NEXT: [[TMP59:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[TMP58]] 6234; UNROLL-NO-IC-NEXT: store i32 [[TMP5]], i32* [[TMP59]], align 4 6235; UNROLL-NO-IC-NEXT: br label [[PRED_STORE_CONTINUE27]] 6236; UNROLL-NO-IC: pred.store.continue27: 6237; UNROLL-NO-IC-NEXT: [[TMP60:%.*]] = extractelement <4 x i1> [[TMP11]], i32 0 6238; UNROLL-NO-IC-NEXT: br i1 [[TMP60]], label [[PRED_STORE_IF28:%.*]], label [[PRED_STORE_CONTINUE29:%.*]] 6239; UNROLL-NO-IC: pred.store.if28: 6240; UNROLL-NO-IC-NEXT: [[TMP61:%.*]] = add i32 [[INDEX]], 4 6241; UNROLL-NO-IC-NEXT: [[TMP62:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[TMP61]] 6242; UNROLL-NO-IC-NEXT: store i32 [[TMP6]], i32* [[TMP62]], align 4 6243; UNROLL-NO-IC-NEXT: br label [[PRED_STORE_CONTINUE29]] 6244; UNROLL-NO-IC: pred.store.continue29: 6245; UNROLL-NO-IC-NEXT: [[TMP63:%.*]] = extractelement <4 x i1> [[TMP11]], i32 1 6246; UNROLL-NO-IC-NEXT: br i1 [[TMP63]], label [[PRED_STORE_IF30:%.*]], label [[PRED_STORE_CONTINUE31:%.*]] 6247; UNROLL-NO-IC: pred.store.if30: 6248; UNROLL-NO-IC-NEXT: [[TMP64:%.*]] = add i32 [[INDEX]], 5 6249; UNROLL-NO-IC-NEXT: [[TMP65:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[TMP64]] 6250; UNROLL-NO-IC-NEXT: store i32 [[TMP7]], i32* [[TMP65]], align 4 6251; UNROLL-NO-IC-NEXT: br label [[PRED_STORE_CONTINUE31]] 6252; UNROLL-NO-IC: pred.store.continue31: 6253; UNROLL-NO-IC-NEXT: [[TMP66:%.*]] = extractelement <4 x i1> [[TMP11]], i32 2 6254; UNROLL-NO-IC-NEXT: br i1 [[TMP66]], label [[PRED_STORE_IF32:%.*]], label [[PRED_STORE_CONTINUE33:%.*]] 6255; UNROLL-NO-IC: pred.store.if32: 6256; UNROLL-NO-IC-NEXT: [[TMP67:%.*]] = add i32 [[INDEX]], 6 6257; UNROLL-NO-IC-NEXT: [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[TMP67]] 6258; UNROLL-NO-IC-NEXT: store i32 [[TMP8]], i32* [[TMP68]], align 4 6259; UNROLL-NO-IC-NEXT: br label [[PRED_STORE_CONTINUE33]] 6260; UNROLL-NO-IC: pred.store.continue33: 6261; UNROLL-NO-IC-NEXT: [[TMP69:%.*]] = extractelement <4 x i1> [[TMP11]], i32 3 6262; UNROLL-NO-IC-NEXT: br i1 [[TMP69]], label [[PRED_STORE_IF34:%.*]], label [[PRED_STORE_CONTINUE35]] 6263; UNROLL-NO-IC: pred.store.if34: 6264; UNROLL-NO-IC-NEXT: [[TMP70:%.*]] = add i32 [[INDEX]], 7 6265; UNROLL-NO-IC-NEXT: [[TMP71:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[TMP70]] 6266; UNROLL-NO-IC-NEXT: store i32 [[TMP9]], i32* [[TMP71]], align 4 6267; UNROLL-NO-IC-NEXT: br label [[PRED_STORE_CONTINUE35]] 6268; UNROLL-NO-IC: pred.store.continue35: 6269; UNROLL-NO-IC-NEXT: [[TMP72:%.*]] = select <4 x i1> [[TMP10]], <4 x i32> [[TMP46]], <4 x i32> [[VEC_PHI]] 6270; UNROLL-NO-IC-NEXT: [[TMP73:%.*]] = select <4 x i1> [[TMP11]], <4 x i32> [[TMP47]], <4 x i32> [[VEC_PHI7]] 6271; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8 6272; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], <i32 -4, i32 -4, i32 -4, i32 -4> 6273; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT6]] = add <4 x i32> [[STEP_ADD4]], <i32 4, i32 4, i32 4, i32 4> 6274; UNROLL-NO-IC-NEXT: [[TMP74:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 6275; UNROLL-NO-IC-NEXT: br i1 [[TMP74]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF52]], !llvm.loop [[LOOP56:![0-9]+]] 6276; UNROLL-NO-IC: middle.block: 6277; UNROLL-NO-IC-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP73]], [[TMP72]] 6278; UNROLL-NO-IC-NEXT: [[TMP75:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]]) 6279; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP43]], i32 3 6280; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP43]], i32 2 6281; UNROLL-NO-IC-NEXT: br i1 true, label [[BB1:%.*]], label [[SCALAR_PH]] 6282; UNROLL-NO-IC: scalar.ph: 6283; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[BB:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 6284; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[Y]], [[BB]] ] 6285; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[BB]] ] 6286; UNROLL-NO-IC-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[BB]] ], [ [[TMP75]], [[MIDDLE_BLOCK]] ] 6287; UNROLL-NO-IC-NEXT: br label [[BB2:%.*]] 6288; UNROLL-NO-IC: bb1: 6289; UNROLL-NO-IC-NEXT: [[VAR:%.*]] = phi i32 [ [[VAR6:%.*]], [[BB2]] ], [ [[TMP75]], [[MIDDLE_BLOCK]] ] 6290; UNROLL-NO-IC-NEXT: ret i32 [[VAR]] 6291; UNROLL-NO-IC: bb2: 6292; UNROLL-NO-IC-NEXT: [[VAR3:%.*]] = phi i32 [ [[VAR8:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 6293; UNROLL-NO-IC-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ] 6294; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[VAR7:%.*]], [[BB2]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ] 6295; UNROLL-NO-IC-NEXT: [[VAR5:%.*]] = phi i32 [ [[VAR6]], [[BB2]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 6296; UNROLL-NO-IC-NEXT: [[G:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[IV]] 6297; UNROLL-NO-IC-NEXT: [[VAR6]] = add i32 [[VAR5]], [[SCALAR_RECUR]] 6298; UNROLL-NO-IC-NEXT: [[VAR7]] = udiv i32 219220132, [[VAR3]] 6299; UNROLL-NO-IC-NEXT: store i32 [[VAR3]], i32* [[G]], align 4 6300; UNROLL-NO-IC-NEXT: [[VAR8]] = add nsw i32 [[VAR3]], -1 6301; UNROLL-NO-IC-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], 1 6302; UNROLL-NO-IC-NEXT: [[VAR9:%.*]] = icmp slt i32 [[VAR3]], 2 6303; UNROLL-NO-IC-NEXT: br i1 [[VAR9]], label [[BB1]], label [[BB2]], !prof [[PROF54]], !llvm.loop [[LOOP57:![0-9]+]] 6304; 6305; UNROLL-NO-VF-LABEL: @sink_into_replication_region_multiple( 6306; UNROLL-NO-VF-NEXT: bb: 6307; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i32 [[Y:%.*]], 1 6308; UNROLL-NO-VF-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[Y]], i32 1) 6309; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]] 6310; UNROLL-NO-VF-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 6311; UNROLL-NO-VF: vector.ph: 6312; UNROLL-NO-VF-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 1 6313; UNROLL-NO-VF-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2 6314; UNROLL-NO-VF-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]] 6315; UNROLL-NO-VF-NEXT: [[IND_END:%.*]] = sub i32 [[Y]], [[N_VEC]] 6316; UNROLL-NO-VF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[TMP1]], 1 6317; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 6318; UNROLL-NO-VF: vector.body: 6319; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE10:%.*]] ] 6320; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[PRED_STORE_CONTINUE10]] ] 6321; UNROLL-NO-VF-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[PRED_STORE_CONTINUE10]] ] 6322; UNROLL-NO-VF-NEXT: [[VEC_PHI5:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[PRED_STORE_CONTINUE10]] ] 6323; UNROLL-NO-VF-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]] 6324; UNROLL-NO-VF-NEXT: [[INDUCTION:%.*]] = add i32 [[OFFSET_IDX]], 0 6325; UNROLL-NO-VF-NEXT: [[INDUCTION2:%.*]] = add i32 [[OFFSET_IDX]], -1 6326; UNROLL-NO-VF-NEXT: [[VEC_IV:%.*]] = add i32 [[INDEX]], 0 6327; UNROLL-NO-VF-NEXT: [[VEC_IV6:%.*]] = add i32 [[INDEX]], 1 6328; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = icmp ule i32 [[VEC_IV]], [[TRIP_COUNT_MINUS_1]] 6329; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = icmp ule i32 [[VEC_IV6]], [[TRIP_COUNT_MINUS_1]] 6330; UNROLL-NO-VF-NEXT: br i1 [[TMP2]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 6331; UNROLL-NO-VF: pred.udiv.if: 6332; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = udiv i32 219220132, [[INDUCTION]] 6333; UNROLL-NO-VF-NEXT: br label [[PRED_UDIV_CONTINUE]] 6334; UNROLL-NO-VF: pred.udiv.continue: 6335; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP4]], [[PRED_UDIV_IF]] ] 6336; UNROLL-NO-VF-NEXT: br i1 [[TMP3]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8:%.*]] 6337; UNROLL-NO-VF: pred.udiv.if7: 6338; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = udiv i32 219220132, [[INDUCTION2]] 6339; UNROLL-NO-VF-NEXT: br label [[PRED_UDIV_CONTINUE8]] 6340; UNROLL-NO-VF: pred.udiv.continue8: 6341; UNROLL-NO-VF-NEXT: [[TMP7]] = phi i32 [ poison, [[PRED_UDIV_CONTINUE]] ], [ [[TMP6]], [[PRED_UDIV_IF7]] ] 6342; UNROLL-NO-VF-NEXT: [[TMP8]] = add i32 [[VEC_PHI]], [[VECTOR_RECUR]] 6343; UNROLL-NO-VF-NEXT: [[TMP9]] = add i32 [[VEC_PHI5]], [[TMP5]] 6344; UNROLL-NO-VF-NEXT: br i1 [[TMP2]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] 6345; UNROLL-NO-VF: pred.store.if: 6346; UNROLL-NO-VF-NEXT: [[INDUCTION3:%.*]] = add i32 [[INDEX]], 0 6347; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i32 [[INDUCTION3]] 6348; UNROLL-NO-VF-NEXT: store i32 [[INDUCTION]], i32* [[TMP10]], align 4 6349; UNROLL-NO-VF-NEXT: br label [[PRED_STORE_CONTINUE]] 6350; UNROLL-NO-VF: pred.store.continue: 6351; UNROLL-NO-VF-NEXT: br i1 [[TMP3]], label [[PRED_STORE_IF9:%.*]], label [[PRED_STORE_CONTINUE10]] 6352; UNROLL-NO-VF: pred.store.if9: 6353; UNROLL-NO-VF-NEXT: [[INDUCTION4:%.*]] = add i32 [[INDEX]], 1 6354; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[INDUCTION4]] 6355; UNROLL-NO-VF-NEXT: store i32 [[INDUCTION2]], i32* [[TMP11]], align 4 6356; UNROLL-NO-VF-NEXT: br label [[PRED_STORE_CONTINUE10]] 6357; UNROLL-NO-VF: pred.store.continue10: 6358; UNROLL-NO-VF-NEXT: [[TMP12:%.*]] = select i1 [[TMP2]], i32 [[TMP8]], i32 [[VEC_PHI]] 6359; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = select i1 [[TMP3]], i32 [[TMP9]], i32 [[VEC_PHI5]] 6360; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2 6361; UNROLL-NO-VF-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 6362; UNROLL-NO-VF-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF51]], !llvm.loop [[LOOP55:![0-9]+]] 6363; UNROLL-NO-VF: middle.block: 6364; UNROLL-NO-VF-NEXT: [[BIN_RDX:%.*]] = add i32 [[TMP13]], [[TMP12]] 6365; UNROLL-NO-VF-NEXT: br i1 true, label [[BB1:%.*]], label [[SCALAR_PH]] 6366; UNROLL-NO-VF: scalar.ph: 6367; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[BB:%.*]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ] 6368; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[Y]], [[BB]] ] 6369; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[BB]] ] 6370; UNROLL-NO-VF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[BB]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ] 6371; UNROLL-NO-VF-NEXT: br label [[BB2:%.*]] 6372; UNROLL-NO-VF: bb1: 6373; UNROLL-NO-VF-NEXT: [[VAR:%.*]] = phi i32 [ [[VAR6:%.*]], [[BB2]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ] 6374; UNROLL-NO-VF-NEXT: ret i32 [[VAR]] 6375; UNROLL-NO-VF: bb2: 6376; UNROLL-NO-VF-NEXT: [[VAR3:%.*]] = phi i32 [ [[VAR8:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 6377; UNROLL-NO-VF-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ] 6378; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[VAR7:%.*]], [[BB2]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ] 6379; UNROLL-NO-VF-NEXT: [[VAR5:%.*]] = phi i32 [ [[VAR6]], [[BB2]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 6380; UNROLL-NO-VF-NEXT: [[G:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[IV]] 6381; UNROLL-NO-VF-NEXT: [[VAR6]] = add i32 [[VAR5]], [[SCALAR_RECUR]] 6382; UNROLL-NO-VF-NEXT: [[VAR7]] = udiv i32 219220132, [[VAR3]] 6383; UNROLL-NO-VF-NEXT: store i32 [[VAR3]], i32* [[G]], align 4 6384; UNROLL-NO-VF-NEXT: [[VAR8]] = add nsw i32 [[VAR3]], -1 6385; UNROLL-NO-VF-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], 1 6386; UNROLL-NO-VF-NEXT: [[VAR9:%.*]] = icmp slt i32 [[VAR3]], 2 6387; UNROLL-NO-VF-NEXT: br i1 [[VAR9]], label [[BB1]], label [[BB2]], !prof [[PROF53]], !llvm.loop [[LOOP56:![0-9]+]] 6388; 6389; SINK-AFTER-LABEL: @sink_into_replication_region_multiple( 6390; SINK-AFTER-NEXT: bb: 6391; SINK-AFTER-NEXT: [[TMP0:%.*]] = add i32 [[Y:%.*]], 1 6392; SINK-AFTER-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[Y]], i32 1) 6393; SINK-AFTER-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]] 6394; SINK-AFTER-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 6395; SINK-AFTER: vector.ph: 6396; SINK-AFTER-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 3 6397; SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 4 6398; SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]] 6399; SINK-AFTER-NEXT: [[IND_END:%.*]] = sub i32 [[Y]], [[N_VEC]] 6400; SINK-AFTER-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[TMP1]], 1 6401; SINK-AFTER-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i32 0 6402; SINK-AFTER-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 6403; SINK-AFTER-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i32 0 6404; SINK-AFTER-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 6405; SINK-AFTER-NEXT: [[INDUCTION:%.*]] = add <4 x i32> [[DOTSPLAT]], <i32 0, i32 -1, i32 -2, i32 -3> 6406; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 6407; SINK-AFTER: vector.body: 6408; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE15:%.*]] ] 6409; SINK-AFTER-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE15]] ] 6410; SINK-AFTER-NEXT: [[VEC_IND2:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[PRED_STORE_CONTINUE15]] ] 6411; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[PRED_STORE_CONTINUE15]] ] 6412; SINK-AFTER-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[PRED_STORE_CONTINUE15]] ] 6413; SINK-AFTER-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]] 6414; SINK-AFTER-NEXT: [[TMP2:%.*]] = add i32 [[OFFSET_IDX]], 0 6415; SINK-AFTER-NEXT: [[TMP3:%.*]] = add i32 [[OFFSET_IDX]], -1 6416; SINK-AFTER-NEXT: [[TMP4:%.*]] = add i32 [[OFFSET_IDX]], -2 6417; SINK-AFTER-NEXT: [[TMP5:%.*]] = add i32 [[OFFSET_IDX]], -3 6418; SINK-AFTER-NEXT: [[TMP6:%.*]] = icmp ule <4 x i32> [[VEC_IND2]], [[BROADCAST_SPLAT]] 6419; SINK-AFTER-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i32 0 6420; SINK-AFTER-NEXT: br i1 [[TMP7]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 6421; SINK-AFTER: pred.udiv.if: 6422; SINK-AFTER-NEXT: [[TMP8:%.*]] = udiv i32 219220132, [[TMP2]] 6423; SINK-AFTER-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> poison, i32 [[TMP8]], i32 0 6424; SINK-AFTER-NEXT: br label [[PRED_UDIV_CONTINUE]] 6425; SINK-AFTER: pred.udiv.continue: 6426; SINK-AFTER-NEXT: [[TMP10:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP9]], [[PRED_UDIV_IF]] ] 6427; SINK-AFTER-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP6]], i32 1 6428; SINK-AFTER-NEXT: br i1 [[TMP11]], label [[PRED_UDIV_IF4:%.*]], label [[PRED_UDIV_CONTINUE5:%.*]] 6429; SINK-AFTER: pred.udiv.if4: 6430; SINK-AFTER-NEXT: [[TMP12:%.*]] = udiv i32 219220132, [[TMP3]] 6431; SINK-AFTER-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP12]], i32 1 6432; SINK-AFTER-NEXT: br label [[PRED_UDIV_CONTINUE5]] 6433; SINK-AFTER: pred.udiv.continue5: 6434; SINK-AFTER-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP10]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP13]], [[PRED_UDIV_IF4]] ] 6435; SINK-AFTER-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP6]], i32 2 6436; SINK-AFTER-NEXT: br i1 [[TMP15]], label [[PRED_UDIV_IF6:%.*]], label [[PRED_UDIV_CONTINUE7:%.*]] 6437; SINK-AFTER: pred.udiv.if6: 6438; SINK-AFTER-NEXT: [[TMP16:%.*]] = udiv i32 219220132, [[TMP4]] 6439; SINK-AFTER-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP16]], i32 2 6440; SINK-AFTER-NEXT: br label [[PRED_UDIV_CONTINUE7]] 6441; SINK-AFTER: pred.udiv.continue7: 6442; SINK-AFTER-NEXT: [[TMP18:%.*]] = phi <4 x i32> [ [[TMP14]], [[PRED_UDIV_CONTINUE5]] ], [ [[TMP17]], [[PRED_UDIV_IF6]] ] 6443; SINK-AFTER-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP6]], i32 3 6444; SINK-AFTER-NEXT: br i1 [[TMP19]], label [[PRED_UDIV_IF8:%.*]], label [[PRED_UDIV_CONTINUE9:%.*]] 6445; SINK-AFTER: pred.udiv.if8: 6446; SINK-AFTER-NEXT: [[TMP20:%.*]] = udiv i32 219220132, [[TMP5]] 6447; SINK-AFTER-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP18]], i32 [[TMP20]], i32 3 6448; SINK-AFTER-NEXT: br label [[PRED_UDIV_CONTINUE9]] 6449; SINK-AFTER: pred.udiv.continue9: 6450; SINK-AFTER-NEXT: [[TMP22]] = phi <4 x i32> [ [[TMP18]], [[PRED_UDIV_CONTINUE7]] ], [ [[TMP21]], [[PRED_UDIV_IF8]] ] 6451; SINK-AFTER-NEXT: [[TMP23:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP22]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 6452; SINK-AFTER-NEXT: [[TMP24]] = add <4 x i32> [[VEC_PHI]], [[TMP23]] 6453; SINK-AFTER-NEXT: [[TMP25:%.*]] = extractelement <4 x i1> [[TMP6]], i32 0 6454; SINK-AFTER-NEXT: br i1 [[TMP25]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] 6455; SINK-AFTER: pred.store.if: 6456; SINK-AFTER-NEXT: [[TMP26:%.*]] = add i32 [[INDEX]], 0 6457; SINK-AFTER-NEXT: [[TMP27:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i32 [[TMP26]] 6458; SINK-AFTER-NEXT: store i32 [[TMP2]], i32* [[TMP27]], align 4 6459; SINK-AFTER-NEXT: br label [[PRED_STORE_CONTINUE]] 6460; SINK-AFTER: pred.store.continue: 6461; SINK-AFTER-NEXT: [[TMP28:%.*]] = extractelement <4 x i1> [[TMP6]], i32 1 6462; SINK-AFTER-NEXT: br i1 [[TMP28]], label [[PRED_STORE_IF10:%.*]], label [[PRED_STORE_CONTINUE11:%.*]] 6463; SINK-AFTER: pred.store.if10: 6464; SINK-AFTER-NEXT: [[TMP29:%.*]] = add i32 [[INDEX]], 1 6465; SINK-AFTER-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[TMP29]] 6466; SINK-AFTER-NEXT: store i32 [[TMP3]], i32* [[TMP30]], align 4 6467; SINK-AFTER-NEXT: br label [[PRED_STORE_CONTINUE11]] 6468; SINK-AFTER: pred.store.continue11: 6469; SINK-AFTER-NEXT: [[TMP31:%.*]] = extractelement <4 x i1> [[TMP6]], i32 2 6470; SINK-AFTER-NEXT: br i1 [[TMP31]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE13:%.*]] 6471; SINK-AFTER: pred.store.if12: 6472; SINK-AFTER-NEXT: [[TMP32:%.*]] = add i32 [[INDEX]], 2 6473; SINK-AFTER-NEXT: [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[TMP32]] 6474; SINK-AFTER-NEXT: store i32 [[TMP4]], i32* [[TMP33]], align 4 6475; SINK-AFTER-NEXT: br label [[PRED_STORE_CONTINUE13]] 6476; SINK-AFTER: pred.store.continue13: 6477; SINK-AFTER-NEXT: [[TMP34:%.*]] = extractelement <4 x i1> [[TMP6]], i32 3 6478; SINK-AFTER-NEXT: br i1 [[TMP34]], label [[PRED_STORE_IF14:%.*]], label [[PRED_STORE_CONTINUE15]] 6479; SINK-AFTER: pred.store.if14: 6480; SINK-AFTER-NEXT: [[TMP35:%.*]] = add i32 [[INDEX]], 3 6481; SINK-AFTER-NEXT: [[TMP36:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[TMP35]] 6482; SINK-AFTER-NEXT: store i32 [[TMP5]], i32* [[TMP36]], align 4 6483; SINK-AFTER-NEXT: br label [[PRED_STORE_CONTINUE15]] 6484; SINK-AFTER: pred.store.continue15: 6485; SINK-AFTER-NEXT: [[TMP37:%.*]] = select <4 x i1> [[TMP6]], <4 x i32> [[TMP24]], <4 x i32> [[VEC_PHI]] 6486; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4 6487; SINK-AFTER-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 -4, i32 -4, i32 -4, i32 -4> 6488; SINK-AFTER-NEXT: [[VEC_IND_NEXT3]] = add <4 x i32> [[VEC_IND2]], <i32 4, i32 4, i32 4, i32 4> 6489; SINK-AFTER-NEXT: [[TMP38:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 6490; SINK-AFTER-NEXT: br i1 [[TMP38]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF52]], !llvm.loop [[LOOP56:![0-9]+]] 6491; SINK-AFTER: middle.block: 6492; SINK-AFTER-NEXT: [[TMP39:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP37]]) 6493; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP22]], i32 3 6494; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP22]], i32 2 6495; SINK-AFTER-NEXT: br i1 true, label [[BB1:%.*]], label [[SCALAR_PH]] 6496; SINK-AFTER: scalar.ph: 6497; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[BB:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 6498; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[Y]], [[BB]] ] 6499; SINK-AFTER-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[BB]] ] 6500; SINK-AFTER-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[BB]] ], [ [[TMP39]], [[MIDDLE_BLOCK]] ] 6501; SINK-AFTER-NEXT: br label [[BB2:%.*]] 6502; SINK-AFTER: bb1: 6503; SINK-AFTER-NEXT: [[VAR:%.*]] = phi i32 [ [[VAR6:%.*]], [[BB2]] ], [ [[TMP39]], [[MIDDLE_BLOCK]] ] 6504; SINK-AFTER-NEXT: ret i32 [[VAR]] 6505; SINK-AFTER: bb2: 6506; SINK-AFTER-NEXT: [[VAR3:%.*]] = phi i32 [ [[VAR8:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 6507; SINK-AFTER-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ] 6508; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[VAR7:%.*]], [[BB2]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ] 6509; SINK-AFTER-NEXT: [[VAR5:%.*]] = phi i32 [ [[VAR6]], [[BB2]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 6510; SINK-AFTER-NEXT: [[G:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[IV]] 6511; SINK-AFTER-NEXT: [[VAR6]] = add i32 [[VAR5]], [[SCALAR_RECUR]] 6512; SINK-AFTER-NEXT: [[VAR7]] = udiv i32 219220132, [[VAR3]] 6513; SINK-AFTER-NEXT: store i32 [[VAR3]], i32* [[G]], align 4 6514; SINK-AFTER-NEXT: [[VAR8]] = add nsw i32 [[VAR3]], -1 6515; SINK-AFTER-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], 1 6516; SINK-AFTER-NEXT: [[VAR9:%.*]] = icmp slt i32 [[VAR3]], 2 6517; SINK-AFTER-NEXT: br i1 [[VAR9]], label [[BB1]], label [[BB2]], !prof [[PROF54]], !llvm.loop [[LOOP57:![0-9]+]] 6518; 6519; NO-SINK-AFTER-LABEL: @sink_into_replication_region_multiple( 6520; NO-SINK-AFTER-NEXT: bb: 6521; NO-SINK-AFTER-NEXT: [[TMP0:%.*]] = add i32 [[Y:%.*]], 1 6522; NO-SINK-AFTER-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[Y]], i32 1) 6523; NO-SINK-AFTER-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]] 6524; NO-SINK-AFTER-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 6525; NO-SINK-AFTER: vector.ph: 6526; NO-SINK-AFTER-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 3 6527; NO-SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 4 6528; NO-SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]] 6529; NO-SINK-AFTER-NEXT: [[IND_END:%.*]] = sub i32 [[Y]], [[N_VEC]] 6530; NO-SINK-AFTER-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[TMP1]], 1 6531; NO-SINK-AFTER-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i32 0 6532; NO-SINK-AFTER-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 6533; NO-SINK-AFTER-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i32 0 6534; NO-SINK-AFTER-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 6535; NO-SINK-AFTER-NEXT: [[INDUCTION:%.*]] = add <4 x i32> [[DOTSPLAT]], <i32 0, i32 -1, i32 -2, i32 -3> 6536; NO-SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 6537; NO-SINK-AFTER: vector.body: 6538; NO-SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE15:%.*]] ] 6539; NO-SINK-AFTER-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE15]] ] 6540; NO-SINK-AFTER-NEXT: [[VEC_IND2:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[PRED_STORE_CONTINUE15]] ] 6541; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[PRED_STORE_CONTINUE15]] ] 6542; NO-SINK-AFTER-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[PRED_STORE_CONTINUE15]] ] 6543; NO-SINK-AFTER-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]] 6544; NO-SINK-AFTER-NEXT: [[TMP2:%.*]] = add i32 [[OFFSET_IDX]], 0 6545; NO-SINK-AFTER-NEXT: [[TMP3:%.*]] = add i32 [[OFFSET_IDX]], -1 6546; NO-SINK-AFTER-NEXT: [[TMP4:%.*]] = add i32 [[OFFSET_IDX]], -2 6547; NO-SINK-AFTER-NEXT: [[TMP5:%.*]] = add i32 [[OFFSET_IDX]], -3 6548; NO-SINK-AFTER-NEXT: [[TMP6:%.*]] = icmp ule <4 x i32> [[VEC_IND2]], [[BROADCAST_SPLAT]] 6549; NO-SINK-AFTER-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i32 0 6550; NO-SINK-AFTER-NEXT: br i1 [[TMP7]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 6551; NO-SINK-AFTER: pred.udiv.if: 6552; NO-SINK-AFTER-NEXT: [[TMP8:%.*]] = udiv i32 219220132, [[TMP2]] 6553; NO-SINK-AFTER-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> poison, i32 [[TMP8]], i32 0 6554; NO-SINK-AFTER-NEXT: br label [[PRED_UDIV_CONTINUE]] 6555; NO-SINK-AFTER: pred.udiv.continue: 6556; NO-SINK-AFTER-NEXT: [[TMP10:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP9]], [[PRED_UDIV_IF]] ] 6557; NO-SINK-AFTER-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP6]], i32 1 6558; NO-SINK-AFTER-NEXT: br i1 [[TMP11]], label [[PRED_UDIV_IF4:%.*]], label [[PRED_UDIV_CONTINUE5:%.*]] 6559; NO-SINK-AFTER: pred.udiv.if4: 6560; NO-SINK-AFTER-NEXT: [[TMP12:%.*]] = udiv i32 219220132, [[TMP3]] 6561; NO-SINK-AFTER-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP12]], i32 1 6562; NO-SINK-AFTER-NEXT: br label [[PRED_UDIV_CONTINUE5]] 6563; NO-SINK-AFTER: pred.udiv.continue5: 6564; NO-SINK-AFTER-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP10]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP13]], [[PRED_UDIV_IF4]] ] 6565; NO-SINK-AFTER-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP6]], i32 2 6566; NO-SINK-AFTER-NEXT: br i1 [[TMP15]], label [[PRED_UDIV_IF6:%.*]], label [[PRED_UDIV_CONTINUE7:%.*]] 6567; NO-SINK-AFTER: pred.udiv.if6: 6568; NO-SINK-AFTER-NEXT: [[TMP16:%.*]] = udiv i32 219220132, [[TMP4]] 6569; NO-SINK-AFTER-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP16]], i32 2 6570; NO-SINK-AFTER-NEXT: br label [[PRED_UDIV_CONTINUE7]] 6571; NO-SINK-AFTER: pred.udiv.continue7: 6572; NO-SINK-AFTER-NEXT: [[TMP18:%.*]] = phi <4 x i32> [ [[TMP14]], [[PRED_UDIV_CONTINUE5]] ], [ [[TMP17]], [[PRED_UDIV_IF6]] ] 6573; NO-SINK-AFTER-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP6]], i32 3 6574; NO-SINK-AFTER-NEXT: br i1 [[TMP19]], label [[PRED_UDIV_IF8:%.*]], label [[PRED_UDIV_CONTINUE9:%.*]] 6575; NO-SINK-AFTER: pred.udiv.if8: 6576; NO-SINK-AFTER-NEXT: [[TMP20:%.*]] = udiv i32 219220132, [[TMP5]] 6577; NO-SINK-AFTER-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP18]], i32 [[TMP20]], i32 3 6578; NO-SINK-AFTER-NEXT: br label [[PRED_UDIV_CONTINUE9]] 6579; NO-SINK-AFTER: pred.udiv.continue9: 6580; NO-SINK-AFTER-NEXT: [[TMP22]] = phi <4 x i32> [ [[TMP18]], [[PRED_UDIV_CONTINUE7]] ], [ [[TMP21]], [[PRED_UDIV_IF8]] ] 6581; NO-SINK-AFTER-NEXT: [[TMP23:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP22]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 6582; NO-SINK-AFTER-NEXT: [[TMP24]] = add <4 x i32> [[VEC_PHI]], [[TMP23]] 6583; NO-SINK-AFTER-NEXT: [[TMP25:%.*]] = extractelement <4 x i1> [[TMP6]], i32 0 6584; NO-SINK-AFTER-NEXT: br i1 [[TMP25]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] 6585; NO-SINK-AFTER: pred.store.if: 6586; NO-SINK-AFTER-NEXT: [[TMP26:%.*]] = add i32 [[INDEX]], 0 6587; NO-SINK-AFTER-NEXT: [[TMP27:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i32 [[TMP26]] 6588; NO-SINK-AFTER-NEXT: store i32 [[TMP2]], i32* [[TMP27]], align 4 6589; NO-SINK-AFTER-NEXT: br label [[PRED_STORE_CONTINUE]] 6590; NO-SINK-AFTER: pred.store.continue: 6591; NO-SINK-AFTER-NEXT: [[TMP28:%.*]] = extractelement <4 x i1> [[TMP6]], i32 1 6592; NO-SINK-AFTER-NEXT: br i1 [[TMP28]], label [[PRED_STORE_IF10:%.*]], label [[PRED_STORE_CONTINUE11:%.*]] 6593; NO-SINK-AFTER: pred.store.if10: 6594; NO-SINK-AFTER-NEXT: [[TMP29:%.*]] = add i32 [[INDEX]], 1 6595; NO-SINK-AFTER-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[TMP29]] 6596; NO-SINK-AFTER-NEXT: store i32 [[TMP3]], i32* [[TMP30]], align 4 6597; NO-SINK-AFTER-NEXT: br label [[PRED_STORE_CONTINUE11]] 6598; NO-SINK-AFTER: pred.store.continue11: 6599; NO-SINK-AFTER-NEXT: [[TMP31:%.*]] = extractelement <4 x i1> [[TMP6]], i32 2 6600; NO-SINK-AFTER-NEXT: br i1 [[TMP31]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE13:%.*]] 6601; NO-SINK-AFTER: pred.store.if12: 6602; NO-SINK-AFTER-NEXT: [[TMP32:%.*]] = add i32 [[INDEX]], 2 6603; NO-SINK-AFTER-NEXT: [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[TMP32]] 6604; NO-SINK-AFTER-NEXT: store i32 [[TMP4]], i32* [[TMP33]], align 4 6605; NO-SINK-AFTER-NEXT: br label [[PRED_STORE_CONTINUE13]] 6606; NO-SINK-AFTER: pred.store.continue13: 6607; NO-SINK-AFTER-NEXT: [[TMP34:%.*]] = extractelement <4 x i1> [[TMP6]], i32 3 6608; NO-SINK-AFTER-NEXT: br i1 [[TMP34]], label [[PRED_STORE_IF14:%.*]], label [[PRED_STORE_CONTINUE15]] 6609; NO-SINK-AFTER: pred.store.if14: 6610; NO-SINK-AFTER-NEXT: [[TMP35:%.*]] = add i32 [[INDEX]], 3 6611; NO-SINK-AFTER-NEXT: [[TMP36:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[TMP35]] 6612; NO-SINK-AFTER-NEXT: store i32 [[TMP5]], i32* [[TMP36]], align 4 6613; NO-SINK-AFTER-NEXT: br label [[PRED_STORE_CONTINUE15]] 6614; NO-SINK-AFTER: pred.store.continue15: 6615; NO-SINK-AFTER-NEXT: [[TMP37:%.*]] = select <4 x i1> [[TMP6]], <4 x i32> [[TMP24]], <4 x i32> [[VEC_PHI]] 6616; NO-SINK-AFTER-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4 6617; NO-SINK-AFTER-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 -4, i32 -4, i32 -4, i32 -4> 6618; NO-SINK-AFTER-NEXT: [[VEC_IND_NEXT3]] = add <4 x i32> [[VEC_IND2]], <i32 4, i32 4, i32 4, i32 4> 6619; NO-SINK-AFTER-NEXT: [[TMP38:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 6620; NO-SINK-AFTER-NEXT: br i1 [[TMP38]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF52]], !llvm.loop [[LOOP56:![0-9]+]] 6621; NO-SINK-AFTER: middle.block: 6622; NO-SINK-AFTER-NEXT: [[TMP39:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP37]]) 6623; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP22]], i32 3 6624; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP22]], i32 2 6625; NO-SINK-AFTER-NEXT: br i1 true, label [[BB1:%.*]], label [[SCALAR_PH]] 6626; NO-SINK-AFTER: scalar.ph: 6627; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[BB:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 6628; NO-SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[Y]], [[BB]] ] 6629; NO-SINK-AFTER-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[BB]] ] 6630; NO-SINK-AFTER-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[BB]] ], [ [[TMP39]], [[MIDDLE_BLOCK]] ] 6631; NO-SINK-AFTER-NEXT: br label [[BB2:%.*]] 6632; NO-SINK-AFTER: bb1: 6633; NO-SINK-AFTER-NEXT: [[VAR:%.*]] = phi i32 [ [[VAR6:%.*]], [[BB2]] ], [ [[TMP39]], [[MIDDLE_BLOCK]] ] 6634; NO-SINK-AFTER-NEXT: ret i32 [[VAR]] 6635; NO-SINK-AFTER: bb2: 6636; NO-SINK-AFTER-NEXT: [[VAR3:%.*]] = phi i32 [ [[VAR8:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 6637; NO-SINK-AFTER-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ] 6638; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[VAR7:%.*]], [[BB2]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ] 6639; NO-SINK-AFTER-NEXT: [[VAR5:%.*]] = phi i32 [ [[VAR6]], [[BB2]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 6640; NO-SINK-AFTER-NEXT: [[G:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[IV]] 6641; NO-SINK-AFTER-NEXT: [[VAR6]] = add i32 [[VAR5]], [[SCALAR_RECUR]] 6642; NO-SINK-AFTER-NEXT: [[VAR7]] = udiv i32 219220132, [[VAR3]] 6643; NO-SINK-AFTER-NEXT: store i32 [[VAR3]], i32* [[G]], align 4 6644; NO-SINK-AFTER-NEXT: [[VAR8]] = add nsw i32 [[VAR3]], -1 6645; NO-SINK-AFTER-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], 1 6646; NO-SINK-AFTER-NEXT: [[VAR9:%.*]] = icmp slt i32 [[VAR3]], 2 6647; NO-SINK-AFTER-NEXT: br i1 [[VAR9]], label [[BB1]], label [[BB2]], !prof [[PROF54]], !llvm.loop [[LOOP57:![0-9]+]] 6648; 6649bb: 6650 br label %bb2 6651 6652 bb1: ; preds = %bb2 6653 %var = phi i32 [ %var6, %bb2 ] 6654 ret i32 %var 6655 6656 bb2: ; preds = %bb2, %bb 6657 %var3 = phi i32 [ %var8, %bb2 ], [ %y, %bb ] 6658 %iv = phi i32 [ %iv.next, %bb2 ], [ 0, %bb ] 6659 %var4 = phi i32 [ %var7, %bb2 ], [ 0, %bb ] 6660 %var5 = phi i32 [ %var6, %bb2 ], [ 0, %bb ] 6661 %g = getelementptr inbounds i32, i32* %x, i32 %iv 6662 %var6 = add i32 %var5, %var4 6663 %var7 = udiv i32 219220132, %var3 6664 store i32 %var3, i32* %g, align 4 6665 %var8 = add nsw i32 %var3, -1 6666 %iv.next = add nsw i32 %iv, 1 6667 %var9 = icmp slt i32 %var3, 2 6668 br i1 %var9, label %bb1, label %bb2, !prof !2 6669} 6670 6671; %vec.dead will be marked as dead instruction in the vector loop and no recipe 6672; will be created for it. Make sure a valid sink target is used. 6673define void @sink_after_dead_inst(i32* %A.ptr) { 6674; CHECK-LABEL: @sink_after_dead_inst( 6675; CHECK-NEXT: entry: 6676; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 6677; CHECK: vector.ph: 6678; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 6679; CHECK: vector.body: 6680; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 6681; CHECK-NEXT: [[OFFSET_IDX:%.*]] = zext i32 [[INDEX]] to i64 6682; CHECK-NEXT: [[SEXT:%.*]] = shl i64 [[OFFSET_IDX]], 48 6683; CHECK-NEXT: [[TMP0:%.*]] = ashr exact i64 [[SEXT]], 48 6684; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, i32* [[A_PTR:%.*]], i64 [[TMP0]] 6685; CHECK-NEXT: [[TMP2:%.*]] = bitcast i32* [[TMP1]] to <4 x i32>* 6686; CHECK-NEXT: store <4 x i32> zeroinitializer, <4 x i32>* [[TMP2]], align 4 6687; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 6688; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16 6689; CHECK-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP58:![0-9]+]] 6690; CHECK: middle.block: 6691; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]] 6692; CHECK: scalar.ph: 6693; CHECK-NEXT: br label [[LOOP:%.*]] 6694; CHECK: loop: 6695; CHECK-NEXT: br i1 undef, label [[FOR_END]], label [[LOOP]], !llvm.loop [[LOOP59:![0-9]+]] 6696; CHECK: for.end: 6697; CHECK-NEXT: ret void 6698; 6699; UNROLL-LABEL: @sink_after_dead_inst( 6700; UNROLL-NEXT: entry: 6701; UNROLL-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 6702; UNROLL: vector.ph: 6703; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 6704; UNROLL: vector.body: 6705; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 6706; UNROLL-NEXT: [[OFFSET_IDX:%.*]] = zext i32 [[INDEX]] to i64 6707; UNROLL-NEXT: [[SEXT:%.*]] = shl i64 [[OFFSET_IDX]], 48 6708; UNROLL-NEXT: [[TMP0:%.*]] = ashr exact i64 [[SEXT]], 48 6709; UNROLL-NEXT: [[TMP1:%.*]] = getelementptr i32, i32* [[A_PTR:%.*]], i64 [[TMP0]] 6710; UNROLL-NEXT: [[TMP2:%.*]] = bitcast i32* [[TMP1]] to <4 x i32>* 6711; UNROLL-NEXT: store <4 x i32> zeroinitializer, <4 x i32>* [[TMP2]], align 4 6712; UNROLL-NEXT: [[TMP3:%.*]] = getelementptr i32, i32* [[TMP1]], i64 4 6713; UNROLL-NEXT: [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>* 6714; UNROLL-NEXT: store <4 x i32> zeroinitializer, <4 x i32>* [[TMP4]], align 4 6715; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 6716; UNROLL-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16 6717; UNROLL-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP58:![0-9]+]] 6718; UNROLL: middle.block: 6719; UNROLL-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]] 6720; UNROLL: scalar.ph: 6721; UNROLL-NEXT: br label [[LOOP:%.*]] 6722; UNROLL: loop: 6723; UNROLL-NEXT: br i1 undef, label [[FOR_END]], label [[LOOP]], !llvm.loop [[LOOP59:![0-9]+]] 6724; UNROLL: for.end: 6725; UNROLL-NEXT: ret void 6726; 6727; UNROLL-NO-IC-LABEL: @sink_after_dead_inst( 6728; UNROLL-NO-IC-NEXT: entry: 6729; UNROLL-NO-IC-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 6730; UNROLL-NO-IC: vector.ph: 6731; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 6732; UNROLL-NO-IC: vector.body: 6733; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 6734; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 6735; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ] 6736; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add <4 x i16> [[VEC_IND]], <i16 4, i16 4, i16 4, i16 4> 6737; UNROLL-NO-IC-NEXT: [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16 6738; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i16 [[OFFSET_IDX]], 0 6739; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = add i16 [[OFFSET_IDX]], 1 6740; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add i16 [[OFFSET_IDX]], 2 6741; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add i16 [[OFFSET_IDX]], 3 6742; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = add i16 [[OFFSET_IDX]], 4 6743; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = add i16 [[OFFSET_IDX]], 5 6744; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = add i16 [[OFFSET_IDX]], 6 6745; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = add i16 [[OFFSET_IDX]], 7 6746; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = add <4 x i16> [[VEC_IND]], <i16 1, i16 1, i16 1, i16 1> 6747; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = add <4 x i16> [[STEP_ADD]], <i16 1, i16 1, i16 1, i16 1> 6748; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = or <4 x i16> [[TMP8]], [[TMP8]] 6749; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = or <4 x i16> [[TMP9]], [[TMP9]] 6750; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = zext <4 x i16> [[TMP10]] to <4 x i32> 6751; UNROLL-NO-IC-NEXT: [[TMP13]] = zext <4 x i16> [[TMP11]] to <4 x i32> 6752; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP12]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 6753; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = shufflevector <4 x i32> [[TMP12]], <4 x i32> [[TMP13]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 6754; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = icmp eq <4 x i32> [[TMP14]], <i32 15, i32 15, i32 15, i32 15> 6755; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = icmp eq <4 x i32> [[TMP15]], <i32 15, i32 15, i32 15, i32 15> 6756; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = icmp eq <4 x i1> [[TMP16]], <i1 true, i1 true, i1 true, i1 true> 6757; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = icmp eq <4 x i1> [[TMP17]], <i1 true, i1 true, i1 true, i1 true> 6758; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = and <4 x i1> [[TMP16]], [[TMP18]] 6759; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = and <4 x i1> [[TMP17]], [[TMP19]] 6760; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = zext <4 x i1> [[TMP20]] to <4 x i32> 6761; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = zext <4 x i1> [[TMP21]] to <4 x i32> 6762; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = getelementptr i32, i32* [[A_PTR:%.*]], i16 [[TMP0]] 6763; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = getelementptr i32, i32* [[A_PTR]], i16 [[TMP4]] 6764; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = getelementptr i32, i32* [[TMP24]], i32 0 6765; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = bitcast i32* [[TMP26]] to <4 x i32>* 6766; UNROLL-NO-IC-NEXT: store <4 x i32> zeroinitializer, <4 x i32>* [[TMP27]], align 4 6767; UNROLL-NO-IC-NEXT: [[TMP28:%.*]] = getelementptr i32, i32* [[TMP24]], i32 4 6768; UNROLL-NO-IC-NEXT: [[TMP29:%.*]] = bitcast i32* [[TMP28]] to <4 x i32>* 6769; UNROLL-NO-IC-NEXT: store <4 x i32> zeroinitializer, <4 x i32>* [[TMP29]], align 4 6770; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 6771; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT]] = add <4 x i16> [[STEP_ADD]], <i16 4, i16 4, i16 4, i16 4> 6772; UNROLL-NO-IC-NEXT: [[TMP30:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16 6773; UNROLL-NO-IC-NEXT: br i1 [[TMP30]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP58:![0-9]+]] 6774; UNROLL-NO-IC: middle.block: 6775; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i32 16, 16 6776; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP13]], i32 3 6777; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP13]], i32 2 6778; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 6779; UNROLL-NO-IC: scalar.ph: 6780; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 6781; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ 16, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 6782; UNROLL-NO-IC-NEXT: br label [[LOOP:%.*]] 6783; UNROLL-NO-IC: loop: 6784; UNROLL-NO-IC-NEXT: [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ] 6785; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[FOR_PREV:%.*]], [[LOOP]] ] 6786; UNROLL-NO-IC-NEXT: [[CMP:%.*]] = icmp eq i32 [[SCALAR_RECUR]], 15 6787; UNROLL-NO-IC-NEXT: [[C:%.*]] = icmp eq i1 [[CMP]], true 6788; UNROLL-NO-IC-NEXT: [[VEC_DEAD:%.*]] = and i1 [[C]], true 6789; UNROLL-NO-IC-NEXT: [[IV_NEXT]] = add i16 [[IV]], 1 6790; UNROLL-NO-IC-NEXT: [[B1:%.*]] = or i16 [[IV_NEXT]], [[IV_NEXT]] 6791; UNROLL-NO-IC-NEXT: [[B3:%.*]] = and i1 [[CMP]], [[C]] 6792; UNROLL-NO-IC-NEXT: [[FOR_PREV]] = zext i16 [[B1]] to i32 6793; UNROLL-NO-IC-NEXT: [[EXT:%.*]] = zext i1 [[B3]] to i32 6794; UNROLL-NO-IC-NEXT: [[A_GEP:%.*]] = getelementptr i32, i32* [[A_PTR]], i16 [[IV]] 6795; UNROLL-NO-IC-NEXT: store i32 0, i32* [[A_GEP]], align 4 6796; UNROLL-NO-IC-NEXT: br i1 [[VEC_DEAD]], label [[FOR_END]], label [[LOOP]], !llvm.loop [[LOOP59:![0-9]+]] 6797; UNROLL-NO-IC: for.end: 6798; UNROLL-NO-IC-NEXT: ret void 6799; 6800; UNROLL-NO-VF-LABEL: @sink_after_dead_inst( 6801; UNROLL-NO-VF-NEXT: entry: 6802; UNROLL-NO-VF-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 6803; UNROLL-NO-VF: vector.ph: 6804; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 6805; UNROLL-NO-VF: vector.body: 6806; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 6807; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ] 6808; UNROLL-NO-VF-NEXT: [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16 6809; UNROLL-NO-VF-NEXT: [[INDUCTION:%.*]] = add i16 [[OFFSET_IDX]], 0 6810; UNROLL-NO-VF-NEXT: [[INDUCTION1:%.*]] = add i16 [[OFFSET_IDX]], 1 6811; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i16 [[INDUCTION]], 1 6812; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = add i16 [[INDUCTION1]], 1 6813; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = or i16 [[TMP0]], [[TMP0]] 6814; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = or i16 [[TMP1]], [[TMP1]] 6815; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = zext i16 [[TMP2]] to i32 6816; UNROLL-NO-VF-NEXT: [[TMP5]] = zext i16 [[TMP3]] to i32 6817; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = icmp eq i32 [[VECTOR_RECUR]], 15 6818; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP4]], 15 6819; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = icmp eq i1 [[TMP6]], true 6820; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = icmp eq i1 [[TMP7]], true 6821; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = and i1 [[TMP6]], [[TMP8]] 6822; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = and i1 [[TMP7]], [[TMP9]] 6823; UNROLL-NO-VF-NEXT: [[TMP12:%.*]] = zext i1 [[TMP10]] to i32 6824; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = zext i1 [[TMP11]] to i32 6825; UNROLL-NO-VF-NEXT: [[TMP14:%.*]] = getelementptr i32, i32* [[A_PTR:%.*]], i16 [[INDUCTION]] 6826; UNROLL-NO-VF-NEXT: [[TMP15:%.*]] = getelementptr i32, i32* [[A_PTR]], i16 [[INDUCTION1]] 6827; UNROLL-NO-VF-NEXT: store i32 0, i32* [[TMP14]], align 4 6828; UNROLL-NO-VF-NEXT: store i32 0, i32* [[TMP15]], align 4 6829; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 6830; UNROLL-NO-VF-NEXT: [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16 6831; UNROLL-NO-VF-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP57:![0-9]+]] 6832; UNROLL-NO-VF: middle.block: 6833; UNROLL-NO-VF-NEXT: [[CMP_N:%.*]] = icmp eq i32 16, 16 6834; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 6835; UNROLL-NO-VF: scalar.ph: 6836; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ] 6837; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ 16, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 6838; UNROLL-NO-VF-NEXT: br label [[LOOP:%.*]] 6839; UNROLL-NO-VF: loop: 6840; UNROLL-NO-VF-NEXT: [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ] 6841; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[FOR_PREV:%.*]], [[LOOP]] ] 6842; UNROLL-NO-VF-NEXT: [[CMP:%.*]] = icmp eq i32 [[SCALAR_RECUR]], 15 6843; UNROLL-NO-VF-NEXT: [[C:%.*]] = icmp eq i1 [[CMP]], true 6844; UNROLL-NO-VF-NEXT: [[VEC_DEAD:%.*]] = and i1 [[C]], true 6845; UNROLL-NO-VF-NEXT: [[IV_NEXT]] = add i16 [[IV]], 1 6846; UNROLL-NO-VF-NEXT: [[B1:%.*]] = or i16 [[IV_NEXT]], [[IV_NEXT]] 6847; UNROLL-NO-VF-NEXT: [[B3:%.*]] = and i1 [[CMP]], [[C]] 6848; UNROLL-NO-VF-NEXT: [[FOR_PREV]] = zext i16 [[B1]] to i32 6849; UNROLL-NO-VF-NEXT: [[EXT:%.*]] = zext i1 [[B3]] to i32 6850; UNROLL-NO-VF-NEXT: [[A_GEP:%.*]] = getelementptr i32, i32* [[A_PTR]], i16 [[IV]] 6851; UNROLL-NO-VF-NEXT: store i32 0, i32* [[A_GEP]], align 4 6852; UNROLL-NO-VF-NEXT: br i1 [[VEC_DEAD]], label [[FOR_END]], label [[LOOP]], !llvm.loop [[LOOP58:![0-9]+]] 6853; UNROLL-NO-VF: for.end: 6854; UNROLL-NO-VF-NEXT: ret void 6855; 6856; SINK-AFTER-LABEL: @sink_after_dead_inst( 6857; SINK-AFTER-NEXT: entry: 6858; SINK-AFTER-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 6859; SINK-AFTER: vector.ph: 6860; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 6861; SINK-AFTER: vector.body: 6862; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 6863; SINK-AFTER-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 6864; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ] 6865; SINK-AFTER-NEXT: [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16 6866; SINK-AFTER-NEXT: [[TMP0:%.*]] = add i16 [[OFFSET_IDX]], 0 6867; SINK-AFTER-NEXT: [[TMP1:%.*]] = add i16 [[OFFSET_IDX]], 1 6868; SINK-AFTER-NEXT: [[TMP2:%.*]] = add i16 [[OFFSET_IDX]], 2 6869; SINK-AFTER-NEXT: [[TMP3:%.*]] = add i16 [[OFFSET_IDX]], 3 6870; SINK-AFTER-NEXT: [[TMP4:%.*]] = add <4 x i16> [[VEC_IND]], <i16 1, i16 1, i16 1, i16 1> 6871; SINK-AFTER-NEXT: [[TMP5:%.*]] = or <4 x i16> [[TMP4]], [[TMP4]] 6872; SINK-AFTER-NEXT: [[TMP6]] = zext <4 x i16> [[TMP5]] to <4 x i32> 6873; SINK-AFTER-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP6]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 6874; SINK-AFTER-NEXT: [[TMP8:%.*]] = icmp eq <4 x i32> [[TMP7]], <i32 15, i32 15, i32 15, i32 15> 6875; SINK-AFTER-NEXT: [[TMP9:%.*]] = icmp eq <4 x i1> [[TMP8]], <i1 true, i1 true, i1 true, i1 true> 6876; SINK-AFTER-NEXT: [[TMP10:%.*]] = and <4 x i1> [[TMP8]], [[TMP9]] 6877; SINK-AFTER-NEXT: [[TMP11:%.*]] = zext <4 x i1> [[TMP10]] to <4 x i32> 6878; SINK-AFTER-NEXT: [[TMP12:%.*]] = getelementptr i32, i32* [[A_PTR:%.*]], i16 [[TMP0]] 6879; SINK-AFTER-NEXT: [[TMP13:%.*]] = getelementptr i32, i32* [[TMP12]], i32 0 6880; SINK-AFTER-NEXT: [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <4 x i32>* 6881; SINK-AFTER-NEXT: store <4 x i32> zeroinitializer, <4 x i32>* [[TMP14]], align 4 6882; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 6883; SINK-AFTER-NEXT: [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], <i16 4, i16 4, i16 4, i16 4> 6884; SINK-AFTER-NEXT: [[TMP15:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16 6885; SINK-AFTER-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP58:![0-9]+]] 6886; SINK-AFTER: middle.block: 6887; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i32 16, 16 6888; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP6]], i32 3 6889; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP6]], i32 2 6890; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 6891; SINK-AFTER: scalar.ph: 6892; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 6893; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ 16, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 6894; SINK-AFTER-NEXT: br label [[LOOP:%.*]] 6895; SINK-AFTER: loop: 6896; SINK-AFTER-NEXT: [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ] 6897; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[FOR_PREV:%.*]], [[LOOP]] ] 6898; SINK-AFTER-NEXT: [[CMP:%.*]] = icmp eq i32 [[SCALAR_RECUR]], 15 6899; SINK-AFTER-NEXT: [[C:%.*]] = icmp eq i1 [[CMP]], true 6900; SINK-AFTER-NEXT: [[VEC_DEAD:%.*]] = and i1 [[C]], true 6901; SINK-AFTER-NEXT: [[IV_NEXT]] = add i16 [[IV]], 1 6902; SINK-AFTER-NEXT: [[B1:%.*]] = or i16 [[IV_NEXT]], [[IV_NEXT]] 6903; SINK-AFTER-NEXT: [[B3:%.*]] = and i1 [[CMP]], [[C]] 6904; SINK-AFTER-NEXT: [[FOR_PREV]] = zext i16 [[B1]] to i32 6905; SINK-AFTER-NEXT: [[EXT:%.*]] = zext i1 [[B3]] to i32 6906; SINK-AFTER-NEXT: [[A_GEP:%.*]] = getelementptr i32, i32* [[A_PTR]], i16 [[IV]] 6907; SINK-AFTER-NEXT: store i32 0, i32* [[A_GEP]], align 4 6908; SINK-AFTER-NEXT: br i1 [[VEC_DEAD]], label [[FOR_END]], label [[LOOP]], !llvm.loop [[LOOP59:![0-9]+]] 6909; SINK-AFTER: for.end: 6910; SINK-AFTER-NEXT: ret void 6911; 6912; NO-SINK-AFTER-LABEL: @sink_after_dead_inst( 6913; NO-SINK-AFTER-NEXT: entry: 6914; NO-SINK-AFTER-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 6915; NO-SINK-AFTER: vector.ph: 6916; NO-SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 6917; NO-SINK-AFTER: vector.body: 6918; NO-SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 6919; NO-SINK-AFTER-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 6920; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ] 6921; NO-SINK-AFTER-NEXT: [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16 6922; NO-SINK-AFTER-NEXT: [[TMP0:%.*]] = add i16 [[OFFSET_IDX]], 0 6923; NO-SINK-AFTER-NEXT: [[TMP1:%.*]] = add i16 [[OFFSET_IDX]], 1 6924; NO-SINK-AFTER-NEXT: [[TMP2:%.*]] = add i16 [[OFFSET_IDX]], 2 6925; NO-SINK-AFTER-NEXT: [[TMP3:%.*]] = add i16 [[OFFSET_IDX]], 3 6926; NO-SINK-AFTER-NEXT: [[TMP4:%.*]] = add <4 x i16> [[VEC_IND]], <i16 1, i16 1, i16 1, i16 1> 6927; NO-SINK-AFTER-NEXT: [[TMP5:%.*]] = or <4 x i16> [[TMP4]], [[TMP4]] 6928; NO-SINK-AFTER-NEXT: [[TMP6]] = zext <4 x i16> [[TMP5]] to <4 x i32> 6929; NO-SINK-AFTER-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP6]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 6930; NO-SINK-AFTER-NEXT: [[TMP8:%.*]] = icmp eq <4 x i32> [[TMP7]], <i32 15, i32 15, i32 15, i32 15> 6931; NO-SINK-AFTER-NEXT: [[TMP9:%.*]] = icmp eq <4 x i1> [[TMP8]], <i1 true, i1 true, i1 true, i1 true> 6932; NO-SINK-AFTER-NEXT: [[TMP10:%.*]] = and <4 x i1> [[TMP8]], [[TMP9]] 6933; NO-SINK-AFTER-NEXT: [[TMP11:%.*]] = zext <4 x i1> [[TMP10]] to <4 x i32> 6934; NO-SINK-AFTER-NEXT: [[TMP12:%.*]] = getelementptr i32, i32* [[A_PTR:%.*]], i16 [[TMP0]] 6935; NO-SINK-AFTER-NEXT: [[TMP13:%.*]] = getelementptr i32, i32* [[TMP12]], i32 0 6936; NO-SINK-AFTER-NEXT: [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <4 x i32>* 6937; NO-SINK-AFTER-NEXT: store <4 x i32> zeroinitializer, <4 x i32>* [[TMP14]], align 4 6938; NO-SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 6939; NO-SINK-AFTER-NEXT: [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], <i16 4, i16 4, i16 4, i16 4> 6940; NO-SINK-AFTER-NEXT: [[TMP15:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16 6941; NO-SINK-AFTER-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP58:![0-9]+]] 6942; NO-SINK-AFTER: middle.block: 6943; NO-SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i32 16, 16 6944; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP6]], i32 3 6945; NO-SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP6]], i32 2 6946; NO-SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 6947; NO-SINK-AFTER: scalar.ph: 6948; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 6949; NO-SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ 16, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 6950; NO-SINK-AFTER-NEXT: br label [[LOOP:%.*]] 6951; NO-SINK-AFTER: loop: 6952; NO-SINK-AFTER-NEXT: [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ] 6953; NO-SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[FOR_PREV:%.*]], [[LOOP]] ] 6954; NO-SINK-AFTER-NEXT: [[CMP:%.*]] = icmp eq i32 [[SCALAR_RECUR]], 15 6955; NO-SINK-AFTER-NEXT: [[C:%.*]] = icmp eq i1 [[CMP]], true 6956; NO-SINK-AFTER-NEXT: [[VEC_DEAD:%.*]] = and i1 [[C]], true 6957; NO-SINK-AFTER-NEXT: [[IV_NEXT]] = add i16 [[IV]], 1 6958; NO-SINK-AFTER-NEXT: [[B1:%.*]] = or i16 [[IV_NEXT]], [[IV_NEXT]] 6959; NO-SINK-AFTER-NEXT: [[B3:%.*]] = and i1 [[CMP]], [[C]] 6960; NO-SINK-AFTER-NEXT: [[FOR_PREV]] = zext i16 [[B1]] to i32 6961; NO-SINK-AFTER-NEXT: [[EXT:%.*]] = zext i1 [[B3]] to i32 6962; NO-SINK-AFTER-NEXT: [[A_GEP:%.*]] = getelementptr i32, i32* [[A_PTR]], i16 [[IV]] 6963; NO-SINK-AFTER-NEXT: store i32 0, i32* [[A_GEP]], align 4 6964; NO-SINK-AFTER-NEXT: br i1 [[VEC_DEAD]], label [[FOR_END]], label [[LOOP]], !llvm.loop [[LOOP59:![0-9]+]] 6965; NO-SINK-AFTER: for.end: 6966; NO-SINK-AFTER-NEXT: ret void 6967; 6968entry: 6969 br label %loop 6970 6971loop: 6972 %iv = phi i16 [ 0, %entry ], [ %iv.next, %loop ] 6973 %for = phi i32 [ 0, %entry ], [ %for.prev, %loop ] 6974 %cmp = icmp eq i32 %for, 15 6975 %C = icmp eq i1 %cmp, true 6976 %vec.dead = and i1 %C, 1 6977 %iv.next = add i16 %iv, 1 6978 %B1 = or i16 %iv.next, %iv.next 6979 %B3 = and i1 %cmp, %C 6980 %for.prev = zext i16 %B1 to i32 6981 6982 %ext = zext i1 %B3 to i32 6983 %A.gep = getelementptr i32, i32* %A.ptr, i16 %iv 6984 store i32 0, i32* %A.gep 6985 br i1 %vec.dead, label %for.end, label %loop 6986 6987for.end: 6988 ret void 6989} 6990 6991!2 = !{!"branch_weights", i32 1, i32 1} 6992