1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py 2; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -dce -instcombine -S | FileCheck %s 3; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -dce -instcombine -S | FileCheck %s --check-prefix=UNROLL 4; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S | FileCheck %s --check-prefix=UNROLL-NO-IC 5; RUN: opt < %s -loop-vectorize -force-vector-width=1 -force-vector-interleave=2 -S | FileCheck %s --check-prefix=UNROLL-NO-VF 6; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s --check-prefix=SINK-AFTER 7 8target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" 9 10; void recurrence_1(int *a, int *b, int n) { 11; for(int i = 0; i < n; i++) 12; b[i] = a[i] + a[i - 1] 13; } 14; 15; 16; 17define void @recurrence_1(i32* nocapture readonly %a, i32* nocapture %b, i32 %n) { 18; CHECK-LABEL: @recurrence_1( 19; CHECK-NEXT: entry: 20; CHECK-NEXT: [[A2:%.*]] = ptrtoint i32* [[A:%.*]] to i64 21; CHECK-NEXT: [[B1:%.*]] = ptrtoint i32* [[B:%.*]] to i64 22; CHECK-NEXT: br label [[FOR_PREHEADER:%.*]] 23; CHECK: for.preheader: 24; CHECK-NEXT: [[PRE_LOAD:%.*]] = load i32, i32* [[A]], align 4 25; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 26; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 27; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 28; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 3 29; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 30; CHECK: vector.memcheck: 31; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[A2]], 4 32; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[TMP3]] 33; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], 16 34; CHECK-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 35; CHECK: vector.ph: 36; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934588 37; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[PRE_LOAD]], i64 3 38; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 39; CHECK: vector.body: 40; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 41; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ] 42; CHECK-NEXT: [[TMP7:%.*]] = or i64 [[INDEX]], 1 43; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP7]] 44; CHECK-NEXT: [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>* 45; CHECK-NEXT: [[WIDE_LOAD]] = load <4 x i32>, <4 x i32>* [[TMP9]], align 4 46; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 47; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]] 48; CHECK-NEXT: [[TMP12:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[TMP10]] 49; CHECK-NEXT: [[TMP13:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>* 50; CHECK-NEXT: store <4 x i32> [[TMP12]], <4 x i32>* [[TMP13]], align 4 51; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 52; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 53; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] 54; CHECK: middle.block: 55; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 56; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 3 57; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]] 58; CHECK: scalar.ph: 59; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[PRE_LOAD]], [[VECTOR_MEMCHECK]] ], [ [[PRE_LOAD]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 60; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[FOR_PREHEADER]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ] 61; CHECK-NEXT: br label [[SCALAR_BODY:%.*]] 62; CHECK: scalar.body: 63; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP15:%.*]], [[SCALAR_BODY]] ] 64; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ] 65; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 66; CHECK-NEXT: [[ARRAYIDX32:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT]] 67; CHECK-NEXT: [[TMP15]] = load i32, i32* [[ARRAYIDX32]], align 4 68; CHECK-NEXT: [[ARRAYIDX34:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 69; CHECK-NEXT: [[ADD35:%.*]] = add i32 [[TMP15]], [[SCALAR_RECUR]] 70; CHECK-NEXT: store i32 [[ADD35]], i32* [[ARRAYIDX34]], align 4 71; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 72; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 73; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] 74; CHECK: for.exit: 75; CHECK-NEXT: ret void 76; 77; UNROLL-LABEL: @recurrence_1( 78; UNROLL-NEXT: entry: 79; UNROLL-NEXT: [[A2:%.*]] = ptrtoint i32* [[A:%.*]] to i64 80; UNROLL-NEXT: [[B1:%.*]] = ptrtoint i32* [[B:%.*]] to i64 81; UNROLL-NEXT: br label [[FOR_PREHEADER:%.*]] 82; UNROLL: for.preheader: 83; UNROLL-NEXT: [[PRE_LOAD:%.*]] = load i32, i32* [[A]], align 4 84; UNROLL-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 85; UNROLL-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 86; UNROLL-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 87; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 7 88; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 89; UNROLL: vector.memcheck: 90; UNROLL-NEXT: [[TMP3:%.*]] = add i64 [[A2]], 4 91; UNROLL-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[TMP3]] 92; UNROLL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], 32 93; UNROLL-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 94; UNROLL: vector.ph: 95; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934584 96; UNROLL-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[PRE_LOAD]], i64 3 97; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 98; UNROLL: vector.body: 99; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 100; UNROLL-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD7:%.*]], [[VECTOR_BODY]] ] 101; UNROLL-NEXT: [[TMP7:%.*]] = or i64 [[INDEX]], 1 102; UNROLL-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP7]] 103; UNROLL-NEXT: [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>* 104; UNROLL-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP9]], align 4 105; UNROLL-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP8]], i64 4 106; UNROLL-NEXT: [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <4 x i32>* 107; UNROLL-NEXT: [[WIDE_LOAD7]] = load <4 x i32>, <4 x i32>* [[TMP11]], align 4 108; UNROLL-NEXT: [[TMP12:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 109; UNROLL-NEXT: [[TMP13:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> [[WIDE_LOAD7]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 110; UNROLL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]] 111; UNROLL-NEXT: [[TMP15:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[TMP12]] 112; UNROLL-NEXT: [[TMP16:%.*]] = add <4 x i32> [[WIDE_LOAD7]], [[TMP13]] 113; UNROLL-NEXT: [[TMP17:%.*]] = bitcast i32* [[TMP14]] to <4 x i32>* 114; UNROLL-NEXT: store <4 x i32> [[TMP15]], <4 x i32>* [[TMP17]], align 4 115; UNROLL-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[TMP14]], i64 4 116; UNROLL-NEXT: [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <4 x i32>* 117; UNROLL-NEXT: store <4 x i32> [[TMP16]], <4 x i32>* [[TMP19]], align 4 118; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 119; UNROLL-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 120; UNROLL-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] 121; UNROLL: middle.block: 122; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 123; UNROLL-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[WIDE_LOAD7]], i64 3 124; UNROLL-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]] 125; UNROLL: scalar.ph: 126; UNROLL-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[PRE_LOAD]], [[VECTOR_MEMCHECK]] ], [ [[PRE_LOAD]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 127; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[FOR_PREHEADER]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ] 128; UNROLL-NEXT: br label [[SCALAR_BODY:%.*]] 129; UNROLL: scalar.body: 130; UNROLL-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP21:%.*]], [[SCALAR_BODY]] ] 131; UNROLL-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ] 132; UNROLL-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 133; UNROLL-NEXT: [[ARRAYIDX32:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT]] 134; UNROLL-NEXT: [[TMP21]] = load i32, i32* [[ARRAYIDX32]], align 4 135; UNROLL-NEXT: [[ARRAYIDX34:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 136; UNROLL-NEXT: [[ADD35:%.*]] = add i32 [[TMP21]], [[SCALAR_RECUR]] 137; UNROLL-NEXT: store i32 [[ADD35]], i32* [[ARRAYIDX34]], align 4 138; UNROLL-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 139; UNROLL-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 140; UNROLL-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] 141; UNROLL: for.exit: 142; UNROLL-NEXT: ret void 143; 144; UNROLL-NO-IC-LABEL: @recurrence_1( 145; UNROLL-NO-IC-NEXT: entry: 146; UNROLL-NO-IC-NEXT: [[A2:%.*]] = ptrtoint i32* [[A:%.*]] to i64 147; UNROLL-NO-IC-NEXT: [[B1:%.*]] = ptrtoint i32* [[B:%.*]] to i64 148; UNROLL-NO-IC-NEXT: br label [[FOR_PREHEADER:%.*]] 149; UNROLL-NO-IC: for.preheader: 150; UNROLL-NO-IC-NEXT: [[ARRAYIDX_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 0 151; UNROLL-NO-IC-NEXT: [[PRE_LOAD:%.*]] = load i32, i32* [[ARRAYIDX_PHI_TRANS_INSERT]], align 4 152; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 153; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 154; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 155; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 8 156; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 157; UNROLL-NO-IC: vector.memcheck: 158; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add i64 [[A2]], 4 159; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[TMP3]] 160; UNROLL-NO-IC-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], 32 161; UNROLL-NO-IC-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 162; UNROLL-NO-IC: vector.ph: 163; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 8 164; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 165; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[PRE_LOAD]], i32 3 166; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 167; UNROLL-NO-IC: vector.body: 168; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 169; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD7:%.*]], [[VECTOR_BODY]] ] 170; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 0 171; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 4 172; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = add nuw nsw i64 [[TMP7]], 1 173; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = add nuw nsw i64 [[TMP8]], 1 174; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP9]] 175; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP10]] 176; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[TMP11]], i32 0 177; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <4 x i32>* 178; UNROLL-NO-IC-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP14]], align 4 179; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, i32* [[TMP11]], i32 4 180; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = bitcast i32* [[TMP15]] to <4 x i32>* 181; UNROLL-NO-IC-NEXT: [[WIDE_LOAD7]] = load <4 x i32>, <4 x i32>* [[TMP16]], align 4 182; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 183; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> [[WIDE_LOAD7]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 184; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP7]] 185; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP8]] 186; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[TMP17]] 187; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = add <4 x i32> [[WIDE_LOAD7]], [[TMP18]] 188; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[TMP19]], i32 0 189; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <4 x i32>* 190; UNROLL-NO-IC-NEXT: store <4 x i32> [[TMP21]], <4 x i32>* [[TMP24]], align 4 191; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, i32* [[TMP19]], i32 4 192; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = bitcast i32* [[TMP25]] to <4 x i32>* 193; UNROLL-NO-IC-NEXT: store <4 x i32> [[TMP22]], <4 x i32>* [[TMP26]], align 4 194; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 195; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 196; UNROLL-NO-IC-NEXT: br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] 197; UNROLL-NO-IC: middle.block: 198; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 199; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[WIDE_LOAD7]], i32 3 200; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[WIDE_LOAD7]], i32 2 201; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]] 202; UNROLL-NO-IC: scalar.ph: 203; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[PRE_LOAD]], [[VECTOR_MEMCHECK]] ], [ [[PRE_LOAD]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 204; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ] 205; UNROLL-NO-IC-NEXT: br label [[SCALAR_BODY:%.*]] 206; UNROLL-NO-IC: scalar.body: 207; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP28:%.*]], [[SCALAR_BODY]] ] 208; UNROLL-NO-IC-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ] 209; UNROLL-NO-IC-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 210; UNROLL-NO-IC-NEXT: [[ARRAYIDX32:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT]] 211; UNROLL-NO-IC-NEXT: [[TMP28]] = load i32, i32* [[ARRAYIDX32]], align 4 212; UNROLL-NO-IC-NEXT: [[ARRAYIDX34:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 213; UNROLL-NO-IC-NEXT: [[ADD35:%.*]] = add i32 [[TMP28]], [[SCALAR_RECUR]] 214; UNROLL-NO-IC-NEXT: store i32 [[ADD35]], i32* [[ARRAYIDX34]], align 4 215; UNROLL-NO-IC-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 216; UNROLL-NO-IC-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 217; UNROLL-NO-IC-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] 218; UNROLL-NO-IC: for.exit: 219; UNROLL-NO-IC-NEXT: ret void 220; 221; UNROLL-NO-VF-LABEL: @recurrence_1( 222; UNROLL-NO-VF-NEXT: entry: 223; UNROLL-NO-VF-NEXT: [[A2:%.*]] = ptrtoint i32* [[A:%.*]] to i64 224; UNROLL-NO-VF-NEXT: [[B1:%.*]] = ptrtoint i32* [[B:%.*]] to i64 225; UNROLL-NO-VF-NEXT: br label [[FOR_PREHEADER:%.*]] 226; UNROLL-NO-VF: for.preheader: 227; UNROLL-NO-VF-NEXT: [[ARRAYIDX_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 0 228; UNROLL-NO-VF-NEXT: [[PRE_LOAD:%.*]] = load i32, i32* [[ARRAYIDX_PHI_TRANS_INSERT]], align 4 229; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 230; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 231; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 232; UNROLL-NO-VF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 2 233; UNROLL-NO-VF-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 234; UNROLL-NO-VF: vector.memcheck: 235; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = add i64 [[A2]], 4 236; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[TMP3]] 237; UNROLL-NO-VF-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], 8 238; UNROLL-NO-VF-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 239; UNROLL-NO-VF: vector.ph: 240; UNROLL-NO-VF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 2 241; UNROLL-NO-VF-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 242; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 243; UNROLL-NO-VF: vector.body: 244; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 245; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i32 [ [[PRE_LOAD]], [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ] 246; UNROLL-NO-VF-NEXT: [[INDUCTION:%.*]] = add i64 [[INDEX]], 0 247; UNROLL-NO-VF-NEXT: [[INDUCTION7:%.*]] = add i64 [[INDEX]], 1 248; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = add nuw nsw i64 [[INDUCTION]], 1 249; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = add nuw nsw i64 [[INDUCTION7]], 1 250; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP7]] 251; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP8]] 252; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = load i32, i32* [[TMP9]], align 4 253; UNROLL-NO-VF-NEXT: [[TMP12]] = load i32, i32* [[TMP10]], align 4 254; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDUCTION]] 255; UNROLL-NO-VF-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDUCTION7]] 256; UNROLL-NO-VF-NEXT: [[TMP15:%.*]] = add i32 [[TMP11]], [[VECTOR_RECUR]] 257; UNROLL-NO-VF-NEXT: [[TMP16:%.*]] = add i32 [[TMP12]], [[TMP11]] 258; UNROLL-NO-VF-NEXT: store i32 [[TMP15]], i32* [[TMP13]], align 4 259; UNROLL-NO-VF-NEXT: store i32 [[TMP16]], i32* [[TMP14]], align 4 260; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 261; UNROLL-NO-VF-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 262; UNROLL-NO-VF-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] 263; UNROLL-NO-VF: middle.block: 264; UNROLL-NO-VF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 265; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]] 266; UNROLL-NO-VF: scalar.ph: 267; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[PRE_LOAD]], [[VECTOR_MEMCHECK]] ], [ [[PRE_LOAD]], [[FOR_PREHEADER]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ] 268; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ] 269; UNROLL-NO-VF-NEXT: br label [[SCALAR_BODY:%.*]] 270; UNROLL-NO-VF: scalar.body: 271; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP18:%.*]], [[SCALAR_BODY]] ] 272; UNROLL-NO-VF-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ] 273; UNROLL-NO-VF-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 274; UNROLL-NO-VF-NEXT: [[ARRAYIDX32:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT]] 275; UNROLL-NO-VF-NEXT: [[TMP18]] = load i32, i32* [[ARRAYIDX32]], align 4 276; UNROLL-NO-VF-NEXT: [[ARRAYIDX34:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 277; UNROLL-NO-VF-NEXT: [[ADD35:%.*]] = add i32 [[TMP18]], [[SCALAR_RECUR]] 278; UNROLL-NO-VF-NEXT: store i32 [[ADD35]], i32* [[ARRAYIDX34]], align 4 279; UNROLL-NO-VF-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 280; UNROLL-NO-VF-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 281; UNROLL-NO-VF-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] 282; UNROLL-NO-VF: for.exit: 283; UNROLL-NO-VF-NEXT: ret void 284; 285; SINK-AFTER-LABEL: @recurrence_1( 286; SINK-AFTER-NEXT: entry: 287; SINK-AFTER-NEXT: [[A2:%.*]] = ptrtoint i32* [[A:%.*]] to i64 288; SINK-AFTER-NEXT: [[B1:%.*]] = ptrtoint i32* [[B:%.*]] to i64 289; SINK-AFTER-NEXT: br label [[FOR_PREHEADER:%.*]] 290; SINK-AFTER: for.preheader: 291; SINK-AFTER-NEXT: [[ARRAYIDX_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 0 292; SINK-AFTER-NEXT: [[PRE_LOAD:%.*]] = load i32, i32* [[ARRAYIDX_PHI_TRANS_INSERT]], align 4 293; SINK-AFTER-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 294; SINK-AFTER-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 295; SINK-AFTER-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 296; SINK-AFTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4 297; SINK-AFTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 298; SINK-AFTER: vector.memcheck: 299; SINK-AFTER-NEXT: [[TMP3:%.*]] = add i64 [[A2]], 4 300; SINK-AFTER-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[TMP3]] 301; SINK-AFTER-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], 16 302; SINK-AFTER-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 303; SINK-AFTER: vector.ph: 304; SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4 305; SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 306; SINK-AFTER-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[PRE_LOAD]], i32 3 307; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 308; SINK-AFTER: vector.body: 309; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 310; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ] 311; SINK-AFTER-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 0 312; SINK-AFTER-NEXT: [[TMP8:%.*]] = add nuw nsw i64 [[TMP7]], 1 313; SINK-AFTER-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP8]] 314; SINK-AFTER-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP9]], i32 0 315; SINK-AFTER-NEXT: [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <4 x i32>* 316; SINK-AFTER-NEXT: [[WIDE_LOAD]] = load <4 x i32>, <4 x i32>* [[TMP11]], align 4 317; SINK-AFTER-NEXT: [[TMP12:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 318; SINK-AFTER-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP7]] 319; SINK-AFTER-NEXT: [[TMP14:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[TMP12]] 320; SINK-AFTER-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, i32* [[TMP13]], i32 0 321; SINK-AFTER-NEXT: [[TMP16:%.*]] = bitcast i32* [[TMP15]] to <4 x i32>* 322; SINK-AFTER-NEXT: store <4 x i32> [[TMP14]], <4 x i32>* [[TMP16]], align 4 323; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 324; SINK-AFTER-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 325; SINK-AFTER-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] 326; SINK-AFTER: middle.block: 327; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 328; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i32 3 329; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i32 2 330; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]] 331; SINK-AFTER: scalar.ph: 332; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[PRE_LOAD]], [[VECTOR_MEMCHECK]] ], [ [[PRE_LOAD]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 333; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ] 334; SINK-AFTER-NEXT: br label [[SCALAR_BODY:%.*]] 335; SINK-AFTER: scalar.body: 336; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP18:%.*]], [[SCALAR_BODY]] ] 337; SINK-AFTER-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ] 338; SINK-AFTER-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 339; SINK-AFTER-NEXT: [[ARRAYIDX32:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT]] 340; SINK-AFTER-NEXT: [[TMP18]] = load i32, i32* [[ARRAYIDX32]], align 4 341; SINK-AFTER-NEXT: [[ARRAYIDX34:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 342; SINK-AFTER-NEXT: [[ADD35:%.*]] = add i32 [[TMP18]], [[SCALAR_RECUR]] 343; SINK-AFTER-NEXT: store i32 [[ADD35]], i32* [[ARRAYIDX34]], align 4 344; SINK-AFTER-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 345; SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 346; SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] 347; SINK-AFTER: for.exit: 348; SINK-AFTER-NEXT: ret void 349; 350entry: 351 br label %for.preheader 352 353for.preheader: 354 %arrayidx.phi.trans.insert = getelementptr inbounds i32, i32* %a, i64 0 355 %pre_load = load i32, i32* %arrayidx.phi.trans.insert 356 br label %scalar.body 357 358scalar.body: 359 %0 = phi i32 [ %pre_load, %for.preheader ], [ %1, %scalar.body ] 360 %indvars.iv = phi i64 [ 0, %for.preheader ], [ %indvars.iv.next, %scalar.body ] 361 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 362 %arrayidx32 = getelementptr inbounds i32, i32* %a, i64 %indvars.iv.next 363 %1 = load i32, i32* %arrayidx32 364 %arrayidx34 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv 365 %add35 = add i32 %1, %0 366 store i32 %add35, i32* %arrayidx34 367 %lftr.wideiv = trunc i64 %indvars.iv.next to i32 368 %exitcond = icmp eq i32 %lftr.wideiv, %n 369 br i1 %exitcond, label %for.exit, label %scalar.body 370 371for.exit: 372 ret void 373} 374 375; int recurrence_2(int *a, int n) { 376; int minmax; 377; for (int i = 0; i < n; ++i) 378; minmax = min(minmax, max(a[i] - a[i-1], 0)); 379; return minmax; 380; } 381; 382; 383; 384define i32 @recurrence_2(i32* nocapture readonly %a, i32 %n) { 385; CHECK-LABEL: @recurrence_2( 386; CHECK-NEXT: entry: 387; CHECK-NEXT: [[CMP27:%.*]] = icmp sgt i32 [[N:%.*]], 0 388; CHECK-NEXT: br i1 [[CMP27]], label [[FOR_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]] 389; CHECK: for.preheader: 390; CHECK-NEXT: [[ARRAYIDX2_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 -1 391; CHECK-NEXT: [[DOTPRE:%.*]] = load i32, i32* [[ARRAYIDX2_PHI_TRANS_INSERT]], align 4 392; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1 393; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 394; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 395; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 3 396; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 397; CHECK: vector.ph: 398; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934588 399; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[DOTPRE]], i64 3 400; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 401; CHECK: vector.body: 402; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 403; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ] 404; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ poison, [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[VECTOR_BODY]] ] 405; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDEX]] 406; CHECK-NEXT: [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>* 407; CHECK-NEXT: [[WIDE_LOAD]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4 408; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 409; CHECK-NEXT: [[TMP6:%.*]] = sub nsw <4 x i32> [[WIDE_LOAD]], [[TMP5]] 410; CHECK-NEXT: [[TMP7:%.*]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[TMP6]], <4 x i32> zeroinitializer) 411; CHECK-NEXT: [[TMP8]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[TMP7]]) 412; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 413; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 414; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] 415; CHECK: middle.block: 416; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[TMP8]]) 417; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 418; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 3 419; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]] 420; CHECK: scalar.ph: 421; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[DOTPRE]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 422; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[FOR_PREHEADER]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ] 423; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ poison, [[FOR_PREHEADER]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ] 424; CHECK-NEXT: br label [[SCALAR_BODY:%.*]] 425; CHECK: for.cond.cleanup.loopexit: 426; CHECK-NEXT: [[MINMAX_0_COND_LCSSA:%.*]] = phi i32 [ [[TMP13:%.*]], [[SCALAR_BODY]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ] 427; CHECK-NEXT: br label [[FOR_COND_CLEANUP]] 428; CHECK: for.cond.cleanup: 429; CHECK-NEXT: [[MINMAX_0_LCSSA:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[MINMAX_0_COND_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ] 430; CHECK-NEXT: ret i32 [[MINMAX_0_LCSSA]] 431; CHECK: scalar.body: 432; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP11:%.*]], [[SCALAR_BODY]] ] 433; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ] 434; CHECK-NEXT: [[MINMAX_028:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[TMP13]], [[SCALAR_BODY]] ] 435; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]] 436; CHECK-NEXT: [[TMP11]] = load i32, i32* [[ARRAYIDX]], align 4 437; CHECK-NEXT: [[SUB3:%.*]] = sub nsw i32 [[TMP11]], [[SCALAR_RECUR]] 438; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.smax.i32(i32 [[SUB3]], i32 0) 439; CHECK-NEXT: [[TMP13]] = call i32 @llvm.smin.i32(i32 [[MINMAX_028]], i32 [[TMP12]]) 440; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 441; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 442; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 443; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] 444; 445; UNROLL-LABEL: @recurrence_2( 446; UNROLL-NEXT: entry: 447; UNROLL-NEXT: [[CMP27:%.*]] = icmp sgt i32 [[N:%.*]], 0 448; UNROLL-NEXT: br i1 [[CMP27]], label [[FOR_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]] 449; UNROLL: for.preheader: 450; UNROLL-NEXT: [[ARRAYIDX2_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 -1 451; UNROLL-NEXT: [[DOTPRE:%.*]] = load i32, i32* [[ARRAYIDX2_PHI_TRANS_INSERT]], align 4 452; UNROLL-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1 453; UNROLL-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 454; UNROLL-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 455; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 7 456; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 457; UNROLL: vector.ph: 458; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934584 459; UNROLL-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[DOTPRE]], i64 3 460; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 461; UNROLL: vector.body: 462; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 463; UNROLL-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD2:%.*]], [[VECTOR_BODY]] ] 464; UNROLL-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ poison, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ] 465; UNROLL-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ poison, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ] 466; UNROLL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDEX]] 467; UNROLL-NEXT: [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>* 468; UNROLL-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4 469; UNROLL-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP3]], i64 4 470; UNROLL-NEXT: [[TMP6:%.*]] = bitcast i32* [[TMP5]] to <4 x i32>* 471; UNROLL-NEXT: [[WIDE_LOAD2]] = load <4 x i32>, <4 x i32>* [[TMP6]], align 4 472; UNROLL-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 473; UNROLL-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> [[WIDE_LOAD2]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 474; UNROLL-NEXT: [[TMP9:%.*]] = sub nsw <4 x i32> [[WIDE_LOAD]], [[TMP7]] 475; UNROLL-NEXT: [[TMP10:%.*]] = sub nsw <4 x i32> [[WIDE_LOAD2]], [[TMP8]] 476; UNROLL-NEXT: [[TMP11:%.*]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[TMP9]], <4 x i32> zeroinitializer) 477; UNROLL-NEXT: [[TMP12:%.*]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[TMP10]], <4 x i32> zeroinitializer) 478; UNROLL-NEXT: [[TMP13]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[TMP11]]) 479; UNROLL-NEXT: [[TMP14]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[VEC_PHI1]], <4 x i32> [[TMP12]]) 480; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 481; UNROLL-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 482; UNROLL-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] 483; UNROLL: middle.block: 484; UNROLL-NEXT: [[TMP16:%.*]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[TMP13]], <4 x i32> [[TMP14]]) 485; UNROLL-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[TMP16]]) 486; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 487; UNROLL-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i64 3 488; UNROLL-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]] 489; UNROLL: scalar.ph: 490; UNROLL-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[DOTPRE]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 491; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[FOR_PREHEADER]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ] 492; UNROLL-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ poison, [[FOR_PREHEADER]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ] 493; UNROLL-NEXT: br label [[SCALAR_BODY:%.*]] 494; UNROLL: for.cond.cleanup.loopexit: 495; UNROLL-NEXT: [[MINMAX_0_COND_LCSSA:%.*]] = phi i32 [ [[TMP20:%.*]], [[SCALAR_BODY]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ] 496; UNROLL-NEXT: br label [[FOR_COND_CLEANUP]] 497; UNROLL: for.cond.cleanup: 498; UNROLL-NEXT: [[MINMAX_0_LCSSA:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[MINMAX_0_COND_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ] 499; UNROLL-NEXT: ret i32 [[MINMAX_0_LCSSA]] 500; UNROLL: scalar.body: 501; UNROLL-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP18:%.*]], [[SCALAR_BODY]] ] 502; UNROLL-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ] 503; UNROLL-NEXT: [[MINMAX_028:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[TMP20]], [[SCALAR_BODY]] ] 504; UNROLL-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]] 505; UNROLL-NEXT: [[TMP18]] = load i32, i32* [[ARRAYIDX]], align 4 506; UNROLL-NEXT: [[SUB3:%.*]] = sub nsw i32 [[TMP18]], [[SCALAR_RECUR]] 507; UNROLL-NEXT: [[TMP19:%.*]] = call i32 @llvm.smax.i32(i32 [[SUB3]], i32 0) 508; UNROLL-NEXT: [[TMP20]] = call i32 @llvm.smin.i32(i32 [[MINMAX_028]], i32 [[TMP19]]) 509; UNROLL-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 510; UNROLL-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 511; UNROLL-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 512; UNROLL-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] 513; 514; UNROLL-NO-IC-LABEL: @recurrence_2( 515; UNROLL-NO-IC-NEXT: entry: 516; UNROLL-NO-IC-NEXT: [[CMP27:%.*]] = icmp sgt i32 [[N:%.*]], 0 517; UNROLL-NO-IC-NEXT: br i1 [[CMP27]], label [[FOR_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]] 518; UNROLL-NO-IC: for.preheader: 519; UNROLL-NO-IC-NEXT: [[ARRAYIDX2_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 -1 520; UNROLL-NO-IC-NEXT: [[DOTPRE:%.*]] = load i32, i32* [[ARRAYIDX2_PHI_TRANS_INSERT]], align 4 521; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1 522; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 523; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 524; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 8 525; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 526; UNROLL-NO-IC: vector.ph: 527; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 8 528; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 529; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[DOTPRE]], i32 3 530; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 531; UNROLL-NO-IC: vector.body: 532; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 533; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD2:%.*]], [[VECTOR_BODY]] ] 534; UNROLL-NO-IC-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ poison, [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ] 535; UNROLL-NO-IC-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ poison, [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ] 536; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 0 537; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 4 538; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP3]] 539; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP4]] 540; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[TMP5]], i32 0 541; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = bitcast i32* [[TMP7]] to <4 x i32>* 542; UNROLL-NO-IC-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP8]], align 4 543; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP5]], i32 4 544; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = bitcast i32* [[TMP9]] to <4 x i32>* 545; UNROLL-NO-IC-NEXT: [[WIDE_LOAD2]] = load <4 x i32>, <4 x i32>* [[TMP10]], align 4 546; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 547; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> [[WIDE_LOAD2]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 548; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = sub nsw <4 x i32> [[WIDE_LOAD]], [[TMP11]] 549; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = sub nsw <4 x i32> [[WIDE_LOAD2]], [[TMP12]] 550; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = icmp sgt <4 x i32> [[TMP13]], zeroinitializer 551; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = icmp sgt <4 x i32> [[TMP14]], zeroinitializer 552; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = select <4 x i1> [[TMP15]], <4 x i32> [[TMP13]], <4 x i32> zeroinitializer 553; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = select <4 x i1> [[TMP16]], <4 x i32> [[TMP14]], <4 x i32> zeroinitializer 554; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = icmp slt <4 x i32> [[VEC_PHI]], [[TMP17]] 555; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = icmp slt <4 x i32> [[VEC_PHI1]], [[TMP18]] 556; UNROLL-NO-IC-NEXT: [[TMP21]] = select <4 x i1> [[TMP19]], <4 x i32> [[VEC_PHI]], <4 x i32> [[TMP17]] 557; UNROLL-NO-IC-NEXT: [[TMP22]] = select <4 x i1> [[TMP20]], <4 x i32> [[VEC_PHI1]], <4 x i32> [[TMP18]] 558; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 559; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 560; UNROLL-NO-IC-NEXT: br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] 561; UNROLL-NO-IC: middle.block: 562; UNROLL-NO-IC-NEXT: [[RDX_MINMAX_CMP:%.*]] = icmp slt <4 x i32> [[TMP21]], [[TMP22]] 563; UNROLL-NO-IC-NEXT: [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP21]], <4 x i32> [[TMP22]] 564; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[RDX_MINMAX_SELECT]]) 565; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 566; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i32 3 567; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i32 2 568; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]] 569; UNROLL-NO-IC: scalar.ph: 570; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[DOTPRE]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 571; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_PREHEADER]] ] 572; UNROLL-NO-IC-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ poison, [[FOR_PREHEADER]] ], [ [[TMP24]], [[MIDDLE_BLOCK]] ] 573; UNROLL-NO-IC-NEXT: br label [[SCALAR_BODY:%.*]] 574; UNROLL-NO-IC: for.cond.cleanup.loopexit: 575; UNROLL-NO-IC-NEXT: [[MINMAX_0_COND_LCSSA:%.*]] = phi i32 [ [[MINMAX_0_COND:%.*]], [[SCALAR_BODY]] ], [ [[TMP24]], [[MIDDLE_BLOCK]] ] 576; UNROLL-NO-IC-NEXT: br label [[FOR_COND_CLEANUP]] 577; UNROLL-NO-IC: for.cond.cleanup: 578; UNROLL-NO-IC-NEXT: [[MINMAX_0_LCSSA:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[MINMAX_0_COND_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ] 579; UNROLL-NO-IC-NEXT: ret i32 [[MINMAX_0_LCSSA]] 580; UNROLL-NO-IC: scalar.body: 581; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP25:%.*]], [[SCALAR_BODY]] ] 582; UNROLL-NO-IC-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ] 583; UNROLL-NO-IC-NEXT: [[MINMAX_028:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MINMAX_0_COND]], [[SCALAR_BODY]] ] 584; UNROLL-NO-IC-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]] 585; UNROLL-NO-IC-NEXT: [[TMP25]] = load i32, i32* [[ARRAYIDX]], align 4 586; UNROLL-NO-IC-NEXT: [[SUB3:%.*]] = sub nsw i32 [[TMP25]], [[SCALAR_RECUR]] 587; UNROLL-NO-IC-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[SUB3]], 0 588; UNROLL-NO-IC-NEXT: [[COND:%.*]] = select i1 [[CMP4]], i32 [[SUB3]], i32 0 589; UNROLL-NO-IC-NEXT: [[CMP5:%.*]] = icmp slt i32 [[MINMAX_028]], [[COND]] 590; UNROLL-NO-IC-NEXT: [[MINMAX_0_COND]] = select i1 [[CMP5]], i32 [[MINMAX_028]], i32 [[COND]] 591; UNROLL-NO-IC-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 592; UNROLL-NO-IC-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 593; UNROLL-NO-IC-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 594; UNROLL-NO-IC-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] 595; 596; UNROLL-NO-VF-LABEL: @recurrence_2( 597; UNROLL-NO-VF-NEXT: entry: 598; UNROLL-NO-VF-NEXT: [[CMP27:%.*]] = icmp sgt i32 [[N:%.*]], 0 599; UNROLL-NO-VF-NEXT: br i1 [[CMP27]], label [[FOR_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]] 600; UNROLL-NO-VF: for.preheader: 601; UNROLL-NO-VF-NEXT: [[ARRAYIDX2_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 -1 602; UNROLL-NO-VF-NEXT: [[DOTPRE:%.*]] = load i32, i32* [[ARRAYIDX2_PHI_TRANS_INSERT]], align 4 603; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1 604; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 605; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 606; UNROLL-NO-VF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 2 607; UNROLL-NO-VF-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 608; UNROLL-NO-VF: vector.ph: 609; UNROLL-NO-VF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 2 610; UNROLL-NO-VF-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 611; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 612; UNROLL-NO-VF: vector.body: 613; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 614; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i32 [ [[DOTPRE]], [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ] 615; UNROLL-NO-VF-NEXT: [[VEC_PHI:%.*]] = phi i32 [ poison, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ] 616; UNROLL-NO-VF-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ poison, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ] 617; UNROLL-NO-VF-NEXT: [[INDUCTION:%.*]] = add i64 [[INDEX]], 0 618; UNROLL-NO-VF-NEXT: [[INDUCTION2:%.*]] = add i64 [[INDEX]], 1 619; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDUCTION]] 620; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDUCTION2]] 621; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = load i32, i32* [[TMP3]], align 4 622; UNROLL-NO-VF-NEXT: [[TMP6]] = load i32, i32* [[TMP4]], align 4 623; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = sub nsw i32 [[TMP5]], [[VECTOR_RECUR]] 624; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = sub nsw i32 [[TMP6]], [[TMP5]] 625; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = icmp sgt i32 [[TMP7]], 0 626; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = icmp sgt i32 [[TMP8]], 0 627; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = select i1 [[TMP9]], i32 [[TMP7]], i32 0 628; UNROLL-NO-VF-NEXT: [[TMP12:%.*]] = select i1 [[TMP10]], i32 [[TMP8]], i32 0 629; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = icmp slt i32 [[VEC_PHI]], [[TMP11]] 630; UNROLL-NO-VF-NEXT: [[TMP14:%.*]] = icmp slt i32 [[VEC_PHI1]], [[TMP12]] 631; UNROLL-NO-VF-NEXT: [[TMP15]] = select i1 [[TMP13]], i32 [[VEC_PHI]], i32 [[TMP11]] 632; UNROLL-NO-VF-NEXT: [[TMP16]] = select i1 [[TMP14]], i32 [[VEC_PHI1]], i32 [[TMP12]] 633; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 634; UNROLL-NO-VF-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 635; UNROLL-NO-VF-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] 636; UNROLL-NO-VF: middle.block: 637; UNROLL-NO-VF-NEXT: [[RDX_MINMAX_CMP:%.*]] = icmp slt i32 [[TMP15]], [[TMP16]] 638; UNROLL-NO-VF-NEXT: [[RDX_MINMAX_SELECT:%.*]] = select i1 [[RDX_MINMAX_CMP]], i32 [[TMP15]], i32 [[TMP16]] 639; UNROLL-NO-VF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 640; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]] 641; UNROLL-NO-VF: scalar.ph: 642; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[DOTPRE]], [[FOR_PREHEADER]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ] 643; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_PREHEADER]] ] 644; UNROLL-NO-VF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ poison, [[FOR_PREHEADER]] ], [ [[RDX_MINMAX_SELECT]], [[MIDDLE_BLOCK]] ] 645; UNROLL-NO-VF-NEXT: br label [[SCALAR_BODY:%.*]] 646; UNROLL-NO-VF: for.cond.cleanup.loopexit: 647; UNROLL-NO-VF-NEXT: [[MINMAX_0_COND_LCSSA:%.*]] = phi i32 [ [[MINMAX_0_COND:%.*]], [[SCALAR_BODY]] ], [ [[RDX_MINMAX_SELECT]], [[MIDDLE_BLOCK]] ] 648; UNROLL-NO-VF-NEXT: br label [[FOR_COND_CLEANUP]] 649; UNROLL-NO-VF: for.cond.cleanup: 650; UNROLL-NO-VF-NEXT: [[MINMAX_0_LCSSA:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[MINMAX_0_COND_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ] 651; UNROLL-NO-VF-NEXT: ret i32 [[MINMAX_0_LCSSA]] 652; UNROLL-NO-VF: scalar.body: 653; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP18:%.*]], [[SCALAR_BODY]] ] 654; UNROLL-NO-VF-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ] 655; UNROLL-NO-VF-NEXT: [[MINMAX_028:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MINMAX_0_COND]], [[SCALAR_BODY]] ] 656; UNROLL-NO-VF-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]] 657; UNROLL-NO-VF-NEXT: [[TMP18]] = load i32, i32* [[ARRAYIDX]], align 4 658; UNROLL-NO-VF-NEXT: [[SUB3:%.*]] = sub nsw i32 [[TMP18]], [[SCALAR_RECUR]] 659; UNROLL-NO-VF-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[SUB3]], 0 660; UNROLL-NO-VF-NEXT: [[COND:%.*]] = select i1 [[CMP4]], i32 [[SUB3]], i32 0 661; UNROLL-NO-VF-NEXT: [[CMP5:%.*]] = icmp slt i32 [[MINMAX_028]], [[COND]] 662; UNROLL-NO-VF-NEXT: [[MINMAX_0_COND]] = select i1 [[CMP5]], i32 [[MINMAX_028]], i32 [[COND]] 663; UNROLL-NO-VF-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 664; UNROLL-NO-VF-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 665; UNROLL-NO-VF-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 666; UNROLL-NO-VF-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] 667; 668; SINK-AFTER-LABEL: @recurrence_2( 669; SINK-AFTER-NEXT: entry: 670; SINK-AFTER-NEXT: [[CMP27:%.*]] = icmp sgt i32 [[N:%.*]], 0 671; SINK-AFTER-NEXT: br i1 [[CMP27]], label [[FOR_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]] 672; SINK-AFTER: for.preheader: 673; SINK-AFTER-NEXT: [[ARRAYIDX2_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 -1 674; SINK-AFTER-NEXT: [[DOTPRE:%.*]] = load i32, i32* [[ARRAYIDX2_PHI_TRANS_INSERT]], align 4 675; SINK-AFTER-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1 676; SINK-AFTER-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 677; SINK-AFTER-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 678; SINK-AFTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4 679; SINK-AFTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 680; SINK-AFTER: vector.ph: 681; SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4 682; SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 683; SINK-AFTER-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[DOTPRE]], i32 3 684; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 685; SINK-AFTER: vector.body: 686; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 687; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ] 688; SINK-AFTER-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ poison, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ] 689; SINK-AFTER-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 0 690; SINK-AFTER-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP3]] 691; SINK-AFTER-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 0 692; SINK-AFTER-NEXT: [[TMP6:%.*]] = bitcast i32* [[TMP5]] to <4 x i32>* 693; SINK-AFTER-NEXT: [[WIDE_LOAD]] = load <4 x i32>, <4 x i32>* [[TMP6]], align 4 694; SINK-AFTER-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 695; SINK-AFTER-NEXT: [[TMP8:%.*]] = sub nsw <4 x i32> [[WIDE_LOAD]], [[TMP7]] 696; SINK-AFTER-NEXT: [[TMP9:%.*]] = icmp sgt <4 x i32> [[TMP8]], zeroinitializer 697; SINK-AFTER-NEXT: [[TMP10:%.*]] = select <4 x i1> [[TMP9]], <4 x i32> [[TMP8]], <4 x i32> zeroinitializer 698; SINK-AFTER-NEXT: [[TMP11:%.*]] = icmp slt <4 x i32> [[VEC_PHI]], [[TMP10]] 699; SINK-AFTER-NEXT: [[TMP12]] = select <4 x i1> [[TMP11]], <4 x i32> [[VEC_PHI]], <4 x i32> [[TMP10]] 700; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 701; SINK-AFTER-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 702; SINK-AFTER-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] 703; SINK-AFTER: middle.block: 704; SINK-AFTER-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[TMP12]]) 705; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 706; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i32 3 707; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i32 2 708; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]] 709; SINK-AFTER: scalar.ph: 710; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[DOTPRE]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 711; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_PREHEADER]] ] 712; SINK-AFTER-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ poison, [[FOR_PREHEADER]] ], [ [[TMP14]], [[MIDDLE_BLOCK]] ] 713; SINK-AFTER-NEXT: br label [[SCALAR_BODY:%.*]] 714; SINK-AFTER: for.cond.cleanup.loopexit: 715; SINK-AFTER-NEXT: [[MINMAX_0_COND_LCSSA:%.*]] = phi i32 [ [[MINMAX_0_COND:%.*]], [[SCALAR_BODY]] ], [ [[TMP14]], [[MIDDLE_BLOCK]] ] 716; SINK-AFTER-NEXT: br label [[FOR_COND_CLEANUP]] 717; SINK-AFTER: for.cond.cleanup: 718; SINK-AFTER-NEXT: [[MINMAX_0_LCSSA:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[MINMAX_0_COND_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ] 719; SINK-AFTER-NEXT: ret i32 [[MINMAX_0_LCSSA]] 720; SINK-AFTER: scalar.body: 721; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP15:%.*]], [[SCALAR_BODY]] ] 722; SINK-AFTER-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ] 723; SINK-AFTER-NEXT: [[MINMAX_028:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MINMAX_0_COND]], [[SCALAR_BODY]] ] 724; SINK-AFTER-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]] 725; SINK-AFTER-NEXT: [[TMP15]] = load i32, i32* [[ARRAYIDX]], align 4 726; SINK-AFTER-NEXT: [[SUB3:%.*]] = sub nsw i32 [[TMP15]], [[SCALAR_RECUR]] 727; SINK-AFTER-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[SUB3]], 0 728; SINK-AFTER-NEXT: [[COND:%.*]] = select i1 [[CMP4]], i32 [[SUB3]], i32 0 729; SINK-AFTER-NEXT: [[CMP5:%.*]] = icmp slt i32 [[MINMAX_028]], [[COND]] 730; SINK-AFTER-NEXT: [[MINMAX_0_COND]] = select i1 [[CMP5]], i32 [[MINMAX_028]], i32 [[COND]] 731; SINK-AFTER-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 732; SINK-AFTER-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 733; SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 734; SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] 735; 736entry: 737 %cmp27 = icmp sgt i32 %n, 0 738 br i1 %cmp27, label %for.preheader, label %for.cond.cleanup 739 740for.preheader: 741 %arrayidx2.phi.trans.insert = getelementptr inbounds i32, i32* %a, i64 -1 742 %.pre = load i32, i32* %arrayidx2.phi.trans.insert, align 4 743 br label %scalar.body 744 745for.cond.cleanup.loopexit: 746 %minmax.0.cond.lcssa = phi i32 [ %minmax.0.cond, %scalar.body ] 747 br label %for.cond.cleanup 748 749for.cond.cleanup: 750 %minmax.0.lcssa = phi i32 [ poison, %entry ], [ %minmax.0.cond.lcssa, %for.cond.cleanup.loopexit ] 751 ret i32 %minmax.0.lcssa 752 753scalar.body: 754 %0 = phi i32 [ %.pre, %for.preheader ], [ %1, %scalar.body ] 755 %indvars.iv = phi i64 [ 0, %for.preheader ], [ %indvars.iv.next, %scalar.body ] 756 %minmax.028 = phi i32 [ poison, %for.preheader ], [ %minmax.0.cond, %scalar.body ] 757 %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv 758 %1 = load i32, i32* %arrayidx, align 4 759 %sub3 = sub nsw i32 %1, %0 760 %cmp4 = icmp sgt i32 %sub3, 0 761 %cond = select i1 %cmp4, i32 %sub3, i32 0 762 %cmp5 = icmp slt i32 %minmax.028, %cond 763 %minmax.0.cond = select i1 %cmp5, i32 %minmax.028, i32 %cond 764 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 765 %lftr.wideiv = trunc i64 %indvars.iv.next to i32 766 %exitcond = icmp eq i32 %lftr.wideiv, %n 767 br i1 %exitcond, label %for.cond.cleanup.loopexit, label %scalar.body 768} 769 770; void recurrence_3(short *a, double *b, int n, float f, short p) { 771; b[0] = (double)a[0] - f * (double)p; 772; for (int i = 1; i < n; i++) 773; b[i] = (double)a[i] - f * (double)a[i - 1]; 774; } 775; 776; Check also that the casts were not moved needlessly. 777; 778; 779define void @recurrence_3(i16* nocapture readonly %a, double* nocapture %b, i32 %n, float %f, i16 %p) { 780; CHECK-LABEL: @recurrence_3( 781; CHECK-NEXT: entry: 782; CHECK-NEXT: [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2 783; CHECK-NEXT: [[CONV:%.*]] = sitofp i16 [[TMP0]] to double 784; CHECK-NEXT: [[CONV1:%.*]] = fpext float [[F:%.*]] to double 785; CHECK-NEXT: [[CONV2:%.*]] = sitofp i16 [[P:%.*]] to double 786; CHECK-NEXT: [[MUL:%.*]] = fmul fast double [[CONV2]], [[CONV1]] 787; CHECK-NEXT: [[SUB:%.*]] = fsub fast double [[CONV]], [[MUL]] 788; CHECK-NEXT: store double [[SUB]], double* [[B:%.*]], align 8 789; CHECK-NEXT: [[CMP25:%.*]] = icmp sgt i32 [[N:%.*]], 1 790; CHECK-NEXT: br i1 [[CMP25]], label [[FOR_PREHEADER:%.*]], label [[FOR_END:%.*]] 791; CHECK: for.preheader: 792; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[N]], -2 793; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64 794; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1 795; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], 3 796; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 797; CHECK: vector.memcheck: 798; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr double, double* [[B]], i64 1 799; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[N]], -2 800; CHECK-NEXT: [[TMP5:%.*]] = zext i32 [[TMP4]] to i64 801; CHECK-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[TMP5]], 2 802; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr double, double* [[B]], i64 [[TMP6]] 803; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i16, i16* [[A]], i64 1 804; CHECK-NEXT: [[SCEVGEP6:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP6]] 805; CHECK-NEXT: [[TMP7:%.*]] = bitcast i16* [[SCEVGEP6]] to double* 806; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult double* [[SCEVGEP]], [[TMP7]] 807; CHECK-NEXT: [[TMP8:%.*]] = bitcast double* [[SCEVGEP2]] to i16* 808; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult i16* [[SCEVGEP4]], [[TMP8]] 809; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 810; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 811; CHECK: vector.ph: 812; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP3]], 8589934588 813; CHECK-NEXT: [[IND_END:%.*]] = or i64 [[N_VEC]], 1 814; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[TMP0]], i64 3 815; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[CONV1]], i64 0 816; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer 817; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 818; CHECK: vector.body: 819; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 820; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ] 821; CHECK-NEXT: [[OFFSET_IDX:%.*]] = or i64 [[INDEX]], 1 822; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[OFFSET_IDX]] 823; CHECK-NEXT: [[TMP10:%.*]] = bitcast i16* [[TMP9]] to <4 x i16>* 824; CHECK-NEXT: [[WIDE_LOAD]] = load <4 x i16>, <4 x i16>* [[TMP10]], align 2 825; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 826; CHECK-NEXT: [[TMP12:%.*]] = sitofp <4 x i16> [[WIDE_LOAD]] to <4 x double> 827; CHECK-NEXT: [[TMP13:%.*]] = sitofp <4 x i16> [[TMP11]] to <4 x double> 828; CHECK-NEXT: [[TMP14:%.*]] = fmul fast <4 x double> [[BROADCAST_SPLAT]], [[TMP13]] 829; CHECK-NEXT: [[TMP15:%.*]] = fsub fast <4 x double> [[TMP12]], [[TMP14]] 830; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[OFFSET_IDX]] 831; CHECK-NEXT: [[TMP17:%.*]] = bitcast double* [[TMP16]] to <4 x double>* 832; CHECK-NEXT: store <4 x double> [[TMP15]], <4 x double>* [[TMP17]], align 8 833; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 834; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 835; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] 836; CHECK: middle.block: 837; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]] 838; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 3 839; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] 840; CHECK: scalar.ph: 841; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[TMP0]], [[VECTOR_MEMCHECK]] ], [ [[TMP0]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 842; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1, [[VECTOR_MEMCHECK]] ], [ 1, [[FOR_PREHEADER]] ], [ [[IND_END]], [[MIDDLE_BLOCK]] ] 843; CHECK-NEXT: br label [[SCALAR_BODY:%.*]] 844; CHECK: scalar.body: 845; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP19:%.*]], [[SCALAR_BODY]] ] 846; CHECK-NEXT: [[ADVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ] 847; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[ADVARS_IV]] 848; CHECK-NEXT: [[TMP19]] = load i16, i16* [[ARRAYIDX5]], align 2 849; CHECK-NEXT: [[CONV6:%.*]] = sitofp i16 [[TMP19]] to double 850; CHECK-NEXT: [[CONV11:%.*]] = sitofp i16 [[SCALAR_RECUR]] to double 851; CHECK-NEXT: [[MUL12:%.*]] = fmul fast double [[CONV11]], [[CONV1]] 852; CHECK-NEXT: [[SUB13:%.*]] = fsub fast double [[CONV6]], [[MUL12]] 853; CHECK-NEXT: [[ARRAYIDX15:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[ADVARS_IV]] 854; CHECK-NEXT: store double [[SUB13]], double* [[ARRAYIDX15]], align 8 855; CHECK-NEXT: [[ADVARS_IV_NEXT]] = add nuw nsw i64 [[ADVARS_IV]], 1 856; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[ADVARS_IV_NEXT]] to i32 857; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 858; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP17:![0-9]+]] 859; CHECK: for.end.loopexit: 860; CHECK-NEXT: br label [[FOR_END]] 861; CHECK: for.end: 862; CHECK-NEXT: ret void 863; 864; UNROLL-LABEL: @recurrence_3( 865; UNROLL-NEXT: entry: 866; UNROLL-NEXT: [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2 867; UNROLL-NEXT: [[CONV:%.*]] = sitofp i16 [[TMP0]] to double 868; UNROLL-NEXT: [[CONV1:%.*]] = fpext float [[F:%.*]] to double 869; UNROLL-NEXT: [[CONV2:%.*]] = sitofp i16 [[P:%.*]] to double 870; UNROLL-NEXT: [[MUL:%.*]] = fmul fast double [[CONV2]], [[CONV1]] 871; UNROLL-NEXT: [[SUB:%.*]] = fsub fast double [[CONV]], [[MUL]] 872; UNROLL-NEXT: store double [[SUB]], double* [[B:%.*]], align 8 873; UNROLL-NEXT: [[CMP25:%.*]] = icmp sgt i32 [[N:%.*]], 1 874; UNROLL-NEXT: br i1 [[CMP25]], label [[FOR_PREHEADER:%.*]], label [[FOR_END:%.*]] 875; UNROLL: for.preheader: 876; UNROLL-NEXT: [[TMP1:%.*]] = add i32 [[N]], -2 877; UNROLL-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64 878; UNROLL-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1 879; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], 7 880; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 881; UNROLL: vector.memcheck: 882; UNROLL-NEXT: [[SCEVGEP:%.*]] = getelementptr double, double* [[B]], i64 1 883; UNROLL-NEXT: [[TMP4:%.*]] = add i32 [[N]], -2 884; UNROLL-NEXT: [[TMP5:%.*]] = zext i32 [[TMP4]] to i64 885; UNROLL-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[TMP5]], 2 886; UNROLL-NEXT: [[SCEVGEP2:%.*]] = getelementptr double, double* [[B]], i64 [[TMP6]] 887; UNROLL-NEXT: [[SCEVGEP4:%.*]] = getelementptr i16, i16* [[A]], i64 1 888; UNROLL-NEXT: [[SCEVGEP6:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP6]] 889; UNROLL-NEXT: [[TMP7:%.*]] = bitcast i16* [[SCEVGEP6]] to double* 890; UNROLL-NEXT: [[BOUND0:%.*]] = icmp ult double* [[SCEVGEP]], [[TMP7]] 891; UNROLL-NEXT: [[TMP8:%.*]] = bitcast double* [[SCEVGEP2]] to i16* 892; UNROLL-NEXT: [[BOUND1:%.*]] = icmp ult i16* [[SCEVGEP4]], [[TMP8]] 893; UNROLL-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 894; UNROLL-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 895; UNROLL: vector.ph: 896; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[TMP3]], 8589934584 897; UNROLL-NEXT: [[IND_END:%.*]] = or i64 [[N_VEC]], 1 898; UNROLL-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[TMP0]], i64 3 899; UNROLL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[CONV1]], i64 0 900; UNROLL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer 901; UNROLL-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <4 x double> poison, double [[CONV1]], i64 0 902; UNROLL-NEXT: [[BROADCAST_SPLAT10:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT9]], <4 x double> poison, <4 x i32> zeroinitializer 903; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 904; UNROLL: vector.body: 905; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 906; UNROLL-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD8:%.*]], [[VECTOR_BODY]] ] 907; UNROLL-NEXT: [[OFFSET_IDX:%.*]] = or i64 [[INDEX]], 1 908; UNROLL-NEXT: [[TMP9:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[OFFSET_IDX]] 909; UNROLL-NEXT: [[TMP10:%.*]] = bitcast i16* [[TMP9]] to <4 x i16>* 910; UNROLL-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, <4 x i16>* [[TMP10]], align 2 911; UNROLL-NEXT: [[TMP11:%.*]] = getelementptr inbounds i16, i16* [[TMP9]], i64 4 912; UNROLL-NEXT: [[TMP12:%.*]] = bitcast i16* [[TMP11]] to <4 x i16>* 913; UNROLL-NEXT: [[WIDE_LOAD8]] = load <4 x i16>, <4 x i16>* [[TMP12]], align 2 914; UNROLL-NEXT: [[TMP13:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 915; UNROLL-NEXT: [[TMP14:%.*]] = shufflevector <4 x i16> [[WIDE_LOAD]], <4 x i16> [[WIDE_LOAD8]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 916; UNROLL-NEXT: [[TMP15:%.*]] = sitofp <4 x i16> [[WIDE_LOAD]] to <4 x double> 917; UNROLL-NEXT: [[TMP16:%.*]] = sitofp <4 x i16> [[WIDE_LOAD8]] to <4 x double> 918; UNROLL-NEXT: [[TMP17:%.*]] = sitofp <4 x i16> [[TMP13]] to <4 x double> 919; UNROLL-NEXT: [[TMP18:%.*]] = sitofp <4 x i16> [[TMP14]] to <4 x double> 920; UNROLL-NEXT: [[TMP19:%.*]] = fmul fast <4 x double> [[BROADCAST_SPLAT]], [[TMP17]] 921; UNROLL-NEXT: [[TMP20:%.*]] = fmul fast <4 x double> [[BROADCAST_SPLAT10]], [[TMP18]] 922; UNROLL-NEXT: [[TMP21:%.*]] = fsub fast <4 x double> [[TMP15]], [[TMP19]] 923; UNROLL-NEXT: [[TMP22:%.*]] = fsub fast <4 x double> [[TMP16]], [[TMP20]] 924; UNROLL-NEXT: [[TMP23:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[OFFSET_IDX]] 925; UNROLL-NEXT: [[TMP24:%.*]] = bitcast double* [[TMP23]] to <4 x double>* 926; UNROLL-NEXT: store <4 x double> [[TMP21]], <4 x double>* [[TMP24]], align 8 927; UNROLL-NEXT: [[TMP25:%.*]] = getelementptr inbounds double, double* [[TMP23]], i64 4 928; UNROLL-NEXT: [[TMP26:%.*]] = bitcast double* [[TMP25]] to <4 x double>* 929; UNROLL-NEXT: store <4 x double> [[TMP22]], <4 x double>* [[TMP26]], align 8 930; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 931; UNROLL-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 932; UNROLL-NEXT: br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] 933; UNROLL: middle.block: 934; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]] 935; UNROLL-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD8]], i64 3 936; UNROLL-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] 937; UNROLL: scalar.ph: 938; UNROLL-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[TMP0]], [[VECTOR_MEMCHECK]] ], [ [[TMP0]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 939; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1, [[VECTOR_MEMCHECK]] ], [ 1, [[FOR_PREHEADER]] ], [ [[IND_END]], [[MIDDLE_BLOCK]] ] 940; UNROLL-NEXT: br label [[SCALAR_BODY:%.*]] 941; UNROLL: scalar.body: 942; UNROLL-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP28:%.*]], [[SCALAR_BODY]] ] 943; UNROLL-NEXT: [[ADVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ] 944; UNROLL-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[ADVARS_IV]] 945; UNROLL-NEXT: [[TMP28]] = load i16, i16* [[ARRAYIDX5]], align 2 946; UNROLL-NEXT: [[CONV6:%.*]] = sitofp i16 [[TMP28]] to double 947; UNROLL-NEXT: [[CONV11:%.*]] = sitofp i16 [[SCALAR_RECUR]] to double 948; UNROLL-NEXT: [[MUL12:%.*]] = fmul fast double [[CONV11]], [[CONV1]] 949; UNROLL-NEXT: [[SUB13:%.*]] = fsub fast double [[CONV6]], [[MUL12]] 950; UNROLL-NEXT: [[ARRAYIDX15:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[ADVARS_IV]] 951; UNROLL-NEXT: store double [[SUB13]], double* [[ARRAYIDX15]], align 8 952; UNROLL-NEXT: [[ADVARS_IV_NEXT]] = add nuw nsw i64 [[ADVARS_IV]], 1 953; UNROLL-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[ADVARS_IV_NEXT]] to i32 954; UNROLL-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 955; UNROLL-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP17:![0-9]+]] 956; UNROLL: for.end.loopexit: 957; UNROLL-NEXT: br label [[FOR_END]] 958; UNROLL: for.end: 959; UNROLL-NEXT: ret void 960; 961; UNROLL-NO-IC-LABEL: @recurrence_3( 962; UNROLL-NO-IC-NEXT: entry: 963; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2 964; UNROLL-NO-IC-NEXT: [[CONV:%.*]] = sitofp i16 [[TMP0]] to double 965; UNROLL-NO-IC-NEXT: [[CONV1:%.*]] = fpext float [[F:%.*]] to double 966; UNROLL-NO-IC-NEXT: [[CONV2:%.*]] = sitofp i16 [[P:%.*]] to double 967; UNROLL-NO-IC-NEXT: [[MUL:%.*]] = fmul fast double [[CONV2]], [[CONV1]] 968; UNROLL-NO-IC-NEXT: [[SUB:%.*]] = fsub fast double [[CONV]], [[MUL]] 969; UNROLL-NO-IC-NEXT: store double [[SUB]], double* [[B:%.*]], align 8 970; UNROLL-NO-IC-NEXT: [[CMP25:%.*]] = icmp sgt i32 [[N:%.*]], 1 971; UNROLL-NO-IC-NEXT: br i1 [[CMP25]], label [[FOR_PREHEADER:%.*]], label [[FOR_END:%.*]] 972; UNROLL-NO-IC: for.preheader: 973; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = add i32 [[N]], -2 974; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64 975; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1 976; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 8 977; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 978; UNROLL-NO-IC: vector.memcheck: 979; UNROLL-NO-IC-NEXT: [[SCEVGEP:%.*]] = getelementptr double, double* [[B]], i64 1 980; UNROLL-NO-IC-NEXT: [[SCEVGEP1:%.*]] = bitcast double* [[SCEVGEP]] to i8* 981; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = add i32 [[N]], -2 982; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = zext i32 [[TMP4]] to i64 983; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[TMP5]], 2 984; UNROLL-NO-IC-NEXT: [[SCEVGEP2:%.*]] = getelementptr double, double* [[B]], i64 [[TMP6]] 985; UNROLL-NO-IC-NEXT: [[SCEVGEP23:%.*]] = bitcast double* [[SCEVGEP2]] to i8* 986; UNROLL-NO-IC-NEXT: [[SCEVGEP4:%.*]] = getelementptr i16, i16* [[A]], i64 1 987; UNROLL-NO-IC-NEXT: [[SCEVGEP45:%.*]] = bitcast i16* [[SCEVGEP4]] to i8* 988; UNROLL-NO-IC-NEXT: [[SCEVGEP6:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP6]] 989; UNROLL-NO-IC-NEXT: [[SCEVGEP67:%.*]] = bitcast i16* [[SCEVGEP6]] to i8* 990; UNROLL-NO-IC-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[SCEVGEP1]], [[SCEVGEP67]] 991; UNROLL-NO-IC-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP45]], [[SCEVGEP23]] 992; UNROLL-NO-IC-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 993; UNROLL-NO-IC-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 994; UNROLL-NO-IC: vector.ph: 995; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 8 996; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]] 997; UNROLL-NO-IC-NEXT: [[IND_END:%.*]] = add i64 1, [[N_VEC]] 998; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[TMP0]], i32 3 999; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[CONV1]], i32 0 1000; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer 1001; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <4 x double> poison, double [[CONV1]], i32 0 1002; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLAT10:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT9]], <4 x double> poison, <4 x i32> zeroinitializer 1003; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 1004; UNROLL-NO-IC: vector.body: 1005; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1006; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD8:%.*]], [[VECTOR_BODY]] ] 1007; UNROLL-NO-IC-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]] 1008; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 0 1009; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = add i64 [[OFFSET_IDX]], 4 1010; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP7]] 1011; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP8]] 1012; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = getelementptr inbounds i16, i16* [[TMP9]], i32 0 1013; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = bitcast i16* [[TMP11]] to <4 x i16>* 1014; UNROLL-NO-IC-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, <4 x i16>* [[TMP12]], align 2 1015; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = getelementptr inbounds i16, i16* [[TMP9]], i32 4 1016; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = bitcast i16* [[TMP13]] to <4 x i16>* 1017; UNROLL-NO-IC-NEXT: [[WIDE_LOAD8]] = load <4 x i16>, <4 x i16>* [[TMP14]], align 2 1018; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 1019; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = shufflevector <4 x i16> [[WIDE_LOAD]], <4 x i16> [[WIDE_LOAD8]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 1020; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = sitofp <4 x i16> [[WIDE_LOAD]] to <4 x double> 1021; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = sitofp <4 x i16> [[WIDE_LOAD8]] to <4 x double> 1022; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = sitofp <4 x i16> [[TMP15]] to <4 x double> 1023; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = sitofp <4 x i16> [[TMP16]] to <4 x double> 1024; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = fmul fast <4 x double> [[TMP19]], [[BROADCAST_SPLAT]] 1025; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = fmul fast <4 x double> [[TMP20]], [[BROADCAST_SPLAT10]] 1026; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = fsub fast <4 x double> [[TMP17]], [[TMP21]] 1027; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = fsub fast <4 x double> [[TMP18]], [[TMP22]] 1028; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[TMP7]] 1029; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[TMP8]] 1030; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = getelementptr inbounds double, double* [[TMP25]], i32 0 1031; UNROLL-NO-IC-NEXT: [[TMP28:%.*]] = bitcast double* [[TMP27]] to <4 x double>* 1032; UNROLL-NO-IC-NEXT: store <4 x double> [[TMP23]], <4 x double>* [[TMP28]], align 8 1033; UNROLL-NO-IC-NEXT: [[TMP29:%.*]] = getelementptr inbounds double, double* [[TMP25]], i32 4 1034; UNROLL-NO-IC-NEXT: [[TMP30:%.*]] = bitcast double* [[TMP29]] to <4 x double>* 1035; UNROLL-NO-IC-NEXT: store <4 x double> [[TMP24]], <4 x double>* [[TMP30]], align 8 1036; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 1037; UNROLL-NO-IC-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1038; UNROLL-NO-IC-NEXT: br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] 1039; UNROLL-NO-IC: middle.block: 1040; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]] 1041; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD8]], i32 3 1042; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD8]], i32 2 1043; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] 1044; UNROLL-NO-IC: scalar.ph: 1045; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[TMP0]], [[VECTOR_MEMCHECK]] ], [ [[TMP0]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 1046; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[FOR_PREHEADER]] ], [ 1, [[VECTOR_MEMCHECK]] ] 1047; UNROLL-NO-IC-NEXT: br label [[SCALAR_BODY:%.*]] 1048; UNROLL-NO-IC: scalar.body: 1049; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP32:%.*]], [[SCALAR_BODY]] ] 1050; UNROLL-NO-IC-NEXT: [[ADVARS_IV:%.*]] = phi i64 [ [[ADVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 1051; UNROLL-NO-IC-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[ADVARS_IV]] 1052; UNROLL-NO-IC-NEXT: [[TMP32]] = load i16, i16* [[ARRAYIDX5]], align 2 1053; UNROLL-NO-IC-NEXT: [[CONV6:%.*]] = sitofp i16 [[TMP32]] to double 1054; UNROLL-NO-IC-NEXT: [[CONV11:%.*]] = sitofp i16 [[SCALAR_RECUR]] to double 1055; UNROLL-NO-IC-NEXT: [[MUL12:%.*]] = fmul fast double [[CONV11]], [[CONV1]] 1056; UNROLL-NO-IC-NEXT: [[SUB13:%.*]] = fsub fast double [[CONV6]], [[MUL12]] 1057; UNROLL-NO-IC-NEXT: [[ARRAYIDX15:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[ADVARS_IV]] 1058; UNROLL-NO-IC-NEXT: store double [[SUB13]], double* [[ARRAYIDX15]], align 8 1059; UNROLL-NO-IC-NEXT: [[ADVARS_IV_NEXT]] = add nuw nsw i64 [[ADVARS_IV]], 1 1060; UNROLL-NO-IC-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[ADVARS_IV_NEXT]] to i32 1061; UNROLL-NO-IC-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 1062; UNROLL-NO-IC-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP17:![0-9]+]] 1063; UNROLL-NO-IC: for.end.loopexit: 1064; UNROLL-NO-IC-NEXT: br label [[FOR_END]] 1065; UNROLL-NO-IC: for.end: 1066; UNROLL-NO-IC-NEXT: ret void 1067; 1068; UNROLL-NO-VF-LABEL: @recurrence_3( 1069; UNROLL-NO-VF-NEXT: entry: 1070; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2 1071; UNROLL-NO-VF-NEXT: [[CONV:%.*]] = sitofp i16 [[TMP0]] to double 1072; UNROLL-NO-VF-NEXT: [[CONV1:%.*]] = fpext float [[F:%.*]] to double 1073; UNROLL-NO-VF-NEXT: [[CONV2:%.*]] = sitofp i16 [[P:%.*]] to double 1074; UNROLL-NO-VF-NEXT: [[MUL:%.*]] = fmul fast double [[CONV2]], [[CONV1]] 1075; UNROLL-NO-VF-NEXT: [[SUB:%.*]] = fsub fast double [[CONV]], [[MUL]] 1076; UNROLL-NO-VF-NEXT: store double [[SUB]], double* [[B:%.*]], align 8 1077; UNROLL-NO-VF-NEXT: [[CMP25:%.*]] = icmp sgt i32 [[N:%.*]], 1 1078; UNROLL-NO-VF-NEXT: br i1 [[CMP25]], label [[FOR_PREHEADER:%.*]], label [[FOR_END:%.*]] 1079; UNROLL-NO-VF: for.preheader: 1080; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = add i32 [[N]], -2 1081; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64 1082; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1 1083; UNROLL-NO-VF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 2 1084; UNROLL-NO-VF-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 1085; UNROLL-NO-VF: vector.memcheck: 1086; UNROLL-NO-VF-NEXT: [[SCEVGEP:%.*]] = getelementptr double, double* [[B]], i64 1 1087; UNROLL-NO-VF-NEXT: [[SCEVGEP1:%.*]] = bitcast double* [[SCEVGEP]] to i8* 1088; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = add i32 [[N]], -2 1089; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = zext i32 [[TMP4]] to i64 1090; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[TMP5]], 2 1091; UNROLL-NO-VF-NEXT: [[SCEVGEP2:%.*]] = getelementptr double, double* [[B]], i64 [[TMP6]] 1092; UNROLL-NO-VF-NEXT: [[SCEVGEP23:%.*]] = bitcast double* [[SCEVGEP2]] to i8* 1093; UNROLL-NO-VF-NEXT: [[SCEVGEP4:%.*]] = getelementptr i16, i16* [[A]], i64 1 1094; UNROLL-NO-VF-NEXT: [[SCEVGEP45:%.*]] = bitcast i16* [[SCEVGEP4]] to i8* 1095; UNROLL-NO-VF-NEXT: [[SCEVGEP6:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP6]] 1096; UNROLL-NO-VF-NEXT: [[SCEVGEP67:%.*]] = bitcast i16* [[SCEVGEP6]] to i8* 1097; UNROLL-NO-VF-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[SCEVGEP1]], [[SCEVGEP67]] 1098; UNROLL-NO-VF-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP45]], [[SCEVGEP23]] 1099; UNROLL-NO-VF-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 1100; UNROLL-NO-VF-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 1101; UNROLL-NO-VF: vector.ph: 1102; UNROLL-NO-VF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 2 1103; UNROLL-NO-VF-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]] 1104; UNROLL-NO-VF-NEXT: [[IND_END:%.*]] = add i64 1, [[N_VEC]] 1105; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 1106; UNROLL-NO-VF: vector.body: 1107; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1108; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i16 [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ] 1109; UNROLL-NO-VF-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]] 1110; UNROLL-NO-VF-NEXT: [[INDUCTION:%.*]] = add i64 [[OFFSET_IDX]], 0 1111; UNROLL-NO-VF-NEXT: [[INDUCTION8:%.*]] = add i64 [[OFFSET_IDX]], 1 1112; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[INDUCTION]] 1113; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[INDUCTION8]] 1114; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = load i16, i16* [[TMP7]], align 2 1115; UNROLL-NO-VF-NEXT: [[TMP10]] = load i16, i16* [[TMP8]], align 2 1116; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = sitofp i16 [[TMP9]] to double 1117; UNROLL-NO-VF-NEXT: [[TMP12:%.*]] = sitofp i16 [[TMP10]] to double 1118; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = sitofp i16 [[VECTOR_RECUR]] to double 1119; UNROLL-NO-VF-NEXT: [[TMP14:%.*]] = sitofp i16 [[TMP9]] to double 1120; UNROLL-NO-VF-NEXT: [[TMP15:%.*]] = fmul fast double [[TMP13]], [[CONV1]] 1121; UNROLL-NO-VF-NEXT: [[TMP16:%.*]] = fmul fast double [[TMP14]], [[CONV1]] 1122; UNROLL-NO-VF-NEXT: [[TMP17:%.*]] = fsub fast double [[TMP11]], [[TMP15]] 1123; UNROLL-NO-VF-NEXT: [[TMP18:%.*]] = fsub fast double [[TMP12]], [[TMP16]] 1124; UNROLL-NO-VF-NEXT: [[TMP19:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[INDUCTION]] 1125; UNROLL-NO-VF-NEXT: [[TMP20:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[INDUCTION8]] 1126; UNROLL-NO-VF-NEXT: store double [[TMP17]], double* [[TMP19]], align 8 1127; UNROLL-NO-VF-NEXT: store double [[TMP18]], double* [[TMP20]], align 8 1128; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 1129; UNROLL-NO-VF-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1130; UNROLL-NO-VF-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]] 1131; UNROLL-NO-VF: middle.block: 1132; UNROLL-NO-VF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]] 1133; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] 1134; UNROLL-NO-VF: scalar.ph: 1135; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[TMP0]], [[VECTOR_MEMCHECK]] ], [ [[TMP0]], [[FOR_PREHEADER]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ] 1136; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[FOR_PREHEADER]] ], [ 1, [[VECTOR_MEMCHECK]] ] 1137; UNROLL-NO-VF-NEXT: br label [[SCALAR_BODY:%.*]] 1138; UNROLL-NO-VF: scalar.body: 1139; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP22:%.*]], [[SCALAR_BODY]] ] 1140; UNROLL-NO-VF-NEXT: [[ADVARS_IV:%.*]] = phi i64 [ [[ADVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 1141; UNROLL-NO-VF-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[ADVARS_IV]] 1142; UNROLL-NO-VF-NEXT: [[TMP22]] = load i16, i16* [[ARRAYIDX5]], align 2 1143; UNROLL-NO-VF-NEXT: [[CONV6:%.*]] = sitofp i16 [[TMP22]] to double 1144; UNROLL-NO-VF-NEXT: [[CONV11:%.*]] = sitofp i16 [[SCALAR_RECUR]] to double 1145; UNROLL-NO-VF-NEXT: [[MUL12:%.*]] = fmul fast double [[CONV11]], [[CONV1]] 1146; UNROLL-NO-VF-NEXT: [[SUB13:%.*]] = fsub fast double [[CONV6]], [[MUL12]] 1147; UNROLL-NO-VF-NEXT: [[ARRAYIDX15:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[ADVARS_IV]] 1148; UNROLL-NO-VF-NEXT: store double [[SUB13]], double* [[ARRAYIDX15]], align 8 1149; UNROLL-NO-VF-NEXT: [[ADVARS_IV_NEXT]] = add nuw nsw i64 [[ADVARS_IV]], 1 1150; UNROLL-NO-VF-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[ADVARS_IV_NEXT]] to i32 1151; UNROLL-NO-VF-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 1152; UNROLL-NO-VF-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] 1153; UNROLL-NO-VF: for.end.loopexit: 1154; UNROLL-NO-VF-NEXT: br label [[FOR_END]] 1155; UNROLL-NO-VF: for.end: 1156; UNROLL-NO-VF-NEXT: ret void 1157; 1158; SINK-AFTER-LABEL: @recurrence_3( 1159; SINK-AFTER-NEXT: entry: 1160; SINK-AFTER-NEXT: [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2 1161; SINK-AFTER-NEXT: [[CONV:%.*]] = sitofp i16 [[TMP0]] to double 1162; SINK-AFTER-NEXT: [[CONV1:%.*]] = fpext float [[F:%.*]] to double 1163; SINK-AFTER-NEXT: [[CONV2:%.*]] = sitofp i16 [[P:%.*]] to double 1164; SINK-AFTER-NEXT: [[MUL:%.*]] = fmul fast double [[CONV2]], [[CONV1]] 1165; SINK-AFTER-NEXT: [[SUB:%.*]] = fsub fast double [[CONV]], [[MUL]] 1166; SINK-AFTER-NEXT: store double [[SUB]], double* [[B:%.*]], align 8 1167; SINK-AFTER-NEXT: [[CMP25:%.*]] = icmp sgt i32 [[N:%.*]], 1 1168; SINK-AFTER-NEXT: br i1 [[CMP25]], label [[FOR_PREHEADER:%.*]], label [[FOR_END:%.*]] 1169; SINK-AFTER: for.preheader: 1170; SINK-AFTER-NEXT: [[TMP1:%.*]] = add i32 [[N]], -2 1171; SINK-AFTER-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64 1172; SINK-AFTER-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1 1173; SINK-AFTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4 1174; SINK-AFTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 1175; SINK-AFTER: vector.memcheck: 1176; SINK-AFTER-NEXT: [[SCEVGEP:%.*]] = getelementptr double, double* [[B]], i64 1 1177; SINK-AFTER-NEXT: [[SCEVGEP1:%.*]] = bitcast double* [[SCEVGEP]] to i8* 1178; SINK-AFTER-NEXT: [[TMP4:%.*]] = add i32 [[N]], -2 1179; SINK-AFTER-NEXT: [[TMP5:%.*]] = zext i32 [[TMP4]] to i64 1180; SINK-AFTER-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[TMP5]], 2 1181; SINK-AFTER-NEXT: [[SCEVGEP2:%.*]] = getelementptr double, double* [[B]], i64 [[TMP6]] 1182; SINK-AFTER-NEXT: [[SCEVGEP23:%.*]] = bitcast double* [[SCEVGEP2]] to i8* 1183; SINK-AFTER-NEXT: [[SCEVGEP4:%.*]] = getelementptr i16, i16* [[A]], i64 1 1184; SINK-AFTER-NEXT: [[SCEVGEP45:%.*]] = bitcast i16* [[SCEVGEP4]] to i8* 1185; SINK-AFTER-NEXT: [[SCEVGEP6:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP6]] 1186; SINK-AFTER-NEXT: [[SCEVGEP67:%.*]] = bitcast i16* [[SCEVGEP6]] to i8* 1187; SINK-AFTER-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[SCEVGEP1]], [[SCEVGEP67]] 1188; SINK-AFTER-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP45]], [[SCEVGEP23]] 1189; SINK-AFTER-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 1190; SINK-AFTER-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 1191; SINK-AFTER: vector.ph: 1192; SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4 1193; SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]] 1194; SINK-AFTER-NEXT: [[IND_END:%.*]] = add i64 1, [[N_VEC]] 1195; SINK-AFTER-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[TMP0]], i32 3 1196; SINK-AFTER-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[CONV1]], i32 0 1197; SINK-AFTER-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer 1198; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 1199; SINK-AFTER: vector.body: 1200; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1201; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ] 1202; SINK-AFTER-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]] 1203; SINK-AFTER-NEXT: [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 0 1204; SINK-AFTER-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP7]] 1205; SINK-AFTER-NEXT: [[TMP9:%.*]] = getelementptr inbounds i16, i16* [[TMP8]], i32 0 1206; SINK-AFTER-NEXT: [[TMP10:%.*]] = bitcast i16* [[TMP9]] to <4 x i16>* 1207; SINK-AFTER-NEXT: [[WIDE_LOAD]] = load <4 x i16>, <4 x i16>* [[TMP10]], align 2 1208; SINK-AFTER-NEXT: [[TMP11:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 1209; SINK-AFTER-NEXT: [[TMP12:%.*]] = sitofp <4 x i16> [[WIDE_LOAD]] to <4 x double> 1210; SINK-AFTER-NEXT: [[TMP13:%.*]] = sitofp <4 x i16> [[TMP11]] to <4 x double> 1211; SINK-AFTER-NEXT: [[TMP14:%.*]] = fmul fast <4 x double> [[TMP13]], [[BROADCAST_SPLAT]] 1212; SINK-AFTER-NEXT: [[TMP15:%.*]] = fsub fast <4 x double> [[TMP12]], [[TMP14]] 1213; SINK-AFTER-NEXT: [[TMP16:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[TMP7]] 1214; SINK-AFTER-NEXT: [[TMP17:%.*]] = getelementptr inbounds double, double* [[TMP16]], i32 0 1215; SINK-AFTER-NEXT: [[TMP18:%.*]] = bitcast double* [[TMP17]] to <4 x double>* 1216; SINK-AFTER-NEXT: store <4 x double> [[TMP15]], <4 x double>* [[TMP18]], align 8 1217; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 1218; SINK-AFTER-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1219; SINK-AFTER-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] 1220; SINK-AFTER: middle.block: 1221; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]] 1222; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3 1223; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2 1224; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] 1225; SINK-AFTER: scalar.ph: 1226; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[TMP0]], [[VECTOR_MEMCHECK]] ], [ [[TMP0]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 1227; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[FOR_PREHEADER]] ], [ 1, [[VECTOR_MEMCHECK]] ] 1228; SINK-AFTER-NEXT: br label [[SCALAR_BODY:%.*]] 1229; SINK-AFTER: scalar.body: 1230; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP20:%.*]], [[SCALAR_BODY]] ] 1231; SINK-AFTER-NEXT: [[ADVARS_IV:%.*]] = phi i64 [ [[ADVARS_IV_NEXT:%.*]], [[SCALAR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 1232; SINK-AFTER-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[ADVARS_IV]] 1233; SINK-AFTER-NEXT: [[TMP20]] = load i16, i16* [[ARRAYIDX5]], align 2 1234; SINK-AFTER-NEXT: [[CONV6:%.*]] = sitofp i16 [[TMP20]] to double 1235; SINK-AFTER-NEXT: [[CONV11:%.*]] = sitofp i16 [[SCALAR_RECUR]] to double 1236; SINK-AFTER-NEXT: [[MUL12:%.*]] = fmul fast double [[CONV11]], [[CONV1]] 1237; SINK-AFTER-NEXT: [[SUB13:%.*]] = fsub fast double [[CONV6]], [[MUL12]] 1238; SINK-AFTER-NEXT: [[ARRAYIDX15:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[ADVARS_IV]] 1239; SINK-AFTER-NEXT: store double [[SUB13]], double* [[ARRAYIDX15]], align 8 1240; SINK-AFTER-NEXT: [[ADVARS_IV_NEXT]] = add nuw nsw i64 [[ADVARS_IV]], 1 1241; SINK-AFTER-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[ADVARS_IV_NEXT]] to i32 1242; SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 1243; SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[SCALAR_BODY]], !llvm.loop [[LOOP17:![0-9]+]] 1244; SINK-AFTER: for.end.loopexit: 1245; SINK-AFTER-NEXT: br label [[FOR_END]] 1246; SINK-AFTER: for.end: 1247; SINK-AFTER-NEXT: ret void 1248; 1249entry: 1250 %0 = load i16, i16* %a, align 2 1251 %conv = sitofp i16 %0 to double 1252 %conv1 = fpext float %f to double 1253 %conv2 = sitofp i16 %p to double 1254 %mul = fmul fast double %conv2, %conv1 1255 %sub = fsub fast double %conv, %mul 1256 store double %sub, double* %b, align 8 1257 %cmp25 = icmp sgt i32 %n, 1 1258 br i1 %cmp25, label %for.preheader, label %for.end 1259 1260for.preheader: 1261 br label %scalar.body 1262 1263scalar.body: 1264 %1 = phi i16 [ %0, %for.preheader ], [ %2, %scalar.body ] 1265 %advars.iv = phi i64 [ %advars.iv.next, %scalar.body ], [ 1, %for.preheader ] 1266 %arrayidx5 = getelementptr inbounds i16, i16* %a, i64 %advars.iv 1267 %2 = load i16, i16* %arrayidx5, align 2 1268 %conv6 = sitofp i16 %2 to double 1269 %conv11 = sitofp i16 %1 to double 1270 %mul12 = fmul fast double %conv11, %conv1 1271 %sub13 = fsub fast double %conv6, %mul12 1272 %arrayidx15 = getelementptr inbounds double, double* %b, i64 %advars.iv 1273 store double %sub13, double* %arrayidx15, align 8 1274 %advars.iv.next = add nuw nsw i64 %advars.iv, 1 1275 %lftr.wideiv = trunc i64 %advars.iv.next to i32 1276 %exitcond = icmp eq i32 %lftr.wideiv, %n 1277 br i1 %exitcond, label %for.end.loopexit, label %scalar.body 1278 1279for.end.loopexit: 1280 br label %for.end 1281 1282for.end: 1283 ret void 1284} 1285 1286; void PR26734(short *a, int *b, int *c, int d, short *e) { 1287; for (; d != 21; d++) { 1288; *b &= *c; 1289; *e = *a - 6; 1290; *c = *e; 1291; } 1292; } 1293; 1294; 1295define void @PR26734(i16* %a, i32* %b, i32* %c, i32 %d, i16* %e) { 1296; CHECK-LABEL: @PR26734( 1297; CHECK-NEXT: entry: 1298; CHECK-NEXT: [[CMP4:%.*]] = icmp eq i32 [[D:%.*]], 21 1299; CHECK-NEXT: br i1 [[CMP4]], label [[ENTRY_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY_LR_PH:%.*]] 1300; CHECK: entry.for.end_crit_edge: 1301; CHECK-NEXT: br label [[FOR_END:%.*]] 1302; CHECK: for.body.lr.ph: 1303; CHECK-NEXT: [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2 1304; CHECK-NEXT: [[SUB:%.*]] = add i16 [[TMP0]], -6 1305; CHECK-NEXT: [[CONV2:%.*]] = sext i16 [[SUB]] to i32 1306; CHECK-NEXT: [[C_PROMOTED:%.*]] = load i32, i32* [[C:%.*]], align 4 1307; CHECK-NEXT: [[B_PROMOTED:%.*]] = load i32, i32* [[B:%.*]], align 4 1308; CHECK-NEXT: br label [[FOR_BODY:%.*]] 1309; CHECK: for.body: 1310; CHECK-NEXT: [[INC7:%.*]] = phi i32 [ [[D]], [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ] 1311; CHECK-NEXT: [[AND6:%.*]] = phi i32 [ [[B_PROMOTED]], [[FOR_BODY_LR_PH]] ], [ [[AND:%.*]], [[FOR_BODY]] ] 1312; CHECK-NEXT: [[CONV25:%.*]] = phi i32 [ [[C_PROMOTED]], [[FOR_BODY_LR_PH]] ], [ [[CONV2]], [[FOR_BODY]] ] 1313; CHECK-NEXT: [[AND]] = and i32 [[AND6]], [[CONV25]] 1314; CHECK-NEXT: [[INC]] = add nsw i32 [[INC7]], 1 1315; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], 21 1316; CHECK-NEXT: br i1 [[CMP]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY]] 1317; CHECK: for.cond.for.end_crit_edge: 1318; CHECK-NEXT: store i32 [[CONV2]], i32* [[C]], align 4 1319; CHECK-NEXT: store i32 [[AND]], i32* [[B]], align 4 1320; CHECK-NEXT: store i16 [[SUB]], i16* [[E:%.*]], align 2 1321; CHECK-NEXT: br label [[FOR_END]] 1322; CHECK: for.end: 1323; CHECK-NEXT: ret void 1324; 1325; UNROLL-LABEL: @PR26734( 1326; UNROLL-NEXT: entry: 1327; UNROLL-NEXT: [[CMP4:%.*]] = icmp eq i32 [[D:%.*]], 21 1328; UNROLL-NEXT: br i1 [[CMP4]], label [[ENTRY_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY_LR_PH:%.*]] 1329; UNROLL: entry.for.end_crit_edge: 1330; UNROLL-NEXT: br label [[FOR_END:%.*]] 1331; UNROLL: for.body.lr.ph: 1332; UNROLL-NEXT: [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2 1333; UNROLL-NEXT: [[SUB:%.*]] = add i16 [[TMP0]], -6 1334; UNROLL-NEXT: [[CONV2:%.*]] = sext i16 [[SUB]] to i32 1335; UNROLL-NEXT: [[C_PROMOTED:%.*]] = load i32, i32* [[C:%.*]], align 4 1336; UNROLL-NEXT: [[B_PROMOTED:%.*]] = load i32, i32* [[B:%.*]], align 4 1337; UNROLL-NEXT: br label [[FOR_BODY:%.*]] 1338; UNROLL: for.body: 1339; UNROLL-NEXT: [[INC7:%.*]] = phi i32 [ [[D]], [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ] 1340; UNROLL-NEXT: [[AND6:%.*]] = phi i32 [ [[B_PROMOTED]], [[FOR_BODY_LR_PH]] ], [ [[AND:%.*]], [[FOR_BODY]] ] 1341; UNROLL-NEXT: [[CONV25:%.*]] = phi i32 [ [[C_PROMOTED]], [[FOR_BODY_LR_PH]] ], [ [[CONV2]], [[FOR_BODY]] ] 1342; UNROLL-NEXT: [[AND]] = and i32 [[AND6]], [[CONV25]] 1343; UNROLL-NEXT: [[INC]] = add nsw i32 [[INC7]], 1 1344; UNROLL-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], 21 1345; UNROLL-NEXT: br i1 [[CMP]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY]] 1346; UNROLL: for.cond.for.end_crit_edge: 1347; UNROLL-NEXT: store i32 [[CONV2]], i32* [[C]], align 4 1348; UNROLL-NEXT: store i32 [[AND]], i32* [[B]], align 4 1349; UNROLL-NEXT: store i16 [[SUB]], i16* [[E:%.*]], align 2 1350; UNROLL-NEXT: br label [[FOR_END]] 1351; UNROLL: for.end: 1352; UNROLL-NEXT: ret void 1353; 1354; UNROLL-NO-IC-LABEL: @PR26734( 1355; UNROLL-NO-IC-NEXT: entry: 1356; UNROLL-NO-IC-NEXT: [[CMP4:%.*]] = icmp eq i32 [[D:%.*]], 21 1357; UNROLL-NO-IC-NEXT: br i1 [[CMP4]], label [[ENTRY_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY_LR_PH:%.*]] 1358; UNROLL-NO-IC: entry.for.end_crit_edge: 1359; UNROLL-NO-IC-NEXT: [[DOTPRE:%.*]] = load i32, i32* [[B:%.*]], align 4 1360; UNROLL-NO-IC-NEXT: br label [[FOR_END:%.*]] 1361; UNROLL-NO-IC: for.body.lr.ph: 1362; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2 1363; UNROLL-NO-IC-NEXT: [[SUB:%.*]] = add i16 [[TMP0]], -6 1364; UNROLL-NO-IC-NEXT: [[CONV2:%.*]] = sext i16 [[SUB]] to i32 1365; UNROLL-NO-IC-NEXT: [[C_PROMOTED:%.*]] = load i32, i32* [[C:%.*]], align 4 1366; UNROLL-NO-IC-NEXT: [[B_PROMOTED:%.*]] = load i32, i32* [[B]], align 4 1367; UNROLL-NO-IC-NEXT: br label [[FOR_BODY:%.*]] 1368; UNROLL-NO-IC: for.body: 1369; UNROLL-NO-IC-NEXT: [[INC7:%.*]] = phi i32 [ [[D]], [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ] 1370; UNROLL-NO-IC-NEXT: [[AND6:%.*]] = phi i32 [ [[B_PROMOTED]], [[FOR_BODY_LR_PH]] ], [ [[AND:%.*]], [[FOR_BODY]] ] 1371; UNROLL-NO-IC-NEXT: [[CONV25:%.*]] = phi i32 [ [[C_PROMOTED]], [[FOR_BODY_LR_PH]] ], [ [[CONV2]], [[FOR_BODY]] ] 1372; UNROLL-NO-IC-NEXT: [[AND]] = and i32 [[AND6]], [[CONV25]] 1373; UNROLL-NO-IC-NEXT: [[INC]] = add nsw i32 [[INC7]], 1 1374; UNROLL-NO-IC-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], 21 1375; UNROLL-NO-IC-NEXT: br i1 [[CMP]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY]] 1376; UNROLL-NO-IC: for.cond.for.end_crit_edge: 1377; UNROLL-NO-IC-NEXT: [[AND_LCSSA:%.*]] = phi i32 [ [[AND]], [[FOR_BODY]] ] 1378; UNROLL-NO-IC-NEXT: store i32 [[CONV2]], i32* [[C]], align 4 1379; UNROLL-NO-IC-NEXT: store i32 [[AND_LCSSA]], i32* [[B]], align 4 1380; UNROLL-NO-IC-NEXT: store i16 [[SUB]], i16* [[E:%.*]], align 2 1381; UNROLL-NO-IC-NEXT: br label [[FOR_END]] 1382; UNROLL-NO-IC: for.end: 1383; UNROLL-NO-IC-NEXT: ret void 1384; 1385; UNROLL-NO-VF-LABEL: @PR26734( 1386; UNROLL-NO-VF-NEXT: entry: 1387; UNROLL-NO-VF-NEXT: [[CMP4:%.*]] = icmp eq i32 [[D:%.*]], 21 1388; UNROLL-NO-VF-NEXT: br i1 [[CMP4]], label [[ENTRY_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY_LR_PH:%.*]] 1389; UNROLL-NO-VF: entry.for.end_crit_edge: 1390; UNROLL-NO-VF-NEXT: [[DOTPRE:%.*]] = load i32, i32* [[B:%.*]], align 4 1391; UNROLL-NO-VF-NEXT: br label [[FOR_END:%.*]] 1392; UNROLL-NO-VF: for.body.lr.ph: 1393; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2 1394; UNROLL-NO-VF-NEXT: [[SUB:%.*]] = add i16 [[TMP0]], -6 1395; UNROLL-NO-VF-NEXT: [[CONV2:%.*]] = sext i16 [[SUB]] to i32 1396; UNROLL-NO-VF-NEXT: [[C_PROMOTED:%.*]] = load i32, i32* [[C:%.*]], align 4 1397; UNROLL-NO-VF-NEXT: [[B_PROMOTED:%.*]] = load i32, i32* [[B]], align 4 1398; UNROLL-NO-VF-NEXT: br label [[FOR_BODY:%.*]] 1399; UNROLL-NO-VF: for.body: 1400; UNROLL-NO-VF-NEXT: [[INC7:%.*]] = phi i32 [ [[D]], [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ] 1401; UNROLL-NO-VF-NEXT: [[AND6:%.*]] = phi i32 [ [[B_PROMOTED]], [[FOR_BODY_LR_PH]] ], [ [[AND:%.*]], [[FOR_BODY]] ] 1402; UNROLL-NO-VF-NEXT: [[CONV25:%.*]] = phi i32 [ [[C_PROMOTED]], [[FOR_BODY_LR_PH]] ], [ [[CONV2]], [[FOR_BODY]] ] 1403; UNROLL-NO-VF-NEXT: [[AND]] = and i32 [[AND6]], [[CONV25]] 1404; UNROLL-NO-VF-NEXT: [[INC]] = add nsw i32 [[INC7]], 1 1405; UNROLL-NO-VF-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], 21 1406; UNROLL-NO-VF-NEXT: br i1 [[CMP]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY]] 1407; UNROLL-NO-VF: for.cond.for.end_crit_edge: 1408; UNROLL-NO-VF-NEXT: [[AND_LCSSA:%.*]] = phi i32 [ [[AND]], [[FOR_BODY]] ] 1409; UNROLL-NO-VF-NEXT: store i32 [[CONV2]], i32* [[C]], align 4 1410; UNROLL-NO-VF-NEXT: store i32 [[AND_LCSSA]], i32* [[B]], align 4 1411; UNROLL-NO-VF-NEXT: store i16 [[SUB]], i16* [[E:%.*]], align 2 1412; UNROLL-NO-VF-NEXT: br label [[FOR_END]] 1413; UNROLL-NO-VF: for.end: 1414; UNROLL-NO-VF-NEXT: ret void 1415; 1416; SINK-AFTER-LABEL: @PR26734( 1417; SINK-AFTER-NEXT: entry: 1418; SINK-AFTER-NEXT: [[CMP4:%.*]] = icmp eq i32 [[D:%.*]], 21 1419; SINK-AFTER-NEXT: br i1 [[CMP4]], label [[ENTRY_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY_LR_PH:%.*]] 1420; SINK-AFTER: entry.for.end_crit_edge: 1421; SINK-AFTER-NEXT: [[DOTPRE:%.*]] = load i32, i32* [[B:%.*]], align 4 1422; SINK-AFTER-NEXT: br label [[FOR_END:%.*]] 1423; SINK-AFTER: for.body.lr.ph: 1424; SINK-AFTER-NEXT: [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2 1425; SINK-AFTER-NEXT: [[SUB:%.*]] = add i16 [[TMP0]], -6 1426; SINK-AFTER-NEXT: [[CONV2:%.*]] = sext i16 [[SUB]] to i32 1427; SINK-AFTER-NEXT: [[C_PROMOTED:%.*]] = load i32, i32* [[C:%.*]], align 4 1428; SINK-AFTER-NEXT: [[B_PROMOTED:%.*]] = load i32, i32* [[B]], align 4 1429; SINK-AFTER-NEXT: br label [[FOR_BODY:%.*]] 1430; SINK-AFTER: for.body: 1431; SINK-AFTER-NEXT: [[INC7:%.*]] = phi i32 [ [[D]], [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ] 1432; SINK-AFTER-NEXT: [[AND6:%.*]] = phi i32 [ [[B_PROMOTED]], [[FOR_BODY_LR_PH]] ], [ [[AND:%.*]], [[FOR_BODY]] ] 1433; SINK-AFTER-NEXT: [[CONV25:%.*]] = phi i32 [ [[C_PROMOTED]], [[FOR_BODY_LR_PH]] ], [ [[CONV2]], [[FOR_BODY]] ] 1434; SINK-AFTER-NEXT: [[AND]] = and i32 [[AND6]], [[CONV25]] 1435; SINK-AFTER-NEXT: [[INC]] = add nsw i32 [[INC7]], 1 1436; SINK-AFTER-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], 21 1437; SINK-AFTER-NEXT: br i1 [[CMP]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY]] 1438; SINK-AFTER: for.cond.for.end_crit_edge: 1439; SINK-AFTER-NEXT: [[AND_LCSSA:%.*]] = phi i32 [ [[AND]], [[FOR_BODY]] ] 1440; SINK-AFTER-NEXT: store i32 [[CONV2]], i32* [[C]], align 4 1441; SINK-AFTER-NEXT: store i32 [[AND_LCSSA]], i32* [[B]], align 4 1442; SINK-AFTER-NEXT: store i16 [[SUB]], i16* [[E:%.*]], align 2 1443; SINK-AFTER-NEXT: br label [[FOR_END]] 1444; SINK-AFTER: for.end: 1445; SINK-AFTER-NEXT: ret void 1446; 1447entry: 1448 %cmp4 = icmp eq i32 %d, 21 1449 br i1 %cmp4, label %entry.for.end_crit_edge, label %for.body.lr.ph 1450 1451entry.for.end_crit_edge: 1452 %.pre = load i32, i32* %b, align 4 1453 br label %for.end 1454 1455for.body.lr.ph: 1456 %0 = load i16, i16* %a, align 2 1457 %sub = add i16 %0, -6 1458 %conv2 = sext i16 %sub to i32 1459 %c.promoted = load i32, i32* %c, align 4 1460 %b.promoted = load i32, i32* %b, align 4 1461 br label %for.body 1462 1463for.body: 1464 %inc7 = phi i32 [ %d, %for.body.lr.ph ], [ %inc, %for.body ] 1465 %and6 = phi i32 [ %b.promoted, %for.body.lr.ph ], [ %and, %for.body ] 1466 %conv25 = phi i32 [ %c.promoted, %for.body.lr.ph ], [ %conv2, %for.body ] 1467 %and = and i32 %and6, %conv25 1468 %inc = add nsw i32 %inc7, 1 1469 %cmp = icmp eq i32 %inc, 21 1470 br i1 %cmp, label %for.cond.for.end_crit_edge, label %for.body 1471 1472for.cond.for.end_crit_edge: 1473 %and.lcssa = phi i32 [ %and, %for.body ] 1474 store i32 %conv2, i32* %c, align 4 1475 store i32 %and.lcssa, i32* %b, align 4 1476 store i16 %sub, i16* %e, align 2 1477 br label %for.end 1478 1479for.end: 1480 ret void 1481} 1482 1483; int PR27246() { 1484; unsigned int e, n; 1485; for (int i = 1; i < 49; ++i) { 1486; for (int k = i; k > 1; --k) 1487; e = k; 1488; n = e; 1489; } 1490; return n; 1491; } 1492; 1493; 1494define i32 @PR27246() { 1495; CHECK-LABEL: @PR27246( 1496; CHECK-NEXT: entry: 1497; CHECK-NEXT: br label [[FOR_COND1_PREHEADER:%.*]] 1498; CHECK: for.cond1.preheader: 1499; CHECK-NEXT: [[I_016:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_COND_CLEANUP3:%.*]] ] 1500; CHECK-NEXT: [[E_015:%.*]] = phi i32 [ poison, [[ENTRY]] ], [ [[E_1:%.*]], [[FOR_COND_CLEANUP3]] ] 1501; CHECK-NEXT: br label [[FOR_COND1:%.*]] 1502; CHECK: for.cond.cleanup: 1503; CHECK-NEXT: ret i32 [[E_1]] 1504; CHECK: for.cond1: 1505; CHECK-NEXT: [[E_1]] = phi i32 [ [[K_0:%.*]], [[FOR_COND1]] ], [ [[E_015]], [[FOR_COND1_PREHEADER]] ] 1506; CHECK-NEXT: [[K_0]] = phi i32 [ [[DEC:%.*]], [[FOR_COND1]] ], [ [[I_016]], [[FOR_COND1_PREHEADER]] ] 1507; CHECK-NEXT: [[CMP2:%.*]] = icmp sgt i32 [[K_0]], 1 1508; CHECK-NEXT: [[DEC]] = add nsw i32 [[K_0]], -1 1509; CHECK-NEXT: br i1 [[CMP2]], label [[FOR_COND1]], label [[FOR_COND_CLEANUP3]] 1510; CHECK: for.cond.cleanup3: 1511; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_016]], 1 1512; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 49 1513; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER]] 1514; 1515; UNROLL-LABEL: @PR27246( 1516; UNROLL-NEXT: entry: 1517; UNROLL-NEXT: br label [[FOR_COND1_PREHEADER:%.*]] 1518; UNROLL: for.cond1.preheader: 1519; UNROLL-NEXT: [[I_016:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_COND_CLEANUP3:%.*]] ] 1520; UNROLL-NEXT: [[E_015:%.*]] = phi i32 [ poison, [[ENTRY]] ], [ [[E_1:%.*]], [[FOR_COND_CLEANUP3]] ] 1521; UNROLL-NEXT: br label [[FOR_COND1:%.*]] 1522; UNROLL: for.cond.cleanup: 1523; UNROLL-NEXT: ret i32 [[E_1]] 1524; UNROLL: for.cond1: 1525; UNROLL-NEXT: [[E_1]] = phi i32 [ [[K_0:%.*]], [[FOR_COND1]] ], [ [[E_015]], [[FOR_COND1_PREHEADER]] ] 1526; UNROLL-NEXT: [[K_0]] = phi i32 [ [[DEC:%.*]], [[FOR_COND1]] ], [ [[I_016]], [[FOR_COND1_PREHEADER]] ] 1527; UNROLL-NEXT: [[CMP2:%.*]] = icmp sgt i32 [[K_0]], 1 1528; UNROLL-NEXT: [[DEC]] = add nsw i32 [[K_0]], -1 1529; UNROLL-NEXT: br i1 [[CMP2]], label [[FOR_COND1]], label [[FOR_COND_CLEANUP3]] 1530; UNROLL: for.cond.cleanup3: 1531; UNROLL-NEXT: [[INC]] = add nuw nsw i32 [[I_016]], 1 1532; UNROLL-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 49 1533; UNROLL-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER]] 1534; 1535; UNROLL-NO-IC-LABEL: @PR27246( 1536; UNROLL-NO-IC-NEXT: entry: 1537; UNROLL-NO-IC-NEXT: br label [[FOR_COND1_PREHEADER:%.*]] 1538; UNROLL-NO-IC: for.cond1.preheader: 1539; UNROLL-NO-IC-NEXT: [[I_016:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_COND_CLEANUP3:%.*]] ] 1540; UNROLL-NO-IC-NEXT: [[E_015:%.*]] = phi i32 [ poison, [[ENTRY]] ], [ [[E_1_LCSSA:%.*]], [[FOR_COND_CLEANUP3]] ] 1541; UNROLL-NO-IC-NEXT: br label [[FOR_COND1:%.*]] 1542; UNROLL-NO-IC: for.cond.cleanup: 1543; UNROLL-NO-IC-NEXT: [[E_1_LCSSA_LCSSA:%.*]] = phi i32 [ [[E_1_LCSSA]], [[FOR_COND_CLEANUP3]] ] 1544; UNROLL-NO-IC-NEXT: ret i32 [[E_1_LCSSA_LCSSA]] 1545; UNROLL-NO-IC: for.cond1: 1546; UNROLL-NO-IC-NEXT: [[E_1:%.*]] = phi i32 [ [[K_0:%.*]], [[FOR_COND1]] ], [ [[E_015]], [[FOR_COND1_PREHEADER]] ] 1547; UNROLL-NO-IC-NEXT: [[K_0]] = phi i32 [ [[DEC:%.*]], [[FOR_COND1]] ], [ [[I_016]], [[FOR_COND1_PREHEADER]] ] 1548; UNROLL-NO-IC-NEXT: [[CMP2:%.*]] = icmp sgt i32 [[K_0]], 1 1549; UNROLL-NO-IC-NEXT: [[DEC]] = add nsw i32 [[K_0]], -1 1550; UNROLL-NO-IC-NEXT: br i1 [[CMP2]], label [[FOR_COND1]], label [[FOR_COND_CLEANUP3]] 1551; UNROLL-NO-IC: for.cond.cleanup3: 1552; UNROLL-NO-IC-NEXT: [[E_1_LCSSA]] = phi i32 [ [[E_1]], [[FOR_COND1]] ] 1553; UNROLL-NO-IC-NEXT: [[INC]] = add nuw nsw i32 [[I_016]], 1 1554; UNROLL-NO-IC-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 49 1555; UNROLL-NO-IC-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER]] 1556; 1557; UNROLL-NO-VF-LABEL: @PR27246( 1558; UNROLL-NO-VF-NEXT: entry: 1559; UNROLL-NO-VF-NEXT: br label [[FOR_COND1_PREHEADER:%.*]] 1560; UNROLL-NO-VF: for.cond1.preheader: 1561; UNROLL-NO-VF-NEXT: [[I_016:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_COND_CLEANUP3:%.*]] ] 1562; UNROLL-NO-VF-NEXT: [[E_015:%.*]] = phi i32 [ poison, [[ENTRY]] ], [ [[E_1_LCSSA:%.*]], [[FOR_COND_CLEANUP3]] ] 1563; UNROLL-NO-VF-NEXT: br label [[FOR_COND1:%.*]] 1564; UNROLL-NO-VF: for.cond.cleanup: 1565; UNROLL-NO-VF-NEXT: [[E_1_LCSSA_LCSSA:%.*]] = phi i32 [ [[E_1_LCSSA]], [[FOR_COND_CLEANUP3]] ] 1566; UNROLL-NO-VF-NEXT: ret i32 [[E_1_LCSSA_LCSSA]] 1567; UNROLL-NO-VF: for.cond1: 1568; UNROLL-NO-VF-NEXT: [[E_1:%.*]] = phi i32 [ [[K_0:%.*]], [[FOR_COND1]] ], [ [[E_015]], [[FOR_COND1_PREHEADER]] ] 1569; UNROLL-NO-VF-NEXT: [[K_0]] = phi i32 [ [[DEC:%.*]], [[FOR_COND1]] ], [ [[I_016]], [[FOR_COND1_PREHEADER]] ] 1570; UNROLL-NO-VF-NEXT: [[CMP2:%.*]] = icmp sgt i32 [[K_0]], 1 1571; UNROLL-NO-VF-NEXT: [[DEC]] = add nsw i32 [[K_0]], -1 1572; UNROLL-NO-VF-NEXT: br i1 [[CMP2]], label [[FOR_COND1]], label [[FOR_COND_CLEANUP3]] 1573; UNROLL-NO-VF: for.cond.cleanup3: 1574; UNROLL-NO-VF-NEXT: [[E_1_LCSSA]] = phi i32 [ [[E_1]], [[FOR_COND1]] ] 1575; UNROLL-NO-VF-NEXT: [[INC]] = add nuw nsw i32 [[I_016]], 1 1576; UNROLL-NO-VF-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 49 1577; UNROLL-NO-VF-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER]] 1578; 1579; SINK-AFTER-LABEL: @PR27246( 1580; SINK-AFTER-NEXT: entry: 1581; SINK-AFTER-NEXT: br label [[FOR_COND1_PREHEADER:%.*]] 1582; SINK-AFTER: for.cond1.preheader: 1583; SINK-AFTER-NEXT: [[I_016:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_COND_CLEANUP3:%.*]] ] 1584; SINK-AFTER-NEXT: [[E_015:%.*]] = phi i32 [ poison, [[ENTRY]] ], [ [[E_1_LCSSA:%.*]], [[FOR_COND_CLEANUP3]] ] 1585; SINK-AFTER-NEXT: br label [[FOR_COND1:%.*]] 1586; SINK-AFTER: for.cond.cleanup: 1587; SINK-AFTER-NEXT: [[E_1_LCSSA_LCSSA:%.*]] = phi i32 [ [[E_1_LCSSA]], [[FOR_COND_CLEANUP3]] ] 1588; SINK-AFTER-NEXT: ret i32 [[E_1_LCSSA_LCSSA]] 1589; SINK-AFTER: for.cond1: 1590; SINK-AFTER-NEXT: [[E_1:%.*]] = phi i32 [ [[K_0:%.*]], [[FOR_COND1]] ], [ [[E_015]], [[FOR_COND1_PREHEADER]] ] 1591; SINK-AFTER-NEXT: [[K_0]] = phi i32 [ [[DEC:%.*]], [[FOR_COND1]] ], [ [[I_016]], [[FOR_COND1_PREHEADER]] ] 1592; SINK-AFTER-NEXT: [[CMP2:%.*]] = icmp sgt i32 [[K_0]], 1 1593; SINK-AFTER-NEXT: [[DEC]] = add nsw i32 [[K_0]], -1 1594; SINK-AFTER-NEXT: br i1 [[CMP2]], label [[FOR_COND1]], label [[FOR_COND_CLEANUP3]] 1595; SINK-AFTER: for.cond.cleanup3: 1596; SINK-AFTER-NEXT: [[E_1_LCSSA]] = phi i32 [ [[E_1]], [[FOR_COND1]] ] 1597; SINK-AFTER-NEXT: [[INC]] = add nuw nsw i32 [[I_016]], 1 1598; SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 49 1599; SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER]] 1600; 1601entry: 1602 br label %for.cond1.preheader 1603 1604for.cond1.preheader: 1605 %i.016 = phi i32 [ 1, %entry ], [ %inc, %for.cond.cleanup3 ] 1606 %e.015 = phi i32 [ poison, %entry ], [ %e.1.lcssa, %for.cond.cleanup3 ] 1607 br label %for.cond1 1608 1609for.cond.cleanup: 1610 %e.1.lcssa.lcssa = phi i32 [ %e.1.lcssa, %for.cond.cleanup3 ] 1611 ret i32 %e.1.lcssa.lcssa 1612 1613for.cond1: 1614 %e.1 = phi i32 [ %k.0, %for.cond1 ], [ %e.015, %for.cond1.preheader ] 1615 %k.0 = phi i32 [ %dec, %for.cond1 ], [ %i.016, %for.cond1.preheader ] 1616 %cmp2 = icmp sgt i32 %k.0, 1 1617 %dec = add nsw i32 %k.0, -1 1618 br i1 %cmp2, label %for.cond1, label %for.cond.cleanup3 1619 1620for.cond.cleanup3: 1621 %e.1.lcssa = phi i32 [ %e.1, %for.cond1 ] 1622 %inc = add nuw nsw i32 %i.016, 1 1623 %exitcond = icmp eq i32 %inc, 49 1624 br i1 %exitcond, label %for.cond.cleanup, label %for.cond1.preheader 1625} 1626 1627; 1628define void @PR30183(i32 %pre_load, i32* %a, i32* %b, i64 %n) { 1629; CHECK-LABEL: @PR30183( 1630; CHECK-NEXT: entry: 1631; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], -2 1632; CHECK-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 1 1633; CHECK-NEXT: [[TMP2:%.*]] = add nuw i64 [[TMP1]], 1 1634; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 6 1635; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 1636; CHECK: vector.ph: 1637; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], -4 1638; CHECK-NEXT: [[IND_END:%.*]] = shl i64 [[N_VEC]], 1 1639; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 1640; CHECK: vector.body: 1641; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1642; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 1643; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1644; CHECK-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] 1645; CHECK: middle.block: 1646; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 1647; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 1648; CHECK: scalar.ph: 1649; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 1650; CHECK-NEXT: br label [[SCALAR_BODY:%.*]] 1651; CHECK: scalar.body: 1652; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ] 1653; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 2 1654; CHECK-NEXT: [[COND:%.*]] = icmp eq i64 [[I_NEXT]], [[N]] 1655; CHECK-NEXT: br i1 [[COND]], label [[FOR_END]], label [[SCALAR_BODY]], !llvm.loop [[LOOP19:![0-9]+]] 1656; CHECK: for.end: 1657; CHECK-NEXT: ret void 1658; 1659; UNROLL-LABEL: @PR30183( 1660; UNROLL-NEXT: entry: 1661; UNROLL-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], -2 1662; UNROLL-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 1 1663; UNROLL-NEXT: [[TMP2:%.*]] = add nuw i64 [[TMP1]], 1 1664; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 14 1665; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 1666; UNROLL: vector.ph: 1667; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], -8 1668; UNROLL-NEXT: [[IND_END:%.*]] = shl i64 [[N_VEC]], 1 1669; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 1670; UNROLL: vector.body: 1671; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1672; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 1673; UNROLL-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1674; UNROLL-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] 1675; UNROLL: middle.block: 1676; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 1677; UNROLL-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 1678; UNROLL: scalar.ph: 1679; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 1680; UNROLL-NEXT: br label [[SCALAR_BODY:%.*]] 1681; UNROLL: scalar.body: 1682; UNROLL-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ] 1683; UNROLL-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 2 1684; UNROLL-NEXT: [[COND:%.*]] = icmp eq i64 [[I_NEXT]], [[N]] 1685; UNROLL-NEXT: br i1 [[COND]], label [[FOR_END]], label [[SCALAR_BODY]], !llvm.loop [[LOOP19:![0-9]+]] 1686; UNROLL: for.end: 1687; UNROLL-NEXT: ret void 1688; 1689; UNROLL-NO-IC-LABEL: @PR30183( 1690; UNROLL-NO-IC-NEXT: entry: 1691; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], -2 1692; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 1 1693; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add nuw i64 [[TMP1]], 1 1694; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 8 1695; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 1696; UNROLL-NO-IC: vector.ph: 1697; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 8 1698; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 1699; UNROLL-NO-IC-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 2 1700; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[PRE_LOAD:%.*]], i32 3 1701; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 1702; UNROLL-NO-IC: vector.body: 1703; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1704; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP42:%.*]], [[VECTOR_BODY]] ] 1705; UNROLL-NO-IC-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2 1706; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 0 1707; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 2 1708; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 4 1709; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 6 1710; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 8 1711; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = add i64 [[OFFSET_IDX]], 10 1712; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = add i64 [[OFFSET_IDX]], 12 1713; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = add i64 [[OFFSET_IDX]], 14 1714; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = add nuw nsw i64 [[TMP3]], 2 1715; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = add nuw nsw i64 [[TMP4]], 2 1716; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = add nuw nsw i64 [[TMP5]], 2 1717; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = add nuw nsw i64 [[TMP6]], 2 1718; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = add nuw nsw i64 [[TMP7]], 2 1719; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = add nuw nsw i64 [[TMP8]], 2 1720; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = add nuw nsw i64 [[TMP9]], 2 1721; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = add nuw nsw i64 [[TMP10]], 2 1722; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP11]] 1723; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP12]] 1724; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP13]] 1725; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP14]] 1726; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP15]] 1727; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP16]] 1728; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP17]] 1729; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP18]] 1730; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = load i32, i32* [[TMP19]], align 4 1731; UNROLL-NO-IC-NEXT: [[TMP28:%.*]] = load i32, i32* [[TMP20]], align 4 1732; UNROLL-NO-IC-NEXT: [[TMP29:%.*]] = load i32, i32* [[TMP21]], align 4 1733; UNROLL-NO-IC-NEXT: [[TMP30:%.*]] = load i32, i32* [[TMP22]], align 4 1734; UNROLL-NO-IC-NEXT: [[TMP31:%.*]] = insertelement <4 x i32> poison, i32 [[TMP27]], i32 0 1735; UNROLL-NO-IC-NEXT: [[TMP32:%.*]] = insertelement <4 x i32> [[TMP31]], i32 [[TMP28]], i32 1 1736; UNROLL-NO-IC-NEXT: [[TMP33:%.*]] = insertelement <4 x i32> [[TMP32]], i32 [[TMP29]], i32 2 1737; UNROLL-NO-IC-NEXT: [[TMP34:%.*]] = insertelement <4 x i32> [[TMP33]], i32 [[TMP30]], i32 3 1738; UNROLL-NO-IC-NEXT: [[TMP35:%.*]] = load i32, i32* [[TMP23]], align 4 1739; UNROLL-NO-IC-NEXT: [[TMP36:%.*]] = load i32, i32* [[TMP24]], align 4 1740; UNROLL-NO-IC-NEXT: [[TMP37:%.*]] = load i32, i32* [[TMP25]], align 4 1741; UNROLL-NO-IC-NEXT: [[TMP38:%.*]] = load i32, i32* [[TMP26]], align 4 1742; UNROLL-NO-IC-NEXT: [[TMP39:%.*]] = insertelement <4 x i32> poison, i32 [[TMP35]], i32 0 1743; UNROLL-NO-IC-NEXT: [[TMP40:%.*]] = insertelement <4 x i32> [[TMP39]], i32 [[TMP36]], i32 1 1744; UNROLL-NO-IC-NEXT: [[TMP41:%.*]] = insertelement <4 x i32> [[TMP40]], i32 [[TMP37]], i32 2 1745; UNROLL-NO-IC-NEXT: [[TMP42]] = insertelement <4 x i32> [[TMP41]], i32 [[TMP38]], i32 3 1746; UNROLL-NO-IC-NEXT: [[TMP43:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP34]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 1747; UNROLL-NO-IC-NEXT: [[TMP44:%.*]] = shufflevector <4 x i32> [[TMP34]], <4 x i32> [[TMP42]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 1748; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 1749; UNROLL-NO-IC-NEXT: [[TMP45:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1750; UNROLL-NO-IC-NEXT: br i1 [[TMP45]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] 1751; UNROLL-NO-IC: middle.block: 1752; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 1753; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP42]], i32 3 1754; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP42]], i32 2 1755; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 1756; UNROLL-NO-IC: scalar.ph: 1757; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[PRE_LOAD]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 1758; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 1759; UNROLL-NO-IC-NEXT: br label [[SCALAR_BODY:%.*]] 1760; UNROLL-NO-IC: scalar.body: 1761; UNROLL-NO-IC-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ] 1762; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[VAR2:%.*]], [[SCALAR_BODY]] ] 1763; UNROLL-NO-IC-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 2 1764; UNROLL-NO-IC-NEXT: [[VAR1:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[I_NEXT]] 1765; UNROLL-NO-IC-NEXT: [[VAR2]] = load i32, i32* [[VAR1]], align 4 1766; UNROLL-NO-IC-NEXT: [[COND:%.*]] = icmp eq i64 [[I_NEXT]], [[N]] 1767; UNROLL-NO-IC-NEXT: br i1 [[COND]], label [[FOR_END]], label [[SCALAR_BODY]], !llvm.loop [[LOOP19:![0-9]+]] 1768; UNROLL-NO-IC: for.end: 1769; UNROLL-NO-IC-NEXT: ret void 1770; 1771; UNROLL-NO-VF-LABEL: @PR30183( 1772; UNROLL-NO-VF-NEXT: entry: 1773; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], -2 1774; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 1 1775; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = add nuw i64 [[TMP1]], 1 1776; UNROLL-NO-VF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 2 1777; UNROLL-NO-VF-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 1778; UNROLL-NO-VF: vector.ph: 1779; UNROLL-NO-VF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 2 1780; UNROLL-NO-VF-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 1781; UNROLL-NO-VF-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 2 1782; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 1783; UNROLL-NO-VF: vector.body: 1784; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1785; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i32 [ [[PRE_LOAD:%.*]], [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[VECTOR_BODY]] ] 1786; UNROLL-NO-VF-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2 1787; UNROLL-NO-VF-NEXT: [[INDUCTION:%.*]] = add i64 [[OFFSET_IDX]], 0 1788; UNROLL-NO-VF-NEXT: [[INDUCTION1:%.*]] = add i64 [[OFFSET_IDX]], 2 1789; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[INDUCTION]], 2 1790; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = add nuw nsw i64 [[INDUCTION1]], 2 1791; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP3]] 1792; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP4]] 1793; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = load i32, i32* [[TMP5]], align 4 1794; UNROLL-NO-VF-NEXT: [[TMP8]] = load i32, i32* [[TMP6]], align 4 1795; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 1796; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1797; UNROLL-NO-VF-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]] 1798; UNROLL-NO-VF: middle.block: 1799; UNROLL-NO-VF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 1800; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 1801; UNROLL-NO-VF: scalar.ph: 1802; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[PRE_LOAD]], [[ENTRY:%.*]] ], [ [[TMP8]], [[MIDDLE_BLOCK]] ] 1803; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 1804; UNROLL-NO-VF-NEXT: br label [[SCALAR_BODY:%.*]] 1805; UNROLL-NO-VF: scalar.body: 1806; UNROLL-NO-VF-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ] 1807; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[VAR2:%.*]], [[SCALAR_BODY]] ] 1808; UNROLL-NO-VF-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 2 1809; UNROLL-NO-VF-NEXT: [[VAR1:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[I_NEXT]] 1810; UNROLL-NO-VF-NEXT: [[VAR2]] = load i32, i32* [[VAR1]], align 4 1811; UNROLL-NO-VF-NEXT: [[COND:%.*]] = icmp eq i64 [[I_NEXT]], [[N]] 1812; UNROLL-NO-VF-NEXT: br i1 [[COND]], label [[FOR_END]], label [[SCALAR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] 1813; UNROLL-NO-VF: for.end: 1814; UNROLL-NO-VF-NEXT: ret void 1815; 1816; SINK-AFTER-LABEL: @PR30183( 1817; SINK-AFTER-NEXT: entry: 1818; SINK-AFTER-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], -2 1819; SINK-AFTER-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 1 1820; SINK-AFTER-NEXT: [[TMP2:%.*]] = add nuw i64 [[TMP1]], 1 1821; SINK-AFTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4 1822; SINK-AFTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 1823; SINK-AFTER: vector.ph: 1824; SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4 1825; SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 1826; SINK-AFTER-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 2 1827; SINK-AFTER-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[PRE_LOAD:%.*]], i32 3 1828; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 1829; SINK-AFTER: vector.body: 1830; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1831; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ] 1832; SINK-AFTER-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2 1833; SINK-AFTER-NEXT: [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 0 1834; SINK-AFTER-NEXT: [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 2 1835; SINK-AFTER-NEXT: [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 4 1836; SINK-AFTER-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 6 1837; SINK-AFTER-NEXT: [[TMP7:%.*]] = add nuw nsw i64 [[TMP3]], 2 1838; SINK-AFTER-NEXT: [[TMP8:%.*]] = add nuw nsw i64 [[TMP4]], 2 1839; SINK-AFTER-NEXT: [[TMP9:%.*]] = add nuw nsw i64 [[TMP5]], 2 1840; SINK-AFTER-NEXT: [[TMP10:%.*]] = add nuw nsw i64 [[TMP6]], 2 1841; SINK-AFTER-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP7]] 1842; SINK-AFTER-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP8]] 1843; SINK-AFTER-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP9]] 1844; SINK-AFTER-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP10]] 1845; SINK-AFTER-NEXT: [[TMP15:%.*]] = load i32, i32* [[TMP11]], align 4 1846; SINK-AFTER-NEXT: [[TMP16:%.*]] = load i32, i32* [[TMP12]], align 4 1847; SINK-AFTER-NEXT: [[TMP17:%.*]] = load i32, i32* [[TMP13]], align 4 1848; SINK-AFTER-NEXT: [[TMP18:%.*]] = load i32, i32* [[TMP14]], align 4 1849; SINK-AFTER-NEXT: [[TMP19:%.*]] = insertelement <4 x i32> poison, i32 [[TMP15]], i32 0 1850; SINK-AFTER-NEXT: [[TMP20:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP16]], i32 1 1851; SINK-AFTER-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP17]], i32 2 1852; SINK-AFTER-NEXT: [[TMP22]] = insertelement <4 x i32> [[TMP21]], i32 [[TMP18]], i32 3 1853; SINK-AFTER-NEXT: [[TMP23:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP22]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 1854; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 1855; SINK-AFTER-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1856; SINK-AFTER-NEXT: br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] 1857; SINK-AFTER: middle.block: 1858; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 1859; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP22]], i32 3 1860; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP22]], i32 2 1861; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 1862; SINK-AFTER: scalar.ph: 1863; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[PRE_LOAD]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 1864; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 1865; SINK-AFTER-NEXT: br label [[SCALAR_BODY:%.*]] 1866; SINK-AFTER: scalar.body: 1867; SINK-AFTER-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ] 1868; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[VAR2:%.*]], [[SCALAR_BODY]] ] 1869; SINK-AFTER-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 2 1870; SINK-AFTER-NEXT: [[VAR1:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[I_NEXT]] 1871; SINK-AFTER-NEXT: [[VAR2]] = load i32, i32* [[VAR1]], align 4 1872; SINK-AFTER-NEXT: [[COND:%.*]] = icmp eq i64 [[I_NEXT]], [[N]] 1873; SINK-AFTER-NEXT: br i1 [[COND]], label [[FOR_END]], label [[SCALAR_BODY]], !llvm.loop [[LOOP19:![0-9]+]] 1874; SINK-AFTER: for.end: 1875; SINK-AFTER-NEXT: ret void 1876; 1877entry: 1878 br label %scalar.body 1879 1880scalar.body: 1881 %i = phi i64 [ 0, %entry ], [ %i.next, %scalar.body ] 1882 %var0 = phi i32 [ %pre_load, %entry ], [ %var2, %scalar.body ] 1883 %i.next = add nuw nsw i64 %i, 2 1884 %var1 = getelementptr inbounds i32, i32* %a, i64 %i.next 1885 %var2 = load i32, i32* %var1 1886 %cond = icmp eq i64 %i.next,%n 1887 br i1 %cond, label %for.end, label %scalar.body 1888 1889for.end: 1890 ret void 1891} 1892 1893; 1894define void @constant_folded_previous_value() { 1895; CHECK-LABEL: @constant_folded_previous_value( 1896; CHECK-NEXT: entry: 1897; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 1898; CHECK: vector.ph: 1899; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 1900; CHECK: vector.body: 1901; CHECK-NEXT: br i1 undef, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] 1902; CHECK: middle.block: 1903; CHECK-NEXT: br i1 undef, label [[FOR_END:%.*]], label [[SCALAR_PH]] 1904; CHECK: scalar.ph: 1905; CHECK-NEXT: br label [[SCALAR_BODY:%.*]] 1906; CHECK: scalar.body: 1907; CHECK-NEXT: br i1 undef, label [[FOR_END]], label [[SCALAR_BODY]], !llvm.loop [[LOOP21:![0-9]+]] 1908; CHECK: for.end: 1909; CHECK-NEXT: ret void 1910; 1911; UNROLL-LABEL: @constant_folded_previous_value( 1912; UNROLL-NEXT: entry: 1913; UNROLL-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 1914; UNROLL: vector.ph: 1915; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 1916; UNROLL: vector.body: 1917; UNROLL-NEXT: br i1 undef, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] 1918; UNROLL: middle.block: 1919; UNROLL-NEXT: br i1 undef, label [[FOR_END:%.*]], label [[SCALAR_PH]] 1920; UNROLL: scalar.ph: 1921; UNROLL-NEXT: br label [[SCALAR_BODY:%.*]] 1922; UNROLL: scalar.body: 1923; UNROLL-NEXT: br i1 undef, label [[FOR_END]], label [[SCALAR_BODY]], !llvm.loop [[LOOP21:![0-9]+]] 1924; UNROLL: for.end: 1925; UNROLL-NEXT: ret void 1926; 1927; UNROLL-NO-IC-LABEL: @constant_folded_previous_value( 1928; UNROLL-NO-IC-NEXT: entry: 1929; UNROLL-NO-IC-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 1930; UNROLL-NO-IC: vector.ph: 1931; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 1932; UNROLL-NO-IC: vector.body: 1933; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1934; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 0>, [[VECTOR_PH]] ], [ <i64 1, i64 1, i64 1, i64 1>, [[VECTOR_BODY]] ] 1935; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> <i64 1, i64 1, i64 1, i64 1>, <4 x i32> <i32 3, i32 4, i32 5, i32 6> 1936; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 1937; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], undef 1938; UNROLL-NO-IC-NEXT: br i1 [[TMP1]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] 1939; UNROLL-NO-IC: middle.block: 1940; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 undef, undef 1941; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 1942; UNROLL-NO-IC: scalar.ph: 1943; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ 1, [[MIDDLE_BLOCK]] ] 1944; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ undef, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 1945; UNROLL-NO-IC-NEXT: br label [[SCALAR_BODY:%.*]] 1946; UNROLL-NO-IC: scalar.body: 1947; UNROLL-NO-IC-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ] 1948; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[VAR3:%.*]], [[SCALAR_BODY]] ] 1949; UNROLL-NO-IC-NEXT: [[VAR3]] = add i64 0, 1 1950; UNROLL-NO-IC-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 1951; UNROLL-NO-IC-NEXT: [[COND:%.*]] = icmp eq i64 [[I_NEXT]], undef 1952; UNROLL-NO-IC-NEXT: br i1 [[COND]], label [[FOR_END]], label [[SCALAR_BODY]], !llvm.loop [[LOOP21:![0-9]+]] 1953; UNROLL-NO-IC: for.end: 1954; UNROLL-NO-IC-NEXT: ret void 1955; 1956; UNROLL-NO-VF-LABEL: @constant_folded_previous_value( 1957; UNROLL-NO-VF-NEXT: entry: 1958; UNROLL-NO-VF-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 1959; UNROLL-NO-VF: vector.ph: 1960; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 1961; UNROLL-NO-VF: vector.body: 1962; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1963; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ] 1964; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i64 0, 1 1965; UNROLL-NO-VF-NEXT: [[TMP1]] = add i64 0, 1 1966; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 1967; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], undef 1968; UNROLL-NO-VF-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]] 1969; UNROLL-NO-VF: middle.block: 1970; UNROLL-NO-VF-NEXT: [[CMP_N:%.*]] = icmp eq i64 undef, undef 1971; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 1972; UNROLL-NO-VF: scalar.ph: 1973; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[TMP1]], [[MIDDLE_BLOCK]] ] 1974; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ undef, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 1975; UNROLL-NO-VF-NEXT: br label [[SCALAR_BODY:%.*]] 1976; UNROLL-NO-VF: scalar.body: 1977; UNROLL-NO-VF-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ] 1978; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[VAR3:%.*]], [[SCALAR_BODY]] ] 1979; UNROLL-NO-VF-NEXT: [[VAR3]] = add i64 0, 1 1980; UNROLL-NO-VF-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 1981; UNROLL-NO-VF-NEXT: [[COND:%.*]] = icmp eq i64 [[I_NEXT]], undef 1982; UNROLL-NO-VF-NEXT: br i1 [[COND]], label [[FOR_END]], label [[SCALAR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] 1983; UNROLL-NO-VF: for.end: 1984; UNROLL-NO-VF-NEXT: ret void 1985; 1986; SINK-AFTER-LABEL: @constant_folded_previous_value( 1987; SINK-AFTER-NEXT: entry: 1988; SINK-AFTER-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 1989; SINK-AFTER: vector.ph: 1990; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 1991; SINK-AFTER: vector.body: 1992; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1993; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 0>, [[VECTOR_PH]] ], [ <i64 1, i64 1, i64 1, i64 1>, [[VECTOR_BODY]] ] 1994; SINK-AFTER-NEXT: [[TMP0:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> <i64 1, i64 1, i64 1, i64 1>, <4 x i32> <i32 3, i32 4, i32 5, i32 6> 1995; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 1996; SINK-AFTER-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], undef 1997; SINK-AFTER-NEXT: br i1 [[TMP1]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] 1998; SINK-AFTER: middle.block: 1999; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 undef, undef 2000; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2001; SINK-AFTER: scalar.ph: 2002; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ 1, [[MIDDLE_BLOCK]] ] 2003; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ undef, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2004; SINK-AFTER-NEXT: br label [[SCALAR_BODY:%.*]] 2005; SINK-AFTER: scalar.body: 2006; SINK-AFTER-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ] 2007; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[VAR3:%.*]], [[SCALAR_BODY]] ] 2008; SINK-AFTER-NEXT: [[VAR3]] = add i64 0, 1 2009; SINK-AFTER-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 2010; SINK-AFTER-NEXT: [[COND:%.*]] = icmp eq i64 [[I_NEXT]], undef 2011; SINK-AFTER-NEXT: br i1 [[COND]], label [[FOR_END]], label [[SCALAR_BODY]], !llvm.loop [[LOOP21:![0-9]+]] 2012; SINK-AFTER: for.end: 2013; SINK-AFTER-NEXT: ret void 2014; 2015entry: 2016 br label %scalar.body 2017 2018scalar.body: 2019 %i = phi i64 [ 0, %entry ], [ %i.next, %scalar.body ] 2020 %var2 = phi i64 [ 0, %entry ], [ %var3, %scalar.body ] 2021 %var3 = add i64 0, 1 2022 %i.next = add nuw nsw i64 %i, 1 2023 %cond = icmp eq i64 %i.next, undef 2024 br i1 %cond, label %for.end, label %scalar.body 2025 2026for.end: 2027 ret void 2028} 2029 2030; We vectorize this first order recurrence, by generating two 2031; extracts for the phi `val.phi` - one at the last index and 2032; another at the second last index. We need these 2 extracts because 2033; the first order recurrence phi is used outside the loop, so we require the phi 2034; itself and not its update (addx). 2035; Check the case when unrolled but not vectorized. 2036define i32 @extract_second_last_iteration(i32* %cval, i32 %x) { 2037; CHECK-LABEL: @extract_second_last_iteration( 2038; CHECK-NEXT: entry: 2039; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2040; CHECK: vector.ph: 2041; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 2042; CHECK: vector.body: 2043; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2044; CHECK-NEXT: [[TMP0:%.*]] = phi i32 [ 2, [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ] 2045; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 2046; CHECK-NEXT: [[TMP1]] = add i32 [[TMP0]], 4 2047; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96 2048; CHECK-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] 2049; CHECK: middle.block: 2050; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = add i32 [[TMP0]], [[X:%.*]] 2051; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]] 2052; CHECK: scalar.ph: 2053; CHECK-NEXT: br label [[FOR_BODY:%.*]] 2054; CHECK: for.body: 2055; CHECK-NEXT: br i1 undef, label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]] 2056; CHECK: for.end: 2057; CHECK-NEXT: [[VAL_PHI_LCSSA:%.*]] = phi i32 [ undef, [[FOR_BODY]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], [[MIDDLE_BLOCK]] ] 2058; CHECK-NEXT: ret i32 [[VAL_PHI_LCSSA]] 2059; 2060; UNROLL-LABEL: @extract_second_last_iteration( 2061; UNROLL-NEXT: entry: 2062; UNROLL-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2063; UNROLL: vector.ph: 2064; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 2065; UNROLL: vector.body: 2066; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2067; UNROLL-NEXT: [[TMP0:%.*]] = phi i32 [ 2, [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ] 2068; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 2069; UNROLL-NEXT: [[TMP1]] = add i32 [[TMP0]], 8 2070; UNROLL-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96 2071; UNROLL-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] 2072; UNROLL: middle.block: 2073; UNROLL-NEXT: [[TMP3:%.*]] = add i32 [[TMP0]], 4 2074; UNROLL-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = add i32 [[TMP3]], [[X:%.*]] 2075; UNROLL-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]] 2076; UNROLL: scalar.ph: 2077; UNROLL-NEXT: br label [[FOR_BODY:%.*]] 2078; UNROLL: for.body: 2079; UNROLL-NEXT: br i1 undef, label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]] 2080; UNROLL: for.end: 2081; UNROLL-NEXT: [[VAL_PHI_LCSSA:%.*]] = phi i32 [ undef, [[FOR_BODY]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], [[MIDDLE_BLOCK]] ] 2082; UNROLL-NEXT: ret i32 [[VAL_PHI_LCSSA]] 2083; 2084; UNROLL-NO-IC-LABEL: @extract_second_last_iteration( 2085; UNROLL-NO-IC-NEXT: entry: 2086; UNROLL-NO-IC-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2087; UNROLL-NO-IC: vector.ph: 2088; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i32 0 2089; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 2090; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i32 0 2091; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT2]], <4 x i32> poison, <4 x i32> zeroinitializer 2092; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 2093; UNROLL-NO-IC: vector.body: 2094; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2095; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 2096; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ] 2097; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4> 2098; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]] 2099; UNROLL-NO-IC-NEXT: [[TMP1]] = add <4 x i32> [[STEP_ADD]], [[BROADCAST_SPLAT3]] 2100; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP0]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2101; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> [[TMP1]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2102; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 2103; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], <i32 4, i32 4, i32 4, i32 4> 2104; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96 2105; UNROLL-NO-IC-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] 2106; UNROLL-NO-IC: middle.block: 2107; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i32 96, 96 2108; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3 2109; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2 2110; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2111; UNROLL-NO-IC: scalar.ph: 2112; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 2113; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 96, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2114; UNROLL-NO-IC-NEXT: br label [[FOR_BODY:%.*]] 2115; UNROLL-NO-IC: for.body: 2116; UNROLL-NO-IC-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ] 2117; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[ADDX:%.*]], [[FOR_BODY]] ] 2118; UNROLL-NO-IC-NEXT: [[INC]] = add i32 [[INC_PHI]], 1 2119; UNROLL-NO-IC-NEXT: [[BC:%.*]] = zext i32 [[INC_PHI]] to i64 2120; UNROLL-NO-IC-NEXT: [[ADDX]] = add i32 [[INC_PHI]], [[X]] 2121; UNROLL-NO-IC-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC_PHI]], 95 2122; UNROLL-NO-IC-NEXT: br i1 [[CMP]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]] 2123; UNROLL-NO-IC: for.end: 2124; UNROLL-NO-IC-NEXT: [[VAL_PHI_LCSSA:%.*]] = phi i32 [ [[SCALAR_RECUR]], [[FOR_BODY]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], [[MIDDLE_BLOCK]] ] 2125; UNROLL-NO-IC-NEXT: ret i32 [[VAL_PHI_LCSSA]] 2126; 2127; UNROLL-NO-VF-LABEL: @extract_second_last_iteration( 2128; UNROLL-NO-VF-NEXT: entry: 2129; UNROLL-NO-VF-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2130; UNROLL-NO-VF: vector.ph: 2131; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 2132; UNROLL-NO-VF: vector.body: 2133; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2134; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ] 2135; UNROLL-NO-VF-NEXT: [[INDUCTION:%.*]] = add i32 [[INDEX]], 0 2136; UNROLL-NO-VF-NEXT: [[INDUCTION1:%.*]] = add i32 [[INDEX]], 1 2137; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i32 [[INDUCTION]], [[X:%.*]] 2138; UNROLL-NO-VF-NEXT: [[TMP1]] = add i32 [[INDUCTION1]], [[X]] 2139; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 2140; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96 2141; UNROLL-NO-VF-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]] 2142; UNROLL-NO-VF: middle.block: 2143; UNROLL-NO-VF-NEXT: [[CMP_N:%.*]] = icmp eq i32 96, 96 2144; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2145; UNROLL-NO-VF: scalar.ph: 2146; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP1]], [[MIDDLE_BLOCK]] ] 2147; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 96, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2148; UNROLL-NO-VF-NEXT: br label [[FOR_BODY:%.*]] 2149; UNROLL-NO-VF: for.body: 2150; UNROLL-NO-VF-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ] 2151; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[ADDX:%.*]], [[FOR_BODY]] ] 2152; UNROLL-NO-VF-NEXT: [[INC]] = add i32 [[INC_PHI]], 1 2153; UNROLL-NO-VF-NEXT: [[BC:%.*]] = zext i32 [[INC_PHI]] to i64 2154; UNROLL-NO-VF-NEXT: [[ADDX]] = add i32 [[INC_PHI]], [[X]] 2155; UNROLL-NO-VF-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC_PHI]], 95 2156; UNROLL-NO-VF-NEXT: br i1 [[CMP]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] 2157; UNROLL-NO-VF: for.end: 2158; UNROLL-NO-VF-NEXT: [[VAL_PHI_LCSSA:%.*]] = phi i32 [ [[SCALAR_RECUR]], [[FOR_BODY]] ], [ [[TMP0]], [[MIDDLE_BLOCK]] ] 2159; UNROLL-NO-VF-NEXT: ret i32 [[VAL_PHI_LCSSA]] 2160; 2161; SINK-AFTER-LABEL: @extract_second_last_iteration( 2162; SINK-AFTER-NEXT: entry: 2163; SINK-AFTER-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2164; SINK-AFTER: vector.ph: 2165; SINK-AFTER-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i32 0 2166; SINK-AFTER-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 2167; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 2168; SINK-AFTER: vector.body: 2169; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2170; SINK-AFTER-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 2171; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP0:%.*]], [[VECTOR_BODY]] ] 2172; SINK-AFTER-NEXT: [[TMP0]] = add <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]] 2173; SINK-AFTER-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP0]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2174; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 2175; SINK-AFTER-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4> 2176; SINK-AFTER-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96 2177; SINK-AFTER-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] 2178; SINK-AFTER: middle.block: 2179; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i32 96, 96 2180; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP0]], i32 3 2181; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP0]], i32 2 2182; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2183; SINK-AFTER: scalar.ph: 2184; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 2185; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 96, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2186; SINK-AFTER-NEXT: br label [[FOR_BODY:%.*]] 2187; SINK-AFTER: for.body: 2188; SINK-AFTER-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ] 2189; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[ADDX:%.*]], [[FOR_BODY]] ] 2190; SINK-AFTER-NEXT: [[INC]] = add i32 [[INC_PHI]], 1 2191; SINK-AFTER-NEXT: [[BC:%.*]] = zext i32 [[INC_PHI]] to i64 2192; SINK-AFTER-NEXT: [[ADDX]] = add i32 [[INC_PHI]], [[X]] 2193; SINK-AFTER-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC_PHI]], 95 2194; SINK-AFTER-NEXT: br i1 [[CMP]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]] 2195; SINK-AFTER: for.end: 2196; SINK-AFTER-NEXT: [[VAL_PHI_LCSSA:%.*]] = phi i32 [ [[SCALAR_RECUR]], [[FOR_BODY]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], [[MIDDLE_BLOCK]] ] 2197; SINK-AFTER-NEXT: ret i32 [[VAL_PHI_LCSSA]] 2198; 2199entry: 2200 br label %for.body 2201 2202for.body: 2203 %inc.phi = phi i32 [ 0, %entry ], [ %inc, %for.body ] 2204 %val.phi = phi i32 [ 0, %entry ], [ %addx, %for.body ] 2205 %inc = add i32 %inc.phi, 1 2206 %bc = zext i32 %inc.phi to i64 2207 %addx = add i32 %inc.phi, %x 2208 %cmp = icmp eq i32 %inc.phi, 95 2209 br i1 %cmp, label %for.end, label %for.body 2210 2211for.end: 2212 ret i32 %val.phi 2213} 2214 2215; We vectorize this first order recurrence, with a set of insertelements for 2216; each unrolled part. Make sure these insertelements are generated in-order, 2217; because the shuffle of the first order recurrence will be added after the 2218; insertelement of the last part UF - 1, assuming the latter appears after the 2219; insertelements of all other parts. 2220; 2221; int PR33613(double *b, double j, int d) { 2222; int a = 0; 2223; for(int i = 0; i < 10240; i++, b+=25) { 2224; double f = b[d]; // Scalarize to form insertelements 2225; if (j * f) 2226; a++; 2227; j = f; 2228; } 2229; return a; 2230; } 2231; 2232; 2233define i32 @PR33613(double* %b, double %j, i32 %d) { 2234; CHECK-LABEL: @PR33613( 2235; CHECK-NEXT: entry: 2236; CHECK-NEXT: [[IDXPROM:%.*]] = sext i32 [[D:%.*]] to i64 2237; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2238; CHECK: vector.ph: 2239; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x double> poison, double [[J:%.*]], i64 3 2240; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 2241; CHECK: vector.body: 2242; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2243; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[VECTOR_BODY]] ] 2244; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x double> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[VECTOR_BODY]] ] 2245; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[INDEX]], 25 2246; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr double, double* [[B:%.*]], i64 [[IDXPROM]] 2247; CHECK-NEXT: [[TMP1:%.*]] = or i64 [[INDEX]], 1 2248; CHECK-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 25 2249; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr double, double* [[B]], i64 [[IDXPROM]] 2250; CHECK-NEXT: [[TMP3:%.*]] = or i64 [[INDEX]], 2 2251; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], 25 2252; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr double, double* [[B]], i64 [[IDXPROM]] 2253; CHECK-NEXT: [[TMP5:%.*]] = or i64 [[INDEX]], 3 2254; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 25 2255; CHECK-NEXT: [[NEXT_GEP4:%.*]] = getelementptr double, double* [[B]], i64 [[IDXPROM]] 2256; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP]], i64 [[TMP0]] 2257; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP2]], i64 [[TMP2]] 2258; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP3]], i64 [[TMP4]] 2259; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP4]], i64 [[TMP6]] 2260; CHECK-NEXT: [[TMP11:%.*]] = load double, double* [[TMP7]], align 8 2261; CHECK-NEXT: [[TMP12:%.*]] = load double, double* [[TMP8]], align 8 2262; CHECK-NEXT: [[TMP13:%.*]] = load double, double* [[TMP9]], align 8 2263; CHECK-NEXT: [[TMP14:%.*]] = load double, double* [[TMP10]], align 8 2264; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x double> poison, double [[TMP11]], i64 0 2265; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x double> [[TMP15]], double [[TMP12]], i64 1 2266; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x double> [[TMP16]], double [[TMP13]], i64 2 2267; CHECK-NEXT: [[TMP18]] = insertelement <4 x double> [[TMP17]], double [[TMP14]], i64 3 2268; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR]], <4 x double> [[TMP17]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2269; CHECK-NEXT: [[TMP20:%.*]] = fmul <4 x double> [[TMP19]], [[TMP18]] 2270; CHECK-NEXT: [[TMP21:%.*]] = fcmp une <4 x double> [[TMP20]], zeroinitializer 2271; CHECK-NEXT: [[TMP22:%.*]] = zext <4 x i1> [[TMP21]] to <4 x i32> 2272; CHECK-NEXT: [[TMP23]] = add <4 x i32> [[VEC_PHI]], [[TMP22]] 2273; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 2274; CHECK-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10240 2275; CHECK-NEXT: br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]] 2276; CHECK: middle.block: 2277; CHECK-NEXT: [[TMP25:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP23]]) 2278; CHECK-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]] 2279; CHECK: scalar.ph: 2280; CHECK-NEXT: br label [[FOR_BODY:%.*]] 2281; CHECK: for.cond.cleanup: 2282; CHECK-NEXT: [[A_1_LCSSA:%.*]] = phi i32 [ undef, [[FOR_BODY]] ], [ [[TMP25]], [[MIDDLE_BLOCK]] ] 2283; CHECK-NEXT: ret i32 [[A_1_LCSSA]] 2284; CHECK: for.body: 2285; CHECK-NEXT: br i1 undef, label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]] 2286; 2287; UNROLL-LABEL: @PR33613( 2288; UNROLL-NEXT: entry: 2289; UNROLL-NEXT: [[IDXPROM:%.*]] = sext i32 [[D:%.*]] to i64 2290; UNROLL-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2291; UNROLL: vector.ph: 2292; UNROLL-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x double> poison, double [[J:%.*]], i64 3 2293; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 2294; UNROLL: vector.body: 2295; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2296; UNROLL-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP47:%.*]], [[VECTOR_BODY]] ] 2297; UNROLL-NEXT: [[VEC_PHI9:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP48:%.*]], [[VECTOR_BODY]] ] 2298; UNROLL-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x double> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP38:%.*]], [[VECTOR_BODY]] ] 2299; UNROLL-NEXT: [[TMP0:%.*]] = mul i64 [[INDEX]], 25 2300; UNROLL-NEXT: [[NEXT_GEP:%.*]] = getelementptr double, double* [[B:%.*]], i64 [[IDXPROM]] 2301; UNROLL-NEXT: [[TMP1:%.*]] = or i64 [[INDEX]], 1 2302; UNROLL-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 25 2303; UNROLL-NEXT: [[NEXT_GEP2:%.*]] = getelementptr double, double* [[B]], i64 [[IDXPROM]] 2304; UNROLL-NEXT: [[TMP3:%.*]] = or i64 [[INDEX]], 2 2305; UNROLL-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], 25 2306; UNROLL-NEXT: [[NEXT_GEP3:%.*]] = getelementptr double, double* [[B]], i64 [[IDXPROM]] 2307; UNROLL-NEXT: [[TMP5:%.*]] = or i64 [[INDEX]], 3 2308; UNROLL-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 25 2309; UNROLL-NEXT: [[NEXT_GEP4:%.*]] = getelementptr double, double* [[B]], i64 [[IDXPROM]] 2310; UNROLL-NEXT: [[TMP7:%.*]] = or i64 [[INDEX]], 4 2311; UNROLL-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 25 2312; UNROLL-NEXT: [[NEXT_GEP5:%.*]] = getelementptr double, double* [[B]], i64 [[IDXPROM]] 2313; UNROLL-NEXT: [[TMP9:%.*]] = or i64 [[INDEX]], 5 2314; UNROLL-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 25 2315; UNROLL-NEXT: [[NEXT_GEP6:%.*]] = getelementptr double, double* [[B]], i64 [[IDXPROM]] 2316; UNROLL-NEXT: [[TMP11:%.*]] = or i64 [[INDEX]], 6 2317; UNROLL-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 25 2318; UNROLL-NEXT: [[NEXT_GEP7:%.*]] = getelementptr double, double* [[B]], i64 [[IDXPROM]] 2319; UNROLL-NEXT: [[TMP13:%.*]] = or i64 [[INDEX]], 7 2320; UNROLL-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 25 2321; UNROLL-NEXT: [[NEXT_GEP8:%.*]] = getelementptr double, double* [[B]], i64 [[IDXPROM]] 2322; UNROLL-NEXT: [[TMP15:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP]], i64 [[TMP0]] 2323; UNROLL-NEXT: [[TMP16:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP2]], i64 [[TMP2]] 2324; UNROLL-NEXT: [[TMP17:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP3]], i64 [[TMP4]] 2325; UNROLL-NEXT: [[TMP18:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP4]], i64 [[TMP6]] 2326; UNROLL-NEXT: [[TMP19:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP5]], i64 [[TMP8]] 2327; UNROLL-NEXT: [[TMP20:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP6]], i64 [[TMP10]] 2328; UNROLL-NEXT: [[TMP21:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP7]], i64 [[TMP12]] 2329; UNROLL-NEXT: [[TMP22:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP8]], i64 [[TMP14]] 2330; UNROLL-NEXT: [[TMP23:%.*]] = load double, double* [[TMP15]], align 8 2331; UNROLL-NEXT: [[TMP24:%.*]] = load double, double* [[TMP16]], align 8 2332; UNROLL-NEXT: [[TMP25:%.*]] = load double, double* [[TMP17]], align 8 2333; UNROLL-NEXT: [[TMP26:%.*]] = load double, double* [[TMP18]], align 8 2334; UNROLL-NEXT: [[TMP27:%.*]] = insertelement <4 x double> poison, double [[TMP23]], i64 0 2335; UNROLL-NEXT: [[TMP28:%.*]] = insertelement <4 x double> [[TMP27]], double [[TMP24]], i64 1 2336; UNROLL-NEXT: [[TMP29:%.*]] = insertelement <4 x double> [[TMP28]], double [[TMP25]], i64 2 2337; UNROLL-NEXT: [[TMP30:%.*]] = insertelement <4 x double> [[TMP29]], double [[TMP26]], i64 3 2338; UNROLL-NEXT: [[TMP31:%.*]] = load double, double* [[TMP19]], align 8 2339; UNROLL-NEXT: [[TMP32:%.*]] = load double, double* [[TMP20]], align 8 2340; UNROLL-NEXT: [[TMP33:%.*]] = load double, double* [[TMP21]], align 8 2341; UNROLL-NEXT: [[TMP34:%.*]] = load double, double* [[TMP22]], align 8 2342; UNROLL-NEXT: [[TMP35:%.*]] = insertelement <4 x double> poison, double [[TMP31]], i64 0 2343; UNROLL-NEXT: [[TMP36:%.*]] = insertelement <4 x double> [[TMP35]], double [[TMP32]], i64 1 2344; UNROLL-NEXT: [[TMP37:%.*]] = insertelement <4 x double> [[TMP36]], double [[TMP33]], i64 2 2345; UNROLL-NEXT: [[TMP38]] = insertelement <4 x double> [[TMP37]], double [[TMP34]], i64 3 2346; UNROLL-NEXT: [[TMP39:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR]], <4 x double> [[TMP29]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2347; UNROLL-NEXT: [[TMP40:%.*]] = shufflevector <4 x double> [[TMP30]], <4 x double> [[TMP37]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2348; UNROLL-NEXT: [[TMP41:%.*]] = fmul <4 x double> [[TMP39]], [[TMP30]] 2349; UNROLL-NEXT: [[TMP42:%.*]] = fmul <4 x double> [[TMP40]], [[TMP38]] 2350; UNROLL-NEXT: [[TMP43:%.*]] = fcmp une <4 x double> [[TMP41]], zeroinitializer 2351; UNROLL-NEXT: [[TMP44:%.*]] = fcmp une <4 x double> [[TMP42]], zeroinitializer 2352; UNROLL-NEXT: [[TMP45:%.*]] = zext <4 x i1> [[TMP43]] to <4 x i32> 2353; UNROLL-NEXT: [[TMP46:%.*]] = zext <4 x i1> [[TMP44]] to <4 x i32> 2354; UNROLL-NEXT: [[TMP47]] = add <4 x i32> [[VEC_PHI]], [[TMP45]] 2355; UNROLL-NEXT: [[TMP48]] = add <4 x i32> [[VEC_PHI9]], [[TMP46]] 2356; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 2357; UNROLL-NEXT: [[TMP49:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10240 2358; UNROLL-NEXT: br i1 [[TMP49]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]] 2359; UNROLL: middle.block: 2360; UNROLL-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP48]], [[TMP47]] 2361; UNROLL-NEXT: [[TMP50:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]]) 2362; UNROLL-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]] 2363; UNROLL: scalar.ph: 2364; UNROLL-NEXT: br label [[FOR_BODY:%.*]] 2365; UNROLL: for.cond.cleanup: 2366; UNROLL-NEXT: [[A_1_LCSSA:%.*]] = phi i32 [ undef, [[FOR_BODY]] ], [ [[TMP50]], [[MIDDLE_BLOCK]] ] 2367; UNROLL-NEXT: ret i32 [[A_1_LCSSA]] 2368; UNROLL: for.body: 2369; UNROLL-NEXT: br i1 undef, label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]] 2370; 2371; UNROLL-NO-IC-LABEL: @PR33613( 2372; UNROLL-NO-IC-NEXT: entry: 2373; UNROLL-NO-IC-NEXT: [[IDXPROM:%.*]] = sext i32 [[D:%.*]] to i64 2374; UNROLL-NO-IC-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2375; UNROLL-NO-IC: vector.ph: 2376; UNROLL-NO-IC-NEXT: [[IND_END:%.*]] = getelementptr double, double* [[B:%.*]], i64 256000 2377; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x double> poison, double [[J:%.*]], i32 3 2378; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 2379; UNROLL-NO-IC: vector.body: 2380; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2381; UNROLL-NO-IC-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP48:%.*]], [[VECTOR_BODY]] ] 2382; UNROLL-NO-IC-NEXT: [[VEC_PHI9:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP49:%.*]], [[VECTOR_BODY]] ] 2383; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x double> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP39:%.*]], [[VECTOR_BODY]] ] 2384; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 2385; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 25 2386; UNROLL-NO-IC-NEXT: [[NEXT_GEP:%.*]] = getelementptr double, double* [[B]], i64 [[TMP1]] 2387; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1 2388; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 25 2389; UNROLL-NO-IC-NEXT: [[NEXT_GEP2:%.*]] = getelementptr double, double* [[B]], i64 [[TMP3]] 2390; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 2 2391; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 25 2392; UNROLL-NO-IC-NEXT: [[NEXT_GEP3:%.*]] = getelementptr double, double* [[B]], i64 [[TMP5]] 2393; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 3 2394; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 25 2395; UNROLL-NO-IC-NEXT: [[NEXT_GEP4:%.*]] = getelementptr double, double* [[B]], i64 [[TMP7]] 2396; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 4 2397; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 25 2398; UNROLL-NO-IC-NEXT: [[NEXT_GEP5:%.*]] = getelementptr double, double* [[B]], i64 [[TMP9]] 2399; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 5 2400; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 25 2401; UNROLL-NO-IC-NEXT: [[NEXT_GEP6:%.*]] = getelementptr double, double* [[B]], i64 [[TMP11]] 2402; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 6 2403; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 25 2404; UNROLL-NO-IC-NEXT: [[NEXT_GEP7:%.*]] = getelementptr double, double* [[B]], i64 [[TMP13]] 2405; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = add i64 [[INDEX]], 7 2406; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 25 2407; UNROLL-NO-IC-NEXT: [[NEXT_GEP8:%.*]] = getelementptr double, double* [[B]], i64 [[TMP15]] 2408; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP]], i64 [[IDXPROM]] 2409; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP2]], i64 [[IDXPROM]] 2410; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP3]], i64 [[IDXPROM]] 2411; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP4]], i64 [[IDXPROM]] 2412; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP5]], i64 [[IDXPROM]] 2413; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP6]], i64 [[IDXPROM]] 2414; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP7]], i64 [[IDXPROM]] 2415; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP8]], i64 [[IDXPROM]] 2416; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = load double, double* [[TMP16]], align 8 2417; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = load double, double* [[TMP17]], align 8 2418; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = load double, double* [[TMP18]], align 8 2419; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = load double, double* [[TMP19]], align 8 2420; UNROLL-NO-IC-NEXT: [[TMP28:%.*]] = insertelement <4 x double> poison, double [[TMP24]], i32 0 2421; UNROLL-NO-IC-NEXT: [[TMP29:%.*]] = insertelement <4 x double> [[TMP28]], double [[TMP25]], i32 1 2422; UNROLL-NO-IC-NEXT: [[TMP30:%.*]] = insertelement <4 x double> [[TMP29]], double [[TMP26]], i32 2 2423; UNROLL-NO-IC-NEXT: [[TMP31:%.*]] = insertelement <4 x double> [[TMP30]], double [[TMP27]], i32 3 2424; UNROLL-NO-IC-NEXT: [[TMP32:%.*]] = load double, double* [[TMP20]], align 8 2425; UNROLL-NO-IC-NEXT: [[TMP33:%.*]] = load double, double* [[TMP21]], align 8 2426; UNROLL-NO-IC-NEXT: [[TMP34:%.*]] = load double, double* [[TMP22]], align 8 2427; UNROLL-NO-IC-NEXT: [[TMP35:%.*]] = load double, double* [[TMP23]], align 8 2428; UNROLL-NO-IC-NEXT: [[TMP36:%.*]] = insertelement <4 x double> poison, double [[TMP32]], i32 0 2429; UNROLL-NO-IC-NEXT: [[TMP37:%.*]] = insertelement <4 x double> [[TMP36]], double [[TMP33]], i32 1 2430; UNROLL-NO-IC-NEXT: [[TMP38:%.*]] = insertelement <4 x double> [[TMP37]], double [[TMP34]], i32 2 2431; UNROLL-NO-IC-NEXT: [[TMP39]] = insertelement <4 x double> [[TMP38]], double [[TMP35]], i32 3 2432; UNROLL-NO-IC-NEXT: [[TMP40:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR]], <4 x double> [[TMP31]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2433; UNROLL-NO-IC-NEXT: [[TMP41:%.*]] = shufflevector <4 x double> [[TMP31]], <4 x double> [[TMP39]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2434; UNROLL-NO-IC-NEXT: [[TMP42:%.*]] = fmul <4 x double> [[TMP40]], [[TMP31]] 2435; UNROLL-NO-IC-NEXT: [[TMP43:%.*]] = fmul <4 x double> [[TMP41]], [[TMP39]] 2436; UNROLL-NO-IC-NEXT: [[TMP44:%.*]] = fcmp une <4 x double> [[TMP42]], zeroinitializer 2437; UNROLL-NO-IC-NEXT: [[TMP45:%.*]] = fcmp une <4 x double> [[TMP43]], zeroinitializer 2438; UNROLL-NO-IC-NEXT: [[TMP46:%.*]] = zext <4 x i1> [[TMP44]] to <4 x i32> 2439; UNROLL-NO-IC-NEXT: [[TMP47:%.*]] = zext <4 x i1> [[TMP45]] to <4 x i32> 2440; UNROLL-NO-IC-NEXT: [[TMP48]] = add <4 x i32> [[VEC_PHI]], [[TMP46]] 2441; UNROLL-NO-IC-NEXT: [[TMP49]] = add <4 x i32> [[VEC_PHI9]], [[TMP47]] 2442; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 2443; UNROLL-NO-IC-NEXT: [[TMP50:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10240 2444; UNROLL-NO-IC-NEXT: br i1 [[TMP50]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]] 2445; UNROLL-NO-IC: middle.block: 2446; UNROLL-NO-IC-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP49]], [[TMP48]] 2447; UNROLL-NO-IC-NEXT: [[TMP51:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]]) 2448; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 10240, 10240 2449; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x double> [[TMP39]], i32 3 2450; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x double> [[TMP39]], i32 2 2451; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]] 2452; UNROLL-NO-IC: scalar.ph: 2453; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi double [ [[J]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 2454; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi double* [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[B]], [[ENTRY]] ] 2455; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ 10240, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2456; UNROLL-NO-IC-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP51]], [[MIDDLE_BLOCK]] ] 2457; UNROLL-NO-IC-NEXT: br label [[FOR_BODY:%.*]] 2458; UNROLL-NO-IC: for.cond.cleanup: 2459; UNROLL-NO-IC-NEXT: [[A_1_LCSSA:%.*]] = phi i32 [ [[A_1:%.*]], [[FOR_BODY]] ], [ [[TMP51]], [[MIDDLE_BLOCK]] ] 2460; UNROLL-NO-IC-NEXT: ret i32 [[A_1_LCSSA]] 2461; UNROLL-NO-IC: for.body: 2462; UNROLL-NO-IC-NEXT: [[B_ADDR_012:%.*]] = phi double* [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADD_PTR:%.*]], [[FOR_BODY]] ] 2463; UNROLL-NO-IC-NEXT: [[I_011:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[INC1:%.*]], [[FOR_BODY]] ] 2464; UNROLL-NO-IC-NEXT: [[A_010:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[A_1]], [[FOR_BODY]] ] 2465; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi double [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP52:%.*]], [[FOR_BODY]] ] 2466; UNROLL-NO-IC-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[B_ADDR_012]], i64 [[IDXPROM]] 2467; UNROLL-NO-IC-NEXT: [[TMP52]] = load double, double* [[ARRAYIDX]], align 8 2468; UNROLL-NO-IC-NEXT: [[MUL:%.*]] = fmul double [[SCALAR_RECUR]], [[TMP52]] 2469; UNROLL-NO-IC-NEXT: [[TOBOOL:%.*]] = fcmp une double [[MUL]], 0.000000e+00 2470; UNROLL-NO-IC-NEXT: [[INC:%.*]] = zext i1 [[TOBOOL]] to i32 2471; UNROLL-NO-IC-NEXT: [[A_1]] = add nsw i32 [[A_010]], [[INC]] 2472; UNROLL-NO-IC-NEXT: [[INC1]] = add nuw nsw i32 [[I_011]], 1 2473; UNROLL-NO-IC-NEXT: [[ADD_PTR]] = getelementptr inbounds double, double* [[B_ADDR_012]], i64 25 2474; UNROLL-NO-IC-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC1]], 10240 2475; UNROLL-NO-IC-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]] 2476; 2477; UNROLL-NO-VF-LABEL: @PR33613( 2478; UNROLL-NO-VF-NEXT: entry: 2479; UNROLL-NO-VF-NEXT: [[IDXPROM:%.*]] = sext i32 [[D:%.*]] to i64 2480; UNROLL-NO-VF-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2481; UNROLL-NO-VF: vector.ph: 2482; UNROLL-NO-VF-NEXT: [[IND_END:%.*]] = getelementptr double, double* [[B:%.*]], i64 256000 2483; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 2484; UNROLL-NO-VF: vector.body: 2485; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2486; UNROLL-NO-VF-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ] 2487; UNROLL-NO-VF-NEXT: [[VEC_PHI3:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ] 2488; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi double [ [[J:%.*]], [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ] 2489; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 2490; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 25 2491; UNROLL-NO-VF-NEXT: [[NEXT_GEP:%.*]] = getelementptr double, double* [[B]], i64 [[TMP1]] 2492; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1 2493; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 25 2494; UNROLL-NO-VF-NEXT: [[NEXT_GEP2:%.*]] = getelementptr double, double* [[B]], i64 [[TMP3]] 2495; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP]], i64 [[IDXPROM]] 2496; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP2]], i64 [[IDXPROM]] 2497; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = load double, double* [[TMP4]], align 8 2498; UNROLL-NO-VF-NEXT: [[TMP7]] = load double, double* [[TMP5]], align 8 2499; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = fmul double [[VECTOR_RECUR]], [[TMP6]] 2500; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = fmul double [[TMP6]], [[TMP7]] 2501; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = fcmp une double [[TMP8]], 0.000000e+00 2502; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = fcmp une double [[TMP9]], 0.000000e+00 2503; UNROLL-NO-VF-NEXT: [[TMP12:%.*]] = zext i1 [[TMP10]] to i32 2504; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = zext i1 [[TMP11]] to i32 2505; UNROLL-NO-VF-NEXT: [[TMP14]] = add i32 [[VEC_PHI]], [[TMP12]] 2506; UNROLL-NO-VF-NEXT: [[TMP15]] = add i32 [[VEC_PHI3]], [[TMP13]] 2507; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 2508; UNROLL-NO-VF-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10240 2509; UNROLL-NO-VF-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]] 2510; UNROLL-NO-VF: middle.block: 2511; UNROLL-NO-VF-NEXT: [[BIN_RDX:%.*]] = add i32 [[TMP15]], [[TMP14]] 2512; UNROLL-NO-VF-NEXT: [[CMP_N:%.*]] = icmp eq i64 10240, 10240 2513; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]] 2514; UNROLL-NO-VF: scalar.ph: 2515; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi double [ [[J]], [[ENTRY:%.*]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ] 2516; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi double* [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[B]], [[ENTRY]] ] 2517; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ 10240, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2518; UNROLL-NO-VF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ] 2519; UNROLL-NO-VF-NEXT: br label [[FOR_BODY:%.*]] 2520; UNROLL-NO-VF: for.cond.cleanup: 2521; UNROLL-NO-VF-NEXT: [[A_1_LCSSA:%.*]] = phi i32 [ [[A_1:%.*]], [[FOR_BODY]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ] 2522; UNROLL-NO-VF-NEXT: ret i32 [[A_1_LCSSA]] 2523; UNROLL-NO-VF: for.body: 2524; UNROLL-NO-VF-NEXT: [[B_ADDR_012:%.*]] = phi double* [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADD_PTR:%.*]], [[FOR_BODY]] ] 2525; UNROLL-NO-VF-NEXT: [[I_011:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[INC1:%.*]], [[FOR_BODY]] ] 2526; UNROLL-NO-VF-NEXT: [[A_010:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[A_1]], [[FOR_BODY]] ] 2527; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi double [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP17:%.*]], [[FOR_BODY]] ] 2528; UNROLL-NO-VF-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[B_ADDR_012]], i64 [[IDXPROM]] 2529; UNROLL-NO-VF-NEXT: [[TMP17]] = load double, double* [[ARRAYIDX]], align 8 2530; UNROLL-NO-VF-NEXT: [[MUL:%.*]] = fmul double [[SCALAR_RECUR]], [[TMP17]] 2531; UNROLL-NO-VF-NEXT: [[TOBOOL:%.*]] = fcmp une double [[MUL]], 0.000000e+00 2532; UNROLL-NO-VF-NEXT: [[INC:%.*]] = zext i1 [[TOBOOL]] to i32 2533; UNROLL-NO-VF-NEXT: [[A_1]] = add nsw i32 [[A_010]], [[INC]] 2534; UNROLL-NO-VF-NEXT: [[INC1]] = add nuw nsw i32 [[I_011]], 1 2535; UNROLL-NO-VF-NEXT: [[ADD_PTR]] = getelementptr inbounds double, double* [[B_ADDR_012]], i64 25 2536; UNROLL-NO-VF-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC1]], 10240 2537; UNROLL-NO-VF-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]] 2538; 2539; SINK-AFTER-LABEL: @PR33613( 2540; SINK-AFTER-NEXT: entry: 2541; SINK-AFTER-NEXT: [[IDXPROM:%.*]] = sext i32 [[D:%.*]] to i64 2542; SINK-AFTER-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2543; SINK-AFTER: vector.ph: 2544; SINK-AFTER-NEXT: [[IND_END:%.*]] = getelementptr double, double* [[B:%.*]], i64 256000 2545; SINK-AFTER-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x double> poison, double [[J:%.*]], i32 3 2546; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 2547; SINK-AFTER: vector.body: 2548; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2549; SINK-AFTER-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ] 2550; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x double> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ] 2551; SINK-AFTER-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 2552; SINK-AFTER-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 25 2553; SINK-AFTER-NEXT: [[NEXT_GEP:%.*]] = getelementptr double, double* [[B]], i64 [[TMP1]] 2554; SINK-AFTER-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1 2555; SINK-AFTER-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 25 2556; SINK-AFTER-NEXT: [[NEXT_GEP2:%.*]] = getelementptr double, double* [[B]], i64 [[TMP3]] 2557; SINK-AFTER-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 2 2558; SINK-AFTER-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 25 2559; SINK-AFTER-NEXT: [[NEXT_GEP3:%.*]] = getelementptr double, double* [[B]], i64 [[TMP5]] 2560; SINK-AFTER-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 3 2561; SINK-AFTER-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 25 2562; SINK-AFTER-NEXT: [[NEXT_GEP4:%.*]] = getelementptr double, double* [[B]], i64 [[TMP7]] 2563; SINK-AFTER-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP]], i64 [[IDXPROM]] 2564; SINK-AFTER-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP2]], i64 [[IDXPROM]] 2565; SINK-AFTER-NEXT: [[TMP10:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP3]], i64 [[IDXPROM]] 2566; SINK-AFTER-NEXT: [[TMP11:%.*]] = getelementptr inbounds double, double* [[NEXT_GEP4]], i64 [[IDXPROM]] 2567; SINK-AFTER-NEXT: [[TMP12:%.*]] = load double, double* [[TMP8]], align 8 2568; SINK-AFTER-NEXT: [[TMP13:%.*]] = load double, double* [[TMP9]], align 8 2569; SINK-AFTER-NEXT: [[TMP14:%.*]] = load double, double* [[TMP10]], align 8 2570; SINK-AFTER-NEXT: [[TMP15:%.*]] = load double, double* [[TMP11]], align 8 2571; SINK-AFTER-NEXT: [[TMP16:%.*]] = insertelement <4 x double> poison, double [[TMP12]], i32 0 2572; SINK-AFTER-NEXT: [[TMP17:%.*]] = insertelement <4 x double> [[TMP16]], double [[TMP13]], i32 1 2573; SINK-AFTER-NEXT: [[TMP18:%.*]] = insertelement <4 x double> [[TMP17]], double [[TMP14]], i32 2 2574; SINK-AFTER-NEXT: [[TMP19]] = insertelement <4 x double> [[TMP18]], double [[TMP15]], i32 3 2575; SINK-AFTER-NEXT: [[TMP20:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR]], <4 x double> [[TMP19]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2576; SINK-AFTER-NEXT: [[TMP21:%.*]] = fmul <4 x double> [[TMP20]], [[TMP19]] 2577; SINK-AFTER-NEXT: [[TMP22:%.*]] = fcmp une <4 x double> [[TMP21]], zeroinitializer 2578; SINK-AFTER-NEXT: [[TMP23:%.*]] = zext <4 x i1> [[TMP22]] to <4 x i32> 2579; SINK-AFTER-NEXT: [[TMP24]] = add <4 x i32> [[VEC_PHI]], [[TMP23]] 2580; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 2581; SINK-AFTER-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10240 2582; SINK-AFTER-NEXT: br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]] 2583; SINK-AFTER: middle.block: 2584; SINK-AFTER-NEXT: [[TMP26:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP24]]) 2585; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 10240, 10240 2586; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x double> [[TMP19]], i32 3 2587; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x double> [[TMP19]], i32 2 2588; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]] 2589; SINK-AFTER: scalar.ph: 2590; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi double [ [[J]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 2591; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi double* [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[B]], [[ENTRY]] ] 2592; SINK-AFTER-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ 10240, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2593; SINK-AFTER-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP26]], [[MIDDLE_BLOCK]] ] 2594; SINK-AFTER-NEXT: br label [[FOR_BODY:%.*]] 2595; SINK-AFTER: for.cond.cleanup: 2596; SINK-AFTER-NEXT: [[A_1_LCSSA:%.*]] = phi i32 [ [[A_1:%.*]], [[FOR_BODY]] ], [ [[TMP26]], [[MIDDLE_BLOCK]] ] 2597; SINK-AFTER-NEXT: ret i32 [[A_1_LCSSA]] 2598; SINK-AFTER: for.body: 2599; SINK-AFTER-NEXT: [[B_ADDR_012:%.*]] = phi double* [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADD_PTR:%.*]], [[FOR_BODY]] ] 2600; SINK-AFTER-NEXT: [[I_011:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[INC1:%.*]], [[FOR_BODY]] ] 2601; SINK-AFTER-NEXT: [[A_010:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[A_1]], [[FOR_BODY]] ] 2602; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi double [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP27:%.*]], [[FOR_BODY]] ] 2603; SINK-AFTER-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[B_ADDR_012]], i64 [[IDXPROM]] 2604; SINK-AFTER-NEXT: [[TMP27]] = load double, double* [[ARRAYIDX]], align 8 2605; SINK-AFTER-NEXT: [[MUL:%.*]] = fmul double [[SCALAR_RECUR]], [[TMP27]] 2606; SINK-AFTER-NEXT: [[TOBOOL:%.*]] = fcmp une double [[MUL]], 0.000000e+00 2607; SINK-AFTER-NEXT: [[INC:%.*]] = zext i1 [[TOBOOL]] to i32 2608; SINK-AFTER-NEXT: [[A_1]] = add nsw i32 [[A_010]], [[INC]] 2609; SINK-AFTER-NEXT: [[INC1]] = add nuw nsw i32 [[I_011]], 1 2610; SINK-AFTER-NEXT: [[ADD_PTR]] = getelementptr inbounds double, double* [[B_ADDR_012]], i64 25 2611; SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC1]], 10240 2612; SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]] 2613; 2614entry: 2615 %idxprom = sext i32 %d to i64 2616 br label %for.body 2617 2618for.cond.cleanup: 2619 %a.1.lcssa = phi i32 [ %a.1, %for.body ] 2620 ret i32 %a.1.lcssa 2621 2622for.body: 2623 %b.addr.012 = phi double* [ %b, %entry ], [ %add.ptr, %for.body ] 2624 %i.011 = phi i32 [ 0, %entry ], [ %inc1, %for.body ] 2625 %a.010 = phi i32 [ 0, %entry ], [ %a.1, %for.body ] 2626 %j.addr.09 = phi double [ %j, %entry ], [ %0, %for.body ] 2627 %arrayidx = getelementptr inbounds double, double* %b.addr.012, i64 %idxprom 2628 %0 = load double, double* %arrayidx, align 8 2629 %mul = fmul double %j.addr.09, %0 2630 %tobool = fcmp une double %mul, 0.000000e+00 2631 %inc = zext i1 %tobool to i32 2632 %a.1 = add nsw i32 %a.010, %inc 2633 %inc1 = add nuw nsw i32 %i.011, 1 2634 %add.ptr = getelementptr inbounds double, double* %b.addr.012, i64 25 2635 %exitcond = icmp eq i32 %inc1, 10240 2636 br i1 %exitcond, label %for.cond.cleanup, label %for.body 2637} 2638 2639; void sink_after(short *a, int n, int *b) { 2640; for(int i = 0; i < n; i++) 2641; b[i] = (a[i] * a[i + 1]); 2642; } 2643; 2644; Check that the sext sank after the load in the vector loop. 2645; 2646define void @sink_after(i16* %a, i32* %b, i64 %n) { 2647; CHECK-LABEL: @sink_after( 2648; CHECK-NEXT: entry: 2649; CHECK-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[A:%.*]], align 2 2650; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 2651; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 2652; CHECK: vector.memcheck: 2653; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B:%.*]], i64 [[N]] 2654; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i16, i16* [[A]], i64 1 2655; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 2656; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP0]] 2657; CHECK-NEXT: [[TMP1:%.*]] = bitcast i16* [[SCEVGEP5]] to i32* 2658; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt i32* [[TMP1]], [[B]] 2659; CHECK-NEXT: [[TMP2:%.*]] = bitcast i32* [[SCEVGEP]] to i16* 2660; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult i16* [[SCEVGEP3]], [[TMP2]] 2661; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 2662; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 2663; CHECK: vector.ph: 2664; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4 2665; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i64 3 2666; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 2667; CHECK: vector.body: 2668; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2669; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ] 2670; CHECK-NEXT: [[TMP3:%.*]] = or i64 [[INDEX]], 1 2671; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP3]] 2672; CHECK-NEXT: [[TMP5:%.*]] = bitcast i16* [[TMP4]] to <4 x i16>* 2673; CHECK-NEXT: [[WIDE_LOAD]] = load <4 x i16>, <4 x i16>* [[TMP5]], align 2 2674; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2675; CHECK-NEXT: [[TMP7:%.*]] = sext <4 x i16> [[TMP6]] to <4 x i32> 2676; CHECK-NEXT: [[TMP8:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i32> 2677; CHECK-NEXT: [[TMP9:%.*]] = mul nsw <4 x i32> [[TMP8]], [[TMP7]] 2678; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]] 2679; CHECK-NEXT: [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <4 x i32>* 2680; CHECK-NEXT: store <4 x i32> [[TMP9]], <4 x i32>* [[TMP11]], align 4 2681; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 2682; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 2683; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]] 2684; CHECK: middle.block: 2685; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] 2686; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 3 2687; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2688; CHECK: scalar.ph: 2689; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 2690; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ] 2691; CHECK-NEXT: br label [[FOR_BODY:%.*]] 2692; CHECK: for.body: 2693; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP13:%.*]], [[FOR_BODY]] ] 2694; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 2695; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 2696; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 2697; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[INDVARS_IV_NEXT]] 2698; CHECK-NEXT: [[TMP13]] = load i16, i16* [[ARRAYIDX2]], align 2 2699; CHECK-NEXT: [[CONV3:%.*]] = sext i16 [[TMP13]] to i32 2700; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV3]], [[CONV]] 2701; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 2702; CHECK-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 2703; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 2704; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]] 2705; CHECK: for.end: 2706; CHECK-NEXT: ret void 2707; 2708; UNROLL-LABEL: @sink_after( 2709; UNROLL-NEXT: entry: 2710; UNROLL-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[A:%.*]], align 2 2711; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 8 2712; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 2713; UNROLL: vector.memcheck: 2714; UNROLL-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B:%.*]], i64 [[N]] 2715; UNROLL-NEXT: [[SCEVGEP3:%.*]] = getelementptr i16, i16* [[A]], i64 1 2716; UNROLL-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 2717; UNROLL-NEXT: [[SCEVGEP5:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP0]] 2718; UNROLL-NEXT: [[TMP1:%.*]] = bitcast i16* [[SCEVGEP5]] to i32* 2719; UNROLL-NEXT: [[BOUND0:%.*]] = icmp ugt i32* [[TMP1]], [[B]] 2720; UNROLL-NEXT: [[TMP2:%.*]] = bitcast i32* [[SCEVGEP]] to i16* 2721; UNROLL-NEXT: [[BOUND1:%.*]] = icmp ult i16* [[SCEVGEP3]], [[TMP2]] 2722; UNROLL-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 2723; UNROLL-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 2724; UNROLL: vector.ph: 2725; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -8 2726; UNROLL-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i64 3 2727; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 2728; UNROLL: vector.body: 2729; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2730; UNROLL-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD7:%.*]], [[VECTOR_BODY]] ] 2731; UNROLL-NEXT: [[TMP3:%.*]] = or i64 [[INDEX]], 1 2732; UNROLL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP3]] 2733; UNROLL-NEXT: [[TMP5:%.*]] = bitcast i16* [[TMP4]] to <4 x i16>* 2734; UNROLL-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, <4 x i16>* [[TMP5]], align 2 2735; UNROLL-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, i16* [[TMP4]], i64 4 2736; UNROLL-NEXT: [[TMP7:%.*]] = bitcast i16* [[TMP6]] to <4 x i16>* 2737; UNROLL-NEXT: [[WIDE_LOAD7]] = load <4 x i16>, <4 x i16>* [[TMP7]], align 2 2738; UNROLL-NEXT: [[TMP8:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2739; UNROLL-NEXT: [[TMP9:%.*]] = shufflevector <4 x i16> [[WIDE_LOAD]], <4 x i16> [[WIDE_LOAD7]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2740; UNROLL-NEXT: [[TMP10:%.*]] = sext <4 x i16> [[TMP8]] to <4 x i32> 2741; UNROLL-NEXT: [[TMP11:%.*]] = sext <4 x i16> [[TMP9]] to <4 x i32> 2742; UNROLL-NEXT: [[TMP12:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i32> 2743; UNROLL-NEXT: [[TMP13:%.*]] = sext <4 x i16> [[WIDE_LOAD7]] to <4 x i32> 2744; UNROLL-NEXT: [[TMP14:%.*]] = mul nsw <4 x i32> [[TMP12]], [[TMP10]] 2745; UNROLL-NEXT: [[TMP15:%.*]] = mul nsw <4 x i32> [[TMP13]], [[TMP11]] 2746; UNROLL-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]] 2747; UNROLL-NEXT: [[TMP17:%.*]] = bitcast i32* [[TMP16]] to <4 x i32>* 2748; UNROLL-NEXT: store <4 x i32> [[TMP14]], <4 x i32>* [[TMP17]], align 4 2749; UNROLL-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[TMP16]], i64 4 2750; UNROLL-NEXT: [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <4 x i32>* 2751; UNROLL-NEXT: store <4 x i32> [[TMP15]], <4 x i32>* [[TMP19]], align 4 2752; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 2753; UNROLL-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 2754; UNROLL-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]] 2755; UNROLL: middle.block: 2756; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] 2757; UNROLL-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD7]], i64 3 2758; UNROLL-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2759; UNROLL: scalar.ph: 2760; UNROLL-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 2761; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ] 2762; UNROLL-NEXT: br label [[FOR_BODY:%.*]] 2763; UNROLL: for.body: 2764; UNROLL-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP21:%.*]], [[FOR_BODY]] ] 2765; UNROLL-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 2766; UNROLL-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 2767; UNROLL-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 2768; UNROLL-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[INDVARS_IV_NEXT]] 2769; UNROLL-NEXT: [[TMP21]] = load i16, i16* [[ARRAYIDX2]], align 2 2770; UNROLL-NEXT: [[CONV3:%.*]] = sext i16 [[TMP21]] to i32 2771; UNROLL-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV3]], [[CONV]] 2772; UNROLL-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 2773; UNROLL-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 2774; UNROLL-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 2775; UNROLL-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]] 2776; UNROLL: for.end: 2777; UNROLL-NEXT: ret void 2778; 2779; UNROLL-NO-IC-LABEL: @sink_after( 2780; UNROLL-NO-IC-NEXT: entry: 2781; UNROLL-NO-IC-NEXT: [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8* 2782; UNROLL-NO-IC-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[A:%.*]], align 2 2783; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 8 2784; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 2785; UNROLL-NO-IC: vector.memcheck: 2786; UNROLL-NO-IC-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[N]] 2787; UNROLL-NO-IC-NEXT: [[SCEVGEP2:%.*]] = bitcast i32* [[SCEVGEP]] to i8* 2788; UNROLL-NO-IC-NEXT: [[SCEVGEP3:%.*]] = getelementptr i16, i16* [[A]], i64 1 2789; UNROLL-NO-IC-NEXT: [[SCEVGEP34:%.*]] = bitcast i16* [[SCEVGEP3]] to i8* 2790; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 2791; UNROLL-NO-IC-NEXT: [[SCEVGEP5:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP0]] 2792; UNROLL-NO-IC-NEXT: [[SCEVGEP56:%.*]] = bitcast i16* [[SCEVGEP5]] to i8* 2793; UNROLL-NO-IC-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[B1]], [[SCEVGEP56]] 2794; UNROLL-NO-IC-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP34]], [[SCEVGEP2]] 2795; UNROLL-NO-IC-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 2796; UNROLL-NO-IC-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 2797; UNROLL-NO-IC: vector.ph: 2798; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8 2799; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 2800; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i32 3 2801; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 2802; UNROLL-NO-IC: vector.body: 2803; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2804; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD7:%.*]], [[VECTOR_BODY]] ] 2805; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 0 2806; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 4 2807; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP1]], 1 2808; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = add nuw nsw i64 [[TMP2]], 1 2809; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP3]] 2810; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP4]] 2811; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, i16* [[TMP5]], i32 0 2812; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = bitcast i16* [[TMP7]] to <4 x i16>* 2813; UNROLL-NO-IC-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, <4 x i16>* [[TMP8]], align 2 2814; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = getelementptr inbounds i16, i16* [[TMP5]], i32 4 2815; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = bitcast i16* [[TMP9]] to <4 x i16>* 2816; UNROLL-NO-IC-NEXT: [[WIDE_LOAD7]] = load <4 x i16>, <4 x i16>* [[TMP10]], align 2 2817; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2818; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = shufflevector <4 x i16> [[WIDE_LOAD]], <4 x i16> [[WIDE_LOAD7]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2819; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = sext <4 x i16> [[TMP11]] to <4 x i32> 2820; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = sext <4 x i16> [[TMP12]] to <4 x i32> 2821; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i32> 2822; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = sext <4 x i16> [[WIDE_LOAD7]] to <4 x i32> 2823; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = mul nsw <4 x i32> [[TMP15]], [[TMP13]] 2824; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = mul nsw <4 x i32> [[TMP16]], [[TMP14]] 2825; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP1]] 2826; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP2]] 2827; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[TMP19]], i32 0 2828; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = bitcast i32* [[TMP21]] to <4 x i32>* 2829; UNROLL-NO-IC-NEXT: store <4 x i32> [[TMP17]], <4 x i32>* [[TMP22]], align 4 2830; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[TMP19]], i32 4 2831; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <4 x i32>* 2832; UNROLL-NO-IC-NEXT: store <4 x i32> [[TMP18]], <4 x i32>* [[TMP24]], align 4 2833; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 2834; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 2835; UNROLL-NO-IC-NEXT: br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]] 2836; UNROLL-NO-IC: middle.block: 2837; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 2838; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD7]], i32 3 2839; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD7]], i32 2 2840; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2841; UNROLL-NO-IC: scalar.ph: 2842; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 2843; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ] 2844; UNROLL-NO-IC-NEXT: br label [[FOR_BODY:%.*]] 2845; UNROLL-NO-IC: for.body: 2846; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP26:%.*]], [[FOR_BODY]] ] 2847; UNROLL-NO-IC-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 2848; UNROLL-NO-IC-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 2849; UNROLL-NO-IC-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 2850; UNROLL-NO-IC-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[INDVARS_IV_NEXT]] 2851; UNROLL-NO-IC-NEXT: [[TMP26]] = load i16, i16* [[ARRAYIDX2]], align 2 2852; UNROLL-NO-IC-NEXT: [[CONV3:%.*]] = sext i16 [[TMP26]] to i32 2853; UNROLL-NO-IC-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV3]], [[CONV]] 2854; UNROLL-NO-IC-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 2855; UNROLL-NO-IC-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 2856; UNROLL-NO-IC-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 2857; UNROLL-NO-IC-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]] 2858; UNROLL-NO-IC: for.end: 2859; UNROLL-NO-IC-NEXT: ret void 2860; 2861; UNROLL-NO-VF-LABEL: @sink_after( 2862; UNROLL-NO-VF-NEXT: entry: 2863; UNROLL-NO-VF-NEXT: [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8* 2864; UNROLL-NO-VF-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[A:%.*]], align 2 2865; UNROLL-NO-VF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 2 2866; UNROLL-NO-VF-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 2867; UNROLL-NO-VF: vector.memcheck: 2868; UNROLL-NO-VF-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[N]] 2869; UNROLL-NO-VF-NEXT: [[SCEVGEP2:%.*]] = bitcast i32* [[SCEVGEP]] to i8* 2870; UNROLL-NO-VF-NEXT: [[SCEVGEP3:%.*]] = getelementptr i16, i16* [[A]], i64 1 2871; UNROLL-NO-VF-NEXT: [[SCEVGEP34:%.*]] = bitcast i16* [[SCEVGEP3]] to i8* 2872; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 2873; UNROLL-NO-VF-NEXT: [[SCEVGEP5:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP0]] 2874; UNROLL-NO-VF-NEXT: [[SCEVGEP56:%.*]] = bitcast i16* [[SCEVGEP5]] to i8* 2875; UNROLL-NO-VF-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[B1]], [[SCEVGEP56]] 2876; UNROLL-NO-VF-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP34]], [[SCEVGEP2]] 2877; UNROLL-NO-VF-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 2878; UNROLL-NO-VF-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 2879; UNROLL-NO-VF: vector.ph: 2880; UNROLL-NO-VF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2 2881; UNROLL-NO-VF-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 2882; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 2883; UNROLL-NO-VF: vector.body: 2884; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2885; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ] 2886; UNROLL-NO-VF-NEXT: [[INDUCTION:%.*]] = add i64 [[INDEX]], 0 2887; UNROLL-NO-VF-NEXT: [[INDUCTION7:%.*]] = add i64 [[INDEX]], 1 2888; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[INDUCTION]], 1 2889; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[INDUCTION7]], 1 2890; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP1]] 2891; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP2]] 2892; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = load i16, i16* [[TMP3]], align 2 2893; UNROLL-NO-VF-NEXT: [[TMP6]] = load i16, i16* [[TMP4]], align 2 2894; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = sext i16 [[VECTOR_RECUR]] to i32 2895; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = sext i16 [[TMP5]] to i32 2896; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = sext i16 [[TMP5]] to i32 2897; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = sext i16 [[TMP6]] to i32 2898; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = mul nsw i32 [[TMP9]], [[TMP7]] 2899; UNROLL-NO-VF-NEXT: [[TMP12:%.*]] = mul nsw i32 [[TMP10]], [[TMP8]] 2900; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDUCTION]] 2901; UNROLL-NO-VF-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDUCTION7]] 2902; UNROLL-NO-VF-NEXT: store i32 [[TMP11]], i32* [[TMP13]], align 4 2903; UNROLL-NO-VF-NEXT: store i32 [[TMP12]], i32* [[TMP14]], align 4 2904; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 2905; UNROLL-NO-VF-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 2906; UNROLL-NO-VF-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]] 2907; UNROLL-NO-VF: middle.block: 2908; UNROLL-NO-VF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 2909; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2910; UNROLL-NO-VF: scalar.ph: 2911; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ] 2912; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ] 2913; UNROLL-NO-VF-NEXT: br label [[FOR_BODY:%.*]] 2914; UNROLL-NO-VF: for.body: 2915; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP16:%.*]], [[FOR_BODY]] ] 2916; UNROLL-NO-VF-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 2917; UNROLL-NO-VF-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 2918; UNROLL-NO-VF-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 2919; UNROLL-NO-VF-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[INDVARS_IV_NEXT]] 2920; UNROLL-NO-VF-NEXT: [[TMP16]] = load i16, i16* [[ARRAYIDX2]], align 2 2921; UNROLL-NO-VF-NEXT: [[CONV3:%.*]] = sext i16 [[TMP16]] to i32 2922; UNROLL-NO-VF-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV3]], [[CONV]] 2923; UNROLL-NO-VF-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 2924; UNROLL-NO-VF-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 2925; UNROLL-NO-VF-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 2926; UNROLL-NO-VF-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]] 2927; UNROLL-NO-VF: for.end: 2928; UNROLL-NO-VF-NEXT: ret void 2929; 2930; SINK-AFTER-LABEL: @sink_after( 2931; SINK-AFTER-NEXT: entry: 2932; SINK-AFTER-NEXT: [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8* 2933; SINK-AFTER-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[A:%.*]], align 2 2934; SINK-AFTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 2935; SINK-AFTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 2936; SINK-AFTER: vector.memcheck: 2937; SINK-AFTER-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[N]] 2938; SINK-AFTER-NEXT: [[SCEVGEP2:%.*]] = bitcast i32* [[SCEVGEP]] to i8* 2939; SINK-AFTER-NEXT: [[SCEVGEP3:%.*]] = getelementptr i16, i16* [[A]], i64 1 2940; SINK-AFTER-NEXT: [[SCEVGEP34:%.*]] = bitcast i16* [[SCEVGEP3]] to i8* 2941; SINK-AFTER-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 2942; SINK-AFTER-NEXT: [[SCEVGEP5:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP0]] 2943; SINK-AFTER-NEXT: [[SCEVGEP56:%.*]] = bitcast i16* [[SCEVGEP5]] to i8* 2944; SINK-AFTER-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[B1]], [[SCEVGEP56]] 2945; SINK-AFTER-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP34]], [[SCEVGEP2]] 2946; SINK-AFTER-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 2947; SINK-AFTER-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 2948; SINK-AFTER: vector.ph: 2949; SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4 2950; SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 2951; SINK-AFTER-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i32 3 2952; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 2953; SINK-AFTER: vector.body: 2954; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2955; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ] 2956; SINK-AFTER-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 0 2957; SINK-AFTER-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 2958; SINK-AFTER-NEXT: [[TMP3:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP2]] 2959; SINK-AFTER-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, i16* [[TMP3]], i32 0 2960; SINK-AFTER-NEXT: [[TMP5:%.*]] = bitcast i16* [[TMP4]] to <4 x i16>* 2961; SINK-AFTER-NEXT: [[WIDE_LOAD]] = load <4 x i16>, <4 x i16>* [[TMP5]], align 2 2962; SINK-AFTER-NEXT: [[TMP6:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 2963; SINK-AFTER-NEXT: [[TMP7:%.*]] = sext <4 x i16> [[TMP6]] to <4 x i32> 2964; SINK-AFTER-NEXT: [[TMP8:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i32> 2965; SINK-AFTER-NEXT: [[TMP9:%.*]] = mul nsw <4 x i32> [[TMP8]], [[TMP7]] 2966; SINK-AFTER-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP1]] 2967; SINK-AFTER-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP10]], i32 0 2968; SINK-AFTER-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>* 2969; SINK-AFTER-NEXT: store <4 x i32> [[TMP9]], <4 x i32>* [[TMP12]], align 4 2970; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 2971; SINK-AFTER-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 2972; SINK-AFTER-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]] 2973; SINK-AFTER: middle.block: 2974; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 2975; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3 2976; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2 2977; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2978; SINK-AFTER: scalar.ph: 2979; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 2980; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ] 2981; SINK-AFTER-NEXT: br label [[FOR_BODY:%.*]] 2982; SINK-AFTER: for.body: 2983; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP14:%.*]], [[FOR_BODY]] ] 2984; SINK-AFTER-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 2985; SINK-AFTER-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 2986; SINK-AFTER-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 2987; SINK-AFTER-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[INDVARS_IV_NEXT]] 2988; SINK-AFTER-NEXT: [[TMP14]] = load i16, i16* [[ARRAYIDX2]], align 2 2989; SINK-AFTER-NEXT: [[CONV3:%.*]] = sext i16 [[TMP14]] to i32 2990; SINK-AFTER-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV3]], [[CONV]] 2991; SINK-AFTER-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 2992; SINK-AFTER-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 2993; SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 2994; SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]] 2995; SINK-AFTER: for.end: 2996; SINK-AFTER-NEXT: ret void 2997; 2998entry: 2999 %.pre = load i16, i16* %a 3000 br label %for.body 3001 3002for.body: 3003 %0 = phi i16 [ %.pre, %entry ], [ %1, %for.body ] 3004 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 3005 %conv = sext i16 %0 to i32 3006 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 3007 %arrayidx2 = getelementptr inbounds i16, i16* %a, i64 %indvars.iv.next 3008 %1 = load i16, i16* %arrayidx2 3009 %conv3 = sext i16 %1 to i32 3010 %mul = mul nsw i32 %conv3, %conv 3011 %arrayidx5 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv 3012 store i32 %mul, i32* %arrayidx5 3013 %exitcond = icmp eq i64 %indvars.iv.next, %n 3014 br i1 %exitcond, label %for.end, label %for.body 3015 3016for.end: 3017 ret void 3018} 3019 3020; PR34711: given three consecutive instructions such that the first will be 3021; widened, the second is a cast that will be widened and needs to sink after the 3022; third, and the third is a first-order-recurring load that will be replicated 3023; instead of widened. Although the cast and the first instruction will both be 3024; widened, and are originally adjacent to each other, make sure the replicated 3025; load ends up appearing between them. 3026; 3027; void PR34711(short[2] *a, int *b, int *c, int n) { 3028; for(int i = 0; i < n; i++) { 3029; c[i] = 7; 3030; b[i] = (a[i][0] * a[i][1]); 3031; } 3032; } 3033; 3034; Check that the sext sank after the load in the vector loop. 3035; 3036define void @PR34711([2 x i16]* %a, i32* %b, i32* %c, i64 %n) { 3037; CHECK-LABEL: @PR34711( 3038; CHECK-NEXT: entry: 3039; CHECK-NEXT: [[PRE_INDEX:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A:%.*]], i64 0, i64 0 3040; CHECK-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[PRE_INDEX]], align 2 3041; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 3042; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 3043; CHECK: vector.memcheck: 3044; CHECK-NEXT: [[B1:%.*]] = ptrtoint i32* [[B:%.*]] to i64 3045; CHECK-NEXT: [[C2:%.*]] = ptrtoint i32* [[C:%.*]] to i64 3046; CHECK-NEXT: [[A3:%.*]] = ptrtoint [2 x i16]* [[A]] to i64 3047; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[B1]], [[C2]] 3048; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 16 3049; CHECK-NEXT: [[TMP1:%.*]] = add nuw i64 [[A3]], 2 3050; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[TMP1]], [[C2]] 3051; CHECK-NEXT: [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP2]], 16 3052; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]] 3053; CHECK-NEXT: [[TMP3:%.*]] = sub i64 [[B1]], [[TMP1]] 3054; CHECK-NEXT: [[DIFF_CHECK5:%.*]] = icmp ult i64 [[TMP3]], 16 3055; CHECK-NEXT: [[CONFLICT_RDX6:%.*]] = or i1 [[CONFLICT_RDX]], [[DIFF_CHECK5]] 3056; CHECK-NEXT: br i1 [[CONFLICT_RDX6]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3057; CHECK: vector.ph: 3058; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4 3059; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i64 3 3060; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 3061; CHECK: vector.body: 3062; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3063; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ] 3064; CHECK-NEXT: [[TMP4:%.*]] = or i64 [[INDEX]], 1 3065; CHECK-NEXT: [[TMP5:%.*]] = or i64 [[INDEX]], 2 3066; CHECK-NEXT: [[TMP6:%.*]] = or i64 [[INDEX]], 3 3067; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDEX]] 3068; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[INDEX]], i64 1 3069; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP4]], i64 1 3070; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP5]], i64 1 3071; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP6]], i64 1 3072; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP7]] to <4 x i32>* 3073; CHECK-NEXT: store <4 x i32> <i32 7, i32 7, i32 7, i32 7>, <4 x i32>* [[TMP12]], align 4 3074; CHECK-NEXT: [[TMP13:%.*]] = load i16, i16* [[TMP8]], align 2 3075; CHECK-NEXT: [[TMP14:%.*]] = load i16, i16* [[TMP9]], align 2 3076; CHECK-NEXT: [[TMP15:%.*]] = load i16, i16* [[TMP10]], align 2 3077; CHECK-NEXT: [[TMP16:%.*]] = load i16, i16* [[TMP11]], align 2 3078; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i16> poison, i16 [[TMP13]], i64 0 3079; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i16> [[TMP17]], i16 [[TMP14]], i64 1 3080; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i16> [[TMP18]], i16 [[TMP15]], i64 2 3081; CHECK-NEXT: [[TMP20]] = insertelement <4 x i16> [[TMP19]], i16 [[TMP16]], i64 3 3082; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP19]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 3083; CHECK-NEXT: [[TMP22:%.*]] = sext <4 x i16> [[TMP21]] to <4 x i32> 3084; CHECK-NEXT: [[TMP23:%.*]] = sext <4 x i16> [[TMP20]] to <4 x i32> 3085; CHECK-NEXT: [[TMP24:%.*]] = mul nsw <4 x i32> [[TMP23]], [[TMP22]] 3086; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]] 3087; CHECK-NEXT: [[TMP26:%.*]] = bitcast i32* [[TMP25]] to <4 x i32>* 3088; CHECK-NEXT: store <4 x i32> [[TMP24]], <4 x i32>* [[TMP26]], align 4 3089; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 3090; CHECK-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 3091; CHECK-NEXT: br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]] 3092; CHECK: middle.block: 3093; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] 3094; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 3095; CHECK: scalar.ph: 3096; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[TMP16]], [[MIDDLE_BLOCK]] ] 3097; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ] 3098; CHECK-NEXT: br label [[FOR_BODY:%.*]] 3099; CHECK: for.body: 3100; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP28:%.*]], [[FOR_BODY]] ] 3101; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 3102; CHECK-NEXT: [[ARRAYCIDX:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDVARS_IV]] 3103; CHECK-NEXT: [[CUR_INDEX:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[INDVARS_IV]], i64 1 3104; CHECK-NEXT: store i32 7, i32* [[ARRAYCIDX]], align 4 3105; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 3106; CHECK-NEXT: [[TMP28]] = load i16, i16* [[CUR_INDEX]], align 2 3107; CHECK-NEXT: [[CONV3:%.*]] = sext i16 [[TMP28]] to i32 3108; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV3]], [[CONV]] 3109; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 3110; CHECK-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 3111; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 3112; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 3113; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]] 3114; CHECK: for.end: 3115; CHECK-NEXT: ret void 3116; 3117; UNROLL-LABEL: @PR34711( 3118; UNROLL-NEXT: entry: 3119; UNROLL-NEXT: [[PRE_INDEX:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A:%.*]], i64 0, i64 0 3120; UNROLL-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[PRE_INDEX]], align 2 3121; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 8 3122; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 3123; UNROLL: vector.memcheck: 3124; UNROLL-NEXT: [[B1:%.*]] = ptrtoint i32* [[B:%.*]] to i64 3125; UNROLL-NEXT: [[C2:%.*]] = ptrtoint i32* [[C:%.*]] to i64 3126; UNROLL-NEXT: [[A3:%.*]] = ptrtoint [2 x i16]* [[A]] to i64 3127; UNROLL-NEXT: [[TMP0:%.*]] = sub i64 [[B1]], [[C2]] 3128; UNROLL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 32 3129; UNROLL-NEXT: [[TMP1:%.*]] = add nuw i64 [[A3]], 2 3130; UNROLL-NEXT: [[TMP2:%.*]] = sub i64 [[TMP1]], [[C2]] 3131; UNROLL-NEXT: [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP2]], 32 3132; UNROLL-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]] 3133; UNROLL-NEXT: [[TMP3:%.*]] = sub i64 [[B1]], [[TMP1]] 3134; UNROLL-NEXT: [[DIFF_CHECK5:%.*]] = icmp ult i64 [[TMP3]], 32 3135; UNROLL-NEXT: [[CONFLICT_RDX6:%.*]] = or i1 [[CONFLICT_RDX]], [[DIFF_CHECK5]] 3136 3137; UNROLL: vector.ph: 3138; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -8 3139; UNROLL-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i64 3 3140; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 3141; UNROLL: vector.body: 3142; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3143; UNROLL-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP38:%.*]], [[VECTOR_BODY]] ] 3144; UNROLL-NEXT: [[TMP4:%.*]] = or i64 [[INDEX]], 1 3145; UNROLL-NEXT: [[TMP5:%.*]] = or i64 [[INDEX]], 2 3146; UNROLL-NEXT: [[TMP6:%.*]] = or i64 [[INDEX]], 3 3147; UNROLL-NEXT: [[TMP7:%.*]] = or i64 [[INDEX]], 4 3148; UNROLL-NEXT: [[TMP8:%.*]] = or i64 [[INDEX]], 5 3149; UNROLL-NEXT: [[TMP9:%.*]] = or i64 [[INDEX]], 6 3150; UNROLL-NEXT: [[TMP10:%.*]] = or i64 [[INDEX]], 7 3151; UNROLL-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDEX]] 3152; UNROLL-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[INDEX]], i64 1 3153; UNROLL-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP4]], i64 1 3154; UNROLL-NEXT: [[TMP14:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP5]], i64 1 3155; UNROLL-NEXT: [[TMP15:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP6]], i64 1 3156; UNROLL-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP7]], i64 1 3157; UNROLL-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP8]], i64 1 3158; UNROLL-NEXT: [[TMP18:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP9]], i64 1 3159; UNROLL-NEXT: [[TMP19:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP10]], i64 1 3160; UNROLL-NEXT: [[TMP20:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>* 3161; UNROLL-NEXT: store <4 x i32> <i32 7, i32 7, i32 7, i32 7>, <4 x i32>* [[TMP20]], align 4 3162; UNROLL-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[TMP11]], i64 4 3163; UNROLL-NEXT: [[TMP22:%.*]] = bitcast i32* [[TMP21]] to <4 x i32>* 3164; UNROLL-NEXT: store <4 x i32> <i32 7, i32 7, i32 7, i32 7>, <4 x i32>* [[TMP22]], align 4 3165; UNROLL-NEXT: [[TMP23:%.*]] = load i16, i16* [[TMP12]], align 2 3166; UNROLL-NEXT: [[TMP24:%.*]] = load i16, i16* [[TMP13]], align 2 3167; UNROLL-NEXT: [[TMP25:%.*]] = load i16, i16* [[TMP14]], align 2 3168; UNROLL-NEXT: [[TMP26:%.*]] = load i16, i16* [[TMP15]], align 2 3169; UNROLL-NEXT: [[TMP27:%.*]] = insertelement <4 x i16> poison, i16 [[TMP23]], i64 0 3170; UNROLL-NEXT: [[TMP28:%.*]] = insertelement <4 x i16> [[TMP27]], i16 [[TMP24]], i64 1 3171; UNROLL-NEXT: [[TMP29:%.*]] = insertelement <4 x i16> [[TMP28]], i16 [[TMP25]], i64 2 3172; UNROLL-NEXT: [[TMP30:%.*]] = insertelement <4 x i16> [[TMP29]], i16 [[TMP26]], i64 3 3173; UNROLL-NEXT: [[TMP31:%.*]] = load i16, i16* [[TMP16]], align 2 3174; UNROLL-NEXT: [[TMP32:%.*]] = load i16, i16* [[TMP17]], align 2 3175; UNROLL-NEXT: [[TMP33:%.*]] = load i16, i16* [[TMP18]], align 2 3176; UNROLL-NEXT: [[TMP34:%.*]] = load i16, i16* [[TMP19]], align 2 3177; UNROLL-NEXT: [[TMP35:%.*]] = insertelement <4 x i16> poison, i16 [[TMP31]], i64 0 3178; UNROLL-NEXT: [[TMP36:%.*]] = insertelement <4 x i16> [[TMP35]], i16 [[TMP32]], i64 1 3179; UNROLL-NEXT: [[TMP37:%.*]] = insertelement <4 x i16> [[TMP36]], i16 [[TMP33]], i64 2 3180; UNROLL-NEXT: [[TMP38]] = insertelement <4 x i16> [[TMP37]], i16 [[TMP34]], i64 3 3181; UNROLL-NEXT: [[TMP39:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP29]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 3182; UNROLL-NEXT: [[TMP40:%.*]] = shufflevector <4 x i16> [[TMP30]], <4 x i16> [[TMP37]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 3183; UNROLL-NEXT: [[TMP41:%.*]] = sext <4 x i16> [[TMP39]] to <4 x i32> 3184; UNROLL-NEXT: [[TMP42:%.*]] = sext <4 x i16> [[TMP40]] to <4 x i32> 3185; UNROLL-NEXT: [[TMP43:%.*]] = sext <4 x i16> [[TMP30]] to <4 x i32> 3186; UNROLL-NEXT: [[TMP44:%.*]] = sext <4 x i16> [[TMP38]] to <4 x i32> 3187; UNROLL-NEXT: [[TMP45:%.*]] = mul nsw <4 x i32> [[TMP43]], [[TMP41]] 3188; UNROLL-NEXT: [[TMP46:%.*]] = mul nsw <4 x i32> [[TMP44]], [[TMP42]] 3189; UNROLL-NEXT: [[TMP47:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]] 3190; UNROLL-NEXT: [[TMP48:%.*]] = bitcast i32* [[TMP47]] to <4 x i32>* 3191; UNROLL-NEXT: store <4 x i32> [[TMP45]], <4 x i32>* [[TMP48]], align 4 3192; UNROLL-NEXT: [[TMP49:%.*]] = getelementptr inbounds i32, i32* [[TMP47]], i64 4 3193; UNROLL-NEXT: [[TMP50:%.*]] = bitcast i32* [[TMP49]] to <4 x i32>* 3194; UNROLL-NEXT: store <4 x i32> [[TMP46]], <4 x i32>* [[TMP50]], align 4 3195; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 3196; UNROLL-NEXT: [[TMP51:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 3197; UNROLL-NEXT: br i1 [[TMP51]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]] 3198; UNROLL: middle.block: 3199; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] 3200; UNROLL-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 3201; UNROLL: scalar.ph: 3202; UNROLL-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[TMP34]], [[MIDDLE_BLOCK]] ] 3203; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ] 3204; UNROLL-NEXT: br label [[FOR_BODY:%.*]] 3205; UNROLL: for.body: 3206; UNROLL-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP52:%.*]], [[FOR_BODY]] ] 3207; UNROLL-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 3208; UNROLL-NEXT: [[ARRAYCIDX:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDVARS_IV]] 3209; UNROLL-NEXT: [[CUR_INDEX:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[INDVARS_IV]], i64 1 3210; UNROLL-NEXT: store i32 7, i32* [[ARRAYCIDX]], align 4 3211; UNROLL-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 3212; UNROLL-NEXT: [[TMP52]] = load i16, i16* [[CUR_INDEX]], align 2 3213; UNROLL-NEXT: [[CONV3:%.*]] = sext i16 [[TMP52]] to i32 3214; UNROLL-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV3]], [[CONV]] 3215; UNROLL-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 3216; UNROLL-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 3217; UNROLL-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 3218; UNROLL-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 3219; UNROLL-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]] 3220; UNROLL: for.end: 3221; UNROLL-NEXT: ret void 3222; 3223; UNROLL-NO-IC-LABEL: @PR34711( 3224; UNROLL-NO-IC-NEXT: entry: 3225; UNROLL-NO-IC-NEXT: [[A3:%.*]] = ptrtoint [2 x i16]* [[A:%.*]] to i64 3226; UNROLL-NO-IC-NEXT: [[C2:%.*]] = ptrtoint i32* [[C:%.*]] to i64 3227; UNROLL-NO-IC-NEXT: [[B1:%.*]] = ptrtoint i32* [[B:%.*]] to i64 3228; UNROLL-NO-IC-NEXT: [[PRE_INDEX:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A:%.*]], i64 0, i64 0 3229; UNROLL-NO-IC-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[PRE_INDEX]], align 2 3230; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 8 3231; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 3232; UNROLL-NO-IC: vector.memcheck: 3233; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = sub i64 [[B1]], [[C2]] 3234; UNROLL-NO-IC-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 32 3235; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = add nuw i64 [[A3]], 2 3236; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = sub i64 [[TMP1]], [[C2]] 3237; UNROLL-NO-IC-NEXT: [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP2]], 32 3238; UNROLL-NO-IC-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]] 3239; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = sub i64 [[B1]], [[TMP1]] 3240; UNROLL-NO-IC-NEXT: [[DIFF_CHECK5:%.*]] = icmp ult i64 [[TMP3]], 32 3241; UNROLL-NO-IC-NEXT: [[CONFLICT_RDX6:%.*]] = or i1 [[CONFLICT_RDX]], [[DIFF_CHECK5]] 3242; UNROLL-NO-IC-NEXT: br i1 [[CONFLICT_RDX6]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3243; UNROLL-NO-IC: vector.ph: 3244; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8 3245; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 3246; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i32 3 3247; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 3248; UNROLL-NO-IC: vector.body: 3249; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3250; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP37:%.*]], [[VECTOR_BODY]] ] 3251; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 3252; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1 3253; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2 3254; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3 3255; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 4 3256; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 5 3257; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 6 3258; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 7 3259; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[TMP0]] 3260; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[TMP4]] 3261; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP0]], i64 1 3262; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP1]], i64 1 3263; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP2]], i64 1 3264; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP3]], i64 1 3265; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP4]], i64 1 3266; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP5]], i64 1 3267; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP6]], i64 1 3268; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP7]], i64 1 3269; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[TMP8]], i32 0 3270; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <4 x i32>* 3271; UNROLL-NO-IC-NEXT: store <4 x i32> <i32 7, i32 7, i32 7, i32 7>, <4 x i32>* [[TMP19]], align 4 3272; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[TMP8]], i32 4 3273; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = bitcast i32* [[TMP20]] to <4 x i32>* 3274; UNROLL-NO-IC-NEXT: store <4 x i32> <i32 7, i32 7, i32 7, i32 7>, <4 x i32>* [[TMP21]], align 4 3275; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = load i16, i16* [[TMP10]], align 2 3276; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = load i16, i16* [[TMP11]], align 2 3277; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = load i16, i16* [[TMP12]], align 2 3278; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = load i16, i16* [[TMP13]], align 2 3279; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = insertelement <4 x i16> poison, i16 [[TMP22]], i32 0 3280; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = insertelement <4 x i16> [[TMP26]], i16 [[TMP23]], i32 1 3281; UNROLL-NO-IC-NEXT: [[TMP28:%.*]] = insertelement <4 x i16> [[TMP27]], i16 [[TMP24]], i32 2 3282; UNROLL-NO-IC-NEXT: [[TMP29:%.*]] = insertelement <4 x i16> [[TMP28]], i16 [[TMP25]], i32 3 3283; UNROLL-NO-IC-NEXT: [[TMP30:%.*]] = load i16, i16* [[TMP14]], align 2 3284; UNROLL-NO-IC-NEXT: [[TMP31:%.*]] = load i16, i16* [[TMP15]], align 2 3285; UNROLL-NO-IC-NEXT: [[TMP32:%.*]] = load i16, i16* [[TMP16]], align 2 3286; UNROLL-NO-IC-NEXT: [[TMP33:%.*]] = load i16, i16* [[TMP17]], align 2 3287; UNROLL-NO-IC-NEXT: [[TMP34:%.*]] = insertelement <4 x i16> poison, i16 [[TMP30]], i32 0 3288; UNROLL-NO-IC-NEXT: [[TMP35:%.*]] = insertelement <4 x i16> [[TMP34]], i16 [[TMP31]], i32 1 3289; UNROLL-NO-IC-NEXT: [[TMP36:%.*]] = insertelement <4 x i16> [[TMP35]], i16 [[TMP32]], i32 2 3290; UNROLL-NO-IC-NEXT: [[TMP37]] = insertelement <4 x i16> [[TMP36]], i16 [[TMP33]], i32 3 3291; UNROLL-NO-IC-NEXT: [[TMP38:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP29]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 3292; UNROLL-NO-IC-NEXT: [[TMP39:%.*]] = shufflevector <4 x i16> [[TMP29]], <4 x i16> [[TMP37]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 3293; UNROLL-NO-IC-NEXT: [[TMP40:%.*]] = sext <4 x i16> [[TMP38]] to <4 x i32> 3294; UNROLL-NO-IC-NEXT: [[TMP41:%.*]] = sext <4 x i16> [[TMP39]] to <4 x i32> 3295; UNROLL-NO-IC-NEXT: [[TMP42:%.*]] = sext <4 x i16> [[TMP29]] to <4 x i32> 3296; UNROLL-NO-IC-NEXT: [[TMP43:%.*]] = sext <4 x i16> [[TMP37]] to <4 x i32> 3297; UNROLL-NO-IC-NEXT: [[TMP44:%.*]] = mul nsw <4 x i32> [[TMP42]], [[TMP40]] 3298; UNROLL-NO-IC-NEXT: [[TMP45:%.*]] = mul nsw <4 x i32> [[TMP43]], [[TMP41]] 3299; UNROLL-NO-IC-NEXT: [[TMP46:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP0]] 3300; UNROLL-NO-IC-NEXT: [[TMP47:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP4]] 3301; UNROLL-NO-IC-NEXT: [[TMP48:%.*]] = getelementptr inbounds i32, i32* [[TMP46]], i32 0 3302; UNROLL-NO-IC-NEXT: [[TMP49:%.*]] = bitcast i32* [[TMP48]] to <4 x i32>* 3303; UNROLL-NO-IC-NEXT: store <4 x i32> [[TMP44]], <4 x i32>* [[TMP49]], align 4 3304; UNROLL-NO-IC-NEXT: [[TMP50:%.*]] = getelementptr inbounds i32, i32* [[TMP46]], i32 4 3305; UNROLL-NO-IC-NEXT: [[TMP51:%.*]] = bitcast i32* [[TMP50]] to <4 x i32>* 3306; UNROLL-NO-IC-NEXT: store <4 x i32> [[TMP45]], <4 x i32>* [[TMP51]], align 4 3307; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 3308; UNROLL-NO-IC-NEXT: [[TMP52:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 3309; UNROLL-NO-IC-NEXT: br i1 [[TMP52]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]] 3310; UNROLL-NO-IC: middle.block: 3311; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 3312; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[TMP37]], i32 3 3313; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[TMP37]], i32 2 3314; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 3315; UNROLL-NO-IC: scalar.ph: 3316; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 3317; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ] 3318; UNROLL-NO-IC-NEXT: br label [[FOR_BODY:%.*]] 3319; UNROLL-NO-IC: for.body: 3320; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP53:%.*]], [[FOR_BODY]] ] 3321; UNROLL-NO-IC-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 3322; UNROLL-NO-IC-NEXT: [[ARRAYCIDX:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDVARS_IV]] 3323; UNROLL-NO-IC-NEXT: [[CUR_INDEX:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[INDVARS_IV]], i64 1 3324; UNROLL-NO-IC-NEXT: store i32 7, i32* [[ARRAYCIDX]], align 4 3325; UNROLL-NO-IC-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 3326; UNROLL-NO-IC-NEXT: [[TMP53]] = load i16, i16* [[CUR_INDEX]], align 2 3327; UNROLL-NO-IC-NEXT: [[CONV3:%.*]] = sext i16 [[TMP53]] to i32 3328; UNROLL-NO-IC-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV3]], [[CONV]] 3329; UNROLL-NO-IC-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 3330; UNROLL-NO-IC-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 3331; UNROLL-NO-IC-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 3332; UNROLL-NO-IC-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 3333; UNROLL-NO-IC-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]] 3334; UNROLL-NO-IC: for.end: 3335; UNROLL-NO-IC-NEXT: ret void 3336; 3337; UNROLL-NO-VF-LABEL: @PR34711( 3338; UNROLL-NO-VF-NEXT: entry: 3339; UNROLL-NO-VF-NEXT: [[A3:%.*]] = ptrtoint [2 x i16]* [[A:%.*]] to i64 3340; UNROLL-NO-VF-NEXT: [[C2:%.*]] = ptrtoint i32* [[C:%.*]] to i64 3341; UNROLL-NO-VF-NEXT: [[B1:%.*]] = ptrtoint i32* [[B:%.*]] to i64 3342; UNROLL-NO-VF-NEXT: [[PRE_INDEX:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A:%.*]], i64 0, i64 0 3343; UNROLL-NO-VF-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[PRE_INDEX]], align 2 3344; UNROLL-NO-VF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 2 3345; UNROLL-NO-VF-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 3346; UNROLL-NO-VF: vector.memcheck: 3347; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = sub i64 [[B1]], [[C2]] 3348; UNROLL-NO-VF-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8 3349; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = add nuw i64 [[A3]], 2 3350; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = sub i64 [[TMP1]], [[C2]] 3351; UNROLL-NO-VF-NEXT: [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP2]], 8 3352; UNROLL-NO-VF-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]] 3353; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = sub i64 [[B1]], [[TMP1]] 3354; UNROLL-NO-VF-NEXT: [[DIFF_CHECK5:%.*]] = icmp ult i64 [[TMP3]], 8 3355; UNROLL-NO-VF-NEXT: [[CONFLICT_RDX6:%.*]] = or i1 [[CONFLICT_RDX]], [[DIFF_CHECK5]] 3356; UNROLL-NO-VF-NEXT: br i1 [[CONFLICT_RDX6]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3357; UNROLL-NO-VF: vector.ph: 3358; UNROLL-NO-VF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2 3359; UNROLL-NO-VF-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 3360; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 3361; UNROLL-NO-VF: vector.body: 3362; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3363; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ] 3364; UNROLL-NO-VF-NEXT: [[INDUCTION:%.*]] = add i64 [[INDEX]], 0 3365; UNROLL-NO-VF-NEXT: [[INDUCTION17:%.*]] = add i64 [[INDEX]], 1 3366; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDUCTION]] 3367; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDUCTION17]] 3368; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[INDUCTION]], i64 1 3369; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[INDUCTION17]], i64 1 3370; UNROLL-NO-VF-NEXT: store i32 7, i32* [[TMP0]], align 4 3371; UNROLL-NO-VF-NEXT: store i32 7, i32* [[TMP1]], align 4 3372; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = load i16, i16* [[TMP2]], align 2 3373; UNROLL-NO-VF-NEXT: [[TMP5]] = load i16, i16* [[TMP3]], align 2 3374; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = sext i16 [[VECTOR_RECUR]] to i32 3375; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = sext i16 [[TMP4]] to i32 3376; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = sext i16 [[TMP4]] to i32 3377; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = sext i16 [[TMP5]] to i32 3378; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = mul nsw i32 [[TMP8]], [[TMP6]] 3379; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = mul nsw i32 [[TMP9]], [[TMP7]] 3380; UNROLL-NO-VF-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDUCTION]] 3381; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDUCTION17]] 3382; UNROLL-NO-VF-NEXT: store i32 [[TMP10]], i32* [[TMP12]], align 4 3383; UNROLL-NO-VF-NEXT: store i32 [[TMP11]], i32* [[TMP13]], align 4 3384; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 3385; UNROLL-NO-VF-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 3386; UNROLL-NO-VF-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]] 3387; UNROLL-NO-VF: middle.block: 3388; UNROLL-NO-VF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 3389; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 3390; UNROLL-NO-VF: scalar.ph: 3391; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ] 3392; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ] 3393; UNROLL-NO-VF-NEXT: br label [[FOR_BODY:%.*]] 3394; UNROLL-NO-VF: for.body: 3395; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP15:%.*]], [[FOR_BODY]] ] 3396; UNROLL-NO-VF-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 3397; UNROLL-NO-VF-NEXT: [[ARRAYCIDX:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDVARS_IV]] 3398; UNROLL-NO-VF-NEXT: [[CUR_INDEX:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[INDVARS_IV]], i64 1 3399; UNROLL-NO-VF-NEXT: store i32 7, i32* [[ARRAYCIDX]], align 4 3400; UNROLL-NO-VF-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 3401; UNROLL-NO-VF-NEXT: [[TMP15]] = load i16, i16* [[CUR_INDEX]], align 2 3402; UNROLL-NO-VF-NEXT: [[CONV3:%.*]] = sext i16 [[TMP15]] to i32 3403; UNROLL-NO-VF-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV3]], [[CONV]] 3404; UNROLL-NO-VF-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 3405; UNROLL-NO-VF-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 3406; UNROLL-NO-VF-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 3407; UNROLL-NO-VF-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 3408; UNROLL-NO-VF-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]] 3409; UNROLL-NO-VF: for.end: 3410; UNROLL-NO-VF-NEXT: ret void 3411; 3412; SINK-AFTER-LABEL: @PR34711( 3413; SINK-AFTER-NEXT: entry: 3414; SINK-AFTER-NEXT: [[A3:%.*]] = ptrtoint [2 x i16]* [[A:%.*]] to i64 3415; SINK-AFTER-NEXT: [[C2:%.*]] = ptrtoint i32* [[C:%.*]] to i64 3416; SINK-AFTER-NEXT: [[B1:%.*]] = ptrtoint i32* [[B:%.*]] to i64 3417; SINK-AFTER-NEXT: [[PRE_INDEX:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A:%.*]], i64 0, i64 0 3418; SINK-AFTER-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[PRE_INDEX]], align 2 3419; SINK-AFTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 3420; SINK-AFTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 3421; SINK-AFTER: vector.memcheck: 3422; SINK-AFTER-NEXT: [[TMP0:%.*]] = sub i64 [[B1]], [[C2]] 3423; SINK-AFTER-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 16 3424; SINK-AFTER-NEXT: [[TMP1:%.*]] = add nuw i64 [[A3]], 2 3425; SINK-AFTER-NEXT: [[TMP2:%.*]] = sub i64 [[TMP1]], [[C2]] 3426; SINK-AFTER-NEXT: [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP2]], 16 3427; SINK-AFTER-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]] 3428; SINK-AFTER-NEXT: [[TMP3:%.*]] = sub i64 [[B1]], [[TMP1]] 3429; SINK-AFTER-NEXT: [[DIFF_CHECK5:%.*]] = icmp ult i64 [[TMP3]], 16 3430; SINK-AFTER-NEXT: [[CONFLICT_RDX6:%.*]] = or i1 [[CONFLICT_RDX]], [[DIFF_CHECK5]] 3431; SINK-AFTER-NEXT: br i1 [[CONFLICT_RDX6]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3432; SINK-AFTER: vector.ph: 3433; SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4 3434; SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 3435; SINK-AFTER-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i32 3 3436; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 3437; SINK-AFTER: vector.body: 3438; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3439; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[VECTOR_BODY]] ] 3440; SINK-AFTER-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 3441; SINK-AFTER-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1 3442; SINK-AFTER-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2 3443; SINK-AFTER-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3 3444; SINK-AFTER-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[TMP0]] 3445; SINK-AFTER-NEXT: [[TMP5:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP0]], i64 1 3446; SINK-AFTER-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP1]], i64 1 3447; SINK-AFTER-NEXT: [[TMP7:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP2]], i64 1 3448; SINK-AFTER-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[TMP3]], i64 1 3449; SINK-AFTER-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 0 3450; SINK-AFTER-NEXT: [[TMP10:%.*]] = bitcast i32* [[TMP9]] to <4 x i32>* 3451; SINK-AFTER-NEXT: store <4 x i32> <i32 7, i32 7, i32 7, i32 7>, <4 x i32>* [[TMP10]], align 4 3452; SINK-AFTER-NEXT: [[TMP11:%.*]] = load i16, i16* [[TMP5]], align 2 3453; SINK-AFTER-NEXT: [[TMP12:%.*]] = load i16, i16* [[TMP6]], align 2 3454; SINK-AFTER-NEXT: [[TMP13:%.*]] = load i16, i16* [[TMP7]], align 2 3455; SINK-AFTER-NEXT: [[TMP14:%.*]] = load i16, i16* [[TMP8]], align 2 3456; SINK-AFTER-NEXT: [[TMP15:%.*]] = insertelement <4 x i16> poison, i16 [[TMP11]], i32 0 3457; SINK-AFTER-NEXT: [[TMP16:%.*]] = insertelement <4 x i16> [[TMP15]], i16 [[TMP12]], i32 1 3458; SINK-AFTER-NEXT: [[TMP17:%.*]] = insertelement <4 x i16> [[TMP16]], i16 [[TMP13]], i32 2 3459; SINK-AFTER-NEXT: [[TMP18]] = insertelement <4 x i16> [[TMP17]], i16 [[TMP14]], i32 3 3460; SINK-AFTER-NEXT: [[TMP19:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP18]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 3461; SINK-AFTER-NEXT: [[TMP20:%.*]] = sext <4 x i16> [[TMP19]] to <4 x i32> 3462; SINK-AFTER-NEXT: [[TMP21:%.*]] = sext <4 x i16> [[TMP18]] to <4 x i32> 3463; SINK-AFTER-NEXT: [[TMP22:%.*]] = mul nsw <4 x i32> [[TMP21]], [[TMP20]] 3464; SINK-AFTER-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP0]] 3465; SINK-AFTER-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, i32* [[TMP23]], i32 0 3466; SINK-AFTER-NEXT: [[TMP25:%.*]] = bitcast i32* [[TMP24]] to <4 x i32>* 3467; SINK-AFTER-NEXT: store <4 x i32> [[TMP22]], <4 x i32>* [[TMP25]], align 4 3468; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 3469; SINK-AFTER-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 3470; SINK-AFTER-NEXT: br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]] 3471; SINK-AFTER: middle.block: 3472; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 3473; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[TMP18]], i32 3 3474; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[TMP18]], i32 2 3475; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 3476; SINK-AFTER: scalar.ph: 3477; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 3478; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ] 3479; SINK-AFTER-NEXT: br label [[FOR_BODY:%.*]] 3480; SINK-AFTER: for.body: 3481; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP27:%.*]], [[FOR_BODY]] ] 3482; SINK-AFTER-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 3483; SINK-AFTER-NEXT: [[ARRAYCIDX:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDVARS_IV]] 3484; SINK-AFTER-NEXT: [[CUR_INDEX:%.*]] = getelementptr inbounds [2 x i16], [2 x i16]* [[A]], i64 [[INDVARS_IV]], i64 1 3485; SINK-AFTER-NEXT: store i32 7, i32* [[ARRAYCIDX]], align 4 3486; SINK-AFTER-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 3487; SINK-AFTER-NEXT: [[TMP27]] = load i16, i16* [[CUR_INDEX]], align 2 3488; SINK-AFTER-NEXT: [[CONV3:%.*]] = sext i16 [[TMP27]] to i32 3489; SINK-AFTER-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV3]], [[CONV]] 3490; SINK-AFTER-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 3491; SINK-AFTER-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 3492; SINK-AFTER-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 3493; SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 3494; SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]] 3495; SINK-AFTER: for.end: 3496; SINK-AFTER-NEXT: ret void 3497; 3498entry: 3499 %pre.index = getelementptr inbounds [2 x i16], [2 x i16]* %a, i64 0, i64 0 3500 %.pre = load i16, i16* %pre.index 3501 br label %for.body 3502 3503for.body: 3504 %0 = phi i16 [ %.pre, %entry ], [ %1, %for.body ] 3505 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 3506 %arraycidx = getelementptr inbounds i32, i32* %c, i64 %indvars.iv 3507 %cur.index = getelementptr inbounds [2 x i16], [2 x i16]* %a, i64 %indvars.iv, i64 1 3508 store i32 7, i32* %arraycidx ; 1st instruction, to be widened. 3509 %conv = sext i16 %0 to i32 ; 2nd, cast to sink after third. 3510 %1 = load i16, i16* %cur.index ; 3rd, first-order-recurring load not widened. 3511 %conv3 = sext i16 %1 to i32 3512 %mul = mul nsw i32 %conv3, %conv 3513 %arrayidx5 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv 3514 store i32 %mul, i32* %arrayidx5 3515 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 3516 %exitcond = icmp eq i64 %indvars.iv.next, %n 3517 br i1 %exitcond, label %for.end, label %for.body 3518 3519for.end: 3520 ret void 3521} 3522 3523; void no_sink_after(short *a, int n, int *b) { 3524; for(int i = 0; i < n; i++) 3525; b[i] = ((a[i] + 2) * a[i + 1]); 3526; } 3527; 3528 3529; 3530define void @sink_after_with_multiple_users(i16* %a, i32* %b, i64 %n) { 3531; CHECK-LABEL: @sink_after_with_multiple_users( 3532; CHECK-NEXT: entry: 3533; CHECK-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[A:%.*]], align 2 3534; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 3535; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 3536; CHECK: vector.memcheck: 3537; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B:%.*]], i64 [[N]] 3538; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i16, i16* [[A]], i64 1 3539; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 3540; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP0]] 3541; CHECK-NEXT: [[TMP1:%.*]] = bitcast i16* [[SCEVGEP5]] to i32* 3542; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt i32* [[TMP1]], [[B]] 3543; CHECK-NEXT: [[TMP2:%.*]] = bitcast i32* [[SCEVGEP]] to i16* 3544; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult i16* [[SCEVGEP3]], [[TMP2]] 3545; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 3546; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3547; CHECK: vector.ph: 3548; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4 3549; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i64 3 3550; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 3551; CHECK: vector.body: 3552; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3553; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ] 3554; CHECK-NEXT: [[TMP3:%.*]] = or i64 [[INDEX]], 1 3555; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP3]] 3556; CHECK-NEXT: [[TMP5:%.*]] = bitcast i16* [[TMP4]] to <4 x i16>* 3557; CHECK-NEXT: [[WIDE_LOAD]] = load <4 x i16>, <4 x i16>* [[TMP5]], align 2 3558; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 3559; CHECK-NEXT: [[TMP7:%.*]] = sext <4 x i16> [[TMP6]] to <4 x i32> 3560; CHECK-NEXT: [[TMP8:%.*]] = add nsw <4 x i32> [[TMP7]], <i32 2, i32 2, i32 2, i32 2> 3561; CHECK-NEXT: [[TMP9:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i32> 3562; CHECK-NEXT: [[TMP10:%.*]] = mul nsw <4 x i32> [[TMP8]], [[TMP9]] 3563; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]] 3564; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>* 3565; CHECK-NEXT: store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4 3566; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 3567; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 3568; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]] 3569; CHECK: middle.block: 3570; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] 3571; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 3 3572; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 3573; CHECK: scalar.ph: 3574; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 3575; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ] 3576; CHECK-NEXT: br label [[FOR_BODY:%.*]] 3577; CHECK: for.body: 3578; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP14:%.*]], [[FOR_BODY]] ] 3579; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 3580; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 3581; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[CONV]], 2 3582; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 3583; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[INDVARS_IV_NEXT]] 3584; CHECK-NEXT: [[TMP14]] = load i16, i16* [[ARRAYIDX2]], align 2 3585; CHECK-NEXT: [[CONV3:%.*]] = sext i16 [[TMP14]] to i32 3586; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[ADD]], [[CONV3]] 3587; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 3588; CHECK-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 3589; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 3590; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]] 3591; CHECK: for.end: 3592; CHECK-NEXT: ret void 3593; 3594; UNROLL-LABEL: @sink_after_with_multiple_users( 3595; UNROLL-NEXT: entry: 3596; UNROLL-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[A:%.*]], align 2 3597; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 8 3598; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 3599; UNROLL: vector.memcheck: 3600; UNROLL-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B:%.*]], i64 [[N]] 3601; UNROLL-NEXT: [[SCEVGEP3:%.*]] = getelementptr i16, i16* [[A]], i64 1 3602; UNROLL-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 3603; UNROLL-NEXT: [[SCEVGEP5:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP0]] 3604; UNROLL-NEXT: [[TMP1:%.*]] = bitcast i16* [[SCEVGEP5]] to i32* 3605; UNROLL-NEXT: [[BOUND0:%.*]] = icmp ugt i32* [[TMP1]], [[B]] 3606; UNROLL-NEXT: [[TMP2:%.*]] = bitcast i32* [[SCEVGEP]] to i16* 3607; UNROLL-NEXT: [[BOUND1:%.*]] = icmp ult i16* [[SCEVGEP3]], [[TMP2]] 3608; UNROLL-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 3609; UNROLL-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3610; UNROLL: vector.ph: 3611; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -8 3612; UNROLL-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i64 3 3613; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 3614; UNROLL: vector.body: 3615; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3616; UNROLL-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD7:%.*]], [[VECTOR_BODY]] ] 3617; UNROLL-NEXT: [[TMP3:%.*]] = or i64 [[INDEX]], 1 3618; UNROLL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP3]] 3619; UNROLL-NEXT: [[TMP5:%.*]] = bitcast i16* [[TMP4]] to <4 x i16>* 3620; UNROLL-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, <4 x i16>* [[TMP5]], align 2 3621; UNROLL-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, i16* [[TMP4]], i64 4 3622; UNROLL-NEXT: [[TMP7:%.*]] = bitcast i16* [[TMP6]] to <4 x i16>* 3623; UNROLL-NEXT: [[WIDE_LOAD7]] = load <4 x i16>, <4 x i16>* [[TMP7]], align 2 3624; UNROLL-NEXT: [[TMP8:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 3625; UNROLL-NEXT: [[TMP9:%.*]] = shufflevector <4 x i16> [[WIDE_LOAD]], <4 x i16> [[WIDE_LOAD7]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 3626; UNROLL-NEXT: [[TMP10:%.*]] = sext <4 x i16> [[TMP8]] to <4 x i32> 3627; UNROLL-NEXT: [[TMP11:%.*]] = sext <4 x i16> [[TMP9]] to <4 x i32> 3628; UNROLL-NEXT: [[TMP12:%.*]] = add nsw <4 x i32> [[TMP10]], <i32 2, i32 2, i32 2, i32 2> 3629; UNROLL-NEXT: [[TMP13:%.*]] = add nsw <4 x i32> [[TMP11]], <i32 2, i32 2, i32 2, i32 2> 3630; UNROLL-NEXT: [[TMP14:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i32> 3631; UNROLL-NEXT: [[TMP15:%.*]] = sext <4 x i16> [[WIDE_LOAD7]] to <4 x i32> 3632; UNROLL-NEXT: [[TMP16:%.*]] = mul nsw <4 x i32> [[TMP12]], [[TMP14]] 3633; UNROLL-NEXT: [[TMP17:%.*]] = mul nsw <4 x i32> [[TMP13]], [[TMP15]] 3634; UNROLL-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]] 3635; UNROLL-NEXT: [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <4 x i32>* 3636; UNROLL-NEXT: store <4 x i32> [[TMP16]], <4 x i32>* [[TMP19]], align 4 3637; UNROLL-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[TMP18]], i64 4 3638; UNROLL-NEXT: [[TMP21:%.*]] = bitcast i32* [[TMP20]] to <4 x i32>* 3639; UNROLL-NEXT: store <4 x i32> [[TMP17]], <4 x i32>* [[TMP21]], align 4 3640; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 3641; UNROLL-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 3642; UNROLL-NEXT: br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]] 3643; UNROLL: middle.block: 3644; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] 3645; UNROLL-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD7]], i64 3 3646; UNROLL-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 3647; UNROLL: scalar.ph: 3648; UNROLL-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 3649; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ] 3650; UNROLL-NEXT: br label [[FOR_BODY:%.*]] 3651; UNROLL: for.body: 3652; UNROLL-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP23:%.*]], [[FOR_BODY]] ] 3653; UNROLL-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 3654; UNROLL-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 3655; UNROLL-NEXT: [[ADD:%.*]] = add nsw i32 [[CONV]], 2 3656; UNROLL-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 3657; UNROLL-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[INDVARS_IV_NEXT]] 3658; UNROLL-NEXT: [[TMP23]] = load i16, i16* [[ARRAYIDX2]], align 2 3659; UNROLL-NEXT: [[CONV3:%.*]] = sext i16 [[TMP23]] to i32 3660; UNROLL-NEXT: [[MUL:%.*]] = mul nsw i32 [[ADD]], [[CONV3]] 3661; UNROLL-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 3662; UNROLL-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 3663; UNROLL-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 3664; UNROLL-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]] 3665; UNROLL: for.end: 3666; UNROLL-NEXT: ret void 3667; 3668; UNROLL-NO-IC-LABEL: @sink_after_with_multiple_users( 3669; UNROLL-NO-IC-NEXT: entry: 3670; UNROLL-NO-IC-NEXT: [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8* 3671; UNROLL-NO-IC-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[A:%.*]], align 2 3672; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 8 3673; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 3674; UNROLL-NO-IC: vector.memcheck: 3675; UNROLL-NO-IC-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[N]] 3676; UNROLL-NO-IC-NEXT: [[SCEVGEP2:%.*]] = bitcast i32* [[SCEVGEP]] to i8* 3677; UNROLL-NO-IC-NEXT: [[SCEVGEP3:%.*]] = getelementptr i16, i16* [[A]], i64 1 3678; UNROLL-NO-IC-NEXT: [[SCEVGEP34:%.*]] = bitcast i16* [[SCEVGEP3]] to i8* 3679; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 3680; UNROLL-NO-IC-NEXT: [[SCEVGEP5:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP0]] 3681; UNROLL-NO-IC-NEXT: [[SCEVGEP56:%.*]] = bitcast i16* [[SCEVGEP5]] to i8* 3682; UNROLL-NO-IC-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[B1]], [[SCEVGEP56]] 3683; UNROLL-NO-IC-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP34]], [[SCEVGEP2]] 3684; UNROLL-NO-IC-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 3685; UNROLL-NO-IC-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3686; UNROLL-NO-IC: vector.ph: 3687; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8 3688; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 3689; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i32 3 3690; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 3691; UNROLL-NO-IC: vector.body: 3692; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3693; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD7:%.*]], [[VECTOR_BODY]] ] 3694; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 0 3695; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 4 3696; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP1]], 1 3697; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = add nuw nsw i64 [[TMP2]], 1 3698; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP3]] 3699; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP4]] 3700; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, i16* [[TMP5]], i32 0 3701; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = bitcast i16* [[TMP7]] to <4 x i16>* 3702; UNROLL-NO-IC-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, <4 x i16>* [[TMP8]], align 2 3703; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = getelementptr inbounds i16, i16* [[TMP5]], i32 4 3704; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = bitcast i16* [[TMP9]] to <4 x i16>* 3705; UNROLL-NO-IC-NEXT: [[WIDE_LOAD7]] = load <4 x i16>, <4 x i16>* [[TMP10]], align 2 3706; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 3707; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = shufflevector <4 x i16> [[WIDE_LOAD]], <4 x i16> [[WIDE_LOAD7]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 3708; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = sext <4 x i16> [[TMP11]] to <4 x i32> 3709; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = sext <4 x i16> [[TMP12]] to <4 x i32> 3710; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = add nsw <4 x i32> [[TMP13]], <i32 2, i32 2, i32 2, i32 2> 3711; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = add nsw <4 x i32> [[TMP14]], <i32 2, i32 2, i32 2, i32 2> 3712; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i32> 3713; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = sext <4 x i16> [[WIDE_LOAD7]] to <4 x i32> 3714; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = mul nsw <4 x i32> [[TMP15]], [[TMP17]] 3715; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = mul nsw <4 x i32> [[TMP16]], [[TMP18]] 3716; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP1]] 3717; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP2]] 3718; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[TMP21]], i32 0 3719; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <4 x i32>* 3720; UNROLL-NO-IC-NEXT: store <4 x i32> [[TMP19]], <4 x i32>* [[TMP24]], align 4 3721; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, i32* [[TMP21]], i32 4 3722; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = bitcast i32* [[TMP25]] to <4 x i32>* 3723; UNROLL-NO-IC-NEXT: store <4 x i32> [[TMP20]], <4 x i32>* [[TMP26]], align 4 3724; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 3725; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 3726; UNROLL-NO-IC-NEXT: br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]] 3727; UNROLL-NO-IC: middle.block: 3728; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 3729; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD7]], i32 3 3730; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD7]], i32 2 3731; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 3732; UNROLL-NO-IC: scalar.ph: 3733; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 3734; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ] 3735; UNROLL-NO-IC-NEXT: br label [[FOR_BODY:%.*]] 3736; UNROLL-NO-IC: for.body: 3737; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP28:%.*]], [[FOR_BODY]] ] 3738; UNROLL-NO-IC-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 3739; UNROLL-NO-IC-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 3740; UNROLL-NO-IC-NEXT: [[ADD:%.*]] = add nsw i32 [[CONV]], 2 3741; UNROLL-NO-IC-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 3742; UNROLL-NO-IC-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[INDVARS_IV_NEXT]] 3743; UNROLL-NO-IC-NEXT: [[TMP28]] = load i16, i16* [[ARRAYIDX2]], align 2 3744; UNROLL-NO-IC-NEXT: [[CONV3:%.*]] = sext i16 [[TMP28]] to i32 3745; UNROLL-NO-IC-NEXT: [[MUL:%.*]] = mul nsw i32 [[ADD]], [[CONV3]] 3746; UNROLL-NO-IC-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 3747; UNROLL-NO-IC-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 3748; UNROLL-NO-IC-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 3749; UNROLL-NO-IC-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]] 3750; UNROLL-NO-IC: for.end: 3751; UNROLL-NO-IC-NEXT: ret void 3752; 3753; UNROLL-NO-VF-LABEL: @sink_after_with_multiple_users( 3754; UNROLL-NO-VF-NEXT: entry: 3755; UNROLL-NO-VF-NEXT: [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8* 3756; UNROLL-NO-VF-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[A:%.*]], align 2 3757; UNROLL-NO-VF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 2 3758; UNROLL-NO-VF-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 3759; UNROLL-NO-VF: vector.memcheck: 3760; UNROLL-NO-VF-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[N]] 3761; UNROLL-NO-VF-NEXT: [[SCEVGEP2:%.*]] = bitcast i32* [[SCEVGEP]] to i8* 3762; UNROLL-NO-VF-NEXT: [[SCEVGEP3:%.*]] = getelementptr i16, i16* [[A]], i64 1 3763; UNROLL-NO-VF-NEXT: [[SCEVGEP34:%.*]] = bitcast i16* [[SCEVGEP3]] to i8* 3764; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 3765; UNROLL-NO-VF-NEXT: [[SCEVGEP5:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP0]] 3766; UNROLL-NO-VF-NEXT: [[SCEVGEP56:%.*]] = bitcast i16* [[SCEVGEP5]] to i8* 3767; UNROLL-NO-VF-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[B1]], [[SCEVGEP56]] 3768; UNROLL-NO-VF-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP34]], [[SCEVGEP2]] 3769; UNROLL-NO-VF-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 3770; UNROLL-NO-VF-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3771; UNROLL-NO-VF: vector.ph: 3772; UNROLL-NO-VF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2 3773; UNROLL-NO-VF-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 3774; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 3775; UNROLL-NO-VF: vector.body: 3776; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3777; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ] 3778; UNROLL-NO-VF-NEXT: [[INDUCTION:%.*]] = add i64 [[INDEX]], 0 3779; UNROLL-NO-VF-NEXT: [[INDUCTION7:%.*]] = add i64 [[INDEX]], 1 3780; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[INDUCTION]], 1 3781; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[INDUCTION7]], 1 3782; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP1]] 3783; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP2]] 3784; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = load i16, i16* [[TMP3]], align 2 3785; UNROLL-NO-VF-NEXT: [[TMP6]] = load i16, i16* [[TMP4]], align 2 3786; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = sext i16 [[VECTOR_RECUR]] to i32 3787; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = sext i16 [[TMP5]] to i32 3788; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = add nsw i32 [[TMP7]], 2 3789; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = add nsw i32 [[TMP8]], 2 3790; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = sext i16 [[TMP5]] to i32 3791; UNROLL-NO-VF-NEXT: [[TMP12:%.*]] = sext i16 [[TMP6]] to i32 3792; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = mul nsw i32 [[TMP9]], [[TMP11]] 3793; UNROLL-NO-VF-NEXT: [[TMP14:%.*]] = mul nsw i32 [[TMP10]], [[TMP12]] 3794; UNROLL-NO-VF-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDUCTION]] 3795; UNROLL-NO-VF-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDUCTION7]] 3796; UNROLL-NO-VF-NEXT: store i32 [[TMP13]], i32* [[TMP15]], align 4 3797; UNROLL-NO-VF-NEXT: store i32 [[TMP14]], i32* [[TMP16]], align 4 3798; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 3799; UNROLL-NO-VF-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 3800; UNROLL-NO-VF-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP47:![0-9]+]] 3801; UNROLL-NO-VF: middle.block: 3802; UNROLL-NO-VF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 3803; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 3804; UNROLL-NO-VF: scalar.ph: 3805; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ] 3806; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ] 3807; UNROLL-NO-VF-NEXT: br label [[FOR_BODY:%.*]] 3808; UNROLL-NO-VF: for.body: 3809; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP18:%.*]], [[FOR_BODY]] ] 3810; UNROLL-NO-VF-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 3811; UNROLL-NO-VF-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 3812; UNROLL-NO-VF-NEXT: [[ADD:%.*]] = add nsw i32 [[CONV]], 2 3813; UNROLL-NO-VF-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 3814; UNROLL-NO-VF-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[INDVARS_IV_NEXT]] 3815; UNROLL-NO-VF-NEXT: [[TMP18]] = load i16, i16* [[ARRAYIDX2]], align 2 3816; UNROLL-NO-VF-NEXT: [[CONV3:%.*]] = sext i16 [[TMP18]] to i32 3817; UNROLL-NO-VF-NEXT: [[MUL:%.*]] = mul nsw i32 [[ADD]], [[CONV3]] 3818; UNROLL-NO-VF-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 3819; UNROLL-NO-VF-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 3820; UNROLL-NO-VF-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 3821; UNROLL-NO-VF-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]] 3822; UNROLL-NO-VF: for.end: 3823; UNROLL-NO-VF-NEXT: ret void 3824; 3825; SINK-AFTER-LABEL: @sink_after_with_multiple_users( 3826; SINK-AFTER-NEXT: entry: 3827; SINK-AFTER-NEXT: [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8* 3828; SINK-AFTER-NEXT: [[DOTPRE:%.*]] = load i16, i16* [[A:%.*]], align 2 3829; SINK-AFTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 3830; SINK-AFTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 3831; SINK-AFTER: vector.memcheck: 3832; SINK-AFTER-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[N]] 3833; SINK-AFTER-NEXT: [[SCEVGEP2:%.*]] = bitcast i32* [[SCEVGEP]] to i8* 3834; SINK-AFTER-NEXT: [[SCEVGEP3:%.*]] = getelementptr i16, i16* [[A]], i64 1 3835; SINK-AFTER-NEXT: [[SCEVGEP34:%.*]] = bitcast i16* [[SCEVGEP3]] to i8* 3836; SINK-AFTER-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 3837; SINK-AFTER-NEXT: [[SCEVGEP5:%.*]] = getelementptr i16, i16* [[A]], i64 [[TMP0]] 3838; SINK-AFTER-NEXT: [[SCEVGEP56:%.*]] = bitcast i16* [[SCEVGEP5]] to i8* 3839; SINK-AFTER-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[B1]], [[SCEVGEP56]] 3840; SINK-AFTER-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP34]], [[SCEVGEP2]] 3841; SINK-AFTER-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 3842; SINK-AFTER-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3843; SINK-AFTER: vector.ph: 3844; SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4 3845; SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 3846; SINK-AFTER-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i32 3 3847; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 3848; SINK-AFTER: vector.body: 3849; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3850; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ] 3851; SINK-AFTER-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 0 3852; SINK-AFTER-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 3853; SINK-AFTER-NEXT: [[TMP3:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[TMP2]] 3854; SINK-AFTER-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, i16* [[TMP3]], i32 0 3855; SINK-AFTER-NEXT: [[TMP5:%.*]] = bitcast i16* [[TMP4]] to <4 x i16>* 3856; SINK-AFTER-NEXT: [[WIDE_LOAD]] = load <4 x i16>, <4 x i16>* [[TMP5]], align 2 3857; SINK-AFTER-NEXT: [[TMP6:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 3858; SINK-AFTER-NEXT: [[TMP7:%.*]] = sext <4 x i16> [[TMP6]] to <4 x i32> 3859; SINK-AFTER-NEXT: [[TMP8:%.*]] = add nsw <4 x i32> [[TMP7]], <i32 2, i32 2, i32 2, i32 2> 3860; SINK-AFTER-NEXT: [[TMP9:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i32> 3861; SINK-AFTER-NEXT: [[TMP10:%.*]] = mul nsw <4 x i32> [[TMP8]], [[TMP9]] 3862; SINK-AFTER-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP1]] 3863; SINK-AFTER-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TMP11]], i32 0 3864; SINK-AFTER-NEXT: [[TMP13:%.*]] = bitcast i32* [[TMP12]] to <4 x i32>* 3865; SINK-AFTER-NEXT: store <4 x i32> [[TMP10]], <4 x i32>* [[TMP13]], align 4 3866; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 3867; SINK-AFTER-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 3868; SINK-AFTER-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]] 3869; SINK-AFTER: middle.block: 3870; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 3871; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3 3872; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2 3873; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 3874; SINK-AFTER: scalar.ph: 3875; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 3876; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ] 3877; SINK-AFTER-NEXT: br label [[FOR_BODY:%.*]] 3878; SINK-AFTER: for.body: 3879; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP15:%.*]], [[FOR_BODY]] ] 3880; SINK-AFTER-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 3881; SINK-AFTER-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32 3882; SINK-AFTER-NEXT: [[ADD:%.*]] = add nsw i32 [[CONV]], 2 3883; SINK-AFTER-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 3884; SINK-AFTER-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 [[INDVARS_IV_NEXT]] 3885; SINK-AFTER-NEXT: [[TMP15]] = load i16, i16* [[ARRAYIDX2]], align 2 3886; SINK-AFTER-NEXT: [[CONV3:%.*]] = sext i16 [[TMP15]] to i32 3887; SINK-AFTER-NEXT: [[MUL:%.*]] = mul nsw i32 [[ADD]], [[CONV3]] 3888; SINK-AFTER-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]] 3889; SINK-AFTER-NEXT: store i32 [[MUL]], i32* [[ARRAYIDX5]], align 4 3890; SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 3891; SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]] 3892; SINK-AFTER: for.end: 3893; SINK-AFTER-NEXT: ret void 3894; 3895entry: 3896 %.pre = load i16, i16* %a 3897 br label %for.body 3898 3899for.body: 3900 %0 = phi i16 [ %.pre, %entry ], [ %1, %for.body ] 3901 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 3902 %conv = sext i16 %0 to i32 3903 %add = add nsw i32 %conv, 2 3904 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 3905 %arrayidx2 = getelementptr inbounds i16, i16* %a, i64 %indvars.iv.next 3906 %1 = load i16, i16* %arrayidx2 3907 %conv3 = sext i16 %1 to i32 3908 %mul = mul nsw i32 %add, %conv3 3909 %arrayidx5 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv 3910 store i32 %mul, i32* %arrayidx5 3911 %exitcond = icmp eq i64 %indvars.iv.next, %n 3912 br i1 %exitcond, label %for.end, label %for.body 3913 3914for.end: 3915 ret void 3916} 3917 3918; Do not sink branches: While branches are if-converted and do not require 3919; sinking, instructions with side effects (e.g. loads) conditioned by those 3920; branches will become users of the condition bit after vectorization and would 3921; need to be sunk if the loop is vectorized. 3922define void @do_not_sink_branch(i32 %x, i32* %in, i32* %out, i32 %tc) local_unnamed_addr #0 { 3923; CHECK-LABEL: @do_not_sink_branch( 3924; CHECK-NEXT: entry: 3925; CHECK-NEXT: [[CMP530:%.*]] = icmp sgt i32 [[TC:%.*]], 0 3926; CHECK-NEXT: br label [[FOR_BODY4:%.*]] 3927; CHECK: for.body4: 3928; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[COND_END:%.*]] ] 3929; CHECK-NEXT: [[CMP534:%.*]] = phi i1 [ [[CMP530]], [[ENTRY]] ], [ [[CMP5:%.*]], [[COND_END]] ] 3930; CHECK-NEXT: br i1 [[CMP534]], label [[COND_TRUE:%.*]], label [[COND_END]] 3931; CHECK: cond.true: 3932; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[INDVARS_IV]] to i64 3933; CHECK-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[IN:%.*]], i64 [[TMP0]] 3934; CHECK-NEXT: [[IN_VAL:%.*]] = load i32, i32* [[ARRAYIDX7]], align 4 3935; CHECK-NEXT: br label [[COND_END]] 3936; CHECK: cond.end: 3937; CHECK-NEXT: [[COND:%.*]] = phi i32 [ [[IN_VAL]], [[COND_TRUE]] ], [ 0, [[FOR_BODY4]] ] 3938; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[INDVARS_IV]] to i64 3939; CHECK-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, i32* [[OUT:%.*]], i64 [[TMP1]] 3940; CHECK-NEXT: store i32 [[COND]], i32* [[ARRAYIDX8]], align 4 3941; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[INDVARS_IV]], 1 3942; CHECK-NEXT: [[CMP5]] = icmp slt i32 [[INDVARS_IV_NEXT]], [[TC]] 3943; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], [[X:%.*]] 3944; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END12_LOOPEXIT:%.*]], label [[FOR_BODY4]] 3945; CHECK: for.end12.loopexit: 3946; CHECK-NEXT: ret void 3947; 3948; UNROLL-LABEL: @do_not_sink_branch( 3949; UNROLL-NEXT: entry: 3950; UNROLL-NEXT: [[CMP530:%.*]] = icmp sgt i32 [[TC:%.*]], 0 3951; UNROLL-NEXT: br label [[FOR_BODY4:%.*]] 3952; UNROLL: for.body4: 3953; UNROLL-NEXT: [[INDVARS_IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[COND_END:%.*]] ] 3954; UNROLL-NEXT: [[CMP534:%.*]] = phi i1 [ [[CMP530]], [[ENTRY]] ], [ [[CMP5:%.*]], [[COND_END]] ] 3955; UNROLL-NEXT: br i1 [[CMP534]], label [[COND_TRUE:%.*]], label [[COND_END]] 3956; UNROLL: cond.true: 3957; UNROLL-NEXT: [[TMP0:%.*]] = zext i32 [[INDVARS_IV]] to i64 3958; UNROLL-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[IN:%.*]], i64 [[TMP0]] 3959; UNROLL-NEXT: [[IN_VAL:%.*]] = load i32, i32* [[ARRAYIDX7]], align 4 3960; UNROLL-NEXT: br label [[COND_END]] 3961; UNROLL: cond.end: 3962; UNROLL-NEXT: [[COND:%.*]] = phi i32 [ [[IN_VAL]], [[COND_TRUE]] ], [ 0, [[FOR_BODY4]] ] 3963; UNROLL-NEXT: [[TMP1:%.*]] = zext i32 [[INDVARS_IV]] to i64 3964; UNROLL-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, i32* [[OUT:%.*]], i64 [[TMP1]] 3965; UNROLL-NEXT: store i32 [[COND]], i32* [[ARRAYIDX8]], align 4 3966; UNROLL-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[INDVARS_IV]], 1 3967; UNROLL-NEXT: [[CMP5]] = icmp slt i32 [[INDVARS_IV_NEXT]], [[TC]] 3968; UNROLL-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], [[X:%.*]] 3969; UNROLL-NEXT: br i1 [[EXITCOND]], label [[FOR_END12_LOOPEXIT:%.*]], label [[FOR_BODY4]] 3970; UNROLL: for.end12.loopexit: 3971; UNROLL-NEXT: ret void 3972; 3973; UNROLL-NO-IC-LABEL: @do_not_sink_branch( 3974; UNROLL-NO-IC-NEXT: entry: 3975; UNROLL-NO-IC-NEXT: [[CMP530:%.*]] = icmp slt i32 0, [[TC:%.*]] 3976; UNROLL-NO-IC-NEXT: br label [[FOR_BODY4:%.*]] 3977; UNROLL-NO-IC: for.body4: 3978; UNROLL-NO-IC-NEXT: [[INDVARS_IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[COND_END:%.*]] ] 3979; UNROLL-NO-IC-NEXT: [[CMP534:%.*]] = phi i1 [ [[CMP530]], [[ENTRY]] ], [ [[CMP5:%.*]], [[COND_END]] ] 3980; UNROLL-NO-IC-NEXT: br i1 [[CMP534]], label [[COND_TRUE:%.*]], label [[COND_END]] 3981; UNROLL-NO-IC: cond.true: 3982; UNROLL-NO-IC-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[IN:%.*]], i32 [[INDVARS_IV]] 3983; UNROLL-NO-IC-NEXT: [[IN_VAL:%.*]] = load i32, i32* [[ARRAYIDX7]], align 4 3984; UNROLL-NO-IC-NEXT: br label [[COND_END]] 3985; UNROLL-NO-IC: cond.end: 3986; UNROLL-NO-IC-NEXT: [[COND:%.*]] = phi i32 [ [[IN_VAL]], [[COND_TRUE]] ], [ 0, [[FOR_BODY4]] ] 3987; UNROLL-NO-IC-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, i32* [[OUT:%.*]], i32 [[INDVARS_IV]] 3988; UNROLL-NO-IC-NEXT: store i32 [[COND]], i32* [[ARRAYIDX8]], align 4 3989; UNROLL-NO-IC-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[INDVARS_IV]], 1 3990; UNROLL-NO-IC-NEXT: [[CMP5]] = icmp slt i32 [[INDVARS_IV_NEXT]], [[TC]] 3991; UNROLL-NO-IC-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], [[X:%.*]] 3992; UNROLL-NO-IC-NEXT: br i1 [[EXITCOND]], label [[FOR_END12_LOOPEXIT:%.*]], label [[FOR_BODY4]] 3993; UNROLL-NO-IC: for.end12.loopexit: 3994; UNROLL-NO-IC-NEXT: ret void 3995; 3996; UNROLL-NO-VF-LABEL: @do_not_sink_branch( 3997; UNROLL-NO-VF-NEXT: entry: 3998; UNROLL-NO-VF-NEXT: [[CMP530:%.*]] = icmp slt i32 0, [[TC:%.*]] 3999; UNROLL-NO-VF-NEXT: br label [[FOR_BODY4:%.*]] 4000; UNROLL-NO-VF: for.body4: 4001; UNROLL-NO-VF-NEXT: [[INDVARS_IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[COND_END:%.*]] ] 4002; UNROLL-NO-VF-NEXT: [[CMP534:%.*]] = phi i1 [ [[CMP530]], [[ENTRY]] ], [ [[CMP5:%.*]], [[COND_END]] ] 4003; UNROLL-NO-VF-NEXT: br i1 [[CMP534]], label [[COND_TRUE:%.*]], label [[COND_END]] 4004; UNROLL-NO-VF: cond.true: 4005; UNROLL-NO-VF-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[IN:%.*]], i32 [[INDVARS_IV]] 4006; UNROLL-NO-VF-NEXT: [[IN_VAL:%.*]] = load i32, i32* [[ARRAYIDX7]], align 4 4007; UNROLL-NO-VF-NEXT: br label [[COND_END]] 4008; UNROLL-NO-VF: cond.end: 4009; UNROLL-NO-VF-NEXT: [[COND:%.*]] = phi i32 [ [[IN_VAL]], [[COND_TRUE]] ], [ 0, [[FOR_BODY4]] ] 4010; UNROLL-NO-VF-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, i32* [[OUT:%.*]], i32 [[INDVARS_IV]] 4011; UNROLL-NO-VF-NEXT: store i32 [[COND]], i32* [[ARRAYIDX8]], align 4 4012; UNROLL-NO-VF-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[INDVARS_IV]], 1 4013; UNROLL-NO-VF-NEXT: [[CMP5]] = icmp slt i32 [[INDVARS_IV_NEXT]], [[TC]] 4014; UNROLL-NO-VF-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], [[X:%.*]] 4015; UNROLL-NO-VF-NEXT: br i1 [[EXITCOND]], label [[FOR_END12_LOOPEXIT:%.*]], label [[FOR_BODY4]] 4016; UNROLL-NO-VF: for.end12.loopexit: 4017; UNROLL-NO-VF-NEXT: ret void 4018; 4019; SINK-AFTER-LABEL: @do_not_sink_branch( 4020; SINK-AFTER-NEXT: entry: 4021; SINK-AFTER-NEXT: [[CMP530:%.*]] = icmp slt i32 0, [[TC:%.*]] 4022; SINK-AFTER-NEXT: br label [[FOR_BODY4:%.*]] 4023; SINK-AFTER: for.body4: 4024; SINK-AFTER-NEXT: [[INDVARS_IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[COND_END:%.*]] ] 4025; SINK-AFTER-NEXT: [[CMP534:%.*]] = phi i1 [ [[CMP530]], [[ENTRY]] ], [ [[CMP5:%.*]], [[COND_END]] ] 4026; SINK-AFTER-NEXT: br i1 [[CMP534]], label [[COND_TRUE:%.*]], label [[COND_END]] 4027; SINK-AFTER: cond.true: 4028; SINK-AFTER-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[IN:%.*]], i32 [[INDVARS_IV]] 4029; SINK-AFTER-NEXT: [[IN_VAL:%.*]] = load i32, i32* [[ARRAYIDX7]], align 4 4030; SINK-AFTER-NEXT: br label [[COND_END]] 4031; SINK-AFTER: cond.end: 4032; SINK-AFTER-NEXT: [[COND:%.*]] = phi i32 [ [[IN_VAL]], [[COND_TRUE]] ], [ 0, [[FOR_BODY4]] ] 4033; SINK-AFTER-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, i32* [[OUT:%.*]], i32 [[INDVARS_IV]] 4034; SINK-AFTER-NEXT: store i32 [[COND]], i32* [[ARRAYIDX8]], align 4 4035; SINK-AFTER-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[INDVARS_IV]], 1 4036; SINK-AFTER-NEXT: [[CMP5]] = icmp slt i32 [[INDVARS_IV_NEXT]], [[TC]] 4037; SINK-AFTER-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], [[X:%.*]] 4038; SINK-AFTER-NEXT: br i1 [[EXITCOND]], label [[FOR_END12_LOOPEXIT:%.*]], label [[FOR_BODY4]] 4039; SINK-AFTER: for.end12.loopexit: 4040; SINK-AFTER-NEXT: ret void 4041; 4042entry: 4043 %cmp530 = icmp slt i32 0, %tc 4044 br label %for.body4 4045 4046for.body4: ; preds = %cond.end, %entry 4047 %indvars.iv = phi i32 [ 0, %entry ], [ %indvars.iv.next, %cond.end ] 4048 %cmp534 = phi i1 [ %cmp530, %entry ], [ %cmp5, %cond.end ] 4049 br i1 %cmp534, label %cond.true, label %cond.end 4050 4051cond.true: ; preds = %for.body4 4052 %arrayidx7 = getelementptr inbounds i32, i32* %in, i32 %indvars.iv 4053 %in.val = load i32, i32* %arrayidx7, align 4 4054 br label %cond.end 4055 4056cond.end: ; preds = %for.body4, %cond.true 4057 %cond = phi i32 [ %in.val, %cond.true ], [ 0, %for.body4 ] 4058 %arrayidx8 = getelementptr inbounds i32, i32* %out, i32 %indvars.iv 4059 store i32 %cond, i32* %arrayidx8, align 4 4060 %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1 4061 %cmp5 = icmp slt i32 %indvars.iv.next, %tc 4062 %exitcond = icmp eq i32 %indvars.iv.next, %x 4063 br i1 %exitcond, label %for.end12.loopexit, label %for.body4 4064 4065for.end12.loopexit: ; preds = %cond.end 4066 ret void 4067} 4068 4069; Dead instructions, like the exit condition are not part of the actual VPlan 4070; and do not need to be sunk. PR44634. 4071define void @sink_dead_inst() { 4072; 4073; CHECK-LABEL: @sink_dead_inst( 4074; CHECK-NEXT: entry: 4075; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4076; CHECK: vector.ph: 4077; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 4078; CHECK: vector.body: 4079; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4080; CHECK-NEXT: [[TMP0:%.*]] = phi i16 [ -24, [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ] 4081; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 4082; CHECK-NEXT: [[TMP1]] = add i16 [[TMP0]], 4 4083; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 40 4084; CHECK-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP50:![0-9]+]] 4085; CHECK: middle.block: 4086; CHECK-NEXT: [[TMP3:%.*]] = or i16 [[TMP0]], 1 4087; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT2:%.*]] = zext i16 [[TMP3]] to i32 4088; CHECK-NEXT: br i1 false, label [[FOR_END:%.*]], label [[SCALAR_PH]] 4089; CHECK: scalar.ph: 4090; CHECK-NEXT: [[SCALAR_RECUR_INIT4:%.*]] = phi i32 [ -27, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT2]], [[MIDDLE_BLOCK]] ] 4091; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ -27, [[ENTRY]] ], [ 13, [[MIDDLE_BLOCK]] ] 4092; CHECK-NEXT: br label [[FOR_COND:%.*]] 4093; CHECK: for.cond: 4094; CHECK-NEXT: [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_COND]] ] 4095; CHECK-NEXT: [[SCALAR_RECUR5:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT4]], [[SCALAR_PH]] ], [ [[REC_2_PREV:%.*]], [[FOR_COND]] ] 4096; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[SCALAR_RECUR5]], 15 4097; CHECK-NEXT: [[IV_NEXT]] = add i16 [[IV]], 1 4098; CHECK-NEXT: [[REC_2_PREV]] = zext i16 [[IV_NEXT]] to i32 4099; CHECK-NEXT: br i1 [[CMP]], label [[FOR_END]], label [[FOR_COND]], !llvm.loop [[LOOP51:![0-9]+]] 4100; CHECK: for.end: 4101; CHECK-NEXT: ret void 4102; 4103; UNROLL-LABEL: @sink_dead_inst( 4104; UNROLL-NEXT: entry: 4105; UNROLL-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4106; UNROLL: vector.ph: 4107; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 4108; UNROLL: vector.body: 4109; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4110; UNROLL-NEXT: [[TMP0:%.*]] = phi i16 [ -24, [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ] 4111; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 4112; UNROLL-NEXT: [[TMP1]] = add i16 [[TMP0]], 8 4113; UNROLL-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 40 4114; UNROLL-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP50:![0-9]+]] 4115; UNROLL: middle.block: 4116; UNROLL-NEXT: [[TMP3:%.*]] = or i16 [[TMP0]], 5 4117; UNROLL-NEXT: [[VECTOR_RECUR_EXTRACT3:%.*]] = zext i16 [[TMP3]] to i32 4118; UNROLL-NEXT: br i1 false, label [[FOR_END:%.*]], label [[SCALAR_PH]] 4119; UNROLL: scalar.ph: 4120; UNROLL-NEXT: [[SCALAR_RECUR_INIT5:%.*]] = phi i32 [ -27, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT3]], [[MIDDLE_BLOCK]] ] 4121; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ -27, [[ENTRY]] ], [ 13, [[MIDDLE_BLOCK]] ] 4122; UNROLL-NEXT: br label [[FOR_COND:%.*]] 4123; UNROLL: for.cond: 4124; UNROLL-NEXT: [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_COND]] ] 4125; UNROLL-NEXT: [[SCALAR_RECUR6:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT5]], [[SCALAR_PH]] ], [ [[REC_2_PREV:%.*]], [[FOR_COND]] ] 4126; UNROLL-NEXT: [[CMP:%.*]] = icmp eq i32 [[SCALAR_RECUR6]], 15 4127; UNROLL-NEXT: [[IV_NEXT]] = add i16 [[IV]], 1 4128; UNROLL-NEXT: [[REC_2_PREV]] = zext i16 [[IV_NEXT]] to i32 4129; UNROLL-NEXT: br i1 [[CMP]], label [[FOR_END]], label [[FOR_COND]], !llvm.loop [[LOOP51:![0-9]+]] 4130; UNROLL: for.end: 4131; UNROLL-NEXT: ret void 4132; 4133; UNROLL-NO-IC-LABEL: @sink_dead_inst( 4134; UNROLL-NO-IC-NEXT: entry: 4135; UNROLL-NO-IC-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4136; UNROLL-NO-IC: vector.ph: 4137; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 4138; UNROLL-NO-IC: vector.body: 4139; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4140; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 -27, i16 -26, i16 -25, i16 -24>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 4141; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 0>, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ] 4142; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR2:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 -27>, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ] 4143; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add <4 x i16> [[VEC_IND]], <i16 4, i16 4, i16 4, i16 4> 4144; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add <4 x i16> [[VEC_IND]], <i16 1, i16 1, i16 1, i16 1> 4145; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = add <4 x i16> [[STEP_ADD]], <i16 1, i16 1, i16 1, i16 1> 4146; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = zext <4 x i16> [[TMP0]] to <4 x i32> 4147; UNROLL-NO-IC-NEXT: [[TMP3]] = zext <4 x i16> [[TMP1]] to <4 x i32> 4148; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR2]], <4 x i32> [[TMP2]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 4149; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 4150; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = add <4 x i16> [[TMP0]], <i16 5, i16 5, i16 5, i16 5> 4151; UNROLL-NO-IC-NEXT: [[TMP7]] = add <4 x i16> [[TMP1]], <i16 5, i16 5, i16 5, i16 5> 4152; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP6]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 4153; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = shufflevector <4 x i16> [[TMP6]], <4 x i16> [[TMP7]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 4154; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 4155; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT]] = add <4 x i16> [[STEP_ADD]], <i16 4, i16 4, i16 4, i16 4> 4156; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], 40 4157; UNROLL-NO-IC-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP50:![0-9]+]] 4158; UNROLL-NO-IC: middle.block: 4159; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i32 43, 40 4160; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[TMP7]], i32 3 4161; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[TMP7]], i32 2 4162; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT3:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3 4163; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2 4164; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 4165; UNROLL-NO-IC: scalar.ph: 4166; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT5:%.*]] = phi i32 [ -27, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT3]], [[MIDDLE_BLOCK]] ] 4167; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ 0, [[ENTRY]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 4168; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ 13, [[MIDDLE_BLOCK]] ], [ -27, [[ENTRY]] ] 4169; UNROLL-NO-IC-NEXT: br label [[FOR_COND:%.*]] 4170; UNROLL-NO-IC: for.cond: 4171; UNROLL-NO-IC-NEXT: [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_COND]] ] 4172; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[REC_1_PREV:%.*]], [[FOR_COND]] ] 4173; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR6:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT5]], [[SCALAR_PH]] ], [ [[REC_2_PREV:%.*]], [[FOR_COND]] ] 4174; UNROLL-NO-IC-NEXT: [[USE_REC_1:%.*]] = sub i16 [[SCALAR_RECUR]], 10 4175; UNROLL-NO-IC-NEXT: [[CMP:%.*]] = icmp eq i32 [[SCALAR_RECUR6]], 15 4176; UNROLL-NO-IC-NEXT: [[IV_NEXT]] = add i16 [[IV]], 1 4177; UNROLL-NO-IC-NEXT: [[REC_2_PREV]] = zext i16 [[IV_NEXT]] to i32 4178; UNROLL-NO-IC-NEXT: [[REC_1_PREV]] = add i16 [[IV_NEXT]], 5 4179; UNROLL-NO-IC-NEXT: br i1 [[CMP]], label [[FOR_END]], label [[FOR_COND]], !llvm.loop [[LOOP51:![0-9]+]] 4180; UNROLL-NO-IC: for.end: 4181; UNROLL-NO-IC-NEXT: ret void 4182; 4183; UNROLL-NO-VF-LABEL: @sink_dead_inst( 4184; UNROLL-NO-VF-NEXT: entry: 4185; UNROLL-NO-VF-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4186; UNROLL-NO-VF: vector.ph: 4187; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 4188; UNROLL-NO-VF: vector.body: 4189; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4190; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i16 [ 0, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ] 4191; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR1:%.*]] = phi i32 [ -27, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ] 4192; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = trunc i32 [[INDEX]] to i16 4193; UNROLL-NO-VF-NEXT: [[OFFSET_IDX:%.*]] = add i16 -27, [[TMP0]] 4194; UNROLL-NO-VF-NEXT: [[INDUCTION:%.*]] = add i16 [[OFFSET_IDX]], 0 4195; UNROLL-NO-VF-NEXT: [[INDUCTION2:%.*]] = add i16 [[OFFSET_IDX]], 1 4196; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = add i16 [[INDUCTION]], 1 4197; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = add i16 [[INDUCTION2]], 1 4198; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = zext i16 [[TMP1]] to i32 4199; UNROLL-NO-VF-NEXT: [[TMP4]] = zext i16 [[TMP2]] to i32 4200; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = add i16 [[TMP1]], 5 4201; UNROLL-NO-VF-NEXT: [[TMP6]] = add i16 [[TMP2]], 5 4202; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 4203; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 42 4204; UNROLL-NO-VF-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]] 4205; UNROLL-NO-VF: middle.block: 4206; UNROLL-NO-VF-NEXT: [[CMP_N:%.*]] = icmp eq i32 43, 42 4207; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 4208; UNROLL-NO-VF: scalar.ph: 4209; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT3:%.*]] = phi i32 [ -27, [[ENTRY:%.*]] ], [ [[TMP4]], [[MIDDLE_BLOCK]] ] 4210; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ 0, [[ENTRY]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ] 4211; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ 15, [[MIDDLE_BLOCK]] ], [ -27, [[ENTRY]] ] 4212; UNROLL-NO-VF-NEXT: br label [[FOR_COND:%.*]] 4213; UNROLL-NO-VF: for.cond: 4214; UNROLL-NO-VF-NEXT: [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_COND]] ] 4215; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[REC_1_PREV:%.*]], [[FOR_COND]] ] 4216; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR4:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT3]], [[SCALAR_PH]] ], [ [[REC_2_PREV:%.*]], [[FOR_COND]] ] 4217; UNROLL-NO-VF-NEXT: [[USE_REC_1:%.*]] = sub i16 [[SCALAR_RECUR]], 10 4218; UNROLL-NO-VF-NEXT: [[CMP:%.*]] = icmp eq i32 [[SCALAR_RECUR4]], 15 4219; UNROLL-NO-VF-NEXT: [[IV_NEXT]] = add i16 [[IV]], 1 4220; UNROLL-NO-VF-NEXT: [[REC_2_PREV]] = zext i16 [[IV_NEXT]] to i32 4221; UNROLL-NO-VF-NEXT: [[REC_1_PREV]] = add i16 [[IV_NEXT]], 5 4222; UNROLL-NO-VF-NEXT: br i1 [[CMP]], label [[FOR_END]], label [[FOR_COND]], !llvm.loop [[LOOP50:![0-9]+]] 4223; UNROLL-NO-VF: for.end: 4224; UNROLL-NO-VF-NEXT: ret void 4225; 4226; SINK-AFTER-LABEL: @sink_dead_inst( 4227; SINK-AFTER-NEXT: entry: 4228; SINK-AFTER-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4229; SINK-AFTER: vector.ph: 4230; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 4231; SINK-AFTER: vector.body: 4232; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4233; SINK-AFTER-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 -27, i16 -26, i16 -25, i16 -24>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 4234; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 0>, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ] 4235; SINK-AFTER-NEXT: [[VECTOR_RECUR1:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 -27>, [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ] 4236; SINK-AFTER-NEXT: [[TMP0:%.*]] = add <4 x i16> [[VEC_IND]], <i16 1, i16 1, i16 1, i16 1> 4237; SINK-AFTER-NEXT: [[TMP1]] = zext <4 x i16> [[TMP0]] to <4 x i32> 4238; SINK-AFTER-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR1]], <4 x i32> [[TMP1]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 4239; SINK-AFTER-NEXT: [[TMP3]] = add <4 x i16> [[TMP0]], <i16 5, i16 5, i16 5, i16 5> 4240; SINK-AFTER-NEXT: [[TMP4:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP3]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 4241; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 4242; SINK-AFTER-NEXT: [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], <i16 4, i16 4, i16 4, i16 4> 4243; SINK-AFTER-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 40 4244; SINK-AFTER-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP50:![0-9]+]] 4245; SINK-AFTER: middle.block: 4246; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i32 43, 40 4247; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[TMP3]], i32 3 4248; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[TMP3]], i32 2 4249; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT2:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3 4250; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI3:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2 4251; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 4252; SINK-AFTER: scalar.ph: 4253; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT4:%.*]] = phi i32 [ -27, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT2]], [[MIDDLE_BLOCK]] ] 4254; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ 0, [[ENTRY]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 4255; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ 13, [[MIDDLE_BLOCK]] ], [ -27, [[ENTRY]] ] 4256; SINK-AFTER-NEXT: br label [[FOR_COND:%.*]] 4257; SINK-AFTER: for.cond: 4258; SINK-AFTER-NEXT: [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_COND]] ] 4259; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[REC_1_PREV:%.*]], [[FOR_COND]] ] 4260; SINK-AFTER-NEXT: [[SCALAR_RECUR5:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT4]], [[SCALAR_PH]] ], [ [[REC_2_PREV:%.*]], [[FOR_COND]] ] 4261; SINK-AFTER-NEXT: [[USE_REC_1:%.*]] = sub i16 [[SCALAR_RECUR]], 10 4262; SINK-AFTER-NEXT: [[CMP:%.*]] = icmp eq i32 [[SCALAR_RECUR5]], 15 4263; SINK-AFTER-NEXT: [[IV_NEXT]] = add i16 [[IV]], 1 4264; SINK-AFTER-NEXT: [[REC_2_PREV]] = zext i16 [[IV_NEXT]] to i32 4265; SINK-AFTER-NEXT: [[REC_1_PREV]] = add i16 [[IV_NEXT]], 5 4266; SINK-AFTER-NEXT: br i1 [[CMP]], label [[FOR_END]], label [[FOR_COND]], !llvm.loop [[LOOP51:![0-9]+]] 4267; SINK-AFTER: for.end: 4268; SINK-AFTER-NEXT: ret void 4269; 4270entry: 4271 br label %for.cond 4272 4273for.cond: 4274 %iv = phi i16 [ -27, %entry ], [ %iv.next, %for.cond ] 4275 %rec.1 = phi i16 [ 0, %entry ], [ %rec.1.prev, %for.cond ] 4276 %rec.2 = phi i32 [ -27, %entry ], [ %rec.2.prev, %for.cond ] 4277 %use.rec.1 = sub i16 %rec.1, 10 4278 %cmp = icmp eq i32 %rec.2, 15 4279 %iv.next = add i16 %iv, 1 4280 %rec.2.prev = zext i16 %iv.next to i32 4281 %rec.1.prev = add i16 %iv.next, 5 4282 br i1 %cmp, label %for.end, label %for.cond 4283 4284for.end: 4285 ret void 4286} 4287 4288define i32 @sink_into_replication_region(i32 %y) { 4289; 4290; CHECK-LABEL: @sink_into_replication_region( 4291; CHECK-NEXT: bb: 4292; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4293; CHECK: vector.ph: 4294; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.smax.i32(i32 [[Y:%.*]], i32 1) 4295; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[TMP0]], 3 4296; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -4 4297; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i32 [[TMP0]], -1 4298; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0 4299; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 4300; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 4301; CHECK: vector.body: 4302; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE8:%.*]] ] 4303; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[PRED_UDIV_CONTINUE8]] ] 4304; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[PRED_UDIV_CONTINUE8]] ] 4305; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]] 4306; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[INDEX]], i64 0 4307; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer 4308; CHECK-NEXT: [[VEC_IV:%.*]] = or <4 x i32> [[BROADCAST_SPLAT2]], <i32 0, i32 1, i32 2, i32 3> 4309; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <4 x i32> [[VEC_IV]], [[BROADCAST_SPLAT]] 4310; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0 4311; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 4312; CHECK: pred.udiv.if: 4313; CHECK-NEXT: [[TMP3:%.*]] = udiv i32 219220132, [[OFFSET_IDX]] 4314; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i32> poison, i32 [[TMP3]], i64 0 4315; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE]] 4316; CHECK: pred.udiv.continue: 4317; CHECK-NEXT: [[TMP5:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP4]], [[PRED_UDIV_IF]] ] 4318; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1 4319; CHECK-NEXT: br i1 [[TMP6]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]] 4320; CHECK: pred.udiv.if3: 4321; CHECK-NEXT: [[TMP7:%.*]] = add i32 [[OFFSET_IDX]], -1 4322; CHECK-NEXT: [[TMP8:%.*]] = udiv i32 219220132, [[TMP7]] 4323; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[TMP8]], i64 1 4324; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE4]] 4325; CHECK: pred.udiv.continue4: 4326; CHECK-NEXT: [[TMP10:%.*]] = phi <4 x i32> [ [[TMP5]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP9]], [[PRED_UDIV_IF3]] ] 4327; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2 4328; CHECK-NEXT: br i1 [[TMP11]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]] 4329; CHECK: pred.udiv.if5: 4330; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[OFFSET_IDX]], -2 4331; CHECK-NEXT: [[TMP13:%.*]] = udiv i32 219220132, [[TMP12]] 4332; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP13]], i64 2 4333; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE6]] 4334; CHECK: pred.udiv.continue6: 4335; CHECK-NEXT: [[TMP15:%.*]] = phi <4 x i32> [ [[TMP10]], [[PRED_UDIV_CONTINUE4]] ], [ [[TMP14]], [[PRED_UDIV_IF5]] ] 4336; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3 4337; CHECK-NEXT: br i1 [[TMP16]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8]] 4338; CHECK: pred.udiv.if7: 4339; CHECK-NEXT: [[TMP17:%.*]] = add i32 [[OFFSET_IDX]], -3 4340; CHECK-NEXT: [[TMP18:%.*]] = udiv i32 219220132, [[TMP17]] 4341; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP18]], i64 3 4342; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE8]] 4343; CHECK: pred.udiv.continue8: 4344; CHECK-NEXT: [[TMP20]] = phi <4 x i32> [ [[TMP15]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP19]], [[PRED_UDIV_IF7]] ] 4345; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP20]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 4346; CHECK-NEXT: [[TMP22]] = add <4 x i32> [[VEC_PHI]], [[TMP21]] 4347; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4 4348; CHECK-NEXT: [[TMP23:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 4349; CHECK-NEXT: br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF52:![0-9]+]], !llvm.loop [[LOOP53:![0-9]+]] 4350; CHECK: middle.block: 4351; CHECK-NEXT: [[TMP24:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[TMP22]], <4 x i32> [[VEC_PHI]] 4352; CHECK-NEXT: [[TMP25:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP24]]) 4353; CHECK-NEXT: br i1 true, label [[BB1:%.*]], label [[SCALAR_PH]] 4354; CHECK: scalar.ph: 4355; CHECK-NEXT: br label [[BB2:%.*]] 4356; CHECK: bb1: 4357; CHECK-NEXT: [[VAR:%.*]] = phi i32 [ undef, [[BB2]] ], [ [[TMP25]], [[MIDDLE_BLOCK]] ] 4358; CHECK-NEXT: ret i32 [[VAR]] 4359; CHECK: bb2: 4360; CHECK-NEXT: br i1 undef, label [[BB1]], label [[BB2]], !prof [[PROF54:![0-9]+]], !llvm.loop [[LOOP55:![0-9]+]] 4361; 4362; UNROLL-LABEL: @sink_into_replication_region( 4363; UNROLL-NEXT: bb: 4364; UNROLL-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4365; UNROLL: vector.ph: 4366; UNROLL-NEXT: [[TMP0:%.*]] = call i32 @llvm.smax.i32(i32 [[Y:%.*]], i32 1) 4367; UNROLL-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[TMP0]], 7 4368; UNROLL-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8 4369; UNROLL-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i32 [[TMP0]], -1 4370; UNROLL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0 4371; UNROLL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 4372; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 4373; UNROLL: vector.body: 4374; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE18:%.*]] ] 4375; UNROLL-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP41:%.*]], [[PRED_UDIV_CONTINUE18]] ] 4376; UNROLL-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP44:%.*]], [[PRED_UDIV_CONTINUE18]] ] 4377; UNROLL-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP45:%.*]], [[PRED_UDIV_CONTINUE18]] ] 4378; UNROLL-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]] 4379; UNROLL-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x i32> poison, i32 [[INDEX]], i64 0 4380; UNROLL-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT2]], <4 x i32> poison, <4 x i32> zeroinitializer 4381; UNROLL-NEXT: [[VEC_IV:%.*]] = or <4 x i32> [[BROADCAST_SPLAT3]], <i32 0, i32 1, i32 2, i32 3> 4382; UNROLL-NEXT: [[VEC_IV4:%.*]] = or <4 x i32> [[BROADCAST_SPLAT3]], <i32 4, i32 5, i32 6, i32 7> 4383; UNROLL-NEXT: [[TMP1:%.*]] = icmp ule <4 x i32> [[VEC_IV]], [[BROADCAST_SPLAT]] 4384; UNROLL-NEXT: [[TMP2:%.*]] = icmp ule <4 x i32> [[VEC_IV4]], [[BROADCAST_SPLAT]] 4385; UNROLL-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0 4386; UNROLL-NEXT: br i1 [[TMP3]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 4387; UNROLL: pred.udiv.if: 4388; UNROLL-NEXT: [[TMP4:%.*]] = udiv i32 219220132, [[OFFSET_IDX]] 4389; UNROLL-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0 4390; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE]] 4391; UNROLL: pred.udiv.continue: 4392; UNROLL-NEXT: [[TMP6:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_UDIV_IF]] ] 4393; UNROLL-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1 4394; UNROLL-NEXT: br i1 [[TMP7]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]] 4395; UNROLL: pred.udiv.if5: 4396; UNROLL-NEXT: [[TMP8:%.*]] = add i32 [[OFFSET_IDX]], -1 4397; UNROLL-NEXT: [[TMP9:%.*]] = udiv i32 219220132, [[TMP8]] 4398; UNROLL-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[TMP9]], i64 1 4399; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE6]] 4400; UNROLL: pred.udiv.continue6: 4401; UNROLL-NEXT: [[TMP11:%.*]] = phi <4 x i32> [ [[TMP6]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP10]], [[PRED_UDIV_IF5]] ] 4402; UNROLL-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2 4403; UNROLL-NEXT: br i1 [[TMP12]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8:%.*]] 4404; UNROLL: pred.udiv.if7: 4405; UNROLL-NEXT: [[TMP13:%.*]] = add i32 [[OFFSET_IDX]], -2 4406; UNROLL-NEXT: [[TMP14:%.*]] = udiv i32 219220132, [[TMP13]] 4407; UNROLL-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP14]], i64 2 4408; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE8]] 4409; UNROLL: pred.udiv.continue8: 4410; UNROLL-NEXT: [[TMP16:%.*]] = phi <4 x i32> [ [[TMP11]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP15]], [[PRED_UDIV_IF7]] ] 4411; UNROLL-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3 4412; UNROLL-NEXT: br i1 [[TMP17]], label [[PRED_UDIV_IF9:%.*]], label [[PRED_UDIV_CONTINUE10:%.*]] 4413; UNROLL: pred.udiv.if9: 4414; UNROLL-NEXT: [[TMP18:%.*]] = add i32 [[OFFSET_IDX]], -3 4415; UNROLL-NEXT: [[TMP19:%.*]] = udiv i32 219220132, [[TMP18]] 4416; UNROLL-NEXT: [[TMP20:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP19]], i64 3 4417; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE10]] 4418; UNROLL: pred.udiv.continue10: 4419; UNROLL-NEXT: [[TMP21:%.*]] = phi <4 x i32> [ [[TMP16]], [[PRED_UDIV_CONTINUE8]] ], [ [[TMP20]], [[PRED_UDIV_IF9]] ] 4420; UNROLL-NEXT: [[TMP22:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0 4421; UNROLL-NEXT: br i1 [[TMP22]], label [[PRED_UDIV_IF11:%.*]], label [[PRED_UDIV_CONTINUE12:%.*]] 4422; UNROLL: pred.udiv.if11: 4423; UNROLL-NEXT: [[TMP23:%.*]] = add i32 [[OFFSET_IDX]], -4 4424; UNROLL-NEXT: [[TMP24:%.*]] = udiv i32 219220132, [[TMP23]] 4425; UNROLL-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> poison, i32 [[TMP24]], i64 0 4426; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE12]] 4427; UNROLL: pred.udiv.continue12: 4428; UNROLL-NEXT: [[TMP26:%.*]] = phi <4 x i32> [ poison, [[PRED_UDIV_CONTINUE10]] ], [ [[TMP25]], [[PRED_UDIV_IF11]] ] 4429; UNROLL-NEXT: [[TMP27:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1 4430; UNROLL-NEXT: br i1 [[TMP27]], label [[PRED_UDIV_IF13:%.*]], label [[PRED_UDIV_CONTINUE14:%.*]] 4431; UNROLL: pred.udiv.if13: 4432; UNROLL-NEXT: [[TMP28:%.*]] = add i32 [[OFFSET_IDX]], -5 4433; UNROLL-NEXT: [[TMP29:%.*]] = udiv i32 219220132, [[TMP28]] 4434; UNROLL-NEXT: [[TMP30:%.*]] = insertelement <4 x i32> [[TMP26]], i32 [[TMP29]], i64 1 4435; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE14]] 4436; UNROLL: pred.udiv.continue14: 4437; UNROLL-NEXT: [[TMP31:%.*]] = phi <4 x i32> [ [[TMP26]], [[PRED_UDIV_CONTINUE12]] ], [ [[TMP30]], [[PRED_UDIV_IF13]] ] 4438; UNROLL-NEXT: [[TMP32:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2 4439; UNROLL-NEXT: br i1 [[TMP32]], label [[PRED_UDIV_IF15:%.*]], label [[PRED_UDIV_CONTINUE16:%.*]] 4440; UNROLL: pred.udiv.if15: 4441; UNROLL-NEXT: [[TMP33:%.*]] = add i32 [[OFFSET_IDX]], -6 4442; UNROLL-NEXT: [[TMP34:%.*]] = udiv i32 219220132, [[TMP33]] 4443; UNROLL-NEXT: [[TMP35:%.*]] = insertelement <4 x i32> [[TMP31]], i32 [[TMP34]], i64 2 4444; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE16]] 4445; UNROLL: pred.udiv.continue16: 4446; UNROLL-NEXT: [[TMP36:%.*]] = phi <4 x i32> [ [[TMP31]], [[PRED_UDIV_CONTINUE14]] ], [ [[TMP35]], [[PRED_UDIV_IF15]] ] 4447; UNROLL-NEXT: [[TMP37:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3 4448; UNROLL-NEXT: br i1 [[TMP37]], label [[PRED_UDIV_IF17:%.*]], label [[PRED_UDIV_CONTINUE18]] 4449; UNROLL: pred.udiv.if17: 4450; UNROLL-NEXT: [[TMP38:%.*]] = add i32 [[OFFSET_IDX]], -7 4451; UNROLL-NEXT: [[TMP39:%.*]] = udiv i32 219220132, [[TMP38]] 4452; UNROLL-NEXT: [[TMP40:%.*]] = insertelement <4 x i32> [[TMP36]], i32 [[TMP39]], i64 3 4453; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE18]] 4454; UNROLL: pred.udiv.continue18: 4455; UNROLL-NEXT: [[TMP41]] = phi <4 x i32> [ [[TMP36]], [[PRED_UDIV_CONTINUE16]] ], [ [[TMP40]], [[PRED_UDIV_IF17]] ] 4456; UNROLL-NEXT: [[TMP42:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP21]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 4457; UNROLL-NEXT: [[TMP43:%.*]] = shufflevector <4 x i32> [[TMP21]], <4 x i32> [[TMP41]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 4458; UNROLL-NEXT: [[TMP44]] = add <4 x i32> [[VEC_PHI]], [[TMP42]] 4459; UNROLL-NEXT: [[TMP45]] = add <4 x i32> [[VEC_PHI1]], [[TMP43]] 4460; UNROLL-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8 4461; UNROLL-NEXT: [[TMP46:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 4462; UNROLL-NEXT: br i1 [[TMP46]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF52:![0-9]+]], !llvm.loop [[LOOP53:![0-9]+]] 4463; UNROLL: middle.block: 4464; UNROLL-NEXT: [[TMP47:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP45]], <4 x i32> [[VEC_PHI1]] 4465; UNROLL-NEXT: [[TMP48:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[TMP44]], <4 x i32> [[VEC_PHI]] 4466; UNROLL-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP47]], [[TMP48]] 4467; UNROLL-NEXT: [[TMP49:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]]) 4468; UNROLL-NEXT: br i1 true, label [[BB1:%.*]], label [[SCALAR_PH]] 4469; UNROLL: scalar.ph: 4470; UNROLL-NEXT: br label [[BB2:%.*]] 4471; UNROLL: bb1: 4472; UNROLL-NEXT: [[VAR:%.*]] = phi i32 [ undef, [[BB2]] ], [ [[TMP49]], [[MIDDLE_BLOCK]] ] 4473; UNROLL-NEXT: ret i32 [[VAR]] 4474; UNROLL: bb2: 4475; UNROLL-NEXT: br i1 undef, label [[BB1]], label [[BB2]], !prof [[PROF54:![0-9]+]], !llvm.loop [[LOOP55:![0-9]+]] 4476; 4477; UNROLL-NO-IC-LABEL: @sink_into_replication_region( 4478; UNROLL-NO-IC-NEXT: bb: 4479; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i32 [[Y:%.*]], 1 4480; UNROLL-NO-IC-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[Y]], i32 1) 4481; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]] 4482; UNROLL-NO-IC-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4483; UNROLL-NO-IC: vector.ph: 4484; UNROLL-NO-IC-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 7 4485; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 8 4486; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]] 4487; UNROLL-NO-IC-NEXT: [[IND_END:%.*]] = sub i32 [[Y]], [[N_VEC]] 4488; UNROLL-NO-IC-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[TMP1]], 1 4489; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i32 0 4490; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 4491; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 4492; UNROLL-NO-IC: vector.body: 4493; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE18:%.*]] ] 4494; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP43:%.*]], [[PRED_UDIV_CONTINUE18]] ] 4495; UNROLL-NO-IC-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP46:%.*]], [[PRED_UDIV_CONTINUE18]] ] 4496; UNROLL-NO-IC-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP47:%.*]], [[PRED_UDIV_CONTINUE18]] ] 4497; UNROLL-NO-IC-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]] 4498; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x i32> poison, i32 [[INDEX]], i32 0 4499; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT2]], <4 x i32> poison, <4 x i32> zeroinitializer 4500; UNROLL-NO-IC-NEXT: [[VEC_IV:%.*]] = add <4 x i32> [[BROADCAST_SPLAT3]], <i32 0, i32 1, i32 2, i32 3> 4501; UNROLL-NO-IC-NEXT: [[VEC_IV4:%.*]] = add <4 x i32> [[BROADCAST_SPLAT3]], <i32 4, i32 5, i32 6, i32 7> 4502; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = icmp ule <4 x i32> [[VEC_IV]], [[BROADCAST_SPLAT]] 4503; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = icmp ule <4 x i32> [[VEC_IV4]], [[BROADCAST_SPLAT]] 4504; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP2]], i32 0 4505; UNROLL-NO-IC-NEXT: br i1 [[TMP4]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 4506; UNROLL-NO-IC: pred.udiv.if: 4507; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = add i32 [[OFFSET_IDX]], 0 4508; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = udiv i32 219220132, [[TMP5]] 4509; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[TMP6]], i32 0 4510; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE]] 4511; UNROLL-NO-IC: pred.udiv.continue: 4512; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP7]], [[PRED_UDIV_IF]] ] 4513; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP2]], i32 1 4514; UNROLL-NO-IC-NEXT: br i1 [[TMP9]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]] 4515; UNROLL-NO-IC: pred.udiv.if5: 4516; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = add i32 [[OFFSET_IDX]], -1 4517; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = udiv i32 219220132, [[TMP10]] 4518; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP11]], i32 1 4519; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE6]] 4520; UNROLL-NO-IC: pred.udiv.continue6: 4521; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = phi <4 x i32> [ [[TMP8]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP12]], [[PRED_UDIV_IF5]] ] 4522; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP2]], i32 2 4523; UNROLL-NO-IC-NEXT: br i1 [[TMP14]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8:%.*]] 4524; UNROLL-NO-IC: pred.udiv.if7: 4525; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = add i32 [[OFFSET_IDX]], -2 4526; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = udiv i32 219220132, [[TMP15]] 4527; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP16]], i32 2 4528; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE8]] 4529; UNROLL-NO-IC: pred.udiv.continue8: 4530; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = phi <4 x i32> [ [[TMP13]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP17]], [[PRED_UDIV_IF7]] ] 4531; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP2]], i32 3 4532; UNROLL-NO-IC-NEXT: br i1 [[TMP19]], label [[PRED_UDIV_IF9:%.*]], label [[PRED_UDIV_CONTINUE10:%.*]] 4533; UNROLL-NO-IC: pred.udiv.if9: 4534; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = add i32 [[OFFSET_IDX]], -3 4535; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = udiv i32 219220132, [[TMP20]] 4536; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = insertelement <4 x i32> [[TMP18]], i32 [[TMP21]], i32 3 4537; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE10]] 4538; UNROLL-NO-IC: pred.udiv.continue10: 4539; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = phi <4 x i32> [ [[TMP18]], [[PRED_UDIV_CONTINUE8]] ], [ [[TMP22]], [[PRED_UDIV_IF9]] ] 4540; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = extractelement <4 x i1> [[TMP3]], i32 0 4541; UNROLL-NO-IC-NEXT: br i1 [[TMP24]], label [[PRED_UDIV_IF11:%.*]], label [[PRED_UDIV_CONTINUE12:%.*]] 4542; UNROLL-NO-IC: pred.udiv.if11: 4543; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = add i32 [[OFFSET_IDX]], -4 4544; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = udiv i32 219220132, [[TMP25]] 4545; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = insertelement <4 x i32> poison, i32 [[TMP26]], i32 0 4546; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE12]] 4547; UNROLL-NO-IC: pred.udiv.continue12: 4548; UNROLL-NO-IC-NEXT: [[TMP28:%.*]] = phi <4 x i32> [ poison, [[PRED_UDIV_CONTINUE10]] ], [ [[TMP27]], [[PRED_UDIV_IF11]] ] 4549; UNROLL-NO-IC-NEXT: [[TMP29:%.*]] = extractelement <4 x i1> [[TMP3]], i32 1 4550; UNROLL-NO-IC-NEXT: br i1 [[TMP29]], label [[PRED_UDIV_IF13:%.*]], label [[PRED_UDIV_CONTINUE14:%.*]] 4551; UNROLL-NO-IC: pred.udiv.if13: 4552; UNROLL-NO-IC-NEXT: [[TMP30:%.*]] = add i32 [[OFFSET_IDX]], -5 4553; UNROLL-NO-IC-NEXT: [[TMP31:%.*]] = udiv i32 219220132, [[TMP30]] 4554; UNROLL-NO-IC-NEXT: [[TMP32:%.*]] = insertelement <4 x i32> [[TMP28]], i32 [[TMP31]], i32 1 4555; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE14]] 4556; UNROLL-NO-IC: pred.udiv.continue14: 4557; UNROLL-NO-IC-NEXT: [[TMP33:%.*]] = phi <4 x i32> [ [[TMP28]], [[PRED_UDIV_CONTINUE12]] ], [ [[TMP32]], [[PRED_UDIV_IF13]] ] 4558; UNROLL-NO-IC-NEXT: [[TMP34:%.*]] = extractelement <4 x i1> [[TMP3]], i32 2 4559; UNROLL-NO-IC-NEXT: br i1 [[TMP34]], label [[PRED_UDIV_IF15:%.*]], label [[PRED_UDIV_CONTINUE16:%.*]] 4560; UNROLL-NO-IC: pred.udiv.if15: 4561; UNROLL-NO-IC-NEXT: [[TMP35:%.*]] = add i32 [[OFFSET_IDX]], -6 4562; UNROLL-NO-IC-NEXT: [[TMP36:%.*]] = udiv i32 219220132, [[TMP35]] 4563; UNROLL-NO-IC-NEXT: [[TMP37:%.*]] = insertelement <4 x i32> [[TMP33]], i32 [[TMP36]], i32 2 4564; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE16]] 4565; UNROLL-NO-IC: pred.udiv.continue16: 4566; UNROLL-NO-IC-NEXT: [[TMP38:%.*]] = phi <4 x i32> [ [[TMP33]], [[PRED_UDIV_CONTINUE14]] ], [ [[TMP37]], [[PRED_UDIV_IF15]] ] 4567; UNROLL-NO-IC-NEXT: [[TMP39:%.*]] = extractelement <4 x i1> [[TMP3]], i32 3 4568; UNROLL-NO-IC-NEXT: br i1 [[TMP39]], label [[PRED_UDIV_IF17:%.*]], label [[PRED_UDIV_CONTINUE18]] 4569; UNROLL-NO-IC: pred.udiv.if17: 4570; UNROLL-NO-IC-NEXT: [[TMP40:%.*]] = add i32 [[OFFSET_IDX]], -7 4571; UNROLL-NO-IC-NEXT: [[TMP41:%.*]] = udiv i32 219220132, [[TMP40]] 4572; UNROLL-NO-IC-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> [[TMP38]], i32 [[TMP41]], i32 3 4573; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE18]] 4574; UNROLL-NO-IC: pred.udiv.continue18: 4575; UNROLL-NO-IC-NEXT: [[TMP43]] = phi <4 x i32> [ [[TMP38]], [[PRED_UDIV_CONTINUE16]] ], [ [[TMP42]], [[PRED_UDIV_IF17]] ] 4576; UNROLL-NO-IC-NEXT: [[TMP44:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP23]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 4577; UNROLL-NO-IC-NEXT: [[TMP45:%.*]] = shufflevector <4 x i32> [[TMP23]], <4 x i32> [[TMP43]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 4578; UNROLL-NO-IC-NEXT: [[TMP46]] = add <4 x i32> [[VEC_PHI]], [[TMP44]] 4579; UNROLL-NO-IC-NEXT: [[TMP47]] = add <4 x i32> [[VEC_PHI1]], [[TMP45]] 4580; UNROLL-NO-IC-NEXT: [[TMP48:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP46]], <4 x i32> [[VEC_PHI]] 4581; UNROLL-NO-IC-NEXT: [[TMP49:%.*]] = select <4 x i1> [[TMP3]], <4 x i32> [[TMP47]], <4 x i32> [[VEC_PHI1]] 4582; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8 4583; UNROLL-NO-IC-NEXT: [[TMP50:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 4584; UNROLL-NO-IC-NEXT: br i1 [[TMP50]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF52:![0-9]+]], !llvm.loop [[LOOP53:![0-9]+]] 4585; UNROLL-NO-IC: middle.block: 4586; UNROLL-NO-IC-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP49]], [[TMP48]] 4587; UNROLL-NO-IC-NEXT: [[TMP51:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]]) 4588; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP43]], i32 3 4589; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP43]], i32 2 4590; UNROLL-NO-IC-NEXT: br i1 true, label [[BB1:%.*]], label [[SCALAR_PH]] 4591; UNROLL-NO-IC: scalar.ph: 4592; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[BB:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 4593; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[Y]], [[BB]] ] 4594; UNROLL-NO-IC-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[BB]] ], [ [[TMP51]], [[MIDDLE_BLOCK]] ] 4595; UNROLL-NO-IC-NEXT: br label [[BB2:%.*]] 4596; UNROLL-NO-IC: bb1: 4597; UNROLL-NO-IC-NEXT: [[VAR:%.*]] = phi i32 [ [[VAR6:%.*]], [[BB2]] ], [ [[TMP51]], [[MIDDLE_BLOCK]] ] 4598; UNROLL-NO-IC-NEXT: ret i32 [[VAR]] 4599; UNROLL-NO-IC: bb2: 4600; UNROLL-NO-IC-NEXT: [[VAR3:%.*]] = phi i32 [ [[VAR8:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 4601; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[VAR7:%.*]], [[BB2]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ] 4602; UNROLL-NO-IC-NEXT: [[VAR5:%.*]] = phi i32 [ [[VAR6]], [[BB2]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 4603; UNROLL-NO-IC-NEXT: [[VAR6]] = add i32 [[VAR5]], [[SCALAR_RECUR]] 4604; UNROLL-NO-IC-NEXT: [[VAR7]] = udiv i32 219220132, [[VAR3]] 4605; UNROLL-NO-IC-NEXT: [[VAR8]] = add nsw i32 [[VAR3]], -1 4606; UNROLL-NO-IC-NEXT: [[VAR9:%.*]] = icmp slt i32 [[VAR3]], 2 4607; UNROLL-NO-IC-NEXT: br i1 [[VAR9]], label [[BB1]], label [[BB2]], !prof [[PROF54:![0-9]+]], !llvm.loop [[LOOP55:![0-9]+]] 4608; 4609; UNROLL-NO-VF-LABEL: @sink_into_replication_region( 4610; UNROLL-NO-VF-NEXT: bb: 4611; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i32 [[Y:%.*]], 1 4612; UNROLL-NO-VF-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[Y]], i32 1) 4613; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]] 4614; UNROLL-NO-VF-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4615; UNROLL-NO-VF: vector.ph: 4616; UNROLL-NO-VF-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 1 4617; UNROLL-NO-VF-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2 4618; UNROLL-NO-VF-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]] 4619; UNROLL-NO-VF-NEXT: [[IND_END:%.*]] = sub i32 [[Y]], [[N_VEC]] 4620; UNROLL-NO-VF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[TMP1]], 1 4621; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 4622; UNROLL-NO-VF: vector.body: 4623; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE5:%.*]] ] 4624; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[PRED_UDIV_CONTINUE5]] ] 4625; UNROLL-NO-VF-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[PRED_UDIV_CONTINUE5]] ] 4626; UNROLL-NO-VF-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[PRED_UDIV_CONTINUE5]] ] 4627; UNROLL-NO-VF-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]] 4628; UNROLL-NO-VF-NEXT: [[VEC_IV:%.*]] = add i32 [[INDEX]], 0 4629; UNROLL-NO-VF-NEXT: [[VEC_IV3:%.*]] = add i32 [[INDEX]], 1 4630; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = icmp ule i32 [[VEC_IV]], [[TRIP_COUNT_MINUS_1]] 4631; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = icmp ule i32 [[VEC_IV3]], [[TRIP_COUNT_MINUS_1]] 4632; UNROLL-NO-VF-NEXT: br i1 [[TMP2]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 4633; UNROLL-NO-VF: pred.udiv.if: 4634; UNROLL-NO-VF-NEXT: [[INDUCTION:%.*]] = add i32 [[OFFSET_IDX]], 0 4635; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = udiv i32 219220132, [[INDUCTION]] 4636; UNROLL-NO-VF-NEXT: br label [[PRED_UDIV_CONTINUE]] 4637; UNROLL-NO-VF: pred.udiv.continue: 4638; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP4]], [[PRED_UDIV_IF]] ] 4639; UNROLL-NO-VF-NEXT: br i1 [[TMP3]], label [[PRED_UDIV_IF4:%.*]], label [[PRED_UDIV_CONTINUE5]] 4640; UNROLL-NO-VF: pred.udiv.if4: 4641; UNROLL-NO-VF-NEXT: [[INDUCTION2:%.*]] = add i32 [[OFFSET_IDX]], -1 4642; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = udiv i32 219220132, [[INDUCTION2]] 4643; UNROLL-NO-VF-NEXT: br label [[PRED_UDIV_CONTINUE5]] 4644; UNROLL-NO-VF: pred.udiv.continue5: 4645; UNROLL-NO-VF-NEXT: [[TMP7]] = phi i32 [ poison, [[PRED_UDIV_CONTINUE]] ], [ [[TMP6]], [[PRED_UDIV_IF4]] ] 4646; UNROLL-NO-VF-NEXT: [[TMP8]] = add i32 [[VEC_PHI]], [[VECTOR_RECUR]] 4647; UNROLL-NO-VF-NEXT: [[TMP9]] = add i32 [[VEC_PHI1]], [[TMP5]] 4648; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = select i1 [[TMP2]], i32 [[TMP8]], i32 [[VEC_PHI]] 4649; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = select i1 [[TMP3]], i32 [[TMP9]], i32 [[VEC_PHI1]] 4650; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2 4651; UNROLL-NO-VF-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 4652; UNROLL-NO-VF-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF51:![0-9]+]], !llvm.loop [[LOOP52:![0-9]+]] 4653; UNROLL-NO-VF: middle.block: 4654; UNROLL-NO-VF-NEXT: [[BIN_RDX:%.*]] = add i32 [[TMP11]], [[TMP10]] 4655; UNROLL-NO-VF-NEXT: br i1 true, label [[BB1:%.*]], label [[SCALAR_PH]] 4656; UNROLL-NO-VF: scalar.ph: 4657; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[BB:%.*]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ] 4658; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[Y]], [[BB]] ] 4659; UNROLL-NO-VF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[BB]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ] 4660; UNROLL-NO-VF-NEXT: br label [[BB2:%.*]] 4661; UNROLL-NO-VF: bb1: 4662; UNROLL-NO-VF-NEXT: [[VAR:%.*]] = phi i32 [ [[VAR6:%.*]], [[BB2]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ] 4663; UNROLL-NO-VF-NEXT: ret i32 [[VAR]] 4664; UNROLL-NO-VF: bb2: 4665; UNROLL-NO-VF-NEXT: [[VAR3:%.*]] = phi i32 [ [[VAR8:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 4666; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[VAR7:%.*]], [[BB2]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ] 4667; UNROLL-NO-VF-NEXT: [[VAR5:%.*]] = phi i32 [ [[VAR6]], [[BB2]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 4668; UNROLL-NO-VF-NEXT: [[VAR6]] = add i32 [[VAR5]], [[SCALAR_RECUR]] 4669; UNROLL-NO-VF-NEXT: [[VAR7]] = udiv i32 219220132, [[VAR3]] 4670; UNROLL-NO-VF-NEXT: [[VAR8]] = add nsw i32 [[VAR3]], -1 4671; UNROLL-NO-VF-NEXT: [[VAR9:%.*]] = icmp slt i32 [[VAR3]], 2 4672; UNROLL-NO-VF-NEXT: br i1 [[VAR9]], label [[BB1]], label [[BB2]], !prof [[PROF53:![0-9]+]], !llvm.loop [[LOOP54:![0-9]+]] 4673; 4674; SINK-AFTER-LABEL: @sink_into_replication_region( 4675; SINK-AFTER-NEXT: bb: 4676; SINK-AFTER-NEXT: [[TMP0:%.*]] = add i32 [[Y:%.*]], 1 4677; SINK-AFTER-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[Y]], i32 1) 4678; SINK-AFTER-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]] 4679; SINK-AFTER-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4680; SINK-AFTER: vector.ph: 4681; SINK-AFTER-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 3 4682; SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 4 4683; SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]] 4684; SINK-AFTER-NEXT: [[IND_END:%.*]] = sub i32 [[Y]], [[N_VEC]] 4685; SINK-AFTER-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[TMP1]], 1 4686; SINK-AFTER-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i32 0 4687; SINK-AFTER-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 4688; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 4689; SINK-AFTER: vector.body: 4690; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE8:%.*]] ] 4691; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[PRED_UDIV_CONTINUE8]] ] 4692; SINK-AFTER-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[PRED_UDIV_CONTINUE8]] ] 4693; SINK-AFTER-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]] 4694; SINK-AFTER-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[INDEX]], i32 0 4695; SINK-AFTER-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer 4696; SINK-AFTER-NEXT: [[VEC_IV:%.*]] = add <4 x i32> [[BROADCAST_SPLAT2]], <i32 0, i32 1, i32 2, i32 3> 4697; SINK-AFTER-NEXT: [[TMP2:%.*]] = icmp ule <4 x i32> [[VEC_IV]], [[BROADCAST_SPLAT]] 4698; SINK-AFTER-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i32 0 4699; SINK-AFTER-NEXT: br i1 [[TMP3]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 4700; SINK-AFTER: pred.udiv.if: 4701; SINK-AFTER-NEXT: [[TMP4:%.*]] = add i32 [[OFFSET_IDX]], 0 4702; SINK-AFTER-NEXT: [[TMP5:%.*]] = udiv i32 219220132, [[TMP4]] 4703; SINK-AFTER-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i32 0 4704; SINK-AFTER-NEXT: br label [[PRED_UDIV_CONTINUE]] 4705; SINK-AFTER: pred.udiv.continue: 4706; SINK-AFTER-NEXT: [[TMP7:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_UDIV_IF]] ] 4707; SINK-AFTER-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP2]], i32 1 4708; SINK-AFTER-NEXT: br i1 [[TMP8]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]] 4709; SINK-AFTER: pred.udiv.if3: 4710; SINK-AFTER-NEXT: [[TMP9:%.*]] = add i32 [[OFFSET_IDX]], -1 4711; SINK-AFTER-NEXT: [[TMP10:%.*]] = udiv i32 219220132, [[TMP9]] 4712; SINK-AFTER-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[TMP10]], i32 1 4713; SINK-AFTER-NEXT: br label [[PRED_UDIV_CONTINUE4]] 4714; SINK-AFTER: pred.udiv.continue4: 4715; SINK-AFTER-NEXT: [[TMP12:%.*]] = phi <4 x i32> [ [[TMP7]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP11]], [[PRED_UDIV_IF3]] ] 4716; SINK-AFTER-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP2]], i32 2 4717; SINK-AFTER-NEXT: br i1 [[TMP13]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]] 4718; SINK-AFTER: pred.udiv.if5: 4719; SINK-AFTER-NEXT: [[TMP14:%.*]] = add i32 [[OFFSET_IDX]], -2 4720; SINK-AFTER-NEXT: [[TMP15:%.*]] = udiv i32 219220132, [[TMP14]] 4721; SINK-AFTER-NEXT: [[TMP16:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP15]], i32 2 4722; SINK-AFTER-NEXT: br label [[PRED_UDIV_CONTINUE6]] 4723; SINK-AFTER: pred.udiv.continue6: 4724; SINK-AFTER-NEXT: [[TMP17:%.*]] = phi <4 x i32> [ [[TMP12]], [[PRED_UDIV_CONTINUE4]] ], [ [[TMP16]], [[PRED_UDIV_IF5]] ] 4725; SINK-AFTER-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[TMP2]], i32 3 4726; SINK-AFTER-NEXT: br i1 [[TMP18]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8]] 4727; SINK-AFTER: pred.udiv.if7: 4728; SINK-AFTER-NEXT: [[TMP19:%.*]] = add i32 [[OFFSET_IDX]], -3 4729; SINK-AFTER-NEXT: [[TMP20:%.*]] = udiv i32 219220132, [[TMP19]] 4730; SINK-AFTER-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP20]], i32 3 4731; SINK-AFTER-NEXT: br label [[PRED_UDIV_CONTINUE8]] 4732; SINK-AFTER: pred.udiv.continue8: 4733; SINK-AFTER-NEXT: [[TMP22]] = phi <4 x i32> [ [[TMP17]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP21]], [[PRED_UDIV_IF7]] ] 4734; SINK-AFTER-NEXT: [[TMP23:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP22]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 4735; SINK-AFTER-NEXT: [[TMP24]] = add <4 x i32> [[VEC_PHI]], [[TMP23]] 4736; SINK-AFTER-NEXT: [[TMP25:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP24]], <4 x i32> [[VEC_PHI]] 4737; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4 4738; SINK-AFTER-NEXT: [[TMP26:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 4739; SINK-AFTER-NEXT: br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF52:![0-9]+]], !llvm.loop [[LOOP53:![0-9]+]] 4740; SINK-AFTER: middle.block: 4741; SINK-AFTER-NEXT: [[TMP27:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP25]]) 4742; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP22]], i32 3 4743; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP22]], i32 2 4744; SINK-AFTER-NEXT: br i1 true, label [[BB1:%.*]], label [[SCALAR_PH]] 4745; SINK-AFTER: scalar.ph: 4746; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[BB:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 4747; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[Y]], [[BB]] ] 4748; SINK-AFTER-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[BB]] ], [ [[TMP27]], [[MIDDLE_BLOCK]] ] 4749; SINK-AFTER-NEXT: br label [[BB2:%.*]] 4750; SINK-AFTER: bb1: 4751; SINK-AFTER-NEXT: [[VAR:%.*]] = phi i32 [ [[VAR6:%.*]], [[BB2]] ], [ [[TMP27]], [[MIDDLE_BLOCK]] ] 4752; SINK-AFTER-NEXT: ret i32 [[VAR]] 4753; SINK-AFTER: bb2: 4754; SINK-AFTER-NEXT: [[VAR3:%.*]] = phi i32 [ [[VAR8:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 4755; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[VAR7:%.*]], [[BB2]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ] 4756; SINK-AFTER-NEXT: [[VAR5:%.*]] = phi i32 [ [[VAR6]], [[BB2]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 4757; SINK-AFTER-NEXT: [[VAR6]] = add i32 [[VAR5]], [[SCALAR_RECUR]] 4758; SINK-AFTER-NEXT: [[VAR7]] = udiv i32 219220132, [[VAR3]] 4759; SINK-AFTER-NEXT: [[VAR8]] = add nsw i32 [[VAR3]], -1 4760; SINK-AFTER-NEXT: [[VAR9:%.*]] = icmp slt i32 [[VAR3]], 2 4761; SINK-AFTER-NEXT: br i1 [[VAR9]], label [[BB1]], label [[BB2]], !prof [[PROF54:![0-9]+]], !llvm.loop [[LOOP55:![0-9]+]] 4762; 4763bb: 4764 br label %bb2 4765 4766 bb1: ; preds = %bb2 4767 %var = phi i32 [ %var6, %bb2 ] 4768 ret i32 %var 4769 4770 bb2: ; preds = %bb2, %bb 4771 %var3 = phi i32 [ %var8, %bb2 ], [ %y, %bb ] 4772 %var4 = phi i32 [ %var7, %bb2 ], [ 0, %bb ] 4773 %var5 = phi i32 [ %var6, %bb2 ], [ 0, %bb ] 4774 %var6 = add i32 %var5, %var4 4775 %var7 = udiv i32 219220132, %var3 4776 %var8 = add nsw i32 %var3, -1 4777 %var9 = icmp slt i32 %var3, 2 4778 br i1 %var9, label %bb1, label %bb2, !prof !2 4779} 4780 4781define i32 @sink_into_replication_region_multiple(i32 *%x, i32 %y) { 4782; 4783; CHECK-LABEL: @sink_into_replication_region_multiple( 4784; CHECK-NEXT: bb: 4785; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4786; CHECK: vector.ph: 4787; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.smax.i32(i32 [[Y:%.*]], i32 1) 4788; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[TMP0]], 3 4789; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -4 4790; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i32 [[TMP0]], -1 4791; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0 4792; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 4793; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 4794; CHECK: vector.body: 4795; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE13:%.*]] ] 4796; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE13]] ] 4797; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[PRED_STORE_CONTINUE13]] ] 4798; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[PRED_STORE_CONTINUE13]] ] 4799; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]] 4800; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFFSET_IDX]], -1 4801; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[OFFSET_IDX]], -2 4802; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[OFFSET_IDX]], -3 4803; CHECK-NEXT: [[TMP4:%.*]] = icmp ule <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]] 4804; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i64 0 4805; CHECK-NEXT: br i1 [[TMP5]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 4806; CHECK: pred.udiv.if: 4807; CHECK-NEXT: [[TMP6:%.*]] = udiv i32 219220132, [[OFFSET_IDX]] 4808; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[TMP6]], i64 0 4809; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE]] 4810; CHECK: pred.udiv.continue: 4811; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP7]], [[PRED_UDIV_IF]] ] 4812; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP4]], i64 1 4813; CHECK-NEXT: br i1 [[TMP9]], label [[PRED_UDIV_IF2:%.*]], label [[PRED_UDIV_CONTINUE3:%.*]] 4814; CHECK: pred.udiv.if2: 4815; CHECK-NEXT: [[TMP10:%.*]] = udiv i32 219220132, [[TMP1]] 4816; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP10]], i64 1 4817; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE3]] 4818; CHECK: pred.udiv.continue3: 4819; CHECK-NEXT: [[TMP12:%.*]] = phi <4 x i32> [ [[TMP8]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP11]], [[PRED_UDIV_IF2]] ] 4820; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP4]], i64 2 4821; CHECK-NEXT: br i1 [[TMP13]], label [[PRED_UDIV_IF4:%.*]], label [[PRED_UDIV_CONTINUE5:%.*]] 4822; CHECK: pred.udiv.if4: 4823; CHECK-NEXT: [[TMP14:%.*]] = udiv i32 219220132, [[TMP2]] 4824; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP14]], i64 2 4825; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE5]] 4826; CHECK: pred.udiv.continue5: 4827; CHECK-NEXT: [[TMP16:%.*]] = phi <4 x i32> [ [[TMP12]], [[PRED_UDIV_CONTINUE3]] ], [ [[TMP15]], [[PRED_UDIV_IF4]] ] 4828; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP4]], i64 3 4829; CHECK-NEXT: br i1 [[TMP17]], label [[PRED_UDIV_IF6:%.*]], label [[PRED_UDIV_CONTINUE7:%.*]] 4830; CHECK: pred.udiv.if6: 4831; CHECK-NEXT: [[TMP18:%.*]] = udiv i32 219220132, [[TMP3]] 4832; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP18]], i64 3 4833; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE7]] 4834; CHECK: pred.udiv.continue7: 4835; CHECK-NEXT: [[TMP20]] = phi <4 x i32> [ [[TMP16]], [[PRED_UDIV_CONTINUE5]] ], [ [[TMP19]], [[PRED_UDIV_IF6]] ] 4836; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP20]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 4837; CHECK-NEXT: [[TMP22]] = add <4 x i32> [[VEC_PHI]], [[TMP21]] 4838; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x i1> [[TMP4]], i64 0 4839; CHECK-NEXT: br i1 [[TMP23]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] 4840; CHECK: pred.store.if: 4841; CHECK-NEXT: [[TMP24:%.*]] = sext i32 [[INDEX]] to i64 4842; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 [[TMP24]] 4843; CHECK-NEXT: store i32 [[OFFSET_IDX]], i32* [[TMP25]], align 4 4844; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]] 4845; CHECK: pred.store.continue: 4846; CHECK-NEXT: [[TMP26:%.*]] = extractelement <4 x i1> [[TMP4]], i64 1 4847; CHECK-NEXT: br i1 [[TMP26]], label [[PRED_STORE_IF8:%.*]], label [[PRED_STORE_CONTINUE9:%.*]] 4848; CHECK: pred.store.if8: 4849; CHECK-NEXT: [[TMP27:%.*]] = or i32 [[INDEX]], 1 4850; CHECK-NEXT: [[TMP28:%.*]] = sext i32 [[TMP27]] to i64 4851; CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 [[TMP28]] 4852; CHECK-NEXT: store i32 [[TMP1]], i32* [[TMP29]], align 4 4853; CHECK-NEXT: br label [[PRED_STORE_CONTINUE9]] 4854; CHECK: pred.store.continue9: 4855; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i1> [[TMP4]], i64 2 4856; CHECK-NEXT: br i1 [[TMP30]], label [[PRED_STORE_IF10:%.*]], label [[PRED_STORE_CONTINUE11:%.*]] 4857; CHECK: pred.store.if10: 4858; CHECK-NEXT: [[TMP31:%.*]] = or i32 [[INDEX]], 2 4859; CHECK-NEXT: [[TMP32:%.*]] = sext i32 [[TMP31]] to i64 4860; CHECK-NEXT: [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 [[TMP32]] 4861; CHECK-NEXT: store i32 [[TMP2]], i32* [[TMP33]], align 4 4862; CHECK-NEXT: br label [[PRED_STORE_CONTINUE11]] 4863; CHECK: pred.store.continue11: 4864; CHECK-NEXT: [[TMP34:%.*]] = extractelement <4 x i1> [[TMP4]], i64 3 4865; CHECK-NEXT: br i1 [[TMP34]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE13]] 4866; CHECK: pred.store.if12: 4867; CHECK-NEXT: [[TMP35:%.*]] = or i32 [[INDEX]], 3 4868; CHECK-NEXT: [[TMP36:%.*]] = sext i32 [[TMP35]] to i64 4869; CHECK-NEXT: [[TMP37:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 [[TMP36]] 4870; CHECK-NEXT: store i32 [[TMP3]], i32* [[TMP37]], align 4 4871; CHECK-NEXT: br label [[PRED_STORE_CONTINUE13]] 4872; CHECK: pred.store.continue13: 4873; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4 4874; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4> 4875; CHECK-NEXT: [[TMP38:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 4876; CHECK-NEXT: br i1 [[TMP38]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF52]], !llvm.loop [[LOOP56:![0-9]+]] 4877; CHECK: middle.block: 4878; CHECK-NEXT: [[TMP39:%.*]] = select <4 x i1> [[TMP4]], <4 x i32> [[TMP22]], <4 x i32> [[VEC_PHI]] 4879; CHECK-NEXT: [[TMP40:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP39]]) 4880; CHECK-NEXT: br i1 true, label [[BB1:%.*]], label [[SCALAR_PH]] 4881; CHECK: scalar.ph: 4882; CHECK-NEXT: br label [[BB2:%.*]] 4883; CHECK: bb1: 4884; CHECK-NEXT: [[VAR:%.*]] = phi i32 [ undef, [[BB2]] ], [ [[TMP40]], [[MIDDLE_BLOCK]] ] 4885; CHECK-NEXT: ret i32 [[VAR]] 4886; CHECK: bb2: 4887; CHECK-NEXT: br i1 undef, label [[BB1]], label [[BB2]], !prof [[PROF54]], !llvm.loop [[LOOP57:![0-9]+]] 4888; 4889; UNROLL-LABEL: @sink_into_replication_region_multiple( 4890; UNROLL-NEXT: bb: 4891; UNROLL-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4892; UNROLL: vector.ph: 4893; UNROLL-NEXT: [[TMP0:%.*]] = call i32 @llvm.smax.i32(i32 [[Y:%.*]], i32 1) 4894; UNROLL-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[TMP0]], 7 4895; UNROLL-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8 4896; UNROLL-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i32 [[TMP0]], -1 4897; UNROLL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0 4898; UNROLL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 4899; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 4900; UNROLL: vector.body: 4901; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE31:%.*]] ] 4902; UNROLL-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE31]] ] 4903; UNROLL-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP41:%.*]], [[PRED_STORE_CONTINUE31]] ] 4904; UNROLL-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP44:%.*]], [[PRED_STORE_CONTINUE31]] ] 4905; UNROLL-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP45:%.*]], [[PRED_STORE_CONTINUE31]] ] 4906; UNROLL-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4> 4907; UNROLL-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]] 4908; UNROLL-NEXT: [[TMP1:%.*]] = add i32 [[OFFSET_IDX]], -1 4909; UNROLL-NEXT: [[TMP2:%.*]] = add i32 [[OFFSET_IDX]], -2 4910; UNROLL-NEXT: [[TMP3:%.*]] = add i32 [[OFFSET_IDX]], -3 4911; UNROLL-NEXT: [[TMP4:%.*]] = add i32 [[OFFSET_IDX]], -4 4912; UNROLL-NEXT: [[TMP5:%.*]] = add i32 [[OFFSET_IDX]], -5 4913; UNROLL-NEXT: [[TMP6:%.*]] = add i32 [[OFFSET_IDX]], -6 4914; UNROLL-NEXT: [[TMP7:%.*]] = add i32 [[OFFSET_IDX]], -7 4915; UNROLL-NEXT: [[TMP8:%.*]] = icmp ule <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]] 4916; UNROLL-NEXT: [[TMP9:%.*]] = icmp ule <4 x i32> [[STEP_ADD]], [[BROADCAST_SPLAT]] 4917; UNROLL-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP8]], i64 0 4918; UNROLL-NEXT: br i1 [[TMP10]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 4919; UNROLL: pred.udiv.if: 4920; UNROLL-NEXT: [[TMP11:%.*]] = udiv i32 219220132, [[OFFSET_IDX]] 4921; UNROLL-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> poison, i32 [[TMP11]], i64 0 4922; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE]] 4923; UNROLL: pred.udiv.continue: 4924; UNROLL-NEXT: [[TMP13:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP12]], [[PRED_UDIV_IF]] ] 4925; UNROLL-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP8]], i64 1 4926; UNROLL-NEXT: br i1 [[TMP14]], label [[PRED_UDIV_IF4:%.*]], label [[PRED_UDIV_CONTINUE5:%.*]] 4927; UNROLL: pred.udiv.if4: 4928; UNROLL-NEXT: [[TMP15:%.*]] = udiv i32 219220132, [[TMP1]] 4929; UNROLL-NEXT: [[TMP16:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP15]], i64 1 4930; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE5]] 4931; UNROLL: pred.udiv.continue5: 4932; UNROLL-NEXT: [[TMP17:%.*]] = phi <4 x i32> [ [[TMP13]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP16]], [[PRED_UDIV_IF4]] ] 4933; UNROLL-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[TMP8]], i64 2 4934; UNROLL-NEXT: br i1 [[TMP18]], label [[PRED_UDIV_IF6:%.*]], label [[PRED_UDIV_CONTINUE7:%.*]] 4935; UNROLL: pred.udiv.if6: 4936; UNROLL-NEXT: [[TMP19:%.*]] = udiv i32 219220132, [[TMP2]] 4937; UNROLL-NEXT: [[TMP20:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP19]], i64 2 4938; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE7]] 4939; UNROLL: pred.udiv.continue7: 4940; UNROLL-NEXT: [[TMP21:%.*]] = phi <4 x i32> [ [[TMP17]], [[PRED_UDIV_CONTINUE5]] ], [ [[TMP20]], [[PRED_UDIV_IF6]] ] 4941; UNROLL-NEXT: [[TMP22:%.*]] = extractelement <4 x i1> [[TMP8]], i64 3 4942; UNROLL-NEXT: br i1 [[TMP22]], label [[PRED_UDIV_IF8:%.*]], label [[PRED_UDIV_CONTINUE9:%.*]] 4943; UNROLL: pred.udiv.if8: 4944; UNROLL-NEXT: [[TMP23:%.*]] = udiv i32 219220132, [[TMP3]] 4945; UNROLL-NEXT: [[TMP24:%.*]] = insertelement <4 x i32> [[TMP21]], i32 [[TMP23]], i64 3 4946; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE9]] 4947; UNROLL: pred.udiv.continue9: 4948; UNROLL-NEXT: [[TMP25:%.*]] = phi <4 x i32> [ [[TMP21]], [[PRED_UDIV_CONTINUE7]] ], [ [[TMP24]], [[PRED_UDIV_IF8]] ] 4949; UNROLL-NEXT: [[TMP26:%.*]] = extractelement <4 x i1> [[TMP9]], i64 0 4950; UNROLL-NEXT: br i1 [[TMP26]], label [[PRED_UDIV_IF10:%.*]], label [[PRED_UDIV_CONTINUE11:%.*]] 4951; UNROLL: pred.udiv.if10: 4952; UNROLL-NEXT: [[TMP27:%.*]] = udiv i32 219220132, [[TMP4]] 4953; UNROLL-NEXT: [[TMP28:%.*]] = insertelement <4 x i32> poison, i32 [[TMP27]], i64 0 4954; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE11]] 4955; UNROLL: pred.udiv.continue11: 4956; UNROLL-NEXT: [[TMP29:%.*]] = phi <4 x i32> [ poison, [[PRED_UDIV_CONTINUE9]] ], [ [[TMP28]], [[PRED_UDIV_IF10]] ] 4957; UNROLL-NEXT: [[TMP30:%.*]] = extractelement <4 x i1> [[TMP9]], i64 1 4958; UNROLL-NEXT: br i1 [[TMP30]], label [[PRED_UDIV_IF12:%.*]], label [[PRED_UDIV_CONTINUE13:%.*]] 4959; UNROLL: pred.udiv.if12: 4960; UNROLL-NEXT: [[TMP31:%.*]] = udiv i32 219220132, [[TMP5]] 4961; UNROLL-NEXT: [[TMP32:%.*]] = insertelement <4 x i32> [[TMP29]], i32 [[TMP31]], i64 1 4962; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE13]] 4963; UNROLL: pred.udiv.continue13: 4964; UNROLL-NEXT: [[TMP33:%.*]] = phi <4 x i32> [ [[TMP29]], [[PRED_UDIV_CONTINUE11]] ], [ [[TMP32]], [[PRED_UDIV_IF12]] ] 4965; UNROLL-NEXT: [[TMP34:%.*]] = extractelement <4 x i1> [[TMP9]], i64 2 4966; UNROLL-NEXT: br i1 [[TMP34]], label [[PRED_UDIV_IF14:%.*]], label [[PRED_UDIV_CONTINUE15:%.*]] 4967; UNROLL: pred.udiv.if14: 4968; UNROLL-NEXT: [[TMP35:%.*]] = udiv i32 219220132, [[TMP6]] 4969; UNROLL-NEXT: [[TMP36:%.*]] = insertelement <4 x i32> [[TMP33]], i32 [[TMP35]], i64 2 4970; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE15]] 4971; UNROLL: pred.udiv.continue15: 4972; UNROLL-NEXT: [[TMP37:%.*]] = phi <4 x i32> [ [[TMP33]], [[PRED_UDIV_CONTINUE13]] ], [ [[TMP36]], [[PRED_UDIV_IF14]] ] 4973; UNROLL-NEXT: [[TMP38:%.*]] = extractelement <4 x i1> [[TMP9]], i64 3 4974; UNROLL-NEXT: br i1 [[TMP38]], label [[PRED_UDIV_IF16:%.*]], label [[PRED_UDIV_CONTINUE17:%.*]] 4975; UNROLL: pred.udiv.if16: 4976; UNROLL-NEXT: [[TMP39:%.*]] = udiv i32 219220132, [[TMP7]] 4977; UNROLL-NEXT: [[TMP40:%.*]] = insertelement <4 x i32> [[TMP37]], i32 [[TMP39]], i64 3 4978; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE17]] 4979; UNROLL: pred.udiv.continue17: 4980; UNROLL-NEXT: [[TMP41]] = phi <4 x i32> [ [[TMP37]], [[PRED_UDIV_CONTINUE15]] ], [ [[TMP40]], [[PRED_UDIV_IF16]] ] 4981; UNROLL-NEXT: [[TMP42:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP25]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 4982; UNROLL-NEXT: [[TMP43:%.*]] = shufflevector <4 x i32> [[TMP25]], <4 x i32> [[TMP41]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 4983; UNROLL-NEXT: [[TMP44]] = add <4 x i32> [[VEC_PHI]], [[TMP42]] 4984; UNROLL-NEXT: [[TMP45]] = add <4 x i32> [[VEC_PHI3]], [[TMP43]] 4985; UNROLL-NEXT: [[TMP46:%.*]] = extractelement <4 x i1> [[TMP8]], i64 0 4986; UNROLL-NEXT: br i1 [[TMP46]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] 4987; UNROLL: pred.store.if: 4988; UNROLL-NEXT: [[TMP47:%.*]] = sext i32 [[INDEX]] to i64 4989; UNROLL-NEXT: [[TMP48:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 [[TMP47]] 4990; UNROLL-NEXT: store i32 [[OFFSET_IDX]], i32* [[TMP48]], align 4 4991; UNROLL-NEXT: br label [[PRED_STORE_CONTINUE]] 4992; UNROLL: pred.store.continue: 4993; UNROLL-NEXT: [[TMP49:%.*]] = extractelement <4 x i1> [[TMP8]], i64 1 4994; UNROLL-NEXT: br i1 [[TMP49]], label [[PRED_STORE_IF18:%.*]], label [[PRED_STORE_CONTINUE19:%.*]] 4995; UNROLL: pred.store.if18: 4996; UNROLL-NEXT: [[TMP50:%.*]] = or i32 [[INDEX]], 1 4997; UNROLL-NEXT: [[TMP51:%.*]] = sext i32 [[TMP50]] to i64 4998; UNROLL-NEXT: [[TMP52:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 [[TMP51]] 4999; UNROLL-NEXT: store i32 [[TMP1]], i32* [[TMP52]], align 4 5000; UNROLL-NEXT: br label [[PRED_STORE_CONTINUE19]] 5001; UNROLL: pred.store.continue19: 5002; UNROLL-NEXT: [[TMP53:%.*]] = extractelement <4 x i1> [[TMP8]], i64 2 5003; UNROLL-NEXT: br i1 [[TMP53]], label [[PRED_STORE_IF20:%.*]], label [[PRED_STORE_CONTINUE21:%.*]] 5004; UNROLL: pred.store.if20: 5005; UNROLL-NEXT: [[TMP54:%.*]] = or i32 [[INDEX]], 2 5006; UNROLL-NEXT: [[TMP55:%.*]] = sext i32 [[TMP54]] to i64 5007; UNROLL-NEXT: [[TMP56:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 [[TMP55]] 5008; UNROLL-NEXT: store i32 [[TMP2]], i32* [[TMP56]], align 4 5009; UNROLL-NEXT: br label [[PRED_STORE_CONTINUE21]] 5010; UNROLL: pred.store.continue21: 5011; UNROLL-NEXT: [[TMP57:%.*]] = extractelement <4 x i1> [[TMP8]], i64 3 5012; UNROLL-NEXT: br i1 [[TMP57]], label [[PRED_STORE_IF22:%.*]], label [[PRED_STORE_CONTINUE23:%.*]] 5013; UNROLL: pred.store.if22: 5014; UNROLL-NEXT: [[TMP58:%.*]] = or i32 [[INDEX]], 3 5015; UNROLL-NEXT: [[TMP59:%.*]] = sext i32 [[TMP58]] to i64 5016; UNROLL-NEXT: [[TMP60:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 [[TMP59]] 5017; UNROLL-NEXT: store i32 [[TMP3]], i32* [[TMP60]], align 4 5018; UNROLL-NEXT: br label [[PRED_STORE_CONTINUE23]] 5019; UNROLL: pred.store.continue23: 5020; UNROLL-NEXT: [[TMP61:%.*]] = extractelement <4 x i1> [[TMP9]], i64 0 5021; UNROLL-NEXT: br i1 [[TMP61]], label [[PRED_STORE_IF24:%.*]], label [[PRED_STORE_CONTINUE25:%.*]] 5022; UNROLL: pred.store.if24: 5023; UNROLL-NEXT: [[TMP62:%.*]] = or i32 [[INDEX]], 4 5024; UNROLL-NEXT: [[TMP63:%.*]] = sext i32 [[TMP62]] to i64 5025; UNROLL-NEXT: [[TMP64:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 [[TMP63]] 5026; UNROLL-NEXT: store i32 [[TMP4]], i32* [[TMP64]], align 4 5027; UNROLL-NEXT: br label [[PRED_STORE_CONTINUE25]] 5028; UNROLL: pred.store.continue25: 5029; UNROLL-NEXT: [[TMP65:%.*]] = extractelement <4 x i1> [[TMP9]], i64 1 5030; UNROLL-NEXT: br i1 [[TMP65]], label [[PRED_STORE_IF26:%.*]], label [[PRED_STORE_CONTINUE27:%.*]] 5031; UNROLL: pred.store.if26: 5032; UNROLL-NEXT: [[TMP66:%.*]] = or i32 [[INDEX]], 5 5033; UNROLL-NEXT: [[TMP67:%.*]] = sext i32 [[TMP66]] to i64 5034; UNROLL-NEXT: [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 [[TMP67]] 5035; UNROLL-NEXT: store i32 [[TMP5]], i32* [[TMP68]], align 4 5036; UNROLL-NEXT: br label [[PRED_STORE_CONTINUE27]] 5037; UNROLL: pred.store.continue27: 5038; UNROLL-NEXT: [[TMP69:%.*]] = extractelement <4 x i1> [[TMP9]], i64 2 5039; UNROLL-NEXT: br i1 [[TMP69]], label [[PRED_STORE_IF28:%.*]], label [[PRED_STORE_CONTINUE29:%.*]] 5040; UNROLL: pred.store.if28: 5041; UNROLL-NEXT: [[TMP70:%.*]] = or i32 [[INDEX]], 6 5042; UNROLL-NEXT: [[TMP71:%.*]] = sext i32 [[TMP70]] to i64 5043; UNROLL-NEXT: [[TMP72:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 [[TMP71]] 5044; UNROLL-NEXT: store i32 [[TMP6]], i32* [[TMP72]], align 4 5045; UNROLL-NEXT: br label [[PRED_STORE_CONTINUE29]] 5046; UNROLL: pred.store.continue29: 5047; UNROLL-NEXT: [[TMP73:%.*]] = extractelement <4 x i1> [[TMP9]], i64 3 5048; UNROLL-NEXT: br i1 [[TMP73]], label [[PRED_STORE_IF30:%.*]], label [[PRED_STORE_CONTINUE31]] 5049; UNROLL: pred.store.if30: 5050; UNROLL-NEXT: [[TMP74:%.*]] = or i32 [[INDEX]], 7 5051; UNROLL-NEXT: [[TMP75:%.*]] = sext i32 [[TMP74]] to i64 5052; UNROLL-NEXT: [[TMP76:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 [[TMP75]] 5053; UNROLL-NEXT: store i32 [[TMP7]], i32* [[TMP76]], align 4 5054; UNROLL-NEXT: br label [[PRED_STORE_CONTINUE31]] 5055; UNROLL: pred.store.continue31: 5056; UNROLL-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8 5057; UNROLL-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 8, i32 8, i32 8, i32 8> 5058; UNROLL-NEXT: [[TMP77:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 5059; UNROLL-NEXT: br i1 [[TMP77]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF52]], !llvm.loop [[LOOP56:![0-9]+]] 5060; UNROLL: middle.block: 5061; UNROLL-NEXT: [[TMP78:%.*]] = select <4 x i1> [[TMP9]], <4 x i32> [[TMP45]], <4 x i32> [[VEC_PHI3]] 5062; UNROLL-NEXT: [[TMP79:%.*]] = select <4 x i1> [[TMP8]], <4 x i32> [[TMP44]], <4 x i32> [[VEC_PHI]] 5063; UNROLL-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP78]], [[TMP79]] 5064; UNROLL-NEXT: [[TMP80:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]]) 5065; UNROLL-NEXT: br i1 true, label [[BB1:%.*]], label [[SCALAR_PH]] 5066; UNROLL: scalar.ph: 5067; UNROLL-NEXT: br label [[BB2:%.*]] 5068; UNROLL: bb1: 5069; UNROLL-NEXT: [[VAR:%.*]] = phi i32 [ undef, [[BB2]] ], [ [[TMP80]], [[MIDDLE_BLOCK]] ] 5070; UNROLL-NEXT: ret i32 [[VAR]] 5071; UNROLL: bb2: 5072; UNROLL-NEXT: br i1 undef, label [[BB1]], label [[BB2]], !prof [[PROF54]], !llvm.loop [[LOOP57:![0-9]+]] 5073; 5074; UNROLL-NO-IC-LABEL: @sink_into_replication_region_multiple( 5075; UNROLL-NO-IC-NEXT: bb: 5076; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i32 [[Y:%.*]], 1 5077; UNROLL-NO-IC-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[Y]], i32 1) 5078; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]] 5079; UNROLL-NO-IC-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5080; UNROLL-NO-IC: vector.ph: 5081; UNROLL-NO-IC-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 7 5082; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 8 5083; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]] 5084; UNROLL-NO-IC-NEXT: [[IND_END:%.*]] = sub i32 [[Y]], [[N_VEC]] 5085; UNROLL-NO-IC-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[TMP1]], 1 5086; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i32 0 5087; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 5088; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 5089; UNROLL-NO-IC: vector.body: 5090; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE31:%.*]] ] 5091; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE31]] ] 5092; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP43:%.*]], [[PRED_STORE_CONTINUE31]] ] 5093; UNROLL-NO-IC-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP46:%.*]], [[PRED_STORE_CONTINUE31]] ] 5094; UNROLL-NO-IC-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP47:%.*]], [[PRED_STORE_CONTINUE31]] ] 5095; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4> 5096; UNROLL-NO-IC-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]] 5097; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add i32 [[OFFSET_IDX]], 0 5098; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add i32 [[OFFSET_IDX]], -1 5099; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = add i32 [[OFFSET_IDX]], -2 5100; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = add i32 [[OFFSET_IDX]], -3 5101; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = add i32 [[OFFSET_IDX]], -4 5102; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = add i32 [[OFFSET_IDX]], -5 5103; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = add i32 [[OFFSET_IDX]], -6 5104; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = add i32 [[OFFSET_IDX]], -7 5105; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = icmp ule <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]] 5106; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = icmp ule <4 x i32> [[STEP_ADD]], [[BROADCAST_SPLAT]] 5107; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP10]], i32 0 5108; UNROLL-NO-IC-NEXT: br i1 [[TMP12]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 5109; UNROLL-NO-IC: pred.udiv.if: 5110; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = udiv i32 219220132, [[TMP2]] 5111; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = insertelement <4 x i32> poison, i32 [[TMP13]], i32 0 5112; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE]] 5113; UNROLL-NO-IC: pred.udiv.continue: 5114; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP14]], [[PRED_UDIV_IF]] ] 5115; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP10]], i32 1 5116; UNROLL-NO-IC-NEXT: br i1 [[TMP16]], label [[PRED_UDIV_IF4:%.*]], label [[PRED_UDIV_CONTINUE5:%.*]] 5117; UNROLL-NO-IC: pred.udiv.if4: 5118; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = udiv i32 219220132, [[TMP3]] 5119; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP17]], i32 1 5120; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE5]] 5121; UNROLL-NO-IC: pred.udiv.continue5: 5122; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = phi <4 x i32> [ [[TMP15]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP18]], [[PRED_UDIV_IF4]] ] 5123; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP10]], i32 2 5124; UNROLL-NO-IC-NEXT: br i1 [[TMP20]], label [[PRED_UDIV_IF6:%.*]], label [[PRED_UDIV_CONTINUE7:%.*]] 5125; UNROLL-NO-IC: pred.udiv.if6: 5126; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = udiv i32 219220132, [[TMP4]] 5127; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP21]], i32 2 5128; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE7]] 5129; UNROLL-NO-IC: pred.udiv.continue7: 5130; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = phi <4 x i32> [ [[TMP19]], [[PRED_UDIV_CONTINUE5]] ], [ [[TMP22]], [[PRED_UDIV_IF6]] ] 5131; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = extractelement <4 x i1> [[TMP10]], i32 3 5132; UNROLL-NO-IC-NEXT: br i1 [[TMP24]], label [[PRED_UDIV_IF8:%.*]], label [[PRED_UDIV_CONTINUE9:%.*]] 5133; UNROLL-NO-IC: pred.udiv.if8: 5134; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = udiv i32 219220132, [[TMP5]] 5135; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = insertelement <4 x i32> [[TMP23]], i32 [[TMP25]], i32 3 5136; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE9]] 5137; UNROLL-NO-IC: pred.udiv.continue9: 5138; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = phi <4 x i32> [ [[TMP23]], [[PRED_UDIV_CONTINUE7]] ], [ [[TMP26]], [[PRED_UDIV_IF8]] ] 5139; UNROLL-NO-IC-NEXT: [[TMP28:%.*]] = extractelement <4 x i1> [[TMP11]], i32 0 5140; UNROLL-NO-IC-NEXT: br i1 [[TMP28]], label [[PRED_UDIV_IF10:%.*]], label [[PRED_UDIV_CONTINUE11:%.*]] 5141; UNROLL-NO-IC: pred.udiv.if10: 5142; UNROLL-NO-IC-NEXT: [[TMP29:%.*]] = udiv i32 219220132, [[TMP6]] 5143; UNROLL-NO-IC-NEXT: [[TMP30:%.*]] = insertelement <4 x i32> poison, i32 [[TMP29]], i32 0 5144; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE11]] 5145; UNROLL-NO-IC: pred.udiv.continue11: 5146; UNROLL-NO-IC-NEXT: [[TMP31:%.*]] = phi <4 x i32> [ poison, [[PRED_UDIV_CONTINUE9]] ], [ [[TMP30]], [[PRED_UDIV_IF10]] ] 5147; UNROLL-NO-IC-NEXT: [[TMP32:%.*]] = extractelement <4 x i1> [[TMP11]], i32 1 5148; UNROLL-NO-IC-NEXT: br i1 [[TMP32]], label [[PRED_UDIV_IF12:%.*]], label [[PRED_UDIV_CONTINUE13:%.*]] 5149; UNROLL-NO-IC: pred.udiv.if12: 5150; UNROLL-NO-IC-NEXT: [[TMP33:%.*]] = udiv i32 219220132, [[TMP7]] 5151; UNROLL-NO-IC-NEXT: [[TMP34:%.*]] = insertelement <4 x i32> [[TMP31]], i32 [[TMP33]], i32 1 5152; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE13]] 5153; UNROLL-NO-IC: pred.udiv.continue13: 5154; UNROLL-NO-IC-NEXT: [[TMP35:%.*]] = phi <4 x i32> [ [[TMP31]], [[PRED_UDIV_CONTINUE11]] ], [ [[TMP34]], [[PRED_UDIV_IF12]] ] 5155; UNROLL-NO-IC-NEXT: [[TMP36:%.*]] = extractelement <4 x i1> [[TMP11]], i32 2 5156; UNROLL-NO-IC-NEXT: br i1 [[TMP36]], label [[PRED_UDIV_IF14:%.*]], label [[PRED_UDIV_CONTINUE15:%.*]] 5157; UNROLL-NO-IC: pred.udiv.if14: 5158; UNROLL-NO-IC-NEXT: [[TMP37:%.*]] = udiv i32 219220132, [[TMP8]] 5159; UNROLL-NO-IC-NEXT: [[TMP38:%.*]] = insertelement <4 x i32> [[TMP35]], i32 [[TMP37]], i32 2 5160; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE15]] 5161; UNROLL-NO-IC: pred.udiv.continue15: 5162; UNROLL-NO-IC-NEXT: [[TMP39:%.*]] = phi <4 x i32> [ [[TMP35]], [[PRED_UDIV_CONTINUE13]] ], [ [[TMP38]], [[PRED_UDIV_IF14]] ] 5163; UNROLL-NO-IC-NEXT: [[TMP40:%.*]] = extractelement <4 x i1> [[TMP11]], i32 3 5164; UNROLL-NO-IC-NEXT: br i1 [[TMP40]], label [[PRED_UDIV_IF16:%.*]], label [[PRED_UDIV_CONTINUE17:%.*]] 5165; UNROLL-NO-IC: pred.udiv.if16: 5166; UNROLL-NO-IC-NEXT: [[TMP41:%.*]] = udiv i32 219220132, [[TMP9]] 5167; UNROLL-NO-IC-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> [[TMP39]], i32 [[TMP41]], i32 3 5168; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE17]] 5169; UNROLL-NO-IC: pred.udiv.continue17: 5170; UNROLL-NO-IC-NEXT: [[TMP43]] = phi <4 x i32> [ [[TMP39]], [[PRED_UDIV_CONTINUE15]] ], [ [[TMP42]], [[PRED_UDIV_IF16]] ] 5171; UNROLL-NO-IC-NEXT: [[TMP44:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP27]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 5172; UNROLL-NO-IC-NEXT: [[TMP45:%.*]] = shufflevector <4 x i32> [[TMP27]], <4 x i32> [[TMP43]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 5173; UNROLL-NO-IC-NEXT: [[TMP46]] = add <4 x i32> [[VEC_PHI]], [[TMP44]] 5174; UNROLL-NO-IC-NEXT: [[TMP47]] = add <4 x i32> [[VEC_PHI3]], [[TMP45]] 5175; UNROLL-NO-IC-NEXT: [[TMP48:%.*]] = extractelement <4 x i1> [[TMP10]], i32 0 5176; UNROLL-NO-IC-NEXT: br i1 [[TMP48]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] 5177; UNROLL-NO-IC: pred.store.if: 5178; UNROLL-NO-IC-NEXT: [[TMP49:%.*]] = add i32 [[INDEX]], 0 5179; UNROLL-NO-IC-NEXT: [[TMP50:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i32 [[TMP49]] 5180; UNROLL-NO-IC-NEXT: store i32 [[TMP2]], i32* [[TMP50]], align 4 5181; UNROLL-NO-IC-NEXT: br label [[PRED_STORE_CONTINUE]] 5182; UNROLL-NO-IC: pred.store.continue: 5183; UNROLL-NO-IC-NEXT: [[TMP51:%.*]] = extractelement <4 x i1> [[TMP10]], i32 1 5184; UNROLL-NO-IC-NEXT: br i1 [[TMP51]], label [[PRED_STORE_IF18:%.*]], label [[PRED_STORE_CONTINUE19:%.*]] 5185; UNROLL-NO-IC: pred.store.if18: 5186; UNROLL-NO-IC-NEXT: [[TMP52:%.*]] = add i32 [[INDEX]], 1 5187; UNROLL-NO-IC-NEXT: [[TMP53:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[TMP52]] 5188; UNROLL-NO-IC-NEXT: store i32 [[TMP3]], i32* [[TMP53]], align 4 5189; UNROLL-NO-IC-NEXT: br label [[PRED_STORE_CONTINUE19]] 5190; UNROLL-NO-IC: pred.store.continue19: 5191; UNROLL-NO-IC-NEXT: [[TMP54:%.*]] = extractelement <4 x i1> [[TMP10]], i32 2 5192; UNROLL-NO-IC-NEXT: br i1 [[TMP54]], label [[PRED_STORE_IF20:%.*]], label [[PRED_STORE_CONTINUE21:%.*]] 5193; UNROLL-NO-IC: pred.store.if20: 5194; UNROLL-NO-IC-NEXT: [[TMP55:%.*]] = add i32 [[INDEX]], 2 5195; UNROLL-NO-IC-NEXT: [[TMP56:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[TMP55]] 5196; UNROLL-NO-IC-NEXT: store i32 [[TMP4]], i32* [[TMP56]], align 4 5197; UNROLL-NO-IC-NEXT: br label [[PRED_STORE_CONTINUE21]] 5198; UNROLL-NO-IC: pred.store.continue21: 5199; UNROLL-NO-IC-NEXT: [[TMP57:%.*]] = extractelement <4 x i1> [[TMP10]], i32 3 5200; UNROLL-NO-IC-NEXT: br i1 [[TMP57]], label [[PRED_STORE_IF22:%.*]], label [[PRED_STORE_CONTINUE23:%.*]] 5201; UNROLL-NO-IC: pred.store.if22: 5202; UNROLL-NO-IC-NEXT: [[TMP58:%.*]] = add i32 [[INDEX]], 3 5203; UNROLL-NO-IC-NEXT: [[TMP59:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[TMP58]] 5204; UNROLL-NO-IC-NEXT: store i32 [[TMP5]], i32* [[TMP59]], align 4 5205; UNROLL-NO-IC-NEXT: br label [[PRED_STORE_CONTINUE23]] 5206; UNROLL-NO-IC: pred.store.continue23: 5207; UNROLL-NO-IC-NEXT: [[TMP60:%.*]] = extractelement <4 x i1> [[TMP11]], i32 0 5208; UNROLL-NO-IC-NEXT: br i1 [[TMP60]], label [[PRED_STORE_IF24:%.*]], label [[PRED_STORE_CONTINUE25:%.*]] 5209; UNROLL-NO-IC: pred.store.if24: 5210; UNROLL-NO-IC-NEXT: [[TMP61:%.*]] = add i32 [[INDEX]], 4 5211; UNROLL-NO-IC-NEXT: [[TMP62:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[TMP61]] 5212; UNROLL-NO-IC-NEXT: store i32 [[TMP6]], i32* [[TMP62]], align 4 5213; UNROLL-NO-IC-NEXT: br label [[PRED_STORE_CONTINUE25]] 5214; UNROLL-NO-IC: pred.store.continue25: 5215; UNROLL-NO-IC-NEXT: [[TMP63:%.*]] = extractelement <4 x i1> [[TMP11]], i32 1 5216; UNROLL-NO-IC-NEXT: br i1 [[TMP63]], label [[PRED_STORE_IF26:%.*]], label [[PRED_STORE_CONTINUE27:%.*]] 5217; UNROLL-NO-IC: pred.store.if26: 5218; UNROLL-NO-IC-NEXT: [[TMP64:%.*]] = add i32 [[INDEX]], 5 5219; UNROLL-NO-IC-NEXT: [[TMP65:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[TMP64]] 5220; UNROLL-NO-IC-NEXT: store i32 [[TMP7]], i32* [[TMP65]], align 4 5221; UNROLL-NO-IC-NEXT: br label [[PRED_STORE_CONTINUE27]] 5222; UNROLL-NO-IC: pred.store.continue27: 5223; UNROLL-NO-IC-NEXT: [[TMP66:%.*]] = extractelement <4 x i1> [[TMP11]], i32 2 5224; UNROLL-NO-IC-NEXT: br i1 [[TMP66]], label [[PRED_STORE_IF28:%.*]], label [[PRED_STORE_CONTINUE29:%.*]] 5225; UNROLL-NO-IC: pred.store.if28: 5226; UNROLL-NO-IC-NEXT: [[TMP67:%.*]] = add i32 [[INDEX]], 6 5227; UNROLL-NO-IC-NEXT: [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[TMP67]] 5228; UNROLL-NO-IC-NEXT: store i32 [[TMP8]], i32* [[TMP68]], align 4 5229; UNROLL-NO-IC-NEXT: br label [[PRED_STORE_CONTINUE29]] 5230; UNROLL-NO-IC: pred.store.continue29: 5231; UNROLL-NO-IC-NEXT: [[TMP69:%.*]] = extractelement <4 x i1> [[TMP11]], i32 3 5232; UNROLL-NO-IC-NEXT: br i1 [[TMP69]], label [[PRED_STORE_IF30:%.*]], label [[PRED_STORE_CONTINUE31]] 5233; UNROLL-NO-IC: pred.store.if30: 5234; UNROLL-NO-IC-NEXT: [[TMP70:%.*]] = add i32 [[INDEX]], 7 5235; UNROLL-NO-IC-NEXT: [[TMP71:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[TMP70]] 5236; UNROLL-NO-IC-NEXT: store i32 [[TMP9]], i32* [[TMP71]], align 4 5237; UNROLL-NO-IC-NEXT: br label [[PRED_STORE_CONTINUE31]] 5238; UNROLL-NO-IC: pred.store.continue31: 5239; UNROLL-NO-IC-NEXT: [[TMP72:%.*]] = select <4 x i1> [[TMP10]], <4 x i32> [[TMP46]], <4 x i32> [[VEC_PHI]] 5240; UNROLL-NO-IC-NEXT: [[TMP73:%.*]] = select <4 x i1> [[TMP11]], <4 x i32> [[TMP47]], <4 x i32> [[VEC_PHI3]] 5241; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8 5242; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], <i32 4, i32 4, i32 4, i32 4> 5243; UNROLL-NO-IC-NEXT: [[TMP74:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 5244; UNROLL-NO-IC-NEXT: br i1 [[TMP74]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF52]], !llvm.loop [[LOOP56:![0-9]+]] 5245; UNROLL-NO-IC: middle.block: 5246; UNROLL-NO-IC-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP73]], [[TMP72]] 5247; UNROLL-NO-IC-NEXT: [[TMP75:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]]) 5248; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP43]], i32 3 5249; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP43]], i32 2 5250; UNROLL-NO-IC-NEXT: br i1 true, label [[BB1:%.*]], label [[SCALAR_PH]] 5251; UNROLL-NO-IC: scalar.ph: 5252; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[BB:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 5253; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[Y]], [[BB]] ] 5254; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[BB]] ] 5255; UNROLL-NO-IC-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[BB]] ], [ [[TMP75]], [[MIDDLE_BLOCK]] ] 5256; UNROLL-NO-IC-NEXT: br label [[BB2:%.*]] 5257; UNROLL-NO-IC: bb1: 5258; UNROLL-NO-IC-NEXT: [[VAR:%.*]] = phi i32 [ [[VAR6:%.*]], [[BB2]] ], [ [[TMP75]], [[MIDDLE_BLOCK]] ] 5259; UNROLL-NO-IC-NEXT: ret i32 [[VAR]] 5260; UNROLL-NO-IC: bb2: 5261; UNROLL-NO-IC-NEXT: [[VAR3:%.*]] = phi i32 [ [[VAR8:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 5262; UNROLL-NO-IC-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ] 5263; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[VAR7:%.*]], [[BB2]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ] 5264; UNROLL-NO-IC-NEXT: [[VAR5:%.*]] = phi i32 [ [[VAR6]], [[BB2]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 5265; UNROLL-NO-IC-NEXT: [[G:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[IV]] 5266; UNROLL-NO-IC-NEXT: [[VAR6]] = add i32 [[VAR5]], [[SCALAR_RECUR]] 5267; UNROLL-NO-IC-NEXT: [[VAR7]] = udiv i32 219220132, [[VAR3]] 5268; UNROLL-NO-IC-NEXT: store i32 [[VAR3]], i32* [[G]], align 4 5269; UNROLL-NO-IC-NEXT: [[VAR8]] = add nsw i32 [[VAR3]], -1 5270; UNROLL-NO-IC-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], 1 5271; UNROLL-NO-IC-NEXT: [[VAR9:%.*]] = icmp slt i32 [[VAR3]], 2 5272; UNROLL-NO-IC-NEXT: br i1 [[VAR9]], label [[BB1]], label [[BB2]], !prof [[PROF54]], !llvm.loop [[LOOP57:![0-9]+]] 5273; 5274; UNROLL-NO-VF-LABEL: @sink_into_replication_region_multiple( 5275; UNROLL-NO-VF-NEXT: bb: 5276; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i32 [[Y:%.*]], 1 5277; UNROLL-NO-VF-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[Y]], i32 1) 5278; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]] 5279; UNROLL-NO-VF-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5280; UNROLL-NO-VF: vector.ph: 5281; UNROLL-NO-VF-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 1 5282; UNROLL-NO-VF-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2 5283; UNROLL-NO-VF-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]] 5284; UNROLL-NO-VF-NEXT: [[IND_END:%.*]] = sub i32 [[Y]], [[N_VEC]] 5285; UNROLL-NO-VF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[TMP1]], 1 5286; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 5287; UNROLL-NO-VF: vector.body: 5288; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE10:%.*]] ] 5289; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[PRED_STORE_CONTINUE10]] ] 5290; UNROLL-NO-VF-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[PRED_STORE_CONTINUE10]] ] 5291; UNROLL-NO-VF-NEXT: [[VEC_PHI2:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[PRED_STORE_CONTINUE10]] ] 5292; UNROLL-NO-VF-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]] 5293; UNROLL-NO-VF-NEXT: [[INDUCTION4:%.*]] = add i32 [[OFFSET_IDX]], 0 5294; UNROLL-NO-VF-NEXT: [[INDUCTION5:%.*]] = add i32 [[OFFSET_IDX]], -1 5295; UNROLL-NO-VF-NEXT: [[VEC_IV:%.*]] = add i32 [[INDEX]], 0 5296; UNROLL-NO-VF-NEXT: [[VEC_IV6:%.*]] = add i32 [[INDEX]], 1 5297; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = icmp ule i32 [[VEC_IV]], [[TRIP_COUNT_MINUS_1]] 5298; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = icmp ule i32 [[VEC_IV6]], [[TRIP_COUNT_MINUS_1]] 5299; UNROLL-NO-VF-NEXT: br i1 [[TMP2]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 5300; UNROLL-NO-VF: pred.udiv.if: 5301; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = udiv i32 219220132, [[INDUCTION4]] 5302; UNROLL-NO-VF-NEXT: br label [[PRED_UDIV_CONTINUE]] 5303; UNROLL-NO-VF: pred.udiv.continue: 5304; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP4]], [[PRED_UDIV_IF]] ] 5305; UNROLL-NO-VF-NEXT: br i1 [[TMP3]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8:%.*]] 5306; UNROLL-NO-VF: pred.udiv.if7: 5307; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = udiv i32 219220132, [[INDUCTION5]] 5308; UNROLL-NO-VF-NEXT: br label [[PRED_UDIV_CONTINUE8]] 5309; UNROLL-NO-VF: pred.udiv.continue8: 5310; UNROLL-NO-VF-NEXT: [[TMP7]] = phi i32 [ poison, [[PRED_UDIV_CONTINUE]] ], [ [[TMP6]], [[PRED_UDIV_IF7]] ] 5311; UNROLL-NO-VF-NEXT: [[TMP8]] = add i32 [[VEC_PHI]], [[VECTOR_RECUR]] 5312; UNROLL-NO-VF-NEXT: [[TMP9]] = add i32 [[VEC_PHI2]], [[TMP5]] 5313; UNROLL-NO-VF-NEXT: br i1 [[TMP2]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] 5314; UNROLL-NO-VF: pred.store.if: 5315; UNROLL-NO-VF-NEXT: [[INDUCTION:%.*]] = add i32 [[INDEX]], 0 5316; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i32 [[INDUCTION]] 5317; UNROLL-NO-VF-NEXT: store i32 [[INDUCTION4]], i32* [[TMP10]], align 4 5318; UNROLL-NO-VF-NEXT: br label [[PRED_STORE_CONTINUE]] 5319; UNROLL-NO-VF: pred.store.continue: 5320; UNROLL-NO-VF-NEXT: br i1 [[TMP3]], label [[PRED_STORE_IF9:%.*]], label [[PRED_STORE_CONTINUE10]] 5321; UNROLL-NO-VF: pred.store.if9: 5322; UNROLL-NO-VF-NEXT: [[INDUCTION3:%.*]] = add i32 [[INDEX]], 1 5323; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[INDUCTION3]] 5324; UNROLL-NO-VF-NEXT: store i32 [[INDUCTION5]], i32* [[TMP11]], align 4 5325; UNROLL-NO-VF-NEXT: br label [[PRED_STORE_CONTINUE10]] 5326; UNROLL-NO-VF: pred.store.continue10: 5327; UNROLL-NO-VF-NEXT: [[TMP12:%.*]] = select i1 [[TMP2]], i32 [[TMP8]], i32 [[VEC_PHI]] 5328; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = select i1 [[TMP3]], i32 [[TMP9]], i32 [[VEC_PHI2]] 5329; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2 5330; UNROLL-NO-VF-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 5331; UNROLL-NO-VF-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF51]], !llvm.loop [[LOOP55:![0-9]+]] 5332; UNROLL-NO-VF: middle.block: 5333; UNROLL-NO-VF-NEXT: [[BIN_RDX:%.*]] = add i32 [[TMP13]], [[TMP12]] 5334; UNROLL-NO-VF-NEXT: br i1 true, label [[BB1:%.*]], label [[SCALAR_PH]] 5335; UNROLL-NO-VF: scalar.ph: 5336; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[BB:%.*]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ] 5337; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[Y]], [[BB]] ] 5338; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[BB]] ] 5339; UNROLL-NO-VF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[BB]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ] 5340; UNROLL-NO-VF-NEXT: br label [[BB2:%.*]] 5341; UNROLL-NO-VF: bb1: 5342; UNROLL-NO-VF-NEXT: [[VAR:%.*]] = phi i32 [ [[VAR6:%.*]], [[BB2]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ] 5343; UNROLL-NO-VF-NEXT: ret i32 [[VAR]] 5344; UNROLL-NO-VF: bb2: 5345; UNROLL-NO-VF-NEXT: [[VAR3:%.*]] = phi i32 [ [[VAR8:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 5346; UNROLL-NO-VF-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ] 5347; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[VAR7:%.*]], [[BB2]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ] 5348; UNROLL-NO-VF-NEXT: [[VAR5:%.*]] = phi i32 [ [[VAR6]], [[BB2]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 5349; UNROLL-NO-VF-NEXT: [[G:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[IV]] 5350; UNROLL-NO-VF-NEXT: [[VAR6]] = add i32 [[VAR5]], [[SCALAR_RECUR]] 5351; UNROLL-NO-VF-NEXT: [[VAR7]] = udiv i32 219220132, [[VAR3]] 5352; UNROLL-NO-VF-NEXT: store i32 [[VAR3]], i32* [[G]], align 4 5353; UNROLL-NO-VF-NEXT: [[VAR8]] = add nsw i32 [[VAR3]], -1 5354; UNROLL-NO-VF-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], 1 5355; UNROLL-NO-VF-NEXT: [[VAR9:%.*]] = icmp slt i32 [[VAR3]], 2 5356; UNROLL-NO-VF-NEXT: br i1 [[VAR9]], label [[BB1]], label [[BB2]], !prof [[PROF53]], !llvm.loop [[LOOP56:![0-9]+]] 5357; 5358; SINK-AFTER-LABEL: @sink_into_replication_region_multiple( 5359; SINK-AFTER-NEXT: bb: 5360; SINK-AFTER-NEXT: [[TMP0:%.*]] = add i32 [[Y:%.*]], 1 5361; SINK-AFTER-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[Y]], i32 1) 5362; SINK-AFTER-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]] 5363; SINK-AFTER-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5364; SINK-AFTER: vector.ph: 5365; SINK-AFTER-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 3 5366; SINK-AFTER-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 4 5367; SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]] 5368; SINK-AFTER-NEXT: [[IND_END:%.*]] = sub i32 [[Y]], [[N_VEC]] 5369; SINK-AFTER-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[TMP1]], 1 5370; SINK-AFTER-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i32 0 5371; SINK-AFTER-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 5372; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 5373; SINK-AFTER: vector.body: 5374; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE13:%.*]] ] 5375; SINK-AFTER-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE13]] ] 5376; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[PRED_STORE_CONTINUE13]] ] 5377; SINK-AFTER-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[PRED_STORE_CONTINUE13]] ] 5378; SINK-AFTER-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]] 5379; SINK-AFTER-NEXT: [[TMP2:%.*]] = add i32 [[OFFSET_IDX]], 0 5380; SINK-AFTER-NEXT: [[TMP3:%.*]] = add i32 [[OFFSET_IDX]], -1 5381; SINK-AFTER-NEXT: [[TMP4:%.*]] = add i32 [[OFFSET_IDX]], -2 5382; SINK-AFTER-NEXT: [[TMP5:%.*]] = add i32 [[OFFSET_IDX]], -3 5383; SINK-AFTER-NEXT: [[TMP6:%.*]] = icmp ule <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]] 5384; SINK-AFTER-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i32 0 5385; SINK-AFTER-NEXT: br i1 [[TMP7]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 5386; SINK-AFTER: pred.udiv.if: 5387; SINK-AFTER-NEXT: [[TMP8:%.*]] = udiv i32 219220132, [[TMP2]] 5388; SINK-AFTER-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> poison, i32 [[TMP8]], i32 0 5389; SINK-AFTER-NEXT: br label [[PRED_UDIV_CONTINUE]] 5390; SINK-AFTER: pred.udiv.continue: 5391; SINK-AFTER-NEXT: [[TMP10:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP9]], [[PRED_UDIV_IF]] ] 5392; SINK-AFTER-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP6]], i32 1 5393; SINK-AFTER-NEXT: br i1 [[TMP11]], label [[PRED_UDIV_IF2:%.*]], label [[PRED_UDIV_CONTINUE3:%.*]] 5394; SINK-AFTER: pred.udiv.if2: 5395; SINK-AFTER-NEXT: [[TMP12:%.*]] = udiv i32 219220132, [[TMP3]] 5396; SINK-AFTER-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP12]], i32 1 5397; SINK-AFTER-NEXT: br label [[PRED_UDIV_CONTINUE3]] 5398; SINK-AFTER: pred.udiv.continue3: 5399; SINK-AFTER-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP10]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP13]], [[PRED_UDIV_IF2]] ] 5400; SINK-AFTER-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP6]], i32 2 5401; SINK-AFTER-NEXT: br i1 [[TMP15]], label [[PRED_UDIV_IF4:%.*]], label [[PRED_UDIV_CONTINUE5:%.*]] 5402; SINK-AFTER: pred.udiv.if4: 5403; SINK-AFTER-NEXT: [[TMP16:%.*]] = udiv i32 219220132, [[TMP4]] 5404; SINK-AFTER-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP16]], i32 2 5405; SINK-AFTER-NEXT: br label [[PRED_UDIV_CONTINUE5]] 5406; SINK-AFTER: pred.udiv.continue5: 5407; SINK-AFTER-NEXT: [[TMP18:%.*]] = phi <4 x i32> [ [[TMP14]], [[PRED_UDIV_CONTINUE3]] ], [ [[TMP17]], [[PRED_UDIV_IF4]] ] 5408; SINK-AFTER-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP6]], i32 3 5409; SINK-AFTER-NEXT: br i1 [[TMP19]], label [[PRED_UDIV_IF6:%.*]], label [[PRED_UDIV_CONTINUE7:%.*]] 5410; SINK-AFTER: pred.udiv.if6: 5411; SINK-AFTER-NEXT: [[TMP20:%.*]] = udiv i32 219220132, [[TMP5]] 5412; SINK-AFTER-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP18]], i32 [[TMP20]], i32 3 5413; SINK-AFTER-NEXT: br label [[PRED_UDIV_CONTINUE7]] 5414; SINK-AFTER: pred.udiv.continue7: 5415; SINK-AFTER-NEXT: [[TMP22]] = phi <4 x i32> [ [[TMP18]], [[PRED_UDIV_CONTINUE5]] ], [ [[TMP21]], [[PRED_UDIV_IF6]] ] 5416; SINK-AFTER-NEXT: [[TMP23:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP22]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 5417; SINK-AFTER-NEXT: [[TMP24]] = add <4 x i32> [[VEC_PHI]], [[TMP23]] 5418; SINK-AFTER-NEXT: [[TMP25:%.*]] = extractelement <4 x i1> [[TMP6]], i32 0 5419; SINK-AFTER-NEXT: br i1 [[TMP25]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] 5420; SINK-AFTER: pred.store.if: 5421; SINK-AFTER-NEXT: [[TMP26:%.*]] = add i32 [[INDEX]], 0 5422; SINK-AFTER-NEXT: [[TMP27:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i32 [[TMP26]] 5423; SINK-AFTER-NEXT: store i32 [[TMP2]], i32* [[TMP27]], align 4 5424; SINK-AFTER-NEXT: br label [[PRED_STORE_CONTINUE]] 5425; SINK-AFTER: pred.store.continue: 5426; SINK-AFTER-NEXT: [[TMP28:%.*]] = extractelement <4 x i1> [[TMP6]], i32 1 5427; SINK-AFTER-NEXT: br i1 [[TMP28]], label [[PRED_STORE_IF8:%.*]], label [[PRED_STORE_CONTINUE9:%.*]] 5428; SINK-AFTER: pred.store.if8: 5429; SINK-AFTER-NEXT: [[TMP29:%.*]] = add i32 [[INDEX]], 1 5430; SINK-AFTER-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[TMP29]] 5431; SINK-AFTER-NEXT: store i32 [[TMP3]], i32* [[TMP30]], align 4 5432; SINK-AFTER-NEXT: br label [[PRED_STORE_CONTINUE9]] 5433; SINK-AFTER: pred.store.continue9: 5434; SINK-AFTER-NEXT: [[TMP31:%.*]] = extractelement <4 x i1> [[TMP6]], i32 2 5435; SINK-AFTER-NEXT: br i1 [[TMP31]], label [[PRED_STORE_IF10:%.*]], label [[PRED_STORE_CONTINUE11:%.*]] 5436; SINK-AFTER: pred.store.if10: 5437; SINK-AFTER-NEXT: [[TMP32:%.*]] = add i32 [[INDEX]], 2 5438; SINK-AFTER-NEXT: [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[TMP32]] 5439; SINK-AFTER-NEXT: store i32 [[TMP4]], i32* [[TMP33]], align 4 5440; SINK-AFTER-NEXT: br label [[PRED_STORE_CONTINUE11]] 5441; SINK-AFTER: pred.store.continue11: 5442; SINK-AFTER-NEXT: [[TMP34:%.*]] = extractelement <4 x i1> [[TMP6]], i32 3 5443; SINK-AFTER-NEXT: br i1 [[TMP34]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE13]] 5444; SINK-AFTER: pred.store.if12: 5445; SINK-AFTER-NEXT: [[TMP35:%.*]] = add i32 [[INDEX]], 3 5446; SINK-AFTER-NEXT: [[TMP36:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[TMP35]] 5447; SINK-AFTER-NEXT: store i32 [[TMP5]], i32* [[TMP36]], align 4 5448; SINK-AFTER-NEXT: br label [[PRED_STORE_CONTINUE13]] 5449; SINK-AFTER: pred.store.continue13: 5450; SINK-AFTER-NEXT: [[TMP37:%.*]] = select <4 x i1> [[TMP6]], <4 x i32> [[TMP24]], <4 x i32> [[VEC_PHI]] 5451; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4 5452; SINK-AFTER-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4> 5453; SINK-AFTER-NEXT: [[TMP38:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 5454; SINK-AFTER-NEXT: br i1 [[TMP38]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF52]], !llvm.loop [[LOOP56:![0-9]+]] 5455; SINK-AFTER: middle.block: 5456; SINK-AFTER-NEXT: [[TMP39:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP37]]) 5457; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP22]], i32 3 5458; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP22]], i32 2 5459; SINK-AFTER-NEXT: br i1 true, label [[BB1:%.*]], label [[SCALAR_PH]] 5460; SINK-AFTER: scalar.ph: 5461; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[BB:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 5462; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[Y]], [[BB]] ] 5463; SINK-AFTER-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[BB]] ] 5464; SINK-AFTER-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[BB]] ], [ [[TMP39]], [[MIDDLE_BLOCK]] ] 5465; SINK-AFTER-NEXT: br label [[BB2:%.*]] 5466; SINK-AFTER: bb1: 5467; SINK-AFTER-NEXT: [[VAR:%.*]] = phi i32 [ [[VAR6:%.*]], [[BB2]] ], [ [[TMP39]], [[MIDDLE_BLOCK]] ] 5468; SINK-AFTER-NEXT: ret i32 [[VAR]] 5469; SINK-AFTER: bb2: 5470; SINK-AFTER-NEXT: [[VAR3:%.*]] = phi i32 [ [[VAR8:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 5471; SINK-AFTER-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ] 5472; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[VAR7:%.*]], [[BB2]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ] 5473; SINK-AFTER-NEXT: [[VAR5:%.*]] = phi i32 [ [[VAR6]], [[BB2]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 5474; SINK-AFTER-NEXT: [[G:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[IV]] 5475; SINK-AFTER-NEXT: [[VAR6]] = add i32 [[VAR5]], [[SCALAR_RECUR]] 5476; SINK-AFTER-NEXT: [[VAR7]] = udiv i32 219220132, [[VAR3]] 5477; SINK-AFTER-NEXT: store i32 [[VAR3]], i32* [[G]], align 4 5478; SINK-AFTER-NEXT: [[VAR8]] = add nsw i32 [[VAR3]], -1 5479; SINK-AFTER-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], 1 5480; SINK-AFTER-NEXT: [[VAR9:%.*]] = icmp slt i32 [[VAR3]], 2 5481; SINK-AFTER-NEXT: br i1 [[VAR9]], label [[BB1]], label [[BB2]], !prof [[PROF54]], !llvm.loop [[LOOP57:![0-9]+]] 5482; 5483bb: 5484 br label %bb2 5485 5486 bb1: ; preds = %bb2 5487 %var = phi i32 [ %var6, %bb2 ] 5488 ret i32 %var 5489 5490 bb2: ; preds = %bb2, %bb 5491 %var3 = phi i32 [ %var8, %bb2 ], [ %y, %bb ] 5492 %iv = phi i32 [ %iv.next, %bb2 ], [ 0, %bb ] 5493 %var4 = phi i32 [ %var7, %bb2 ], [ 0, %bb ] 5494 %var5 = phi i32 [ %var6, %bb2 ], [ 0, %bb ] 5495 %g = getelementptr inbounds i32, i32* %x, i32 %iv 5496 %var6 = add i32 %var5, %var4 5497 %var7 = udiv i32 219220132, %var3 5498 store i32 %var3, i32* %g, align 4 5499 %var8 = add nsw i32 %var3, -1 5500 %iv.next = add nsw i32 %iv, 1 5501 %var9 = icmp slt i32 %var3, 2 5502 br i1 %var9, label %bb1, label %bb2, !prof !2 5503} 5504 5505; %vec.dead will be marked as dead instruction in the vector loop and no recipe 5506; will be created for it. Make sure a valid sink target is used. 5507define void @sink_after_dead_inst(i32* %A.ptr) { 5508; CHECK-LABEL: @sink_after_dead_inst( 5509; CHECK-NEXT: entry: 5510; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5511; CHECK: vector.ph: 5512; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 5513; CHECK: vector.body: 5514; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 5515; CHECK-NEXT: [[OFFSET_IDX:%.*]] = zext i32 [[INDEX]] to i64 5516; CHECK-NEXT: [[SEXT:%.*]] = shl i64 [[OFFSET_IDX]], 48 5517; CHECK-NEXT: [[TMP0:%.*]] = ashr exact i64 [[SEXT]], 48 5518; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, i32* [[A_PTR:%.*]], i64 [[TMP0]] 5519; CHECK-NEXT: [[TMP2:%.*]] = bitcast i32* [[TMP1]] to <4 x i32>* 5520; CHECK-NEXT: store <4 x i32> zeroinitializer, <4 x i32>* [[TMP2]], align 4 5521; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 5522; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16 5523; CHECK-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP58:![0-9]+]] 5524; CHECK: middle.block: 5525; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]] 5526; CHECK: scalar.ph: 5527; CHECK-NEXT: br label [[LOOP:%.*]] 5528; CHECK: loop: 5529; CHECK-NEXT: br i1 undef, label [[FOR_END]], label [[LOOP]], !llvm.loop [[LOOP59:![0-9]+]] 5530; CHECK: for.end: 5531; CHECK-NEXT: ret void 5532; 5533; UNROLL-LABEL: @sink_after_dead_inst( 5534; UNROLL-NEXT: entry: 5535; UNROLL-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5536; UNROLL: vector.ph: 5537; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 5538; UNROLL: vector.body: 5539; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 5540; UNROLL-NEXT: [[OFFSET_IDX:%.*]] = zext i32 [[INDEX]] to i64 5541; UNROLL-NEXT: [[SEXT:%.*]] = shl i64 [[OFFSET_IDX]], 48 5542; UNROLL-NEXT: [[TMP0:%.*]] = ashr exact i64 [[SEXT]], 48 5543; UNROLL-NEXT: [[TMP1:%.*]] = getelementptr i32, i32* [[A_PTR:%.*]], i64 [[TMP0]] 5544; UNROLL-NEXT: [[TMP2:%.*]] = bitcast i32* [[TMP1]] to <4 x i32>* 5545; UNROLL-NEXT: store <4 x i32> zeroinitializer, <4 x i32>* [[TMP2]], align 4 5546; UNROLL-NEXT: [[TMP3:%.*]] = getelementptr i32, i32* [[TMP1]], i64 4 5547; UNROLL-NEXT: [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>* 5548; UNROLL-NEXT: store <4 x i32> zeroinitializer, <4 x i32>* [[TMP4]], align 4 5549; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 5550; UNROLL-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16 5551; UNROLL-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP58:![0-9]+]] 5552; UNROLL: middle.block: 5553; UNROLL-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]] 5554; UNROLL: scalar.ph: 5555; UNROLL-NEXT: br label [[LOOP:%.*]] 5556; UNROLL: loop: 5557; UNROLL-NEXT: br i1 undef, label [[FOR_END]], label [[LOOP]], !llvm.loop [[LOOP59:![0-9]+]] 5558; UNROLL: for.end: 5559; UNROLL-NEXT: ret void 5560; 5561; UNROLL-NO-IC-LABEL: @sink_after_dead_inst( 5562; UNROLL-NO-IC-NEXT: entry: 5563; UNROLL-NO-IC-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5564; UNROLL-NO-IC: vector.ph: 5565; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 5566; UNROLL-NO-IC: vector.body: 5567; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 5568; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 5569; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ] 5570; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add <4 x i16> [[VEC_IND]], <i16 4, i16 4, i16 4, i16 4> 5571; UNROLL-NO-IC-NEXT: [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16 5572; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i16 [[OFFSET_IDX]], 0 5573; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = add i16 [[OFFSET_IDX]], 4 5574; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add <4 x i16> [[VEC_IND]], <i16 1, i16 1, i16 1, i16 1> 5575; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add <4 x i16> [[STEP_ADD]], <i16 1, i16 1, i16 1, i16 1> 5576; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = or <4 x i16> [[TMP2]], [[TMP2]] 5577; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = or <4 x i16> [[TMP3]], [[TMP3]] 5578; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = zext <4 x i16> [[TMP4]] to <4 x i32> 5579; UNROLL-NO-IC-NEXT: [[TMP7]] = zext <4 x i16> [[TMP5]] to <4 x i32> 5580; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP6]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 5581; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> [[TMP7]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 5582; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = getelementptr i32, i32* [[A_PTR:%.*]], i16 [[TMP0]] 5583; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = getelementptr i32, i32* [[A_PTR]], i16 [[TMP1]] 5584; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = getelementptr i32, i32* [[TMP10]], i32 0 5585; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = bitcast i32* [[TMP12]] to <4 x i32>* 5586; UNROLL-NO-IC-NEXT: store <4 x i32> zeroinitializer, <4 x i32>* [[TMP13]], align 4 5587; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = getelementptr i32, i32* [[TMP10]], i32 4 5588; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = bitcast i32* [[TMP14]] to <4 x i32>* 5589; UNROLL-NO-IC-NEXT: store <4 x i32> zeroinitializer, <4 x i32>* [[TMP15]], align 4 5590; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 5591; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT]] = add <4 x i16> [[STEP_ADD]], <i16 4, i16 4, i16 4, i16 4> 5592; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16 5593; UNROLL-NO-IC-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP58:![0-9]+]] 5594; UNROLL-NO-IC: middle.block: 5595; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i32 16, 16 5596; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP7]], i32 3 5597; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP7]], i32 2 5598; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 5599; UNROLL-NO-IC: scalar.ph: 5600; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 5601; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ 16, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 5602; UNROLL-NO-IC-NEXT: br label [[LOOP:%.*]] 5603; UNROLL-NO-IC: loop: 5604; UNROLL-NO-IC-NEXT: [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ] 5605; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[FOR_PREV:%.*]], [[LOOP]] ] 5606; UNROLL-NO-IC-NEXT: [[CMP:%.*]] = icmp eq i32 [[SCALAR_RECUR]], 15 5607; UNROLL-NO-IC-NEXT: [[C:%.*]] = icmp eq i1 [[CMP]], true 5608; UNROLL-NO-IC-NEXT: [[VEC_DEAD:%.*]] = and i1 [[C]], true 5609; UNROLL-NO-IC-NEXT: [[IV_NEXT]] = add i16 [[IV]], 1 5610; UNROLL-NO-IC-NEXT: [[B1:%.*]] = or i16 [[IV_NEXT]], [[IV_NEXT]] 5611; UNROLL-NO-IC-NEXT: [[B3:%.*]] = and i1 [[CMP]], [[C]] 5612; UNROLL-NO-IC-NEXT: [[FOR_PREV]] = zext i16 [[B1]] to i32 5613; UNROLL-NO-IC-NEXT: [[EXT:%.*]] = zext i1 [[B3]] to i32 5614; UNROLL-NO-IC-NEXT: [[A_GEP:%.*]] = getelementptr i32, i32* [[A_PTR]], i16 [[IV]] 5615; UNROLL-NO-IC-NEXT: store i32 0, i32* [[A_GEP]], align 4 5616; UNROLL-NO-IC-NEXT: br i1 [[VEC_DEAD]], label [[FOR_END]], label [[LOOP]], !llvm.loop [[LOOP59:![0-9]+]] 5617; UNROLL-NO-IC: for.end: 5618; UNROLL-NO-IC-NEXT: ret void 5619; 5620; UNROLL-NO-VF-LABEL: @sink_after_dead_inst( 5621; UNROLL-NO-VF-NEXT: entry: 5622; UNROLL-NO-VF-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5623; UNROLL-NO-VF: vector.ph: 5624; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]] 5625; UNROLL-NO-VF: vector.body: 5626; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 5627; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ] 5628; UNROLL-NO-VF-NEXT: [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16 5629; UNROLL-NO-VF-NEXT: [[INDUCTION:%.*]] = add i16 [[OFFSET_IDX]], 0 5630; UNROLL-NO-VF-NEXT: [[INDUCTION1:%.*]] = add i16 [[OFFSET_IDX]], 1 5631; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i16 [[INDUCTION]], 1 5632; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = add i16 [[INDUCTION1]], 1 5633; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = or i16 [[TMP0]], [[TMP0]] 5634; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = or i16 [[TMP1]], [[TMP1]] 5635; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = zext i16 [[TMP2]] to i32 5636; UNROLL-NO-VF-NEXT: [[TMP5]] = zext i16 [[TMP3]] to i32 5637; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = getelementptr i32, i32* [[A_PTR:%.*]], i16 [[INDUCTION]] 5638; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = getelementptr i32, i32* [[A_PTR]], i16 [[INDUCTION1]] 5639; UNROLL-NO-VF-NEXT: store i32 0, i32* [[TMP6]], align 4 5640; UNROLL-NO-VF-NEXT: store i32 0, i32* [[TMP7]], align 4 5641; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 5642; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16 5643; UNROLL-NO-VF-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP57:![0-9]+]] 5644; UNROLL-NO-VF: middle.block: 5645; UNROLL-NO-VF-NEXT: [[CMP_N:%.*]] = icmp eq i32 16, 16 5646; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 5647; UNROLL-NO-VF: scalar.ph: 5648; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ] 5649; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ 16, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 5650; UNROLL-NO-VF-NEXT: br label [[LOOP:%.*]] 5651; UNROLL-NO-VF: loop: 5652; UNROLL-NO-VF-NEXT: [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ] 5653; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[FOR_PREV:%.*]], [[LOOP]] ] 5654; UNROLL-NO-VF-NEXT: [[CMP:%.*]] = icmp eq i32 [[SCALAR_RECUR]], 15 5655; UNROLL-NO-VF-NEXT: [[C:%.*]] = icmp eq i1 [[CMP]], true 5656; UNROLL-NO-VF-NEXT: [[VEC_DEAD:%.*]] = and i1 [[C]], true 5657; UNROLL-NO-VF-NEXT: [[IV_NEXT]] = add i16 [[IV]], 1 5658; UNROLL-NO-VF-NEXT: [[B1:%.*]] = or i16 [[IV_NEXT]], [[IV_NEXT]] 5659; UNROLL-NO-VF-NEXT: [[B3:%.*]] = and i1 [[CMP]], [[C]] 5660; UNROLL-NO-VF-NEXT: [[FOR_PREV]] = zext i16 [[B1]] to i32 5661; UNROLL-NO-VF-NEXT: [[EXT:%.*]] = zext i1 [[B3]] to i32 5662; UNROLL-NO-VF-NEXT: [[A_GEP:%.*]] = getelementptr i32, i32* [[A_PTR]], i16 [[IV]] 5663; UNROLL-NO-VF-NEXT: store i32 0, i32* [[A_GEP]], align 4 5664; UNROLL-NO-VF-NEXT: br i1 [[VEC_DEAD]], label [[FOR_END]], label [[LOOP]], !llvm.loop [[LOOP58:![0-9]+]] 5665; UNROLL-NO-VF: for.end: 5666; UNROLL-NO-VF-NEXT: ret void 5667; 5668; SINK-AFTER-LABEL: @sink_after_dead_inst( 5669; SINK-AFTER-NEXT: entry: 5670; SINK-AFTER-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5671; SINK-AFTER: vector.ph: 5672; SINK-AFTER-NEXT: br label [[VECTOR_BODY:%.*]] 5673; SINK-AFTER: vector.body: 5674; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 5675; SINK-AFTER-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 5676; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ] 5677; SINK-AFTER-NEXT: [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16 5678; SINK-AFTER-NEXT: [[TMP0:%.*]] = add i16 [[OFFSET_IDX]], 0 5679; SINK-AFTER-NEXT: [[TMP1:%.*]] = add <4 x i16> [[VEC_IND]], <i16 1, i16 1, i16 1, i16 1> 5680; SINK-AFTER-NEXT: [[TMP2:%.*]] = or <4 x i16> [[TMP1]], [[TMP1]] 5681; SINK-AFTER-NEXT: [[TMP3]] = zext <4 x i16> [[TMP2]] to <4 x i32> 5682; SINK-AFTER-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP3]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 5683; SINK-AFTER-NEXT: [[TMP5:%.*]] = getelementptr i32, i32* [[A_PTR:%.*]], i16 [[TMP0]] 5684; SINK-AFTER-NEXT: [[TMP6:%.*]] = getelementptr i32, i32* [[TMP5]], i32 0 5685; SINK-AFTER-NEXT: [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>* 5686; SINK-AFTER-NEXT: store <4 x i32> zeroinitializer, <4 x i32>* [[TMP7]], align 4 5687; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 5688; SINK-AFTER-NEXT: [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], <i16 4, i16 4, i16 4, i16 4> 5689; SINK-AFTER-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16 5690; SINK-AFTER-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP58:![0-9]+]] 5691; SINK-AFTER: middle.block: 5692; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i32 16, 16 5693; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3 5694; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2 5695; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 5696; SINK-AFTER: scalar.ph: 5697; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] 5698; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ 16, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 5699; SINK-AFTER-NEXT: br label [[LOOP:%.*]] 5700; SINK-AFTER: loop: 5701; SINK-AFTER-NEXT: [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ] 5702; SINK-AFTER-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[FOR_PREV:%.*]], [[LOOP]] ] 5703; SINK-AFTER-NEXT: [[CMP:%.*]] = icmp eq i32 [[SCALAR_RECUR]], 15 5704; SINK-AFTER-NEXT: [[C:%.*]] = icmp eq i1 [[CMP]], true 5705; SINK-AFTER-NEXT: [[VEC_DEAD:%.*]] = and i1 [[C]], true 5706; SINK-AFTER-NEXT: [[IV_NEXT]] = add i16 [[IV]], 1 5707; SINK-AFTER-NEXT: [[B1:%.*]] = or i16 [[IV_NEXT]], [[IV_NEXT]] 5708; SINK-AFTER-NEXT: [[B3:%.*]] = and i1 [[CMP]], [[C]] 5709; SINK-AFTER-NEXT: [[FOR_PREV]] = zext i16 [[B1]] to i32 5710; SINK-AFTER-NEXT: [[EXT:%.*]] = zext i1 [[B3]] to i32 5711; SINK-AFTER-NEXT: [[A_GEP:%.*]] = getelementptr i32, i32* [[A_PTR]], i16 [[IV]] 5712; SINK-AFTER-NEXT: store i32 0, i32* [[A_GEP]], align 4 5713; SINK-AFTER-NEXT: br i1 [[VEC_DEAD]], label [[FOR_END]], label [[LOOP]], !llvm.loop [[LOOP59:![0-9]+]] 5714; SINK-AFTER: for.end: 5715; SINK-AFTER-NEXT: ret void 5716; 5717entry: 5718 br label %loop 5719 5720loop: 5721 %iv = phi i16 [ 0, %entry ], [ %iv.next, %loop ] 5722 %for = phi i32 [ 0, %entry ], [ %for.prev, %loop ] 5723 %cmp = icmp eq i32 %for, 15 5724 %C = icmp eq i1 %cmp, true 5725 %vec.dead = and i1 %C, 1 5726 %iv.next = add i16 %iv, 1 5727 %B1 = or i16 %iv.next, %iv.next 5728 %B3 = and i1 %cmp, %C 5729 %for.prev = zext i16 %B1 to i32 5730 5731 %ext = zext i1 %B3 to i32 5732 %A.gep = getelementptr i32, i32* %A.ptr, i16 %iv 5733 store i32 0, i32* %A.gep 5734 br i1 %vec.dead, label %for.end, label %loop 5735 5736for.end: 5737 ret void 5738} 5739 5740!2 = !{!"branch_weights", i32 1, i32 1} 5741