; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -enable-new-pm=0 -loop-vectorize -S | FileCheck %s -check-prefixes=DEFAULT,PGSO
; RUN: opt < %s -enable-new-pm=0 -loop-vectorize -pgso -S | FileCheck %s -check-prefixes=DEFAULT,PGSO
; RUN: opt < %s -enable-new-pm=0 -loop-vectorize -pgso=false -S | FileCheck %s -check-prefixes=DEFAULT,NPGSO
; RUN: opt < %s -passes='require<profile-summary>,loop-vectorize' -S | FileCheck %s -check-prefixes=DEFAULT,PGSO
; RUN: opt < %s -passes='require<profile-summary>,loop-vectorize' -pgso -S | FileCheck %s -check-prefixes=DEFAULT,PGSO
; RUN: opt < %s -passes='require<profile-summary>,loop-vectorize' -pgso=false -S | FileCheck %s -check-prefixes=DEFAULT,NPGSO

; REQUIRES: asserts

; This test verifies that the loop vectorizer will NOT produce a tail
; loop with the optimize for size or the minimize size attributes.

target datalayout = "E-m:e-p:32:32-i64:32-f64:32:64-a:0:32-n32-S128"

@tab = common global [32 x i8] zeroinitializer, align 1

define i32 @foo_optsize() #0 {
; DEFAULT-LABEL: @foo_optsize(
; DEFAULT-NEXT:  entry:
; DEFAULT-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; DEFAULT:       vector.ph:
; DEFAULT-NEXT:    br label [[VECTOR_BODY:%.*]]
; DEFAULT:       vector.body:
; DEFAULT-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE12:%.*]] ]
; DEFAULT-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE12]] ]
; DEFAULT-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
; DEFAULT-NEXT:    [[TMP1:%.*]] = add i32 [[INDEX]], 1
; DEFAULT-NEXT:    [[TMP2:%.*]] = add i32 [[INDEX]], 2
; DEFAULT-NEXT:    [[TMP3:%.*]] = add i32 [[INDEX]], 3
; DEFAULT-NEXT:    [[TMP4:%.*]] = icmp ule <4 x i32> [[VEC_IND]], <i32 202, i32 202, i32 202, i32 202>
; DEFAULT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 [[TMP0]]
; DEFAULT-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 [[TMP1]]
; DEFAULT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 [[TMP2]]
; DEFAULT-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 [[TMP3]]
; DEFAULT-NEXT:    [[TMP9:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
; DEFAULT-NEXT:    br i1 [[TMP9]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
; DEFAULT:       pred.load.if:
; DEFAULT-NEXT:    [[TMP10:%.*]] = load i8, i8* [[TMP5]], align 1
; DEFAULT-NEXT:    [[TMP11:%.*]] = insertelement <4 x i8> poison, i8 [[TMP10]], i32 0
; DEFAULT-NEXT:    br label [[PRED_LOAD_CONTINUE]]
; DEFAULT:       pred.load.continue:
; DEFAULT-NEXT:    [[TMP12:%.*]] = phi <4 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP11]], [[PRED_LOAD_IF]] ]
; DEFAULT-NEXT:    [[TMP13:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
; DEFAULT-NEXT:    br i1 [[TMP13]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
; DEFAULT:       pred.load.if1:
; DEFAULT-NEXT:    [[TMP14:%.*]] = load i8, i8* [[TMP6]], align 1
; DEFAULT-NEXT:    [[TMP15:%.*]] = insertelement <4 x i8> [[TMP12]], i8 [[TMP14]], i32 1
; DEFAULT-NEXT:    br label [[PRED_LOAD_CONTINUE2]]
; DEFAULT:       pred.load.continue2:
; DEFAULT-NEXT:    [[TMP16:%.*]] = phi <4 x i8> [ [[TMP12]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP15]], [[PRED_LOAD_IF1]] ]
; DEFAULT-NEXT:    [[TMP17:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
; DEFAULT-NEXT:    br i1 [[TMP17]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
; DEFAULT:       pred.load.if3:
; DEFAULT-NEXT:    [[TMP18:%.*]] = load i8, i8* [[TMP7]], align 1
; DEFAULT-NEXT:    [[TMP19:%.*]] = insertelement <4 x i8> [[TMP16]], i8 [[TMP18]], i32 2
; DEFAULT-NEXT:    br label [[PRED_LOAD_CONTINUE4]]
; DEFAULT:       pred.load.continue4:
; DEFAULT-NEXT:    [[TMP20:%.*]] = phi <4 x i8> [ [[TMP16]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP19]], [[PRED_LOAD_IF3]] ]
; DEFAULT-NEXT:    [[TMP21:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
; DEFAULT-NEXT:    br i1 [[TMP21]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
; DEFAULT:       pred.load.if5:
; DEFAULT-NEXT:    [[TMP22:%.*]] = load i8, i8* [[TMP8]], align 1
; DEFAULT-NEXT:    [[TMP23:%.*]] = insertelement <4 x i8> [[TMP20]], i8 [[TMP22]], i32 3
; DEFAULT-NEXT:    br label [[PRED_LOAD_CONTINUE6]]
; DEFAULT:       pred.load.continue6:
; DEFAULT-NEXT:    [[TMP24:%.*]] = phi <4 x i8> [ [[TMP20]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP23]], [[PRED_LOAD_IF5]] ]
; DEFAULT-NEXT:    [[TMP25:%.*]] = icmp eq <4 x i8> [[TMP24]], zeroinitializer
; DEFAULT-NEXT:    [[TMP26:%.*]] = select <4 x i1> [[TMP25]], <4 x i8> <i8 2, i8 2, i8 2, i8 2>, <4 x i8> <i8 1, i8 1, i8 1, i8 1>
; DEFAULT-NEXT:    [[TMP27:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
; DEFAULT-NEXT:    br i1 [[TMP27]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; DEFAULT:       pred.store.if:
; DEFAULT-NEXT:    [[TMP28:%.*]] = extractelement <4 x i8> [[TMP26]], i32 0
; DEFAULT-NEXT:    store i8 [[TMP28]], i8* [[TMP5]], align 1
; DEFAULT-NEXT:    br label [[PRED_STORE_CONTINUE]]
; DEFAULT:       pred.store.continue:
; DEFAULT-NEXT:    [[TMP29:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
; DEFAULT-NEXT:    br i1 [[TMP29]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8:%.*]]
; DEFAULT:       pred.store.if7:
; DEFAULT-NEXT:    [[TMP30:%.*]] = extractelement <4 x i8> [[TMP26]], i32 1
; DEFAULT-NEXT:    store i8 [[TMP30]], i8* [[TMP6]], align 1
; DEFAULT-NEXT:    br label [[PRED_STORE_CONTINUE8]]
; DEFAULT:       pred.store.continue8:
; DEFAULT-NEXT:    [[TMP31:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
; DEFAULT-NEXT:    br i1 [[TMP31]], label [[PRED_STORE_IF9:%.*]], label [[PRED_STORE_CONTINUE10:%.*]]
; DEFAULT:       pred.store.if9:
; DEFAULT-NEXT:    [[TMP32:%.*]] = extractelement <4 x i8> [[TMP26]], i32 2
; DEFAULT-NEXT:    store i8 [[TMP32]], i8* [[TMP7]], align 1
; DEFAULT-NEXT:    br label [[PRED_STORE_CONTINUE10]]
; DEFAULT:       pred.store.continue10:
; DEFAULT-NEXT:    [[TMP33:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
; DEFAULT-NEXT:    br i1 [[TMP33]], label [[PRED_STORE_IF11:%.*]], label [[PRED_STORE_CONTINUE12]]
; DEFAULT:       pred.store.if11:
; DEFAULT-NEXT:    [[TMP34:%.*]] = extractelement <4 x i8> [[TMP26]], i32 3
; DEFAULT-NEXT:    store i8 [[TMP34]], i8* [[TMP8]], align 1
; DEFAULT-NEXT:    br label [[PRED_STORE_CONTINUE12]]
; DEFAULT:       pred.store.continue12:
; DEFAULT-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
; DEFAULT-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
; DEFAULT-NEXT:    [[TMP35:%.*]] = icmp eq i32 [[INDEX_NEXT]], 204
; DEFAULT-NEXT:    br i1 [[TMP35]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; DEFAULT:       middle.block:
; DEFAULT-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
; DEFAULT:       scalar.ph:
; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 204, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; DEFAULT-NEXT:    br label [[FOR_BODY:%.*]]
; DEFAULT:       for.body:
; DEFAULT-NEXT:    [[I_08:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
; DEFAULT-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 [[I_08]]
; DEFAULT-NEXT:    [[TMP36:%.*]] = load i8, i8* [[ARRAYIDX]], align 1
; DEFAULT-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[TMP36]], 0
; DEFAULT-NEXT:    [[DOT:%.*]] = select i1 [[CMP1]], i8 2, i8 1
; DEFAULT-NEXT:    store i8 [[DOT]], i8* [[ARRAYIDX]], align 1
; DEFAULT-NEXT:    [[INC]] = add nsw i32 [[I_08]], 1
; DEFAULT-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[I_08]], 202
; DEFAULT-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; DEFAULT:       for.end:
; DEFAULT-NEXT:    ret i32 0
;
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
  %arrayidx = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 %i.08
  %0 = load i8, i8* %arrayidx, align 1
  %cmp1 = icmp eq i8 %0, 0
  %. = select i1 %cmp1, i8 2, i8 1
  store i8 %., i8* %arrayidx, align 1
  %inc = add nsw i32 %i.08, 1
  %exitcond = icmp eq i32 %i.08, 202
  br i1 %exitcond, label %for.end, label %for.body

for.end:                                          ; preds = %for.body
  ret i32 0
}

attributes #0 = { optsize }

define i32 @foo_minsize() #1 {
; DEFAULT-LABEL: @foo_minsize(
; DEFAULT-NEXT:  entry:
; DEFAULT-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; DEFAULT:       vector.ph:
; DEFAULT-NEXT:    br label [[VECTOR_BODY:%.*]]
; DEFAULT:       vector.body:
; DEFAULT-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE12:%.*]] ]
; DEFAULT-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE12]] ]
; DEFAULT-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
; DEFAULT-NEXT:    [[TMP1:%.*]] = add i32 [[INDEX]], 1
; DEFAULT-NEXT:    [[TMP2:%.*]] = add i32 [[INDEX]], 2
; DEFAULT-NEXT:    [[TMP3:%.*]] = add i32 [[INDEX]], 3
; DEFAULT-NEXT:    [[TMP4:%.*]] = icmp ule <4 x i32> [[VEC_IND]], <i32 202, i32 202, i32 202, i32 202>
; DEFAULT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 [[TMP0]]
; DEFAULT-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 [[TMP1]]
; DEFAULT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 [[TMP2]]
; DEFAULT-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 [[TMP3]]
; DEFAULT-NEXT:    [[TMP9:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
; DEFAULT-NEXT:    br i1 [[TMP9]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
; DEFAULT:       pred.load.if:
; DEFAULT-NEXT:    [[TMP10:%.*]] = load i8, i8* [[TMP5]], align 1
; DEFAULT-NEXT:    [[TMP11:%.*]] = insertelement <4 x i8> poison, i8 [[TMP10]], i32 0
; DEFAULT-NEXT:    br label [[PRED_LOAD_CONTINUE]]
; DEFAULT:       pred.load.continue:
; DEFAULT-NEXT:    [[TMP12:%.*]] = phi <4 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP11]], [[PRED_LOAD_IF]] ]
; DEFAULT-NEXT:    [[TMP13:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
; DEFAULT-NEXT:    br i1 [[TMP13]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
; DEFAULT:       pred.load.if1:
; DEFAULT-NEXT:    [[TMP14:%.*]] = load i8, i8* [[TMP6]], align 1
; DEFAULT-NEXT:    [[TMP15:%.*]] = insertelement <4 x i8> [[TMP12]], i8 [[TMP14]], i32 1
; DEFAULT-NEXT:    br label [[PRED_LOAD_CONTINUE2]]
; DEFAULT:       pred.load.continue2:
; DEFAULT-NEXT:    [[TMP16:%.*]] = phi <4 x i8> [ [[TMP12]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP15]], [[PRED_LOAD_IF1]] ]
; DEFAULT-NEXT:    [[TMP17:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
; DEFAULT-NEXT:    br i1 [[TMP17]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
; DEFAULT:       pred.load.if3:
; DEFAULT-NEXT:    [[TMP18:%.*]] = load i8, i8* [[TMP7]], align 1
; DEFAULT-NEXT:    [[TMP19:%.*]] = insertelement <4 x i8> [[TMP16]], i8 [[TMP18]], i32 2
; DEFAULT-NEXT:    br label [[PRED_LOAD_CONTINUE4]]
; DEFAULT:       pred.load.continue4:
; DEFAULT-NEXT:    [[TMP20:%.*]] = phi <4 x i8> [ [[TMP16]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP19]], [[PRED_LOAD_IF3]] ]
; DEFAULT-NEXT:    [[TMP21:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
; DEFAULT-NEXT:    br i1 [[TMP21]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
; DEFAULT:       pred.load.if5:
; DEFAULT-NEXT:    [[TMP22:%.*]] = load i8, i8* [[TMP8]], align 1
; DEFAULT-NEXT:    [[TMP23:%.*]] = insertelement <4 x i8> [[TMP20]], i8 [[TMP22]], i32 3
; DEFAULT-NEXT:    br label [[PRED_LOAD_CONTINUE6]]
; DEFAULT:       pred.load.continue6:
; DEFAULT-NEXT:    [[TMP24:%.*]] = phi <4 x i8> [ [[TMP20]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP23]], [[PRED_LOAD_IF5]] ]
; DEFAULT-NEXT:    [[TMP25:%.*]] = icmp eq <4 x i8> [[TMP24]], zeroinitializer
; DEFAULT-NEXT:    [[TMP26:%.*]] = select <4 x i1> [[TMP25]], <4 x i8> <i8 2, i8 2, i8 2, i8 2>, <4 x i8> <i8 1, i8 1, i8 1, i8 1>
; DEFAULT-NEXT:    [[TMP27:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
; DEFAULT-NEXT:    br i1 [[TMP27]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; DEFAULT:       pred.store.if:
; DEFAULT-NEXT:    [[TMP28:%.*]] = extractelement <4 x i8> [[TMP26]], i32 0
; DEFAULT-NEXT:    store i8 [[TMP28]], i8* [[TMP5]], align 1
; DEFAULT-NEXT:    br label [[PRED_STORE_CONTINUE]]
; DEFAULT:       pred.store.continue:
; DEFAULT-NEXT:    [[TMP29:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
; DEFAULT-NEXT:    br i1 [[TMP29]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8:%.*]]
; DEFAULT:       pred.store.if7:
; DEFAULT-NEXT:    [[TMP30:%.*]] = extractelement <4 x i8> [[TMP26]], i32 1
; DEFAULT-NEXT:    store i8 [[TMP30]], i8* [[TMP6]], align 1
; DEFAULT-NEXT:    br label [[PRED_STORE_CONTINUE8]]
; DEFAULT:       pred.store.continue8:
; DEFAULT-NEXT:    [[TMP31:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
; DEFAULT-NEXT:    br i1 [[TMP31]], label [[PRED_STORE_IF9:%.*]], label [[PRED_STORE_CONTINUE10:%.*]]
; DEFAULT:       pred.store.if9:
; DEFAULT-NEXT:    [[TMP32:%.*]] = extractelement <4 x i8> [[TMP26]], i32 2
; DEFAULT-NEXT:    store i8 [[TMP32]], i8* [[TMP7]], align 1
; DEFAULT-NEXT:    br label [[PRED_STORE_CONTINUE10]]
; DEFAULT:       pred.store.continue10:
; DEFAULT-NEXT:    [[TMP33:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
; DEFAULT-NEXT:    br i1 [[TMP33]], label [[PRED_STORE_IF11:%.*]], label [[PRED_STORE_CONTINUE12]]
; DEFAULT:       pred.store.if11:
; DEFAULT-NEXT:    [[TMP34:%.*]] = extractelement <4 x i8> [[TMP26]], i32 3
; DEFAULT-NEXT:    store i8 [[TMP34]], i8* [[TMP8]], align 1
; DEFAULT-NEXT:    br label [[PRED_STORE_CONTINUE12]]
; DEFAULT:       pred.store.continue12:
; DEFAULT-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
; DEFAULT-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
; DEFAULT-NEXT:    [[TMP35:%.*]] = icmp eq i32 [[INDEX_NEXT]], 204
; DEFAULT-NEXT:    br i1 [[TMP35]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
; DEFAULT:       middle.block:
; DEFAULT-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
; DEFAULT:       scalar.ph:
; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 204, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; DEFAULT-NEXT:    br label [[FOR_BODY:%.*]]
; DEFAULT:       for.body:
; DEFAULT-NEXT:    [[I_08:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
; DEFAULT-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 [[I_08]]
; DEFAULT-NEXT:    [[TMP36:%.*]] = load i8, i8* [[ARRAYIDX]], align 1
; DEFAULT-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[TMP36]], 0
; DEFAULT-NEXT:    [[DOT:%.*]] = select i1 [[CMP1]], i8 2, i8 1
; DEFAULT-NEXT:    store i8 [[DOT]], i8* [[ARRAYIDX]], align 1
; DEFAULT-NEXT:    [[INC]] = add nsw i32 [[I_08]], 1
; DEFAULT-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[I_08]], 202
; DEFAULT-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
; DEFAULT:       for.end:
; DEFAULT-NEXT:    ret i32 0
;
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
  %arrayidx = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 %i.08
  %0 = load i8, i8* %arrayidx, align 1
  %cmp1 = icmp eq i8 %0, 0
  %. = select i1 %cmp1, i8 2, i8 1
  store i8 %., i8* %arrayidx, align 1
  %inc = add nsw i32 %i.08, 1
  %exitcond = icmp eq i32 %i.08, 202
  br i1 %exitcond, label %for.end, label %for.body

for.end:                                          ; preds = %for.body
  ret i32 0
}

attributes #1 = { minsize }

define i32 @foo_pgso() !prof !14 {
; PGSO-LABEL: @foo_pgso(
; PGSO-NEXT:  entry:
; PGSO-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; PGSO:       vector.ph:
; PGSO-NEXT:    br label [[VECTOR_BODY:%.*]]
; PGSO:       vector.body:
; PGSO-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE12:%.*]] ]
; PGSO-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE12]] ]
; PGSO-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
; PGSO-NEXT:    [[TMP1:%.*]] = add i32 [[INDEX]], 1
; PGSO-NEXT:    [[TMP2:%.*]] = add i32 [[INDEX]], 2
; PGSO-NEXT:    [[TMP3:%.*]] = add i32 [[INDEX]], 3
; PGSO-NEXT:    [[TMP4:%.*]] = icmp ule <4 x i32> [[VEC_IND]], <i32 202, i32 202, i32 202, i32 202>
; PGSO-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 [[TMP0]]
; PGSO-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 [[TMP1]]
; PGSO-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 [[TMP2]]
; PGSO-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 [[TMP3]]
; PGSO-NEXT:    [[TMP9:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
; PGSO-NEXT:    br i1 [[TMP9]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
; PGSO:       pred.load.if:
; PGSO-NEXT:    [[TMP10:%.*]] = load i8, i8* [[TMP5]], align 1
; PGSO-NEXT:    [[TMP11:%.*]] = insertelement <4 x i8> poison, i8 [[TMP10]], i32 0
; PGSO-NEXT:    br label [[PRED_LOAD_CONTINUE]]
; PGSO:       pred.load.continue:
; PGSO-NEXT:    [[TMP12:%.*]] = phi <4 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP11]], [[PRED_LOAD_IF]] ]
; PGSO-NEXT:    [[TMP13:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
; PGSO-NEXT:    br i1 [[TMP13]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
; PGSO:       pred.load.if1:
; PGSO-NEXT:    [[TMP14:%.*]] = load i8, i8* [[TMP6]], align 1
; PGSO-NEXT:    [[TMP15:%.*]] = insertelement <4 x i8> [[TMP12]], i8 [[TMP14]], i32 1
; PGSO-NEXT:    br label [[PRED_LOAD_CONTINUE2]]
; PGSO:       pred.load.continue2:
; PGSO-NEXT:    [[TMP16:%.*]] = phi <4 x i8> [ [[TMP12]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP15]], [[PRED_LOAD_IF1]] ]
; PGSO-NEXT:    [[TMP17:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
; PGSO-NEXT:    br i1 [[TMP17]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
; PGSO:       pred.load.if3:
; PGSO-NEXT:    [[TMP18:%.*]] = load i8, i8* [[TMP7]], align 1
; PGSO-NEXT:    [[TMP19:%.*]] = insertelement <4 x i8> [[TMP16]], i8 [[TMP18]], i32 2
; PGSO-NEXT:    br label [[PRED_LOAD_CONTINUE4]]
; PGSO:       pred.load.continue4:
; PGSO-NEXT:    [[TMP20:%.*]] = phi <4 x i8> [ [[TMP16]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP19]], [[PRED_LOAD_IF3]] ]
; PGSO-NEXT:    [[TMP21:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
; PGSO-NEXT:    br i1 [[TMP21]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
; PGSO:       pred.load.if5:
; PGSO-NEXT:    [[TMP22:%.*]] = load i8, i8* [[TMP8]], align 1
; PGSO-NEXT:    [[TMP23:%.*]] = insertelement <4 x i8> [[TMP20]], i8 [[TMP22]], i32 3
; PGSO-NEXT:    br label [[PRED_LOAD_CONTINUE6]]
; PGSO:       pred.load.continue6:
; PGSO-NEXT:    [[TMP24:%.*]] = phi <4 x i8> [ [[TMP20]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP23]], [[PRED_LOAD_IF5]] ]
; PGSO-NEXT:    [[TMP25:%.*]] = icmp eq <4 x i8> [[TMP24]], zeroinitializer
; PGSO-NEXT:    [[TMP26:%.*]] = select <4 x i1> [[TMP25]], <4 x i8> <i8 2, i8 2, i8 2, i8 2>, <4 x i8> <i8 1, i8 1, i8 1, i8 1>
; PGSO-NEXT:    [[TMP27:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
; PGSO-NEXT:    br i1 [[TMP27]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; PGSO:       pred.store.if:
; PGSO-NEXT:    [[TMP28:%.*]] = extractelement <4 x i8> [[TMP26]], i32 0
; PGSO-NEXT:    store i8 [[TMP28]], i8* [[TMP5]], align 1
; PGSO-NEXT:    br label [[PRED_STORE_CONTINUE]]
; PGSO:       pred.store.continue:
; PGSO-NEXT:    [[TMP29:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
; PGSO-NEXT:    br i1 [[TMP29]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8:%.*]]
; PGSO:       pred.store.if7:
; PGSO-NEXT:    [[TMP30:%.*]] = extractelement <4 x i8> [[TMP26]], i32 1
; PGSO-NEXT:    store i8 [[TMP30]], i8* [[TMP6]], align 1
; PGSO-NEXT:    br label [[PRED_STORE_CONTINUE8]]
; PGSO:       pred.store.continue8:
; PGSO-NEXT:    [[TMP31:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
; PGSO-NEXT:    br i1 [[TMP31]], label [[PRED_STORE_IF9:%.*]], label [[PRED_STORE_CONTINUE10:%.*]]
; PGSO:       pred.store.if9:
; PGSO-NEXT:    [[TMP32:%.*]] = extractelement <4 x i8> [[TMP26]], i32 2
; PGSO-NEXT:    store i8 [[TMP32]], i8* [[TMP7]], align 1
; PGSO-NEXT:    br label [[PRED_STORE_CONTINUE10]]
; PGSO:       pred.store.continue10:
; PGSO-NEXT:    [[TMP33:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
; PGSO-NEXT:    br i1 [[TMP33]], label [[PRED_STORE_IF11:%.*]], label [[PRED_STORE_CONTINUE12]]
; PGSO:       pred.store.if11:
; PGSO-NEXT:    [[TMP34:%.*]] = extractelement <4 x i8> [[TMP26]], i32 3
; PGSO-NEXT:    store i8 [[TMP34]], i8* [[TMP8]], align 1
; PGSO-NEXT:    br label [[PRED_STORE_CONTINUE12]]
; PGSO:       pred.store.continue12:
; PGSO-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
; PGSO-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
; PGSO-NEXT:    [[TMP35:%.*]] = icmp eq i32 [[INDEX_NEXT]], 204
; PGSO-NEXT:    br i1 [[TMP35]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
; PGSO:       middle.block:
; PGSO-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
; PGSO:       scalar.ph:
; PGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 204, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; PGSO-NEXT:    br label [[FOR_BODY:%.*]]
; PGSO:       for.body:
; PGSO-NEXT:    [[I_08:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
; PGSO-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 [[I_08]]
; PGSO-NEXT:    [[TMP36:%.*]] = load i8, i8* [[ARRAYIDX]], align 1
; PGSO-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[TMP36]], 0
; PGSO-NEXT:    [[DOT:%.*]] = select i1 [[CMP1]], i8 2, i8 1
; PGSO-NEXT:    store i8 [[DOT]], i8* [[ARRAYIDX]], align 1
; PGSO-NEXT:    [[INC]] = add nsw i32 [[I_08]], 1
; PGSO-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[I_08]], 202
; PGSO-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
; PGSO:       for.end:
; PGSO-NEXT:    ret i32 0
;
; NPGSO-LABEL: @foo_pgso(
; NPGSO-NEXT:  entry:
; NPGSO-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; NPGSO:       vector.ph:
; NPGSO-NEXT:    br label [[VECTOR_BODY:%.*]]
; NPGSO:       vector.body:
; NPGSO-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; NPGSO-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
; NPGSO-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 [[TMP0]]
; NPGSO-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[TMP1]], i32 0
; NPGSO-NEXT:    [[TMP3:%.*]] = bitcast i8* [[TMP2]] to <4 x i8>*
; NPGSO-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, <4 x i8>* [[TMP3]], align 1
; NPGSO-NEXT:    [[TMP4:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], zeroinitializer
; NPGSO-NEXT:    [[TMP5:%.*]] = select <4 x i1> [[TMP4]], <4 x i8> <i8 2, i8 2, i8 2, i8 2>, <4 x i8> <i8 1, i8 1, i8 1, i8 1>
; NPGSO-NEXT:    [[TMP6:%.*]] = bitcast i8* [[TMP2]] to <4 x i8>*
; NPGSO-NEXT:    store <4 x i8> [[TMP5]], <4 x i8>* [[TMP6]], align 1
; NPGSO-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; NPGSO-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 200
; NPGSO-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
; NPGSO:       middle.block:
; NPGSO-NEXT:    [[CMP_N:%.*]] = icmp eq i32 203, 200
; NPGSO-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
; NPGSO:       scalar.ph:
; NPGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 200, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; NPGSO-NEXT:    br label [[FOR_BODY:%.*]]
; NPGSO:       for.body:
; NPGSO-NEXT:    [[I_08:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
; NPGSO-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 [[I_08]]
; NPGSO-NEXT:    [[TMP8:%.*]] = load i8, i8* [[ARRAYIDX]], align 1
; NPGSO-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[TMP8]], 0
; NPGSO-NEXT:    [[DOT:%.*]] = select i1 [[CMP1]], i8 2, i8 1
; NPGSO-NEXT:    store i8 [[DOT]], i8* [[ARRAYIDX]], align 1
; NPGSO-NEXT:    [[INC]] = add nsw i32 [[I_08]], 1
; NPGSO-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[I_08]], 202
; NPGSO-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
; NPGSO:       for.end:
; NPGSO-NEXT:    ret i32 0
;
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
  %arrayidx = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 %i.08
  %0 = load i8, i8* %arrayidx, align 1
  %cmp1 = icmp eq i8 %0, 0
  %. = select i1 %cmp1, i8 2, i8 1
  store i8 %., i8* %arrayidx, align 1
  %inc = add nsw i32 %i.08, 1
  %exitcond = icmp eq i32 %i.08, 202
  br i1 %exitcond, label %for.end, label %for.body

for.end:                                          ; preds = %for.body
  ret i32 0
}

; PR43371: don't run into an assert due to emitting SCEV runtime checks
; with OptForSize.
;
@cm_array = external global [2592 x i16], align 1

define void @pr43371() optsize {
; DEFAULT-LABEL: @pr43371(
; DEFAULT-NEXT:  entry:
; DEFAULT-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; DEFAULT:       vector.ph:
; DEFAULT-NEXT:    br label [[VECTOR_BODY:%.*]]
; DEFAULT:       vector.body:
; DEFAULT-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; DEFAULT-NEXT:    [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 0, i16 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; DEFAULT-NEXT:    [[TMP0:%.*]] = add <2 x i16> undef, [[VEC_IND]]
; DEFAULT-NEXT:    [[TMP1:%.*]] = zext <2 x i16> [[TMP0]] to <2 x i32>
; DEFAULT-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
; DEFAULT-NEXT:    [[TMP3:%.*]] = getelementptr [2592 x i16], [2592 x i16]* @cm_array, i32 0, i32 [[TMP2]]
; DEFAULT-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
; DEFAULT-NEXT:    [[TMP5:%.*]] = getelementptr [2592 x i16], [2592 x i16]* @cm_array, i32 0, i32 [[TMP4]]
; DEFAULT-NEXT:    store i16 0, i16* [[TMP3]], align 1
; DEFAULT-NEXT:    store i16 0, i16* [[TMP5]], align 1
; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; DEFAULT-NEXT:    [[VEC_IND_NEXT]] = add <2 x i16> [[VEC_IND]], <i16 2, i16 2>
; DEFAULT-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 756
; DEFAULT-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
; DEFAULT:       middle.block:
; DEFAULT-NEXT:    [[CMP_N:%.*]] = icmp eq i32 756, 756
; DEFAULT-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP28:%.*]], label [[SCALAR_PH]]
; DEFAULT:       scalar.ph:
; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 756, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; DEFAULT-NEXT:    br label [[FOR_BODY29:%.*]]
; DEFAULT:       for.cond.cleanup28:
; DEFAULT-NEXT:    unreachable
; DEFAULT:       for.body29:
; DEFAULT-NEXT:    [[I24_0170:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC37:%.*]], [[FOR_BODY29]] ]
; DEFAULT-NEXT:    [[ADD33:%.*]] = add i16 undef, [[I24_0170]]
; DEFAULT-NEXT:    [[IDXPROM34:%.*]] = zext i16 [[ADD33]] to i32
; DEFAULT-NEXT:    [[ARRAYIDX35:%.*]] = getelementptr [2592 x i16], [2592 x i16]* @cm_array, i32 0, i32 [[IDXPROM34]]
; DEFAULT-NEXT:    store i16 0, i16* [[ARRAYIDX35]], align 1
; DEFAULT-NEXT:    [[INC37]] = add i16 [[I24_0170]], 1
; DEFAULT-NEXT:    [[CMP26:%.*]] = icmp ult i16 [[INC37]], 756
; DEFAULT-NEXT:    br i1 [[CMP26]], label [[FOR_BODY29]], label [[FOR_COND_CLEANUP28]], !llvm.loop [[LOOP24:![0-9]+]]
;
entry:
  br label %for.body29

for.cond.cleanup28:
  unreachable

for.body29:
  %i24.0170 = phi i16 [ 0, %entry], [ %inc37, %for.body29]
  %add33 = add i16 undef, %i24.0170
  %idxprom34 = zext i16 %add33 to i32
  %arrayidx35 = getelementptr [2592 x i16], [2592 x i16] * @cm_array, i32 0, i32 %idxprom34
  store i16 0, i16 * %arrayidx35, align 1
  %inc37 = add i16 %i24.0170, 1
  %cmp26 = icmp ult i16 %inc37, 756
  br i1 %cmp26, label %for.body29, label %for.cond.cleanup28
}

define void @pr43371_pgso() !prof !14 {
; PGSO-LABEL: @pr43371_pgso(
; PGSO-NEXT:  entry:
; PGSO-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; PGSO:       vector.ph:
; PGSO-NEXT:    br label [[VECTOR_BODY:%.*]]
; PGSO:       vector.body:
; PGSO-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; PGSO-NEXT:    [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 0, i16 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; PGSO-NEXT:    [[TMP0:%.*]] = add <2 x i16> undef, [[VEC_IND]]
; PGSO-NEXT:    [[TMP1:%.*]] = zext <2 x i16> [[TMP0]] to <2 x i32>
; PGSO-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
; PGSO-NEXT:    [[TMP3:%.*]] = getelementptr [2592 x i16], [2592 x i16]* @cm_array, i32 0, i32 [[TMP2]]
; PGSO-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
; PGSO-NEXT:    [[TMP5:%.*]] = getelementptr [2592 x i16], [2592 x i16]* @cm_array, i32 0, i32 [[TMP4]]
; PGSO-NEXT:    store i16 0, i16* [[TMP3]], align 1
; PGSO-NEXT:    store i16 0, i16* [[TMP5]], align 1
; PGSO-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; PGSO-NEXT:    [[VEC_IND_NEXT]] = add <2 x i16> [[VEC_IND]], <i16 2, i16 2>
; PGSO-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 756
; PGSO-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
; PGSO:       middle.block:
; PGSO-NEXT:    [[CMP_N:%.*]] = icmp eq i32 756, 756
; PGSO-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP28:%.*]], label [[SCALAR_PH]]
; PGSO:       scalar.ph:
; PGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 756, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; PGSO-NEXT:    br label [[FOR_BODY29:%.*]]
; PGSO:       for.cond.cleanup28:
; PGSO-NEXT:    unreachable
; PGSO:       for.body29:
; PGSO-NEXT:    [[I24_0170:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC37:%.*]], [[FOR_BODY29]] ]
; PGSO-NEXT:    [[ADD33:%.*]] = add i16 undef, [[I24_0170]]
; PGSO-NEXT:    [[IDXPROM34:%.*]] = zext i16 [[ADD33]] to i32
; PGSO-NEXT:    [[ARRAYIDX35:%.*]] = getelementptr [2592 x i16], [2592 x i16]* @cm_array, i32 0, i32 [[IDXPROM34]]
; PGSO-NEXT:    store i16 0, i16* [[ARRAYIDX35]], align 1
; PGSO-NEXT:    [[INC37]] = add i16 [[I24_0170]], 1
; PGSO-NEXT:    [[CMP26:%.*]] = icmp ult i16 [[INC37]], 756
; PGSO-NEXT:    br i1 [[CMP26]], label [[FOR_BODY29]], label [[FOR_COND_CLEANUP28]], !llvm.loop [[LOOP26:![0-9]+]]
;
; NPGSO-LABEL: @pr43371_pgso(
; NPGSO-NEXT:  entry:
; NPGSO-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
; NPGSO:       vector.scevcheck:
; NPGSO-NEXT:    br i1 undef, label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; NPGSO:       vector.ph:
; NPGSO-NEXT:    br label [[VECTOR_BODY:%.*]]
; NPGSO:       vector.body:
; NPGSO-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; NPGSO-NEXT:    [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16
; NPGSO-NEXT:    [[TMP0:%.*]] = add i16 [[OFFSET_IDX]], 0
; NPGSO-NEXT:    [[TMP1:%.*]] = add i16 undef, [[TMP0]]
; NPGSO-NEXT:    [[TMP2:%.*]] = zext i16 [[TMP1]] to i32
; NPGSO-NEXT:    [[TMP3:%.*]] = getelementptr [2592 x i16], [2592 x i16]* @cm_array, i32 0, i32 [[TMP2]]
; NPGSO-NEXT:    [[TMP4:%.*]] = getelementptr i16, i16* [[TMP3]], i32 0
; NPGSO-NEXT:    [[TMP5:%.*]] = bitcast i16* [[TMP4]] to <2 x i16>*
; NPGSO-NEXT:    store <2 x i16> zeroinitializer, <2 x i16>* [[TMP5]], align 1
; NPGSO-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; NPGSO-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 756
; NPGSO-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
; NPGSO:       middle.block:
; NPGSO-NEXT:    [[CMP_N:%.*]] = icmp eq i32 756, 756
; NPGSO-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP28:%.*]], label [[SCALAR_PH]]
; NPGSO:       scalar.ph:
; NPGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 756, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
; NPGSO-NEXT:    br label [[FOR_BODY29:%.*]]
; NPGSO:       for.cond.cleanup28:
; NPGSO-NEXT:    unreachable
; NPGSO:       for.body29:
; NPGSO-NEXT:    [[I24_0170:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC37:%.*]], [[FOR_BODY29]] ]
; NPGSO-NEXT:    [[ADD33:%.*]] = add i16 undef, [[I24_0170]]
; NPGSO-NEXT:    [[IDXPROM34:%.*]] = zext i16 [[ADD33]] to i32
; NPGSO-NEXT:    [[ARRAYIDX35:%.*]] = getelementptr [2592 x i16], [2592 x i16]* @cm_array, i32 0, i32 [[IDXPROM34]]
; NPGSO-NEXT:    store i16 0, i16* [[ARRAYIDX35]], align 1
; NPGSO-NEXT:    [[INC37]] = add i16 [[I24_0170]], 1
; NPGSO-NEXT:    [[CMP26:%.*]] = icmp ult i16 [[INC37]], 756
; NPGSO-NEXT:    br i1 [[CMP26]], label [[FOR_BODY29]], label [[FOR_COND_CLEANUP28]], !llvm.loop [[LOOP26:![0-9]+]]
;
entry:
  br label %for.body29

for.cond.cleanup28:
  unreachable

for.body29:
  %i24.0170 = phi i16 [ 0, %entry], [ %inc37, %for.body29]
  %add33 = add i16 undef, %i24.0170
  %idxprom34 = zext i16 %add33 to i32
  %arrayidx35 = getelementptr [2592 x i16], [2592 x i16] * @cm_array, i32 0, i32 %idxprom34
  store i16 0, i16 * %arrayidx35, align 1
  %inc37 = add i16 %i24.0170, 1
  %cmp26 = icmp ult i16 %inc37, 756
  br i1 %cmp26, label %for.body29, label %for.cond.cleanup28
}

; PR45526: don't vectorize with fold-tail if first-order-recurrence is live-out.
;
define i32 @pr45526() optsize {
; DEFAULT-LABEL: @pr45526(
; DEFAULT-NEXT:  entry:
; DEFAULT-NEXT:    br label [[LOOP:%.*]]
; DEFAULT:       loop:
; DEFAULT-NEXT:    [[PIV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[PIVPLUS1:%.*]], [[LOOP]] ]
; DEFAULT-NEXT:    [[FOR:%.*]] = phi i32 [ 5, [[ENTRY]] ], [ [[PIVPLUS1]], [[LOOP]] ]
; DEFAULT-NEXT:    [[PIVPLUS1]] = add nuw nsw i32 [[PIV]], 1
; DEFAULT-NEXT:    [[COND:%.*]] = icmp ult i32 [[PIV]], 510
; DEFAULT-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
; DEFAULT:       exit:
; DEFAULT-NEXT:    [[FOR_LCSSA:%.*]] = phi i32 [ [[FOR]], [[LOOP]] ]
; DEFAULT-NEXT:    ret i32 [[FOR_LCSSA]]
;
entry:
  br label %loop

loop:
  %piv = phi i32 [ 0, %entry ], [ %pivPlus1, %loop ]
  %for = phi i32 [ 5, %entry ], [ %pivPlus1, %loop ]
  %pivPlus1 = add nuw nsw i32 %piv, 1
  %cond = icmp ult i32 %piv, 510
  br i1 %cond, label %loop, label %exit

exit:
  ret i32 %for
}

define i32 @pr45526_pgso() !prof !14 {
; PGSO-LABEL: @pr45526_pgso(
; PGSO-NEXT:  entry:
; PGSO-NEXT:    br label [[LOOP:%.*]]
; PGSO:       loop:
; PGSO-NEXT:    [[PIV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[PIVPLUS1:%.*]], [[LOOP]] ]
; PGSO-NEXT:    [[FOR:%.*]] = phi i32 [ 5, [[ENTRY]] ], [ [[PIVPLUS1]], [[LOOP]] ]
; PGSO-NEXT:    [[PIVPLUS1]] = add nuw nsw i32 [[PIV]], 1
; PGSO-NEXT:    [[COND:%.*]] = icmp ult i32 [[PIV]], 510
; PGSO-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
; PGSO:       exit:
; PGSO-NEXT:    [[FOR_LCSSA:%.*]] = phi i32 [ [[FOR]], [[LOOP]] ]
; PGSO-NEXT:    ret i32 [[FOR_LCSSA]]
;
; NPGSO-LABEL: @pr45526_pgso(
; NPGSO-NEXT:  entry:
; NPGSO-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; NPGSO:       vector.ph:
; NPGSO-NEXT:    br label [[VECTOR_BODY:%.*]]
; NPGSO:       vector.body:
; NPGSO-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; NPGSO-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; NPGSO-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 5>, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
; NPGSO-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
; NPGSO-NEXT:    [[TMP1:%.*]] = add i32 [[INDEX]], 1
; NPGSO-NEXT:    [[TMP2:%.*]] = add i32 [[INDEX]], 2
; NPGSO-NEXT:    [[TMP3:%.*]] = add i32 [[INDEX]], 3
; NPGSO-NEXT:    [[TMP4]] = add nuw nsw <4 x i32> [[VEC_IND]], <i32 1, i32 1, i32 1, i32 1>
; NPGSO-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP4]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; NPGSO-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; NPGSO-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
; NPGSO-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 508
; NPGSO-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
; NPGSO:       middle.block:
; NPGSO-NEXT:    [[CMP_N:%.*]] = icmp eq i32 511, 508
; NPGSO-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP4]], i32 3
; NPGSO-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP4]], i32 2
; NPGSO-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
; NPGSO:       scalar.ph:
; NPGSO-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 5, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ]
; NPGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 508, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
; NPGSO-NEXT:    br label [[LOOP:%.*]]
; NPGSO:       loop:
; NPGSO-NEXT:    [[PIV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[PIVPLUS1:%.*]], [[LOOP]] ]
; NPGSO-NEXT:    [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[PIVPLUS1]], [[LOOP]] ]
; NPGSO-NEXT:    [[PIVPLUS1]] = add nuw nsw i32 [[PIV]], 1
; NPGSO-NEXT:    [[COND:%.*]] = icmp ult i32 [[PIV]], 510
; NPGSO-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP28:![0-9]+]]
; NPGSO:       exit:
; NPGSO-NEXT:    [[FOR_LCSSA:%.*]] = phi i32 [ [[SCALAR_RECUR]], [[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], [[MIDDLE_BLOCK]] ]
; NPGSO-NEXT:    ret i32 [[FOR_LCSSA]]
;
entry:
  br label %loop

loop:
  %piv = phi i32 [ 0, %entry ], [ %pivPlus1, %loop ]
  %for = phi i32 [ 5, %entry ], [ %pivPlus1, %loop ]
  %pivPlus1 = add nuw nsw i32 %piv, 1
  %cond = icmp ult i32 %piv, 510
  br i1 %cond, label %loop, label %exit

exit:
  ret i32 %for
}

; PR46228: Vectorize w/o versioning for unit stride under optsize and enabled
; vectorization.

; NOTE: Some assertions have been autogenerated by utils/update_test_checks.py
define void @stride1(i16* noalias %B, i32 %BStride) optsize {
; PGSO-LABEL: @stride1(
; PGSO-NEXT:  entry:
; PGSO-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; PGSO:       vector.ph:
; PGSO-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[BSTRIDE:%.*]], i32 0
; PGSO-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
; PGSO-NEXT:    br label [[VECTOR_BODY:%.*]]
; PGSO:       vector.body:
; PGSO-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
; PGSO-NEXT:    [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE2]] ]
; PGSO-NEXT:    [[TMP0:%.*]] = icmp ule <2 x i32> [[VEC_IND]], <i32 1024, i32 1024>
; PGSO-NEXT:    [[TMP1:%.*]] = mul nsw <2 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; PGSO-NEXT:    [[TMP2:%.*]] = extractelement <2 x i1> [[TMP0]], i32 0
; PGSO-NEXT:    br i1 [[TMP2]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; PGSO:       pred.store.if:
; PGSO-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
; PGSO-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i16, i16* [[B:%.*]], i32 [[TMP3]]
; PGSO-NEXT:    store i16 42, i16* [[TMP4]], align 4
; PGSO-NEXT:    br label [[PRED_STORE_CONTINUE]]
; PGSO:       pred.store.continue:
; PGSO-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP0]], i32 1
; PGSO-NEXT:    br i1 [[TMP5]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
; PGSO:       pred.store.if1:
; PGSO-NEXT:    [[TMP6:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
; PGSO-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i16, i16* [[B]], i32 [[TMP6]]
; PGSO-NEXT:    store i16 42, i16* [[TMP7]], align 4
; PGSO-NEXT:    br label [[PRED_STORE_CONTINUE2]]
; PGSO:       pred.store.continue2:
; PGSO-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
; PGSO-NEXT:    [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], <i32 2, i32 2>
; PGSO-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1026
; PGSO-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
; PGSO:       middle.block:
; PGSO-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
; PGSO:       scalar.ph:
; PGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1026, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; PGSO-NEXT:    br label [[FOR_BODY:%.*]]
; PGSO:       for.body:
; PGSO-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
; PGSO-NEXT:    [[MULB:%.*]] = mul nsw i32 [[IV]], [[BSTRIDE]]
; PGSO-NEXT:    [[GEPOFB:%.*]] = getelementptr inbounds i16, i16* [[B]], i32 [[MULB]]
; PGSO-NEXT:    store i16 42, i16* [[GEPOFB]], align 4
; PGSO-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
; PGSO-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[IV_NEXT]], 1025
; PGSO-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
; PGSO:       for.end:
; PGSO-NEXT:    ret void
;
; NPGSO-LABEL: @stride1(
; NPGSO-NEXT:  entry:
; NPGSO-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; NPGSO:       vector.ph:
; NPGSO-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[BSTRIDE:%.*]], i32 0
; NPGSO-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
; NPGSO-NEXT:    br label [[VECTOR_BODY:%.*]]
; NPGSO:       vector.body:
; NPGSO-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
; NPGSO-NEXT:    [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE2]] ]
; NPGSO-NEXT:    [[TMP0:%.*]] = icmp ule <2 x i32> [[VEC_IND]], <i32 1024, i32 1024>
; NPGSO-NEXT:    [[TMP1:%.*]] = mul nsw <2 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; NPGSO-NEXT:    [[TMP2:%.*]] = extractelement <2 x i1> [[TMP0]], i32 0
; NPGSO-NEXT:    br i1 [[TMP2]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; NPGSO:       pred.store.if:
; NPGSO-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
; NPGSO-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i16, i16* [[B:%.*]], i32 [[TMP3]]
; NPGSO-NEXT:    store i16 42, i16* [[TMP4]], align 4
; NPGSO-NEXT:    br label [[PRED_STORE_CONTINUE]]
; NPGSO:       pred.store.continue:
; NPGSO-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP0]], i32 1
; NPGSO-NEXT:    br i1 [[TMP5]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
; NPGSO:       pred.store.if1:
; NPGSO-NEXT:    [[TMP6:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
; NPGSO-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i16, i16* [[B]], i32 [[TMP6]]
; NPGSO-NEXT:    store i16 42, i16* [[TMP7]], align 4
; NPGSO-NEXT:    br label [[PRED_STORE_CONTINUE2]]
; NPGSO:       pred.store.continue2:
; NPGSO-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
; NPGSO-NEXT:    [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], <i32 2, i32 2>
; NPGSO-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1026
; NPGSO-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
; NPGSO:       middle.block:
; NPGSO-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
; NPGSO:       scalar.ph:
; NPGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1026, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; NPGSO-NEXT:    br label [[FOR_BODY:%.*]]
; NPGSO:       for.body:
; NPGSO-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
; NPGSO-NEXT:    [[MULB:%.*]] = mul nsw i32 [[IV]], [[BSTRIDE]]
; NPGSO-NEXT:    [[GEPOFB:%.*]] = getelementptr inbounds i16, i16* [[B]], i32 [[MULB]]
; NPGSO-NEXT:    store i16 42, i16* [[GEPOFB]], align 4
; NPGSO-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
; NPGSO-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[IV_NEXT]], 1025
; NPGSO-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
; NPGSO:       for.end:
; NPGSO-NEXT:    ret void
;
entry:
  br label %for.body

for.body:
  %iv = phi i32 [ %iv.next, %for.body ], [ 0, %entry ]
  %mulB = mul nsw i32 %iv, %BStride
  %gepOfB = getelementptr inbounds i16, i16* %B, i32 %mulB
  store i16 42, i16* %gepOfB, align 4
  %iv.next = add nuw nsw i32 %iv, 1
  %exitcond = icmp eq i32 %iv.next, 1025
  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !15

for.end:
  ret void
}

; Vectorize with versioning for unit stride for PGSO and enabled vectorization.
;
define void @stride1_pgso(i16* noalias %B, i32 %BStride) !prof !14 {
; PGSO-LABEL: @stride1_pgso(
; PGSO-NEXT:  entry:
; PGSO-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
; PGSO:       vector.scevcheck:
; PGSO-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i32 [[BSTRIDE:%.*]], 1
; PGSO-NEXT:    br i1 [[IDENT_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; PGSO:       vector.ph:
; PGSO-NEXT:    br label [[VECTOR_BODY:%.*]]
; PGSO:       vector.body:
; PGSO-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; PGSO-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
; PGSO-NEXT:    [[TMP1:%.*]] = mul nsw i32 [[TMP0]], [[BSTRIDE]]
; PGSO-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, i16* [[B:%.*]], i32 [[TMP1]]
; PGSO-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i16, i16* [[TMP2]], i32 0
; PGSO-NEXT:    [[TMP4:%.*]] = bitcast i16* [[TMP3]] to <2 x i16>*
; PGSO-NEXT:    store <2 x i16> <i16 42, i16 42>, <2 x i16>* [[TMP4]], align 4
; PGSO-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; PGSO-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
; PGSO-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
; PGSO:       middle.block:
; PGSO-NEXT:    [[CMP_N:%.*]] = icmp eq i32 1025, 1024
; PGSO-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
; PGSO:       scalar.ph:
; PGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
; PGSO-NEXT:    br label [[FOR_BODY:%.*]]
; PGSO:       for.body:
; PGSO-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
; PGSO-NEXT:    [[MULB:%.*]] = mul nsw i32 [[IV]], [[BSTRIDE]]
; PGSO-NEXT:    [[GEPOFB:%.*]] = getelementptr inbounds i16, i16* [[B]], i32 [[MULB]]
; PGSO-NEXT:    store i16 42, i16* [[GEPOFB]], align 4
; PGSO-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
; PGSO-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[IV_NEXT]], 1025
; PGSO-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
; PGSO:       for.end:
; PGSO-NEXT:    ret void
;
; NPGSO-LABEL: @stride1_pgso(
; NPGSO-NEXT:  entry:
; NPGSO-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
; NPGSO:       vector.scevcheck:
; NPGSO-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i32 [[BSTRIDE:%.*]], 1
; NPGSO-NEXT:    br i1 [[IDENT_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; NPGSO:       vector.ph:
; NPGSO-NEXT:    br label [[VECTOR_BODY:%.*]]
; NPGSO:       vector.body:
; NPGSO-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; NPGSO-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
; NPGSO-NEXT:    [[TMP1:%.*]] = mul nsw i32 [[TMP0]], [[BSTRIDE]]
; NPGSO-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, i16* [[B:%.*]], i32 [[TMP1]]
; NPGSO-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i16, i16* [[TMP2]], i32 0
; NPGSO-NEXT:    [[TMP4:%.*]] = bitcast i16* [[TMP3]] to <2 x i16>*
; NPGSO-NEXT:    store <2 x i16> <i16 42, i16 42>, <2 x i16>* [[TMP4]], align 4
; NPGSO-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; NPGSO-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
; NPGSO-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
; NPGSO:       middle.block:
; NPGSO-NEXT:    [[CMP_N:%.*]] = icmp eq i32 1025, 1024
; NPGSO-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
; NPGSO:       scalar.ph:
; NPGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
; NPGSO-NEXT:    br label [[FOR_BODY:%.*]]
; NPGSO:       for.body:
; NPGSO-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
; NPGSO-NEXT:    [[MULB:%.*]] = mul nsw i32 [[IV]], [[BSTRIDE]]
; NPGSO-NEXT:    [[GEPOFB:%.*]] = getelementptr inbounds i16, i16* [[B]], i32 [[MULB]]
; NPGSO-NEXT:    store i16 42, i16* [[GEPOFB]], align 4
; NPGSO-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
; NPGSO-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[IV_NEXT]], 1025
; NPGSO-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
; NPGSO:       for.end:
; NPGSO-NEXT:    ret void
;
entry:
  br label %for.body

for.body:
  %iv = phi i32 [ %iv.next, %for.body ], [ 0, %entry ]
  %mulB = mul nsw i32 %iv, %BStride
  %gepOfB = getelementptr inbounds i16, i16* %B, i32 %mulB
  store i16 42, i16* %gepOfB, align 4
  %iv.next = add nuw nsw i32 %iv, 1
  %exitcond = icmp eq i32 %iv.next, 1025
  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !15

for.end:
  ret void
}

; PR46652: Check that the need for stride==1 check prevents vectorizing a loop
; having tiny trip count, when compiling w/o -Os/-Oz.

@g = external global [1 x i16], align 1

define void @pr46652(i16 %stride) {
; DEFAULT-LABEL: @pr46652(
; DEFAULT-NEXT:  entry:
; DEFAULT-NEXT:    br label [[FOR_BODY:%.*]]
; DEFAULT:       for.body:
; DEFAULT-NEXT:    [[L1_02:%.*]] = phi i16 [ 1, [[ENTRY:%.*]] ], [ [[INC9:%.*]], [[FOR_BODY]] ]
; DEFAULT-NEXT:    [[MUL:%.*]] = mul nsw i16 [[L1_02]], [[STRIDE:%.*]]
; DEFAULT-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds [1 x i16], [1 x i16]* @g, i16 0, i16 [[MUL]]
; DEFAULT-NEXT:    [[TMP0:%.*]] = load i16, i16* [[ARRAYIDX6]], align 1
; DEFAULT-NEXT:    [[INC9]] = add nuw nsw i16 [[L1_02]], 1
; DEFAULT-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i16 [[INC9]], 16
; DEFAULT-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]]
; DEFAULT:       for.end:
; DEFAULT-NEXT:    ret void
;
entry:
  br label %for.body

for.body:                                        ; preds = %for.body, %entry
  %l1.02 = phi i16 [ 1, %entry ], [ %inc9, %for.body ]
  %mul = mul nsw i16 %l1.02, %stride
  %arrayidx6 = getelementptr inbounds [1 x i16], [1 x i16]* @g, i16 0, i16 %mul
  %0 = load i16, i16* %arrayidx6, align 1
  %inc9 = add nuw nsw i16 %l1.02, 1
  %exitcond.not = icmp eq i16 %inc9, 16
  br i1 %exitcond.not, label %for.end, label %for.body

for.end:                                        ; preds = %for.body
  ret void
}

; Make sure we do not crash while building the VPlan for the loop with the
; select below.
define i32 @PR48142(i32* %ptr.start, i32* %ptr.end) optsize {
; DEFAULT-LABEL: @PR48142(
; DEFAULT-NEXT:  entry:
; DEFAULT-NEXT:    br label [[FOR_BODY:%.*]]
; DEFAULT:       for.body:
; DEFAULT-NEXT:    [[I_014:%.*]] = phi i32 [ 20, [[ENTRY:%.*]] ], [ [[COND:%.*]], [[FOR_BODY]] ]
; DEFAULT-NEXT:    [[PTR_IV:%.*]] = phi i32* [ [[PTR_START:%.*]], [[ENTRY]] ], [ [[PTR_NEXT:%.*]], [[FOR_BODY]] ]
; DEFAULT-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[I_014]], 99
; DEFAULT-NEXT:    [[COND]] = select i1 [[CMP4]], i32 99, i32 [[I_014]]
; DEFAULT-NEXT:    store i32 0, i32* [[PTR_IV]], align 4
; DEFAULT-NEXT:    [[PTR_NEXT]] = getelementptr inbounds i32, i32* [[PTR_IV]], i64 1
; DEFAULT-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32* [[PTR_NEXT]], [[PTR_END:%.*]]
; DEFAULT-NEXT:    br i1 [[CMP_NOT]], label [[EXIT:%.*]], label [[FOR_BODY]]
; DEFAULT:       exit:
; DEFAULT-NEXT:    [[RES:%.*]] = phi i32 [ [[COND]], [[FOR_BODY]] ]
; DEFAULT-NEXT:    ret i32 [[RES]]
;
entry:
  br label %for.body

for.body:
  %i.014 = phi i32 [ 20, %entry ], [ %cond, %for.body ]
  %ptr.iv = phi i32* [ %ptr.start, %entry ], [ %ptr.next, %for.body ]
  %cmp4 = icmp slt i32 %i.014, 99
  %cond = select i1 %cmp4, i32 99, i32 %i.014
  store i32 0, i32* %ptr.iv
  %ptr.next = getelementptr inbounds i32, i32* %ptr.iv, i64 1
  %cmp.not = icmp eq i32* %ptr.next, %ptr.end
  br i1 %cmp.not, label %exit, label %for.body

exit:
  %res = phi i32 [ %cond, %for.body ]
  ret i32 %res
}

!llvm.module.flags = !{!0}
!0 = !{i32 1, !"ProfileSummary", !1}
!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
!2 = !{!"ProfileFormat", !"InstrProf"}
!3 = !{!"TotalCount", i64 10000}
!4 = !{!"MaxCount", i64 10}
!5 = !{!"MaxInternalCount", i64 1}
!6 = !{!"MaxFunctionCount", i64 1000}
!7 = !{!"NumCounts", i64 3}
!8 = !{!"NumFunctions", i64 3}
!9 = !{!"DetailedSummary", !10}
!10 = !{!11, !12, !13}
!11 = !{i32 10000, i64 100, i32 1}
!12 = !{i32 999000, i64 100, i32 1}
!13 = !{i32 999999, i64 1, i32 2}
!14 = !{!"function_entry_count", i64 0}
!15 = distinct !{!15, !16}
!16 = !{!"llvm.loop.vectorize.enable", i1 true}