1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py 2; REQUIRES: asserts 3; RUN: opt < %s -passes='loop-vectorize' -force-target-supports-scalable-vectors=true -enable-epilogue-vectorization -epilogue-vectorization-force-VF=2 --debug-only=loop-vectorize -S -scalable-vectorization=on 2>&1 | FileCheck %s 4 5target datalayout = "e-m:e-i64:64-n32:64-v256:256:256-v512:512:512" 6 7; Currently we cannot handle scalable vectorization factors. 8; CHECK: LV: Checking a loop in 'f1' 9; CHECK: LEV: Epilogue vectorization factor is forced. 10; CHECK: Epilogue Loop VF:2, Epilogue Loop UF:1 11 12define void @f1(i8* %A) { 13; CHECK-LABEL: @f1( 14; CHECK-NEXT: iter.check: 15; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]] 16; CHECK: vector.main.loop.iter.check: 17; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 18; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 19; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]] 20; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]] 21; CHECK: vector.ph: 22; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() 23; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 24; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] 25; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] 26; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 27; CHECK: vector.body: 28; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 29; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 30; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, i8* [[A:%.*]], i64 [[TMP4]] 31; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, i8* [[TMP5]], i32 0 32; CHECK-NEXT: [[TMP7:%.*]] = bitcast i8* [[TMP6]] to <vscale x 4 x i8>* 33; CHECK-NEXT: store <vscale x 4 x i8> shufflevector (<vscale x 4 x i8> insertelement (<vscale x 4 x i8> poison, i8 1, i32 0), <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x i8>* [[TMP7]], align 1 34; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() 35; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 36; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] 37; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 38; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] 39; CHECK: middle.block: 40; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]] 41; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]] 42; CHECK: vec.epilog.iter.check: 43; CHECK-NEXT: [[N_VEC_REMAINING:%.*]] = sub i64 1024, [[N_VEC]] 44; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 2 45; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]] 46; CHECK: vec.epilog.ph: 47; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ] 48; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]] 49; CHECK: vec.epilog.vector.body: 50; CHECK-NEXT: [[OFFSET_IDX:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ] 51; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[OFFSET_IDX]], 0 52; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, i8* [[A]], i64 [[TMP11]] 53; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, i8* [[TMP12]], i32 0 54; CHECK-NEXT: [[TMP14:%.*]] = bitcast i8* [[TMP13]] to <2 x i8>* 55; CHECK-NEXT: store <2 x i8> <i8 1, i8 1>, <2 x i8>* [[TMP14]], align 1 56; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[OFFSET_IDX]], 2 57; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 1024 58; CHECK-NEXT: br i1 [[TMP15]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]] 59; CHECK: vec.epilog.middle.block: 60; CHECK-NEXT: [[CMP_N1:%.*]] = icmp eq i64 1024, 1024 61; CHECK-NEXT: br i1 [[CMP_N1]], label [[EXIT_LOOPEXIT:%.*]], label [[VEC_EPILOG_SCALAR_PH]] 62; CHECK: vec.epilog.scalar.ph: 63; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ] 64; CHECK-NEXT: br label [[FOR_BODY:%.*]] 65; CHECK: for.body: 66; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] 67; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, i8* [[A]], i64 [[IV]] 68; CHECK-NEXT: store i8 1, i8* [[ARRAYIDX]], align 1 69; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 70; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[IV_NEXT]], 1024 71; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP4:![0-9]+]] 72; CHECK: exit.loopexit: 73; CHECK-NEXT: br label [[EXIT]] 74; CHECK: exit: 75; CHECK-NEXT: ret void 76; 77entry: 78 br label %for.body 79 80for.body: 81 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] 82 %arrayidx = getelementptr inbounds i8, i8* %A, i64 %iv 83 store i8 1, i8* %arrayidx, align 1 84 %iv.next = add nuw nsw i64 %iv, 1 85 %exitcond = icmp ne i64 %iv.next, 1024 86 br i1 %exitcond, label %for.body, label %exit, !llvm.loop !0 87 88exit: 89 ret void 90} 91 92!0 = !{!0, !1} 93!1 = !{!"llvm.loop.vectorize.scalable.enable", i1 true} 94