Lines Matching refs:EPI
790 OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI, in InnerLoopAndEpilogueVectorizer() argument
795 EPI.MainLoopVF, EPI.MainLoopVF, EPI.MainLoopUF, LVL, in InnerLoopAndEpilogueVectorizer()
797 EPI(EPI) {} in InnerLoopAndEpilogueVectorizer()
817 EpilogueLoopVectorizationInfo &EPI; member in llvm::InnerLoopAndEpilogueVectorizer
829 OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI, in EpilogueVectorizerMainLoop() argument
834 EPI, LVL, CM, BFI, PSI, Check) {} in EpilogueVectorizerMainLoop()
857 OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI, in EpilogueVectorizerEpilogueLoop() argument
862 EPI, LVL, CM, BFI, PSI, Checks) { in EpilogueVectorizerEpilogueLoop()
863 TripCount = EPI.TripCount; in EpilogueVectorizerEpilogueLoop()
7668 EPI.EpilogueIterationCountCheck = in createEpilogueVectorizedLoopSkeleton()
7670 EPI.EpilogueIterationCountCheck->setName("iter.check"); in createEpilogueVectorizedLoopSkeleton()
7674 EPI.SCEVSafetyCheck = emitSCEVChecks(LoopScalarPreHeader); in createEpilogueVectorizedLoopSkeleton()
7679 EPI.MemSafetyCheck = emitMemRuntimeChecks(LoopScalarPreHeader); in createEpilogueVectorizedLoopSkeleton()
7687 EPI.MainLoopIterationCountCheck = in createEpilogueVectorizedLoopSkeleton()
7691 EPI.VectorTripCount = getOrCreateVectorTripCount(LoopVectorPreHeader); in createEpilogueVectorizedLoopSkeleton()
7704 << "Main Loop VF:" << EPI.MainLoopVF in printDebugTracesAtStart()
7705 << ", Main Loop UF:" << EPI.MainLoopUF in printDebugTracesAtStart()
7706 << ", Epilogue Loop VF:" << EPI.EpilogueVF in printDebugTracesAtStart()
7707 << ", Epilogue Loop UF:" << EPI.EpilogueUF << "\n"; in printDebugTracesAtStart()
7722 ElementCount VFactor = ForEpilogue ? EPI.EpilogueVF : VF; in emitIterationCountCheck()
7723 unsigned UFactor = ForEpilogue ? EPI.EpilogueUF : UF; in emitIterationCountCheck()
7732 auto P = Cost->requiresScalarEpilogue(ForEpilogue ? EPI.EpilogueVF : VF) ? in emitIterationCountCheck()
7753 if (!Cost->requiresScalarEpilogue(EPI.EpilogueVF)) in emitIterationCountCheck()
7764 EPI.TripCount = Count; in emitIterationCountCheck()
7797 assert(EPI.MainLoopIterationCountCheck && EPI.EpilogueIterationCountCheck && in createEpilogueVectorizedLoopSkeleton()
7799 EPI.MainLoopIterationCountCheck->getTerminator()->replaceUsesOfWith( in createEpilogueVectorizedLoopSkeleton()
7803 EPI.MainLoopIterationCountCheck); in createEpilogueVectorizedLoopSkeleton()
7805 EPI.EpilogueIterationCountCheck->getTerminator()->replaceUsesOfWith( in createEpilogueVectorizedLoopSkeleton()
7808 if (EPI.SCEVSafetyCheck) in createEpilogueVectorizedLoopSkeleton()
7809 EPI.SCEVSafetyCheck->getTerminator()->replaceUsesOfWith( in createEpilogueVectorizedLoopSkeleton()
7811 if (EPI.MemSafetyCheck) in createEpilogueVectorizedLoopSkeleton()
7812 EPI.MemSafetyCheck->getTerminator()->replaceUsesOfWith( in createEpilogueVectorizedLoopSkeleton()
7820 EPI.EpilogueIterationCountCheck); in createEpilogueVectorizedLoopSkeleton()
7821 if (!Cost->requiresScalarEpilogue(EPI.EpilogueVF)) in createEpilogueVectorizedLoopSkeleton()
7826 EPI.EpilogueIterationCountCheck); in createEpilogueVectorizedLoopSkeleton()
7830 if (EPI.SCEVSafetyCheck) in createEpilogueVectorizedLoopSkeleton()
7831 LoopBypassBlocks.push_back(EPI.SCEVSafetyCheck); in createEpilogueVectorizedLoopSkeleton()
7832 if (EPI.MemSafetyCheck) in createEpilogueVectorizedLoopSkeleton()
7833 LoopBypassBlocks.push_back(EPI.MemSafetyCheck); in createEpilogueVectorizedLoopSkeleton()
7834 LoopBypassBlocks.push_back(EPI.EpilogueIterationCountCheck); in createEpilogueVectorizedLoopSkeleton()
7847 Phi->removeIncomingValue(EPI.EpilogueIterationCountCheck); in createEpilogueVectorizedLoopSkeleton()
7848 if (EPI.SCEVSafetyCheck) in createEpilogueVectorizedLoopSkeleton()
7849 Phi->removeIncomingValue(EPI.SCEVSafetyCheck); in createEpilogueVectorizedLoopSkeleton()
7850 if (EPI.MemSafetyCheck) in createEpilogueVectorizedLoopSkeleton()
7851 Phi->removeIncomingValue(EPI.MemSafetyCheck); in createEpilogueVectorizedLoopSkeleton()
7860 EPResumeVal->addIncoming(EPI.VectorTripCount, VecEpilogueIterationCountCheck); in createEpilogueVectorizedLoopSkeleton()
7862 EPI.MainLoopIterationCountCheck); in createEpilogueVectorizedLoopSkeleton()
7872 EPI.VectorTripCount} /* AdditionalBypass */); in createEpilogueVectorizedLoopSkeleton()
7881 assert(EPI.TripCount && in emitMinimumVectorEpilogueIterCountCheck()
7884 (!isa<Instruction>(EPI.TripCount) || in emitMinimumVectorEpilogueIterCountCheck()
7885 DT->dominates(cast<Instruction>(EPI.TripCount)->getParent(), Insert)) && in emitMinimumVectorEpilogueIterCountCheck()
7887 Value *TC = EPI.TripCount; in emitMinimumVectorEpilogueIterCountCheck()
7889 Value *Count = Builder.CreateSub(TC, EPI.VectorTripCount, "n.vec.remaining"); in emitMinimumVectorEpilogueIterCountCheck()
7893 auto P = Cost->requiresScalarEpilogue(EPI.EpilogueVF) ? in emitMinimumVectorEpilogueIterCountCheck()
7899 EPI.EpilogueVF, EPI.EpilogueUF), in emitMinimumVectorEpilogueIterCountCheck()
7913 << "Epilogue Loop VF:" << EPI.EpilogueVF in printDebugTracesAtStart()
7914 << ", Epilogue Loop UF:" << EPI.EpilogueUF << "\n"; in printDebugTracesAtStart()
10358 EpilogueLoopVectorizationInfo EPI(VF.Width, IC, EpilogueVF.Width, 1); in processLoop() local
10360 EPI, &LVL, &CM, BFI, PSI, Checks); in processLoop()
10362 VPlan &BestMainPlan = LVP.getBestPlanFor(EPI.MainLoopVF); in processLoop()
10363 LVP.executePlan(EPI.MainLoopVF, EPI.MainLoopUF, BestMainPlan, MainILV, in processLoop()
10369 EPI.MainLoopVF = EPI.EpilogueVF; in processLoop()
10370 EPI.MainLoopUF = EPI.EpilogueUF; in processLoop()
10372 ORE, EPI, &LVL, &CM, BFI, PSI, in processLoop()
10375 VPlan &BestEpiPlan = LVP.getBestPlanFor(EPI.EpilogueVF); in processLoop()
10392 LVP.executePlan(EPI.EpilogueVF, EPI.EpilogueUF, BestEpiPlan, EpilogILV, in processLoop()