Lines Matching refs:OrigLoop
440 InnerLoopVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE, in InnerLoopVectorizer() argument
449 : OrigLoop(OrigLoop), PSE(PSE), LI(LI), DT(DT), TLI(TLI), TTI(TTI), in InnerLoopVectorizer()
456 OrigLoop->getHeader(), PSI, BFI, PGSOQueryType::IRPass); in InnerLoopVectorizer()
633 Loop *OrigLoop; member in llvm::InnerLoopVectorizer
737 InnerLoopUnroller(Loop *OrigLoop, PredicatedScalarEvolution &PSE, in InnerLoopUnroller() argument
745 : InnerLoopVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE, in InnerLoopUnroller()
787 Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, in InnerLoopAndEpilogueVectorizer() argument
794 : InnerLoopVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE, in InnerLoopAndEpilogueVectorizer()
826 Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, in EpilogueVectorizerMainLoop() argument
833 : InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE, in EpilogueVectorizerMainLoop()
854 Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, in EpilogueVectorizerEpilogueLoop() argument
861 : InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE, in EpilogueVectorizerEpilogueLoop()
2251 bool SafeToHoist = OrigLoop->isLoopInvariant(V) && in getBroadcastInstrs()
2398 Loop *OrigLoop = nullptr) { in CreateStepValue() argument
2400 assert((!OrigLoop || SE.isLoopInvariant(Step, OrigLoop)) && in CreateStepValue()
3046 OrigLoop->getStartLoc(), in emitMemRuntimeChecks()
3047 OrigLoop->getHeader()) in emitMemRuntimeChecks()
3063 LoopScalarBody = OrigLoop->getHeader(); in createVectorLoopSkeleton()
3064 LoopVectorPreHeader = OrigLoop->getLoopPreheader(); in createVectorLoopSkeleton()
3066 LoopExitBlock = OrigLoop->getUniqueExitBlock(); // may be nullptr in createVectorLoopSkeleton()
3077 auto *ScalarLatchTerm = OrigLoop->getLoopLatch()->getTerminator(); in createVectorLoopSkeleton()
3190 auto *ScalarLatchTerm = OrigLoop->getLoopLatch()->getTerminator(); in completeLoopSkeleton()
3255 MDNode *OrigLoopID = OrigLoop->getLoopID(); in createVectorizedLoopSkeleton()
3264 getOrCreateTripCount(OrigLoop->getLoopPreheader()); in createVectorizedLoopSkeleton()
3306 assert(OrigLoop->getUniqueExitBlock() && "Expected a single exit block"); in fixupIVUsers()
3312 Value *PostInc = OrigPhi->getIncomingValueForBlock(OrigLoop->getLoopLatch()); in fixupIVUsers()
3315 if (!OrigLoop->contains(UI)) { in fixupIVUsers()
3326 if (!OrigLoop->contains(UI)) { in fixupIVUsers()
3660 PSE.getSE()->forgetLoop(OrigLoop); in fixVectorizedLoop()
4033 OrigPhi->getBasicBlockIndex(OrigLoop->getLoopLatch()); in fixReduction()
7391 if (!OrigLoop->isInnermost()) { in planInVPlanNativePath()
7430 assert(OrigLoop->isInnermost() && "Inner loop expected."); in plan()
7436 if (CM.blockNeedsPredicationForAnyReason(OrigLoop->getHeader()) && in plan()
7465 "InvalidCost", ORE, OrigLoop); in plan()
7576 *LAI, LAI->getRuntimePointerChecking()->getChecks(), OrigLoop, LI, DT, in executePlan()
7603 MDNode *OrigLoopID = OrigLoop->getLoopID(); in executePlan()
7617 if (MDNode *LID = OrigLoop->getLoopID()) in executePlan()
7654 MDNode *OrigLoopID = OrigLoop->getLoopID(); in createEpilogueVectorizedLoopSkeleton()
7663 getOrCreateTripCount(OrigLoop->getLoopPreheader()); in createEpilogueVectorizedLoopSkeleton()
7714 << *OrigLoop->getHeader()->getParent() << "\n"; in printDebugTracesAtEnd()
7782 MDNode *OrigLoopID = OrigLoop->getLoopID(); in createEpilogueVectorizedLoopSkeleton()
7920 dbgs() << "final fn:\n" << *OrigLoop->getHeader()->getParent() << "\n"; in printDebugTracesAtEnd()
7976 if (OrigLoop->isLoopExiting(Src)) in createEdgeMask()
8000 assert(OrigLoop->contains(BB) && "Block is not a part of a loop"); in createBlockInMask()
8011 if (OrigLoop->getHeader() == BB) { in createBlockInMask()
8113 VPlan &Plan, ScalarEvolution &SE, Loop &OrigLoop, VFRange &Range) { in createWidenInductionRecipes() argument
8127 Phi->getIncomingValueForBlock(OrigLoop.getLoopPreheader())); in createWidenInductionRecipes()
8128 assert(SE.isLoopInvariant(IndDesc.getStep(), &OrigLoop) && in createWidenInductionRecipes()
8149 *PSE.getSE(), *OrigLoop, Range); in tryToOptimizeInductionPHI()
8187 *PSE.getSE(), *OrigLoop, Range); in tryToOptimizeInductionTruncate()
8347 BasicBlock *OrigLatch = OrigLoop->getLoopLatch(); in fixHeaderPhis()
8483 if (Phi->getParent() != OrigLoop->getHeader()) in tryToCreateWidenRecipe()
8497 Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader())); in tryToCreateWidenRecipe()
8508 Phi->getIncomingValueForBlock(OrigLoop->getLoopLatch()))); in tryToCreateWidenRecipe()
8534 GEP, make_range(Operands.begin(), Operands.end()), OrigLoop)); in tryToCreateWidenRecipe()
8538 PSE.getSE()->isLoopInvariant(PSE.getSCEV(SI->getOperand(0)), OrigLoop); in tryToCreateWidenRecipe()
8548 assert(OrigLoop->isInnermost() && "Inner loop expected."); in buildVPlansWithVPRecipes()
8673 VPBasicBlock *MiddleVPBB, Loop *OrigLoop, in addUsersInExitBlock() argument
8675 BasicBlock *ExitBB = OrigLoop->getUniqueExitBlock(); in addUsersInExitBlock()
8676 BasicBlock *ExitingBB = OrigLoop->getExitingBlock(); in addUsersInExitBlock()
8696 VPRecipeBuilder RecipeBuilder(OrigLoop, TLI, Legal, CM, PSE, Builder); in buildVPlanWithVPRecipes()
8773 LoopBlocksDFS DFS(OrigLoop); in buildVPlanWithVPRecipes()
8798 if (Phi && Phi->getParent() == OrigLoop->getHeader()) { in buildVPlanWithVPRecipes()
8800 Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader()))); in buildVPlanWithVPRecipes()
8871 addUsersInExitBlock(HeaderVPBB, MiddleVPBB, OrigLoop, *Plan); in buildVPlanWithVPRecipes()
9057 assert(!OrigLoop->isInnermost()); in buildVPlan()
9064 VPlanHCFGBuilder HCFGBuilder(OrigLoop, LI, *Plan); in buildVPlan()
9073 OrigLoop, Plan, in buildVPlan()
9186 RecipeBuilder.createBlockInMask(OrigLoop->getHeader(), Plan); in adjustRecipesForReductions()