Lines Matching refs:Plan
481 void fixVectorizedLoop(VPTransformState &State, VPlan &Plan);
515 void fixNonInductionPHIs(VPlan &Plan, VPTransformState &State);
548 VPlan &Plan);
1049 VPBlockRecursiveTraversalWrapper<VPBlockBase *>(State.Plan->getEntry())); in collectPoisonGeneratingRecipes()
3300 BasicBlock *VectorHeader, VPlan &Plan) { in fixupIVUsers() argument
3362 Plan.removeLiveOut(PHI); in fixupIVUsers()
3519 VPValue *Def = State.Plan->getVPValue(KV.first, true); in truncateToMinimalBitwidths()
3627 VPValue *Def = State.Plan->getVPValue(KV.first, true); in truncateToMinimalBitwidths()
3643 VPlan &Plan) { in fixVectorizedLoop() argument
3651 fixNonInductionPHIs(Plan, State); in fixVectorizedLoop()
3662 VPBasicBlock *LatchVPBB = Plan.getVectorLoopRegion()->getExitingBasicBlock(); in fixVectorizedLoop()
3668 Plan.clearLiveOuts(); in fixVectorizedLoop()
3679 VectorLoop->getHeader(), Plan); in fixVectorizedLoop()
3685 for (auto &KV : Plan.getLiveOuts()) in fixVectorizedLoop()
3686 KV.second->fixPhi(Plan, State); in fixVectorizedLoop()
3720 State.Plan->getVectorLoopRegion()->getEntryBasicBlock(); in fixCrossIterationPHIs()
3838 State.Plan->removeLiveOut(&LCSSAPhi); in fixFirstOrderRecurrence()
4027 State.Plan->removeLiveOut(&LCSSAPhi); in fixReduction()
4147 void InnerLoopVectorizer::fixNonInductionPHIs(VPlan &Plan, in fixNonInductionPHIs() argument
4150 VPBlockRecursiveTraversalWrapper<VPBlockBase *>(Plan.getEntry())); in fixNonInductionPHIs()
7503 [VF](const VPlanPtr &Plan) { return Plan->hasVF(VF); }) == in getBestPlanFor() argument
7507 for (const VPlanPtr &Plan : VPlans) { in getBestPlanFor() local
7508 if (Plan->hasVF(VF)) in getBestPlanFor()
7509 return *Plan.get(); in getBestPlanFor()
7636 for (const auto &Plan : VPlans) in printPlans() local
7638 Plan->printDOT(O); in printPlans()
7640 Plan->print(O); in printPlans()
7955 VPlanPtr &Plan) { in createEdgeMask() argument
7964 VPValue *SrcMask = createBlockInMask(Src, Plan); in createEdgeMask()
7979 VPValue *EdgeMask = Plan->getOrAddVPValue(BI->getCondition()); in createEdgeMask()
7990 VPValue *False = Plan->getOrAddVPValue( in createEdgeMask()
7999 VPValue *VPRecipeBuilder::createBlockInMask(BasicBlock *BB, VPlanPtr &Plan) { in createBlockInMask() argument
8021 return BlockMaskCache[BB] = Plan->getActiveLaneMaskPhi(); in createBlockInMask()
8029 Plan->getVectorLoopRegion()->getEntryBasicBlock(); in createBlockInMask()
8031 auto *IV = new VPWidenCanonicalIVRecipe(Plan->getCanonicalIV()); in createBlockInMask()
8037 VPValue *TC = Plan->getOrCreateTripCount(); in createBlockInMask()
8041 VPValue *BTC = Plan->getOrCreateBackedgeTakenCount(); in createBlockInMask()
8049 VPValue *EdgeMask = createEdgeMask(Predecessor, BB, Plan); in createBlockInMask()
8067 VPlanPtr &Plan) { in tryToWidenMemory() argument
8089 Mask = createBlockInMask(I->getParent(), Plan); in tryToWidenMemory()
8113 VPlan &Plan, ScalarEvolution &SE, Loop &OrigLoop, VFRange &Range) { in createWidenInductionRecipes() argument
8132 vputils::getOrCreateVPValueForSCEVExpr(Plan, IndDesc.getStep(), SE); in createWidenInductionRecipes()
8143 PHINode *Phi, ArrayRef<VPValue *> Operands, VPlan &Plan, VFRange &Range) { in tryToOptimizeInductionPHI() argument
8148 return createWidenInductionRecipes(Phi, Phi, Operands[0], *II, CM, Plan, in tryToOptimizeInductionPHI()
8165 TruncInst *I, ArrayRef<VPValue *> Operands, VFRange &Range, VPlan &Plan) { in tryToOptimizeInductionTruncate() argument
8185 VPValue *Start = Plan.getOrAddVPValue(II.getStartValue()); in tryToOptimizeInductionTruncate()
8186 return createWidenInductionRecipes(Phi, I, Start, II, CM, Plan, in tryToOptimizeInductionTruncate()
8194 VPlanPtr &Plan) { in tryToBlend() argument
8231 createEdgeMask(Phi->getIncomingBlock(In), Phi->getParent(), Plan); in tryToBlend()
8358 VPlanPtr &Plan) { in handleReplication() argument
8398 auto *Recipe = new VPReplicateRecipe(I, Plan->mapToVPValues(I->operands()), in handleReplication()
8419 Plan->addVPValue(I, Recipe); in handleReplication()
8430 VPBlockBase *Region = createReplicateRegion(Recipe, Plan); in handleReplication()
8440 VPlanPtr &Plan) { in createReplicateRegion() argument
8445 VPValue *BlockInMask = createBlockInMask(Instr->getParent(), Plan); in createReplicateRegion()
8457 Plan->addVPValue(Instr, PHIRecipe); in createReplicateRegion()
8460 Plan->addVPValue(Instr, PredRecipe); in createReplicateRegion()
8478 VFRange &Range, VPlanPtr &Plan) { in tryToCreateWidenRecipe() argument
8484 return tryToBlend(Phi, Operands, Plan); in tryToCreateWidenRecipe()
8485 if ((Recipe = tryToOptimizeInductionPHI(Phi, Operands, *Plan, Range))) in tryToCreateWidenRecipe()
8515 Range, *Plan))) in tryToCreateWidenRecipe()
8527 return toVPRecipeResult(tryToWidenMemory(Instr, Operands, Range, Plan)); in tryToCreateWidenRecipe()
8592 static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, DebugLoc DL, in addCanonicalIVRecipes() argument
8596 auto *StartV = Plan.getOrAddVPValue(StartIdx); in addCanonicalIVRecipes()
8600 VPRegionBlock *TopRegion = Plan.getVectorLoopRegion(); in addCanonicalIVRecipes()
8617 VPBasicBlock *Preheader = Plan.getEntry()->getEntryBasicBlock(); in addCanonicalIVRecipes()
8629 VPValue *TC = Plan.getOrCreateTripCount(); in addCanonicalIVRecipes()
8665 {CanonicalIVIncrement, &Plan.getVectorTripCount()}, DL); in addCanonicalIVRecipes()
8674 VPlan &Plan) { in addUsersInExitBlock() argument
8685 VPValue *V = Plan.getOrAddVPValue(IncomingValue, true); in addUsersInExitBlock()
8686 Plan.addLiveOut(&ExitPhi, V); in addUsersInExitBlock()
8754 auto Plan = std::make_unique<VPlan>(Preheader); in buildVPlanWithVPRecipes() local
8766 addCanonicalIVRecipes(*Plan, Legal->getWidestInductionType(), in buildVPlanWithVPRecipes()
8799 Operands.push_back(Plan->getOrAddVPValue( in buildVPlanWithVPRecipes()
8802 auto OpRange = Plan->mapToVPValues(Instr->operands()); in buildVPlanWithVPRecipes()
8814 Instr, Operands, Range, Plan)) { in buildVPlanWithVPRecipes()
8818 Plan->addVPValue(Instr, VPV); in buildVPlanWithVPRecipes()
8829 Plan->addVPValue(UV, Def); in buildVPlanWithVPRecipes()
8851 RecipeBuilder.handleReplication(Instr, Range, VPBB, Plan); in buildVPlanWithVPRecipes()
8871 addUsersInExitBlock(HeaderVPBB, MiddleVPBB, OrigLoop, *Plan); in buildVPlanWithVPRecipes()
8873 assert(isa<VPRegionBlock>(Plan->getVectorLoopRegion()) && in buildVPlanWithVPRecipes()
8874 !Plan->getVectorLoopRegion()->getEntryBasicBlock()->empty() && in buildVPlanWithVPRecipes()
8945 VPlanTransforms::removeRedundantCanonicalIVs(*Plan); in buildVPlanWithVPRecipes()
8946 VPlanTransforms::removeRedundantInductionCasts(*Plan); in buildVPlanWithVPRecipes()
8954 adjustRecipesForReductions(cast<VPBasicBlock>(TopRegion->getExiting()), Plan, in buildVPlanWithVPRecipes()
8960 Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) { in buildVPlanWithVPRecipes()
9010 VPValue *OriginalV = Plan->getVPValue(Member); in buildVPlanWithVPRecipes()
9011 Plan->removeVPValueFor(Member); in buildVPlanWithVPRecipes()
9012 Plan->addVPValue(Member, VPIG->getVPValue(J)); in buildVPlanWithVPRecipes()
9023 Plan->addVF(VF); in buildVPlanWithVPRecipes()
9026 Plan->addVF(VF); in buildVPlanWithVPRecipes()
9031 Plan->setName(PlanName); in buildVPlanWithVPRecipes()
9035 Plan->disableValue2VPValue(); in buildVPlanWithVPRecipes()
9037 VPlanTransforms::optimizeInductions(*Plan, *PSE.getSE()); in buildVPlanWithVPRecipes()
9038 VPlanTransforms::sinkScalarOperands(*Plan); in buildVPlanWithVPRecipes()
9039 VPlanTransforms::removeDeadRecipes(*Plan); in buildVPlanWithVPRecipes()
9040 VPlanTransforms::mergeReplicateRegions(*Plan); in buildVPlanWithVPRecipes()
9041 VPlanTransforms::removeRedundantExpandSCEVRecipes(*Plan); in buildVPlanWithVPRecipes()
9048 assert(VPlanVerifier::verifyPlanIsValid(*Plan) && "VPlan is invalid"); in buildVPlanWithVPRecipes()
9049 return Plan; in buildVPlanWithVPRecipes()
9061 auto Plan = std::make_unique<VPlan>(); in buildVPlan() local
9064 VPlanHCFGBuilder HCFGBuilder(OrigLoop, LI, *Plan); in buildVPlan()
9069 Plan->addVF(VF); in buildVPlan()
9073 OrigLoop, Plan, in buildVPlan()
9080 Plan->getVectorLoopRegion()->getExitingBasicBlock()->getTerminator(); in buildVPlan()
9083 addCanonicalIVRecipes(*Plan, Legal->getWidestInductionType(), DebugLoc(), in buildVPlan()
9085 return Plan; in buildVPlan()
9094 VPBasicBlock *LatchVPBB, VPlanPtr &Plan, VPRecipeBuilder &RecipeBuilder, in adjustRecipesForReductions() argument
9114 VPValue *ChainOp = Plan->getVPValue(Chain); in adjustRecipesForReductions()
9134 VPValue *VecOp = Plan->getVPValue(R->getOperand(VecOpId)); in adjustRecipesForReductions()
9137 ? RecipeBuilder.createBlockInMask(R->getParent(), Plan) in adjustRecipesForReductions()
9145 Instruction::FMul, {VecOp, Plan->getVPValue(R->getOperand(1))}); in adjustRecipesForReductions()
9154 Plan->removeVPValueFor(R); in adjustRecipesForReductions()
9155 Plan->addVPValue(R, RedRecipe); in adjustRecipesForReductions()
9181 Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) { in adjustRecipesForReductions()
9186 RecipeBuilder.createBlockInMask(OrigLoop->getHeader(), Plan); in adjustRecipesForReductions()