Lines Matching refs:FC0
633 bool isControlFlowEquivalent(const FusionCandidate &FC0, in isControlFlowEquivalent()
635 assert(FC0.Preheader && FC1.Preheader && "Expecting valid preheaders"); in isControlFlowEquivalent()
637 return ::isControlFlowEquivalent(*FC0.getEntryBlock(), *FC1.getEntryBlock(), in isControlFlowEquivalent()
690 bool isBeneficialFusion(const FusionCandidate &FC0, in isBeneficialFusion()
704 haveIdenticalTripCounts(const FusionCandidate &FC0, in haveIdenticalTripCounts()
707 const SCEV *TripCount0 = SE.getBackedgeTakenCount(FC0.L); in haveIdenticalTripCounts()
734 const unsigned TC0 = SE.getSmallConstantTripCount(FC0.L); in haveIdenticalTripCounts()
763 void peelFusionCandidate(FusionCandidate &FC0, const FusionCandidate &FC1, in peelFusionCandidate()
765 assert(FC0.AbleToPeel && "Should be able to peel loop"); in peelFusionCandidate()
770 FC0.Peeled = peelLoop(FC0.L, PeelCount, &LI, &SE, DT, &AC, true); in peelFusionCandidate()
771 if (FC0.Peeled) { in peelFusionCandidate()
775 auto IdenticalTripCount = haveIdenticalTripCounts(FC0, FC1); in peelFusionCandidate()
781 FC0.PP.PeelCount += PeelCount; in peelFusionCandidate()
784 PDT.recalculate(*FC0.Preheader->getParent()); in peelFusionCandidate()
786 FC0.updateAfterPeeling(); in peelFusionCandidate()
797 FC0.GuardBranch ? FC0.ExitBlock->getUniqueSuccessor() : FC1.Preheader; in peelFusionCandidate()
802 if (Pred != FC0.ExitBlock) { in peelFusionCandidate()
821 dbgs() << "Sucessfully peeled " << FC0.PP.PeelCount in peelFusionCandidate()
844 for (auto FC0 = CandidateSet.begin(); FC0 != CandidateSet.end(); ++FC0) { in fuseCandidates() local
845 assert(!LDT.isRemovedLoop(FC0->L) && in fuseCandidates()
847 auto FC1 = FC0; in fuseCandidates()
852 LLVM_DEBUG(dbgs() << "Attempting to fuse candidate \n"; FC0->dump(); in fuseCandidates()
855 FC0->verify(); in fuseCandidates()
864 haveIdenticalTripCounts(*FC0, *FC1); in fuseCandidates()
870 if (FC0->AbleToPeel && !SameTripCount && TCDifference) { in fuseCandidates()
886 reportLoopFusion<OptimizationRemarkMissed>(*FC0, *FC1, in fuseCandidates()
891 if (!isAdjacent(*FC0, *FC1)) { in fuseCandidates()
894 reportLoopFusion<OptimizationRemarkMissed>(*FC0, *FC1, NonAdjacent); in fuseCandidates()
898 if (!FC0->GuardBranch && FC1->GuardBranch) { in fuseCandidates()
902 *FC0, *FC1, OnlySecondCandidateIsGuarded); in fuseCandidates()
908 if (FC0->GuardBranch && FC1->GuardBranch && in fuseCandidates()
909 !haveIdenticalGuards(*FC0, *FC1) && !TCDifference) { in fuseCandidates()
912 reportLoopFusion<OptimizationRemarkMissed>(*FC0, *FC1, in fuseCandidates()
918 *FC0->Preheader->getTerminator(), DT, &PDT, in fuseCandidates()
922 reportLoopFusion<OptimizationRemarkMissed>(*FC0, *FC1, in fuseCandidates()
927 if (FC0->GuardBranch) { in fuseCandidates()
930 if (!isSafeToMoveBefore(*FC0->ExitBlock, in fuseCandidates()
935 reportLoopFusion<OptimizationRemarkMissed>(*FC0, *FC1, in fuseCandidates()
942 *FC0->GuardBranch->getParent()->getTerminator(), DT, &PDT, in fuseCandidates()
947 reportLoopFusion<OptimizationRemarkMissed>(*FC0, *FC1, in fuseCandidates()
955 if (!dependencesAllowFusion(*FC0, *FC1)) { in fuseCandidates()
957 reportLoopFusion<OptimizationRemarkMissed>(*FC0, *FC1, in fuseCandidates()
962 bool BeneficialToFuse = isBeneficialFusion(*FC0, *FC1); in fuseCandidates()
967 reportLoopFusion<OptimizationRemarkMissed>(*FC0, *FC1, in fuseCandidates()
975 LLVM_DEBUG(dbgs() << "\tFusion is performed: " << *FC0 << " and " in fuseCandidates()
978 FusionCandidate FC0Copy = *FC0; in fuseCandidates()
989 reportLoopFusion<OptimizationRemark>((Peel ? FC0Copy : *FC0), *FC1, in fuseCandidates()
993 performFusion((Peel ? FC0Copy : *FC0), *FC1), DT, &PDT, ORE, in fuseCandidates()
1002 CandidateSet.erase(FC0); in fuseCandidates()
1013 FC0 = FC1 = InsertPos.first; in fuseCandidates()
1118 bool dependencesAllowFusion(const FusionCandidate &FC0, in dependencesAllowFusion()
1130 return accessDiffIsPositive(*FC0.L, *FC1.L, I0, I1, AnyDep); in dependencesAllowFusion()
1155 return dependencesAllowFusion(FC0, FC1, I0, I1, AnyDep, in dependencesAllowFusion()
1157 dependencesAllowFusion(FC0, FC1, I0, I1, AnyDep, in dependencesAllowFusion()
1165 bool dependencesAllowFusion(const FusionCandidate &FC0, in dependencesAllowFusion()
1167 LLVM_DEBUG(dbgs() << "Check if " << FC0 << " can be fused with " << FC1 in dependencesAllowFusion()
1169 assert(FC0.L->getLoopDepth() == FC1.L->getLoopDepth()); in dependencesAllowFusion()
1170 assert(DT.dominates(FC0.getEntryBlock(), FC1.getEntryBlock())); in dependencesAllowFusion()
1172 for (Instruction *WriteL0 : FC0.MemWrites) { in dependencesAllowFusion()
1174 if (!dependencesAllowFusion(FC0, FC1, *WriteL0, *WriteL1, in dependencesAllowFusion()
1181 if (!dependencesAllowFusion(FC0, FC1, *WriteL0, *ReadL1, in dependencesAllowFusion()
1190 for (Instruction *WriteL0 : FC0.MemWrites) in dependencesAllowFusion()
1191 if (!dependencesAllowFusion(FC0, FC1, *WriteL0, *WriteL1, in dependencesAllowFusion()
1197 for (Instruction *ReadL0 : FC0.MemReads) in dependencesAllowFusion()
1198 if (!dependencesAllowFusion(FC0, FC1, *ReadL0, *WriteL1, in dependencesAllowFusion()
1212 if (FC0.L->contains(Def->getParent())) { in dependencesAllowFusion()
1229 bool isAdjacent(const FusionCandidate &FC0, in isAdjacent()
1232 if (FC0.GuardBranch) in isAdjacent()
1233 return FC0.getNonLoopBlock() == FC1.getEntryBlock(); in isAdjacent()
1235 return FC0.ExitBlock == FC1.getEntryBlock(); in isAdjacent()
1250 bool haveIdenticalGuards(const FusionCandidate &FC0, in haveIdenticalGuards()
1252 assert(FC0.GuardBranch && FC1.GuardBranch && in haveIdenticalGuards()
1256 dyn_cast<Instruction>(FC0.GuardBranch->getCondition())) in haveIdenticalGuards()
1265 if (FC0.GuardBranch->getSuccessor(0) == FC0.Preheader) in haveIdenticalGuards()
1287 void mergeLatch(const FusionCandidate &FC0, const FusionCandidate &FC1) { in mergeLatch()
1288 moveInstructionsToTheBeginning(*FC0.Latch, *FC1.Latch, DT, PDT, DI); in mergeLatch()
1289 if (BasicBlock *Succ = FC0.Latch->getUniqueSuccessor()) { in mergeLatch()
1324 Loop *performFusion(const FusionCandidate &FC0, const FusionCandidate &FC1) { in performFusion()
1325 assert(FC0.isValid() && FC1.isValid() && in performFusion()
1328 LLVM_DEBUG(dbgs() << "Fusion Candidate 0: \n"; FC0.dump(); in performFusion()
1333 moveInstructionsToTheEnd(*FC1.Preheader, *FC0.Preheader, DT, PDT, DI); in performFusion()
1338 if (FC0.GuardBranch) in performFusion()
1339 return fuseGuardedLoops(FC0, FC1); in performFusion()
1342 (FC0.Peeled ? FC0.ExitBlock->getUniqueSuccessor() : FC0.ExitBlock)); in performFusion()
1356 if (FC0.ExitingBlock != FC0.Latch) in performFusion()
1357 for (PHINode &PHI : FC0.Header->phis()) in performFusion()
1361 FC1.Preheader->replaceSuccessorsPhiUsesWith(FC0.Preheader); in performFusion()
1362 FC0.Latch->replaceSuccessorsPhiUsesWith(FC1.Latch); in performFusion()
1384 if (!FC0.Peeled) { in performFusion()
1385 FC0.ExitingBlock->getTerminator()->replaceUsesOfWith(FC1.Preheader, in performFusion()
1388 DominatorTree::Delete, FC0.ExitingBlock, FC1.Preheader)); in performFusion()
1390 DominatorTree::Insert, FC0.ExitingBlock, FC1.Header)); in performFusion()
1393 DominatorTree::Delete, FC0.ExitBlock, FC1.Preheader)); in performFusion()
1396 FC0.ExitingBlock->getTerminator()->replaceUsesOfWith(FC0.ExitBlock, in performFusion()
1399 DominatorTree::Delete, FC0.ExitingBlock, FC0.ExitBlock)); in performFusion()
1400 FC0.ExitBlock->getTerminator()->eraseFromParent(); in performFusion()
1402 DominatorTree::Insert, FC0.ExitingBlock, FC1.Header)); in performFusion()
1403 new UnreachableInst(FC0.ExitBlock->getContext(), FC0.ExitBlock); in performFusion()
1418 PHI->moveBefore(&*FC0.Header->getFirstInsertionPt()); in performFusion()
1437 L1HeaderPHI->addIncoming(LCV, FC0.Latch); in performFusion()
1439 FC0.ExitingBlock); in performFusion()
1445 FC0.Latch->getTerminator()->replaceUsesOfWith(FC0.Header, FC1.Header); in performFusion()
1446 FC1.Latch->getTerminator()->replaceUsesOfWith(FC1.Header, FC0.Header); in performFusion()
1450 simplifyLatchBranch(FC0); in performFusion()
1454 if (FC0.Latch != FC0.ExitingBlock) in performFusion()
1456 DominatorTree::Insert, FC0.Latch, FC1.Header)); in performFusion()
1459 FC0.Latch, FC0.Header)); in performFusion()
1461 FC1.Latch, FC0.Header)); in performFusion()
1470 if (FC0.Peeled) { in performFusion()
1471 LI.removeBlock(FC0.ExitBlock); in performFusion()
1472 DTU.deleteBB(FC0.ExitBlock); in performFusion()
1482 SE.forgetLoop(FC0.L); in performFusion()
1486 mergeLatch(FC0, FC1); in performFusion()
1491 FC0.L->addBlockEntry(BB); in performFusion()
1495 LI.changeLoopFor(BB, FC0.L); in performFusion()
1501 FC0.L->addChildLoop(ChildLoop); in performFusion()
1508 assert(!verifyFunction(*FC0.Header->getParent(), &errs())); in performFusion()
1517 return FC0.L; in performFusion()
1533 void reportLoopFusion(const FusionCandidate &FC0, const FusionCandidate &FC1, in reportLoopFusion()
1535 assert(FC0.Preheader && FC1.Preheader && in reportLoopFusion()
1540 ORE.emit(RemarkKind(DEBUG_TYPE, Stat.getName(), FC0.L->getStartLoc(), in reportLoopFusion()
1541 FC0.Preheader) in reportLoopFusion()
1542 << "[" << FC0.Preheader->getParent()->getName() in reportLoopFusion()
1543 << "]: " << NV("Cand1", StringRef(FC0.Preheader->getName())) in reportLoopFusion()
1564 Loop *fuseGuardedLoops(const FusionCandidate &FC0, in fuseGuardedLoops()
1566 assert(FC0.GuardBranch && FC1.GuardBranch && "Expecting guarded loops"); in fuseGuardedLoops()
1568 BasicBlock *FC0GuardBlock = FC0.GuardBranch->getParent(); in fuseGuardedLoops()
1570 BasicBlock *FC0NonLoopBlock = FC0.getNonLoopBlock(); in fuseGuardedLoops()
1572 BasicBlock *FC0ExitBlockSuccessor = FC0.ExitBlock->getUniqueSuccessor(); in fuseGuardedLoops()
1579 (FC0.Peeled ? *FC0ExitBlockSuccessor : *FC0.ExitBlock), *FC1.ExitBlock, in fuseGuardedLoops()
1599 FC0.GuardBranch->replaceUsesOfWith(FC0NonLoopBlock, FC1NonLoopBlock); in fuseGuardedLoops()
1601 BasicBlock *BBToUpdate = FC0.Peeled ? FC0ExitBlockSuccessor : FC0.ExitBlock; in fuseGuardedLoops()
1617 if (FC0.Peeled) { in fuseGuardedLoops()
1644 if (FC0.ExitingBlock != FC0.Latch) in fuseGuardedLoops()
1645 for (PHINode &PHI : FC0.Header->phis()) in fuseGuardedLoops()
1651 FC1.Preheader->replaceSuccessorsPhiUsesWith(FC0.Preheader); in fuseGuardedLoops()
1652 FC0.Latch->replaceSuccessorsPhiUsesWith(FC1.Latch); in fuseGuardedLoops()
1667 FC0.ExitingBlock->getTerminator()->replaceUsesOfWith(FC0.ExitBlock, in fuseGuardedLoops()
1671 DominatorTree::Delete, FC0.ExitingBlock, FC0.ExitBlock)); in fuseGuardedLoops()
1673 DominatorTree::Insert, FC0.ExitingBlock, FC1.Header)); in fuseGuardedLoops()
1682 assert(pred_empty(FC0.ExitBlock) && "Expecting exit block to be empty"); in fuseGuardedLoops()
1683 FC0.ExitBlock->getTerminator()->eraseFromParent(); in fuseGuardedLoops()
1684 new UnreachableInst(FC0.ExitBlock->getContext(), FC0.ExitBlock); in fuseGuardedLoops()
1699 PHI->moveBefore(&*FC0.Header->getFirstInsertionPt()); in fuseGuardedLoops()
1718 L1HeaderPHI->addIncoming(LCV, FC0.Latch); in fuseGuardedLoops()
1720 FC0.ExitingBlock); in fuseGuardedLoops()
1728 FC0.Latch->getTerminator()->replaceUsesOfWith(FC0.Header, FC1.Header); in fuseGuardedLoops()
1729 FC1.Latch->getTerminator()->replaceUsesOfWith(FC1.Header, FC0.Header); in fuseGuardedLoops()
1733 simplifyLatchBranch(FC0); in fuseGuardedLoops()
1737 if (FC0.Latch != FC0.ExitingBlock) in fuseGuardedLoops()
1739 DominatorTree::Insert, FC0.Latch, FC1.Header)); in fuseGuardedLoops()
1742 FC0.Latch, FC0.Header)); in fuseGuardedLoops()
1744 FC1.Latch, FC0.Header)); in fuseGuardedLoops()
1759 LI.removeBlock(FC0.ExitBlock); in fuseGuardedLoops()
1760 if (FC0.Peeled) { in fuseGuardedLoops()
1766 DTU.deleteBB(FC0.ExitBlock); in fuseGuardedLoops()
1774 SE.forgetLoop(FC0.L); in fuseGuardedLoops()
1778 mergeLatch(FC0, FC1); in fuseGuardedLoops()
1783 FC0.L->addBlockEntry(BB); in fuseGuardedLoops()
1787 LI.changeLoopFor(BB, FC0.L); in fuseGuardedLoops()
1793 FC0.L->addChildLoop(ChildLoop); in fuseGuardedLoops()
1800 assert(!verifyFunction(*FC0.Header->getParent(), &errs())); in fuseGuardedLoops()
1809 return FC0.L; in fuseGuardedLoops()