Lines Matching refs:Region
549 collectRegionsConstants(OutlinableRegion &Region, in collectRegionsConstants() argument
554 IRSimilarityCandidate &C = *Region.Candidate; in collectRegionsConstants()
601 for (OutlinableRegion *Region : Regions) in findSameConstants()
602 collectRegionsConstants(*Region, GVNToConstant, NotSame); in findSameConstants()
863 OutlinableRegion &Region, std::vector<unsigned> &InputGVNs, in getCodeExtractorArguments() argument
866 IRSimilarityCandidate &C = *Region.Candidate; in getCodeExtractorArguments()
882 CodeExtractor *CE = Region.CE; in getCodeExtractorArguments()
884 assert(Region.StartBB && "Region must have a start BasicBlock!"); in getCodeExtractorArguments()
885 Function *OrigF = Region.StartBB->getParent(); in getCodeExtractorArguments()
892 Region.IgnoreRegion = true; in getCodeExtractorArguments()
906 Region.IgnoreRegion = true; in getCodeExtractorArguments()
933 findExtractedInputToOverallInputMapping(OutlinableRegion &Region, in findExtractedInputToOverallInputMapping() argument
937 IRSimilarityCandidate &C = *Region.Candidate; in findExtractedInputToOverallInputMapping()
938 OutlinableGroup &Group = *Region.Parent; in findExtractedInputToOverallInputMapping()
983 Region.AggArgToConstant.insert(std::make_pair(AggArgIt->second, CST)); in findExtractedInputToOverallInputMapping()
987 Region.AggArgToConstant.insert(std::make_pair(TypeIndex, CST)); in findExtractedInputToOverallInputMapping()
999 Region.ChangedArgOrder = true; in findExtractedInputToOverallInputMapping()
1000 Region.ExtractedArgToAgg.insert( in findExtractedInputToOverallInputMapping()
1002 Region.AggArgToExtracted.insert( in findExtractedInputToOverallInputMapping()
1007 Region.ExtractedArgToAgg.insert(std::make_pair(OriginalIndex, TypeIndex)); in findExtractedInputToOverallInputMapping()
1008 Region.AggArgToExtracted.insert(std::make_pair(TypeIndex, OriginalIndex)); in findExtractedInputToOverallInputMapping()
1023 Region.NumExtractedInputs = OriginalIndex; in findExtractedInputToOverallInputMapping()
1173 static std::optional<unsigned> getGVNForPHINode(OutlinableRegion &Region, in getGVNForPHINode() argument
1177 OutlinableGroup &Group = *Region.Parent; in getGVNForPHINode()
1178 IRSimilarityCandidate &Cand = *Region.Candidate; in getGVNForPHINode()
1193 Region.IgnoreRegion = true; in getGVNForPHINode()
1275 findExtractedOutputToOverallOutputMapping(Module &M, OutlinableRegion &Region, in findExtractedOutputToOverallOutputMapping() argument
1277 OutlinableGroup &Group = *Region.Parent; in findExtractedOutputToOverallOutputMapping()
1278 IRSimilarityCandidate &C = *Region.Candidate; in findExtractedOutputToOverallOutputMapping()
1302 unsigned OriginalIndex = Region.NumExtractedInputs; in findExtractedOutputToOverallOutputMapping()
1338 Region.ExtractedArgToAgg.insert(std::make_pair(OriginalIndex, Jdx)); in findExtractedOutputToOverallOutputMapping()
1339 Region.AggArgToExtracted.insert(std::make_pair(Jdx, OriginalIndex)); in findExtractedOutputToOverallOutputMapping()
1354 Region.ExtractedArgToAgg.insert( in findExtractedOutputToOverallOutputMapping()
1356 Region.AggArgToExtracted.insert( in findExtractedOutputToOverallOutputMapping()
1375 GVN = getGVNForPHINode(Region, PN, BlocksInRegion, AggArgIdx); in findExtractedOutputToOverallOutputMapping()
1389 Region.GVNStores.push_back(*GVN); in findExtractedOutputToOverallOutputMapping()
1397 stable_sort(Region.GVNStores); in findExtractedOutputToOverallOutputMapping()
1400 void IROutliner::findAddInputsOutputs(Module &M, OutlinableRegion &Region, in findAddInputsOutputs() argument
1405 getCodeExtractorArguments(Region, Inputs, NotSame, OutputMappings, ArgInputs, in findAddInputsOutputs()
1408 if (Region.IgnoreRegion) in findAddInputsOutputs()
1413 findExtractedInputToOverallInputMapping(Region, Inputs, ArgInputs); in findAddInputsOutputs()
1417 findExtractedOutputToOverallOutputMapping(M, Region, Outputs); in findAddInputsOutputs()
1429 CallInst *replaceCalledFunction(Module &M, OutlinableRegion &Region) { in replaceCalledFunction() argument
1433 OutlinableGroup &Group = *Region.Parent; in replaceCalledFunction()
1434 CallInst *Call = Region.Call; in replaceCalledFunction()
1443 if (!Region.ChangedArgOrder && AggFunc->arg_size() == Call->arg_size()) { in replaceCalledFunction()
1462 << Region.OutputBlockNum << "\n"); in replaceCalledFunction()
1464 Region.OutputBlockNum)); in replaceCalledFunction()
1468 ArgPair = Region.AggArgToExtracted.find(AggArgIdx); in replaceCalledFunction()
1469 if (ArgPair != Region.AggArgToExtracted.end()) { in replaceCalledFunction()
1481 if (Region.AggArgToConstant.contains(AggArgIdx)) { in replaceCalledFunction()
1482 Constant *CST = Region.AggArgToConstant.find(AggArgIdx)->second; in replaceCalledFunction()
1507 CallInst *OldCall = Region.Call; in replaceCalledFunction()
1508 if (Region.NewFront->Inst == OldCall) in replaceCalledFunction()
1509 Region.NewFront->Inst = Call; in replaceCalledFunction()
1510 if (Region.NewBack->Inst == OldCall) in replaceCalledFunction()
1511 Region.NewBack->Inst = Call; in replaceCalledFunction()
1514 Call->setDebugLoc(Region.Call->getDebugLoc()); in replaceCalledFunction()
1521 Region.Call = Call; in replaceCalledFunction()
1594 const OutlinableRegion &Region) { in getPassedArgumentInAlreadyOutlinedFunction() argument
1598 return Region.Call->getArgOperand(A->getArgNo()); in getPassedArgumentInAlreadyOutlinedFunction()
1610 const OutlinableRegion &Region) { in getPassedArgumentAndAdjustArgumentLocation() argument
1615 if (Region.AggArgToConstant.count(ArgNum)) in getPassedArgumentAndAdjustArgumentLocation()
1616 return Region.AggArgToConstant.find(ArgNum)->second; in getPassedArgumentAndAdjustArgumentLocation()
1620 ArgNum = Region.AggArgToExtracted.find(ArgNum)->second; in getPassedArgumentAndAdjustArgumentLocation()
1621 return Region.Call->getArgOperand(ArgNum); in getPassedArgumentAndAdjustArgumentLocation()
1635 PHINode *PN, OutlinableRegion &Region, in findCanonNumsForPHI() argument
1647 IVal = getPassedArgumentInAlreadyOutlinedFunction(A, Region); in findCanonNumsForPHI()
1649 IVal = getPassedArgumentAndAdjustArgumentLocation(A, Region); in findCanonNumsForPHI()
1656 std::optional<unsigned> GVN = Region.Candidate->getGVN(IVal); in findCanonNumsForPHI()
1658 std::optional<unsigned> CanonNum = Region.Candidate->getCanonicalNum(*GVN); in findCanonNumsForPHI()
1678 findOrCreatePHIInBlock(PHINode &PN, OutlinableRegion &Region, in findOrCreatePHIInBlock() argument
1682 OutlinableGroup &Group = *Region.Parent; in findOrCreatePHIInBlock()
1693 findCanonNumsForPHI(&PN, Region, OutputMappings, PNCanonNums, in findOrCreatePHIInBlock()
1738 Region.findCorrespondingBlockIn(*FirstRegion, ToAdd.second); in findOrCreatePHIInBlock()
1766 Region.findCorrespondingBlockIn(*FirstRegion, IncomingBlock); in findOrCreatePHIInBlock()
1779 Value *Val = Region.findCorrespondingValueIn(*FirstRegion, IncomingVal); in findOrCreatePHIInBlock()
1800 replaceArgumentUses(OutlinableRegion &Region, in replaceArgumentUses() argument
1804 OutlinableGroup &Group = *Region.Parent; in replaceArgumentUses()
1805 assert(Region.ExtractedFunction && "Region has no extracted function?"); in replaceArgumentUses()
1807 Function *DominatingFunction = Region.ExtractedFunction; in replaceArgumentUses()
1813 for (unsigned ArgIdx = 0; ArgIdx < Region.ExtractedFunction->arg_size(); in replaceArgumentUses()
1815 assert(Region.ExtractedArgToAgg.contains(ArgIdx) && in replaceArgumentUses()
1817 unsigned AggArgIdx = Region.ExtractedArgToAgg.find(ArgIdx)->second; in replaceArgumentUses()
1819 Argument *Arg = Region.ExtractedFunction->getArg(ArgIdx); in replaceArgumentUses()
1822 if (ArgIdx < Region.NumExtractedInputs) { in replaceArgumentUses()
1824 << *Region.ExtractedFunction << " with " << *AggArg in replaceArgumentUses()
1827 Value *V = Region.Call->getArgOperand(ArgIdx); in replaceArgumentUses()
1828 Region.RemappedArguments.insert(std::make_pair(V, AggArg)); in replaceArgumentUses()
1877 Region.Candidate->getGVN(ValueOperand).has_value()) { in replaceArgumentUses()
1881 Region.findCorrespondingValueIn(*Group.Regions[0], ValueOperand); in replaceArgumentUses()
1889 if (Region.Candidate->getGVN(PN)) in replaceArgumentUses()
1894 Region.PHIBlocks.insert(std::make_pair(RetVal, PN->getParent())); in replaceArgumentUses()
1912 PHINode *NewPN = findOrCreatePHIInBlock(*PN, Region, OverallPhiBlock, in replaceArgumentUses()
1924 << *Region.ExtractedFunction << " with " << *AggArg in replaceArgumentUses()
1934 void replaceConstants(OutlinableRegion &Region) { in replaceConstants() argument
1935 OutlinableGroup &Group = *Region.Parent; in replaceConstants()
1937 for (std::pair<unsigned, Constant *> &Const : Region.AggArgToConstant) { in replaceConstants()
2023 OutlinableRegion &Region) { in analyzeAndPruneOutputBlocks() argument
2052 Region.OutputBlockNum = -1; in analyzeAndPruneOutputBlocks()
2070 OutlinableGroup &OG, OutlinableRegion &Region, in alignOutputBlockWithAggFunc() argument
2078 if (analyzeAndPruneOutputBlocks(OutputBBs, Region)) in alignOutputBlockWithAggFunc()
2089 << Region.ExtractedFunction << " to " << *MatchingBB); in alignOutputBlockWithAggFunc()
2091 Region.OutputBlockNum = *MatchingBB; in alignOutputBlockWithAggFunc()
2097 Region.OutputBlockNum = OutputStoreBBs.size(); in alignOutputBlockWithAggFunc()
2108 << Region.ExtractedFunction << " to " in alignOutputBlockWithAggFunc()
2361 const OutlinableRegion &Region) { in isCompatibleWithAlreadyOutlinedCode() argument
2362 IRSimilarityCandidate *IRSC = Region.Candidate; in isCompatibleWithAlreadyOutlinedCode()
2374 if (!Region.Candidate->backInstruction()->isTerminator()) { in isCompatibleWithAlreadyOutlinedCode()
2376 Region.Candidate->backInstruction()->getNextNonDebugInstruction(); in isCompatibleWithAlreadyOutlinedCode()
2378 if (Region.Candidate->end()->Inst != NewEndInst) { in isCompatibleWithAlreadyOutlinedCode()
2379 IRInstructionDataList *IDL = Region.Candidate->front()->IDL; in isCompatibleWithAlreadyOutlinedCode()
2386 IDL->insert(Region.Candidate->end(), *NewEndIRID); in isCompatibleWithAlreadyOutlinedCode()
2484 for (OutlinableRegion *Region : CurrentGroup.Regions) { in findBenefitFromAllRegions()
2485 TargetTransformInfo &TTI = getTTI(*Region->StartBB->getParent()); in findBenefitFromAllRegions()
2488 RegionBenefit += Region->getBenefit(TTI); in findBenefitFromAllRegions()
2504 static Value *findOutputValueInRegion(OutlinableRegion &Region, in findOutputValueInRegion() argument
2506 OutlinableGroup &CurrentGroup = *Region.Parent; in findOutputValueInRegion()
2518 Region.Candidate->fromCanonicalNum(OutputCanon); in findOutputValueInRegion()
2520 std::optional<Value *> OV = Region.Candidate->fromGVN(*OGVN); in findOutputValueInRegion()
2528 for (OutlinableRegion *Region : CurrentGroup.Regions) { in findCostOutputReloads()
2529 TargetTransformInfo &TTI = getTTI(*Region->StartBB->getParent()); in findCostOutputReloads()
2532 for (unsigned OutputCanon : Region->GVNStores) { in findCostOutputReloads()
2533 Value *V = findOutputValueInRegion(*Region, OutputCanon); in findCostOutputReloads()
2676 void IROutliner::updateOutputMapping(OutlinableRegion &Region, in updateOutputMapping() argument
2683 for (unsigned ArgIdx = Region.NumExtractedInputs; in updateOutputMapping()
2684 ArgIdx < Region.Call->arg_size(); ArgIdx++) { in updateOutputMapping()
2685 if (Operand == Region.Call->getArgOperand(ArgIdx)) { in updateOutputMapping()
2686 OutputIdx = ArgIdx - Region.NumExtractedInputs; in updateOutputMapping()
2708 bool IROutliner::extractSection(OutlinableRegion &Region) { in extractSection() argument
2710 assert(Region.StartBB && "StartBB for the OutlinableRegion is nullptr!"); in extractSection()
2711 BasicBlock *InitialStart = Region.StartBB; in extractSection()
2712 Function *OrigF = Region.StartBB->getParent(); in extractSection()
2714 Region.ExtractedFunction = in extractSection()
2715 Region.CE->extractCodeRegion(CEAC, ArgInputs, Outputs); in extractSection()
2719 if (!Region.ExtractedFunction) { in extractSection()
2720 LLVM_DEBUG(dbgs() << "CodeExtractor failed to outline " << Region.StartBB in extractSection()
2722 Region.reattachCandidate(); in extractSection()
2730 User *InstAsUser = Region.ExtractedFunction->user_back(); in extractSection()
2732 Region.PrevBB = RewrittenBB->getSinglePredecessor(); in extractSection()
2733 assert(Region.PrevBB && "PrevBB is nullptr?"); in extractSection()
2734 if (Region.PrevBB == InitialStart) { in extractSection()
2739 Region.PrevBB = NewPrev; in extractSection()
2743 Region.StartBB = RewrittenBB; in extractSection()
2744 Region.EndBB = RewrittenBB; in extractSection()
2751 IRInstructionDataList *IDL = Region.Candidate->front()->IDL; in extractSection()
2754 Region.NewFront = new (InstDataAllocator.Allocate()) IRInstructionData( in extractSection()
2756 Region.NewBack = new (InstDataAllocator.Allocate()) IRInstructionData( in extractSection()
2761 IDL->insert(Region.Candidate->begin(), *Region.NewFront); in extractSection()
2764 IDL->insert(Region.Candidate->end(), *Region.NewBack); in extractSection()
2766 IDL->erase(Region.Candidate->begin(), std::prev(Region.Candidate->end())); in extractSection()
2775 if (Region.ExtractedFunction == CI->getCalledFunction()) in extractSection()
2776 Region.Call = CI; in extractSection()
2778 updateOutputMapping(Region, Outputs.getArrayRef(), LI); in extractSection()
2779 Region.reattachCandidate(); in extractSection()
2889 [&R](OutlinableRegion *Region) { in doOutline() argument
2892 Region->Candidate->frontInstruction()->getDebugLoc()); in doOutline()
2916 for (OutlinableRegion *Region : CurrentGroup.Regions) { in doOutline()
2919 if (!isCompatibleWithAlreadyOutlinedCode(*Region)) in doOutline()
2921 OutlinedRegions.push_back(Region); in doOutline()
2938 for (OutlinableRegion *Region : CurrentGroup.Regions) { in doOutline()
2939 Region->splitCandidate(); in doOutline()
2940 if (!Region->CandidateSplit) in doOutline()
2942 OutlinedRegions.push_back(Region); in doOutline()
2995 [&R](OutlinableRegion *Region) { in doOutline() argument
2997 Region->Candidate->frontInstruction()->getDebugLoc()); in doOutline()