Lines Matching refs:LU
1051 const LSRUse &LU, const Formula &F);
1055 const LSRUse &LU, const Formula &F,
1105 const LSRUse &LU,
1369 const LSRUse &LU, in RateFormula() argument
1403 NumBaseParts - (1 + (F.Scale && isAMCompletelyFolded(*TTI, LU, F))); in RateFormula()
1407 C.ScaleCost += *getScalingFactorCost(*TTI, LU, F, *L).getValue(); in RateFormula()
1410 for (const LSRFixup &Fixup : LU.Fixups) { in RateFormula()
1421 if (LU.Kind == LSRUse::Address && Offset != 0 && in RateFormula()
1422 !isAMCompletelyFolded(*TTI, LSRUse::Address, LU.AccessTy, F.BaseGV, in RateFormula()
1457 if (LU.Kind == LSRUse::ICmpZero && !F.hasZeroEnd() && in RateFormula()
1464 if (LU.Kind != LSRUse::ICmpZero) in RateFormula()
1788 const LSRUse &LU, const Formula &F) { in isAMCompletelyFolded() argument
1790 if (LU.Kind == LSRUse::Address && TTI.LSRWithInstrQueries()) { in isAMCompletelyFolded()
1791 for (const LSRFixup &Fixup : LU.Fixups) in isAMCompletelyFolded()
1792 if (!isAMCompletelyFolded(TTI, LSRUse::Address, LU.AccessTy, F.BaseGV, in isAMCompletelyFolded()
1799 return isAMCompletelyFolded(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, in isAMCompletelyFolded()
1800 LU.AccessTy, F.BaseGV, F.BaseOffset, F.HasBaseReg, in isAMCompletelyFolded()
1805 const LSRUse &LU, const Formula &F, in getScalingFactorCost() argument
1812 if (!isAMCompletelyFolded(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, in getScalingFactorCost()
1813 LU.AccessTy, F, L)) in getScalingFactorCost()
1816 switch (LU.Kind) { in getScalingFactorCost()
1820 LU.AccessTy.MemTy, F.BaseGV, F.BaseOffset + LU.MinOffset, F.HasBaseReg, in getScalingFactorCost()
1821 F.Scale, LU.AccessTy.AddrSpace); in getScalingFactorCost()
1823 LU.AccessTy.MemTy, F.BaseGV, F.BaseOffset + LU.MaxOffset, F.HasBaseReg, in getScalingFactorCost()
1824 F.Scale, LU.AccessTy.AddrSpace); in getScalingFactorCost()
2029 bool reconcileNewOffset(LSRUse &LU, int64_t NewOffset, bool HasBaseReg,
2035 void DeleteUse(LSRUse &LU, size_t LUIdx);
2039 void InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
2040 void InsertSupplementalFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
2042 bool InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F);
2046 void GenerateReassociations(LSRUse &LU, unsigned LUIdx, Formula Base,
2049 void GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
2052 void GenerateCombinations(LSRUse &LU, unsigned LUIdx, Formula Base);
2053 void GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
2056 void GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
2057 void GenerateConstantOffsetsImpl(LSRUse &LU, unsigned LUIdx,
2061 void GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
2062 void GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, Formula Base);
2063 void GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base);
2064 void GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base);
2093 const LSRUse &LU) const;
2095 Value *Expand(const LSRUse &LU, const LSRFixup &LF, const Formula &F,
2098 void RewriteForPHI(PHINode *PN, const LSRUse &LU, const LSRFixup &LF,
2101 void Rewrite(const LSRUse &LU, const LSRFixup &LF, const Formula &F,
2566 bool LSRInstance::reconcileNewOffset(LSRUse &LU, int64_t NewOffset, in reconcileNewOffset() argument
2569 int64_t NewMinOffset = LU.MinOffset; in reconcileNewOffset()
2570 int64_t NewMaxOffset = LU.MaxOffset; in reconcileNewOffset()
2576 if (LU.Kind != Kind) in reconcileNewOffset()
2583 if (AccessTy.MemTy != LU.AccessTy.MemTy) { in reconcileNewOffset()
2590 if (NewOffset < LU.MinOffset) { in reconcileNewOffset()
2592 LU.MaxOffset - NewOffset, HasBaseReg)) in reconcileNewOffset()
2595 } else if (NewOffset > LU.MaxOffset) { in reconcileNewOffset()
2597 NewOffset - LU.MinOffset, HasBaseReg)) in reconcileNewOffset()
2603 LU.MinOffset = NewMinOffset; in reconcileNewOffset()
2604 LU.MaxOffset = NewMaxOffset; in reconcileNewOffset()
2605 LU.AccessTy = NewAccessTy; in reconcileNewOffset()
2630 LSRUse &LU = Uses[LUIdx]; in getUse() local
2631 if (reconcileNewOffset(LU, Offset, /*HasBaseReg=*/true, Kind, AccessTy)) in getUse()
2640 LSRUse &LU = Uses[LUIdx]; in getUse() local
2642 LU.MinOffset = Offset; in getUse()
2643 LU.MaxOffset = Offset; in getUse()
2648 void LSRInstance::DeleteUse(LSRUse &LU, size_t LUIdx) { in DeleteUse() argument
2649 if (&LU != &Uses.back()) in DeleteUse()
2650 std::swap(LU, Uses.back()); in DeleteUse()
2663 for (LSRUse &LU : Uses) { in FindUseWithSimilarFormula()
2669 if (&LU != &OrigLU && in FindUseWithSimilarFormula()
2670 LU.Kind != LSRUse::ICmpZero && in FindUseWithSimilarFormula()
2671 LU.Kind == OrigLU.Kind && OrigLU.AccessTy == LU.AccessTy && in FindUseWithSimilarFormula()
2672 LU.WidestFixupType == OrigLU.WidestFixupType && in FindUseWithSimilarFormula()
2673 LU.HasFormulaWithSameRegs(OrigF)) { in FindUseWithSimilarFormula()
2675 for (const Formula &F : LU.Formulae) { in FindUseWithSimilarFormula()
2684 return &LU; in FindUseWithSimilarFormula()
3383 LSRUse &LU = Uses[LUIdx]; in CollectFixupsAndInitialFormulae() local
3386 LSRFixup &LF = LU.getNewFixup(); in CollectFixupsAndInitialFormulae()
3391 LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L); in CollectFixupsAndInitialFormulae()
3397 BaselineCost.RateFormula(F, Regs, VisitedRegs, LU); in CollectFixupsAndInitialFormulae()
3401 if (!LU.WidestFixupType || in CollectFixupsAndInitialFormulae()
3402 SE.getTypeSizeInBits(LU.WidestFixupType) < in CollectFixupsAndInitialFormulae()
3404 LU.WidestFixupType = LF.OperandValToReplace->getType(); in CollectFixupsAndInitialFormulae()
3407 if (LU.Formulae.empty()) { in CollectFixupsAndInitialFormulae()
3408 InsertInitialFormula(S, LU, LUIdx); in CollectFixupsAndInitialFormulae()
3409 CountRegisters(LU.Formulae.back(), LUIdx); in CollectFixupsAndInitialFormulae()
3418 void LSRInstance::InsertInitialFormula(const SCEV *S, LSRUse &LU, in InsertInitialFormula() argument
3422 LU.RigidFormula = true; in InsertInitialFormula()
3426 bool Inserted = InsertFormula(LU, LUIdx, F); in InsertInitialFormula()
3434 LSRUse &LU, size_t LUIdx) { in InsertSupplementalFormula() argument
3438 bool Inserted = InsertFormula(LU, LUIdx, F); in InsertSupplementalFormula()
3452 bool LSRInstance::InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F) { in InsertFormula() argument
3454 assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F) && in InsertFormula()
3457 if (!LU.InsertFormula(F, *L)) in InsertFormula()
3576 LSRUse &LU = Uses[LUIdx]; in CollectLoopInvariantFixupsAndFormulae() local
3577 LSRFixup &LF = LU.getNewFixup(); in CollectLoopInvariantFixupsAndFormulae()
3581 LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L); in CollectLoopInvariantFixupsAndFormulae()
3582 if (!LU.WidestFixupType || in CollectLoopInvariantFixupsAndFormulae()
3583 SE.getTypeSizeInBits(LU.WidestFixupType) < in CollectLoopInvariantFixupsAndFormulae()
3585 LU.WidestFixupType = LF.OperandValToReplace->getType(); in CollectLoopInvariantFixupsAndFormulae()
3586 InsertSupplementalFormula(US, LU, LUIdx); in CollectLoopInvariantFixupsAndFormulae()
3587 CountRegisters(LU.Formulae.back(), Uses.size() - 1); in CollectLoopInvariantFixupsAndFormulae()
3658 LSRUse &LU, const SCEV *S, const Loop *L, in mayUsePostIncMode() argument
3660 if (LU.Kind != LSRUse::Address || in mayUsePostIncMode()
3661 !LU.AccessTy.getType()->isIntOrIntVectorTy()) in mayUsePostIncMode()
3680 void LSRInstance::GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx, in GenerateReassociationsImpl() argument
3689 if (AMK == TTI::AMK_PostIndexed && mayUsePostIncMode(TTI, LU, BaseReg, L, SE)) in GenerateReassociationsImpl()
3709 if (isAlwaysFoldable(TTI, SE, LU.MinOffset, LU.MaxOffset, LU.Kind, in GenerateReassociationsImpl()
3710 LU.AccessTy, *J, Base.getNumRegs() > 1)) in GenerateReassociationsImpl()
3722 isAlwaysFoldable(TTI, SE, LU.MinOffset, LU.MaxOffset, LU.Kind, in GenerateReassociationsImpl()
3723 LU.AccessTy, InnerAddOps[0], Base.getNumRegs() > 1)) in GenerateReassociationsImpl()
3760 if (InsertFormula(LU, LUIdx, F)) in GenerateReassociationsImpl()
3767 GenerateReassociations(LU, LUIdx, LU.Formulae.back(), in GenerateReassociationsImpl()
3773 void LSRInstance::GenerateReassociations(LSRUse &LU, unsigned LUIdx, in GenerateReassociations() argument
3781 GenerateReassociationsImpl(LU, LUIdx, Base, Depth, i); in GenerateReassociations()
3784 GenerateReassociationsImpl(LU, LUIdx, Base, Depth, in GenerateReassociations()
3790 void LSRInstance::GenerateCombinations(LSRUse &LU, unsigned LUIdx, in GenerateCombinations() argument
3832 (void)InsertFormula(LU, LUIdx, F); in GenerateCombinations()
3853 void LSRInstance::GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx, in GenerateSymbolicOffsetsImpl() argument
3862 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F)) in GenerateSymbolicOffsetsImpl()
3868 (void)InsertFormula(LU, LUIdx, F); in GenerateSymbolicOffsetsImpl()
3872 void LSRInstance::GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx, in GenerateSymbolicOffsets() argument
3878 GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, i); in GenerateSymbolicOffsets()
3880 GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, /* Idx */ -1, in GenerateSymbolicOffsets()
3886 LSRUse &LU, unsigned LUIdx, const Formula &Base, in GenerateConstantOffsetsImpl() argument
3893 if (isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F)) { in GenerateConstantOffsetsImpl()
3909 (void)InsertFormula(LU, LUIdx, F); in GenerateConstantOffsetsImpl()
3923 if (AMK == TTI::AMK_PreIndexed && LU.Kind == LSRUse::Address) { in GenerateConstantOffsetsImpl()
3946 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F)) in GenerateConstantOffsetsImpl()
3956 (void)InsertFormula(LU, LUIdx, F); in GenerateConstantOffsetsImpl()
3960 void LSRInstance::GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx, in GenerateConstantOffsets() argument
3965 Worklist.push_back(LU.MinOffset); in GenerateConstantOffsets()
3966 if (LU.MaxOffset != LU.MinOffset) in GenerateConstantOffsets()
3967 Worklist.push_back(LU.MaxOffset); in GenerateConstantOffsets()
3970 GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, i); in GenerateConstantOffsets()
3972 GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, /* Idx */ -1, in GenerateConstantOffsets()
3978 void LSRInstance::GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, in GenerateICmpZeroScales() argument
3980 if (LU.Kind != LSRUse::ICmpZero) return; in GenerateICmpZeroScales()
3988 if (LU.MinOffset != LU.MaxOffset) return; in GenerateICmpZeroScales()
4016 int64_t Offset = LU.MinOffset; in GenerateICmpZeroScales()
4020 if (Offset / Factor != LU.MinOffset) in GenerateICmpZeroScales()
4031 if (!isLegalUse(TTI, Offset, Offset, LU.Kind, LU.AccessTy, F)) in GenerateICmpZeroScales()
4035 F.BaseOffset = (uint64_t)F.BaseOffset + Offset - LU.MinOffset; in GenerateICmpZeroScales()
4068 (void)InsertFormula(LU, LUIdx, F); in GenerateICmpZeroScales()
4075 void LSRInstance::GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base) { in GenerateScales() argument
4092 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, in GenerateScales()
4096 if (LU.Kind == LSRUse::Basic && in GenerateScales()
4097 isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LSRUse::Special, in GenerateScales()
4098 LU.AccessTy, Base) && in GenerateScales()
4099 LU.AllFixupsOutsideLoop) in GenerateScales()
4100 LU.Kind = LSRUse::Special; in GenerateScales()
4106 if (LU.Kind == LSRUse::ICmpZero && in GenerateScales()
4112 if (AR && (AR->getLoop() == L || LU.AllFixupsOutsideLoop)) { in GenerateScales()
4128 (AR->getLoop() != L && LU.AllFixupsOutsideLoop))) in GenerateScales()
4132 if (F.Scale == 1 && LU.AllFixupsOutsideLoop) in GenerateScales()
4134 (void)InsertFormula(LU, LUIdx, F); in GenerateScales()
4165 void LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) { in GenerateTruncates() argument
4184 for (auto &LF : LU.Fixups) in GenerateTruncates()
4221 (void)InsertFormula(LU, LUIdx, F); in GenerateTruncates()
4342 LSRUse &LU = Uses[LUIdx]; in GenerateCrossUseConstantOffsets() local
4351 for (size_t L = 0, LE = LU.Formulae.size(); L != LE; ++L) { in GenerateCrossUseConstantOffsets()
4352 Formula F = LU.Formulae[L]; in GenerateCrossUseConstantOffsets()
4367 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, in GenerateCrossUseConstantOffsets()
4383 (void)InsertFormula(LU, LUIdx, NewF); in GenerateCrossUseConstantOffsets()
4392 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, in GenerateCrossUseConstantOffsets()
4393 LU.Kind, LU.AccessTy, NewF)) { in GenerateCrossUseConstantOffsets()
4395 mayUsePostIncMode(TTI, LU, OrigReg, this->L, SE)) in GenerateCrossUseConstantOffsets()
4418 (void)InsertFormula(LU, LUIdx, NewF); in GenerateCrossUseConstantOffsets()
4433 LSRUse &LU = Uses[LUIdx]; in GenerateAllReuseFormulae() local
4434 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) in GenerateAllReuseFormulae()
4435 GenerateReassociations(LU, LUIdx, LU.Formulae[i]); in GenerateAllReuseFormulae()
4436 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) in GenerateAllReuseFormulae()
4437 GenerateCombinations(LU, LUIdx, LU.Formulae[i]); in GenerateAllReuseFormulae()
4440 LSRUse &LU = Uses[LUIdx]; in GenerateAllReuseFormulae() local
4441 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) in GenerateAllReuseFormulae()
4442 GenerateSymbolicOffsets(LU, LUIdx, LU.Formulae[i]); in GenerateAllReuseFormulae()
4443 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) in GenerateAllReuseFormulae()
4444 GenerateConstantOffsets(LU, LUIdx, LU.Formulae[i]); in GenerateAllReuseFormulae()
4445 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) in GenerateAllReuseFormulae()
4446 GenerateICmpZeroScales(LU, LUIdx, LU.Formulae[i]); in GenerateAllReuseFormulae()
4447 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) in GenerateAllReuseFormulae()
4448 GenerateScales(LU, LUIdx, LU.Formulae[i]); in GenerateAllReuseFormulae()
4451 LSRUse &LU = Uses[LUIdx]; in GenerateAllReuseFormulae() local
4452 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) in GenerateAllReuseFormulae()
4453 GenerateTruncates(LU, LUIdx, LU.Formulae[i]); in GenerateAllReuseFormulae()
4481 LSRUse &LU = Uses[LUIdx]; in FilterOutUndesirableDedicatedRegisters() local
4482 LLVM_DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs()); in FilterOutUndesirableDedicatedRegisters()
4486 for (size_t FIdx = 0, NumForms = LU.Formulae.size(); in FilterOutUndesirableDedicatedRegisters()
4488 Formula &F = LU.Formulae[FIdx]; in FilterOutUndesirableDedicatedRegisters()
4499 CostF.RateFormula(F, Regs, VisitedRegs, LU, &LoserRegs); in FilterOutUndesirableDedicatedRegisters()
4528 Formula &Best = LU.Formulae[P.first->second]; in FilterOutUndesirableDedicatedRegisters()
4532 CostBest.RateFormula(Best, Regs, VisitedRegs, LU); in FilterOutUndesirableDedicatedRegisters()
4543 LU.DeleteFormula(F); in FilterOutUndesirableDedicatedRegisters()
4551 LU.RecomputeRegs(LUIdx, RegUses); in FilterOutUndesirableDedicatedRegisters()
4569 for (const LSRUse &LU : Uses) { in EstimateSearchSpaceComplexity() local
4570 size_t FSize = LU.Formulae.size(); in EstimateSearchSpaceComplexity()
4594 LSRUse &LU = Uses[LUIdx]; in NarrowSearchSpaceByDetectingSupersets() local
4596 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) { in NarrowSearchSpaceByDetectingSupersets()
4597 Formula &F = LU.Formulae[i]; in NarrowSearchSpaceByDetectingSupersets()
4610 if (LU.HasFormulaWithSameRegs(NewF)) { in NarrowSearchSpaceByDetectingSupersets()
4613 LU.DeleteFormula(F); in NarrowSearchSpaceByDetectingSupersets()
4626 if (LU.HasFormulaWithSameRegs(NewF)) { in NarrowSearchSpaceByDetectingSupersets()
4629 LU.DeleteFormula(F); in NarrowSearchSpaceByDetectingSupersets()
4640 LU.RecomputeRegs(LUIdx, RegUses); in NarrowSearchSpaceByDetectingSupersets()
4661 LSRUse &LU = Uses[LUIdx]; in NarrowSearchSpaceByCollapsingUnrolledCode() local
4662 for (const Formula &F : LU.Formulae) { in NarrowSearchSpaceByCollapsingUnrolledCode()
4666 LSRUse *LUThatHas = FindUseWithSimilarFormula(F, LU); in NarrowSearchSpaceByCollapsingUnrolledCode()
4671 LU.Kind, LU.AccessTy)) in NarrowSearchSpaceByCollapsingUnrolledCode()
4674 LLVM_DEBUG(dbgs() << " Deleting use "; LU.print(dbgs()); dbgs() << '\n'); in NarrowSearchSpaceByCollapsingUnrolledCode()
4676 LUThatHas->AllFixupsOutsideLoop &= LU.AllFixupsOutsideLoop; in NarrowSearchSpaceByCollapsingUnrolledCode()
4679 for (LSRFixup &Fixup : LU.Fixups) { in NarrowSearchSpaceByCollapsingUnrolledCode()
4703 DeleteUse(LU, LUIdx); in NarrowSearchSpaceByCollapsingUnrolledCode()
4758 LSRUse &LU = Uses[LUIdx]; in NarrowSearchSpaceByFilterFormulaWithSameScaledReg() local
4759 LLVM_DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs()); in NarrowSearchSpaceByFilterFormulaWithSameScaledReg()
4786 CostFA.RateFormula(FA, Regs, VisitedRegs, LU); in NarrowSearchSpaceByFilterFormulaWithSameScaledReg()
4788 CostFB.RateFormula(FB, Regs, VisitedRegs, LU); in NarrowSearchSpaceByFilterFormulaWithSameScaledReg()
4793 for (size_t FIdx = 0, NumForms = LU.Formulae.size(); FIdx != NumForms; in NarrowSearchSpaceByFilterFormulaWithSameScaledReg()
4795 Formula &F = LU.Formulae[FIdx]; in NarrowSearchSpaceByFilterFormulaWithSameScaledReg()
4802 Formula &Best = LU.Formulae[P.first->second]; in NarrowSearchSpaceByFilterFormulaWithSameScaledReg()
4812 LU.DeleteFormula(F); in NarrowSearchSpaceByFilterFormulaWithSameScaledReg()
4818 LU.RecomputeRegs(LUIdx, RegUses); in NarrowSearchSpaceByFilterFormulaWithSameScaledReg()
4844 LSRUse &LU = Uses[LUIdx]; in NarrowSearchSpaceByFilterPostInc() local
4846 if (LU.Kind != LSRUse::Address) in NarrowSearchSpaceByFilterPostInc()
4848 if (!TTI.isIndexedLoadLegal(TTI.MIM_PostInc, LU.AccessTy.getType()) && in NarrowSearchSpaceByFilterPostInc()
4849 !TTI.isIndexedStoreLegal(TTI.MIM_PostInc, LU.AccessTy.getType())) in NarrowSearchSpaceByFilterPostInc()
4853 for (const Formula &F : LU.Formulae) in NarrowSearchSpaceByFilterPostInc()
4857 for (size_t FIdx = 0, NumForms = LU.Formulae.size(); FIdx != NumForms; in NarrowSearchSpaceByFilterPostInc()
4859 Formula &F = LU.Formulae[FIdx]; in NarrowSearchSpaceByFilterPostInc()
4863 LU.DeleteFormula(F); in NarrowSearchSpaceByFilterPostInc()
4870 LU.RecomputeRegs(LUIdx, RegUses); in NarrowSearchSpaceByFilterPostInc()
4939 for (const LSRUse &LU : Uses) { in NarrowSearchSpaceByDeletingCostlyFormulas() local
4940 if (!LU.Regs.count(Reg)) in NarrowSearchSpaceByDeletingCostlyFormulas()
4942 float P = LU.getNotSelectedProbability(Reg); in NarrowSearchSpaceByDeletingCostlyFormulas()
4956 LSRUse &LU = Uses[LUIdx]; in NarrowSearchSpaceByDeletingCostlyFormulas() local
4958 if (LU.Formulae.size() < 2) in NarrowSearchSpaceByDeletingCostlyFormulas()
4963 float FMinRegNum = LU.Formulae[0].getNumRegs(); in NarrowSearchSpaceByDeletingCostlyFormulas()
4964 float FMinARegNum = LU.Formulae[0].getNumRegs(); in NarrowSearchSpaceByDeletingCostlyFormulas()
4966 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) { in NarrowSearchSpaceByDeletingCostlyFormulas()
4967 Formula &F = LU.Formulae[i]; in NarrowSearchSpaceByDeletingCostlyFormulas()
4973 FRegNum += RegNumMap[BaseReg] / LU.getNotSelectedProbability(BaseReg); in NarrowSearchSpaceByDeletingCostlyFormulas()
4976 RegNumMap[BaseReg] / LU.getNotSelectedProbability(BaseReg); in NarrowSearchSpaceByDeletingCostlyFormulas()
4981 RegNumMap[ScaledReg] / LU.getNotSelectedProbability(ScaledReg); in NarrowSearchSpaceByDeletingCostlyFormulas()
4984 RegNumMap[ScaledReg] / LU.getNotSelectedProbability(ScaledReg); in NarrowSearchSpaceByDeletingCostlyFormulas()
4994 LLVM_DEBUG(dbgs() << " The formula "; LU.Formulae[MinIdx].print(dbgs()); in NarrowSearchSpaceByDeletingCostlyFormulas()
4997 std::swap(LU.Formulae[MinIdx], LU.Formulae[0]); in NarrowSearchSpaceByDeletingCostlyFormulas()
4998 while (LU.Formulae.size() != 1) { in NarrowSearchSpaceByDeletingCostlyFormulas()
4999 LLVM_DEBUG(dbgs() << " Deleting "; LU.Formulae.back().print(dbgs()); in NarrowSearchSpaceByDeletingCostlyFormulas()
5001 LU.Formulae.pop_back(); in NarrowSearchSpaceByDeletingCostlyFormulas()
5003 LU.RecomputeRegs(LUIdx, RegUses); in NarrowSearchSpaceByDeletingCostlyFormulas()
5004 assert(LU.Formulae.size() == 1 && "Should be exactly 1 min regs formula"); in NarrowSearchSpaceByDeletingCostlyFormulas()
5005 Formula &F = LU.Formulae[0]; in NarrowSearchSpaceByDeletingCostlyFormulas()
5093 LSRUse &LU = Uses[LUIdx]; in NarrowSearchSpaceByPickingWinnerRegs() local
5094 if (!LU.Regs.count(Best)) continue; in NarrowSearchSpaceByPickingWinnerRegs()
5097 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) { in NarrowSearchSpaceByPickingWinnerRegs()
5098 Formula &F = LU.Formulae[i]; in NarrowSearchSpaceByPickingWinnerRegs()
5101 LU.DeleteFormula(F); in NarrowSearchSpaceByPickingWinnerRegs()
5111 LU.RecomputeRegs(LUIdx, RegUses); in NarrowSearchSpaceByPickingWinnerRegs()
5152 const LSRUse &LU = Uses[Workspace.size()]; in SolveRecurse() local
5160 if (LU.Regs.count(S)) in SolveRecurse()
5165 for (const Formula &F : LU.Formulae) { in SolveRecurse()
5172 if (AMK != TTI::AMK_PostIndexed || LU.Kind != LSRUse::Address) { in SolveRecurse()
5193 NewCost.RateFormula(F, NewRegs, VisitedRegs, LU); in SolveRecurse()
5326 BasicBlock::iterator LowestIP, const LSRFixup &LF, const LSRUse &LU) const { in AdjustInsertPositionForExpand()
5333 if (LU.Kind == LSRUse::ICmpZero) in AdjustInsertPositionForExpand()
5387 Value *LSRInstance::Expand(const LSRUse &LU, const LSRFixup &LF, in Expand() argument
5390 if (LU.RigidFormula) in Expand()
5395 IP = AdjustInsertPositionForExpand(IP, LF, LU); in Expand()
5436 if (LU.Kind == LSRUse::ICmpZero) { in Expand()
5455 if (!Ops.empty() && LU.Kind == LSRUse::Address && in Expand()
5456 isAMCompletelyFolded(TTI, LU, F)) { in Expand()
5491 if (LU.Kind == LSRUse::ICmpZero) { in Expand()
5527 if (LU.Kind == LSRUse::ICmpZero) { in Expand()
5568 PHINode *PN, const LSRUse &LU, const LSRFixup &LF, const Formula &F, in RewriteForPHI() argument
5633 Expand(LU, LF, F, BB->getTerminator()->getIterator(), DeadInsts); in RewriteForPHI()
5700 void LSRInstance::Rewrite(const LSRUse &LU, const LSRFixup &LF, in Rewrite() argument
5706 RewriteForPHI(PN, LU, LF, F, DeadInsts); in Rewrite()
5708 Value *FullV = Expand(LU, LF, F, LF.UserInst->getIterator(), DeadInsts); in Rewrite()
5724 if (LU.Kind == LSRUse::ICmpZero) in Rewrite()
5738 const LSRUse &LU, Instruction *IVIncInsertPos, in canHoistIVInc() argument
5740 if (LU.Kind != LSRUse::Address) in canHoistIVInc()
5954 for (const LSRUse &LU : Uses) { in LSRInstance() local
5955 for (const Formula &F : LU.Formulae) in LSRInstance()
5956 assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, in LSRInstance()
5988 for (const LSRUse &LU : Uses) in print_fixups() local
5989 for (const LSRFixup &LF : LU.Fixups) { in print_fixups()
5998 for (const LSRUse &LU : Uses) { in print_uses() local
6000 LU.print(OS); in print_uses()
6002 for (const Formula &F : LU.Formulae) { in print_uses()