Lines Matching refs:SE
395 void initialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE);
424 ScalarEvolution &SE) { in DoInitialMatch() argument
426 if (SE.properlyDominates(S, L->getHeader())) { in DoInitialMatch()
434 DoInitialMatch(S, L, Good, Bad, SE); in DoInitialMatch()
441 DoInitialMatch(AR->getStart(), L, Good, Bad, SE); in DoInitialMatch()
442 DoInitialMatch(SE.getAddRecExpr(SE.getConstant(AR->getType(), 0), in DoInitialMatch()
443 AR->getStepRecurrence(SE), in DoInitialMatch()
446 L, Good, Bad, SE); in DoInitialMatch()
454 const SCEV *NewMul = SE.getMulExpr(Ops); in DoInitialMatch()
458 DoInitialMatch(NewMul, L, MyGood, MyBad, SE); in DoInitialMatch()
459 const SCEV *NegOne = SE.getSCEV(ConstantInt::getAllOnesValue( in DoInitialMatch()
460 SE.getEffectiveSCEVType(NewMul->getType()))); in DoInitialMatch()
462 Good.push_back(SE.getMulExpr(NegOne, S)); in DoInitialMatch()
464 Bad.push_back(SE.getMulExpr(NegOne, S)); in DoInitialMatch()
475 void Formula::initialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE) { in initialMatch() argument
478 DoInitialMatch(S, L, Good, Bad, SE); in initialMatch()
480 const SCEV *Sum = SE.getAddExpr(Good); in initialMatch()
486 const SCEV *Sum = SE.getAddExpr(Bad); in initialMatch()
668 static bool isAddRecSExtable(const SCEVAddRecExpr *AR, ScalarEvolution &SE) { in isAddRecSExtable() argument
670 IntegerType::get(SE.getContext(), SE.getTypeSizeInBits(AR->getType()) + 1); in isAddRecSExtable()
671 return isa<SCEVAddRecExpr>(SE.getSignExtendExpr(AR, WideTy)); in isAddRecSExtable()
676 static bool isAddSExtable(const SCEVAddExpr *A, ScalarEvolution &SE) { in isAddSExtable() argument
678 IntegerType::get(SE.getContext(), SE.getTypeSizeInBits(A->getType()) + 1); in isAddSExtable()
679 return isa<SCEVAddExpr>(SE.getSignExtendExpr(A, WideTy)); in isAddSExtable()
684 static bool isMulSExtable(const SCEVMulExpr *M, ScalarEvolution &SE) { in isMulSExtable() argument
686 IntegerType::get(SE.getContext(), in isMulSExtable()
687 SE.getTypeSizeInBits(M->getType()) * M->getNumOperands()); in isMulSExtable()
688 return isa<SCEVMulExpr>(SE.getSignExtendExpr(M, WideTy)); in isMulSExtable()
697 ScalarEvolution &SE, in getExactSDiv() argument
701 return SE.getConstant(LHS->getType(), 1); in getExactSDiv()
712 return SE.getMulExpr(LHS, RC); in getExactSDiv()
727 return SE.getConstant(LA.sdiv(RA)); in getExactSDiv()
732 if ((IgnoreSignificantBits || isAddRecSExtable(AR, SE)) && AR->isAffine()) { in getExactSDiv()
733 const SCEV *Step = getExactSDiv(AR->getStepRecurrence(SE), RHS, SE, in getExactSDiv()
736 const SCEV *Start = getExactSDiv(AR->getStart(), RHS, SE, in getExactSDiv()
742 return SE.getAddRecExpr(Start, Step, AR->getLoop(), SCEV::FlagAnyWrap); in getExactSDiv()
749 if (IgnoreSignificantBits || isAddSExtable(Add, SE)) { in getExactSDiv()
752 const SCEV *Op = getExactSDiv(S, RHS, SE, IgnoreSignificantBits); in getExactSDiv()
756 return SE.getAddExpr(Ops); in getExactSDiv()
763 if (IgnoreSignificantBits || isMulSExtable(Mul, SE)) { in getExactSDiv()
766 if (IgnoreSignificantBits || isMulSExtable(MulRHS, SE)) { in getExactSDiv()
774 return getExactSDiv(LC, RC, SE, IgnoreSignificantBits); in getExactSDiv()
783 if (const SCEV *Q = getExactSDiv(S, RHS, SE, in getExactSDiv()
790 return Found ? SE.getMulExpr(Ops) : nullptr; in getExactSDiv()
801 static int64_t ExtractImmediate(const SCEV *&S, ScalarEvolution &SE) { in ExtractImmediate() argument
804 S = SE.getConstant(C->getType(), 0); in ExtractImmediate()
809 int64_t Result = ExtractImmediate(NewOps.front(), SE); in ExtractImmediate()
811 S = SE.getAddExpr(NewOps); in ExtractImmediate()
815 int64_t Result = ExtractImmediate(NewOps.front(), SE); in ExtractImmediate()
817 S = SE.getAddRecExpr(NewOps, AR->getLoop(), in ExtractImmediate()
827 static GlobalValue *ExtractSymbol(const SCEV *&S, ScalarEvolution &SE) { in ExtractSymbol() argument
830 S = SE.getConstant(GV->getType(), 0); in ExtractSymbol()
835 GlobalValue *Result = ExtractSymbol(NewOps.back(), SE); in ExtractSymbol()
837 S = SE.getAddExpr(NewOps); in ExtractSymbol()
841 GlobalValue *Result = ExtractSymbol(NewOps.front(), SE); in ExtractSymbol()
843 S = SE.getAddRecExpr(NewOps, AR->getLoop(), in ExtractSymbol()
951 static bool isExistingPhi(const SCEVAddRecExpr *AR, ScalarEvolution &SE) { in isExistingPhi() argument
953 if (SE.isSCEVable(PN.getType()) && in isExistingPhi()
954 (SE.getEffectiveSCEVType(PN.getType()) == in isExistingPhi()
955 SE.getEffectiveSCEVType(AR->getType())) && in isExistingPhi()
956 SE.getSCEV(&PN) == AR) in isExistingPhi()
973 ScalarEvolution &SE) { in isHighCostExpansion() argument
982 Processed, SE); in isHighCostExpansion()
985 Processed, SE); in isHighCostExpansion()
988 Processed, SE); in isHighCostExpansion()
998 if (isHighCostExpansion(S, Processed, SE)) in isHighCostExpansion()
1008 return isHighCostExpansion(Mul->getOperand(1), Processed, SE); in isHighCostExpansion()
1018 SE.isSCEVable(UI->getType())) { in isHighCostExpansion()
1019 return SE.getSCEV(UI) == Mul; in isHighCostExpansion()
1027 if (isExistingPhi(AR, SE)) in isHighCostExpansion()
1063 ScalarEvolution *SE = nullptr; member in __anon926e14ea0811::Cost
1070 Cost(const Loop *L, ScalarEvolution &SE, const TargetTransformInfo &TTI, in Cost() argument
1072 L(L), SE(&SE), TTI(&TTI), AMK(AMK) { in Cost()
1290 if (isExistingPhi(AR, *SE) && AMK != TTI::AMK_PostIndexed) in RateRegister()
1312 if (auto *Step = dyn_cast<SCEVConstant>(AR->getStepRecurrence(*SE))) in RateRegister()
1316 const SCEV *LoopStep = AR->getStepRecurrence(*SE); in RateRegister()
1320 SE->isLoopInvariant(LoopStart, L)) in RateRegister()
1346 SE->hasComputableLoopEvolution(Reg, L); in RateRegister()
1864 ScalarEvolution &SE, int64_t MinOffset, in isAlwaysFoldable() argument
1873 int64_t BaseOffset = ExtractImmediate(S, SE); in isAlwaysFoldable()
1874 GlobalValue *BaseGV = ExtractSymbol(S, SE); in isAlwaysFoldable()
1957 ScalarEvolution &SE; member in __anon926e14ea0a11::LSRInstance
2106 LSRInstance(Loop *L, IVUsers &IU, ScalarEvolution &SE, DominatorTree &DT,
2127 const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L); in OptimizeShadowIV()
2172 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(SE.getSCEV(PH)); in OptimizeShadowIV()
2180 if ((int)SE.getTypeSizeInBits(SrcTy) > Mantissa) in OptimizeShadowIV()
2312 const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L); in OptimizeMax()
2315 const SCEV *One = SE.getConstant(BackedgeTakenCount->getType(), 1); in OptimizeMax()
2318 const SCEV *IterationCount = SE.getAddExpr(One, BackedgeTakenCount); in OptimizeMax()
2319 if (IterationCount != SE.getSCEV(Sel)) return Cond; in OptimizeMax()
2356 const SCEV *IV = SE.getSCEV(Cond->getOperand(0)); in OptimizeMax()
2360 AR->getStepRecurrence(SE) != One) in OptimizeMax()
2373 if (BO1->isOne() && SE.getSCEV(BO->getOperand(0)) == MaxRHS) in OptimizeMax()
2377 if (BO1->isOne() && SE.getSCEV(BO->getOperand(0)) == MaxRHS) in OptimizeMax()
2381 } else if (SE.getSCEV(Sel->getOperand(1)) == MaxRHS) in OptimizeMax()
2383 else if (SE.getSCEV(Sel->getOperand(2)) == MaxRHS) in OptimizeMax()
2486 if (SE.getTypeSizeInBits(A->getType()) != in OptimizeLoopTermCond()
2487 SE.getTypeSizeInBits(B->getType())) { in OptimizeLoopTermCond()
2488 if (SE.getTypeSizeInBits(A->getType()) > in OptimizeLoopTermCond()
2489 SE.getTypeSizeInBits(B->getType())) in OptimizeLoopTermCond()
2490 B = SE.getSignExtendExpr(B, A->getType()); in OptimizeLoopTermCond()
2492 A = SE.getSignExtendExpr(A, B->getType()); in OptimizeLoopTermCond()
2495 dyn_cast_or_null<SCEVConstant>(getExactSDiv(B, A, SE))) { in OptimizeLoopTermCond()
2616 int64_t Offset = ExtractImmediate(Expr, SE); in getUse()
2709 Types.insert(SE.getEffectiveSCEVType(Expr->getType())); in CollectInterestingTypesAndFactors()
2717 Strides.insert(AR->getStepRecurrence(SE)); in CollectInterestingTypesAndFactors()
2733 if (SE.getTypeSizeInBits(OldStride->getType()) != in CollectInterestingTypesAndFactors()
2734 SE.getTypeSizeInBits(NewStride->getType())) { in CollectInterestingTypesAndFactors()
2735 if (SE.getTypeSizeInBits(OldStride->getType()) > in CollectInterestingTypesAndFactors()
2736 SE.getTypeSizeInBits(NewStride->getType())) in CollectInterestingTypesAndFactors()
2737 NewStride = SE.getSignExtendExpr(NewStride, OldStride->getType()); in CollectInterestingTypesAndFactors()
2739 OldStride = SE.getSignExtendExpr(OldStride, NewStride->getType()); in CollectInterestingTypesAndFactors()
2743 SE, true))) { in CollectInterestingTypesAndFactors()
2749 SE, true))) { in CollectInterestingTypesAndFactors()
2768 Loop *L, ScalarEvolution &SE) { in findIVOperand() argument
2771 if (!SE.isSCEVable(Oper->getType())) in findIVOperand()
2775 dyn_cast<SCEVAddRecExpr>(SE.getSCEV(Oper))) { in findIVOperand()
2842 ScalarEvolution &SE) { in isProfitableIncrement() argument
2850 const SCEV *HeadExpr = SE.getSCEV(getWideOperand(Incs[0].IVOperand)); in isProfitableIncrement()
2851 if (isa<SCEVConstant>(SE.getMinusSCEV(OperExpr, HeadExpr))) in isProfitableIncrement()
2856 return !isHighCostExpansion(IncExpr, Processed, SE); in isProfitableIncrement()
2871 ScalarEvolution &SE, in isProfitableChain() argument
2894 && SE.getSCEV(Chain.tailUserInst()) == Chain.Incs[0].IncExpr) { in isProfitableChain()
2953 const SCEV *const OperExpr = SE.getSCEV(NextIV); in ChainInstruction()
2979 const SCEV *PrevExpr = SE.getSCEV(PrevIV); in ChainInstruction()
2980 const SCEV *IncExpr = SE.getMinusSCEV(OperExpr, PrevExpr); in ChainInstruction()
2981 if (isa<SCEVCouldNotCompute>(IncExpr) || !SE.isLoopInvariant(IncExpr, L)) in ChainInstruction()
2984 if (Chain.isProfitableIncrement(OperExpr, IncExpr, SE)) { in ChainInstruction()
3046 if (SE.isSCEVable(OtherUse->getType()) in ChainInstruction()
3047 && !isa<SCEVUnknown>(SE.getSCEV(OtherUse)) in ChainInstruction()
3103 if (SE.isSCEVable(I.getType()) && !isa<SCEVUnknown>(SE.getSCEV(&I))) in CollectChains()
3114 User::op_iterator IVOpIter = findIVOperand(I.op_begin(), IVOpEnd, L, SE); in CollectChains()
3119 IVOpIter = findIVOperand(std::next(IVOpIter), IVOpEnd, L, SE); in CollectChains()
3125 if (!SE.isSCEVable(PN.getType())) in CollectChains()
3138 ChainUsersVec[UsersIdx].FarUsers, SE, TTI)) in CollectChains()
3190 IVOpEnd, L, SE); in GenerateIVChain()
3203 if (SE.getSCEV(*IVOpIter) == Head.IncExpr in GenerateIVChain()
3204 || SE.getSCEV(IVSrc) == Head.IncExpr) { in GenerateIVChain()
3207 IVOpIter = findIVOperand(std::next(IVOpIter), IVOpEnd, L, SE); in GenerateIVChain()
3218 Type *IntTy = SE.getEffectiveSCEVType(IVTy); in GenerateIVChain()
3231 const SCEV *IncExpr = SE.getNoopOrSignExtend(Inc.IncExpr, IntTy); in GenerateIVChain()
3233 SE.getAddExpr(LeftOverExpr, IncExpr) : IncExpr; in GenerateIVChain()
3239 const SCEV *IVOperExpr = SE.getAddExpr(SE.getUnknown(IVSrc), in GenerateIVChain()
3240 SE.getUnknown(IncV)); in GenerateIVChain()
3252 assert(SE.getTypeSizeInBits(IVTy) >= SE.getTypeSizeInBits(OperTy) && in GenerateIVChain()
3269 if (!PostIncV || (SE.getSCEV(PostIncV) != SE.getSCEV(IVSrc))) in GenerateIVChain()
3287 bool SaveCmp = TTI.canSaveCmp(L, &ExitBranch, &SE, &LI, &DT, &AC, &TLI); in CollectFixupsAndInitialFormulae()
3340 const SCEV *N = SE.getSCEV(NV); in CollectFixupsAndInitialFormulae()
3341 if (SE.isLoopInvariant(N, L) && Rewriter.isSafeToExpand(N) && in CollectFixupsAndInitialFormulae()
3343 SE.getPointerBase(N) == SE.getPointerBase(S))) { in CollectFixupsAndInitialFormulae()
3346 N = normalizeForPostIncUse(N, TmpPostIncLoops, SE); in CollectFixupsAndInitialFormulae()
3350 S = SE.getMinusSCEV(N, S); in CollectFixupsAndInitialFormulae()
3361 N = SE.getUnknown(NV); in CollectFixupsAndInitialFormulae()
3362 N = normalizeForPostIncUse(N, TmpPostIncLoops, SE); in CollectFixupsAndInitialFormulae()
3366 S = SE.getMinusSCEV(N, S); in CollectFixupsAndInitialFormulae()
3396 F.initialMatch(S, L, SE); in CollectFixupsAndInitialFormulae()
3402 SE.getTypeSizeInBits(LU.WidestFixupType) < in CollectFixupsAndInitialFormulae()
3403 SE.getTypeSizeInBits(LF.OperandValToReplace->getType())) in CollectFixupsAndInitialFormulae()
3425 F.initialMatch(S, L, SE); in InsertInitialFormula()
3553 if (SE.isSCEVable(UserInst->getType())) { in CollectLoopInvariantFixupsAndFormulae()
3554 const SCEV *UserS = SE.getSCEV(const_cast<Instruction *>(UserInst)); in CollectLoopInvariantFixupsAndFormulae()
3560 SE.getUnknown(const_cast<Instruction *>(UserInst))); in CollectLoopInvariantFixupsAndFormulae()
3568 if (SE.hasComputableLoopEvolution(SE.getSCEV(OtherOp), L)) in CollectLoopInvariantFixupsAndFormulae()
3583 SE.getTypeSizeInBits(LU.WidestFixupType) < in CollectLoopInvariantFixupsAndFormulae()
3584 SE.getTypeSizeInBits(LF.OperandValToReplace->getType())) in CollectLoopInvariantFixupsAndFormulae()
3602 ScalarEvolution &SE, in CollectSubexprs() argument
3611 const SCEV *Remainder = CollectSubexprs(S, C, Ops, L, SE, Depth+1); in CollectSubexprs()
3613 Ops.push_back(C ? SE.getMulExpr(C, Remainder) : Remainder); in CollectSubexprs()
3622 C, Ops, L, SE, Depth+1); in CollectSubexprs()
3626 Ops.push_back(C ? SE.getMulExpr(C, Remainder) : Remainder); in CollectSubexprs()
3631 Remainder = SE.getConstant(AR->getType(), 0); in CollectSubexprs()
3632 return SE.getAddRecExpr(Remainder, in CollectSubexprs()
3633 AR->getStepRecurrence(SE), in CollectSubexprs()
3644 C = C ? cast<SCEVConstant>(SE.getMulExpr(C, Op0)) : Op0; in CollectSubexprs()
3646 CollectSubexprs(Mul->getOperand(1), C, Ops, L, SE, Depth+1); in CollectSubexprs()
3648 Ops.push_back(SE.getMulExpr(C, Remainder)); in CollectSubexprs()
3659 ScalarEvolution &SE) { in mayUsePostIncMode() argument
3666 const SCEV *LoopStep = AR->getStepRecurrence(SE); in mayUsePostIncMode()
3673 if (!isa<SCEVConstant>(LoopStart) && SE.isLoopInvariant(LoopStart, L)) in mayUsePostIncMode()
3689 if (AMK == TTI::AMK_PostIndexed && mayUsePostIncMode(TTI, LU, BaseReg, L, SE)) in GenerateReassociationsImpl()
3692 const SCEV *Remainder = CollectSubexprs(BaseReg, nullptr, AddOps, L, SE); in GenerateReassociationsImpl()
3704 if (isa<SCEVUnknown>(*J) && !SE.isLoopInvariant(*J, L)) in GenerateReassociationsImpl()
3709 if (isAlwaysFoldable(TTI, SE, LU.MinOffset, LU.MaxOffset, LU.Kind, in GenerateReassociationsImpl()
3722 isAlwaysFoldable(TTI, SE, LU.MinOffset, LU.MaxOffset, LU.Kind, in GenerateReassociationsImpl()
3726 const SCEV *InnerSum = SE.getAddExpr(InnerAddOps); in GenerateReassociationsImpl()
3733 if (InnerSumSC && SE.getTypeSizeInBits(InnerSumSC->getType()) <= 64 && in GenerateReassociationsImpl()
3749 if (SC && SE.getTypeSizeInBits(SC->getType()) <= 64 && in GenerateReassociationsImpl()
3805 if (SE.properlyDominates(BaseReg, L->getHeader()) && in GenerateCombinations()
3806 !SE.hasComputableLoopEvolution(BaseReg, L)) { in GenerateCombinations()
3808 CombinedIntegerType = SE.getEffectiveSCEVType(BaseReg->getType()); in GenerateCombinations()
3838 GenerateFormula(SE.getAddExpr(OpsCopy)); in GenerateCombinations()
3845 Ops.push_back(SE.getConstant(CombinedIntegerType, NewBase.UnfoldedOffset, in GenerateCombinations()
3848 GenerateFormula(SE.getAddExpr(Ops)); in GenerateCombinations()
3857 GlobalValue *GV = ExtractSymbol(G, SE); in GenerateSymbolicOffsetsImpl()
3895 const SCEV *NewG = SE.getAddExpr(SE.getConstant(G->getType(), Offset), G); in GenerateConstantOffsetsImpl()
3926 dyn_cast<SCEVConstant>(GAR->getStepRecurrence(SE))) { in GenerateConstantOffsetsImpl()
3941 int64_t Imm = ExtractImmediate(G, SE); in GenerateConstantOffsetsImpl()
3985 if (SE.getTypeSizeInBits(IntTy) > 64) return; in GenerateICmpZeroScales()
4037 const SCEV *FactorS = SE.getConstant(IntTy, Factor); in GenerateICmpZeroScales()
4041 F.BaseRegs[i] = SE.getMulExpr(F.BaseRegs[i], FactorS); in GenerateICmpZeroScales()
4042 if (getExactSDiv(F.BaseRegs[i], FactorS, SE) != Base.BaseRegs[i]) in GenerateICmpZeroScales()
4048 F.ScaledReg = SE.getMulExpr(F.ScaledReg, FactorS); in GenerateICmpZeroScales()
4049 if (getExactSDiv(F.ScaledReg, FactorS, SE) != Base.ScaledReg) in GenerateICmpZeroScales()
4113 const SCEV *FactorS = SE.getConstant(IntTy, Factor); in GenerateScales()
4118 if (const SCEV *Quotient = getExactSDiv(AR, FactorS, SE, true)) in GenerateScales()
4149 ScalarEvolution &SE) { in getAnyExtendConsideringPostIncUses() argument
4152 auto *DenormExpr = denormalizeForPostIncUse(Expr, L, SE); in getAnyExtendConsideringPostIncUses()
4153 const SCEV *NewDenormExpr = SE.getAnyExtendExpr(DenormExpr, ToTy); in getAnyExtendConsideringPostIncUses()
4154 const SCEV *New = normalizeForPostIncUse(NewDenormExpr, L, SE); in getAnyExtendConsideringPostIncUses()
4197 getAnyExtendConsideringPostIncUses(Loops, F.ScaledReg, SrcTy, SE); in GenerateTruncates()
4205 getAnyExtendConsideringPostIncUses(Loops, BaseReg, SrcTy, SE); in GenerateTruncates()
4267 int64_t Imm = ExtractImmediate(Reg, SE); in GenerateCrossUseConstantOffsets()
4346 Type *IntTy = SE.getEffectiveSCEVType(OrigReg->getType()); in GenerateCrossUseConstantOffsets()
4347 const SCEV *NegImmS = SE.getSCEV(ConstantInt::get(IntTy, -(uint64_t)Imm)); in GenerateCrossUseConstantOffsets()
4348 unsigned BitWidth = SE.getTypeSizeInBits(IntTy); in GenerateCrossUseConstantOffsets()
4362 if (F.referencesReg(SE.getSCEV( in GenerateCrossUseConstantOffsets()
4370 NewF.ScaledReg = SE.getAddExpr(NegImmS, NewF.ScaledReg); in GenerateCrossUseConstantOffsets()
4395 mayUsePostIncMode(TTI, LU, OrigReg, this->L, SE)) in GenerateCrossUseConstantOffsets()
4402 NewF.BaseRegs[N] = SE.getAddExpr(NegImmS, BaseReg); in GenerateCrossUseConstantOffsets()
4497 Cost CostF(L, SE, TTI, AMK); in FilterOutUndesirableDedicatedRegisters()
4530 Cost CostBest(L, SE, TTI, AMK); in FilterOutUndesirableDedicatedRegisters()
4783 Cost CostFA(L, SE, TTI, AMK); in NarrowSearchSpaceByFilterFormulaWithSameScaledReg()
4784 Cost CostFB(L, SE, TTI, AMK); in NarrowSearchSpaceByFilterFormulaWithSameScaledReg()
5019 ScalarEvolution &SE, const SCEV *Best, in IsSimplerBaseSCEVForTarget() argument
5027 const auto *Diff = dyn_cast<SCEVConstant>(SE.getMinusSCEV(Best, Reg)); in IsSimplerBaseSCEVForTarget()
5076 IsSimplerBaseSCEVForTarget(TTI, SE, Best, Reg, in NarrowSearchSpaceByPickingWinnerRegs()
5164 Cost NewCost(L, SE, TTI, AMK); in SolveRecurse()
5220 Cost SolutionCost(L, SE, TTI, AMK); in Solve()
5222 Cost CurCost(L, SE, TTI, AMK); in Solve()
5409 else if (SE.getEffectiveSCEVType(Ty) == SE.getEffectiveSCEVType(OpTy)) in Expand()
5413 Type *IntTy = SE.getEffectiveSCEVType(Ty); in Expand()
5423 Reg = denormalizeForPostIncUse(Reg, LF.PostIncLoops, SE); in Expand()
5424 Ops.push_back(SE.getUnknown(Rewriter.expandCodeFor(Reg, nullptr))); in Expand()
5434 ScaledS = denormalizeForPostIncUse(ScaledS, Loops, SE); in Expand()
5440 SE.getUnknown(Rewriter.expandCodeFor(ScaledS, nullptr))); in Expand()
5457 Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), nullptr); in Expand()
5459 Ops.push_back(SE.getUnknown(FullV)); in Expand()
5461 ScaledS = SE.getUnknown(Rewriter.expandCodeFor(ScaledS, nullptr)); in Expand()
5464 SE.getMulExpr(ScaledS, SE.getConstant(ScaledS->getType(), F.Scale)); in Expand()
5473 Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), IntTy); in Expand()
5475 Ops.push_back(SE.getUnknown(FullV)); in Expand()
5477 Ops.push_back(SE.getUnknown(F.BaseGV)); in Expand()
5483 Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), Ty); in Expand()
5485 Ops.push_back(SE.getUnknown(FullV)); in Expand()
5497 Ops.push_back(SE.getUnknown(ICmpScaledV)); in Expand()
5503 Ops.push_back(SE.getUnknown(ConstantInt::getSigned(IntTy, Offset))); in Expand()
5511 Ops.push_back(SE.getUnknown(ConstantInt::getSigned(IntTy, in Expand()
5517 SE.getConstant(IntTy, 0) : in Expand()
5518 SE.getAddExpr(Ops); in Expand()
5548 Constant *C = ConstantInt::getSigned(SE.getEffectiveSCEVType(OpTy), in Expand()
5694 formLCSSAForInstructions(InsertedNonLCSSAInsts, DT, LI, &SE); in RewriteForPHI()
5851 LSRInstance::LSRInstance(Loop *L, IVUsers &IU, ScalarEvolution &SE, in LSRInstance() argument
5855 : IU(IU), SE(SE), DT(DT), LI(LI), AC(AC), TLI(TLI), TTI(TTI), L(L), in LSRInstance()
5858 : TTI.getPreferredAddressingMode(L, &SE)), in LSRInstance()
5859 Rewriter(SE, L->getHeader()->getModule()->getDataLayout(), "lsr", false), in LSRInstance()
5860 BaselineCost(L, SE, TTI, AMK) { in LSRInstance()
6228 bool SCEVToValueExpr(const llvm::SCEVAddRecExpr &SAR, ScalarEvolution &SE) { in SCEVToValueExpr()
6235 const SCEV *Stride = SAR.getStepRecurrence(SE); in SCEVToValueExpr()
6265 ScalarEvolution &SE) { in createIterCountExpr()
6287 if (!SCEVToValueExpr(*Rec, SE)) in createIterCountExpr()
6299 ScalarEvolution &SE) { in SCEVToIterCountExpr()
6307 const SCEV *Stride = SAR.getStepRecurrence(SE); in SCEVToIterCountExpr()
6519 static bool SalvageDVI(llvm::Loop *L, ScalarEvolution &SE, in SalvageDVI() argument
6557 if (SE.containsErasedValue(DVIRec.SCEVs[i]) || in SalvageDVI()
6558 SE.containsUndefs(DVIRec.SCEVs[i])) { in SalvageDVI()
6574 SE.computeConstantDifference(DVIRec.SCEVs[i], SCEVInductionVar)) { in SalvageDVI()
6578 SE)) in SalvageDVI()
6631 llvm::Loop *L, ScalarEvolution &SE, llvm::PHINode *LSRInductionVar, in DbgRewriteSalvageableDVIs() argument
6636 const llvm::SCEV *SCEVInductionVar = SE.getSCEV(LSRInductionVar); in DbgRewriteSalvageableDVIs()
6652 if (!IterCountExpr.SCEVToIterCountExpr(*IVAddRec, SE)) in DbgRewriteSalvageableDVIs()
6659 SalvageDVI(L, SE, LSRInductionVar, *DVIRec, SCEVInductionVar, in DbgRewriteSalvageableDVIs()
6669 Loop *L, ScalarEvolution &SE, in DbgGatherSalvagableDVI() argument
6688 if (!SE.isSCEVable(LocOp->getType())) in DbgGatherSalvagableDVI()
6691 const SCEV *S = SE.getSCEV(LocOp); in DbgGatherSalvagableDVI()
6692 if (SE.containsUndefs(S)) in DbgGatherSalvagableDVI()
6708 NewRec->SCEVs.push_back(SE.getSCEV(LocOp)); in DbgGatherSalvagableDVI()
6731 static llvm::PHINode *GetInductionVariable(const Loop &L, ScalarEvolution &SE, in GetInductionVariable() argument
6735 if (!SE.isSCEVable(P->getType())) in GetInductionVariable()
6737 if (const SCEVAddRecExpr *Rec = dyn_cast<SCEVAddRecExpr>(SE.getSCEV(P))) in GetInductionVariable()
6738 return Rec->isAffine() && !SE.containsUndefs(SE.getSCEV(P)); in GetInductionVariable()
6764 canFoldTermCondOfLoop(Loop *L, ScalarEvolution &SE, DominatorTree &DT, in canFoldTermCondOfLoop() argument
6776 if (!SE.hasLoopInvariantBackedgeTakenCount(L)) { in canFoldTermCondOfLoop()
6816 const SCEV *BECount = SE.getBackedgeTakenCount(L); in canFoldTermCondOfLoop()
6818 SCEVExpander Expander(SE, DL, "lsr_fold_term_cond"); in canFoldTermCondOfLoop()
6827 if (!SE.isSCEVable(PN.getType())) { in canFoldTermCondOfLoop()
6833 const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(SE.getSCEV(&PN)); in canFoldTermCondOfLoop()
6852 !SE.isKnownNonZero(AddRec->getStepRecurrence(SE))) in canFoldTermCondOfLoop()
6855 const SCEVAddRecExpr *PostInc = AddRec->getPostIncExpr(SE); in canFoldTermCondOfLoop()
6856 const SCEV *TermValueSLocal = PostInc->evaluateAtIteration(BECount, SE); in canFoldTermCondOfLoop()
6907 << " BECount (SCEV): " << *SE.getBackedgeTakenCount(L) << "\n" in canFoldTermCondOfLoop()
6918 static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE, in ReduceLoopStrength() argument
6928 DbgGatherSalvagableDVI(L, SE, SalvageableDVIRecords, DVIHandles); in ReduceLoopStrength()
6937 LSRInstance(L, IU, SE, DT, LI, TTI, AC, TLI, MSSAU.get()); in ReduceLoopStrength()
6945 SCEVExpander Rewriter(SE, DL, "lsr", false); in ReduceLoopStrength()
6965 SCEVExpander Rewriter(SE, DL, "lsr", true); in ReduceLoopStrength()
6966 int Rewrites = rewriteLoopExitValues(L, &LI, &TLI, &SE, &TTI, Rewriter, &DT, in ReduceLoopStrength()
6989 if (auto Opt = canFoldTermCondOfLoop(L, SE, DT, LI)) { in ReduceLoopStrength()
7014 SCEVExpander Expander(SE, DL, "lsr_fold_term_cond"); in ReduceLoopStrength()
7060 if (llvm::PHINode *IV = GetInductionVariable(*L, SE, Reducer)) in ReduceLoopStrength()
7061 DbgRewriteSalvageableDVIs(L, SE, IV, SalvageableDVIRecords); in ReduceLoopStrength()
7080 auto &SE = getAnalysis<ScalarEvolutionWrapperPass>().getSE(); in runOnLoop() local
7093 return ReduceLoopStrength(L, IU, SE, DT, LI, TTI, AC, TLI, MSSA); in runOnLoop()
7099 if (!ReduceLoopStrength(&L, AM.getResult<IVUsersAnalysis>(L, AR), AR.SE, in run()