Lines Matching refs:scop
299 int OldDepth = scop->getRelativeLoopDepth(OldL); in adjustDomainDimensions()
300 int NewDepth = scop->getRelativeLoopDepth(NewL); in adjustDomainDimensions()
318 auto &R = scop->getRegion(); in adjustDomainDimensions()
348 PWACtx PWAC = scop->getPwAff(E, BB, NonNegative, &RecordedAssumptions); in getPwAff()
446 getUniqueNonErrorValue(PHI, &scop->getRegion(), &SD)); in buildConditionSets()
488 Region &R = scop->getRegion(); in buildConditionSets()
552 scop->invalidate(COMPLEXITY, TI ? TI->getDebugLoc() : DebugLoc(), in buildConditionSets()
604 if (!scop->isNonAffineSubRegion(SubRegion)) { in propagateDomainConstraints()
612 isl::set &Domain = scop->getOrInitEmptyDomain(BB); in propagateDomainConstraints()
618 Domain = Domain.align_params(scop->getParamSpace()); in propagateDomainConstraints()
621 if (BBLoop && BBLoop->getHeader() == BB && scop->contains(BBLoop)) in propagateDomainConstraints()
635 auto *RI = scop->getRegion().getRegionInfo(); in propagateDomainConstraintsToRegionExit()
638 if (!BBReg || BBReg->getEntry() != BB || !scop->contains(ExitBB)) in propagateDomainConstraintsToRegionExit()
644 while (L && scop->contains(L)) { in propagateDomainConstraintsToRegionExit()
653 isl::set Domain = scop->getOrInitEmptyDomain(BB); in propagateDomainConstraintsToRegionExit()
656 Loop *ExitBBLoop = getFirstNonBoxedLoopFor(ExitBB, LI, scop->getBoxedLoops()); in propagateDomainConstraintsToRegionExit()
661 isl::set &ExitDomain = scop->getOrInitEmptyDomain(ExitBB); in propagateDomainConstraintsToRegionExit()
677 if (scop->getRegion().getEntry() == BB) in getPredecessorDomainConstraints()
681 auto &RI = *scop->getRegion().getRegionInfo(); in getPredecessorDomainConstraints()
683 Loop *BBLoop = getFirstNonBoxedLoopFor(BB, LI, scop->getBoxedLoops()); in getPredecessorDomainConstraints()
719 isl::set PredBBDom = scop->getDomainConditions(PredBB); in getPredecessorDomainConstraints()
721 getFirstNonBoxedLoopFor(PredBB, LI, scop->getBoxedLoops()); in getPredecessorDomainConstraints()
731 int LoopDepth = scop->getRelativeLoopDepth(L); in addLoopBoundsToHeaderDomain()
735 assert(scop->isDomainDefined(HeaderBB)); in addLoopBoundsToHeaderDomain()
736 isl::set &HeaderBBDom = scop->getOrInitEmptyDomain(HeaderBB); in addLoopBoundsToHeaderDomain()
748 if (!scop->isDomainDefined(LatchBB)) in addLoopBoundsToHeaderDomain()
751 isl::set LatchBBDom = scop->getDomainConditions(LatchBB); in addLoopBoundsToHeaderDomain()
774 int LatchLoopDepth = scop->getRelativeLoopDepth(LI.getLoopFor(LatchBB)); in addLoopBoundsToHeaderDomain()
801 bool RequiresRTC = !scop->hasNSWAddRecForLoop(L); in addLoopBoundsToHeaderDomain()
813 const InvariantLoadsSetTy &RIL = scop->getRequiredInvariantLoads(); in buildInvariantEquivalenceClasses()
820 scop->addInvariantLoadMapping(LInst, ClassRep); in buildInvariantEquivalenceClasses()
825 scop->addInvariantEquivClass( in buildInvariantEquivalenceClasses()
832 bool IsOnlyNonAffineRegion = scop->isNonAffineSubRegion(R); in buildDomains()
835 int LD = scop->getRelativeLoopDepth(L); in buildDomains()
837 isl_set_universe(isl_space_set_alloc(scop->getIslCtx().get(), 0, LD + 1)); in buildDomains()
841 scop->setDomain(EntryBB, Domain); in buildDomains()
890 if (!scop->isNonAffineSubRegion(SubRegion)) { in buildDomainsWithBranchConstraints()
897 if (containsErrorBlock(RN, scop->getRegion(), &SD)) in buildDomainsWithBranchConstraints()
898 scop->notifyErrorBlock(); in buildDomainsWithBranchConstraints()
907 if (!scop->isDomainDefined(BB)) in buildDomainsWithBranchConstraints()
909 isl::set Domain = scop->getDomainConditions(BB); in buildDomainsWithBranchConstraints()
911 scop->updateMaxLoopDepth(unsignedFromIslSize(Domain.tuple_dim())); in buildDomainsWithBranchConstraints()
951 if (!scop->contains(SuccBB)) in buildDomainsWithBranchConstraints()
964 getFirstNonBoxedLoopFor(SuccBB, LI, scop->getBoxedLoops()); in buildDomainsWithBranchConstraints()
971 isl::set &SuccDomain = scop->getOrInitEmptyDomain(SuccBB); in buildDomainsWithBranchConstraints()
988 scop->invalidate(COMPLEXITY, DebugLoc()); in buildDomainsWithBranchConstraints()
1007 if (!scop->isNonAffineSubRegion(SubRegion)) { in propagateInvalidStmtDomains()
1013 bool ContainsErrorBlock = containsErrorBlock(RN, scop->getRegion(), &SD); in propagateInvalidStmtDomains()
1015 isl::set &Domain = scop->getOrInitEmptyDomain(BB); in propagateInvalidStmtDomains()
1044 if (!scop->contains(SuccBB)) in propagateInvalidStmtDomains()
1052 getFirstNonBoxedLoopFor(SuccBB, LI, scop->getBoxedLoops()); in propagateInvalidStmtDomains()
1070 scop->invalidate(COMPLEXITY, TI->getDebugLoc(), TI->getParent()); in propagateInvalidStmtDomains()
1092 if (!IsExitBlock && canSynthesize(PHI, *scop, &SE, Scope)) in buildPHIAccesses()
1102 ScopStmt *OpStmt = scop->getIncomingStmtFor(PHI->getOperandUse(u)); in buildPHIAccesses()
1178 Loop *L = getLoopSurroundingScop(*scop, LI); in buildSchedule()
1180 buildSchedule(scop->getRegion().getNode(), LoopStack); in buildSchedule()
1182 scop->setScheduleTree(LoopStack[0].Schedule); in buildSchedule()
1210 Loop *OuterScopLoop = getLoopSurroundingScop(*scop, LI); in buildSchedule()
1237 if (!scop->contains(L)) in buildSchedule()
1256 if (!scop->isNonAffineSubRegion(LocalRegion)) { in buildSchedule()
1266 for (auto *Stmt : scop->getStmtListFor(RN)) { in buildSchedule()
1304 scop->markDisableHeuristics(); in buildSchedule()
1308 isl::id IslLoopId = createIslLoopAttr(scop->getIslCtx(), L); in buildSchedule()
1326 if (scop->isEscaping(Inst)) in buildEscapingDependences()
1334 scop->addAssumption(AS.Kind, AS.Set, AS.Loc, AS.Sign, in addRecordedAssumptions()
1340 isl_set *Dom = scop->getDomainConditions(AS.BB).release(); in addRecordedAssumptions()
1359 scop->addAssumption(AS.Kind, isl::manage(S), AS.Loc, AS_RESTRICTION, AS.BB, in addRecordedAssumptions()
1371 bool InScop = scop->contains(CI); in addUserAssumptions()
1372 if (!InScop && !scop->isDominatedBy(DT, CI->getParent())) in addUserAssumptions()
1378 auto &R = scop->getRegion(); in addUserAssumptions()
1390 Param = scop->getRepresentingInvariantLoadSCEV(Param); in addUserAssumptions()
1391 if (scop->isParam(Param)) in addUserAssumptions()
1399 auto *Dom = InScop ? isl_set_copy(scop->getDomainConditions(BB).get()) in addUserAssumptions()
1400 : isl_set_copy(scop->getContext().get()); in addUserAssumptions()
1436 scop->getContext().intersect(isl::manage(AssumptionCtx)); in addUserAssumptions()
1437 scop->setContext(newContext); in addUserAssumptions()
1475 const InvariantLoadsSetTy &ScopRIL = scop->getRequiredInvariantLoads(); in buildAccessMultiDimFixed()
1480 if (!isAffineExpr(&scop->getRegion(), SurroundingLoop, Subscript, SE, in buildAccessMultiDimFixed()
1521 auto &InsnToMemAcc = scop->getInsnToMemAccMap(); in buildAccessMultiDimParam()
1547 scop->invalidate(DELINEARIZATION, Inst->getDebugLoc(), Inst->getParent()); in buildAccessMultiDimParam()
1566 const InvariantLoadsSetTy &ScopRIL = scop->getRequiredInvariantLoads(); in buildAccessMemIntrinsic()
1569 bool LengthIsAffine = isAffineExpr(&scop->getRegion(), SurroundingLoop, in buildAccessMemIntrinsic()
1716 isAffineExpr(&scop->getRegion(), SurroundingLoop, in buildAccessSingleDim()
1719 const InvariantLoadsSetTy &ScopRIL = scop->getRequiredInvariantLoads(); in buildAccessSingleDim()
1748 for (auto &Stmt : *scop) { in buildAccessFunctions()
1763 for (BasicBlock *BB : scop->getRegion().blocks()) { in buildAccessFunctions()
1771 !canSynthesize(Inst, *scop, &SE, L); in shouldModelInst()
1811 long BBIdx = scop->getNextStmtIdx(); in buildSequentialBlockStmts()
1819 scop->addScopStmt(BB, Name, SurroundingLoop, Instructions); in buildSequentialBlockStmts()
1826 scop->addScopStmt(BB, Name, SurroundingLoop, Instructions); in buildSequentialBlockStmts()
2000 long BBIdx = scop->getNextStmtIdx(); in buildEqivClassBlockStmts()
2008 scop->addScopStmt(BB, Name, L, std::move(InstList)); in buildEqivClassBlockStmts()
2019 scop->addScopStmt(BB, EpilogueName, L, {}); in buildEqivClassBlockStmts()
2023 if (scop->isNonAffineSubRegion(&SR)) { in buildStmts()
2026 getFirstNonBoxedLoopFor(SR.getEntry(), LI, scop->getBoxedLoops()); in buildStmts()
2030 long RIdx = scop->getNextStmtIdx(); in buildStmts()
2032 scop->addScopStmt(&SR, Name, SurroundingLoop, Instructions); in buildStmts()
2064 if (SD.isErrorBlock(BB, scop->getRegion())) in buildAccessFunctions()
2086 const InvariantLoadsSetTy &RIL = scop->getRequiredInvariantLoads(); in buildAccessFunctions()
2140 scop->addAccessFunction(Access); in addMemoryAccess()
2187 isl::union_set Accessed = scop->getAccesses().range(); in foldSizeConstantsToRight()
2189 for (auto Array : scop->arrays()) { in foldSizeConstantsToRight()
2266 for (auto &Access : scop->access_functions()) in foldSizeConstantsToRight()
2294 for (ScopStmt &Stmt : *scop) in updateAccessDimensionality()
2311 for (auto &Stmt : *scop) in updateAccessDimensionality()
2317 for (auto &Stmt : *scop) in foldAccessRelations()
2325 for (auto &Stmt : *scop) in assumeNoOutOfBounds()
2339 ScopStmt *Stmt = scop->getStmtFor(Inst); in ensureValueWrite()
2347 Stmt = scop->getLastStmtFor(Inst->getParent()); in ensureValueWrite()
2373 auto VUse = VirtualUse::create(scop.get(), UserStmt, Scope, V, false); in ensureValueRead()
2414 scop->getOrCreateScopArrayInfo(PHI, PHI->getType(), {}, in ensurePHIWrite()
2451 isl::id Id = isl::id::alloc(scop->getIslCtx(), Stmt.getBaseName(), &Stmt); in buildDomain()
2453 Stmt.Domain = scop->getDomainConditions(&Stmt); in buildDomain()
2569 auto &RIL = scop->getRequiredInvariantLoads(); in verifyInvariantLoads()
2571 assert(LI && scop->contains(LI)); in verifyInvariantLoads()
2574 for (ScopStmt &Stmt : *scop) in verifyInvariantLoads()
2576 scop->invalidate(INVARIANTLOAD, LI->getDebugLoc(), LI->getParent()); in verifyInvariantLoads()
2586 isl::union_map Writes = scop->getWrites(); in hoistInvariantLoads()
2587 for (ScopStmt &Stmt : *scop) { in hoistInvariantLoads()
2631 if (auto *BasePtrMA = scop->lookupBasePtrAccess(MA)) { in hasNonHoistableBasePtrInScop()
2638 return scop->contains(BasePtrInst); in hasNonHoistableBasePtrInScop()
2647 isl::set UserContext = isl::set(scop->getIslCtx(), UserContextStr.c_str()); in addUserContext()
2648 isl::space Space = scop->getParamSpace(); in addUserContext()
2662 scop->getContext().get_dim_name(isl::dim::param, i); in addUserContext()
2680 isl::set newContext = scop->getContext().intersect(UserContext); in addUserContext()
2681 scop->setContext(newContext); in addUserContext()
2721 auto &DL = scop->getFunction().getParent()->getDataLayout(); in getNonHoistableCtx()
2749 scop->addAssumption(INVARIANTLOAD, WrittenCtx, LI->getDebugLoc(), in getNonHoistableCtx()
2769 isAParameter(LInst->getPointerOperand(), scop->getFunction())) in canAlwaysBeHoisted()
2813 scop->invalidate(COMPLEXITY, AccInst->getDebugLoc(), AccInst->getParent()); in addInvariantLoads()
2827 for (const SCEV *Parameter : scop->parameters()) { in addInvariantLoads()
2833 isl::id ParamId = scop->getIdForParam(Parameter); in addInvariantLoads()
2866 MACtx = MACtx.gist_params(scop->getContext()); in addInvariantLoads()
2870 for (auto &IAClass : scop->invariantEquivClasses()) { in addInvariantLoads()
2913 scop->addInvariantEquivClass( in addInvariantLoads()
3005 for (InvariantEquivClassTy &EqClass : scop->InvariantEquivClasses) { in canonicalizeDynamicBasePtrs()
3009 findCanonicalArray(*scop, BasePtrAccesses); in canonicalizeDynamicBasePtrs()
3015 const ScopArrayInfo *BasePtrSAI = scop->getScopArrayInfoOrNull( in canonicalizeDynamicBasePtrs()
3026 if (isUsedForIndirectHoistedLoad(*scop, BasePtrSAI)) in canonicalizeDynamicBasePtrs()
3029 replaceBasePtrArrays(*scop, BasePtrSAI, CanonicalBasePtrSAI); in canonicalizeDynamicBasePtrs()
3054 scop->getPwAff(Size, nullptr, false, &RecordedAssumptions); in buildAccessRelations()
3056 auto *SAI = scop->getOrCreateScopArrayInfo(Access->getOriginalBaseAddr(), in buildAccessRelations()
3065 scop->getPwAff(Subscript, Stmt.getEntryBlock(), false, in buildAccessRelations()
3069 scop->addAccessData(Access); in buildAccessRelations()
3152 isl::union_set Domains = scop->getDomains(); in calculateMinMaxAccess()
3153 isl::union_map Accesses = isl::union_map::empty(scop->getIslCtx()); in calculateMinMaxAccess()
3163 LimitReached |= !buildMinMaxAccess(Set, MinMaxAccesses, *scop); in calculateMinMaxAccess()
3185 if (scop->getAliasGroups().size()) in buildAliasChecks()
3193 scop->invalidate(ALIASING, DebugLoc()); in buildAliasChecks()
3195 LLVM_DEBUG(dbgs() << "\n\nNOTE: Run time checks for " << scop->getNameStr() in buildAliasChecks()
3206 for (ScopStmt &Stmt : *scop) { in buildAliasGroupsForAccesses()
3258 if (!scop->hasFeasibleRuntimeContext()) in buildAliasGroups()
3262 IslMaxOperationsGuard MaxOpGuard(scop->getIslCtx().get(), OptComputeOut); in buildAliasGroups()
3267 if (isl_ctx_last_error(scop->getIslCtx().get()) == isl_error_quota) { in buildAliasGroups()
3268 scop->invalidate(COMPLEXITY, DebugLoc()); in buildAliasGroups()
3313 scop->invalidate(ALIASING, MA->getAccessInstruction()->getDebugLoc(), in buildAliasGroup()
3322 if (MemoryAccess *BasePtrMA = scop->lookupBasePtrAccess(MA)) in buildAliasGroup()
3323 scop->addRequiredInvariantLoad( in buildAliasGroup()
3348 scop->addAliasGroup(MinMaxAccessesReadWrite, MinMaxAccessesReadOnly); in buildAliasGroup()
3450 scop.reset(new Scop(R, SE, LI, DT, *SD.getDetectionContext(&R), ORE, in buildScop()
3458 const InvariantLoadsSetTy &RIL = scop->getRequiredInvariantLoads(); in buildScop()
3459 for (BasicBlock *BB : scop->getRegion().blocks()) { in buildScop()
3460 if (SD.isErrorBlock(*BB, scop->getRegion())) in buildScop()
3475 ArrayRef<ScopStmt *> List = scop->getStmtListFor(BB); in buildScop()
3490 if (!R.isTopLevelRegion() && !scop->hasSingleExitEdge()) { in buildScop()
3524 for (ScopStmt &Stmt : scop->Stmts) in buildScop()
3533 scop->removeStmtNotInDomainMap(); in buildScop()
3534 scop->simplifySCoP(false); in buildScop()
3535 if (scop->isEmpty()) { in buildScop()
3541 for (ScopStmt &Stmt : *scop) { in buildScop()
3552 if (!scop->hasFeasibleRuntimeContext()) { in buildScop()
3559 if (!scop->isProfitable(UnprofitableScalarAccs)) { in buildScop()
3560 scop->invalidate(PROFITABLE, DebugLoc()); in buildScop()
3570 scop->realignParams(); in buildScop()
3578 scop->simplifyContexts(); in buildScop()
3587 scop->simplifySCoP(true); in buildScop()
3591 if (!scop->hasFeasibleRuntimeContext()) { in buildScop()
3597 verifyUses(scop.get(), LI, DT); in buildScop()
3616 LLVM_DEBUG(dbgs() << *scop); in ScopBuilder()
3618 if (!scop->hasFeasibleRuntimeContext()) { in ScopBuilder()
3623 scop.reset(); in ScopBuilder()
3627 if (scop->getMaxLoopDepth() > 0) in ScopBuilder()