Lines Matching refs:forOp
56 getCleanupLoopLowerBound(AffineForOp forOp, unsigned unrollFactor, in getCleanupLoopLowerBound() argument
61 getTripCountMapAndOperands(forOp, &tripCountMap, &tripCountOperands); in getCleanupLoopLowerBound()
68 OpBuilder b(forOp); in getCleanupLoopLowerBound()
69 auto lbMap = forOp.getLowerBoundMap(); in getCleanupLoopLowerBound()
70 auto lb = b.create<AffineApplyOp>(forOp.getLoc(), lbMap, in getCleanupLoopLowerBound()
71 forOp.getLowerBoundOperands()); in getCleanupLoopLowerBound()
80 int64_t step = forOp.getStep(); in getCleanupLoopLowerBound()
87 b.create<AffineApplyOp>(forOp.getLoc(), bumpMap, tripCountOperands); in getCleanupLoopLowerBound()
114 static void replaceIterArgsAndYieldResults(AffineForOp forOp) { in replaceIterArgsAndYieldResults() argument
116 auto iterOperands = forOp.getIterOperands(); in replaceIterArgsAndYieldResults()
117 auto iterArgs = forOp.getRegionIterArgs(); in replaceIterArgsAndYieldResults()
122 auto outerResults = forOp.getResults(); in replaceIterArgsAndYieldResults()
123 auto innerResults = forOp.getBody()->getTerminator()->getOperands(); in replaceIterArgsAndYieldResults()
131 LogicalResult mlir::promoteIfSingleIteration(AffineForOp forOp) { in promoteIfSingleIteration() argument
132 Optional<uint64_t> tripCount = getConstantTripCount(forOp); in promoteIfSingleIteration()
136 if (forOp.getLowerBoundMap().getNumResults() != 1) in promoteIfSingleIteration()
140 auto iv = forOp.getInductionVar(); in promoteIfSingleIteration()
141 auto *parentBlock = forOp->getBlock(); in promoteIfSingleIteration()
143 if (forOp.hasConstantLowerBound()) { in promoteIfSingleIteration()
144 OpBuilder topBuilder(forOp->getParentOfType<func::FuncOp>().getBody()); in promoteIfSingleIteration()
146 forOp.getLoc(), forOp.getConstantLowerBound()); in promoteIfSingleIteration()
149 auto lbOperands = forOp.getLowerBoundOperands(); in promoteIfSingleIteration()
150 auto lbMap = forOp.getLowerBoundMap(); in promoteIfSingleIteration()
151 OpBuilder builder(forOp); in promoteIfSingleIteration()
157 builder.create<AffineApplyOp>(forOp.getLoc(), lbMap, lbOperands); in promoteIfSingleIteration()
163 replaceIterArgsAndYieldResults(forOp); in promoteIfSingleIteration()
167 forOp.getBody()->back().erase(); in promoteIfSingleIteration()
168 parentBlock->getOperations().splice(Block::iterator(forOp), in promoteIfSingleIteration()
169 forOp.getBody()->getOperations()); in promoteIfSingleIteration()
170 forOp.erase(); in promoteIfSingleIteration()
235 LogicalResult mlir::affineForOpBodySkew(AffineForOp forOp, in affineForOpBodySkew() argument
238 assert(forOp.getBody()->getOperations().size() == shifts.size() && in affineForOpBodySkew()
240 if (forOp.getBody()->begin() == std::prev(forOp.getBody()->end())) in affineForOpBodySkew()
247 auto mayBeConstTripCount = getConstantTripCount(forOp); in affineForOpBodySkew()
249 LLVM_DEBUG(forOp.emitRemark("non-constant trip count loop not handled")); in affineForOpBodySkew()
254 assert(isOpwiseShiftValid(forOp, shifts) && in affineForOpBodySkew()
257 int64_t step = forOp.getStep(); in affineForOpBodySkew()
265 forOp.emitWarning("not shifting because shifts are unrealistically large"); in affineForOpBodySkew()
274 for (auto &op : forOp.getBody()->without_terminator()) { in affineForOpBodySkew()
291 auto origLbMap = forOp.getLowerBoundMap(); in affineForOpBodySkew()
293 OpBuilder b(forOp); in affineForOpBodySkew()
309 opGroupQueue, /*offset=*/0, forOp, b); in affineForOpBodySkew()
316 opGroupQueue, /*offset=*/0, forOp, b); in affineForOpBodySkew()
346 opGroupQueue, /*offset=*/i, forOp, b); in affineForOpBodySkew()
353 forOp.erase(); in affineForOpBodySkew()
966 for (AffineForOp forOp : f.getOps<AffineForOp>()) { in getTileableBands()
968 getPerfectlyNestedLoops(band, forOp); in getTileableBands()
974 LogicalResult mlir::loopUnrollFull(AffineForOp forOp) { in loopUnrollFull() argument
975 Optional<uint64_t> mayBeConstantTripCount = getConstantTripCount(forOp); in loopUnrollFull()
981 return promoteIfSingleIteration(forOp); in loopUnrollFull()
982 return loopUnrollByFactor(forOp, tripCount); in loopUnrollFull()
989 LogicalResult mlir::loopUnrollUpToFactor(AffineForOp forOp, in loopUnrollUpToFactor() argument
991 Optional<uint64_t> mayBeConstantTripCount = getConstantTripCount(forOp); in loopUnrollUpToFactor()
994 return loopUnrollByFactor(forOp, mayBeConstantTripCount.value()); in loopUnrollUpToFactor()
995 return loopUnrollByFactor(forOp, unrollFactor); in loopUnrollUpToFactor()
1056 static LogicalResult generateCleanupLoopForUnroll(AffineForOp forOp, in generateCleanupLoopForUnroll() argument
1059 OpBuilder builder(forOp->getBlock(), std::next(Block::iterator(forOp))); in generateCleanupLoopForUnroll()
1060 auto cleanupForOp = cast<AffineForOp>(builder.clone(*forOp)); in generateCleanupLoopForUnroll()
1064 auto results = forOp.getResults(); in generateCleanupLoopForUnroll()
1075 getCleanupLoopLowerBound(forOp, unrollFactor, cleanupMap, cleanupOperands); in generateCleanupLoopForUnroll()
1085 forOp.setUpperBound(cleanupOperands, cleanupMap); in generateCleanupLoopForUnroll()
1092 AffineForOp forOp, uint64_t unrollFactor, in loopUnrollByFactor() argument
1096 Optional<uint64_t> mayBeConstantTripCount = getConstantTripCount(forOp); in loopUnrollByFactor()
1099 failed(promoteIfSingleIteration(forOp))) in loopUnrollByFactor()
1105 if (llvm::hasSingleElement(forOp.getBody()->getOperations())) in loopUnrollByFactor()
1114 if (getLargestDivisorOfTripCount(forOp) % unrollFactor != 0) { in loopUnrollByFactor()
1119 if (forOp.getLowerBoundMap().getNumResults() != 1 || in loopUnrollByFactor()
1120 forOp.getUpperBoundMap().getNumResults() != 1) in loopUnrollByFactor()
1122 if (failed(generateCleanupLoopForUnroll(forOp, unrollFactor))) in loopUnrollByFactor()
1127 ValueRange iterArgs(forOp.getRegionIterArgs()); in loopUnrollByFactor()
1128 auto yieldedValues = forOp.getBody()->getTerminator()->getOperands(); in loopUnrollByFactor()
1131 int64_t step = forOp.getStep(); in loopUnrollByFactor()
1132 forOp.setStep(step * unrollFactor); in loopUnrollByFactor()
1134 forOp.getBody(), forOp.getInductionVar(), unrollFactor, in loopUnrollByFactor()
1139 return b.create<AffineApplyOp>(forOp.getLoc(), bumpMap, iv); in loopUnrollByFactor()
1145 (void)promoteIfSingleIteration(forOp); in loopUnrollByFactor()
1149 LogicalResult mlir::loopUnrollJamUpToFactor(AffineForOp forOp, in loopUnrollJamUpToFactor() argument
1151 Optional<uint64_t> mayBeConstantTripCount = getConstantTripCount(forOp); in loopUnrollJamUpToFactor()
1154 return loopUnrollJamByFactor(forOp, mayBeConstantTripCount.value()); in loopUnrollJamUpToFactor()
1155 return loopUnrollJamByFactor(forOp, unrollJamFactor); in loopUnrollJamUpToFactor()
1160 static bool areInnerBoundsInvariant(AffineForOp forOp) { in areInnerBoundsInvariant() argument
1161 auto walkResult = forOp.walk([&](AffineForOp aForOp) { in areInnerBoundsInvariant()
1163 if (!forOp.isDefinedOutsideOfLoop(controlOperand)) in areInnerBoundsInvariant()
1200 LogicalResult mlir::loopUnrollJamByFactor(AffineForOp forOp, in loopUnrollJamByFactor() argument
1204 Optional<uint64_t> mayBeConstantTripCount = getConstantTripCount(forOp); in loopUnrollJamByFactor()
1207 failed(promoteIfSingleIteration(forOp))) in loopUnrollJamByFactor()
1213 if (llvm::hasSingleElement(forOp.getBody()->getOperations())) in loopUnrollJamByFactor()
1224 if (!areInnerBoundsInvariant(forOp)) in loopUnrollJamByFactor()
1229 jbg.walk(forOp); in loopUnrollJamByFactor()
1234 forOp.walk([&](AffineForOp aForOp) { in loopUnrollJamByFactor()
1241 if (forOp.getNumIterOperands() > 0) in loopUnrollJamByFactor()
1242 getSupportedReductions(forOp, reductions); in loopUnrollJamByFactor()
1246 if (getLargestDivisorOfTripCount(forOp) % unrollJamFactor != 0) { in loopUnrollJamByFactor()
1251 if (forOp.getLowerBoundMap().getNumResults() != 1 || in loopUnrollJamByFactor()
1252 forOp.getUpperBoundMap().getNumResults() != 1) in loopUnrollJamByFactor()
1254 if (failed(generateCleanupLoopForUnroll(forOp, unrollJamFactor))) in loopUnrollJamByFactor()
1267 OpBuilder builder(forOp.getContext()); in loopUnrollJamByFactor()
1287 if (oldForOp == forOp) in loopUnrollJamByFactor()
1288 forOp = newForOp; in loopUnrollJamByFactor()
1312 int64_t step = forOp.getStep(); in loopUnrollJamByFactor()
1313 forOp.setStep(step * unrollJamFactor); in loopUnrollJamByFactor()
1315 auto forOpIV = forOp.getInductionVar(); in loopUnrollJamByFactor()
1330 builder.create<AffineApplyOp>(forOp.getLoc(), bumpMap, forOpIV); in loopUnrollJamByFactor()
1359 if (forOp.getNumResults() > 0) { in loopUnrollJamByFactor()
1364 builder.setInsertionPointAfter(forOp); in loopUnrollJamByFactor()
1365 auto loc = forOp.getLoc(); in loopUnrollJamByFactor()
1366 unsigned oldNumResults = forOp.getNumResults() / unrollJamFactor; in loopUnrollJamByFactor()
1369 Value lhs = forOp.getResult(pos); in loopUnrollJamByFactor()
1373 rhs = forOp.getResult(i * oldNumResults + pos); in loopUnrollJamByFactor()
1383 forOp.getResult(pos).replaceAllUsesExcept(lhs, newOps); in loopUnrollJamByFactor()
1388 (void)promoteIfSingleIteration(forOp); in loopUnrollJamByFactor()
1557 AffineForOp mlir::sinkSequentialLoops(AffineForOp forOp) { in sinkSequentialLoops() argument
1559 getPerfectlyNestedLoops(loops, forOp); in sinkSequentialLoops()
1561 return forOp; in sinkSequentialLoops()
1602 return forOp; in sinkSequentialLoops()
1637 stripmineSink(AffineForOp forOp, uint64_t factor, in stripmineSink() argument
1639 auto originalStep = forOp.getStep(); in stripmineSink()
1641 forOp.setStep(scaledStep); in stripmineSink()
1643 OpBuilder b(forOp->getBlock(), std::next(Block::iterator(forOp))); in stripmineSink()
1646 auto lbMap = forOp.getLowerBoundMap(); in stripmineSink()
1647 SmallVector<Value, 4> lbOperands(forOp.getLowerBoundOperands()); in stripmineSink()
1648 augmentMapAndBounds(b, forOp.getInductionVar(), &lbMap, &lbOperands); in stripmineSink()
1651 auto ubMap = forOp.getUpperBoundMap(); in stripmineSink()
1652 SmallVector<Value, 4> ubOperands(forOp.getUpperBoundOperands()); in stripmineSink()
1653 augmentMapAndBounds(b, forOp.getInductionVar(), &ubMap, &ubOperands, in stripmineSink()
1656 auto iv = forOp.getInductionVar(); in stripmineSink()
1680 static AffineForOp stripmineSink(AffineForOp forOp, SizeType factor, in stripmineSink() argument
1686 auto res = stripmineSink(forOp, factor, ArrayRef<AffineForOp>(target)); in stripmineSink()
1829 void mlir::mapLoopToProcessorIds(scf::ForOp forOp, ArrayRef<Value> processorId, in mapLoopToProcessorIds() argument
1835 OpBuilder b(forOp); in mapLoopToProcessorIds()
1836 Location loc(forOp.getLoc()); in mapLoopToProcessorIds()
1838 bindSymbols(forOp.getContext(), lhs, rhs); in mapLoopToProcessorIds()
1851 loc, mulMap, ValueRange{linearIndex, forOp.getStep()}); in mapLoopToProcessorIds()
1853 loc, addMap, ValueRange{mulApplyOp, forOp.getLowerBound()}); in mapLoopToProcessorIds()
1854 forOp.setLowerBound(lb); in mapLoopToProcessorIds()
1856 Value step = forOp.getStep(); in mapLoopToProcessorIds()
1859 forOp.setStep(step); in mapLoopToProcessorIds()
1970 auto forOp = createCanonicalizedAffineForOp(b, loc, lbOperands, lbMaps[d], in generatePointWiseCopy() local
1973 copyNestRoot = forOp; in generatePointWiseCopy()
1975 b = OpBuilder::atBlockTerminator(forOp.getBody()); in generatePointWiseCopy()
1986 fastBufMapOperands.push_back(forOp.getInductionVar()); in generatePointWiseCopy()
1990 memIndices.push_back(forOp.getInductionVar()); in generatePointWiseCopy()
2540 AffineForOp forOp; in affineDataCopyGenerate() local
2541 if (llvm::DebugFlag && (forOp = dyn_cast<AffineForOp>(&*begin))) { in affineDataCopyGenerate()
2542 LLVM_DEBUG(forOp.emitRemark() in affineDataCopyGenerate()
2558 LogicalResult mlir::affineDataCopyGenerate(AffineForOp forOp, in affineDataCopyGenerate() argument
2562 return affineDataCopyGenerate(forOp.getBody()->begin(), in affineDataCopyGenerate()
2563 std::prev(forOp.getBody()->end()), copyOptions, in affineDataCopyGenerate()
2603 if (auto forOp = dyn_cast<AffineForOp>(op)) { in gatherLoopsInBlock() local
2604 depthToLoops[currLoopDepth].push_back(forOp); in gatherLoopsInBlock()
2605 gatherLoopsInBlock(forOp.getBody(), currLoopDepth + 1, depthToLoops); in gatherLoopsInBlock()