Home
last modified time | relevance | path

Searched refs:forOp (Results 1 – 25 of 55) sorted by relevance

123

/llvm-project-15.0.7/mlir/lib/Dialect/Affine/Transforms/
H A DPipelineDataTransfer.cpp35 void runOnAffineForOp(AffineForOp forOp);
67 auto *forBody = forOp.getBody(); in doubleBuffer()
84 OpBuilder bOuter(forOp); in doubleBuffer()
99 int64_t step = forOp.getStep(); in doubleBuffer()
120 bOuter.setInsertionPointAfter(forOp); in doubleBuffer()
134 getOperation().walk([&](AffineForOp forOp) { forOps.push_back(forOp); }); in runOnOperation() argument
135 for (auto forOp : forOps) in runOnOperation() local
136 runOnAffineForOp(forOp); in runOnOperation()
164 AffineForOp forOp, in findMatchingStartFinishInsts() argument
169 for (auto &op : *forOp.getBody()) { in findMatchingStartFinishInsts()
[all …]
H A DLoopUnroll.cpp76 f.walk([&](AffineForOp forOp) { in gatherInnermostLoops() argument
77 if (isInnermostAffineForOp(forOp)) in gatherInnermostLoops()
78 loops.push_back(forOp); in gatherInnermostLoops()
94 getOperation().walk([&](AffineForOp forOp) { in runOnOperation() argument
97 loops.push_back(forOp); in runOnOperation()
99 for (auto forOp : loops) in runOnOperation() local
100 (void)loopUnrollFull(forOp); in runOnOperation()
112 for (auto forOp : loops) in runOnOperation() local
125 return loopUnrollByFactor(forOp, getUnrollFactor(forOp)); in runOnAffineForOp()
128 return loopUnrollFull(forOp); in runOnAffineForOp()
[all …]
H A DAffineDataCopyGeneration.cpp133 AffineForOp forOp; in runOnBlock() local
135 if ((forOp = dyn_cast<AffineForOp>(&*it)) && copyNests.count(forOp) == 0) { in runOnBlock()
141 auto exceedsCapacity = [&](AffineForOp forOp) { in runOnBlock() argument
143 getMemoryFootprintBytes(forOp, in runOnBlock()
156 bool recurseInner = skipNonUnitStrideLoops ? forOp.getStep() != 1 in runOnBlock()
157 : exceedsCapacity(forOp); in runOnBlock()
161 runOnBlock(forOp.getBody(), copyNests); in runOnBlock()
223 if (auto forOp = dyn_cast<AffineForOp>(op)) in runOnOperation() local
224 (void)promoteIfSingleIteration(forOp); in runOnOperation()
H A DAffineLoopInvariantCodeMotion.cpp49 void runOnAffineForOp(AffineForOp forOp);
78 } else if (auto forOp = dyn_cast<AffineForOp>(op)) { in isOpLoopInvariant() local
79 if (!areAllOpsInTheBlockListInvariant(forOp.getLoopBody(), indVar, iterArgs, in isOpLoopInvariant()
197 void LoopInvariantCodeMotion::runOnAffineForOp(AffineForOp forOp) { in runOnAffineForOp() argument
198 auto *loopBody = forOp.getBody(); in runOnAffineForOp()
199 auto indVar = forOp.getInductionVar(); in runOnAffineForOp()
200 ValueRange iterArgs = forOp.getRegionIterArgs(); in runOnAffineForOp()
203 OpBuilder b(forOp.getOperation()); in runOnAffineForOp()
225 op->moveBefore(forOp); in runOnAffineForOp()
228 LLVM_DEBUG(forOp->print(llvm::dbgs() << "Modified loop\n")); in runOnAffineForOp()
H A DSuperVectorize.cpp891 static void eraseLoopNest(AffineForOp forOp) { in eraseLoopNest() argument
893 forOp.erase(); in eraseLoopNest()
923 auto loop = cast<AffineForOp>(forOp); in isVectorizableLoopPtrFactory()
1077 if (forOp && strategy->loopToVectorDim.count(forOp) == 0) in isUniformDefinition()
1276 if (isLoopVecDim && forOp.getNumIterOperands() > 0 && forOp.getStep() != 1) { in vectorizeAffineForOp()
1280 << forOp.getStep() << "\n"); in vectorizeAffineForOp()
1294 newStep = forOp.getStep(); in vectorizeAffineForOp()
1324 forOp.getLoc(), forOp.getLowerBoundOperands(), forOp.getLowerBoundMap(), in vectorizeAffineForOp()
1325 forOp.getUpperBoundOperands(), forOp.getUpperBoundMap(), newStep, in vectorizeAffineForOp()
1363 Value origInit = forOp.getOperand(forOp.getNumControlOperands() + i); in vectorizeAffineForOp()
[all …]
/llvm-project-15.0.7/mlir/lib/Dialect/SCF/Transforms/
H A DLoopSpecialization.cpp125 auto loc = forOp.getLoc(); in peelForLoop()
130 b.setInsertionPoint(forOp); in peelForLoop()
137 b.setInsertionPointAfter(forOp); in peelForLoop()
145 forOp, [&]() { forOp.getUpperBoundMutable().assign(splitBound); }); in peelForLoop()
154 Value mainIv = forOp.getInductionVar(); in rewriteAffineOpAfterPeeling()
158 Value step = forOp.getStep(); in rewriteAffineOpAfterPeeling()
160 forOp.walk([&](OpTy affineOp) { in rewriteAffineOpAfterPeeling()
178 Value previousUb = forOp.getUpperBound(); in peelAndCanonicalizeForLoop()
200 LogicalResult matchAndRewrite(ForOp forOp, in matchAndRewrite()
203 if (forOp->hasAttr(kPeeledLoopLabel)) in matchAndRewrite()
[all …]
H A DLoopPipelining.cpp37 ForOp forOp; member
86 forOp = op; in initializeLoopInfo()
103 options.getScheduleFn(forOp, schedule); in initializeLoopInfo()
117 if (forOp in initializeLoopInfo()
151 setValueMapping(forOp.getInductionVar(), iv, i); in emitPrologue()
239 Value newUb = forOp.getUpperBound(); in createKernelLoop()
244 rewriter.create<scf::ForOp>(forOp.getLoc(), forOp.getLowerBound(), newUb, in createKernelLoop()
443 ForOp forOp, PatternRewriter &rewriter) const { in returningMatchAndRewrite() argument
478 if (forOp->getNumResults() > 0) in returningMatchAndRewrite()
479 rewriter.replaceOp(forOp, returnValues); in returningMatchAndRewrite()
[all …]
H A DLoopCanonicalization.cpp33 static bool isShapePreserving(ForOp forOp, int64_t arg) { in isShapePreserving() argument
34 auto yieldOp = cast<YieldOp>(forOp.getBody()->getTerminator()); in isShapePreserving()
39 if (value == forOp.getRegionIterArgs()[arg]) in isShapePreserving()
50 .template Case<ForOp>([&](ForOp forOp) { in isShapePreserving() argument
93 if (!forOp) in matchAndRewrite()
95 if (!isShapePreserving(forOp, blockArg.getArgNumber() - 1)) in matchAndRewrite()
135 if (!forOp) in matchAndRewrite()
139 if (!isShapePreserving(forOp, resultNumber)) in matchAndRewrite()
159 lb = forOp.getLowerBound(); in matchAndRewrite()
160 ub = forOp.getUpperBound(); in matchAndRewrite()
[all …]
H A DForToWhile.cpp31 LogicalResult matchAndRewrite(ForOp forOp, in matchAndRewrite()
37 lcvTypes.push_back(forOp.getInductionVar().getType()); in matchAndRewrite()
38 lcvLocs.push_back(forOp.getInductionVar().getLoc()); in matchAndRewrite()
39 for (Value value : forOp.getInitArgs()) { in matchAndRewrite()
46 initArgs.push_back(forOp.getLowerBound()); in matchAndRewrite()
47 llvm::append_range(initArgs, forOp.getInitArgs()); in matchAndRewrite()
49 forOp->getAttrs()); in matchAndRewrite()
58 beforeBlock->getArgument(0), forOp.getUpperBound()); in matchAndRewrite()
79 for (auto &arg : llvm::make_early_inc_range(*forOp.getBody())) in matchAndRewrite()
93 for (const auto &arg : llvm::enumerate(forOp.getResults())) in matchAndRewrite()
[all …]
H A DBufferizableOpInterfaceImpl.cpp383 auto forOp = cast<scf::ForOp>(op); in bufferizesToMemoryRead() local
395 auto forOp = cast<scf::ForOp>(op); in getAliasingOpResult() local
396 return {forOp.getResultForOpOperand(opOperand)}; in getAliasingOpResult()
403 auto forOp = cast<scf::ForOp>(op); in bufferRelation() local
440 auto forOp = cast<scf::ForOp>(op); in resolveConflicts() local
477 auto forOp = cast<scf::ForOp>(op); in getBufferType() local
484 auto forOp = cast<scf::ForOp>(op); in bufferize() local
500 forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(), in bufferize()
501 forOp.getStep(), initArgs); in bufferize()
502 newForOp->setAttrs(forOp->getAttrs()); in bufferize()
[all …]
/llvm-project-15.0.7/mlir/lib/Dialect/Affine/Utils/
H A DLoopFusionUtils.cpp179 gatherLoadsAndStores(AffineForOp forOp, in gatherLoadsAndStores() argument
182 forOp.walk([&](Operation *op) { in gatherLoadsAndStores()
376 auto *parentOp = forOp->getParentOp(); in promoteSingleIterReductionLoop()
411 auto iv = forOp.getInductionVar(); in promoteSingleIterReductionLoop()
421 forOp.getBody()->back().erase(); in promoteSingleIterReductionLoop()
422 auto *parentBlock = forOp->getBlock(); in promoteSingleIterReductionLoop()
425 forOp.erase(); in promoteSingleIterReductionLoop()
447 sliceLoops.push_back(forOp); in fuseLoops()
497 for (auto &op : *forOp.getBody()) { in getLoopNestStats()
540 if (stats.loopMap.count(forOp) > 0) { in getComputeCostHelper()
[all …]
H A DLoopUtils.cpp68 OpBuilder b(forOp); in getCleanupLoopLowerBound()
146 forOp.getLoc(), forOp.getConstantLowerBound()); in promoteIfSingleIteration()
170 forOp.erase(); in promoteIfSingleIteration()
240 if (forOp.getBody()->begin() == std::prev(forOp.getBody()->end())) in affineForOpBodySkew()
293 OpBuilder b(forOp); in affineForOpBodySkew()
353 forOp.erase(); in affineForOpBodySkew()
1134 forOp.getBody(), forOp.getInductionVar(), unrollFactor, in loopUnrollByFactor()
1229 jbg.walk(forOp); in loopUnrollJamByFactor()
1561 return forOp; in sinkSequentialLoops()
1602 return forOp; in sinkSequentialLoops()
[all …]
/llvm-project-15.0.7/mlir/lib/Conversion/SCFToGPU/
H A DSCFToGPU.cpp74 return forOp.getLowerBoundOperands(); in getLowerBoundOperands()
79 return forOp.getUpperBoundOperands(); in getUpperBoundOperands()
92 return lowerAffineLowerBound(forOp, builder); in getOrEmitLowerBound()
98 return lowerAffineUpperBound(forOp, builder); in getOrEmitUpperBound()
109 Region &limit = forOp.getRegion(); in checkAffineLoopNestMappableImpl()
111 Operation *nested = &forOp.getBody()->front(); in checkAffineLoopNestMappableImpl()
114 return forOp.emitError( in checkAffineLoopNestMappableImpl()
123 auto begin = forOp.getBody()->begin(), end = forOp.getBody()->end(); in checkAffineLoopNestMappableImpl()
127 if (!(forOp = dyn_cast<AffineForOp>(nested))) in checkAffineLoopNestMappableImpl()
184 OpBuilder builder(forOp.getOperation()); in collectBounds()
[all …]
/llvm-project-15.0.7/mlir/lib/Dialect/Affine/Analysis/
H A DLoopAnalysis.cpp36 AffineForOp forOp, AffineMap *tripCountMap, in getTripCountMapAndOperands() argument
38 MLIRContext *context = forOp.getContext(); in getTripCountMapAndOperands()
39 int64_t step = forOp.getStep(); in getTripCountMapAndOperands()
41 if (forOp.hasConstantBounds()) { in getTripCountMapAndOperands()
42 int64_t lb = forOp.getConstantLowerBound(); in getTripCountMapAndOperands()
43 int64_t ub = forOp.getConstantUpperBound(); in getTripCountMapAndOperands()
51 auto lbMap = forOp.getLowerBoundMap(); in getTripCountMapAndOperands()
52 auto ubMap = forOp.getUpperBoundMap(); in getTripCountMapAndOperands()
269 auto *forOp = loop.getOperation(); in isVectorizableLoopBodyWithOpCond() local
284 regions.match(forOp, &regionsMatched); in isVectorizableLoopBodyWithOpCond()
[all …]
H A DAffineAnalysis.cpp82 unsigned numIterArgs = forOp.getNumIterOperands(); in getSupportedReductions()
96 bool mlir::isLoopParallel(AffineForOp forOp, in isLoopParallel() argument
98 unsigned numIterArgs = forOp.getNumIterOperands(); in isLoopParallel()
115 return isLoopMemoryParallel(forOp); in isLoopParallel()
134 bool mlir::isLoopMemoryParallel(AffineForOp forOp) { in isLoopMemoryParallel() argument
136 if (llvm::any_of(forOp.getResultTypes(), in isLoopMemoryParallel()
167 unsigned depth = getNestingDepth(forOp) + 1; in isLoopMemoryParallel()
249 forOps.push_back(forOp); in getIndexSet()
339 Block *commonBlock = forOp.getBody(); in getCommonBlock()
665 AffineForOp forOp, unsigned maxLoopDepth, in getDependenceComponents() argument
[all …]
/llvm-project-15.0.7/mlir/lib/Dialect/Linalg/Transforms/
H A DHoistPadding.cpp244 if (backwardSlice.contains(forOp)) in HoistingAnalysis()
245 packingLoops.push_back(forOp); in HoistingAnalysis()
306 if (auto forOp = dyn_cast<scf::ForOp>(op)) { in dropNonIndexDependencies() local
351 for (auto forOp : packingLoops) { in getPackedTensorSizes() local
383 MLIRContext *ctx = forOp->getContext(); in buildLoopIterationCount()
390 Value ivVal = forOp.getInductionVar(), lbVal = forOp.getLowerBound(), in buildLoopIterationCount()
391 stepVal = forOp.getStep(); in buildLoopIterationCount()
392 auto loc = forOp->getLoc(); in buildLoopIterationCount()
459 auto forOp = dyn_cast<scf::ForOp>(op); in hoistPaddingOnTensors() local
460 if (!forOp) { in hoistPaddingOnTensors()
[all …]
H A DHoisting.cpp229 if (!forOp.isDefinedOutsideOfLoop(operand)) in getLoopInvariantTransferWriteOpDefining()
270 << *forOp.getOperation() in hoistReadWrite()
277 forOp.moveOutOfLoop(read.extractSliceOp); in hoistReadWrite()
280 forOp.moveOutOfLoop(read.transferReadOp); in hoistReadWrite()
289 forOp.getInitArgs()[initArgNumber]); in hoistReadWrite()
292 forOp.getInitArgs()[initArgNumber]); in hoistReadWrite()
296 write.insertSliceOp->moveAfter(forOp); in hoistReadWrite()
297 write.transferWriteOp->moveAfter(forOp); in hoistReadWrite()
353 func.walk([&](scf::ForOp forOp) { in hoistRedundantVectorTransfersOnTensor() argument
368 [&forOp](Value index) { in hoistRedundantVectorTransfersOnTensor()
[all …]
/llvm-project-15.0.7/mlir/lib/Dialect/SCF/Utils/
H A DUtils.cpp357 forOp.erase(); in promoteIfSingleIteration()
429 OpBuilder boundsBuilder(forOp); in loopUnrollByFactor()
430 auto loc = forOp.getLoc(); in loopUnrollByFactor()
431 auto step = forOp.getStep(); in loopUnrollByFactor()
519 forOp.setStep(stepUnrolled); in loopUnrollByFactor()
525 forOp.getBody(), forOp.getInductionVar(), unrollFactor, in loopUnrollByFactor()
840 OpBuilder b(forOp); in stripmineSink()
841 forOp.setStep(b.create<arith::MulIOp>(forOp.getLoc(), originalStep, factor)); in stripmineSink()
944 auto forOp = forOps[i]; in extractFixedOuterLoops() local
945 OpBuilder builder(forOp); in extractFixedOuterLoops()
[all …]
/llvm-project-15.0.7/mlir/include/mlir/Dialect/Affine/
H A DLoopUtils.h42 LogicalResult loopUnrollFull(AffineForOp forOp);
49 AffineForOp forOp, uint64_t unrollFactor,
54 LogicalResult loopUnrollUpToFactor(AffineForOp forOp, uint64_t unrollFactor);
71 LogicalResult loopUnrollJamByFactor(AffineForOp forOp,
76 LogicalResult loopUnrollJamUpToFactor(AffineForOp forOp,
81 LogicalResult promoteIfSingleIteration(AffineForOp forOp);
92 LogicalResult affineForOpBodySkew(AffineForOp forOp, ArrayRef<uint64_t> shifts,
139 AffineForOp sinkSequentialLoops(AffineForOp forOp);
190 LogicalResult affineDataCopyGenerate(AffineForOp forOp,
262 void mapLoopToProcessorIds(scf::ForOp forOp, ArrayRef<Value> processorId,
/llvm-project-15.0.7/mlir/lib/Conversion/SCFToControlFlow/
H A DSCFToControlFlow.cpp101 LogicalResult matchAndRewrite(ForOp forOp,
292 Location loc = forOp.getLoc(); in matchAndRewrite()
308 auto *lastBodyBlock = &forOp.getRegion().back(); in matchAndRewrite()
317 auto step = forOp.getStep(); in matchAndRewrite()
330 Value lowerBound = forOp.getLowerBound(); in matchAndRewrite()
331 Value upperBound = forOp.getUpperBound(); in matchAndRewrite()
339 auto iterOperands = forOp.getIterOperands(); in matchAndRewrite()
467 ivs.push_back(forOp.getInductionVar()); in matchAndRewrite()
468 auto iterRange = forOp.getRegionIterArgs(); in matchAndRewrite()
474 loopResults.assign(forOp.result_begin(), forOp.result_end()); in matchAndRewrite()
[all …]
/llvm-project-15.0.7/mlir/test/lib/Dialect/SCF/
H A DTestSCFUtils.cpp48 func.walk([&](scf::ForOp forOp) { in runOnOperation() argument
49 if (forOp.getNumResults() == 0) in runOnOperation()
51 auto newInitValues = forOp.getInitArgs(); in runOnOperation()
65 OpBuilder b(forOp); in runOnOperation()
66 replaceLoopWithNewYields(b, forOp, newInitValues, fn); in runOnOperation()
132 getSchedule(scf::ForOp forOp, in getSchedule()
134 if (!forOp->hasAttr(kTestPipeliningLoopMarker)) in getSchedule()
136 schedule.resize(forOp.getBody()->getOperations().size() - 1); in getSchedule()
137 forOp.walk([&schedule](Operation *op) { in getSchedule()
/llvm-project-15.0.7/mlir/include/mlir/Dialect/Affine/Analysis/
H A DLoopAnalysis.h37 void getTripCountMapAndOperands(AffineForOp forOp, AffineMap *map,
43 Optional<uint64_t> getConstantTripCount(AffineForOp forOp);
48 uint64_t getLargestDivisorOfTripCount(AffineForOp forOp);
84 bool isOpwiseShiftValid(AffineForOp forOp, ArrayRef<uint64_t> shifts);
H A DAffineAnalysis.h46 AffineForOp forOp, SmallVectorImpl<LoopReduction> &supportedReductions);
52 AffineForOp forOp,
60 bool isLoopMemoryParallel(AffineForOp forOp);
184 AffineForOp forOp, unsigned maxLoopDepth,
/llvm-project-15.0.7/mlir/test/lib/Dialect/Affine/
H A DTestLoopPermutation.cpp53 getOperation()->walk([&](AffineForOp forOp) { forOps.push_back(forOp); }); in runOnOperation() argument
55 for (auto forOp : forOps) { in runOnOperation() local
58 getPerfectlyNestedLoops(nest, forOp); in runOnOperation()
/llvm-project-15.0.7/mlir/include/mlir/Dialect/SCF/Transforms/
H A DPatterns.h39 LogicalResult matchAndRewrite(ForOp forOp, in matchAndRewrite() argument
41 return returningMatchAndRewrite(forOp, rewriter); in matchAndRewrite()
44 FailureOr<ForOp> returningMatchAndRewrite(ForOp forOp,

123