149e37000SMatthias Springer //===- BufferizableOpInterfaceImpl.cpp - Impl. of BufferizableOpInterface -===//
249e37000SMatthias Springer //
349e37000SMatthias Springer // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
449e37000SMatthias Springer // See https://llvm.org/LICENSE.txt for license information.
549e37000SMatthias Springer // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
649e37000SMatthias Springer //
749e37000SMatthias Springer //===----------------------------------------------------------------------===//
849e37000SMatthias Springer 
949e37000SMatthias Springer #include "mlir/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.h"
1049e37000SMatthias Springer #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
1149e37000SMatthias Springer #include "mlir/Dialect/MemRef/IR/MemRef.h"
1271bbb78bSMatthias Springer #include "mlir/Dialect/SCF/SCF.h"
1349e37000SMatthias Springer #include "mlir/Dialect/Tensor/IR/Tensor.h"
1449e37000SMatthias Springer #include "mlir/IR/Dialect.h"
1549e37000SMatthias Springer #include "mlir/IR/Operation.h"
1649e37000SMatthias Springer 
1749e37000SMatthias Springer using namespace mlir;
1849e37000SMatthias Springer using namespace mlir::bufferization;
1949e37000SMatthias Springer using namespace mlir::tensor;
2049e37000SMatthias Springer 
2149e37000SMatthias Springer namespace mlir {
2249e37000SMatthias Springer namespace tensor {
2349e37000SMatthias Springer namespace {
2449e37000SMatthias Springer 
2549e37000SMatthias Springer struct CastOpInterface
2649e37000SMatthias Springer     : public BufferizableOpInterface::ExternalModel<CastOpInterface,
2749e37000SMatthias Springer                                                     tensor::CastOp> {
2849e37000SMatthias Springer   bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
2949e37000SMatthias Springer                               const BufferizationState &state) const {
3049e37000SMatthias Springer     return false;
3149e37000SMatthias Springer   }
3249e37000SMatthias Springer 
3349e37000SMatthias Springer   bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand,
3449e37000SMatthias Springer                                const BufferizationState &state) const {
3549e37000SMatthias Springer     return false;
3649e37000SMatthias Springer   }
3749e37000SMatthias Springer 
3849e37000SMatthias Springer   OpResult getAliasingOpResult(Operation *op, OpOperand &opOperand,
3949e37000SMatthias Springer                                const BufferizationState &state) const {
4049e37000SMatthias Springer     return op->getResult(0);
4149e37000SMatthias Springer   }
4249e37000SMatthias Springer 
4349e37000SMatthias Springer   BufferRelation bufferRelation(Operation *op, OpResult opResult,
4449e37000SMatthias Springer                                 const BufferizationState &state) const {
4549e37000SMatthias Springer     return BufferRelation::Equivalent;
4649e37000SMatthias Springer   }
4749e37000SMatthias Springer 
4849e37000SMatthias Springer   LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
4949e37000SMatthias Springer                           const BufferizationState &state) const {
5049e37000SMatthias Springer     auto castOp = cast<tensor::CastOp>(op);
5149e37000SMatthias Springer 
5249e37000SMatthias Springer     // The result buffer still has the old (pre-cast) type.
5349e37000SMatthias Springer     FailureOr<Value> resultBuffer =
5449e37000SMatthias Springer         state.getBuffer(rewriter, castOp->getOpOperand(0) /*source*/);
5549e37000SMatthias Springer     if (failed(resultBuffer))
5649e37000SMatthias Springer       return failure();
5749e37000SMatthias Springer     auto sourceMemRefType = resultBuffer->getType().cast<BaseMemRefType>();
5849e37000SMatthias Springer     Attribute memorySpace = sourceMemRefType.getMemorySpace();
5949e37000SMatthias Springer     TensorType resultTensorType =
6049e37000SMatthias Springer         castOp.getResult().getType().cast<TensorType>();
6149e37000SMatthias Springer     MemRefLayoutAttrInterface layout;
6249e37000SMatthias Springer 
6349e37000SMatthias Springer     if (auto rankedMemRefType = sourceMemRefType.dyn_cast<MemRefType>())
6449e37000SMatthias Springer       if (resultTensorType.isa<RankedTensorType>())
6549e37000SMatthias Springer         layout = rankedMemRefType.getLayout();
6649e37000SMatthias Springer 
6749e37000SMatthias Springer     // Compute the new memref type.
68*26852423SMatthias Springer     Type resultMemRefType = getMemRefType(resultTensorType, state.getOptions(),
69*26852423SMatthias Springer                                           layout, memorySpace);
7049e37000SMatthias Springer 
7149e37000SMatthias Springer     // Replace the op with a memref.cast.
7249e37000SMatthias Springer     assert(memref::CastOp::areCastCompatible(resultBuffer->getType(),
7349e37000SMatthias Springer                                              resultMemRefType) &&
7449e37000SMatthias Springer            "CallOp::bufferize: cast incompatible");
7549e37000SMatthias Springer     replaceOpWithNewBufferizedOp<memref::CastOp>(rewriter, op, resultMemRefType,
7649e37000SMatthias Springer                                                  *resultBuffer);
7749e37000SMatthias Springer 
7849e37000SMatthias Springer     return success();
7949e37000SMatthias Springer   }
8049e37000SMatthias Springer };
8149e37000SMatthias Springer 
8249e37000SMatthias Springer /// Bufferization of tensor.dim. Replace with memref.dim.
8349e37000SMatthias Springer struct DimOpInterface
8449e37000SMatthias Springer     : public BufferizableOpInterface::ExternalModel<DimOpInterface,
8549e37000SMatthias Springer                                                     tensor::DimOp> {
8649e37000SMatthias Springer   bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
8749e37000SMatthias Springer                               const BufferizationState &state) const {
8849e37000SMatthias Springer     return true;
8949e37000SMatthias Springer   }
9049e37000SMatthias Springer 
9149e37000SMatthias Springer   bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand,
9249e37000SMatthias Springer                                const BufferizationState &state) const {
9349e37000SMatthias Springer     return false;
9449e37000SMatthias Springer   }
9549e37000SMatthias Springer 
9649e37000SMatthias Springer   OpResult getAliasingOpResult(Operation *op, OpOperand &opOperand,
9749e37000SMatthias Springer                                const BufferizationState &state) const {
9849e37000SMatthias Springer     return OpResult();
9949e37000SMatthias Springer   }
10049e37000SMatthias Springer 
10149e37000SMatthias Springer   LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
10249e37000SMatthias Springer                           const BufferizationState &state) const {
10349e37000SMatthias Springer     auto dimOp = cast<tensor::DimOp>(op);
10449e37000SMatthias Springer     Value v = *state.getBuffer(rewriter, dimOp->getOpOperand(0) /*source*/);
10549e37000SMatthias Springer     replaceOpWithNewBufferizedOp<memref::DimOp>(rewriter, op, v, dimOp.index());
10649e37000SMatthias Springer     return success();
10749e37000SMatthias Springer   }
10849e37000SMatthias Springer };
10949e37000SMatthias Springer 
11049e37000SMatthias Springer /// Bufferization of tensor.extract_slice. Replace with memref.subview.
11149e37000SMatthias Springer struct ExtractSliceOpInterface
11249e37000SMatthias Springer     : public BufferizableOpInterface::ExternalModel<ExtractSliceOpInterface,
11349e37000SMatthias Springer                                                     tensor::ExtractSliceOp> {
11449e37000SMatthias Springer   bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
11549e37000SMatthias Springer                               const BufferizationState &state) const {
11649e37000SMatthias Springer     return false;
11749e37000SMatthias Springer   }
11849e37000SMatthias Springer 
11949e37000SMatthias Springer   bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand,
12049e37000SMatthias Springer                                const BufferizationState &state) const {
12149e37000SMatthias Springer     return false;
12249e37000SMatthias Springer   }
12349e37000SMatthias Springer 
12449e37000SMatthias Springer   OpResult getAliasingOpResult(Operation *op, OpOperand &opOperand,
12549e37000SMatthias Springer                                const BufferizationState &state) const {
12649e37000SMatthias Springer     return &opOperand == &op->getOpOperand(0) /*source*/
12749e37000SMatthias Springer                ? op->getResult(0)
12849e37000SMatthias Springer                : OpResult();
12949e37000SMatthias Springer   }
13049e37000SMatthias Springer 
13149e37000SMatthias Springer   BufferRelation bufferRelation(Operation *op, OpResult opResult,
13249e37000SMatthias Springer                                 const BufferizationState &state) const {
13349e37000SMatthias Springer     return BufferRelation::None;
13449e37000SMatthias Springer   }
13549e37000SMatthias Springer 
13649e37000SMatthias Springer   LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
13749e37000SMatthias Springer                           const BufferizationState &state) const {
13849e37000SMatthias Springer     auto extractSliceOp = cast<tensor::ExtractSliceOp>(op);
13949e37000SMatthias Springer     Location loc = extractSliceOp.getLoc();
14049e37000SMatthias Springer     Value srcMemref =
14149e37000SMatthias Springer         *state.getBuffer(rewriter, extractSliceOp->getOpOperand(0) /*source*/,
14249e37000SMatthias Springer                          /*forceInPlace=*/true);
14349e37000SMatthias Springer     auto srcMemrefType = srcMemref.getType().cast<MemRefType>();
14449e37000SMatthias Springer     auto dstTensorType =
14549e37000SMatthias Springer         extractSliceOp.result().getType().cast<RankedTensorType>();
14649e37000SMatthias Springer 
14749e37000SMatthias Springer     // If not inplaceable, alloc.
14849e37000SMatthias Springer     bool inplace = state.isInPlace(extractSliceOp->getOpOperand(0));
14949e37000SMatthias Springer     Value alloc;
15049e37000SMatthias Springer     if (!inplace) {
15149e37000SMatthias Springer       FailureOr<Value> allocOrFailure =
15249e37000SMatthias Springer           createAlloc(rewriter, loc, extractSliceOp.result(),
15349e37000SMatthias Springer                       state.getOptions().createDeallocs, state.getOptions());
15449e37000SMatthias Springer       if (failed(allocOrFailure))
15549e37000SMatthias Springer         return failure();
15649e37000SMatthias Springer       alloc = *allocOrFailure;
15749e37000SMatthias Springer     }
15849e37000SMatthias Springer 
15949e37000SMatthias Springer     // Expand offsets, sizes and strides to the full rank to handle the
16049e37000SMatthias Springer     // rank-reducing case.
16149e37000SMatthias Springer     SmallVector<OpFoldResult> mixedOffsets = extractSliceOp.getMixedOffsets();
16249e37000SMatthias Springer     SmallVector<OpFoldResult> mixedSizes = extractSliceOp.getMixedSizes();
16349e37000SMatthias Springer     SmallVector<OpFoldResult> mixedStrides = extractSliceOp.getMixedStrides();
16449e37000SMatthias Springer     OffsetSizeAndStrideOpInterface::expandToRank(
16549e37000SMatthias Springer         srcMemref, mixedOffsets, mixedSizes, mixedStrides,
16649e37000SMatthias Springer         [&](Value target, int64_t dim) -> OpFoldResult {
16749e37000SMatthias Springer           auto shapedType = target.getType().cast<ShapedType>();
16849e37000SMatthias Springer           if (shapedType.isDynamicDim(dim))
16949e37000SMatthias Springer             return rewriter.create<memref::DimOp>(loc, target, dim).result();
17049e37000SMatthias Springer           return rewriter.getIndexAttr(shapedType.getDimSize(dim));
17149e37000SMatthias Springer         });
17249e37000SMatthias Springer     // Bufferize to subview.
17349e37000SMatthias Springer     auto subviewMemRefType = memref::SubViewOp::inferRankReducedResultType(
17449e37000SMatthias Springer                                  dstTensorType.getRank(), srcMemrefType,
17549e37000SMatthias Springer                                  mixedOffsets, mixedSizes, mixedStrides)
17649e37000SMatthias Springer                                  .cast<MemRefType>();
17749e37000SMatthias Springer     Value subView = rewriter.create<memref::SubViewOp>(
17849e37000SMatthias Springer         loc, subviewMemRefType, srcMemref, mixedOffsets, mixedSizes,
17949e37000SMatthias Springer         mixedStrides);
18049e37000SMatthias Springer 
18149e37000SMatthias Springer     // If not inplaceable, copy.
18249e37000SMatthias Springer     if (!inplace) {
18349e37000SMatthias Springer       // Do not copy if the copied data is never read.
18449e37000SMatthias Springer       if (state.isValueRead(extractSliceOp.result()))
18549e37000SMatthias Springer         if (failed(createMemCpy(rewriter, extractSliceOp.getLoc(), subView,
18649e37000SMatthias Springer                                 alloc, state.getOptions())))
18749e37000SMatthias Springer           return failure();
18849e37000SMatthias Springer       subView = alloc;
18949e37000SMatthias Springer     }
19049e37000SMatthias Springer 
19149e37000SMatthias Springer     replaceOpWithBufferizedValues(rewriter, op, subView);
19249e37000SMatthias Springer     return success();
19349e37000SMatthias Springer   }
19449e37000SMatthias Springer };
19549e37000SMatthias Springer 
19649e37000SMatthias Springer /// Bufferization of tensor.extract. Replace with memref.load.
19749e37000SMatthias Springer struct ExtractOpInterface
19849e37000SMatthias Springer     : public BufferizableOpInterface::ExternalModel<ExtractOpInterface,
19949e37000SMatthias Springer                                                     tensor::ExtractOp> {
20049e37000SMatthias Springer   bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
20149e37000SMatthias Springer                               const BufferizationState &state) const {
20249e37000SMatthias Springer     return true;
20349e37000SMatthias Springer   }
20449e37000SMatthias Springer 
20549e37000SMatthias Springer   bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand,
20649e37000SMatthias Springer                                const BufferizationState &state) const {
20749e37000SMatthias Springer     return false;
20849e37000SMatthias Springer   }
20949e37000SMatthias Springer 
21049e37000SMatthias Springer   OpResult getAliasingOpResult(Operation *op, OpOperand &opOperand,
21149e37000SMatthias Springer                                const BufferizationState &state) const {
21249e37000SMatthias Springer     return OpResult();
21349e37000SMatthias Springer   }
21449e37000SMatthias Springer 
21549e37000SMatthias Springer   LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
21649e37000SMatthias Springer                           const BufferizationState &state) const {
21749e37000SMatthias Springer     auto extractOp = cast<tensor::ExtractOp>(op);
21849e37000SMatthias Springer     Value srcMemref =
21949e37000SMatthias Springer         *state.getBuffer(rewriter, extractOp->getOpOperand(0) /*tensor*/);
22049e37000SMatthias Springer     replaceOpWithNewBufferizedOp<memref::LoadOp>(rewriter, op, srcMemref,
22149e37000SMatthias Springer                                                  extractOp.indices());
22249e37000SMatthias Springer     return success();
22349e37000SMatthias Springer   }
22449e37000SMatthias Springer };
22549e37000SMatthias Springer 
226d581c94dSMatthias Springer // Implements backtracking to traverse indices of the output buffer while
227d581c94dSMatthias Springer // iterating over op.elements().
228d581c94dSMatthias Springer static void createStores(RewriterBase &rewriter, Location loc, int dim,
229d581c94dSMatthias Springer                          Value buffer, ArrayRef<int64_t> shape,
230d581c94dSMatthias Springer                          ArrayRef<Value> constants,
231d581c94dSMatthias Springer                          OperandRange::iterator &elementIt,
232d581c94dSMatthias Springer                          SmallVectorImpl<Value> &indices) {
233d581c94dSMatthias Springer   if (dim == static_cast<int>(shape.size()) - 1) {
234d581c94dSMatthias Springer     for (int i = 0; i < shape.back(); ++i) {
235d581c94dSMatthias Springer       indices.back() = constants[i];
236d581c94dSMatthias Springer       rewriter.create<memref::StoreOp>(loc, *elementIt, buffer, indices);
237d581c94dSMatthias Springer       ++elementIt;
238d581c94dSMatthias Springer     }
239d581c94dSMatthias Springer     return;
240d581c94dSMatthias Springer   }
241d581c94dSMatthias Springer   for (int i = 0; i < shape[dim]; ++i) {
242d581c94dSMatthias Springer     indices[dim] = constants[i];
243d581c94dSMatthias Springer     createStores(rewriter, loc, dim + 1, buffer, shape, constants, elementIt,
244d581c94dSMatthias Springer                  indices);
245d581c94dSMatthias Springer   }
246d581c94dSMatthias Springer }
247d581c94dSMatthias Springer 
248d581c94dSMatthias Springer /// Bufferization of tensor.from_elements.
249d581c94dSMatthias Springer struct FromElementsOpInterface
250d581c94dSMatthias Springer     : public BufferizableOpInterface::ExternalModel<FromElementsOpInterface,
251d581c94dSMatthias Springer                                                     tensor::FromElementsOp> {
252d581c94dSMatthias Springer   LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
253d581c94dSMatthias Springer                           const BufferizationState &state) const {
254d581c94dSMatthias Springer     auto fromElementsOp = cast<tensor::FromElementsOp>(op);
255d581c94dSMatthias Springer 
256d581c94dSMatthias Springer     // Allocate a buffer for the result.
257d581c94dSMatthias Springer     Location loc = op->getLoc();
258d581c94dSMatthias Springer     auto tensorType = fromElementsOp.getType().cast<RankedTensorType>();
259d581c94dSMatthias Springer     auto shape = tensorType.getShape();
260*26852423SMatthias Springer     MemRefType resultType = getContiguousMemRefType(tensorType);
261d581c94dSMatthias Springer     FailureOr<Value> maybeBuffer =
262d581c94dSMatthias Springer         createAlloc(rewriter, loc, resultType, {},
263d581c94dSMatthias Springer                     /*deallocMemref=*/state.getOptions().createDeallocs,
264d581c94dSMatthias Springer                     state.getOptions());
265d581c94dSMatthias Springer     if (failed(maybeBuffer))
266d581c94dSMatthias Springer       return failure();
267d581c94dSMatthias Springer     Value buffer = *maybeBuffer;
268d581c94dSMatthias Springer 
269d581c94dSMatthias Springer     // Case: tensor<0xelem_type>.
270d581c94dSMatthias Springer     if (fromElementsOp.elements().empty()) {
271d581c94dSMatthias Springer       replaceOpWithBufferizedValues(rewriter, op, buffer);
272d581c94dSMatthias Springer       return success();
273d581c94dSMatthias Springer     }
274d581c94dSMatthias Springer 
275d581c94dSMatthias Springer     // Case: tensor<elem_type>.
276d581c94dSMatthias Springer     if (shape.empty()) {
277d581c94dSMatthias Springer       rewriter.create<memref::StoreOp>(loc, fromElementsOp.elements().front(),
278d581c94dSMatthias Springer                                        buffer);
279d581c94dSMatthias Springer       replaceOpWithBufferizedValues(rewriter, op, buffer);
280d581c94dSMatthias Springer       return success();
281d581c94dSMatthias Springer     }
282d581c94dSMatthias Springer 
283d581c94dSMatthias Springer     // Create constants for the range of possible indices [0, max{shape_i}).
284d581c94dSMatthias Springer     auto maxDim = *std::max_element(shape.begin(), shape.end());
285d581c94dSMatthias Springer     SmallVector<Value, 2> constants;
286d581c94dSMatthias Springer     constants.reserve(maxDim);
287d581c94dSMatthias Springer     for (int i = 0; i < maxDim; ++i)
288d581c94dSMatthias Springer       constants.push_back(rewriter.create<arith::ConstantIndexOp>(loc, i));
289d581c94dSMatthias Springer 
290d581c94dSMatthias Springer     // Traverse all `elements` and create `memref.store` ops.
291d581c94dSMatthias Springer     auto elementIt = fromElementsOp.elements().begin();
292d581c94dSMatthias Springer     SmallVector<Value, 2> indices(tensorType.getRank(), constants[0]);
293d581c94dSMatthias Springer     createStores(rewriter, loc, /*dim=*/0, buffer, shape, constants, elementIt,
294d581c94dSMatthias Springer                  indices);
295d581c94dSMatthias Springer 
296d581c94dSMatthias Springer     replaceOpWithBufferizedValues(rewriter, op, buffer);
297d581c94dSMatthias Springer     return success();
298d581c94dSMatthias Springer   }
299d581c94dSMatthias Springer };
300d581c94dSMatthias Springer 
30171bbb78bSMatthias Springer /// Bufferization of tensor.generate.
30271bbb78bSMatthias Springer struct GenerateOpInterface
30371bbb78bSMatthias Springer     : public BufferizableOpInterface::ExternalModel<GenerateOpInterface,
30471bbb78bSMatthias Springer                                                     tensor::GenerateOp> {
30571bbb78bSMatthias Springer   LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
30671bbb78bSMatthias Springer                           const BufferizationState &state) const {
30771bbb78bSMatthias Springer     auto generateOp = cast<tensor::GenerateOp>(op);
30871bbb78bSMatthias Springer 
30971bbb78bSMatthias Springer     // Allocate memory.
31071bbb78bSMatthias Springer     Location loc = op->getLoc();
31171bbb78bSMatthias Springer     MemRefType memrefType =
31271bbb78bSMatthias Springer         getContiguousMemRefType(generateOp.getType().cast<RankedTensorType>());
31371bbb78bSMatthias Springer     FailureOr<Value> maybeResult =
31471bbb78bSMatthias Springer         createAlloc(rewriter, loc, memrefType, generateOp.dynamicExtents(),
31571bbb78bSMatthias Springer                     /*deallocMemref=*/state.getOptions().createDeallocs,
31671bbb78bSMatthias Springer                     state.getOptions());
31771bbb78bSMatthias Springer     if (failed(maybeResult))
31871bbb78bSMatthias Springer       return failure();
31971bbb78bSMatthias Springer     Value result = *maybeResult;
32071bbb78bSMatthias Springer 
32171bbb78bSMatthias Springer     // Collect loop bounds.
32271bbb78bSMatthias Springer     int64_t rank = memrefType.getRank();
32371bbb78bSMatthias Springer     Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
32471bbb78bSMatthias Springer     Value one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
32571bbb78bSMatthias Springer     SmallVector<Value, 4> lowerBounds(rank, zero);
32671bbb78bSMatthias Springer     SmallVector<Value, 4> steps(rank, one);
32771bbb78bSMatthias Springer     SmallVector<Value, 4> upperBounds;
32871bbb78bSMatthias Springer     int nextDynamicIndex = 0;
32971bbb78bSMatthias Springer     for (int i = 0; i < rank; i++) {
33071bbb78bSMatthias Springer       Value upperBound = memrefType.isDynamicDim(i)
33171bbb78bSMatthias Springer                              ? generateOp.dynamicExtents()[nextDynamicIndex++]
33271bbb78bSMatthias Springer                              : rewriter.create<arith::ConstantIndexOp>(
33371bbb78bSMatthias Springer                                    loc, memrefType.getDimSize(i));
33471bbb78bSMatthias Springer       upperBounds.push_back(upperBound);
33571bbb78bSMatthias Springer     }
33671bbb78bSMatthias Springer 
33771bbb78bSMatthias Springer     // Generate tensor elements with a parallel loop that stores into
33871bbb78bSMatthias Springer     // each element of the resulting memref. We use mergeBlockBefore to "move"
33971bbb78bSMatthias Springer     // this op's body into the scf.parallel's body.
34071bbb78bSMatthias Springer     auto parallel =
34171bbb78bSMatthias Springer         rewriter.create<scf::ParallelOp>(loc, lowerBounds, upperBounds, steps);
34271bbb78bSMatthias Springer     Block *parallelBody = parallel.getBody();
34371bbb78bSMatthias Springer     rewriter.mergeBlockBefore(generateOp.getBody(),
34471bbb78bSMatthias Springer                               parallelBody->getTerminator(),
34571bbb78bSMatthias Springer                               parallelBody->getArguments());
34671bbb78bSMatthias Springer     // Replace the inlined yield op with a store op. The scf.parallel's builder
34771bbb78bSMatthias Springer     // already populated an scf.yield at the end, so we don't need to worry
34871bbb78bSMatthias Springer     // about creating that.
34971bbb78bSMatthias Springer     Operation *elementYield = parallelBody->getTerminator()->getPrevNode();
35071bbb78bSMatthias Springer     rewriter.setInsertionPointAfter(elementYield);
35171bbb78bSMatthias Springer     rewriter.replaceOpWithNewOp<memref::StoreOp>(
35271bbb78bSMatthias Springer         elementYield, elementYield->getOperands()[0], result,
35371bbb78bSMatthias Springer         parallelBody->getArguments());
35471bbb78bSMatthias Springer 
35571bbb78bSMatthias Springer     replaceOpWithBufferizedValues(rewriter, op, result);
35671bbb78bSMatthias Springer     return success();
35771bbb78bSMatthias Springer   }
35871bbb78bSMatthias Springer };
35971bbb78bSMatthias Springer 
36049e37000SMatthias Springer /// Bufferization of tensor.insert. Replace with memref.store.
36149e37000SMatthias Springer struct InsertOpInterface
36249e37000SMatthias Springer     : public BufferizableOpInterface::ExternalModel<InsertOpInterface,
36349e37000SMatthias Springer                                                     tensor::InsertOp> {
36449e37000SMatthias Springer   bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
36549e37000SMatthias Springer                               const BufferizationState &state) const {
36649e37000SMatthias Springer     return true;
36749e37000SMatthias Springer   }
36849e37000SMatthias Springer 
36949e37000SMatthias Springer   bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand,
37049e37000SMatthias Springer                                const BufferizationState &state) const {
37149e37000SMatthias Springer     return true;
37249e37000SMatthias Springer   }
37349e37000SMatthias Springer 
37449e37000SMatthias Springer   OpResult getAliasingOpResult(Operation *op, OpOperand &opOperand,
37549e37000SMatthias Springer                                const BufferizationState &state) const {
37649e37000SMatthias Springer     assert(&opOperand == &op->getOpOperand(1) /*dest*/ &&
37749e37000SMatthias Springer            "expected dest OpOperand");
37849e37000SMatthias Springer     return op->getOpResult(0);
37949e37000SMatthias Springer   }
38049e37000SMatthias Springer 
38149e37000SMatthias Springer   SmallVector<OpOperand *>
38249e37000SMatthias Springer   getAliasingOpOperand(Operation *op, OpResult opResult,
38349e37000SMatthias Springer                        const BufferizationState &state) const {
38449e37000SMatthias Springer     return {&op->getOpOperand(1) /*dest*/};
38549e37000SMatthias Springer   }
38649e37000SMatthias Springer 
38749e37000SMatthias Springer   LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
38849e37000SMatthias Springer                           const BufferizationState &state) const {
38949e37000SMatthias Springer     auto insertOp = cast<tensor::InsertOp>(op);
39049e37000SMatthias Springer     FailureOr<Value> destMemref =
39149e37000SMatthias Springer         state.getBuffer(rewriter, insertOp->getOpOperand(1) /*dest*/);
39249e37000SMatthias Springer     if (failed(destMemref))
39349e37000SMatthias Springer       return failure();
39449e37000SMatthias Springer     rewriter.create<memref::StoreOp>(insertOp.getLoc(), insertOp.scalar(),
39549e37000SMatthias Springer                                      *destMemref, insertOp.indices());
39649e37000SMatthias Springer     replaceOpWithBufferizedValues(rewriter, op, *destMemref);
39749e37000SMatthias Springer     return success();
39849e37000SMatthias Springer   }
39949e37000SMatthias Springer 
40049e37000SMatthias Springer   BufferRelation bufferRelation(Operation *op, OpResult opResult,
40149e37000SMatthias Springer                                 const BufferizationState &state) const {
40249e37000SMatthias Springer     return BufferRelation::Equivalent;
40349e37000SMatthias Springer   }
40449e37000SMatthias Springer };
40549e37000SMatthias Springer 
40649e37000SMatthias Springer /// Return true if the (ExtractSliceOp, InsertSliceOp) pair match (i.e.
40749e37000SMatthias Springer /// equivalent operand / result and same offset/sizes/strides specification).
40849e37000SMatthias Springer ///
40949e37000SMatthias Springer /// This is one particular type of relationship between ops on tensors that
41049e37000SMatthias Springer /// reduce to an equivalence on buffers. This should be generalized and
41149e37000SMatthias Springer /// exposed as interfaces on the proper types.
41249e37000SMatthias Springer static bool areEquivalentExtractSliceOps(const BufferizationState &state,
41349e37000SMatthias Springer                                          ExtractSliceOp st, InsertSliceOp sti) {
41449e37000SMatthias Springer   if (!st || !sti)
41549e37000SMatthias Springer     return false;
41649e37000SMatthias Springer   if (sti != sti &&
41749e37000SMatthias Springer       !state.areEquivalentBufferizedValues(st.source(), sti.dest()))
41849e37000SMatthias Springer     return false;
41949e37000SMatthias Springer   if (!sameOffsetsSizesAndStrides(st, sti, isEqualConstantIntOrValue))
42049e37000SMatthias Springer     return false;
42149e37000SMatthias Springer   return true;
42249e37000SMatthias Springer }
42349e37000SMatthias Springer 
42449e37000SMatthias Springer /// Return true if `value` is originating from an ExtractSliceOp that matches
42549e37000SMatthias Springer /// the given InsertSliceOp.
42649e37000SMatthias Springer static bool hasMatchingExtractSliceOp(const BufferizationState &state,
42749e37000SMatthias Springer                                       Value value, InsertSliceOp insertOp) {
42849e37000SMatthias Springer   auto condition = [&](Value val) {
42949e37000SMatthias Springer     if (auto extractOp = val.getDefiningOp<ExtractSliceOp>())
43049e37000SMatthias Springer       if (areEquivalentExtractSliceOps(state, extractOp, insertOp))
43149e37000SMatthias Springer         return true;
43249e37000SMatthias Springer     return false;
43349e37000SMatthias Springer   };
43449e37000SMatthias Springer 
43549e37000SMatthias Springer   return llvm::all_of(state.findValueInReverseUseDefChain(value, condition),
43649e37000SMatthias Springer                       condition);
43749e37000SMatthias Springer }
43849e37000SMatthias Springer 
43949e37000SMatthias Springer /// Bufferization of tensor.insert_slice. Replace with a memory copy. Under
44049e37000SMatthias Springer /// certain circumstances, this op can also be a no-op.
44149e37000SMatthias Springer struct InsertSliceOpInterface
44249e37000SMatthias Springer     : public BufferizableOpInterface::ExternalModel<InsertSliceOpInterface,
44349e37000SMatthias Springer                                                     tensor::InsertSliceOp> {
44449e37000SMatthias Springer   bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
44549e37000SMatthias Springer                               const BufferizationState &state) const {
44649e37000SMatthias Springer     return true;
44749e37000SMatthias Springer   }
44849e37000SMatthias Springer 
44949e37000SMatthias Springer   bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand,
45049e37000SMatthias Springer                                const BufferizationState &state) const {
45149e37000SMatthias Springer     return &opOperand == &op->getOpOperand(1) /*dest*/;
45249e37000SMatthias Springer   }
45349e37000SMatthias Springer 
45449e37000SMatthias Springer   OpResult getAliasingOpResult(Operation *op, OpOperand &opOperand,
45549e37000SMatthias Springer                                const BufferizationState &state) const {
45649e37000SMatthias Springer     return &opOperand == &op->getOpOperand(1) /*dest*/
45749e37000SMatthias Springer                ? op->getResult(0)
45849e37000SMatthias Springer                : OpResult();
45949e37000SMatthias Springer   }
46049e37000SMatthias Springer 
46149e37000SMatthias Springer   BufferRelation bufferRelation(Operation *op, OpResult opResult,
46249e37000SMatthias Springer                                 const BufferizationState &state) const {
46349e37000SMatthias Springer     return BufferRelation::Equivalent;
46449e37000SMatthias Springer   }
46549e37000SMatthias Springer 
46649e37000SMatthias Springer   bool isNotConflicting(Operation *op, OpOperand *uRead,
46749e37000SMatthias Springer                         OpOperand *uConflictingWrite,
46849e37000SMatthias Springer                         const BufferizationState &state) const {
46949e37000SMatthias Springer     Operation *readingOp = uRead->getOwner();
47049e37000SMatthias Springer     Operation *conflictingWritingOp = uConflictingWrite->getOwner();
47149e37000SMatthias Springer 
47249e37000SMatthias Springer     // Special rules for matching ExtractSliceOp/InsertSliceOp pairs. If
47349e37000SMatthias Springer     // uRead is an InsertSliceOp...
47449e37000SMatthias Springer     if (auto insertSliceOp = dyn_cast<InsertSliceOp>(readingOp)) {
47549e37000SMatthias Springer       // As an example, consider the following IR.
47649e37000SMatthias Springer       //
47749e37000SMatthias Springer       // %0 = tensor.extract_slice %t[%a, %b][%c, %d][1, 1] {inplace = [true] }
47849e37000SMatthias Springer       // %1 = linalg.fill %cst, %0 {inplace= [true] }
47949e37000SMatthias Springer       // %2 = tensor.insert_slice %1 into %t[%a, %b][%c, %d][1, 1]
48049e37000SMatthias Springer       //     {inplace= [true] }
48149e37000SMatthias Springer 
48249e37000SMatthias Springer       // TODO: Use insertSliceOp.getDestOpOperand etc. when available.
48349e37000SMatthias Springer       if (uRead == &insertSliceOp->getOpOperand(1) /*dest*/ &&
48449e37000SMatthias Springer           hasMatchingExtractSliceOp(state, uConflictingWrite->get(),
48549e37000SMatthias Springer                                     insertSliceOp))
48649e37000SMatthias Springer         // Case 1: The main insight is that InsertSliceOp reads only part of
48749e37000SMatthias Springer         // the destination tensor. The overwritten area is not read. If
48849e37000SMatthias Springer         // uConflictingWrite writes into exactly the memory location that is
48949e37000SMatthias Springer         // being read by uRead, this is not a conflict.
49049e37000SMatthias Springer         //
49149e37000SMatthias Springer         // In the above example:
49249e37000SMatthias Springer         // uRead             = OpOperand 1 (%t) of tensor.insert_slice
49349e37000SMatthias Springer         // uConflictingWrite = OpOperand 1 (%0) of linalg.fill
49449e37000SMatthias Springer         //
49549e37000SMatthias Springer         // The read of %t does not conflict with the write of the FillOp
49649e37000SMatthias Springer         // (same aliases!) because the area that the FillOp operates on is
49749e37000SMatthias Springer         // exactly the one that is *not* read via %t.
49849e37000SMatthias Springer         return true;
49949e37000SMatthias Springer 
50049e37000SMatthias Springer       if (uRead == &insertSliceOp->getOpOperand(0) /*source*/ &&
50149e37000SMatthias Springer           uConflictingWrite == &insertSliceOp->getOpOperand(1) /*dest*/ &&
50249e37000SMatthias Springer           hasMatchingExtractSliceOp(state, uRead->get(), insertSliceOp))
50349e37000SMatthias Springer         // Case 2: The read of the source tensor and the write to the dest
50449e37000SMatthias Springer         // tensor via an InsertSliceOp is not a conflict if the read is
50549e37000SMatthias Springer         // reading exactly that part of an equivalent tensor that the
50649e37000SMatthias Springer         // InsertSliceOp is writing.
50749e37000SMatthias Springer         //
50849e37000SMatthias Springer         // In the above example:
50949e37000SMatthias Springer         // uRead             = OpOperand 0 (%1) of tensor.insert_slice
51049e37000SMatthias Springer         // uConflictingWrite = OpOperand 1 (%t) of tensor.insert_slice
51149e37000SMatthias Springer         return true;
51249e37000SMatthias Springer     }
51349e37000SMatthias Springer 
51449e37000SMatthias Springer     // If uConflictingWrite is an InsertSliceOp...
51549e37000SMatthias Springer     if (auto insertSliceOp = dyn_cast<InsertSliceOp>(conflictingWritingOp))
51649e37000SMatthias Springer       // As an example, consider the following IR.
51749e37000SMatthias Springer       //
51849e37000SMatthias Springer       // %0 = tensor.extract_slice %t[%a, %b][%c, %d][1, 1] {inplace = [true] }
51949e37000SMatthias Springer       // %1 = linalg.fill %cst, %0 {inplace= [true] }
52049e37000SMatthias Springer       // %2 = tensor.insert_slice %1 into %t[%a, %b][%c, %d][1, 1]
52149e37000SMatthias Springer       //     {inplace= [true] }
52249e37000SMatthias Springer       // %3 = vector.transfer_read %1, %cst
52349e37000SMatthias Springer       //
52449e37000SMatthias Springer       // In the above example:
52549e37000SMatthias Springer       // uRead             = OpOperand 0 (%1) of vector.transfer_read
52649e37000SMatthias Springer       // uConflictingWrite = OpOperand 1 (%t) of tensor.insert_slice
52749e37000SMatthias Springer       // lastWrite         = %1
52849e37000SMatthias Springer       //
52949e37000SMatthias Springer       // This is not a conflict because the InsertSliceOp overwrites the
53049e37000SMatthias Springer       // memory segment of %1 with the exact same data. (Effectively, there
53149e37000SMatthias Springer       // is no memory write here.)
53249e37000SMatthias Springer       if (uConflictingWrite == &insertSliceOp->getOpOperand(1) /*dest*/ &&
53349e37000SMatthias Springer           state.areEquivalentBufferizedValues(uRead->get(),
53449e37000SMatthias Springer                                               insertSliceOp.source()) &&
53549e37000SMatthias Springer           hasMatchingExtractSliceOp(state, insertSliceOp.source(),
53649e37000SMatthias Springer                                     insertSliceOp))
53749e37000SMatthias Springer         return true;
53849e37000SMatthias Springer 
53949e37000SMatthias Springer     return false;
54049e37000SMatthias Springer   }
54149e37000SMatthias Springer 
54249e37000SMatthias Springer   LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
54349e37000SMatthias Springer                           const BufferizationState &state) const {
54449e37000SMatthias Springer     // insert_slice ops arise from tiling and bufferizing them out-of-place is
54549e37000SMatthias Springer     // generally a deal breaker. When used with loops, this ends up cloning the
54649e37000SMatthias Springer     // whole tensor on every single iteration and is a symptom of a
54749e37000SMatthias Springer     // catastrophically bad scheduling decision.
54849e37000SMatthias Springer     // TODO: be very loud about it or even consider failing the pass.
54949e37000SMatthias Springer     auto insertSliceOp = cast<tensor::InsertSliceOp>(op);
55049e37000SMatthias Springer     Location loc = insertSliceOp.getLoc();
55149e37000SMatthias Springer 
55249e37000SMatthias Springer     // When bufferizing out-of-place, `getResultBuffer` allocates.
55349e37000SMatthias Springer     FailureOr<Value> dstMemref =
55449e37000SMatthias Springer         state.getBuffer(rewriter, insertSliceOp->getOpOperand(1) /*dest*/);
55549e37000SMatthias Springer     if (failed(dstMemref))
55649e37000SMatthias Springer       return failure();
55749e37000SMatthias Springer 
55849e37000SMatthias Springer     // Expand offsets, sizes and strides to the full rank to handle the
55949e37000SMatthias Springer     // rank-reducing case.
56049e37000SMatthias Springer     SmallVector<OpFoldResult> mixedOffsets = insertSliceOp.getMixedOffsets();
56149e37000SMatthias Springer     SmallVector<OpFoldResult> mixedSizes = insertSliceOp.getMixedSizes();
56249e37000SMatthias Springer     SmallVector<OpFoldResult> mixedStrides = insertSliceOp.getMixedStrides();
56349e37000SMatthias Springer     OffsetSizeAndStrideOpInterface::expandToRank(
56449e37000SMatthias Springer         *dstMemref, mixedOffsets, mixedSizes, mixedStrides,
56549e37000SMatthias Springer         [&](Value target, int64_t dim) -> OpFoldResult {
56649e37000SMatthias Springer           auto shapedType = target.getType().cast<ShapedType>();
56749e37000SMatthias Springer           if (shapedType.isDynamicDim(dim))
56849e37000SMatthias Springer             return rewriter.create<memref::DimOp>(loc, target, dim).result();
56949e37000SMatthias Springer           return rewriter.getIndexAttr(shapedType.getDimSize(dim));
57049e37000SMatthias Springer         });
57149e37000SMatthias Springer     // Take a subview of the dst.
57249e37000SMatthias Springer     auto dstMemrefType = dstMemref->getType().cast<MemRefType>();
57349e37000SMatthias Springer     auto subviewMemRefType =
57449e37000SMatthias Springer         memref::SubViewOp::inferRankReducedResultType(
57549e37000SMatthias Springer             insertSliceOp.getSourceType().getRank(), dstMemrefType,
57649e37000SMatthias Springer             mixedOffsets, mixedSizes, mixedStrides)
57749e37000SMatthias Springer             .cast<MemRefType>();
57849e37000SMatthias Springer     Value subView = rewriter.create<memref::SubViewOp>(
57949e37000SMatthias Springer         loc, subviewMemRefType, *dstMemref, mixedOffsets, mixedSizes,
58049e37000SMatthias Springer         mixedStrides);
58149e37000SMatthias Springer 
58249e37000SMatthias Springer     // Copy tensor. If this tensor.insert_slice has a matching
58349e37000SMatthias Springer     // tensor.extract_slice, the copy operation will eventually fold away.
58449e37000SMatthias Springer     Value srcMemref =
58549e37000SMatthias Springer         *state.getBuffer(rewriter, insertSliceOp->getOpOperand(0) /*source*/);
58649e37000SMatthias Springer     if (failed(createMemCpy(rewriter, loc, srcMemref, subView,
58749e37000SMatthias Springer                             state.getOptions())))
58849e37000SMatthias Springer       return failure();
58949e37000SMatthias Springer 
59049e37000SMatthias Springer     replaceOpWithBufferizedValues(rewriter, op, *dstMemref);
59149e37000SMatthias Springer     return success();
59249e37000SMatthias Springer   }
59349e37000SMatthias Springer };
59449e37000SMatthias Springer 
595fc08d1c2SMatthias Springer /// Bufferization of tensor.rank. Replace with memref.rank.
596fc08d1c2SMatthias Springer struct RankOpInterface
597fc08d1c2SMatthias Springer     : public BufferizableOpInterface::ExternalModel<RankOpInterface,
598fc08d1c2SMatthias Springer                                                     tensor::RankOp> {
599fc08d1c2SMatthias Springer   bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
600fc08d1c2SMatthias Springer                               const BufferizationState &state) const {
601fc08d1c2SMatthias Springer     return true;
602fc08d1c2SMatthias Springer   }
603fc08d1c2SMatthias Springer 
604fc08d1c2SMatthias Springer   bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand,
605fc08d1c2SMatthias Springer                                const BufferizationState &state) const {
606fc08d1c2SMatthias Springer     return false;
607fc08d1c2SMatthias Springer   }
608fc08d1c2SMatthias Springer 
609fc08d1c2SMatthias Springer   OpResult getAliasingOpResult(Operation *op, OpOperand &opOperand,
610fc08d1c2SMatthias Springer                                const BufferizationState &state) const {
611fc08d1c2SMatthias Springer     return OpResult();
612fc08d1c2SMatthias Springer   }
613fc08d1c2SMatthias Springer 
614fc08d1c2SMatthias Springer   LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
615fc08d1c2SMatthias Springer                           const BufferizationState &state) const {
616fc08d1c2SMatthias Springer     auto rankOp = cast<tensor::RankOp>(op);
617fc08d1c2SMatthias Springer     Value v = *state.getBuffer(rewriter, rankOp->getOpOperand(0) /*source*/);
618fc08d1c2SMatthias Springer     replaceOpWithNewBufferizedOp<memref::RankOp>(rewriter, op, rankOp.getType(),
619fc08d1c2SMatthias Springer                                                  v);
620fc08d1c2SMatthias Springer     return success();
621fc08d1c2SMatthias Springer   }
622fc08d1c2SMatthias Springer };
623fc08d1c2SMatthias Springer 
62449e37000SMatthias Springer } // namespace
62549e37000SMatthias Springer } // namespace tensor
62649e37000SMatthias Springer } // namespace mlir
62749e37000SMatthias Springer 
62849e37000SMatthias Springer void mlir::tensor::registerBufferizableOpInterfaceExternalModels(
62949e37000SMatthias Springer     DialectRegistry &registry) {
63049e37000SMatthias Springer   registry.addOpInterface<CastOp, CastOpInterface>();
63149e37000SMatthias Springer   registry.addOpInterface<DimOp, DimOpInterface>();
63249e37000SMatthias Springer   registry.addOpInterface<ExtractSliceOp, ExtractSliceOpInterface>();
63349e37000SMatthias Springer   registry.addOpInterface<ExtractOp, ExtractOpInterface>();
634d581c94dSMatthias Springer   registry.addOpInterface<FromElementsOp, FromElementsOpInterface>();
63571bbb78bSMatthias Springer   registry.addOpInterface<GenerateOp, GenerateOpInterface>();
63649e37000SMatthias Springer   registry.addOpInterface<InsertOp, InsertOpInterface>();
63749e37000SMatthias Springer   registry.addOpInterface<InsertSliceOp, InsertSliceOpInterface>();
638fc08d1c2SMatthias Springer   registry.addOpInterface<RankOp, RankOpInterface>();
63949e37000SMatthias Springer }
640