149e37000SMatthias Springer //===- BufferizableOpInterfaceImpl.cpp - Impl. of BufferizableOpInterface -===// 249e37000SMatthias Springer // 349e37000SMatthias Springer // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 449e37000SMatthias Springer // See https://llvm.org/LICENSE.txt for license information. 549e37000SMatthias Springer // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 649e37000SMatthias Springer // 749e37000SMatthias Springer //===----------------------------------------------------------------------===// 849e37000SMatthias Springer 949e37000SMatthias Springer #include "mlir/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.h" 1049e37000SMatthias Springer #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h" 1149e37000SMatthias Springer #include "mlir/Dialect/MemRef/IR/MemRef.h" 1271bbb78bSMatthias Springer #include "mlir/Dialect/SCF/SCF.h" 1349e37000SMatthias Springer #include "mlir/Dialect/Tensor/IR/Tensor.h" 1449e37000SMatthias Springer #include "mlir/IR/Dialect.h" 1549e37000SMatthias Springer #include "mlir/IR/Operation.h" 1649e37000SMatthias Springer 1749e37000SMatthias Springer using namespace mlir; 1849e37000SMatthias Springer using namespace mlir::bufferization; 1949e37000SMatthias Springer using namespace mlir::tensor; 2049e37000SMatthias Springer 2149e37000SMatthias Springer namespace mlir { 2249e37000SMatthias Springer namespace tensor { 2349e37000SMatthias Springer namespace { 2449e37000SMatthias Springer 2549e37000SMatthias Springer struct CastOpInterface 2649e37000SMatthias Springer : public BufferizableOpInterface::ExternalModel<CastOpInterface, 2749e37000SMatthias Springer tensor::CastOp> { 2849e37000SMatthias Springer bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand, 2949e37000SMatthias Springer const BufferizationState &state) const { 3049e37000SMatthias Springer return false; 3149e37000SMatthias Springer } 3249e37000SMatthias Springer 3349e37000SMatthias Springer bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand, 3449e37000SMatthias Springer const BufferizationState &state) const { 3549e37000SMatthias Springer return false; 3649e37000SMatthias Springer } 3749e37000SMatthias Springer 3849e37000SMatthias Springer OpResult getAliasingOpResult(Operation *op, OpOperand &opOperand, 3949e37000SMatthias Springer const BufferizationState &state) const { 4049e37000SMatthias Springer return op->getResult(0); 4149e37000SMatthias Springer } 4249e37000SMatthias Springer 4349e37000SMatthias Springer BufferRelation bufferRelation(Operation *op, OpResult opResult, 4449e37000SMatthias Springer const BufferizationState &state) const { 4549e37000SMatthias Springer return BufferRelation::Equivalent; 4649e37000SMatthias Springer } 4749e37000SMatthias Springer 4849e37000SMatthias Springer LogicalResult bufferize(Operation *op, RewriterBase &rewriter, 4949e37000SMatthias Springer const BufferizationState &state) const { 5049e37000SMatthias Springer auto castOp = cast<tensor::CastOp>(op); 5149e37000SMatthias Springer 5249e37000SMatthias Springer // The result buffer still has the old (pre-cast) type. 5349e37000SMatthias Springer FailureOr<Value> resultBuffer = 5449e37000SMatthias Springer state.getBuffer(rewriter, castOp->getOpOperand(0) /*source*/); 5549e37000SMatthias Springer if (failed(resultBuffer)) 5649e37000SMatthias Springer return failure(); 5749e37000SMatthias Springer auto sourceMemRefType = resultBuffer->getType().cast<BaseMemRefType>(); 5849e37000SMatthias Springer Attribute memorySpace = sourceMemRefType.getMemorySpace(); 5949e37000SMatthias Springer TensorType resultTensorType = 6049e37000SMatthias Springer castOp.getResult().getType().cast<TensorType>(); 6149e37000SMatthias Springer MemRefLayoutAttrInterface layout; 6249e37000SMatthias Springer 6349e37000SMatthias Springer if (auto rankedMemRefType = sourceMemRefType.dyn_cast<MemRefType>()) 6449e37000SMatthias Springer if (resultTensorType.isa<RankedTensorType>()) 6549e37000SMatthias Springer layout = rankedMemRefType.getLayout(); 6649e37000SMatthias Springer 6749e37000SMatthias Springer // Compute the new memref type. 68*26852423SMatthias Springer Type resultMemRefType = getMemRefType(resultTensorType, state.getOptions(), 69*26852423SMatthias Springer layout, memorySpace); 7049e37000SMatthias Springer 7149e37000SMatthias Springer // Replace the op with a memref.cast. 7249e37000SMatthias Springer assert(memref::CastOp::areCastCompatible(resultBuffer->getType(), 7349e37000SMatthias Springer resultMemRefType) && 7449e37000SMatthias Springer "CallOp::bufferize: cast incompatible"); 7549e37000SMatthias Springer replaceOpWithNewBufferizedOp<memref::CastOp>(rewriter, op, resultMemRefType, 7649e37000SMatthias Springer *resultBuffer); 7749e37000SMatthias Springer 7849e37000SMatthias Springer return success(); 7949e37000SMatthias Springer } 8049e37000SMatthias Springer }; 8149e37000SMatthias Springer 8249e37000SMatthias Springer /// Bufferization of tensor.dim. Replace with memref.dim. 8349e37000SMatthias Springer struct DimOpInterface 8449e37000SMatthias Springer : public BufferizableOpInterface::ExternalModel<DimOpInterface, 8549e37000SMatthias Springer tensor::DimOp> { 8649e37000SMatthias Springer bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand, 8749e37000SMatthias Springer const BufferizationState &state) const { 8849e37000SMatthias Springer return true; 8949e37000SMatthias Springer } 9049e37000SMatthias Springer 9149e37000SMatthias Springer bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand, 9249e37000SMatthias Springer const BufferizationState &state) const { 9349e37000SMatthias Springer return false; 9449e37000SMatthias Springer } 9549e37000SMatthias Springer 9649e37000SMatthias Springer OpResult getAliasingOpResult(Operation *op, OpOperand &opOperand, 9749e37000SMatthias Springer const BufferizationState &state) const { 9849e37000SMatthias Springer return OpResult(); 9949e37000SMatthias Springer } 10049e37000SMatthias Springer 10149e37000SMatthias Springer LogicalResult bufferize(Operation *op, RewriterBase &rewriter, 10249e37000SMatthias Springer const BufferizationState &state) const { 10349e37000SMatthias Springer auto dimOp = cast<tensor::DimOp>(op); 10449e37000SMatthias Springer Value v = *state.getBuffer(rewriter, dimOp->getOpOperand(0) /*source*/); 10549e37000SMatthias Springer replaceOpWithNewBufferizedOp<memref::DimOp>(rewriter, op, v, dimOp.index()); 10649e37000SMatthias Springer return success(); 10749e37000SMatthias Springer } 10849e37000SMatthias Springer }; 10949e37000SMatthias Springer 11049e37000SMatthias Springer /// Bufferization of tensor.extract_slice. Replace with memref.subview. 11149e37000SMatthias Springer struct ExtractSliceOpInterface 11249e37000SMatthias Springer : public BufferizableOpInterface::ExternalModel<ExtractSliceOpInterface, 11349e37000SMatthias Springer tensor::ExtractSliceOp> { 11449e37000SMatthias Springer bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand, 11549e37000SMatthias Springer const BufferizationState &state) const { 11649e37000SMatthias Springer return false; 11749e37000SMatthias Springer } 11849e37000SMatthias Springer 11949e37000SMatthias Springer bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand, 12049e37000SMatthias Springer const BufferizationState &state) const { 12149e37000SMatthias Springer return false; 12249e37000SMatthias Springer } 12349e37000SMatthias Springer 12449e37000SMatthias Springer OpResult getAliasingOpResult(Operation *op, OpOperand &opOperand, 12549e37000SMatthias Springer const BufferizationState &state) const { 12649e37000SMatthias Springer return &opOperand == &op->getOpOperand(0) /*source*/ 12749e37000SMatthias Springer ? op->getResult(0) 12849e37000SMatthias Springer : OpResult(); 12949e37000SMatthias Springer } 13049e37000SMatthias Springer 13149e37000SMatthias Springer BufferRelation bufferRelation(Operation *op, OpResult opResult, 13249e37000SMatthias Springer const BufferizationState &state) const { 13349e37000SMatthias Springer return BufferRelation::None; 13449e37000SMatthias Springer } 13549e37000SMatthias Springer 13649e37000SMatthias Springer LogicalResult bufferize(Operation *op, RewriterBase &rewriter, 13749e37000SMatthias Springer const BufferizationState &state) const { 13849e37000SMatthias Springer auto extractSliceOp = cast<tensor::ExtractSliceOp>(op); 13949e37000SMatthias Springer Location loc = extractSliceOp.getLoc(); 14049e37000SMatthias Springer Value srcMemref = 14149e37000SMatthias Springer *state.getBuffer(rewriter, extractSliceOp->getOpOperand(0) /*source*/, 14249e37000SMatthias Springer /*forceInPlace=*/true); 14349e37000SMatthias Springer auto srcMemrefType = srcMemref.getType().cast<MemRefType>(); 14449e37000SMatthias Springer auto dstTensorType = 14549e37000SMatthias Springer extractSliceOp.result().getType().cast<RankedTensorType>(); 14649e37000SMatthias Springer 14749e37000SMatthias Springer // If not inplaceable, alloc. 14849e37000SMatthias Springer bool inplace = state.isInPlace(extractSliceOp->getOpOperand(0)); 14949e37000SMatthias Springer Value alloc; 15049e37000SMatthias Springer if (!inplace) { 15149e37000SMatthias Springer FailureOr<Value> allocOrFailure = 15249e37000SMatthias Springer createAlloc(rewriter, loc, extractSliceOp.result(), 15349e37000SMatthias Springer state.getOptions().createDeallocs, state.getOptions()); 15449e37000SMatthias Springer if (failed(allocOrFailure)) 15549e37000SMatthias Springer return failure(); 15649e37000SMatthias Springer alloc = *allocOrFailure; 15749e37000SMatthias Springer } 15849e37000SMatthias Springer 15949e37000SMatthias Springer // Expand offsets, sizes and strides to the full rank to handle the 16049e37000SMatthias Springer // rank-reducing case. 16149e37000SMatthias Springer SmallVector<OpFoldResult> mixedOffsets = extractSliceOp.getMixedOffsets(); 16249e37000SMatthias Springer SmallVector<OpFoldResult> mixedSizes = extractSliceOp.getMixedSizes(); 16349e37000SMatthias Springer SmallVector<OpFoldResult> mixedStrides = extractSliceOp.getMixedStrides(); 16449e37000SMatthias Springer OffsetSizeAndStrideOpInterface::expandToRank( 16549e37000SMatthias Springer srcMemref, mixedOffsets, mixedSizes, mixedStrides, 16649e37000SMatthias Springer [&](Value target, int64_t dim) -> OpFoldResult { 16749e37000SMatthias Springer auto shapedType = target.getType().cast<ShapedType>(); 16849e37000SMatthias Springer if (shapedType.isDynamicDim(dim)) 16949e37000SMatthias Springer return rewriter.create<memref::DimOp>(loc, target, dim).result(); 17049e37000SMatthias Springer return rewriter.getIndexAttr(shapedType.getDimSize(dim)); 17149e37000SMatthias Springer }); 17249e37000SMatthias Springer // Bufferize to subview. 17349e37000SMatthias Springer auto subviewMemRefType = memref::SubViewOp::inferRankReducedResultType( 17449e37000SMatthias Springer dstTensorType.getRank(), srcMemrefType, 17549e37000SMatthias Springer mixedOffsets, mixedSizes, mixedStrides) 17649e37000SMatthias Springer .cast<MemRefType>(); 17749e37000SMatthias Springer Value subView = rewriter.create<memref::SubViewOp>( 17849e37000SMatthias Springer loc, subviewMemRefType, srcMemref, mixedOffsets, mixedSizes, 17949e37000SMatthias Springer mixedStrides); 18049e37000SMatthias Springer 18149e37000SMatthias Springer // If not inplaceable, copy. 18249e37000SMatthias Springer if (!inplace) { 18349e37000SMatthias Springer // Do not copy if the copied data is never read. 18449e37000SMatthias Springer if (state.isValueRead(extractSliceOp.result())) 18549e37000SMatthias Springer if (failed(createMemCpy(rewriter, extractSliceOp.getLoc(), subView, 18649e37000SMatthias Springer alloc, state.getOptions()))) 18749e37000SMatthias Springer return failure(); 18849e37000SMatthias Springer subView = alloc; 18949e37000SMatthias Springer } 19049e37000SMatthias Springer 19149e37000SMatthias Springer replaceOpWithBufferizedValues(rewriter, op, subView); 19249e37000SMatthias Springer return success(); 19349e37000SMatthias Springer } 19449e37000SMatthias Springer }; 19549e37000SMatthias Springer 19649e37000SMatthias Springer /// Bufferization of tensor.extract. Replace with memref.load. 19749e37000SMatthias Springer struct ExtractOpInterface 19849e37000SMatthias Springer : public BufferizableOpInterface::ExternalModel<ExtractOpInterface, 19949e37000SMatthias Springer tensor::ExtractOp> { 20049e37000SMatthias Springer bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand, 20149e37000SMatthias Springer const BufferizationState &state) const { 20249e37000SMatthias Springer return true; 20349e37000SMatthias Springer } 20449e37000SMatthias Springer 20549e37000SMatthias Springer bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand, 20649e37000SMatthias Springer const BufferizationState &state) const { 20749e37000SMatthias Springer return false; 20849e37000SMatthias Springer } 20949e37000SMatthias Springer 21049e37000SMatthias Springer OpResult getAliasingOpResult(Operation *op, OpOperand &opOperand, 21149e37000SMatthias Springer const BufferizationState &state) const { 21249e37000SMatthias Springer return OpResult(); 21349e37000SMatthias Springer } 21449e37000SMatthias Springer 21549e37000SMatthias Springer LogicalResult bufferize(Operation *op, RewriterBase &rewriter, 21649e37000SMatthias Springer const BufferizationState &state) const { 21749e37000SMatthias Springer auto extractOp = cast<tensor::ExtractOp>(op); 21849e37000SMatthias Springer Value srcMemref = 21949e37000SMatthias Springer *state.getBuffer(rewriter, extractOp->getOpOperand(0) /*tensor*/); 22049e37000SMatthias Springer replaceOpWithNewBufferizedOp<memref::LoadOp>(rewriter, op, srcMemref, 22149e37000SMatthias Springer extractOp.indices()); 22249e37000SMatthias Springer return success(); 22349e37000SMatthias Springer } 22449e37000SMatthias Springer }; 22549e37000SMatthias Springer 226d581c94dSMatthias Springer // Implements backtracking to traverse indices of the output buffer while 227d581c94dSMatthias Springer // iterating over op.elements(). 228d581c94dSMatthias Springer static void createStores(RewriterBase &rewriter, Location loc, int dim, 229d581c94dSMatthias Springer Value buffer, ArrayRef<int64_t> shape, 230d581c94dSMatthias Springer ArrayRef<Value> constants, 231d581c94dSMatthias Springer OperandRange::iterator &elementIt, 232d581c94dSMatthias Springer SmallVectorImpl<Value> &indices) { 233d581c94dSMatthias Springer if (dim == static_cast<int>(shape.size()) - 1) { 234d581c94dSMatthias Springer for (int i = 0; i < shape.back(); ++i) { 235d581c94dSMatthias Springer indices.back() = constants[i]; 236d581c94dSMatthias Springer rewriter.create<memref::StoreOp>(loc, *elementIt, buffer, indices); 237d581c94dSMatthias Springer ++elementIt; 238d581c94dSMatthias Springer } 239d581c94dSMatthias Springer return; 240d581c94dSMatthias Springer } 241d581c94dSMatthias Springer for (int i = 0; i < shape[dim]; ++i) { 242d581c94dSMatthias Springer indices[dim] = constants[i]; 243d581c94dSMatthias Springer createStores(rewriter, loc, dim + 1, buffer, shape, constants, elementIt, 244d581c94dSMatthias Springer indices); 245d581c94dSMatthias Springer } 246d581c94dSMatthias Springer } 247d581c94dSMatthias Springer 248d581c94dSMatthias Springer /// Bufferization of tensor.from_elements. 249d581c94dSMatthias Springer struct FromElementsOpInterface 250d581c94dSMatthias Springer : public BufferizableOpInterface::ExternalModel<FromElementsOpInterface, 251d581c94dSMatthias Springer tensor::FromElementsOp> { 252d581c94dSMatthias Springer LogicalResult bufferize(Operation *op, RewriterBase &rewriter, 253d581c94dSMatthias Springer const BufferizationState &state) const { 254d581c94dSMatthias Springer auto fromElementsOp = cast<tensor::FromElementsOp>(op); 255d581c94dSMatthias Springer 256d581c94dSMatthias Springer // Allocate a buffer for the result. 257d581c94dSMatthias Springer Location loc = op->getLoc(); 258d581c94dSMatthias Springer auto tensorType = fromElementsOp.getType().cast<RankedTensorType>(); 259d581c94dSMatthias Springer auto shape = tensorType.getShape(); 260*26852423SMatthias Springer MemRefType resultType = getContiguousMemRefType(tensorType); 261d581c94dSMatthias Springer FailureOr<Value> maybeBuffer = 262d581c94dSMatthias Springer createAlloc(rewriter, loc, resultType, {}, 263d581c94dSMatthias Springer /*deallocMemref=*/state.getOptions().createDeallocs, 264d581c94dSMatthias Springer state.getOptions()); 265d581c94dSMatthias Springer if (failed(maybeBuffer)) 266d581c94dSMatthias Springer return failure(); 267d581c94dSMatthias Springer Value buffer = *maybeBuffer; 268d581c94dSMatthias Springer 269d581c94dSMatthias Springer // Case: tensor<0xelem_type>. 270d581c94dSMatthias Springer if (fromElementsOp.elements().empty()) { 271d581c94dSMatthias Springer replaceOpWithBufferizedValues(rewriter, op, buffer); 272d581c94dSMatthias Springer return success(); 273d581c94dSMatthias Springer } 274d581c94dSMatthias Springer 275d581c94dSMatthias Springer // Case: tensor<elem_type>. 276d581c94dSMatthias Springer if (shape.empty()) { 277d581c94dSMatthias Springer rewriter.create<memref::StoreOp>(loc, fromElementsOp.elements().front(), 278d581c94dSMatthias Springer buffer); 279d581c94dSMatthias Springer replaceOpWithBufferizedValues(rewriter, op, buffer); 280d581c94dSMatthias Springer return success(); 281d581c94dSMatthias Springer } 282d581c94dSMatthias Springer 283d581c94dSMatthias Springer // Create constants for the range of possible indices [0, max{shape_i}). 284d581c94dSMatthias Springer auto maxDim = *std::max_element(shape.begin(), shape.end()); 285d581c94dSMatthias Springer SmallVector<Value, 2> constants; 286d581c94dSMatthias Springer constants.reserve(maxDim); 287d581c94dSMatthias Springer for (int i = 0; i < maxDim; ++i) 288d581c94dSMatthias Springer constants.push_back(rewriter.create<arith::ConstantIndexOp>(loc, i)); 289d581c94dSMatthias Springer 290d581c94dSMatthias Springer // Traverse all `elements` and create `memref.store` ops. 291d581c94dSMatthias Springer auto elementIt = fromElementsOp.elements().begin(); 292d581c94dSMatthias Springer SmallVector<Value, 2> indices(tensorType.getRank(), constants[0]); 293d581c94dSMatthias Springer createStores(rewriter, loc, /*dim=*/0, buffer, shape, constants, elementIt, 294d581c94dSMatthias Springer indices); 295d581c94dSMatthias Springer 296d581c94dSMatthias Springer replaceOpWithBufferizedValues(rewriter, op, buffer); 297d581c94dSMatthias Springer return success(); 298d581c94dSMatthias Springer } 299d581c94dSMatthias Springer }; 300d581c94dSMatthias Springer 30171bbb78bSMatthias Springer /// Bufferization of tensor.generate. 30271bbb78bSMatthias Springer struct GenerateOpInterface 30371bbb78bSMatthias Springer : public BufferizableOpInterface::ExternalModel<GenerateOpInterface, 30471bbb78bSMatthias Springer tensor::GenerateOp> { 30571bbb78bSMatthias Springer LogicalResult bufferize(Operation *op, RewriterBase &rewriter, 30671bbb78bSMatthias Springer const BufferizationState &state) const { 30771bbb78bSMatthias Springer auto generateOp = cast<tensor::GenerateOp>(op); 30871bbb78bSMatthias Springer 30971bbb78bSMatthias Springer // Allocate memory. 31071bbb78bSMatthias Springer Location loc = op->getLoc(); 31171bbb78bSMatthias Springer MemRefType memrefType = 31271bbb78bSMatthias Springer getContiguousMemRefType(generateOp.getType().cast<RankedTensorType>()); 31371bbb78bSMatthias Springer FailureOr<Value> maybeResult = 31471bbb78bSMatthias Springer createAlloc(rewriter, loc, memrefType, generateOp.dynamicExtents(), 31571bbb78bSMatthias Springer /*deallocMemref=*/state.getOptions().createDeallocs, 31671bbb78bSMatthias Springer state.getOptions()); 31771bbb78bSMatthias Springer if (failed(maybeResult)) 31871bbb78bSMatthias Springer return failure(); 31971bbb78bSMatthias Springer Value result = *maybeResult; 32071bbb78bSMatthias Springer 32171bbb78bSMatthias Springer // Collect loop bounds. 32271bbb78bSMatthias Springer int64_t rank = memrefType.getRank(); 32371bbb78bSMatthias Springer Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0); 32471bbb78bSMatthias Springer Value one = rewriter.create<arith::ConstantIndexOp>(loc, 1); 32571bbb78bSMatthias Springer SmallVector<Value, 4> lowerBounds(rank, zero); 32671bbb78bSMatthias Springer SmallVector<Value, 4> steps(rank, one); 32771bbb78bSMatthias Springer SmallVector<Value, 4> upperBounds; 32871bbb78bSMatthias Springer int nextDynamicIndex = 0; 32971bbb78bSMatthias Springer for (int i = 0; i < rank; i++) { 33071bbb78bSMatthias Springer Value upperBound = memrefType.isDynamicDim(i) 33171bbb78bSMatthias Springer ? generateOp.dynamicExtents()[nextDynamicIndex++] 33271bbb78bSMatthias Springer : rewriter.create<arith::ConstantIndexOp>( 33371bbb78bSMatthias Springer loc, memrefType.getDimSize(i)); 33471bbb78bSMatthias Springer upperBounds.push_back(upperBound); 33571bbb78bSMatthias Springer } 33671bbb78bSMatthias Springer 33771bbb78bSMatthias Springer // Generate tensor elements with a parallel loop that stores into 33871bbb78bSMatthias Springer // each element of the resulting memref. We use mergeBlockBefore to "move" 33971bbb78bSMatthias Springer // this op's body into the scf.parallel's body. 34071bbb78bSMatthias Springer auto parallel = 34171bbb78bSMatthias Springer rewriter.create<scf::ParallelOp>(loc, lowerBounds, upperBounds, steps); 34271bbb78bSMatthias Springer Block *parallelBody = parallel.getBody(); 34371bbb78bSMatthias Springer rewriter.mergeBlockBefore(generateOp.getBody(), 34471bbb78bSMatthias Springer parallelBody->getTerminator(), 34571bbb78bSMatthias Springer parallelBody->getArguments()); 34671bbb78bSMatthias Springer // Replace the inlined yield op with a store op. The scf.parallel's builder 34771bbb78bSMatthias Springer // already populated an scf.yield at the end, so we don't need to worry 34871bbb78bSMatthias Springer // about creating that. 34971bbb78bSMatthias Springer Operation *elementYield = parallelBody->getTerminator()->getPrevNode(); 35071bbb78bSMatthias Springer rewriter.setInsertionPointAfter(elementYield); 35171bbb78bSMatthias Springer rewriter.replaceOpWithNewOp<memref::StoreOp>( 35271bbb78bSMatthias Springer elementYield, elementYield->getOperands()[0], result, 35371bbb78bSMatthias Springer parallelBody->getArguments()); 35471bbb78bSMatthias Springer 35571bbb78bSMatthias Springer replaceOpWithBufferizedValues(rewriter, op, result); 35671bbb78bSMatthias Springer return success(); 35771bbb78bSMatthias Springer } 35871bbb78bSMatthias Springer }; 35971bbb78bSMatthias Springer 36049e37000SMatthias Springer /// Bufferization of tensor.insert. Replace with memref.store. 36149e37000SMatthias Springer struct InsertOpInterface 36249e37000SMatthias Springer : public BufferizableOpInterface::ExternalModel<InsertOpInterface, 36349e37000SMatthias Springer tensor::InsertOp> { 36449e37000SMatthias Springer bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand, 36549e37000SMatthias Springer const BufferizationState &state) const { 36649e37000SMatthias Springer return true; 36749e37000SMatthias Springer } 36849e37000SMatthias Springer 36949e37000SMatthias Springer bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand, 37049e37000SMatthias Springer const BufferizationState &state) const { 37149e37000SMatthias Springer return true; 37249e37000SMatthias Springer } 37349e37000SMatthias Springer 37449e37000SMatthias Springer OpResult getAliasingOpResult(Operation *op, OpOperand &opOperand, 37549e37000SMatthias Springer const BufferizationState &state) const { 37649e37000SMatthias Springer assert(&opOperand == &op->getOpOperand(1) /*dest*/ && 37749e37000SMatthias Springer "expected dest OpOperand"); 37849e37000SMatthias Springer return op->getOpResult(0); 37949e37000SMatthias Springer } 38049e37000SMatthias Springer 38149e37000SMatthias Springer SmallVector<OpOperand *> 38249e37000SMatthias Springer getAliasingOpOperand(Operation *op, OpResult opResult, 38349e37000SMatthias Springer const BufferizationState &state) const { 38449e37000SMatthias Springer return {&op->getOpOperand(1) /*dest*/}; 38549e37000SMatthias Springer } 38649e37000SMatthias Springer 38749e37000SMatthias Springer LogicalResult bufferize(Operation *op, RewriterBase &rewriter, 38849e37000SMatthias Springer const BufferizationState &state) const { 38949e37000SMatthias Springer auto insertOp = cast<tensor::InsertOp>(op); 39049e37000SMatthias Springer FailureOr<Value> destMemref = 39149e37000SMatthias Springer state.getBuffer(rewriter, insertOp->getOpOperand(1) /*dest*/); 39249e37000SMatthias Springer if (failed(destMemref)) 39349e37000SMatthias Springer return failure(); 39449e37000SMatthias Springer rewriter.create<memref::StoreOp>(insertOp.getLoc(), insertOp.scalar(), 39549e37000SMatthias Springer *destMemref, insertOp.indices()); 39649e37000SMatthias Springer replaceOpWithBufferizedValues(rewriter, op, *destMemref); 39749e37000SMatthias Springer return success(); 39849e37000SMatthias Springer } 39949e37000SMatthias Springer 40049e37000SMatthias Springer BufferRelation bufferRelation(Operation *op, OpResult opResult, 40149e37000SMatthias Springer const BufferizationState &state) const { 40249e37000SMatthias Springer return BufferRelation::Equivalent; 40349e37000SMatthias Springer } 40449e37000SMatthias Springer }; 40549e37000SMatthias Springer 40649e37000SMatthias Springer /// Return true if the (ExtractSliceOp, InsertSliceOp) pair match (i.e. 40749e37000SMatthias Springer /// equivalent operand / result and same offset/sizes/strides specification). 40849e37000SMatthias Springer /// 40949e37000SMatthias Springer /// This is one particular type of relationship between ops on tensors that 41049e37000SMatthias Springer /// reduce to an equivalence on buffers. This should be generalized and 41149e37000SMatthias Springer /// exposed as interfaces on the proper types. 41249e37000SMatthias Springer static bool areEquivalentExtractSliceOps(const BufferizationState &state, 41349e37000SMatthias Springer ExtractSliceOp st, InsertSliceOp sti) { 41449e37000SMatthias Springer if (!st || !sti) 41549e37000SMatthias Springer return false; 41649e37000SMatthias Springer if (sti != sti && 41749e37000SMatthias Springer !state.areEquivalentBufferizedValues(st.source(), sti.dest())) 41849e37000SMatthias Springer return false; 41949e37000SMatthias Springer if (!sameOffsetsSizesAndStrides(st, sti, isEqualConstantIntOrValue)) 42049e37000SMatthias Springer return false; 42149e37000SMatthias Springer return true; 42249e37000SMatthias Springer } 42349e37000SMatthias Springer 42449e37000SMatthias Springer /// Return true if `value` is originating from an ExtractSliceOp that matches 42549e37000SMatthias Springer /// the given InsertSliceOp. 42649e37000SMatthias Springer static bool hasMatchingExtractSliceOp(const BufferizationState &state, 42749e37000SMatthias Springer Value value, InsertSliceOp insertOp) { 42849e37000SMatthias Springer auto condition = [&](Value val) { 42949e37000SMatthias Springer if (auto extractOp = val.getDefiningOp<ExtractSliceOp>()) 43049e37000SMatthias Springer if (areEquivalentExtractSliceOps(state, extractOp, insertOp)) 43149e37000SMatthias Springer return true; 43249e37000SMatthias Springer return false; 43349e37000SMatthias Springer }; 43449e37000SMatthias Springer 43549e37000SMatthias Springer return llvm::all_of(state.findValueInReverseUseDefChain(value, condition), 43649e37000SMatthias Springer condition); 43749e37000SMatthias Springer } 43849e37000SMatthias Springer 43949e37000SMatthias Springer /// Bufferization of tensor.insert_slice. Replace with a memory copy. Under 44049e37000SMatthias Springer /// certain circumstances, this op can also be a no-op. 44149e37000SMatthias Springer struct InsertSliceOpInterface 44249e37000SMatthias Springer : public BufferizableOpInterface::ExternalModel<InsertSliceOpInterface, 44349e37000SMatthias Springer tensor::InsertSliceOp> { 44449e37000SMatthias Springer bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand, 44549e37000SMatthias Springer const BufferizationState &state) const { 44649e37000SMatthias Springer return true; 44749e37000SMatthias Springer } 44849e37000SMatthias Springer 44949e37000SMatthias Springer bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand, 45049e37000SMatthias Springer const BufferizationState &state) const { 45149e37000SMatthias Springer return &opOperand == &op->getOpOperand(1) /*dest*/; 45249e37000SMatthias Springer } 45349e37000SMatthias Springer 45449e37000SMatthias Springer OpResult getAliasingOpResult(Operation *op, OpOperand &opOperand, 45549e37000SMatthias Springer const BufferizationState &state) const { 45649e37000SMatthias Springer return &opOperand == &op->getOpOperand(1) /*dest*/ 45749e37000SMatthias Springer ? op->getResult(0) 45849e37000SMatthias Springer : OpResult(); 45949e37000SMatthias Springer } 46049e37000SMatthias Springer 46149e37000SMatthias Springer BufferRelation bufferRelation(Operation *op, OpResult opResult, 46249e37000SMatthias Springer const BufferizationState &state) const { 46349e37000SMatthias Springer return BufferRelation::Equivalent; 46449e37000SMatthias Springer } 46549e37000SMatthias Springer 46649e37000SMatthias Springer bool isNotConflicting(Operation *op, OpOperand *uRead, 46749e37000SMatthias Springer OpOperand *uConflictingWrite, 46849e37000SMatthias Springer const BufferizationState &state) const { 46949e37000SMatthias Springer Operation *readingOp = uRead->getOwner(); 47049e37000SMatthias Springer Operation *conflictingWritingOp = uConflictingWrite->getOwner(); 47149e37000SMatthias Springer 47249e37000SMatthias Springer // Special rules for matching ExtractSliceOp/InsertSliceOp pairs. If 47349e37000SMatthias Springer // uRead is an InsertSliceOp... 47449e37000SMatthias Springer if (auto insertSliceOp = dyn_cast<InsertSliceOp>(readingOp)) { 47549e37000SMatthias Springer // As an example, consider the following IR. 47649e37000SMatthias Springer // 47749e37000SMatthias Springer // %0 = tensor.extract_slice %t[%a, %b][%c, %d][1, 1] {inplace = [true] } 47849e37000SMatthias Springer // %1 = linalg.fill %cst, %0 {inplace= [true] } 47949e37000SMatthias Springer // %2 = tensor.insert_slice %1 into %t[%a, %b][%c, %d][1, 1] 48049e37000SMatthias Springer // {inplace= [true] } 48149e37000SMatthias Springer 48249e37000SMatthias Springer // TODO: Use insertSliceOp.getDestOpOperand etc. when available. 48349e37000SMatthias Springer if (uRead == &insertSliceOp->getOpOperand(1) /*dest*/ && 48449e37000SMatthias Springer hasMatchingExtractSliceOp(state, uConflictingWrite->get(), 48549e37000SMatthias Springer insertSliceOp)) 48649e37000SMatthias Springer // Case 1: The main insight is that InsertSliceOp reads only part of 48749e37000SMatthias Springer // the destination tensor. The overwritten area is not read. If 48849e37000SMatthias Springer // uConflictingWrite writes into exactly the memory location that is 48949e37000SMatthias Springer // being read by uRead, this is not a conflict. 49049e37000SMatthias Springer // 49149e37000SMatthias Springer // In the above example: 49249e37000SMatthias Springer // uRead = OpOperand 1 (%t) of tensor.insert_slice 49349e37000SMatthias Springer // uConflictingWrite = OpOperand 1 (%0) of linalg.fill 49449e37000SMatthias Springer // 49549e37000SMatthias Springer // The read of %t does not conflict with the write of the FillOp 49649e37000SMatthias Springer // (same aliases!) because the area that the FillOp operates on is 49749e37000SMatthias Springer // exactly the one that is *not* read via %t. 49849e37000SMatthias Springer return true; 49949e37000SMatthias Springer 50049e37000SMatthias Springer if (uRead == &insertSliceOp->getOpOperand(0) /*source*/ && 50149e37000SMatthias Springer uConflictingWrite == &insertSliceOp->getOpOperand(1) /*dest*/ && 50249e37000SMatthias Springer hasMatchingExtractSliceOp(state, uRead->get(), insertSliceOp)) 50349e37000SMatthias Springer // Case 2: The read of the source tensor and the write to the dest 50449e37000SMatthias Springer // tensor via an InsertSliceOp is not a conflict if the read is 50549e37000SMatthias Springer // reading exactly that part of an equivalent tensor that the 50649e37000SMatthias Springer // InsertSliceOp is writing. 50749e37000SMatthias Springer // 50849e37000SMatthias Springer // In the above example: 50949e37000SMatthias Springer // uRead = OpOperand 0 (%1) of tensor.insert_slice 51049e37000SMatthias Springer // uConflictingWrite = OpOperand 1 (%t) of tensor.insert_slice 51149e37000SMatthias Springer return true; 51249e37000SMatthias Springer } 51349e37000SMatthias Springer 51449e37000SMatthias Springer // If uConflictingWrite is an InsertSliceOp... 51549e37000SMatthias Springer if (auto insertSliceOp = dyn_cast<InsertSliceOp>(conflictingWritingOp)) 51649e37000SMatthias Springer // As an example, consider the following IR. 51749e37000SMatthias Springer // 51849e37000SMatthias Springer // %0 = tensor.extract_slice %t[%a, %b][%c, %d][1, 1] {inplace = [true] } 51949e37000SMatthias Springer // %1 = linalg.fill %cst, %0 {inplace= [true] } 52049e37000SMatthias Springer // %2 = tensor.insert_slice %1 into %t[%a, %b][%c, %d][1, 1] 52149e37000SMatthias Springer // {inplace= [true] } 52249e37000SMatthias Springer // %3 = vector.transfer_read %1, %cst 52349e37000SMatthias Springer // 52449e37000SMatthias Springer // In the above example: 52549e37000SMatthias Springer // uRead = OpOperand 0 (%1) of vector.transfer_read 52649e37000SMatthias Springer // uConflictingWrite = OpOperand 1 (%t) of tensor.insert_slice 52749e37000SMatthias Springer // lastWrite = %1 52849e37000SMatthias Springer // 52949e37000SMatthias Springer // This is not a conflict because the InsertSliceOp overwrites the 53049e37000SMatthias Springer // memory segment of %1 with the exact same data. (Effectively, there 53149e37000SMatthias Springer // is no memory write here.) 53249e37000SMatthias Springer if (uConflictingWrite == &insertSliceOp->getOpOperand(1) /*dest*/ && 53349e37000SMatthias Springer state.areEquivalentBufferizedValues(uRead->get(), 53449e37000SMatthias Springer insertSliceOp.source()) && 53549e37000SMatthias Springer hasMatchingExtractSliceOp(state, insertSliceOp.source(), 53649e37000SMatthias Springer insertSliceOp)) 53749e37000SMatthias Springer return true; 53849e37000SMatthias Springer 53949e37000SMatthias Springer return false; 54049e37000SMatthias Springer } 54149e37000SMatthias Springer 54249e37000SMatthias Springer LogicalResult bufferize(Operation *op, RewriterBase &rewriter, 54349e37000SMatthias Springer const BufferizationState &state) const { 54449e37000SMatthias Springer // insert_slice ops arise from tiling and bufferizing them out-of-place is 54549e37000SMatthias Springer // generally a deal breaker. When used with loops, this ends up cloning the 54649e37000SMatthias Springer // whole tensor on every single iteration and is a symptom of a 54749e37000SMatthias Springer // catastrophically bad scheduling decision. 54849e37000SMatthias Springer // TODO: be very loud about it or even consider failing the pass. 54949e37000SMatthias Springer auto insertSliceOp = cast<tensor::InsertSliceOp>(op); 55049e37000SMatthias Springer Location loc = insertSliceOp.getLoc(); 55149e37000SMatthias Springer 55249e37000SMatthias Springer // When bufferizing out-of-place, `getResultBuffer` allocates. 55349e37000SMatthias Springer FailureOr<Value> dstMemref = 55449e37000SMatthias Springer state.getBuffer(rewriter, insertSliceOp->getOpOperand(1) /*dest*/); 55549e37000SMatthias Springer if (failed(dstMemref)) 55649e37000SMatthias Springer return failure(); 55749e37000SMatthias Springer 55849e37000SMatthias Springer // Expand offsets, sizes and strides to the full rank to handle the 55949e37000SMatthias Springer // rank-reducing case. 56049e37000SMatthias Springer SmallVector<OpFoldResult> mixedOffsets = insertSliceOp.getMixedOffsets(); 56149e37000SMatthias Springer SmallVector<OpFoldResult> mixedSizes = insertSliceOp.getMixedSizes(); 56249e37000SMatthias Springer SmallVector<OpFoldResult> mixedStrides = insertSliceOp.getMixedStrides(); 56349e37000SMatthias Springer OffsetSizeAndStrideOpInterface::expandToRank( 56449e37000SMatthias Springer *dstMemref, mixedOffsets, mixedSizes, mixedStrides, 56549e37000SMatthias Springer [&](Value target, int64_t dim) -> OpFoldResult { 56649e37000SMatthias Springer auto shapedType = target.getType().cast<ShapedType>(); 56749e37000SMatthias Springer if (shapedType.isDynamicDim(dim)) 56849e37000SMatthias Springer return rewriter.create<memref::DimOp>(loc, target, dim).result(); 56949e37000SMatthias Springer return rewriter.getIndexAttr(shapedType.getDimSize(dim)); 57049e37000SMatthias Springer }); 57149e37000SMatthias Springer // Take a subview of the dst. 57249e37000SMatthias Springer auto dstMemrefType = dstMemref->getType().cast<MemRefType>(); 57349e37000SMatthias Springer auto subviewMemRefType = 57449e37000SMatthias Springer memref::SubViewOp::inferRankReducedResultType( 57549e37000SMatthias Springer insertSliceOp.getSourceType().getRank(), dstMemrefType, 57649e37000SMatthias Springer mixedOffsets, mixedSizes, mixedStrides) 57749e37000SMatthias Springer .cast<MemRefType>(); 57849e37000SMatthias Springer Value subView = rewriter.create<memref::SubViewOp>( 57949e37000SMatthias Springer loc, subviewMemRefType, *dstMemref, mixedOffsets, mixedSizes, 58049e37000SMatthias Springer mixedStrides); 58149e37000SMatthias Springer 58249e37000SMatthias Springer // Copy tensor. If this tensor.insert_slice has a matching 58349e37000SMatthias Springer // tensor.extract_slice, the copy operation will eventually fold away. 58449e37000SMatthias Springer Value srcMemref = 58549e37000SMatthias Springer *state.getBuffer(rewriter, insertSliceOp->getOpOperand(0) /*source*/); 58649e37000SMatthias Springer if (failed(createMemCpy(rewriter, loc, srcMemref, subView, 58749e37000SMatthias Springer state.getOptions()))) 58849e37000SMatthias Springer return failure(); 58949e37000SMatthias Springer 59049e37000SMatthias Springer replaceOpWithBufferizedValues(rewriter, op, *dstMemref); 59149e37000SMatthias Springer return success(); 59249e37000SMatthias Springer } 59349e37000SMatthias Springer }; 59449e37000SMatthias Springer 595fc08d1c2SMatthias Springer /// Bufferization of tensor.rank. Replace with memref.rank. 596fc08d1c2SMatthias Springer struct RankOpInterface 597fc08d1c2SMatthias Springer : public BufferizableOpInterface::ExternalModel<RankOpInterface, 598fc08d1c2SMatthias Springer tensor::RankOp> { 599fc08d1c2SMatthias Springer bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand, 600fc08d1c2SMatthias Springer const BufferizationState &state) const { 601fc08d1c2SMatthias Springer return true; 602fc08d1c2SMatthias Springer } 603fc08d1c2SMatthias Springer 604fc08d1c2SMatthias Springer bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand, 605fc08d1c2SMatthias Springer const BufferizationState &state) const { 606fc08d1c2SMatthias Springer return false; 607fc08d1c2SMatthias Springer } 608fc08d1c2SMatthias Springer 609fc08d1c2SMatthias Springer OpResult getAliasingOpResult(Operation *op, OpOperand &opOperand, 610fc08d1c2SMatthias Springer const BufferizationState &state) const { 611fc08d1c2SMatthias Springer return OpResult(); 612fc08d1c2SMatthias Springer } 613fc08d1c2SMatthias Springer 614fc08d1c2SMatthias Springer LogicalResult bufferize(Operation *op, RewriterBase &rewriter, 615fc08d1c2SMatthias Springer const BufferizationState &state) const { 616fc08d1c2SMatthias Springer auto rankOp = cast<tensor::RankOp>(op); 617fc08d1c2SMatthias Springer Value v = *state.getBuffer(rewriter, rankOp->getOpOperand(0) /*source*/); 618fc08d1c2SMatthias Springer replaceOpWithNewBufferizedOp<memref::RankOp>(rewriter, op, rankOp.getType(), 619fc08d1c2SMatthias Springer v); 620fc08d1c2SMatthias Springer return success(); 621fc08d1c2SMatthias Springer } 622fc08d1c2SMatthias Springer }; 623fc08d1c2SMatthias Springer 62449e37000SMatthias Springer } // namespace 62549e37000SMatthias Springer } // namespace tensor 62649e37000SMatthias Springer } // namespace mlir 62749e37000SMatthias Springer 62849e37000SMatthias Springer void mlir::tensor::registerBufferizableOpInterfaceExternalModels( 62949e37000SMatthias Springer DialectRegistry ®istry) { 63049e37000SMatthias Springer registry.addOpInterface<CastOp, CastOpInterface>(); 63149e37000SMatthias Springer registry.addOpInterface<DimOp, DimOpInterface>(); 63249e37000SMatthias Springer registry.addOpInterface<ExtractSliceOp, ExtractSliceOpInterface>(); 63349e37000SMatthias Springer registry.addOpInterface<ExtractOp, ExtractOpInterface>(); 634d581c94dSMatthias Springer registry.addOpInterface<FromElementsOp, FromElementsOpInterface>(); 63571bbb78bSMatthias Springer registry.addOpInterface<GenerateOp, GenerateOpInterface>(); 63649e37000SMatthias Springer registry.addOpInterface<InsertOp, InsertOpInterface>(); 63749e37000SMatthias Springer registry.addOpInterface<InsertSliceOp, InsertSliceOpInterface>(); 638fc08d1c2SMatthias Springer registry.addOpInterface<RankOp, RankOpInterface>(); 63949e37000SMatthias Springer } 640