1 //===----------------------------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
10 #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
11 #include "mlir/Dialect/Bufferization/IR/Bufferization.h"
12 #include "mlir/Dialect/MemRef/IR/MemRef.h"
13 #include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
14 #include "mlir/Dialect/Tensor/IR/Tensor.h"
15 #include "mlir/IR/Matchers.h"
16 
17 using namespace mlir;
18 using namespace mlir::bufferization;
19 
20 //===----------------------------------------------------------------------===//
21 // Helper functions
22 //===----------------------------------------------------------------------===//
23 
24 FailureOr<Value>
25 mlir::bufferization::castOrReallocMemRefValue(OpBuilder &b, Value value,
26                                               MemRefType destType) {
27   auto srcType = value.getType().cast<MemRefType>();
28 
29   // Element type, rank and memory space must match.
30   if (srcType.getElementType() != destType.getElementType())
31     return failure();
32   if (srcType.getMemorySpaceAsInt() != destType.getMemorySpaceAsInt())
33     return failure();
34   if (srcType.getRank() != destType.getRank())
35     return failure();
36 
37   // In case the affine maps are different, we may need to use a copy if we go
38   // from dynamic to static offset or stride (the canonicalization cannot know
39   // at this point that it is really cast compatible).
40   auto isGuaranteedCastCompatible = [](MemRefType source, MemRefType target) {
41     int64_t sourceOffset, targetOffset;
42     SmallVector<int64_t, 4> sourceStrides, targetStrides;
43     if (failed(getStridesAndOffset(source, sourceStrides, sourceOffset)) ||
44         failed(getStridesAndOffset(target, targetStrides, targetOffset)))
45       return false;
46     auto dynamicToStatic = [](int64_t a, int64_t b) {
47       return a == MemRefType::getDynamicStrideOrOffset() &&
48              b != MemRefType::getDynamicStrideOrOffset();
49     };
50     if (dynamicToStatic(sourceOffset, targetOffset))
51       return false;
52     for (auto it : zip(sourceStrides, targetStrides))
53       if (dynamicToStatic(std::get<0>(it), std::get<1>(it)))
54         return false;
55     return true;
56   };
57 
58   // Note: If `areCastCompatible`, a cast is valid, but may fail at runtime. To
59   // ensure that we only generate casts that always succeed at runtime, we check
60   // a fix extra conditions in `isGuaranteedCastCompatible`.
61   if (memref::CastOp::areCastCompatible(srcType, destType) &&
62       isGuaranteedCastCompatible(srcType, destType)) {
63     Value casted = b.create<memref::CastOp>(value.getLoc(), destType, value);
64     return casted;
65   }
66 
67   auto loc = value.getLoc();
68   SmallVector<Value, 4> dynamicOperands;
69   for (int i = 0; i < destType.getRank(); ++i) {
70     if (destType.getShape()[i] != ShapedType::kDynamicSize)
71       continue;
72     auto index = b.createOrFold<arith::ConstantIndexOp>(loc, i);
73     Value size = b.create<memref::DimOp>(loc, value, index);
74     dynamicOperands.push_back(size);
75   }
76   // TODO: Use alloc/memcpy callback from BufferizationOptions if called via
77   // BufferizableOpInterface impl of ToMemrefOp.
78   Value copy = b.create<memref::AllocOp>(loc, destType, dynamicOperands);
79   b.create<memref::CopyOp>(loc, value, copy);
80   return copy;
81 }
82 
83 /// Try to fold to_memref(to_tensor(x)). If x's type and the result type of the
84 /// to_memref op are different, a memref.cast is needed.
85 LogicalResult mlir::bufferization::foldToMemrefToTensorPair(
86     RewriterBase &rewriter, ToMemrefOp toMemref, bool allowSameType) {
87   auto memrefToTensor = toMemref.getTensor().getDefiningOp<ToTensorOp>();
88   if (!memrefToTensor)
89     return failure();
90 
91   Type srcType = memrefToTensor.getMemref().getType();
92   Type destType = toMemref.getType();
93 
94   // Directly rewrite if the type did not change.
95   if (srcType == destType) {
96     // Function can be configured to only handle cases where a cast is needed.
97     if (!allowSameType)
98       return failure();
99     rewriter.replaceOp(toMemref, memrefToTensor.getMemref());
100     return success();
101   }
102 
103   auto rankedSrcType = srcType.dyn_cast<MemRefType>();
104   auto rankedDestType = destType.dyn_cast<MemRefType>();
105   auto unrankedSrcType = srcType.dyn_cast<UnrankedMemRefType>();
106 
107   // Ranked memref -> Ranked memref cast.
108   if (rankedSrcType && rankedDestType) {
109     FailureOr<Value> replacement = castOrReallocMemRefValue(
110         rewriter, memrefToTensor.getMemref(), rankedDestType);
111     if (failed(replacement))
112       return failure();
113 
114     rewriter.replaceOp(toMemref, *replacement);
115     return success();
116   }
117 
118   // Unranked memref -> Ranked memref cast: May require a copy.
119   // TODO: Not implemented at the moment.
120   if (unrankedSrcType && rankedDestType)
121     return failure();
122 
123   // Unranked memref -> unranked memref cast
124   // Ranked memref -> unranked memref cast: No copy needed.
125   assert(memref::CastOp::areCastCompatible(srcType, destType) &&
126          "expected that types are cast compatible");
127   rewriter.replaceOpWithNewOp<memref::CastOp>(toMemref, destType,
128                                               memrefToTensor.getMemref());
129   return success();
130 }
131 
132 void mlir::bufferization::populateDynamicDimSizes(
133     OpBuilder &b, Location loc, Value shapedValue,
134     SmallVector<Value> &dynamicDims) {
135   auto shapedType = shapedValue.getType().cast<ShapedType>();
136   for (int64_t i = 0; i < shapedType.getRank(); ++i) {
137     if (shapedType.isDynamicDim(i)) {
138       if (shapedType.isa<MemRefType>()) {
139         dynamicDims.push_back(b.create<memref::DimOp>(loc, shapedValue, i));
140       } else {
141         assert(shapedType.isa<RankedTensorType>() && "expected tensor");
142         dynamicDims.push_back(b.create<tensor::DimOp>(loc, shapedValue, i));
143       }
144     }
145   }
146 }
147 
148 //===----------------------------------------------------------------------===//
149 // AllocTensorOp
150 //===----------------------------------------------------------------------===//
151 
152 LogicalResult AllocTensorOp::bufferize(RewriterBase &rewriter,
153                                        const BufferizationOptions &options) {
154   OpBuilder::InsertionGuard g(rewriter);
155   Operation *op = this->getOperation();
156   Location loc = getLoc();
157 
158   // Nothing to do for dead AllocTensorOps.
159   if (getOperation()->getUses().empty()) {
160     rewriter.eraseOp(getOperation());
161     return success();
162   }
163 
164   // Create buffer allocation.
165   Value copyBuffer;
166   if (getCopy())
167     copyBuffer = getBuffer(rewriter, getCopy(), options);
168   auto allocType =
169       MemRefType::get(getType().getShape(), getType().getElementType());
170   SmallVector<Value> dynamicDims = getDynamicSizes();
171   if (getCopy()) {
172     assert(dynamicDims.empty() && "expected either `copy` or `dynamicDims`");
173     populateDynamicDimSizes(rewriter, loc, copyBuffer, dynamicDims);
174   }
175   FailureOr<Value> alloc =
176       options.createAlloc(rewriter, loc, allocType, dynamicDims);
177   if (failed(alloc))
178     return failure();
179 
180   // Create memory copy (if any).
181   if (getCopy()) {
182     if (failed(options.createMemCpy(rewriter, loc, copyBuffer, *alloc)))
183       return failure();
184   }
185 
186   // Should the buffer be deallocated?
187   AnalysisState analysisState(options);
188   bool dealloc;
189   if (op->hasAttr(BufferizationDialect::kEscapeAttrName)) {
190     // AllocTensorOp has one result.
191     ArrayAttr escapeAttr =
192         op->getAttr(BufferizationDialect::kEscapeAttrName).cast<ArrayAttr>();
193     dealloc = !escapeAttr[0].cast<BoolAttr>().getValue();
194   } else {
195     // No "escape" annotation found.
196     if (options.createDeallocs) {
197       // Perform an ad-hoc analysis.
198       dealloc = !analysisState.isTensorYielded(getResult());
199     } else {
200       dealloc = false;
201     }
202   }
203 
204   // Replace op.
205   replaceOpWithBufferizedValues(rewriter, getOperation(), *alloc);
206 
207   // Create buffer deallocation (if requested).
208   if (!dealloc)
209     return success();
210 
211   rewriter.setInsertionPoint(rewriter.getInsertionBlock()->getTerminator());
212   if (failed(options.createDealloc(rewriter, loc, *alloc)))
213     return failure();
214   return success();
215 }
216 
217 bool AllocTensorOp::isMemoryWrite(OpResult opResult,
218                                   const AnalysisState &state) {
219   // AllocTensorOps do not write unless they have a `copy` value.
220   return static_cast<bool>(getCopy());
221 }
222 
223 bool AllocTensorOp::bufferizesToMemoryRead(OpOperand &opOperand,
224                                            const AnalysisState &state) {
225   assert(opOperand.getOperandNumber() == getNumOperands() - 1 &&
226          "expected copy operand");
227   return true;
228 }
229 
230 bool AllocTensorOp::bufferizesToMemoryWrite(OpOperand &opOperand,
231                                             const AnalysisState &state) {
232   assert(opOperand.getOperandNumber() == getNumOperands() - 1 &&
233          "expected copy operand");
234   return false;
235 }
236 
237 SmallVector<OpResult>
238 AllocTensorOp::getAliasingOpResult(OpOperand &opOperand,
239                                    const AnalysisState &state) {
240   // This is a new allocation. It does not alias with any other buffer.
241   return {};
242 }
243 
244 LogicalResult AllocTensorOp::verify() {
245   if (getCopy() && !getDynamicSizes().empty())
246     return emitError("dynamic sizes not needed when copying a tensor");
247   if (!getCopy() && getType().getNumDynamicDims() !=
248                         static_cast<int64_t>(getDynamicSizes().size()))
249     return emitError("expected ")
250            << getType().getNumDynamicDims() << " dynamic sizes";
251   if (getCopy() && getCopy().getType() != getType())
252     return emitError("expected that `copy` and return type match");
253   return success();
254 }
255 
256 void AllocTensorOp::build(OpBuilder &builder, OperationState &result,
257                           RankedTensorType type, ValueRange dynamicSizes) {
258   build(builder, result, type, dynamicSizes, /*copy=*/Value());
259 }
260 
261 namespace {
262 /// Change the type of the result of a `bufferization.alloc_tensor` by making
263 /// the result type statically sized along dimension that in the original
264 /// operation where defined as dynamic, but the size was defined using a
265 /// `constant` op. For example:
266 ///
267 ///  %c5 = arith.constant 5: index
268 ///  %0 = bufferization.alloc_tensor(%arg0, %c5) : tensor<?x?xf32>
269 ///
270 ///  to
271 ///
272 ///  %0 = bufferization.alloc_tensor(%arg0) : tensor<?x5xf32>
273 struct ReplaceStaticShapeDims : OpRewritePattern<AllocTensorOp> {
274   using OpRewritePattern<AllocTensorOp>::OpRewritePattern;
275 
276   LogicalResult matchAndRewrite(AllocTensorOp op,
277                                 PatternRewriter &rewriter) const override {
278     if (op.getCopy())
279       return failure();
280     SmallVector<int64_t> newShape = llvm::to_vector(op.getType().getShape());
281     SmallVector<Value> newDynamicSizes;
282     unsigned int dynValCounter = 0;
283     for (int64_t i = 0; i < op.getType().getRank(); ++i) {
284       if (!op.isDynamicDim(i))
285         continue;
286       Value value = op.getDynamicSizes()[dynValCounter++];
287       APInt intVal;
288       if (matchPattern(value, m_ConstantInt(&intVal))) {
289         newShape[i] = intVal.getSExtValue();
290       } else {
291         newDynamicSizes.push_back(value);
292       }
293     }
294     RankedTensorType newType = RankedTensorType::get(
295         newShape, op.getType().getElementType(), op.getType().getEncoding());
296     if (newType == op.getType())
297       return failure();
298     auto newOp = rewriter.create<AllocTensorOp>(
299         op.getLoc(), newType, newDynamicSizes, /*copy=*/Value());
300     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(), newOp);
301     return success();
302   }
303 };
304 
305 struct FoldDimOfAllocTensorOp : public OpRewritePattern<tensor::DimOp> {
306   using OpRewritePattern<tensor::DimOp>::OpRewritePattern;
307 
308   LogicalResult matchAndRewrite(tensor::DimOp dimOp,
309                                 PatternRewriter &rewriter) const override {
310     Optional<int64_t> maybeConstantIndex = dimOp.getConstantIndex();
311     auto allocTensorOp = dimOp.source().getDefiningOp<AllocTensorOp>();
312     if (!allocTensorOp || !maybeConstantIndex)
313       return failure();
314     if (!allocTensorOp.getType().isDynamicDim(*maybeConstantIndex))
315       return failure();
316     rewriter.replaceOp(
317         dimOp, allocTensorOp.getDynamicSize(rewriter, *maybeConstantIndex));
318     return success();
319   }
320 };
321 } // namespace
322 
323 void AllocTensorOp::getCanonicalizationPatterns(RewritePatternSet &results,
324                                                 MLIRContext *ctx) {
325   results.add<FoldDimOfAllocTensorOp, ReplaceStaticShapeDims>(ctx);
326 }
327 
328 LogicalResult AllocTensorOp::reifyResultShapes(
329     OpBuilder &builder, ReifiedRankedShapedTypeDims &reifiedReturnShapes) {
330   auto shapes = llvm::to_vector<4>(llvm::map_range(
331       llvm::seq<int64_t>(0, getType().getRank()), [&](int64_t dim) -> Value {
332         if (isDynamicDim(dim))
333           return getDynamicSize(builder, dim);
334         return builder.create<arith::ConstantIndexOp>(getLoc(),
335                                                       getStaticSize(dim));
336       }));
337   reifiedReturnShapes.emplace_back(std::move(shapes));
338   return success();
339 }
340 
341 ParseResult AllocTensorOp::parse(OpAsmParser &parser, OperationState &result) {
342   SmallVector<OpAsmParser::UnresolvedOperand> dynamicSizesOperands;
343   if (parser.parseLParen() || parser.parseOperandList(dynamicSizesOperands) ||
344       parser.parseRParen())
345     return failure();
346   ParseResult copyKeyword = parser.parseOptionalKeyword("copy");
347   OpAsmParser::UnresolvedOperand copyOperand;
348   if (copyKeyword.succeeded())
349     if (parser.parseLParen() || parser.parseOperand(copyOperand) ||
350         parser.parseRParen())
351       return failure();
352   if (parser.parseOptionalAttrDict(result.attributes) || parser.parseColon())
353     return failure();
354 
355   TensorType type;
356   if (parser.parseCustomTypeWithFallback(type))
357     return failure();
358   result.addTypes(type);
359 
360   Type indexType = parser.getBuilder().getIndexType();
361   if (parser.resolveOperands(dynamicSizesOperands, indexType, result.operands))
362     return failure();
363   if (copyKeyword.succeeded())
364     if (parser.resolveOperand(copyOperand, type, result.operands))
365       return failure();
366   result.addAttribute(AllocTensorOp::getOperandSegmentSizeAttr(),
367                       parser.getBuilder().getI32VectorAttr(
368                           {static_cast<int32_t>(dynamicSizesOperands.size()),
369                            static_cast<int32_t>(copyKeyword.succeeded())}));
370   return success();
371 }
372 
373 void AllocTensorOp::print(OpAsmPrinter &p) {
374   p << "(" << getDynamicSizes() << ")";
375   if (getCopy())
376     p << " copy(" << getCopy() << ")";
377   p.printOptionalAttrDict((*this)->getAttrs(), /*elidedAttrs=*/{
378                               AllocTensorOp::getOperandSegmentSizeAttr()});
379   p << " : ";
380   auto type = getResult().getType();
381   if (auto validType = type.dyn_cast<::mlir::TensorType>())
382     p.printStrippedAttrOrType(validType);
383   else
384     p << type;
385 }
386 
387 Value AllocTensorOp::getDynamicSize(OpBuilder &b, unsigned idx) {
388   assert(isDynamicDim(idx) && "expected dynamic dim");
389   if (getCopy())
390     return b.create<tensor::DimOp>(getLoc(), getCopy(), idx);
391   return getOperand(getIndexOfDynamicSize(idx));
392 }
393 
394 //===----------------------------------------------------------------------===//
395 // CloneOp
396 //===----------------------------------------------------------------------===//
397 
398 void CloneOp::getEffects(
399     SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
400         &effects) {
401   effects.emplace_back(MemoryEffects::Read::get(), getInput(),
402                        SideEffects::DefaultResource::get());
403   effects.emplace_back(MemoryEffects::Write::get(), getOutput(),
404                        SideEffects::DefaultResource::get());
405   effects.emplace_back(MemoryEffects::Allocate::get(), getOutput(),
406                        SideEffects::DefaultResource::get());
407 }
408 
409 OpFoldResult CloneOp::fold(ArrayRef<Attribute> operands) {
410   return succeeded(memref::foldMemRefCast(*this)) ? getResult() : Value();
411 }
412 
413 namespace {
414 
415 /// Merge the clone and its source (by converting the clone to a cast) when
416 /// possible.
417 struct SimplifyClones : public OpRewritePattern<CloneOp> {
418   using OpRewritePattern<CloneOp>::OpRewritePattern;
419 
420   LogicalResult matchAndRewrite(CloneOp cloneOp,
421                                 PatternRewriter &rewriter) const override {
422     if (cloneOp.use_empty()) {
423       rewriter.eraseOp(cloneOp);
424       return success();
425     }
426 
427     Value source = cloneOp.getInput();
428 
429     // This only finds dealloc operations for the immediate value. It should
430     // also consider aliases. That would also make the safety check below
431     // redundant.
432     llvm::Optional<Operation *> maybeCloneDeallocOp =
433         memref::findDealloc(cloneOp.getOutput());
434     // Skip if either of them has > 1 deallocate operations.
435     if (!maybeCloneDeallocOp.hasValue())
436       return failure();
437     llvm::Optional<Operation *> maybeSourceDeallocOp =
438         memref::findDealloc(source);
439     if (!maybeSourceDeallocOp.hasValue())
440       return failure();
441     Operation *cloneDeallocOp = *maybeCloneDeallocOp;
442     Operation *sourceDeallocOp = *maybeSourceDeallocOp;
443 
444     // If both are deallocated in the same block, their in-block lifetimes
445     // might not fully overlap, so we cannot decide which one to drop.
446     if (cloneDeallocOp && sourceDeallocOp &&
447         cloneDeallocOp->getBlock() == sourceDeallocOp->getBlock())
448       return failure();
449 
450     Block *currentBlock = cloneOp->getBlock();
451     Operation *redundantDealloc = nullptr;
452     if (cloneDeallocOp && cloneDeallocOp->getBlock() == currentBlock) {
453       redundantDealloc = cloneDeallocOp;
454     } else if (sourceDeallocOp && sourceDeallocOp->getBlock() == currentBlock) {
455       redundantDealloc = sourceDeallocOp;
456     }
457 
458     if (!redundantDealloc)
459       return failure();
460 
461     // Safety check that there are no other deallocations inbetween
462     // cloneOp and redundantDealloc, as otherwise we might deallocate an alias
463     // of source before the uses of the clone. With alias information, we could
464     // restrict this to only fail of the dealloc's operand is an alias
465     // of the source.
466     for (Operation *pos = cloneOp->getNextNode(); pos != redundantDealloc;
467          pos = pos->getNextNode()) {
468       auto effectInterface = dyn_cast<MemoryEffectOpInterface>(pos);
469       if (!effectInterface)
470         continue;
471       if (effectInterface.hasEffect<MemoryEffects::Free>())
472         return failure();
473     }
474 
475     rewriter.replaceOpWithNewOp<memref::CastOp>(cloneOp, cloneOp.getType(),
476                                                 source);
477     rewriter.eraseOp(redundantDealloc);
478     return success();
479   }
480 };
481 
482 } // namespace
483 
484 void CloneOp::getCanonicalizationPatterns(RewritePatternSet &results,
485                                           MLIRContext *context) {
486   results.add<SimplifyClones>(context);
487 }
488 
489 //===----------------------------------------------------------------------===//
490 // ToTensorOp
491 //===----------------------------------------------------------------------===//
492 
493 OpFoldResult ToTensorOp::fold(ArrayRef<Attribute>) {
494   if (auto toMemref = getMemref().getDefiningOp<ToMemrefOp>())
495     // Approximate alias analysis by conservatively folding only when no there
496     // is no interleaved operation.
497     if (toMemref->getBlock() == this->getOperation()->getBlock() &&
498         toMemref->getNextNode() == this->getOperation())
499       return toMemref.getTensor();
500   return {};
501 }
502 
503 namespace {
504 
505 struct DimOfToTensorFolder : public OpRewritePattern<tensor::DimOp> {
506   using OpRewritePattern<tensor::DimOp>::OpRewritePattern;
507 
508   LogicalResult matchAndRewrite(tensor::DimOp dimOp,
509                                 PatternRewriter &rewriter) const override {
510     auto memrefToTensorOp = dimOp.source().getDefiningOp<ToTensorOp>();
511     if (!memrefToTensorOp)
512       return failure();
513 
514     rewriter.replaceOpWithNewOp<memref::DimOp>(
515         dimOp, memrefToTensorOp.getMemref(), dimOp.index());
516     return success();
517   }
518 };
519 
520 } // namespace
521 
522 void ToTensorOp::getCanonicalizationPatterns(RewritePatternSet &results,
523                                              MLIRContext *context) {
524   results.add<DimOfToTensorFolder>(context);
525 }
526 
527 //===----------------------------------------------------------------------===//
528 // ToMemrefOp
529 //===----------------------------------------------------------------------===//
530 
531 OpFoldResult ToMemrefOp::fold(ArrayRef<Attribute>) {
532   if (auto memrefToTensor = getTensor().getDefiningOp<ToTensorOp>())
533     if (memrefToTensor.getMemref().getType() == getType())
534       return memrefToTensor.getMemref();
535   return {};
536 }
537 
538 namespace {
539 
540 /// Replace tensor.cast + to_memref by to_memref + memref.cast.
541 struct ToMemrefOfCast : public OpRewritePattern<ToMemrefOp> {
542   using OpRewritePattern<ToMemrefOp>::OpRewritePattern;
543 
544   LogicalResult matchAndRewrite(ToMemrefOp toMemref,
545                                 PatternRewriter &rewriter) const final {
546     auto tensorCastOperand =
547         toMemref.getOperand().getDefiningOp<tensor::CastOp>();
548     if (!tensorCastOperand)
549       return failure();
550     auto srcTensorType =
551         tensorCastOperand.getOperand().getType().dyn_cast<RankedTensorType>();
552     if (!srcTensorType)
553       return failure();
554     auto memrefType = MemRefType::get(srcTensorType.getShape(),
555                                       srcTensorType.getElementType());
556     Value memref = rewriter.create<ToMemrefOp>(toMemref.getLoc(), memrefType,
557                                                tensorCastOperand.getOperand());
558     rewriter.replaceOpWithNewOp<memref::CastOp>(toMemref, toMemref.getType(),
559                                                 memref);
560     return success();
561   }
562 };
563 
564 /// Canonicalize bufferization.to_tensor + bufferization.to_memref to
565 /// memref.cast when type mismatches prevent `ToMemrefOp::fold` to kick in.
566 struct TensorLoadToMemref : public OpRewritePattern<ToMemrefOp> {
567   using OpRewritePattern<ToMemrefOp>::OpRewritePattern;
568 
569   LogicalResult matchAndRewrite(ToMemrefOp toMemref,
570                                 PatternRewriter &rewriter) const final {
571     // Only handle cases where a cast is needed. The other case is handled by
572     // the folder.
573     return foldToMemrefToTensorPair(rewriter, toMemref,
574                                     /*allowSameType=*/false);
575   }
576 };
577 
578 /// Fold a load on a to_memref operation into an tensor.extract on the
579 /// corresponding tensor.
580 struct LoadOfToMemref : public OpRewritePattern<memref::LoadOp> {
581   using OpRewritePattern<memref::LoadOp>::OpRewritePattern;
582 
583   LogicalResult matchAndRewrite(memref::LoadOp load,
584                                 PatternRewriter &rewriter) const override {
585     auto toMemref = load.memref().getDefiningOp<ToMemrefOp>();
586     if (!toMemref)
587       return failure();
588 
589     rewriter.replaceOpWithNewOp<tensor::ExtractOp>(load, toMemref.getTensor(),
590                                                    load.indices());
591     return success();
592   }
593 };
594 
595 /// Fold dim of a to_memref into the dim of the tensor.
596 struct DimOfCastOp : public OpRewritePattern<memref::DimOp> {
597   using OpRewritePattern<memref::DimOp>::OpRewritePattern;
598 
599   LogicalResult matchAndRewrite(memref::DimOp dimOp,
600                                 PatternRewriter &rewriter) const override {
601     auto castOp = dimOp.source().getDefiningOp<ToMemrefOp>();
602     if (!castOp)
603       return failure();
604     Value newSource = castOp.getOperand();
605     rewriter.replaceOpWithNewOp<tensor::DimOp>(dimOp, newSource, dimOp.index());
606     return success();
607   }
608 };
609 
610 } // namespace
611 
612 void ToMemrefOp::getCanonicalizationPatterns(RewritePatternSet &results,
613                                              MLIRContext *context) {
614   results.add<DimOfCastOp, LoadOfToMemref, ToMemrefOfCast, TensorLoadToMemref>(
615       context);
616 }
617 
618 LogicalResult ToMemrefOp::bufferize(RewriterBase &rewriter,
619                                     const BufferizationOptions &options) {
620   // Fold to_memref(to_tensor(x)) to x. Insert a cast if necessary.
621   (void)foldToMemrefToTensorPair(rewriter, *this);
622   // Note: The return value of `bufferize` indicates whether there was an error
623   // or not. (And not whether the pattern matched or not.)
624   return success();
625 }
626 
627 Optional<Operation *> CloneOp::buildDealloc(OpBuilder &builder, Value alloc) {
628   return builder.create<memref::DeallocOp>(alloc.getLoc(), alloc)
629       .getOperation();
630 }
631 
632 Optional<Value> CloneOp::buildClone(OpBuilder &builder, Value alloc) {
633   return builder.create<CloneOp>(alloc.getLoc(), alloc).getResult();
634 }
635 
636 //===----------------------------------------------------------------------===//
637 // TableGen'd op method definitions
638 //===----------------------------------------------------------------------===//
639 
640 #define GET_OP_CLASSES
641 #include "mlir/Dialect/Bufferization/IR/BufferizationOps.cpp.inc"
642