Lines Matching refs:mlir
28 using namespace mlir;
30 using OperationUseMapT = llvm::DenseMap<mlir::Operation *, mlir::Operation *>;
68 using ConflictSetT = llvm::SmallPtrSet<mlir::Operation *, 16>;
69 using UseSetT = llvm::SmallPtrSet<mlir::OpOperand *, 8>;
70 using LoadMapSetsT = llvm::DenseMap<mlir::Operation *, UseSetT>;
71 using AmendAccessSetT = llvm::SmallPtrSet<mlir::Operation *, 4>;
73 ArrayCopyAnalysis(mlir::Operation *op) : operation{op} { construct(op); } in ArrayCopyAnalysis()
75 mlir::Operation *getOperation() const { return operation; } in getOperation()
78 bool hasPotentialConflict(mlir::Operation *op) const { in hasPotentialConflict()
94 bool inAmendAccessSet(mlir::Operation *op) const { in inAmendAccessSet()
99 UseSetT getLoadUseSet(mlir::Operation *load) const { in getLoadUseSet()
104 void arrayMentions(llvm::SmallVectorImpl<mlir::Operation *> &mentions,
108 void construct(mlir::Operation *topLevelOp);
110 mlir::Operation *operation; // operation that analysis ran upon
123 ReachCollector(llvm::SmallVectorImpl<mlir::Operation *> &reach, in ReachCollector()
124 mlir::Region *loopRegion) in ReachCollector()
127 void collectArrayMentionFrom(mlir::Operation *op, mlir::ValueRange range) { in collectArrayMentionFrom()
129 collectArrayMentionFrom(op, mlir::Value{}); in collectArrayMentionFrom()
132 for (mlir::Value v : range) in collectArrayMentionFrom()
144 mlir::Block *block) { in collectAccesses()
146 if (auto access = mlir::dyn_cast<ArrayAccessOp>(op)) { in collectAccesses()
157 void collectArrayMentionFrom(mlir::Operation *op, mlir::Value val) { in collectArrayMentionFrom()
164 auto resNum = val.cast<mlir::OpResult>().getResultNumber(); in collectArrayMentionFrom()
165 llvm::SmallVector<mlir::Value> results; in collectArrayMentionFrom()
170 if (auto rop = mlir::dyn_cast<DoLoopOp>(op)) { in collectArrayMentionFrom()
174 if (auto rop = mlir::dyn_cast<IterWhileOp>(op)) { in collectArrayMentionFrom()
178 if (auto rop = mlir::dyn_cast<fir::IfOp>(op)) { in collectArrayMentionFrom()
182 if (auto box = mlir::dyn_cast<EmboxOp>(op)) { in collectArrayMentionFrom()
188 if (auto mergeStore = mlir::dyn_cast<ArrayMergeStoreOp>(op)) { in collectArrayMentionFrom()
194 if (mlir::isa<AllocaOp, AllocMemOp>(op)) { in collectArrayMentionFrom()
198 if (auto store = mlir::dyn_cast<fir::StoreOp>(user)) in collectArrayMentionFrom()
205 if (auto amend = mlir::dyn_cast<ArrayAmendOp>(op)) { in collectArrayMentionFrom()
214 if (mlir::isa<ArrayAccessOp, ArrayLoadOp, ArrayUpdateOp, ArrayModifyOp, in collectArrayMentionFrom()
221 if (auto arrLd = mlir::dyn_cast<ArrayLoadOp>(op)) in collectArrayMentionFrom()
223 if (mlir::isa<ArrayAccessOp>(user)) { in collectArrayMentionFrom()
230 if (mlir::isa<ArrayModifyOp>(op)) in collectArrayMentionFrom()
234 if (mlir::isa<fir::CallOp>(op)) { in collectArrayMentionFrom()
243 void collectArrayMentionFrom(mlir::BlockArgument ba) { in collectArrayMentionFrom()
250 if (auto rop = mlir::dyn_cast<DoLoopOp>(parent)) { in collectArrayMentionFrom()
254 if (auto rop = mlir::dyn_cast<IterWhileOp>(parent)) { in collectArrayMentionFrom()
267 void collectArrayMentionFrom(mlir::Value val) { in collectArrayMentionFrom()
273 if (auto ba = val.dyn_cast<mlir::BlockArgument>()) { in collectArrayMentionFrom()
289 static void reachingValues(llvm::SmallVectorImpl<mlir::Operation *> &reach, in reachingValues()
290 mlir::Value seq) { in reachingValues()
292 mlir::Region *loopRegion = nullptr; in reachingValues()
293 if (auto doLoop = mlir::dyn_cast_or_null<DoLoopOp>(seq.getDefiningOp())) in reachingValues()
302 bool opIsInsideLoops(mlir::Operation *op) const { in opIsInsideLoops()
314 void followUsers(mlir::Operation *op) { in followUsers()
322 llvm::SmallVectorImpl<mlir::Operation *> &reach;
323 llvm::SmallPtrSet<mlir::Value, 16> visited;
325 mlir::Region *loopRegion;
332 llvm::SmallVectorImpl<mlir::Operation *> &mentions, ArrayLoadOp load) { in arrayMentions()
338 if (mlir::isa<ArrayAccessOp, ArrayAmendOp, ArrayFetchOp, ArrayUpdateOp, in arrayMentions()
346 llvm::SmallVector<mlir::OpOperand *> queue; // uses of ArrayLoad[orig] in arrayMentions()
348 auto appendToQueue = [&](mlir::Value val) { in arrayMentions()
362 mlir::OpOperand *operand = queue.pop_back_val(); in arrayMentions()
363 mlir::Operation *owner = operand->getOwner(); in arrayMentions()
369 mlir::Value output = ro.getResult(ro.getFinalValue() ? arg : arg - 1); in arrayMentions()
375 auto branchOp = [&](mlir::Block *dest, OperandRange operands) { in arrayMentions()
391 if (auto ro = mlir::dyn_cast<DoLoopOp>(owner)) { in arrayMentions()
393 } else if (auto ro = mlir::dyn_cast<IterWhileOp>(owner)) { in arrayMentions()
395 } else if (auto rs = mlir::dyn_cast<ResultOp>(owner)) { in arrayMentions()
397 mlir::Operation *parent = rs->getParentRegion()->getParentOp(); in arrayMentions()
398 if (auto ifOp = mlir::dyn_cast<fir::IfOp>(parent)) in arrayMentions()
400 } else if (mlir::isa<ArrayFetchOp>(owner)) { in arrayMentions()
405 } else if (auto update = mlir::dyn_cast<ArrayUpdateOp>(owner)) { in arrayMentions()
411 } else if (auto update = mlir::dyn_cast<ArrayModifyOp>(owner)) { in arrayMentions()
418 } else if (auto mention = mlir::dyn_cast<ArrayAccessOp>(owner)) { in arrayMentions()
420 } else if (auto amend = mlir::dyn_cast<ArrayAmendOp>(owner)) { in arrayMentions()
423 } else if (auto br = mlir::dyn_cast<mlir::cf::BranchOp>(owner)) { in arrayMentions()
425 } else if (auto br = mlir::dyn_cast<mlir::cf::CondBranchOp>(owner)) { in arrayMentions()
428 } else if (mlir::isa<ArrayMergeStoreOp>(owner)) { in arrayMentions()
437 static bool hasPointerType(mlir::Type type) { in hasPointerType()
450 auto getSliceOp = [](mlir::Value val) -> SliceOp { in mutuallyExclusiveSliceRange()
453 auto sliceOp = mlir::dyn_cast_or_null<SliceOp>(val.getDefiningOp()); in mutuallyExclusiveSliceRange()
478 auto displacedByConstant = [](mlir::Value v1, mlir::Value v2) { in mutuallyExclusiveSliceRange()
479 auto removeConvert = [](mlir::Value v) -> mlir::Operation * { in mutuallyExclusiveSliceRange()
481 while (auto conv = mlir::dyn_cast_or_null<ConvertOp>(op)) in mutuallyExclusiveSliceRange()
486 auto isPositiveConstant = [](mlir::Value v) -> bool { in mutuallyExclusiveSliceRange()
488 mlir::dyn_cast<mlir::arith::ConstantOp>(v.getDefiningOp())) in mutuallyExclusiveSliceRange()
489 if (auto iattr = conOp.getValue().dyn_cast<mlir::IntegerAttr>()) in mutuallyExclusiveSliceRange()
498 if (auto addi = mlir::dyn_cast<mlir::arith::AddIOp>(op2)) in mutuallyExclusiveSliceRange()
504 if (auto subi = mlir::dyn_cast<mlir::arith::SubIOp>(op1)) in mutuallyExclusiveSliceRange()
513 if (mlir::isa_and_nonnull<fir::UndefOp>(ldTriples[i + 1].getDefiningOp()) && in mutuallyExclusiveSliceRange()
514 mlir::isa_and_nonnull<fir::UndefOp>(stTriples[i + 1].getDefiningOp())) { in mutuallyExclusiveSliceRange()
516 if (mlir::isa_and_nonnull<fir::UndefOp>(ldTriples[i].getDefiningOp()) || in mutuallyExclusiveSliceRange()
517 mlir::isa_and_nonnull<fir::UndefOp>(stTriples[i].getDefiningOp())) in mutuallyExclusiveSliceRange()
540 static bool conflictOnLoad(llvm::ArrayRef<mlir::Operation *> reach, in conflictOnLoad()
542 mlir::Value load; in conflictOnLoad()
543 mlir::Value addr = st.getMemref(); in conflictOnLoad()
546 if (auto ld = mlir::dyn_cast<ArrayLoadOp>(op)) { in conflictOnLoad()
547 mlir::Type ldTy = ld.getMemref().getType(); in conflictOnLoad()
578 static bool conflictOnMerge(llvm::ArrayRef<mlir::Operation *> mentions) { in conflictOnMerge()
581 llvm::SmallVector<mlir::Value> indices; in conflictOnMerge()
587 llvm::SmallVector<mlir::Value> compareVector; in conflictOnMerge()
588 if (auto u = mlir::dyn_cast<ArrayUpdateOp>(op)) { in conflictOnMerge()
595 } else if (auto f = mlir::dyn_cast<ArrayModifyOp>(op)) { in conflictOnMerge()
602 } else if (auto f = mlir::dyn_cast<ArrayFetchOp>(op)) { in conflictOnMerge()
609 } else if (auto f = mlir::dyn_cast<ArrayAccessOp>(op)) { in conflictOnMerge()
616 } else if (mlir::isa<ArrayAmendOp>(op)) { in conflictOnMerge()
620 mlir::emitError(op->getLoc(), "unexpected operation in analysis"); in conflictOnMerge()
635 static bool conflictOnReference(llvm::ArrayRef<mlir::Operation *> mentions) { in conflictOnReference()
643 if (mlir::isa<ArrayAmendOp>(op) && ++amendCount > 1) { in conflictOnReference()
647 if (mlir::isa<ArrayAccessOp>(op) && ++accessCount > 1) { in conflictOnReference()
652 if (mlir::isa<ArrayFetchOp, ArrayUpdateOp, ArrayModifyOp>(op)) { in conflictOnReference()
662 static mlir::Operation *
663 amendingAccess(llvm::ArrayRef<mlir::Operation *> mentions) { in amendingAccess()
665 if (auto amend = mlir::dyn_cast<ArrayAmendOp>(op)) in amendingAccess()
671 static bool conflictDetected(llvm::ArrayRef<mlir::Operation *> reach, in conflictDetected()
672 llvm::ArrayRef<mlir::Operation *> mentions, in conflictDetected()
680 conservativeCallConflict(llvm::ArrayRef<mlir::Operation *> reaches) { in conservativeCallConflict()
681 return llvm::any_of(reaches, [](mlir::Operation *op) { in conservativeCallConflict()
682 if (auto call = mlir::dyn_cast<fir::CallOp>(op)) in conservativeCallConflict()
684 call.getCallableForCallee().dyn_cast<mlir::SymbolRefAttr>()) { in conservativeCallConflict()
685 auto module = op->getParentOfType<mlir::ModuleOp>(); in conservativeCallConflict()
687 module.lookupSymbol<mlir::func::FuncOp>(callee)); in conservativeCallConflict()
695 void ArrayCopyAnalysis::construct(mlir::Operation *topLevelOp) { in construct()
697 if (auto st = mlir::dyn_cast<fir::ArrayMergeStoreOp>(op)) { in construct()
698 llvm::SmallVector<mlir::Operation *> values; in construct()
701 llvm::SmallVector<mlir::Operation *> mentions; in construct()
703 mlir::cast<ArrayLoadOp>(st.getOriginal().getDefiningOp())); in construct()
720 } else if (auto load = mlir::dyn_cast<ArrayLoadOp>(op)) { in construct()
721 llvm::SmallVector<mlir::Operation *> mentions; in construct()
727 if (mlir::isa<ArrayAccessOp, ArrayAmendOp, ArrayFetchOp, ArrayUpdateOp, in construct()
730 mlir::emitError( in construct()
750 class ArrayLoadConversion : public mlir::OpRewritePattern<ArrayLoadOp> {
754 mlir::LogicalResult
756 mlir::PatternRewriter &rewriter) const override { in matchAndRewrite()
759 return mlir::success(); in matchAndRewrite()
764 : public mlir::OpRewritePattern<ArrayMergeStoreOp> {
768 mlir::LogicalResult
770 mlir::PatternRewriter &rewriter) const override { in matchAndRewrite()
773 return mlir::success(); in matchAndRewrite()
778 static mlir::Type getEleTy(mlir::Type ty) { in getEleTy()
785 static bool getAdjustedExtents(mlir::Location loc, in getAdjustedExtents()
786 mlir::PatternRewriter &rewriter, in getAdjustedExtents()
788 llvm::SmallVectorImpl<mlir::Value> &result, in getAdjustedExtents()
789 mlir::Value shape) { in getAdjustedExtents()
792 if (auto s = mlir::dyn_cast_or_null<ShapeOp>(shapeOp)) { in getAdjustedExtents()
795 } else if (auto s = mlir::dyn_cast_or_null<ShapeShiftOp>(shapeOp)) { in getAdjustedExtents()
804 auto one = rewriter.create<mlir::arith::ConstantIndexOp>(loc, 1); in getAdjustedExtents()
805 mlir::Value size = one; in getAdjustedExtents()
806 if (mlir::Value sliceArg = arrLoad.getSlice()) { in getAdjustedExtents()
808 mlir::dyn_cast_or_null<SliceOp>(sliceArg.getDefiningOp())) { in getAdjustedExtents()
811 auto module = arrLoad->getParentOfType<mlir::ModuleOp>(); in getAdjustedExtents()
833 static mlir::Value getOrReadExtentsAndShapeOp( in getOrReadExtentsAndShapeOp()
834 mlir::Location loc, mlir::PatternRewriter &rewriter, ArrayLoadOp arrLoad, in getOrReadExtentsAndShapeOp()
835 llvm::SmallVectorImpl<mlir::Value> &result, bool ©UsingSlice) { in getOrReadExtentsAndShapeOp()
845 auto dimVal = rewriter.create<mlir::arith::ConstantIndexOp>(loc, dim); in getOrReadExtentsAndShapeOp()
856 llvm::SmallVector<mlir::Value> shapeShiftOperands; in getOrReadExtentsAndShapeOp()
869 static mlir::Type toRefType(mlir::Type ty) { in toRefType()
875 static llvm::SmallVector<mlir::Value>
876 getTypeParamsIfRawData(mlir::Location loc, FirOpBuilder &builder, in getTypeParamsIfRawData()
877 ArrayLoadOp arrLoad, mlir::Type ty) { in getTypeParamsIfRawData()
883 static mlir::Value genCoorOp(mlir::PatternRewriter &rewriter, in genCoorOp()
884 mlir::Location loc, mlir::Type eleTy, in genCoorOp()
885 mlir::Type resTy, mlir::Value alloc, in genCoorOp()
886 mlir::Value shape, mlir::Value slice, in genCoorOp()
887 mlir::ValueRange indices, ArrayLoadOp load, in genCoorOp()
889 llvm::SmallVector<mlir::Value> originated; in genCoorOp()
898 auto module = load->getParentOfType<mlir::ModuleOp>(); in genCoorOp()
902 mlir::Value result = rewriter.create<ArrayCoorOp>( in genCoorOp()
904 llvm::ArrayRef<mlir::Value>{originated}.take_front(dimension), in genCoorOp()
909 llvm::ArrayRef<mlir::Value>{originated}.drop_front(dimension)); in genCoorOp()
913 static mlir::Value getCharacterLen(mlir::Location loc, FirOpBuilder &builder, in getCharacterLen()
926 return builder.create<mlir::arith::DivSIOp>(loc, eleSzInBytes, in getCharacterLen()
942 void genArrayCopy(mlir::Location loc, mlir::PatternRewriter &rewriter, in genArrayCopy()
943 mlir::Value dst, mlir::Value src, mlir::Value shapeOp, in genArrayCopy()
944 mlir::Value sliceOp, ArrayLoadOp arrLoad) { in genArrayCopy()
946 llvm::SmallVector<mlir::Value> indices; in genArrayCopy()
947 llvm::SmallVector<mlir::Value> extents; in genArrayCopy()
954 auto zero = rewriter.create<mlir::arith::ConstantIndexOp>(loc, 0); in genArrayCopy()
955 auto one = rewriter.create<mlir::arith::ConstantIndexOp>(loc, 1); in genArrayCopy()
956 auto ub = rewriter.create<mlir::arith::SubIOp>(loc, idxTy, ubi, one); in genArrayCopy()
963 auto module = arrLoad->getParentOfType<mlir::ModuleOp>(); in genArrayCopy()
968 CopyIn && copyUsingSlice ? sliceOp : mlir::Value{}, in genArrayCopy()
973 !CopyIn && copyUsingSlice ? sliceOp : mlir::Value{}, in genArrayCopy()
993 static llvm::SmallVector<mlir::Value>
994 genArrayLoadTypeParameters(mlir::Location loc, mlir::PatternRewriter &rewriter, in genArrayLoadTypeParameters()
1002 auto module = load->getParentOfType<mlir::ModuleOp>(); in genArrayLoadTypeParameters()
1014 static llvm::SmallVector<mlir::Value>
1015 findNonconstantExtents(mlir::Type memrefTy, in findNonconstantExtents()
1016 llvm::ArrayRef<mlir::Value> extents) { in findNonconstantExtents()
1017 llvm::SmallVector<mlir::Value> nce; in findNonconstantExtents()
1035 static std::pair<mlir::Value,
1036 std::function<void(mlir::PatternRewriter &rewriter)>>
1037 allocateArrayTemp(mlir::Location loc, mlir::PatternRewriter &rewriter, in allocateArrayTemp()
1038 ArrayLoadOp load, llvm::ArrayRef<mlir::Value> extents, in allocateArrayTemp()
1039 mlir::Value shape) { in allocateArrayTemp()
1040 mlir::Type baseType = load.getMemref().getType(); in allocateArrayTemp()
1041 llvm::SmallVector<mlir::Value> nonconstantExtents = in allocateArrayTemp()
1043 llvm::SmallVector<mlir::Value> typeParams = in allocateArrayTemp()
1045 mlir::Value allocmem = rewriter.create<AllocMemOp>( in allocateArrayTemp()
1047 mlir::Type eleType = in allocateArrayTemp()
1052 mlir::Value box = rewriter.create<fir::EmboxOp>( in allocateArrayTemp()
1054 /*slice=*/mlir::Value{}, typeParams); in allocateArrayTemp()
1055 auto module = load->getParentOfType<mlir::ModuleOp>(); in allocateArrayTemp()
1061 auto cleanup = [=](mlir::PatternRewriter &r) { in allocateArrayTemp()
1069 auto cleanup = [=](mlir::PatternRewriter &r) { in allocateArrayTemp()
1081 class ArrayUpdateConversionBase : public mlir::OpRewritePattern<ArrayOp> {
1084 explicit ArrayUpdateConversionBase(mlir::MLIRContext *ctx, in ArrayUpdateConversionBase()
1087 : mlir::OpRewritePattern<ArrayOp>{ctx}, analysis{a}, useMap{m} {} in ArrayUpdateConversionBase()
1091 mlir::Value referenceToClone(mlir::Location loc, in referenceToClone()
1092 mlir::PatternRewriter &rewriter, in referenceToClone()
1098 auto load = mlir::cast<ArrayLoadOp>(loadOp); in referenceToClone()
1102 llvm::SmallVector<mlir::Value> extents; in referenceToClone()
1115 copyUsingSlice ? mlir::Value{} : load.getSlice(), access.getIndices(), in referenceToClone()
1119 auto store = mlir::cast<ArrayMergeStoreOp>(storeOp); in referenceToClone()
1134 std::pair<mlir::Value, mlir::Value>
1135 materializeAssignment(mlir::Location loc, mlir::PatternRewriter &rewriter, in materializeAssignment()
1137 const std::function<void(mlir::Value)> &assignElement, in materializeAssignment()
1138 mlir::Type lhsEltRefType) const { in materializeAssignment()
1141 auto load = mlir::cast<ArrayLoadOp>(loadOp); in materializeAssignment()
1150 llvm::SmallVector<mlir::Value> extents; in materializeAssignment()
1163 shapeOp, copyUsingSlice ? mlir::Value{} : load.getSlice(), in materializeAssignment()
1168 auto store = mlir::cast<ArrayMergeStoreOp>(storeOp); in materializeAssignment()
1197 explicit ArrayUpdateConversion(mlir::MLIRContext *ctx, in ArrayUpdateConversion()
1202 mlir::LogicalResult
1204 mlir::PatternRewriter &rewriter) const override { in matchAndRewrite()
1206 auto assignElement = [&](mlir::Value coor) { in matchAndRewrite()
1219 return mlir::success(); in matchAndRewrite()
1225 explicit ArrayModifyConversion(mlir::MLIRContext *ctx, in ArrayModifyConversion()
1230 mlir::LogicalResult
1232 mlir::PatternRewriter &rewriter) const override { in matchAndRewrite()
1234 auto assignElement = [](mlir::Value) { in matchAndRewrite()
1240 modify.replaceAllUsesWith(mlir::ValueRange{lhsEltCoor, lhsLoadResult}); in matchAndRewrite()
1241 rewriter.replaceOp(modify, mlir::ValueRange{lhsEltCoor, lhsLoadResult}); in matchAndRewrite()
1242 return mlir::success(); in matchAndRewrite()
1246 class ArrayFetchConversion : public mlir::OpRewritePattern<ArrayFetchOp> {
1248 explicit ArrayFetchConversion(mlir::MLIRContext *ctx, in ArrayFetchConversion()
1252 mlir::LogicalResult
1254 mlir::PatternRewriter &rewriter) const override { in matchAndRewrite()
1257 auto load = mlir::cast<ArrayLoadOp>(useMap.lookup(op)); in matchAndRewrite()
1267 return mlir::success(); in matchAndRewrite()
1278 explicit ArrayAccessConversion(mlir::MLIRContext *ctx, in ArrayAccessConversion()
1283 mlir::LogicalResult
1285 mlir::PatternRewriter &rewriter) const override { in matchAndRewrite()
1294 return mlir::success(); in matchAndRewrite()
1297 auto load = mlir::cast<ArrayLoadOp>(useMap.lookup(op)); in matchAndRewrite()
1303 return mlir::success(); in matchAndRewrite()
1311 class ArrayAmendConversion : public mlir::OpRewritePattern<ArrayAmendOp> {
1313 explicit ArrayAmendConversion(mlir::MLIRContext *ctx) in ArrayAmendConversion()
1316 mlir::LogicalResult
1318 mlir::PatternRewriter &rewriter) const override { in matchAndRewrite()
1324 return mlir::success(); in matchAndRewrite()
1341 mlir::RewritePatternSet patterns1(context); in runOnOperation()
1347 mlir::ConversionTarget target(*context); in runOnOperation()
1348 target.addLegalDialect<FIROpsDialect, mlir::scf::SCFDialect, in runOnOperation()
1349 mlir::arith::ArithmeticDialect, in runOnOperation()
1350 mlir::func::FuncDialect>(); in runOnOperation()
1354 if (mlir::failed( in runOnOperation()
1355 mlir::applyPartialConversion(func, target, std::move(patterns1)))) { in runOnOperation()
1356 mlir::emitError(mlir::UnknownLoc::get(context), in runOnOperation()
1361 mlir::RewritePatternSet patterns2(context); in runOnOperation()
1365 if (mlir::failed( in runOnOperation()
1366 mlir::applyPartialConversion(func, target, std::move(patterns2)))) { in runOnOperation()
1367 mlir::emitError(mlir::UnknownLoc::get(context), in runOnOperation()
1375 std::unique_ptr<mlir::Pass> fir::createArrayValueCopyPass() { in createArrayValueCopyPass()