Lines Matching refs:rewriter
86 static func::CallOp replaceOpWithFuncCall(RewriterBase &rewriter, Operation *op, in replaceOpWithFuncCall() argument
91 return rewriter.replaceOpWithNewOp<func::CallOp>(op, resultType, fn, in replaceOpWithFuncCall()
164 static Value genAlloc(RewriterBase &rewriter, Location loc, Value sz, Type tp) { in genAlloc() argument
166 return rewriter.create<memref::AllocOp>(loc, memTp, ValueRange{sz}); in genAlloc()
393 static void translateIndices(Location loc, ConversionPatternRewriter &rewriter, in translateIndices() argument
412 Value idx = constantIndex(rewriter, loc, i++); in translateIndices()
415 val = rewriter.create<memref::LoadOp>(loc, srcIdx, idx); in translateIndices()
419 Value stride = constantIndex(rewriter, loc, linear); in translateIndices()
420 Value jdx = constantIndex(rewriter, loc, j); in translateIndices()
422 Value old = rewriter.create<memref::LoadOp>(loc, srcIdx, jdx); in translateIndices()
425 : rewriter.create<arith::MulIOp>(loc, old, stride); in translateIndices()
426 val = val ? rewriter.create<arith::AddIOp>(loc, val, mul) : mul; in translateIndices()
430 val = rewriter.create<arith::DivUIOp>(loc, val, stride); in translateIndices()
431 rewriter.create<memref::StoreOp>(loc, val, dstIdx, jdx); in translateIndices()
433 val = rewriter.create<arith::RemUIOp>(loc, old, stride); in translateIndices()
438 rewriter.create<memref::StoreOp>(loc, val, dstIdx, idx); in translateIndices()
460 genSparse2SparseReshape(Operation *op, ConversionPatternRewriter &rewriter, in genSparse2SparseReshape() argument
478 sizesFromPtr(rewriter, sizes, op, noPerm, srcTp, src); in genSparse2SparseReshape()
479 newParams(rewriter, params, op, srcTp, noPerm, Action::kToIterator, sizes, in genSparse2SparseReshape()
481 Value iter = genNewCall(rewriter, op, params); in genSparse2SparseReshape()
485 sizesFromPtr(rewriter, sizes, op, encDst, dstTp, src); in genSparse2SparseReshape()
486 newParams(rewriter, params, op, dstTp, encDst, Action::kEmptyCOO, sizes); in genSparse2SparseReshape()
487 Value coo = genNewCall(rewriter, op, params); in genSparse2SparseReshape()
490 Value srcIdx = genAlloca(rewriter, loc, srcRank, rewriter.getIndexType()); in genSparse2SparseReshape()
491 Value dstIdx = genAlloca(rewriter, loc, dstRank, rewriter.getIndexType()); in genSparse2SparseReshape()
492 Value elemPtr = genAllocaScalar(rewriter, loc, elemTp); in genSparse2SparseReshape()
495 auto whileOp = rewriter.create<scf::WhileOp>(loc, noTypes, noArgs); in genSparse2SparseReshape()
496 Block *before = rewriter.createBlock(&whileOp.getBefore(), {}, noTypes); in genSparse2SparseReshape()
497 rewriter.setInsertionPointToEnd(before); in genSparse2SparseReshape()
498 Value cond = genGetNextCall(rewriter, op, iter, srcIdx, elemPtr); in genSparse2SparseReshape()
499 rewriter.create<scf::ConditionOp>(loc, cond, before->getArguments()); in genSparse2SparseReshape()
502 Block *after = rewriter.createBlock(&whileOp.getAfter(), {}, noTypes); in genSparse2SparseReshape()
503 rewriter.setInsertionPointToStart(after); in genSparse2SparseReshape()
504 translateIndices(loc, rewriter, reassociation, dstTp, srcTp, dstIdx, srcIdx); in genSparse2SparseReshape()
505 genAddEltCall(rewriter, op, elemTp, coo, elemPtr, dstIdx, dstPerm); in genSparse2SparseReshape()
506 rewriter.create<scf::YieldOp>(loc); in genSparse2SparseReshape()
508 rewriter.setInsertionPointAfter(whileOp); in genSparse2SparseReshape()
509 params[6] = constantAction(rewriter, loc, Action::kFromCOO); in genSparse2SparseReshape()
511 Value dst = genNewCall(rewriter, op, params); in genSparse2SparseReshape()
512 genDelCOOCall(rewriter, op, elemTp, coo); in genSparse2SparseReshape()
513 genDelCOOCall(rewriter, op, elemTp, iter); in genSparse2SparseReshape()
514 rewriter.replaceOp(op, dst); in genSparse2SparseReshape()
528 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
529 rewriter.replaceOpWithNewOp<func::ReturnOp>(op, adaptor.getOperands()); in matchAndRewrite()
541 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
552 rewriter.replaceOp(op, genDimSizeCall(rewriter, op, enc, src, idx)); in matchAndRewrite()
563 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
569 rewriter.replaceOp(op, adaptor.getOperands()); in matchAndRewrite()
582 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
589 op, rewriter, op.getReassociationIndices(), adaptor.getOperands()[0], in matchAndRewrite()
601 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
611 sizesFromType(rewriter, sizes, op.getLoc(), stp); in matchAndRewrite()
613 newParams(rewriter, params, op, stp, enc, Action::kFromFile, sizes, ptr); in matchAndRewrite()
614 rewriter.replaceOp(op, genNewCall(rewriter, op, params)); in matchAndRewrite()
626 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
628 return rewriter.notifyMatchFailure(op, in matchAndRewrite()
641 sizes.push_back(rewriter.create<arith::ConstantIndexOp>( in matchAndRewrite()
649 newParams(rewriter, params, op, stp, enc, Action::kEmpty, sizes); in matchAndRewrite()
650 rewriter.replaceOp(op, genNewCall(rewriter, op, params)); in matchAndRewrite()
668 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
683 rewriter.replaceOp(op, adaptor.getOperands()); // hidden nop cast in matchAndRewrite()
689 sizesFromPtr(rewriter, sizes, op, encSrc, stp, src); in matchAndRewrite()
705 newParams(rewriter, params, op, stp, encDst, Action::kSparseToSparse, in matchAndRewrite()
707 rewriter.replaceOp(op, genNewCall(rewriter, op, params)); in matchAndRewrite()
715 newParams(rewriter, params, op, stp, enc, Action::kToCOO, sizes, src); in matchAndRewrite()
716 Value coo = genNewCall(rewriter, op, params); in matchAndRewrite()
717 params[3] = constantPointerTypeEncoding(rewriter, loc, encDst); in matchAndRewrite()
718 params[4] = constantIndexTypeEncoding(rewriter, loc, encDst); in matchAndRewrite()
719 params[6] = constantAction(rewriter, loc, Action::kFromCOO); in matchAndRewrite()
721 Value dst = genNewCall(rewriter, op, params); in matchAndRewrite()
722 genDelCOOCall(rewriter, op, stp.getElementType(), coo); in matchAndRewrite()
723 rewriter.replaceOp(op, dst); in matchAndRewrite()
749 sizesFromPtr(rewriter, sizes, op, encSrc, srcTensorTp, src); in matchAndRewrite()
750 newParams(rewriter, params, op, dstTensorTp, encDst, Action::kToIterator, in matchAndRewrite()
752 Value iter = genNewCall(rewriter, op, params); in matchAndRewrite()
753 Value ind = genAlloca(rewriter, loc, rank, rewriter.getIndexType()); in matchAndRewrite()
754 Value elemPtr = genAllocaScalar(rewriter, loc, elemTp); in matchAndRewrite()
755 Block *insertionBlock = rewriter.getInsertionBlock(); in matchAndRewrite()
758 Value dst = allocDenseTensor(rewriter, loc, dstTensorTp, sizes); in matchAndRewrite()
761 auto whileOp = rewriter.create<scf::WhileOp>(loc, noTypes, noArgs); in matchAndRewrite()
762 Block *before = rewriter.createBlock(&whileOp.getBefore(), {}, noTypes); in matchAndRewrite()
763 rewriter.setInsertionPointToEnd(before); in matchAndRewrite()
764 Value cond = genGetNextCall(rewriter, op, iter, ind, elemPtr); in matchAndRewrite()
765 rewriter.create<scf::ConditionOp>(loc, cond, before->getArguments()); in matchAndRewrite()
766 Block *after = rewriter.createBlock(&whileOp.getAfter(), {}, noTypes); in matchAndRewrite()
767 rewriter.setInsertionPointToStart(after); in matchAndRewrite()
768 insertScalarIntoDenseTensor(rewriter, loc, elemPtr, dst, rank, ind); in matchAndRewrite()
769 rewriter.create<scf::YieldOp>(loc); in matchAndRewrite()
770 rewriter.setInsertionPointAfter(whileOp); in matchAndRewrite()
771 genDelCOOCall(rewriter, op, elemTp, iter); in matchAndRewrite()
772 rewriter.replaceOpWithNewOp<bufferization::ToTensorOp>(op, resType, dst); in matchAndRewrite()
775 rewriter.setInsertionPoint(insertionBlock->getTerminator()); in matchAndRewrite()
776 deallocDenseTensor(rewriter, loc, dst); in matchAndRewrite()
814 sizesFromSrc(rewriter, sizes, loc, src); in matchAndRewrite()
815 newParams(rewriter, params, op, stp, encDst, Action::kEmptyCOO, sizes); in matchAndRewrite()
816 Value coo = genNewCall(rewriter, op, params); in matchAndRewrite()
817 Value ind = genAlloca(rewriter, loc, rank, rewriter.getIndexType()); in matchAndRewrite()
822 Value zero = constantIndex(rewriter, loc, 0); in matchAndRewrite()
823 Value one = constantIndex(rewriter, loc, 1); in matchAndRewrite()
824 auto indicesValues = genSplitSparseConstant(rewriter, loc, src); in matchAndRewrite()
832 hi.push_back(linalg::createOrFoldDimOp(rewriter, loc, values, 0)); in matchAndRewrite()
837 hi.push_back(linalg::createOrFoldDimOp(rewriter, loc, src, i)); in matchAndRewrite()
842 Value elemPtr = genAllocaScalar(rewriter, loc, eltType); in matchAndRewrite()
844 rewriter, op.getLoc(), lo, hi, st, {}, in matchAndRewrite()
849 val = genIndexAndValueForSparse(rewriter, loc, indices, values, ind, in matchAndRewrite()
852 val = genIndexAndValueForDense(rewriter, loc, src, ind, ivs); in matchAndRewrite()
854 genAddEltCall(rewriter, op, eltType, coo, elemPtr, ind, perm); in matchAndRewrite()
858 params[6] = constantAction(rewriter, loc, Action::kFromCOO); in matchAndRewrite()
860 Value dst = genNewCall(rewriter, op, params); in matchAndRewrite()
861 genDelCOOCall(rewriter, op, eltType, coo); in matchAndRewrite()
862 rewriter.replaceOp(op, dst); in matchAndRewrite()
878 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
884 createFuncCall(rewriter, op, name, noTp, adaptor.getOperands(), in matchAndRewrite()
886 rewriter.eraseOp(op); in matchAndRewrite()
898 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
902 replaceOpWithFuncCall(rewriter, op, name, resType, adaptor.getOperands(), in matchAndRewrite()
914 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
918 replaceOpWithFuncCall(rewriter, op, name, resType, adaptor.getOperands(), in matchAndRewrite()
930 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
934 replaceOpWithFuncCall(rewriter, op, name, resType, adaptor.getOperands(), in matchAndRewrite()
946 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
951 createFuncCall(rewriter, op, name, noTp, adaptor.getOperands(), in matchAndRewrite()
954 rewriter.replaceOp(op, adaptor.getOperands()); in matchAndRewrite()
965 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
969 replaceOpWithFuncCall(rewriter, op, name, noTp, adaptor.getOperands(), in matchAndRewrite()
981 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
985 Type boolType = rewriter.getIntegerType(1); in matchAndRewrite()
986 Type idxType = rewriter.getIndexType(); in matchAndRewrite()
988 rewriter.setInsertionPointAfter(op.getTensor().getDefiningOp()); in matchAndRewrite()
992 Value sz = genDimSizeCall(rewriter, op, enc, src, srcType.getRank() - 1); in matchAndRewrite()
996 Value values = genAlloc(rewriter, loc, sz, eltType); in matchAndRewrite()
997 Value filled = genAlloc(rewriter, loc, sz, boolType); in matchAndRewrite()
998 Value indices = genAlloc(rewriter, loc, sz, idxType); in matchAndRewrite()
999 Value zero = constantZero(rewriter, loc, idxType); in matchAndRewrite()
1005 rewriter.create<linalg::FillOp>( in matchAndRewrite()
1006 loc, ValueRange{constantZero(rewriter, loc, eltType)}, in matchAndRewrite()
1008 rewriter.create<linalg::FillOp>( in matchAndRewrite()
1009 loc, ValueRange{constantZero(rewriter, loc, boolType)}, in matchAndRewrite()
1013 rewriter.replaceOp(op, {values, filled, indices, zero}); in matchAndRewrite()
1024 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
1033 replaceOpWithFuncCall(rewriter, op, name, noTp, adaptor.getOperands(), in matchAndRewrite()
1043 rewriter.setInsertionPointAfter(parent); in matchAndRewrite()
1044 rewriter.create<memref::DeallocOp>(loc, adaptor.getOperands()[2]); in matchAndRewrite()
1045 rewriter.create<memref::DeallocOp>(loc, adaptor.getOperands()[3]); in matchAndRewrite()
1046 rewriter.create<memref::DeallocOp>(loc, adaptor.getOperands()[4]); in matchAndRewrite()
1057 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
1065 sizesFromPtr(rewriter, sizes, op, encSrc, srcType, src); in matchAndRewrite()
1069 newParams(rewriter, params, op, srcType, enc, Action::kToCOO, sizes, src); in matchAndRewrite()
1070 Value coo = genNewCall(rewriter, op, params); in matchAndRewrite()
1081 params.push_back(constantI1(rewriter, loc, sort)); in matchAndRewrite()
1085 createFuncCall(rewriter, op, name, noTp, params, EmitCInterface::Off); in matchAndRewrite()
1086 genDelCOOCall(rewriter, op, eltType, coo); in matchAndRewrite()
1087 rewriter.eraseOp(op); in matchAndRewrite()