| /llvm-project-15.0.7/mlir/lib/Conversion/VectorToSPIRV/ |
| H A D | VectorToSPIRV.cpp | 49 if (dstType == adaptor.getSource().getType()) in matchAndRewrite() 70 adaptor.getSource()); in matchAndRewrite() 100 extractOp, adaptor.getVector(), id); in matchAndRewrite() 122 Value srcVector = adaptor.getOperands().front(); in matchAndRewrite() 152 fmaOp, dstType, adaptor.getLhs(), adaptor.getRhs(), adaptor.getAcc()); in matchAndRewrite() 176 insertOp, adaptor.getSource(), adaptor.getDest(), id); in matchAndRewrite() 221 Value dstVector = adaptor.getOperands().back(); in matchAndRewrite() 279 if (Value acc = adaptor.getAcc()) in matchAndRewrite() 328 rewriter.replaceOp(op, adaptor.getInput()); in matchAndRewrite() 359 shuffleOp, newResultType, adaptor.getV1(), adaptor.getV2(), in matchAndRewrite() [all …]
|
| /llvm-project-15.0.7/mlir/lib/Conversion/ArithmeticToSPIRV/ |
| H A D | ArithmeticToSPIRV.cpp | 482 op.getLoc(), adaptor.getOperands()[0], adaptor.getOperands()[1], in matchAndRewrite() 483 adaptor.getOperands()[0], rewriter); in matchAndRewrite() 497 op.getLoc(), adaptor.getOperands()[0], adaptor.getOperands()[1], in matchAndRewrite() 498 adaptor.getOperands()[0], rewriter); in matchAndRewrite() 511 Op op, typename Op::Adaptor adaptor, in matchAndRewrite() argument 533 arith::XOrIOp op, OpAdaptor adaptor, in matchAndRewrite() argument 554 arith::XOrIOp op, OpAdaptor adaptor, in matchAndRewrite() argument 643 Op op, typename Op::Adaptor adaptor, in matchAndRewrite() argument 667 arith::CmpIOp op, OpAdaptor adaptor, in matchAndRewrite() argument 795 arith::CmpFOp op, OpAdaptor adaptor, in matchAndRewrite() argument [all …]
|
| /llvm-project-15.0.7/mlir/lib/Conversion/ArithmeticToLLVM/ |
| H A D | ArithmeticToLLVM.cpp | 101 matchAndRewrite(arith::CmpIOp op, OpAdaptor adaptor, 132 arith::IndexCastOp op, OpAdaptor adaptor, in matchAndRewrite() argument 144 rewriter.replaceOp(op, adaptor.getIn()); in matchAndRewrite() 166 auto operandType = adaptor.getLhs().getType(); in matchAndRewrite() 174 adaptor.getLhs(), adaptor.getRhs()); in matchAndRewrite() 185 OpAdaptor adaptor(operands); in matchAndRewrite() local 189 adaptor.getLhs(), adaptor.getRhs()); in matchAndRewrite() 201 auto operandType = adaptor.getLhs().getType(); in matchAndRewrite() 209 adaptor.getLhs(), adaptor.getRhs()); in matchAndRewrite() 220 OpAdaptor adaptor(operands); in matchAndRewrite() local [all …]
|
| /llvm-project-15.0.7/mlir/lib/Dialect/AMX/Transforms/ |
| H A D | LegalizeForLLVMExport.cpp | 86 matchAndRewrite(TileZeroOp op, OpAdaptor adaptor, in matchAndRewrite() 104 matchAndRewrite(TileLoadOp op, OpAdaptor adaptor, in matchAndRewrite() 115 adaptor.getBase(), op.getLoc()); in matchAndRewrite() 131 matchAndRewrite(TileStoreOp op, OpAdaptor adaptor, in matchAndRewrite() 156 matchAndRewrite(TileMulFOp op, OpAdaptor adaptor, in matchAndRewrite() 170 adaptor.getLhs(), adaptor.getRhs()); in matchAndRewrite() 178 matchAndRewrite(TileMulIOp op, OpAdaptor adaptor, in matchAndRewrite() 195 adaptor.getLhs(), adaptor.getRhs()); in matchAndRewrite() 199 adaptor.getLhs(), adaptor.getRhs()); in matchAndRewrite() 203 adaptor.getLhs(), adaptor.getRhs()); in matchAndRewrite() [all …]
|
| /llvm-project-15.0.7/mlir/lib/Conversion/VectorToLLVM/ |
| H A D | ConvertVectorToLLVM.cpp | 160 adaptor.getLhs(), adaptor.getRhs(), matmulOp.getLhsRows(), in matchAndRewrite() 198 loadOp, vectorTy, ptr, adaptor.getMask(), adaptor.getPassThru(), align); in replaceLoadOrStoreOp() 214 storeOp, adaptor.getValueToStore(), ptr, adaptor.getMask(), align); in replaceLoadOrStoreOp() 315 scatter, adaptor.getValueToStore(), ptrs, adaptor.getMask(), in matchAndRewrite() 339 expand, vtype, ptr, adaptor.getMask(), adaptor.getPassThru()); in matchAndRewrite() 361 compress, adaptor.getValueToStore(), ptr, adaptor.getMask()); in matchAndRewrite() 577 loc, adaptor.getV1(), adaptor.getV2(), maskArrayAttr); in matchAndRewrite() 635 extractEltOp, llvmType, adaptor.getVector(), adaptor.getPosition()); in matchAndRewrite() 722 fmaOp, adaptor.getLhs(), adaptor.getRhs(), adaptor.getAcc()); in matchAndRewrite() 754 insertEltOp, llvmType, adaptor.getDest(), adaptor.getSource(), in matchAndRewrite() [all …]
|
| /llvm-project-15.0.7/mlir/lib/Conversion/GPUCommon/ |
| H A D | GPUToLLVMConversion.cpp | 446 gpu::AllocOp allocOp, OpAdaptor adaptor, in matchAndRewrite() argument 514 async::YieldOp yieldOp, OpAdaptor adaptor, in matchAndRewrite() argument 526 auto stream = adaptor.getOperands()[idx]; in matchAndRewrite() 553 gpu::WaitOp waitOp, OpAdaptor adaptor, in matchAndRewrite() argument 583 gpu::WaitOp waitOp, OpAdaptor adaptor, in matchAndRewrite() argument 765 adaptor.asyncDependencies().empty() in matchAndRewrite() 776 {function.getResult(0), adaptor.gridSizeX(), adaptor.gridSizeY(), in matchAndRewrite() 777 adaptor.gridSizeZ(), adaptor.blockSizeX(), adaptor.blockSizeY(), in matchAndRewrite() 809 MemRefDescriptor srcDesc(adaptor.src()); in matchAndRewrite() 845 Type valueType = adaptor.value().getType(); in matchAndRewrite() [all …]
|
| H A D | GPUOpsLowering.h | 25 matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor, 45 matchAndRewrite(gpu::PrintfOp gpuPrintfOp, gpu::PrintfOpAdaptor adaptor, 63 matchAndRewrite(gpu::PrintfOp gpuPrintfOp, gpu::PrintfOpAdaptor adaptor, 74 matchAndRewrite(gpu::ReturnOp op, OpAdaptor adaptor, in matchAndRewrite() 76 rewriter.replaceOpWithNewOp<LLVM::ReturnOp>(op, adaptor.getOperands()); in matchAndRewrite()
|
| /llvm-project-15.0.7/mlir/lib/Dialect/X86Vector/Transforms/ |
| H A D | LegalizeForLLVMExport.cpp | 49 matchAndRewrite(OpTy op, typename OpTy::Adaptor adaptor, in matchAndRewrite() 55 adaptor.getOperands(), in matchAndRewrite() 59 adaptor.getOperands(), in matchAndRewrite() 71 matchAndRewrite(MaskCompressOp op, OpAdaptor adaptor, in matchAndRewrite() 73 auto opType = adaptor.getA().getType(); in matchAndRewrite() 77 src = adaptor.getSrc(); in matchAndRewrite() 97 matchAndRewrite(RsqrtOp op, OpAdaptor adaptor, in matchAndRewrite() 99 auto opType = adaptor.getA().getType(); in matchAndRewrite() 109 matchAndRewrite(DotOp op, OpAdaptor adaptor, in matchAndRewrite() 111 auto opType = adaptor.getA().getType(); in matchAndRewrite() [all …]
|
| /llvm-project-15.0.7/mlir/lib/Conversion/ShapeToStandard/ |
| H A D | ShapeToStandard.cpp | 33 matchAndRewrite(AnyOp op, OpAdaptor adaptor, 72 matchAndRewrite(BroadcastOp op, OpAdaptor adaptor, 121 BroadcastOp op, OpAdaptor adaptor, in matchAndRewrite() argument 182 ConstShapeOp op, OpAdaptor adaptor, in matchAndRewrite() argument 216 ConstSizeOp op, OpAdaptor adaptor, in matchAndRewrite() argument 235 IsBroadcastableOp op, OpAdaptor adaptor, in matchAndRewrite() argument 336 GetExtentOp op, OpAdaptor adaptor, in matchAndRewrite() argument 542 ShapeOfOp op, OpAdaptor adaptor, in matchAndRewrite() argument 551 Value tensor = adaptor.getArg(); in matchAndRewrite() 605 SplitAtOp op, OpAdaptor adaptor, in matchAndRewrite() argument [all …]
|
| /llvm-project-15.0.7/mlir/lib/Conversion/ComplexToStandard/ |
| H A D | ComplexToStandard.cpp | 29 matchAndRewrite(complex::AbsOp op, OpAdaptor adaptor, in matchAndRewrite() 52 matchAndRewrite(complex::Atan2Op op, OpAdaptor adaptor, in matchAndRewrite() 59 Value lhs = adaptor.getLhs(); in matchAndRewrite() 60 Value rhs = adaptor.getRhs(); in matchAndRewrite() 101 auto type = adaptor.getLhs() in matchAndRewrite() 156 matchAndRewrite(TrigonometricOp op, OpAdaptor adaptor, in matchAndRewrite() 218 matchAndRewrite(complex::DivOp op, OpAdaptor adaptor, in matchAndRewrite() 437 matchAndRewrite(complex::ExpOp op, OpAdaptor adaptor, in matchAndRewrite() 487 matchAndRewrite(complex::LogOp op, OpAdaptor adaptor, in matchAndRewrite() 529 matchAndRewrite(complex::MulOp op, OpAdaptor adaptor, in matchAndRewrite() [all …]
|
| /llvm-project-15.0.7/mlir/lib/Conversion/AsyncToLLVM/ |
| H A D | AsyncToLLVM.cpp | 429 auto coroHdl = adaptor.handle(); in matchAndRewrite() 506 auto coroState = adaptor.state(); in matchAndRewrite() 606 adaptor.getOperands()); in matchAndRewrite() 735 Value operand = adaptor.operand(); in matchAndRewrite() 736 Value handle = adaptor.handle(); in matchAndRewrite() 773 auto coroHdl = adaptor.handle(); in matchAndRewrite() 798 auto storage = adaptor.storage(); in matchAndRewrite() 814 auto value = adaptor.value(); in matchAndRewrite() 841 auto storage = adaptor.storage(); in matchAndRewrite() 935 auto operand = adaptor.operand(); in matchAndRewrite() [all …]
|
| /llvm-project-15.0.7/mlir/lib/Conversion/ComplexToLLVM/ |
| H A D | ComplexToLLVM.cpp | 62 matchAndRewrite(complex::AbsOp op, OpAdaptor adaptor, in matchAndRewrite() 66 ComplexStructBuilder complexStruct(adaptor.getComplex()); in matchAndRewrite() 84 matchAndRewrite(complex::ConstantOp op, OpAdaptor adaptor, in matchAndRewrite() 114 matchAndRewrite(complex::ReOp op, OpAdaptor adaptor, in matchAndRewrite() 129 matchAndRewrite(complex::ImOp op, OpAdaptor adaptor, in matchAndRewrite() 153 ComplexStructBuilder lhs(adaptor.getLhs()); in unpackBinaryComplexOperands() 156 ComplexStructBuilder rhs(adaptor.getRhs()); in unpackBinaryComplexOperands() 167 matchAndRewrite(complex::AddOp op, OpAdaptor adaptor, in matchAndRewrite() 195 matchAndRewrite(complex::DivOp op, OpAdaptor adaptor, in matchAndRewrite() 240 matchAndRewrite(complex::MulOp op, OpAdaptor adaptor, in matchAndRewrite() [all …]
|
| /llvm-project-15.0.7/mlir/lib/Conversion/GPUToNVVM/ |
| H A D | WmmaOpsToNvvm.cpp | 74 OpAdaptor adaptor, in matchAndRewrite() 116 adaptor.srcMemref(), adaptor.indices(), rewriter); in matchAndRewrite() 138 OpAdaptor adaptor, in matchAndRewrite() 170 adaptor.dstMemref(), adaptor.indices(), rewriter); in matchAndRewrite() 189 OpAdaptor adaptor, in matchAndRewrite() 230 unpackOp(adaptor.opA()); in matchAndRewrite() 231 unpackOp(adaptor.opB()); in matchAndRewrite() 232 unpackOp(adaptor.opC()); in matchAndRewrite() 249 OpAdaptor adaptor, in matchAndRewrite() 255 Value cst = adaptor.getOperands()[0]; in matchAndRewrite() [all …]
|
| /llvm-project-15.0.7/mlir/lib/Conversion/SCFToSPIRV/ |
| H A D | SCFToSPIRV.cpp | 87 matchAndRewrite(scf::ForOp forOp, OpAdaptor adaptor, 98 matchAndRewrite(scf::IfOp ifOp, OpAdaptor adaptor, 107 matchAndRewrite(scf::YieldOp terminatorOp, OpAdaptor adaptor, 116 matchAndRewrite(scf::WhileOp forOp, OpAdaptor adaptor, 180 Value adapLowerBound = adaptor.getLowerBound(); in matchAndRewrite() 183 for (Value arg : adaptor.getInitArgs()) in matchAndRewrite() 203 SmallVector<Value, 8> args(1, adaptor.getLowerBound()); in matchAndRewrite() 204 args.append(adaptor.getInitArgs().begin(), adaptor.getInitArgs().end()); in matchAndRewrite() 233 for (auto arg : adaptor.getInitArgs()) in matchAndRewrite() 304 scf::YieldOp terminatorOp, OpAdaptor adaptor, in matchAndRewrite() argument [all …]
|
| /llvm-project-15.0.7/mlir/lib/Dialect/SCF/Transforms/ |
| H A D | StructuralTypeConversions.cpp | 23 matchAndRewrite(ForOp op, OpAdaptor adaptor, in matchAndRewrite() argument 65 newOp->setOperands(adaptor.getOperands()); in matchAndRewrite() 81 matchAndRewrite(IfOp op, OpAdaptor adaptor, in matchAndRewrite() argument 110 newOp->setOperands(adaptor.getOperands()); in matchAndRewrite() 127 matchAndRewrite(scf::YieldOp op, OpAdaptor adaptor, in matchAndRewrite() argument 129 rewriter.replaceOpWithNewOp<scf::YieldOp>(op, adaptor.getOperands()); in matchAndRewrite() 141 matchAndRewrite(WhileOp op, OpAdaptor adaptor, in matchAndRewrite() argument 150 adaptor.getOperands()); in matchAndRewrite() 168 matchAndRewrite(ConditionOp op, OpAdaptor adaptor, in matchAndRewrite() argument 171 op, [&]() { op->setOperands(adaptor.getOperands()); }); in matchAndRewrite()
|
| /llvm-project-15.0.7/mlir/lib/Conversion/MathToLLVM/ |
| H A D | MathToLLVM.cpp | 50 matchAndRewrite(MathOp op, typename MathOp::Adaptor adaptor, in matchAndRewrite() 52 auto operandType = adaptor.getOperand().getType(); in matchAndRewrite() 96 matchAndRewrite(math::ExpM1Op op, OpAdaptor adaptor, in matchAndRewrite() 98 auto operandType = adaptor.getOperand().getType(); in matchAndRewrite() 127 op.getOperation(), adaptor.getOperands(), *getTypeConverter(), in matchAndRewrite() 149 matchAndRewrite(math::Log1pOp op, OpAdaptor adaptor, in matchAndRewrite() 151 auto operandType = adaptor.getOperand().getType(); in matchAndRewrite() 171 adaptor.getOperand()); in matchAndRewrite() 181 op.getOperation(), adaptor.getOperands(), *getTypeConverter(), in matchAndRewrite() 203 matchAndRewrite(math::RsqrtOp op, OpAdaptor adaptor, in matchAndRewrite() [all …]
|
| /llvm-project-15.0.7/mlir/lib/Conversion/MemRefToSPIRV/ |
| H A D | MemRefToSPIRV.cpp | 171 matchAndRewrite(memref::AllocaOp allocaOp, OpAdaptor adaptor, 184 matchAndRewrite(memref::AllocOp operation, OpAdaptor adaptor, 205 matchAndRewrite(memref::LoadOp loadOp, OpAdaptor adaptor, 215 matchAndRewrite(memref::LoadOp loadOp, OpAdaptor adaptor, 225 matchAndRewrite(memref::StoreOp storeOp, OpAdaptor adaptor, 235 matchAndRewrite(memref::StoreOp storeOp, OpAdaptor adaptor, 304 OpAdaptor adaptor, in matchAndRewrite() argument 421 adaptor.getIndices(), loadOp.getLoc(), rewriter); in matchAndRewrite() 466 Value storeVal = adaptor.getValue(); in matchAndRewrite() 497 Value storeVal = adaptor.getValue(); in matchAndRewrite() [all …]
|
| /llvm-project-15.0.7/mlir/lib/Conversion/GPUToSPIRV/ |
| H A D | GPUToSPIRV.cpp | 62 matchAndRewrite(gpu::BlockDimOp op, OpAdaptor adaptor, 72 matchAndRewrite(gpu::GPUFuncOp funcOp, OpAdaptor adaptor, 95 matchAndRewrite(gpu::ModuleEndOp endOp, OpAdaptor adaptor, in matchAndRewrite() argument 131 SourceOp op, typename SourceOp::Adaptor adaptor, in matchAndRewrite() argument 148 SourceOp op, typename SourceOp::Adaptor adaptor, in matchAndRewrite() argument 160 gpu::BlockDimOp op, OpAdaptor adaptor, in matchAndRewrite() argument 263 gpu::GPUFuncOp funcOp, OpAdaptor adaptor, in matchAndRewrite() argument 305 gpu::GPUModuleOp moduleOp, OpAdaptor adaptor, in matchAndRewrite() argument 335 gpu::ReturnOp returnOp, OpAdaptor adaptor, in matchAndRewrite() argument 337 if (!adaptor.getOperands().empty()) in matchAndRewrite() [all …]
|
| /llvm-project-15.0.7/mlir/lib/Conversion/MathToSPIRV/ |
| H A D | MathToSPIRV.cpp | 63 matchAndRewrite(math::CopySignOp copySignOp, OpAdaptor adaptor, in matchAndRewrite() argument 129 matchAndRewrite(math::CountLeadingZerosOp countOp, OpAdaptor adaptor, in matchAndRewrite() argument 145 Value input = adaptor.getOperand(); in matchAndRewrite() 177 matchAndRewrite(math::ExpM1Op operation, OpAdaptor adaptor, in matchAndRewrite() 179 assert(adaptor.getOperands().size() == 1); in matchAndRewrite() 182 auto exp = rewriter.create<ExpOp>(loc, type, adaptor.getOperand()); in matchAndRewrite() 198 matchAndRewrite(math::Log1pOp operation, OpAdaptor adaptor, in matchAndRewrite() 200 assert(adaptor.getOperands().size() == 1); in matchAndRewrite() 205 rewriter.create<spirv::FAddOp>(loc, one, adaptor.getOperand()); in matchAndRewrite() 216 matchAndRewrite(math::PowFOp powfOp, OpAdaptor adaptor, in matchAndRewrite() [all …]
|
| /llvm-project-15.0.7/mlir/lib/Conversion/MemRefToLLVM/ |
| H A D | MemRefToLLVM.cpp | 278 Value memref = adaptor.getMemref(); in matchAndRewrite() 332 MemRefDescriptor memref(adaptor.getMemref()); in matchAndRewrite() 422 OpAdaptor adaptor, in extractSizeOfRankedMemRef() 439 Value index = adaptor.getIndex(); in extractSizeOfRankedMemRef() 823 loc, adaptor.getSource(), rewriter); in rewrite() 876 MemRefDescriptor srcDesc(adaptor.getSource()); in lowerToMemCopyIntrinsic() 1208 MemRefDescriptor shapeDesc(adaptor.getShape()); in convertSourceMemRefToDescriptor() 1590 MemRefDescriptor srcDesc(adaptor.getSrc()); in matchAndRewrite() 1815 MemRefDescriptor viewMemRef(adaptor.getIn()); in matchAndRewrite() 1958 adaptor.getSizes(), i); in matchAndRewrite() [all …]
|
| /llvm-project-15.0.7/mlir/lib/Conversion/SPIRVToLLVM/ |
| H A D | SPIRVToLLVM.cpp | 235 spirv::StoreOpAdaptor adaptor(operands); in replaceWithLoadOrStore() local 311 auto indices = llvm::to_vector<4>(adaptor.indices()); in matchAndRewrite() 330 matchAndRewrite(spirv::AddressOfOp op, OpAdaptor adaptor, in matchAndRewrite() argument 574 op, dstType, adaptor.composite(), index); in matchAndRewrite() 578 op, dstType, adaptor.composite(), op.indices()); in matchAndRewrite() 604 op, dstType, adaptor.composite(), adaptor.object(), index); in matchAndRewrite() 608 op, dstType, adaptor.composite(), adaptor.object(), op.indices()); in matchAndRewrite() 1188 loc, dstType, adaptor.operand1(), extended); in matchAndRewrite() 1376 auto components = adaptor.components(); in matchAndRewrite() 1377 auto vector1 = adaptor.vector1(); in matchAndRewrite() [all …]
|
| /llvm-project-15.0.7/mlir/lib/Conversion/ControlFlowToSPIRV/ |
| H A D | ControlFlowToSPIRV.cpp | 40 matchAndRewrite(cf::BranchOp op, OpAdaptor adaptor, in matchAndRewrite() 43 adaptor.getDestOperands()); in matchAndRewrite() 54 matchAndRewrite(cf::CondBranchOp op, OpAdaptor adaptor, in matchAndRewrite() 57 op, op.getCondition(), op.getTrueDest(), adaptor.getTrueDestOperands(), in matchAndRewrite() 58 op.getFalseDest(), adaptor.getFalseDestOperands()); in matchAndRewrite()
|
| /llvm-project-15.0.7/mlir/lib/Pass/ |
| H A D | PassStatistics.cpp | 62 auto *adaptor = dyn_cast<OpToOpPassAdaptor>(pass); in printResultsAsList() local 65 if (!adaptor) { in printResultsAsList() 84 for (auto &mgr : adaptor->getPassManagers()) in printResultsAsList() 111 if (auto *adaptor = dyn_cast<OpToOpPassAdaptor>(pass)) { in printResultsAsPipeline() local 114 auto mgrs = adaptor->getPassManagers(); in printResultsAsPipeline() 116 printPassEntry(os, indent, adaptor->getAdaptorName()); in printResultsAsPipeline() 223 OpToOpPassAdaptor *adaptor = dyn_cast<OpToOpPassAdaptor>(&pass); in prepareStatistics() local 224 if (!adaptor) in prepareStatistics() 226 MutableArrayRef<OpPassManager> nestedPms = adaptor->getPassManagers(); in prepareStatistics() 230 for (auto &asyncPM : adaptor->getParallelPassManagers()) { in prepareStatistics()
|
| /llvm-project-15.0.7/mlir/lib/Conversion/FuncToSPIRV/ |
| H A D | FuncToSPIRV.cpp | 45 matchAndRewrite(func::ReturnOp returnOp, OpAdaptor adaptor, in matchAndRewrite() argument 52 returnOp, adaptor.getOperands()[0]); in matchAndRewrite() 66 matchAndRewrite(func::CallOp callOp, OpAdaptor adaptor, in matchAndRewrite() argument 77 callOp, resultType, adaptor.getOperands(), callOp->getAttrs()); in matchAndRewrite() 80 callOp, TypeRange(), adaptor.getOperands(), callOp->getAttrs()); in matchAndRewrite()
|
| /llvm-project-15.0.7/mlir/lib/Conversion/NVGPUToNVVM/ |
| H A D | NVGPUToNVVM.cpp | 196 matchAndRewrite(nvgpu::LdMatrixOp op, OpAdaptor adaptor, in matchAndRewrite() 226 getStridedElementPtr(loc, srcMemrefType, adaptor.getSrcMemref(), in matchAndRewrite() 227 adaptor.getIndices(), rewriter); in matchAndRewrite() 261 matchAndRewrite(nvgpu::MmaSyncOp op, OpAdaptor adaptor, in matchAndRewrite() 306 unpackOperandVector(rewriter, loc, adaptor.getMatrixA(), ptxTypeA); in matchAndRewrite() 360 matchAndRewrite(nvgpu::DeviceAsyncCopyOp op, OpAdaptor adaptor, in matchAndRewrite() 383 int64_t numElements = adaptor.getNumElements().getZExtValue(); in matchAndRewrite() 389 sizeInBytes == 16 ? adaptor.getBypassL1Attr() : UnitAttr(); in matchAndRewrite() 408 matchAndRewrite(nvgpu::DeviceAsyncCreateGroupOp op, OpAdaptor adaptor, in matchAndRewrite() 426 matchAndRewrite(nvgpu::DeviceAsyncWaitOp op, OpAdaptor adaptor, in matchAndRewrite() [all …]
|