Lines Matching refs:rewriter
62 RewriterBase &rewriter) { in convertIntrinsicResult() argument
63 MLIRContext *ctx = rewriter.getContext(); in convertIntrinsicResult()
66 Type i32Ty = rewriter.getI32Type(); in convertIntrinsicResult()
67 Type f32Ty = rewriter.getF32Type(); in convertIntrinsicResult()
68 Type f64Ty = rewriter.getF64Type(); in convertIntrinsicResult()
69 Type f16x2Ty = LLVM::getFixedVectorType(rewriter.getF16Type(), 2); in convertIntrinsicResult()
76 return rewriter.create<LLVM::ConstantOp>(loc, IntegerType::get(ctx, 32), in convertIntrinsicResult()
77 rewriter.getI32IntegerAttr(index)); in convertIntrinsicResult()
88 Value el = rewriter.create<LLVM::ExtractValueOp>( in convertIntrinsicResult()
90 rewriter.getI64ArrayAttr(i)); in convertIntrinsicResult()
91 el = rewriter.createOrFold<LLVM::BitcastOp>( in convertIntrinsicResult()
107 rewriter.create<LLVM::UndefOp>(loc, arrayType.getElementType()); in convertIntrinsicResult()
108 Value x1 = rewriter.create<LLVM::ExtractValueOp>( in convertIntrinsicResult()
110 rewriter.getI64ArrayAttr(i * 2)); in convertIntrinsicResult()
111 Value x2 = rewriter.create<LLVM::ExtractValueOp>( in convertIntrinsicResult()
113 rewriter.getI64ArrayAttr(i * 2 + 1)); in convertIntrinsicResult()
114 vec = rewriter.create<LLVM::InsertElementOp>(loc, vec.getType(), vec, in convertIntrinsicResult()
116 vec = rewriter.create<LLVM::InsertElementOp>(loc, vec.getType(), vec, in convertIntrinsicResult()
123 Value result = rewriter.create<LLVM::UndefOp>(loc, arrayType); in convertIntrinsicResult()
125 result = rewriter.create<LLVM::InsertValueOp>( in convertIntrinsicResult()
127 rewriter.getI64ArrayAttr(el.index())); in convertIntrinsicResult()
140 static SmallVector<Value> unpackOperandVector(RewriterBase &rewriter, in unpackOperandVector() argument
144 Type i32Ty = rewriter.getI32Type(); in unpackOperandVector()
145 Type f64Ty = rewriter.getF64Type(); in unpackOperandVector()
146 Type f32Ty = rewriter.getF32Type(); in unpackOperandVector()
147 Type i8Ty = rewriter.getI8Type(); in unpackOperandVector()
148 Type i4Ty = rewriter.getIntegerType(4); in unpackOperandVector()
155 Value toUse = rewriter.create<LLVM::ExtractValueOp>( in unpackOperandVector()
156 loc, arrayTy.getElementType(), operand, rewriter.getI64ArrayAttr(i)); in unpackOperandVector()
165 rewriter.create<LLVM::BitcastOp>(loc, rewriter.getI32Type(), toUse)); in unpackOperandVector()
178 result.push_back(rewriter.create<LLVM::ExtractElementOp>( in unpackOperandVector()
180 rewriter.create<LLVM::ConstantOp>( in unpackOperandVector()
181 loc, rewriter.getI64Type(), rewriter.getI64IntegerAttr(idx)))); in unpackOperandVector()
197 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
219 ctx, SmallVector<Type>(num32BitRegs, rewriter.getI32Type())); in matchAndRewrite()
221 ldMatrixResultType = rewriter.getI32Type(); in matchAndRewrite()
227 adaptor.getIndices(), rewriter); in matchAndRewrite()
228 Value ldMatrixResult = rewriter.create<NVVM::LdMatrixOp>( in matchAndRewrite()
239 Value result = rewriter.create<LLVM::UndefOp>(loc, finalResultType); in matchAndRewrite()
242 ? rewriter.create<LLVM::ExtractValueOp>( in matchAndRewrite()
243 loc, rewriter.getI32Type(), ldMatrixResult, in matchAndRewrite()
244 rewriter.getI64ArrayAttr(i)) in matchAndRewrite()
247 rewriter.create<LLVM::BitcastOp>(loc, innerVectorType, i32Register); in matchAndRewrite()
248 result = rewriter.create<LLVM::InsertValueOp>( in matchAndRewrite()
249 loc, finalResultType, result, casted, rewriter.getI64ArrayAttr(i)); in matchAndRewrite()
252 rewriter.replaceOp(op, result); in matchAndRewrite()
262 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
306 unpackOperandVector(rewriter, loc, adaptor.getMatrixA(), ptxTypeA); in matchAndRewrite()
308 unpackOperandVector(rewriter, loc, adaptor.getMatrixB(), ptxTypeB); in matchAndRewrite()
310 unpackOperandVector(rewriter, loc, adaptor.getMatrixC(), *ptxTypeC); in matchAndRewrite()
315 Value intrinsicResult = rewriter.create<NVVM::MmaOp>( in matchAndRewrite()
325 rewriter.replaceOp(op, convertIntrinsicResult(op.getLoc(), intrinsicResTy, in matchAndRewrite()
327 rewriter)); in matchAndRewrite()
361 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
365 adaptor.getDstIndices(), rewriter); in matchAndRewrite()
369 dstPtr = rewriter.create<LLVM::BitcastOp>(loc, dstPointerType, dstPtr); in matchAndRewrite()
374 adaptor.getSrcIndices(), rewriter); in matchAndRewrite()
377 scrPtr = rewriter.create<LLVM::BitcastOp>(loc, srcPointerType, scrPtr); in matchAndRewrite()
381 scrPtr = rewriter.create<LLVM::AddrSpaceCastOp>(loc, srcPointerGlobalType, in matchAndRewrite()
390 rewriter.create<NVVM::CpAsyncOp>( in matchAndRewrite()
391 loc, dstPtr, scrPtr, rewriter.getI32IntegerAttr(sizeInBytes), bypassL1); in matchAndRewrite()
394 Value zero = rewriter.create<LLVM::ConstantOp>( in matchAndRewrite()
396 rewriter.getI32IntegerAttr(0)); in matchAndRewrite()
397 rewriter.replaceOp(op, zero); in matchAndRewrite()
409 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
410 rewriter.create<NVVM::CpAsyncCommitGroupOp>(op.getLoc()); in matchAndRewrite()
412 Value zero = rewriter.create<LLVM::ConstantOp>( in matchAndRewrite()
414 rewriter.getI32IntegerAttr(0)); in matchAndRewrite()
415 rewriter.replaceOp(op, zero); in matchAndRewrite()
427 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
430 rewriter.create<NVVM::CpAsyncWaitGroupOp>(op.getLoc(), numGroups); in matchAndRewrite()
431 rewriter.eraseOp(op); in matchAndRewrite()