Lines Matching refs:rewriter
48 static Value insertOne(ConversionPatternRewriter &rewriter, in insertOne() argument
54 auto idxType = rewriter.getIndexType(); in insertOne()
55 auto constant = rewriter.create<LLVM::ConstantOp>( in insertOne()
57 rewriter.getIntegerAttr(idxType, pos)); in insertOne()
58 return rewriter.create<LLVM::InsertElementOp>(loc, llvmType, val1, val2, in insertOne()
61 return rewriter.create<LLVM::InsertValueOp>(loc, llvmType, val1, val2, in insertOne()
62 rewriter.getI64ArrayAttr(pos)); in insertOne()
66 static Value extractOne(ConversionPatternRewriter &rewriter, in extractOne() argument
70 auto idxType = rewriter.getIndexType(); in extractOne()
71 auto constant = rewriter.create<LLVM::ConstantOp>( in extractOne()
73 rewriter.getIntegerAttr(idxType, pos)); in extractOne()
74 return rewriter.create<LLVM::ExtractElementOp>(loc, llvmType, val, in extractOne()
77 return rewriter.create<LLVM::ExtractValueOp>(loc, llvmType, val, in extractOne()
78 rewriter.getI64ArrayAttr(pos)); in extractOne()
98 static LogicalResult getIndexedPtrs(ConversionPatternRewriter &rewriter, in getIndexedPtrs() argument
110 ptrs = rewriter.create<LLVM::GEPOp>(loc, ptrsType, base, index); in getIndexedPtrs()
116 static Value castDataPtr(ConversionPatternRewriter &rewriter, Location loc, in castDataPtr() argument
119 return rewriter.create<LLVM::BitcastOp>(loc, pType, ptr); in castDataPtr()
136 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
142 rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(bitCastOp, newResultTy, in matchAndRewrite()
157 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
158 rewriter.replaceOpWithNewOp<LLVM::MatrixMultiplyOp>( in matchAndRewrite()
175 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
176 rewriter.replaceOpWithNewOp<LLVM::MatrixTransposeOp>( in matchAndRewrite()
189 ConversionPatternRewriter &rewriter) { in replaceLoadOrStoreOp() argument
190 rewriter.replaceOpWithNewOp<LLVM::LoadOp>(loadOp, ptr, align); in replaceLoadOrStoreOp()
196 ConversionPatternRewriter &rewriter) { in replaceLoadOrStoreOp() argument
197 rewriter.replaceOpWithNewOp<LLVM::MaskedLoadOp>( in replaceLoadOrStoreOp()
204 ConversionPatternRewriter &rewriter) { in replaceLoadOrStoreOp() argument
205 rewriter.replaceOpWithNewOp<LLVM::StoreOp>(storeOp, adaptor.getValueToStore(), in replaceLoadOrStoreOp()
212 ConversionPatternRewriter &rewriter) { in replaceLoadOrStoreOp() argument
213 rewriter.replaceOpWithNewOp<LLVM::MaskedStoreOp>( in replaceLoadOrStoreOp()
227 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
245 adaptor.getIndices(), rewriter); in matchAndRewrite()
246 Value ptr = castDataPtr(rewriter, loc, dataPtr, memRefTy, vtype); in matchAndRewrite()
248 replaceLoadOrStoreOp(loadOrStoreOp, adaptor, vtype, ptr, align, rewriter); in matchAndRewrite()
261 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
274 adaptor.getIndices(), rewriter); in matchAndRewrite()
275 if (failed(getIndexedPtrs(rewriter, loc, adaptor.getBase(), ptr, in matchAndRewrite()
280 rewriter.replaceOpWithNewOp<LLVM::masked_gather>( in matchAndRewrite()
282 adaptor.getPassThru(), rewriter.getI32IntegerAttr(align)); in matchAndRewrite()
295 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
308 adaptor.getIndices(), rewriter); in matchAndRewrite()
309 if (failed(getIndexedPtrs(rewriter, loc, adaptor.getBase(), ptr, in matchAndRewrite()
314 rewriter.replaceOpWithNewOp<LLVM::masked_scatter>( in matchAndRewrite()
316 rewriter.getI32IntegerAttr(align)); in matchAndRewrite()
329 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
336 adaptor.getIndices(), rewriter); in matchAndRewrite()
338 rewriter.replaceOpWithNewOp<LLVM::masked_expandload>( in matchAndRewrite()
352 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
358 adaptor.getIndices(), rewriter); in matchAndRewrite()
360 rewriter.replaceOpWithNewOp<LLVM::masked_compressstore>( in matchAndRewrite()
372 ConversionPatternRewriter &rewriter, Location loc, Type llvmType, in createIntegerReductionArithmeticOpLowering() argument
374 Value result = rewriter.create<VectorOp>(loc, llvmType, vectorOperand); in createIntegerReductionArithmeticOpLowering()
376 result = rewriter.create<ScalarOp>(loc, accumulator, result); in createIntegerReductionArithmeticOpLowering()
386 ConversionPatternRewriter &rewriter, Location loc, Type llvmType, in createIntegerReductionComparisonOpLowering() argument
388 Value result = rewriter.create<VectorOp>(loc, llvmType, vectorOperand); in createIntegerReductionComparisonOpLowering()
391 rewriter.create<LLVM::ICmpOp>(loc, predicate, accumulator, result); in createIntegerReductionComparisonOpLowering()
392 result = rewriter.create<LLVM::SelectOp>(loc, cmp, accumulator, result); in createIntegerReductionComparisonOpLowering()
429 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
444 rewriter, loc, llvmType, operand, acc); in matchAndRewrite()
450 rewriter, loc, llvmType, operand, acc); in matchAndRewrite()
454 LLVM::vector_reduce_umin>(rewriter, loc, llvmType, operand, acc, in matchAndRewrite()
459 LLVM::vector_reduce_smin>(rewriter, loc, llvmType, operand, acc, in matchAndRewrite()
464 LLVM::vector_reduce_umax>(rewriter, loc, llvmType, operand, acc, in matchAndRewrite()
469 LLVM::vector_reduce_smax>(rewriter, loc, llvmType, operand, acc, in matchAndRewrite()
476 rewriter, loc, llvmType, operand, acc); in matchAndRewrite()
482 rewriter, loc, llvmType, operand, acc); in matchAndRewrite()
488 rewriter, loc, llvmType, operand, acc); in matchAndRewrite()
493 rewriter.replaceOp(reductionOp, result); in matchAndRewrite()
506 : rewriter.create<LLVM::ConstantOp>( in matchAndRewrite()
508 rewriter.getZeroAttr(eltType)); in matchAndRewrite()
509 rewriter.replaceOpWithNewOp<LLVM::vector_reduce_fadd>( in matchAndRewrite()
511 rewriter.getBoolAttr(reassociateFPReductions)); in matchAndRewrite()
516 : rewriter.create<LLVM::ConstantOp>( in matchAndRewrite()
518 rewriter.getFloatAttr(eltType, 1.0)); in matchAndRewrite()
519 rewriter.replaceOpWithNewOp<LLVM::vector_reduce_fmul>( in matchAndRewrite()
521 rewriter.getBoolAttr(reassociateFPReductions)); in matchAndRewrite()
526 rewriter.create<LLVM::vector_reduce_fmin>(loc, llvmType, operand); in matchAndRewrite()
528 result = createMinMaxF(rewriter, loc, result, acc, /*isMin=*/true); in matchAndRewrite()
529 rewriter.replaceOp(reductionOp, result); in matchAndRewrite()
534 rewriter.create<LLVM::vector_reduce_fmax>(loc, llvmType, operand); in matchAndRewrite()
536 result = createMinMaxF(rewriter, loc, result, acc, /*isMin=*/false); in matchAndRewrite()
537 rewriter.replaceOp(reductionOp, result); in matchAndRewrite()
555 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
576 Value llvmShuffleOp = rewriter.create<LLVM::ShuffleVectorOp>( in matchAndRewrite()
578 rewriter.replaceOp(shuffleOp, llvmShuffleOp); in matchAndRewrite()
588 Value insert = rewriter.create<LLVM::UndefOp>(loc, llvmType); in matchAndRewrite()
597 Value extract = extractOne(rewriter, *getTypeConverter(), loc, value, in matchAndRewrite()
599 insert = insertOne(rewriter, *getTypeConverter(), loc, insert, extract, in matchAndRewrite()
602 rewriter.replaceOp(shuffleOp, insert); in matchAndRewrite()
615 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
625 auto idxType = rewriter.getIndexType(); in matchAndRewrite()
626 auto zero = rewriter.create<LLVM::ConstantOp>( in matchAndRewrite()
628 rewriter.getIntegerAttr(idxType, 0)); in matchAndRewrite()
629 rewriter.replaceOpWithNewOp<LLVM::ExtractElementOp>( in matchAndRewrite()
634 rewriter.replaceOpWithNewOp<LLVM::ExtractElementOp>( in matchAndRewrite()
647 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
660 rewriter.replaceOp(extractOp, adaptor.getVector()); in matchAndRewrite()
666 Value extracted = rewriter.create<LLVM::ExtractValueOp>( in matchAndRewrite()
668 rewriter.replaceOp(extractOp, extracted); in matchAndRewrite()
680 extracted = rewriter.create<LLVM::ExtractValueOp>( in matchAndRewrite()
687 auto i64Type = IntegerType::get(rewriter.getContext(), 64); in matchAndRewrite()
688 auto constant = rewriter.create<LLVM::ConstantOp>(loc, i64Type, position); in matchAndRewrite()
690 rewriter.create<LLVM::ExtractElementOp>(loc, extracted, constant); in matchAndRewrite()
691 rewriter.replaceOp(extractOp, extracted); in matchAndRewrite()
717 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
721 rewriter.replaceOpWithNewOp<LLVM::FMulAddOp>( in matchAndRewrite()
734 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
744 auto idxType = rewriter.getIndexType(); in matchAndRewrite()
745 auto zero = rewriter.create<LLVM::ConstantOp>( in matchAndRewrite()
747 rewriter.getIntegerAttr(idxType, 0)); in matchAndRewrite()
748 rewriter.replaceOpWithNewOp<LLVM::InsertElementOp>( in matchAndRewrite()
753 rewriter.replaceOpWithNewOp<LLVM::InsertElementOp>( in matchAndRewrite()
767 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
781 rewriter.replaceOp(insertOp, adaptor.getSource()); in matchAndRewrite()
787 Value inserted = rewriter.create<LLVM::InsertValueOp>( in matchAndRewrite()
790 rewriter.replaceOp(insertOp, inserted); in matchAndRewrite()
804 extracted = rewriter.create<LLVM::ExtractValueOp>( in matchAndRewrite()
810 auto i64Type = IntegerType::get(rewriter.getContext(), 64); in matchAndRewrite()
811 auto constant = rewriter.create<LLVM::ConstantOp>(loc, i64Type, position); in matchAndRewrite()
812 Value inserted = rewriter.create<LLVM::InsertElementOp>( in matchAndRewrite()
820 inserted = rewriter.create<LLVM::InsertValueOp>( in matchAndRewrite()
825 rewriter.replaceOp(insertOp, inserted); in matchAndRewrite()
862 PatternRewriter &rewriter) const override { in matchAndRewrite()
869 Value zero = rewriter.create<arith::ConstantOp>( in matchAndRewrite()
870 loc, elemType, rewriter.getZeroAttr(elemType)); in matchAndRewrite()
871 Value desc = rewriter.create<vector::SplatOp>(loc, vType, zero); in matchAndRewrite()
873 Value extrLHS = rewriter.create<ExtractOp>(loc, op.getLhs(), i); in matchAndRewrite()
874 Value extrRHS = rewriter.create<ExtractOp>(loc, op.getRhs(), i); in matchAndRewrite()
875 Value extrACC = rewriter.create<ExtractOp>(loc, op.getAcc(), i); in matchAndRewrite()
876 Value fma = rewriter.create<FMAOp>(loc, extrLHS, extrRHS, extrACC); in matchAndRewrite()
877 desc = rewriter.create<InsertOp>(loc, fma, desc, i); in matchAndRewrite()
879 rewriter.replaceOp(op, desc); in matchAndRewrite()
921 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
954 auto int64Ty = IntegerType::get(rewriter.getContext(), 64); in matchAndRewrite()
957 auto desc = MemRefDescriptor::undef(rewriter, loc, llvmTargetDescriptorTy); in matchAndRewrite()
960 Value allocated = sourceMemRef.allocatedPtr(rewriter, loc); in matchAndRewrite()
962 rewriter.create<LLVM::BitcastOp>(loc, llvmTargetElementTy, allocated); in matchAndRewrite()
963 desc.setAllocatedPtr(rewriter, loc, allocated); in matchAndRewrite()
965 Value ptr = sourceMemRef.alignedPtr(rewriter, loc); in matchAndRewrite()
966 ptr = rewriter.create<LLVM::BitcastOp>(loc, llvmTargetElementTy, ptr); in matchAndRewrite()
967 desc.setAlignedPtr(rewriter, loc, ptr); in matchAndRewrite()
969 auto attr = rewriter.getIntegerAttr(rewriter.getIndexType(), 0); in matchAndRewrite()
970 auto zero = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, attr); in matchAndRewrite()
971 desc.setOffset(rewriter, loc, zero); in matchAndRewrite()
978 rewriter.getIntegerAttr(rewriter.getIndexType(), indexedSize.value()); in matchAndRewrite()
979 auto size = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, sizeAttr); in matchAndRewrite()
980 desc.setSize(rewriter, loc, index, size); in matchAndRewrite()
981 auto strideAttr = rewriter.getIntegerAttr(rewriter.getIndexType(), in matchAndRewrite()
983 auto stride = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, strideAttr); in matchAndRewrite()
984 desc.setStride(rewriter, loc, index, stride); in matchAndRewrite()
987 rewriter.replaceOp(castOp, {desc}); in matchAndRewrite()
1003 PatternRewriter &rewriter) const override { in matchAndRewrite()
1008 force32BitVectorIndices ? rewriter.getI32Type() : rewriter.getI64Type(); in matchAndRewrite()
1010 Value indices = rewriter.create<LLVM::StepVectorOp>( in matchAndRewrite()
1013 auto bound = getValueOrCreateCastToIndexLike(rewriter, loc, idxType, in matchAndRewrite()
1015 Value bounds = rewriter.create<SplatOp>(loc, indices.getType(), bound); in matchAndRewrite()
1016 Value comp = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt, in matchAndRewrite()
1018 rewriter.replaceOp(op, comp); in matchAndRewrite()
1044 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
1100 emitRanks(rewriter, printOp, adaptor.getSource(), type, printer, rank, in matchAndRewrite()
1102 emitCall(rewriter, printOp->getLoc(), in matchAndRewrite()
1105 rewriter.eraseOp(printOp); in matchAndRewrite()
1118 void emitRanks(ConversionPatternRewriter &rewriter, Operation *op, in emitRanks() argument
1127 value = rewriter.create<arith::ExtUIOp>( in emitRanks()
1128 loc, IntegerType::get(rewriter.getContext(), 64), value); in emitRanks()
1131 value = rewriter.create<arith::ExtSIOp>( in emitRanks()
1132 loc, IntegerType::get(rewriter.getContext(), 64), value); in emitRanks()
1137 emitCall(rewriter, loc, printer, value); in emitRanks()
1141 emitCall(rewriter, loc, in emitRanks()
1151 Value nestedVal = extractOne(rewriter, *getTypeConverter(), loc, value, in emitRanks()
1153 emitRanks(rewriter, op, nestedVal, reducedType, printer, /*rank=*/0, in emitRanks()
1156 emitCall(rewriter, loc, printComma); in emitRanks()
1159 rewriter, loc, in emitRanks()
1168 Value nestedVal = extractOne(rewriter, *getTypeConverter(), loc, value, in emitRanks()
1170 emitRanks(rewriter, op, nestedVal, reducedType, printer, rank - 1, in emitRanks()
1173 emitCall(rewriter, loc, printComma); in emitRanks()
1175 emitCall(rewriter, loc, in emitRanks()
1180 static void emitCall(ConversionPatternRewriter &rewriter, Location loc, in emitCall() argument
1182 rewriter.create<LLVM::CallOp>(loc, TypeRange(), SymbolRefAttr::get(ref), in emitCall()
1194 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
1201 Value undef = rewriter.create<LLVM::UndefOp>(splatOp.getLoc(), vectorType); in matchAndRewrite()
1202 auto zero = rewriter.create<LLVM::ConstantOp>( in matchAndRewrite()
1204 typeConverter->convertType(rewriter.getIntegerType(32)), in matchAndRewrite()
1205 rewriter.getZeroAttr(rewriter.getIntegerType(32))); in matchAndRewrite()
1209 rewriter.replaceOpWithNewOp<LLVM::InsertElementOp>( in matchAndRewrite()
1215 auto v = rewriter.create<LLVM::InsertElementOp>( in matchAndRewrite()
1222 ArrayAttr zeroAttrs = rewriter.getI32ArrayAttr(zeroValues); in matchAndRewrite()
1223 rewriter.replaceOpWithNewOp<LLVM::ShuffleVectorOp>(splatOp, v, undef, in matchAndRewrite()
1237 ConversionPatternRewriter &rewriter) const override { in matchAndRewrite()
1252 Value desc = rewriter.create<LLVM::UndefOp>(loc, llvmNDVectorTy); in matchAndRewrite()
1256 Value vdesc = rewriter.create<LLVM::UndefOp>(loc, llvm1DVectorTy); in matchAndRewrite()
1257 auto zero = rewriter.create<LLVM::ConstantOp>( in matchAndRewrite()
1258 loc, typeConverter->convertType(rewriter.getIntegerType(32)), in matchAndRewrite()
1259 rewriter.getZeroAttr(rewriter.getIntegerType(32))); in matchAndRewrite()
1260 Value v = rewriter.create<LLVM::InsertElementOp>(loc, llvm1DVectorTy, vdesc, in matchAndRewrite()
1266 ArrayAttr zeroAttrs = rewriter.getI32ArrayAttr(zeroValues); in matchAndRewrite()
1267 v = rewriter.create<LLVM::ShuffleVectorOp>(loc, v, v, zeroAttrs); in matchAndRewrite()
1271 nDVectorIterate(vectorTypeInfo, rewriter, [&](ArrayAttr position) { in matchAndRewrite()
1272 desc = rewriter.create<LLVM::InsertValueOp>(loc, llvmNDVectorTy, desc, v, in matchAndRewrite()
1275 rewriter.replaceOp(splatOp, desc); in matchAndRewrite()