1 //===- VectorToLLVM.cpp - Conversion from Vector to the LLVM dialect ------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h"
10 
11 #include "../PassDetail.h"
12 #include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h"
13 #include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h"
14 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
15 #include "mlir/Dialect/StandardOps/IR/Ops.h"
16 #include "mlir/Dialect/Vector/VectorOps.h"
17 #include "mlir/IR/AffineMap.h"
18 #include "mlir/IR/Attributes.h"
19 #include "mlir/IR/Builders.h"
20 #include "mlir/IR/MLIRContext.h"
21 #include "mlir/IR/Module.h"
22 #include "mlir/IR/Operation.h"
23 #include "mlir/IR/PatternMatch.h"
24 #include "mlir/IR/StandardTypes.h"
25 #include "mlir/IR/Types.h"
26 #include "mlir/Target/LLVMIR/TypeTranslation.h"
27 #include "mlir/Transforms/DialectConversion.h"
28 #include "mlir/Transforms/Passes.h"
29 #include "llvm/IR/DerivedTypes.h"
30 #include "llvm/IR/Module.h"
31 #include "llvm/IR/Type.h"
32 #include "llvm/Support/Allocator.h"
33 #include "llvm/Support/ErrorHandling.h"
34 
35 using namespace mlir;
36 using namespace mlir::vector;
37 
38 // Helper to reduce vector type by one rank at front.
39 static VectorType reducedVectorTypeFront(VectorType tp) {
40   assert((tp.getRank() > 1) && "unlowerable vector type");
41   return VectorType::get(tp.getShape().drop_front(), tp.getElementType());
42 }
43 
44 // Helper to reduce vector type by *all* but one rank at back.
45 static VectorType reducedVectorTypeBack(VectorType tp) {
46   assert((tp.getRank() > 1) && "unlowerable vector type");
47   return VectorType::get(tp.getShape().take_back(), tp.getElementType());
48 }
49 
50 // Helper that picks the proper sequence for inserting.
51 static Value insertOne(ConversionPatternRewriter &rewriter,
52                        LLVMTypeConverter &typeConverter, Location loc,
53                        Value val1, Value val2, Type llvmType, int64_t rank,
54                        int64_t pos) {
55   if (rank == 1) {
56     auto idxType = rewriter.getIndexType();
57     auto constant = rewriter.create<LLVM::ConstantOp>(
58         loc, typeConverter.convertType(idxType),
59         rewriter.getIntegerAttr(idxType, pos));
60     return rewriter.create<LLVM::InsertElementOp>(loc, llvmType, val1, val2,
61                                                   constant);
62   }
63   return rewriter.create<LLVM::InsertValueOp>(loc, llvmType, val1, val2,
64                                               rewriter.getI64ArrayAttr(pos));
65 }
66 
67 // Helper that picks the proper sequence for inserting.
68 static Value insertOne(PatternRewriter &rewriter, Location loc, Value from,
69                        Value into, int64_t offset) {
70   auto vectorType = into.getType().cast<VectorType>();
71   if (vectorType.getRank() > 1)
72     return rewriter.create<InsertOp>(loc, from, into, offset);
73   return rewriter.create<vector::InsertElementOp>(
74       loc, vectorType, from, into,
75       rewriter.create<ConstantIndexOp>(loc, offset));
76 }
77 
78 // Helper that picks the proper sequence for extracting.
79 static Value extractOne(ConversionPatternRewriter &rewriter,
80                         LLVMTypeConverter &typeConverter, Location loc,
81                         Value val, Type llvmType, int64_t rank, int64_t pos) {
82   if (rank == 1) {
83     auto idxType = rewriter.getIndexType();
84     auto constant = rewriter.create<LLVM::ConstantOp>(
85         loc, typeConverter.convertType(idxType),
86         rewriter.getIntegerAttr(idxType, pos));
87     return rewriter.create<LLVM::ExtractElementOp>(loc, llvmType, val,
88                                                    constant);
89   }
90   return rewriter.create<LLVM::ExtractValueOp>(loc, llvmType, val,
91                                                rewriter.getI64ArrayAttr(pos));
92 }
93 
94 // Helper that picks the proper sequence for extracting.
95 static Value extractOne(PatternRewriter &rewriter, Location loc, Value vector,
96                         int64_t offset) {
97   auto vectorType = vector.getType().cast<VectorType>();
98   if (vectorType.getRank() > 1)
99     return rewriter.create<ExtractOp>(loc, vector, offset);
100   return rewriter.create<vector::ExtractElementOp>(
101       loc, vectorType.getElementType(), vector,
102       rewriter.create<ConstantIndexOp>(loc, offset));
103 }
104 
105 // Helper that returns a subset of `arrayAttr` as a vector of int64_t.
106 // TODO: Better support for attribute subtype forwarding + slicing.
107 static SmallVector<int64_t, 4> getI64SubArray(ArrayAttr arrayAttr,
108                                               unsigned dropFront = 0,
109                                               unsigned dropBack = 0) {
110   assert(arrayAttr.size() > dropFront + dropBack && "Out of bounds");
111   auto range = arrayAttr.getAsRange<IntegerAttr>();
112   SmallVector<int64_t, 4> res;
113   res.reserve(arrayAttr.size() - dropFront - dropBack);
114   for (auto it = range.begin() + dropFront, eit = range.end() - dropBack;
115        it != eit; ++it)
116     res.push_back((*it).getValue().getSExtValue());
117   return res;
118 }
119 
120 // Helper that returns data layout alignment of an operation with memref.
121 template <typename T>
122 LogicalResult getMemRefAlignment(LLVMTypeConverter &typeConverter, T op,
123                                  unsigned &align) {
124   Type elementTy =
125       typeConverter.convertType(op.getMemRefType().getElementType());
126   if (!elementTy)
127     return failure();
128 
129   // TODO: this should use the MLIR data layout when it becomes available and
130   // stop depending on translation.
131   LLVM::LLVMDialect *dialect = typeConverter.getDialect();
132   align = LLVM::TypeToLLVMIRTranslator(dialect->getLLVMContext())
133               .getPreferredAlignment(elementTy.cast<LLVM::LLVMType>(),
134                                      dialect->getLLVMModule().getDataLayout());
135   return success();
136 }
137 
138 // Helper that returns the base address of a memref.
139 LogicalResult getBase(ConversionPatternRewriter &rewriter, Location loc,
140                       Value memref, MemRefType memRefType, Value &base) {
141   // Inspect stride and offset structure.
142   //
143   // TODO: flat memory only for now, generalize
144   //
145   int64_t offset;
146   SmallVector<int64_t, 4> strides;
147   auto successStrides = getStridesAndOffset(memRefType, strides, offset);
148   if (failed(successStrides) || strides.size() != 1 || strides[0] != 1 ||
149       offset != 0 || memRefType.getMemorySpace() != 0)
150     return failure();
151   base = MemRefDescriptor(memref).alignedPtr(rewriter, loc);
152   return success();
153 }
154 
155 // Helper that returns a pointer given a memref base.
156 LogicalResult getBasePtr(ConversionPatternRewriter &rewriter, Location loc,
157                          Value memref, MemRefType memRefType, Value &ptr) {
158   Value base;
159   if (failed(getBase(rewriter, loc, memref, memRefType, base)))
160     return failure();
161   auto pType = MemRefDescriptor(memref).getElementType();
162   ptr = rewriter.create<LLVM::GEPOp>(loc, pType, base);
163   return success();
164 }
165 
166 // Helper that returns vector of pointers given a memref base and an index
167 // vector.
168 LogicalResult getIndexedPtrs(ConversionPatternRewriter &rewriter, Location loc,
169                              Value memref, Value indices, MemRefType memRefType,
170                              VectorType vType, Type iType, Value &ptrs) {
171   Value base;
172   if (failed(getBase(rewriter, loc, memref, memRefType, base)))
173     return failure();
174   auto pType = MemRefDescriptor(memref).getElementType();
175   auto ptrsType = LLVM::LLVMType::getVectorTy(pType, vType.getDimSize(0));
176   ptrs = rewriter.create<LLVM::GEPOp>(loc, ptrsType, base, indices);
177   return success();
178 }
179 
180 static LogicalResult
181 replaceTransferOpWithLoadOrStore(ConversionPatternRewriter &rewriter,
182                                  LLVMTypeConverter &typeConverter, Location loc,
183                                  TransferReadOp xferOp,
184                                  ArrayRef<Value> operands, Value dataPtr) {
185   unsigned align;
186   if (failed(getMemRefAlignment(typeConverter, xferOp, align)))
187     return failure();
188   rewriter.replaceOpWithNewOp<LLVM::LoadOp>(xferOp, dataPtr, align);
189   return success();
190 }
191 
192 static LogicalResult
193 replaceTransferOpWithMasked(ConversionPatternRewriter &rewriter,
194                             LLVMTypeConverter &typeConverter, Location loc,
195                             TransferReadOp xferOp, ArrayRef<Value> operands,
196                             Value dataPtr, Value mask) {
197   auto toLLVMTy = [&](Type t) { return typeConverter.convertType(t); };
198   VectorType fillType = xferOp.getVectorType();
199   Value fill = rewriter.create<SplatOp>(loc, fillType, xferOp.padding());
200   fill = rewriter.create<LLVM::DialectCastOp>(loc, toLLVMTy(fillType), fill);
201 
202   Type vecTy = typeConverter.convertType(xferOp.getVectorType());
203   if (!vecTy)
204     return failure();
205 
206   unsigned align;
207   if (failed(getMemRefAlignment(typeConverter, xferOp, align)))
208     return failure();
209 
210   rewriter.replaceOpWithNewOp<LLVM::MaskedLoadOp>(
211       xferOp, vecTy, dataPtr, mask, ValueRange{fill},
212       rewriter.getI32IntegerAttr(align));
213   return success();
214 }
215 
216 static LogicalResult
217 replaceTransferOpWithLoadOrStore(ConversionPatternRewriter &rewriter,
218                                  LLVMTypeConverter &typeConverter, Location loc,
219                                  TransferWriteOp xferOp,
220                                  ArrayRef<Value> operands, Value dataPtr) {
221   unsigned align;
222   if (failed(getMemRefAlignment(typeConverter, xferOp, align)))
223     return failure();
224   auto adaptor = TransferWriteOpAdaptor(operands);
225   rewriter.replaceOpWithNewOp<LLVM::StoreOp>(xferOp, adaptor.vector(), dataPtr,
226                                              align);
227   return success();
228 }
229 
230 static LogicalResult
231 replaceTransferOpWithMasked(ConversionPatternRewriter &rewriter,
232                             LLVMTypeConverter &typeConverter, Location loc,
233                             TransferWriteOp xferOp, ArrayRef<Value> operands,
234                             Value dataPtr, Value mask) {
235   unsigned align;
236   if (failed(getMemRefAlignment(typeConverter, xferOp, align)))
237     return failure();
238 
239   auto adaptor = TransferWriteOpAdaptor(operands);
240   rewriter.replaceOpWithNewOp<LLVM::MaskedStoreOp>(
241       xferOp, adaptor.vector(), dataPtr, mask,
242       rewriter.getI32IntegerAttr(align));
243   return success();
244 }
245 
246 static TransferReadOpAdaptor getTransferOpAdapter(TransferReadOp xferOp,
247                                                   ArrayRef<Value> operands) {
248   return TransferReadOpAdaptor(operands);
249 }
250 
251 static TransferWriteOpAdaptor getTransferOpAdapter(TransferWriteOp xferOp,
252                                                    ArrayRef<Value> operands) {
253   return TransferWriteOpAdaptor(operands);
254 }
255 
256 namespace {
257 
258 /// Conversion pattern for a vector.matrix_multiply.
259 /// This is lowered directly to the proper llvm.intr.matrix.multiply.
260 class VectorMatmulOpConversion : public ConvertToLLVMPattern {
261 public:
262   explicit VectorMatmulOpConversion(MLIRContext *context,
263                                     LLVMTypeConverter &typeConverter)
264       : ConvertToLLVMPattern(vector::MatmulOp::getOperationName(), context,
265                              typeConverter) {}
266 
267   LogicalResult
268   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
269                   ConversionPatternRewriter &rewriter) const override {
270     auto matmulOp = cast<vector::MatmulOp>(op);
271     auto adaptor = vector::MatmulOpAdaptor(operands);
272     rewriter.replaceOpWithNewOp<LLVM::MatrixMultiplyOp>(
273         op, typeConverter.convertType(matmulOp.res().getType()), adaptor.lhs(),
274         adaptor.rhs(), matmulOp.lhs_rows(), matmulOp.lhs_columns(),
275         matmulOp.rhs_columns());
276     return success();
277   }
278 };
279 
280 /// Conversion pattern for a vector.flat_transpose.
281 /// This is lowered directly to the proper llvm.intr.matrix.transpose.
282 class VectorFlatTransposeOpConversion : public ConvertToLLVMPattern {
283 public:
284   explicit VectorFlatTransposeOpConversion(MLIRContext *context,
285                                            LLVMTypeConverter &typeConverter)
286       : ConvertToLLVMPattern(vector::FlatTransposeOp::getOperationName(),
287                              context, typeConverter) {}
288 
289   LogicalResult
290   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
291                   ConversionPatternRewriter &rewriter) const override {
292     auto transOp = cast<vector::FlatTransposeOp>(op);
293     auto adaptor = vector::FlatTransposeOpAdaptor(operands);
294     rewriter.replaceOpWithNewOp<LLVM::MatrixTransposeOp>(
295         transOp, typeConverter.convertType(transOp.res().getType()),
296         adaptor.matrix(), transOp.rows(), transOp.columns());
297     return success();
298   }
299 };
300 
301 /// Conversion pattern for a vector.gather.
302 class VectorGatherOpConversion : public ConvertToLLVMPattern {
303 public:
304   explicit VectorGatherOpConversion(MLIRContext *context,
305                                     LLVMTypeConverter &typeConverter)
306       : ConvertToLLVMPattern(vector::GatherOp::getOperationName(), context,
307                              typeConverter) {}
308 
309   LogicalResult
310   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
311                   ConversionPatternRewriter &rewriter) const override {
312     auto loc = op->getLoc();
313     auto gather = cast<vector::GatherOp>(op);
314     auto adaptor = vector::GatherOpAdaptor(operands);
315 
316     // Resolve alignment.
317     unsigned align;
318     if (failed(getMemRefAlignment(typeConverter, gather, align)))
319       return failure();
320 
321     // Get index ptrs.
322     VectorType vType = gather.getResultVectorType();
323     Type iType = gather.getIndicesVectorType().getElementType();
324     Value ptrs;
325     if (failed(getIndexedPtrs(rewriter, loc, adaptor.base(), adaptor.indices(),
326                               gather.getMemRefType(), vType, iType, ptrs)))
327       return failure();
328 
329     // Replace with the gather intrinsic.
330     ValueRange v = (llvm::size(adaptor.pass_thru()) == 0) ? ValueRange({})
331                                                           : adaptor.pass_thru();
332     rewriter.replaceOpWithNewOp<LLVM::masked_gather>(
333         gather, typeConverter.convertType(vType), ptrs, adaptor.mask(), v,
334         rewriter.getI32IntegerAttr(align));
335     return success();
336   }
337 };
338 
339 /// Conversion pattern for a vector.scatter.
340 class VectorScatterOpConversion : public ConvertToLLVMPattern {
341 public:
342   explicit VectorScatterOpConversion(MLIRContext *context,
343                                      LLVMTypeConverter &typeConverter)
344       : ConvertToLLVMPattern(vector::ScatterOp::getOperationName(), context,
345                              typeConverter) {}
346 
347   LogicalResult
348   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
349                   ConversionPatternRewriter &rewriter) const override {
350     auto loc = op->getLoc();
351     auto scatter = cast<vector::ScatterOp>(op);
352     auto adaptor = vector::ScatterOpAdaptor(operands);
353 
354     // Resolve alignment.
355     unsigned align;
356     if (failed(getMemRefAlignment(typeConverter, scatter, align)))
357       return failure();
358 
359     // Get index ptrs.
360     VectorType vType = scatter.getValueVectorType();
361     Type iType = scatter.getIndicesVectorType().getElementType();
362     Value ptrs;
363     if (failed(getIndexedPtrs(rewriter, loc, adaptor.base(), adaptor.indices(),
364                               scatter.getMemRefType(), vType, iType, ptrs)))
365       return failure();
366 
367     // Replace with the scatter intrinsic.
368     rewriter.replaceOpWithNewOp<LLVM::masked_scatter>(
369         scatter, adaptor.value(), ptrs, adaptor.mask(),
370         rewriter.getI32IntegerAttr(align));
371     return success();
372   }
373 };
374 
375 /// Conversion pattern for a vector.expandload.
376 class VectorExpandLoadOpConversion : public ConvertToLLVMPattern {
377 public:
378   explicit VectorExpandLoadOpConversion(MLIRContext *context,
379                                         LLVMTypeConverter &typeConverter)
380       : ConvertToLLVMPattern(vector::ExpandLoadOp::getOperationName(), context,
381                              typeConverter) {}
382 
383   LogicalResult
384   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
385                   ConversionPatternRewriter &rewriter) const override {
386     auto loc = op->getLoc();
387     auto expand = cast<vector::ExpandLoadOp>(op);
388     auto adaptor = vector::ExpandLoadOpAdaptor(operands);
389 
390     Value ptr;
391     if (failed(getBasePtr(rewriter, loc, adaptor.base(), expand.getMemRefType(),
392                           ptr)))
393       return failure();
394 
395     auto vType = expand.getResultVectorType();
396     rewriter.replaceOpWithNewOp<LLVM::masked_expandload>(
397         op, typeConverter.convertType(vType), ptr, adaptor.mask(),
398         adaptor.pass_thru());
399     return success();
400   }
401 };
402 
403 /// Conversion pattern for a vector.compressstore.
404 class VectorCompressStoreOpConversion : public ConvertToLLVMPattern {
405 public:
406   explicit VectorCompressStoreOpConversion(MLIRContext *context,
407                                            LLVMTypeConverter &typeConverter)
408       : ConvertToLLVMPattern(vector::CompressStoreOp::getOperationName(),
409                              context, typeConverter) {}
410 
411   LogicalResult
412   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
413                   ConversionPatternRewriter &rewriter) const override {
414     auto loc = op->getLoc();
415     auto compress = cast<vector::CompressStoreOp>(op);
416     auto adaptor = vector::CompressStoreOpAdaptor(operands);
417 
418     Value ptr;
419     if (failed(getBasePtr(rewriter, loc, adaptor.base(),
420                           compress.getMemRefType(), ptr)))
421       return failure();
422 
423     rewriter.replaceOpWithNewOp<LLVM::masked_compressstore>(
424         op, adaptor.value(), ptr, adaptor.mask());
425     return success();
426   }
427 };
428 
429 /// Conversion pattern for all vector reductions.
430 class VectorReductionOpConversion : public ConvertToLLVMPattern {
431 public:
432   explicit VectorReductionOpConversion(MLIRContext *context,
433                                        LLVMTypeConverter &typeConverter,
434                                        bool reassociateFP)
435       : ConvertToLLVMPattern(vector::ReductionOp::getOperationName(), context,
436                              typeConverter),
437         reassociateFPReductions(reassociateFP) {}
438 
439   LogicalResult
440   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
441                   ConversionPatternRewriter &rewriter) const override {
442     auto reductionOp = cast<vector::ReductionOp>(op);
443     auto kind = reductionOp.kind();
444     Type eltType = reductionOp.dest().getType();
445     Type llvmType = typeConverter.convertType(eltType);
446     if (eltType.isSignlessInteger(32) || eltType.isSignlessInteger(64)) {
447       // Integer reductions: add/mul/min/max/and/or/xor.
448       if (kind == "add")
449         rewriter.replaceOpWithNewOp<LLVM::experimental_vector_reduce_add>(
450             op, llvmType, operands[0]);
451       else if (kind == "mul")
452         rewriter.replaceOpWithNewOp<LLVM::experimental_vector_reduce_mul>(
453             op, llvmType, operands[0]);
454       else if (kind == "min")
455         rewriter.replaceOpWithNewOp<LLVM::experimental_vector_reduce_smin>(
456             op, llvmType, operands[0]);
457       else if (kind == "max")
458         rewriter.replaceOpWithNewOp<LLVM::experimental_vector_reduce_smax>(
459             op, llvmType, operands[0]);
460       else if (kind == "and")
461         rewriter.replaceOpWithNewOp<LLVM::experimental_vector_reduce_and>(
462             op, llvmType, operands[0]);
463       else if (kind == "or")
464         rewriter.replaceOpWithNewOp<LLVM::experimental_vector_reduce_or>(
465             op, llvmType, operands[0]);
466       else if (kind == "xor")
467         rewriter.replaceOpWithNewOp<LLVM::experimental_vector_reduce_xor>(
468             op, llvmType, operands[0]);
469       else
470         return failure();
471       return success();
472 
473     } else if (eltType.isF32() || eltType.isF64()) {
474       // Floating-point reductions: add/mul/min/max
475       if (kind == "add") {
476         // Optional accumulator (or zero).
477         Value acc = operands.size() > 1 ? operands[1]
478                                         : rewriter.create<LLVM::ConstantOp>(
479                                               op->getLoc(), llvmType,
480                                               rewriter.getZeroAttr(eltType));
481         rewriter.replaceOpWithNewOp<LLVM::experimental_vector_reduce_v2_fadd>(
482             op, llvmType, acc, operands[0],
483             rewriter.getBoolAttr(reassociateFPReductions));
484       } else if (kind == "mul") {
485         // Optional accumulator (or one).
486         Value acc = operands.size() > 1
487                         ? operands[1]
488                         : rewriter.create<LLVM::ConstantOp>(
489                               op->getLoc(), llvmType,
490                               rewriter.getFloatAttr(eltType, 1.0));
491         rewriter.replaceOpWithNewOp<LLVM::experimental_vector_reduce_v2_fmul>(
492             op, llvmType, acc, operands[0],
493             rewriter.getBoolAttr(reassociateFPReductions));
494       } else if (kind == "min")
495         rewriter.replaceOpWithNewOp<LLVM::experimental_vector_reduce_fmin>(
496             op, llvmType, operands[0]);
497       else if (kind == "max")
498         rewriter.replaceOpWithNewOp<LLVM::experimental_vector_reduce_fmax>(
499             op, llvmType, operands[0]);
500       else
501         return failure();
502       return success();
503     }
504     return failure();
505   }
506 
507 private:
508   const bool reassociateFPReductions;
509 };
510 
511 class VectorShuffleOpConversion : public ConvertToLLVMPattern {
512 public:
513   explicit VectorShuffleOpConversion(MLIRContext *context,
514                                      LLVMTypeConverter &typeConverter)
515       : ConvertToLLVMPattern(vector::ShuffleOp::getOperationName(), context,
516                              typeConverter) {}
517 
518   LogicalResult
519   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
520                   ConversionPatternRewriter &rewriter) const override {
521     auto loc = op->getLoc();
522     auto adaptor = vector::ShuffleOpAdaptor(operands);
523     auto shuffleOp = cast<vector::ShuffleOp>(op);
524     auto v1Type = shuffleOp.getV1VectorType();
525     auto v2Type = shuffleOp.getV2VectorType();
526     auto vectorType = shuffleOp.getVectorType();
527     Type llvmType = typeConverter.convertType(vectorType);
528     auto maskArrayAttr = shuffleOp.mask();
529 
530     // Bail if result type cannot be lowered.
531     if (!llvmType)
532       return failure();
533 
534     // Get rank and dimension sizes.
535     int64_t rank = vectorType.getRank();
536     assert(v1Type.getRank() == rank);
537     assert(v2Type.getRank() == rank);
538     int64_t v1Dim = v1Type.getDimSize(0);
539 
540     // For rank 1, where both operands have *exactly* the same vector type,
541     // there is direct shuffle support in LLVM. Use it!
542     if (rank == 1 && v1Type == v2Type) {
543       Value shuffle = rewriter.create<LLVM::ShuffleVectorOp>(
544           loc, adaptor.v1(), adaptor.v2(), maskArrayAttr);
545       rewriter.replaceOp(op, shuffle);
546       return success();
547     }
548 
549     // For all other cases, insert the individual values individually.
550     Value insert = rewriter.create<LLVM::UndefOp>(loc, llvmType);
551     int64_t insPos = 0;
552     for (auto en : llvm::enumerate(maskArrayAttr)) {
553       int64_t extPos = en.value().cast<IntegerAttr>().getInt();
554       Value value = adaptor.v1();
555       if (extPos >= v1Dim) {
556         extPos -= v1Dim;
557         value = adaptor.v2();
558       }
559       Value extract = extractOne(rewriter, typeConverter, loc, value, llvmType,
560                                  rank, extPos);
561       insert = insertOne(rewriter, typeConverter, loc, insert, extract,
562                          llvmType, rank, insPos++);
563     }
564     rewriter.replaceOp(op, insert);
565     return success();
566   }
567 };
568 
569 class VectorExtractElementOpConversion : public ConvertToLLVMPattern {
570 public:
571   explicit VectorExtractElementOpConversion(MLIRContext *context,
572                                             LLVMTypeConverter &typeConverter)
573       : ConvertToLLVMPattern(vector::ExtractElementOp::getOperationName(),
574                              context, typeConverter) {}
575 
576   LogicalResult
577   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
578                   ConversionPatternRewriter &rewriter) const override {
579     auto adaptor = vector::ExtractElementOpAdaptor(operands);
580     auto extractEltOp = cast<vector::ExtractElementOp>(op);
581     auto vectorType = extractEltOp.getVectorType();
582     auto llvmType = typeConverter.convertType(vectorType.getElementType());
583 
584     // Bail if result type cannot be lowered.
585     if (!llvmType)
586       return failure();
587 
588     rewriter.replaceOpWithNewOp<LLVM::ExtractElementOp>(
589         op, llvmType, adaptor.vector(), adaptor.position());
590     return success();
591   }
592 };
593 
594 class VectorExtractOpConversion : public ConvertToLLVMPattern {
595 public:
596   explicit VectorExtractOpConversion(MLIRContext *context,
597                                      LLVMTypeConverter &typeConverter)
598       : ConvertToLLVMPattern(vector::ExtractOp::getOperationName(), context,
599                              typeConverter) {}
600 
601   LogicalResult
602   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
603                   ConversionPatternRewriter &rewriter) const override {
604     auto loc = op->getLoc();
605     auto adaptor = vector::ExtractOpAdaptor(operands);
606     auto extractOp = cast<vector::ExtractOp>(op);
607     auto vectorType = extractOp.getVectorType();
608     auto resultType = extractOp.getResult().getType();
609     auto llvmResultType = typeConverter.convertType(resultType);
610     auto positionArrayAttr = extractOp.position();
611 
612     // Bail if result type cannot be lowered.
613     if (!llvmResultType)
614       return failure();
615 
616     // One-shot extraction of vector from array (only requires extractvalue).
617     if (resultType.isa<VectorType>()) {
618       Value extracted = rewriter.create<LLVM::ExtractValueOp>(
619           loc, llvmResultType, adaptor.vector(), positionArrayAttr);
620       rewriter.replaceOp(op, extracted);
621       return success();
622     }
623 
624     // Potential extraction of 1-D vector from array.
625     auto *context = op->getContext();
626     Value extracted = adaptor.vector();
627     auto positionAttrs = positionArrayAttr.getValue();
628     if (positionAttrs.size() > 1) {
629       auto oneDVectorType = reducedVectorTypeBack(vectorType);
630       auto nMinusOnePositionAttrs =
631           ArrayAttr::get(positionAttrs.drop_back(), context);
632       extracted = rewriter.create<LLVM::ExtractValueOp>(
633           loc, typeConverter.convertType(oneDVectorType), extracted,
634           nMinusOnePositionAttrs);
635     }
636 
637     // Remaining extraction of element from 1-D LLVM vector
638     auto position = positionAttrs.back().cast<IntegerAttr>();
639     auto i64Type = LLVM::LLVMType::getInt64Ty(typeConverter.getDialect());
640     auto constant = rewriter.create<LLVM::ConstantOp>(loc, i64Type, position);
641     extracted =
642         rewriter.create<LLVM::ExtractElementOp>(loc, extracted, constant);
643     rewriter.replaceOp(op, extracted);
644 
645     return success();
646   }
647 };
648 
649 /// Conversion pattern that turns a vector.fma on a 1-D vector
650 /// into an llvm.intr.fmuladd. This is a trivial 1-1 conversion.
651 /// This does not match vectors of n >= 2 rank.
652 ///
653 /// Example:
654 /// ```
655 ///  vector.fma %a, %a, %a : vector<8xf32>
656 /// ```
657 /// is converted to:
658 /// ```
659 ///  llvm.intr.fmuladd %va, %va, %va:
660 ///    (!llvm<"<8 x float>">, !llvm<"<8 x float>">, !llvm<"<8 x float>">)
661 ///    -> !llvm<"<8 x float>">
662 /// ```
663 class VectorFMAOp1DConversion : public ConvertToLLVMPattern {
664 public:
665   explicit VectorFMAOp1DConversion(MLIRContext *context,
666                                    LLVMTypeConverter &typeConverter)
667       : ConvertToLLVMPattern(vector::FMAOp::getOperationName(), context,
668                              typeConverter) {}
669 
670   LogicalResult
671   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
672                   ConversionPatternRewriter &rewriter) const override {
673     auto adaptor = vector::FMAOpAdaptor(operands);
674     vector::FMAOp fmaOp = cast<vector::FMAOp>(op);
675     VectorType vType = fmaOp.getVectorType();
676     if (vType.getRank() != 1)
677       return failure();
678     rewriter.replaceOpWithNewOp<LLVM::FMulAddOp>(op, adaptor.lhs(),
679                                                  adaptor.rhs(), adaptor.acc());
680     return success();
681   }
682 };
683 
684 class VectorInsertElementOpConversion : public ConvertToLLVMPattern {
685 public:
686   explicit VectorInsertElementOpConversion(MLIRContext *context,
687                                            LLVMTypeConverter &typeConverter)
688       : ConvertToLLVMPattern(vector::InsertElementOp::getOperationName(),
689                              context, typeConverter) {}
690 
691   LogicalResult
692   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
693                   ConversionPatternRewriter &rewriter) const override {
694     auto adaptor = vector::InsertElementOpAdaptor(operands);
695     auto insertEltOp = cast<vector::InsertElementOp>(op);
696     auto vectorType = insertEltOp.getDestVectorType();
697     auto llvmType = typeConverter.convertType(vectorType);
698 
699     // Bail if result type cannot be lowered.
700     if (!llvmType)
701       return failure();
702 
703     rewriter.replaceOpWithNewOp<LLVM::InsertElementOp>(
704         op, llvmType, adaptor.dest(), adaptor.source(), adaptor.position());
705     return success();
706   }
707 };
708 
709 class VectorInsertOpConversion : public ConvertToLLVMPattern {
710 public:
711   explicit VectorInsertOpConversion(MLIRContext *context,
712                                     LLVMTypeConverter &typeConverter)
713       : ConvertToLLVMPattern(vector::InsertOp::getOperationName(), context,
714                              typeConverter) {}
715 
716   LogicalResult
717   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
718                   ConversionPatternRewriter &rewriter) const override {
719     auto loc = op->getLoc();
720     auto adaptor = vector::InsertOpAdaptor(operands);
721     auto insertOp = cast<vector::InsertOp>(op);
722     auto sourceType = insertOp.getSourceType();
723     auto destVectorType = insertOp.getDestVectorType();
724     auto llvmResultType = typeConverter.convertType(destVectorType);
725     auto positionArrayAttr = insertOp.position();
726 
727     // Bail if result type cannot be lowered.
728     if (!llvmResultType)
729       return failure();
730 
731     // One-shot insertion of a vector into an array (only requires insertvalue).
732     if (sourceType.isa<VectorType>()) {
733       Value inserted = rewriter.create<LLVM::InsertValueOp>(
734           loc, llvmResultType, adaptor.dest(), adaptor.source(),
735           positionArrayAttr);
736       rewriter.replaceOp(op, inserted);
737       return success();
738     }
739 
740     // Potential extraction of 1-D vector from array.
741     auto *context = op->getContext();
742     Value extracted = adaptor.dest();
743     auto positionAttrs = positionArrayAttr.getValue();
744     auto position = positionAttrs.back().cast<IntegerAttr>();
745     auto oneDVectorType = destVectorType;
746     if (positionAttrs.size() > 1) {
747       oneDVectorType = reducedVectorTypeBack(destVectorType);
748       auto nMinusOnePositionAttrs =
749           ArrayAttr::get(positionAttrs.drop_back(), context);
750       extracted = rewriter.create<LLVM::ExtractValueOp>(
751           loc, typeConverter.convertType(oneDVectorType), extracted,
752           nMinusOnePositionAttrs);
753     }
754 
755     // Insertion of an element into a 1-D LLVM vector.
756     auto i64Type = LLVM::LLVMType::getInt64Ty(typeConverter.getDialect());
757     auto constant = rewriter.create<LLVM::ConstantOp>(loc, i64Type, position);
758     Value inserted = rewriter.create<LLVM::InsertElementOp>(
759         loc, typeConverter.convertType(oneDVectorType), extracted,
760         adaptor.source(), constant);
761 
762     // Potential insertion of resulting 1-D vector into array.
763     if (positionAttrs.size() > 1) {
764       auto nMinusOnePositionAttrs =
765           ArrayAttr::get(positionAttrs.drop_back(), context);
766       inserted = rewriter.create<LLVM::InsertValueOp>(loc, llvmResultType,
767                                                       adaptor.dest(), inserted,
768                                                       nMinusOnePositionAttrs);
769     }
770 
771     rewriter.replaceOp(op, inserted);
772     return success();
773   }
774 };
775 
776 /// Rank reducing rewrite for n-D FMA into (n-1)-D FMA where n > 1.
777 ///
778 /// Example:
779 /// ```
780 ///   %d = vector.fma %a, %b, %c : vector<2x4xf32>
781 /// ```
782 /// is rewritten into:
783 /// ```
784 ///  %r = splat %f0: vector<2x4xf32>
785 ///  %va = vector.extractvalue %a[0] : vector<2x4xf32>
786 ///  %vb = vector.extractvalue %b[0] : vector<2x4xf32>
787 ///  %vc = vector.extractvalue %c[0] : vector<2x4xf32>
788 ///  %vd = vector.fma %va, %vb, %vc : vector<4xf32>
789 ///  %r2 = vector.insertvalue %vd, %r[0] : vector<4xf32> into vector<2x4xf32>
790 ///  %va2 = vector.extractvalue %a2[1] : vector<2x4xf32>
791 ///  %vb2 = vector.extractvalue %b2[1] : vector<2x4xf32>
792 ///  %vc2 = vector.extractvalue %c2[1] : vector<2x4xf32>
793 ///  %vd2 = vector.fma %va2, %vb2, %vc2 : vector<4xf32>
794 ///  %r3 = vector.insertvalue %vd2, %r2[1] : vector<4xf32> into vector<2x4xf32>
795 ///  // %r3 holds the final value.
796 /// ```
797 class VectorFMAOpNDRewritePattern : public OpRewritePattern<FMAOp> {
798 public:
799   using OpRewritePattern<FMAOp>::OpRewritePattern;
800 
801   LogicalResult matchAndRewrite(FMAOp op,
802                                 PatternRewriter &rewriter) const override {
803     auto vType = op.getVectorType();
804     if (vType.getRank() < 2)
805       return failure();
806 
807     auto loc = op.getLoc();
808     auto elemType = vType.getElementType();
809     Value zero = rewriter.create<ConstantOp>(loc, elemType,
810                                              rewriter.getZeroAttr(elemType));
811     Value desc = rewriter.create<SplatOp>(loc, vType, zero);
812     for (int64_t i = 0, e = vType.getShape().front(); i != e; ++i) {
813       Value extrLHS = rewriter.create<ExtractOp>(loc, op.lhs(), i);
814       Value extrRHS = rewriter.create<ExtractOp>(loc, op.rhs(), i);
815       Value extrACC = rewriter.create<ExtractOp>(loc, op.acc(), i);
816       Value fma = rewriter.create<FMAOp>(loc, extrLHS, extrRHS, extrACC);
817       desc = rewriter.create<InsertOp>(loc, fma, desc, i);
818     }
819     rewriter.replaceOp(op, desc);
820     return success();
821   }
822 };
823 
824 // When ranks are different, InsertStridedSlice needs to extract a properly
825 // ranked vector from the destination vector into which to insert. This pattern
826 // only takes care of this part and forwards the rest of the conversion to
827 // another pattern that converts InsertStridedSlice for operands of the same
828 // rank.
829 //
830 // RewritePattern for InsertStridedSliceOp where source and destination vectors
831 // have different ranks. In this case:
832 //   1. the proper subvector is extracted from the destination vector
833 //   2. a new InsertStridedSlice op is created to insert the source in the
834 //   destination subvector
835 //   3. the destination subvector is inserted back in the proper place
836 //   4. the op is replaced by the result of step 3.
837 // The new InsertStridedSlice from step 2. will be picked up by a
838 // `VectorInsertStridedSliceOpSameRankRewritePattern`.
839 class VectorInsertStridedSliceOpDifferentRankRewritePattern
840     : public OpRewritePattern<InsertStridedSliceOp> {
841 public:
842   using OpRewritePattern<InsertStridedSliceOp>::OpRewritePattern;
843 
844   LogicalResult matchAndRewrite(InsertStridedSliceOp op,
845                                 PatternRewriter &rewriter) const override {
846     auto srcType = op.getSourceVectorType();
847     auto dstType = op.getDestVectorType();
848 
849     if (op.offsets().getValue().empty())
850       return failure();
851 
852     auto loc = op.getLoc();
853     int64_t rankDiff = dstType.getRank() - srcType.getRank();
854     assert(rankDiff >= 0);
855     if (rankDiff == 0)
856       return failure();
857 
858     int64_t rankRest = dstType.getRank() - rankDiff;
859     // Extract / insert the subvector of matching rank and InsertStridedSlice
860     // on it.
861     Value extracted =
862         rewriter.create<ExtractOp>(loc, op.dest(),
863                                    getI64SubArray(op.offsets(), /*dropFront=*/0,
864                                                   /*dropFront=*/rankRest));
865     // A different pattern will kick in for InsertStridedSlice with matching
866     // ranks.
867     auto stridedSliceInnerOp = rewriter.create<InsertStridedSliceOp>(
868         loc, op.source(), extracted,
869         getI64SubArray(op.offsets(), /*dropFront=*/rankDiff),
870         getI64SubArray(op.strides(), /*dropFront=*/0));
871     rewriter.replaceOpWithNewOp<InsertOp>(
872         op, stridedSliceInnerOp.getResult(), op.dest(),
873         getI64SubArray(op.offsets(), /*dropFront=*/0,
874                        /*dropFront=*/rankRest));
875     return success();
876   }
877 };
878 
879 // RewritePattern for InsertStridedSliceOp where source and destination vectors
880 // have the same rank. In this case, we reduce
881 //   1. the proper subvector is extracted from the destination vector
882 //   2. a new InsertStridedSlice op is created to insert the source in the
883 //   destination subvector
884 //   3. the destination subvector is inserted back in the proper place
885 //   4. the op is replaced by the result of step 3.
886 // The new InsertStridedSlice from step 2. will be picked up by a
887 // `VectorInsertStridedSliceOpSameRankRewritePattern`.
888 class VectorInsertStridedSliceOpSameRankRewritePattern
889     : public OpRewritePattern<InsertStridedSliceOp> {
890 public:
891   using OpRewritePattern<InsertStridedSliceOp>::OpRewritePattern;
892 
893   LogicalResult matchAndRewrite(InsertStridedSliceOp op,
894                                 PatternRewriter &rewriter) const override {
895     auto srcType = op.getSourceVectorType();
896     auto dstType = op.getDestVectorType();
897 
898     if (op.offsets().getValue().empty())
899       return failure();
900 
901     int64_t rankDiff = dstType.getRank() - srcType.getRank();
902     assert(rankDiff >= 0);
903     if (rankDiff != 0)
904       return failure();
905 
906     if (srcType == dstType) {
907       rewriter.replaceOp(op, op.source());
908       return success();
909     }
910 
911     int64_t offset =
912         op.offsets().getValue().front().cast<IntegerAttr>().getInt();
913     int64_t size = srcType.getShape().front();
914     int64_t stride =
915         op.strides().getValue().front().cast<IntegerAttr>().getInt();
916 
917     auto loc = op.getLoc();
918     Value res = op.dest();
919     // For each slice of the source vector along the most major dimension.
920     for (int64_t off = offset, e = offset + size * stride, idx = 0; off < e;
921          off += stride, ++idx) {
922       // 1. extract the proper subvector (or element) from source
923       Value extractedSource = extractOne(rewriter, loc, op.source(), idx);
924       if (extractedSource.getType().isa<VectorType>()) {
925         // 2. If we have a vector, extract the proper subvector from destination
926         // Otherwise we are at the element level and no need to recurse.
927         Value extractedDest = extractOne(rewriter, loc, op.dest(), off);
928         // 3. Reduce the problem to lowering a new InsertStridedSlice op with
929         // smaller rank.
930         extractedSource = rewriter.create<InsertStridedSliceOp>(
931             loc, extractedSource, extractedDest,
932             getI64SubArray(op.offsets(), /* dropFront=*/1),
933             getI64SubArray(op.strides(), /* dropFront=*/1));
934       }
935       // 4. Insert the extractedSource into the res vector.
936       res = insertOne(rewriter, loc, extractedSource, res, off);
937     }
938 
939     rewriter.replaceOp(op, res);
940     return success();
941   }
942   /// This pattern creates recursive InsertStridedSliceOp, but the recursion is
943   /// bounded as the rank is strictly decreasing.
944   bool hasBoundedRewriteRecursion() const final { return true; }
945 };
946 
947 class VectorTypeCastOpConversion : public ConvertToLLVMPattern {
948 public:
949   explicit VectorTypeCastOpConversion(MLIRContext *context,
950                                       LLVMTypeConverter &typeConverter)
951       : ConvertToLLVMPattern(vector::TypeCastOp::getOperationName(), context,
952                              typeConverter) {}
953 
954   LogicalResult
955   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
956                   ConversionPatternRewriter &rewriter) const override {
957     auto loc = op->getLoc();
958     vector::TypeCastOp castOp = cast<vector::TypeCastOp>(op);
959     MemRefType sourceMemRefType =
960         castOp.getOperand().getType().cast<MemRefType>();
961     MemRefType targetMemRefType =
962         castOp.getResult().getType().cast<MemRefType>();
963 
964     // Only static shape casts supported atm.
965     if (!sourceMemRefType.hasStaticShape() ||
966         !targetMemRefType.hasStaticShape())
967       return failure();
968 
969     auto llvmSourceDescriptorTy =
970         operands[0].getType().dyn_cast<LLVM::LLVMType>();
971     if (!llvmSourceDescriptorTy || !llvmSourceDescriptorTy.isStructTy())
972       return failure();
973     MemRefDescriptor sourceMemRef(operands[0]);
974 
975     auto llvmTargetDescriptorTy = typeConverter.convertType(targetMemRefType)
976                                       .dyn_cast_or_null<LLVM::LLVMType>();
977     if (!llvmTargetDescriptorTy || !llvmTargetDescriptorTy.isStructTy())
978       return failure();
979 
980     int64_t offset;
981     SmallVector<int64_t, 4> strides;
982     auto successStrides =
983         getStridesAndOffset(sourceMemRefType, strides, offset);
984     bool isContiguous = (strides.back() == 1);
985     if (isContiguous) {
986       auto sizes = sourceMemRefType.getShape();
987       for (int index = 0, e = strides.size() - 2; index < e; ++index) {
988         if (strides[index] != strides[index + 1] * sizes[index + 1]) {
989           isContiguous = false;
990           break;
991         }
992       }
993     }
994     // Only contiguous source tensors supported atm.
995     if (failed(successStrides) || !isContiguous)
996       return failure();
997 
998     auto int64Ty = LLVM::LLVMType::getInt64Ty(typeConverter.getDialect());
999 
1000     // Create descriptor.
1001     auto desc = MemRefDescriptor::undef(rewriter, loc, llvmTargetDescriptorTy);
1002     Type llvmTargetElementTy = desc.getElementType();
1003     // Set allocated ptr.
1004     Value allocated = sourceMemRef.allocatedPtr(rewriter, loc);
1005     allocated =
1006         rewriter.create<LLVM::BitcastOp>(loc, llvmTargetElementTy, allocated);
1007     desc.setAllocatedPtr(rewriter, loc, allocated);
1008     // Set aligned ptr.
1009     Value ptr = sourceMemRef.alignedPtr(rewriter, loc);
1010     ptr = rewriter.create<LLVM::BitcastOp>(loc, llvmTargetElementTy, ptr);
1011     desc.setAlignedPtr(rewriter, loc, ptr);
1012     // Fill offset 0.
1013     auto attr = rewriter.getIntegerAttr(rewriter.getIndexType(), 0);
1014     auto zero = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, attr);
1015     desc.setOffset(rewriter, loc, zero);
1016 
1017     // Fill size and stride descriptors in memref.
1018     for (auto indexedSize : llvm::enumerate(targetMemRefType.getShape())) {
1019       int64_t index = indexedSize.index();
1020       auto sizeAttr =
1021           rewriter.getIntegerAttr(rewriter.getIndexType(), indexedSize.value());
1022       auto size = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, sizeAttr);
1023       desc.setSize(rewriter, loc, index, size);
1024       auto strideAttr =
1025           rewriter.getIntegerAttr(rewriter.getIndexType(), strides[index]);
1026       auto stride = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, strideAttr);
1027       desc.setStride(rewriter, loc, index, stride);
1028     }
1029 
1030     rewriter.replaceOp(op, {desc});
1031     return success();
1032   }
1033 };
1034 
1035 /// Conversion pattern that converts a 1-D vector transfer read/write op in a
1036 /// sequence of:
1037 /// 1. Bitcast or addrspacecast to vector form.
1038 /// 2. Create an offsetVector = [ offset + 0 .. offset + vector_length - 1 ].
1039 /// 3. Create a mask where offsetVector is compared against memref upper bound.
1040 /// 4. Rewrite op as a masked read or write.
1041 template <typename ConcreteOp>
1042 class VectorTransferConversion : public ConvertToLLVMPattern {
1043 public:
1044   explicit VectorTransferConversion(MLIRContext *context,
1045                                     LLVMTypeConverter &typeConv)
1046       : ConvertToLLVMPattern(ConcreteOp::getOperationName(), context,
1047                              typeConv) {}
1048 
1049   LogicalResult
1050   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
1051                   ConversionPatternRewriter &rewriter) const override {
1052     auto xferOp = cast<ConcreteOp>(op);
1053     auto adaptor = getTransferOpAdapter(xferOp, operands);
1054 
1055     if (xferOp.getVectorType().getRank() > 1 ||
1056         llvm::size(xferOp.indices()) == 0)
1057       return failure();
1058     if (xferOp.permutation_map() !=
1059         AffineMap::getMinorIdentityMap(xferOp.permutation_map().getNumInputs(),
1060                                        xferOp.getVectorType().getRank(),
1061                                        op->getContext()))
1062       return failure();
1063 
1064     auto toLLVMTy = [&](Type t) { return typeConverter.convertType(t); };
1065 
1066     Location loc = op->getLoc();
1067     Type i64Type = rewriter.getIntegerType(64);
1068     MemRefType memRefType = xferOp.getMemRefType();
1069 
1070     // 1. Get the source/dst address as an LLVM vector pointer.
1071     //    The vector pointer would always be on address space 0, therefore
1072     //    addrspacecast shall be used when source/dst memrefs are not on
1073     //    address space 0.
1074     // TODO: support alignment when possible.
1075     Value dataPtr = getDataPtr(loc, memRefType, adaptor.memref(),
1076                                adaptor.indices(), rewriter, getModule());
1077     auto vecTy =
1078         toLLVMTy(xferOp.getVectorType()).template cast<LLVM::LLVMType>();
1079     Value vectorDataPtr;
1080     if (memRefType.getMemorySpace() == 0)
1081       vectorDataPtr =
1082           rewriter.create<LLVM::BitcastOp>(loc, vecTy.getPointerTo(), dataPtr);
1083     else
1084       vectorDataPtr = rewriter.create<LLVM::AddrSpaceCastOp>(
1085           loc, vecTy.getPointerTo(), dataPtr);
1086 
1087     if (!xferOp.isMaskedDim(0))
1088       return replaceTransferOpWithLoadOrStore(rewriter, typeConverter, loc,
1089                                               xferOp, operands, vectorDataPtr);
1090 
1091     // 2. Create a vector with linear indices [ 0 .. vector_length - 1 ].
1092     unsigned vecWidth = vecTy.getVectorNumElements();
1093     VectorType vectorCmpType = VectorType::get(vecWidth, i64Type);
1094     SmallVector<int64_t, 8> indices;
1095     indices.reserve(vecWidth);
1096     for (unsigned i = 0; i < vecWidth; ++i)
1097       indices.push_back(i);
1098     Value linearIndices = rewriter.create<ConstantOp>(
1099         loc, vectorCmpType,
1100         DenseElementsAttr::get(vectorCmpType, ArrayRef<int64_t>(indices)));
1101     linearIndices = rewriter.create<LLVM::DialectCastOp>(
1102         loc, toLLVMTy(vectorCmpType), linearIndices);
1103 
1104     // 3. Create offsetVector = [ offset + 0 .. offset + vector_length - 1 ].
1105     // TODO: when the leaf transfer rank is k > 1 we need the last
1106     // `k` dimensions here.
1107     unsigned lastIndex = llvm::size(xferOp.indices()) - 1;
1108     Value offsetIndex = *(xferOp.indices().begin() + lastIndex);
1109     offsetIndex = rewriter.create<IndexCastOp>(loc, i64Type, offsetIndex);
1110     Value base = rewriter.create<SplatOp>(loc, vectorCmpType, offsetIndex);
1111     Value offsetVector = rewriter.create<AddIOp>(loc, base, linearIndices);
1112 
1113     // 4. Let dim the memref dimension, compute the vector comparison mask:
1114     //   [ offset + 0 .. offset + vector_length - 1 ] < [ dim .. dim ]
1115     Value dim = rewriter.create<DimOp>(loc, xferOp.memref(), lastIndex);
1116     dim = rewriter.create<IndexCastOp>(loc, i64Type, dim);
1117     dim = rewriter.create<SplatOp>(loc, vectorCmpType, dim);
1118     Value mask =
1119         rewriter.create<CmpIOp>(loc, CmpIPredicate::slt, offsetVector, dim);
1120     mask = rewriter.create<LLVM::DialectCastOp>(loc, toLLVMTy(mask.getType()),
1121                                                 mask);
1122 
1123     // 5. Rewrite as a masked read / write.
1124     return replaceTransferOpWithMasked(rewriter, typeConverter, loc, xferOp,
1125                                        operands, vectorDataPtr, mask);
1126   }
1127 };
1128 
1129 class VectorPrintOpConversion : public ConvertToLLVMPattern {
1130 public:
1131   explicit VectorPrintOpConversion(MLIRContext *context,
1132                                    LLVMTypeConverter &typeConverter)
1133       : ConvertToLLVMPattern(vector::PrintOp::getOperationName(), context,
1134                              typeConverter) {}
1135 
1136   // Proof-of-concept lowering implementation that relies on a small
1137   // runtime support library, which only needs to provide a few
1138   // printing methods (single value for all data types, opening/closing
1139   // bracket, comma, newline). The lowering fully unrolls a vector
1140   // in terms of these elementary printing operations. The advantage
1141   // of this approach is that the library can remain unaware of all
1142   // low-level implementation details of vectors while still supporting
1143   // output of any shaped and dimensioned vector. Due to full unrolling,
1144   // this approach is less suited for very large vectors though.
1145   //
1146   // TODO: rely solely on libc in future? something else?
1147   //
1148   LogicalResult
1149   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
1150                   ConversionPatternRewriter &rewriter) const override {
1151     auto printOp = cast<vector::PrintOp>(op);
1152     auto adaptor = vector::PrintOpAdaptor(operands);
1153     Type printType = printOp.getPrintType();
1154 
1155     if (typeConverter.convertType(printType) == nullptr)
1156       return failure();
1157 
1158     // Make sure element type has runtime support (currently just Float/Double).
1159     VectorType vectorType = printType.dyn_cast<VectorType>();
1160     Type eltType = vectorType ? vectorType.getElementType() : printType;
1161     int64_t rank = vectorType ? vectorType.getRank() : 0;
1162     Operation *printer;
1163     if (eltType.isSignlessInteger(1) || eltType.isSignlessInteger(32))
1164       printer = getPrintI32(op);
1165     else if (eltType.isSignlessInteger(64))
1166       printer = getPrintI64(op);
1167     else if (eltType.isF32())
1168       printer = getPrintFloat(op);
1169     else if (eltType.isF64())
1170       printer = getPrintDouble(op);
1171     else
1172       return failure();
1173 
1174     // Unroll vector into elementary print calls.
1175     emitRanks(rewriter, op, adaptor.source(), vectorType, printer, rank);
1176     emitCall(rewriter, op->getLoc(), getPrintNewline(op));
1177     rewriter.eraseOp(op);
1178     return success();
1179   }
1180 
1181 private:
1182   void emitRanks(ConversionPatternRewriter &rewriter, Operation *op,
1183                  Value value, VectorType vectorType, Operation *printer,
1184                  int64_t rank) const {
1185     Location loc = op->getLoc();
1186     if (rank == 0) {
1187       if (value.getType() ==
1188           LLVM::LLVMType::getInt1Ty(typeConverter.getDialect())) {
1189         // Convert i1 (bool) to i32 so we can use the print_i32 method.
1190         // This avoids the need for a print_i1 method with an unclear ABI.
1191         auto i32Type = LLVM::LLVMType::getInt32Ty(typeConverter.getDialect());
1192         auto trueVal = rewriter.create<ConstantOp>(
1193             loc, i32Type, rewriter.getI32IntegerAttr(1));
1194         auto falseVal = rewriter.create<ConstantOp>(
1195             loc, i32Type, rewriter.getI32IntegerAttr(0));
1196         value = rewriter.create<SelectOp>(loc, value, trueVal, falseVal);
1197       }
1198       emitCall(rewriter, loc, printer, value);
1199       return;
1200     }
1201 
1202     emitCall(rewriter, loc, getPrintOpen(op));
1203     Operation *printComma = getPrintComma(op);
1204     int64_t dim = vectorType.getDimSize(0);
1205     for (int64_t d = 0; d < dim; ++d) {
1206       auto reducedType =
1207           rank > 1 ? reducedVectorTypeFront(vectorType) : nullptr;
1208       auto llvmType = typeConverter.convertType(
1209           rank > 1 ? reducedType : vectorType.getElementType());
1210       Value nestedVal =
1211           extractOne(rewriter, typeConverter, loc, value, llvmType, rank, d);
1212       emitRanks(rewriter, op, nestedVal, reducedType, printer, rank - 1);
1213       if (d != dim - 1)
1214         emitCall(rewriter, loc, printComma);
1215     }
1216     emitCall(rewriter, loc, getPrintClose(op));
1217   }
1218 
1219   // Helper to emit a call.
1220   static void emitCall(ConversionPatternRewriter &rewriter, Location loc,
1221                        Operation *ref, ValueRange params = ValueRange()) {
1222     rewriter.create<LLVM::CallOp>(loc, ArrayRef<Type>{},
1223                                   rewriter.getSymbolRefAttr(ref), params);
1224   }
1225 
1226   // Helper for printer method declaration (first hit) and lookup.
1227   static Operation *getPrint(Operation *op, LLVM::LLVMDialect *dialect,
1228                              StringRef name, ArrayRef<LLVM::LLVMType> params) {
1229     auto module = op->getParentOfType<ModuleOp>();
1230     auto func = module.lookupSymbol<LLVM::LLVMFuncOp>(name);
1231     if (func)
1232       return func;
1233     OpBuilder moduleBuilder(module.getBodyRegion());
1234     return moduleBuilder.create<LLVM::LLVMFuncOp>(
1235         op->getLoc(), name,
1236         LLVM::LLVMType::getFunctionTy(LLVM::LLVMType::getVoidTy(dialect),
1237                                       params, /*isVarArg=*/false));
1238   }
1239 
1240   // Helpers for method names.
1241   Operation *getPrintI32(Operation *op) const {
1242     LLVM::LLVMDialect *dialect = typeConverter.getDialect();
1243     return getPrint(op, dialect, "print_i32",
1244                     LLVM::LLVMType::getInt32Ty(dialect));
1245   }
1246   Operation *getPrintI64(Operation *op) const {
1247     LLVM::LLVMDialect *dialect = typeConverter.getDialect();
1248     return getPrint(op, dialect, "print_i64",
1249                     LLVM::LLVMType::getInt64Ty(dialect));
1250   }
1251   Operation *getPrintFloat(Operation *op) const {
1252     LLVM::LLVMDialect *dialect = typeConverter.getDialect();
1253     return getPrint(op, dialect, "print_f32",
1254                     LLVM::LLVMType::getFloatTy(dialect));
1255   }
1256   Operation *getPrintDouble(Operation *op) const {
1257     LLVM::LLVMDialect *dialect = typeConverter.getDialect();
1258     return getPrint(op, dialect, "print_f64",
1259                     LLVM::LLVMType::getDoubleTy(dialect));
1260   }
1261   Operation *getPrintOpen(Operation *op) const {
1262     return getPrint(op, typeConverter.getDialect(), "print_open", {});
1263   }
1264   Operation *getPrintClose(Operation *op) const {
1265     return getPrint(op, typeConverter.getDialect(), "print_close", {});
1266   }
1267   Operation *getPrintComma(Operation *op) const {
1268     return getPrint(op, typeConverter.getDialect(), "print_comma", {});
1269   }
1270   Operation *getPrintNewline(Operation *op) const {
1271     return getPrint(op, typeConverter.getDialect(), "print_newline", {});
1272   }
1273 };
1274 
1275 /// Progressive lowering of ExtractStridedSliceOp to either:
1276 ///   1. extractelement + insertelement for the 1-D case
1277 ///   2. extract + optional strided_slice + insert for the n-D case.
1278 class VectorStridedSliceOpConversion
1279     : public OpRewritePattern<ExtractStridedSliceOp> {
1280 public:
1281   using OpRewritePattern<ExtractStridedSliceOp>::OpRewritePattern;
1282 
1283   LogicalResult matchAndRewrite(ExtractStridedSliceOp op,
1284                                 PatternRewriter &rewriter) const override {
1285     auto dstType = op.getResult().getType().cast<VectorType>();
1286 
1287     assert(!op.offsets().getValue().empty() && "Unexpected empty offsets");
1288 
1289     int64_t offset =
1290         op.offsets().getValue().front().cast<IntegerAttr>().getInt();
1291     int64_t size = op.sizes().getValue().front().cast<IntegerAttr>().getInt();
1292     int64_t stride =
1293         op.strides().getValue().front().cast<IntegerAttr>().getInt();
1294 
1295     auto loc = op.getLoc();
1296     auto elemType = dstType.getElementType();
1297     assert(elemType.isSignlessIntOrIndexOrFloat());
1298     Value zero = rewriter.create<ConstantOp>(loc, elemType,
1299                                              rewriter.getZeroAttr(elemType));
1300     Value res = rewriter.create<SplatOp>(loc, dstType, zero);
1301     for (int64_t off = offset, e = offset + size * stride, idx = 0; off < e;
1302          off += stride, ++idx) {
1303       Value extracted = extractOne(rewriter, loc, op.vector(), off);
1304       if (op.offsets().getValue().size() > 1) {
1305         extracted = rewriter.create<ExtractStridedSliceOp>(
1306             loc, extracted, getI64SubArray(op.offsets(), /* dropFront=*/1),
1307             getI64SubArray(op.sizes(), /* dropFront=*/1),
1308             getI64SubArray(op.strides(), /* dropFront=*/1));
1309       }
1310       res = insertOne(rewriter, loc, extracted, res, idx);
1311     }
1312     rewriter.replaceOp(op, {res});
1313     return success();
1314   }
1315   /// This pattern creates recursive ExtractStridedSliceOp, but the recursion is
1316   /// bounded as the rank is strictly decreasing.
1317   bool hasBoundedRewriteRecursion() const final { return true; }
1318 };
1319 
1320 } // namespace
1321 
1322 /// Populate the given list with patterns that convert from Vector to LLVM.
1323 void mlir::populateVectorToLLVMConversionPatterns(
1324     LLVMTypeConverter &converter, OwningRewritePatternList &patterns,
1325     bool reassociateFPReductions) {
1326   MLIRContext *ctx = converter.getDialect()->getContext();
1327   // clang-format off
1328   patterns.insert<VectorFMAOpNDRewritePattern,
1329                   VectorInsertStridedSliceOpDifferentRankRewritePattern,
1330                   VectorInsertStridedSliceOpSameRankRewritePattern,
1331                   VectorStridedSliceOpConversion>(ctx);
1332   patterns.insert<VectorReductionOpConversion>(
1333       ctx, converter, reassociateFPReductions);
1334   patterns
1335       .insert<VectorShuffleOpConversion,
1336               VectorExtractElementOpConversion,
1337               VectorExtractOpConversion,
1338               VectorFMAOp1DConversion,
1339               VectorInsertElementOpConversion,
1340               VectorInsertOpConversion,
1341               VectorPrintOpConversion,
1342               VectorTransferConversion<TransferReadOp>,
1343               VectorTransferConversion<TransferWriteOp>,
1344               VectorTypeCastOpConversion,
1345               VectorGatherOpConversion,
1346               VectorScatterOpConversion,
1347               VectorExpandLoadOpConversion,
1348               VectorCompressStoreOpConversion>(ctx, converter);
1349   // clang-format on
1350 }
1351 
1352 void mlir::populateVectorToLLVMMatrixConversionPatterns(
1353     LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
1354   MLIRContext *ctx = converter.getDialect()->getContext();
1355   patterns.insert<VectorMatmulOpConversion>(ctx, converter);
1356   patterns.insert<VectorFlatTransposeOpConversion>(ctx, converter);
1357 }
1358 
1359 namespace {
1360 struct LowerVectorToLLVMPass
1361     : public ConvertVectorToLLVMBase<LowerVectorToLLVMPass> {
1362   LowerVectorToLLVMPass(const LowerVectorToLLVMOptions &options) {
1363     this->reassociateFPReductions = options.reassociateFPReductions;
1364   }
1365   void runOnOperation() override;
1366 };
1367 } // namespace
1368 
1369 void LowerVectorToLLVMPass::runOnOperation() {
1370   // Perform progressive lowering of operations on slices and
1371   // all contraction operations. Also applies folding and DCE.
1372   {
1373     OwningRewritePatternList patterns;
1374     populateVectorToVectorCanonicalizationPatterns(patterns, &getContext());
1375     populateVectorSlicesLoweringPatterns(patterns, &getContext());
1376     populateVectorContractLoweringPatterns(patterns, &getContext());
1377     applyPatternsAndFoldGreedily(getOperation(), patterns);
1378   }
1379 
1380   // Convert to the LLVM IR dialect.
1381   LLVMTypeConverter converter(&getContext());
1382   OwningRewritePatternList patterns;
1383   populateVectorToLLVMMatrixConversionPatterns(converter, patterns);
1384   populateVectorToLLVMConversionPatterns(converter, patterns,
1385                                          reassociateFPReductions);
1386   populateVectorToLLVMMatrixConversionPatterns(converter, patterns);
1387   populateStdToLLVMConversionPatterns(converter, patterns);
1388 
1389   LLVMConversionTarget target(getContext());
1390   if (failed(applyPartialConversion(getOperation(), target, patterns))) {
1391     signalPassFailure();
1392   }
1393 }
1394 
1395 std::unique_ptr<OperationPass<ModuleOp>>
1396 mlir::createConvertVectorToLLVMPass(const LowerVectorToLLVMOptions &options) {
1397   return std::make_unique<LowerVectorToLLVMPass>(options);
1398 }
1399