1 //===-- RISCVISelDAGToDAG.cpp - A dag to dag inst selector for RISCV ------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines an instruction selector for the RISCV target.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "RISCVISelDAGToDAG.h"
14 #include "MCTargetDesc/RISCVMCTargetDesc.h"
15 #include "MCTargetDesc/RISCVMatInt.h"
16 #include "RISCVISelLowering.h"
17 #include "llvm/CodeGen/MachineFrameInfo.h"
18 #include "llvm/IR/IntrinsicsRISCV.h"
19 #include "llvm/Support/Alignment.h"
20 #include "llvm/Support/Debug.h"
21 #include "llvm/Support/KnownBits.h"
22 #include "llvm/Support/MathExtras.h"
23 #include "llvm/Support/raw_ostream.h"
24 
25 using namespace llvm;
26 
27 #define DEBUG_TYPE "riscv-isel"
28 
29 namespace llvm {
30 namespace RISCV {
31 #define GET_RISCVVSSEGTable_IMPL
32 #define GET_RISCVVLSEGTable_IMPL
33 #define GET_RISCVVLXSEGTable_IMPL
34 #define GET_RISCVVSXSEGTable_IMPL
35 #define GET_RISCVVLETable_IMPL
36 #define GET_RISCVVSETable_IMPL
37 #define GET_RISCVVLXTable_IMPL
38 #define GET_RISCVVSXTable_IMPL
39 #include "RISCVGenSearchableTables.inc"
40 } // namespace RISCV
41 } // namespace llvm
42 
43 void RISCVDAGToDAGISel::PostprocessISelDAG() {
44   doPeepholeLoadStoreADDI();
45 }
46 
47 static SDNode *selectImm(SelectionDAG *CurDAG, const SDLoc &DL, int64_t Imm,
48                          MVT XLenVT) {
49   RISCVMatInt::InstSeq Seq;
50   RISCVMatInt::generateInstSeq(Imm, XLenVT == MVT::i64, Seq);
51 
52   SDNode *Result = nullptr;
53   SDValue SrcReg = CurDAG->getRegister(RISCV::X0, XLenVT);
54   for (RISCVMatInt::Inst &Inst : Seq) {
55     SDValue SDImm = CurDAG->getTargetConstant(Inst.Imm, DL, XLenVT);
56     if (Inst.Opc == RISCV::LUI)
57       Result = CurDAG->getMachineNode(RISCV::LUI, DL, XLenVT, SDImm);
58     else
59       Result = CurDAG->getMachineNode(Inst.Opc, DL, XLenVT, SrcReg, SDImm);
60 
61     // Only the first instruction has X0 as its source.
62     SrcReg = SDValue(Result, 0);
63   }
64 
65   return Result;
66 }
67 
68 static SDValue createTupleImpl(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
69                                unsigned RegClassID, unsigned SubReg0) {
70   assert(Regs.size() >= 2 && Regs.size() <= 8);
71 
72   SDLoc DL(Regs[0]);
73   SmallVector<SDValue, 8> Ops;
74 
75   Ops.push_back(CurDAG.getTargetConstant(RegClassID, DL, MVT::i32));
76 
77   for (unsigned I = 0; I < Regs.size(); ++I) {
78     Ops.push_back(Regs[I]);
79     Ops.push_back(CurDAG.getTargetConstant(SubReg0 + I, DL, MVT::i32));
80   }
81   SDNode *N =
82       CurDAG.getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops);
83   return SDValue(N, 0);
84 }
85 
86 static SDValue createM1Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
87                              unsigned NF) {
88   static const unsigned RegClassIDs[] = {
89       RISCV::VRN2M1RegClassID, RISCV::VRN3M1RegClassID, RISCV::VRN4M1RegClassID,
90       RISCV::VRN5M1RegClassID, RISCV::VRN6M1RegClassID, RISCV::VRN7M1RegClassID,
91       RISCV::VRN8M1RegClassID};
92 
93   return createTupleImpl(CurDAG, Regs, RegClassIDs[NF - 2], RISCV::sub_vrm1_0);
94 }
95 
96 static SDValue createM2Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
97                              unsigned NF) {
98   static const unsigned RegClassIDs[] = {RISCV::VRN2M2RegClassID,
99                                          RISCV::VRN3M2RegClassID,
100                                          RISCV::VRN4M2RegClassID};
101 
102   return createTupleImpl(CurDAG, Regs, RegClassIDs[NF - 2], RISCV::sub_vrm2_0);
103 }
104 
105 static SDValue createM4Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
106                              unsigned NF) {
107   return createTupleImpl(CurDAG, Regs, RISCV::VRN2M4RegClassID,
108                          RISCV::sub_vrm4_0);
109 }
110 
111 static SDValue createTuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
112                            unsigned NF, RISCVVLMUL LMUL) {
113   switch (LMUL) {
114   default:
115     llvm_unreachable("Invalid LMUL.");
116   case RISCVVLMUL::LMUL_F8:
117   case RISCVVLMUL::LMUL_F4:
118   case RISCVVLMUL::LMUL_F2:
119   case RISCVVLMUL::LMUL_1:
120     return createM1Tuple(CurDAG, Regs, NF);
121   case RISCVVLMUL::LMUL_2:
122     return createM2Tuple(CurDAG, Regs, NF);
123   case RISCVVLMUL::LMUL_4:
124     return createM4Tuple(CurDAG, Regs, NF);
125   }
126 }
127 
128 void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, bool IsMasked,
129                                     bool IsStrided) {
130   SDLoc DL(Node);
131   unsigned NF = Node->getNumValues() - 1;
132   MVT VT = Node->getSimpleValueType(0);
133   unsigned ScalarSize = VT.getScalarSizeInBits();
134   MVT XLenVT = Subtarget->getXLenVT();
135   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
136   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
137   unsigned CurOp = 2;
138   SmallVector<SDValue, 7> Operands;
139   if (IsMasked) {
140     SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
141                                  Node->op_begin() + CurOp + NF);
142     SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
143     Operands.push_back(MaskedOff);
144     CurOp += NF;
145   }
146   SDValue Base;
147   SelectBaseAddr(Node->getOperand(CurOp++), Base);
148   Operands.push_back(Base); // Base pointer.
149   if (IsStrided)
150     Operands.push_back(Node->getOperand(CurOp++)); // Stride.
151   if (IsMasked)
152     Operands.push_back(Node->getOperand(CurOp++)); // Mask.
153   SDValue VL;
154   selectVLOp(Node->getOperand(CurOp++), VL);
155   Operands.push_back(VL);
156   Operands.push_back(SEW);
157   Operands.push_back(Node->getOperand(0)); // Chain.
158   const RISCV::VLSEGPseudo *P =
159       RISCV::getVLSEGPseudo(NF, IsMasked, IsStrided, /*FF*/ false, ScalarSize,
160                             static_cast<unsigned>(LMUL));
161   MachineSDNode *Load =
162       CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
163 
164   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
165     CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
166 
167   SDValue SuperReg = SDValue(Load, 0);
168   for (unsigned I = 0; I < NF; ++I) {
169     unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
170     ReplaceUses(SDValue(Node, I),
171                 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
172   }
173 
174   ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
175   CurDAG->RemoveDeadNode(Node);
176 }
177 
178 void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node, bool IsMasked) {
179   SDLoc DL(Node);
180   unsigned NF = Node->getNumValues() - 2; // Do not count VL and Chain.
181   MVT VT = Node->getSimpleValueType(0);
182   MVT XLenVT = Subtarget->getXLenVT();
183   unsigned ScalarSize = VT.getScalarSizeInBits();
184   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
185   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
186 
187   unsigned CurOp = 2;
188   SmallVector<SDValue, 7> Operands;
189   if (IsMasked) {
190     SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
191                                  Node->op_begin() + CurOp + NF);
192     SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
193     Operands.push_back(MaskedOff);
194     CurOp += NF;
195   }
196   SDValue Base;
197   SelectBaseAddr(Node->getOperand(CurOp++), Base);
198   Operands.push_back(Base); // Base pointer.
199   if (IsMasked)
200     Operands.push_back(Node->getOperand(CurOp++)); // Mask.
201   SDValue VL;
202   selectVLOp(Node->getOperand(CurOp++), VL);
203   Operands.push_back(VL);
204   Operands.push_back(SEW);
205   Operands.push_back(Node->getOperand(0)); // Chain.
206   const RISCV::VLSEGPseudo *P =
207       RISCV::getVLSEGPseudo(NF, IsMasked, /*Strided*/ false, /*FF*/ true,
208                             ScalarSize, static_cast<unsigned>(LMUL));
209   MachineSDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped,
210                                                MVT::Other, MVT::Glue, Operands);
211   SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT,
212                                           /*Glue*/ SDValue(Load, 2));
213 
214   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
215     CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
216 
217   SDValue SuperReg = SDValue(Load, 0);
218   for (unsigned I = 0; I < NF; ++I) {
219     unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
220     ReplaceUses(SDValue(Node, I),
221                 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
222   }
223 
224   ReplaceUses(SDValue(Node, NF), SDValue(ReadVL, 0));   // VL
225   ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 1)); // Chain
226   CurDAG->RemoveDeadNode(Node);
227 }
228 
229 void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, bool IsMasked,
230                                      bool IsOrdered) {
231   SDLoc DL(Node);
232   unsigned NF = Node->getNumValues() - 1;
233   MVT VT = Node->getSimpleValueType(0);
234   unsigned ScalarSize = VT.getScalarSizeInBits();
235   MVT XLenVT = Subtarget->getXLenVT();
236   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
237   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
238   unsigned CurOp = 2;
239   SmallVector<SDValue, 7> Operands;
240   if (IsMasked) {
241     SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
242                                  Node->op_begin() + CurOp + NF);
243     SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
244     Operands.push_back(MaskedOff);
245     CurOp += NF;
246   }
247   SDValue Base;
248   SelectBaseAddr(Node->getOperand(CurOp++), Base);
249   Operands.push_back(Base); // Base pointer.
250   Operands.push_back(Node->getOperand(CurOp++)); // Index.
251   MVT IndexVT = Operands.back()->getSimpleValueType(0);
252   if (IsMasked)
253     Operands.push_back(Node->getOperand(CurOp++)); // Mask.
254   SDValue VL;
255   selectVLOp(Node->getOperand(CurOp++), VL);
256   Operands.push_back(VL);
257   Operands.push_back(SEW);
258   Operands.push_back(Node->getOperand(0)); // Chain.
259 
260   assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
261          "Element count mismatch");
262 
263   RISCVVLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
264   unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
265   const RISCV::VLXSEGPseudo *P = RISCV::getVLXSEGPseudo(
266       NF, IsMasked, IsOrdered, IndexScalarSize, static_cast<unsigned>(LMUL),
267       static_cast<unsigned>(IndexLMUL));
268   MachineSDNode *Load =
269       CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
270 
271   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
272     CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
273 
274   SDValue SuperReg = SDValue(Load, 0);
275   for (unsigned I = 0; I < NF; ++I) {
276     unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
277     ReplaceUses(SDValue(Node, I),
278                 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
279   }
280 
281   ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
282   CurDAG->RemoveDeadNode(Node);
283 }
284 
285 void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, bool IsMasked,
286                                     bool IsStrided) {
287   SDLoc DL(Node);
288   unsigned NF = Node->getNumOperands() - 4;
289   if (IsStrided)
290     NF--;
291   if (IsMasked)
292     NF--;
293   MVT VT = Node->getOperand(2)->getSimpleValueType(0);
294   unsigned ScalarSize = VT.getScalarSizeInBits();
295   MVT XLenVT = Subtarget->getXLenVT();
296   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
297   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
298   SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
299   SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
300   SmallVector<SDValue, 7> Operands;
301   Operands.push_back(StoreVal);
302   unsigned CurOp = 2 + NF;
303   SDValue Base;
304   SelectBaseAddr(Node->getOperand(CurOp++), Base);
305   Operands.push_back(Base); // Base pointer.
306   if (IsStrided)
307     Operands.push_back(Node->getOperand(CurOp++)); // Stride.
308   if (IsMasked)
309     Operands.push_back(Node->getOperand(CurOp++)); // Mask.
310   SDValue VL;
311   selectVLOp(Node->getOperand(CurOp++), VL);
312   Operands.push_back(VL);
313   Operands.push_back(SEW);
314   Operands.push_back(Node->getOperand(0)); // Chain.
315   const RISCV::VSSEGPseudo *P = RISCV::getVSSEGPseudo(
316       NF, IsMasked, IsStrided, ScalarSize, static_cast<unsigned>(LMUL));
317   MachineSDNode *Store =
318       CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
319 
320   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
321     CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
322 
323   ReplaceNode(Node, Store);
324 }
325 
326 void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, bool IsMasked,
327                                      bool IsOrdered) {
328   SDLoc DL(Node);
329   unsigned NF = Node->getNumOperands() - 5;
330   if (IsMasked)
331     --NF;
332   MVT VT = Node->getOperand(2)->getSimpleValueType(0);
333   unsigned ScalarSize = VT.getScalarSizeInBits();
334   MVT XLenVT = Subtarget->getXLenVT();
335   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
336   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
337   SmallVector<SDValue, 7> Operands;
338   SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
339   SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
340   Operands.push_back(StoreVal);
341   unsigned CurOp = 2 + NF;
342   SDValue Base;
343   SelectBaseAddr(Node->getOperand(CurOp++), Base);
344   Operands.push_back(Base); // Base pointer.
345   Operands.push_back(Node->getOperand(CurOp++)); // Index.
346   MVT IndexVT = Operands.back()->getSimpleValueType(0);
347   if (IsMasked)
348     Operands.push_back(Node->getOperand(CurOp++)); // Mask.
349   SDValue VL;
350   selectVLOp(Node->getOperand(CurOp++), VL);
351   Operands.push_back(VL);
352   Operands.push_back(SEW);
353   Operands.push_back(Node->getOperand(0)); // Chain.
354 
355   assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
356          "Element count mismatch");
357 
358   RISCVVLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
359   unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
360   const RISCV::VSXSEGPseudo *P = RISCV::getVSXSEGPseudo(
361       NF, IsMasked, IsOrdered, IndexScalarSize, static_cast<unsigned>(LMUL),
362       static_cast<unsigned>(IndexLMUL));
363   MachineSDNode *Store =
364       CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
365 
366   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
367     CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
368 
369   ReplaceNode(Node, Store);
370 }
371 
372 
373 void RISCVDAGToDAGISel::Select(SDNode *Node) {
374   // If we have a custom node, we have already selected.
375   if (Node->isMachineOpcode()) {
376     LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n");
377     Node->setNodeId(-1);
378     return;
379   }
380 
381   // Instruction Selection not handled by the auto-generated tablegen selection
382   // should be handled here.
383   unsigned Opcode = Node->getOpcode();
384   MVT XLenVT = Subtarget->getXLenVT();
385   SDLoc DL(Node);
386   MVT VT = Node->getSimpleValueType(0);
387 
388   switch (Opcode) {
389   case ISD::ADD: {
390     // Optimize (add r, imm) to (addi (addi r, imm0) imm1) if applicable. The
391     // immediate must be in specific ranges and have a single use.
392     if (auto *ConstOp = dyn_cast<ConstantSDNode>(Node->getOperand(1))) {
393       if (!(ConstOp->hasOneUse()))
394         break;
395       // The imm must be in range [-4096,-2049] or [2048,4094].
396       int64_t Imm = ConstOp->getSExtValue();
397       if (!(-4096 <= Imm && Imm <= -2049) && !(2048 <= Imm && Imm <= 4094))
398         break;
399       // Break the imm to imm0+imm1.
400       const SDValue ImmOp0 = CurDAG->getTargetConstant(Imm - Imm / 2, DL, VT);
401       const SDValue ImmOp1 = CurDAG->getTargetConstant(Imm / 2, DL, VT);
402       auto *NodeAddi0 = CurDAG->getMachineNode(RISCV::ADDI, DL, VT,
403                                                Node->getOperand(0), ImmOp0);
404       auto *NodeAddi1 = CurDAG->getMachineNode(RISCV::ADDI, DL, VT,
405                                                SDValue(NodeAddi0, 0), ImmOp1);
406       ReplaceNode(Node, NodeAddi1);
407       return;
408     }
409     break;
410   }
411   case ISD::Constant: {
412     auto *ConstNode = cast<ConstantSDNode>(Node);
413     if (VT == XLenVT && ConstNode->isNullValue()) {
414       SDValue New =
415           CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, RISCV::X0, XLenVT);
416       ReplaceNode(Node, New.getNode());
417       return;
418     }
419     ReplaceNode(Node, selectImm(CurDAG, DL, ConstNode->getSExtValue(), XLenVT));
420     return;
421     break;
422   }
423   case ISD::FrameIndex: {
424     SDValue Imm = CurDAG->getTargetConstant(0, DL, XLenVT);
425     int FI = cast<FrameIndexSDNode>(Node)->getIndex();
426     SDValue TFI = CurDAG->getTargetFrameIndex(FI, VT);
427     ReplaceNode(Node, CurDAG->getMachineNode(RISCV::ADDI, DL, VT, TFI, Imm));
428     return;
429   }
430   case ISD::SRL: {
431     // Optimize (srl (and X, 0xffff), C) -> (srli (slli X, 16), 16 + C).
432     // Taking into account that the 0xffff may have had lower bits unset by
433     // SimplifyDemandedBits. This avoids materializing the 0xffff immediate.
434     // This pattern occurs when type legalizing i16 right shifts.
435     // FIXME: This could be extended to other AND masks.
436     auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
437     if (N1C) {
438       uint64_t ShAmt = N1C->getZExtValue();
439       SDValue N0 = Node->getOperand(0);
440       if (ShAmt < 16 && N0.getOpcode() == ISD::AND && N0.hasOneUse() &&
441           isa<ConstantSDNode>(N0.getOperand(1))) {
442         uint64_t Mask = N0.getConstantOperandVal(1);
443         Mask |= maskTrailingOnes<uint64_t>(ShAmt);
444         if (Mask == 0xffff) {
445           unsigned SLLOpc = Subtarget->is64Bit() ? RISCV::SLLIW : RISCV::SLLI;
446           unsigned SRLOpc = Subtarget->is64Bit() ? RISCV::SRLIW : RISCV::SRLI;
447           SDNode *SLLI =
448               CurDAG->getMachineNode(SLLOpc, DL, VT, N0->getOperand(0),
449                                      CurDAG->getTargetConstant(16, DL, VT));
450           SDNode *SRLI = CurDAG->getMachineNode(
451               SRLOpc, DL, VT, SDValue(SLLI, 0),
452               CurDAG->getTargetConstant(16 + ShAmt, DL, VT));
453           ReplaceNode(Node, SRLI);
454           return;
455         }
456       }
457     }
458 
459     break;
460   }
461   case ISD::INTRINSIC_W_CHAIN: {
462     unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
463     switch (IntNo) {
464       // By default we do not custom select any intrinsic.
465     default:
466       break;
467 
468     case Intrinsic::riscv_vsetvli:
469     case Intrinsic::riscv_vsetvlimax: {
470       if (!Subtarget->hasStdExtV())
471         break;
472 
473       bool VLMax = IntNo == Intrinsic::riscv_vsetvlimax;
474       unsigned Offset = VLMax ? 2 : 3;
475 
476       assert(Node->getNumOperands() == Offset + 2 &&
477              "Unexpected number of operands");
478 
479       RISCVVSEW VSEW =
480           static_cast<RISCVVSEW>(Node->getConstantOperandVal(Offset) & 0x7);
481       RISCVVLMUL VLMul = static_cast<RISCVVLMUL>(
482           Node->getConstantOperandVal(Offset + 1) & 0x7);
483 
484       unsigned VTypeI = RISCVVType::encodeVTYPE(
485           VLMul, VSEW, /*TailAgnostic*/ true, /*MaskAgnostic*/ false);
486       SDValue VTypeIOp = CurDAG->getTargetConstant(VTypeI, DL, XLenVT);
487 
488       SDValue VLOperand;
489       if (VLMax) {
490         VLOperand = CurDAG->getRegister(RISCV::X0, XLenVT);
491       } else {
492         VLOperand = Node->getOperand(2);
493 
494         if (auto *C = dyn_cast<ConstantSDNode>(VLOperand)) {
495           uint64_t AVL = C->getZExtValue();
496           if (isUInt<5>(AVL)) {
497             SDValue VLImm = CurDAG->getTargetConstant(AVL, DL, XLenVT);
498             ReplaceNode(
499                 Node, CurDAG->getMachineNode(RISCV::PseudoVSETIVLI, DL, XLenVT,
500                                              MVT::Other, VLImm, VTypeIOp,
501                                              /* Chain */ Node->getOperand(0)));
502             return;
503           }
504         }
505       }
506 
507       ReplaceNode(Node,
508                   CurDAG->getMachineNode(RISCV::PseudoVSETVLI, DL, XLenVT,
509                                          MVT::Other, VLOperand, VTypeIOp,
510                                          /* Chain */ Node->getOperand(0)));
511       return;
512     }
513     case Intrinsic::riscv_vlseg2:
514     case Intrinsic::riscv_vlseg3:
515     case Intrinsic::riscv_vlseg4:
516     case Intrinsic::riscv_vlseg5:
517     case Intrinsic::riscv_vlseg6:
518     case Intrinsic::riscv_vlseg7:
519     case Intrinsic::riscv_vlseg8: {
520       selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false);
521       return;
522     }
523     case Intrinsic::riscv_vlseg2_mask:
524     case Intrinsic::riscv_vlseg3_mask:
525     case Intrinsic::riscv_vlseg4_mask:
526     case Intrinsic::riscv_vlseg5_mask:
527     case Intrinsic::riscv_vlseg6_mask:
528     case Intrinsic::riscv_vlseg7_mask:
529     case Intrinsic::riscv_vlseg8_mask: {
530       selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false);
531       return;
532     }
533     case Intrinsic::riscv_vlsseg2:
534     case Intrinsic::riscv_vlsseg3:
535     case Intrinsic::riscv_vlsseg4:
536     case Intrinsic::riscv_vlsseg5:
537     case Intrinsic::riscv_vlsseg6:
538     case Intrinsic::riscv_vlsseg7:
539     case Intrinsic::riscv_vlsseg8: {
540       selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true);
541       return;
542     }
543     case Intrinsic::riscv_vlsseg2_mask:
544     case Intrinsic::riscv_vlsseg3_mask:
545     case Intrinsic::riscv_vlsseg4_mask:
546     case Intrinsic::riscv_vlsseg5_mask:
547     case Intrinsic::riscv_vlsseg6_mask:
548     case Intrinsic::riscv_vlsseg7_mask:
549     case Intrinsic::riscv_vlsseg8_mask: {
550       selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true);
551       return;
552     }
553     case Intrinsic::riscv_vloxseg2:
554     case Intrinsic::riscv_vloxseg3:
555     case Intrinsic::riscv_vloxseg4:
556     case Intrinsic::riscv_vloxseg5:
557     case Intrinsic::riscv_vloxseg6:
558     case Intrinsic::riscv_vloxseg7:
559     case Intrinsic::riscv_vloxseg8:
560       selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true);
561       return;
562     case Intrinsic::riscv_vluxseg2:
563     case Intrinsic::riscv_vluxseg3:
564     case Intrinsic::riscv_vluxseg4:
565     case Intrinsic::riscv_vluxseg5:
566     case Intrinsic::riscv_vluxseg6:
567     case Intrinsic::riscv_vluxseg7:
568     case Intrinsic::riscv_vluxseg8:
569       selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false);
570       return;
571     case Intrinsic::riscv_vloxseg2_mask:
572     case Intrinsic::riscv_vloxseg3_mask:
573     case Intrinsic::riscv_vloxseg4_mask:
574     case Intrinsic::riscv_vloxseg5_mask:
575     case Intrinsic::riscv_vloxseg6_mask:
576     case Intrinsic::riscv_vloxseg7_mask:
577     case Intrinsic::riscv_vloxseg8_mask:
578       selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true);
579       return;
580     case Intrinsic::riscv_vluxseg2_mask:
581     case Intrinsic::riscv_vluxseg3_mask:
582     case Intrinsic::riscv_vluxseg4_mask:
583     case Intrinsic::riscv_vluxseg5_mask:
584     case Intrinsic::riscv_vluxseg6_mask:
585     case Intrinsic::riscv_vluxseg7_mask:
586     case Intrinsic::riscv_vluxseg8_mask:
587       selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false);
588       return;
589     case Intrinsic::riscv_vlseg8ff:
590     case Intrinsic::riscv_vlseg7ff:
591     case Intrinsic::riscv_vlseg6ff:
592     case Intrinsic::riscv_vlseg5ff:
593     case Intrinsic::riscv_vlseg4ff:
594     case Intrinsic::riscv_vlseg3ff:
595     case Intrinsic::riscv_vlseg2ff: {
596       selectVLSEGFF(Node, /*IsMasked*/ false);
597       return;
598     }
599     case Intrinsic::riscv_vlseg8ff_mask:
600     case Intrinsic::riscv_vlseg7ff_mask:
601     case Intrinsic::riscv_vlseg6ff_mask:
602     case Intrinsic::riscv_vlseg5ff_mask:
603     case Intrinsic::riscv_vlseg4ff_mask:
604     case Intrinsic::riscv_vlseg3ff_mask:
605     case Intrinsic::riscv_vlseg2ff_mask: {
606       selectVLSEGFF(Node, /*IsMasked*/ true);
607       return;
608     }
609     case Intrinsic::riscv_vloxei:
610     case Intrinsic::riscv_vloxei_mask:
611     case Intrinsic::riscv_vluxei:
612     case Intrinsic::riscv_vluxei_mask: {
613       bool IsMasked = IntNo == Intrinsic::riscv_vloxei_mask ||
614                       IntNo == Intrinsic::riscv_vluxei_mask;
615       bool IsOrdered = IntNo == Intrinsic::riscv_vloxei ||
616                        IntNo == Intrinsic::riscv_vloxei_mask;
617 
618       MVT VT = Node->getSimpleValueType(0);
619       unsigned ScalarSize = VT.getScalarSizeInBits();
620       MVT XLenVT = Subtarget->getXLenVT();
621       SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
622 
623       unsigned CurOp = 2;
624       SmallVector<SDValue, 7> Operands;
625       if (IsMasked)
626         Operands.push_back(Node->getOperand(CurOp++));
627       SDValue Base;
628       SelectBaseAddr(Node->getOperand(CurOp++), Base);
629       Operands.push_back(Base); // Base pointer.
630       Operands.push_back(Node->getOperand(CurOp++)); // Index.
631       MVT IndexVT = Operands.back()->getSimpleValueType(0);
632       if (IsMasked)
633         Operands.push_back(Node->getOperand(CurOp++)); // Mask.
634       SDValue VL;
635       selectVLOp(Node->getOperand(CurOp++), VL);
636       Operands.push_back(VL);
637       Operands.push_back(SEW);
638       Operands.push_back(Node->getOperand(0)); // Chain.
639 
640       assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
641              "Element count mismatch");
642 
643       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
644       RISCVVLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
645       unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
646       const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
647           IsMasked, IsOrdered, IndexScalarSize, static_cast<unsigned>(LMUL),
648           static_cast<unsigned>(IndexLMUL));
649       MachineSDNode *Load =
650           CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
651 
652       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
653         CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
654 
655       ReplaceNode(Node, Load);
656       return;
657     }
658     case Intrinsic::riscv_vle1:
659     case Intrinsic::riscv_vle:
660     case Intrinsic::riscv_vle_mask:
661     case Intrinsic::riscv_vlse:
662     case Intrinsic::riscv_vlse_mask: {
663       bool IsMasked = IntNo == Intrinsic::riscv_vle_mask ||
664                       IntNo == Intrinsic::riscv_vlse_mask;
665       bool IsStrided =
666           IntNo == Intrinsic::riscv_vlse || IntNo == Intrinsic::riscv_vlse_mask;
667 
668       MVT VT = Node->getSimpleValueType(0);
669       unsigned ScalarSize = VT.getScalarSizeInBits();
670       MVT XLenVT = Subtarget->getXLenVT();
671       // VLE1 uses an SEW of 8.
672       unsigned SEWImm = (IntNo == Intrinsic::riscv_vle1) ? 8 : ScalarSize;
673       SDValue SEW = CurDAG->getTargetConstant(SEWImm, DL, XLenVT);
674 
675       unsigned CurOp = 2;
676       SmallVector<SDValue, 7> Operands;
677       if (IsMasked)
678         Operands.push_back(Node->getOperand(CurOp++));
679       SDValue Base;
680       SelectBaseAddr(Node->getOperand(CurOp++), Base);
681       Operands.push_back(Base); // Base pointer.
682       if (IsStrided)
683         Operands.push_back(Node->getOperand(CurOp++)); // Stride.
684       if (IsMasked)
685         Operands.push_back(Node->getOperand(CurOp++)); // Mask.
686       SDValue VL;
687       selectVLOp(Node->getOperand(CurOp++), VL);
688       Operands.push_back(VL);
689       Operands.push_back(SEW);
690       Operands.push_back(Node->getOperand(0)); // Chain.
691 
692       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
693       const RISCV::VLEPseudo *P =
694           RISCV::getVLEPseudo(IsMasked, IsStrided, /*FF*/ false, ScalarSize,
695                               static_cast<unsigned>(LMUL));
696       MachineSDNode *Load =
697           CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
698 
699       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
700         CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
701 
702       ReplaceNode(Node, Load);
703       return;
704     }
705     case Intrinsic::riscv_vleff:
706     case Intrinsic::riscv_vleff_mask: {
707       bool IsMasked = IntNo == Intrinsic::riscv_vleff_mask;
708 
709       MVT VT = Node->getSimpleValueType(0);
710       unsigned ScalarSize = VT.getScalarSizeInBits();
711       MVT XLenVT = Subtarget->getXLenVT();
712       SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
713 
714       unsigned CurOp = 2;
715       SmallVector<SDValue, 7> Operands;
716       if (IsMasked)
717         Operands.push_back(Node->getOperand(CurOp++));
718       SDValue Base;
719       SelectBaseAddr(Node->getOperand(CurOp++), Base);
720       Operands.push_back(Base); // Base pointer.
721       if (IsMasked)
722         Operands.push_back(Node->getOperand(CurOp++)); // Mask.
723       SDValue VL;
724       selectVLOp(Node->getOperand(CurOp++), VL);
725       Operands.push_back(VL);
726       Operands.push_back(SEW);
727       Operands.push_back(Node->getOperand(0)); // Chain.
728 
729       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
730       const RISCV::VLEPseudo *P =
731           RISCV::getVLEPseudo(IsMasked, /*Strided*/ false, /*FF*/ true,
732                               ScalarSize, static_cast<unsigned>(LMUL));
733       MachineSDNode *Load =
734           CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0),
735                                  MVT::Other, MVT::Glue, Operands);
736       SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT,
737                                               /*Glue*/ SDValue(Load, 2));
738 
739       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
740         CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
741 
742       ReplaceUses(SDValue(Node, 0), SDValue(Load, 0));
743       ReplaceUses(SDValue(Node, 1), SDValue(ReadVL, 0)); // VL
744       ReplaceUses(SDValue(Node, 2), SDValue(Load, 1));   // Chain
745       CurDAG->RemoveDeadNode(Node);
746       return;
747     }
748     }
749     break;
750   }
751   case ISD::INTRINSIC_VOID: {
752     unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
753     switch (IntNo) {
754     case Intrinsic::riscv_vsseg2:
755     case Intrinsic::riscv_vsseg3:
756     case Intrinsic::riscv_vsseg4:
757     case Intrinsic::riscv_vsseg5:
758     case Intrinsic::riscv_vsseg6:
759     case Intrinsic::riscv_vsseg7:
760     case Intrinsic::riscv_vsseg8: {
761       selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false);
762       return;
763     }
764     case Intrinsic::riscv_vsseg2_mask:
765     case Intrinsic::riscv_vsseg3_mask:
766     case Intrinsic::riscv_vsseg4_mask:
767     case Intrinsic::riscv_vsseg5_mask:
768     case Intrinsic::riscv_vsseg6_mask:
769     case Intrinsic::riscv_vsseg7_mask:
770     case Intrinsic::riscv_vsseg8_mask: {
771       selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false);
772       return;
773     }
774     case Intrinsic::riscv_vssseg2:
775     case Intrinsic::riscv_vssseg3:
776     case Intrinsic::riscv_vssseg4:
777     case Intrinsic::riscv_vssseg5:
778     case Intrinsic::riscv_vssseg6:
779     case Intrinsic::riscv_vssseg7:
780     case Intrinsic::riscv_vssseg8: {
781       selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true);
782       return;
783     }
784     case Intrinsic::riscv_vssseg2_mask:
785     case Intrinsic::riscv_vssseg3_mask:
786     case Intrinsic::riscv_vssseg4_mask:
787     case Intrinsic::riscv_vssseg5_mask:
788     case Intrinsic::riscv_vssseg6_mask:
789     case Intrinsic::riscv_vssseg7_mask:
790     case Intrinsic::riscv_vssseg8_mask: {
791       selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true);
792       return;
793     }
794     case Intrinsic::riscv_vsoxseg2:
795     case Intrinsic::riscv_vsoxseg3:
796     case Intrinsic::riscv_vsoxseg4:
797     case Intrinsic::riscv_vsoxseg5:
798     case Intrinsic::riscv_vsoxseg6:
799     case Intrinsic::riscv_vsoxseg7:
800     case Intrinsic::riscv_vsoxseg8:
801       selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true);
802       return;
803     case Intrinsic::riscv_vsuxseg2:
804     case Intrinsic::riscv_vsuxseg3:
805     case Intrinsic::riscv_vsuxseg4:
806     case Intrinsic::riscv_vsuxseg5:
807     case Intrinsic::riscv_vsuxseg6:
808     case Intrinsic::riscv_vsuxseg7:
809     case Intrinsic::riscv_vsuxseg8:
810       selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false);
811       return;
812     case Intrinsic::riscv_vsoxseg2_mask:
813     case Intrinsic::riscv_vsoxseg3_mask:
814     case Intrinsic::riscv_vsoxseg4_mask:
815     case Intrinsic::riscv_vsoxseg5_mask:
816     case Intrinsic::riscv_vsoxseg6_mask:
817     case Intrinsic::riscv_vsoxseg7_mask:
818     case Intrinsic::riscv_vsoxseg8_mask:
819       selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true);
820       return;
821     case Intrinsic::riscv_vsuxseg2_mask:
822     case Intrinsic::riscv_vsuxseg3_mask:
823     case Intrinsic::riscv_vsuxseg4_mask:
824     case Intrinsic::riscv_vsuxseg5_mask:
825     case Intrinsic::riscv_vsuxseg6_mask:
826     case Intrinsic::riscv_vsuxseg7_mask:
827     case Intrinsic::riscv_vsuxseg8_mask:
828       selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false);
829       return;
830     case Intrinsic::riscv_vsoxei:
831     case Intrinsic::riscv_vsoxei_mask:
832     case Intrinsic::riscv_vsuxei:
833     case Intrinsic::riscv_vsuxei_mask: {
834       bool IsMasked = IntNo == Intrinsic::riscv_vsoxei_mask ||
835                       IntNo == Intrinsic::riscv_vsuxei_mask;
836       bool IsOrdered = IntNo == Intrinsic::riscv_vsoxei ||
837                        IntNo == Intrinsic::riscv_vsoxei_mask;
838 
839       MVT VT = Node->getOperand(2)->getSimpleValueType(0);
840       unsigned ScalarSize = VT.getScalarSizeInBits();
841       MVT XLenVT = Subtarget->getXLenVT();
842       SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
843 
844       unsigned CurOp = 2;
845       SmallVector<SDValue, 6> Operands;
846       Operands.push_back(Node->getOperand(CurOp++)); // Store value.
847       SDValue Base;
848       SelectBaseAddr(Node->getOperand(CurOp++), Base);
849       Operands.push_back(Base); // Base pointer.
850       Operands.push_back(Node->getOperand(CurOp++)); // Index.
851       MVT IndexVT = Operands.back()->getSimpleValueType(0);
852       if (IsMasked)
853         Operands.push_back(Node->getOperand(CurOp++)); // Mask.
854       SDValue VL;
855       selectVLOp(Node->getOperand(CurOp++), VL);
856       Operands.push_back(VL);
857       Operands.push_back(SEW);
858       Operands.push_back(Node->getOperand(0)); // Chain.
859 
860       assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
861              "Element count mismatch");
862 
863       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
864       RISCVVLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
865       unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
866       const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
867           IsMasked, IsOrdered, IndexScalarSize, static_cast<unsigned>(LMUL),
868           static_cast<unsigned>(IndexLMUL));
869       MachineSDNode *Store =
870           CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
871 
872       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
873         CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
874 
875       ReplaceNode(Node, Store);
876       return;
877     }
878     case Intrinsic::riscv_vse1:
879     case Intrinsic::riscv_vse:
880     case Intrinsic::riscv_vse_mask:
881     case Intrinsic::riscv_vsse:
882     case Intrinsic::riscv_vsse_mask: {
883       bool IsMasked = IntNo == Intrinsic::riscv_vse_mask ||
884                       IntNo == Intrinsic::riscv_vsse_mask;
885       bool IsStrided =
886           IntNo == Intrinsic::riscv_vsse || IntNo == Intrinsic::riscv_vsse_mask;
887 
888       MVT VT = Node->getOperand(2)->getSimpleValueType(0);
889       unsigned ScalarSize = VT.getScalarSizeInBits();
890       MVT XLenVT = Subtarget->getXLenVT();
891       // VSE1 uses an SEW of 8.
892       unsigned SEWImm = (IntNo == Intrinsic::riscv_vse1) ? 8 : ScalarSize;
893       SDValue SEW = CurDAG->getTargetConstant(SEWImm, DL, XLenVT);
894 
895       unsigned CurOp = 2;
896       SmallVector<SDValue, 6> Operands;
897       Operands.push_back(Node->getOperand(CurOp++)); // Store value.
898       SDValue Base;
899       SelectBaseAddr(Node->getOperand(CurOp++), Base);
900       Operands.push_back(Base); // Base pointer.
901       if (IsStrided)
902         Operands.push_back(Node->getOperand(CurOp++)); // Stride.
903       if (IsMasked)
904         Operands.push_back(Node->getOperand(CurOp++)); // Mask.
905       SDValue VL;
906       selectVLOp(Node->getOperand(CurOp++), VL);
907       Operands.push_back(VL);
908       Operands.push_back(SEW);
909       Operands.push_back(Node->getOperand(0)); // Chain.
910 
911       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
912       const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(
913           IsMasked, IsStrided, ScalarSize, static_cast<unsigned>(LMUL));
914       MachineSDNode *Store =
915           CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
916 
917       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
918         CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
919 
920       ReplaceNode(Node, Store);
921       return;
922     }
923     }
924     break;
925   }
926   case ISD::BITCAST: {
927     MVT SrcVT = Node->getOperand(0).getSimpleValueType();
928     // Just drop bitcasts between vectors if both are fixed or both are
929     // scalable.
930     if ((VT.isScalableVector() && SrcVT.isScalableVector()) ||
931         (VT.isFixedLengthVector() && SrcVT.isFixedLengthVector())) {
932       ReplaceUses(SDValue(Node, 0), Node->getOperand(0));
933       CurDAG->RemoveDeadNode(Node);
934       return;
935     }
936     break;
937   }
938   case ISD::INSERT_SUBVECTOR: {
939     SDValue V = Node->getOperand(0);
940     SDValue SubV = Node->getOperand(1);
941     SDLoc DL(SubV);
942     auto Idx = Node->getConstantOperandVal(2);
943     MVT SubVecVT = SubV.getSimpleValueType();
944 
945     MVT SubVecContainerVT = SubVecVT;
946     // Establish the correct scalable-vector types for any fixed-length type.
947     if (SubVecVT.isFixedLengthVector())
948       SubVecContainerVT = RISCVTargetLowering::getContainerForFixedLengthVector(
949           *CurDAG, SubVecVT, *Subtarget);
950     if (VT.isFixedLengthVector())
951       VT = RISCVTargetLowering::getContainerForFixedLengthVector(*CurDAG, VT,
952                                                                  *Subtarget);
953 
954     const auto *TRI = Subtarget->getRegisterInfo();
955     unsigned SubRegIdx;
956     std::tie(SubRegIdx, Idx) =
957         RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
958             VT, SubVecContainerVT, Idx, TRI);
959 
960     // If the Idx hasn't been completely eliminated then this is a subvector
961     // insert which doesn't naturally align to a vector register. These must
962     // be handled using instructions to manipulate the vector registers.
963     if (Idx != 0)
964       break;
965 
966     RISCVVLMUL SubVecLMUL = RISCVTargetLowering::getLMUL(SubVecContainerVT);
967     bool IsSubVecPartReg = SubVecLMUL == RISCVVLMUL::LMUL_F2 ||
968                            SubVecLMUL == RISCVVLMUL::LMUL_F4 ||
969                            SubVecLMUL == RISCVVLMUL::LMUL_F8;
970     (void)IsSubVecPartReg; // Silence unused variable warning without asserts.
971     assert((!IsSubVecPartReg || V.isUndef()) &&
972            "Expecting lowering to have created legal INSERT_SUBVECTORs when "
973            "the subvector is smaller than a full-sized register");
974 
975     // If we haven't set a SubRegIdx, then we must be going between
976     // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy.
977     if (SubRegIdx == RISCV::NoSubRegister) {
978       unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT);
979       assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) ==
980                  InRegClassID &&
981              "Unexpected subvector extraction");
982       SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT);
983       SDNode *NewNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
984                                                DL, VT, SubV, RC);
985       return ReplaceNode(Node, NewNode);
986     }
987 
988     SDValue Insert = CurDAG->getTargetInsertSubreg(SubRegIdx, DL, VT, V, SubV);
989     return ReplaceNode(Node, Insert.getNode());
990   }
991   case ISD::EXTRACT_SUBVECTOR: {
992     SDValue V = Node->getOperand(0);
993     auto Idx = Node->getConstantOperandVal(1);
994     MVT InVT = V.getSimpleValueType();
995     SDLoc DL(V);
996 
997     MVT SubVecContainerVT = VT;
998     // Establish the correct scalable-vector types for any fixed-length type.
999     if (VT.isFixedLengthVector())
1000       SubVecContainerVT = RISCVTargetLowering::getContainerForFixedLengthVector(
1001           *CurDAG, VT, *Subtarget);
1002     if (InVT.isFixedLengthVector())
1003       InVT = RISCVTargetLowering::getContainerForFixedLengthVector(
1004           *CurDAG, InVT, *Subtarget);
1005 
1006     const auto *TRI = Subtarget->getRegisterInfo();
1007     unsigned SubRegIdx;
1008     std::tie(SubRegIdx, Idx) =
1009         RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1010             InVT, SubVecContainerVT, Idx, TRI);
1011 
1012     // If the Idx hasn't been completely eliminated then this is a subvector
1013     // extract which doesn't naturally align to a vector register. These must
1014     // be handled using instructions to manipulate the vector registers.
1015     if (Idx != 0)
1016       break;
1017 
1018     // If we haven't set a SubRegIdx, then we must be going between
1019     // equally-sized LMUL types (e.g. VR -> VR). This can be done as a copy.
1020     if (SubRegIdx == RISCV::NoSubRegister) {
1021       unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(InVT);
1022       assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) ==
1023                  InRegClassID &&
1024              "Unexpected subvector extraction");
1025       SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT);
1026       SDNode *NewNode =
1027           CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, DL, VT, V, RC);
1028       return ReplaceNode(Node, NewNode);
1029     }
1030 
1031     SDValue Extract = CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, V);
1032     return ReplaceNode(Node, Extract.getNode());
1033   }
1034   }
1035 
1036   // Select the default instruction.
1037   SelectCode(Node);
1038 }
1039 
1040 bool RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand(
1041     const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {
1042   switch (ConstraintID) {
1043   case InlineAsm::Constraint_m:
1044     // We just support simple memory operands that have a single address
1045     // operand and need no special handling.
1046     OutOps.push_back(Op);
1047     return false;
1048   case InlineAsm::Constraint_A:
1049     OutOps.push_back(Op);
1050     return false;
1051   default:
1052     break;
1053   }
1054 
1055   return true;
1056 }
1057 
1058 bool RISCVDAGToDAGISel::SelectAddrFI(SDValue Addr, SDValue &Base) {
1059   if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
1060     Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT());
1061     return true;
1062   }
1063   return false;
1064 }
1065 
1066 bool RISCVDAGToDAGISel::SelectBaseAddr(SDValue Addr, SDValue &Base) {
1067   // If this is FrameIndex, select it directly. Otherwise just let it get
1068   // selected to a register independently.
1069   if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr))
1070     Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT());
1071   else
1072     Base = Addr;
1073   return true;
1074 }
1075 
1076 bool RISCVDAGToDAGISel::selectShiftMask(SDValue N, unsigned ShiftWidth,
1077                                         SDValue &ShAmt) {
1078   // Shift instructions on RISCV only read the lower 5 or 6 bits of the shift
1079   // amount. If there is an AND on the shift amount, we can bypass it if it
1080   // doesn't affect any of those bits.
1081   if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(N.getOperand(1))) {
1082     const APInt &AndMask = N->getConstantOperandAPInt(1);
1083 
1084     // Since the max shift amount is a power of 2 we can subtract 1 to make a
1085     // mask that covers the bits needed to represent all shift amounts.
1086     assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!");
1087     APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
1088 
1089     if (ShMask.isSubsetOf(AndMask)) {
1090       ShAmt = N.getOperand(0);
1091       return true;
1092     }
1093 
1094     // SimplifyDemandedBits may have optimized the mask so try restoring any
1095     // bits that are known zero.
1096     KnownBits Known = CurDAG->computeKnownBits(N->getOperand(0));
1097     if (ShMask.isSubsetOf(AndMask | Known.Zero)) {
1098       ShAmt = N.getOperand(0);
1099       return true;
1100     }
1101   }
1102 
1103   ShAmt = N;
1104   return true;
1105 }
1106 
1107 bool RISCVDAGToDAGISel::selectSExti32(SDValue N, SDValue &Val) {
1108   if (N.getOpcode() == ISD::SIGN_EXTEND_INREG &&
1109       cast<VTSDNode>(N.getOperand(1))->getVT() == MVT::i32) {
1110     Val = N.getOperand(0);
1111     return true;
1112   }
1113   // FIXME: Should we just call computeNumSignBits here?
1114   if (N.getOpcode() == ISD::AssertSext &&
1115       cast<VTSDNode>(N->getOperand(1))->getVT().bitsLE(MVT::i32)) {
1116     Val = N;
1117     return true;
1118   }
1119   if (N.getOpcode() == ISD::AssertZext &&
1120       cast<VTSDNode>(N->getOperand(1))->getVT().bitsLT(MVT::i32)) {
1121     Val = N;
1122     return true;
1123   }
1124 
1125   return false;
1126 }
1127 
1128 bool RISCVDAGToDAGISel::selectZExti32(SDValue N, SDValue &Val) {
1129   if (N.getOpcode() == ISD::AND) {
1130     auto *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
1131     if (C && CheckAndMask(N.getOperand(0), C, UINT64_C(0xFFFFFFFF))) {
1132       Val = N.getOperand(0);
1133       return true;
1134     }
1135   }
1136   // FIXME: Should we just call computeKnownBits here?
1137   if (N.getOpcode() == ISD::AssertZext &&
1138       cast<VTSDNode>(N->getOperand(1))->getVT().bitsLE(MVT::i32)) {
1139     Val = N;
1140     return true;
1141   }
1142 
1143   return false;
1144 }
1145 
1146 // Match (srl (and val, mask), imm) where the result would be a
1147 // zero-extended 32-bit integer. i.e. the mask is 0xffffffff or the result
1148 // is equivalent to this (SimplifyDemandedBits may have removed lower bits
1149 // from the mask that aren't necessary due to the right-shifting).
1150 bool RISCVDAGToDAGISel::MatchSRLIW(SDNode *N) const {
1151   assert(N->getOpcode() == ISD::SRL);
1152   assert(N->getOperand(0).getOpcode() == ISD::AND);
1153   assert(isa<ConstantSDNode>(N->getOperand(1)));
1154   assert(isa<ConstantSDNode>(N->getOperand(0).getOperand(1)));
1155 
1156   // The IsRV64 predicate is checked after PatFrag predicates so we can get
1157   // here even on RV32.
1158   if (!Subtarget->is64Bit())
1159     return false;
1160 
1161   SDValue And = N->getOperand(0);
1162   uint64_t ShAmt = N->getConstantOperandVal(1);
1163   uint64_t Mask = And.getConstantOperandVal(1);
1164   return (Mask | maskTrailingOnes<uint64_t>(ShAmt)) == 0xffffffff;
1165 }
1166 
1167 // Check that it is a SLLIUW (Shift Logical Left Immediate Unsigned i32
1168 // on RV64).
1169 // SLLIUW is the same as SLLI except for the fact that it clears the bits
1170 // XLEN-1:32 of the input RS1 before shifting.
1171 // A PatFrag has already checked that it has the right structure:
1172 //
1173 //  (AND (SHL RS1, VC2), VC1)
1174 //
1175 // We check that VC2, the shamt is less than 32, otherwise the pattern is
1176 // exactly the same as SLLI and we give priority to that.
1177 // Eventually we check that VC1, the mask used to clear the upper 32 bits
1178 // of RS1, is correct:
1179 //
1180 //  VC1 == (0xFFFFFFFF << VC2)
1181 //
1182 bool RISCVDAGToDAGISel::MatchSLLIUW(SDNode *N) const {
1183   assert(N->getOpcode() == ISD::AND);
1184   assert(N->getOperand(0).getOpcode() == ISD::SHL);
1185   assert(isa<ConstantSDNode>(N->getOperand(1)));
1186   assert(isa<ConstantSDNode>(N->getOperand(0).getOperand(1)));
1187 
1188   // The IsRV64 predicate is checked after PatFrag predicates so we can get
1189   // here even on RV32.
1190   if (!Subtarget->is64Bit())
1191     return false;
1192 
1193   SDValue Shl = N->getOperand(0);
1194   uint64_t VC1 = N->getConstantOperandVal(1);
1195   uint64_t VC2 = Shl.getConstantOperandVal(1);
1196 
1197   // Immediate range should be enforced by uimm5 predicate.
1198   assert(VC2 < 32 && "Unexpected immediate");
1199   return (VC1 >> VC2) == UINT64_C(0xFFFFFFFF);
1200 }
1201 
1202 // X0 has special meaning for vsetvl/vsetvli.
1203 //  rd | rs1 |   AVL value | Effect on vl
1204 //--------------------------------------------------------------
1205 // !X0 |  X0 |       VLMAX | Set vl to VLMAX
1206 //  X0 |  X0 | Value in vl | Keep current vl, just change vtype.
1207 bool RISCVDAGToDAGISel::selectVLOp(SDValue N, SDValue &VL) {
1208   // If the VL value is a constant 0, manually select it to an ADDI with 0
1209   // immediate to prevent the default selection path from matching it to X0.
1210   auto *C = dyn_cast<ConstantSDNode>(N);
1211   if (C && C->isNullValue())
1212     VL = SDValue(selectImm(CurDAG, SDLoc(N), 0, Subtarget->getXLenVT()), 0);
1213   else
1214     VL = N;
1215 
1216   return true;
1217 }
1218 
1219 bool RISCVDAGToDAGISel::selectVSplat(SDValue N, SDValue &SplatVal) {
1220   if (N.getOpcode() != ISD::SPLAT_VECTOR &&
1221       N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64 &&
1222       N.getOpcode() != RISCVISD::VMV_V_X_VL)
1223     return false;
1224   SplatVal = N.getOperand(0);
1225   return true;
1226 }
1227 
1228 bool RISCVDAGToDAGISel::selectVSplatSimm5(SDValue N, SDValue &SplatVal) {
1229   if ((N.getOpcode() != ISD::SPLAT_VECTOR &&
1230        N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64 &&
1231        N.getOpcode() != RISCVISD::VMV_V_X_VL) ||
1232       !isa<ConstantSDNode>(N.getOperand(0)))
1233     return false;
1234 
1235   int64_t SplatImm = cast<ConstantSDNode>(N.getOperand(0))->getSExtValue();
1236 
1237   // Both ISD::SPLAT_VECTOR and RISCVISD::SPLAT_VECTOR_I64 share semantics when
1238   // the operand type is wider than the resulting vector element type: an
1239   // implicit truncation first takes place. Therefore, perform a manual
1240   // truncation/sign-extension in order to ignore any truncated bits and catch
1241   // any zero-extended immediate.
1242   // For example, we wish to match (i8 -1) -> (XLenVT 255) as a simm5 by first
1243   // sign-extending to (XLenVT -1).
1244   MVT XLenVT = Subtarget->getXLenVT();
1245   assert(XLenVT == N.getOperand(0).getSimpleValueType() &&
1246          "Unexpected splat operand type");
1247   MVT EltVT = N.getSimpleValueType().getVectorElementType();
1248   if (EltVT.bitsLT(XLenVT)) {
1249     SplatImm = SignExtend64(SplatImm, EltVT.getSizeInBits());
1250   }
1251 
1252   if (!isInt<5>(SplatImm))
1253     return false;
1254 
1255   SplatVal = CurDAG->getTargetConstant(SplatImm, SDLoc(N), XLenVT);
1256   return true;
1257 }
1258 
1259 bool RISCVDAGToDAGISel::selectVSplatUimm5(SDValue N, SDValue &SplatVal) {
1260   if ((N.getOpcode() != ISD::SPLAT_VECTOR &&
1261        N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64 &&
1262        N.getOpcode() != RISCVISD::VMV_V_X_VL) ||
1263       !isa<ConstantSDNode>(N.getOperand(0)))
1264     return false;
1265 
1266   int64_t SplatImm = cast<ConstantSDNode>(N.getOperand(0))->getSExtValue();
1267 
1268   if (!isUInt<5>(SplatImm))
1269     return false;
1270 
1271   SplatVal =
1272       CurDAG->getTargetConstant(SplatImm, SDLoc(N), Subtarget->getXLenVT());
1273 
1274   return true;
1275 }
1276 
1277 bool RISCVDAGToDAGISel::selectRVVSimm5(SDValue N, unsigned Width,
1278                                        SDValue &Imm) {
1279   if (auto *C = dyn_cast<ConstantSDNode>(N)) {
1280     int64_t ImmVal = SignExtend64(C->getSExtValue(), Width);
1281 
1282     if (!isInt<5>(ImmVal))
1283       return false;
1284 
1285     Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), Subtarget->getXLenVT());
1286     return true;
1287   }
1288 
1289   return false;
1290 }
1291 
1292 bool RISCVDAGToDAGISel::selectRVVUimm5(SDValue N, unsigned Width,
1293                                        SDValue &Imm) {
1294   if (auto *C = dyn_cast<ConstantSDNode>(N)) {
1295     int64_t ImmVal = C->getSExtValue();
1296 
1297     if (!isUInt<5>(ImmVal))
1298       return false;
1299 
1300     Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), Subtarget->getXLenVT());
1301     return true;
1302   }
1303 
1304   return false;
1305 }
1306 
1307 // Merge an ADDI into the offset of a load/store instruction where possible.
1308 // (load (addi base, off1), off2) -> (load base, off1+off2)
1309 // (store val, (addi base, off1), off2) -> (store val, base, off1+off2)
1310 // This is possible when off1+off2 fits a 12-bit immediate.
1311 void RISCVDAGToDAGISel::doPeepholeLoadStoreADDI() {
1312   SelectionDAG::allnodes_iterator Position(CurDAG->getRoot().getNode());
1313   ++Position;
1314 
1315   while (Position != CurDAG->allnodes_begin()) {
1316     SDNode *N = &*--Position;
1317     // Skip dead nodes and any non-machine opcodes.
1318     if (N->use_empty() || !N->isMachineOpcode())
1319       continue;
1320 
1321     int OffsetOpIdx;
1322     int BaseOpIdx;
1323 
1324     // Only attempt this optimisation for I-type loads and S-type stores.
1325     switch (N->getMachineOpcode()) {
1326     default:
1327       continue;
1328     case RISCV::LB:
1329     case RISCV::LH:
1330     case RISCV::LW:
1331     case RISCV::LBU:
1332     case RISCV::LHU:
1333     case RISCV::LWU:
1334     case RISCV::LD:
1335     case RISCV::FLH:
1336     case RISCV::FLW:
1337     case RISCV::FLD:
1338       BaseOpIdx = 0;
1339       OffsetOpIdx = 1;
1340       break;
1341     case RISCV::SB:
1342     case RISCV::SH:
1343     case RISCV::SW:
1344     case RISCV::SD:
1345     case RISCV::FSH:
1346     case RISCV::FSW:
1347     case RISCV::FSD:
1348       BaseOpIdx = 1;
1349       OffsetOpIdx = 2;
1350       break;
1351     }
1352 
1353     if (!isa<ConstantSDNode>(N->getOperand(OffsetOpIdx)))
1354       continue;
1355 
1356     SDValue Base = N->getOperand(BaseOpIdx);
1357 
1358     // If the base is an ADDI, we can merge it in to the load/store.
1359     if (!Base.isMachineOpcode() || Base.getMachineOpcode() != RISCV::ADDI)
1360       continue;
1361 
1362     SDValue ImmOperand = Base.getOperand(1);
1363     uint64_t Offset2 = N->getConstantOperandVal(OffsetOpIdx);
1364 
1365     if (auto *Const = dyn_cast<ConstantSDNode>(ImmOperand)) {
1366       int64_t Offset1 = Const->getSExtValue();
1367       int64_t CombinedOffset = Offset1 + Offset2;
1368       if (!isInt<12>(CombinedOffset))
1369         continue;
1370       ImmOperand = CurDAG->getTargetConstant(CombinedOffset, SDLoc(ImmOperand),
1371                                              ImmOperand.getValueType());
1372     } else if (auto *GA = dyn_cast<GlobalAddressSDNode>(ImmOperand)) {
1373       // If the off1 in (addi base, off1) is a global variable's address (its
1374       // low part, really), then we can rely on the alignment of that variable
1375       // to provide a margin of safety before off1 can overflow the 12 bits.
1376       // Check if off2 falls within that margin; if so off1+off2 can't overflow.
1377       const DataLayout &DL = CurDAG->getDataLayout();
1378       Align Alignment = GA->getGlobal()->getPointerAlignment(DL);
1379       if (Offset2 != 0 && Alignment <= Offset2)
1380         continue;
1381       int64_t Offset1 = GA->getOffset();
1382       int64_t CombinedOffset = Offset1 + Offset2;
1383       ImmOperand = CurDAG->getTargetGlobalAddress(
1384           GA->getGlobal(), SDLoc(ImmOperand), ImmOperand.getValueType(),
1385           CombinedOffset, GA->getTargetFlags());
1386     } else if (auto *CP = dyn_cast<ConstantPoolSDNode>(ImmOperand)) {
1387       // Ditto.
1388       Align Alignment = CP->getAlign();
1389       if (Offset2 != 0 && Alignment <= Offset2)
1390         continue;
1391       int64_t Offset1 = CP->getOffset();
1392       int64_t CombinedOffset = Offset1 + Offset2;
1393       ImmOperand = CurDAG->getTargetConstantPool(
1394           CP->getConstVal(), ImmOperand.getValueType(), CP->getAlign(),
1395           CombinedOffset, CP->getTargetFlags());
1396     } else {
1397       continue;
1398     }
1399 
1400     LLVM_DEBUG(dbgs() << "Folding add-immediate into mem-op:\nBase:    ");
1401     LLVM_DEBUG(Base->dump(CurDAG));
1402     LLVM_DEBUG(dbgs() << "\nN: ");
1403     LLVM_DEBUG(N->dump(CurDAG));
1404     LLVM_DEBUG(dbgs() << "\n");
1405 
1406     // Modify the offset operand of the load/store.
1407     if (BaseOpIdx == 0) // Load
1408       CurDAG->UpdateNodeOperands(N, Base.getOperand(0), ImmOperand,
1409                                  N->getOperand(2));
1410     else // Store
1411       CurDAG->UpdateNodeOperands(N, N->getOperand(0), Base.getOperand(0),
1412                                  ImmOperand, N->getOperand(3));
1413 
1414     // The add-immediate may now be dead, in which case remove it.
1415     if (Base.getNode()->use_empty())
1416       CurDAG->RemoveDeadNode(Base.getNode());
1417   }
1418 }
1419 
1420 // This pass converts a legalized DAG into a RISCV-specific DAG, ready
1421 // for instruction scheduling.
1422 FunctionPass *llvm::createRISCVISelDag(RISCVTargetMachine &TM) {
1423   return new RISCVDAGToDAGISel(TM);
1424 }
1425