1 //===-- RISCVISelDAGToDAG.cpp - A dag to dag inst selector for RISCV ------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines an instruction selector for the RISCV target.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "RISCVISelDAGToDAG.h"
14 #include "MCTargetDesc/RISCVMCTargetDesc.h"
15 #include "MCTargetDesc/RISCVMatInt.h"
16 #include "RISCVISelLowering.h"
17 #include "llvm/CodeGen/MachineFrameInfo.h"
18 #include "llvm/IR/IntrinsicsRISCV.h"
19 #include "llvm/Support/Alignment.h"
20 #include "llvm/Support/Debug.h"
21 #include "llvm/Support/KnownBits.h"
22 #include "llvm/Support/MathExtras.h"
23 #include "llvm/Support/raw_ostream.h"
24 
25 using namespace llvm;
26 
27 #define DEBUG_TYPE "riscv-isel"
28 
29 namespace llvm {
30 namespace RISCV {
31 #define GET_RISCVVSSEGTable_IMPL
32 #define GET_RISCVVLSEGTable_IMPL
33 #define GET_RISCVVLXSEGTable_IMPL
34 #define GET_RISCVVSXSEGTable_IMPL
35 #define GET_RISCVVLETable_IMPL
36 #define GET_RISCVVSETable_IMPL
37 #define GET_RISCVVLXTable_IMPL
38 #define GET_RISCVVSXTable_IMPL
39 #include "RISCVGenSearchableTables.inc"
40 } // namespace RISCV
41 } // namespace llvm
42 
43 void RISCVDAGToDAGISel::PostprocessISelDAG() {
44   doPeepholeLoadStoreADDI();
45 }
46 
47 static SDNode *selectImm(SelectionDAG *CurDAG, const SDLoc &DL, int64_t Imm,
48                          MVT XLenVT) {
49   RISCVMatInt::InstSeq Seq;
50   RISCVMatInt::generateInstSeq(Imm, XLenVT == MVT::i64, Seq);
51 
52   SDNode *Result = nullptr;
53   SDValue SrcReg = CurDAG->getRegister(RISCV::X0, XLenVT);
54   for (RISCVMatInt::Inst &Inst : Seq) {
55     SDValue SDImm = CurDAG->getTargetConstant(Inst.Imm, DL, XLenVT);
56     if (Inst.Opc == RISCV::LUI)
57       Result = CurDAG->getMachineNode(RISCV::LUI, DL, XLenVT, SDImm);
58     else
59       Result = CurDAG->getMachineNode(Inst.Opc, DL, XLenVT, SrcReg, SDImm);
60 
61     // Only the first instruction has X0 as its source.
62     SrcReg = SDValue(Result, 0);
63   }
64 
65   return Result;
66 }
67 
68 static SDValue createTupleImpl(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
69                                unsigned RegClassID, unsigned SubReg0) {
70   assert(Regs.size() >= 2 && Regs.size() <= 8);
71 
72   SDLoc DL(Regs[0]);
73   SmallVector<SDValue, 8> Ops;
74 
75   Ops.push_back(CurDAG.getTargetConstant(RegClassID, DL, MVT::i32));
76 
77   for (unsigned I = 0; I < Regs.size(); ++I) {
78     Ops.push_back(Regs[I]);
79     Ops.push_back(CurDAG.getTargetConstant(SubReg0 + I, DL, MVT::i32));
80   }
81   SDNode *N =
82       CurDAG.getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops);
83   return SDValue(N, 0);
84 }
85 
86 static SDValue createM1Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
87                              unsigned NF) {
88   static const unsigned RegClassIDs[] = {
89       RISCV::VRN2M1RegClassID, RISCV::VRN3M1RegClassID, RISCV::VRN4M1RegClassID,
90       RISCV::VRN5M1RegClassID, RISCV::VRN6M1RegClassID, RISCV::VRN7M1RegClassID,
91       RISCV::VRN8M1RegClassID};
92 
93   return createTupleImpl(CurDAG, Regs, RegClassIDs[NF - 2], RISCV::sub_vrm1_0);
94 }
95 
96 static SDValue createM2Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
97                              unsigned NF) {
98   static const unsigned RegClassIDs[] = {RISCV::VRN2M2RegClassID,
99                                          RISCV::VRN3M2RegClassID,
100                                          RISCV::VRN4M2RegClassID};
101 
102   return createTupleImpl(CurDAG, Regs, RegClassIDs[NF - 2], RISCV::sub_vrm2_0);
103 }
104 
105 static SDValue createM4Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
106                              unsigned NF) {
107   return createTupleImpl(CurDAG, Regs, RISCV::VRN2M4RegClassID,
108                          RISCV::sub_vrm4_0);
109 }
110 
111 static SDValue createTuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
112                            unsigned NF, RISCVVLMUL LMUL) {
113   switch (LMUL) {
114   default:
115     llvm_unreachable("Invalid LMUL.");
116   case RISCVVLMUL::LMUL_F8:
117   case RISCVVLMUL::LMUL_F4:
118   case RISCVVLMUL::LMUL_F2:
119   case RISCVVLMUL::LMUL_1:
120     return createM1Tuple(CurDAG, Regs, NF);
121   case RISCVVLMUL::LMUL_2:
122     return createM2Tuple(CurDAG, Regs, NF);
123   case RISCVVLMUL::LMUL_4:
124     return createM4Tuple(CurDAG, Regs, NF);
125   }
126 }
127 
128 void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, bool IsMasked,
129                                     bool IsStrided) {
130   SDLoc DL(Node);
131   unsigned NF = Node->getNumValues() - 1;
132   MVT VT = Node->getSimpleValueType(0);
133   unsigned ScalarSize = VT.getScalarSizeInBits();
134   MVT XLenVT = Subtarget->getXLenVT();
135   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
136   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
137   unsigned CurOp = 2;
138   SmallVector<SDValue, 7> Operands;
139   if (IsMasked) {
140     SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
141                                  Node->op_begin() + CurOp + NF);
142     SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
143     Operands.push_back(MaskedOff);
144     CurOp += NF;
145   }
146   Operands.push_back(Node->getOperand(CurOp++)); // Base pointer.
147   if (IsStrided)
148     Operands.push_back(Node->getOperand(CurOp++)); // Stride.
149   if (IsMasked)
150     Operands.push_back(Node->getOperand(CurOp++)); // Mask.
151   SDValue VL;
152   selectVLOp(Node->getOperand(CurOp++), VL);
153   Operands.push_back(VL);
154   Operands.push_back(SEW);
155   Operands.push_back(Node->getOperand(0)); // Chain.
156   const RISCV::VLSEGPseudo *P =
157       RISCV::getVLSEGPseudo(NF, IsMasked, IsStrided, /*FF*/ false, ScalarSize,
158                             static_cast<unsigned>(LMUL));
159   MachineSDNode *Load =
160       CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
161 
162   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
163     CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
164 
165   SDValue SuperReg = SDValue(Load, 0);
166   for (unsigned I = 0; I < NF; ++I) {
167     unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
168     ReplaceUses(SDValue(Node, I),
169                 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
170   }
171 
172   ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
173   CurDAG->RemoveDeadNode(Node);
174 }
175 
176 void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node, bool IsMasked) {
177   SDLoc DL(Node);
178   unsigned NF = Node->getNumValues() - 2; // Do not count VL and Chain.
179   MVT VT = Node->getSimpleValueType(0);
180   MVT XLenVT = Subtarget->getXLenVT();
181   unsigned ScalarSize = VT.getScalarSizeInBits();
182   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
183   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
184 
185   unsigned CurOp = 2;
186   SmallVector<SDValue, 7> Operands;
187   if (IsMasked) {
188     SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
189                                  Node->op_begin() + CurOp + NF);
190     SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
191     Operands.push_back(MaskedOff);
192     CurOp += NF;
193   }
194   Operands.push_back(Node->getOperand(CurOp++)); // Base pointer.
195   if (IsMasked)
196     Operands.push_back(Node->getOperand(CurOp++)); // Mask.
197   SDValue VL;
198   selectVLOp(Node->getOperand(CurOp++), VL);
199   Operands.push_back(VL);
200   Operands.push_back(SEW);
201   Operands.push_back(Node->getOperand(0)); // Chain.
202   const RISCV::VLSEGPseudo *P =
203       RISCV::getVLSEGPseudo(NF, IsMasked, /*Strided*/ false, /*FF*/ true,
204                             ScalarSize, static_cast<unsigned>(LMUL));
205   MachineSDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped,
206                                                MVT::Other, MVT::Glue, Operands);
207   SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT,
208                                           /*Glue*/ SDValue(Load, 2));
209 
210   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
211     CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
212 
213   SDValue SuperReg = SDValue(Load, 0);
214   for (unsigned I = 0; I < NF; ++I) {
215     unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
216     ReplaceUses(SDValue(Node, I),
217                 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
218   }
219 
220   ReplaceUses(SDValue(Node, NF), SDValue(ReadVL, 0));   // VL
221   ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 1)); // Chain
222   CurDAG->RemoveDeadNode(Node);
223 }
224 
225 void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, bool IsMasked,
226                                      bool IsOrdered) {
227   SDLoc DL(Node);
228   unsigned NF = Node->getNumValues() - 1;
229   MVT VT = Node->getSimpleValueType(0);
230   unsigned ScalarSize = VT.getScalarSizeInBits();
231   MVT XLenVT = Subtarget->getXLenVT();
232   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
233   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
234   unsigned CurOp = 2;
235   SmallVector<SDValue, 7> Operands;
236   if (IsMasked) {
237     SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
238                                  Node->op_begin() + CurOp + NF);
239     SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
240     Operands.push_back(MaskedOff);
241     CurOp += NF;
242   }
243   Operands.push_back(Node->getOperand(CurOp++)); // Base pointer.
244   Operands.push_back(Node->getOperand(CurOp++)); // Index.
245   MVT IndexVT = Operands.back()->getSimpleValueType(0);
246   if (IsMasked)
247     Operands.push_back(Node->getOperand(CurOp++)); // Mask.
248   SDValue VL;
249   selectVLOp(Node->getOperand(CurOp++), VL);
250   Operands.push_back(VL);
251   Operands.push_back(SEW);
252   Operands.push_back(Node->getOperand(0)); // Chain.
253 
254   assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
255          "Element count mismatch");
256 
257   RISCVVLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
258   unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
259   const RISCV::VLXSEGPseudo *P = RISCV::getVLXSEGPseudo(
260       NF, IsMasked, IsOrdered, IndexScalarSize, static_cast<unsigned>(LMUL),
261       static_cast<unsigned>(IndexLMUL));
262   MachineSDNode *Load =
263       CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
264 
265   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
266     CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
267 
268   SDValue SuperReg = SDValue(Load, 0);
269   for (unsigned I = 0; I < NF; ++I) {
270     unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
271     ReplaceUses(SDValue(Node, I),
272                 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
273   }
274 
275   ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
276   CurDAG->RemoveDeadNode(Node);
277 }
278 
279 void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, bool IsMasked,
280                                     bool IsStrided) {
281   SDLoc DL(Node);
282   unsigned NF = Node->getNumOperands() - 4;
283   if (IsStrided)
284     NF--;
285   if (IsMasked)
286     NF--;
287   MVT VT = Node->getOperand(2)->getSimpleValueType(0);
288   unsigned ScalarSize = VT.getScalarSizeInBits();
289   MVT XLenVT = Subtarget->getXLenVT();
290   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
291   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
292   SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
293   SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
294   SmallVector<SDValue, 7> Operands;
295   Operands.push_back(StoreVal);
296   unsigned CurOp = 2 + NF;
297   Operands.push_back(Node->getOperand(CurOp++)); // Base pointer.
298   if (IsStrided)
299     Operands.push_back(Node->getOperand(CurOp++)); // Stride.
300   if (IsMasked)
301     Operands.push_back(Node->getOperand(CurOp++)); // Mask.
302   SDValue VL;
303   selectVLOp(Node->getOperand(CurOp++), VL);
304   Operands.push_back(VL);
305   Operands.push_back(SEW);
306   Operands.push_back(Node->getOperand(0)); // Chain.
307   const RISCV::VSSEGPseudo *P = RISCV::getVSSEGPseudo(
308       NF, IsMasked, IsStrided, ScalarSize, static_cast<unsigned>(LMUL));
309   MachineSDNode *Store =
310       CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
311 
312   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
313     CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
314 
315   ReplaceNode(Node, Store);
316 }
317 
318 void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, bool IsMasked,
319                                      bool IsOrdered) {
320   SDLoc DL(Node);
321   unsigned NF = Node->getNumOperands() - 5;
322   if (IsMasked)
323     --NF;
324   MVT VT = Node->getOperand(2)->getSimpleValueType(0);
325   unsigned ScalarSize = VT.getScalarSizeInBits();
326   MVT XLenVT = Subtarget->getXLenVT();
327   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
328   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
329   SmallVector<SDValue, 7> Operands;
330   SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
331   SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
332   Operands.push_back(StoreVal);
333   unsigned CurOp = 2 + NF;
334   Operands.push_back(Node->getOperand(CurOp++)); // Base pointer.
335   Operands.push_back(Node->getOperand(CurOp++)); // Index.
336   MVT IndexVT = Operands.back()->getSimpleValueType(0);
337   if (IsMasked)
338     Operands.push_back(Node->getOperand(CurOp++)); // Mask.
339   SDValue VL;
340   selectVLOp(Node->getOperand(CurOp++), VL);
341   Operands.push_back(VL);
342   Operands.push_back(SEW);
343   Operands.push_back(Node->getOperand(0)); // Chain.
344 
345   assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
346          "Element count mismatch");
347 
348   RISCVVLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
349   unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
350   const RISCV::VSXSEGPseudo *P = RISCV::getVSXSEGPseudo(
351       NF, IsMasked, IsOrdered, IndexScalarSize, static_cast<unsigned>(LMUL),
352       static_cast<unsigned>(IndexLMUL));
353   MachineSDNode *Store =
354       CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
355 
356   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
357     CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
358 
359   ReplaceNode(Node, Store);
360 }
361 
362 
363 void RISCVDAGToDAGISel::Select(SDNode *Node) {
364   // If we have a custom node, we have already selected.
365   if (Node->isMachineOpcode()) {
366     LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n");
367     Node->setNodeId(-1);
368     return;
369   }
370 
371   // Instruction Selection not handled by the auto-generated tablegen selection
372   // should be handled here.
373   unsigned Opcode = Node->getOpcode();
374   MVT XLenVT = Subtarget->getXLenVT();
375   SDLoc DL(Node);
376   MVT VT = Node->getSimpleValueType(0);
377 
378   switch (Opcode) {
379   case ISD::ADD: {
380     // Optimize (add r, imm) to (addi (addi r, imm0) imm1) if applicable. The
381     // immediate must be in specific ranges and have a single use.
382     if (auto *ConstOp = dyn_cast<ConstantSDNode>(Node->getOperand(1))) {
383       if (!(ConstOp->hasOneUse()))
384         break;
385       // The imm must be in range [-4096,-2049] or [2048,4094].
386       int64_t Imm = ConstOp->getSExtValue();
387       if (!(-4096 <= Imm && Imm <= -2049) && !(2048 <= Imm && Imm <= 4094))
388         break;
389       // Break the imm to imm0+imm1.
390       const SDValue ImmOp0 = CurDAG->getTargetConstant(Imm - Imm / 2, DL, VT);
391       const SDValue ImmOp1 = CurDAG->getTargetConstant(Imm / 2, DL, VT);
392       auto *NodeAddi0 = CurDAG->getMachineNode(RISCV::ADDI, DL, VT,
393                                                Node->getOperand(0), ImmOp0);
394       auto *NodeAddi1 = CurDAG->getMachineNode(RISCV::ADDI, DL, VT,
395                                                SDValue(NodeAddi0, 0), ImmOp1);
396       ReplaceNode(Node, NodeAddi1);
397       return;
398     }
399     break;
400   }
401   case ISD::Constant: {
402     auto *ConstNode = cast<ConstantSDNode>(Node);
403     if (VT == XLenVT && ConstNode->isNullValue()) {
404       SDValue New =
405           CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, RISCV::X0, XLenVT);
406       ReplaceNode(Node, New.getNode());
407       return;
408     }
409     int64_t Imm = ConstNode->getSExtValue();
410     if (XLenVT == MVT::i64) {
411       ReplaceNode(Node, selectImm(CurDAG, DL, Imm, XLenVT));
412       return;
413     }
414     break;
415   }
416   case ISD::FrameIndex: {
417     SDValue Imm = CurDAG->getTargetConstant(0, DL, XLenVT);
418     int FI = cast<FrameIndexSDNode>(Node)->getIndex();
419     SDValue TFI = CurDAG->getTargetFrameIndex(FI, VT);
420     ReplaceNode(Node, CurDAG->getMachineNode(RISCV::ADDI, DL, VT, TFI, Imm));
421     return;
422   }
423   case ISD::SRL: {
424     // Optimize (srl (and X, 0xffff), C) -> (srli (slli X, 16), 16 + C).
425     // Taking into account that the 0xffff may have had lower bits unset by
426     // SimplifyDemandedBits. This avoids materializing the 0xffff immediate.
427     // This pattern occurs when type legalizing i16 right shifts.
428     // FIXME: This could be extended to other AND masks.
429     auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
430     if (N1C) {
431       uint64_t ShAmt = N1C->getZExtValue();
432       SDValue N0 = Node->getOperand(0);
433       if (ShAmt < 16 && N0.getOpcode() == ISD::AND && N0.hasOneUse() &&
434           isa<ConstantSDNode>(N0.getOperand(1))) {
435         uint64_t Mask = N0.getConstantOperandVal(1);
436         Mask |= maskTrailingOnes<uint64_t>(ShAmt);
437         if (Mask == 0xffff) {
438           SDLoc DL(Node);
439           unsigned SLLOpc = Subtarget->is64Bit() ? RISCV::SLLIW : RISCV::SLLI;
440           unsigned SRLOpc = Subtarget->is64Bit() ? RISCV::SRLIW : RISCV::SRLI;
441           SDNode *SLLI =
442               CurDAG->getMachineNode(SLLOpc, DL, VT, N0->getOperand(0),
443                                      CurDAG->getTargetConstant(16, DL, VT));
444           SDNode *SRLI = CurDAG->getMachineNode(
445               SRLOpc, DL, VT, SDValue(SLLI, 0),
446               CurDAG->getTargetConstant(16 + ShAmt, DL, VT));
447           ReplaceNode(Node, SRLI);
448           return;
449         }
450       }
451     }
452 
453     break;
454   }
455   case ISD::INTRINSIC_W_CHAIN: {
456     unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
457     switch (IntNo) {
458       // By default we do not custom select any intrinsic.
459     default:
460       break;
461 
462     case Intrinsic::riscv_vsetvli:
463     case Intrinsic::riscv_vsetvlimax: {
464       if (!Subtarget->hasStdExtV())
465         break;
466 
467       bool VLMax = IntNo == Intrinsic::riscv_vsetvlimax;
468       unsigned Offset = VLMax ? 2 : 3;
469 
470       assert(Node->getNumOperands() == Offset + 2 &&
471              "Unexpected number of operands");
472 
473       RISCVVSEW VSEW =
474           static_cast<RISCVVSEW>(Node->getConstantOperandVal(Offset) & 0x7);
475       RISCVVLMUL VLMul = static_cast<RISCVVLMUL>(
476           Node->getConstantOperandVal(Offset + 1) & 0x7);
477 
478       unsigned VTypeI = RISCVVType::encodeVTYPE(
479           VLMul, VSEW, /*TailAgnostic*/ true, /*MaskAgnostic*/ false);
480       SDValue VTypeIOp = CurDAG->getTargetConstant(VTypeI, DL, XLenVT);
481 
482       SDValue VLOperand;
483       if (VLMax) {
484         VLOperand = CurDAG->getRegister(RISCV::X0, XLenVT);
485       } else {
486         VLOperand = Node->getOperand(2);
487 
488         if (auto *C = dyn_cast<ConstantSDNode>(VLOperand)) {
489           uint64_t AVL = C->getZExtValue();
490           if (isUInt<5>(AVL)) {
491             SDValue VLImm = CurDAG->getTargetConstant(AVL, DL, XLenVT);
492             ReplaceNode(
493                 Node, CurDAG->getMachineNode(RISCV::PseudoVSETIVLI, DL, XLenVT,
494                                              MVT::Other, VLImm, VTypeIOp,
495                                              /* Chain */ Node->getOperand(0)));
496             return;
497           }
498         }
499       }
500 
501       ReplaceNode(Node,
502                   CurDAG->getMachineNode(RISCV::PseudoVSETVLI, DL, XLenVT,
503                                          MVT::Other, VLOperand, VTypeIOp,
504                                          /* Chain */ Node->getOperand(0)));
505       return;
506     }
507     case Intrinsic::riscv_vlseg2:
508     case Intrinsic::riscv_vlseg3:
509     case Intrinsic::riscv_vlseg4:
510     case Intrinsic::riscv_vlseg5:
511     case Intrinsic::riscv_vlseg6:
512     case Intrinsic::riscv_vlseg7:
513     case Intrinsic::riscv_vlseg8: {
514       selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false);
515       return;
516     }
517     case Intrinsic::riscv_vlseg2_mask:
518     case Intrinsic::riscv_vlseg3_mask:
519     case Intrinsic::riscv_vlseg4_mask:
520     case Intrinsic::riscv_vlseg5_mask:
521     case Intrinsic::riscv_vlseg6_mask:
522     case Intrinsic::riscv_vlseg7_mask:
523     case Intrinsic::riscv_vlseg8_mask: {
524       selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false);
525       return;
526     }
527     case Intrinsic::riscv_vlsseg2:
528     case Intrinsic::riscv_vlsseg3:
529     case Intrinsic::riscv_vlsseg4:
530     case Intrinsic::riscv_vlsseg5:
531     case Intrinsic::riscv_vlsseg6:
532     case Intrinsic::riscv_vlsseg7:
533     case Intrinsic::riscv_vlsseg8: {
534       selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true);
535       return;
536     }
537     case Intrinsic::riscv_vlsseg2_mask:
538     case Intrinsic::riscv_vlsseg3_mask:
539     case Intrinsic::riscv_vlsseg4_mask:
540     case Intrinsic::riscv_vlsseg5_mask:
541     case Intrinsic::riscv_vlsseg6_mask:
542     case Intrinsic::riscv_vlsseg7_mask:
543     case Intrinsic::riscv_vlsseg8_mask: {
544       selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true);
545       return;
546     }
547     case Intrinsic::riscv_vloxseg2:
548     case Intrinsic::riscv_vloxseg3:
549     case Intrinsic::riscv_vloxseg4:
550     case Intrinsic::riscv_vloxseg5:
551     case Intrinsic::riscv_vloxseg6:
552     case Intrinsic::riscv_vloxseg7:
553     case Intrinsic::riscv_vloxseg8:
554       selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true);
555       return;
556     case Intrinsic::riscv_vluxseg2:
557     case Intrinsic::riscv_vluxseg3:
558     case Intrinsic::riscv_vluxseg4:
559     case Intrinsic::riscv_vluxseg5:
560     case Intrinsic::riscv_vluxseg6:
561     case Intrinsic::riscv_vluxseg7:
562     case Intrinsic::riscv_vluxseg8:
563       selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false);
564       return;
565     case Intrinsic::riscv_vloxseg2_mask:
566     case Intrinsic::riscv_vloxseg3_mask:
567     case Intrinsic::riscv_vloxseg4_mask:
568     case Intrinsic::riscv_vloxseg5_mask:
569     case Intrinsic::riscv_vloxseg6_mask:
570     case Intrinsic::riscv_vloxseg7_mask:
571     case Intrinsic::riscv_vloxseg8_mask:
572       selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true);
573       return;
574     case Intrinsic::riscv_vluxseg2_mask:
575     case Intrinsic::riscv_vluxseg3_mask:
576     case Intrinsic::riscv_vluxseg4_mask:
577     case Intrinsic::riscv_vluxseg5_mask:
578     case Intrinsic::riscv_vluxseg6_mask:
579     case Intrinsic::riscv_vluxseg7_mask:
580     case Intrinsic::riscv_vluxseg8_mask:
581       selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false);
582       return;
583     case Intrinsic::riscv_vlseg8ff:
584     case Intrinsic::riscv_vlseg7ff:
585     case Intrinsic::riscv_vlseg6ff:
586     case Intrinsic::riscv_vlseg5ff:
587     case Intrinsic::riscv_vlseg4ff:
588     case Intrinsic::riscv_vlseg3ff:
589     case Intrinsic::riscv_vlseg2ff: {
590       selectVLSEGFF(Node, /*IsMasked*/ false);
591       return;
592     }
593     case Intrinsic::riscv_vlseg8ff_mask:
594     case Intrinsic::riscv_vlseg7ff_mask:
595     case Intrinsic::riscv_vlseg6ff_mask:
596     case Intrinsic::riscv_vlseg5ff_mask:
597     case Intrinsic::riscv_vlseg4ff_mask:
598     case Intrinsic::riscv_vlseg3ff_mask:
599     case Intrinsic::riscv_vlseg2ff_mask: {
600       selectVLSEGFF(Node, /*IsMasked*/ true);
601       return;
602     }
603     case Intrinsic::riscv_vloxei:
604     case Intrinsic::riscv_vloxei_mask:
605     case Intrinsic::riscv_vluxei:
606     case Intrinsic::riscv_vluxei_mask: {
607       bool IsMasked = IntNo == Intrinsic::riscv_vloxei_mask ||
608                       IntNo == Intrinsic::riscv_vluxei_mask;
609       bool IsOrdered = IntNo == Intrinsic::riscv_vloxei ||
610                        IntNo == Intrinsic::riscv_vloxei_mask;
611 
612       SDLoc DL(Node);
613       MVT VT = Node->getSimpleValueType(0);
614       unsigned ScalarSize = VT.getScalarSizeInBits();
615       MVT XLenVT = Subtarget->getXLenVT();
616       SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
617 
618       unsigned CurOp = 2;
619       SmallVector<SDValue, 7> Operands;
620       if (IsMasked)
621         Operands.push_back(Node->getOperand(CurOp++));
622       Operands.push_back(Node->getOperand(CurOp++)); // Base pointer.
623       Operands.push_back(Node->getOperand(CurOp++)); // Index.
624       MVT IndexVT = Operands.back()->getSimpleValueType(0);
625       if (IsMasked)
626         Operands.push_back(Node->getOperand(CurOp++)); // Mask.
627       SDValue VL;
628       selectVLOp(Node->getOperand(CurOp++), VL);
629       Operands.push_back(VL);
630       Operands.push_back(SEW);
631       Operands.push_back(Node->getOperand(0)); // Chain.
632 
633       assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
634              "Element count mismatch");
635 
636       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
637       RISCVVLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
638       unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
639       const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
640           IsMasked, IsOrdered, IndexScalarSize, static_cast<unsigned>(LMUL),
641           static_cast<unsigned>(IndexLMUL));
642       MachineSDNode *Load =
643           CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
644 
645       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
646         CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
647 
648       ReplaceNode(Node, Load);
649       return;
650     }
651     case Intrinsic::riscv_vle1:
652     case Intrinsic::riscv_vle:
653     case Intrinsic::riscv_vle_mask:
654     case Intrinsic::riscv_vlse:
655     case Intrinsic::riscv_vlse_mask: {
656       bool IsMasked = IntNo == Intrinsic::riscv_vle_mask ||
657                       IntNo == Intrinsic::riscv_vlse_mask;
658       bool IsStrided =
659           IntNo == Intrinsic::riscv_vlse || IntNo == Intrinsic::riscv_vlse_mask;
660 
661       SDLoc DL(Node);
662       MVT VT = Node->getSimpleValueType(0);
663       unsigned ScalarSize = VT.getScalarSizeInBits();
664       MVT XLenVT = Subtarget->getXLenVT();
665       // VLE1 uses an SEW of 8.
666       unsigned SEWImm = (IntNo == Intrinsic::riscv_vle1) ? 8 : ScalarSize;
667       SDValue SEW = CurDAG->getTargetConstant(SEWImm, DL, XLenVT);
668 
669       unsigned CurOp = 2;
670       SmallVector<SDValue, 7> Operands;
671       if (IsMasked)
672         Operands.push_back(Node->getOperand(CurOp++));
673       Operands.push_back(Node->getOperand(CurOp++)); // Base pointer.
674       if (IsStrided)
675         Operands.push_back(Node->getOperand(CurOp++)); // Stride.
676       if (IsMasked)
677         Operands.push_back(Node->getOperand(CurOp++)); // Mask.
678       SDValue VL;
679       selectVLOp(Node->getOperand(CurOp++), VL);
680       Operands.push_back(VL);
681       Operands.push_back(SEW);
682       Operands.push_back(Node->getOperand(0)); // Chain.
683 
684       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
685       const RISCV::VLEPseudo *P =
686           RISCV::getVLEPseudo(IsMasked, IsStrided, /*FF*/ false, ScalarSize,
687                               static_cast<unsigned>(LMUL));
688       MachineSDNode *Load =
689           CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
690 
691       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
692         CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
693 
694       ReplaceNode(Node, Load);
695       return;
696     }
697     case Intrinsic::riscv_vleff:
698     case Intrinsic::riscv_vleff_mask: {
699       bool IsMasked = IntNo == Intrinsic::riscv_vleff_mask;
700 
701       SDLoc DL(Node);
702       MVT VT = Node->getSimpleValueType(0);
703       unsigned ScalarSize = VT.getScalarSizeInBits();
704       MVT XLenVT = Subtarget->getXLenVT();
705       SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
706 
707       unsigned CurOp = 2;
708       SmallVector<SDValue, 7> Operands;
709       if (IsMasked)
710         Operands.push_back(Node->getOperand(CurOp++));
711       Operands.push_back(Node->getOperand(CurOp++)); // Base pointer.
712       if (IsMasked)
713         Operands.push_back(Node->getOperand(CurOp++)); // Mask.
714       SDValue VL;
715       selectVLOp(Node->getOperand(CurOp++), VL);
716       Operands.push_back(VL);
717       Operands.push_back(SEW);
718       Operands.push_back(Node->getOperand(0)); // Chain.
719 
720       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
721       const RISCV::VLEPseudo *P =
722           RISCV::getVLEPseudo(IsMasked, /*Strided*/ false, /*FF*/ true,
723                               ScalarSize, static_cast<unsigned>(LMUL));
724       MachineSDNode *Load =
725           CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0),
726                                  MVT::Other, MVT::Glue, Operands);
727       SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT,
728                                               /*Glue*/ SDValue(Load, 2));
729 
730       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
731         CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
732 
733       ReplaceUses(SDValue(Node, 0), SDValue(Load, 0));
734       ReplaceUses(SDValue(Node, 1), SDValue(ReadVL, 0)); // VL
735       ReplaceUses(SDValue(Node, 2), SDValue(Load, 1));   // Chain
736       CurDAG->RemoveDeadNode(Node);
737       return;
738     }
739     }
740     break;
741   }
742   case ISD::INTRINSIC_VOID: {
743     unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
744     switch (IntNo) {
745     case Intrinsic::riscv_vsseg2:
746     case Intrinsic::riscv_vsseg3:
747     case Intrinsic::riscv_vsseg4:
748     case Intrinsic::riscv_vsseg5:
749     case Intrinsic::riscv_vsseg6:
750     case Intrinsic::riscv_vsseg7:
751     case Intrinsic::riscv_vsseg8: {
752       selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false);
753       return;
754     }
755     case Intrinsic::riscv_vsseg2_mask:
756     case Intrinsic::riscv_vsseg3_mask:
757     case Intrinsic::riscv_vsseg4_mask:
758     case Intrinsic::riscv_vsseg5_mask:
759     case Intrinsic::riscv_vsseg6_mask:
760     case Intrinsic::riscv_vsseg7_mask:
761     case Intrinsic::riscv_vsseg8_mask: {
762       selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false);
763       return;
764     }
765     case Intrinsic::riscv_vssseg2:
766     case Intrinsic::riscv_vssseg3:
767     case Intrinsic::riscv_vssseg4:
768     case Intrinsic::riscv_vssseg5:
769     case Intrinsic::riscv_vssseg6:
770     case Intrinsic::riscv_vssseg7:
771     case Intrinsic::riscv_vssseg8: {
772       selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true);
773       return;
774     }
775     case Intrinsic::riscv_vssseg2_mask:
776     case Intrinsic::riscv_vssseg3_mask:
777     case Intrinsic::riscv_vssseg4_mask:
778     case Intrinsic::riscv_vssseg5_mask:
779     case Intrinsic::riscv_vssseg6_mask:
780     case Intrinsic::riscv_vssseg7_mask:
781     case Intrinsic::riscv_vssseg8_mask: {
782       selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true);
783       return;
784     }
785     case Intrinsic::riscv_vsoxseg2:
786     case Intrinsic::riscv_vsoxseg3:
787     case Intrinsic::riscv_vsoxseg4:
788     case Intrinsic::riscv_vsoxseg5:
789     case Intrinsic::riscv_vsoxseg6:
790     case Intrinsic::riscv_vsoxseg7:
791     case Intrinsic::riscv_vsoxseg8:
792       selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true);
793       return;
794     case Intrinsic::riscv_vsuxseg2:
795     case Intrinsic::riscv_vsuxseg3:
796     case Intrinsic::riscv_vsuxseg4:
797     case Intrinsic::riscv_vsuxseg5:
798     case Intrinsic::riscv_vsuxseg6:
799     case Intrinsic::riscv_vsuxseg7:
800     case Intrinsic::riscv_vsuxseg8:
801       selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false);
802       return;
803     case Intrinsic::riscv_vsoxseg2_mask:
804     case Intrinsic::riscv_vsoxseg3_mask:
805     case Intrinsic::riscv_vsoxseg4_mask:
806     case Intrinsic::riscv_vsoxseg5_mask:
807     case Intrinsic::riscv_vsoxseg6_mask:
808     case Intrinsic::riscv_vsoxseg7_mask:
809     case Intrinsic::riscv_vsoxseg8_mask:
810       selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true);
811       return;
812     case Intrinsic::riscv_vsuxseg2_mask:
813     case Intrinsic::riscv_vsuxseg3_mask:
814     case Intrinsic::riscv_vsuxseg4_mask:
815     case Intrinsic::riscv_vsuxseg5_mask:
816     case Intrinsic::riscv_vsuxseg6_mask:
817     case Intrinsic::riscv_vsuxseg7_mask:
818     case Intrinsic::riscv_vsuxseg8_mask:
819       selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false);
820       return;
821     case Intrinsic::riscv_vsoxei:
822     case Intrinsic::riscv_vsoxei_mask:
823     case Intrinsic::riscv_vsuxei:
824     case Intrinsic::riscv_vsuxei_mask: {
825       bool IsMasked = IntNo == Intrinsic::riscv_vsoxei_mask ||
826                       IntNo == Intrinsic::riscv_vsuxei_mask;
827       bool IsOrdered = IntNo == Intrinsic::riscv_vsoxei ||
828                        IntNo == Intrinsic::riscv_vsoxei_mask;
829 
830       SDLoc DL(Node);
831       MVT VT = Node->getOperand(2)->getSimpleValueType(0);
832       unsigned ScalarSize = VT.getScalarSizeInBits();
833       MVT XLenVT = Subtarget->getXLenVT();
834       SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
835 
836       unsigned CurOp = 2;
837       SmallVector<SDValue, 6> Operands;
838       Operands.push_back(Node->getOperand(CurOp++)); // Store value.
839       Operands.push_back(Node->getOperand(CurOp++)); // Base pointer.
840       Operands.push_back(Node->getOperand(CurOp++)); // Index.
841       MVT IndexVT = Operands.back()->getSimpleValueType(0);
842       if (IsMasked)
843         Operands.push_back(Node->getOperand(CurOp++)); // Mask.
844       SDValue VL;
845       selectVLOp(Node->getOperand(CurOp++), VL);
846       Operands.push_back(VL);
847       Operands.push_back(SEW);
848       Operands.push_back(Node->getOperand(0)); // Chain.
849 
850       assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
851              "Element count mismatch");
852 
853       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
854       RISCVVLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
855       unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
856       const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
857           IsMasked, IsOrdered, IndexScalarSize, static_cast<unsigned>(LMUL),
858           static_cast<unsigned>(IndexLMUL));
859       MachineSDNode *Store =
860           CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
861 
862       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
863         CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
864 
865       ReplaceNode(Node, Store);
866       return;
867     }
868     case Intrinsic::riscv_vse1:
869     case Intrinsic::riscv_vse:
870     case Intrinsic::riscv_vse_mask:
871     case Intrinsic::riscv_vsse:
872     case Intrinsic::riscv_vsse_mask: {
873       bool IsMasked = IntNo == Intrinsic::riscv_vse_mask ||
874                       IntNo == Intrinsic::riscv_vsse_mask;
875       bool IsStrided =
876           IntNo == Intrinsic::riscv_vsse || IntNo == Intrinsic::riscv_vsse_mask;
877 
878       SDLoc DL(Node);
879       MVT VT = Node->getOperand(2)->getSimpleValueType(0);
880       unsigned ScalarSize = VT.getScalarSizeInBits();
881       MVT XLenVT = Subtarget->getXLenVT();
882       // VSE1 uses an SEW of 8.
883       unsigned SEWImm = (IntNo == Intrinsic::riscv_vse1) ? 8 : ScalarSize;
884       SDValue SEW = CurDAG->getTargetConstant(SEWImm, DL, XLenVT);
885 
886       unsigned CurOp = 2;
887       SmallVector<SDValue, 6> Operands;
888       Operands.push_back(Node->getOperand(CurOp++)); // Store value.
889       Operands.push_back(Node->getOperand(CurOp++)); // Base pointer.
890       if (IsStrided)
891         Operands.push_back(Node->getOperand(CurOp++)); // Stride.
892       if (IsMasked)
893         Operands.push_back(Node->getOperand(CurOp++)); // Mask.
894       SDValue VL;
895       selectVLOp(Node->getOperand(CurOp++), VL);
896       Operands.push_back(VL);
897       Operands.push_back(SEW);
898       Operands.push_back(Node->getOperand(0)); // Chain.
899 
900       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
901       const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(
902           IsMasked, IsStrided, ScalarSize, static_cast<unsigned>(LMUL));
903       MachineSDNode *Store =
904           CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
905 
906       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
907         CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
908 
909       ReplaceNode(Node, Store);
910       return;
911     }
912     }
913     break;
914   }
915   case ISD::BITCAST: {
916     MVT SrcVT = Node->getOperand(0).getSimpleValueType();
917     // Just drop bitcasts between vectors if both are fixed or both are
918     // scalable.
919     if ((VT.isScalableVector() && SrcVT.isScalableVector()) ||
920         (VT.isFixedLengthVector() && SrcVT.isFixedLengthVector())) {
921       ReplaceUses(SDValue(Node, 0), Node->getOperand(0));
922       CurDAG->RemoveDeadNode(Node);
923       return;
924     }
925     break;
926   }
927   case ISD::INSERT_SUBVECTOR: {
928     SDValue V = Node->getOperand(0);
929     SDValue SubV = Node->getOperand(1);
930     SDLoc DL(SubV);
931     auto Idx = Node->getConstantOperandVal(2);
932     MVT SubVecVT = Node->getOperand(1).getSimpleValueType();
933 
934     // TODO: This method of selecting INSERT_SUBVECTOR should work
935     // with any type of insertion (fixed <-> scalable) but we don't yet
936     // correctly identify the canonical register class for fixed-length types.
937     // For now, keep the two paths separate.
938     if (VT.isScalableVector() && SubVecVT.isScalableVector()) {
939       bool IsFullVecReg = false;
940       switch (RISCVTargetLowering::getLMUL(SubVecVT)) {
941       default:
942         break;
943       case RISCVVLMUL::LMUL_1:
944       case RISCVVLMUL::LMUL_2:
945       case RISCVVLMUL::LMUL_4:
946       case RISCVVLMUL::LMUL_8:
947         IsFullVecReg = true;
948         break;
949       }
950 
951       // If the subvector doesn't occupy a full vector register then we can't
952       // insert it purely using subregister manipulation. We must not clobber
953       // the untouched elements (say, in the upper half of the VR register).
954       if (!IsFullVecReg)
955         break;
956 
957       const auto *TRI = Subtarget->getRegisterInfo();
958       unsigned SubRegIdx;
959       std::tie(SubRegIdx, Idx) =
960           RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
961               VT, SubVecVT, Idx, TRI);
962 
963       // If the Idx hasn't been completely eliminated then this is a subvector
964       // insert which doesn't naturally align to a vector register. These must
965       // be handled using instructions to manipulate the vector registers.
966       if (Idx != 0)
967         break;
968 
969       SDNode *NewNode = CurDAG->getMachineNode(
970           TargetOpcode::INSERT_SUBREG, DL, VT, V, SubV,
971           CurDAG->getTargetConstant(SubRegIdx, DL, Subtarget->getXLenVT()));
972       return ReplaceNode(Node, NewNode);
973     }
974 
975     if (VT.isScalableVector() && SubVecVT.isFixedLengthVector()) {
976       // Bail when not a "cast" like insert_subvector.
977       if (Idx != 0)
978         break;
979       if (!Node->getOperand(0).isUndef())
980         break;
981 
982       unsigned RegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT);
983 
984       SDValue RC =
985           CurDAG->getTargetConstant(RegClassID, DL, Subtarget->getXLenVT());
986       SDNode *NewNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
987                                                DL, VT, SubV, RC);
988       ReplaceNode(Node, NewNode);
989       return;
990     }
991     break;
992   }
993   case ISD::EXTRACT_SUBVECTOR: {
994     SDValue V = Node->getOperand(0);
995     auto Idx = Node->getConstantOperandVal(1);
996     MVT InVT = Node->getOperand(0).getSimpleValueType();
997     SDLoc DL(V);
998 
999     // TODO: This method of selecting EXTRACT_SUBVECTOR should work
1000     // with any type of extraction (fixed <-> scalable) but we don't yet
1001     // correctly identify the canonical register class for fixed-length types.
1002     // For now, keep the two paths separate.
1003     if (VT.isScalableVector() && InVT.isScalableVector()) {
1004       const auto *TRI = Subtarget->getRegisterInfo();
1005       unsigned SubRegIdx;
1006       std::tie(SubRegIdx, Idx) =
1007           RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1008               InVT, VT, Idx, TRI);
1009 
1010       // If the Idx hasn't been completely eliminated then this is a subvector
1011       // extract which doesn't naturally align to a vector register. These must
1012       // be handled using instructions to manipulate the vector registers.
1013       if (Idx != 0)
1014         break;
1015 
1016       // If we haven't set a SubRegIdx, then we must be going between LMUL<=1
1017       // types (VR -> VR). This can be done as a copy.
1018       if (SubRegIdx == RISCV::NoSubRegister) {
1019         unsigned InRegClassID =
1020             RISCVTargetLowering::getRegClassIDForVecVT(InVT);
1021         assert(RISCVTargetLowering::getRegClassIDForVecVT(VT) ==
1022                    RISCV::VRRegClassID &&
1023                InRegClassID == RISCV::VRRegClassID &&
1024                "Unexpected subvector extraction");
1025         SDValue RC =
1026             CurDAG->getTargetConstant(InRegClassID, DL, Subtarget->getXLenVT());
1027         SDNode *NewNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1028                                                  DL, VT, V, RC);
1029         return ReplaceNode(Node, NewNode);
1030       }
1031       SDNode *NewNode = CurDAG->getMachineNode(
1032           TargetOpcode::EXTRACT_SUBREG, DL, VT, V,
1033           CurDAG->getTargetConstant(SubRegIdx, DL, Subtarget->getXLenVT()));
1034       return ReplaceNode(Node, NewNode);
1035     }
1036 
1037     if (VT.isFixedLengthVector() && InVT.isScalableVector()) {
1038       // Bail when not a "cast" like extract_subvector.
1039       if (Idx != 0)
1040         break;
1041 
1042       unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(InVT);
1043 
1044       SDValue RC =
1045           CurDAG->getTargetConstant(InRegClassID, DL, Subtarget->getXLenVT());
1046       SDNode *NewNode =
1047           CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, DL, VT, V, RC);
1048       ReplaceNode(Node, NewNode);
1049       return;
1050     }
1051     break;
1052   }
1053   }
1054 
1055   // Select the default instruction.
1056   SelectCode(Node);
1057 }
1058 
1059 bool RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand(
1060     const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {
1061   switch (ConstraintID) {
1062   case InlineAsm::Constraint_m:
1063     // We just support simple memory operands that have a single address
1064     // operand and need no special handling.
1065     OutOps.push_back(Op);
1066     return false;
1067   case InlineAsm::Constraint_A:
1068     OutOps.push_back(Op);
1069     return false;
1070   default:
1071     break;
1072   }
1073 
1074   return true;
1075 }
1076 
1077 bool RISCVDAGToDAGISel::SelectAddrFI(SDValue Addr, SDValue &Base) {
1078   if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
1079     Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT());
1080     return true;
1081   }
1082   return false;
1083 }
1084 
1085 bool RISCVDAGToDAGISel::SelectBaseAddr(SDValue Addr, SDValue &Base) {
1086   // If this is FrameIndex, select it directly. Otherwise just let it get
1087   // selected to a register independently.
1088   if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr))
1089     Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT());
1090   else
1091     Base = Addr;
1092   return true;
1093 }
1094 
1095 bool RISCVDAGToDAGISel::selectShiftMask(SDValue N, unsigned ShiftWidth,
1096                                         SDValue &ShAmt) {
1097   // Shift instructions on RISCV only read the lower 5 or 6 bits of the shift
1098   // amount. If there is an AND on the shift amount, we can bypass it if it
1099   // doesn't affect any of those bits.
1100   if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(N.getOperand(1))) {
1101     const APInt &AndMask = N->getConstantOperandAPInt(1);
1102 
1103     // Since the max shift amount is a power of 2 we can subtract 1 to make a
1104     // mask that covers the bits needed to represent all shift amounts.
1105     assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!");
1106     APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
1107 
1108     if (ShMask.isSubsetOf(AndMask)) {
1109       ShAmt = N.getOperand(0);
1110       return true;
1111     }
1112 
1113     // SimplifyDemandedBits may have optimized the mask so try restoring any
1114     // bits that are known zero.
1115     KnownBits Known = CurDAG->computeKnownBits(N->getOperand(0));
1116     if (ShMask.isSubsetOf(AndMask | Known.Zero)) {
1117       ShAmt = N.getOperand(0);
1118       return true;
1119     }
1120   }
1121 
1122   ShAmt = N;
1123   return true;
1124 }
1125 
1126 // Match (srl (and val, mask), imm) where the result would be a
1127 // zero-extended 32-bit integer. i.e. the mask is 0xffffffff or the result
1128 // is equivalent to this (SimplifyDemandedBits may have removed lower bits
1129 // from the mask that aren't necessary due to the right-shifting).
1130 bool RISCVDAGToDAGISel::MatchSRLIW(SDNode *N) const {
1131   assert(N->getOpcode() == ISD::SRL);
1132   assert(N->getOperand(0).getOpcode() == ISD::AND);
1133   assert(isa<ConstantSDNode>(N->getOperand(1)));
1134   assert(isa<ConstantSDNode>(N->getOperand(0).getOperand(1)));
1135 
1136   // The IsRV64 predicate is checked after PatFrag predicates so we can get
1137   // here even on RV32.
1138   if (!Subtarget->is64Bit())
1139     return false;
1140 
1141   SDValue And = N->getOperand(0);
1142   uint64_t ShAmt = N->getConstantOperandVal(1);
1143   uint64_t Mask = And.getConstantOperandVal(1);
1144   return (Mask | maskTrailingOnes<uint64_t>(ShAmt)) == 0xffffffff;
1145 }
1146 
1147 // Check that it is a SLLIUW (Shift Logical Left Immediate Unsigned i32
1148 // on RV64).
1149 // SLLIUW is the same as SLLI except for the fact that it clears the bits
1150 // XLEN-1:32 of the input RS1 before shifting.
1151 // A PatFrag has already checked that it has the right structure:
1152 //
1153 //  (AND (SHL RS1, VC2), VC1)
1154 //
1155 // We check that VC2, the shamt is less than 32, otherwise the pattern is
1156 // exactly the same as SLLI and we give priority to that.
1157 // Eventually we check that VC1, the mask used to clear the upper 32 bits
1158 // of RS1, is correct:
1159 //
1160 //  VC1 == (0xFFFFFFFF << VC2)
1161 //
1162 bool RISCVDAGToDAGISel::MatchSLLIUW(SDNode *N) const {
1163   assert(N->getOpcode() == ISD::AND);
1164   assert(N->getOperand(0).getOpcode() == ISD::SHL);
1165   assert(isa<ConstantSDNode>(N->getOperand(1)));
1166   assert(isa<ConstantSDNode>(N->getOperand(0).getOperand(1)));
1167 
1168   // The IsRV64 predicate is checked after PatFrag predicates so we can get
1169   // here even on RV32.
1170   if (!Subtarget->is64Bit())
1171     return false;
1172 
1173   SDValue Shl = N->getOperand(0);
1174   uint64_t VC1 = N->getConstantOperandVal(1);
1175   uint64_t VC2 = Shl.getConstantOperandVal(1);
1176 
1177   // Immediate range should be enforced by uimm5 predicate.
1178   assert(VC2 < 32 && "Unexpected immediate");
1179   return (VC1 >> VC2) == UINT64_C(0xFFFFFFFF);
1180 }
1181 
1182 // X0 has special meaning for vsetvl/vsetvli.
1183 //  rd | rs1 |   AVL value | Effect on vl
1184 //--------------------------------------------------------------
1185 // !X0 |  X0 |       VLMAX | Set vl to VLMAX
1186 //  X0 |  X0 | Value in vl | Keep current vl, just change vtype.
1187 bool RISCVDAGToDAGISel::selectVLOp(SDValue N, SDValue &VL) {
1188   // If the VL value is a constant 0, manually select it to an ADDI with 0
1189   // immediate to prevent the default selection path from matching it to X0.
1190   auto *C = dyn_cast<ConstantSDNode>(N);
1191   if (C && C->isNullValue())
1192     VL = SDValue(selectImm(CurDAG, SDLoc(N), 0, Subtarget->getXLenVT()), 0);
1193   else
1194     VL = N;
1195 
1196   return true;
1197 }
1198 
1199 bool RISCVDAGToDAGISel::selectVSplat(SDValue N, SDValue &SplatVal) {
1200   if (N.getOpcode() != ISD::SPLAT_VECTOR &&
1201       N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64 &&
1202       N.getOpcode() != RISCVISD::VMV_V_X_VL)
1203     return false;
1204   SplatVal = N.getOperand(0);
1205   return true;
1206 }
1207 
1208 bool RISCVDAGToDAGISel::selectVSplatSimm5(SDValue N, SDValue &SplatVal) {
1209   if ((N.getOpcode() != ISD::SPLAT_VECTOR &&
1210        N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64 &&
1211        N.getOpcode() != RISCVISD::VMV_V_X_VL) ||
1212       !isa<ConstantSDNode>(N.getOperand(0)))
1213     return false;
1214 
1215   int64_t SplatImm = cast<ConstantSDNode>(N.getOperand(0))->getSExtValue();
1216 
1217   // Both ISD::SPLAT_VECTOR and RISCVISD::SPLAT_VECTOR_I64 share semantics when
1218   // the operand type is wider than the resulting vector element type: an
1219   // implicit truncation first takes place. Therefore, perform a manual
1220   // truncation/sign-extension in order to ignore any truncated bits and catch
1221   // any zero-extended immediate.
1222   // For example, we wish to match (i8 -1) -> (XLenVT 255) as a simm5 by first
1223   // sign-extending to (XLenVT -1).
1224   MVT XLenVT = Subtarget->getXLenVT();
1225   assert(XLenVT == N.getOperand(0).getSimpleValueType() &&
1226          "Unexpected splat operand type");
1227   MVT EltVT = N.getSimpleValueType().getVectorElementType();
1228   if (EltVT.bitsLT(XLenVT)) {
1229     SplatImm = SignExtend64(SplatImm, EltVT.getSizeInBits());
1230   }
1231 
1232   if (!isInt<5>(SplatImm))
1233     return false;
1234 
1235   SplatVal = CurDAG->getTargetConstant(SplatImm, SDLoc(N), XLenVT);
1236   return true;
1237 }
1238 
1239 bool RISCVDAGToDAGISel::selectVSplatUimm5(SDValue N, SDValue &SplatVal) {
1240   if ((N.getOpcode() != ISD::SPLAT_VECTOR &&
1241        N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64 &&
1242        N.getOpcode() != RISCVISD::VMV_V_X_VL) ||
1243       !isa<ConstantSDNode>(N.getOperand(0)))
1244     return false;
1245 
1246   int64_t SplatImm = cast<ConstantSDNode>(N.getOperand(0))->getSExtValue();
1247 
1248   if (!isUInt<5>(SplatImm))
1249     return false;
1250 
1251   SplatVal =
1252       CurDAG->getTargetConstant(SplatImm, SDLoc(N), Subtarget->getXLenVT());
1253 
1254   return true;
1255 }
1256 
1257 bool RISCVDAGToDAGISel::selectRVVSimm5(SDValue N, unsigned Width,
1258                                        SDValue &Imm) {
1259   if (auto *C = dyn_cast<ConstantSDNode>(N)) {
1260     int64_t ImmVal = SignExtend64(C->getSExtValue(), Width);
1261 
1262     if (!isInt<5>(ImmVal))
1263       return false;
1264 
1265     Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), Subtarget->getXLenVT());
1266     return true;
1267   }
1268 
1269   return false;
1270 }
1271 
1272 bool RISCVDAGToDAGISel::selectRVVUimm5(SDValue N, unsigned Width,
1273                                        SDValue &Imm) {
1274   if (auto *C = dyn_cast<ConstantSDNode>(N)) {
1275     int64_t ImmVal = C->getSExtValue();
1276 
1277     if (!isUInt<5>(ImmVal))
1278       return false;
1279 
1280     Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), Subtarget->getXLenVT());
1281     return true;
1282   }
1283 
1284   return false;
1285 }
1286 
1287 // Merge an ADDI into the offset of a load/store instruction where possible.
1288 // (load (addi base, off1), off2) -> (load base, off1+off2)
1289 // (store val, (addi base, off1), off2) -> (store val, base, off1+off2)
1290 // This is possible when off1+off2 fits a 12-bit immediate.
1291 void RISCVDAGToDAGISel::doPeepholeLoadStoreADDI() {
1292   SelectionDAG::allnodes_iterator Position(CurDAG->getRoot().getNode());
1293   ++Position;
1294 
1295   while (Position != CurDAG->allnodes_begin()) {
1296     SDNode *N = &*--Position;
1297     // Skip dead nodes and any non-machine opcodes.
1298     if (N->use_empty() || !N->isMachineOpcode())
1299       continue;
1300 
1301     int OffsetOpIdx;
1302     int BaseOpIdx;
1303 
1304     // Only attempt this optimisation for I-type loads and S-type stores.
1305     switch (N->getMachineOpcode()) {
1306     default:
1307       continue;
1308     case RISCV::LB:
1309     case RISCV::LH:
1310     case RISCV::LW:
1311     case RISCV::LBU:
1312     case RISCV::LHU:
1313     case RISCV::LWU:
1314     case RISCV::LD:
1315     case RISCV::FLH:
1316     case RISCV::FLW:
1317     case RISCV::FLD:
1318       BaseOpIdx = 0;
1319       OffsetOpIdx = 1;
1320       break;
1321     case RISCV::SB:
1322     case RISCV::SH:
1323     case RISCV::SW:
1324     case RISCV::SD:
1325     case RISCV::FSH:
1326     case RISCV::FSW:
1327     case RISCV::FSD:
1328       BaseOpIdx = 1;
1329       OffsetOpIdx = 2;
1330       break;
1331     }
1332 
1333     if (!isa<ConstantSDNode>(N->getOperand(OffsetOpIdx)))
1334       continue;
1335 
1336     SDValue Base = N->getOperand(BaseOpIdx);
1337 
1338     // If the base is an ADDI, we can merge it in to the load/store.
1339     if (!Base.isMachineOpcode() || Base.getMachineOpcode() != RISCV::ADDI)
1340       continue;
1341 
1342     SDValue ImmOperand = Base.getOperand(1);
1343     uint64_t Offset2 = N->getConstantOperandVal(OffsetOpIdx);
1344 
1345     if (auto *Const = dyn_cast<ConstantSDNode>(ImmOperand)) {
1346       int64_t Offset1 = Const->getSExtValue();
1347       int64_t CombinedOffset = Offset1 + Offset2;
1348       if (!isInt<12>(CombinedOffset))
1349         continue;
1350       ImmOperand = CurDAG->getTargetConstant(CombinedOffset, SDLoc(ImmOperand),
1351                                              ImmOperand.getValueType());
1352     } else if (auto *GA = dyn_cast<GlobalAddressSDNode>(ImmOperand)) {
1353       // If the off1 in (addi base, off1) is a global variable's address (its
1354       // low part, really), then we can rely on the alignment of that variable
1355       // to provide a margin of safety before off1 can overflow the 12 bits.
1356       // Check if off2 falls within that margin; if so off1+off2 can't overflow.
1357       const DataLayout &DL = CurDAG->getDataLayout();
1358       Align Alignment = GA->getGlobal()->getPointerAlignment(DL);
1359       if (Offset2 != 0 && Alignment <= Offset2)
1360         continue;
1361       int64_t Offset1 = GA->getOffset();
1362       int64_t CombinedOffset = Offset1 + Offset2;
1363       ImmOperand = CurDAG->getTargetGlobalAddress(
1364           GA->getGlobal(), SDLoc(ImmOperand), ImmOperand.getValueType(),
1365           CombinedOffset, GA->getTargetFlags());
1366     } else if (auto *CP = dyn_cast<ConstantPoolSDNode>(ImmOperand)) {
1367       // Ditto.
1368       Align Alignment = CP->getAlign();
1369       if (Offset2 != 0 && Alignment <= Offset2)
1370         continue;
1371       int64_t Offset1 = CP->getOffset();
1372       int64_t CombinedOffset = Offset1 + Offset2;
1373       ImmOperand = CurDAG->getTargetConstantPool(
1374           CP->getConstVal(), ImmOperand.getValueType(), CP->getAlign(),
1375           CombinedOffset, CP->getTargetFlags());
1376     } else {
1377       continue;
1378     }
1379 
1380     LLVM_DEBUG(dbgs() << "Folding add-immediate into mem-op:\nBase:    ");
1381     LLVM_DEBUG(Base->dump(CurDAG));
1382     LLVM_DEBUG(dbgs() << "\nN: ");
1383     LLVM_DEBUG(N->dump(CurDAG));
1384     LLVM_DEBUG(dbgs() << "\n");
1385 
1386     // Modify the offset operand of the load/store.
1387     if (BaseOpIdx == 0) // Load
1388       CurDAG->UpdateNodeOperands(N, Base.getOperand(0), ImmOperand,
1389                                  N->getOperand(2));
1390     else // Store
1391       CurDAG->UpdateNodeOperands(N, N->getOperand(0), Base.getOperand(0),
1392                                  ImmOperand, N->getOperand(3));
1393 
1394     // The add-immediate may now be dead, in which case remove it.
1395     if (Base.getNode()->use_empty())
1396       CurDAG->RemoveDeadNode(Base.getNode());
1397   }
1398 }
1399 
1400 // This pass converts a legalized DAG into a RISCV-specific DAG, ready
1401 // for instruction scheduling.
1402 FunctionPass *llvm::createRISCVISelDag(RISCVTargetMachine &TM) {
1403   return new RISCVDAGToDAGISel(TM);
1404 }
1405