1 //===-- RISCVISelDAGToDAG.cpp - A dag to dag inst selector for RISCV ------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines an instruction selector for the RISCV target.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "RISCVISelDAGToDAG.h"
14 #include "MCTargetDesc/RISCVMCTargetDesc.h"
15 #include "MCTargetDesc/RISCVMatInt.h"
16 #include "RISCVISelLowering.h"
17 #include "llvm/CodeGen/MachineFrameInfo.h"
18 #include "llvm/IR/IntrinsicsRISCV.h"
19 #include "llvm/Support/Alignment.h"
20 #include "llvm/Support/Debug.h"
21 #include "llvm/Support/KnownBits.h"
22 #include "llvm/Support/MathExtras.h"
23 #include "llvm/Support/raw_ostream.h"
24 
25 using namespace llvm;
26 
27 #define DEBUG_TYPE "riscv-isel"
28 
29 namespace llvm {
30 namespace RISCV {
31 #define GET_RISCVVSSEGTable_IMPL
32 #define GET_RISCVVLSEGTable_IMPL
33 #define GET_RISCVVLXSEGTable_IMPL
34 #define GET_RISCVVSXSEGTable_IMPL
35 #define GET_RISCVVLETable_IMPL
36 #define GET_RISCVVSETable_IMPL
37 #define GET_RISCVVLXTable_IMPL
38 #define GET_RISCVVSXTable_IMPL
39 #include "RISCVGenSearchableTables.inc"
40 } // namespace RISCV
41 } // namespace llvm
42 
43 void RISCVDAGToDAGISel::PostprocessISelDAG() {
44   doPeepholeLoadStoreADDI();
45 }
46 
47 static SDNode *selectImm(SelectionDAG *CurDAG, const SDLoc &DL, int64_t Imm,
48                          MVT XLenVT) {
49   RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Imm, XLenVT == MVT::i64);
50 
51   SDNode *Result = nullptr;
52   SDValue SrcReg = CurDAG->getRegister(RISCV::X0, XLenVT);
53   for (RISCVMatInt::Inst &Inst : Seq) {
54     SDValue SDImm = CurDAG->getTargetConstant(Inst.Imm, DL, XLenVT);
55     if (Inst.Opc == RISCV::LUI)
56       Result = CurDAG->getMachineNode(RISCV::LUI, DL, XLenVT, SDImm);
57     else
58       Result = CurDAG->getMachineNode(Inst.Opc, DL, XLenVT, SrcReg, SDImm);
59 
60     // Only the first instruction has X0 as its source.
61     SrcReg = SDValue(Result, 0);
62   }
63 
64   return Result;
65 }
66 
67 static SDValue createTupleImpl(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
68                                unsigned RegClassID, unsigned SubReg0) {
69   assert(Regs.size() >= 2 && Regs.size() <= 8);
70 
71   SDLoc DL(Regs[0]);
72   SmallVector<SDValue, 8> Ops;
73 
74   Ops.push_back(CurDAG.getTargetConstant(RegClassID, DL, MVT::i32));
75 
76   for (unsigned I = 0; I < Regs.size(); ++I) {
77     Ops.push_back(Regs[I]);
78     Ops.push_back(CurDAG.getTargetConstant(SubReg0 + I, DL, MVT::i32));
79   }
80   SDNode *N =
81       CurDAG.getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops);
82   return SDValue(N, 0);
83 }
84 
85 static SDValue createM1Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
86                              unsigned NF) {
87   static const unsigned RegClassIDs[] = {
88       RISCV::VRN2M1RegClassID, RISCV::VRN3M1RegClassID, RISCV::VRN4M1RegClassID,
89       RISCV::VRN5M1RegClassID, RISCV::VRN6M1RegClassID, RISCV::VRN7M1RegClassID,
90       RISCV::VRN8M1RegClassID};
91 
92   return createTupleImpl(CurDAG, Regs, RegClassIDs[NF - 2], RISCV::sub_vrm1_0);
93 }
94 
95 static SDValue createM2Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
96                              unsigned NF) {
97   static const unsigned RegClassIDs[] = {RISCV::VRN2M2RegClassID,
98                                          RISCV::VRN3M2RegClassID,
99                                          RISCV::VRN4M2RegClassID};
100 
101   return createTupleImpl(CurDAG, Regs, RegClassIDs[NF - 2], RISCV::sub_vrm2_0);
102 }
103 
104 static SDValue createM4Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
105                              unsigned NF) {
106   return createTupleImpl(CurDAG, Regs, RISCV::VRN2M4RegClassID,
107                          RISCV::sub_vrm4_0);
108 }
109 
110 static SDValue createTuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
111                            unsigned NF, RISCVVLMUL LMUL) {
112   switch (LMUL) {
113   default:
114     llvm_unreachable("Invalid LMUL.");
115   case RISCVVLMUL::LMUL_F8:
116   case RISCVVLMUL::LMUL_F4:
117   case RISCVVLMUL::LMUL_F2:
118   case RISCVVLMUL::LMUL_1:
119     return createM1Tuple(CurDAG, Regs, NF);
120   case RISCVVLMUL::LMUL_2:
121     return createM2Tuple(CurDAG, Regs, NF);
122   case RISCVVLMUL::LMUL_4:
123     return createM4Tuple(CurDAG, Regs, NF);
124   }
125 }
126 
127 void RISCVDAGToDAGISel::addVectorLoadStoreOperands(
128     SDNode *Node, unsigned SEWImm, const SDLoc &DL, unsigned CurOp,
129     bool IsMasked, bool IsStridedOrIndexed, SmallVectorImpl<SDValue> &Operands,
130     MVT *IndexVT) {
131   SDValue Chain = Node->getOperand(0);
132   SDValue Glue;
133 
134   SDValue Base;
135   SelectBaseAddr(Node->getOperand(CurOp++), Base);
136   Operands.push_back(Base); // Base pointer.
137 
138   if (IsStridedOrIndexed) {
139     Operands.push_back(Node->getOperand(CurOp++)); // Index.
140     if (IndexVT)
141       *IndexVT = Operands.back()->getSimpleValueType(0);
142   }
143 
144   if (IsMasked) {
145     // Mask needs to be copied to V0.
146     SDValue Mask = Node->getOperand(CurOp++);
147     Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
148     Glue = Chain.getValue(1);
149     Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
150   }
151   SDValue VL;
152   selectVLOp(Node->getOperand(CurOp++), VL);
153   Operands.push_back(VL);
154 
155   MVT XLenVT = Subtarget->getXLenVT();
156   SDValue SEW = CurDAG->getTargetConstant(SEWImm, DL, XLenVT);
157   Operands.push_back(SEW);
158 
159   Operands.push_back(Chain); // Chain.
160   if (Glue)
161     Operands.push_back(Glue);
162 }
163 
164 void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, bool IsMasked,
165                                     bool IsStrided) {
166   SDLoc DL(Node);
167   unsigned NF = Node->getNumValues() - 1;
168   MVT VT = Node->getSimpleValueType(0);
169   unsigned ScalarSize = VT.getScalarSizeInBits();
170   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
171 
172   unsigned CurOp = 2;
173   SmallVector<SDValue, 8> Operands;
174   if (IsMasked) {
175     SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
176                                  Node->op_begin() + CurOp + NF);
177     SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
178     Operands.push_back(MaskedOff);
179     CurOp += NF;
180   }
181 
182   addVectorLoadStoreOperands(Node, ScalarSize, DL, CurOp, IsMasked, IsStrided,
183                              Operands);
184 
185   const RISCV::VLSEGPseudo *P =
186       RISCV::getVLSEGPseudo(NF, IsMasked, IsStrided, /*FF*/ false, ScalarSize,
187                             static_cast<unsigned>(LMUL));
188   MachineSDNode *Load =
189       CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
190 
191   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
192     CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
193 
194   SDValue SuperReg = SDValue(Load, 0);
195   for (unsigned I = 0; I < NF; ++I) {
196     unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
197     ReplaceUses(SDValue(Node, I),
198                 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
199   }
200 
201   ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
202   CurDAG->RemoveDeadNode(Node);
203 }
204 
205 void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node, bool IsMasked) {
206   SDLoc DL(Node);
207   unsigned NF = Node->getNumValues() - 2; // Do not count VL and Chain.
208   MVT VT = Node->getSimpleValueType(0);
209   MVT XLenVT = Subtarget->getXLenVT();
210   unsigned ScalarSize = VT.getScalarSizeInBits();
211   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
212 
213   unsigned CurOp = 2;
214   SmallVector<SDValue, 7> Operands;
215   if (IsMasked) {
216     SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
217                                  Node->op_begin() + CurOp + NF);
218     SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
219     Operands.push_back(MaskedOff);
220     CurOp += NF;
221   }
222 
223   addVectorLoadStoreOperands(Node, ScalarSize, DL, CurOp, IsMasked,
224                              /*IsStridedOrIndexed*/ false, Operands);
225 
226   const RISCV::VLSEGPseudo *P =
227       RISCV::getVLSEGPseudo(NF, IsMasked, /*Strided*/ false, /*FF*/ true,
228                             ScalarSize, static_cast<unsigned>(LMUL));
229   MachineSDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped,
230                                                MVT::Other, MVT::Glue, Operands);
231   SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT,
232                                           /*Glue*/ SDValue(Load, 2));
233 
234   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
235     CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
236 
237   SDValue SuperReg = SDValue(Load, 0);
238   for (unsigned I = 0; I < NF; ++I) {
239     unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
240     ReplaceUses(SDValue(Node, I),
241                 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
242   }
243 
244   ReplaceUses(SDValue(Node, NF), SDValue(ReadVL, 0));   // VL
245   ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 1)); // Chain
246   CurDAG->RemoveDeadNode(Node);
247 }
248 
249 void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, bool IsMasked,
250                                      bool IsOrdered) {
251   SDLoc DL(Node);
252   unsigned NF = Node->getNumValues() - 1;
253   MVT VT = Node->getSimpleValueType(0);
254   unsigned ScalarSize = VT.getScalarSizeInBits();
255   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
256 
257   unsigned CurOp = 2;
258   SmallVector<SDValue, 8> Operands;
259   if (IsMasked) {
260     SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
261                                  Node->op_begin() + CurOp + NF);
262     SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
263     Operands.push_back(MaskedOff);
264     CurOp += NF;
265   }
266 
267   MVT IndexVT;
268   addVectorLoadStoreOperands(Node, ScalarSize, DL, CurOp, IsMasked,
269                              /*IsStridedOrIndexed*/ true, Operands, &IndexVT);
270 
271   assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
272          "Element count mismatch");
273 
274   RISCVVLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
275   unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
276   const RISCV::VLXSEGPseudo *P = RISCV::getVLXSEGPseudo(
277       NF, IsMasked, IsOrdered, IndexScalarSize, static_cast<unsigned>(LMUL),
278       static_cast<unsigned>(IndexLMUL));
279   MachineSDNode *Load =
280       CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
281 
282   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
283     CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
284 
285   SDValue SuperReg = SDValue(Load, 0);
286   for (unsigned I = 0; I < NF; ++I) {
287     unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
288     ReplaceUses(SDValue(Node, I),
289                 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
290   }
291 
292   ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
293   CurDAG->RemoveDeadNode(Node);
294 }
295 
296 void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, bool IsMasked,
297                                     bool IsStrided) {
298   SDLoc DL(Node);
299   unsigned NF = Node->getNumOperands() - 4;
300   if (IsStrided)
301     NF--;
302   if (IsMasked)
303     NF--;
304   MVT VT = Node->getOperand(2)->getSimpleValueType(0);
305   unsigned ScalarSize = VT.getScalarSizeInBits();
306   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
307   SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
308   SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
309 
310   SmallVector<SDValue, 8> Operands;
311   Operands.push_back(StoreVal);
312   unsigned CurOp = 2 + NF;
313 
314   addVectorLoadStoreOperands(Node, ScalarSize, DL, CurOp, IsMasked, IsStrided,
315                              Operands);
316 
317   const RISCV::VSSEGPseudo *P = RISCV::getVSSEGPseudo(
318       NF, IsMasked, IsStrided, ScalarSize, static_cast<unsigned>(LMUL));
319   MachineSDNode *Store =
320       CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
321 
322   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
323     CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
324 
325   ReplaceNode(Node, Store);
326 }
327 
328 void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, bool IsMasked,
329                                      bool IsOrdered) {
330   SDLoc DL(Node);
331   unsigned NF = Node->getNumOperands() - 5;
332   if (IsMasked)
333     --NF;
334   MVT VT = Node->getOperand(2)->getSimpleValueType(0);
335   unsigned ScalarSize = VT.getScalarSizeInBits();
336   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
337   SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
338   SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
339 
340   SmallVector<SDValue, 8> Operands;
341   Operands.push_back(StoreVal);
342   unsigned CurOp = 2 + NF;
343 
344   MVT IndexVT;
345   addVectorLoadStoreOperands(Node, ScalarSize, DL, CurOp, IsMasked,
346                              /*IsStridedOrIndexed*/ true, Operands, &IndexVT);
347 
348   assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
349          "Element count mismatch");
350 
351   RISCVVLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
352   unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
353   const RISCV::VSXSEGPseudo *P = RISCV::getVSXSEGPseudo(
354       NF, IsMasked, IsOrdered, IndexScalarSize, static_cast<unsigned>(LMUL),
355       static_cast<unsigned>(IndexLMUL));
356   MachineSDNode *Store =
357       CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
358 
359   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
360     CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
361 
362   ReplaceNode(Node, Store);
363 }
364 
365 
366 void RISCVDAGToDAGISel::Select(SDNode *Node) {
367   // If we have a custom node, we have already selected.
368   if (Node->isMachineOpcode()) {
369     LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n");
370     Node->setNodeId(-1);
371     return;
372   }
373 
374   // Instruction Selection not handled by the auto-generated tablegen selection
375   // should be handled here.
376   unsigned Opcode = Node->getOpcode();
377   MVT XLenVT = Subtarget->getXLenVT();
378   SDLoc DL(Node);
379   MVT VT = Node->getSimpleValueType(0);
380 
381   switch (Opcode) {
382   case ISD::ADD: {
383     // Optimize (add r, imm) to (addi (addi r, imm0) imm1) if applicable. The
384     // immediate must be in specific ranges and have a single use.
385     if (auto *ConstOp = dyn_cast<ConstantSDNode>(Node->getOperand(1))) {
386       if (!(ConstOp->hasOneUse()))
387         break;
388       // The imm must be in range [-4096,-2049] or [2048,4094].
389       int64_t Imm = ConstOp->getSExtValue();
390       if (!(-4096 <= Imm && Imm <= -2049) && !(2048 <= Imm && Imm <= 4094))
391         break;
392       // Break the imm to imm0+imm1.
393       const SDValue ImmOp0 = CurDAG->getTargetConstant(Imm - Imm / 2, DL, VT);
394       const SDValue ImmOp1 = CurDAG->getTargetConstant(Imm / 2, DL, VT);
395       auto *NodeAddi0 = CurDAG->getMachineNode(RISCV::ADDI, DL, VT,
396                                                Node->getOperand(0), ImmOp0);
397       auto *NodeAddi1 = CurDAG->getMachineNode(RISCV::ADDI, DL, VT,
398                                                SDValue(NodeAddi0, 0), ImmOp1);
399       ReplaceNode(Node, NodeAddi1);
400       return;
401     }
402     break;
403   }
404   case ISD::Constant: {
405     auto *ConstNode = cast<ConstantSDNode>(Node);
406     if (VT == XLenVT && ConstNode->isNullValue()) {
407       SDValue New =
408           CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, RISCV::X0, XLenVT);
409       ReplaceNode(Node, New.getNode());
410       return;
411     }
412     ReplaceNode(Node, selectImm(CurDAG, DL, ConstNode->getSExtValue(), XLenVT));
413     return;
414   }
415   case ISD::FrameIndex: {
416     SDValue Imm = CurDAG->getTargetConstant(0, DL, XLenVT);
417     int FI = cast<FrameIndexSDNode>(Node)->getIndex();
418     SDValue TFI = CurDAG->getTargetFrameIndex(FI, VT);
419     ReplaceNode(Node, CurDAG->getMachineNode(RISCV::ADDI, DL, VT, TFI, Imm));
420     return;
421   }
422   case ISD::SRL: {
423     // Optimize (srl (and X, 0xffff), C) -> (srli (slli X, 16), 16 + C).
424     // Taking into account that the 0xffff may have had lower bits unset by
425     // SimplifyDemandedBits. This avoids materializing the 0xffff immediate.
426     // This pattern occurs when type legalizing i16 right shifts.
427     // FIXME: This could be extended to other AND masks.
428     auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
429     if (N1C) {
430       uint64_t ShAmt = N1C->getZExtValue();
431       SDValue N0 = Node->getOperand(0);
432       if (ShAmt < 16 && N0.getOpcode() == ISD::AND && N0.hasOneUse() &&
433           isa<ConstantSDNode>(N0.getOperand(1))) {
434         uint64_t Mask = N0.getConstantOperandVal(1);
435         Mask |= maskTrailingOnes<uint64_t>(ShAmt);
436         if (Mask == 0xffff) {
437           unsigned SLLOpc = Subtarget->is64Bit() ? RISCV::SLLIW : RISCV::SLLI;
438           unsigned SRLOpc = Subtarget->is64Bit() ? RISCV::SRLIW : RISCV::SRLI;
439           SDNode *SLLI =
440               CurDAG->getMachineNode(SLLOpc, DL, VT, N0->getOperand(0),
441                                      CurDAG->getTargetConstant(16, DL, VT));
442           SDNode *SRLI = CurDAG->getMachineNode(
443               SRLOpc, DL, VT, SDValue(SLLI, 0),
444               CurDAG->getTargetConstant(16 + ShAmt, DL, VT));
445           ReplaceNode(Node, SRLI);
446           return;
447         }
448       }
449     }
450 
451     break;
452   }
453   case ISD::INTRINSIC_W_CHAIN: {
454     unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
455     switch (IntNo) {
456       // By default we do not custom select any intrinsic.
457     default:
458       break;
459 
460     case Intrinsic::riscv_vsetvli:
461     case Intrinsic::riscv_vsetvlimax: {
462       if (!Subtarget->hasStdExtV())
463         break;
464 
465       bool VLMax = IntNo == Intrinsic::riscv_vsetvlimax;
466       unsigned Offset = VLMax ? 2 : 3;
467 
468       assert(Node->getNumOperands() == Offset + 2 &&
469              "Unexpected number of operands");
470 
471       RISCVVSEW VSEW =
472           static_cast<RISCVVSEW>(Node->getConstantOperandVal(Offset) & 0x7);
473       RISCVVLMUL VLMul = static_cast<RISCVVLMUL>(
474           Node->getConstantOperandVal(Offset + 1) & 0x7);
475 
476       unsigned VTypeI = RISCVVType::encodeVTYPE(
477           VLMul, VSEW, /*TailAgnostic*/ true, /*MaskAgnostic*/ false);
478       SDValue VTypeIOp = CurDAG->getTargetConstant(VTypeI, DL, XLenVT);
479 
480       SDValue VLOperand;
481       if (VLMax) {
482         VLOperand = CurDAG->getRegister(RISCV::X0, XLenVT);
483       } else {
484         VLOperand = Node->getOperand(2);
485 
486         if (auto *C = dyn_cast<ConstantSDNode>(VLOperand)) {
487           uint64_t AVL = C->getZExtValue();
488           if (isUInt<5>(AVL)) {
489             SDValue VLImm = CurDAG->getTargetConstant(AVL, DL, XLenVT);
490             ReplaceNode(
491                 Node, CurDAG->getMachineNode(RISCV::PseudoVSETIVLI, DL, XLenVT,
492                                              MVT::Other, VLImm, VTypeIOp,
493                                              /* Chain */ Node->getOperand(0)));
494             return;
495           }
496         }
497       }
498 
499       ReplaceNode(Node,
500                   CurDAG->getMachineNode(RISCV::PseudoVSETVLI, DL, XLenVT,
501                                          MVT::Other, VLOperand, VTypeIOp,
502                                          /* Chain */ Node->getOperand(0)));
503       return;
504     }
505     case Intrinsic::riscv_vlseg2:
506     case Intrinsic::riscv_vlseg3:
507     case Intrinsic::riscv_vlseg4:
508     case Intrinsic::riscv_vlseg5:
509     case Intrinsic::riscv_vlseg6:
510     case Intrinsic::riscv_vlseg7:
511     case Intrinsic::riscv_vlseg8: {
512       selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false);
513       return;
514     }
515     case Intrinsic::riscv_vlseg2_mask:
516     case Intrinsic::riscv_vlseg3_mask:
517     case Intrinsic::riscv_vlseg4_mask:
518     case Intrinsic::riscv_vlseg5_mask:
519     case Intrinsic::riscv_vlseg6_mask:
520     case Intrinsic::riscv_vlseg7_mask:
521     case Intrinsic::riscv_vlseg8_mask: {
522       selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false);
523       return;
524     }
525     case Intrinsic::riscv_vlsseg2:
526     case Intrinsic::riscv_vlsseg3:
527     case Intrinsic::riscv_vlsseg4:
528     case Intrinsic::riscv_vlsseg5:
529     case Intrinsic::riscv_vlsseg6:
530     case Intrinsic::riscv_vlsseg7:
531     case Intrinsic::riscv_vlsseg8: {
532       selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true);
533       return;
534     }
535     case Intrinsic::riscv_vlsseg2_mask:
536     case Intrinsic::riscv_vlsseg3_mask:
537     case Intrinsic::riscv_vlsseg4_mask:
538     case Intrinsic::riscv_vlsseg5_mask:
539     case Intrinsic::riscv_vlsseg6_mask:
540     case Intrinsic::riscv_vlsseg7_mask:
541     case Intrinsic::riscv_vlsseg8_mask: {
542       selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true);
543       return;
544     }
545     case Intrinsic::riscv_vloxseg2:
546     case Intrinsic::riscv_vloxseg3:
547     case Intrinsic::riscv_vloxseg4:
548     case Intrinsic::riscv_vloxseg5:
549     case Intrinsic::riscv_vloxseg6:
550     case Intrinsic::riscv_vloxseg7:
551     case Intrinsic::riscv_vloxseg8:
552       selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true);
553       return;
554     case Intrinsic::riscv_vluxseg2:
555     case Intrinsic::riscv_vluxseg3:
556     case Intrinsic::riscv_vluxseg4:
557     case Intrinsic::riscv_vluxseg5:
558     case Intrinsic::riscv_vluxseg6:
559     case Intrinsic::riscv_vluxseg7:
560     case Intrinsic::riscv_vluxseg8:
561       selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false);
562       return;
563     case Intrinsic::riscv_vloxseg2_mask:
564     case Intrinsic::riscv_vloxseg3_mask:
565     case Intrinsic::riscv_vloxseg4_mask:
566     case Intrinsic::riscv_vloxseg5_mask:
567     case Intrinsic::riscv_vloxseg6_mask:
568     case Intrinsic::riscv_vloxseg7_mask:
569     case Intrinsic::riscv_vloxseg8_mask:
570       selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true);
571       return;
572     case Intrinsic::riscv_vluxseg2_mask:
573     case Intrinsic::riscv_vluxseg3_mask:
574     case Intrinsic::riscv_vluxseg4_mask:
575     case Intrinsic::riscv_vluxseg5_mask:
576     case Intrinsic::riscv_vluxseg6_mask:
577     case Intrinsic::riscv_vluxseg7_mask:
578     case Intrinsic::riscv_vluxseg8_mask:
579       selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false);
580       return;
581     case Intrinsic::riscv_vlseg8ff:
582     case Intrinsic::riscv_vlseg7ff:
583     case Intrinsic::riscv_vlseg6ff:
584     case Intrinsic::riscv_vlseg5ff:
585     case Intrinsic::riscv_vlseg4ff:
586     case Intrinsic::riscv_vlseg3ff:
587     case Intrinsic::riscv_vlseg2ff: {
588       selectVLSEGFF(Node, /*IsMasked*/ false);
589       return;
590     }
591     case Intrinsic::riscv_vlseg8ff_mask:
592     case Intrinsic::riscv_vlseg7ff_mask:
593     case Intrinsic::riscv_vlseg6ff_mask:
594     case Intrinsic::riscv_vlseg5ff_mask:
595     case Intrinsic::riscv_vlseg4ff_mask:
596     case Intrinsic::riscv_vlseg3ff_mask:
597     case Intrinsic::riscv_vlseg2ff_mask: {
598       selectVLSEGFF(Node, /*IsMasked*/ true);
599       return;
600     }
601     case Intrinsic::riscv_vloxei:
602     case Intrinsic::riscv_vloxei_mask:
603     case Intrinsic::riscv_vluxei:
604     case Intrinsic::riscv_vluxei_mask: {
605       bool IsMasked = IntNo == Intrinsic::riscv_vloxei_mask ||
606                       IntNo == Intrinsic::riscv_vluxei_mask;
607       bool IsOrdered = IntNo == Intrinsic::riscv_vloxei ||
608                        IntNo == Intrinsic::riscv_vloxei_mask;
609 
610       MVT VT = Node->getSimpleValueType(0);
611       unsigned ScalarSize = VT.getScalarSizeInBits();
612 
613       unsigned CurOp = 2;
614       SmallVector<SDValue, 8> Operands;
615       if (IsMasked)
616         Operands.push_back(Node->getOperand(CurOp++));
617 
618       MVT IndexVT;
619       addVectorLoadStoreOperands(Node, ScalarSize, DL, CurOp, IsMasked,
620                                  /*IsStridedOrIndexed*/ true, Operands,
621                                  &IndexVT);
622 
623       assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
624              "Element count mismatch");
625 
626       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
627       RISCVVLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
628       unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
629       const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
630           IsMasked, IsOrdered, IndexScalarSize, static_cast<unsigned>(LMUL),
631           static_cast<unsigned>(IndexLMUL));
632       MachineSDNode *Load =
633           CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
634 
635       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
636         CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
637 
638       ReplaceNode(Node, Load);
639       return;
640     }
641     case Intrinsic::riscv_vle1:
642     case Intrinsic::riscv_vle:
643     case Intrinsic::riscv_vle_mask:
644     case Intrinsic::riscv_vlse:
645     case Intrinsic::riscv_vlse_mask: {
646       bool IsMasked = IntNo == Intrinsic::riscv_vle_mask ||
647                       IntNo == Intrinsic::riscv_vlse_mask;
648       bool IsStrided =
649           IntNo == Intrinsic::riscv_vlse || IntNo == Intrinsic::riscv_vlse_mask;
650 
651       MVT VT = Node->getSimpleValueType(0);
652       unsigned ScalarSize = VT.getScalarSizeInBits();
653       // VLE1 uses an SEW of 8.
654       unsigned SEWImm = (IntNo == Intrinsic::riscv_vle1) ? 8 : ScalarSize;
655 
656       unsigned CurOp = 2;
657       SmallVector<SDValue, 8> Operands;
658       if (IsMasked)
659         Operands.push_back(Node->getOperand(CurOp++));
660 
661       addVectorLoadStoreOperands(Node, SEWImm, DL, CurOp, IsMasked, IsStrided,
662                                  Operands);
663 
664       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
665       const RISCV::VLEPseudo *P =
666           RISCV::getVLEPseudo(IsMasked, IsStrided, /*FF*/ false, ScalarSize,
667                               static_cast<unsigned>(LMUL));
668       MachineSDNode *Load =
669           CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
670 
671       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
672         CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
673 
674       ReplaceNode(Node, Load);
675       return;
676     }
677     case Intrinsic::riscv_vleff:
678     case Intrinsic::riscv_vleff_mask: {
679       bool IsMasked = IntNo == Intrinsic::riscv_vleff_mask;
680 
681       MVT VT = Node->getSimpleValueType(0);
682       unsigned ScalarSize = VT.getScalarSizeInBits();
683 
684       unsigned CurOp = 2;
685       SmallVector<SDValue, 7> Operands;
686       if (IsMasked)
687         Operands.push_back(Node->getOperand(CurOp++));
688 
689       addVectorLoadStoreOperands(Node, ScalarSize, DL, CurOp, IsMasked,
690                                  /*IsStridedOrIndexed*/ false, Operands);
691 
692       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
693       const RISCV::VLEPseudo *P =
694           RISCV::getVLEPseudo(IsMasked, /*Strided*/ false, /*FF*/ true,
695                               ScalarSize, static_cast<unsigned>(LMUL));
696       MachineSDNode *Load =
697           CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0),
698                                  MVT::Other, MVT::Glue, Operands);
699       SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT,
700                                               /*Glue*/ SDValue(Load, 2));
701 
702       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
703         CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
704 
705       ReplaceUses(SDValue(Node, 0), SDValue(Load, 0));
706       ReplaceUses(SDValue(Node, 1), SDValue(ReadVL, 0)); // VL
707       ReplaceUses(SDValue(Node, 2), SDValue(Load, 1));   // Chain
708       CurDAG->RemoveDeadNode(Node);
709       return;
710     }
711     }
712     break;
713   }
714   case ISD::INTRINSIC_VOID: {
715     unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
716     switch (IntNo) {
717     case Intrinsic::riscv_vsseg2:
718     case Intrinsic::riscv_vsseg3:
719     case Intrinsic::riscv_vsseg4:
720     case Intrinsic::riscv_vsseg5:
721     case Intrinsic::riscv_vsseg6:
722     case Intrinsic::riscv_vsseg7:
723     case Intrinsic::riscv_vsseg8: {
724       selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false);
725       return;
726     }
727     case Intrinsic::riscv_vsseg2_mask:
728     case Intrinsic::riscv_vsseg3_mask:
729     case Intrinsic::riscv_vsseg4_mask:
730     case Intrinsic::riscv_vsseg5_mask:
731     case Intrinsic::riscv_vsseg6_mask:
732     case Intrinsic::riscv_vsseg7_mask:
733     case Intrinsic::riscv_vsseg8_mask: {
734       selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false);
735       return;
736     }
737     case Intrinsic::riscv_vssseg2:
738     case Intrinsic::riscv_vssseg3:
739     case Intrinsic::riscv_vssseg4:
740     case Intrinsic::riscv_vssseg5:
741     case Intrinsic::riscv_vssseg6:
742     case Intrinsic::riscv_vssseg7:
743     case Intrinsic::riscv_vssseg8: {
744       selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true);
745       return;
746     }
747     case Intrinsic::riscv_vssseg2_mask:
748     case Intrinsic::riscv_vssseg3_mask:
749     case Intrinsic::riscv_vssseg4_mask:
750     case Intrinsic::riscv_vssseg5_mask:
751     case Intrinsic::riscv_vssseg6_mask:
752     case Intrinsic::riscv_vssseg7_mask:
753     case Intrinsic::riscv_vssseg8_mask: {
754       selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true);
755       return;
756     }
757     case Intrinsic::riscv_vsoxseg2:
758     case Intrinsic::riscv_vsoxseg3:
759     case Intrinsic::riscv_vsoxseg4:
760     case Intrinsic::riscv_vsoxseg5:
761     case Intrinsic::riscv_vsoxseg6:
762     case Intrinsic::riscv_vsoxseg7:
763     case Intrinsic::riscv_vsoxseg8:
764       selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true);
765       return;
766     case Intrinsic::riscv_vsuxseg2:
767     case Intrinsic::riscv_vsuxseg3:
768     case Intrinsic::riscv_vsuxseg4:
769     case Intrinsic::riscv_vsuxseg5:
770     case Intrinsic::riscv_vsuxseg6:
771     case Intrinsic::riscv_vsuxseg7:
772     case Intrinsic::riscv_vsuxseg8:
773       selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false);
774       return;
775     case Intrinsic::riscv_vsoxseg2_mask:
776     case Intrinsic::riscv_vsoxseg3_mask:
777     case Intrinsic::riscv_vsoxseg4_mask:
778     case Intrinsic::riscv_vsoxseg5_mask:
779     case Intrinsic::riscv_vsoxseg6_mask:
780     case Intrinsic::riscv_vsoxseg7_mask:
781     case Intrinsic::riscv_vsoxseg8_mask:
782       selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true);
783       return;
784     case Intrinsic::riscv_vsuxseg2_mask:
785     case Intrinsic::riscv_vsuxseg3_mask:
786     case Intrinsic::riscv_vsuxseg4_mask:
787     case Intrinsic::riscv_vsuxseg5_mask:
788     case Intrinsic::riscv_vsuxseg6_mask:
789     case Intrinsic::riscv_vsuxseg7_mask:
790     case Intrinsic::riscv_vsuxseg8_mask:
791       selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false);
792       return;
793     case Intrinsic::riscv_vsoxei:
794     case Intrinsic::riscv_vsoxei_mask:
795     case Intrinsic::riscv_vsuxei:
796     case Intrinsic::riscv_vsuxei_mask: {
797       bool IsMasked = IntNo == Intrinsic::riscv_vsoxei_mask ||
798                       IntNo == Intrinsic::riscv_vsuxei_mask;
799       bool IsOrdered = IntNo == Intrinsic::riscv_vsoxei ||
800                        IntNo == Intrinsic::riscv_vsoxei_mask;
801 
802       MVT VT = Node->getOperand(2)->getSimpleValueType(0);
803       unsigned ScalarSize = VT.getScalarSizeInBits();
804 
805       unsigned CurOp = 2;
806       SmallVector<SDValue, 8> Operands;
807       Operands.push_back(Node->getOperand(CurOp++)); // Store value.
808 
809       MVT IndexVT;
810       addVectorLoadStoreOperands(Node, ScalarSize, DL, CurOp, IsMasked,
811                                  /*IsStridedOrIndexed*/ true, Operands,
812                                  &IndexVT);
813 
814       assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
815              "Element count mismatch");
816 
817       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
818       RISCVVLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
819       unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
820       const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
821           IsMasked, IsOrdered, IndexScalarSize, static_cast<unsigned>(LMUL),
822           static_cast<unsigned>(IndexLMUL));
823       MachineSDNode *Store =
824           CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
825 
826       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
827         CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
828 
829       ReplaceNode(Node, Store);
830       return;
831     }
832     case Intrinsic::riscv_vse1:
833     case Intrinsic::riscv_vse:
834     case Intrinsic::riscv_vse_mask:
835     case Intrinsic::riscv_vsse:
836     case Intrinsic::riscv_vsse_mask: {
837       bool IsMasked = IntNo == Intrinsic::riscv_vse_mask ||
838                       IntNo == Intrinsic::riscv_vsse_mask;
839       bool IsStrided =
840           IntNo == Intrinsic::riscv_vsse || IntNo == Intrinsic::riscv_vsse_mask;
841 
842       MVT VT = Node->getOperand(2)->getSimpleValueType(0);
843       unsigned ScalarSize = VT.getScalarSizeInBits();
844       // VSE1 uses an SEW of 8.
845       unsigned SEWImm = (IntNo == Intrinsic::riscv_vse1) ? 8 : ScalarSize;
846 
847       unsigned CurOp = 2;
848       SmallVector<SDValue, 8> Operands;
849       Operands.push_back(Node->getOperand(CurOp++)); // Store value.
850 
851       addVectorLoadStoreOperands(Node, SEWImm, DL, CurOp, IsMasked, IsStrided,
852                                  Operands);
853 
854       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
855       const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(
856           IsMasked, IsStrided, ScalarSize, static_cast<unsigned>(LMUL));
857       MachineSDNode *Store =
858           CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
859       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
860         CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
861 
862       ReplaceNode(Node, Store);
863       return;
864     }
865     }
866     break;
867   }
868   case ISD::BITCAST: {
869     MVT SrcVT = Node->getOperand(0).getSimpleValueType();
870     // Just drop bitcasts between vectors if both are fixed or both are
871     // scalable.
872     if ((VT.isScalableVector() && SrcVT.isScalableVector()) ||
873         (VT.isFixedLengthVector() && SrcVT.isFixedLengthVector())) {
874       ReplaceUses(SDValue(Node, 0), Node->getOperand(0));
875       CurDAG->RemoveDeadNode(Node);
876       return;
877     }
878     break;
879   }
880   case ISD::INSERT_SUBVECTOR: {
881     SDValue V = Node->getOperand(0);
882     SDValue SubV = Node->getOperand(1);
883     SDLoc DL(SubV);
884     auto Idx = Node->getConstantOperandVal(2);
885     MVT SubVecVT = SubV.getSimpleValueType();
886 
887     MVT SubVecContainerVT = SubVecVT;
888     // Establish the correct scalable-vector types for any fixed-length type.
889     if (SubVecVT.isFixedLengthVector())
890       SubVecContainerVT = RISCVTargetLowering::getContainerForFixedLengthVector(
891           *CurDAG, SubVecVT, *Subtarget);
892     if (VT.isFixedLengthVector())
893       VT = RISCVTargetLowering::getContainerForFixedLengthVector(*CurDAG, VT,
894                                                                  *Subtarget);
895 
896     const auto *TRI = Subtarget->getRegisterInfo();
897     unsigned SubRegIdx;
898     std::tie(SubRegIdx, Idx) =
899         RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
900             VT, SubVecContainerVT, Idx, TRI);
901 
902     // If the Idx hasn't been completely eliminated then this is a subvector
903     // insert which doesn't naturally align to a vector register. These must
904     // be handled using instructions to manipulate the vector registers.
905     if (Idx != 0)
906       break;
907 
908     RISCVVLMUL SubVecLMUL = RISCVTargetLowering::getLMUL(SubVecContainerVT);
909     bool IsSubVecPartReg = SubVecLMUL == RISCVVLMUL::LMUL_F2 ||
910                            SubVecLMUL == RISCVVLMUL::LMUL_F4 ||
911                            SubVecLMUL == RISCVVLMUL::LMUL_F8;
912     (void)IsSubVecPartReg; // Silence unused variable warning without asserts.
913     assert((!IsSubVecPartReg || V.isUndef()) &&
914            "Expecting lowering to have created legal INSERT_SUBVECTORs when "
915            "the subvector is smaller than a full-sized register");
916 
917     // If we haven't set a SubRegIdx, then we must be going between
918     // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy.
919     if (SubRegIdx == RISCV::NoSubRegister) {
920       unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT);
921       assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) ==
922                  InRegClassID &&
923              "Unexpected subvector extraction");
924       SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT);
925       SDNode *NewNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
926                                                DL, VT, SubV, RC);
927       ReplaceNode(Node, NewNode);
928       return;
929     }
930 
931     SDValue Insert = CurDAG->getTargetInsertSubreg(SubRegIdx, DL, VT, V, SubV);
932     ReplaceNode(Node, Insert.getNode());
933     return;
934   }
935   case ISD::EXTRACT_SUBVECTOR: {
936     SDValue V = Node->getOperand(0);
937     auto Idx = Node->getConstantOperandVal(1);
938     MVT InVT = V.getSimpleValueType();
939     SDLoc DL(V);
940 
941     MVT SubVecContainerVT = VT;
942     // Establish the correct scalable-vector types for any fixed-length type.
943     if (VT.isFixedLengthVector())
944       SubVecContainerVT = RISCVTargetLowering::getContainerForFixedLengthVector(
945           *CurDAG, VT, *Subtarget);
946     if (InVT.isFixedLengthVector())
947       InVT = RISCVTargetLowering::getContainerForFixedLengthVector(
948           *CurDAG, InVT, *Subtarget);
949 
950     const auto *TRI = Subtarget->getRegisterInfo();
951     unsigned SubRegIdx;
952     std::tie(SubRegIdx, Idx) =
953         RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
954             InVT, SubVecContainerVT, Idx, TRI);
955 
956     // If the Idx hasn't been completely eliminated then this is a subvector
957     // extract which doesn't naturally align to a vector register. These must
958     // be handled using instructions to manipulate the vector registers.
959     if (Idx != 0)
960       break;
961 
962     // If we haven't set a SubRegIdx, then we must be going between
963     // equally-sized LMUL types (e.g. VR -> VR). This can be done as a copy.
964     if (SubRegIdx == RISCV::NoSubRegister) {
965       unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(InVT);
966       assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) ==
967                  InRegClassID &&
968              "Unexpected subvector extraction");
969       SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT);
970       SDNode *NewNode =
971           CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, DL, VT, V, RC);
972       ReplaceNode(Node, NewNode);
973       return;
974     }
975 
976     SDValue Extract = CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, V);
977     ReplaceNode(Node, Extract.getNode());
978     return;
979   }
980   }
981 
982   // Select the default instruction.
983   SelectCode(Node);
984 }
985 
986 bool RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand(
987     const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {
988   switch (ConstraintID) {
989   case InlineAsm::Constraint_m:
990     // We just support simple memory operands that have a single address
991     // operand and need no special handling.
992     OutOps.push_back(Op);
993     return false;
994   case InlineAsm::Constraint_A:
995     OutOps.push_back(Op);
996     return false;
997   default:
998     break;
999   }
1000 
1001   return true;
1002 }
1003 
1004 bool RISCVDAGToDAGISel::SelectAddrFI(SDValue Addr, SDValue &Base) {
1005   if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
1006     Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT());
1007     return true;
1008   }
1009   return false;
1010 }
1011 
1012 bool RISCVDAGToDAGISel::SelectBaseAddr(SDValue Addr, SDValue &Base) {
1013   // If this is FrameIndex, select it directly. Otherwise just let it get
1014   // selected to a register independently.
1015   if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr))
1016     Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT());
1017   else
1018     Base = Addr;
1019   return true;
1020 }
1021 
1022 bool RISCVDAGToDAGISel::selectShiftMask(SDValue N, unsigned ShiftWidth,
1023                                         SDValue &ShAmt) {
1024   // Shift instructions on RISCV only read the lower 5 or 6 bits of the shift
1025   // amount. If there is an AND on the shift amount, we can bypass it if it
1026   // doesn't affect any of those bits.
1027   if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(N.getOperand(1))) {
1028     const APInt &AndMask = N->getConstantOperandAPInt(1);
1029 
1030     // Since the max shift amount is a power of 2 we can subtract 1 to make a
1031     // mask that covers the bits needed to represent all shift amounts.
1032     assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!");
1033     APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
1034 
1035     if (ShMask.isSubsetOf(AndMask)) {
1036       ShAmt = N.getOperand(0);
1037       return true;
1038     }
1039 
1040     // SimplifyDemandedBits may have optimized the mask so try restoring any
1041     // bits that are known zero.
1042     KnownBits Known = CurDAG->computeKnownBits(N->getOperand(0));
1043     if (ShMask.isSubsetOf(AndMask | Known.Zero)) {
1044       ShAmt = N.getOperand(0);
1045       return true;
1046     }
1047   }
1048 
1049   ShAmt = N;
1050   return true;
1051 }
1052 
1053 bool RISCVDAGToDAGISel::selectSExti32(SDValue N, SDValue &Val) {
1054   if (N.getOpcode() == ISD::SIGN_EXTEND_INREG &&
1055       cast<VTSDNode>(N.getOperand(1))->getVT() == MVT::i32) {
1056     Val = N.getOperand(0);
1057     return true;
1058   }
1059   // FIXME: Should we just call computeNumSignBits here?
1060   if (N.getOpcode() == ISD::AssertSext &&
1061       cast<VTSDNode>(N->getOperand(1))->getVT().bitsLE(MVT::i32)) {
1062     Val = N;
1063     return true;
1064   }
1065   if (N.getOpcode() == ISD::AssertZext &&
1066       cast<VTSDNode>(N->getOperand(1))->getVT().bitsLT(MVT::i32)) {
1067     Val = N;
1068     return true;
1069   }
1070 
1071   return false;
1072 }
1073 
1074 bool RISCVDAGToDAGISel::selectZExti32(SDValue N, SDValue &Val) {
1075   if (N.getOpcode() == ISD::AND) {
1076     auto *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
1077     if (C && C->getZExtValue() == UINT64_C(0xFFFFFFFF)) {
1078       Val = N.getOperand(0);
1079       return true;
1080     }
1081   }
1082   // FIXME: Should we just call computeKnownBits here?
1083   if (N.getOpcode() == ISD::AssertZext &&
1084       cast<VTSDNode>(N->getOperand(1))->getVT().bitsLE(MVT::i32)) {
1085     Val = N;
1086     return true;
1087   }
1088 
1089   return false;
1090 }
1091 
1092 // Check that it is a SLLIUW (Shift Logical Left Immediate Unsigned i32
1093 // on RV64).
1094 // SLLIUW is the same as SLLI except for the fact that it clears the bits
1095 // XLEN-1:32 of the input RS1 before shifting.
1096 // A PatFrag has already checked that it has the right structure:
1097 //
1098 //  (AND (SHL RS1, VC2), VC1)
1099 //
1100 // We check that VC2, the shamt is less than 32, otherwise the pattern is
1101 // exactly the same as SLLI and we give priority to that.
1102 // Eventually we check that VC1, the mask used to clear the upper 32 bits
1103 // of RS1, is correct:
1104 //
1105 //  VC1 == (0xFFFFFFFF << VC2)
1106 //
1107 bool RISCVDAGToDAGISel::MatchSLLIUW(SDNode *N) const {
1108   assert(N->getOpcode() == ISD::AND);
1109   assert(N->getOperand(0).getOpcode() == ISD::SHL);
1110   assert(isa<ConstantSDNode>(N->getOperand(1)));
1111   assert(isa<ConstantSDNode>(N->getOperand(0).getOperand(1)));
1112 
1113   // The IsRV64 predicate is checked after PatFrag predicates so we can get
1114   // here even on RV32.
1115   if (!Subtarget->is64Bit())
1116     return false;
1117 
1118   SDValue Shl = N->getOperand(0);
1119   uint64_t VC1 = N->getConstantOperandVal(1);
1120   uint64_t VC2 = Shl.getConstantOperandVal(1);
1121 
1122   // Immediate range should be enforced by uimm5 predicate.
1123   assert(VC2 < 32 && "Unexpected immediate");
1124   return (VC1 >> VC2) == UINT64_C(0xFFFFFFFF);
1125 }
1126 
1127 // X0 has special meaning for vsetvl/vsetvli.
1128 //  rd | rs1 |   AVL value | Effect on vl
1129 //--------------------------------------------------------------
1130 // !X0 |  X0 |       VLMAX | Set vl to VLMAX
1131 //  X0 |  X0 | Value in vl | Keep current vl, just change vtype.
1132 bool RISCVDAGToDAGISel::selectVLOp(SDValue N, SDValue &VL) {
1133   // If the VL value is a constant 0, manually select it to an ADDI with 0
1134   // immediate to prevent the default selection path from matching it to X0.
1135   auto *C = dyn_cast<ConstantSDNode>(N);
1136   if (C && C->isNullValue())
1137     VL = SDValue(selectImm(CurDAG, SDLoc(N), 0, Subtarget->getXLenVT()), 0);
1138   else
1139     VL = N;
1140 
1141   return true;
1142 }
1143 
1144 bool RISCVDAGToDAGISel::selectVSplat(SDValue N, SDValue &SplatVal) {
1145   if (N.getOpcode() != ISD::SPLAT_VECTOR &&
1146       N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64 &&
1147       N.getOpcode() != RISCVISD::VMV_V_X_VL)
1148     return false;
1149   SplatVal = N.getOperand(0);
1150   return true;
1151 }
1152 
1153 bool RISCVDAGToDAGISel::selectVSplatSimm5(SDValue N, SDValue &SplatVal) {
1154   if ((N.getOpcode() != ISD::SPLAT_VECTOR &&
1155        N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64 &&
1156        N.getOpcode() != RISCVISD::VMV_V_X_VL) ||
1157       !isa<ConstantSDNode>(N.getOperand(0)))
1158     return false;
1159 
1160   int64_t SplatImm = cast<ConstantSDNode>(N.getOperand(0))->getSExtValue();
1161 
1162   // Both ISD::SPLAT_VECTOR and RISCVISD::SPLAT_VECTOR_I64 share semantics when
1163   // the operand type is wider than the resulting vector element type: an
1164   // implicit truncation first takes place. Therefore, perform a manual
1165   // truncation/sign-extension in order to ignore any truncated bits and catch
1166   // any zero-extended immediate.
1167   // For example, we wish to match (i8 -1) -> (XLenVT 255) as a simm5 by first
1168   // sign-extending to (XLenVT -1).
1169   MVT XLenVT = Subtarget->getXLenVT();
1170   assert(XLenVT == N.getOperand(0).getSimpleValueType() &&
1171          "Unexpected splat operand type");
1172   MVT EltVT = N.getSimpleValueType().getVectorElementType();
1173   if (EltVT.bitsLT(XLenVT)) {
1174     SplatImm = SignExtend64(SplatImm, EltVT.getSizeInBits());
1175   }
1176 
1177   if (!isInt<5>(SplatImm))
1178     return false;
1179 
1180   SplatVal = CurDAG->getTargetConstant(SplatImm, SDLoc(N), XLenVT);
1181   return true;
1182 }
1183 
1184 bool RISCVDAGToDAGISel::selectVSplatUimm5(SDValue N, SDValue &SplatVal) {
1185   if ((N.getOpcode() != ISD::SPLAT_VECTOR &&
1186        N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64 &&
1187        N.getOpcode() != RISCVISD::VMV_V_X_VL) ||
1188       !isa<ConstantSDNode>(N.getOperand(0)))
1189     return false;
1190 
1191   int64_t SplatImm = cast<ConstantSDNode>(N.getOperand(0))->getSExtValue();
1192 
1193   if (!isUInt<5>(SplatImm))
1194     return false;
1195 
1196   SplatVal =
1197       CurDAG->getTargetConstant(SplatImm, SDLoc(N), Subtarget->getXLenVT());
1198 
1199   return true;
1200 }
1201 
1202 bool RISCVDAGToDAGISel::selectRVVSimm5(SDValue N, unsigned Width,
1203                                        SDValue &Imm) {
1204   if (auto *C = dyn_cast<ConstantSDNode>(N)) {
1205     int64_t ImmVal = SignExtend64(C->getSExtValue(), Width);
1206 
1207     if (!isInt<5>(ImmVal))
1208       return false;
1209 
1210     Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), Subtarget->getXLenVT());
1211     return true;
1212   }
1213 
1214   return false;
1215 }
1216 
1217 bool RISCVDAGToDAGISel::selectRVVUimm5(SDValue N, unsigned Width,
1218                                        SDValue &Imm) {
1219   if (auto *C = dyn_cast<ConstantSDNode>(N)) {
1220     int64_t ImmVal = C->getSExtValue();
1221 
1222     if (!isUInt<5>(ImmVal))
1223       return false;
1224 
1225     Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), Subtarget->getXLenVT());
1226     return true;
1227   }
1228 
1229   return false;
1230 }
1231 
1232 // Merge an ADDI into the offset of a load/store instruction where possible.
1233 // (load (addi base, off1), off2) -> (load base, off1+off2)
1234 // (store val, (addi base, off1), off2) -> (store val, base, off1+off2)
1235 // This is possible when off1+off2 fits a 12-bit immediate.
1236 void RISCVDAGToDAGISel::doPeepholeLoadStoreADDI() {
1237   SelectionDAG::allnodes_iterator Position(CurDAG->getRoot().getNode());
1238   ++Position;
1239 
1240   while (Position != CurDAG->allnodes_begin()) {
1241     SDNode *N = &*--Position;
1242     // Skip dead nodes and any non-machine opcodes.
1243     if (N->use_empty() || !N->isMachineOpcode())
1244       continue;
1245 
1246     int OffsetOpIdx;
1247     int BaseOpIdx;
1248 
1249     // Only attempt this optimisation for I-type loads and S-type stores.
1250     switch (N->getMachineOpcode()) {
1251     default:
1252       continue;
1253     case RISCV::LB:
1254     case RISCV::LH:
1255     case RISCV::LW:
1256     case RISCV::LBU:
1257     case RISCV::LHU:
1258     case RISCV::LWU:
1259     case RISCV::LD:
1260     case RISCV::FLH:
1261     case RISCV::FLW:
1262     case RISCV::FLD:
1263       BaseOpIdx = 0;
1264       OffsetOpIdx = 1;
1265       break;
1266     case RISCV::SB:
1267     case RISCV::SH:
1268     case RISCV::SW:
1269     case RISCV::SD:
1270     case RISCV::FSH:
1271     case RISCV::FSW:
1272     case RISCV::FSD:
1273       BaseOpIdx = 1;
1274       OffsetOpIdx = 2;
1275       break;
1276     }
1277 
1278     if (!isa<ConstantSDNode>(N->getOperand(OffsetOpIdx)))
1279       continue;
1280 
1281     SDValue Base = N->getOperand(BaseOpIdx);
1282 
1283     // If the base is an ADDI, we can merge it in to the load/store.
1284     if (!Base.isMachineOpcode() || Base.getMachineOpcode() != RISCV::ADDI)
1285       continue;
1286 
1287     SDValue ImmOperand = Base.getOperand(1);
1288     uint64_t Offset2 = N->getConstantOperandVal(OffsetOpIdx);
1289 
1290     if (auto *Const = dyn_cast<ConstantSDNode>(ImmOperand)) {
1291       int64_t Offset1 = Const->getSExtValue();
1292       int64_t CombinedOffset = Offset1 + Offset2;
1293       if (!isInt<12>(CombinedOffset))
1294         continue;
1295       ImmOperand = CurDAG->getTargetConstant(CombinedOffset, SDLoc(ImmOperand),
1296                                              ImmOperand.getValueType());
1297     } else if (auto *GA = dyn_cast<GlobalAddressSDNode>(ImmOperand)) {
1298       // If the off1 in (addi base, off1) is a global variable's address (its
1299       // low part, really), then we can rely on the alignment of that variable
1300       // to provide a margin of safety before off1 can overflow the 12 bits.
1301       // Check if off2 falls within that margin; if so off1+off2 can't overflow.
1302       const DataLayout &DL = CurDAG->getDataLayout();
1303       Align Alignment = GA->getGlobal()->getPointerAlignment(DL);
1304       if (Offset2 != 0 && Alignment <= Offset2)
1305         continue;
1306       int64_t Offset1 = GA->getOffset();
1307       int64_t CombinedOffset = Offset1 + Offset2;
1308       ImmOperand = CurDAG->getTargetGlobalAddress(
1309           GA->getGlobal(), SDLoc(ImmOperand), ImmOperand.getValueType(),
1310           CombinedOffset, GA->getTargetFlags());
1311     } else if (auto *CP = dyn_cast<ConstantPoolSDNode>(ImmOperand)) {
1312       // Ditto.
1313       Align Alignment = CP->getAlign();
1314       if (Offset2 != 0 && Alignment <= Offset2)
1315         continue;
1316       int64_t Offset1 = CP->getOffset();
1317       int64_t CombinedOffset = Offset1 + Offset2;
1318       ImmOperand = CurDAG->getTargetConstantPool(
1319           CP->getConstVal(), ImmOperand.getValueType(), CP->getAlign(),
1320           CombinedOffset, CP->getTargetFlags());
1321     } else {
1322       continue;
1323     }
1324 
1325     LLVM_DEBUG(dbgs() << "Folding add-immediate into mem-op:\nBase:    ");
1326     LLVM_DEBUG(Base->dump(CurDAG));
1327     LLVM_DEBUG(dbgs() << "\nN: ");
1328     LLVM_DEBUG(N->dump(CurDAG));
1329     LLVM_DEBUG(dbgs() << "\n");
1330 
1331     // Modify the offset operand of the load/store.
1332     if (BaseOpIdx == 0) // Load
1333       CurDAG->UpdateNodeOperands(N, Base.getOperand(0), ImmOperand,
1334                                  N->getOperand(2));
1335     else // Store
1336       CurDAG->UpdateNodeOperands(N, N->getOperand(0), Base.getOperand(0),
1337                                  ImmOperand, N->getOperand(3));
1338 
1339     // The add-immediate may now be dead, in which case remove it.
1340     if (Base.getNode()->use_empty())
1341       CurDAG->RemoveDeadNode(Base.getNode());
1342   }
1343 }
1344 
1345 // This pass converts a legalized DAG into a RISCV-specific DAG, ready
1346 // for instruction scheduling.
1347 FunctionPass *llvm::createRISCVISelDag(RISCVTargetMachine &TM) {
1348   return new RISCVDAGToDAGISel(TM);
1349 }
1350