1 //===-- RISCVISelDAGToDAG.cpp - A dag to dag inst selector for RISCV ------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines an instruction selector for the RISCV target.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "RISCVISelDAGToDAG.h"
14 #include "MCTargetDesc/RISCVMCTargetDesc.h"
15 #include "MCTargetDesc/RISCVMatInt.h"
16 #include "RISCVISelLowering.h"
17 #include "llvm/CodeGen/MachineFrameInfo.h"
18 #include "llvm/IR/IntrinsicsRISCV.h"
19 #include "llvm/Support/Alignment.h"
20 #include "llvm/Support/Debug.h"
21 #include "llvm/Support/KnownBits.h"
22 #include "llvm/Support/MathExtras.h"
23 #include "llvm/Support/raw_ostream.h"
24 
25 using namespace llvm;
26 
27 #define DEBUG_TYPE "riscv-isel"
28 
29 namespace llvm {
30 namespace RISCV {
31 #define GET_RISCVVSSEGTable_IMPL
32 #define GET_RISCVVLSEGTable_IMPL
33 #define GET_RISCVVLXSEGTable_IMPL
34 #define GET_RISCVVSXSEGTable_IMPL
35 #define GET_RISCVVLETable_IMPL
36 #define GET_RISCVVSETable_IMPL
37 #define GET_RISCVVLXTable_IMPL
38 #define GET_RISCVVSXTable_IMPL
39 #include "RISCVGenSearchableTables.inc"
40 } // namespace RISCV
41 } // namespace llvm
42 
43 void RISCVDAGToDAGISel::PostprocessISelDAG() {
44   doPeepholeLoadStoreADDI();
45 }
46 
47 static SDNode *selectImm(SelectionDAG *CurDAG, const SDLoc &DL, int64_t Imm,
48                          MVT XLenVT) {
49   RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Imm, XLenVT == MVT::i64);
50 
51   SDNode *Result = nullptr;
52   SDValue SrcReg = CurDAG->getRegister(RISCV::X0, XLenVT);
53   for (RISCVMatInt::Inst &Inst : Seq) {
54     SDValue SDImm = CurDAG->getTargetConstant(Inst.Imm, DL, XLenVT);
55     if (Inst.Opc == RISCV::LUI)
56       Result = CurDAG->getMachineNode(RISCV::LUI, DL, XLenVT, SDImm);
57     else
58       Result = CurDAG->getMachineNode(Inst.Opc, DL, XLenVT, SrcReg, SDImm);
59 
60     // Only the first instruction has X0 as its source.
61     SrcReg = SDValue(Result, 0);
62   }
63 
64   return Result;
65 }
66 
67 static SDValue createTupleImpl(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
68                                unsigned RegClassID, unsigned SubReg0) {
69   assert(Regs.size() >= 2 && Regs.size() <= 8);
70 
71   SDLoc DL(Regs[0]);
72   SmallVector<SDValue, 8> Ops;
73 
74   Ops.push_back(CurDAG.getTargetConstant(RegClassID, DL, MVT::i32));
75 
76   for (unsigned I = 0; I < Regs.size(); ++I) {
77     Ops.push_back(Regs[I]);
78     Ops.push_back(CurDAG.getTargetConstant(SubReg0 + I, DL, MVT::i32));
79   }
80   SDNode *N =
81       CurDAG.getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops);
82   return SDValue(N, 0);
83 }
84 
85 static SDValue createM1Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
86                              unsigned NF) {
87   static const unsigned RegClassIDs[] = {
88       RISCV::VRN2M1RegClassID, RISCV::VRN3M1RegClassID, RISCV::VRN4M1RegClassID,
89       RISCV::VRN5M1RegClassID, RISCV::VRN6M1RegClassID, RISCV::VRN7M1RegClassID,
90       RISCV::VRN8M1RegClassID};
91 
92   return createTupleImpl(CurDAG, Regs, RegClassIDs[NF - 2], RISCV::sub_vrm1_0);
93 }
94 
95 static SDValue createM2Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
96                              unsigned NF) {
97   static const unsigned RegClassIDs[] = {RISCV::VRN2M2RegClassID,
98                                          RISCV::VRN3M2RegClassID,
99                                          RISCV::VRN4M2RegClassID};
100 
101   return createTupleImpl(CurDAG, Regs, RegClassIDs[NF - 2], RISCV::sub_vrm2_0);
102 }
103 
104 static SDValue createM4Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
105                              unsigned NF) {
106   return createTupleImpl(CurDAG, Regs, RISCV::VRN2M4RegClassID,
107                          RISCV::sub_vrm4_0);
108 }
109 
110 static SDValue createTuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
111                            unsigned NF, RISCVVLMUL LMUL) {
112   switch (LMUL) {
113   default:
114     llvm_unreachable("Invalid LMUL.");
115   case RISCVVLMUL::LMUL_F8:
116   case RISCVVLMUL::LMUL_F4:
117   case RISCVVLMUL::LMUL_F2:
118   case RISCVVLMUL::LMUL_1:
119     return createM1Tuple(CurDAG, Regs, NF);
120   case RISCVVLMUL::LMUL_2:
121     return createM2Tuple(CurDAG, Regs, NF);
122   case RISCVVLMUL::LMUL_4:
123     return createM4Tuple(CurDAG, Regs, NF);
124   }
125 }
126 
127 void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, bool IsMasked,
128                                     bool IsStrided) {
129   SDLoc DL(Node);
130   unsigned NF = Node->getNumValues() - 1;
131   MVT VT = Node->getSimpleValueType(0);
132   unsigned ScalarSize = VT.getScalarSizeInBits();
133   MVT XLenVT = Subtarget->getXLenVT();
134   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
135   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
136 
137   SDValue Chain = Node->getOperand(0);
138   SDValue Glue;
139 
140   unsigned CurOp = 2;
141   SmallVector<SDValue, 8> Operands;
142   if (IsMasked) {
143     SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
144                                  Node->op_begin() + CurOp + NF);
145     SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
146     Operands.push_back(MaskedOff);
147     CurOp += NF;
148   }
149   SDValue Base;
150   SelectBaseAddr(Node->getOperand(CurOp++), Base);
151   Operands.push_back(Base); // Base pointer.
152   if (IsStrided)
153     Operands.push_back(Node->getOperand(CurOp++)); // Stride.
154   if (IsMasked) {
155     // Mask needs to be copied to V0.
156     SDValue Mask = Node->getOperand(CurOp++);
157     Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
158     Glue = Chain.getValue(1);
159     Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
160   }
161   SDValue VL;
162   selectVLOp(Node->getOperand(CurOp++), VL);
163   Operands.push_back(VL);
164   Operands.push_back(SEW);
165   Operands.push_back(Chain); // Chain.
166   if (Glue)
167     Operands.push_back(Glue);
168   const RISCV::VLSEGPseudo *P =
169       RISCV::getVLSEGPseudo(NF, IsMasked, IsStrided, /*FF*/ false, ScalarSize,
170                             static_cast<unsigned>(LMUL));
171   MachineSDNode *Load =
172       CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
173 
174   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
175     CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
176 
177   SDValue SuperReg = SDValue(Load, 0);
178   for (unsigned I = 0; I < NF; ++I) {
179     unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
180     ReplaceUses(SDValue(Node, I),
181                 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
182   }
183 
184   ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
185   CurDAG->RemoveDeadNode(Node);
186 }
187 
188 void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node, bool IsMasked) {
189   SDLoc DL(Node);
190   unsigned NF = Node->getNumValues() - 2; // Do not count VL and Chain.
191   MVT VT = Node->getSimpleValueType(0);
192   MVT XLenVT = Subtarget->getXLenVT();
193   unsigned ScalarSize = VT.getScalarSizeInBits();
194   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
195   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
196 
197   SDValue Chain = Node->getOperand(0);
198   SDValue Glue;
199 
200   unsigned CurOp = 2;
201   SmallVector<SDValue, 7> Operands;
202   if (IsMasked) {
203     SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
204                                  Node->op_begin() + CurOp + NF);
205     SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
206     Operands.push_back(MaskedOff);
207     CurOp += NF;
208   }
209   SDValue Base;
210   SelectBaseAddr(Node->getOperand(CurOp++), Base);
211   Operands.push_back(Base); // Base pointer.
212   if (IsMasked) {
213     // Mask needs to be copied to V0.
214     SDValue Mask = Node->getOperand(CurOp++);
215     Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
216     Glue = Chain.getValue(1);
217     Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
218   }
219   SDValue VL;
220   selectVLOp(Node->getOperand(CurOp++), VL);
221   Operands.push_back(VL);
222   Operands.push_back(SEW);
223   Operands.push_back(Chain); // Chain.
224   if (Glue)
225     Operands.push_back(Glue);
226   const RISCV::VLSEGPseudo *P =
227       RISCV::getVLSEGPseudo(NF, IsMasked, /*Strided*/ false, /*FF*/ true,
228                             ScalarSize, static_cast<unsigned>(LMUL));
229   MachineSDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped,
230                                                MVT::Other, MVT::Glue, Operands);
231   SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT,
232                                           /*Glue*/ SDValue(Load, 2));
233 
234   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
235     CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
236 
237   SDValue SuperReg = SDValue(Load, 0);
238   for (unsigned I = 0; I < NF; ++I) {
239     unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
240     ReplaceUses(SDValue(Node, I),
241                 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
242   }
243 
244   ReplaceUses(SDValue(Node, NF), SDValue(ReadVL, 0));   // VL
245   ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 1)); // Chain
246   CurDAG->RemoveDeadNode(Node);
247 }
248 
249 void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, bool IsMasked,
250                                      bool IsOrdered) {
251   SDLoc DL(Node);
252   unsigned NF = Node->getNumValues() - 1;
253   MVT VT = Node->getSimpleValueType(0);
254   unsigned ScalarSize = VT.getScalarSizeInBits();
255   MVT XLenVT = Subtarget->getXLenVT();
256   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
257   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
258 
259   SDValue Chain = Node->getOperand(0);
260   SDValue Glue;
261 
262   unsigned CurOp = 2;
263   SmallVector<SDValue, 8> Operands;
264   if (IsMasked) {
265     SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
266                                  Node->op_begin() + CurOp + NF);
267     SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
268     Operands.push_back(MaskedOff);
269     CurOp += NF;
270   }
271   SDValue Base;
272   SelectBaseAddr(Node->getOperand(CurOp++), Base);
273   Operands.push_back(Base); // Base pointer.
274   Operands.push_back(Node->getOperand(CurOp++)); // Index.
275   MVT IndexVT = Operands.back()->getSimpleValueType(0);
276   if (IsMasked) {
277     // Mask needs to be copied to V0.
278     SDValue Mask = Node->getOperand(CurOp++);
279     Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
280     Glue = Chain.getValue(1);
281     Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
282   }
283   SDValue VL;
284   selectVLOp(Node->getOperand(CurOp++), VL);
285   Operands.push_back(VL);
286   Operands.push_back(SEW);
287   Operands.push_back(Chain); // Chain.
288   if (Glue)
289     Operands.push_back(Glue);
290 
291   assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
292          "Element count mismatch");
293 
294   RISCVVLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
295   unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
296   const RISCV::VLXSEGPseudo *P = RISCV::getVLXSEGPseudo(
297       NF, IsMasked, IsOrdered, IndexScalarSize, static_cast<unsigned>(LMUL),
298       static_cast<unsigned>(IndexLMUL));
299   MachineSDNode *Load =
300       CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
301 
302   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
303     CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
304 
305   SDValue SuperReg = SDValue(Load, 0);
306   for (unsigned I = 0; I < NF; ++I) {
307     unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
308     ReplaceUses(SDValue(Node, I),
309                 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
310   }
311 
312   ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
313   CurDAG->RemoveDeadNode(Node);
314 }
315 
316 void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, bool IsMasked,
317                                     bool IsStrided) {
318   SDLoc DL(Node);
319   unsigned NF = Node->getNumOperands() - 4;
320   if (IsStrided)
321     NF--;
322   if (IsMasked)
323     NF--;
324   MVT VT = Node->getOperand(2)->getSimpleValueType(0);
325   unsigned ScalarSize = VT.getScalarSizeInBits();
326   MVT XLenVT = Subtarget->getXLenVT();
327   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
328   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
329   SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
330   SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
331 
332   SDValue Chain = Node->getOperand(0);
333   SDValue Glue;
334 
335   SmallVector<SDValue, 8> Operands;
336   Operands.push_back(StoreVal);
337   unsigned CurOp = 2 + NF;
338   SDValue Base;
339   SelectBaseAddr(Node->getOperand(CurOp++), Base);
340   Operands.push_back(Base); // Base pointer.
341   if (IsStrided)
342     Operands.push_back(Node->getOperand(CurOp++)); // Stride.
343   if (IsMasked) {
344     // Mask needs to be copied to V0.
345     SDValue Mask = Node->getOperand(CurOp++);
346     Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
347     Glue = Chain.getValue(1);
348     Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
349   }
350   SDValue VL;
351   selectVLOp(Node->getOperand(CurOp++), VL);
352   Operands.push_back(VL);
353   Operands.push_back(SEW);
354   Operands.push_back(Chain); // Chain.
355   if (Glue)
356     Operands.push_back(Glue);
357   const RISCV::VSSEGPseudo *P = RISCV::getVSSEGPseudo(
358       NF, IsMasked, IsStrided, ScalarSize, static_cast<unsigned>(LMUL));
359   MachineSDNode *Store =
360       CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
361 
362   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
363     CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
364 
365   ReplaceNode(Node, Store);
366 }
367 
368 void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, bool IsMasked,
369                                      bool IsOrdered) {
370   SDLoc DL(Node);
371   unsigned NF = Node->getNumOperands() - 5;
372   if (IsMasked)
373     --NF;
374   MVT VT = Node->getOperand(2)->getSimpleValueType(0);
375   unsigned ScalarSize = VT.getScalarSizeInBits();
376   MVT XLenVT = Subtarget->getXLenVT();
377   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
378   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
379   SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
380   SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
381 
382   SDValue Chain = Node->getOperand(0);
383   SDValue Glue;
384 
385   SmallVector<SDValue, 8> Operands;
386   Operands.push_back(StoreVal);
387   unsigned CurOp = 2 + NF;
388   SDValue Base;
389   SelectBaseAddr(Node->getOperand(CurOp++), Base);
390   Operands.push_back(Base); // Base pointer.
391   Operands.push_back(Node->getOperand(CurOp++)); // Index.
392   MVT IndexVT = Operands.back()->getSimpleValueType(0);
393   if (IsMasked) {
394     // Mask needs to be copied to V0.
395     SDValue Mask = Node->getOperand(CurOp++);
396     Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
397     Glue = Chain.getValue(1);
398     Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
399   }
400   SDValue VL;
401   selectVLOp(Node->getOperand(CurOp++), VL);
402   Operands.push_back(VL);
403   Operands.push_back(SEW);
404   Operands.push_back(Chain); // Chain.
405   if (Glue)
406     Operands.push_back(Glue);
407 
408   assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
409          "Element count mismatch");
410 
411   RISCVVLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
412   unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
413   const RISCV::VSXSEGPseudo *P = RISCV::getVSXSEGPseudo(
414       NF, IsMasked, IsOrdered, IndexScalarSize, static_cast<unsigned>(LMUL),
415       static_cast<unsigned>(IndexLMUL));
416   MachineSDNode *Store =
417       CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
418 
419   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
420     CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
421 
422   ReplaceNode(Node, Store);
423 }
424 
425 
426 void RISCVDAGToDAGISel::Select(SDNode *Node) {
427   // If we have a custom node, we have already selected.
428   if (Node->isMachineOpcode()) {
429     LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n");
430     Node->setNodeId(-1);
431     return;
432   }
433 
434   // Instruction Selection not handled by the auto-generated tablegen selection
435   // should be handled here.
436   unsigned Opcode = Node->getOpcode();
437   MVT XLenVT = Subtarget->getXLenVT();
438   SDLoc DL(Node);
439   MVT VT = Node->getSimpleValueType(0);
440 
441   switch (Opcode) {
442   case ISD::ADD: {
443     // Optimize (add r, imm) to (addi (addi r, imm0) imm1) if applicable. The
444     // immediate must be in specific ranges and have a single use.
445     if (auto *ConstOp = dyn_cast<ConstantSDNode>(Node->getOperand(1))) {
446       if (!(ConstOp->hasOneUse()))
447         break;
448       // The imm must be in range [-4096,-2049] or [2048,4094].
449       int64_t Imm = ConstOp->getSExtValue();
450       if (!(-4096 <= Imm && Imm <= -2049) && !(2048 <= Imm && Imm <= 4094))
451         break;
452       // Break the imm to imm0+imm1.
453       const SDValue ImmOp0 = CurDAG->getTargetConstant(Imm - Imm / 2, DL, VT);
454       const SDValue ImmOp1 = CurDAG->getTargetConstant(Imm / 2, DL, VT);
455       auto *NodeAddi0 = CurDAG->getMachineNode(RISCV::ADDI, DL, VT,
456                                                Node->getOperand(0), ImmOp0);
457       auto *NodeAddi1 = CurDAG->getMachineNode(RISCV::ADDI, DL, VT,
458                                                SDValue(NodeAddi0, 0), ImmOp1);
459       ReplaceNode(Node, NodeAddi1);
460       return;
461     }
462     break;
463   }
464   case ISD::Constant: {
465     auto *ConstNode = cast<ConstantSDNode>(Node);
466     if (VT == XLenVT && ConstNode->isNullValue()) {
467       SDValue New =
468           CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, RISCV::X0, XLenVT);
469       ReplaceNode(Node, New.getNode());
470       return;
471     }
472     ReplaceNode(Node, selectImm(CurDAG, DL, ConstNode->getSExtValue(), XLenVT));
473     return;
474     break;
475   }
476   case ISD::FrameIndex: {
477     SDValue Imm = CurDAG->getTargetConstant(0, DL, XLenVT);
478     int FI = cast<FrameIndexSDNode>(Node)->getIndex();
479     SDValue TFI = CurDAG->getTargetFrameIndex(FI, VT);
480     ReplaceNode(Node, CurDAG->getMachineNode(RISCV::ADDI, DL, VT, TFI, Imm));
481     return;
482   }
483   case ISD::SRL: {
484     // Optimize (srl (and X, 0xffff), C) -> (srli (slli X, 16), 16 + C).
485     // Taking into account that the 0xffff may have had lower bits unset by
486     // SimplifyDemandedBits. This avoids materializing the 0xffff immediate.
487     // This pattern occurs when type legalizing i16 right shifts.
488     // FIXME: This could be extended to other AND masks.
489     auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
490     if (N1C) {
491       uint64_t ShAmt = N1C->getZExtValue();
492       SDValue N0 = Node->getOperand(0);
493       if (ShAmt < 16 && N0.getOpcode() == ISD::AND && N0.hasOneUse() &&
494           isa<ConstantSDNode>(N0.getOperand(1))) {
495         uint64_t Mask = N0.getConstantOperandVal(1);
496         Mask |= maskTrailingOnes<uint64_t>(ShAmt);
497         if (Mask == 0xffff) {
498           unsigned SLLOpc = Subtarget->is64Bit() ? RISCV::SLLIW : RISCV::SLLI;
499           unsigned SRLOpc = Subtarget->is64Bit() ? RISCV::SRLIW : RISCV::SRLI;
500           SDNode *SLLI =
501               CurDAG->getMachineNode(SLLOpc, DL, VT, N0->getOperand(0),
502                                      CurDAG->getTargetConstant(16, DL, VT));
503           SDNode *SRLI = CurDAG->getMachineNode(
504               SRLOpc, DL, VT, SDValue(SLLI, 0),
505               CurDAG->getTargetConstant(16 + ShAmt, DL, VT));
506           ReplaceNode(Node, SRLI);
507           return;
508         }
509       }
510     }
511 
512     break;
513   }
514   case ISD::INTRINSIC_W_CHAIN: {
515     unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
516     switch (IntNo) {
517       // By default we do not custom select any intrinsic.
518     default:
519       break;
520 
521     case Intrinsic::riscv_vsetvli:
522     case Intrinsic::riscv_vsetvlimax: {
523       if (!Subtarget->hasStdExtV())
524         break;
525 
526       bool VLMax = IntNo == Intrinsic::riscv_vsetvlimax;
527       unsigned Offset = VLMax ? 2 : 3;
528 
529       assert(Node->getNumOperands() == Offset + 2 &&
530              "Unexpected number of operands");
531 
532       RISCVVSEW VSEW =
533           static_cast<RISCVVSEW>(Node->getConstantOperandVal(Offset) & 0x7);
534       RISCVVLMUL VLMul = static_cast<RISCVVLMUL>(
535           Node->getConstantOperandVal(Offset + 1) & 0x7);
536 
537       unsigned VTypeI = RISCVVType::encodeVTYPE(
538           VLMul, VSEW, /*TailAgnostic*/ true, /*MaskAgnostic*/ false);
539       SDValue VTypeIOp = CurDAG->getTargetConstant(VTypeI, DL, XLenVT);
540 
541       SDValue VLOperand;
542       if (VLMax) {
543         VLOperand = CurDAG->getRegister(RISCV::X0, XLenVT);
544       } else {
545         VLOperand = Node->getOperand(2);
546 
547         if (auto *C = dyn_cast<ConstantSDNode>(VLOperand)) {
548           uint64_t AVL = C->getZExtValue();
549           if (isUInt<5>(AVL)) {
550             SDValue VLImm = CurDAG->getTargetConstant(AVL, DL, XLenVT);
551             ReplaceNode(
552                 Node, CurDAG->getMachineNode(RISCV::PseudoVSETIVLI, DL, XLenVT,
553                                              MVT::Other, VLImm, VTypeIOp,
554                                              /* Chain */ Node->getOperand(0)));
555             return;
556           }
557         }
558       }
559 
560       ReplaceNode(Node,
561                   CurDAG->getMachineNode(RISCV::PseudoVSETVLI, DL, XLenVT,
562                                          MVT::Other, VLOperand, VTypeIOp,
563                                          /* Chain */ Node->getOperand(0)));
564       return;
565     }
566     case Intrinsic::riscv_vlseg2:
567     case Intrinsic::riscv_vlseg3:
568     case Intrinsic::riscv_vlseg4:
569     case Intrinsic::riscv_vlseg5:
570     case Intrinsic::riscv_vlseg6:
571     case Intrinsic::riscv_vlseg7:
572     case Intrinsic::riscv_vlseg8: {
573       selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false);
574       return;
575     }
576     case Intrinsic::riscv_vlseg2_mask:
577     case Intrinsic::riscv_vlseg3_mask:
578     case Intrinsic::riscv_vlseg4_mask:
579     case Intrinsic::riscv_vlseg5_mask:
580     case Intrinsic::riscv_vlseg6_mask:
581     case Intrinsic::riscv_vlseg7_mask:
582     case Intrinsic::riscv_vlseg8_mask: {
583       selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false);
584       return;
585     }
586     case Intrinsic::riscv_vlsseg2:
587     case Intrinsic::riscv_vlsseg3:
588     case Intrinsic::riscv_vlsseg4:
589     case Intrinsic::riscv_vlsseg5:
590     case Intrinsic::riscv_vlsseg6:
591     case Intrinsic::riscv_vlsseg7:
592     case Intrinsic::riscv_vlsseg8: {
593       selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true);
594       return;
595     }
596     case Intrinsic::riscv_vlsseg2_mask:
597     case Intrinsic::riscv_vlsseg3_mask:
598     case Intrinsic::riscv_vlsseg4_mask:
599     case Intrinsic::riscv_vlsseg5_mask:
600     case Intrinsic::riscv_vlsseg6_mask:
601     case Intrinsic::riscv_vlsseg7_mask:
602     case Intrinsic::riscv_vlsseg8_mask: {
603       selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true);
604       return;
605     }
606     case Intrinsic::riscv_vloxseg2:
607     case Intrinsic::riscv_vloxseg3:
608     case Intrinsic::riscv_vloxseg4:
609     case Intrinsic::riscv_vloxseg5:
610     case Intrinsic::riscv_vloxseg6:
611     case Intrinsic::riscv_vloxseg7:
612     case Intrinsic::riscv_vloxseg8:
613       selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true);
614       return;
615     case Intrinsic::riscv_vluxseg2:
616     case Intrinsic::riscv_vluxseg3:
617     case Intrinsic::riscv_vluxseg4:
618     case Intrinsic::riscv_vluxseg5:
619     case Intrinsic::riscv_vluxseg6:
620     case Intrinsic::riscv_vluxseg7:
621     case Intrinsic::riscv_vluxseg8:
622       selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false);
623       return;
624     case Intrinsic::riscv_vloxseg2_mask:
625     case Intrinsic::riscv_vloxseg3_mask:
626     case Intrinsic::riscv_vloxseg4_mask:
627     case Intrinsic::riscv_vloxseg5_mask:
628     case Intrinsic::riscv_vloxseg6_mask:
629     case Intrinsic::riscv_vloxseg7_mask:
630     case Intrinsic::riscv_vloxseg8_mask:
631       selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true);
632       return;
633     case Intrinsic::riscv_vluxseg2_mask:
634     case Intrinsic::riscv_vluxseg3_mask:
635     case Intrinsic::riscv_vluxseg4_mask:
636     case Intrinsic::riscv_vluxseg5_mask:
637     case Intrinsic::riscv_vluxseg6_mask:
638     case Intrinsic::riscv_vluxseg7_mask:
639     case Intrinsic::riscv_vluxseg8_mask:
640       selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false);
641       return;
642     case Intrinsic::riscv_vlseg8ff:
643     case Intrinsic::riscv_vlseg7ff:
644     case Intrinsic::riscv_vlseg6ff:
645     case Intrinsic::riscv_vlseg5ff:
646     case Intrinsic::riscv_vlseg4ff:
647     case Intrinsic::riscv_vlseg3ff:
648     case Intrinsic::riscv_vlseg2ff: {
649       selectVLSEGFF(Node, /*IsMasked*/ false);
650       return;
651     }
652     case Intrinsic::riscv_vlseg8ff_mask:
653     case Intrinsic::riscv_vlseg7ff_mask:
654     case Intrinsic::riscv_vlseg6ff_mask:
655     case Intrinsic::riscv_vlseg5ff_mask:
656     case Intrinsic::riscv_vlseg4ff_mask:
657     case Intrinsic::riscv_vlseg3ff_mask:
658     case Intrinsic::riscv_vlseg2ff_mask: {
659       selectVLSEGFF(Node, /*IsMasked*/ true);
660       return;
661     }
662     case Intrinsic::riscv_vloxei:
663     case Intrinsic::riscv_vloxei_mask:
664     case Intrinsic::riscv_vluxei:
665     case Intrinsic::riscv_vluxei_mask: {
666       bool IsMasked = IntNo == Intrinsic::riscv_vloxei_mask ||
667                       IntNo == Intrinsic::riscv_vluxei_mask;
668       bool IsOrdered = IntNo == Intrinsic::riscv_vloxei ||
669                        IntNo == Intrinsic::riscv_vloxei_mask;
670 
671       MVT VT = Node->getSimpleValueType(0);
672       unsigned ScalarSize = VT.getScalarSizeInBits();
673       MVT XLenVT = Subtarget->getXLenVT();
674       SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
675 
676       SDValue Chain = Node->getOperand(0);
677       SDValue Glue;
678 
679       unsigned CurOp = 2;
680       SmallVector<SDValue, 8> Operands;
681       if (IsMasked)
682         Operands.push_back(Node->getOperand(CurOp++));
683       SDValue Base;
684       SelectBaseAddr(Node->getOperand(CurOp++), Base);
685       Operands.push_back(Base); // Base pointer.
686       Operands.push_back(Node->getOperand(CurOp++)); // Index.
687       MVT IndexVT = Operands.back()->getSimpleValueType(0);
688       if (IsMasked) {
689         // Mask needs to be copied to V0.
690         SDValue Mask = Node->getOperand(CurOp++);
691         Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
692         Glue = Chain.getValue(1);
693         Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
694       }
695       SDValue VL;
696       selectVLOp(Node->getOperand(CurOp++), VL);
697       Operands.push_back(VL);
698       Operands.push_back(SEW);
699       Operands.push_back(Chain); // Chain.
700       if (Glue)
701         Operands.push_back(Glue);
702 
703       assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
704              "Element count mismatch");
705 
706       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
707       RISCVVLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
708       unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
709       const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
710           IsMasked, IsOrdered, IndexScalarSize, static_cast<unsigned>(LMUL),
711           static_cast<unsigned>(IndexLMUL));
712       MachineSDNode *Load =
713           CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
714 
715       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
716         CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
717 
718       ReplaceNode(Node, Load);
719       return;
720     }
721     case Intrinsic::riscv_vle1:
722     case Intrinsic::riscv_vle:
723     case Intrinsic::riscv_vle_mask:
724     case Intrinsic::riscv_vlse:
725     case Intrinsic::riscv_vlse_mask: {
726       bool IsMasked = IntNo == Intrinsic::riscv_vle_mask ||
727                       IntNo == Intrinsic::riscv_vlse_mask;
728       bool IsStrided =
729           IntNo == Intrinsic::riscv_vlse || IntNo == Intrinsic::riscv_vlse_mask;
730 
731       MVT VT = Node->getSimpleValueType(0);
732       unsigned ScalarSize = VT.getScalarSizeInBits();
733       MVT XLenVT = Subtarget->getXLenVT();
734       // VLE1 uses an SEW of 8.
735       unsigned SEWImm = (IntNo == Intrinsic::riscv_vle1) ? 8 : ScalarSize;
736       SDValue SEW = CurDAG->getTargetConstant(SEWImm, DL, XLenVT);
737 
738       SDValue Chain = Node->getOperand(0);
739       SDValue Glue;
740 
741       unsigned CurOp = 2;
742       SmallVector<SDValue, 8> Operands;
743       if (IsMasked)
744         Operands.push_back(Node->getOperand(CurOp++));
745       SDValue Base;
746       SelectBaseAddr(Node->getOperand(CurOp++), Base);
747       Operands.push_back(Base); // Base pointer.
748       if (IsStrided)
749         Operands.push_back(Node->getOperand(CurOp++)); // Stride.
750       if (IsMasked) {
751         // Mask needs to be copied to V0.
752         SDValue Mask = Node->getOperand(CurOp++);
753         Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
754         Glue = Chain.getValue(1);
755         Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
756       }
757       SDValue VL;
758       selectVLOp(Node->getOperand(CurOp++), VL);
759       Operands.push_back(VL);
760       Operands.push_back(SEW);
761       Operands.push_back(Chain); // Chain.
762       if (Glue)
763         Operands.push_back(Glue);
764 
765       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
766       const RISCV::VLEPseudo *P =
767           RISCV::getVLEPseudo(IsMasked, IsStrided, /*FF*/ false, ScalarSize,
768                               static_cast<unsigned>(LMUL));
769       MachineSDNode *Load =
770           CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
771 
772       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
773         CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
774 
775       ReplaceNode(Node, Load);
776       return;
777     }
778     case Intrinsic::riscv_vleff:
779     case Intrinsic::riscv_vleff_mask: {
780       bool IsMasked = IntNo == Intrinsic::riscv_vleff_mask;
781 
782       MVT VT = Node->getSimpleValueType(0);
783       unsigned ScalarSize = VT.getScalarSizeInBits();
784       MVT XLenVT = Subtarget->getXLenVT();
785       SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
786 
787       SDValue Chain = Node->getOperand(0);
788       SDValue Glue;
789 
790       unsigned CurOp = 2;
791       SmallVector<SDValue, 7> Operands;
792       if (IsMasked)
793         Operands.push_back(Node->getOperand(CurOp++));
794       SDValue Base;
795       SelectBaseAddr(Node->getOperand(CurOp++), Base);
796       Operands.push_back(Base); // Base pointer.
797       if (IsMasked) {
798         // Mask needs to be copied to V0.
799         SDValue Mask = Node->getOperand(CurOp++);
800         Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
801         Glue = Chain.getValue(1);
802         Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
803       }
804       SDValue VL;
805       selectVLOp(Node->getOperand(CurOp++), VL);
806       Operands.push_back(VL);
807       Operands.push_back(SEW);
808       Operands.push_back(Chain); // Chain.
809       if (Glue)
810         Operands.push_back(Glue);
811 
812       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
813       const RISCV::VLEPseudo *P =
814           RISCV::getVLEPseudo(IsMasked, /*Strided*/ false, /*FF*/ true,
815                               ScalarSize, static_cast<unsigned>(LMUL));
816       MachineSDNode *Load =
817           CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0),
818                                  MVT::Other, MVT::Glue, Operands);
819       SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT,
820                                               /*Glue*/ SDValue(Load, 2));
821 
822       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
823         CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
824 
825       ReplaceUses(SDValue(Node, 0), SDValue(Load, 0));
826       ReplaceUses(SDValue(Node, 1), SDValue(ReadVL, 0)); // VL
827       ReplaceUses(SDValue(Node, 2), SDValue(Load, 1));   // Chain
828       CurDAG->RemoveDeadNode(Node);
829       return;
830     }
831     }
832     break;
833   }
834   case ISD::INTRINSIC_VOID: {
835     unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
836     switch (IntNo) {
837     case Intrinsic::riscv_vsseg2:
838     case Intrinsic::riscv_vsseg3:
839     case Intrinsic::riscv_vsseg4:
840     case Intrinsic::riscv_vsseg5:
841     case Intrinsic::riscv_vsseg6:
842     case Intrinsic::riscv_vsseg7:
843     case Intrinsic::riscv_vsseg8: {
844       selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false);
845       return;
846     }
847     case Intrinsic::riscv_vsseg2_mask:
848     case Intrinsic::riscv_vsseg3_mask:
849     case Intrinsic::riscv_vsseg4_mask:
850     case Intrinsic::riscv_vsseg5_mask:
851     case Intrinsic::riscv_vsseg6_mask:
852     case Intrinsic::riscv_vsseg7_mask:
853     case Intrinsic::riscv_vsseg8_mask: {
854       selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false);
855       return;
856     }
857     case Intrinsic::riscv_vssseg2:
858     case Intrinsic::riscv_vssseg3:
859     case Intrinsic::riscv_vssseg4:
860     case Intrinsic::riscv_vssseg5:
861     case Intrinsic::riscv_vssseg6:
862     case Intrinsic::riscv_vssseg7:
863     case Intrinsic::riscv_vssseg8: {
864       selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true);
865       return;
866     }
867     case Intrinsic::riscv_vssseg2_mask:
868     case Intrinsic::riscv_vssseg3_mask:
869     case Intrinsic::riscv_vssseg4_mask:
870     case Intrinsic::riscv_vssseg5_mask:
871     case Intrinsic::riscv_vssseg6_mask:
872     case Intrinsic::riscv_vssseg7_mask:
873     case Intrinsic::riscv_vssseg8_mask: {
874       selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true);
875       return;
876     }
877     case Intrinsic::riscv_vsoxseg2:
878     case Intrinsic::riscv_vsoxseg3:
879     case Intrinsic::riscv_vsoxseg4:
880     case Intrinsic::riscv_vsoxseg5:
881     case Intrinsic::riscv_vsoxseg6:
882     case Intrinsic::riscv_vsoxseg7:
883     case Intrinsic::riscv_vsoxseg8:
884       selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true);
885       return;
886     case Intrinsic::riscv_vsuxseg2:
887     case Intrinsic::riscv_vsuxseg3:
888     case Intrinsic::riscv_vsuxseg4:
889     case Intrinsic::riscv_vsuxseg5:
890     case Intrinsic::riscv_vsuxseg6:
891     case Intrinsic::riscv_vsuxseg7:
892     case Intrinsic::riscv_vsuxseg8:
893       selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false);
894       return;
895     case Intrinsic::riscv_vsoxseg2_mask:
896     case Intrinsic::riscv_vsoxseg3_mask:
897     case Intrinsic::riscv_vsoxseg4_mask:
898     case Intrinsic::riscv_vsoxseg5_mask:
899     case Intrinsic::riscv_vsoxseg6_mask:
900     case Intrinsic::riscv_vsoxseg7_mask:
901     case Intrinsic::riscv_vsoxseg8_mask:
902       selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true);
903       return;
904     case Intrinsic::riscv_vsuxseg2_mask:
905     case Intrinsic::riscv_vsuxseg3_mask:
906     case Intrinsic::riscv_vsuxseg4_mask:
907     case Intrinsic::riscv_vsuxseg5_mask:
908     case Intrinsic::riscv_vsuxseg6_mask:
909     case Intrinsic::riscv_vsuxseg7_mask:
910     case Intrinsic::riscv_vsuxseg8_mask:
911       selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false);
912       return;
913     case Intrinsic::riscv_vsoxei:
914     case Intrinsic::riscv_vsoxei_mask:
915     case Intrinsic::riscv_vsuxei:
916     case Intrinsic::riscv_vsuxei_mask: {
917       bool IsMasked = IntNo == Intrinsic::riscv_vsoxei_mask ||
918                       IntNo == Intrinsic::riscv_vsuxei_mask;
919       bool IsOrdered = IntNo == Intrinsic::riscv_vsoxei ||
920                        IntNo == Intrinsic::riscv_vsoxei_mask;
921 
922       MVT VT = Node->getOperand(2)->getSimpleValueType(0);
923       unsigned ScalarSize = VT.getScalarSizeInBits();
924       MVT XLenVT = Subtarget->getXLenVT();
925       SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
926 
927       unsigned CurOp = 2;
928       SmallVector<SDValue, 7> Operands;
929       Operands.push_back(Node->getOperand(CurOp++)); // Store value.
930       SDValue Base;
931       SelectBaseAddr(Node->getOperand(CurOp++), Base);
932       Operands.push_back(Base); // Base pointer.
933       Operands.push_back(Node->getOperand(CurOp++)); // Index.
934       MVT IndexVT = Operands.back()->getSimpleValueType(0);
935       if (IsMasked)
936         Operands.push_back(Node->getOperand(CurOp++)); // Mask.
937       SDValue VL;
938       selectVLOp(Node->getOperand(CurOp++), VL);
939       Operands.push_back(VL);
940       Operands.push_back(SEW);
941       Operands.push_back(Node->getOperand(0)); // Chain.
942 
943       assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
944              "Element count mismatch");
945 
946       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
947       RISCVVLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
948       unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
949       const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
950           IsMasked, IsOrdered, IndexScalarSize, static_cast<unsigned>(LMUL),
951           static_cast<unsigned>(IndexLMUL));
952       MachineSDNode *Store =
953           CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
954 
955       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
956         CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
957 
958       ReplaceNode(Node, Store);
959       return;
960     }
961     case Intrinsic::riscv_vse1:
962     case Intrinsic::riscv_vse:
963     case Intrinsic::riscv_vse_mask:
964     case Intrinsic::riscv_vsse:
965     case Intrinsic::riscv_vsse_mask: {
966       bool IsMasked = IntNo == Intrinsic::riscv_vse_mask ||
967                       IntNo == Intrinsic::riscv_vsse_mask;
968       bool IsStrided =
969           IntNo == Intrinsic::riscv_vsse || IntNo == Intrinsic::riscv_vsse_mask;
970 
971       MVT VT = Node->getOperand(2)->getSimpleValueType(0);
972       unsigned ScalarSize = VT.getScalarSizeInBits();
973       MVT XLenVT = Subtarget->getXLenVT();
974       // VSE1 uses an SEW of 8.
975       unsigned SEWImm = (IntNo == Intrinsic::riscv_vse1) ? 8 : ScalarSize;
976       SDValue SEW = CurDAG->getTargetConstant(SEWImm, DL, XLenVT);
977 
978       unsigned CurOp = 2;
979       SmallVector<SDValue, 7> Operands;
980       Operands.push_back(Node->getOperand(CurOp++)); // Store value.
981       SDValue Base;
982       SelectBaseAddr(Node->getOperand(CurOp++), Base);
983       Operands.push_back(Base); // Base pointer.
984       if (IsStrided)
985         Operands.push_back(Node->getOperand(CurOp++)); // Stride.
986       if (IsMasked)
987         Operands.push_back(Node->getOperand(CurOp++)); // Mask.
988       SDValue VL;
989       selectVLOp(Node->getOperand(CurOp++), VL);
990       Operands.push_back(VL);
991       Operands.push_back(SEW);
992       Operands.push_back(Node->getOperand(0)); // Chain.
993 
994       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
995       const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(
996           IsMasked, IsStrided, ScalarSize, static_cast<unsigned>(LMUL));
997       MachineSDNode *Store =
998           CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
999 
1000       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1001         CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
1002 
1003       ReplaceNode(Node, Store);
1004       return;
1005     }
1006     }
1007     break;
1008   }
1009   case ISD::BITCAST: {
1010     MVT SrcVT = Node->getOperand(0).getSimpleValueType();
1011     // Just drop bitcasts between vectors if both are fixed or both are
1012     // scalable.
1013     if ((VT.isScalableVector() && SrcVT.isScalableVector()) ||
1014         (VT.isFixedLengthVector() && SrcVT.isFixedLengthVector())) {
1015       ReplaceUses(SDValue(Node, 0), Node->getOperand(0));
1016       CurDAG->RemoveDeadNode(Node);
1017       return;
1018     }
1019     break;
1020   }
1021   case ISD::INSERT_SUBVECTOR: {
1022     SDValue V = Node->getOperand(0);
1023     SDValue SubV = Node->getOperand(1);
1024     SDLoc DL(SubV);
1025     auto Idx = Node->getConstantOperandVal(2);
1026     MVT SubVecVT = SubV.getSimpleValueType();
1027 
1028     MVT SubVecContainerVT = SubVecVT;
1029     // Establish the correct scalable-vector types for any fixed-length type.
1030     if (SubVecVT.isFixedLengthVector())
1031       SubVecContainerVT = RISCVTargetLowering::getContainerForFixedLengthVector(
1032           *CurDAG, SubVecVT, *Subtarget);
1033     if (VT.isFixedLengthVector())
1034       VT = RISCVTargetLowering::getContainerForFixedLengthVector(*CurDAG, VT,
1035                                                                  *Subtarget);
1036 
1037     const auto *TRI = Subtarget->getRegisterInfo();
1038     unsigned SubRegIdx;
1039     std::tie(SubRegIdx, Idx) =
1040         RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1041             VT, SubVecContainerVT, Idx, TRI);
1042 
1043     // If the Idx hasn't been completely eliminated then this is a subvector
1044     // insert which doesn't naturally align to a vector register. These must
1045     // be handled using instructions to manipulate the vector registers.
1046     if (Idx != 0)
1047       break;
1048 
1049     RISCVVLMUL SubVecLMUL = RISCVTargetLowering::getLMUL(SubVecContainerVT);
1050     bool IsSubVecPartReg = SubVecLMUL == RISCVVLMUL::LMUL_F2 ||
1051                            SubVecLMUL == RISCVVLMUL::LMUL_F4 ||
1052                            SubVecLMUL == RISCVVLMUL::LMUL_F8;
1053     (void)IsSubVecPartReg; // Silence unused variable warning without asserts.
1054     assert((!IsSubVecPartReg || V.isUndef()) &&
1055            "Expecting lowering to have created legal INSERT_SUBVECTORs when "
1056            "the subvector is smaller than a full-sized register");
1057 
1058     // If we haven't set a SubRegIdx, then we must be going between
1059     // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy.
1060     if (SubRegIdx == RISCV::NoSubRegister) {
1061       unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT);
1062       assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) ==
1063                  InRegClassID &&
1064              "Unexpected subvector extraction");
1065       SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT);
1066       SDNode *NewNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1067                                                DL, VT, SubV, RC);
1068       return ReplaceNode(Node, NewNode);
1069     }
1070 
1071     SDValue Insert = CurDAG->getTargetInsertSubreg(SubRegIdx, DL, VT, V, SubV);
1072     return ReplaceNode(Node, Insert.getNode());
1073   }
1074   case ISD::EXTRACT_SUBVECTOR: {
1075     SDValue V = Node->getOperand(0);
1076     auto Idx = Node->getConstantOperandVal(1);
1077     MVT InVT = V.getSimpleValueType();
1078     SDLoc DL(V);
1079 
1080     MVT SubVecContainerVT = VT;
1081     // Establish the correct scalable-vector types for any fixed-length type.
1082     if (VT.isFixedLengthVector())
1083       SubVecContainerVT = RISCVTargetLowering::getContainerForFixedLengthVector(
1084           *CurDAG, VT, *Subtarget);
1085     if (InVT.isFixedLengthVector())
1086       InVT = RISCVTargetLowering::getContainerForFixedLengthVector(
1087           *CurDAG, InVT, *Subtarget);
1088 
1089     const auto *TRI = Subtarget->getRegisterInfo();
1090     unsigned SubRegIdx;
1091     std::tie(SubRegIdx, Idx) =
1092         RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1093             InVT, SubVecContainerVT, Idx, TRI);
1094 
1095     // If the Idx hasn't been completely eliminated then this is a subvector
1096     // extract which doesn't naturally align to a vector register. These must
1097     // be handled using instructions to manipulate the vector registers.
1098     if (Idx != 0)
1099       break;
1100 
1101     // If we haven't set a SubRegIdx, then we must be going between
1102     // equally-sized LMUL types (e.g. VR -> VR). This can be done as a copy.
1103     if (SubRegIdx == RISCV::NoSubRegister) {
1104       unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(InVT);
1105       assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) ==
1106                  InRegClassID &&
1107              "Unexpected subvector extraction");
1108       SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT);
1109       SDNode *NewNode =
1110           CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, DL, VT, V, RC);
1111       return ReplaceNode(Node, NewNode);
1112     }
1113 
1114     SDValue Extract = CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, V);
1115     return ReplaceNode(Node, Extract.getNode());
1116   }
1117   }
1118 
1119   // Select the default instruction.
1120   SelectCode(Node);
1121 }
1122 
1123 bool RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand(
1124     const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {
1125   switch (ConstraintID) {
1126   case InlineAsm::Constraint_m:
1127     // We just support simple memory operands that have a single address
1128     // operand and need no special handling.
1129     OutOps.push_back(Op);
1130     return false;
1131   case InlineAsm::Constraint_A:
1132     OutOps.push_back(Op);
1133     return false;
1134   default:
1135     break;
1136   }
1137 
1138   return true;
1139 }
1140 
1141 bool RISCVDAGToDAGISel::SelectAddrFI(SDValue Addr, SDValue &Base) {
1142   if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
1143     Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT());
1144     return true;
1145   }
1146   return false;
1147 }
1148 
1149 bool RISCVDAGToDAGISel::SelectBaseAddr(SDValue Addr, SDValue &Base) {
1150   // If this is FrameIndex, select it directly. Otherwise just let it get
1151   // selected to a register independently.
1152   if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr))
1153     Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT());
1154   else
1155     Base = Addr;
1156   return true;
1157 }
1158 
1159 bool RISCVDAGToDAGISel::selectShiftMask(SDValue N, unsigned ShiftWidth,
1160                                         SDValue &ShAmt) {
1161   // Shift instructions on RISCV only read the lower 5 or 6 bits of the shift
1162   // amount. If there is an AND on the shift amount, we can bypass it if it
1163   // doesn't affect any of those bits.
1164   if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(N.getOperand(1))) {
1165     const APInt &AndMask = N->getConstantOperandAPInt(1);
1166 
1167     // Since the max shift amount is a power of 2 we can subtract 1 to make a
1168     // mask that covers the bits needed to represent all shift amounts.
1169     assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!");
1170     APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
1171 
1172     if (ShMask.isSubsetOf(AndMask)) {
1173       ShAmt = N.getOperand(0);
1174       return true;
1175     }
1176 
1177     // SimplifyDemandedBits may have optimized the mask so try restoring any
1178     // bits that are known zero.
1179     KnownBits Known = CurDAG->computeKnownBits(N->getOperand(0));
1180     if (ShMask.isSubsetOf(AndMask | Known.Zero)) {
1181       ShAmt = N.getOperand(0);
1182       return true;
1183     }
1184   }
1185 
1186   ShAmt = N;
1187   return true;
1188 }
1189 
1190 bool RISCVDAGToDAGISel::selectSExti32(SDValue N, SDValue &Val) {
1191   if (N.getOpcode() == ISD::SIGN_EXTEND_INREG &&
1192       cast<VTSDNode>(N.getOperand(1))->getVT() == MVT::i32) {
1193     Val = N.getOperand(0);
1194     return true;
1195   }
1196   // FIXME: Should we just call computeNumSignBits here?
1197   if (N.getOpcode() == ISD::AssertSext &&
1198       cast<VTSDNode>(N->getOperand(1))->getVT().bitsLE(MVT::i32)) {
1199     Val = N;
1200     return true;
1201   }
1202   if (N.getOpcode() == ISD::AssertZext &&
1203       cast<VTSDNode>(N->getOperand(1))->getVT().bitsLT(MVT::i32)) {
1204     Val = N;
1205     return true;
1206   }
1207 
1208   return false;
1209 }
1210 
1211 bool RISCVDAGToDAGISel::selectZExti32(SDValue N, SDValue &Val) {
1212   if (N.getOpcode() == ISD::AND) {
1213     auto *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
1214     if (C && C->getZExtValue() == UINT64_C(0xFFFFFFFF)) {
1215       Val = N.getOperand(0);
1216       return true;
1217     }
1218   }
1219   // FIXME: Should we just call computeKnownBits here?
1220   if (N.getOpcode() == ISD::AssertZext &&
1221       cast<VTSDNode>(N->getOperand(1))->getVT().bitsLE(MVT::i32)) {
1222     Val = N;
1223     return true;
1224   }
1225 
1226   return false;
1227 }
1228 
1229 // Check that it is a SLLIUW (Shift Logical Left Immediate Unsigned i32
1230 // on RV64).
1231 // SLLIUW is the same as SLLI except for the fact that it clears the bits
1232 // XLEN-1:32 of the input RS1 before shifting.
1233 // A PatFrag has already checked that it has the right structure:
1234 //
1235 //  (AND (SHL RS1, VC2), VC1)
1236 //
1237 // We check that VC2, the shamt is less than 32, otherwise the pattern is
1238 // exactly the same as SLLI and we give priority to that.
1239 // Eventually we check that VC1, the mask used to clear the upper 32 bits
1240 // of RS1, is correct:
1241 //
1242 //  VC1 == (0xFFFFFFFF << VC2)
1243 //
1244 bool RISCVDAGToDAGISel::MatchSLLIUW(SDNode *N) const {
1245   assert(N->getOpcode() == ISD::AND);
1246   assert(N->getOperand(0).getOpcode() == ISD::SHL);
1247   assert(isa<ConstantSDNode>(N->getOperand(1)));
1248   assert(isa<ConstantSDNode>(N->getOperand(0).getOperand(1)));
1249 
1250   // The IsRV64 predicate is checked after PatFrag predicates so we can get
1251   // here even on RV32.
1252   if (!Subtarget->is64Bit())
1253     return false;
1254 
1255   SDValue Shl = N->getOperand(0);
1256   uint64_t VC1 = N->getConstantOperandVal(1);
1257   uint64_t VC2 = Shl.getConstantOperandVal(1);
1258 
1259   // Immediate range should be enforced by uimm5 predicate.
1260   assert(VC2 < 32 && "Unexpected immediate");
1261   return (VC1 >> VC2) == UINT64_C(0xFFFFFFFF);
1262 }
1263 
1264 // X0 has special meaning for vsetvl/vsetvli.
1265 //  rd | rs1 |   AVL value | Effect on vl
1266 //--------------------------------------------------------------
1267 // !X0 |  X0 |       VLMAX | Set vl to VLMAX
1268 //  X0 |  X0 | Value in vl | Keep current vl, just change vtype.
1269 bool RISCVDAGToDAGISel::selectVLOp(SDValue N, SDValue &VL) {
1270   // If the VL value is a constant 0, manually select it to an ADDI with 0
1271   // immediate to prevent the default selection path from matching it to X0.
1272   auto *C = dyn_cast<ConstantSDNode>(N);
1273   if (C && C->isNullValue())
1274     VL = SDValue(selectImm(CurDAG, SDLoc(N), 0, Subtarget->getXLenVT()), 0);
1275   else
1276     VL = N;
1277 
1278   return true;
1279 }
1280 
1281 bool RISCVDAGToDAGISel::selectVSplat(SDValue N, SDValue &SplatVal) {
1282   if (N.getOpcode() != ISD::SPLAT_VECTOR &&
1283       N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64 &&
1284       N.getOpcode() != RISCVISD::VMV_V_X_VL)
1285     return false;
1286   SplatVal = N.getOperand(0);
1287   return true;
1288 }
1289 
1290 bool RISCVDAGToDAGISel::selectVSplatSimm5(SDValue N, SDValue &SplatVal) {
1291   if ((N.getOpcode() != ISD::SPLAT_VECTOR &&
1292        N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64 &&
1293        N.getOpcode() != RISCVISD::VMV_V_X_VL) ||
1294       !isa<ConstantSDNode>(N.getOperand(0)))
1295     return false;
1296 
1297   int64_t SplatImm = cast<ConstantSDNode>(N.getOperand(0))->getSExtValue();
1298 
1299   // Both ISD::SPLAT_VECTOR and RISCVISD::SPLAT_VECTOR_I64 share semantics when
1300   // the operand type is wider than the resulting vector element type: an
1301   // implicit truncation first takes place. Therefore, perform a manual
1302   // truncation/sign-extension in order to ignore any truncated bits and catch
1303   // any zero-extended immediate.
1304   // For example, we wish to match (i8 -1) -> (XLenVT 255) as a simm5 by first
1305   // sign-extending to (XLenVT -1).
1306   MVT XLenVT = Subtarget->getXLenVT();
1307   assert(XLenVT == N.getOperand(0).getSimpleValueType() &&
1308          "Unexpected splat operand type");
1309   MVT EltVT = N.getSimpleValueType().getVectorElementType();
1310   if (EltVT.bitsLT(XLenVT)) {
1311     SplatImm = SignExtend64(SplatImm, EltVT.getSizeInBits());
1312   }
1313 
1314   if (!isInt<5>(SplatImm))
1315     return false;
1316 
1317   SplatVal = CurDAG->getTargetConstant(SplatImm, SDLoc(N), XLenVT);
1318   return true;
1319 }
1320 
1321 bool RISCVDAGToDAGISel::selectVSplatUimm5(SDValue N, SDValue &SplatVal) {
1322   if ((N.getOpcode() != ISD::SPLAT_VECTOR &&
1323        N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64 &&
1324        N.getOpcode() != RISCVISD::VMV_V_X_VL) ||
1325       !isa<ConstantSDNode>(N.getOperand(0)))
1326     return false;
1327 
1328   int64_t SplatImm = cast<ConstantSDNode>(N.getOperand(0))->getSExtValue();
1329 
1330   if (!isUInt<5>(SplatImm))
1331     return false;
1332 
1333   SplatVal =
1334       CurDAG->getTargetConstant(SplatImm, SDLoc(N), Subtarget->getXLenVT());
1335 
1336   return true;
1337 }
1338 
1339 bool RISCVDAGToDAGISel::selectRVVSimm5(SDValue N, unsigned Width,
1340                                        SDValue &Imm) {
1341   if (auto *C = dyn_cast<ConstantSDNode>(N)) {
1342     int64_t ImmVal = SignExtend64(C->getSExtValue(), Width);
1343 
1344     if (!isInt<5>(ImmVal))
1345       return false;
1346 
1347     Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), Subtarget->getXLenVT());
1348     return true;
1349   }
1350 
1351   return false;
1352 }
1353 
1354 bool RISCVDAGToDAGISel::selectRVVUimm5(SDValue N, unsigned Width,
1355                                        SDValue &Imm) {
1356   if (auto *C = dyn_cast<ConstantSDNode>(N)) {
1357     int64_t ImmVal = C->getSExtValue();
1358 
1359     if (!isUInt<5>(ImmVal))
1360       return false;
1361 
1362     Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), Subtarget->getXLenVT());
1363     return true;
1364   }
1365 
1366   return false;
1367 }
1368 
1369 // Merge an ADDI into the offset of a load/store instruction where possible.
1370 // (load (addi base, off1), off2) -> (load base, off1+off2)
1371 // (store val, (addi base, off1), off2) -> (store val, base, off1+off2)
1372 // This is possible when off1+off2 fits a 12-bit immediate.
1373 void RISCVDAGToDAGISel::doPeepholeLoadStoreADDI() {
1374   SelectionDAG::allnodes_iterator Position(CurDAG->getRoot().getNode());
1375   ++Position;
1376 
1377   while (Position != CurDAG->allnodes_begin()) {
1378     SDNode *N = &*--Position;
1379     // Skip dead nodes and any non-machine opcodes.
1380     if (N->use_empty() || !N->isMachineOpcode())
1381       continue;
1382 
1383     int OffsetOpIdx;
1384     int BaseOpIdx;
1385 
1386     // Only attempt this optimisation for I-type loads and S-type stores.
1387     switch (N->getMachineOpcode()) {
1388     default:
1389       continue;
1390     case RISCV::LB:
1391     case RISCV::LH:
1392     case RISCV::LW:
1393     case RISCV::LBU:
1394     case RISCV::LHU:
1395     case RISCV::LWU:
1396     case RISCV::LD:
1397     case RISCV::FLH:
1398     case RISCV::FLW:
1399     case RISCV::FLD:
1400       BaseOpIdx = 0;
1401       OffsetOpIdx = 1;
1402       break;
1403     case RISCV::SB:
1404     case RISCV::SH:
1405     case RISCV::SW:
1406     case RISCV::SD:
1407     case RISCV::FSH:
1408     case RISCV::FSW:
1409     case RISCV::FSD:
1410       BaseOpIdx = 1;
1411       OffsetOpIdx = 2;
1412       break;
1413     }
1414 
1415     if (!isa<ConstantSDNode>(N->getOperand(OffsetOpIdx)))
1416       continue;
1417 
1418     SDValue Base = N->getOperand(BaseOpIdx);
1419 
1420     // If the base is an ADDI, we can merge it in to the load/store.
1421     if (!Base.isMachineOpcode() || Base.getMachineOpcode() != RISCV::ADDI)
1422       continue;
1423 
1424     SDValue ImmOperand = Base.getOperand(1);
1425     uint64_t Offset2 = N->getConstantOperandVal(OffsetOpIdx);
1426 
1427     if (auto *Const = dyn_cast<ConstantSDNode>(ImmOperand)) {
1428       int64_t Offset1 = Const->getSExtValue();
1429       int64_t CombinedOffset = Offset1 + Offset2;
1430       if (!isInt<12>(CombinedOffset))
1431         continue;
1432       ImmOperand = CurDAG->getTargetConstant(CombinedOffset, SDLoc(ImmOperand),
1433                                              ImmOperand.getValueType());
1434     } else if (auto *GA = dyn_cast<GlobalAddressSDNode>(ImmOperand)) {
1435       // If the off1 in (addi base, off1) is a global variable's address (its
1436       // low part, really), then we can rely on the alignment of that variable
1437       // to provide a margin of safety before off1 can overflow the 12 bits.
1438       // Check if off2 falls within that margin; if so off1+off2 can't overflow.
1439       const DataLayout &DL = CurDAG->getDataLayout();
1440       Align Alignment = GA->getGlobal()->getPointerAlignment(DL);
1441       if (Offset2 != 0 && Alignment <= Offset2)
1442         continue;
1443       int64_t Offset1 = GA->getOffset();
1444       int64_t CombinedOffset = Offset1 + Offset2;
1445       ImmOperand = CurDAG->getTargetGlobalAddress(
1446           GA->getGlobal(), SDLoc(ImmOperand), ImmOperand.getValueType(),
1447           CombinedOffset, GA->getTargetFlags());
1448     } else if (auto *CP = dyn_cast<ConstantPoolSDNode>(ImmOperand)) {
1449       // Ditto.
1450       Align Alignment = CP->getAlign();
1451       if (Offset2 != 0 && Alignment <= Offset2)
1452         continue;
1453       int64_t Offset1 = CP->getOffset();
1454       int64_t CombinedOffset = Offset1 + Offset2;
1455       ImmOperand = CurDAG->getTargetConstantPool(
1456           CP->getConstVal(), ImmOperand.getValueType(), CP->getAlign(),
1457           CombinedOffset, CP->getTargetFlags());
1458     } else {
1459       continue;
1460     }
1461 
1462     LLVM_DEBUG(dbgs() << "Folding add-immediate into mem-op:\nBase:    ");
1463     LLVM_DEBUG(Base->dump(CurDAG));
1464     LLVM_DEBUG(dbgs() << "\nN: ");
1465     LLVM_DEBUG(N->dump(CurDAG));
1466     LLVM_DEBUG(dbgs() << "\n");
1467 
1468     // Modify the offset operand of the load/store.
1469     if (BaseOpIdx == 0) // Load
1470       CurDAG->UpdateNodeOperands(N, Base.getOperand(0), ImmOperand,
1471                                  N->getOperand(2));
1472     else // Store
1473       CurDAG->UpdateNodeOperands(N, N->getOperand(0), Base.getOperand(0),
1474                                  ImmOperand, N->getOperand(3));
1475 
1476     // The add-immediate may now be dead, in which case remove it.
1477     if (Base.getNode()->use_empty())
1478       CurDAG->RemoveDeadNode(Base.getNode());
1479   }
1480 }
1481 
1482 // This pass converts a legalized DAG into a RISCV-specific DAG, ready
1483 // for instruction scheduling.
1484 FunctionPass *llvm::createRISCVISelDag(RISCVTargetMachine &TM) {
1485   return new RISCVDAGToDAGISel(TM);
1486 }
1487