1 //===-- RISCVISelDAGToDAG.cpp - A dag to dag inst selector for RISCV ------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines an instruction selector for the RISCV target.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "RISCVISelDAGToDAG.h"
14 #include "MCTargetDesc/RISCVMCTargetDesc.h"
15 #include "MCTargetDesc/RISCVMatInt.h"
16 #include "RISCVISelLowering.h"
17 #include "llvm/CodeGen/MachineFrameInfo.h"
18 #include "llvm/IR/IntrinsicsRISCV.h"
19 #include "llvm/Support/Alignment.h"
20 #include "llvm/Support/Debug.h"
21 #include "llvm/Support/KnownBits.h"
22 #include "llvm/Support/MathExtras.h"
23 #include "llvm/Support/raw_ostream.h"
24 
25 using namespace llvm;
26 
27 #define DEBUG_TYPE "riscv-isel"
28 
29 namespace llvm {
30 namespace RISCV {
31 #define GET_RISCVVSSEGTable_IMPL
32 #define GET_RISCVVLSEGTable_IMPL
33 #define GET_RISCVVLXSEGTable_IMPL
34 #define GET_RISCVVSXSEGTable_IMPL
35 #define GET_RISCVVLETable_IMPL
36 #define GET_RISCVVSETable_IMPL
37 #define GET_RISCVVLXTable_IMPL
38 #define GET_RISCVVSXTable_IMPL
39 #include "RISCVGenSearchableTables.inc"
40 } // namespace RISCV
41 } // namespace llvm
42 
43 void RISCVDAGToDAGISel::PostprocessISelDAG() {
44   doPeepholeLoadStoreADDI();
45 }
46 
47 static SDNode *selectImm(SelectionDAG *CurDAG, const SDLoc &DL, int64_t Imm,
48                          MVT XLenVT) {
49   RISCVMatInt::InstSeq Seq;
50   RISCVMatInt::generateInstSeq(Imm, XLenVT == MVT::i64, Seq);
51 
52   SDNode *Result = nullptr;
53   SDValue SrcReg = CurDAG->getRegister(RISCV::X0, XLenVT);
54   for (RISCVMatInt::Inst &Inst : Seq) {
55     SDValue SDImm = CurDAG->getTargetConstant(Inst.Imm, DL, XLenVT);
56     if (Inst.Opc == RISCV::LUI)
57       Result = CurDAG->getMachineNode(RISCV::LUI, DL, XLenVT, SDImm);
58     else
59       Result = CurDAG->getMachineNode(Inst.Opc, DL, XLenVT, SrcReg, SDImm);
60 
61     // Only the first instruction has X0 as its source.
62     SrcReg = SDValue(Result, 0);
63   }
64 
65   return Result;
66 }
67 
68 static SDValue createTupleImpl(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
69                                unsigned RegClassID, unsigned SubReg0) {
70   assert(Regs.size() >= 2 && Regs.size() <= 8);
71 
72   SDLoc DL(Regs[0]);
73   SmallVector<SDValue, 8> Ops;
74 
75   Ops.push_back(CurDAG.getTargetConstant(RegClassID, DL, MVT::i32));
76 
77   for (unsigned I = 0; I < Regs.size(); ++I) {
78     Ops.push_back(Regs[I]);
79     Ops.push_back(CurDAG.getTargetConstant(SubReg0 + I, DL, MVT::i32));
80   }
81   SDNode *N =
82       CurDAG.getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops);
83   return SDValue(N, 0);
84 }
85 
86 static SDValue createM1Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
87                              unsigned NF) {
88   static const unsigned RegClassIDs[] = {
89       RISCV::VRN2M1RegClassID, RISCV::VRN3M1RegClassID, RISCV::VRN4M1RegClassID,
90       RISCV::VRN5M1RegClassID, RISCV::VRN6M1RegClassID, RISCV::VRN7M1RegClassID,
91       RISCV::VRN8M1RegClassID};
92 
93   return createTupleImpl(CurDAG, Regs, RegClassIDs[NF - 2], RISCV::sub_vrm1_0);
94 }
95 
96 static SDValue createM2Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
97                              unsigned NF) {
98   static const unsigned RegClassIDs[] = {RISCV::VRN2M2RegClassID,
99                                          RISCV::VRN3M2RegClassID,
100                                          RISCV::VRN4M2RegClassID};
101 
102   return createTupleImpl(CurDAG, Regs, RegClassIDs[NF - 2], RISCV::sub_vrm2_0);
103 }
104 
105 static SDValue createM4Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
106                              unsigned NF) {
107   return createTupleImpl(CurDAG, Regs, RISCV::VRN2M4RegClassID,
108                          RISCV::sub_vrm4_0);
109 }
110 
111 static SDValue createTuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
112                            unsigned NF, RISCVVLMUL LMUL) {
113   switch (LMUL) {
114   default:
115     llvm_unreachable("Invalid LMUL.");
116   case RISCVVLMUL::LMUL_F8:
117   case RISCVVLMUL::LMUL_F4:
118   case RISCVVLMUL::LMUL_F2:
119   case RISCVVLMUL::LMUL_1:
120     return createM1Tuple(CurDAG, Regs, NF);
121   case RISCVVLMUL::LMUL_2:
122     return createM2Tuple(CurDAG, Regs, NF);
123   case RISCVVLMUL::LMUL_4:
124     return createM4Tuple(CurDAG, Regs, NF);
125   }
126 }
127 
128 void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, bool IsMasked,
129                                     bool IsStrided) {
130   SDLoc DL(Node);
131   unsigned NF = Node->getNumValues() - 1;
132   MVT VT = Node->getSimpleValueType(0);
133   unsigned ScalarSize = VT.getScalarSizeInBits();
134   MVT XLenVT = Subtarget->getXLenVT();
135   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
136   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
137 
138   SDValue Chain = Node->getOperand(0);
139   SDValue Glue;
140 
141   unsigned CurOp = 2;
142   SmallVector<SDValue, 8> Operands;
143   if (IsMasked) {
144     SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
145                                  Node->op_begin() + CurOp + NF);
146     SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
147     Operands.push_back(MaskedOff);
148     CurOp += NF;
149   }
150   SDValue Base;
151   SelectBaseAddr(Node->getOperand(CurOp++), Base);
152   Operands.push_back(Base); // Base pointer.
153   if (IsStrided)
154     Operands.push_back(Node->getOperand(CurOp++)); // Stride.
155   if (IsMasked) {
156     // Mask needs to be copied to V0.
157     SDValue Mask = Node->getOperand(CurOp++);
158     Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
159     Glue = Chain.getValue(1);
160     Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
161   }
162   SDValue VL;
163   selectVLOp(Node->getOperand(CurOp++), VL);
164   Operands.push_back(VL);
165   Operands.push_back(SEW);
166   Operands.push_back(Chain); // Chain.
167   if (Glue)
168     Operands.push_back(Glue);
169   const RISCV::VLSEGPseudo *P =
170       RISCV::getVLSEGPseudo(NF, IsMasked, IsStrided, /*FF*/ false, ScalarSize,
171                             static_cast<unsigned>(LMUL));
172   MachineSDNode *Load =
173       CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
174 
175   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
176     CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
177 
178   SDValue SuperReg = SDValue(Load, 0);
179   for (unsigned I = 0; I < NF; ++I) {
180     unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
181     ReplaceUses(SDValue(Node, I),
182                 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
183   }
184 
185   ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
186   CurDAG->RemoveDeadNode(Node);
187 }
188 
189 void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node, bool IsMasked) {
190   SDLoc DL(Node);
191   unsigned NF = Node->getNumValues() - 2; // Do not count VL and Chain.
192   MVT VT = Node->getSimpleValueType(0);
193   MVT XLenVT = Subtarget->getXLenVT();
194   unsigned ScalarSize = VT.getScalarSizeInBits();
195   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
196   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
197 
198   SDValue Chain = Node->getOperand(0);
199   SDValue Glue;
200 
201   unsigned CurOp = 2;
202   SmallVector<SDValue, 7> Operands;
203   if (IsMasked) {
204     SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
205                                  Node->op_begin() + CurOp + NF);
206     SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
207     Operands.push_back(MaskedOff);
208     CurOp += NF;
209   }
210   SDValue Base;
211   SelectBaseAddr(Node->getOperand(CurOp++), Base);
212   Operands.push_back(Base); // Base pointer.
213   if (IsMasked) {
214     // Mask needs to be copied to V0.
215     SDValue Mask = Node->getOperand(CurOp++);
216     Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
217     Glue = Chain.getValue(1);
218     Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
219   }
220   SDValue VL;
221   selectVLOp(Node->getOperand(CurOp++), VL);
222   Operands.push_back(VL);
223   Operands.push_back(SEW);
224   Operands.push_back(Chain); // Chain.
225   if (Glue)
226     Operands.push_back(Glue);
227   const RISCV::VLSEGPseudo *P =
228       RISCV::getVLSEGPseudo(NF, IsMasked, /*Strided*/ false, /*FF*/ true,
229                             ScalarSize, static_cast<unsigned>(LMUL));
230   MachineSDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped,
231                                                MVT::Other, MVT::Glue, Operands);
232   SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT,
233                                           /*Glue*/ SDValue(Load, 2));
234 
235   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
236     CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
237 
238   SDValue SuperReg = SDValue(Load, 0);
239   for (unsigned I = 0; I < NF; ++I) {
240     unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
241     ReplaceUses(SDValue(Node, I),
242                 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
243   }
244 
245   ReplaceUses(SDValue(Node, NF), SDValue(ReadVL, 0));   // VL
246   ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 1)); // Chain
247   CurDAG->RemoveDeadNode(Node);
248 }
249 
250 void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, bool IsMasked,
251                                      bool IsOrdered) {
252   SDLoc DL(Node);
253   unsigned NF = Node->getNumValues() - 1;
254   MVT VT = Node->getSimpleValueType(0);
255   unsigned ScalarSize = VT.getScalarSizeInBits();
256   MVT XLenVT = Subtarget->getXLenVT();
257   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
258   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
259 
260   SDValue Chain = Node->getOperand(0);
261   SDValue Glue;
262 
263   unsigned CurOp = 2;
264   SmallVector<SDValue, 8> Operands;
265   if (IsMasked) {
266     SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
267                                  Node->op_begin() + CurOp + NF);
268     SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
269     Operands.push_back(MaskedOff);
270     CurOp += NF;
271   }
272   SDValue Base;
273   SelectBaseAddr(Node->getOperand(CurOp++), Base);
274   Operands.push_back(Base); // Base pointer.
275   Operands.push_back(Node->getOperand(CurOp++)); // Index.
276   MVT IndexVT = Operands.back()->getSimpleValueType(0);
277   if (IsMasked) {
278     // Mask needs to be copied to V0.
279     SDValue Mask = Node->getOperand(CurOp++);
280     Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
281     Glue = Chain.getValue(1);
282     Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
283   }
284   SDValue VL;
285   selectVLOp(Node->getOperand(CurOp++), VL);
286   Operands.push_back(VL);
287   Operands.push_back(SEW);
288   Operands.push_back(Chain); // Chain.
289   if (Glue)
290     Operands.push_back(Glue);
291 
292   assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
293          "Element count mismatch");
294 
295   RISCVVLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
296   unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
297   const RISCV::VLXSEGPseudo *P = RISCV::getVLXSEGPseudo(
298       NF, IsMasked, IsOrdered, IndexScalarSize, static_cast<unsigned>(LMUL),
299       static_cast<unsigned>(IndexLMUL));
300   MachineSDNode *Load =
301       CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
302 
303   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
304     CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
305 
306   SDValue SuperReg = SDValue(Load, 0);
307   for (unsigned I = 0; I < NF; ++I) {
308     unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
309     ReplaceUses(SDValue(Node, I),
310                 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
311   }
312 
313   ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
314   CurDAG->RemoveDeadNode(Node);
315 }
316 
317 void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, bool IsMasked,
318                                     bool IsStrided) {
319   SDLoc DL(Node);
320   unsigned NF = Node->getNumOperands() - 4;
321   if (IsStrided)
322     NF--;
323   if (IsMasked)
324     NF--;
325   MVT VT = Node->getOperand(2)->getSimpleValueType(0);
326   unsigned ScalarSize = VT.getScalarSizeInBits();
327   MVT XLenVT = Subtarget->getXLenVT();
328   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
329   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
330   SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
331   SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
332 
333   SDValue Chain = Node->getOperand(0);
334   SDValue Glue;
335 
336   SmallVector<SDValue, 8> Operands;
337   Operands.push_back(StoreVal);
338   unsigned CurOp = 2 + NF;
339   SDValue Base;
340   SelectBaseAddr(Node->getOperand(CurOp++), Base);
341   Operands.push_back(Base); // Base pointer.
342   if (IsStrided)
343     Operands.push_back(Node->getOperand(CurOp++)); // Stride.
344   if (IsMasked) {
345     // Mask needs to be copied to V0.
346     SDValue Mask = Node->getOperand(CurOp++);
347     Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
348     Glue = Chain.getValue(1);
349     Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
350   }
351   SDValue VL;
352   selectVLOp(Node->getOperand(CurOp++), VL);
353   Operands.push_back(VL);
354   Operands.push_back(SEW);
355   Operands.push_back(Chain); // Chain.
356   if (Glue)
357     Operands.push_back(Glue);
358   const RISCV::VSSEGPseudo *P = RISCV::getVSSEGPseudo(
359       NF, IsMasked, IsStrided, ScalarSize, static_cast<unsigned>(LMUL));
360   MachineSDNode *Store =
361       CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
362 
363   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
364     CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
365 
366   ReplaceNode(Node, Store);
367 }
368 
369 void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, bool IsMasked,
370                                      bool IsOrdered) {
371   SDLoc DL(Node);
372   unsigned NF = Node->getNumOperands() - 5;
373   if (IsMasked)
374     --NF;
375   MVT VT = Node->getOperand(2)->getSimpleValueType(0);
376   unsigned ScalarSize = VT.getScalarSizeInBits();
377   MVT XLenVT = Subtarget->getXLenVT();
378   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
379   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
380   SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
381   SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
382 
383   SDValue Chain = Node->getOperand(0);
384   SDValue Glue;
385 
386   SmallVector<SDValue, 8> Operands;
387   Operands.push_back(StoreVal);
388   unsigned CurOp = 2 + NF;
389   SDValue Base;
390   SelectBaseAddr(Node->getOperand(CurOp++), Base);
391   Operands.push_back(Base); // Base pointer.
392   Operands.push_back(Node->getOperand(CurOp++)); // Index.
393   MVT IndexVT = Operands.back()->getSimpleValueType(0);
394   if (IsMasked) {
395     // Mask needs to be copied to V0.
396     SDValue Mask = Node->getOperand(CurOp++);
397     Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
398     Glue = Chain.getValue(1);
399     Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
400   }
401   SDValue VL;
402   selectVLOp(Node->getOperand(CurOp++), VL);
403   Operands.push_back(VL);
404   Operands.push_back(SEW);
405   Operands.push_back(Chain); // Chain.
406   if (Glue)
407     Operands.push_back(Glue);
408 
409   assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
410          "Element count mismatch");
411 
412   RISCVVLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
413   unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
414   const RISCV::VSXSEGPseudo *P = RISCV::getVSXSEGPseudo(
415       NF, IsMasked, IsOrdered, IndexScalarSize, static_cast<unsigned>(LMUL),
416       static_cast<unsigned>(IndexLMUL));
417   MachineSDNode *Store =
418       CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
419 
420   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
421     CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
422 
423   ReplaceNode(Node, Store);
424 }
425 
426 
427 void RISCVDAGToDAGISel::Select(SDNode *Node) {
428   // If we have a custom node, we have already selected.
429   if (Node->isMachineOpcode()) {
430     LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n");
431     Node->setNodeId(-1);
432     return;
433   }
434 
435   // Instruction Selection not handled by the auto-generated tablegen selection
436   // should be handled here.
437   unsigned Opcode = Node->getOpcode();
438   MVT XLenVT = Subtarget->getXLenVT();
439   SDLoc DL(Node);
440   MVT VT = Node->getSimpleValueType(0);
441 
442   switch (Opcode) {
443   case ISD::ADD: {
444     // Optimize (add r, imm) to (addi (addi r, imm0) imm1) if applicable. The
445     // immediate must be in specific ranges and have a single use.
446     if (auto *ConstOp = dyn_cast<ConstantSDNode>(Node->getOperand(1))) {
447       if (!(ConstOp->hasOneUse()))
448         break;
449       // The imm must be in range [-4096,-2049] or [2048,4094].
450       int64_t Imm = ConstOp->getSExtValue();
451       if (!(-4096 <= Imm && Imm <= -2049) && !(2048 <= Imm && Imm <= 4094))
452         break;
453       // Break the imm to imm0+imm1.
454       const SDValue ImmOp0 = CurDAG->getTargetConstant(Imm - Imm / 2, DL, VT);
455       const SDValue ImmOp1 = CurDAG->getTargetConstant(Imm / 2, DL, VT);
456       auto *NodeAddi0 = CurDAG->getMachineNode(RISCV::ADDI, DL, VT,
457                                                Node->getOperand(0), ImmOp0);
458       auto *NodeAddi1 = CurDAG->getMachineNode(RISCV::ADDI, DL, VT,
459                                                SDValue(NodeAddi0, 0), ImmOp1);
460       ReplaceNode(Node, NodeAddi1);
461       return;
462     }
463     break;
464   }
465   case ISD::Constant: {
466     auto *ConstNode = cast<ConstantSDNode>(Node);
467     if (VT == XLenVT && ConstNode->isNullValue()) {
468       SDValue New =
469           CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, RISCV::X0, XLenVT);
470       ReplaceNode(Node, New.getNode());
471       return;
472     }
473     ReplaceNode(Node, selectImm(CurDAG, DL, ConstNode->getSExtValue(), XLenVT));
474     return;
475     break;
476   }
477   case ISD::FrameIndex: {
478     SDValue Imm = CurDAG->getTargetConstant(0, DL, XLenVT);
479     int FI = cast<FrameIndexSDNode>(Node)->getIndex();
480     SDValue TFI = CurDAG->getTargetFrameIndex(FI, VT);
481     ReplaceNode(Node, CurDAG->getMachineNode(RISCV::ADDI, DL, VT, TFI, Imm));
482     return;
483   }
484   case ISD::SRL: {
485     // Optimize (srl (and X, 0xffff), C) -> (srli (slli X, 16), 16 + C).
486     // Taking into account that the 0xffff may have had lower bits unset by
487     // SimplifyDemandedBits. This avoids materializing the 0xffff immediate.
488     // This pattern occurs when type legalizing i16 right shifts.
489     // FIXME: This could be extended to other AND masks.
490     auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
491     if (N1C) {
492       uint64_t ShAmt = N1C->getZExtValue();
493       SDValue N0 = Node->getOperand(0);
494       if (ShAmt < 16 && N0.getOpcode() == ISD::AND && N0.hasOneUse() &&
495           isa<ConstantSDNode>(N0.getOperand(1))) {
496         uint64_t Mask = N0.getConstantOperandVal(1);
497         Mask |= maskTrailingOnes<uint64_t>(ShAmt);
498         if (Mask == 0xffff) {
499           unsigned SLLOpc = Subtarget->is64Bit() ? RISCV::SLLIW : RISCV::SLLI;
500           unsigned SRLOpc = Subtarget->is64Bit() ? RISCV::SRLIW : RISCV::SRLI;
501           SDNode *SLLI =
502               CurDAG->getMachineNode(SLLOpc, DL, VT, N0->getOperand(0),
503                                      CurDAG->getTargetConstant(16, DL, VT));
504           SDNode *SRLI = CurDAG->getMachineNode(
505               SRLOpc, DL, VT, SDValue(SLLI, 0),
506               CurDAG->getTargetConstant(16 + ShAmt, DL, VT));
507           ReplaceNode(Node, SRLI);
508           return;
509         }
510       }
511     }
512 
513     break;
514   }
515   case ISD::INTRINSIC_W_CHAIN: {
516     unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
517     switch (IntNo) {
518       // By default we do not custom select any intrinsic.
519     default:
520       break;
521 
522     case Intrinsic::riscv_vsetvli:
523     case Intrinsic::riscv_vsetvlimax: {
524       if (!Subtarget->hasStdExtV())
525         break;
526 
527       bool VLMax = IntNo == Intrinsic::riscv_vsetvlimax;
528       unsigned Offset = VLMax ? 2 : 3;
529 
530       assert(Node->getNumOperands() == Offset + 2 &&
531              "Unexpected number of operands");
532 
533       RISCVVSEW VSEW =
534           static_cast<RISCVVSEW>(Node->getConstantOperandVal(Offset) & 0x7);
535       RISCVVLMUL VLMul = static_cast<RISCVVLMUL>(
536           Node->getConstantOperandVal(Offset + 1) & 0x7);
537 
538       unsigned VTypeI = RISCVVType::encodeVTYPE(
539           VLMul, VSEW, /*TailAgnostic*/ true, /*MaskAgnostic*/ false);
540       SDValue VTypeIOp = CurDAG->getTargetConstant(VTypeI, DL, XLenVT);
541 
542       SDValue VLOperand;
543       if (VLMax) {
544         VLOperand = CurDAG->getRegister(RISCV::X0, XLenVT);
545       } else {
546         VLOperand = Node->getOperand(2);
547 
548         if (auto *C = dyn_cast<ConstantSDNode>(VLOperand)) {
549           uint64_t AVL = C->getZExtValue();
550           if (isUInt<5>(AVL)) {
551             SDValue VLImm = CurDAG->getTargetConstant(AVL, DL, XLenVT);
552             ReplaceNode(
553                 Node, CurDAG->getMachineNode(RISCV::PseudoVSETIVLI, DL, XLenVT,
554                                              MVT::Other, VLImm, VTypeIOp,
555                                              /* Chain */ Node->getOperand(0)));
556             return;
557           }
558         }
559       }
560 
561       ReplaceNode(Node,
562                   CurDAG->getMachineNode(RISCV::PseudoVSETVLI, DL, XLenVT,
563                                          MVT::Other, VLOperand, VTypeIOp,
564                                          /* Chain */ Node->getOperand(0)));
565       return;
566     }
567     case Intrinsic::riscv_vlseg2:
568     case Intrinsic::riscv_vlseg3:
569     case Intrinsic::riscv_vlseg4:
570     case Intrinsic::riscv_vlseg5:
571     case Intrinsic::riscv_vlseg6:
572     case Intrinsic::riscv_vlseg7:
573     case Intrinsic::riscv_vlseg8: {
574       selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false);
575       return;
576     }
577     case Intrinsic::riscv_vlseg2_mask:
578     case Intrinsic::riscv_vlseg3_mask:
579     case Intrinsic::riscv_vlseg4_mask:
580     case Intrinsic::riscv_vlseg5_mask:
581     case Intrinsic::riscv_vlseg6_mask:
582     case Intrinsic::riscv_vlseg7_mask:
583     case Intrinsic::riscv_vlseg8_mask: {
584       selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false);
585       return;
586     }
587     case Intrinsic::riscv_vlsseg2:
588     case Intrinsic::riscv_vlsseg3:
589     case Intrinsic::riscv_vlsseg4:
590     case Intrinsic::riscv_vlsseg5:
591     case Intrinsic::riscv_vlsseg6:
592     case Intrinsic::riscv_vlsseg7:
593     case Intrinsic::riscv_vlsseg8: {
594       selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true);
595       return;
596     }
597     case Intrinsic::riscv_vlsseg2_mask:
598     case Intrinsic::riscv_vlsseg3_mask:
599     case Intrinsic::riscv_vlsseg4_mask:
600     case Intrinsic::riscv_vlsseg5_mask:
601     case Intrinsic::riscv_vlsseg6_mask:
602     case Intrinsic::riscv_vlsseg7_mask:
603     case Intrinsic::riscv_vlsseg8_mask: {
604       selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true);
605       return;
606     }
607     case Intrinsic::riscv_vloxseg2:
608     case Intrinsic::riscv_vloxseg3:
609     case Intrinsic::riscv_vloxseg4:
610     case Intrinsic::riscv_vloxseg5:
611     case Intrinsic::riscv_vloxseg6:
612     case Intrinsic::riscv_vloxseg7:
613     case Intrinsic::riscv_vloxseg8:
614       selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true);
615       return;
616     case Intrinsic::riscv_vluxseg2:
617     case Intrinsic::riscv_vluxseg3:
618     case Intrinsic::riscv_vluxseg4:
619     case Intrinsic::riscv_vluxseg5:
620     case Intrinsic::riscv_vluxseg6:
621     case Intrinsic::riscv_vluxseg7:
622     case Intrinsic::riscv_vluxseg8:
623       selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false);
624       return;
625     case Intrinsic::riscv_vloxseg2_mask:
626     case Intrinsic::riscv_vloxseg3_mask:
627     case Intrinsic::riscv_vloxseg4_mask:
628     case Intrinsic::riscv_vloxseg5_mask:
629     case Intrinsic::riscv_vloxseg6_mask:
630     case Intrinsic::riscv_vloxseg7_mask:
631     case Intrinsic::riscv_vloxseg8_mask:
632       selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true);
633       return;
634     case Intrinsic::riscv_vluxseg2_mask:
635     case Intrinsic::riscv_vluxseg3_mask:
636     case Intrinsic::riscv_vluxseg4_mask:
637     case Intrinsic::riscv_vluxseg5_mask:
638     case Intrinsic::riscv_vluxseg6_mask:
639     case Intrinsic::riscv_vluxseg7_mask:
640     case Intrinsic::riscv_vluxseg8_mask:
641       selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false);
642       return;
643     case Intrinsic::riscv_vlseg8ff:
644     case Intrinsic::riscv_vlseg7ff:
645     case Intrinsic::riscv_vlseg6ff:
646     case Intrinsic::riscv_vlseg5ff:
647     case Intrinsic::riscv_vlseg4ff:
648     case Intrinsic::riscv_vlseg3ff:
649     case Intrinsic::riscv_vlseg2ff: {
650       selectVLSEGFF(Node, /*IsMasked*/ false);
651       return;
652     }
653     case Intrinsic::riscv_vlseg8ff_mask:
654     case Intrinsic::riscv_vlseg7ff_mask:
655     case Intrinsic::riscv_vlseg6ff_mask:
656     case Intrinsic::riscv_vlseg5ff_mask:
657     case Intrinsic::riscv_vlseg4ff_mask:
658     case Intrinsic::riscv_vlseg3ff_mask:
659     case Intrinsic::riscv_vlseg2ff_mask: {
660       selectVLSEGFF(Node, /*IsMasked*/ true);
661       return;
662     }
663     case Intrinsic::riscv_vloxei:
664     case Intrinsic::riscv_vloxei_mask:
665     case Intrinsic::riscv_vluxei:
666     case Intrinsic::riscv_vluxei_mask: {
667       bool IsMasked = IntNo == Intrinsic::riscv_vloxei_mask ||
668                       IntNo == Intrinsic::riscv_vluxei_mask;
669       bool IsOrdered = IntNo == Intrinsic::riscv_vloxei ||
670                        IntNo == Intrinsic::riscv_vloxei_mask;
671 
672       MVT VT = Node->getSimpleValueType(0);
673       unsigned ScalarSize = VT.getScalarSizeInBits();
674       MVT XLenVT = Subtarget->getXLenVT();
675       SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
676 
677       SDValue Chain = Node->getOperand(0);
678       SDValue Glue;
679 
680       unsigned CurOp = 2;
681       SmallVector<SDValue, 8> Operands;
682       if (IsMasked)
683         Operands.push_back(Node->getOperand(CurOp++));
684       SDValue Base;
685       SelectBaseAddr(Node->getOperand(CurOp++), Base);
686       Operands.push_back(Base); // Base pointer.
687       Operands.push_back(Node->getOperand(CurOp++)); // Index.
688       MVT IndexVT = Operands.back()->getSimpleValueType(0);
689       if (IsMasked) {
690         // Mask needs to be copied to V0.
691         SDValue Mask = Node->getOperand(CurOp++);
692         Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
693         Glue = Chain.getValue(1);
694         Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
695       }
696       SDValue VL;
697       selectVLOp(Node->getOperand(CurOp++), VL);
698       Operands.push_back(VL);
699       Operands.push_back(SEW);
700       Operands.push_back(Chain); // Chain.
701       if (Glue)
702         Operands.push_back(Glue);
703 
704       assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
705              "Element count mismatch");
706 
707       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
708       RISCVVLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
709       unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
710       const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
711           IsMasked, IsOrdered, IndexScalarSize, static_cast<unsigned>(LMUL),
712           static_cast<unsigned>(IndexLMUL));
713       MachineSDNode *Load =
714           CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
715 
716       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
717         CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
718 
719       ReplaceNode(Node, Load);
720       return;
721     }
722     case Intrinsic::riscv_vle1:
723     case Intrinsic::riscv_vle:
724     case Intrinsic::riscv_vle_mask:
725     case Intrinsic::riscv_vlse:
726     case Intrinsic::riscv_vlse_mask: {
727       bool IsMasked = IntNo == Intrinsic::riscv_vle_mask ||
728                       IntNo == Intrinsic::riscv_vlse_mask;
729       bool IsStrided =
730           IntNo == Intrinsic::riscv_vlse || IntNo == Intrinsic::riscv_vlse_mask;
731 
732       MVT VT = Node->getSimpleValueType(0);
733       unsigned ScalarSize = VT.getScalarSizeInBits();
734       MVT XLenVT = Subtarget->getXLenVT();
735       // VLE1 uses an SEW of 8.
736       unsigned SEWImm = (IntNo == Intrinsic::riscv_vle1) ? 8 : ScalarSize;
737       SDValue SEW = CurDAG->getTargetConstant(SEWImm, DL, XLenVT);
738 
739       SDValue Chain = Node->getOperand(0);
740       SDValue Glue;
741 
742       unsigned CurOp = 2;
743       SmallVector<SDValue, 8> Operands;
744       if (IsMasked)
745         Operands.push_back(Node->getOperand(CurOp++));
746       SDValue Base;
747       SelectBaseAddr(Node->getOperand(CurOp++), Base);
748       Operands.push_back(Base); // Base pointer.
749       if (IsStrided)
750         Operands.push_back(Node->getOperand(CurOp++)); // Stride.
751       if (IsMasked) {
752         // Mask needs to be copied to V0.
753         SDValue Mask = Node->getOperand(CurOp++);
754         Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
755         Glue = Chain.getValue(1);
756         Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
757       }
758       SDValue VL;
759       selectVLOp(Node->getOperand(CurOp++), VL);
760       Operands.push_back(VL);
761       Operands.push_back(SEW);
762       Operands.push_back(Chain); // Chain.
763       if (Glue)
764         Operands.push_back(Glue);
765 
766       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
767       const RISCV::VLEPseudo *P =
768           RISCV::getVLEPseudo(IsMasked, IsStrided, /*FF*/ false, ScalarSize,
769                               static_cast<unsigned>(LMUL));
770       MachineSDNode *Load =
771           CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
772 
773       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
774         CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
775 
776       ReplaceNode(Node, Load);
777       return;
778     }
779     case Intrinsic::riscv_vleff:
780     case Intrinsic::riscv_vleff_mask: {
781       bool IsMasked = IntNo == Intrinsic::riscv_vleff_mask;
782 
783       MVT VT = Node->getSimpleValueType(0);
784       unsigned ScalarSize = VT.getScalarSizeInBits();
785       MVT XLenVT = Subtarget->getXLenVT();
786       SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
787 
788       SDValue Chain = Node->getOperand(0);
789       SDValue Glue;
790 
791       unsigned CurOp = 2;
792       SmallVector<SDValue, 7> Operands;
793       if (IsMasked)
794         Operands.push_back(Node->getOperand(CurOp++));
795       SDValue Base;
796       SelectBaseAddr(Node->getOperand(CurOp++), Base);
797       Operands.push_back(Base); // Base pointer.
798       if (IsMasked) {
799         // Mask needs to be copied to V0.
800         SDValue Mask = Node->getOperand(CurOp++);
801         Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
802         Glue = Chain.getValue(1);
803         Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
804       }
805       SDValue VL;
806       selectVLOp(Node->getOperand(CurOp++), VL);
807       Operands.push_back(VL);
808       Operands.push_back(SEW);
809       Operands.push_back(Chain); // Chain.
810       if (Glue)
811         Operands.push_back(Glue);
812 
813       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
814       const RISCV::VLEPseudo *P =
815           RISCV::getVLEPseudo(IsMasked, /*Strided*/ false, /*FF*/ true,
816                               ScalarSize, static_cast<unsigned>(LMUL));
817       MachineSDNode *Load =
818           CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0),
819                                  MVT::Other, MVT::Glue, Operands);
820       SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT,
821                                               /*Glue*/ SDValue(Load, 2));
822 
823       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
824         CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
825 
826       ReplaceUses(SDValue(Node, 0), SDValue(Load, 0));
827       ReplaceUses(SDValue(Node, 1), SDValue(ReadVL, 0)); // VL
828       ReplaceUses(SDValue(Node, 2), SDValue(Load, 1));   // Chain
829       CurDAG->RemoveDeadNode(Node);
830       return;
831     }
832     }
833     break;
834   }
835   case ISD::INTRINSIC_VOID: {
836     unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
837     switch (IntNo) {
838     case Intrinsic::riscv_vsseg2:
839     case Intrinsic::riscv_vsseg3:
840     case Intrinsic::riscv_vsseg4:
841     case Intrinsic::riscv_vsseg5:
842     case Intrinsic::riscv_vsseg6:
843     case Intrinsic::riscv_vsseg7:
844     case Intrinsic::riscv_vsseg8: {
845       selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false);
846       return;
847     }
848     case Intrinsic::riscv_vsseg2_mask:
849     case Intrinsic::riscv_vsseg3_mask:
850     case Intrinsic::riscv_vsseg4_mask:
851     case Intrinsic::riscv_vsseg5_mask:
852     case Intrinsic::riscv_vsseg6_mask:
853     case Intrinsic::riscv_vsseg7_mask:
854     case Intrinsic::riscv_vsseg8_mask: {
855       selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false);
856       return;
857     }
858     case Intrinsic::riscv_vssseg2:
859     case Intrinsic::riscv_vssseg3:
860     case Intrinsic::riscv_vssseg4:
861     case Intrinsic::riscv_vssseg5:
862     case Intrinsic::riscv_vssseg6:
863     case Intrinsic::riscv_vssseg7:
864     case Intrinsic::riscv_vssseg8: {
865       selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true);
866       return;
867     }
868     case Intrinsic::riscv_vssseg2_mask:
869     case Intrinsic::riscv_vssseg3_mask:
870     case Intrinsic::riscv_vssseg4_mask:
871     case Intrinsic::riscv_vssseg5_mask:
872     case Intrinsic::riscv_vssseg6_mask:
873     case Intrinsic::riscv_vssseg7_mask:
874     case Intrinsic::riscv_vssseg8_mask: {
875       selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true);
876       return;
877     }
878     case Intrinsic::riscv_vsoxseg2:
879     case Intrinsic::riscv_vsoxseg3:
880     case Intrinsic::riscv_vsoxseg4:
881     case Intrinsic::riscv_vsoxseg5:
882     case Intrinsic::riscv_vsoxseg6:
883     case Intrinsic::riscv_vsoxseg7:
884     case Intrinsic::riscv_vsoxseg8:
885       selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true);
886       return;
887     case Intrinsic::riscv_vsuxseg2:
888     case Intrinsic::riscv_vsuxseg3:
889     case Intrinsic::riscv_vsuxseg4:
890     case Intrinsic::riscv_vsuxseg5:
891     case Intrinsic::riscv_vsuxseg6:
892     case Intrinsic::riscv_vsuxseg7:
893     case Intrinsic::riscv_vsuxseg8:
894       selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false);
895       return;
896     case Intrinsic::riscv_vsoxseg2_mask:
897     case Intrinsic::riscv_vsoxseg3_mask:
898     case Intrinsic::riscv_vsoxseg4_mask:
899     case Intrinsic::riscv_vsoxseg5_mask:
900     case Intrinsic::riscv_vsoxseg6_mask:
901     case Intrinsic::riscv_vsoxseg7_mask:
902     case Intrinsic::riscv_vsoxseg8_mask:
903       selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true);
904       return;
905     case Intrinsic::riscv_vsuxseg2_mask:
906     case Intrinsic::riscv_vsuxseg3_mask:
907     case Intrinsic::riscv_vsuxseg4_mask:
908     case Intrinsic::riscv_vsuxseg5_mask:
909     case Intrinsic::riscv_vsuxseg6_mask:
910     case Intrinsic::riscv_vsuxseg7_mask:
911     case Intrinsic::riscv_vsuxseg8_mask:
912       selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false);
913       return;
914     case Intrinsic::riscv_vsoxei:
915     case Intrinsic::riscv_vsoxei_mask:
916     case Intrinsic::riscv_vsuxei:
917     case Intrinsic::riscv_vsuxei_mask: {
918       bool IsMasked = IntNo == Intrinsic::riscv_vsoxei_mask ||
919                       IntNo == Intrinsic::riscv_vsuxei_mask;
920       bool IsOrdered = IntNo == Intrinsic::riscv_vsoxei ||
921                        IntNo == Intrinsic::riscv_vsoxei_mask;
922 
923       MVT VT = Node->getOperand(2)->getSimpleValueType(0);
924       unsigned ScalarSize = VT.getScalarSizeInBits();
925       MVT XLenVT = Subtarget->getXLenVT();
926       SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
927 
928       unsigned CurOp = 2;
929       SmallVector<SDValue, 7> Operands;
930       Operands.push_back(Node->getOperand(CurOp++)); // Store value.
931       SDValue Base;
932       SelectBaseAddr(Node->getOperand(CurOp++), Base);
933       Operands.push_back(Base); // Base pointer.
934       Operands.push_back(Node->getOperand(CurOp++)); // Index.
935       MVT IndexVT = Operands.back()->getSimpleValueType(0);
936       if (IsMasked)
937         Operands.push_back(Node->getOperand(CurOp++)); // Mask.
938       SDValue VL;
939       selectVLOp(Node->getOperand(CurOp++), VL);
940       Operands.push_back(VL);
941       Operands.push_back(SEW);
942       Operands.push_back(Node->getOperand(0)); // Chain.
943 
944       assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
945              "Element count mismatch");
946 
947       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
948       RISCVVLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
949       unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
950       const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
951           IsMasked, IsOrdered, IndexScalarSize, static_cast<unsigned>(LMUL),
952           static_cast<unsigned>(IndexLMUL));
953       MachineSDNode *Store =
954           CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
955 
956       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
957         CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
958 
959       ReplaceNode(Node, Store);
960       return;
961     }
962     case Intrinsic::riscv_vse1:
963     case Intrinsic::riscv_vse:
964     case Intrinsic::riscv_vse_mask:
965     case Intrinsic::riscv_vsse:
966     case Intrinsic::riscv_vsse_mask: {
967       bool IsMasked = IntNo == Intrinsic::riscv_vse_mask ||
968                       IntNo == Intrinsic::riscv_vsse_mask;
969       bool IsStrided =
970           IntNo == Intrinsic::riscv_vsse || IntNo == Intrinsic::riscv_vsse_mask;
971 
972       MVT VT = Node->getOperand(2)->getSimpleValueType(0);
973       unsigned ScalarSize = VT.getScalarSizeInBits();
974       MVT XLenVT = Subtarget->getXLenVT();
975       // VSE1 uses an SEW of 8.
976       unsigned SEWImm = (IntNo == Intrinsic::riscv_vse1) ? 8 : ScalarSize;
977       SDValue SEW = CurDAG->getTargetConstant(SEWImm, DL, XLenVT);
978 
979       unsigned CurOp = 2;
980       SmallVector<SDValue, 7> Operands;
981       Operands.push_back(Node->getOperand(CurOp++)); // Store value.
982       SDValue Base;
983       SelectBaseAddr(Node->getOperand(CurOp++), Base);
984       Operands.push_back(Base); // Base pointer.
985       if (IsStrided)
986         Operands.push_back(Node->getOperand(CurOp++)); // Stride.
987       if (IsMasked)
988         Operands.push_back(Node->getOperand(CurOp++)); // Mask.
989       SDValue VL;
990       selectVLOp(Node->getOperand(CurOp++), VL);
991       Operands.push_back(VL);
992       Operands.push_back(SEW);
993       Operands.push_back(Node->getOperand(0)); // Chain.
994 
995       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
996       const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(
997           IsMasked, IsStrided, ScalarSize, static_cast<unsigned>(LMUL));
998       MachineSDNode *Store =
999           CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
1000 
1001       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1002         CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
1003 
1004       ReplaceNode(Node, Store);
1005       return;
1006     }
1007     }
1008     break;
1009   }
1010   case ISD::BITCAST: {
1011     MVT SrcVT = Node->getOperand(0).getSimpleValueType();
1012     // Just drop bitcasts between vectors if both are fixed or both are
1013     // scalable.
1014     if ((VT.isScalableVector() && SrcVT.isScalableVector()) ||
1015         (VT.isFixedLengthVector() && SrcVT.isFixedLengthVector())) {
1016       ReplaceUses(SDValue(Node, 0), Node->getOperand(0));
1017       CurDAG->RemoveDeadNode(Node);
1018       return;
1019     }
1020     break;
1021   }
1022   case ISD::INSERT_SUBVECTOR: {
1023     SDValue V = Node->getOperand(0);
1024     SDValue SubV = Node->getOperand(1);
1025     SDLoc DL(SubV);
1026     auto Idx = Node->getConstantOperandVal(2);
1027     MVT SubVecVT = SubV.getSimpleValueType();
1028 
1029     MVT SubVecContainerVT = SubVecVT;
1030     // Establish the correct scalable-vector types for any fixed-length type.
1031     if (SubVecVT.isFixedLengthVector())
1032       SubVecContainerVT = RISCVTargetLowering::getContainerForFixedLengthVector(
1033           *CurDAG, SubVecVT, *Subtarget);
1034     if (VT.isFixedLengthVector())
1035       VT = RISCVTargetLowering::getContainerForFixedLengthVector(*CurDAG, VT,
1036                                                                  *Subtarget);
1037 
1038     const auto *TRI = Subtarget->getRegisterInfo();
1039     unsigned SubRegIdx;
1040     std::tie(SubRegIdx, Idx) =
1041         RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1042             VT, SubVecContainerVT, Idx, TRI);
1043 
1044     // If the Idx hasn't been completely eliminated then this is a subvector
1045     // insert which doesn't naturally align to a vector register. These must
1046     // be handled using instructions to manipulate the vector registers.
1047     if (Idx != 0)
1048       break;
1049 
1050     RISCVVLMUL SubVecLMUL = RISCVTargetLowering::getLMUL(SubVecContainerVT);
1051     bool IsSubVecPartReg = SubVecLMUL == RISCVVLMUL::LMUL_F2 ||
1052                            SubVecLMUL == RISCVVLMUL::LMUL_F4 ||
1053                            SubVecLMUL == RISCVVLMUL::LMUL_F8;
1054     (void)IsSubVecPartReg; // Silence unused variable warning without asserts.
1055     assert((!IsSubVecPartReg || V.isUndef()) &&
1056            "Expecting lowering to have created legal INSERT_SUBVECTORs when "
1057            "the subvector is smaller than a full-sized register");
1058 
1059     // If we haven't set a SubRegIdx, then we must be going between
1060     // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy.
1061     if (SubRegIdx == RISCV::NoSubRegister) {
1062       unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT);
1063       assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) ==
1064                  InRegClassID &&
1065              "Unexpected subvector extraction");
1066       SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT);
1067       SDNode *NewNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1068                                                DL, VT, SubV, RC);
1069       return ReplaceNode(Node, NewNode);
1070     }
1071 
1072     SDValue Insert = CurDAG->getTargetInsertSubreg(SubRegIdx, DL, VT, V, SubV);
1073     return ReplaceNode(Node, Insert.getNode());
1074   }
1075   case ISD::EXTRACT_SUBVECTOR: {
1076     SDValue V = Node->getOperand(0);
1077     auto Idx = Node->getConstantOperandVal(1);
1078     MVT InVT = V.getSimpleValueType();
1079     SDLoc DL(V);
1080 
1081     MVT SubVecContainerVT = VT;
1082     // Establish the correct scalable-vector types for any fixed-length type.
1083     if (VT.isFixedLengthVector())
1084       SubVecContainerVT = RISCVTargetLowering::getContainerForFixedLengthVector(
1085           *CurDAG, VT, *Subtarget);
1086     if (InVT.isFixedLengthVector())
1087       InVT = RISCVTargetLowering::getContainerForFixedLengthVector(
1088           *CurDAG, InVT, *Subtarget);
1089 
1090     const auto *TRI = Subtarget->getRegisterInfo();
1091     unsigned SubRegIdx;
1092     std::tie(SubRegIdx, Idx) =
1093         RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1094             InVT, SubVecContainerVT, Idx, TRI);
1095 
1096     // If the Idx hasn't been completely eliminated then this is a subvector
1097     // extract which doesn't naturally align to a vector register. These must
1098     // be handled using instructions to manipulate the vector registers.
1099     if (Idx != 0)
1100       break;
1101 
1102     // If we haven't set a SubRegIdx, then we must be going between
1103     // equally-sized LMUL types (e.g. VR -> VR). This can be done as a copy.
1104     if (SubRegIdx == RISCV::NoSubRegister) {
1105       unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(InVT);
1106       assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) ==
1107                  InRegClassID &&
1108              "Unexpected subvector extraction");
1109       SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT);
1110       SDNode *NewNode =
1111           CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, DL, VT, V, RC);
1112       return ReplaceNode(Node, NewNode);
1113     }
1114 
1115     SDValue Extract = CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, V);
1116     return ReplaceNode(Node, Extract.getNode());
1117   }
1118   }
1119 
1120   // Select the default instruction.
1121   SelectCode(Node);
1122 }
1123 
1124 bool RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand(
1125     const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {
1126   switch (ConstraintID) {
1127   case InlineAsm::Constraint_m:
1128     // We just support simple memory operands that have a single address
1129     // operand and need no special handling.
1130     OutOps.push_back(Op);
1131     return false;
1132   case InlineAsm::Constraint_A:
1133     OutOps.push_back(Op);
1134     return false;
1135   default:
1136     break;
1137   }
1138 
1139   return true;
1140 }
1141 
1142 bool RISCVDAGToDAGISel::SelectAddrFI(SDValue Addr, SDValue &Base) {
1143   if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
1144     Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT());
1145     return true;
1146   }
1147   return false;
1148 }
1149 
1150 bool RISCVDAGToDAGISel::SelectBaseAddr(SDValue Addr, SDValue &Base) {
1151   // If this is FrameIndex, select it directly. Otherwise just let it get
1152   // selected to a register independently.
1153   if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr))
1154     Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT());
1155   else
1156     Base = Addr;
1157   return true;
1158 }
1159 
1160 bool RISCVDAGToDAGISel::selectShiftMask(SDValue N, unsigned ShiftWidth,
1161                                         SDValue &ShAmt) {
1162   // Shift instructions on RISCV only read the lower 5 or 6 bits of the shift
1163   // amount. If there is an AND on the shift amount, we can bypass it if it
1164   // doesn't affect any of those bits.
1165   if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(N.getOperand(1))) {
1166     const APInt &AndMask = N->getConstantOperandAPInt(1);
1167 
1168     // Since the max shift amount is a power of 2 we can subtract 1 to make a
1169     // mask that covers the bits needed to represent all shift amounts.
1170     assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!");
1171     APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
1172 
1173     if (ShMask.isSubsetOf(AndMask)) {
1174       ShAmt = N.getOperand(0);
1175       return true;
1176     }
1177 
1178     // SimplifyDemandedBits may have optimized the mask so try restoring any
1179     // bits that are known zero.
1180     KnownBits Known = CurDAG->computeKnownBits(N->getOperand(0));
1181     if (ShMask.isSubsetOf(AndMask | Known.Zero)) {
1182       ShAmt = N.getOperand(0);
1183       return true;
1184     }
1185   }
1186 
1187   ShAmt = N;
1188   return true;
1189 }
1190 
1191 bool RISCVDAGToDAGISel::selectSExti32(SDValue N, SDValue &Val) {
1192   if (N.getOpcode() == ISD::SIGN_EXTEND_INREG &&
1193       cast<VTSDNode>(N.getOperand(1))->getVT() == MVT::i32) {
1194     Val = N.getOperand(0);
1195     return true;
1196   }
1197   // FIXME: Should we just call computeNumSignBits here?
1198   if (N.getOpcode() == ISD::AssertSext &&
1199       cast<VTSDNode>(N->getOperand(1))->getVT().bitsLE(MVT::i32)) {
1200     Val = N;
1201     return true;
1202   }
1203   if (N.getOpcode() == ISD::AssertZext &&
1204       cast<VTSDNode>(N->getOperand(1))->getVT().bitsLT(MVT::i32)) {
1205     Val = N;
1206     return true;
1207   }
1208 
1209   return false;
1210 }
1211 
1212 bool RISCVDAGToDAGISel::selectZExti32(SDValue N, SDValue &Val) {
1213   if (N.getOpcode() == ISD::AND) {
1214     auto *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
1215     if (C && C->getZExtValue() == UINT64_C(0xFFFFFFFF)) {
1216       Val = N.getOperand(0);
1217       return true;
1218     }
1219   }
1220   // FIXME: Should we just call computeKnownBits here?
1221   if (N.getOpcode() == ISD::AssertZext &&
1222       cast<VTSDNode>(N->getOperand(1))->getVT().bitsLE(MVT::i32)) {
1223     Val = N;
1224     return true;
1225   }
1226 
1227   return false;
1228 }
1229 
1230 // Check that it is a SLLIUW (Shift Logical Left Immediate Unsigned i32
1231 // on RV64).
1232 // SLLIUW is the same as SLLI except for the fact that it clears the bits
1233 // XLEN-1:32 of the input RS1 before shifting.
1234 // A PatFrag has already checked that it has the right structure:
1235 //
1236 //  (AND (SHL RS1, VC2), VC1)
1237 //
1238 // We check that VC2, the shamt is less than 32, otherwise the pattern is
1239 // exactly the same as SLLI and we give priority to that.
1240 // Eventually we check that VC1, the mask used to clear the upper 32 bits
1241 // of RS1, is correct:
1242 //
1243 //  VC1 == (0xFFFFFFFF << VC2)
1244 //
1245 bool RISCVDAGToDAGISel::MatchSLLIUW(SDNode *N) const {
1246   assert(N->getOpcode() == ISD::AND);
1247   assert(N->getOperand(0).getOpcode() == ISD::SHL);
1248   assert(isa<ConstantSDNode>(N->getOperand(1)));
1249   assert(isa<ConstantSDNode>(N->getOperand(0).getOperand(1)));
1250 
1251   // The IsRV64 predicate is checked after PatFrag predicates so we can get
1252   // here even on RV32.
1253   if (!Subtarget->is64Bit())
1254     return false;
1255 
1256   SDValue Shl = N->getOperand(0);
1257   uint64_t VC1 = N->getConstantOperandVal(1);
1258   uint64_t VC2 = Shl.getConstantOperandVal(1);
1259 
1260   // Immediate range should be enforced by uimm5 predicate.
1261   assert(VC2 < 32 && "Unexpected immediate");
1262   return (VC1 >> VC2) == UINT64_C(0xFFFFFFFF);
1263 }
1264 
1265 // X0 has special meaning for vsetvl/vsetvli.
1266 //  rd | rs1 |   AVL value | Effect on vl
1267 //--------------------------------------------------------------
1268 // !X0 |  X0 |       VLMAX | Set vl to VLMAX
1269 //  X0 |  X0 | Value in vl | Keep current vl, just change vtype.
1270 bool RISCVDAGToDAGISel::selectVLOp(SDValue N, SDValue &VL) {
1271   // If the VL value is a constant 0, manually select it to an ADDI with 0
1272   // immediate to prevent the default selection path from matching it to X0.
1273   auto *C = dyn_cast<ConstantSDNode>(N);
1274   if (C && C->isNullValue())
1275     VL = SDValue(selectImm(CurDAG, SDLoc(N), 0, Subtarget->getXLenVT()), 0);
1276   else
1277     VL = N;
1278 
1279   return true;
1280 }
1281 
1282 bool RISCVDAGToDAGISel::selectVSplat(SDValue N, SDValue &SplatVal) {
1283   if (N.getOpcode() != ISD::SPLAT_VECTOR &&
1284       N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64 &&
1285       N.getOpcode() != RISCVISD::VMV_V_X_VL)
1286     return false;
1287   SplatVal = N.getOperand(0);
1288   return true;
1289 }
1290 
1291 bool RISCVDAGToDAGISel::selectVSplatSimm5(SDValue N, SDValue &SplatVal) {
1292   if ((N.getOpcode() != ISD::SPLAT_VECTOR &&
1293        N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64 &&
1294        N.getOpcode() != RISCVISD::VMV_V_X_VL) ||
1295       !isa<ConstantSDNode>(N.getOperand(0)))
1296     return false;
1297 
1298   int64_t SplatImm = cast<ConstantSDNode>(N.getOperand(0))->getSExtValue();
1299 
1300   // Both ISD::SPLAT_VECTOR and RISCVISD::SPLAT_VECTOR_I64 share semantics when
1301   // the operand type is wider than the resulting vector element type: an
1302   // implicit truncation first takes place. Therefore, perform a manual
1303   // truncation/sign-extension in order to ignore any truncated bits and catch
1304   // any zero-extended immediate.
1305   // For example, we wish to match (i8 -1) -> (XLenVT 255) as a simm5 by first
1306   // sign-extending to (XLenVT -1).
1307   MVT XLenVT = Subtarget->getXLenVT();
1308   assert(XLenVT == N.getOperand(0).getSimpleValueType() &&
1309          "Unexpected splat operand type");
1310   MVT EltVT = N.getSimpleValueType().getVectorElementType();
1311   if (EltVT.bitsLT(XLenVT)) {
1312     SplatImm = SignExtend64(SplatImm, EltVT.getSizeInBits());
1313   }
1314 
1315   if (!isInt<5>(SplatImm))
1316     return false;
1317 
1318   SplatVal = CurDAG->getTargetConstant(SplatImm, SDLoc(N), XLenVT);
1319   return true;
1320 }
1321 
1322 bool RISCVDAGToDAGISel::selectVSplatUimm5(SDValue N, SDValue &SplatVal) {
1323   if ((N.getOpcode() != ISD::SPLAT_VECTOR &&
1324        N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64 &&
1325        N.getOpcode() != RISCVISD::VMV_V_X_VL) ||
1326       !isa<ConstantSDNode>(N.getOperand(0)))
1327     return false;
1328 
1329   int64_t SplatImm = cast<ConstantSDNode>(N.getOperand(0))->getSExtValue();
1330 
1331   if (!isUInt<5>(SplatImm))
1332     return false;
1333 
1334   SplatVal =
1335       CurDAG->getTargetConstant(SplatImm, SDLoc(N), Subtarget->getXLenVT());
1336 
1337   return true;
1338 }
1339 
1340 bool RISCVDAGToDAGISel::selectRVVSimm5(SDValue N, unsigned Width,
1341                                        SDValue &Imm) {
1342   if (auto *C = dyn_cast<ConstantSDNode>(N)) {
1343     int64_t ImmVal = SignExtend64(C->getSExtValue(), Width);
1344 
1345     if (!isInt<5>(ImmVal))
1346       return false;
1347 
1348     Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), Subtarget->getXLenVT());
1349     return true;
1350   }
1351 
1352   return false;
1353 }
1354 
1355 bool RISCVDAGToDAGISel::selectRVVUimm5(SDValue N, unsigned Width,
1356                                        SDValue &Imm) {
1357   if (auto *C = dyn_cast<ConstantSDNode>(N)) {
1358     int64_t ImmVal = C->getSExtValue();
1359 
1360     if (!isUInt<5>(ImmVal))
1361       return false;
1362 
1363     Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), Subtarget->getXLenVT());
1364     return true;
1365   }
1366 
1367   return false;
1368 }
1369 
1370 // Merge an ADDI into the offset of a load/store instruction where possible.
1371 // (load (addi base, off1), off2) -> (load base, off1+off2)
1372 // (store val, (addi base, off1), off2) -> (store val, base, off1+off2)
1373 // This is possible when off1+off2 fits a 12-bit immediate.
1374 void RISCVDAGToDAGISel::doPeepholeLoadStoreADDI() {
1375   SelectionDAG::allnodes_iterator Position(CurDAG->getRoot().getNode());
1376   ++Position;
1377 
1378   while (Position != CurDAG->allnodes_begin()) {
1379     SDNode *N = &*--Position;
1380     // Skip dead nodes and any non-machine opcodes.
1381     if (N->use_empty() || !N->isMachineOpcode())
1382       continue;
1383 
1384     int OffsetOpIdx;
1385     int BaseOpIdx;
1386 
1387     // Only attempt this optimisation for I-type loads and S-type stores.
1388     switch (N->getMachineOpcode()) {
1389     default:
1390       continue;
1391     case RISCV::LB:
1392     case RISCV::LH:
1393     case RISCV::LW:
1394     case RISCV::LBU:
1395     case RISCV::LHU:
1396     case RISCV::LWU:
1397     case RISCV::LD:
1398     case RISCV::FLH:
1399     case RISCV::FLW:
1400     case RISCV::FLD:
1401       BaseOpIdx = 0;
1402       OffsetOpIdx = 1;
1403       break;
1404     case RISCV::SB:
1405     case RISCV::SH:
1406     case RISCV::SW:
1407     case RISCV::SD:
1408     case RISCV::FSH:
1409     case RISCV::FSW:
1410     case RISCV::FSD:
1411       BaseOpIdx = 1;
1412       OffsetOpIdx = 2;
1413       break;
1414     }
1415 
1416     if (!isa<ConstantSDNode>(N->getOperand(OffsetOpIdx)))
1417       continue;
1418 
1419     SDValue Base = N->getOperand(BaseOpIdx);
1420 
1421     // If the base is an ADDI, we can merge it in to the load/store.
1422     if (!Base.isMachineOpcode() || Base.getMachineOpcode() != RISCV::ADDI)
1423       continue;
1424 
1425     SDValue ImmOperand = Base.getOperand(1);
1426     uint64_t Offset2 = N->getConstantOperandVal(OffsetOpIdx);
1427 
1428     if (auto *Const = dyn_cast<ConstantSDNode>(ImmOperand)) {
1429       int64_t Offset1 = Const->getSExtValue();
1430       int64_t CombinedOffset = Offset1 + Offset2;
1431       if (!isInt<12>(CombinedOffset))
1432         continue;
1433       ImmOperand = CurDAG->getTargetConstant(CombinedOffset, SDLoc(ImmOperand),
1434                                              ImmOperand.getValueType());
1435     } else if (auto *GA = dyn_cast<GlobalAddressSDNode>(ImmOperand)) {
1436       // If the off1 in (addi base, off1) is a global variable's address (its
1437       // low part, really), then we can rely on the alignment of that variable
1438       // to provide a margin of safety before off1 can overflow the 12 bits.
1439       // Check if off2 falls within that margin; if so off1+off2 can't overflow.
1440       const DataLayout &DL = CurDAG->getDataLayout();
1441       Align Alignment = GA->getGlobal()->getPointerAlignment(DL);
1442       if (Offset2 != 0 && Alignment <= Offset2)
1443         continue;
1444       int64_t Offset1 = GA->getOffset();
1445       int64_t CombinedOffset = Offset1 + Offset2;
1446       ImmOperand = CurDAG->getTargetGlobalAddress(
1447           GA->getGlobal(), SDLoc(ImmOperand), ImmOperand.getValueType(),
1448           CombinedOffset, GA->getTargetFlags());
1449     } else if (auto *CP = dyn_cast<ConstantPoolSDNode>(ImmOperand)) {
1450       // Ditto.
1451       Align Alignment = CP->getAlign();
1452       if (Offset2 != 0 && Alignment <= Offset2)
1453         continue;
1454       int64_t Offset1 = CP->getOffset();
1455       int64_t CombinedOffset = Offset1 + Offset2;
1456       ImmOperand = CurDAG->getTargetConstantPool(
1457           CP->getConstVal(), ImmOperand.getValueType(), CP->getAlign(),
1458           CombinedOffset, CP->getTargetFlags());
1459     } else {
1460       continue;
1461     }
1462 
1463     LLVM_DEBUG(dbgs() << "Folding add-immediate into mem-op:\nBase:    ");
1464     LLVM_DEBUG(Base->dump(CurDAG));
1465     LLVM_DEBUG(dbgs() << "\nN: ");
1466     LLVM_DEBUG(N->dump(CurDAG));
1467     LLVM_DEBUG(dbgs() << "\n");
1468 
1469     // Modify the offset operand of the load/store.
1470     if (BaseOpIdx == 0) // Load
1471       CurDAG->UpdateNodeOperands(N, Base.getOperand(0), ImmOperand,
1472                                  N->getOperand(2));
1473     else // Store
1474       CurDAG->UpdateNodeOperands(N, N->getOperand(0), Base.getOperand(0),
1475                                  ImmOperand, N->getOperand(3));
1476 
1477     // The add-immediate may now be dead, in which case remove it.
1478     if (Base.getNode()->use_empty())
1479       CurDAG->RemoveDeadNode(Base.getNode());
1480   }
1481 }
1482 
1483 // This pass converts a legalized DAG into a RISCV-specific DAG, ready
1484 // for instruction scheduling.
1485 FunctionPass *llvm::createRISCVISelDag(RISCVTargetMachine &TM) {
1486   return new RISCVDAGToDAGISel(TM);
1487 }
1488