1 //===-- RISCVISelDAGToDAG.cpp - A dag to dag inst selector for RISCV ------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines an instruction selector for the RISCV target.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "RISCVISelDAGToDAG.h"
14 #include "MCTargetDesc/RISCVMCTargetDesc.h"
15 #include "MCTargetDesc/RISCVMatInt.h"
16 #include "RISCVISelLowering.h"
17 #include "llvm/CodeGen/MachineFrameInfo.h"
18 #include "llvm/IR/IntrinsicsRISCV.h"
19 #include "llvm/Support/Alignment.h"
20 #include "llvm/Support/Debug.h"
21 #include "llvm/Support/KnownBits.h"
22 #include "llvm/Support/MathExtras.h"
23 #include "llvm/Support/raw_ostream.h"
24 
25 using namespace llvm;
26 
27 #define DEBUG_TYPE "riscv-isel"
28 
29 namespace llvm {
30 namespace RISCV {
31 #define GET_RISCVVSSEGTable_IMPL
32 #define GET_RISCVVLSEGTable_IMPL
33 #define GET_RISCVVLXSEGTable_IMPL
34 #define GET_RISCVVSXSEGTable_IMPL
35 #define GET_RISCVVLETable_IMPL
36 #define GET_RISCVVSETable_IMPL
37 #define GET_RISCVVLXTable_IMPL
38 #define GET_RISCVVSXTable_IMPL
39 #include "RISCVGenSearchableTables.inc"
40 } // namespace RISCV
41 } // namespace llvm
42 
43 void RISCVDAGToDAGISel::PostprocessISelDAG() {
44   doPeepholeLoadStoreADDI();
45 }
46 
47 static SDNode *selectImm(SelectionDAG *CurDAG, const SDLoc &DL, int64_t Imm,
48                          MVT XLenVT) {
49   RISCVMatInt::InstSeq Seq;
50   RISCVMatInt::generateInstSeq(Imm, XLenVT == MVT::i64, Seq);
51 
52   SDNode *Result = nullptr;
53   SDValue SrcReg = CurDAG->getRegister(RISCV::X0, XLenVT);
54   for (RISCVMatInt::Inst &Inst : Seq) {
55     SDValue SDImm = CurDAG->getTargetConstant(Inst.Imm, DL, XLenVT);
56     if (Inst.Opc == RISCV::LUI)
57       Result = CurDAG->getMachineNode(RISCV::LUI, DL, XLenVT, SDImm);
58     else
59       Result = CurDAG->getMachineNode(Inst.Opc, DL, XLenVT, SrcReg, SDImm);
60 
61     // Only the first instruction has X0 as its source.
62     SrcReg = SDValue(Result, 0);
63   }
64 
65   return Result;
66 }
67 
68 static SDValue createTupleImpl(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
69                                unsigned RegClassID, unsigned SubReg0) {
70   assert(Regs.size() >= 2 && Regs.size() <= 8);
71 
72   SDLoc DL(Regs[0]);
73   SmallVector<SDValue, 8> Ops;
74 
75   Ops.push_back(CurDAG.getTargetConstant(RegClassID, DL, MVT::i32));
76 
77   for (unsigned I = 0; I < Regs.size(); ++I) {
78     Ops.push_back(Regs[I]);
79     Ops.push_back(CurDAG.getTargetConstant(SubReg0 + I, DL, MVT::i32));
80   }
81   SDNode *N =
82       CurDAG.getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops);
83   return SDValue(N, 0);
84 }
85 
86 static SDValue createM1Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
87                              unsigned NF) {
88   static const unsigned RegClassIDs[] = {
89       RISCV::VRN2M1RegClassID, RISCV::VRN3M1RegClassID, RISCV::VRN4M1RegClassID,
90       RISCV::VRN5M1RegClassID, RISCV::VRN6M1RegClassID, RISCV::VRN7M1RegClassID,
91       RISCV::VRN8M1RegClassID};
92 
93   return createTupleImpl(CurDAG, Regs, RegClassIDs[NF - 2], RISCV::sub_vrm1_0);
94 }
95 
96 static SDValue createM2Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
97                              unsigned NF) {
98   static const unsigned RegClassIDs[] = {RISCV::VRN2M2RegClassID,
99                                          RISCV::VRN3M2RegClassID,
100                                          RISCV::VRN4M2RegClassID};
101 
102   return createTupleImpl(CurDAG, Regs, RegClassIDs[NF - 2], RISCV::sub_vrm2_0);
103 }
104 
105 static SDValue createM4Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
106                              unsigned NF) {
107   return createTupleImpl(CurDAG, Regs, RISCV::VRN2M4RegClassID,
108                          RISCV::sub_vrm4_0);
109 }
110 
111 static SDValue createTuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
112                            unsigned NF, RISCVVLMUL LMUL) {
113   switch (LMUL) {
114   default:
115     llvm_unreachable("Invalid LMUL.");
116   case RISCVVLMUL::LMUL_F8:
117   case RISCVVLMUL::LMUL_F4:
118   case RISCVVLMUL::LMUL_F2:
119   case RISCVVLMUL::LMUL_1:
120     return createM1Tuple(CurDAG, Regs, NF);
121   case RISCVVLMUL::LMUL_2:
122     return createM2Tuple(CurDAG, Regs, NF);
123   case RISCVVLMUL::LMUL_4:
124     return createM4Tuple(CurDAG, Regs, NF);
125   }
126 }
127 
128 void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, bool IsMasked,
129                                     bool IsStrided) {
130   SDLoc DL(Node);
131   unsigned NF = Node->getNumValues() - 1;
132   MVT VT = Node->getSimpleValueType(0);
133   unsigned ScalarSize = VT.getScalarSizeInBits();
134   MVT XLenVT = Subtarget->getXLenVT();
135   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
136   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
137   unsigned CurOp = 2;
138   SmallVector<SDValue, 7> Operands;
139   if (IsMasked) {
140     SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
141                                  Node->op_begin() + CurOp + NF);
142     SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
143     Operands.push_back(MaskedOff);
144     CurOp += NF;
145   }
146   SDValue Base;
147   SelectBaseAddr(Node->getOperand(CurOp++), Base);
148   Operands.push_back(Base); // Base pointer.
149   if (IsStrided)
150     Operands.push_back(Node->getOperand(CurOp++)); // Stride.
151   if (IsMasked)
152     Operands.push_back(Node->getOperand(CurOp++)); // Mask.
153   SDValue VL;
154   selectVLOp(Node->getOperand(CurOp++), VL);
155   Operands.push_back(VL);
156   Operands.push_back(SEW);
157   Operands.push_back(Node->getOperand(0)); // Chain.
158   const RISCV::VLSEGPseudo *P =
159       RISCV::getVLSEGPseudo(NF, IsMasked, IsStrided, /*FF*/ false, ScalarSize,
160                             static_cast<unsigned>(LMUL));
161   MachineSDNode *Load =
162       CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
163 
164   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
165     CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
166 
167   SDValue SuperReg = SDValue(Load, 0);
168   for (unsigned I = 0; I < NF; ++I) {
169     unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
170     ReplaceUses(SDValue(Node, I),
171                 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
172   }
173 
174   ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
175   CurDAG->RemoveDeadNode(Node);
176 }
177 
178 void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node, bool IsMasked) {
179   SDLoc DL(Node);
180   unsigned NF = Node->getNumValues() - 2; // Do not count VL and Chain.
181   MVT VT = Node->getSimpleValueType(0);
182   MVT XLenVT = Subtarget->getXLenVT();
183   unsigned ScalarSize = VT.getScalarSizeInBits();
184   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
185   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
186 
187   unsigned CurOp = 2;
188   SmallVector<SDValue, 7> Operands;
189   if (IsMasked) {
190     SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
191                                  Node->op_begin() + CurOp + NF);
192     SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
193     Operands.push_back(MaskedOff);
194     CurOp += NF;
195   }
196   SDValue Base;
197   SelectBaseAddr(Node->getOperand(CurOp++), Base);
198   Operands.push_back(Base); // Base pointer.
199   if (IsMasked)
200     Operands.push_back(Node->getOperand(CurOp++)); // Mask.
201   SDValue VL;
202   selectVLOp(Node->getOperand(CurOp++), VL);
203   Operands.push_back(VL);
204   Operands.push_back(SEW);
205   Operands.push_back(Node->getOperand(0)); // Chain.
206   const RISCV::VLSEGPseudo *P =
207       RISCV::getVLSEGPseudo(NF, IsMasked, /*Strided*/ false, /*FF*/ true,
208                             ScalarSize, static_cast<unsigned>(LMUL));
209   MachineSDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped,
210                                                MVT::Other, MVT::Glue, Operands);
211   SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT,
212                                           /*Glue*/ SDValue(Load, 2));
213 
214   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
215     CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
216 
217   SDValue SuperReg = SDValue(Load, 0);
218   for (unsigned I = 0; I < NF; ++I) {
219     unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
220     ReplaceUses(SDValue(Node, I),
221                 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
222   }
223 
224   ReplaceUses(SDValue(Node, NF), SDValue(ReadVL, 0));   // VL
225   ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 1)); // Chain
226   CurDAG->RemoveDeadNode(Node);
227 }
228 
229 void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, bool IsMasked,
230                                      bool IsOrdered) {
231   SDLoc DL(Node);
232   unsigned NF = Node->getNumValues() - 1;
233   MVT VT = Node->getSimpleValueType(0);
234   unsigned ScalarSize = VT.getScalarSizeInBits();
235   MVT XLenVT = Subtarget->getXLenVT();
236   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
237   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
238   unsigned CurOp = 2;
239   SmallVector<SDValue, 7> Operands;
240   if (IsMasked) {
241     SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
242                                  Node->op_begin() + CurOp + NF);
243     SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
244     Operands.push_back(MaskedOff);
245     CurOp += NF;
246   }
247   SDValue Base;
248   SelectBaseAddr(Node->getOperand(CurOp++), Base);
249   Operands.push_back(Base); // Base pointer.
250   Operands.push_back(Node->getOperand(CurOp++)); // Index.
251   MVT IndexVT = Operands.back()->getSimpleValueType(0);
252   if (IsMasked)
253     Operands.push_back(Node->getOperand(CurOp++)); // Mask.
254   SDValue VL;
255   selectVLOp(Node->getOperand(CurOp++), VL);
256   Operands.push_back(VL);
257   Operands.push_back(SEW);
258   Operands.push_back(Node->getOperand(0)); // Chain.
259 
260   assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
261          "Element count mismatch");
262 
263   RISCVVLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
264   unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
265   const RISCV::VLXSEGPseudo *P = RISCV::getVLXSEGPseudo(
266       NF, IsMasked, IsOrdered, IndexScalarSize, static_cast<unsigned>(LMUL),
267       static_cast<unsigned>(IndexLMUL));
268   MachineSDNode *Load =
269       CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
270 
271   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
272     CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
273 
274   SDValue SuperReg = SDValue(Load, 0);
275   for (unsigned I = 0; I < NF; ++I) {
276     unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
277     ReplaceUses(SDValue(Node, I),
278                 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
279   }
280 
281   ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
282   CurDAG->RemoveDeadNode(Node);
283 }
284 
285 void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, bool IsMasked,
286                                     bool IsStrided) {
287   SDLoc DL(Node);
288   unsigned NF = Node->getNumOperands() - 4;
289   if (IsStrided)
290     NF--;
291   if (IsMasked)
292     NF--;
293   MVT VT = Node->getOperand(2)->getSimpleValueType(0);
294   unsigned ScalarSize = VT.getScalarSizeInBits();
295   MVT XLenVT = Subtarget->getXLenVT();
296   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
297   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
298   SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
299   SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
300   SmallVector<SDValue, 7> Operands;
301   Operands.push_back(StoreVal);
302   unsigned CurOp = 2 + NF;
303   SDValue Base;
304   SelectBaseAddr(Node->getOperand(CurOp++), Base);
305   Operands.push_back(Base); // Base pointer.
306   if (IsStrided)
307     Operands.push_back(Node->getOperand(CurOp++)); // Stride.
308   if (IsMasked)
309     Operands.push_back(Node->getOperand(CurOp++)); // Mask.
310   SDValue VL;
311   selectVLOp(Node->getOperand(CurOp++), VL);
312   Operands.push_back(VL);
313   Operands.push_back(SEW);
314   Operands.push_back(Node->getOperand(0)); // Chain.
315   const RISCV::VSSEGPseudo *P = RISCV::getVSSEGPseudo(
316       NF, IsMasked, IsStrided, ScalarSize, static_cast<unsigned>(LMUL));
317   MachineSDNode *Store =
318       CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
319 
320   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
321     CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
322 
323   ReplaceNode(Node, Store);
324 }
325 
326 void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, bool IsMasked,
327                                      bool IsOrdered) {
328   SDLoc DL(Node);
329   unsigned NF = Node->getNumOperands() - 5;
330   if (IsMasked)
331     --NF;
332   MVT VT = Node->getOperand(2)->getSimpleValueType(0);
333   unsigned ScalarSize = VT.getScalarSizeInBits();
334   MVT XLenVT = Subtarget->getXLenVT();
335   RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
336   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
337   SmallVector<SDValue, 7> Operands;
338   SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
339   SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
340   Operands.push_back(StoreVal);
341   unsigned CurOp = 2 + NF;
342   SDValue Base;
343   SelectBaseAddr(Node->getOperand(CurOp++), Base);
344   Operands.push_back(Base); // Base pointer.
345   Operands.push_back(Node->getOperand(CurOp++)); // Index.
346   MVT IndexVT = Operands.back()->getSimpleValueType(0);
347   if (IsMasked)
348     Operands.push_back(Node->getOperand(CurOp++)); // Mask.
349   SDValue VL;
350   selectVLOp(Node->getOperand(CurOp++), VL);
351   Operands.push_back(VL);
352   Operands.push_back(SEW);
353   Operands.push_back(Node->getOperand(0)); // Chain.
354 
355   assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
356          "Element count mismatch");
357 
358   RISCVVLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
359   unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
360   const RISCV::VSXSEGPseudo *P = RISCV::getVSXSEGPseudo(
361       NF, IsMasked, IsOrdered, IndexScalarSize, static_cast<unsigned>(LMUL),
362       static_cast<unsigned>(IndexLMUL));
363   MachineSDNode *Store =
364       CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
365 
366   if (auto *MemOp = dyn_cast<MemSDNode>(Node))
367     CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
368 
369   ReplaceNode(Node, Store);
370 }
371 
372 
373 void RISCVDAGToDAGISel::Select(SDNode *Node) {
374   // If we have a custom node, we have already selected.
375   if (Node->isMachineOpcode()) {
376     LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n");
377     Node->setNodeId(-1);
378     return;
379   }
380 
381   // Instruction Selection not handled by the auto-generated tablegen selection
382   // should be handled here.
383   unsigned Opcode = Node->getOpcode();
384   MVT XLenVT = Subtarget->getXLenVT();
385   SDLoc DL(Node);
386   MVT VT = Node->getSimpleValueType(0);
387 
388   switch (Opcode) {
389   case ISD::ADD: {
390     // Optimize (add r, imm) to (addi (addi r, imm0) imm1) if applicable. The
391     // immediate must be in specific ranges and have a single use.
392     if (auto *ConstOp = dyn_cast<ConstantSDNode>(Node->getOperand(1))) {
393       if (!(ConstOp->hasOneUse()))
394         break;
395       // The imm must be in range [-4096,-2049] or [2048,4094].
396       int64_t Imm = ConstOp->getSExtValue();
397       if (!(-4096 <= Imm && Imm <= -2049) && !(2048 <= Imm && Imm <= 4094))
398         break;
399       // Break the imm to imm0+imm1.
400       const SDValue ImmOp0 = CurDAG->getTargetConstant(Imm - Imm / 2, DL, VT);
401       const SDValue ImmOp1 = CurDAG->getTargetConstant(Imm / 2, DL, VT);
402       auto *NodeAddi0 = CurDAG->getMachineNode(RISCV::ADDI, DL, VT,
403                                                Node->getOperand(0), ImmOp0);
404       auto *NodeAddi1 = CurDAG->getMachineNode(RISCV::ADDI, DL, VT,
405                                                SDValue(NodeAddi0, 0), ImmOp1);
406       ReplaceNode(Node, NodeAddi1);
407       return;
408     }
409     break;
410   }
411   case ISD::Constant: {
412     auto *ConstNode = cast<ConstantSDNode>(Node);
413     if (VT == XLenVT && ConstNode->isNullValue()) {
414       SDValue New =
415           CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, RISCV::X0, XLenVT);
416       ReplaceNode(Node, New.getNode());
417       return;
418     }
419     int64_t Imm = ConstNode->getSExtValue();
420     if (XLenVT == MVT::i64) {
421       ReplaceNode(Node, selectImm(CurDAG, DL, Imm, XLenVT));
422       return;
423     }
424     break;
425   }
426   case ISD::FrameIndex: {
427     SDValue Imm = CurDAG->getTargetConstant(0, DL, XLenVT);
428     int FI = cast<FrameIndexSDNode>(Node)->getIndex();
429     SDValue TFI = CurDAG->getTargetFrameIndex(FI, VT);
430     ReplaceNode(Node, CurDAG->getMachineNode(RISCV::ADDI, DL, VT, TFI, Imm));
431     return;
432   }
433   case ISD::SRL: {
434     // Optimize (srl (and X, 0xffff), C) -> (srli (slli X, 16), 16 + C).
435     // Taking into account that the 0xffff may have had lower bits unset by
436     // SimplifyDemandedBits. This avoids materializing the 0xffff immediate.
437     // This pattern occurs when type legalizing i16 right shifts.
438     // FIXME: This could be extended to other AND masks.
439     auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
440     if (N1C) {
441       uint64_t ShAmt = N1C->getZExtValue();
442       SDValue N0 = Node->getOperand(0);
443       if (ShAmt < 16 && N0.getOpcode() == ISD::AND && N0.hasOneUse() &&
444           isa<ConstantSDNode>(N0.getOperand(1))) {
445         uint64_t Mask = N0.getConstantOperandVal(1);
446         Mask |= maskTrailingOnes<uint64_t>(ShAmt);
447         if (Mask == 0xffff) {
448           unsigned SLLOpc = Subtarget->is64Bit() ? RISCV::SLLIW : RISCV::SLLI;
449           unsigned SRLOpc = Subtarget->is64Bit() ? RISCV::SRLIW : RISCV::SRLI;
450           SDNode *SLLI =
451               CurDAG->getMachineNode(SLLOpc, DL, VT, N0->getOperand(0),
452                                      CurDAG->getTargetConstant(16, DL, VT));
453           SDNode *SRLI = CurDAG->getMachineNode(
454               SRLOpc, DL, VT, SDValue(SLLI, 0),
455               CurDAG->getTargetConstant(16 + ShAmt, DL, VT));
456           ReplaceNode(Node, SRLI);
457           return;
458         }
459       }
460     }
461 
462     break;
463   }
464   case ISD::INTRINSIC_W_CHAIN: {
465     unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
466     switch (IntNo) {
467       // By default we do not custom select any intrinsic.
468     default:
469       break;
470 
471     case Intrinsic::riscv_vsetvli:
472     case Intrinsic::riscv_vsetvlimax: {
473       if (!Subtarget->hasStdExtV())
474         break;
475 
476       bool VLMax = IntNo == Intrinsic::riscv_vsetvlimax;
477       unsigned Offset = VLMax ? 2 : 3;
478 
479       assert(Node->getNumOperands() == Offset + 2 &&
480              "Unexpected number of operands");
481 
482       RISCVVSEW VSEW =
483           static_cast<RISCVVSEW>(Node->getConstantOperandVal(Offset) & 0x7);
484       RISCVVLMUL VLMul = static_cast<RISCVVLMUL>(
485           Node->getConstantOperandVal(Offset + 1) & 0x7);
486 
487       unsigned VTypeI = RISCVVType::encodeVTYPE(
488           VLMul, VSEW, /*TailAgnostic*/ true, /*MaskAgnostic*/ false);
489       SDValue VTypeIOp = CurDAG->getTargetConstant(VTypeI, DL, XLenVT);
490 
491       SDValue VLOperand;
492       if (VLMax) {
493         VLOperand = CurDAG->getRegister(RISCV::X0, XLenVT);
494       } else {
495         VLOperand = Node->getOperand(2);
496 
497         if (auto *C = dyn_cast<ConstantSDNode>(VLOperand)) {
498           uint64_t AVL = C->getZExtValue();
499           if (isUInt<5>(AVL)) {
500             SDValue VLImm = CurDAG->getTargetConstant(AVL, DL, XLenVT);
501             ReplaceNode(
502                 Node, CurDAG->getMachineNode(RISCV::PseudoVSETIVLI, DL, XLenVT,
503                                              MVT::Other, VLImm, VTypeIOp,
504                                              /* Chain */ Node->getOperand(0)));
505             return;
506           }
507         }
508       }
509 
510       ReplaceNode(Node,
511                   CurDAG->getMachineNode(RISCV::PseudoVSETVLI, DL, XLenVT,
512                                          MVT::Other, VLOperand, VTypeIOp,
513                                          /* Chain */ Node->getOperand(0)));
514       return;
515     }
516     case Intrinsic::riscv_vlseg2:
517     case Intrinsic::riscv_vlseg3:
518     case Intrinsic::riscv_vlseg4:
519     case Intrinsic::riscv_vlseg5:
520     case Intrinsic::riscv_vlseg6:
521     case Intrinsic::riscv_vlseg7:
522     case Intrinsic::riscv_vlseg8: {
523       selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false);
524       return;
525     }
526     case Intrinsic::riscv_vlseg2_mask:
527     case Intrinsic::riscv_vlseg3_mask:
528     case Intrinsic::riscv_vlseg4_mask:
529     case Intrinsic::riscv_vlseg5_mask:
530     case Intrinsic::riscv_vlseg6_mask:
531     case Intrinsic::riscv_vlseg7_mask:
532     case Intrinsic::riscv_vlseg8_mask: {
533       selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false);
534       return;
535     }
536     case Intrinsic::riscv_vlsseg2:
537     case Intrinsic::riscv_vlsseg3:
538     case Intrinsic::riscv_vlsseg4:
539     case Intrinsic::riscv_vlsseg5:
540     case Intrinsic::riscv_vlsseg6:
541     case Intrinsic::riscv_vlsseg7:
542     case Intrinsic::riscv_vlsseg8: {
543       selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true);
544       return;
545     }
546     case Intrinsic::riscv_vlsseg2_mask:
547     case Intrinsic::riscv_vlsseg3_mask:
548     case Intrinsic::riscv_vlsseg4_mask:
549     case Intrinsic::riscv_vlsseg5_mask:
550     case Intrinsic::riscv_vlsseg6_mask:
551     case Intrinsic::riscv_vlsseg7_mask:
552     case Intrinsic::riscv_vlsseg8_mask: {
553       selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true);
554       return;
555     }
556     case Intrinsic::riscv_vloxseg2:
557     case Intrinsic::riscv_vloxseg3:
558     case Intrinsic::riscv_vloxseg4:
559     case Intrinsic::riscv_vloxseg5:
560     case Intrinsic::riscv_vloxseg6:
561     case Intrinsic::riscv_vloxseg7:
562     case Intrinsic::riscv_vloxseg8:
563       selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true);
564       return;
565     case Intrinsic::riscv_vluxseg2:
566     case Intrinsic::riscv_vluxseg3:
567     case Intrinsic::riscv_vluxseg4:
568     case Intrinsic::riscv_vluxseg5:
569     case Intrinsic::riscv_vluxseg6:
570     case Intrinsic::riscv_vluxseg7:
571     case Intrinsic::riscv_vluxseg8:
572       selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false);
573       return;
574     case Intrinsic::riscv_vloxseg2_mask:
575     case Intrinsic::riscv_vloxseg3_mask:
576     case Intrinsic::riscv_vloxseg4_mask:
577     case Intrinsic::riscv_vloxseg5_mask:
578     case Intrinsic::riscv_vloxseg6_mask:
579     case Intrinsic::riscv_vloxseg7_mask:
580     case Intrinsic::riscv_vloxseg8_mask:
581       selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true);
582       return;
583     case Intrinsic::riscv_vluxseg2_mask:
584     case Intrinsic::riscv_vluxseg3_mask:
585     case Intrinsic::riscv_vluxseg4_mask:
586     case Intrinsic::riscv_vluxseg5_mask:
587     case Intrinsic::riscv_vluxseg6_mask:
588     case Intrinsic::riscv_vluxseg7_mask:
589     case Intrinsic::riscv_vluxseg8_mask:
590       selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false);
591       return;
592     case Intrinsic::riscv_vlseg8ff:
593     case Intrinsic::riscv_vlseg7ff:
594     case Intrinsic::riscv_vlseg6ff:
595     case Intrinsic::riscv_vlseg5ff:
596     case Intrinsic::riscv_vlseg4ff:
597     case Intrinsic::riscv_vlseg3ff:
598     case Intrinsic::riscv_vlseg2ff: {
599       selectVLSEGFF(Node, /*IsMasked*/ false);
600       return;
601     }
602     case Intrinsic::riscv_vlseg8ff_mask:
603     case Intrinsic::riscv_vlseg7ff_mask:
604     case Intrinsic::riscv_vlseg6ff_mask:
605     case Intrinsic::riscv_vlseg5ff_mask:
606     case Intrinsic::riscv_vlseg4ff_mask:
607     case Intrinsic::riscv_vlseg3ff_mask:
608     case Intrinsic::riscv_vlseg2ff_mask: {
609       selectVLSEGFF(Node, /*IsMasked*/ true);
610       return;
611     }
612     case Intrinsic::riscv_vloxei:
613     case Intrinsic::riscv_vloxei_mask:
614     case Intrinsic::riscv_vluxei:
615     case Intrinsic::riscv_vluxei_mask: {
616       bool IsMasked = IntNo == Intrinsic::riscv_vloxei_mask ||
617                       IntNo == Intrinsic::riscv_vluxei_mask;
618       bool IsOrdered = IntNo == Intrinsic::riscv_vloxei ||
619                        IntNo == Intrinsic::riscv_vloxei_mask;
620 
621       MVT VT = Node->getSimpleValueType(0);
622       unsigned ScalarSize = VT.getScalarSizeInBits();
623       MVT XLenVT = Subtarget->getXLenVT();
624       SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
625 
626       unsigned CurOp = 2;
627       SmallVector<SDValue, 7> Operands;
628       if (IsMasked)
629         Operands.push_back(Node->getOperand(CurOp++));
630       SDValue Base;
631       SelectBaseAddr(Node->getOperand(CurOp++), Base);
632       Operands.push_back(Base); // Base pointer.
633       Operands.push_back(Node->getOperand(CurOp++)); // Index.
634       MVT IndexVT = Operands.back()->getSimpleValueType(0);
635       if (IsMasked)
636         Operands.push_back(Node->getOperand(CurOp++)); // Mask.
637       SDValue VL;
638       selectVLOp(Node->getOperand(CurOp++), VL);
639       Operands.push_back(VL);
640       Operands.push_back(SEW);
641       Operands.push_back(Node->getOperand(0)); // Chain.
642 
643       assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
644              "Element count mismatch");
645 
646       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
647       RISCVVLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
648       unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
649       const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
650           IsMasked, IsOrdered, IndexScalarSize, static_cast<unsigned>(LMUL),
651           static_cast<unsigned>(IndexLMUL));
652       MachineSDNode *Load =
653           CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
654 
655       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
656         CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
657 
658       ReplaceNode(Node, Load);
659       return;
660     }
661     case Intrinsic::riscv_vle1:
662     case Intrinsic::riscv_vle:
663     case Intrinsic::riscv_vle_mask:
664     case Intrinsic::riscv_vlse:
665     case Intrinsic::riscv_vlse_mask: {
666       bool IsMasked = IntNo == Intrinsic::riscv_vle_mask ||
667                       IntNo == Intrinsic::riscv_vlse_mask;
668       bool IsStrided =
669           IntNo == Intrinsic::riscv_vlse || IntNo == Intrinsic::riscv_vlse_mask;
670 
671       MVT VT = Node->getSimpleValueType(0);
672       unsigned ScalarSize = VT.getScalarSizeInBits();
673       MVT XLenVT = Subtarget->getXLenVT();
674       // VLE1 uses an SEW of 8.
675       unsigned SEWImm = (IntNo == Intrinsic::riscv_vle1) ? 8 : ScalarSize;
676       SDValue SEW = CurDAG->getTargetConstant(SEWImm, DL, XLenVT);
677 
678       unsigned CurOp = 2;
679       SmallVector<SDValue, 7> Operands;
680       if (IsMasked)
681         Operands.push_back(Node->getOperand(CurOp++));
682       SDValue Base;
683       SelectBaseAddr(Node->getOperand(CurOp++), Base);
684       Operands.push_back(Base); // Base pointer.
685       if (IsStrided)
686         Operands.push_back(Node->getOperand(CurOp++)); // Stride.
687       if (IsMasked)
688         Operands.push_back(Node->getOperand(CurOp++)); // Mask.
689       SDValue VL;
690       selectVLOp(Node->getOperand(CurOp++), VL);
691       Operands.push_back(VL);
692       Operands.push_back(SEW);
693       Operands.push_back(Node->getOperand(0)); // Chain.
694 
695       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
696       const RISCV::VLEPseudo *P =
697           RISCV::getVLEPseudo(IsMasked, IsStrided, /*FF*/ false, ScalarSize,
698                               static_cast<unsigned>(LMUL));
699       MachineSDNode *Load =
700           CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
701 
702       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
703         CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
704 
705       ReplaceNode(Node, Load);
706       return;
707     }
708     case Intrinsic::riscv_vleff:
709     case Intrinsic::riscv_vleff_mask: {
710       bool IsMasked = IntNo == Intrinsic::riscv_vleff_mask;
711 
712       MVT VT = Node->getSimpleValueType(0);
713       unsigned ScalarSize = VT.getScalarSizeInBits();
714       MVT XLenVT = Subtarget->getXLenVT();
715       SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
716 
717       unsigned CurOp = 2;
718       SmallVector<SDValue, 7> Operands;
719       if (IsMasked)
720         Operands.push_back(Node->getOperand(CurOp++));
721       SDValue Base;
722       SelectBaseAddr(Node->getOperand(CurOp++), Base);
723       Operands.push_back(Base); // Base pointer.
724       if (IsMasked)
725         Operands.push_back(Node->getOperand(CurOp++)); // Mask.
726       SDValue VL;
727       selectVLOp(Node->getOperand(CurOp++), VL);
728       Operands.push_back(VL);
729       Operands.push_back(SEW);
730       Operands.push_back(Node->getOperand(0)); // Chain.
731 
732       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
733       const RISCV::VLEPseudo *P =
734           RISCV::getVLEPseudo(IsMasked, /*Strided*/ false, /*FF*/ true,
735                               ScalarSize, static_cast<unsigned>(LMUL));
736       MachineSDNode *Load =
737           CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0),
738                                  MVT::Other, MVT::Glue, Operands);
739       SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT,
740                                               /*Glue*/ SDValue(Load, 2));
741 
742       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
743         CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
744 
745       ReplaceUses(SDValue(Node, 0), SDValue(Load, 0));
746       ReplaceUses(SDValue(Node, 1), SDValue(ReadVL, 0)); // VL
747       ReplaceUses(SDValue(Node, 2), SDValue(Load, 1));   // Chain
748       CurDAG->RemoveDeadNode(Node);
749       return;
750     }
751     }
752     break;
753   }
754   case ISD::INTRINSIC_VOID: {
755     unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
756     switch (IntNo) {
757     case Intrinsic::riscv_vsseg2:
758     case Intrinsic::riscv_vsseg3:
759     case Intrinsic::riscv_vsseg4:
760     case Intrinsic::riscv_vsseg5:
761     case Intrinsic::riscv_vsseg6:
762     case Intrinsic::riscv_vsseg7:
763     case Intrinsic::riscv_vsseg8: {
764       selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false);
765       return;
766     }
767     case Intrinsic::riscv_vsseg2_mask:
768     case Intrinsic::riscv_vsseg3_mask:
769     case Intrinsic::riscv_vsseg4_mask:
770     case Intrinsic::riscv_vsseg5_mask:
771     case Intrinsic::riscv_vsseg6_mask:
772     case Intrinsic::riscv_vsseg7_mask:
773     case Intrinsic::riscv_vsseg8_mask: {
774       selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false);
775       return;
776     }
777     case Intrinsic::riscv_vssseg2:
778     case Intrinsic::riscv_vssseg3:
779     case Intrinsic::riscv_vssseg4:
780     case Intrinsic::riscv_vssseg5:
781     case Intrinsic::riscv_vssseg6:
782     case Intrinsic::riscv_vssseg7:
783     case Intrinsic::riscv_vssseg8: {
784       selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true);
785       return;
786     }
787     case Intrinsic::riscv_vssseg2_mask:
788     case Intrinsic::riscv_vssseg3_mask:
789     case Intrinsic::riscv_vssseg4_mask:
790     case Intrinsic::riscv_vssseg5_mask:
791     case Intrinsic::riscv_vssseg6_mask:
792     case Intrinsic::riscv_vssseg7_mask:
793     case Intrinsic::riscv_vssseg8_mask: {
794       selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true);
795       return;
796     }
797     case Intrinsic::riscv_vsoxseg2:
798     case Intrinsic::riscv_vsoxseg3:
799     case Intrinsic::riscv_vsoxseg4:
800     case Intrinsic::riscv_vsoxseg5:
801     case Intrinsic::riscv_vsoxseg6:
802     case Intrinsic::riscv_vsoxseg7:
803     case Intrinsic::riscv_vsoxseg8:
804       selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true);
805       return;
806     case Intrinsic::riscv_vsuxseg2:
807     case Intrinsic::riscv_vsuxseg3:
808     case Intrinsic::riscv_vsuxseg4:
809     case Intrinsic::riscv_vsuxseg5:
810     case Intrinsic::riscv_vsuxseg6:
811     case Intrinsic::riscv_vsuxseg7:
812     case Intrinsic::riscv_vsuxseg8:
813       selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false);
814       return;
815     case Intrinsic::riscv_vsoxseg2_mask:
816     case Intrinsic::riscv_vsoxseg3_mask:
817     case Intrinsic::riscv_vsoxseg4_mask:
818     case Intrinsic::riscv_vsoxseg5_mask:
819     case Intrinsic::riscv_vsoxseg6_mask:
820     case Intrinsic::riscv_vsoxseg7_mask:
821     case Intrinsic::riscv_vsoxseg8_mask:
822       selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true);
823       return;
824     case Intrinsic::riscv_vsuxseg2_mask:
825     case Intrinsic::riscv_vsuxseg3_mask:
826     case Intrinsic::riscv_vsuxseg4_mask:
827     case Intrinsic::riscv_vsuxseg5_mask:
828     case Intrinsic::riscv_vsuxseg6_mask:
829     case Intrinsic::riscv_vsuxseg7_mask:
830     case Intrinsic::riscv_vsuxseg8_mask:
831       selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false);
832       return;
833     case Intrinsic::riscv_vsoxei:
834     case Intrinsic::riscv_vsoxei_mask:
835     case Intrinsic::riscv_vsuxei:
836     case Intrinsic::riscv_vsuxei_mask: {
837       bool IsMasked = IntNo == Intrinsic::riscv_vsoxei_mask ||
838                       IntNo == Intrinsic::riscv_vsuxei_mask;
839       bool IsOrdered = IntNo == Intrinsic::riscv_vsoxei ||
840                        IntNo == Intrinsic::riscv_vsoxei_mask;
841 
842       MVT VT = Node->getOperand(2)->getSimpleValueType(0);
843       unsigned ScalarSize = VT.getScalarSizeInBits();
844       MVT XLenVT = Subtarget->getXLenVT();
845       SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
846 
847       unsigned CurOp = 2;
848       SmallVector<SDValue, 6> Operands;
849       Operands.push_back(Node->getOperand(CurOp++)); // Store value.
850       SDValue Base;
851       SelectBaseAddr(Node->getOperand(CurOp++), Base);
852       Operands.push_back(Base); // Base pointer.
853       Operands.push_back(Node->getOperand(CurOp++)); // Index.
854       MVT IndexVT = Operands.back()->getSimpleValueType(0);
855       if (IsMasked)
856         Operands.push_back(Node->getOperand(CurOp++)); // Mask.
857       SDValue VL;
858       selectVLOp(Node->getOperand(CurOp++), VL);
859       Operands.push_back(VL);
860       Operands.push_back(SEW);
861       Operands.push_back(Node->getOperand(0)); // Chain.
862 
863       assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
864              "Element count mismatch");
865 
866       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
867       RISCVVLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
868       unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
869       const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
870           IsMasked, IsOrdered, IndexScalarSize, static_cast<unsigned>(LMUL),
871           static_cast<unsigned>(IndexLMUL));
872       MachineSDNode *Store =
873           CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
874 
875       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
876         CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
877 
878       ReplaceNode(Node, Store);
879       return;
880     }
881     case Intrinsic::riscv_vse1:
882     case Intrinsic::riscv_vse:
883     case Intrinsic::riscv_vse_mask:
884     case Intrinsic::riscv_vsse:
885     case Intrinsic::riscv_vsse_mask: {
886       bool IsMasked = IntNo == Intrinsic::riscv_vse_mask ||
887                       IntNo == Intrinsic::riscv_vsse_mask;
888       bool IsStrided =
889           IntNo == Intrinsic::riscv_vsse || IntNo == Intrinsic::riscv_vsse_mask;
890 
891       MVT VT = Node->getOperand(2)->getSimpleValueType(0);
892       unsigned ScalarSize = VT.getScalarSizeInBits();
893       MVT XLenVT = Subtarget->getXLenVT();
894       // VSE1 uses an SEW of 8.
895       unsigned SEWImm = (IntNo == Intrinsic::riscv_vse1) ? 8 : ScalarSize;
896       SDValue SEW = CurDAG->getTargetConstant(SEWImm, DL, XLenVT);
897 
898       unsigned CurOp = 2;
899       SmallVector<SDValue, 6> Operands;
900       Operands.push_back(Node->getOperand(CurOp++)); // Store value.
901       SDValue Base;
902       SelectBaseAddr(Node->getOperand(CurOp++), Base);
903       Operands.push_back(Base); // Base pointer.
904       if (IsStrided)
905         Operands.push_back(Node->getOperand(CurOp++)); // Stride.
906       if (IsMasked)
907         Operands.push_back(Node->getOperand(CurOp++)); // Mask.
908       SDValue VL;
909       selectVLOp(Node->getOperand(CurOp++), VL);
910       Operands.push_back(VL);
911       Operands.push_back(SEW);
912       Operands.push_back(Node->getOperand(0)); // Chain.
913 
914       RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
915       const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(
916           IsMasked, IsStrided, ScalarSize, static_cast<unsigned>(LMUL));
917       MachineSDNode *Store =
918           CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
919 
920       if (auto *MemOp = dyn_cast<MemSDNode>(Node))
921         CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
922 
923       ReplaceNode(Node, Store);
924       return;
925     }
926     }
927     break;
928   }
929   case ISD::BITCAST: {
930     MVT SrcVT = Node->getOperand(0).getSimpleValueType();
931     // Just drop bitcasts between vectors if both are fixed or both are
932     // scalable.
933     if ((VT.isScalableVector() && SrcVT.isScalableVector()) ||
934         (VT.isFixedLengthVector() && SrcVT.isFixedLengthVector())) {
935       ReplaceUses(SDValue(Node, 0), Node->getOperand(0));
936       CurDAG->RemoveDeadNode(Node);
937       return;
938     }
939     break;
940   }
941   case ISD::INSERT_SUBVECTOR: {
942     SDValue V = Node->getOperand(0);
943     SDValue SubV = Node->getOperand(1);
944     SDLoc DL(SubV);
945     auto Idx = Node->getConstantOperandVal(2);
946     MVT SubVecVT = SubV.getSimpleValueType();
947 
948     MVT SubVecContainerVT = SubVecVT;
949     // Establish the correct scalable-vector types for any fixed-length type.
950     if (SubVecVT.isFixedLengthVector())
951       SubVecContainerVT = RISCVTargetLowering::getContainerForFixedLengthVector(
952           *CurDAG, SubVecVT, *Subtarget);
953     if (VT.isFixedLengthVector())
954       VT = RISCVTargetLowering::getContainerForFixedLengthVector(*CurDAG, VT,
955                                                                  *Subtarget);
956 
957     const auto *TRI = Subtarget->getRegisterInfo();
958     unsigned SubRegIdx;
959     std::tie(SubRegIdx, Idx) =
960         RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
961             VT, SubVecContainerVT, Idx, TRI);
962 
963     // If the Idx hasn't been completely eliminated then this is a subvector
964     // insert which doesn't naturally align to a vector register. These must
965     // be handled using instructions to manipulate the vector registers.
966     if (Idx != 0)
967       break;
968 
969     RISCVVLMUL SubVecLMUL = RISCVTargetLowering::getLMUL(SubVecContainerVT);
970     bool IsSubVecPartReg = SubVecLMUL == RISCVVLMUL::LMUL_F2 ||
971                            SubVecLMUL == RISCVVLMUL::LMUL_F4 ||
972                            SubVecLMUL == RISCVVLMUL::LMUL_F8;
973     (void)IsSubVecPartReg; // Silence unused variable warning without asserts.
974     assert((!IsSubVecPartReg || V.isUndef()) &&
975            "Expecting lowering to have created legal INSERT_SUBVECTORs when "
976            "the subvector is smaller than a full-sized register");
977 
978     // If we haven't set a SubRegIdx, then we must be going between
979     // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy.
980     if (SubRegIdx == RISCV::NoSubRegister) {
981       unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT);
982       assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) ==
983                  InRegClassID &&
984              "Unexpected subvector extraction");
985       SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT);
986       SDNode *NewNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
987                                                DL, VT, SubV, RC);
988       return ReplaceNode(Node, NewNode);
989     }
990 
991     SDNode *NewNode = CurDAG->getMachineNode(
992         TargetOpcode::INSERT_SUBREG, DL, VT, V, SubV,
993         CurDAG->getTargetConstant(SubRegIdx, DL, XLenVT));
994     return ReplaceNode(Node, NewNode);
995   }
996   case ISD::EXTRACT_SUBVECTOR: {
997     SDValue V = Node->getOperand(0);
998     auto Idx = Node->getConstantOperandVal(1);
999     MVT InVT = V.getSimpleValueType();
1000     SDLoc DL(V);
1001 
1002     MVT SubVecContainerVT = VT;
1003     // Establish the correct scalable-vector types for any fixed-length type.
1004     if (VT.isFixedLengthVector())
1005       SubVecContainerVT = RISCVTargetLowering::getContainerForFixedLengthVector(
1006           *CurDAG, VT, *Subtarget);
1007     if (InVT.isFixedLengthVector())
1008       InVT = RISCVTargetLowering::getContainerForFixedLengthVector(
1009           *CurDAG, InVT, *Subtarget);
1010 
1011     const auto *TRI = Subtarget->getRegisterInfo();
1012     unsigned SubRegIdx;
1013     std::tie(SubRegIdx, Idx) =
1014         RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1015             InVT, SubVecContainerVT, Idx, TRI);
1016 
1017     // If the Idx hasn't been completely eliminated then this is a subvector
1018     // extract which doesn't naturally align to a vector register. These must
1019     // be handled using instructions to manipulate the vector registers.
1020     if (Idx != 0)
1021       break;
1022 
1023     // If we haven't set a SubRegIdx, then we must be going between
1024     // equally-sized LMUL types (e.g. VR -> VR). This can be done as a copy.
1025     if (SubRegIdx == RISCV::NoSubRegister) {
1026       unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(InVT);
1027       assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) ==
1028                  InRegClassID &&
1029              "Unexpected subvector extraction");
1030       SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT);
1031       SDNode *NewNode =
1032           CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, DL, VT, V, RC);
1033       return ReplaceNode(Node, NewNode);
1034     }
1035 
1036     SDNode *NewNode = CurDAG->getMachineNode(
1037         TargetOpcode::EXTRACT_SUBREG, DL, VT, V,
1038         CurDAG->getTargetConstant(SubRegIdx, DL, XLenVT));
1039     return ReplaceNode(Node, NewNode);
1040   }
1041   }
1042 
1043   // Select the default instruction.
1044   SelectCode(Node);
1045 }
1046 
1047 bool RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand(
1048     const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {
1049   switch (ConstraintID) {
1050   case InlineAsm::Constraint_m:
1051     // We just support simple memory operands that have a single address
1052     // operand and need no special handling.
1053     OutOps.push_back(Op);
1054     return false;
1055   case InlineAsm::Constraint_A:
1056     OutOps.push_back(Op);
1057     return false;
1058   default:
1059     break;
1060   }
1061 
1062   return true;
1063 }
1064 
1065 bool RISCVDAGToDAGISel::SelectAddrFI(SDValue Addr, SDValue &Base) {
1066   if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
1067     Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT());
1068     return true;
1069   }
1070   return false;
1071 }
1072 
1073 bool RISCVDAGToDAGISel::SelectBaseAddr(SDValue Addr, SDValue &Base) {
1074   // If this is FrameIndex, select it directly. Otherwise just let it get
1075   // selected to a register independently.
1076   if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr))
1077     Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT());
1078   else
1079     Base = Addr;
1080   return true;
1081 }
1082 
1083 bool RISCVDAGToDAGISel::selectShiftMask(SDValue N, unsigned ShiftWidth,
1084                                         SDValue &ShAmt) {
1085   // Shift instructions on RISCV only read the lower 5 or 6 bits of the shift
1086   // amount. If there is an AND on the shift amount, we can bypass it if it
1087   // doesn't affect any of those bits.
1088   if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(N.getOperand(1))) {
1089     const APInt &AndMask = N->getConstantOperandAPInt(1);
1090 
1091     // Since the max shift amount is a power of 2 we can subtract 1 to make a
1092     // mask that covers the bits needed to represent all shift amounts.
1093     assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!");
1094     APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
1095 
1096     if (ShMask.isSubsetOf(AndMask)) {
1097       ShAmt = N.getOperand(0);
1098       return true;
1099     }
1100 
1101     // SimplifyDemandedBits may have optimized the mask so try restoring any
1102     // bits that are known zero.
1103     KnownBits Known = CurDAG->computeKnownBits(N->getOperand(0));
1104     if (ShMask.isSubsetOf(AndMask | Known.Zero)) {
1105       ShAmt = N.getOperand(0);
1106       return true;
1107     }
1108   }
1109 
1110   ShAmt = N;
1111   return true;
1112 }
1113 
1114 bool RISCVDAGToDAGISel::selectSExti32(SDValue N, SDValue &Val) {
1115   if (N.getOpcode() == ISD::SIGN_EXTEND_INREG &&
1116       cast<VTSDNode>(N.getOperand(1))->getVT() == MVT::i32) {
1117     Val = N.getOperand(0);
1118     return true;
1119   }
1120   // FIXME: Should we just call computeNumSignBits here?
1121   if (N.getOpcode() == ISD::AssertSext &&
1122       cast<VTSDNode>(N->getOperand(1))->getVT().bitsLE(MVT::i32)) {
1123     Val = N;
1124     return true;
1125   }
1126   if (N.getOpcode() == ISD::AssertZext &&
1127       cast<VTSDNode>(N->getOperand(1))->getVT().bitsLT(MVT::i32)) {
1128     Val = N;
1129     return true;
1130   }
1131 
1132   return false;
1133 }
1134 
1135 bool RISCVDAGToDAGISel::selectZExti32(SDValue N, SDValue &Val) {
1136   if (N.getOpcode() == ISD::AND) {
1137     auto *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
1138     if (C && CheckAndMask(N.getOperand(0), C, UINT64_C(0xFFFFFFFF))) {
1139       Val = N.getOperand(0);
1140       return true;
1141     }
1142   }
1143   // FIXME: Should we just call computeKnownBits here?
1144   if (N.getOpcode() == ISD::AssertZext &&
1145       cast<VTSDNode>(N->getOperand(1))->getVT().bitsLE(MVT::i32)) {
1146     Val = N;
1147     return true;
1148   }
1149 
1150   return false;
1151 }
1152 
1153 // Match (srl (and val, mask), imm) where the result would be a
1154 // zero-extended 32-bit integer. i.e. the mask is 0xffffffff or the result
1155 // is equivalent to this (SimplifyDemandedBits may have removed lower bits
1156 // from the mask that aren't necessary due to the right-shifting).
1157 bool RISCVDAGToDAGISel::MatchSRLIW(SDNode *N) const {
1158   assert(N->getOpcode() == ISD::SRL);
1159   assert(N->getOperand(0).getOpcode() == ISD::AND);
1160   assert(isa<ConstantSDNode>(N->getOperand(1)));
1161   assert(isa<ConstantSDNode>(N->getOperand(0).getOperand(1)));
1162 
1163   // The IsRV64 predicate is checked after PatFrag predicates so we can get
1164   // here even on RV32.
1165   if (!Subtarget->is64Bit())
1166     return false;
1167 
1168   SDValue And = N->getOperand(0);
1169   uint64_t ShAmt = N->getConstantOperandVal(1);
1170   uint64_t Mask = And.getConstantOperandVal(1);
1171   return (Mask | maskTrailingOnes<uint64_t>(ShAmt)) == 0xffffffff;
1172 }
1173 
1174 // Check that it is a SLLIUW (Shift Logical Left Immediate Unsigned i32
1175 // on RV64).
1176 // SLLIUW is the same as SLLI except for the fact that it clears the bits
1177 // XLEN-1:32 of the input RS1 before shifting.
1178 // A PatFrag has already checked that it has the right structure:
1179 //
1180 //  (AND (SHL RS1, VC2), VC1)
1181 //
1182 // We check that VC2, the shamt is less than 32, otherwise the pattern is
1183 // exactly the same as SLLI and we give priority to that.
1184 // Eventually we check that VC1, the mask used to clear the upper 32 bits
1185 // of RS1, is correct:
1186 //
1187 //  VC1 == (0xFFFFFFFF << VC2)
1188 //
1189 bool RISCVDAGToDAGISel::MatchSLLIUW(SDNode *N) const {
1190   assert(N->getOpcode() == ISD::AND);
1191   assert(N->getOperand(0).getOpcode() == ISD::SHL);
1192   assert(isa<ConstantSDNode>(N->getOperand(1)));
1193   assert(isa<ConstantSDNode>(N->getOperand(0).getOperand(1)));
1194 
1195   // The IsRV64 predicate is checked after PatFrag predicates so we can get
1196   // here even on RV32.
1197   if (!Subtarget->is64Bit())
1198     return false;
1199 
1200   SDValue Shl = N->getOperand(0);
1201   uint64_t VC1 = N->getConstantOperandVal(1);
1202   uint64_t VC2 = Shl.getConstantOperandVal(1);
1203 
1204   // Immediate range should be enforced by uimm5 predicate.
1205   assert(VC2 < 32 && "Unexpected immediate");
1206   return (VC1 >> VC2) == UINT64_C(0xFFFFFFFF);
1207 }
1208 
1209 // X0 has special meaning for vsetvl/vsetvli.
1210 //  rd | rs1 |   AVL value | Effect on vl
1211 //--------------------------------------------------------------
1212 // !X0 |  X0 |       VLMAX | Set vl to VLMAX
1213 //  X0 |  X0 | Value in vl | Keep current vl, just change vtype.
1214 bool RISCVDAGToDAGISel::selectVLOp(SDValue N, SDValue &VL) {
1215   // If the VL value is a constant 0, manually select it to an ADDI with 0
1216   // immediate to prevent the default selection path from matching it to X0.
1217   auto *C = dyn_cast<ConstantSDNode>(N);
1218   if (C && C->isNullValue())
1219     VL = SDValue(selectImm(CurDAG, SDLoc(N), 0, Subtarget->getXLenVT()), 0);
1220   else
1221     VL = N;
1222 
1223   return true;
1224 }
1225 
1226 bool RISCVDAGToDAGISel::selectVSplat(SDValue N, SDValue &SplatVal) {
1227   if (N.getOpcode() != ISD::SPLAT_VECTOR &&
1228       N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64 &&
1229       N.getOpcode() != RISCVISD::VMV_V_X_VL)
1230     return false;
1231   SplatVal = N.getOperand(0);
1232   return true;
1233 }
1234 
1235 bool RISCVDAGToDAGISel::selectVSplatSimm5(SDValue N, SDValue &SplatVal) {
1236   if ((N.getOpcode() != ISD::SPLAT_VECTOR &&
1237        N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64 &&
1238        N.getOpcode() != RISCVISD::VMV_V_X_VL) ||
1239       !isa<ConstantSDNode>(N.getOperand(0)))
1240     return false;
1241 
1242   int64_t SplatImm = cast<ConstantSDNode>(N.getOperand(0))->getSExtValue();
1243 
1244   // Both ISD::SPLAT_VECTOR and RISCVISD::SPLAT_VECTOR_I64 share semantics when
1245   // the operand type is wider than the resulting vector element type: an
1246   // implicit truncation first takes place. Therefore, perform a manual
1247   // truncation/sign-extension in order to ignore any truncated bits and catch
1248   // any zero-extended immediate.
1249   // For example, we wish to match (i8 -1) -> (XLenVT 255) as a simm5 by first
1250   // sign-extending to (XLenVT -1).
1251   MVT XLenVT = Subtarget->getXLenVT();
1252   assert(XLenVT == N.getOperand(0).getSimpleValueType() &&
1253          "Unexpected splat operand type");
1254   MVT EltVT = N.getSimpleValueType().getVectorElementType();
1255   if (EltVT.bitsLT(XLenVT)) {
1256     SplatImm = SignExtend64(SplatImm, EltVT.getSizeInBits());
1257   }
1258 
1259   if (!isInt<5>(SplatImm))
1260     return false;
1261 
1262   SplatVal = CurDAG->getTargetConstant(SplatImm, SDLoc(N), XLenVT);
1263   return true;
1264 }
1265 
1266 bool RISCVDAGToDAGISel::selectVSplatUimm5(SDValue N, SDValue &SplatVal) {
1267   if ((N.getOpcode() != ISD::SPLAT_VECTOR &&
1268        N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64 &&
1269        N.getOpcode() != RISCVISD::VMV_V_X_VL) ||
1270       !isa<ConstantSDNode>(N.getOperand(0)))
1271     return false;
1272 
1273   int64_t SplatImm = cast<ConstantSDNode>(N.getOperand(0))->getSExtValue();
1274 
1275   if (!isUInt<5>(SplatImm))
1276     return false;
1277 
1278   SplatVal =
1279       CurDAG->getTargetConstant(SplatImm, SDLoc(N), Subtarget->getXLenVT());
1280 
1281   return true;
1282 }
1283 
1284 bool RISCVDAGToDAGISel::selectRVVSimm5(SDValue N, unsigned Width,
1285                                        SDValue &Imm) {
1286   if (auto *C = dyn_cast<ConstantSDNode>(N)) {
1287     int64_t ImmVal = SignExtend64(C->getSExtValue(), Width);
1288 
1289     if (!isInt<5>(ImmVal))
1290       return false;
1291 
1292     Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), Subtarget->getXLenVT());
1293     return true;
1294   }
1295 
1296   return false;
1297 }
1298 
1299 bool RISCVDAGToDAGISel::selectRVVUimm5(SDValue N, unsigned Width,
1300                                        SDValue &Imm) {
1301   if (auto *C = dyn_cast<ConstantSDNode>(N)) {
1302     int64_t ImmVal = C->getSExtValue();
1303 
1304     if (!isUInt<5>(ImmVal))
1305       return false;
1306 
1307     Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), Subtarget->getXLenVT());
1308     return true;
1309   }
1310 
1311   return false;
1312 }
1313 
1314 // Merge an ADDI into the offset of a load/store instruction where possible.
1315 // (load (addi base, off1), off2) -> (load base, off1+off2)
1316 // (store val, (addi base, off1), off2) -> (store val, base, off1+off2)
1317 // This is possible when off1+off2 fits a 12-bit immediate.
1318 void RISCVDAGToDAGISel::doPeepholeLoadStoreADDI() {
1319   SelectionDAG::allnodes_iterator Position(CurDAG->getRoot().getNode());
1320   ++Position;
1321 
1322   while (Position != CurDAG->allnodes_begin()) {
1323     SDNode *N = &*--Position;
1324     // Skip dead nodes and any non-machine opcodes.
1325     if (N->use_empty() || !N->isMachineOpcode())
1326       continue;
1327 
1328     int OffsetOpIdx;
1329     int BaseOpIdx;
1330 
1331     // Only attempt this optimisation for I-type loads and S-type stores.
1332     switch (N->getMachineOpcode()) {
1333     default:
1334       continue;
1335     case RISCV::LB:
1336     case RISCV::LH:
1337     case RISCV::LW:
1338     case RISCV::LBU:
1339     case RISCV::LHU:
1340     case RISCV::LWU:
1341     case RISCV::LD:
1342     case RISCV::FLH:
1343     case RISCV::FLW:
1344     case RISCV::FLD:
1345       BaseOpIdx = 0;
1346       OffsetOpIdx = 1;
1347       break;
1348     case RISCV::SB:
1349     case RISCV::SH:
1350     case RISCV::SW:
1351     case RISCV::SD:
1352     case RISCV::FSH:
1353     case RISCV::FSW:
1354     case RISCV::FSD:
1355       BaseOpIdx = 1;
1356       OffsetOpIdx = 2;
1357       break;
1358     }
1359 
1360     if (!isa<ConstantSDNode>(N->getOperand(OffsetOpIdx)))
1361       continue;
1362 
1363     SDValue Base = N->getOperand(BaseOpIdx);
1364 
1365     // If the base is an ADDI, we can merge it in to the load/store.
1366     if (!Base.isMachineOpcode() || Base.getMachineOpcode() != RISCV::ADDI)
1367       continue;
1368 
1369     SDValue ImmOperand = Base.getOperand(1);
1370     uint64_t Offset2 = N->getConstantOperandVal(OffsetOpIdx);
1371 
1372     if (auto *Const = dyn_cast<ConstantSDNode>(ImmOperand)) {
1373       int64_t Offset1 = Const->getSExtValue();
1374       int64_t CombinedOffset = Offset1 + Offset2;
1375       if (!isInt<12>(CombinedOffset))
1376         continue;
1377       ImmOperand = CurDAG->getTargetConstant(CombinedOffset, SDLoc(ImmOperand),
1378                                              ImmOperand.getValueType());
1379     } else if (auto *GA = dyn_cast<GlobalAddressSDNode>(ImmOperand)) {
1380       // If the off1 in (addi base, off1) is a global variable's address (its
1381       // low part, really), then we can rely on the alignment of that variable
1382       // to provide a margin of safety before off1 can overflow the 12 bits.
1383       // Check if off2 falls within that margin; if so off1+off2 can't overflow.
1384       const DataLayout &DL = CurDAG->getDataLayout();
1385       Align Alignment = GA->getGlobal()->getPointerAlignment(DL);
1386       if (Offset2 != 0 && Alignment <= Offset2)
1387         continue;
1388       int64_t Offset1 = GA->getOffset();
1389       int64_t CombinedOffset = Offset1 + Offset2;
1390       ImmOperand = CurDAG->getTargetGlobalAddress(
1391           GA->getGlobal(), SDLoc(ImmOperand), ImmOperand.getValueType(),
1392           CombinedOffset, GA->getTargetFlags());
1393     } else if (auto *CP = dyn_cast<ConstantPoolSDNode>(ImmOperand)) {
1394       // Ditto.
1395       Align Alignment = CP->getAlign();
1396       if (Offset2 != 0 && Alignment <= Offset2)
1397         continue;
1398       int64_t Offset1 = CP->getOffset();
1399       int64_t CombinedOffset = Offset1 + Offset2;
1400       ImmOperand = CurDAG->getTargetConstantPool(
1401           CP->getConstVal(), ImmOperand.getValueType(), CP->getAlign(),
1402           CombinedOffset, CP->getTargetFlags());
1403     } else {
1404       continue;
1405     }
1406 
1407     LLVM_DEBUG(dbgs() << "Folding add-immediate into mem-op:\nBase:    ");
1408     LLVM_DEBUG(Base->dump(CurDAG));
1409     LLVM_DEBUG(dbgs() << "\nN: ");
1410     LLVM_DEBUG(N->dump(CurDAG));
1411     LLVM_DEBUG(dbgs() << "\n");
1412 
1413     // Modify the offset operand of the load/store.
1414     if (BaseOpIdx == 0) // Load
1415       CurDAG->UpdateNodeOperands(N, Base.getOperand(0), ImmOperand,
1416                                  N->getOperand(2));
1417     else // Store
1418       CurDAG->UpdateNodeOperands(N, N->getOperand(0), Base.getOperand(0),
1419                                  ImmOperand, N->getOperand(3));
1420 
1421     // The add-immediate may now be dead, in which case remove it.
1422     if (Base.getNode()->use_empty())
1423       CurDAG->RemoveDeadNode(Base.getNode());
1424   }
1425 }
1426 
1427 // This pass converts a legalized DAG into a RISCV-specific DAG, ready
1428 // for instruction scheduling.
1429 FunctionPass *llvm::createRISCVISelDag(RISCVTargetMachine &TM) {
1430   return new RISCVDAGToDAGISel(TM);
1431 }
1432