1 //===-- RISCVISelLowering.cpp - RISCV DAG Lowering Implementation  --------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file defines the interfaces that RISCV uses to lower LLVM code into a
11 // selection DAG.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "RISCVISelLowering.h"
16 #include "RISCV.h"
17 #include "RISCVRegisterInfo.h"
18 #include "RISCVSubtarget.h"
19 #include "RISCVTargetMachine.h"
20 #include "llvm/CodeGen/CallingConvLower.h"
21 #include "llvm/CodeGen/MachineFrameInfo.h"
22 #include "llvm/CodeGen/MachineFunction.h"
23 #include "llvm/CodeGen/MachineInstrBuilder.h"
24 #include "llvm/CodeGen/MachineRegisterInfo.h"
25 #include "llvm/CodeGen/SelectionDAGISel.h"
26 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
27 #include "llvm/CodeGen/ValueTypes.h"
28 #include "llvm/IR/DiagnosticInfo.h"
29 #include "llvm/IR/DiagnosticPrinter.h"
30 #include "llvm/Support/Debug.h"
31 #include "llvm/Support/ErrorHandling.h"
32 #include "llvm/Support/raw_ostream.h"
33 
34 using namespace llvm;
35 
36 #define DEBUG_TYPE "riscv-lower"
37 
38 RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
39                                          const RISCVSubtarget &STI)
40     : TargetLowering(TM), Subtarget(STI) {
41 
42   MVT XLenVT = Subtarget.getXLenVT();
43 
44   // Set up the register classes.
45   addRegisterClass(XLenVT, &RISCV::GPRRegClass);
46 
47   // Compute derived properties from the register classes.
48   computeRegisterProperties(STI.getRegisterInfo());
49 
50   setStackPointerRegisterToSaveRestore(RISCV::X2);
51 
52   for (auto N : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD})
53     setLoadExtAction(N, XLenVT, MVT::i1, Promote);
54 
55   // TODO: add all necessary setOperationAction calls.
56   setOperationAction(ISD::DYNAMIC_STACKALLOC, XLenVT, Expand);
57 
58   setOperationAction(ISD::BR_JT, MVT::Other, Expand);
59   setOperationAction(ISD::BR_CC, XLenVT, Expand);
60   setOperationAction(ISD::SELECT, XLenVT, Custom);
61   setOperationAction(ISD::SELECT_CC, XLenVT, Expand);
62 
63   setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
64   setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
65 
66   for (auto VT : {MVT::i1, MVT::i8, MVT::i16})
67     setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
68 
69   setOperationAction(ISD::ADDC, XLenVT, Expand);
70   setOperationAction(ISD::ADDE, XLenVT, Expand);
71   setOperationAction(ISD::SUBC, XLenVT, Expand);
72   setOperationAction(ISD::SUBE, XLenVT, Expand);
73 
74   setOperationAction(ISD::SREM, XLenVT, Expand);
75   setOperationAction(ISD::SDIVREM, XLenVT, Expand);
76   setOperationAction(ISD::SDIV, XLenVT, Expand);
77   setOperationAction(ISD::UREM, XLenVT, Expand);
78   setOperationAction(ISD::UDIVREM, XLenVT, Expand);
79   setOperationAction(ISD::UDIV, XLenVT, Expand);
80 
81   setOperationAction(ISD::MUL, XLenVT, Expand);
82   setOperationAction(ISD::SMUL_LOHI, XLenVT, Expand);
83   setOperationAction(ISD::UMUL_LOHI, XLenVT, Expand);
84   setOperationAction(ISD::MULHS, XLenVT, Expand);
85   setOperationAction(ISD::MULHU, XLenVT, Expand);
86 
87   setOperationAction(ISD::SHL_PARTS, XLenVT, Expand);
88   setOperationAction(ISD::SRL_PARTS, XLenVT, Expand);
89   setOperationAction(ISD::SRA_PARTS, XLenVT, Expand);
90 
91   setOperationAction(ISD::ROTL, XLenVT, Expand);
92   setOperationAction(ISD::ROTR, XLenVT, Expand);
93   setOperationAction(ISD::BSWAP, XLenVT, Expand);
94   setOperationAction(ISD::CTTZ, XLenVT, Expand);
95   setOperationAction(ISD::CTLZ, XLenVT, Expand);
96   setOperationAction(ISD::CTPOP, XLenVT, Expand);
97 
98   setOperationAction(ISD::GlobalAddress, XLenVT, Custom);
99   setOperationAction(ISD::BlockAddress, XLenVT, Custom);
100 
101   setBooleanContents(ZeroOrOneBooleanContent);
102 
103   // Function alignments (log2).
104   setMinFunctionAlignment(3);
105   setPrefFunctionAlignment(3);
106 
107   // Effectively disable jump table generation.
108   setMinimumJumpTableEntries(INT_MAX);
109 }
110 
111 // Changes the condition code and swaps operands if necessary, so the SetCC
112 // operation matches one of the comparisons supported directly in the RISC-V
113 // ISA.
114 static void normaliseSetCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC) {
115   switch (CC) {
116   default:
117     break;
118   case ISD::SETGT:
119   case ISD::SETLE:
120   case ISD::SETUGT:
121   case ISD::SETULE:
122     CC = ISD::getSetCCSwappedOperands(CC);
123     std::swap(LHS, RHS);
124     break;
125   }
126 }
127 
128 // Return the RISC-V branch opcode that matches the given DAG integer
129 // condition code. The CondCode must be one of those supported by the RISC-V
130 // ISA (see normaliseSetCC).
131 static unsigned getBranchOpcodeForIntCondCode(ISD::CondCode CC) {
132   switch (CC) {
133   default:
134     llvm_unreachable("Unsupported CondCode");
135   case ISD::SETEQ:
136     return RISCV::BEQ;
137   case ISD::SETNE:
138     return RISCV::BNE;
139   case ISD::SETLT:
140     return RISCV::BLT;
141   case ISD::SETGE:
142     return RISCV::BGE;
143   case ISD::SETULT:
144     return RISCV::BLTU;
145   case ISD::SETUGE:
146     return RISCV::BGEU;
147   }
148 }
149 
150 SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
151                                             SelectionDAG &DAG) const {
152   switch (Op.getOpcode()) {
153   default:
154     report_fatal_error("unimplemented operand");
155   case ISD::GlobalAddress:
156     return lowerGlobalAddress(Op, DAG);
157   case ISD::BlockAddress:
158     return lowerBlockAddress(Op, DAG);
159   case ISD::SELECT:
160     return lowerSELECT(Op, DAG);
161   }
162 }
163 
164 SDValue RISCVTargetLowering::lowerGlobalAddress(SDValue Op,
165                                                 SelectionDAG &DAG) const {
166   SDLoc DL(Op);
167   EVT Ty = Op.getValueType();
168   GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
169   const GlobalValue *GV = N->getGlobal();
170   int64_t Offset = N->getOffset();
171 
172   if (isPositionIndependent() || Subtarget.is64Bit())
173     report_fatal_error("Unable to lowerGlobalAddress");
174 
175   SDValue GAHi =
176     DAG.getTargetGlobalAddress(GV, DL, Ty, Offset, RISCVII::MO_HI);
177   SDValue GALo =
178     DAG.getTargetGlobalAddress(GV, DL, Ty, Offset, RISCVII::MO_LO);
179   SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, GAHi), 0);
180   SDValue MNLo =
181     SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNHi, GALo), 0);
182   return MNLo;
183 }
184 
185 SDValue RISCVTargetLowering::lowerBlockAddress(SDValue Op,
186                                                SelectionDAG &DAG) const {
187   SDLoc DL(Op);
188   EVT Ty = Op.getValueType();
189   BlockAddressSDNode *N = cast<BlockAddressSDNode>(Op);
190   const BlockAddress *BA = N->getBlockAddress();
191   int64_t Offset = N->getOffset();
192 
193   if (isPositionIndependent() || Subtarget.is64Bit())
194     report_fatal_error("Unable to lowerBlockAddress");
195 
196   SDValue BAHi = DAG.getTargetBlockAddress(BA, Ty, Offset, RISCVII::MO_HI);
197   SDValue BALo = DAG.getTargetBlockAddress(BA, Ty, Offset, RISCVII::MO_LO);
198   SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, BAHi), 0);
199   SDValue MNLo =
200     SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNHi, BALo), 0);
201   return MNLo;
202 }
203 
204 SDValue RISCVTargetLowering::lowerExternalSymbol(SDValue Op,
205                                                  SelectionDAG &DAG) const {
206   SDLoc DL(Op);
207   EVT Ty = Op.getValueType();
208   ExternalSymbolSDNode *N = cast<ExternalSymbolSDNode>(Op);
209   const char *Sym = N->getSymbol();
210 
211   // TODO: should also handle gp-relative loads.
212 
213   if (isPositionIndependent() || Subtarget.is64Bit())
214     report_fatal_error("Unable to lowerExternalSymbol");
215 
216   SDValue GAHi = DAG.getTargetExternalSymbol(Sym, Ty, RISCVII::MO_HI);
217   SDValue GALo = DAG.getTargetExternalSymbol(Sym, Ty, RISCVII::MO_LO);
218   SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, GAHi), 0);
219   SDValue MNLo =
220     SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNHi, GALo), 0);
221   return MNLo;
222 }
223 
224 SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
225   SDValue CondV = Op.getOperand(0);
226   SDValue TrueV = Op.getOperand(1);
227   SDValue FalseV = Op.getOperand(2);
228   SDLoc DL(Op);
229   MVT XLenVT = Subtarget.getXLenVT();
230 
231   // If the result type is XLenVT and CondV is the output of a SETCC node
232   // which also operated on XLenVT inputs, then merge the SETCC node into the
233   // lowered RISCVISD::SELECT_CC to take advantage of the integer
234   // compare+branch instructions. i.e.:
235   // (select (setcc lhs, rhs, cc), truev, falsev)
236   // -> (riscvisd::select_cc lhs, rhs, cc, truev, falsev)
237   if (Op.getSimpleValueType() == XLenVT && CondV.getOpcode() == ISD::SETCC &&
238       CondV.getOperand(0).getSimpleValueType() == XLenVT) {
239     SDValue LHS = CondV.getOperand(0);
240     SDValue RHS = CondV.getOperand(1);
241     auto CC = cast<CondCodeSDNode>(CondV.getOperand(2));
242     ISD::CondCode CCVal = CC->get();
243 
244     normaliseSetCC(LHS, RHS, CCVal);
245 
246     SDValue TargetCC = DAG.getConstant(CCVal, DL, XLenVT);
247     SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
248     SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
249     return DAG.getNode(RISCVISD::SELECT_CC, DL, VTs, Ops);
250   }
251 
252   // Otherwise:
253   // (select condv, truev, falsev)
254   // -> (riscvisd::select_cc condv, zero, setne, truev, falsev)
255   SDValue Zero = DAG.getConstant(0, DL, XLenVT);
256   SDValue SetNE = DAG.getConstant(ISD::SETNE, DL, XLenVT);
257 
258   SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
259   SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};
260 
261   return DAG.getNode(RISCVISD::SELECT_CC, DL, VTs, Ops);
262 }
263 
264 MachineBasicBlock *
265 RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
266                                                  MachineBasicBlock *BB) const {
267   const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
268   DebugLoc DL = MI.getDebugLoc();
269 
270   assert(MI.getOpcode() == RISCV::Select_GPR_Using_CC_GPR &&
271          "Unexpected instr type to insert");
272 
273   // To "insert" a SELECT instruction, we actually have to insert the triangle
274   // control-flow pattern.  The incoming instruction knows the destination vreg
275   // to set, the condition code register to branch on, the true/false values to
276   // select between, and the condcode to use to select the appropriate branch.
277   //
278   // We produce the following control flow:
279   //     HeadMBB
280   //     |  \
281   //     |  IfFalseMBB
282   //     | /
283   //    TailMBB
284   const BasicBlock *LLVM_BB = BB->getBasicBlock();
285   MachineFunction::iterator I = ++BB->getIterator();
286 
287   MachineBasicBlock *HeadMBB = BB;
288   MachineFunction *F = BB->getParent();
289   MachineBasicBlock *TailMBB = F->CreateMachineBasicBlock(LLVM_BB);
290   MachineBasicBlock *IfFalseMBB = F->CreateMachineBasicBlock(LLVM_BB);
291 
292   F->insert(I, IfFalseMBB);
293   F->insert(I, TailMBB);
294   // Move all remaining instructions to TailMBB.
295   TailMBB->splice(TailMBB->begin(), HeadMBB,
296                   std::next(MachineBasicBlock::iterator(MI)), HeadMBB->end());
297   // Update machine-CFG edges by transferring all successors of the current
298   // block to the new block which will contain the Phi node for the select.
299   TailMBB->transferSuccessorsAndUpdatePHIs(HeadMBB);
300   // Set the successors for HeadMBB.
301   HeadMBB->addSuccessor(IfFalseMBB);
302   HeadMBB->addSuccessor(TailMBB);
303 
304   // Insert appropriate branch.
305   unsigned LHS = MI.getOperand(1).getReg();
306   unsigned RHS = MI.getOperand(2).getReg();
307   auto CC = static_cast<ISD::CondCode>(MI.getOperand(3).getImm());
308   unsigned Opcode = getBranchOpcodeForIntCondCode(CC);
309 
310   BuildMI(HeadMBB, DL, TII.get(Opcode))
311     .addReg(LHS)
312     .addReg(RHS)
313     .addMBB(TailMBB);
314 
315   // IfFalseMBB just falls through to TailMBB.
316   IfFalseMBB->addSuccessor(TailMBB);
317 
318   // %Result = phi [ %TrueValue, HeadMBB ], [ %FalseValue, IfFalseMBB ]
319   BuildMI(*TailMBB, TailMBB->begin(), DL, TII.get(RISCV::PHI),
320           MI.getOperand(0).getReg())
321       .addReg(MI.getOperand(4).getReg())
322       .addMBB(HeadMBB)
323       .addReg(MI.getOperand(5).getReg())
324       .addMBB(IfFalseMBB);
325 
326   MI.eraseFromParent(); // The pseudo instruction is gone now.
327   return TailMBB;
328 }
329 
330 // Calling Convention Implementation.
331 // The expectations for frontend ABI lowering vary from target to target.
332 // Ideally, an LLVM frontend would be able to avoid worrying about many ABI
333 // details, but this is a longer term goal. For now, we simply try to keep the
334 // role of the frontend as simple and well-defined as possible. The rules can
335 // be summarised as:
336 // * Never split up large scalar arguments. We handle them here.
337 // * If a hardfloat calling convention is being used, and the struct may be
338 // passed in a pair of registers (fp+fp, int+fp), and both registers are
339 // available, then pass as two separate arguments. If either the GPRs or FPRs
340 // are exhausted, then pass according to the rule below.
341 // * If a struct could never be passed in registers or directly in a stack
342 // slot (as it is larger than 2*XLEN and the floating point rules don't
343 // apply), then pass it using a pointer with the byval attribute.
344 // * If a struct is less than 2*XLEN, then coerce to either a two-element
345 // word-sized array or a 2*XLEN scalar (depending on alignment).
346 // * The frontend can determine whether a struct is returned by reference or
347 // not based on its size and fields. If it will be returned by reference, the
348 // frontend must modify the prototype so a pointer with the sret annotation is
349 // passed as the first argument. This is not necessary for large scalar
350 // returns.
351 // * Struct return values and varargs should be coerced to structs containing
352 // register-size fields in the same situations they would be for fixed
353 // arguments.
354 
355 static const MCPhysReg ArgGPRs[] = {
356   RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13,
357   RISCV::X14, RISCV::X15, RISCV::X16, RISCV::X17
358 };
359 
360 // Pass a 2*XLEN argument that has been split into two XLEN values through
361 // registers or the stack as necessary.
362 static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1,
363                                 ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2,
364                                 MVT ValVT2, MVT LocVT2,
365                                 ISD::ArgFlagsTy ArgFlags2) {
366   unsigned XLenInBytes = XLen / 8;
367   if (unsigned Reg = State.AllocateReg(ArgGPRs)) {
368     // At least one half can be passed via register.
369     State.addLoc(CCValAssign::getReg(VA1.getValNo(), VA1.getValVT(), Reg,
370                                      VA1.getLocVT(), CCValAssign::Full));
371   } else {
372     // Both halves must be passed on the stack, with proper alignment.
373     unsigned StackAlign = std::max(XLenInBytes, ArgFlags1.getOrigAlign());
374     State.addLoc(
375         CCValAssign::getMem(VA1.getValNo(), VA1.getValVT(),
376                             State.AllocateStack(XLenInBytes, StackAlign),
377                             VA1.getLocVT(), CCValAssign::Full));
378     State.addLoc(CCValAssign::getMem(
379         ValNo2, ValVT2, State.AllocateStack(XLenInBytes, XLenInBytes), LocVT2,
380         CCValAssign::Full));
381     return false;
382   }
383 
384   if (unsigned Reg = State.AllocateReg(ArgGPRs)) {
385     // The second half can also be passed via register.
386     State.addLoc(
387         CCValAssign::getReg(ValNo2, ValVT2, Reg, LocVT2, CCValAssign::Full));
388   } else {
389     // The second half is passed via the stack, without additional alignment.
390     State.addLoc(CCValAssign::getMem(
391         ValNo2, ValVT2, State.AllocateStack(XLenInBytes, XLenInBytes), LocVT2,
392         CCValAssign::Full));
393   }
394 
395   return false;
396 }
397 
398 // Implements the RISC-V calling convention. Returns true upon failure.
399 static bool CC_RISCV(const DataLayout &DL, unsigned ValNo, MVT ValVT, MVT LocVT,
400                      CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
401                      CCState &State, bool IsFixed, bool IsRet) {
402   unsigned XLen = DL.getLargestLegalIntTypeSizeInBits();
403   assert(XLen == 32 || XLen == 64);
404   MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;
405   assert(ValVT == XLenVT && "Unexpected ValVT");
406   assert(LocVT == XLenVT && "Unexpected LocVT");
407   assert(IsFixed && "Vararg support not yet implemented");
408 
409   // Any return value split in to more than two values can't be returned
410   // directly.
411   if (IsRet && ValNo > 1)
412     return true;
413 
414   SmallVectorImpl<CCValAssign> &PendingLocs = State.getPendingLocs();
415   SmallVectorImpl<ISD::ArgFlagsTy> &PendingArgFlags =
416       State.getPendingArgFlags();
417 
418   assert(PendingLocs.size() == PendingArgFlags.size() &&
419          "PendingLocs and PendingArgFlags out of sync");
420 
421   // Split arguments might be passed indirectly, so keep track of the pending
422   // values.
423   if (ArgFlags.isSplit() || !PendingLocs.empty()) {
424     LocVT = XLenVT;
425     LocInfo = CCValAssign::Indirect;
426     PendingLocs.push_back(
427         CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo));
428     PendingArgFlags.push_back(ArgFlags);
429     if (!ArgFlags.isSplitEnd()) {
430       return false;
431     }
432   }
433 
434   // If the split argument only had two elements, it should be passed directly
435   // in registers or on the stack.
436   if (ArgFlags.isSplitEnd() && PendingLocs.size() <= 2) {
437     assert(PendingLocs.size() == 2 && "Unexpected PendingLocs.size()");
438     // Apply the normal calling convention rules to the first half of the
439     // split argument.
440     CCValAssign VA = PendingLocs[0];
441     ISD::ArgFlagsTy AF = PendingArgFlags[0];
442     PendingLocs.clear();
443     PendingArgFlags.clear();
444     return CC_RISCVAssign2XLen(XLen, State, VA, AF, ValNo, ValVT, LocVT,
445                                ArgFlags);
446   }
447 
448   // Allocate to a register if possible, or else a stack slot.
449   unsigned Reg = State.AllocateReg(ArgGPRs);
450   unsigned StackOffset = Reg ? 0 : State.AllocateStack(XLen / 8, XLen / 8);
451 
452   // If we reach this point and PendingLocs is non-empty, we must be at the
453   // end of a split argument that must be passed indirectly.
454   if (!PendingLocs.empty()) {
455     assert(ArgFlags.isSplitEnd() && "Expected ArgFlags.isSplitEnd()");
456     assert(PendingLocs.size() > 2 && "Unexpected PendingLocs.size()");
457 
458     for (auto &It : PendingLocs) {
459       if (Reg)
460         It.convertToReg(Reg);
461       else
462         It.convertToMem(StackOffset);
463       State.addLoc(It);
464     }
465     PendingLocs.clear();
466     PendingArgFlags.clear();
467     return false;
468   }
469 
470   assert(LocVT == XLenVT && "Expected an XLenVT at this stage");
471 
472   if (Reg) {
473     State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
474   } else {
475     State.addLoc(
476         CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
477   }
478   return false;
479 }
480 
481 void RISCVTargetLowering::analyzeInputArgs(
482     MachineFunction &MF, CCState &CCInfo,
483     const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet) const {
484   unsigned NumArgs = Ins.size();
485 
486   for (unsigned i = 0; i != NumArgs; ++i) {
487     MVT ArgVT = Ins[i].VT;
488     ISD::ArgFlagsTy ArgFlags = Ins[i].Flags;
489 
490     if (CC_RISCV(MF.getDataLayout(), i, ArgVT, ArgVT, CCValAssign::Full,
491                  ArgFlags, CCInfo, /*IsRet=*/true, IsRet)) {
492       DEBUG(dbgs() << "InputArg #" << i << " has unhandled type "
493                    << EVT(ArgVT).getEVTString() << '\n');
494       llvm_unreachable(nullptr);
495     }
496   }
497 }
498 
499 void RISCVTargetLowering::analyzeOutputArgs(
500     MachineFunction &MF, CCState &CCInfo,
501     const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet) const {
502   unsigned NumArgs = Outs.size();
503 
504   for (unsigned i = 0; i != NumArgs; i++) {
505     MVT ArgVT = Outs[i].VT;
506     ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
507 
508     if (CC_RISCV(MF.getDataLayout(), i, ArgVT, ArgVT, CCValAssign::Full,
509                  ArgFlags, CCInfo, Outs[i].IsFixed, IsRet)) {
510       DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type "
511                    << EVT(ArgVT).getEVTString() << "\n");
512       llvm_unreachable(nullptr);
513     }
514   }
515 }
516 
517 // The caller is responsible for loading the full value if the argument is
518 // passed with CCValAssign::Indirect.
519 static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain,
520                                 const CCValAssign &VA, const SDLoc &DL) {
521   MachineFunction &MF = DAG.getMachineFunction();
522   MachineRegisterInfo &RegInfo = MF.getRegInfo();
523   EVT LocVT = VA.getLocVT();
524   SDValue Val;
525 
526   unsigned VReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
527   RegInfo.addLiveIn(VA.getLocReg(), VReg);
528   Val = DAG.getCopyFromReg(Chain, DL, VReg, LocVT);
529 
530   switch (VA.getLocInfo()) {
531   default:
532     llvm_unreachable("Unexpected CCValAssign::LocInfo");
533   case CCValAssign::Full:
534   case CCValAssign::Indirect:
535     return Val;
536   }
537 }
538 
539 // The caller is responsible for loading the full value if the argument is
540 // passed with CCValAssign::Indirect.
541 static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain,
542                                 const CCValAssign &VA, const SDLoc &DL) {
543   MachineFunction &MF = DAG.getMachineFunction();
544   MachineFrameInfo &MFI = MF.getFrameInfo();
545   EVT LocVT = VA.getLocVT();
546   EVT ValVT = VA.getValVT();
547   EVT PtrVT = MVT::getIntegerVT(DAG.getDataLayout().getPointerSizeInBits(0));
548   int FI = MFI.CreateFixedObject(ValVT.getSizeInBits() / 8,
549                                  VA.getLocMemOffset(), /*Immutable=*/true);
550   SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
551   SDValue Val;
552 
553   ISD::LoadExtType ExtType;
554   switch (VA.getLocInfo()) {
555   default:
556     llvm_unreachable("Unexpected CCValAssign::LocInfo");
557   case CCValAssign::Full:
558   case CCValAssign::Indirect:
559     ExtType = ISD::NON_EXTLOAD;
560     break;
561   }
562   Val = DAG.getExtLoad(
563       ExtType, DL, LocVT, Chain, FIN,
564       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), ValVT);
565   return Val;
566 }
567 
568 // Transform physical registers into virtual registers.
569 SDValue RISCVTargetLowering::LowerFormalArguments(
570     SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
571     const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
572     SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
573 
574   switch (CallConv) {
575   default:
576     report_fatal_error("Unsupported calling convention");
577   case CallingConv::C:
578   case CallingConv::Fast:
579     break;
580   }
581 
582   MachineFunction &MF = DAG.getMachineFunction();
583   EVT PtrVT = getPointerTy(DAG.getDataLayout());
584 
585   if (IsVarArg)
586     report_fatal_error("VarArg not supported");
587 
588   // Assign locations to all of the incoming arguments.
589   SmallVector<CCValAssign, 16> ArgLocs;
590   CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
591   analyzeInputArgs(MF, CCInfo, Ins, /*IsRet=*/false);
592 
593   for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
594     CCValAssign &VA = ArgLocs[i];
595     assert(VA.getLocVT() == Subtarget.getXLenVT() && "Unhandled argument type");
596     SDValue ArgValue;
597     if (VA.isRegLoc())
598       ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL);
599     else
600       ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL);
601 
602     if (VA.getLocInfo() == CCValAssign::Indirect) {
603       // If the original argument was split and passed by reference (e.g. i128
604       // on RV32), we need to load all parts of it here (using the same
605       // address).
606       InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
607                                    MachinePointerInfo()));
608       unsigned ArgIndex = Ins[i].OrigArgIndex;
609       assert(Ins[i].PartOffset == 0);
610       while (i + 1 != e && Ins[i + 1].OrigArgIndex == ArgIndex) {
611         CCValAssign &PartVA = ArgLocs[i + 1];
612         unsigned PartOffset = Ins[i + 1].PartOffset;
613         SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue,
614                                       DAG.getIntPtrConstant(PartOffset, DL));
615         InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
616                                      MachinePointerInfo()));
617         ++i;
618       }
619       continue;
620     }
621     InVals.push_back(ArgValue);
622   }
623   return Chain;
624 }
625 
626 // Lower a call to a callseq_start + CALL + callseq_end chain, and add input
627 // and output parameter nodes.
628 SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
629                                        SmallVectorImpl<SDValue> &InVals) const {
630   SelectionDAG &DAG = CLI.DAG;
631   SDLoc &DL = CLI.DL;
632   SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
633   SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
634   SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
635   SDValue Chain = CLI.Chain;
636   SDValue Callee = CLI.Callee;
637   CLI.IsTailCall = false;
638   CallingConv::ID CallConv = CLI.CallConv;
639   bool IsVarArg = CLI.IsVarArg;
640   EVT PtrVT = getPointerTy(DAG.getDataLayout());
641   MVT XLenVT = Subtarget.getXLenVT();
642 
643   if (IsVarArg) {
644     report_fatal_error("LowerCall with varargs not implemented");
645   }
646 
647   MachineFunction &MF = DAG.getMachineFunction();
648 
649   // Analyze the operands of the call, assigning locations to each operand.
650   SmallVector<CCValAssign, 16> ArgLocs;
651   CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
652   analyzeOutputArgs(MF, ArgCCInfo, Outs, /*IsRet=*/false);
653 
654   // Get a count of how many bytes are to be pushed on the stack.
655   unsigned NumBytes = ArgCCInfo.getNextStackOffset();
656 
657   // Create local copies for byval args
658   SmallVector<SDValue, 8> ByValArgs;
659   for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
660     ISD::ArgFlagsTy Flags = Outs[i].Flags;
661     if (!Flags.isByVal())
662       continue;
663 
664     SDValue Arg = OutVals[i];
665     unsigned Size = Flags.getByValSize();
666     unsigned Align = Flags.getByValAlign();
667 
668     int FI = MF.getFrameInfo().CreateStackObject(Size, Align, /*isSS=*/false);
669     SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
670     SDValue SizeNode = DAG.getConstant(Size, DL, XLenVT);
671 
672     Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Align,
673                           /*IsVolatile=*/false,
674                           /*AlwaysInline=*/false,
675                           /*isTailCall=*/false, MachinePointerInfo(),
676                           MachinePointerInfo());
677     ByValArgs.push_back(FIPtr);
678   }
679 
680   Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL);
681 
682   // Copy argument values to their designated locations.
683   SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
684   SmallVector<SDValue, 8> MemOpChains;
685   SDValue StackPtr;
686   for (unsigned i = 0, j = 0, e = ArgLocs.size(); i != e; ++i) {
687     CCValAssign &VA = ArgLocs[i];
688     SDValue ArgValue = OutVals[i];
689     ISD::ArgFlagsTy Flags = Outs[i].Flags;
690 
691     // Promote the value if needed.
692     // For now, only handle fully promoted and indirect arguments.
693     switch (VA.getLocInfo()) {
694     case CCValAssign::Full:
695       break;
696     case CCValAssign::Indirect: {
697       // Store the argument in a stack slot and pass its address.
698       SDValue SpillSlot = DAG.CreateStackTemporary(Outs[i].ArgVT);
699       int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
700       MemOpChains.push_back(
701           DAG.getStore(Chain, DL, ArgValue, SpillSlot,
702                        MachinePointerInfo::getFixedStack(MF, FI)));
703       // If the original argument was split (e.g. i128), we need
704       // to store all parts of it here (and pass just one address).
705       unsigned ArgIndex = Outs[i].OrigArgIndex;
706       assert(Outs[i].PartOffset == 0);
707       while (i + 1 != e && Outs[i + 1].OrigArgIndex == ArgIndex) {
708         SDValue PartValue = OutVals[i + 1];
709         unsigned PartOffset = Outs[i + 1].PartOffset;
710         SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot,
711                                       DAG.getIntPtrConstant(PartOffset, DL));
712         MemOpChains.push_back(
713             DAG.getStore(Chain, DL, PartValue, Address,
714                          MachinePointerInfo::getFixedStack(MF, FI)));
715         ++i;
716       }
717       ArgValue = SpillSlot;
718       break;
719     }
720     default:
721       llvm_unreachable("Unknown loc info!");
722     }
723 
724     // Use local copy if it is a byval arg.
725     if (Flags.isByVal())
726       ArgValue = ByValArgs[j++];
727 
728     if (VA.isRegLoc()) {
729       // Queue up the argument copies and emit them at the end.
730       RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue));
731     } else {
732       assert(VA.isMemLoc() && "Argument not register or memory");
733 
734       // Work out the address of the stack slot.
735       if (!StackPtr.getNode())
736         StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
737       SDValue Address =
738           DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
739                       DAG.getIntPtrConstant(VA.getLocMemOffset(), DL));
740 
741       // Emit the store.
742       MemOpChains.push_back(
743           DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo()));
744     }
745   }
746 
747   // Join the stores, which are independent of one another.
748   if (!MemOpChains.empty())
749     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
750 
751   SDValue Glue;
752 
753   // Build a sequence of copy-to-reg nodes, chained and glued together.
754   for (auto &Reg : RegsToPass) {
755     Chain = DAG.getCopyToReg(Chain, DL, Reg.first, Reg.second, Glue);
756     Glue = Chain.getValue(1);
757   }
758 
759   if (isa<GlobalAddressSDNode>(Callee)) {
760     Callee = lowerGlobalAddress(Callee, DAG);
761   } else if (isa<ExternalSymbolSDNode>(Callee)) {
762     Callee = lowerExternalSymbol(Callee, DAG);
763   }
764 
765   // The first call operand is the chain and the second is the target address.
766   SmallVector<SDValue, 8> Ops;
767   Ops.push_back(Chain);
768   Ops.push_back(Callee);
769 
770   // Add argument registers to the end of the list so that they are
771   // known live into the call.
772   for (auto &Reg : RegsToPass)
773     Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));
774 
775   // Add a register mask operand representing the call-preserved registers.
776   const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
777   const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
778   assert(Mask && "Missing call preserved mask for calling convention");
779   Ops.push_back(DAG.getRegisterMask(Mask));
780 
781   // Glue the call to the argument copies, if any.
782   if (Glue.getNode())
783     Ops.push_back(Glue);
784 
785   // Emit the call.
786   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
787   Chain = DAG.getNode(RISCVISD::CALL, DL, NodeTys, Ops);
788   Glue = Chain.getValue(1);
789 
790   // Mark the end of the call, which is glued to the call itself.
791   Chain = DAG.getCALLSEQ_END(Chain,
792                              DAG.getConstant(NumBytes, DL, PtrVT, true),
793                              DAG.getConstant(0, DL, PtrVT, true),
794                              Glue, DL);
795   Glue = Chain.getValue(1);
796 
797   // Assign locations to each value returned by this call.
798   SmallVector<CCValAssign, 16> RVLocs;
799   CCState RetCCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
800   analyzeInputArgs(MF, RetCCInfo, Ins, /*IsRet=*/true);
801 
802   // Copy all of the result registers out of their specified physreg.
803   for (auto &VA : RVLocs) {
804     // Copy the value out, gluing the copy to the end of the call sequence.
805     SDValue RetValue = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(),
806                                           VA.getLocVT(), Glue);
807     Chain = RetValue.getValue(1);
808     Glue = RetValue.getValue(2);
809 
810     assert(VA.getLocInfo() == CCValAssign::Full && "Unknown loc info!");
811     InVals.push_back(RetValue);
812   }
813 
814   return Chain;
815 }
816 
817 bool RISCVTargetLowering::CanLowerReturn(
818     CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
819     const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
820   SmallVector<CCValAssign, 16> RVLocs;
821   CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
822   for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
823     MVT VT = Outs[i].VT;
824     ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
825     if (CC_RISCV(MF.getDataLayout(), i, VT, VT, CCValAssign::Full, ArgFlags,
826                  CCInfo, /*IsFixed=*/true, /*IsRet=*/true))
827       return false;
828   }
829   return true;
830 }
831 
832 SDValue
833 RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
834                                  bool IsVarArg,
835                                  const SmallVectorImpl<ISD::OutputArg> &Outs,
836                                  const SmallVectorImpl<SDValue> &OutVals,
837                                  const SDLoc &DL, SelectionDAG &DAG) const {
838   if (IsVarArg) {
839     report_fatal_error("VarArg not supported");
840   }
841 
842   // Stores the assignment of the return value to a location.
843   SmallVector<CCValAssign, 16> RVLocs;
844 
845   // Info about the registers and stack slot.
846   CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
847                  *DAG.getContext());
848 
849   analyzeOutputArgs(DAG.getMachineFunction(), CCInfo, Outs, /*IsRet=*/true);
850 
851   SDValue Flag;
852   SmallVector<SDValue, 4> RetOps(1, Chain);
853 
854   // Copy the result values into the output registers.
855   for (unsigned i = 0, e = RVLocs.size(); i < e; ++i) {
856     SDValue Val = OutVals[i];
857     CCValAssign &VA = RVLocs[i];
858     assert(VA.isRegLoc() && "Can only return in registers!");
859     assert(VA.getLocInfo() == CCValAssign::Full &&
860            "Unexpected CCValAssign::LocInfo");
861 
862     Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Flag);
863 
864     // Guarantee that all emitted copies are stuck together.
865     Flag = Chain.getValue(1);
866     RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
867   }
868 
869   RetOps[0] = Chain; // Update chain.
870 
871   // Add the flag if we have it.
872   if (Flag.getNode()) {
873     RetOps.push_back(Flag);
874   }
875 
876   return DAG.getNode(RISCVISD::RET_FLAG, DL, MVT::Other, RetOps);
877 }
878 
879 const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
880   switch ((RISCVISD::NodeType)Opcode) {
881   case RISCVISD::FIRST_NUMBER:
882     break;
883   case RISCVISD::RET_FLAG:
884     return "RISCVISD::RET_FLAG";
885   case RISCVISD::CALL:
886     return "RISCVISD::CALL";
887   case RISCVISD::SELECT_CC:
888     return "RISCVISD::SELECT_CC";
889   }
890   return nullptr;
891 }
892