1 //===- MipsISelLowering.cpp - Mips DAG Lowering Implementation ------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file defines the interfaces that Mips uses to lower LLVM code into a
11 // selection DAG.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "MipsISelLowering.h"
16 #include "InstPrinter/MipsInstPrinter.h"
17 #include "MCTargetDesc/MipsBaseInfo.h"
18 #include "MCTargetDesc/MipsMCTargetDesc.h"
19 #include "MipsCCState.h"
20 #include "MipsInstrInfo.h"
21 #include "MipsMachineFunction.h"
22 #include "MipsRegisterInfo.h"
23 #include "MipsSubtarget.h"
24 #include "MipsTargetMachine.h"
25 #include "MipsTargetObjectFile.h"
26 #include "llvm/ADT/APFloat.h"
27 #include "llvm/ADT/ArrayRef.h"
28 #include "llvm/ADT/SmallVector.h"
29 #include "llvm/ADT/Statistic.h"
30 #include "llvm/ADT/StringRef.h"
31 #include "llvm/ADT/StringSwitch.h"
32 #include "llvm/CodeGen/CallingConvLower.h"
33 #include "llvm/CodeGen/FunctionLoweringInfo.h"
34 #include "llvm/CodeGen/ISDOpcodes.h"
35 #include "llvm/CodeGen/MachineBasicBlock.h"
36 #include "llvm/CodeGen/MachineFrameInfo.h"
37 #include "llvm/CodeGen/MachineFunction.h"
38 #include "llvm/CodeGen/MachineInstr.h"
39 #include "llvm/CodeGen/MachineInstrBuilder.h"
40 #include "llvm/CodeGen/MachineJumpTableInfo.h"
41 #include "llvm/CodeGen/MachineMemOperand.h"
42 #include "llvm/CodeGen/MachineOperand.h"
43 #include "llvm/CodeGen/MachineRegisterInfo.h"
44 #include "llvm/CodeGen/RuntimeLibcalls.h"
45 #include "llvm/CodeGen/SelectionDAG.h"
46 #include "llvm/CodeGen/SelectionDAGNodes.h"
47 #include "llvm/CodeGen/TargetFrameLowering.h"
48 #include "llvm/CodeGen/TargetInstrInfo.h"
49 #include "llvm/CodeGen/TargetRegisterInfo.h"
50 #include "llvm/CodeGen/ValueTypes.h"
51 #include "llvm/IR/CallingConv.h"
52 #include "llvm/IR/Constants.h"
53 #include "llvm/IR/DataLayout.h"
54 #include "llvm/IR/DebugLoc.h"
55 #include "llvm/IR/DerivedTypes.h"
56 #include "llvm/IR/Function.h"
57 #include "llvm/IR/GlobalValue.h"
58 #include "llvm/IR/Type.h"
59 #include "llvm/IR/Value.h"
60 #include "llvm/MC/MCRegisterInfo.h"
61 #include "llvm/Support/Casting.h"
62 #include "llvm/Support/CodeGen.h"
63 #include "llvm/Support/CommandLine.h"
64 #include "llvm/Support/Compiler.h"
65 #include "llvm/Support/ErrorHandling.h"
66 #include "llvm/Support/MachineValueType.h"
67 #include "llvm/Support/MathExtras.h"
68 #include "llvm/Target/TargetMachine.h"
69 #include "llvm/Target/TargetOptions.h"
70 #include <algorithm>
71 #include <cassert>
72 #include <cctype>
73 #include <cstdint>
74 #include <deque>
75 #include <iterator>
76 #include <utility>
77 #include <vector>
78 
79 using namespace llvm;
80 
81 #define DEBUG_TYPE "mips-lower"
82 
83 STATISTIC(NumTailCalls, "Number of tail calls");
84 
85 static cl::opt<bool>
86 LargeGOT("mxgot", cl::Hidden,
87          cl::desc("MIPS: Enable GOT larger than 64k."), cl::init(false));
88 
89 static cl::opt<bool>
90 NoZeroDivCheck("mno-check-zero-division", cl::Hidden,
91                cl::desc("MIPS: Don't trap on integer division by zero."),
92                cl::init(false));
93 
94 static const MCPhysReg Mips64DPRegs[8] = {
95   Mips::D12_64, Mips::D13_64, Mips::D14_64, Mips::D15_64,
96   Mips::D16_64, Mips::D17_64, Mips::D18_64, Mips::D19_64
97 };
98 
99 // If I is a shifted mask, set the size (Size) and the first bit of the
100 // mask (Pos), and return true.
101 // For example, if I is 0x003ff800, (Pos, Size) = (11, 11).
102 static bool isShiftedMask(uint64_t I, uint64_t &Pos, uint64_t &Size) {
103   if (!isShiftedMask_64(I))
104     return false;
105 
106   Size = countPopulation(I);
107   Pos = countTrailingZeros(I);
108   return true;
109 }
110 
111 // The MIPS MSA ABI passes vector arguments in the integer register set.
112 // The number of integer registers used is dependant on the ABI used.
113 MVT MipsTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
114                                                       EVT VT) const {
115   if (VT.isVector()) {
116       if (Subtarget.isABI_O32()) {
117         return MVT::i32;
118       } else {
119         return (VT.getSizeInBits() == 32) ? MVT::i32 : MVT::i64;
120       }
121   }
122   return MipsTargetLowering::getRegisterType(Context, VT);
123 }
124 
125 unsigned MipsTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
126                                                            EVT VT) const {
127   if (VT.isVector())
128     return std::max((VT.getSizeInBits() / (Subtarget.isABI_O32() ? 32 : 64)),
129                     1U);
130   return MipsTargetLowering::getNumRegisters(Context, VT);
131 }
132 
133 unsigned MipsTargetLowering::getVectorTypeBreakdownForCallingConv(
134     LLVMContext &Context, EVT VT, EVT &IntermediateVT,
135     unsigned &NumIntermediates, MVT &RegisterVT) const {
136   // Break down vector types to either 2 i64s or 4 i32s.
137   RegisterVT = getRegisterTypeForCallingConv(Context, VT) ;
138   IntermediateVT = RegisterVT;
139   NumIntermediates = VT.getSizeInBits() < RegisterVT.getSizeInBits()
140                          ? VT.getVectorNumElements()
141                          : VT.getSizeInBits() / RegisterVT.getSizeInBits();
142 
143   return NumIntermediates;
144 }
145 
146 SDValue MipsTargetLowering::getGlobalReg(SelectionDAG &DAG, EVT Ty) const {
147   MipsFunctionInfo *FI = DAG.getMachineFunction().getInfo<MipsFunctionInfo>();
148   return DAG.getRegister(FI->getGlobalBaseReg(), Ty);
149 }
150 
151 SDValue MipsTargetLowering::getTargetNode(GlobalAddressSDNode *N, EVT Ty,
152                                           SelectionDAG &DAG,
153                                           unsigned Flag) const {
154   return DAG.getTargetGlobalAddress(N->getGlobal(), SDLoc(N), Ty, 0, Flag);
155 }
156 
157 SDValue MipsTargetLowering::getTargetNode(ExternalSymbolSDNode *N, EVT Ty,
158                                           SelectionDAG &DAG,
159                                           unsigned Flag) const {
160   return DAG.getTargetExternalSymbol(N->getSymbol(), Ty, Flag);
161 }
162 
163 SDValue MipsTargetLowering::getTargetNode(BlockAddressSDNode *N, EVT Ty,
164                                           SelectionDAG &DAG,
165                                           unsigned Flag) const {
166   return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, 0, Flag);
167 }
168 
169 SDValue MipsTargetLowering::getTargetNode(JumpTableSDNode *N, EVT Ty,
170                                           SelectionDAG &DAG,
171                                           unsigned Flag) const {
172   return DAG.getTargetJumpTable(N->getIndex(), Ty, Flag);
173 }
174 
175 SDValue MipsTargetLowering::getTargetNode(ConstantPoolSDNode *N, EVT Ty,
176                                           SelectionDAG &DAG,
177                                           unsigned Flag) const {
178   return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlignment(),
179                                    N->getOffset(), Flag);
180 }
181 
182 const char *MipsTargetLowering::getTargetNodeName(unsigned Opcode) const {
183   switch ((MipsISD::NodeType)Opcode) {
184   case MipsISD::FIRST_NUMBER:      break;
185   case MipsISD::JmpLink:           return "MipsISD::JmpLink";
186   case MipsISD::TailCall:          return "MipsISD::TailCall";
187   case MipsISD::Highest:           return "MipsISD::Highest";
188   case MipsISD::Higher:            return "MipsISD::Higher";
189   case MipsISD::Hi:                return "MipsISD::Hi";
190   case MipsISD::Lo:                return "MipsISD::Lo";
191   case MipsISD::GotHi:             return "MipsISD::GotHi";
192   case MipsISD::GPRel:             return "MipsISD::GPRel";
193   case MipsISD::ThreadPointer:     return "MipsISD::ThreadPointer";
194   case MipsISD::Ret:               return "MipsISD::Ret";
195   case MipsISD::ERet:              return "MipsISD::ERet";
196   case MipsISD::EH_RETURN:         return "MipsISD::EH_RETURN";
197   case MipsISD::FMS:               return "MipsISD::FMS";
198   case MipsISD::FPBrcond:          return "MipsISD::FPBrcond";
199   case MipsISD::FPCmp:             return "MipsISD::FPCmp";
200   case MipsISD::FSELECT:           return "MipsISD::FSELECT";
201   case MipsISD::MTC1_D64:          return "MipsISD::MTC1_D64";
202   case MipsISD::CMovFP_T:          return "MipsISD::CMovFP_T";
203   case MipsISD::CMovFP_F:          return "MipsISD::CMovFP_F";
204   case MipsISD::TruncIntFP:        return "MipsISD::TruncIntFP";
205   case MipsISD::MFHI:              return "MipsISD::MFHI";
206   case MipsISD::MFLO:              return "MipsISD::MFLO";
207   case MipsISD::MTLOHI:            return "MipsISD::MTLOHI";
208   case MipsISD::Mult:              return "MipsISD::Mult";
209   case MipsISD::Multu:             return "MipsISD::Multu";
210   case MipsISD::MAdd:              return "MipsISD::MAdd";
211   case MipsISD::MAddu:             return "MipsISD::MAddu";
212   case MipsISD::MSub:              return "MipsISD::MSub";
213   case MipsISD::MSubu:             return "MipsISD::MSubu";
214   case MipsISD::DivRem:            return "MipsISD::DivRem";
215   case MipsISD::DivRemU:           return "MipsISD::DivRemU";
216   case MipsISD::DivRem16:          return "MipsISD::DivRem16";
217   case MipsISD::DivRemU16:         return "MipsISD::DivRemU16";
218   case MipsISD::BuildPairF64:      return "MipsISD::BuildPairF64";
219   case MipsISD::ExtractElementF64: return "MipsISD::ExtractElementF64";
220   case MipsISD::Wrapper:           return "MipsISD::Wrapper";
221   case MipsISD::DynAlloc:          return "MipsISD::DynAlloc";
222   case MipsISD::Sync:              return "MipsISD::Sync";
223   case MipsISD::Ext:               return "MipsISD::Ext";
224   case MipsISD::Ins:               return "MipsISD::Ins";
225   case MipsISD::CIns:              return "MipsISD::CIns";
226   case MipsISD::LWL:               return "MipsISD::LWL";
227   case MipsISD::LWR:               return "MipsISD::LWR";
228   case MipsISD::SWL:               return "MipsISD::SWL";
229   case MipsISD::SWR:               return "MipsISD::SWR";
230   case MipsISD::LDL:               return "MipsISD::LDL";
231   case MipsISD::LDR:               return "MipsISD::LDR";
232   case MipsISD::SDL:               return "MipsISD::SDL";
233   case MipsISD::SDR:               return "MipsISD::SDR";
234   case MipsISD::EXTP:              return "MipsISD::EXTP";
235   case MipsISD::EXTPDP:            return "MipsISD::EXTPDP";
236   case MipsISD::EXTR_S_H:          return "MipsISD::EXTR_S_H";
237   case MipsISD::EXTR_W:            return "MipsISD::EXTR_W";
238   case MipsISD::EXTR_R_W:          return "MipsISD::EXTR_R_W";
239   case MipsISD::EXTR_RS_W:         return "MipsISD::EXTR_RS_W";
240   case MipsISD::SHILO:             return "MipsISD::SHILO";
241   case MipsISD::MTHLIP:            return "MipsISD::MTHLIP";
242   case MipsISD::MULSAQ_S_W_PH:     return "MipsISD::MULSAQ_S_W_PH";
243   case MipsISD::MAQ_S_W_PHL:       return "MipsISD::MAQ_S_W_PHL";
244   case MipsISD::MAQ_S_W_PHR:       return "MipsISD::MAQ_S_W_PHR";
245   case MipsISD::MAQ_SA_W_PHL:      return "MipsISD::MAQ_SA_W_PHL";
246   case MipsISD::MAQ_SA_W_PHR:      return "MipsISD::MAQ_SA_W_PHR";
247   case MipsISD::DPAU_H_QBL:        return "MipsISD::DPAU_H_QBL";
248   case MipsISD::DPAU_H_QBR:        return "MipsISD::DPAU_H_QBR";
249   case MipsISD::DPSU_H_QBL:        return "MipsISD::DPSU_H_QBL";
250   case MipsISD::DPSU_H_QBR:        return "MipsISD::DPSU_H_QBR";
251   case MipsISD::DPAQ_S_W_PH:       return "MipsISD::DPAQ_S_W_PH";
252   case MipsISD::DPSQ_S_W_PH:       return "MipsISD::DPSQ_S_W_PH";
253   case MipsISD::DPAQ_SA_L_W:       return "MipsISD::DPAQ_SA_L_W";
254   case MipsISD::DPSQ_SA_L_W:       return "MipsISD::DPSQ_SA_L_W";
255   case MipsISD::DPA_W_PH:          return "MipsISD::DPA_W_PH";
256   case MipsISD::DPS_W_PH:          return "MipsISD::DPS_W_PH";
257   case MipsISD::DPAQX_S_W_PH:      return "MipsISD::DPAQX_S_W_PH";
258   case MipsISD::DPAQX_SA_W_PH:     return "MipsISD::DPAQX_SA_W_PH";
259   case MipsISD::DPAX_W_PH:         return "MipsISD::DPAX_W_PH";
260   case MipsISD::DPSX_W_PH:         return "MipsISD::DPSX_W_PH";
261   case MipsISD::DPSQX_S_W_PH:      return "MipsISD::DPSQX_S_W_PH";
262   case MipsISD::DPSQX_SA_W_PH:     return "MipsISD::DPSQX_SA_W_PH";
263   case MipsISD::MULSA_W_PH:        return "MipsISD::MULSA_W_PH";
264   case MipsISD::MULT:              return "MipsISD::MULT";
265   case MipsISD::MULTU:             return "MipsISD::MULTU";
266   case MipsISD::MADD_DSP:          return "MipsISD::MADD_DSP";
267   case MipsISD::MADDU_DSP:         return "MipsISD::MADDU_DSP";
268   case MipsISD::MSUB_DSP:          return "MipsISD::MSUB_DSP";
269   case MipsISD::MSUBU_DSP:         return "MipsISD::MSUBU_DSP";
270   case MipsISD::SHLL_DSP:          return "MipsISD::SHLL_DSP";
271   case MipsISD::SHRA_DSP:          return "MipsISD::SHRA_DSP";
272   case MipsISD::SHRL_DSP:          return "MipsISD::SHRL_DSP";
273   case MipsISD::SETCC_DSP:         return "MipsISD::SETCC_DSP";
274   case MipsISD::SELECT_CC_DSP:     return "MipsISD::SELECT_CC_DSP";
275   case MipsISD::VALL_ZERO:         return "MipsISD::VALL_ZERO";
276   case MipsISD::VANY_ZERO:         return "MipsISD::VANY_ZERO";
277   case MipsISD::VALL_NONZERO:      return "MipsISD::VALL_NONZERO";
278   case MipsISD::VANY_NONZERO:      return "MipsISD::VANY_NONZERO";
279   case MipsISD::VCEQ:              return "MipsISD::VCEQ";
280   case MipsISD::VCLE_S:            return "MipsISD::VCLE_S";
281   case MipsISD::VCLE_U:            return "MipsISD::VCLE_U";
282   case MipsISD::VCLT_S:            return "MipsISD::VCLT_S";
283   case MipsISD::VCLT_U:            return "MipsISD::VCLT_U";
284   case MipsISD::VEXTRACT_SEXT_ELT: return "MipsISD::VEXTRACT_SEXT_ELT";
285   case MipsISD::VEXTRACT_ZEXT_ELT: return "MipsISD::VEXTRACT_ZEXT_ELT";
286   case MipsISD::VNOR:              return "MipsISD::VNOR";
287   case MipsISD::VSHF:              return "MipsISD::VSHF";
288   case MipsISD::SHF:               return "MipsISD::SHF";
289   case MipsISD::ILVEV:             return "MipsISD::ILVEV";
290   case MipsISD::ILVOD:             return "MipsISD::ILVOD";
291   case MipsISD::ILVL:              return "MipsISD::ILVL";
292   case MipsISD::ILVR:              return "MipsISD::ILVR";
293   case MipsISD::PCKEV:             return "MipsISD::PCKEV";
294   case MipsISD::PCKOD:             return "MipsISD::PCKOD";
295   case MipsISD::INSVE:             return "MipsISD::INSVE";
296   }
297   return nullptr;
298 }
299 
300 MipsTargetLowering::MipsTargetLowering(const MipsTargetMachine &TM,
301                                        const MipsSubtarget &STI)
302     : TargetLowering(TM), Subtarget(STI), ABI(TM.getABI()) {
303   // Mips does not have i1 type, so use i32 for
304   // setcc operations results (slt, sgt, ...).
305   setBooleanContents(ZeroOrOneBooleanContent);
306   setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
307   // The cmp.cond.fmt instruction in MIPS32r6/MIPS64r6 uses 0 and -1 like MSA
308   // does. Integer booleans still use 0 and 1.
309   if (Subtarget.hasMips32r6())
310     setBooleanContents(ZeroOrOneBooleanContent,
311                        ZeroOrNegativeOneBooleanContent);
312 
313   // Load extented operations for i1 types must be promoted
314   for (MVT VT : MVT::integer_valuetypes()) {
315     setLoadExtAction(ISD::EXTLOAD,  VT, MVT::i1,  Promote);
316     setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1,  Promote);
317     setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1,  Promote);
318   }
319 
320   // MIPS doesn't have extending float->double load/store.  Set LoadExtAction
321   // for f32, f16
322   for (MVT VT : MVT::fp_valuetypes()) {
323     setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);
324     setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
325   }
326 
327   // Set LoadExtAction for f16 vectors to Expand
328   for (MVT VT : MVT::fp_vector_valuetypes()) {
329     MVT F16VT = MVT::getVectorVT(MVT::f16, VT.getVectorNumElements());
330     if (F16VT.isValid())
331       setLoadExtAction(ISD::EXTLOAD, VT, F16VT, Expand);
332   }
333 
334   setTruncStoreAction(MVT::f32, MVT::f16, Expand);
335   setTruncStoreAction(MVT::f64, MVT::f16, Expand);
336 
337   setTruncStoreAction(MVT::f64, MVT::f32, Expand);
338 
339   // Used by legalize types to correctly generate the setcc result.
340   // Without this, every float setcc comes with a AND/OR with the result,
341   // we don't want this, since the fpcmp result goes to a flag register,
342   // which is used implicitly by brcond and select operations.
343   AddPromotedToType(ISD::SETCC, MVT::i1, MVT::i32);
344 
345   // Mips Custom Operations
346   setOperationAction(ISD::BR_JT,              MVT::Other, Expand);
347   setOperationAction(ISD::GlobalAddress,      MVT::i32,   Custom);
348   setOperationAction(ISD::BlockAddress,       MVT::i32,   Custom);
349   setOperationAction(ISD::GlobalTLSAddress,   MVT::i32,   Custom);
350   setOperationAction(ISD::JumpTable,          MVT::i32,   Custom);
351   setOperationAction(ISD::ConstantPool,       MVT::i32,   Custom);
352   setOperationAction(ISD::SELECT,             MVT::f32,   Custom);
353   setOperationAction(ISD::SELECT,             MVT::f64,   Custom);
354   setOperationAction(ISD::SELECT,             MVT::i32,   Custom);
355   setOperationAction(ISD::SETCC,              MVT::f32,   Custom);
356   setOperationAction(ISD::SETCC,              MVT::f64,   Custom);
357   setOperationAction(ISD::BRCOND,             MVT::Other, Custom);
358   setOperationAction(ISD::FCOPYSIGN,          MVT::f32,   Custom);
359   setOperationAction(ISD::FCOPYSIGN,          MVT::f64,   Custom);
360   setOperationAction(ISD::FP_TO_SINT,         MVT::i32,   Custom);
361 
362   if (Subtarget.isGP64bit()) {
363     setOperationAction(ISD::GlobalAddress,      MVT::i64,   Custom);
364     setOperationAction(ISD::BlockAddress,       MVT::i64,   Custom);
365     setOperationAction(ISD::GlobalTLSAddress,   MVT::i64,   Custom);
366     setOperationAction(ISD::JumpTable,          MVT::i64,   Custom);
367     setOperationAction(ISD::ConstantPool,       MVT::i64,   Custom);
368     setOperationAction(ISD::SELECT,             MVT::i64,   Custom);
369     setOperationAction(ISD::LOAD,               MVT::i64,   Custom);
370     setOperationAction(ISD::STORE,              MVT::i64,   Custom);
371     setOperationAction(ISD::FP_TO_SINT,         MVT::i64,   Custom);
372     setOperationAction(ISD::SHL_PARTS,          MVT::i64,   Custom);
373     setOperationAction(ISD::SRA_PARTS,          MVT::i64,   Custom);
374     setOperationAction(ISD::SRL_PARTS,          MVT::i64,   Custom);
375   }
376 
377   if (!Subtarget.isGP64bit()) {
378     setOperationAction(ISD::SHL_PARTS,          MVT::i32,   Custom);
379     setOperationAction(ISD::SRA_PARTS,          MVT::i32,   Custom);
380     setOperationAction(ISD::SRL_PARTS,          MVT::i32,   Custom);
381   }
382 
383   setOperationAction(ISD::EH_DWARF_CFA,         MVT::i32,   Custom);
384   if (Subtarget.isGP64bit())
385     setOperationAction(ISD::EH_DWARF_CFA,       MVT::i64,   Custom);
386 
387   setOperationAction(ISD::SDIV, MVT::i32, Expand);
388   setOperationAction(ISD::SREM, MVT::i32, Expand);
389   setOperationAction(ISD::UDIV, MVT::i32, Expand);
390   setOperationAction(ISD::UREM, MVT::i32, Expand);
391   setOperationAction(ISD::SDIV, MVT::i64, Expand);
392   setOperationAction(ISD::SREM, MVT::i64, Expand);
393   setOperationAction(ISD::UDIV, MVT::i64, Expand);
394   setOperationAction(ISD::UREM, MVT::i64, Expand);
395 
396   // Operations not directly supported by Mips.
397   setOperationAction(ISD::BR_CC,             MVT::f32,   Expand);
398   setOperationAction(ISD::BR_CC,             MVT::f64,   Expand);
399   setOperationAction(ISD::BR_CC,             MVT::i32,   Expand);
400   setOperationAction(ISD::BR_CC,             MVT::i64,   Expand);
401   setOperationAction(ISD::SELECT_CC,         MVT::i32,   Expand);
402   setOperationAction(ISD::SELECT_CC,         MVT::i64,   Expand);
403   setOperationAction(ISD::SELECT_CC,         MVT::f32,   Expand);
404   setOperationAction(ISD::SELECT_CC,         MVT::f64,   Expand);
405   setOperationAction(ISD::UINT_TO_FP,        MVT::i32,   Expand);
406   setOperationAction(ISD::UINT_TO_FP,        MVT::i64,   Expand);
407   setOperationAction(ISD::FP_TO_UINT,        MVT::i32,   Expand);
408   setOperationAction(ISD::FP_TO_UINT,        MVT::i64,   Expand);
409   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1,    Expand);
410   if (Subtarget.hasCnMips()) {
411     setOperationAction(ISD::CTPOP,           MVT::i32,   Legal);
412     setOperationAction(ISD::CTPOP,           MVT::i64,   Legal);
413   } else {
414     setOperationAction(ISD::CTPOP,           MVT::i32,   Expand);
415     setOperationAction(ISD::CTPOP,           MVT::i64,   Expand);
416   }
417   setOperationAction(ISD::CTTZ,              MVT::i32,   Expand);
418   setOperationAction(ISD::CTTZ,              MVT::i64,   Expand);
419   setOperationAction(ISD::ROTL,              MVT::i32,   Expand);
420   setOperationAction(ISD::ROTL,              MVT::i64,   Expand);
421   setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32,  Expand);
422   setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64,  Expand);
423 
424   if (!Subtarget.hasMips32r2())
425     setOperationAction(ISD::ROTR, MVT::i32,   Expand);
426 
427   if (!Subtarget.hasMips64r2())
428     setOperationAction(ISD::ROTR, MVT::i64,   Expand);
429 
430   setOperationAction(ISD::FSIN,              MVT::f32,   Expand);
431   setOperationAction(ISD::FSIN,              MVT::f64,   Expand);
432   setOperationAction(ISD::FCOS,              MVT::f32,   Expand);
433   setOperationAction(ISD::FCOS,              MVT::f64,   Expand);
434   setOperationAction(ISD::FSINCOS,           MVT::f32,   Expand);
435   setOperationAction(ISD::FSINCOS,           MVT::f64,   Expand);
436   setOperationAction(ISD::FPOW,              MVT::f32,   Expand);
437   setOperationAction(ISD::FPOW,              MVT::f64,   Expand);
438   setOperationAction(ISD::FLOG,              MVT::f32,   Expand);
439   setOperationAction(ISD::FLOG2,             MVT::f32,   Expand);
440   setOperationAction(ISD::FLOG10,            MVT::f32,   Expand);
441   setOperationAction(ISD::FEXP,              MVT::f32,   Expand);
442   setOperationAction(ISD::FMA,               MVT::f32,   Expand);
443   setOperationAction(ISD::FMA,               MVT::f64,   Expand);
444   setOperationAction(ISD::FREM,              MVT::f32,   Expand);
445   setOperationAction(ISD::FREM,              MVT::f64,   Expand);
446 
447   // Lower f16 conversion operations into library calls
448   setOperationAction(ISD::FP16_TO_FP,        MVT::f32,   Expand);
449   setOperationAction(ISD::FP_TO_FP16,        MVT::f32,   Expand);
450   setOperationAction(ISD::FP16_TO_FP,        MVT::f64,   Expand);
451   setOperationAction(ISD::FP_TO_FP16,        MVT::f64,   Expand);
452 
453   setOperationAction(ISD::EH_RETURN, MVT::Other, Custom);
454 
455   setOperationAction(ISD::VASTART,           MVT::Other, Custom);
456   setOperationAction(ISD::VAARG,             MVT::Other, Custom);
457   setOperationAction(ISD::VACOPY,            MVT::Other, Expand);
458   setOperationAction(ISD::VAEND,             MVT::Other, Expand);
459 
460   // Use the default for now
461   setOperationAction(ISD::STACKSAVE,         MVT::Other, Expand);
462   setOperationAction(ISD::STACKRESTORE,      MVT::Other, Expand);
463 
464   if (!Subtarget.isGP64bit()) {
465     setOperationAction(ISD::ATOMIC_LOAD,     MVT::i64,   Expand);
466     setOperationAction(ISD::ATOMIC_STORE,    MVT::i64,   Expand);
467   }
468 
469   if (!Subtarget.hasMips32r2()) {
470     setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8,  Expand);
471     setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand);
472   }
473 
474   // MIPS16 lacks MIPS32's clz and clo instructions.
475   if (!Subtarget.hasMips32() || Subtarget.inMips16Mode())
476     setOperationAction(ISD::CTLZ, MVT::i32, Expand);
477   if (!Subtarget.hasMips64())
478     setOperationAction(ISD::CTLZ, MVT::i64, Expand);
479 
480   if (!Subtarget.hasMips32r2())
481     setOperationAction(ISD::BSWAP, MVT::i32, Expand);
482   if (!Subtarget.hasMips64r2())
483     setOperationAction(ISD::BSWAP, MVT::i64, Expand);
484 
485   if (Subtarget.isGP64bit()) {
486     setLoadExtAction(ISD::SEXTLOAD, MVT::i64, MVT::i32, Custom);
487     setLoadExtAction(ISD::ZEXTLOAD, MVT::i64, MVT::i32, Custom);
488     setLoadExtAction(ISD::EXTLOAD, MVT::i64, MVT::i32, Custom);
489     setTruncStoreAction(MVT::i64, MVT::i32, Custom);
490   }
491 
492   setOperationAction(ISD::TRAP, MVT::Other, Legal);
493 
494   setTargetDAGCombine(ISD::SDIVREM);
495   setTargetDAGCombine(ISD::UDIVREM);
496   setTargetDAGCombine(ISD::SELECT);
497   setTargetDAGCombine(ISD::AND);
498   setTargetDAGCombine(ISD::OR);
499   setTargetDAGCombine(ISD::ADD);
500   setTargetDAGCombine(ISD::SUB);
501   setTargetDAGCombine(ISD::AssertZext);
502   setTargetDAGCombine(ISD::SHL);
503 
504   if (ABI.IsO32()) {
505     // These libcalls are not available in 32-bit.
506     setLibcallName(RTLIB::SHL_I128, nullptr);
507     setLibcallName(RTLIB::SRL_I128, nullptr);
508     setLibcallName(RTLIB::SRA_I128, nullptr);
509   }
510 
511   setMinFunctionAlignment(Subtarget.isGP64bit() ? 3 : 2);
512 
513   // The arguments on the stack are defined in terms of 4-byte slots on O32
514   // and 8-byte slots on N32/N64.
515   setMinStackArgumentAlignment((ABI.IsN32() || ABI.IsN64()) ? 8 : 4);
516 
517   setStackPointerRegisterToSaveRestore(ABI.IsN64() ? Mips::SP_64 : Mips::SP);
518 
519   MaxStoresPerMemcpy = 16;
520 
521   isMicroMips = Subtarget.inMicroMipsMode();
522 }
523 
524 const MipsTargetLowering *MipsTargetLowering::create(const MipsTargetMachine &TM,
525                                                      const MipsSubtarget &STI) {
526   if (STI.inMips16Mode())
527     return createMips16TargetLowering(TM, STI);
528 
529   return createMipsSETargetLowering(TM, STI);
530 }
531 
532 // Create a fast isel object.
533 FastISel *
534 MipsTargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
535                                   const TargetLibraryInfo *libInfo) const {
536   const MipsTargetMachine &TM =
537       static_cast<const MipsTargetMachine &>(funcInfo.MF->getTarget());
538 
539   // We support only the standard encoding [MIPS32,MIPS32R5] ISAs.
540   bool UseFastISel = TM.Options.EnableFastISel && Subtarget.hasMips32() &&
541                      !Subtarget.hasMips32r6() && !Subtarget.inMips16Mode() &&
542                      !Subtarget.inMicroMipsMode();
543 
544   // Disable if either of the following is true:
545   // We do not generate PIC, the ABI is not O32, LargeGOT is being used.
546   if (!TM.isPositionIndependent() || !TM.getABI().IsO32() || LargeGOT)
547     UseFastISel = false;
548 
549   return UseFastISel ? Mips::createFastISel(funcInfo, libInfo) : nullptr;
550 }
551 
552 EVT MipsTargetLowering::getSetCCResultType(const DataLayout &, LLVMContext &,
553                                            EVT VT) const {
554   if (!VT.isVector())
555     return MVT::i32;
556   return VT.changeVectorElementTypeToInteger();
557 }
558 
559 static SDValue performDivRemCombine(SDNode *N, SelectionDAG &DAG,
560                                     TargetLowering::DAGCombinerInfo &DCI,
561                                     const MipsSubtarget &Subtarget) {
562   if (DCI.isBeforeLegalizeOps())
563     return SDValue();
564 
565   EVT Ty = N->getValueType(0);
566   unsigned LO = (Ty == MVT::i32) ? Mips::LO0 : Mips::LO0_64;
567   unsigned HI = (Ty == MVT::i32) ? Mips::HI0 : Mips::HI0_64;
568   unsigned Opc = N->getOpcode() == ISD::SDIVREM ? MipsISD::DivRem16 :
569                                                   MipsISD::DivRemU16;
570   SDLoc DL(N);
571 
572   SDValue DivRem = DAG.getNode(Opc, DL, MVT::Glue,
573                                N->getOperand(0), N->getOperand(1));
574   SDValue InChain = DAG.getEntryNode();
575   SDValue InGlue = DivRem;
576 
577   // insert MFLO
578   if (N->hasAnyUseOfValue(0)) {
579     SDValue CopyFromLo = DAG.getCopyFromReg(InChain, DL, LO, Ty,
580                                             InGlue);
581     DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), CopyFromLo);
582     InChain = CopyFromLo.getValue(1);
583     InGlue = CopyFromLo.getValue(2);
584   }
585 
586   // insert MFHI
587   if (N->hasAnyUseOfValue(1)) {
588     SDValue CopyFromHi = DAG.getCopyFromReg(InChain, DL,
589                                             HI, Ty, InGlue);
590     DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), CopyFromHi);
591   }
592 
593   return SDValue();
594 }
595 
596 static Mips::CondCode condCodeToFCC(ISD::CondCode CC) {
597   switch (CC) {
598   default: llvm_unreachable("Unknown fp condition code!");
599   case ISD::SETEQ:
600   case ISD::SETOEQ: return Mips::FCOND_OEQ;
601   case ISD::SETUNE: return Mips::FCOND_UNE;
602   case ISD::SETLT:
603   case ISD::SETOLT: return Mips::FCOND_OLT;
604   case ISD::SETGT:
605   case ISD::SETOGT: return Mips::FCOND_OGT;
606   case ISD::SETLE:
607   case ISD::SETOLE: return Mips::FCOND_OLE;
608   case ISD::SETGE:
609   case ISD::SETOGE: return Mips::FCOND_OGE;
610   case ISD::SETULT: return Mips::FCOND_ULT;
611   case ISD::SETULE: return Mips::FCOND_ULE;
612   case ISD::SETUGT: return Mips::FCOND_UGT;
613   case ISD::SETUGE: return Mips::FCOND_UGE;
614   case ISD::SETUO:  return Mips::FCOND_UN;
615   case ISD::SETO:   return Mips::FCOND_OR;
616   case ISD::SETNE:
617   case ISD::SETONE: return Mips::FCOND_ONE;
618   case ISD::SETUEQ: return Mips::FCOND_UEQ;
619   }
620 }
621 
622 /// This function returns true if the floating point conditional branches and
623 /// conditional moves which use condition code CC should be inverted.
624 static bool invertFPCondCodeUser(Mips::CondCode CC) {
625   if (CC >= Mips::FCOND_F && CC <= Mips::FCOND_NGT)
626     return false;
627 
628   assert((CC >= Mips::FCOND_T && CC <= Mips::FCOND_GT) &&
629          "Illegal Condition Code");
630 
631   return true;
632 }
633 
634 // Creates and returns an FPCmp node from a setcc node.
635 // Returns Op if setcc is not a floating point comparison.
636 static SDValue createFPCmp(SelectionDAG &DAG, const SDValue &Op) {
637   // must be a SETCC node
638   if (Op.getOpcode() != ISD::SETCC)
639     return Op;
640 
641   SDValue LHS = Op.getOperand(0);
642 
643   if (!LHS.getValueType().isFloatingPoint())
644     return Op;
645 
646   SDValue RHS = Op.getOperand(1);
647   SDLoc DL(Op);
648 
649   // Assume the 3rd operand is a CondCodeSDNode. Add code to check the type of
650   // node if necessary.
651   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
652 
653   return DAG.getNode(MipsISD::FPCmp, DL, MVT::Glue, LHS, RHS,
654                      DAG.getConstant(condCodeToFCC(CC), DL, MVT::i32));
655 }
656 
657 // Creates and returns a CMovFPT/F node.
658 static SDValue createCMovFP(SelectionDAG &DAG, SDValue Cond, SDValue True,
659                             SDValue False, const SDLoc &DL) {
660   ConstantSDNode *CC = cast<ConstantSDNode>(Cond.getOperand(2));
661   bool invert = invertFPCondCodeUser((Mips::CondCode)CC->getSExtValue());
662   SDValue FCC0 = DAG.getRegister(Mips::FCC0, MVT::i32);
663 
664   return DAG.getNode((invert ? MipsISD::CMovFP_F : MipsISD::CMovFP_T), DL,
665                      True.getValueType(), True, FCC0, False, Cond);
666 }
667 
668 static SDValue performSELECTCombine(SDNode *N, SelectionDAG &DAG,
669                                     TargetLowering::DAGCombinerInfo &DCI,
670                                     const MipsSubtarget &Subtarget) {
671   if (DCI.isBeforeLegalizeOps())
672     return SDValue();
673 
674   SDValue SetCC = N->getOperand(0);
675 
676   if ((SetCC.getOpcode() != ISD::SETCC) ||
677       !SetCC.getOperand(0).getValueType().isInteger())
678     return SDValue();
679 
680   SDValue False = N->getOperand(2);
681   EVT FalseTy = False.getValueType();
682 
683   if (!FalseTy.isInteger())
684     return SDValue();
685 
686   ConstantSDNode *FalseC = dyn_cast<ConstantSDNode>(False);
687 
688   // If the RHS (False) is 0, we swap the order of the operands
689   // of ISD::SELECT (obviously also inverting the condition) so that we can
690   // take advantage of conditional moves using the $0 register.
691   // Example:
692   //   return (a != 0) ? x : 0;
693   //     load $reg, x
694   //     movz $reg, $0, a
695   if (!FalseC)
696     return SDValue();
697 
698   const SDLoc DL(N);
699 
700   if (!FalseC->getZExtValue()) {
701     ISD::CondCode CC = cast<CondCodeSDNode>(SetCC.getOperand(2))->get();
702     SDValue True = N->getOperand(1);
703 
704     SetCC = DAG.getSetCC(DL, SetCC.getValueType(), SetCC.getOperand(0),
705                          SetCC.getOperand(1), ISD::getSetCCInverse(CC, true));
706 
707     return DAG.getNode(ISD::SELECT, DL, FalseTy, SetCC, False, True);
708   }
709 
710   // If both operands are integer constants there's a possibility that we
711   // can do some interesting optimizations.
712   SDValue True = N->getOperand(1);
713   ConstantSDNode *TrueC = dyn_cast<ConstantSDNode>(True);
714 
715   if (!TrueC || !True.getValueType().isInteger())
716     return SDValue();
717 
718   // We'll also ignore MVT::i64 operands as this optimizations proves
719   // to be ineffective because of the required sign extensions as the result
720   // of a SETCC operator is always MVT::i32 for non-vector types.
721   if (True.getValueType() == MVT::i64)
722     return SDValue();
723 
724   int64_t Diff = TrueC->getSExtValue() - FalseC->getSExtValue();
725 
726   // 1)  (a < x) ? y : y-1
727   //  slti $reg1, a, x
728   //  addiu $reg2, $reg1, y-1
729   if (Diff == 1)
730     return DAG.getNode(ISD::ADD, DL, SetCC.getValueType(), SetCC, False);
731 
732   // 2)  (a < x) ? y-1 : y
733   //  slti $reg1, a, x
734   //  xor $reg1, $reg1, 1
735   //  addiu $reg2, $reg1, y-1
736   if (Diff == -1) {
737     ISD::CondCode CC = cast<CondCodeSDNode>(SetCC.getOperand(2))->get();
738     SetCC = DAG.getSetCC(DL, SetCC.getValueType(), SetCC.getOperand(0),
739                          SetCC.getOperand(1), ISD::getSetCCInverse(CC, true));
740     return DAG.getNode(ISD::ADD, DL, SetCC.getValueType(), SetCC, True);
741   }
742 
743   // Could not optimize.
744   return SDValue();
745 }
746 
747 static SDValue performCMovFPCombine(SDNode *N, SelectionDAG &DAG,
748                                     TargetLowering::DAGCombinerInfo &DCI,
749                                     const MipsSubtarget &Subtarget) {
750   if (DCI.isBeforeLegalizeOps())
751     return SDValue();
752 
753   SDValue ValueIfTrue = N->getOperand(0), ValueIfFalse = N->getOperand(2);
754 
755   ConstantSDNode *FalseC = dyn_cast<ConstantSDNode>(ValueIfFalse);
756   if (!FalseC || FalseC->getZExtValue())
757     return SDValue();
758 
759   // Since RHS (False) is 0, we swap the order of the True/False operands
760   // (obviously also inverting the condition) so that we can
761   // take advantage of conditional moves using the $0 register.
762   // Example:
763   //   return (a != 0) ? x : 0;
764   //     load $reg, x
765   //     movz $reg, $0, a
766   unsigned Opc = (N->getOpcode() == MipsISD::CMovFP_T) ? MipsISD::CMovFP_F :
767                                                          MipsISD::CMovFP_T;
768 
769   SDValue FCC = N->getOperand(1), Glue = N->getOperand(3);
770   return DAG.getNode(Opc, SDLoc(N), ValueIfFalse.getValueType(),
771                      ValueIfFalse, FCC, ValueIfTrue, Glue);
772 }
773 
774 static SDValue performANDCombine(SDNode *N, SelectionDAG &DAG,
775                                  TargetLowering::DAGCombinerInfo &DCI,
776                                  const MipsSubtarget &Subtarget) {
777   if (DCI.isBeforeLegalizeOps() || !Subtarget.hasExtractInsert())
778     return SDValue();
779 
780   SDValue FirstOperand = N->getOperand(0);
781   unsigned FirstOperandOpc = FirstOperand.getOpcode();
782   SDValue Mask = N->getOperand(1);
783   EVT ValTy = N->getValueType(0);
784   SDLoc DL(N);
785 
786   uint64_t Pos = 0, SMPos, SMSize;
787   ConstantSDNode *CN;
788   SDValue NewOperand;
789   unsigned Opc;
790 
791   // Op's second operand must be a shifted mask.
792   if (!(CN = dyn_cast<ConstantSDNode>(Mask)) ||
793       !isShiftedMask(CN->getZExtValue(), SMPos, SMSize))
794     return SDValue();
795 
796   if (FirstOperandOpc == ISD::SRA || FirstOperandOpc == ISD::SRL) {
797     // Pattern match EXT.
798     //  $dst = and ((sra or srl) $src , pos), (2**size - 1)
799     //  => ext $dst, $src, pos, size
800 
801     // The second operand of the shift must be an immediate.
802     if (!(CN = dyn_cast<ConstantSDNode>(FirstOperand.getOperand(1))))
803       return SDValue();
804 
805     Pos = CN->getZExtValue();
806 
807     // Return if the shifted mask does not start at bit 0 or the sum of its size
808     // and Pos exceeds the word's size.
809     if (SMPos != 0 || Pos + SMSize > ValTy.getSizeInBits())
810       return SDValue();
811 
812     Opc = MipsISD::Ext;
813     NewOperand = FirstOperand.getOperand(0);
814   } else if (FirstOperandOpc == ISD::SHL && Subtarget.hasCnMips()) {
815     // Pattern match CINS.
816     //  $dst = and (shl $src , pos), mask
817     //  => cins $dst, $src, pos, size
818     // mask is a shifted mask with consecutive 1's, pos = shift amount,
819     // size = population count.
820 
821     // The second operand of the shift must be an immediate.
822     if (!(CN = dyn_cast<ConstantSDNode>(FirstOperand.getOperand(1))))
823       return SDValue();
824 
825     Pos = CN->getZExtValue();
826 
827     if (SMPos != Pos || Pos >= ValTy.getSizeInBits() || SMSize >= 32 ||
828         Pos + SMSize > ValTy.getSizeInBits())
829       return SDValue();
830 
831     NewOperand = FirstOperand.getOperand(0);
832     // SMSize is 'location' (position) in this case, not size.
833     SMSize--;
834     Opc = MipsISD::CIns;
835   } else {
836     // Pattern match EXT.
837     //  $dst = and $src, (2**size - 1) , if size > 16
838     //  => ext $dst, $src, pos, size , pos = 0
839 
840     // If the mask is <= 0xffff, andi can be used instead.
841     if (CN->getZExtValue() <= 0xffff)
842       return SDValue();
843 
844     // Return if the mask doesn't start at position 0.
845     if (SMPos)
846       return SDValue();
847 
848     Opc = MipsISD::Ext;
849     NewOperand = FirstOperand;
850   }
851   return DAG.getNode(Opc, DL, ValTy, NewOperand,
852                      DAG.getConstant(Pos, DL, MVT::i32),
853                      DAG.getConstant(SMSize, DL, MVT::i32));
854 }
855 
856 static SDValue performORCombine(SDNode *N, SelectionDAG &DAG,
857                                 TargetLowering::DAGCombinerInfo &DCI,
858                                 const MipsSubtarget &Subtarget) {
859   // Pattern match INS.
860   //  $dst = or (and $src1 , mask0), (and (shl $src, pos), mask1),
861   //  where mask1 = (2**size - 1) << pos, mask0 = ~mask1
862   //  => ins $dst, $src, size, pos, $src1
863   if (DCI.isBeforeLegalizeOps() || !Subtarget.hasExtractInsert())
864     return SDValue();
865 
866   SDValue And0 = N->getOperand(0), And1 = N->getOperand(1);
867   uint64_t SMPos0, SMSize0, SMPos1, SMSize1;
868   ConstantSDNode *CN, *CN1;
869 
870   // See if Op's first operand matches (and $src1 , mask0).
871   if (And0.getOpcode() != ISD::AND)
872     return SDValue();
873 
874   if (!(CN = dyn_cast<ConstantSDNode>(And0.getOperand(1))) ||
875       !isShiftedMask(~CN->getSExtValue(), SMPos0, SMSize0))
876     return SDValue();
877 
878   // See if Op's second operand matches (and (shl $src, pos), mask1).
879   if (And1.getOpcode() == ISD::AND &&
880       And1.getOperand(0).getOpcode() == ISD::SHL) {
881 
882     if (!(CN = dyn_cast<ConstantSDNode>(And1.getOperand(1))) ||
883         !isShiftedMask(CN->getZExtValue(), SMPos1, SMSize1))
884       return SDValue();
885 
886     // The shift masks must have the same position and size.
887     if (SMPos0 != SMPos1 || SMSize0 != SMSize1)
888       return SDValue();
889 
890     SDValue Shl = And1.getOperand(0);
891 
892     if (!(CN = dyn_cast<ConstantSDNode>(Shl.getOperand(1))))
893       return SDValue();
894 
895     unsigned Shamt = CN->getZExtValue();
896 
897     // Return if the shift amount and the first bit position of mask are not the
898     // same.
899     EVT ValTy = N->getValueType(0);
900     if ((Shamt != SMPos0) || (SMPos0 + SMSize0 > ValTy.getSizeInBits()))
901       return SDValue();
902 
903     SDLoc DL(N);
904     return DAG.getNode(MipsISD::Ins, DL, ValTy, Shl.getOperand(0),
905                        DAG.getConstant(SMPos0, DL, MVT::i32),
906                        DAG.getConstant(SMSize0, DL, MVT::i32),
907                        And0.getOperand(0));
908   } else {
909     // Pattern match DINS.
910     //  $dst = or (and $src, mask0), mask1
911     //  where mask0 = ((1 << SMSize0) -1) << SMPos0
912     //  => dins $dst, $src, pos, size
913     if (~CN->getSExtValue() == ((((int64_t)1 << SMSize0) - 1) << SMPos0) &&
914         ((SMSize0 + SMPos0 <= 64 && Subtarget.hasMips64r2()) ||
915          (SMSize0 + SMPos0 <= 32))) {
916       // Check if AND instruction has constant as argument
917       bool isConstCase = And1.getOpcode() != ISD::AND;
918       if (And1.getOpcode() == ISD::AND) {
919         if (!(CN1 = dyn_cast<ConstantSDNode>(And1->getOperand(1))))
920           return SDValue();
921       } else {
922         if (!(CN1 = dyn_cast<ConstantSDNode>(N->getOperand(1))))
923           return SDValue();
924       }
925       // Don't generate INS if constant OR operand doesn't fit into bits
926       // cleared by constant AND operand.
927       if (CN->getSExtValue() & CN1->getSExtValue())
928         return SDValue();
929 
930       SDLoc DL(N);
931       EVT ValTy = N->getOperand(0)->getValueType(0);
932       SDValue Const1;
933       SDValue SrlX;
934       if (!isConstCase) {
935         Const1 = DAG.getConstant(SMPos0, DL, MVT::i32);
936         SrlX = DAG.getNode(ISD::SRL, DL, And1->getValueType(0), And1, Const1);
937       }
938       return DAG.getNode(
939           MipsISD::Ins, DL, N->getValueType(0),
940           isConstCase
941               ? DAG.getConstant(CN1->getSExtValue() >> SMPos0, DL, ValTy)
942               : SrlX,
943           DAG.getConstant(SMPos0, DL, MVT::i32),
944           DAG.getConstant(ValTy.getSizeInBits() / 8 < 8 ? SMSize0 & 31
945                                                         : SMSize0,
946                           DL, MVT::i32),
947           And0->getOperand(0));
948 
949     }
950     return SDValue();
951   }
952 }
953 
954 static SDValue performMADD_MSUBCombine(SDNode *ROOTNode, SelectionDAG &CurDAG,
955                                        const MipsSubtarget &Subtarget) {
956   // ROOTNode must have a multiplication as an operand for the match to be
957   // successful.
958   if (ROOTNode->getOperand(0).getOpcode() != ISD::MUL &&
959       ROOTNode->getOperand(1).getOpcode() != ISD::MUL)
960     return SDValue();
961 
962   // We don't handle vector types here.
963   if (ROOTNode->getValueType(0).isVector())
964     return SDValue();
965 
966   // For MIPS64, madd / msub instructions are inefficent to use with 64 bit
967   // arithmetic. E.g.
968   // (add (mul a b) c) =>
969   //   let res = (madd (mthi (drotr c 32))x(mtlo c) a b) in
970   //   MIPS64:   (or (dsll (mfhi res) 32) (dsrl (dsll (mflo res) 32) 32)
971   //   or
972   //   MIPS64R2: (dins (mflo res) (mfhi res) 32 32)
973   //
974   // The overhead of setting up the Hi/Lo registers and reassembling the
975   // result makes this a dubious optimzation for MIPS64. The core of the
976   // problem is that Hi/Lo contain the upper and lower 32 bits of the
977   // operand and result.
978   //
979   // It requires a chain of 4 add/mul for MIPS64R2 to get better code
980   // density than doing it naively, 5 for MIPS64. Additionally, using
981   // madd/msub on MIPS64 requires the operands actually be 32 bit sign
982   // extended operands, not true 64 bit values.
983   //
984   // FIXME: For the moment, disable this completely for MIPS64.
985   if (Subtarget.hasMips64())
986     return SDValue();
987 
988   SDValue Mult = ROOTNode->getOperand(0).getOpcode() == ISD::MUL
989                      ? ROOTNode->getOperand(0)
990                      : ROOTNode->getOperand(1);
991 
992   SDValue AddOperand = ROOTNode->getOperand(0).getOpcode() == ISD::MUL
993                      ? ROOTNode->getOperand(1)
994                      : ROOTNode->getOperand(0);
995 
996   // Transform this to a MADD only if the user of this node is the add.
997   // If there are other users of the mul, this function returns here.
998   if (!Mult.hasOneUse())
999     return SDValue();
1000 
1001   // maddu and madd are unusual instructions in that on MIPS64 bits 63..31
1002   // must be in canonical form, i.e. sign extended. For MIPS32, the operands
1003   // of the multiply must have 32 or more sign bits, otherwise we cannot
1004   // perform this optimization. We have to check this here as we're performing
1005   // this optimization pre-legalization.
1006   SDValue MultLHS = Mult->getOperand(0);
1007   SDValue MultRHS = Mult->getOperand(1);
1008 
1009   bool IsSigned = MultLHS->getOpcode() == ISD::SIGN_EXTEND &&
1010                   MultRHS->getOpcode() == ISD::SIGN_EXTEND;
1011   bool IsUnsigned = MultLHS->getOpcode() == ISD::ZERO_EXTEND &&
1012                     MultRHS->getOpcode() == ISD::ZERO_EXTEND;
1013 
1014   if (!IsSigned && !IsUnsigned)
1015     return SDValue();
1016 
1017   // Initialize accumulator.
1018   SDLoc DL(ROOTNode);
1019   SDValue TopHalf;
1020   SDValue BottomHalf;
1021   BottomHalf = CurDAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, AddOperand,
1022                               CurDAG.getIntPtrConstant(0, DL));
1023 
1024   TopHalf = CurDAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, AddOperand,
1025                            CurDAG.getIntPtrConstant(1, DL));
1026   SDValue ACCIn = CurDAG.getNode(MipsISD::MTLOHI, DL, MVT::Untyped,
1027                                   BottomHalf,
1028                                   TopHalf);
1029 
1030   // Create MipsMAdd(u) / MipsMSub(u) node.
1031   bool IsAdd = ROOTNode->getOpcode() == ISD::ADD;
1032   unsigned Opcode = IsAdd ? (IsUnsigned ? MipsISD::MAddu : MipsISD::MAdd)
1033                           : (IsUnsigned ? MipsISD::MSubu : MipsISD::MSub);
1034   SDValue MAddOps[3] = {
1035       CurDAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mult->getOperand(0)),
1036       CurDAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mult->getOperand(1)), ACCIn};
1037   EVT VTs[2] = {MVT::i32, MVT::i32};
1038   SDValue MAdd = CurDAG.getNode(Opcode, DL, VTs, MAddOps);
1039 
1040   SDValue ResLo = CurDAG.getNode(MipsISD::MFLO, DL, MVT::i32, MAdd);
1041   SDValue ResHi = CurDAG.getNode(MipsISD::MFHI, DL, MVT::i32, MAdd);
1042   SDValue Combined =
1043       CurDAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, ResLo, ResHi);
1044   return Combined;
1045 }
1046 
1047 static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG,
1048                                  TargetLowering::DAGCombinerInfo &DCI,
1049                                  const MipsSubtarget &Subtarget) {
1050   // (sub v0 (mul v1, v2)) => (msub v1, v2, v0)
1051   if (DCI.isBeforeLegalizeOps()) {
1052     if (Subtarget.hasMips32() && !Subtarget.hasMips32r6() &&
1053         !Subtarget.inMips16Mode() && N->getValueType(0) == MVT::i64)
1054       return performMADD_MSUBCombine(N, DAG, Subtarget);
1055 
1056     return SDValue();
1057   }
1058 
1059   return SDValue();
1060 }
1061 
1062 static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG,
1063                                  TargetLowering::DAGCombinerInfo &DCI,
1064                                  const MipsSubtarget &Subtarget) {
1065   // (add v0 (mul v1, v2)) => (madd v1, v2, v0)
1066   if (DCI.isBeforeLegalizeOps()) {
1067     if (Subtarget.hasMips32() && !Subtarget.hasMips32r6() &&
1068         !Subtarget.inMips16Mode() && N->getValueType(0) == MVT::i64)
1069       return performMADD_MSUBCombine(N, DAG, Subtarget);
1070 
1071     return SDValue();
1072   }
1073 
1074   // (add v0, (add v1, abs_lo(tjt))) => (add (add v0, v1), abs_lo(tjt))
1075   SDValue Add = N->getOperand(1);
1076 
1077   if (Add.getOpcode() != ISD::ADD)
1078     return SDValue();
1079 
1080   SDValue Lo = Add.getOperand(1);
1081 
1082   if ((Lo.getOpcode() != MipsISD::Lo) ||
1083       (Lo.getOperand(0).getOpcode() != ISD::TargetJumpTable))
1084     return SDValue();
1085 
1086   EVT ValTy = N->getValueType(0);
1087   SDLoc DL(N);
1088 
1089   SDValue Add1 = DAG.getNode(ISD::ADD, DL, ValTy, N->getOperand(0),
1090                              Add.getOperand(0));
1091   return DAG.getNode(ISD::ADD, DL, ValTy, Add1, Lo);
1092 }
1093 
1094 static SDValue performSHLCombine(SDNode *N, SelectionDAG &DAG,
1095                                  TargetLowering::DAGCombinerInfo &DCI,
1096                                  const MipsSubtarget &Subtarget) {
1097   // Pattern match CINS.
1098   //  $dst = shl (and $src , imm), pos
1099   //  => cins $dst, $src, pos, size
1100 
1101   if (DCI.isBeforeLegalizeOps() || !Subtarget.hasCnMips())
1102     return SDValue();
1103 
1104   SDValue FirstOperand = N->getOperand(0);
1105   unsigned FirstOperandOpc = FirstOperand.getOpcode();
1106   SDValue SecondOperand = N->getOperand(1);
1107   EVT ValTy = N->getValueType(0);
1108   SDLoc DL(N);
1109 
1110   uint64_t Pos = 0, SMPos, SMSize;
1111   ConstantSDNode *CN;
1112   SDValue NewOperand;
1113 
1114   // The second operand of the shift must be an immediate.
1115   if (!(CN = dyn_cast<ConstantSDNode>(SecondOperand)))
1116     return SDValue();
1117 
1118   Pos = CN->getZExtValue();
1119 
1120   if (Pos >= ValTy.getSizeInBits())
1121     return SDValue();
1122 
1123   if (FirstOperandOpc != ISD::AND)
1124     return SDValue();
1125 
1126   // AND's second operand must be a shifted mask.
1127   if (!(CN = dyn_cast<ConstantSDNode>(FirstOperand.getOperand(1))) ||
1128       !isShiftedMask(CN->getZExtValue(), SMPos, SMSize))
1129     return SDValue();
1130 
1131   // Return if the shifted mask does not start at bit 0 or the sum of its size
1132   // and Pos exceeds the word's size.
1133   if (SMPos != 0 || SMSize > 32 || Pos + SMSize > ValTy.getSizeInBits())
1134     return SDValue();
1135 
1136   NewOperand = FirstOperand.getOperand(0);
1137   // SMSize is 'location' (position) in this case, not size.
1138   SMSize--;
1139 
1140   return DAG.getNode(MipsISD::CIns, DL, ValTy, NewOperand,
1141                      DAG.getConstant(Pos, DL, MVT::i32),
1142                      DAG.getConstant(SMSize, DL, MVT::i32));
1143 }
1144 
1145 SDValue  MipsTargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI)
1146   const {
1147   SelectionDAG &DAG = DCI.DAG;
1148   unsigned Opc = N->getOpcode();
1149 
1150   switch (Opc) {
1151   default: break;
1152   case ISD::SDIVREM:
1153   case ISD::UDIVREM:
1154     return performDivRemCombine(N, DAG, DCI, Subtarget);
1155   case ISD::SELECT:
1156     return performSELECTCombine(N, DAG, DCI, Subtarget);
1157   case MipsISD::CMovFP_F:
1158   case MipsISD::CMovFP_T:
1159     return performCMovFPCombine(N, DAG, DCI, Subtarget);
1160   case ISD::AND:
1161     return performANDCombine(N, DAG, DCI, Subtarget);
1162   case ISD::OR:
1163     return performORCombine(N, DAG, DCI, Subtarget);
1164   case ISD::ADD:
1165     return performADDCombine(N, DAG, DCI, Subtarget);
1166   case ISD::SHL:
1167     return performSHLCombine(N, DAG, DCI, Subtarget);
1168   case ISD::SUB:
1169     return performSUBCombine(N, DAG, DCI, Subtarget);
1170   }
1171 
1172   return SDValue();
1173 }
1174 
1175 bool MipsTargetLowering::isCheapToSpeculateCttz() const {
1176   return Subtarget.hasMips32();
1177 }
1178 
1179 bool MipsTargetLowering::isCheapToSpeculateCtlz() const {
1180   return Subtarget.hasMips32();
1181 }
1182 
1183 void
1184 MipsTargetLowering::LowerOperationWrapper(SDNode *N,
1185                                           SmallVectorImpl<SDValue> &Results,
1186                                           SelectionDAG &DAG) const {
1187   SDValue Res = LowerOperation(SDValue(N, 0), DAG);
1188 
1189   for (unsigned I = 0, E = Res->getNumValues(); I != E; ++I)
1190     Results.push_back(Res.getValue(I));
1191 }
1192 
1193 void
1194 MipsTargetLowering::ReplaceNodeResults(SDNode *N,
1195                                        SmallVectorImpl<SDValue> &Results,
1196                                        SelectionDAG &DAG) const {
1197   return LowerOperationWrapper(N, Results, DAG);
1198 }
1199 
1200 SDValue MipsTargetLowering::
1201 LowerOperation(SDValue Op, SelectionDAG &DAG) const
1202 {
1203   switch (Op.getOpcode())
1204   {
1205   case ISD::BRCOND:             return lowerBRCOND(Op, DAG);
1206   case ISD::ConstantPool:       return lowerConstantPool(Op, DAG);
1207   case ISD::GlobalAddress:      return lowerGlobalAddress(Op, DAG);
1208   case ISD::BlockAddress:       return lowerBlockAddress(Op, DAG);
1209   case ISD::GlobalTLSAddress:   return lowerGlobalTLSAddress(Op, DAG);
1210   case ISD::JumpTable:          return lowerJumpTable(Op, DAG);
1211   case ISD::SELECT:             return lowerSELECT(Op, DAG);
1212   case ISD::SETCC:              return lowerSETCC(Op, DAG);
1213   case ISD::VASTART:            return lowerVASTART(Op, DAG);
1214   case ISD::VAARG:              return lowerVAARG(Op, DAG);
1215   case ISD::FCOPYSIGN:          return lowerFCOPYSIGN(Op, DAG);
1216   case ISD::FRAMEADDR:          return lowerFRAMEADDR(Op, DAG);
1217   case ISD::RETURNADDR:         return lowerRETURNADDR(Op, DAG);
1218   case ISD::EH_RETURN:          return lowerEH_RETURN(Op, DAG);
1219   case ISD::ATOMIC_FENCE:       return lowerATOMIC_FENCE(Op, DAG);
1220   case ISD::SHL_PARTS:          return lowerShiftLeftParts(Op, DAG);
1221   case ISD::SRA_PARTS:          return lowerShiftRightParts(Op, DAG, true);
1222   case ISD::SRL_PARTS:          return lowerShiftRightParts(Op, DAG, false);
1223   case ISD::LOAD:               return lowerLOAD(Op, DAG);
1224   case ISD::STORE:              return lowerSTORE(Op, DAG);
1225   case ISD::EH_DWARF_CFA:       return lowerEH_DWARF_CFA(Op, DAG);
1226   case ISD::FP_TO_SINT:         return lowerFP_TO_SINT(Op, DAG);
1227   }
1228   return SDValue();
1229 }
1230 
1231 //===----------------------------------------------------------------------===//
1232 //  Lower helper functions
1233 //===----------------------------------------------------------------------===//
1234 
1235 // addLiveIn - This helper function adds the specified physical register to the
1236 // MachineFunction as a live in value.  It also creates a corresponding
1237 // virtual register for it.
1238 static unsigned
1239 addLiveIn(MachineFunction &MF, unsigned PReg, const TargetRegisterClass *RC)
1240 {
1241   unsigned VReg = MF.getRegInfo().createVirtualRegister(RC);
1242   MF.getRegInfo().addLiveIn(PReg, VReg);
1243   return VReg;
1244 }
1245 
1246 static MachineBasicBlock *insertDivByZeroTrap(MachineInstr &MI,
1247                                               MachineBasicBlock &MBB,
1248                                               const TargetInstrInfo &TII,
1249                                               bool Is64Bit, bool IsMicroMips) {
1250   if (NoZeroDivCheck)
1251     return &MBB;
1252 
1253   // Insert instruction "teq $divisor_reg, $zero, 7".
1254   MachineBasicBlock::iterator I(MI);
1255   MachineInstrBuilder MIB;
1256   MachineOperand &Divisor = MI.getOperand(2);
1257   MIB = BuildMI(MBB, std::next(I), MI.getDebugLoc(),
1258                 TII.get(IsMicroMips ? Mips::TEQ_MM : Mips::TEQ))
1259             .addReg(Divisor.getReg(), getKillRegState(Divisor.isKill()))
1260             .addReg(Mips::ZERO)
1261             .addImm(7);
1262 
1263   // Use the 32-bit sub-register if this is a 64-bit division.
1264   if (Is64Bit)
1265     MIB->getOperand(0).setSubReg(Mips::sub_32);
1266 
1267   // Clear Divisor's kill flag.
1268   Divisor.setIsKill(false);
1269 
1270   // We would normally delete the original instruction here but in this case
1271   // we only needed to inject an additional instruction rather than replace it.
1272 
1273   return &MBB;
1274 }
1275 
1276 MachineBasicBlock *
1277 MipsTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
1278                                                 MachineBasicBlock *BB) const {
1279   switch (MI.getOpcode()) {
1280   default:
1281     llvm_unreachable("Unexpected instr type to insert");
1282   case Mips::ATOMIC_LOAD_ADD_I8:
1283     return emitAtomicBinaryPartword(MI, BB, 1, Mips::ADDu);
1284   case Mips::ATOMIC_LOAD_ADD_I16:
1285     return emitAtomicBinaryPartword(MI, BB, 2, Mips::ADDu);
1286   case Mips::ATOMIC_LOAD_ADD_I32:
1287     return emitAtomicBinary(MI, BB, 4, Mips::ADDu);
1288   case Mips::ATOMIC_LOAD_ADD_I64:
1289     return emitAtomicBinary(MI, BB, 8, Mips::DADDu);
1290 
1291   case Mips::ATOMIC_LOAD_AND_I8:
1292     return emitAtomicBinaryPartword(MI, BB, 1, Mips::AND);
1293   case Mips::ATOMIC_LOAD_AND_I16:
1294     return emitAtomicBinaryPartword(MI, BB, 2, Mips::AND);
1295   case Mips::ATOMIC_LOAD_AND_I32:
1296     return emitAtomicBinary(MI, BB, 4, Mips::AND);
1297   case Mips::ATOMIC_LOAD_AND_I64:
1298     return emitAtomicBinary(MI, BB, 8, Mips::AND64);
1299 
1300   case Mips::ATOMIC_LOAD_OR_I8:
1301     return emitAtomicBinaryPartword(MI, BB, 1, Mips::OR);
1302   case Mips::ATOMIC_LOAD_OR_I16:
1303     return emitAtomicBinaryPartword(MI, BB, 2, Mips::OR);
1304   case Mips::ATOMIC_LOAD_OR_I32:
1305     return emitAtomicBinary(MI, BB, 4, Mips::OR);
1306   case Mips::ATOMIC_LOAD_OR_I64:
1307     return emitAtomicBinary(MI, BB, 8, Mips::OR64);
1308 
1309   case Mips::ATOMIC_LOAD_XOR_I8:
1310     return emitAtomicBinaryPartword(MI, BB, 1, Mips::XOR);
1311   case Mips::ATOMIC_LOAD_XOR_I16:
1312     return emitAtomicBinaryPartword(MI, BB, 2, Mips::XOR);
1313   case Mips::ATOMIC_LOAD_XOR_I32:
1314     return emitAtomicBinary(MI, BB, 4, Mips::XOR);
1315   case Mips::ATOMIC_LOAD_XOR_I64:
1316     return emitAtomicBinary(MI, BB, 8, Mips::XOR64);
1317 
1318   case Mips::ATOMIC_LOAD_NAND_I8:
1319     return emitAtomicBinaryPartword(MI, BB, 1, 0, true);
1320   case Mips::ATOMIC_LOAD_NAND_I16:
1321     return emitAtomicBinaryPartword(MI, BB, 2, 0, true);
1322   case Mips::ATOMIC_LOAD_NAND_I32:
1323     return emitAtomicBinary(MI, BB, 4, 0, true);
1324   case Mips::ATOMIC_LOAD_NAND_I64:
1325     return emitAtomicBinary(MI, BB, 8, 0, true);
1326 
1327   case Mips::ATOMIC_LOAD_SUB_I8:
1328     return emitAtomicBinaryPartword(MI, BB, 1, Mips::SUBu);
1329   case Mips::ATOMIC_LOAD_SUB_I16:
1330     return emitAtomicBinaryPartword(MI, BB, 2, Mips::SUBu);
1331   case Mips::ATOMIC_LOAD_SUB_I32:
1332     return emitAtomicBinary(MI, BB, 4, Mips::SUBu);
1333   case Mips::ATOMIC_LOAD_SUB_I64:
1334     return emitAtomicBinary(MI, BB, 8, Mips::DSUBu);
1335 
1336   case Mips::ATOMIC_SWAP_I8:
1337     return emitAtomicBinaryPartword(MI, BB, 1, 0);
1338   case Mips::ATOMIC_SWAP_I16:
1339     return emitAtomicBinaryPartword(MI, BB, 2, 0);
1340   case Mips::ATOMIC_SWAP_I32:
1341     return emitAtomicBinary(MI, BB, 4, 0);
1342   case Mips::ATOMIC_SWAP_I64:
1343     return emitAtomicBinary(MI, BB, 8, 0);
1344 
1345   case Mips::ATOMIC_CMP_SWAP_I8:
1346     return emitAtomicCmpSwapPartword(MI, BB, 1);
1347   case Mips::ATOMIC_CMP_SWAP_I16:
1348     return emitAtomicCmpSwapPartword(MI, BB, 2);
1349   case Mips::ATOMIC_CMP_SWAP_I32:
1350     return emitAtomicCmpSwap(MI, BB, 4);
1351   case Mips::ATOMIC_CMP_SWAP_I64:
1352     return emitAtomicCmpSwap(MI, BB, 8);
1353   case Mips::PseudoSDIV:
1354   case Mips::PseudoUDIV:
1355   case Mips::DIV:
1356   case Mips::DIVU:
1357   case Mips::MOD:
1358   case Mips::MODU:
1359     return insertDivByZeroTrap(MI, *BB, *Subtarget.getInstrInfo(), false,
1360                                false);
1361   case Mips::SDIV_MM_Pseudo:
1362   case Mips::UDIV_MM_Pseudo:
1363   case Mips::SDIV_MM:
1364   case Mips::UDIV_MM:
1365   case Mips::DIV_MMR6:
1366   case Mips::DIVU_MMR6:
1367   case Mips::MOD_MMR6:
1368   case Mips::MODU_MMR6:
1369     return insertDivByZeroTrap(MI, *BB, *Subtarget.getInstrInfo(), false, true);
1370   case Mips::PseudoDSDIV:
1371   case Mips::PseudoDUDIV:
1372   case Mips::DDIV:
1373   case Mips::DDIVU:
1374   case Mips::DMOD:
1375   case Mips::DMODU:
1376     return insertDivByZeroTrap(MI, *BB, *Subtarget.getInstrInfo(), true, false);
1377 
1378   case Mips::PseudoSELECT_I:
1379   case Mips::PseudoSELECT_I64:
1380   case Mips::PseudoSELECT_S:
1381   case Mips::PseudoSELECT_D32:
1382   case Mips::PseudoSELECT_D64:
1383     return emitPseudoSELECT(MI, BB, false, Mips::BNE);
1384   case Mips::PseudoSELECTFP_F_I:
1385   case Mips::PseudoSELECTFP_F_I64:
1386   case Mips::PseudoSELECTFP_F_S:
1387   case Mips::PseudoSELECTFP_F_D32:
1388   case Mips::PseudoSELECTFP_F_D64:
1389     return emitPseudoSELECT(MI, BB, true, Mips::BC1F);
1390   case Mips::PseudoSELECTFP_T_I:
1391   case Mips::PseudoSELECTFP_T_I64:
1392   case Mips::PseudoSELECTFP_T_S:
1393   case Mips::PseudoSELECTFP_T_D32:
1394   case Mips::PseudoSELECTFP_T_D64:
1395     return emitPseudoSELECT(MI, BB, true, Mips::BC1T);
1396   }
1397 }
1398 
1399 // This function also handles Mips::ATOMIC_SWAP_I32 (when BinOpcode == 0), and
1400 // Mips::ATOMIC_LOAD_NAND_I32 (when Nand == true)
1401 MachineBasicBlock *MipsTargetLowering::emitAtomicBinary(MachineInstr &MI,
1402                                                         MachineBasicBlock *BB,
1403                                                         unsigned Size,
1404                                                         unsigned BinOpcode,
1405                                                         bool Nand) const {
1406   assert((Size == 4 || Size == 8) && "Unsupported size for EmitAtomicBinary.");
1407 
1408   MachineFunction *MF = BB->getParent();
1409   MachineRegisterInfo &RegInfo = MF->getRegInfo();
1410   const TargetRegisterClass *RC = getRegClassFor(MVT::getIntegerVT(Size * 8));
1411   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1412   const bool ArePtrs64bit = ABI.ArePtrs64bit();
1413   DebugLoc DL = MI.getDebugLoc();
1414   unsigned LL, SC, AND, NOR, ZERO, BEQ;
1415 
1416   if (Size == 4) {
1417     if (isMicroMips) {
1418       LL = Subtarget.hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
1419       SC = Subtarget.hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
1420     } else {
1421       LL = Subtarget.hasMips32r6()
1422                ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
1423                : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
1424       SC = Subtarget.hasMips32r6()
1425                ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
1426                : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
1427     }
1428 
1429     AND = Mips::AND;
1430     NOR = Mips::NOR;
1431     ZERO = Mips::ZERO;
1432     BEQ = Mips::BEQ;
1433   } else {
1434     LL = Subtarget.hasMips64r6() ? Mips::LLD_R6 : Mips::LLD;
1435     SC = Subtarget.hasMips64r6() ? Mips::SCD_R6 : Mips::SCD;
1436     AND = Mips::AND64;
1437     NOR = Mips::NOR64;
1438     ZERO = Mips::ZERO_64;
1439     BEQ = Mips::BEQ64;
1440   }
1441 
1442   unsigned OldVal = MI.getOperand(0).getReg();
1443   unsigned Ptr = MI.getOperand(1).getReg();
1444   unsigned Incr = MI.getOperand(2).getReg();
1445 
1446   unsigned StoreVal = RegInfo.createVirtualRegister(RC);
1447   unsigned AndRes = RegInfo.createVirtualRegister(RC);
1448   unsigned Success = RegInfo.createVirtualRegister(RC);
1449 
1450   // insert new blocks after the current block
1451   const BasicBlock *LLVM_BB = BB->getBasicBlock();
1452   MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
1453   MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
1454   MachineFunction::iterator It = ++BB->getIterator();
1455   MF->insert(It, loopMBB);
1456   MF->insert(It, exitMBB);
1457 
1458   // Transfer the remainder of BB and its successor edges to exitMBB.
1459   exitMBB->splice(exitMBB->begin(), BB,
1460                   std::next(MachineBasicBlock::iterator(MI)), BB->end());
1461   exitMBB->transferSuccessorsAndUpdatePHIs(BB);
1462 
1463   //  thisMBB:
1464   //    ...
1465   //    fallthrough --> loopMBB
1466   BB->addSuccessor(loopMBB);
1467   loopMBB->addSuccessor(loopMBB);
1468   loopMBB->addSuccessor(exitMBB);
1469 
1470   //  loopMBB:
1471   //    ll oldval, 0(ptr)
1472   //    <binop> storeval, oldval, incr
1473   //    sc success, storeval, 0(ptr)
1474   //    beq success, $0, loopMBB
1475   BB = loopMBB;
1476   BuildMI(BB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0);
1477   if (Nand) {
1478     //  and andres, oldval, incr
1479     //  nor storeval, $0, andres
1480     BuildMI(BB, DL, TII->get(AND), AndRes).addReg(OldVal).addReg(Incr);
1481     BuildMI(BB, DL, TII->get(NOR), StoreVal).addReg(ZERO).addReg(AndRes);
1482   } else if (BinOpcode) {
1483     //  <binop> storeval, oldval, incr
1484     BuildMI(BB, DL, TII->get(BinOpcode), StoreVal).addReg(OldVal).addReg(Incr);
1485   } else {
1486     StoreVal = Incr;
1487   }
1488   BuildMI(BB, DL, TII->get(SC), Success).addReg(StoreVal).addReg(Ptr).addImm(0);
1489   BuildMI(BB, DL, TII->get(BEQ)).addReg(Success).addReg(ZERO).addMBB(loopMBB);
1490 
1491   MI.eraseFromParent(); // The instruction is gone now.
1492 
1493   return exitMBB;
1494 }
1495 
1496 MachineBasicBlock *MipsTargetLowering::emitSignExtendToI32InReg(
1497     MachineInstr &MI, MachineBasicBlock *BB, unsigned Size, unsigned DstReg,
1498     unsigned SrcReg) const {
1499   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1500   const DebugLoc &DL = MI.getDebugLoc();
1501 
1502   if (Subtarget.hasMips32r2() && Size == 1) {
1503     BuildMI(BB, DL, TII->get(Mips::SEB), DstReg).addReg(SrcReg);
1504     return BB;
1505   }
1506 
1507   if (Subtarget.hasMips32r2() && Size == 2) {
1508     BuildMI(BB, DL, TII->get(Mips::SEH), DstReg).addReg(SrcReg);
1509     return BB;
1510   }
1511 
1512   MachineFunction *MF = BB->getParent();
1513   MachineRegisterInfo &RegInfo = MF->getRegInfo();
1514   const TargetRegisterClass *RC = getRegClassFor(MVT::i32);
1515   unsigned ScrReg = RegInfo.createVirtualRegister(RC);
1516 
1517   assert(Size < 32);
1518   int64_t ShiftImm = 32 - (Size * 8);
1519 
1520   BuildMI(BB, DL, TII->get(Mips::SLL), ScrReg).addReg(SrcReg).addImm(ShiftImm);
1521   BuildMI(BB, DL, TII->get(Mips::SRA), DstReg).addReg(ScrReg).addImm(ShiftImm);
1522 
1523   return BB;
1524 }
1525 
1526 MachineBasicBlock *MipsTargetLowering::emitAtomicBinaryPartword(
1527     MachineInstr &MI, MachineBasicBlock *BB, unsigned Size, unsigned BinOpcode,
1528     bool Nand) const {
1529   assert((Size == 1 || Size == 2) &&
1530          "Unsupported size for EmitAtomicBinaryPartial.");
1531 
1532   MachineFunction *MF = BB->getParent();
1533   MachineRegisterInfo &RegInfo = MF->getRegInfo();
1534   const TargetRegisterClass *RC = getRegClassFor(MVT::i32);
1535   const bool ArePtrs64bit = ABI.ArePtrs64bit();
1536   const TargetRegisterClass *RCp =
1537     getRegClassFor(ArePtrs64bit ? MVT::i64 : MVT::i32);
1538   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1539   DebugLoc DL = MI.getDebugLoc();
1540 
1541   unsigned Dest = MI.getOperand(0).getReg();
1542   unsigned Ptr = MI.getOperand(1).getReg();
1543   unsigned Incr = MI.getOperand(2).getReg();
1544 
1545   unsigned AlignedAddr = RegInfo.createVirtualRegister(RCp);
1546   unsigned ShiftAmt = RegInfo.createVirtualRegister(RC);
1547   unsigned Mask = RegInfo.createVirtualRegister(RC);
1548   unsigned Mask2 = RegInfo.createVirtualRegister(RC);
1549   unsigned NewVal = RegInfo.createVirtualRegister(RC);
1550   unsigned OldVal = RegInfo.createVirtualRegister(RC);
1551   unsigned Incr2 = RegInfo.createVirtualRegister(RC);
1552   unsigned MaskLSB2 = RegInfo.createVirtualRegister(RCp);
1553   unsigned PtrLSB2 = RegInfo.createVirtualRegister(RC);
1554   unsigned MaskUpper = RegInfo.createVirtualRegister(RC);
1555   unsigned AndRes = RegInfo.createVirtualRegister(RC);
1556   unsigned BinOpRes = RegInfo.createVirtualRegister(RC);
1557   unsigned MaskedOldVal0 = RegInfo.createVirtualRegister(RC);
1558   unsigned StoreVal = RegInfo.createVirtualRegister(RC);
1559   unsigned MaskedOldVal1 = RegInfo.createVirtualRegister(RC);
1560   unsigned SrlRes = RegInfo.createVirtualRegister(RC);
1561   unsigned Success = RegInfo.createVirtualRegister(RC);
1562 
1563   unsigned LL, SC;
1564   if (isMicroMips) {
1565     LL = Subtarget.hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
1566     SC = Subtarget.hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
1567   } else {
1568     LL = Subtarget.hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
1569                                  : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
1570     SC = Subtarget.hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
1571                                  : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
1572   }
1573 
1574   // insert new blocks after the current block
1575   const BasicBlock *LLVM_BB = BB->getBasicBlock();
1576   MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
1577   MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB);
1578   MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
1579   MachineFunction::iterator It = ++BB->getIterator();
1580   MF->insert(It, loopMBB);
1581   MF->insert(It, sinkMBB);
1582   MF->insert(It, exitMBB);
1583 
1584   // Transfer the remainder of BB and its successor edges to exitMBB.
1585   exitMBB->splice(exitMBB->begin(), BB,
1586                   std::next(MachineBasicBlock::iterator(MI)), BB->end());
1587   exitMBB->transferSuccessorsAndUpdatePHIs(BB);
1588 
1589   BB->addSuccessor(loopMBB);
1590   loopMBB->addSuccessor(loopMBB);
1591   loopMBB->addSuccessor(sinkMBB);
1592   sinkMBB->addSuccessor(exitMBB);
1593 
1594   //  thisMBB:
1595   //    addiu   masklsb2,$0,-4                # 0xfffffffc
1596   //    and     alignedaddr,ptr,masklsb2
1597   //    andi    ptrlsb2,ptr,3
1598   //    sll     shiftamt,ptrlsb2,3
1599   //    ori     maskupper,$0,255               # 0xff
1600   //    sll     mask,maskupper,shiftamt
1601   //    nor     mask2,$0,mask
1602   //    sll     incr2,incr,shiftamt
1603 
1604   int64_t MaskImm = (Size == 1) ? 255 : 65535;
1605   BuildMI(BB, DL, TII->get(ABI.GetPtrAddiuOp()), MaskLSB2)
1606     .addReg(ABI.GetNullPtr()).addImm(-4);
1607   BuildMI(BB, DL, TII->get(ABI.GetPtrAndOp()), AlignedAddr)
1608     .addReg(Ptr).addReg(MaskLSB2);
1609   BuildMI(BB, DL, TII->get(Mips::ANDi), PtrLSB2)
1610       .addReg(Ptr, 0, ArePtrs64bit ? Mips::sub_32 : 0).addImm(3);
1611   if (Subtarget.isLittle()) {
1612     BuildMI(BB, DL, TII->get(Mips::SLL), ShiftAmt).addReg(PtrLSB2).addImm(3);
1613   } else {
1614     unsigned Off = RegInfo.createVirtualRegister(RC);
1615     BuildMI(BB, DL, TII->get(Mips::XORi), Off)
1616       .addReg(PtrLSB2).addImm((Size == 1) ? 3 : 2);
1617     BuildMI(BB, DL, TII->get(Mips::SLL), ShiftAmt).addReg(Off).addImm(3);
1618   }
1619   BuildMI(BB, DL, TII->get(Mips::ORi), MaskUpper)
1620     .addReg(Mips::ZERO).addImm(MaskImm);
1621   BuildMI(BB, DL, TII->get(Mips::SLLV), Mask)
1622     .addReg(MaskUpper).addReg(ShiftAmt);
1623   BuildMI(BB, DL, TII->get(Mips::NOR), Mask2).addReg(Mips::ZERO).addReg(Mask);
1624   BuildMI(BB, DL, TII->get(Mips::SLLV), Incr2).addReg(Incr).addReg(ShiftAmt);
1625 
1626   // atomic.load.binop
1627   // loopMBB:
1628   //   ll      oldval,0(alignedaddr)
1629   //   binop   binopres,oldval,incr2
1630   //   and     newval,binopres,mask
1631   //   and     maskedoldval0,oldval,mask2
1632   //   or      storeval,maskedoldval0,newval
1633   //   sc      success,storeval,0(alignedaddr)
1634   //   beq     success,$0,loopMBB
1635 
1636   // atomic.swap
1637   // loopMBB:
1638   //   ll      oldval,0(alignedaddr)
1639   //   and     newval,incr2,mask
1640   //   and     maskedoldval0,oldval,mask2
1641   //   or      storeval,maskedoldval0,newval
1642   //   sc      success,storeval,0(alignedaddr)
1643   //   beq     success,$0,loopMBB
1644 
1645   BB = loopMBB;
1646   BuildMI(BB, DL, TII->get(LL), OldVal).addReg(AlignedAddr).addImm(0);
1647   if (Nand) {
1648     //  and andres, oldval, incr2
1649     //  nor binopres, $0, andres
1650     //  and newval, binopres, mask
1651     BuildMI(BB, DL, TII->get(Mips::AND), AndRes).addReg(OldVal).addReg(Incr2);
1652     BuildMI(BB, DL, TII->get(Mips::NOR), BinOpRes)
1653       .addReg(Mips::ZERO).addReg(AndRes);
1654     BuildMI(BB, DL, TII->get(Mips::AND), NewVal).addReg(BinOpRes).addReg(Mask);
1655   } else if (BinOpcode) {
1656     //  <binop> binopres, oldval, incr2
1657     //  and newval, binopres, mask
1658     BuildMI(BB, DL, TII->get(BinOpcode), BinOpRes).addReg(OldVal).addReg(Incr2);
1659     BuildMI(BB, DL, TII->get(Mips::AND), NewVal).addReg(BinOpRes).addReg(Mask);
1660   } else { // atomic.swap
1661     //  and newval, incr2, mask
1662     BuildMI(BB, DL, TII->get(Mips::AND), NewVal).addReg(Incr2).addReg(Mask);
1663   }
1664 
1665   BuildMI(BB, DL, TII->get(Mips::AND), MaskedOldVal0)
1666     .addReg(OldVal).addReg(Mask2);
1667   BuildMI(BB, DL, TII->get(Mips::OR), StoreVal)
1668     .addReg(MaskedOldVal0).addReg(NewVal);
1669   BuildMI(BB, DL, TII->get(SC), Success)
1670     .addReg(StoreVal).addReg(AlignedAddr).addImm(0);
1671   BuildMI(BB, DL, TII->get(Mips::BEQ))
1672     .addReg(Success).addReg(Mips::ZERO).addMBB(loopMBB);
1673 
1674   //  sinkMBB:
1675   //    and     maskedoldval1,oldval,mask
1676   //    srl     srlres,maskedoldval1,shiftamt
1677   //    sign_extend dest,srlres
1678   BB = sinkMBB;
1679 
1680   BuildMI(BB, DL, TII->get(Mips::AND), MaskedOldVal1)
1681     .addReg(OldVal).addReg(Mask);
1682   BuildMI(BB, DL, TII->get(Mips::SRLV), SrlRes)
1683       .addReg(MaskedOldVal1).addReg(ShiftAmt);
1684   BB = emitSignExtendToI32InReg(MI, BB, Size, Dest, SrlRes);
1685 
1686   MI.eraseFromParent(); // The instruction is gone now.
1687 
1688   return exitMBB;
1689 }
1690 
1691 MachineBasicBlock *MipsTargetLowering::emitAtomicCmpSwap(MachineInstr &MI,
1692                                                          MachineBasicBlock *BB,
1693                                                          unsigned Size) const {
1694   assert((Size == 4 || Size == 8) && "Unsupported size for EmitAtomicCmpSwap.");
1695 
1696   MachineFunction *MF = BB->getParent();
1697   MachineRegisterInfo &RegInfo = MF->getRegInfo();
1698   const TargetRegisterClass *RC = getRegClassFor(MVT::getIntegerVT(Size * 8));
1699   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1700   const bool ArePtrs64bit = ABI.ArePtrs64bit();
1701   DebugLoc DL = MI.getDebugLoc();
1702   unsigned LL, SC, ZERO, BNE, BEQ;
1703 
1704   if (Size == 4) {
1705     if (isMicroMips) {
1706       LL = Subtarget.hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
1707       SC = Subtarget.hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
1708     } else {
1709       LL = Subtarget.hasMips32r6()
1710                ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
1711                : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
1712       SC = Subtarget.hasMips32r6()
1713                ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
1714                : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
1715     }
1716 
1717     ZERO = Mips::ZERO;
1718     BNE = Mips::BNE;
1719     BEQ = Mips::BEQ;
1720   } else {
1721     LL = Subtarget.hasMips64r6() ? Mips::LLD_R6 : Mips::LLD;
1722     SC = Subtarget.hasMips64r6() ? Mips::SCD_R6 : Mips::SCD;
1723     ZERO = Mips::ZERO_64;
1724     BNE = Mips::BNE64;
1725     BEQ = Mips::BEQ64;
1726   }
1727 
1728   unsigned Dest = MI.getOperand(0).getReg();
1729   unsigned Ptr = MI.getOperand(1).getReg();
1730   unsigned OldVal = MI.getOperand(2).getReg();
1731   unsigned NewVal = MI.getOperand(3).getReg();
1732 
1733   unsigned Success = RegInfo.createVirtualRegister(RC);
1734 
1735   // insert new blocks after the current block
1736   const BasicBlock *LLVM_BB = BB->getBasicBlock();
1737   MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB);
1738   MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB);
1739   MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
1740   MachineFunction::iterator It = ++BB->getIterator();
1741   MF->insert(It, loop1MBB);
1742   MF->insert(It, loop2MBB);
1743   MF->insert(It, exitMBB);
1744 
1745   // Transfer the remainder of BB and its successor edges to exitMBB.
1746   exitMBB->splice(exitMBB->begin(), BB,
1747                   std::next(MachineBasicBlock::iterator(MI)), BB->end());
1748   exitMBB->transferSuccessorsAndUpdatePHIs(BB);
1749 
1750   //  thisMBB:
1751   //    ...
1752   //    fallthrough --> loop1MBB
1753   BB->addSuccessor(loop1MBB);
1754   loop1MBB->addSuccessor(exitMBB);
1755   loop1MBB->addSuccessor(loop2MBB);
1756   loop2MBB->addSuccessor(loop1MBB);
1757   loop2MBB->addSuccessor(exitMBB);
1758 
1759   // loop1MBB:
1760   //   ll dest, 0(ptr)
1761   //   bne dest, oldval, exitMBB
1762   BB = loop1MBB;
1763   BuildMI(BB, DL, TII->get(LL), Dest).addReg(Ptr).addImm(0);
1764   BuildMI(BB, DL, TII->get(BNE))
1765     .addReg(Dest).addReg(OldVal).addMBB(exitMBB);
1766 
1767   // loop2MBB:
1768   //   sc success, newval, 0(ptr)
1769   //   beq success, $0, loop1MBB
1770   BB = loop2MBB;
1771   BuildMI(BB, DL, TII->get(SC), Success)
1772     .addReg(NewVal).addReg(Ptr).addImm(0);
1773   BuildMI(BB, DL, TII->get(BEQ))
1774     .addReg(Success).addReg(ZERO).addMBB(loop1MBB);
1775 
1776   MI.eraseFromParent(); // The instruction is gone now.
1777 
1778   return exitMBB;
1779 }
1780 
1781 MachineBasicBlock *MipsTargetLowering::emitAtomicCmpSwapPartword(
1782     MachineInstr &MI, MachineBasicBlock *BB, unsigned Size) const {
1783   assert((Size == 1 || Size == 2) &&
1784       "Unsupported size for EmitAtomicCmpSwapPartial.");
1785 
1786   MachineFunction *MF = BB->getParent();
1787   MachineRegisterInfo &RegInfo = MF->getRegInfo();
1788   const TargetRegisterClass *RC = getRegClassFor(MVT::i32);
1789   const bool ArePtrs64bit = ABI.ArePtrs64bit();
1790   const TargetRegisterClass *RCp =
1791     getRegClassFor(ArePtrs64bit ? MVT::i64 : MVT::i32);
1792   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1793   DebugLoc DL = MI.getDebugLoc();
1794 
1795   unsigned Dest = MI.getOperand(0).getReg();
1796   unsigned Ptr = MI.getOperand(1).getReg();
1797   unsigned CmpVal = MI.getOperand(2).getReg();
1798   unsigned NewVal = MI.getOperand(3).getReg();
1799 
1800   unsigned AlignedAddr = RegInfo.createVirtualRegister(RCp);
1801   unsigned ShiftAmt = RegInfo.createVirtualRegister(RC);
1802   unsigned Mask = RegInfo.createVirtualRegister(RC);
1803   unsigned Mask2 = RegInfo.createVirtualRegister(RC);
1804   unsigned ShiftedCmpVal = RegInfo.createVirtualRegister(RC);
1805   unsigned OldVal = RegInfo.createVirtualRegister(RC);
1806   unsigned MaskedOldVal0 = RegInfo.createVirtualRegister(RC);
1807   unsigned ShiftedNewVal = RegInfo.createVirtualRegister(RC);
1808   unsigned MaskLSB2 = RegInfo.createVirtualRegister(RCp);
1809   unsigned PtrLSB2 = RegInfo.createVirtualRegister(RC);
1810   unsigned MaskUpper = RegInfo.createVirtualRegister(RC);
1811   unsigned MaskedCmpVal = RegInfo.createVirtualRegister(RC);
1812   unsigned MaskedNewVal = RegInfo.createVirtualRegister(RC);
1813   unsigned MaskedOldVal1 = RegInfo.createVirtualRegister(RC);
1814   unsigned StoreVal = RegInfo.createVirtualRegister(RC);
1815   unsigned SrlRes = RegInfo.createVirtualRegister(RC);
1816   unsigned Success = RegInfo.createVirtualRegister(RC);
1817   unsigned LL, SC;
1818 
1819   if (isMicroMips) {
1820     LL = Subtarget.hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
1821     SC = Subtarget.hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
1822   } else {
1823     LL = Subtarget.hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
1824                                  : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
1825     SC = Subtarget.hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
1826                                  : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
1827   }
1828 
1829   // insert new blocks after the current block
1830   const BasicBlock *LLVM_BB = BB->getBasicBlock();
1831   MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB);
1832   MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB);
1833   MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB);
1834   MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
1835   MachineFunction::iterator It = ++BB->getIterator();
1836   MF->insert(It, loop1MBB);
1837   MF->insert(It, loop2MBB);
1838   MF->insert(It, sinkMBB);
1839   MF->insert(It, exitMBB);
1840 
1841   // Transfer the remainder of BB and its successor edges to exitMBB.
1842   exitMBB->splice(exitMBB->begin(), BB,
1843                   std::next(MachineBasicBlock::iterator(MI)), BB->end());
1844   exitMBB->transferSuccessorsAndUpdatePHIs(BB);
1845 
1846   BB->addSuccessor(loop1MBB);
1847   loop1MBB->addSuccessor(sinkMBB);
1848   loop1MBB->addSuccessor(loop2MBB);
1849   loop2MBB->addSuccessor(loop1MBB);
1850   loop2MBB->addSuccessor(sinkMBB);
1851   sinkMBB->addSuccessor(exitMBB);
1852 
1853   // FIXME: computation of newval2 can be moved to loop2MBB.
1854   //  thisMBB:
1855   //    addiu   masklsb2,$0,-4                # 0xfffffffc
1856   //    and     alignedaddr,ptr,masklsb2
1857   //    andi    ptrlsb2,ptr,3
1858   //    xori    ptrlsb2,ptrlsb2,3              # Only for BE
1859   //    sll     shiftamt,ptrlsb2,3
1860   //    ori     maskupper,$0,255               # 0xff
1861   //    sll     mask,maskupper,shiftamt
1862   //    nor     mask2,$0,mask
1863   //    andi    maskedcmpval,cmpval,255
1864   //    sll     shiftedcmpval,maskedcmpval,shiftamt
1865   //    andi    maskednewval,newval,255
1866   //    sll     shiftednewval,maskednewval,shiftamt
1867   int64_t MaskImm = (Size == 1) ? 255 : 65535;
1868   BuildMI(BB, DL, TII->get(ArePtrs64bit ? Mips::DADDiu : Mips::ADDiu), MaskLSB2)
1869     .addReg(ABI.GetNullPtr()).addImm(-4);
1870   BuildMI(BB, DL, TII->get(ArePtrs64bit ? Mips::AND64 : Mips::AND), AlignedAddr)
1871     .addReg(Ptr).addReg(MaskLSB2);
1872   BuildMI(BB, DL, TII->get(Mips::ANDi), PtrLSB2)
1873       .addReg(Ptr, 0, ArePtrs64bit ? Mips::sub_32 : 0).addImm(3);
1874   if (Subtarget.isLittle()) {
1875     BuildMI(BB, DL, TII->get(Mips::SLL), ShiftAmt).addReg(PtrLSB2).addImm(3);
1876   } else {
1877     unsigned Off = RegInfo.createVirtualRegister(RC);
1878     BuildMI(BB, DL, TII->get(Mips::XORi), Off)
1879       .addReg(PtrLSB2).addImm((Size == 1) ? 3 : 2);
1880     BuildMI(BB, DL, TII->get(Mips::SLL), ShiftAmt).addReg(Off).addImm(3);
1881   }
1882   BuildMI(BB, DL, TII->get(Mips::ORi), MaskUpper)
1883     .addReg(Mips::ZERO).addImm(MaskImm);
1884   BuildMI(BB, DL, TII->get(Mips::SLLV), Mask)
1885     .addReg(MaskUpper).addReg(ShiftAmt);
1886   BuildMI(BB, DL, TII->get(Mips::NOR), Mask2).addReg(Mips::ZERO).addReg(Mask);
1887   BuildMI(BB, DL, TII->get(Mips::ANDi), MaskedCmpVal)
1888     .addReg(CmpVal).addImm(MaskImm);
1889   BuildMI(BB, DL, TII->get(Mips::SLLV), ShiftedCmpVal)
1890     .addReg(MaskedCmpVal).addReg(ShiftAmt);
1891   BuildMI(BB, DL, TII->get(Mips::ANDi), MaskedNewVal)
1892     .addReg(NewVal).addImm(MaskImm);
1893   BuildMI(BB, DL, TII->get(Mips::SLLV), ShiftedNewVal)
1894     .addReg(MaskedNewVal).addReg(ShiftAmt);
1895 
1896   //  loop1MBB:
1897   //    ll      oldval,0(alginedaddr)
1898   //    and     maskedoldval0,oldval,mask
1899   //    bne     maskedoldval0,shiftedcmpval,sinkMBB
1900   BB = loop1MBB;
1901   BuildMI(BB, DL, TII->get(LL), OldVal).addReg(AlignedAddr).addImm(0);
1902   BuildMI(BB, DL, TII->get(Mips::AND), MaskedOldVal0)
1903     .addReg(OldVal).addReg(Mask);
1904   BuildMI(BB, DL, TII->get(Mips::BNE))
1905     .addReg(MaskedOldVal0).addReg(ShiftedCmpVal).addMBB(sinkMBB);
1906 
1907   //  loop2MBB:
1908   //    and     maskedoldval1,oldval,mask2
1909   //    or      storeval,maskedoldval1,shiftednewval
1910   //    sc      success,storeval,0(alignedaddr)
1911   //    beq     success,$0,loop1MBB
1912   BB = loop2MBB;
1913   BuildMI(BB, DL, TII->get(Mips::AND), MaskedOldVal1)
1914     .addReg(OldVal).addReg(Mask2);
1915   BuildMI(BB, DL, TII->get(Mips::OR), StoreVal)
1916     .addReg(MaskedOldVal1).addReg(ShiftedNewVal);
1917   BuildMI(BB, DL, TII->get(SC), Success)
1918       .addReg(StoreVal).addReg(AlignedAddr).addImm(0);
1919   BuildMI(BB, DL, TII->get(Mips::BEQ))
1920       .addReg(Success).addReg(Mips::ZERO).addMBB(loop1MBB);
1921 
1922   //  sinkMBB:
1923   //    srl     srlres,maskedoldval0,shiftamt
1924   //    sign_extend dest,srlres
1925   BB = sinkMBB;
1926 
1927   BuildMI(BB, DL, TII->get(Mips::SRLV), SrlRes)
1928       .addReg(MaskedOldVal0).addReg(ShiftAmt);
1929   BB = emitSignExtendToI32InReg(MI, BB, Size, Dest, SrlRes);
1930 
1931   MI.eraseFromParent(); // The instruction is gone now.
1932 
1933   return exitMBB;
1934 }
1935 
1936 SDValue MipsTargetLowering::lowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
1937   // The first operand is the chain, the second is the condition, the third is
1938   // the block to branch to if the condition is true.
1939   SDValue Chain = Op.getOperand(0);
1940   SDValue Dest = Op.getOperand(2);
1941   SDLoc DL(Op);
1942 
1943   assert(!Subtarget.hasMips32r6() && !Subtarget.hasMips64r6());
1944   SDValue CondRes = createFPCmp(DAG, Op.getOperand(1));
1945 
1946   // Return if flag is not set by a floating point comparison.
1947   if (CondRes.getOpcode() != MipsISD::FPCmp)
1948     return Op;
1949 
1950   SDValue CCNode  = CondRes.getOperand(2);
1951   Mips::CondCode CC =
1952     (Mips::CondCode)cast<ConstantSDNode>(CCNode)->getZExtValue();
1953   unsigned Opc = invertFPCondCodeUser(CC) ? Mips::BRANCH_F : Mips::BRANCH_T;
1954   SDValue BrCode = DAG.getConstant(Opc, DL, MVT::i32);
1955   SDValue FCC0 = DAG.getRegister(Mips::FCC0, MVT::i32);
1956   return DAG.getNode(MipsISD::FPBrcond, DL, Op.getValueType(), Chain, BrCode,
1957                      FCC0, Dest, CondRes);
1958 }
1959 
1960 SDValue MipsTargetLowering::
1961 lowerSELECT(SDValue Op, SelectionDAG &DAG) const
1962 {
1963   assert(!Subtarget.hasMips32r6() && !Subtarget.hasMips64r6());
1964   SDValue Cond = createFPCmp(DAG, Op.getOperand(0));
1965 
1966   // Return if flag is not set by a floating point comparison.
1967   if (Cond.getOpcode() != MipsISD::FPCmp)
1968     return Op;
1969 
1970   return createCMovFP(DAG, Cond, Op.getOperand(1), Op.getOperand(2),
1971                       SDLoc(Op));
1972 }
1973 
1974 SDValue MipsTargetLowering::lowerSETCC(SDValue Op, SelectionDAG &DAG) const {
1975   assert(!Subtarget.hasMips32r6() && !Subtarget.hasMips64r6());
1976   SDValue Cond = createFPCmp(DAG, Op);
1977 
1978   assert(Cond.getOpcode() == MipsISD::FPCmp &&
1979          "Floating point operand expected.");
1980 
1981   SDLoc DL(Op);
1982   SDValue True  = DAG.getConstant(1, DL, MVT::i32);
1983   SDValue False = DAG.getConstant(0, DL, MVT::i32);
1984 
1985   return createCMovFP(DAG, Cond, True, False, DL);
1986 }
1987 
1988 SDValue MipsTargetLowering::lowerGlobalAddress(SDValue Op,
1989                                                SelectionDAG &DAG) const {
1990   EVT Ty = Op.getValueType();
1991   GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
1992   const GlobalValue *GV = N->getGlobal();
1993 
1994   if (!isPositionIndependent()) {
1995     const MipsTargetObjectFile *TLOF =
1996         static_cast<const MipsTargetObjectFile *>(
1997             getTargetMachine().getObjFileLowering());
1998     const GlobalObject *GO = GV->getBaseObject();
1999     if (GO && TLOF->IsGlobalInSmallSection(GO, getTargetMachine()))
2000       // %gp_rel relocation
2001       return getAddrGPRel(N, SDLoc(N), Ty, DAG, ABI.IsN64());
2002 
2003                                  // %hi/%lo relocation
2004     return Subtarget.hasSym32() ? getAddrNonPIC(N, SDLoc(N), Ty, DAG)
2005                                  // %highest/%higher/%hi/%lo relocation
2006                                  : getAddrNonPICSym64(N, SDLoc(N), Ty, DAG);
2007   }
2008 
2009   // Every other architecture would use shouldAssumeDSOLocal in here, but
2010   // mips is special.
2011   // * In PIC code mips requires got loads even for local statics!
2012   // * To save on got entries, for local statics the got entry contains the
2013   //   page and an additional add instruction takes care of the low bits.
2014   // * It is legal to access a hidden symbol with a non hidden undefined,
2015   //   so one cannot guarantee that all access to a hidden symbol will know
2016   //   it is hidden.
2017   // * Mips linkers don't support creating a page and a full got entry for
2018   //   the same symbol.
2019   // * Given all that, we have to use a full got entry for hidden symbols :-(
2020   if (GV->hasLocalLinkage())
2021     return getAddrLocal(N, SDLoc(N), Ty, DAG, ABI.IsN32() || ABI.IsN64());
2022 
2023   if (LargeGOT)
2024     return getAddrGlobalLargeGOT(
2025         N, SDLoc(N), Ty, DAG, MipsII::MO_GOT_HI16, MipsII::MO_GOT_LO16,
2026         DAG.getEntryNode(),
2027         MachinePointerInfo::getGOT(DAG.getMachineFunction()));
2028 
2029   return getAddrGlobal(
2030       N, SDLoc(N), Ty, DAG,
2031       (ABI.IsN32() || ABI.IsN64()) ? MipsII::MO_GOT_DISP : MipsII::MO_GOT,
2032       DAG.getEntryNode(), MachinePointerInfo::getGOT(DAG.getMachineFunction()));
2033 }
2034 
2035 SDValue MipsTargetLowering::lowerBlockAddress(SDValue Op,
2036                                               SelectionDAG &DAG) const {
2037   BlockAddressSDNode *N = cast<BlockAddressSDNode>(Op);
2038   EVT Ty = Op.getValueType();
2039 
2040   if (!isPositionIndependent())
2041     return Subtarget.hasSym32() ? getAddrNonPIC(N, SDLoc(N), Ty, DAG)
2042                                 : getAddrNonPICSym64(N, SDLoc(N), Ty, DAG);
2043 
2044   return getAddrLocal(N, SDLoc(N), Ty, DAG, ABI.IsN32() || ABI.IsN64());
2045 }
2046 
2047 SDValue MipsTargetLowering::
2048 lowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const
2049 {
2050   // If the relocation model is PIC, use the General Dynamic TLS Model or
2051   // Local Dynamic TLS model, otherwise use the Initial Exec or
2052   // Local Exec TLS Model.
2053 
2054   GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
2055   if (DAG.getTarget().useEmulatedTLS())
2056     return LowerToTLSEmulatedModel(GA, DAG);
2057 
2058   SDLoc DL(GA);
2059   const GlobalValue *GV = GA->getGlobal();
2060   EVT PtrVT = getPointerTy(DAG.getDataLayout());
2061 
2062   TLSModel::Model model = getTargetMachine().getTLSModel(GV);
2063 
2064   if (model == TLSModel::GeneralDynamic || model == TLSModel::LocalDynamic) {
2065     // General Dynamic and Local Dynamic TLS Model.
2066     unsigned Flag = (model == TLSModel::LocalDynamic) ? MipsII::MO_TLSLDM
2067                                                       : MipsII::MO_TLSGD;
2068 
2069     SDValue TGA = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, Flag);
2070     SDValue Argument = DAG.getNode(MipsISD::Wrapper, DL, PtrVT,
2071                                    getGlobalReg(DAG, PtrVT), TGA);
2072     unsigned PtrSize = PtrVT.getSizeInBits();
2073     IntegerType *PtrTy = Type::getIntNTy(*DAG.getContext(), PtrSize);
2074 
2075     SDValue TlsGetAddr = DAG.getExternalSymbol("__tls_get_addr", PtrVT);
2076 
2077     ArgListTy Args;
2078     ArgListEntry Entry;
2079     Entry.Node = Argument;
2080     Entry.Ty = PtrTy;
2081     Args.push_back(Entry);
2082 
2083     TargetLowering::CallLoweringInfo CLI(DAG);
2084     CLI.setDebugLoc(DL)
2085         .setChain(DAG.getEntryNode())
2086         .setLibCallee(CallingConv::C, PtrTy, TlsGetAddr, std::move(Args));
2087     std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
2088 
2089     SDValue Ret = CallResult.first;
2090 
2091     if (model != TLSModel::LocalDynamic)
2092       return Ret;
2093 
2094     SDValue TGAHi = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0,
2095                                                MipsII::MO_DTPREL_HI);
2096     SDValue Hi = DAG.getNode(MipsISD::Hi, DL, PtrVT, TGAHi);
2097     SDValue TGALo = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0,
2098                                                MipsII::MO_DTPREL_LO);
2099     SDValue Lo = DAG.getNode(MipsISD::Lo, DL, PtrVT, TGALo);
2100     SDValue Add = DAG.getNode(ISD::ADD, DL, PtrVT, Hi, Ret);
2101     return DAG.getNode(ISD::ADD, DL, PtrVT, Add, Lo);
2102   }
2103 
2104   SDValue Offset;
2105   if (model == TLSModel::InitialExec) {
2106     // Initial Exec TLS Model
2107     SDValue TGA = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0,
2108                                              MipsII::MO_GOTTPREL);
2109     TGA = DAG.getNode(MipsISD::Wrapper, DL, PtrVT, getGlobalReg(DAG, PtrVT),
2110                       TGA);
2111     Offset =
2112         DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), TGA, MachinePointerInfo());
2113   } else {
2114     // Local Exec TLS Model
2115     assert(model == TLSModel::LocalExec);
2116     SDValue TGAHi = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0,
2117                                                MipsII::MO_TPREL_HI);
2118     SDValue TGALo = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0,
2119                                                MipsII::MO_TPREL_LO);
2120     SDValue Hi = DAG.getNode(MipsISD::Hi, DL, PtrVT, TGAHi);
2121     SDValue Lo = DAG.getNode(MipsISD::Lo, DL, PtrVT, TGALo);
2122     Offset = DAG.getNode(ISD::ADD, DL, PtrVT, Hi, Lo);
2123   }
2124 
2125   SDValue ThreadPointer = DAG.getNode(MipsISD::ThreadPointer, DL, PtrVT);
2126   return DAG.getNode(ISD::ADD, DL, PtrVT, ThreadPointer, Offset);
2127 }
2128 
2129 SDValue MipsTargetLowering::
2130 lowerJumpTable(SDValue Op, SelectionDAG &DAG) const
2131 {
2132   JumpTableSDNode *N = cast<JumpTableSDNode>(Op);
2133   EVT Ty = Op.getValueType();
2134 
2135   if (!isPositionIndependent())
2136     return Subtarget.hasSym32() ? getAddrNonPIC(N, SDLoc(N), Ty, DAG)
2137                                 : getAddrNonPICSym64(N, SDLoc(N), Ty, DAG);
2138 
2139   return getAddrLocal(N, SDLoc(N), Ty, DAG, ABI.IsN32() || ABI.IsN64());
2140 }
2141 
2142 SDValue MipsTargetLowering::
2143 lowerConstantPool(SDValue Op, SelectionDAG &DAG) const
2144 {
2145   ConstantPoolSDNode *N = cast<ConstantPoolSDNode>(Op);
2146   EVT Ty = Op.getValueType();
2147 
2148   if (!isPositionIndependent()) {
2149     const MipsTargetObjectFile *TLOF =
2150         static_cast<const MipsTargetObjectFile *>(
2151             getTargetMachine().getObjFileLowering());
2152 
2153     if (TLOF->IsConstantInSmallSection(DAG.getDataLayout(), N->getConstVal(),
2154                                        getTargetMachine()))
2155       // %gp_rel relocation
2156       return getAddrGPRel(N, SDLoc(N), Ty, DAG, ABI.IsN64());
2157 
2158     return Subtarget.hasSym32() ? getAddrNonPIC(N, SDLoc(N), Ty, DAG)
2159                                 : getAddrNonPICSym64(N, SDLoc(N), Ty, DAG);
2160   }
2161 
2162  return getAddrLocal(N, SDLoc(N), Ty, DAG, ABI.IsN32() || ABI.IsN64());
2163 }
2164 
2165 SDValue MipsTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const {
2166   MachineFunction &MF = DAG.getMachineFunction();
2167   MipsFunctionInfo *FuncInfo = MF.getInfo<MipsFunctionInfo>();
2168 
2169   SDLoc DL(Op);
2170   SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
2171                                  getPointerTy(MF.getDataLayout()));
2172 
2173   // vastart just stores the address of the VarArgsFrameIndex slot into the
2174   // memory location argument.
2175   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
2176   return DAG.getStore(Op.getOperand(0), DL, FI, Op.getOperand(1),
2177                       MachinePointerInfo(SV));
2178 }
2179 
2180 SDValue MipsTargetLowering::lowerVAARG(SDValue Op, SelectionDAG &DAG) const {
2181   SDNode *Node = Op.getNode();
2182   EVT VT = Node->getValueType(0);
2183   SDValue Chain = Node->getOperand(0);
2184   SDValue VAListPtr = Node->getOperand(1);
2185   unsigned Align = Node->getConstantOperandVal(3);
2186   const Value *SV = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
2187   SDLoc DL(Node);
2188   unsigned ArgSlotSizeInBytes = (ABI.IsN32() || ABI.IsN64()) ? 8 : 4;
2189 
2190   SDValue VAListLoad = DAG.getLoad(getPointerTy(DAG.getDataLayout()), DL, Chain,
2191                                    VAListPtr, MachinePointerInfo(SV));
2192   SDValue VAList = VAListLoad;
2193 
2194   // Re-align the pointer if necessary.
2195   // It should only ever be necessary for 64-bit types on O32 since the minimum
2196   // argument alignment is the same as the maximum type alignment for N32/N64.
2197   //
2198   // FIXME: We currently align too often. The code generator doesn't notice
2199   //        when the pointer is still aligned from the last va_arg (or pair of
2200   //        va_args for the i64 on O32 case).
2201   if (Align > getMinStackArgumentAlignment()) {
2202     assert(((Align & (Align-1)) == 0) && "Expected Align to be a power of 2");
2203 
2204     VAList = DAG.getNode(ISD::ADD, DL, VAList.getValueType(), VAList,
2205                          DAG.getConstant(Align - 1, DL, VAList.getValueType()));
2206 
2207     VAList = DAG.getNode(ISD::AND, DL, VAList.getValueType(), VAList,
2208                          DAG.getConstant(-(int64_t)Align, DL,
2209                                          VAList.getValueType()));
2210   }
2211 
2212   // Increment the pointer, VAList, to the next vaarg.
2213   auto &TD = DAG.getDataLayout();
2214   unsigned ArgSizeInBytes =
2215       TD.getTypeAllocSize(VT.getTypeForEVT(*DAG.getContext()));
2216   SDValue Tmp3 =
2217       DAG.getNode(ISD::ADD, DL, VAList.getValueType(), VAList,
2218                   DAG.getConstant(alignTo(ArgSizeInBytes, ArgSlotSizeInBytes),
2219                                   DL, VAList.getValueType()));
2220   // Store the incremented VAList to the legalized pointer
2221   Chain = DAG.getStore(VAListLoad.getValue(1), DL, Tmp3, VAListPtr,
2222                        MachinePointerInfo(SV));
2223 
2224   // In big-endian mode we must adjust the pointer when the load size is smaller
2225   // than the argument slot size. We must also reduce the known alignment to
2226   // match. For example in the N64 ABI, we must add 4 bytes to the offset to get
2227   // the correct half of the slot, and reduce the alignment from 8 (slot
2228   // alignment) down to 4 (type alignment).
2229   if (!Subtarget.isLittle() && ArgSizeInBytes < ArgSlotSizeInBytes) {
2230     unsigned Adjustment = ArgSlotSizeInBytes - ArgSizeInBytes;
2231     VAList = DAG.getNode(ISD::ADD, DL, VAListPtr.getValueType(), VAList,
2232                          DAG.getIntPtrConstant(Adjustment, DL));
2233   }
2234   // Load the actual argument out of the pointer VAList
2235   return DAG.getLoad(VT, DL, Chain, VAList, MachinePointerInfo());
2236 }
2237 
2238 static SDValue lowerFCOPYSIGN32(SDValue Op, SelectionDAG &DAG,
2239                                 bool HasExtractInsert) {
2240   EVT TyX = Op.getOperand(0).getValueType();
2241   EVT TyY = Op.getOperand(1).getValueType();
2242   SDLoc DL(Op);
2243   SDValue Const1 = DAG.getConstant(1, DL, MVT::i32);
2244   SDValue Const31 = DAG.getConstant(31, DL, MVT::i32);
2245   SDValue Res;
2246 
2247   // If operand is of type f64, extract the upper 32-bit. Otherwise, bitcast it
2248   // to i32.
2249   SDValue X = (TyX == MVT::f32) ?
2250     DAG.getNode(ISD::BITCAST, DL, MVT::i32, Op.getOperand(0)) :
2251     DAG.getNode(MipsISD::ExtractElementF64, DL, MVT::i32, Op.getOperand(0),
2252                 Const1);
2253   SDValue Y = (TyY == MVT::f32) ?
2254     DAG.getNode(ISD::BITCAST, DL, MVT::i32, Op.getOperand(1)) :
2255     DAG.getNode(MipsISD::ExtractElementF64, DL, MVT::i32, Op.getOperand(1),
2256                 Const1);
2257 
2258   if (HasExtractInsert) {
2259     // ext  E, Y, 31, 1  ; extract bit31 of Y
2260     // ins  X, E, 31, 1  ; insert extracted bit at bit31 of X
2261     SDValue E = DAG.getNode(MipsISD::Ext, DL, MVT::i32, Y, Const31, Const1);
2262     Res = DAG.getNode(MipsISD::Ins, DL, MVT::i32, E, Const31, Const1, X);
2263   } else {
2264     // sll SllX, X, 1
2265     // srl SrlX, SllX, 1
2266     // srl SrlY, Y, 31
2267     // sll SllY, SrlX, 31
2268     // or  Or, SrlX, SllY
2269     SDValue SllX = DAG.getNode(ISD::SHL, DL, MVT::i32, X, Const1);
2270     SDValue SrlX = DAG.getNode(ISD::SRL, DL, MVT::i32, SllX, Const1);
2271     SDValue SrlY = DAG.getNode(ISD::SRL, DL, MVT::i32, Y, Const31);
2272     SDValue SllY = DAG.getNode(ISD::SHL, DL, MVT::i32, SrlY, Const31);
2273     Res = DAG.getNode(ISD::OR, DL, MVT::i32, SrlX, SllY);
2274   }
2275 
2276   if (TyX == MVT::f32)
2277     return DAG.getNode(ISD::BITCAST, DL, Op.getOperand(0).getValueType(), Res);
2278 
2279   SDValue LowX = DAG.getNode(MipsISD::ExtractElementF64, DL, MVT::i32,
2280                              Op.getOperand(0),
2281                              DAG.getConstant(0, DL, MVT::i32));
2282   return DAG.getNode(MipsISD::BuildPairF64, DL, MVT::f64, LowX, Res);
2283 }
2284 
2285 static SDValue lowerFCOPYSIGN64(SDValue Op, SelectionDAG &DAG,
2286                                 bool HasExtractInsert) {
2287   unsigned WidthX = Op.getOperand(0).getValueSizeInBits();
2288   unsigned WidthY = Op.getOperand(1).getValueSizeInBits();
2289   EVT TyX = MVT::getIntegerVT(WidthX), TyY = MVT::getIntegerVT(WidthY);
2290   SDLoc DL(Op);
2291   SDValue Const1 = DAG.getConstant(1, DL, MVT::i32);
2292 
2293   // Bitcast to integer nodes.
2294   SDValue X = DAG.getNode(ISD::BITCAST, DL, TyX, Op.getOperand(0));
2295   SDValue Y = DAG.getNode(ISD::BITCAST, DL, TyY, Op.getOperand(1));
2296 
2297   if (HasExtractInsert) {
2298     // ext  E, Y, width(Y) - 1, 1  ; extract bit width(Y)-1 of Y
2299     // ins  X, E, width(X) - 1, 1  ; insert extracted bit at bit width(X)-1 of X
2300     SDValue E = DAG.getNode(MipsISD::Ext, DL, TyY, Y,
2301                             DAG.getConstant(WidthY - 1, DL, MVT::i32), Const1);
2302 
2303     if (WidthX > WidthY)
2304       E = DAG.getNode(ISD::ZERO_EXTEND, DL, TyX, E);
2305     else if (WidthY > WidthX)
2306       E = DAG.getNode(ISD::TRUNCATE, DL, TyX, E);
2307 
2308     SDValue I = DAG.getNode(MipsISD::Ins, DL, TyX, E,
2309                             DAG.getConstant(WidthX - 1, DL, MVT::i32), Const1,
2310                             X);
2311     return DAG.getNode(ISD::BITCAST, DL, Op.getOperand(0).getValueType(), I);
2312   }
2313 
2314   // (d)sll SllX, X, 1
2315   // (d)srl SrlX, SllX, 1
2316   // (d)srl SrlY, Y, width(Y)-1
2317   // (d)sll SllY, SrlX, width(Y)-1
2318   // or     Or, SrlX, SllY
2319   SDValue SllX = DAG.getNode(ISD::SHL, DL, TyX, X, Const1);
2320   SDValue SrlX = DAG.getNode(ISD::SRL, DL, TyX, SllX, Const1);
2321   SDValue SrlY = DAG.getNode(ISD::SRL, DL, TyY, Y,
2322                              DAG.getConstant(WidthY - 1, DL, MVT::i32));
2323 
2324   if (WidthX > WidthY)
2325     SrlY = DAG.getNode(ISD::ZERO_EXTEND, DL, TyX, SrlY);
2326   else if (WidthY > WidthX)
2327     SrlY = DAG.getNode(ISD::TRUNCATE, DL, TyX, SrlY);
2328 
2329   SDValue SllY = DAG.getNode(ISD::SHL, DL, TyX, SrlY,
2330                              DAG.getConstant(WidthX - 1, DL, MVT::i32));
2331   SDValue Or = DAG.getNode(ISD::OR, DL, TyX, SrlX, SllY);
2332   return DAG.getNode(ISD::BITCAST, DL, Op.getOperand(0).getValueType(), Or);
2333 }
2334 
2335 SDValue
2336 MipsTargetLowering::lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
2337   if (Subtarget.isGP64bit())
2338     return lowerFCOPYSIGN64(Op, DAG, Subtarget.hasExtractInsert());
2339 
2340   return lowerFCOPYSIGN32(Op, DAG, Subtarget.hasExtractInsert());
2341 }
2342 
2343 SDValue MipsTargetLowering::
2344 lowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
2345   // check the depth
2346   assert((cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue() == 0) &&
2347          "Frame address can only be determined for current frame.");
2348 
2349   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
2350   MFI.setFrameAddressIsTaken(true);
2351   EVT VT = Op.getValueType();
2352   SDLoc DL(Op);
2353   SDValue FrameAddr = DAG.getCopyFromReg(
2354       DAG.getEntryNode(), DL, ABI.IsN64() ? Mips::FP_64 : Mips::FP, VT);
2355   return FrameAddr;
2356 }
2357 
2358 SDValue MipsTargetLowering::lowerRETURNADDR(SDValue Op,
2359                                             SelectionDAG &DAG) const {
2360   if (verifyReturnAddressArgumentIsConstant(Op, DAG))
2361     return SDValue();
2362 
2363   // check the depth
2364   assert((cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue() == 0) &&
2365          "Return address can be determined only for current frame.");
2366 
2367   MachineFunction &MF = DAG.getMachineFunction();
2368   MachineFrameInfo &MFI = MF.getFrameInfo();
2369   MVT VT = Op.getSimpleValueType();
2370   unsigned RA = ABI.IsN64() ? Mips::RA_64 : Mips::RA;
2371   MFI.setReturnAddressIsTaken(true);
2372 
2373   // Return RA, which contains the return address. Mark it an implicit live-in.
2374   unsigned Reg = MF.addLiveIn(RA, getRegClassFor(VT));
2375   return DAG.getCopyFromReg(DAG.getEntryNode(), SDLoc(Op), Reg, VT);
2376 }
2377 
2378 // An EH_RETURN is the result of lowering llvm.eh.return which in turn is
2379 // generated from __builtin_eh_return (offset, handler)
2380 // The effect of this is to adjust the stack pointer by "offset"
2381 // and then branch to "handler".
2382 SDValue MipsTargetLowering::lowerEH_RETURN(SDValue Op, SelectionDAG &DAG)
2383                                                                      const {
2384   MachineFunction &MF = DAG.getMachineFunction();
2385   MipsFunctionInfo *MipsFI = MF.getInfo<MipsFunctionInfo>();
2386 
2387   MipsFI->setCallsEhReturn();
2388   SDValue Chain     = Op.getOperand(0);
2389   SDValue Offset    = Op.getOperand(1);
2390   SDValue Handler   = Op.getOperand(2);
2391   SDLoc DL(Op);
2392   EVT Ty = ABI.IsN64() ? MVT::i64 : MVT::i32;
2393 
2394   // Store stack offset in V1, store jump target in V0. Glue CopyToReg and
2395   // EH_RETURN nodes, so that instructions are emitted back-to-back.
2396   unsigned OffsetReg = ABI.IsN64() ? Mips::V1_64 : Mips::V1;
2397   unsigned AddrReg = ABI.IsN64() ? Mips::V0_64 : Mips::V0;
2398   Chain = DAG.getCopyToReg(Chain, DL, OffsetReg, Offset, SDValue());
2399   Chain = DAG.getCopyToReg(Chain, DL, AddrReg, Handler, Chain.getValue(1));
2400   return DAG.getNode(MipsISD::EH_RETURN, DL, MVT::Other, Chain,
2401                      DAG.getRegister(OffsetReg, Ty),
2402                      DAG.getRegister(AddrReg, getPointerTy(MF.getDataLayout())),
2403                      Chain.getValue(1));
2404 }
2405 
2406 SDValue MipsTargetLowering::lowerATOMIC_FENCE(SDValue Op,
2407                                               SelectionDAG &DAG) const {
2408   // FIXME: Need pseudo-fence for 'singlethread' fences
2409   // FIXME: Set SType for weaker fences where supported/appropriate.
2410   unsigned SType = 0;
2411   SDLoc DL(Op);
2412   return DAG.getNode(MipsISD::Sync, DL, MVT::Other, Op.getOperand(0),
2413                      DAG.getConstant(SType, DL, MVT::i32));
2414 }
2415 
2416 SDValue MipsTargetLowering::lowerShiftLeftParts(SDValue Op,
2417                                                 SelectionDAG &DAG) const {
2418   SDLoc DL(Op);
2419   MVT VT = Subtarget.isGP64bit() ? MVT::i64 : MVT::i32;
2420 
2421   SDValue Lo = Op.getOperand(0), Hi = Op.getOperand(1);
2422   SDValue Shamt = Op.getOperand(2);
2423   // if shamt < (VT.bits):
2424   //  lo = (shl lo, shamt)
2425   //  hi = (or (shl hi, shamt) (srl (srl lo, 1), ~shamt))
2426   // else:
2427   //  lo = 0
2428   //  hi = (shl lo, shamt[4:0])
2429   SDValue Not = DAG.getNode(ISD::XOR, DL, MVT::i32, Shamt,
2430                             DAG.getConstant(-1, DL, MVT::i32));
2431   SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo,
2432                                       DAG.getConstant(1, DL, VT));
2433   SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, Not);
2434   SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt);
2435   SDValue Or = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
2436   SDValue ShiftLeftLo = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt);
2437   SDValue Cond = DAG.getNode(ISD::AND, DL, MVT::i32, Shamt,
2438                              DAG.getConstant(VT.getSizeInBits(), DL, MVT::i32));
2439   Lo = DAG.getNode(ISD::SELECT, DL, VT, Cond,
2440                    DAG.getConstant(0, DL, VT), ShiftLeftLo);
2441   Hi = DAG.getNode(ISD::SELECT, DL, VT, Cond, ShiftLeftLo, Or);
2442 
2443   SDValue Ops[2] = {Lo, Hi};
2444   return DAG.getMergeValues(Ops, DL);
2445 }
2446 
2447 SDValue MipsTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
2448                                                  bool IsSRA) const {
2449   SDLoc DL(Op);
2450   SDValue Lo = Op.getOperand(0), Hi = Op.getOperand(1);
2451   SDValue Shamt = Op.getOperand(2);
2452   MVT VT = Subtarget.isGP64bit() ? MVT::i64 : MVT::i32;
2453 
2454   // if shamt < (VT.bits):
2455   //  lo = (or (shl (shl hi, 1), ~shamt) (srl lo, shamt))
2456   //  if isSRA:
2457   //    hi = (sra hi, shamt)
2458   //  else:
2459   //    hi = (srl hi, shamt)
2460   // else:
2461   //  if isSRA:
2462   //   lo = (sra hi, shamt[4:0])
2463   //   hi = (sra hi, 31)
2464   //  else:
2465   //   lo = (srl hi, shamt[4:0])
2466   //   hi = 0
2467   SDValue Not = DAG.getNode(ISD::XOR, DL, MVT::i32, Shamt,
2468                             DAG.getConstant(-1, DL, MVT::i32));
2469   SDValue ShiftLeft1Hi = DAG.getNode(ISD::SHL, DL, VT, Hi,
2470                                      DAG.getConstant(1, DL, VT));
2471   SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, ShiftLeft1Hi, Not);
2472   SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt);
2473   SDValue Or = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
2474   SDValue ShiftRightHi = DAG.getNode(IsSRA ? ISD::SRA : ISD::SRL,
2475                                      DL, VT, Hi, Shamt);
2476   SDValue Cond = DAG.getNode(ISD::AND, DL, MVT::i32, Shamt,
2477                              DAG.getConstant(VT.getSizeInBits(), DL, MVT::i32));
2478   SDValue Ext = DAG.getNode(ISD::SRA, DL, VT, Hi,
2479                             DAG.getConstant(VT.getSizeInBits() - 1, DL, VT));
2480   Lo = DAG.getNode(ISD::SELECT, DL, VT, Cond, ShiftRightHi, Or);
2481   Hi = DAG.getNode(ISD::SELECT, DL, VT, Cond,
2482                    IsSRA ? Ext : DAG.getConstant(0, DL, VT), ShiftRightHi);
2483 
2484   SDValue Ops[2] = {Lo, Hi};
2485   return DAG.getMergeValues(Ops, DL);
2486 }
2487 
2488 static SDValue createLoadLR(unsigned Opc, SelectionDAG &DAG, LoadSDNode *LD,
2489                             SDValue Chain, SDValue Src, unsigned Offset) {
2490   SDValue Ptr = LD->getBasePtr();
2491   EVT VT = LD->getValueType(0), MemVT = LD->getMemoryVT();
2492   EVT BasePtrVT = Ptr.getValueType();
2493   SDLoc DL(LD);
2494   SDVTList VTList = DAG.getVTList(VT, MVT::Other);
2495 
2496   if (Offset)
2497     Ptr = DAG.getNode(ISD::ADD, DL, BasePtrVT, Ptr,
2498                       DAG.getConstant(Offset, DL, BasePtrVT));
2499 
2500   SDValue Ops[] = { Chain, Ptr, Src };
2501   return DAG.getMemIntrinsicNode(Opc, DL, VTList, Ops, MemVT,
2502                                  LD->getMemOperand());
2503 }
2504 
2505 // Expand an unaligned 32 or 64-bit integer load node.
2506 SDValue MipsTargetLowering::lowerLOAD(SDValue Op, SelectionDAG &DAG) const {
2507   LoadSDNode *LD = cast<LoadSDNode>(Op);
2508   EVT MemVT = LD->getMemoryVT();
2509 
2510   if (Subtarget.systemSupportsUnalignedAccess())
2511     return Op;
2512 
2513   // Return if load is aligned or if MemVT is neither i32 nor i64.
2514   if ((LD->getAlignment() >= MemVT.getSizeInBits() / 8) ||
2515       ((MemVT != MVT::i32) && (MemVT != MVT::i64)))
2516     return SDValue();
2517 
2518   bool IsLittle = Subtarget.isLittle();
2519   EVT VT = Op.getValueType();
2520   ISD::LoadExtType ExtType = LD->getExtensionType();
2521   SDValue Chain = LD->getChain(), Undef = DAG.getUNDEF(VT);
2522 
2523   assert((VT == MVT::i32) || (VT == MVT::i64));
2524 
2525   // Expand
2526   //  (set dst, (i64 (load baseptr)))
2527   // to
2528   //  (set tmp, (ldl (add baseptr, 7), undef))
2529   //  (set dst, (ldr baseptr, tmp))
2530   if ((VT == MVT::i64) && (ExtType == ISD::NON_EXTLOAD)) {
2531     SDValue LDL = createLoadLR(MipsISD::LDL, DAG, LD, Chain, Undef,
2532                                IsLittle ? 7 : 0);
2533     return createLoadLR(MipsISD::LDR, DAG, LD, LDL.getValue(1), LDL,
2534                         IsLittle ? 0 : 7);
2535   }
2536 
2537   SDValue LWL = createLoadLR(MipsISD::LWL, DAG, LD, Chain, Undef,
2538                              IsLittle ? 3 : 0);
2539   SDValue LWR = createLoadLR(MipsISD::LWR, DAG, LD, LWL.getValue(1), LWL,
2540                              IsLittle ? 0 : 3);
2541 
2542   // Expand
2543   //  (set dst, (i32 (load baseptr))) or
2544   //  (set dst, (i64 (sextload baseptr))) or
2545   //  (set dst, (i64 (extload baseptr)))
2546   // to
2547   //  (set tmp, (lwl (add baseptr, 3), undef))
2548   //  (set dst, (lwr baseptr, tmp))
2549   if ((VT == MVT::i32) || (ExtType == ISD::SEXTLOAD) ||
2550       (ExtType == ISD::EXTLOAD))
2551     return LWR;
2552 
2553   assert((VT == MVT::i64) && (ExtType == ISD::ZEXTLOAD));
2554 
2555   // Expand
2556   //  (set dst, (i64 (zextload baseptr)))
2557   // to
2558   //  (set tmp0, (lwl (add baseptr, 3), undef))
2559   //  (set tmp1, (lwr baseptr, tmp0))
2560   //  (set tmp2, (shl tmp1, 32))
2561   //  (set dst, (srl tmp2, 32))
2562   SDLoc DL(LD);
2563   SDValue Const32 = DAG.getConstant(32, DL, MVT::i32);
2564   SDValue SLL = DAG.getNode(ISD::SHL, DL, MVT::i64, LWR, Const32);
2565   SDValue SRL = DAG.getNode(ISD::SRL, DL, MVT::i64, SLL, Const32);
2566   SDValue Ops[] = { SRL, LWR.getValue(1) };
2567   return DAG.getMergeValues(Ops, DL);
2568 }
2569 
2570 static SDValue createStoreLR(unsigned Opc, SelectionDAG &DAG, StoreSDNode *SD,
2571                              SDValue Chain, unsigned Offset) {
2572   SDValue Ptr = SD->getBasePtr(), Value = SD->getValue();
2573   EVT MemVT = SD->getMemoryVT(), BasePtrVT = Ptr.getValueType();
2574   SDLoc DL(SD);
2575   SDVTList VTList = DAG.getVTList(MVT::Other);
2576 
2577   if (Offset)
2578     Ptr = DAG.getNode(ISD::ADD, DL, BasePtrVT, Ptr,
2579                       DAG.getConstant(Offset, DL, BasePtrVT));
2580 
2581   SDValue Ops[] = { Chain, Value, Ptr };
2582   return DAG.getMemIntrinsicNode(Opc, DL, VTList, Ops, MemVT,
2583                                  SD->getMemOperand());
2584 }
2585 
2586 // Expand an unaligned 32 or 64-bit integer store node.
2587 static SDValue lowerUnalignedIntStore(StoreSDNode *SD, SelectionDAG &DAG,
2588                                       bool IsLittle) {
2589   SDValue Value = SD->getValue(), Chain = SD->getChain();
2590   EVT VT = Value.getValueType();
2591 
2592   // Expand
2593   //  (store val, baseptr) or
2594   //  (truncstore val, baseptr)
2595   // to
2596   //  (swl val, (add baseptr, 3))
2597   //  (swr val, baseptr)
2598   if ((VT == MVT::i32) || SD->isTruncatingStore()) {
2599     SDValue SWL = createStoreLR(MipsISD::SWL, DAG, SD, Chain,
2600                                 IsLittle ? 3 : 0);
2601     return createStoreLR(MipsISD::SWR, DAG, SD, SWL, IsLittle ? 0 : 3);
2602   }
2603 
2604   assert(VT == MVT::i64);
2605 
2606   // Expand
2607   //  (store val, baseptr)
2608   // to
2609   //  (sdl val, (add baseptr, 7))
2610   //  (sdr val, baseptr)
2611   SDValue SDL = createStoreLR(MipsISD::SDL, DAG, SD, Chain, IsLittle ? 7 : 0);
2612   return createStoreLR(MipsISD::SDR, DAG, SD, SDL, IsLittle ? 0 : 7);
2613 }
2614 
2615 // Lower (store (fp_to_sint $fp) $ptr) to (store (TruncIntFP $fp), $ptr).
2616 static SDValue lowerFP_TO_SINT_STORE(StoreSDNode *SD, SelectionDAG &DAG) {
2617   SDValue Val = SD->getValue();
2618 
2619   if (Val.getOpcode() != ISD::FP_TO_SINT)
2620     return SDValue();
2621 
2622   EVT FPTy = EVT::getFloatingPointVT(Val.getValueSizeInBits());
2623   SDValue Tr = DAG.getNode(MipsISD::TruncIntFP, SDLoc(Val), FPTy,
2624                            Val.getOperand(0));
2625   return DAG.getStore(SD->getChain(), SDLoc(SD), Tr, SD->getBasePtr(),
2626                       SD->getPointerInfo(), SD->getAlignment(),
2627                       SD->getMemOperand()->getFlags());
2628 }
2629 
2630 SDValue MipsTargetLowering::lowerSTORE(SDValue Op, SelectionDAG &DAG) const {
2631   StoreSDNode *SD = cast<StoreSDNode>(Op);
2632   EVT MemVT = SD->getMemoryVT();
2633 
2634   // Lower unaligned integer stores.
2635   if (!Subtarget.systemSupportsUnalignedAccess() &&
2636       (SD->getAlignment() < MemVT.getSizeInBits() / 8) &&
2637       ((MemVT == MVT::i32) || (MemVT == MVT::i64)))
2638     return lowerUnalignedIntStore(SD, DAG, Subtarget.isLittle());
2639 
2640   return lowerFP_TO_SINT_STORE(SD, DAG);
2641 }
2642 
2643 SDValue MipsTargetLowering::lowerEH_DWARF_CFA(SDValue Op,
2644                                               SelectionDAG &DAG) const {
2645 
2646   // Return a fixed StackObject with offset 0 which points to the old stack
2647   // pointer.
2648   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
2649   EVT ValTy = Op->getValueType(0);
2650   int FI = MFI.CreateFixedObject(Op.getValueSizeInBits() / 8, 0, false);
2651   return DAG.getFrameIndex(FI, ValTy);
2652 }
2653 
2654 SDValue MipsTargetLowering::lowerFP_TO_SINT(SDValue Op,
2655                                             SelectionDAG &DAG) const {
2656   EVT FPTy = EVT::getFloatingPointVT(Op.getValueSizeInBits());
2657   SDValue Trunc = DAG.getNode(MipsISD::TruncIntFP, SDLoc(Op), FPTy,
2658                               Op.getOperand(0));
2659   return DAG.getNode(ISD::BITCAST, SDLoc(Op), Op.getValueType(), Trunc);
2660 }
2661 
2662 //===----------------------------------------------------------------------===//
2663 //                      Calling Convention Implementation
2664 //===----------------------------------------------------------------------===//
2665 
2666 //===----------------------------------------------------------------------===//
2667 // TODO: Implement a generic logic using tblgen that can support this.
2668 // Mips O32 ABI rules:
2669 // ---
2670 // i32 - Passed in A0, A1, A2, A3 and stack
2671 // f32 - Only passed in f32 registers if no int reg has been used yet to hold
2672 //       an argument. Otherwise, passed in A1, A2, A3 and stack.
2673 // f64 - Only passed in two aliased f32 registers if no int reg has been used
2674 //       yet to hold an argument. Otherwise, use A2, A3 and stack. If A1 is
2675 //       not used, it must be shadowed. If only A3 is available, shadow it and
2676 //       go to stack.
2677 // vXiX - Received as scalarized i32s, passed in A0 - A3 and the stack.
2678 // vXf32 - Passed in either a pair of registers {A0, A1}, {A2, A3} or {A0 - A3}
2679 //         with the remainder spilled to the stack.
2680 // vXf64 - Passed in either {A0, A1, A2, A3} or {A2, A3} and in both cases
2681 //         spilling the remainder to the stack.
2682 //
2683 //  For vararg functions, all arguments are passed in A0, A1, A2, A3 and stack.
2684 //===----------------------------------------------------------------------===//
2685 
2686 static bool CC_MipsO32(unsigned ValNo, MVT ValVT, MVT LocVT,
2687                        CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
2688                        CCState &State, ArrayRef<MCPhysReg> F64Regs) {
2689   const MipsSubtarget &Subtarget = static_cast<const MipsSubtarget &>(
2690       State.getMachineFunction().getSubtarget());
2691 
2692   static const MCPhysReg IntRegs[] = { Mips::A0, Mips::A1, Mips::A2, Mips::A3 };
2693 
2694   const MipsCCState * MipsState = static_cast<MipsCCState *>(&State);
2695 
2696   static const MCPhysReg F32Regs[] = { Mips::F12, Mips::F14 };
2697 
2698   static const MCPhysReg FloatVectorIntRegs[] = { Mips::A0, Mips::A2 };
2699 
2700   // Do not process byval args here.
2701   if (ArgFlags.isByVal())
2702     return true;
2703 
2704   // Promote i8 and i16
2705   if (ArgFlags.isInReg() && !Subtarget.isLittle()) {
2706     if (LocVT == MVT::i8 || LocVT == MVT::i16 || LocVT == MVT::i32) {
2707       LocVT = MVT::i32;
2708       if (ArgFlags.isSExt())
2709         LocInfo = CCValAssign::SExtUpper;
2710       else if (ArgFlags.isZExt())
2711         LocInfo = CCValAssign::ZExtUpper;
2712       else
2713         LocInfo = CCValAssign::AExtUpper;
2714     }
2715   }
2716 
2717   // Promote i8 and i16
2718   if (LocVT == MVT::i8 || LocVT == MVT::i16) {
2719     LocVT = MVT::i32;
2720     if (ArgFlags.isSExt())
2721       LocInfo = CCValAssign::SExt;
2722     else if (ArgFlags.isZExt())
2723       LocInfo = CCValAssign::ZExt;
2724     else
2725       LocInfo = CCValAssign::AExt;
2726   }
2727 
2728   unsigned Reg;
2729 
2730   // f32 and f64 are allocated in A0, A1, A2, A3 when either of the following
2731   // is true: function is vararg, argument is 3rd or higher, there is previous
2732   // argument which is not f32 or f64.
2733   bool AllocateFloatsInIntReg = State.isVarArg() || ValNo > 1 ||
2734                                 State.getFirstUnallocated(F32Regs) != ValNo;
2735   unsigned OrigAlign = ArgFlags.getOrigAlign();
2736   bool isI64 = (ValVT == MVT::i32 && OrigAlign == 8);
2737   bool isVectorFloat = MipsState->WasOriginalArgVectorFloat(ValNo);
2738 
2739   // The MIPS vector ABI for floats passes them in a pair of registers
2740   if (ValVT == MVT::i32 && isVectorFloat) {
2741     // This is the start of an vector that was scalarized into an unknown number
2742     // of components. It doesn't matter how many there are. Allocate one of the
2743     // notional 8 byte aligned registers which map onto the argument stack, and
2744     // shadow the register lost to alignment requirements.
2745     if (ArgFlags.isSplit()) {
2746       Reg = State.AllocateReg(FloatVectorIntRegs);
2747       if (Reg == Mips::A2)
2748         State.AllocateReg(Mips::A1);
2749       else if (Reg == 0)
2750         State.AllocateReg(Mips::A3);
2751     } else {
2752       // If we're an intermediate component of the split, we can just attempt to
2753       // allocate a register directly.
2754       Reg = State.AllocateReg(IntRegs);
2755     }
2756   } else if (ValVT == MVT::i32 || (ValVT == MVT::f32 && AllocateFloatsInIntReg)) {
2757     Reg = State.AllocateReg(IntRegs);
2758     // If this is the first part of an i64 arg,
2759     // the allocated register must be either A0 or A2.
2760     if (isI64 && (Reg == Mips::A1 || Reg == Mips::A3))
2761       Reg = State.AllocateReg(IntRegs);
2762     LocVT = MVT::i32;
2763   } else if (ValVT == MVT::f64 && AllocateFloatsInIntReg) {
2764     // Allocate int register and shadow next int register. If first
2765     // available register is Mips::A1 or Mips::A3, shadow it too.
2766     Reg = State.AllocateReg(IntRegs);
2767     if (Reg == Mips::A1 || Reg == Mips::A3)
2768       Reg = State.AllocateReg(IntRegs);
2769     State.AllocateReg(IntRegs);
2770     LocVT = MVT::i32;
2771   } else if (ValVT.isFloatingPoint() && !AllocateFloatsInIntReg) {
2772     // we are guaranteed to find an available float register
2773     if (ValVT == MVT::f32) {
2774       Reg = State.AllocateReg(F32Regs);
2775       // Shadow int register
2776       State.AllocateReg(IntRegs);
2777     } else {
2778       Reg = State.AllocateReg(F64Regs);
2779       // Shadow int registers
2780       unsigned Reg2 = State.AllocateReg(IntRegs);
2781       if (Reg2 == Mips::A1 || Reg2 == Mips::A3)
2782         State.AllocateReg(IntRegs);
2783       State.AllocateReg(IntRegs);
2784     }
2785   } else
2786     llvm_unreachable("Cannot handle this ValVT.");
2787 
2788   if (!Reg) {
2789     unsigned Offset = State.AllocateStack(ValVT.getStoreSize(), OrigAlign);
2790     State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo));
2791   } else
2792     State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
2793 
2794   return false;
2795 }
2796 
2797 static bool CC_MipsO32_FP32(unsigned ValNo, MVT ValVT,
2798                             MVT LocVT, CCValAssign::LocInfo LocInfo,
2799                             ISD::ArgFlagsTy ArgFlags, CCState &State) {
2800   static const MCPhysReg F64Regs[] = { Mips::D6, Mips::D7 };
2801 
2802   return CC_MipsO32(ValNo, ValVT, LocVT, LocInfo, ArgFlags, State, F64Regs);
2803 }
2804 
2805 static bool CC_MipsO32_FP64(unsigned ValNo, MVT ValVT,
2806                             MVT LocVT, CCValAssign::LocInfo LocInfo,
2807                             ISD::ArgFlagsTy ArgFlags, CCState &State) {
2808   static const MCPhysReg F64Regs[] = { Mips::D12_64, Mips::D14_64 };
2809 
2810   return CC_MipsO32(ValNo, ValVT, LocVT, LocInfo, ArgFlags, State, F64Regs);
2811 }
2812 
2813 static bool CC_MipsO32(unsigned ValNo, MVT ValVT, MVT LocVT,
2814                        CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
2815                        CCState &State) LLVM_ATTRIBUTE_UNUSED;
2816 
2817 #include "MipsGenCallingConv.inc"
2818 
2819  CCAssignFn *MipsTargetLowering::CCAssignFnForCall() const{
2820    return CC_Mips;
2821  }
2822 
2823  CCAssignFn *MipsTargetLowering::CCAssignFnForReturn() const{
2824    return RetCC_Mips;
2825  }
2826 //===----------------------------------------------------------------------===//
2827 //                  Call Calling Convention Implementation
2828 //===----------------------------------------------------------------------===//
2829 
2830 // Return next O32 integer argument register.
2831 static unsigned getNextIntArgReg(unsigned Reg) {
2832   assert((Reg == Mips::A0) || (Reg == Mips::A2));
2833   return (Reg == Mips::A0) ? Mips::A1 : Mips::A3;
2834 }
2835 
2836 SDValue MipsTargetLowering::passArgOnStack(SDValue StackPtr, unsigned Offset,
2837                                            SDValue Chain, SDValue Arg,
2838                                            const SDLoc &DL, bool IsTailCall,
2839                                            SelectionDAG &DAG) const {
2840   if (!IsTailCall) {
2841     SDValue PtrOff =
2842         DAG.getNode(ISD::ADD, DL, getPointerTy(DAG.getDataLayout()), StackPtr,
2843                     DAG.getIntPtrConstant(Offset, DL));
2844     return DAG.getStore(Chain, DL, Arg, PtrOff, MachinePointerInfo());
2845   }
2846 
2847   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
2848   int FI = MFI.CreateFixedObject(Arg.getValueSizeInBits() / 8, Offset, false);
2849   SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
2850   return DAG.getStore(Chain, DL, Arg, FIN, MachinePointerInfo(),
2851                       /* Alignment = */ 0, MachineMemOperand::MOVolatile);
2852 }
2853 
2854 void MipsTargetLowering::
2855 getOpndList(SmallVectorImpl<SDValue> &Ops,
2856             std::deque<std::pair<unsigned, SDValue>> &RegsToPass,
2857             bool IsPICCall, bool GlobalOrExternal, bool InternalLinkage,
2858             bool IsCallReloc, CallLoweringInfo &CLI, SDValue Callee,
2859             SDValue Chain) const {
2860   // Insert node "GP copy globalreg" before call to function.
2861   //
2862   // R_MIPS_CALL* operators (emitted when non-internal functions are called
2863   // in PIC mode) allow symbols to be resolved via lazy binding.
2864   // The lazy binding stub requires GP to point to the GOT.
2865   // Note that we don't need GP to point to the GOT for indirect calls
2866   // (when R_MIPS_CALL* is not used for the call) because Mips linker generates
2867   // lazy binding stub for a function only when R_MIPS_CALL* are the only relocs
2868   // used for the function (that is, Mips linker doesn't generate lazy binding
2869   // stub for a function whose address is taken in the program).
2870   if (IsPICCall && !InternalLinkage && IsCallReloc) {
2871     unsigned GPReg = ABI.IsN64() ? Mips::GP_64 : Mips::GP;
2872     EVT Ty = ABI.IsN64() ? MVT::i64 : MVT::i32;
2873     RegsToPass.push_back(std::make_pair(GPReg, getGlobalReg(CLI.DAG, Ty)));
2874   }
2875 
2876   // Build a sequence of copy-to-reg nodes chained together with token
2877   // chain and flag operands which copy the outgoing args into registers.
2878   // The InFlag in necessary since all emitted instructions must be
2879   // stuck together.
2880   SDValue InFlag;
2881 
2882   for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
2883     Chain = CLI.DAG.getCopyToReg(Chain, CLI.DL, RegsToPass[i].first,
2884                                  RegsToPass[i].second, InFlag);
2885     InFlag = Chain.getValue(1);
2886   }
2887 
2888   // Add argument registers to the end of the list so that they are
2889   // known live into the call.
2890   for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
2891     Ops.push_back(CLI.DAG.getRegister(RegsToPass[i].first,
2892                                       RegsToPass[i].second.getValueType()));
2893 
2894   // Add a register mask operand representing the call-preserved registers.
2895   const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
2896   const uint32_t *Mask =
2897       TRI->getCallPreservedMask(CLI.DAG.getMachineFunction(), CLI.CallConv);
2898   assert(Mask && "Missing call preserved mask for calling convention");
2899   if (Subtarget.inMips16HardFloat()) {
2900     if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(CLI.Callee)) {
2901       StringRef Sym = G->getGlobal()->getName();
2902       Function *F = G->getGlobal()->getParent()->getFunction(Sym);
2903       if (F && F->hasFnAttribute("__Mips16RetHelper")) {
2904         Mask = MipsRegisterInfo::getMips16RetHelperMask();
2905       }
2906     }
2907   }
2908   Ops.push_back(CLI.DAG.getRegisterMask(Mask));
2909 
2910   if (InFlag.getNode())
2911     Ops.push_back(InFlag);
2912 }
2913 
2914 /// LowerCall - functions arguments are copied from virtual regs to
2915 /// (physical regs)/(stack frame), CALLSEQ_START and CALLSEQ_END are emitted.
2916 SDValue
2917 MipsTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
2918                               SmallVectorImpl<SDValue> &InVals) const {
2919   SelectionDAG &DAG                     = CLI.DAG;
2920   SDLoc DL                              = CLI.DL;
2921   SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
2922   SmallVectorImpl<SDValue> &OutVals     = CLI.OutVals;
2923   SmallVectorImpl<ISD::InputArg> &Ins   = CLI.Ins;
2924   SDValue Chain                         = CLI.Chain;
2925   SDValue Callee                        = CLI.Callee;
2926   bool &IsTailCall                      = CLI.IsTailCall;
2927   CallingConv::ID CallConv              = CLI.CallConv;
2928   bool IsVarArg                         = CLI.IsVarArg;
2929 
2930   MachineFunction &MF = DAG.getMachineFunction();
2931   MachineFrameInfo &MFI = MF.getFrameInfo();
2932   const TargetFrameLowering *TFL = Subtarget.getFrameLowering();
2933   MipsFunctionInfo *FuncInfo = MF.getInfo<MipsFunctionInfo>();
2934   bool IsPIC = isPositionIndependent();
2935 
2936   // Analyze operands of the call, assigning locations to each operand.
2937   SmallVector<CCValAssign, 16> ArgLocs;
2938   MipsCCState CCInfo(
2939       CallConv, IsVarArg, DAG.getMachineFunction(), ArgLocs, *DAG.getContext(),
2940       MipsCCState::getSpecialCallingConvForCallee(Callee.getNode(), Subtarget));
2941 
2942   const ExternalSymbolSDNode *ES =
2943       dyn_cast_or_null<const ExternalSymbolSDNode>(Callee.getNode());
2944 
2945   // There is one case where CALLSEQ_START..CALLSEQ_END can be nested, which
2946   // is during the lowering of a call with a byval argument which produces
2947   // a call to memcpy. For the O32 case, this causes the caller to allocate
2948   // stack space for the reserved argument area for the callee, then recursively
2949   // again for the memcpy call. In the NEWABI case, this doesn't occur as those
2950   // ABIs mandate that the callee allocates the reserved argument area. We do
2951   // still produce nested CALLSEQ_START..CALLSEQ_END with zero space though.
2952   //
2953   // If the callee has a byval argument and memcpy is used, we are mandated
2954   // to already have produced a reserved argument area for the callee for O32.
2955   // Therefore, the reserved argument area can be reused for both calls.
2956   //
2957   // Other cases of calling memcpy cannot have a chain with a CALLSEQ_START
2958   // present, as we have yet to hook that node onto the chain.
2959   //
2960   // Hence, the CALLSEQ_START and CALLSEQ_END nodes can be eliminated in this
2961   // case. GCC does a similar trick, in that wherever possible, it calculates
2962   // the maximum out going argument area (including the reserved area), and
2963   // preallocates the stack space on entrance to the caller.
2964   //
2965   // FIXME: We should do the same for efficency and space.
2966 
2967   // Note: The check on the calling convention below must match
2968   //       MipsABIInfo::GetCalleeAllocdArgSizeInBytes().
2969   bool MemcpyInByVal = ES &&
2970                        StringRef(ES->getSymbol()) == StringRef("memcpy") &&
2971                        CallConv != CallingConv::Fast &&
2972                        Chain.getOpcode() == ISD::CALLSEQ_START;
2973 
2974   // Allocate the reserved argument area. It seems strange to do this from the
2975   // caller side but removing it breaks the frame size calculation.
2976   unsigned ReservedArgArea =
2977       MemcpyInByVal ? 0 : ABI.GetCalleeAllocdArgSizeInBytes(CallConv);
2978   CCInfo.AllocateStack(ReservedArgArea, 1);
2979 
2980   CCInfo.AnalyzeCallOperands(Outs, CC_Mips, CLI.getArgs(),
2981                              ES ? ES->getSymbol() : nullptr);
2982 
2983   // Get a count of how many bytes are to be pushed on the stack.
2984   unsigned NextStackOffset = CCInfo.getNextStackOffset();
2985 
2986   // Check if it's really possible to do a tail call. Restrict it to functions
2987   // that are part of this compilation unit.
2988   bool InternalLinkage = false;
2989   if (IsTailCall) {
2990     IsTailCall = isEligibleForTailCallOptimization(
2991         CCInfo, NextStackOffset, *MF.getInfo<MipsFunctionInfo>());
2992      if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
2993       InternalLinkage = G->getGlobal()->hasInternalLinkage();
2994       IsTailCall &= (InternalLinkage || G->getGlobal()->hasLocalLinkage() ||
2995                      G->getGlobal()->hasPrivateLinkage() ||
2996                      G->getGlobal()->hasHiddenVisibility() ||
2997                      G->getGlobal()->hasProtectedVisibility());
2998      }
2999   }
3000   if (!IsTailCall && CLI.CS && CLI.CS.isMustTailCall())
3001     report_fatal_error("failed to perform tail call elimination on a call "
3002                        "site marked musttail");
3003 
3004   if (IsTailCall)
3005     ++NumTailCalls;
3006 
3007   // Chain is the output chain of the last Load/Store or CopyToReg node.
3008   // ByValChain is the output chain of the last Memcpy node created for copying
3009   // byval arguments to the stack.
3010   unsigned StackAlignment = TFL->getStackAlignment();
3011   NextStackOffset = alignTo(NextStackOffset, StackAlignment);
3012   SDValue NextStackOffsetVal = DAG.getIntPtrConstant(NextStackOffset, DL, true);
3013 
3014   if (!(IsTailCall || MemcpyInByVal))
3015     Chain = DAG.getCALLSEQ_START(Chain, NextStackOffset, 0, DL);
3016 
3017   SDValue StackPtr =
3018       DAG.getCopyFromReg(Chain, DL, ABI.IsN64() ? Mips::SP_64 : Mips::SP,
3019                          getPointerTy(DAG.getDataLayout()));
3020 
3021   std::deque<std::pair<unsigned, SDValue>> RegsToPass;
3022   SmallVector<SDValue, 8> MemOpChains;
3023 
3024   CCInfo.rewindByValRegsInfo();
3025 
3026   // Walk the register/memloc assignments, inserting copies/loads.
3027   for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
3028     SDValue Arg = OutVals[i];
3029     CCValAssign &VA = ArgLocs[i];
3030     MVT ValVT = VA.getValVT(), LocVT = VA.getLocVT();
3031     ISD::ArgFlagsTy Flags = Outs[i].Flags;
3032     bool UseUpperBits = false;
3033 
3034     // ByVal Arg.
3035     if (Flags.isByVal()) {
3036       unsigned FirstByValReg, LastByValReg;
3037       unsigned ByValIdx = CCInfo.getInRegsParamsProcessed();
3038       CCInfo.getInRegsParamInfo(ByValIdx, FirstByValReg, LastByValReg);
3039 
3040       assert(Flags.getByValSize() &&
3041              "ByVal args of size 0 should have been ignored by front-end.");
3042       assert(ByValIdx < CCInfo.getInRegsParamsCount());
3043       assert(!IsTailCall &&
3044              "Do not tail-call optimize if there is a byval argument.");
3045       passByValArg(Chain, DL, RegsToPass, MemOpChains, StackPtr, MFI, DAG, Arg,
3046                    FirstByValReg, LastByValReg, Flags, Subtarget.isLittle(),
3047                    VA);
3048       CCInfo.nextInRegsParam();
3049       continue;
3050     }
3051 
3052     // Promote the value if needed.
3053     switch (VA.getLocInfo()) {
3054     default:
3055       llvm_unreachable("Unknown loc info!");
3056     case CCValAssign::Full:
3057       if (VA.isRegLoc()) {
3058         if ((ValVT == MVT::f32 && LocVT == MVT::i32) ||
3059             (ValVT == MVT::f64 && LocVT == MVT::i64) ||
3060             (ValVT == MVT::i64 && LocVT == MVT::f64))
3061           Arg = DAG.getNode(ISD::BITCAST, DL, LocVT, Arg);
3062         else if (ValVT == MVT::f64 && LocVT == MVT::i32) {
3063           SDValue Lo = DAG.getNode(MipsISD::ExtractElementF64, DL, MVT::i32,
3064                                    Arg, DAG.getConstant(0, DL, MVT::i32));
3065           SDValue Hi = DAG.getNode(MipsISD::ExtractElementF64, DL, MVT::i32,
3066                                    Arg, DAG.getConstant(1, DL, MVT::i32));
3067           if (!Subtarget.isLittle())
3068             std::swap(Lo, Hi);
3069           unsigned LocRegLo = VA.getLocReg();
3070           unsigned LocRegHigh = getNextIntArgReg(LocRegLo);
3071           RegsToPass.push_back(std::make_pair(LocRegLo, Lo));
3072           RegsToPass.push_back(std::make_pair(LocRegHigh, Hi));
3073           continue;
3074         }
3075       }
3076       break;
3077     case CCValAssign::BCvt:
3078       Arg = DAG.getNode(ISD::BITCAST, DL, LocVT, Arg);
3079       break;
3080     case CCValAssign::SExtUpper:
3081       UseUpperBits = true;
3082       LLVM_FALLTHROUGH;
3083     case CCValAssign::SExt:
3084       Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, LocVT, Arg);
3085       break;
3086     case CCValAssign::ZExtUpper:
3087       UseUpperBits = true;
3088       LLVM_FALLTHROUGH;
3089     case CCValAssign::ZExt:
3090       Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, LocVT, Arg);
3091       break;
3092     case CCValAssign::AExtUpper:
3093       UseUpperBits = true;
3094       LLVM_FALLTHROUGH;
3095     case CCValAssign::AExt:
3096       Arg = DAG.getNode(ISD::ANY_EXTEND, DL, LocVT, Arg);
3097       break;
3098     }
3099 
3100     if (UseUpperBits) {
3101       unsigned ValSizeInBits = Outs[i].ArgVT.getSizeInBits();
3102       unsigned LocSizeInBits = VA.getLocVT().getSizeInBits();
3103       Arg = DAG.getNode(
3104           ISD::SHL, DL, VA.getLocVT(), Arg,
3105           DAG.getConstant(LocSizeInBits - ValSizeInBits, DL, VA.getLocVT()));
3106     }
3107 
3108     // Arguments that can be passed on register must be kept at
3109     // RegsToPass vector
3110     if (VA.isRegLoc()) {
3111       RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
3112       continue;
3113     }
3114 
3115     // Register can't get to this point...
3116     assert(VA.isMemLoc());
3117 
3118     // emit ISD::STORE whichs stores the
3119     // parameter value to a stack Location
3120     MemOpChains.push_back(passArgOnStack(StackPtr, VA.getLocMemOffset(),
3121                                          Chain, Arg, DL, IsTailCall, DAG));
3122   }
3123 
3124   // Transform all store nodes into one single node because all store
3125   // nodes are independent of each other.
3126   if (!MemOpChains.empty())
3127     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
3128 
3129   // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
3130   // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
3131   // node so that legalize doesn't hack it.
3132 
3133   EVT Ty = Callee.getValueType();
3134   bool GlobalOrExternal = false, IsCallReloc = false;
3135 
3136   // The long-calls feature is ignored in case of PIC.
3137   // While we do not support -mshared / -mno-shared properly,
3138   // ignore long-calls in case of -mabicalls too.
3139   if (!Subtarget.isABICalls() && !IsPIC) {
3140     // If the function should be called using "long call",
3141     // get its address into a register to prevent using
3142     // of the `jal` instruction for the direct call.
3143     if (auto *N = dyn_cast<ExternalSymbolSDNode>(Callee)) {
3144       if (Subtarget.useLongCalls())
3145         Callee = Subtarget.hasSym32()
3146                      ? getAddrNonPIC(N, SDLoc(N), Ty, DAG)
3147                      : getAddrNonPICSym64(N, SDLoc(N), Ty, DAG);
3148     } else if (auto *N = dyn_cast<GlobalAddressSDNode>(Callee)) {
3149       bool UseLongCalls = Subtarget.useLongCalls();
3150       // If the function has long-call/far/near attribute
3151       // it overrides command line switch pased to the backend.
3152       if (auto *F = dyn_cast<Function>(N->getGlobal())) {
3153         if (F->hasFnAttribute("long-call"))
3154           UseLongCalls = true;
3155         else if (F->hasFnAttribute("short-call"))
3156           UseLongCalls = false;
3157       }
3158       if (UseLongCalls)
3159         Callee = Subtarget.hasSym32()
3160                      ? getAddrNonPIC(N, SDLoc(N), Ty, DAG)
3161                      : getAddrNonPICSym64(N, SDLoc(N), Ty, DAG);
3162     }
3163   }
3164 
3165   if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
3166     if (IsPIC) {
3167       const GlobalValue *Val = G->getGlobal();
3168       InternalLinkage = Val->hasInternalLinkage();
3169 
3170       if (InternalLinkage)
3171         Callee = getAddrLocal(G, DL, Ty, DAG, ABI.IsN32() || ABI.IsN64());
3172       else if (LargeGOT) {
3173         Callee = getAddrGlobalLargeGOT(G, DL, Ty, DAG, MipsII::MO_CALL_HI16,
3174                                        MipsII::MO_CALL_LO16, Chain,
3175                                        FuncInfo->callPtrInfo(Val));
3176         IsCallReloc = true;
3177       } else {
3178         Callee = getAddrGlobal(G, DL, Ty, DAG, MipsII::MO_GOT_CALL, Chain,
3179                                FuncInfo->callPtrInfo(Val));
3180         IsCallReloc = true;
3181       }
3182     } else
3183       Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL,
3184                                           getPointerTy(DAG.getDataLayout()), 0,
3185                                           MipsII::MO_NO_FLAG);
3186     GlobalOrExternal = true;
3187   }
3188   else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
3189     const char *Sym = S->getSymbol();
3190 
3191     if (!IsPIC) // static
3192       Callee = DAG.getTargetExternalSymbol(
3193           Sym, getPointerTy(DAG.getDataLayout()), MipsII::MO_NO_FLAG);
3194     else if (LargeGOT) {
3195       Callee = getAddrGlobalLargeGOT(S, DL, Ty, DAG, MipsII::MO_CALL_HI16,
3196                                      MipsII::MO_CALL_LO16, Chain,
3197                                      FuncInfo->callPtrInfo(Sym));
3198       IsCallReloc = true;
3199     } else { // PIC
3200       Callee = getAddrGlobal(S, DL, Ty, DAG, MipsII::MO_GOT_CALL, Chain,
3201                              FuncInfo->callPtrInfo(Sym));
3202       IsCallReloc = true;
3203     }
3204 
3205     GlobalOrExternal = true;
3206   }
3207 
3208   SmallVector<SDValue, 8> Ops(1, Chain);
3209   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
3210 
3211   getOpndList(Ops, RegsToPass, IsPIC, GlobalOrExternal, InternalLinkage,
3212               IsCallReloc, CLI, Callee, Chain);
3213 
3214   if (IsTailCall) {
3215     MF.getFrameInfo().setHasTailCall();
3216     return DAG.getNode(MipsISD::TailCall, DL, MVT::Other, Ops);
3217   }
3218 
3219   Chain = DAG.getNode(MipsISD::JmpLink, DL, NodeTys, Ops);
3220   SDValue InFlag = Chain.getValue(1);
3221 
3222   // Create the CALLSEQ_END node in the case of where it is not a call to
3223   // memcpy.
3224   if (!(MemcpyInByVal)) {
3225     Chain = DAG.getCALLSEQ_END(Chain, NextStackOffsetVal,
3226                                DAG.getIntPtrConstant(0, DL, true), InFlag, DL);
3227     InFlag = Chain.getValue(1);
3228   }
3229 
3230   // Handle result values, copying them out of physregs into vregs that we
3231   // return.
3232   return LowerCallResult(Chain, InFlag, CallConv, IsVarArg, Ins, DL, DAG,
3233                          InVals, CLI);
3234 }
3235 
3236 /// LowerCallResult - Lower the result values of a call into the
3237 /// appropriate copies out of appropriate physical registers.
3238 SDValue MipsTargetLowering::LowerCallResult(
3239     SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool IsVarArg,
3240     const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
3241     SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
3242     TargetLowering::CallLoweringInfo &CLI) const {
3243   // Assign locations to each value returned by this call.
3244   SmallVector<CCValAssign, 16> RVLocs;
3245   MipsCCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
3246                      *DAG.getContext());
3247 
3248   const ExternalSymbolSDNode *ES =
3249       dyn_cast_or_null<const ExternalSymbolSDNode>(CLI.Callee.getNode());
3250   CCInfo.AnalyzeCallResult(Ins, RetCC_Mips, CLI.RetTy,
3251                            ES ? ES->getSymbol() : nullptr);
3252 
3253   // Copy all of the result registers out of their specified physreg.
3254   for (unsigned i = 0; i != RVLocs.size(); ++i) {
3255     CCValAssign &VA = RVLocs[i];
3256     assert(VA.isRegLoc() && "Can only return in registers!");
3257 
3258     SDValue Val = DAG.getCopyFromReg(Chain, DL, RVLocs[i].getLocReg(),
3259                                      RVLocs[i].getLocVT(), InFlag);
3260     Chain = Val.getValue(1);
3261     InFlag = Val.getValue(2);
3262 
3263     if (VA.isUpperBitsInLoc()) {
3264       unsigned ValSizeInBits = Ins[i].ArgVT.getSizeInBits();
3265       unsigned LocSizeInBits = VA.getLocVT().getSizeInBits();
3266       unsigned Shift =
3267           VA.getLocInfo() == CCValAssign::ZExtUpper ? ISD::SRL : ISD::SRA;
3268       Val = DAG.getNode(
3269           Shift, DL, VA.getLocVT(), Val,
3270           DAG.getConstant(LocSizeInBits - ValSizeInBits, DL, VA.getLocVT()));
3271     }
3272 
3273     switch (VA.getLocInfo()) {
3274     default:
3275       llvm_unreachable("Unknown loc info!");
3276     case CCValAssign::Full:
3277       break;
3278     case CCValAssign::BCvt:
3279       Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
3280       break;
3281     case CCValAssign::AExt:
3282     case CCValAssign::AExtUpper:
3283       Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val);
3284       break;
3285     case CCValAssign::ZExt:
3286     case CCValAssign::ZExtUpper:
3287       Val = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Val,
3288                         DAG.getValueType(VA.getValVT()));
3289       Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val);
3290       break;
3291     case CCValAssign::SExt:
3292     case CCValAssign::SExtUpper:
3293       Val = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Val,
3294                         DAG.getValueType(VA.getValVT()));
3295       Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val);
3296       break;
3297     }
3298 
3299     InVals.push_back(Val);
3300   }
3301 
3302   return Chain;
3303 }
3304 
3305 static SDValue UnpackFromArgumentSlot(SDValue Val, const CCValAssign &VA,
3306                                       EVT ArgVT, const SDLoc &DL,
3307                                       SelectionDAG &DAG) {
3308   MVT LocVT = VA.getLocVT();
3309   EVT ValVT = VA.getValVT();
3310 
3311   // Shift into the upper bits if necessary.
3312   switch (VA.getLocInfo()) {
3313   default:
3314     break;
3315   case CCValAssign::AExtUpper:
3316   case CCValAssign::SExtUpper:
3317   case CCValAssign::ZExtUpper: {
3318     unsigned ValSizeInBits = ArgVT.getSizeInBits();
3319     unsigned LocSizeInBits = VA.getLocVT().getSizeInBits();
3320     unsigned Opcode =
3321         VA.getLocInfo() == CCValAssign::ZExtUpper ? ISD::SRL : ISD::SRA;
3322     Val = DAG.getNode(
3323         Opcode, DL, VA.getLocVT(), Val,
3324         DAG.getConstant(LocSizeInBits - ValSizeInBits, DL, VA.getLocVT()));
3325     break;
3326   }
3327   }
3328 
3329   // If this is an value smaller than the argument slot size (32-bit for O32,
3330   // 64-bit for N32/N64), it has been promoted in some way to the argument slot
3331   // size. Extract the value and insert any appropriate assertions regarding
3332   // sign/zero extension.
3333   switch (VA.getLocInfo()) {
3334   default:
3335     llvm_unreachable("Unknown loc info!");
3336   case CCValAssign::Full:
3337     break;
3338   case CCValAssign::AExtUpper:
3339   case CCValAssign::AExt:
3340     Val = DAG.getNode(ISD::TRUNCATE, DL, ValVT, Val);
3341     break;
3342   case CCValAssign::SExtUpper:
3343   case CCValAssign::SExt:
3344     Val = DAG.getNode(ISD::AssertSext, DL, LocVT, Val, DAG.getValueType(ValVT));
3345     Val = DAG.getNode(ISD::TRUNCATE, DL, ValVT, Val);
3346     break;
3347   case CCValAssign::ZExtUpper:
3348   case CCValAssign::ZExt:
3349     Val = DAG.getNode(ISD::AssertZext, DL, LocVT, Val, DAG.getValueType(ValVT));
3350     Val = DAG.getNode(ISD::TRUNCATE, DL, ValVT, Val);
3351     break;
3352   case CCValAssign::BCvt:
3353     Val = DAG.getNode(ISD::BITCAST, DL, ValVT, Val);
3354     break;
3355   }
3356 
3357   return Val;
3358 }
3359 
3360 //===----------------------------------------------------------------------===//
3361 //             Formal Arguments Calling Convention Implementation
3362 //===----------------------------------------------------------------------===//
3363 /// LowerFormalArguments - transform physical registers into virtual registers
3364 /// and generate load operations for arguments places on the stack.
3365 SDValue MipsTargetLowering::LowerFormalArguments(
3366     SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
3367     const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
3368     SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
3369   MachineFunction &MF = DAG.getMachineFunction();
3370   MachineFrameInfo &MFI = MF.getFrameInfo();
3371   MipsFunctionInfo *MipsFI = MF.getInfo<MipsFunctionInfo>();
3372 
3373   MipsFI->setVarArgsFrameIndex(0);
3374 
3375   // Used with vargs to acumulate store chains.
3376   std::vector<SDValue> OutChains;
3377 
3378   // Assign locations to all of the incoming arguments.
3379   SmallVector<CCValAssign, 16> ArgLocs;
3380   MipsCCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), ArgLocs,
3381                      *DAG.getContext());
3382   CCInfo.AllocateStack(ABI.GetCalleeAllocdArgSizeInBytes(CallConv), 1);
3383   const Function &Func = DAG.getMachineFunction().getFunction();
3384   Function::const_arg_iterator FuncArg = Func.arg_begin();
3385 
3386   if (Func.hasFnAttribute("interrupt") && !Func.arg_empty())
3387     report_fatal_error(
3388         "Functions with the interrupt attribute cannot have arguments!");
3389 
3390   CCInfo.AnalyzeFormalArguments(Ins, CC_Mips_FixedArg);
3391   MipsFI->setFormalArgInfo(CCInfo.getNextStackOffset(),
3392                            CCInfo.getInRegsParamsCount() > 0);
3393 
3394   unsigned CurArgIdx = 0;
3395   CCInfo.rewindByValRegsInfo();
3396 
3397   for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
3398     CCValAssign &VA = ArgLocs[i];
3399     if (Ins[i].isOrigArg()) {
3400       std::advance(FuncArg, Ins[i].getOrigArgIndex() - CurArgIdx);
3401       CurArgIdx = Ins[i].getOrigArgIndex();
3402     }
3403     EVT ValVT = VA.getValVT();
3404     ISD::ArgFlagsTy Flags = Ins[i].Flags;
3405     bool IsRegLoc = VA.isRegLoc();
3406 
3407     if (Flags.isByVal()) {
3408       assert(Ins[i].isOrigArg() && "Byval arguments cannot be implicit");
3409       unsigned FirstByValReg, LastByValReg;
3410       unsigned ByValIdx = CCInfo.getInRegsParamsProcessed();
3411       CCInfo.getInRegsParamInfo(ByValIdx, FirstByValReg, LastByValReg);
3412 
3413       assert(Flags.getByValSize() &&
3414              "ByVal args of size 0 should have been ignored by front-end.");
3415       assert(ByValIdx < CCInfo.getInRegsParamsCount());
3416       copyByValRegs(Chain, DL, OutChains, DAG, Flags, InVals, &*FuncArg,
3417                     FirstByValReg, LastByValReg, VA, CCInfo);
3418       CCInfo.nextInRegsParam();
3419       continue;
3420     }
3421 
3422     // Arguments stored on registers
3423     if (IsRegLoc) {
3424       MVT RegVT = VA.getLocVT();
3425       unsigned ArgReg = VA.getLocReg();
3426       const TargetRegisterClass *RC = getRegClassFor(RegVT);
3427 
3428       // Transform the arguments stored on
3429       // physical registers into virtual ones
3430       unsigned Reg = addLiveIn(DAG.getMachineFunction(), ArgReg, RC);
3431       SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, RegVT);
3432 
3433       ArgValue = UnpackFromArgumentSlot(ArgValue, VA, Ins[i].ArgVT, DL, DAG);
3434 
3435       // Handle floating point arguments passed in integer registers and
3436       // long double arguments passed in floating point registers.
3437       if ((RegVT == MVT::i32 && ValVT == MVT::f32) ||
3438           (RegVT == MVT::i64 && ValVT == MVT::f64) ||
3439           (RegVT == MVT::f64 && ValVT == MVT::i64))
3440         ArgValue = DAG.getNode(ISD::BITCAST, DL, ValVT, ArgValue);
3441       else if (ABI.IsO32() && RegVT == MVT::i32 &&
3442                ValVT == MVT::f64) {
3443         unsigned Reg2 = addLiveIn(DAG.getMachineFunction(),
3444                                   getNextIntArgReg(ArgReg), RC);
3445         SDValue ArgValue2 = DAG.getCopyFromReg(Chain, DL, Reg2, RegVT);
3446         if (!Subtarget.isLittle())
3447           std::swap(ArgValue, ArgValue2);
3448         ArgValue = DAG.getNode(MipsISD::BuildPairF64, DL, MVT::f64,
3449                                ArgValue, ArgValue2);
3450       }
3451 
3452       InVals.push_back(ArgValue);
3453     } else { // VA.isRegLoc()
3454       MVT LocVT = VA.getLocVT();
3455 
3456       if (ABI.IsO32()) {
3457         // We ought to be able to use LocVT directly but O32 sets it to i32
3458         // when allocating floating point values to integer registers.
3459         // This shouldn't influence how we load the value into registers unless
3460         // we are targeting softfloat.
3461         if (VA.getValVT().isFloatingPoint() && !Subtarget.useSoftFloat())
3462           LocVT = VA.getValVT();
3463       }
3464 
3465       // sanity check
3466       assert(VA.isMemLoc());
3467 
3468       // The stack pointer offset is relative to the caller stack frame.
3469       int FI = MFI.CreateFixedObject(LocVT.getSizeInBits() / 8,
3470                                      VA.getLocMemOffset(), true);
3471 
3472       // Create load nodes to retrieve arguments from the stack
3473       SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
3474       SDValue ArgValue = DAG.getLoad(
3475           LocVT, DL, Chain, FIN,
3476           MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI));
3477       OutChains.push_back(ArgValue.getValue(1));
3478 
3479       ArgValue = UnpackFromArgumentSlot(ArgValue, VA, Ins[i].ArgVT, DL, DAG);
3480 
3481       InVals.push_back(ArgValue);
3482     }
3483   }
3484 
3485   for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
3486     // The mips ABIs for returning structs by value requires that we copy
3487     // the sret argument into $v0 for the return. Save the argument into
3488     // a virtual register so that we can access it from the return points.
3489     if (Ins[i].Flags.isSRet()) {
3490       unsigned Reg = MipsFI->getSRetReturnReg();
3491       if (!Reg) {
3492         Reg = MF.getRegInfo().createVirtualRegister(
3493             getRegClassFor(ABI.IsN64() ? MVT::i64 : MVT::i32));
3494         MipsFI->setSRetReturnReg(Reg);
3495       }
3496       SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), DL, Reg, InVals[i]);
3497       Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Copy, Chain);
3498       break;
3499     }
3500   }
3501 
3502   if (IsVarArg)
3503     writeVarArgRegs(OutChains, Chain, DL, DAG, CCInfo);
3504 
3505   // All stores are grouped in one node to allow the matching between
3506   // the size of Ins and InVals. This only happens when on varg functions
3507   if (!OutChains.empty()) {
3508     OutChains.push_back(Chain);
3509     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains);
3510   }
3511 
3512   return Chain;
3513 }
3514 
3515 //===----------------------------------------------------------------------===//
3516 //               Return Value Calling Convention Implementation
3517 //===----------------------------------------------------------------------===//
3518 
3519 bool
3520 MipsTargetLowering::CanLowerReturn(CallingConv::ID CallConv,
3521                                    MachineFunction &MF, bool IsVarArg,
3522                                    const SmallVectorImpl<ISD::OutputArg> &Outs,
3523                                    LLVMContext &Context) const {
3524   SmallVector<CCValAssign, 16> RVLocs;
3525   MipsCCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
3526   return CCInfo.CheckReturn(Outs, RetCC_Mips);
3527 }
3528 
3529 bool
3530 MipsTargetLowering::shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const {
3531   if ((ABI.IsN32() || ABI.IsN64()) && Type == MVT::i32)
3532       return true;
3533 
3534   return IsSigned;
3535 }
3536 
3537 SDValue
3538 MipsTargetLowering::LowerInterruptReturn(SmallVectorImpl<SDValue> &RetOps,
3539                                          const SDLoc &DL,
3540                                          SelectionDAG &DAG) const {
3541   MachineFunction &MF = DAG.getMachineFunction();
3542   MipsFunctionInfo *MipsFI = MF.getInfo<MipsFunctionInfo>();
3543 
3544   MipsFI->setISR();
3545 
3546   return DAG.getNode(MipsISD::ERet, DL, MVT::Other, RetOps);
3547 }
3548 
3549 SDValue
3550 MipsTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
3551                                 bool IsVarArg,
3552                                 const SmallVectorImpl<ISD::OutputArg> &Outs,
3553                                 const SmallVectorImpl<SDValue> &OutVals,
3554                                 const SDLoc &DL, SelectionDAG &DAG) const {
3555   // CCValAssign - represent the assignment of
3556   // the return value to a location
3557   SmallVector<CCValAssign, 16> RVLocs;
3558   MachineFunction &MF = DAG.getMachineFunction();
3559 
3560   // CCState - Info about the registers and stack slot.
3561   MipsCCState CCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
3562 
3563   // Analyze return values.
3564   CCInfo.AnalyzeReturn(Outs, RetCC_Mips);
3565 
3566   SDValue Flag;
3567   SmallVector<SDValue, 4> RetOps(1, Chain);
3568 
3569   // Copy the result values into the output registers.
3570   for (unsigned i = 0; i != RVLocs.size(); ++i) {
3571     SDValue Val = OutVals[i];
3572     CCValAssign &VA = RVLocs[i];
3573     assert(VA.isRegLoc() && "Can only return in registers!");
3574     bool UseUpperBits = false;
3575 
3576     switch (VA.getLocInfo()) {
3577     default:
3578       llvm_unreachable("Unknown loc info!");
3579     case CCValAssign::Full:
3580       break;
3581     case CCValAssign::BCvt:
3582       Val = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Val);
3583       break;
3584     case CCValAssign::AExtUpper:
3585       UseUpperBits = true;
3586       LLVM_FALLTHROUGH;
3587     case CCValAssign::AExt:
3588       Val = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Val);
3589       break;
3590     case CCValAssign::ZExtUpper:
3591       UseUpperBits = true;
3592       LLVM_FALLTHROUGH;
3593     case CCValAssign::ZExt:
3594       Val = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Val);
3595       break;
3596     case CCValAssign::SExtUpper:
3597       UseUpperBits = true;
3598       LLVM_FALLTHROUGH;
3599     case CCValAssign::SExt:
3600       Val = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Val);
3601       break;
3602     }
3603 
3604     if (UseUpperBits) {
3605       unsigned ValSizeInBits = Outs[i].ArgVT.getSizeInBits();
3606       unsigned LocSizeInBits = VA.getLocVT().getSizeInBits();
3607       Val = DAG.getNode(
3608           ISD::SHL, DL, VA.getLocVT(), Val,
3609           DAG.getConstant(LocSizeInBits - ValSizeInBits, DL, VA.getLocVT()));
3610     }
3611 
3612     Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Flag);
3613 
3614     // Guarantee that all emitted copies are stuck together with flags.
3615     Flag = Chain.getValue(1);
3616     RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
3617   }
3618 
3619   // The mips ABIs for returning structs by value requires that we copy
3620   // the sret argument into $v0 for the return. We saved the argument into
3621   // a virtual register in the entry block, so now we copy the value out
3622   // and into $v0.
3623   if (MF.getFunction().hasStructRetAttr()) {
3624     MipsFunctionInfo *MipsFI = MF.getInfo<MipsFunctionInfo>();
3625     unsigned Reg = MipsFI->getSRetReturnReg();
3626 
3627     if (!Reg)
3628       llvm_unreachable("sret virtual register not created in the entry block");
3629     SDValue Val =
3630         DAG.getCopyFromReg(Chain, DL, Reg, getPointerTy(DAG.getDataLayout()));
3631     unsigned V0 = ABI.IsN64() ? Mips::V0_64 : Mips::V0;
3632 
3633     Chain = DAG.getCopyToReg(Chain, DL, V0, Val, Flag);
3634     Flag = Chain.getValue(1);
3635     RetOps.push_back(DAG.getRegister(V0, getPointerTy(DAG.getDataLayout())));
3636   }
3637 
3638   RetOps[0] = Chain;  // Update chain.
3639 
3640   // Add the flag if we have it.
3641   if (Flag.getNode())
3642     RetOps.push_back(Flag);
3643 
3644   // ISRs must use "eret".
3645   if (DAG.getMachineFunction().getFunction().hasFnAttribute("interrupt"))
3646     return LowerInterruptReturn(RetOps, DL, DAG);
3647 
3648   // Standard return on Mips is a "jr $ra"
3649   return DAG.getNode(MipsISD::Ret, DL, MVT::Other, RetOps);
3650 }
3651 
3652 //===----------------------------------------------------------------------===//
3653 //                           Mips Inline Assembly Support
3654 //===----------------------------------------------------------------------===//
3655 
3656 /// getConstraintType - Given a constraint letter, return the type of
3657 /// constraint it is for this target.
3658 MipsTargetLowering::ConstraintType
3659 MipsTargetLowering::getConstraintType(StringRef Constraint) const {
3660   // Mips specific constraints
3661   // GCC config/mips/constraints.md
3662   //
3663   // 'd' : An address register. Equivalent to r
3664   //       unless generating MIPS16 code.
3665   // 'y' : Equivalent to r; retained for
3666   //       backwards compatibility.
3667   // 'c' : A register suitable for use in an indirect
3668   //       jump. This will always be $25 for -mabicalls.
3669   // 'l' : The lo register. 1 word storage.
3670   // 'x' : The hilo register pair. Double word storage.
3671   if (Constraint.size() == 1) {
3672     switch (Constraint[0]) {
3673       default : break;
3674       case 'd':
3675       case 'y':
3676       case 'f':
3677       case 'c':
3678       case 'l':
3679       case 'x':
3680         return C_RegisterClass;
3681       case 'R':
3682         return C_Memory;
3683     }
3684   }
3685 
3686   if (Constraint == "ZC")
3687     return C_Memory;
3688 
3689   return TargetLowering::getConstraintType(Constraint);
3690 }
3691 
3692 /// Examine constraint type and operand type and determine a weight value.
3693 /// This object must already have been set up with the operand type
3694 /// and the current alternative constraint selected.
3695 TargetLowering::ConstraintWeight
3696 MipsTargetLowering::getSingleConstraintMatchWeight(
3697     AsmOperandInfo &info, const char *constraint) const {
3698   ConstraintWeight weight = CW_Invalid;
3699   Value *CallOperandVal = info.CallOperandVal;
3700     // If we don't have a value, we can't do a match,
3701     // but allow it at the lowest weight.
3702   if (!CallOperandVal)
3703     return CW_Default;
3704   Type *type = CallOperandVal->getType();
3705   // Look at the constraint type.
3706   switch (*constraint) {
3707   default:
3708     weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint);
3709     break;
3710   case 'd':
3711   case 'y':
3712     if (type->isIntegerTy())
3713       weight = CW_Register;
3714     break;
3715   case 'f': // FPU or MSA register
3716     if (Subtarget.hasMSA() && type->isVectorTy() &&
3717         cast<VectorType>(type)->getBitWidth() == 128)
3718       weight = CW_Register;
3719     else if (type->isFloatTy())
3720       weight = CW_Register;
3721     break;
3722   case 'c': // $25 for indirect jumps
3723   case 'l': // lo register
3724   case 'x': // hilo register pair
3725     if (type->isIntegerTy())
3726       weight = CW_SpecificReg;
3727     break;
3728   case 'I': // signed 16 bit immediate
3729   case 'J': // integer zero
3730   case 'K': // unsigned 16 bit immediate
3731   case 'L': // signed 32 bit immediate where lower 16 bits are 0
3732   case 'N': // immediate in the range of -65535 to -1 (inclusive)
3733   case 'O': // signed 15 bit immediate (+- 16383)
3734   case 'P': // immediate in the range of 65535 to 1 (inclusive)
3735     if (isa<ConstantInt>(CallOperandVal))
3736       weight = CW_Constant;
3737     break;
3738   case 'R':
3739     weight = CW_Memory;
3740     break;
3741   }
3742   return weight;
3743 }
3744 
3745 /// This is a helper function to parse a physical register string and split it
3746 /// into non-numeric and numeric parts (Prefix and Reg). The first boolean flag
3747 /// that is returned indicates whether parsing was successful. The second flag
3748 /// is true if the numeric part exists.
3749 static std::pair<bool, bool> parsePhysicalReg(StringRef C, StringRef &Prefix,
3750                                               unsigned long long &Reg) {
3751   if (C.front() != '{' || C.back() != '}')
3752     return std::make_pair(false, false);
3753 
3754   // Search for the first numeric character.
3755   StringRef::const_iterator I, B = C.begin() + 1, E = C.end() - 1;
3756   I = std::find_if(B, E, isdigit);
3757 
3758   Prefix = StringRef(B, I - B);
3759 
3760   // The second flag is set to false if no numeric characters were found.
3761   if (I == E)
3762     return std::make_pair(true, false);
3763 
3764   // Parse the numeric characters.
3765   return std::make_pair(!getAsUnsignedInteger(StringRef(I, E - I), 10, Reg),
3766                         true);
3767 }
3768 
3769 std::pair<unsigned, const TargetRegisterClass *> MipsTargetLowering::
3770 parseRegForInlineAsmConstraint(StringRef C, MVT VT) const {
3771   const TargetRegisterInfo *TRI =
3772       Subtarget.getRegisterInfo();
3773   const TargetRegisterClass *RC;
3774   StringRef Prefix;
3775   unsigned long long Reg;
3776 
3777   std::pair<bool, bool> R = parsePhysicalReg(C, Prefix, Reg);
3778 
3779   if (!R.first)
3780     return std::make_pair(0U, nullptr);
3781 
3782   if ((Prefix == "hi" || Prefix == "lo")) { // Parse hi/lo.
3783     // No numeric characters follow "hi" or "lo".
3784     if (R.second)
3785       return std::make_pair(0U, nullptr);
3786 
3787     RC = TRI->getRegClass(Prefix == "hi" ?
3788                           Mips::HI32RegClassID : Mips::LO32RegClassID);
3789     return std::make_pair(*(RC->begin()), RC);
3790   } else if (Prefix.startswith("$msa")) {
3791     // Parse $msa(ir|csr|access|save|modify|request|map|unmap)
3792 
3793     // No numeric characters follow the name.
3794     if (R.second)
3795       return std::make_pair(0U, nullptr);
3796 
3797     Reg = StringSwitch<unsigned long long>(Prefix)
3798               .Case("$msair", Mips::MSAIR)
3799               .Case("$msacsr", Mips::MSACSR)
3800               .Case("$msaaccess", Mips::MSAAccess)
3801               .Case("$msasave", Mips::MSASave)
3802               .Case("$msamodify", Mips::MSAModify)
3803               .Case("$msarequest", Mips::MSARequest)
3804               .Case("$msamap", Mips::MSAMap)
3805               .Case("$msaunmap", Mips::MSAUnmap)
3806               .Default(0);
3807 
3808     if (!Reg)
3809       return std::make_pair(0U, nullptr);
3810 
3811     RC = TRI->getRegClass(Mips::MSACtrlRegClassID);
3812     return std::make_pair(Reg, RC);
3813   }
3814 
3815   if (!R.second)
3816     return std::make_pair(0U, nullptr);
3817 
3818   if (Prefix == "$f") { // Parse $f0-$f31.
3819     // If the size of FP registers is 64-bit or Reg is an even number, select
3820     // the 64-bit register class. Otherwise, select the 32-bit register class.
3821     if (VT == MVT::Other)
3822       VT = (Subtarget.isFP64bit() || !(Reg % 2)) ? MVT::f64 : MVT::f32;
3823 
3824     RC = getRegClassFor(VT);
3825 
3826     if (RC == &Mips::AFGR64RegClass) {
3827       assert(Reg % 2 == 0);
3828       Reg >>= 1;
3829     }
3830   } else if (Prefix == "$fcc") // Parse $fcc0-$fcc7.
3831     RC = TRI->getRegClass(Mips::FCCRegClassID);
3832   else if (Prefix == "$w") { // Parse $w0-$w31.
3833     RC = getRegClassFor((VT == MVT::Other) ? MVT::v16i8 : VT);
3834   } else { // Parse $0-$31.
3835     assert(Prefix == "$");
3836     RC = getRegClassFor((VT == MVT::Other) ? MVT::i32 : VT);
3837   }
3838 
3839   assert(Reg < RC->getNumRegs());
3840   return std::make_pair(*(RC->begin() + Reg), RC);
3841 }
3842 
3843 /// Given a register class constraint, like 'r', if this corresponds directly
3844 /// to an LLVM register class, return a register of 0 and the register class
3845 /// pointer.
3846 std::pair<unsigned, const TargetRegisterClass *>
3847 MipsTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
3848                                                  StringRef Constraint,
3849                                                  MVT VT) const {
3850   if (Constraint.size() == 1) {
3851     switch (Constraint[0]) {
3852     case 'd': // Address register. Same as 'r' unless generating MIPS16 code.
3853     case 'y': // Same as 'r'. Exists for compatibility.
3854     case 'r':
3855       if (VT == MVT::i32 || VT == MVT::i16 || VT == MVT::i8) {
3856         if (Subtarget.inMips16Mode())
3857           return std::make_pair(0U, &Mips::CPU16RegsRegClass);
3858         return std::make_pair(0U, &Mips::GPR32RegClass);
3859       }
3860       if (VT == MVT::i64 && !Subtarget.isGP64bit())
3861         return std::make_pair(0U, &Mips::GPR32RegClass);
3862       if (VT == MVT::i64 && Subtarget.isGP64bit())
3863         return std::make_pair(0U, &Mips::GPR64RegClass);
3864       // This will generate an error message
3865       return std::make_pair(0U, nullptr);
3866     case 'f': // FPU or MSA register
3867       if (VT == MVT::v16i8)
3868         return std::make_pair(0U, &Mips::MSA128BRegClass);
3869       else if (VT == MVT::v8i16 || VT == MVT::v8f16)
3870         return std::make_pair(0U, &Mips::MSA128HRegClass);
3871       else if (VT == MVT::v4i32 || VT == MVT::v4f32)
3872         return std::make_pair(0U, &Mips::MSA128WRegClass);
3873       else if (VT == MVT::v2i64 || VT == MVT::v2f64)
3874         return std::make_pair(0U, &Mips::MSA128DRegClass);
3875       else if (VT == MVT::f32)
3876         return std::make_pair(0U, &Mips::FGR32RegClass);
3877       else if ((VT == MVT::f64) && (!Subtarget.isSingleFloat())) {
3878         if (Subtarget.isFP64bit())
3879           return std::make_pair(0U, &Mips::FGR64RegClass);
3880         return std::make_pair(0U, &Mips::AFGR64RegClass);
3881       }
3882       break;
3883     case 'c': // register suitable for indirect jump
3884       if (VT == MVT::i32)
3885         return std::make_pair((unsigned)Mips::T9, &Mips::GPR32RegClass);
3886       if (VT == MVT::i64)
3887         return std::make_pair((unsigned)Mips::T9_64, &Mips::GPR64RegClass);
3888       // This will generate an error message
3889       return std::make_pair(0U, nullptr);
3890     case 'l': // use the `lo` register to store values
3891               // that are no bigger than a word
3892       if (VT == MVT::i32 || VT == MVT::i16 || VT == MVT::i8)
3893         return std::make_pair((unsigned)Mips::LO0, &Mips::LO32RegClass);
3894       return std::make_pair((unsigned)Mips::LO0_64, &Mips::LO64RegClass);
3895     case 'x': // use the concatenated `hi` and `lo` registers
3896               // to store doubleword values
3897       // Fixme: Not triggering the use of both hi and low
3898       // This will generate an error message
3899       return std::make_pair(0U, nullptr);
3900     }
3901   }
3902 
3903   std::pair<unsigned, const TargetRegisterClass *> R;
3904   R = parseRegForInlineAsmConstraint(Constraint, VT);
3905 
3906   if (R.second)
3907     return R;
3908 
3909   return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
3910 }
3911 
3912 /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
3913 /// vector.  If it is invalid, don't add anything to Ops.
3914 void MipsTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
3915                                                      std::string &Constraint,
3916                                                      std::vector<SDValue>&Ops,
3917                                                      SelectionDAG &DAG) const {
3918   SDLoc DL(Op);
3919   SDValue Result;
3920 
3921   // Only support length 1 constraints for now.
3922   if (Constraint.length() > 1) return;
3923 
3924   char ConstraintLetter = Constraint[0];
3925   switch (ConstraintLetter) {
3926   default: break; // This will fall through to the generic implementation
3927   case 'I': // Signed 16 bit constant
3928     // If this fails, the parent routine will give an error
3929     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
3930       EVT Type = Op.getValueType();
3931       int64_t Val = C->getSExtValue();
3932       if (isInt<16>(Val)) {
3933         Result = DAG.getTargetConstant(Val, DL, Type);
3934         break;
3935       }
3936     }
3937     return;
3938   case 'J': // integer zero
3939     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
3940       EVT Type = Op.getValueType();
3941       int64_t Val = C->getZExtValue();
3942       if (Val == 0) {
3943         Result = DAG.getTargetConstant(0, DL, Type);
3944         break;
3945       }
3946     }
3947     return;
3948   case 'K': // unsigned 16 bit immediate
3949     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
3950       EVT Type = Op.getValueType();
3951       uint64_t Val = (uint64_t)C->getZExtValue();
3952       if (isUInt<16>(Val)) {
3953         Result = DAG.getTargetConstant(Val, DL, Type);
3954         break;
3955       }
3956     }
3957     return;
3958   case 'L': // signed 32 bit immediate where lower 16 bits are 0
3959     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
3960       EVT Type = Op.getValueType();
3961       int64_t Val = C->getSExtValue();
3962       if ((isInt<32>(Val)) && ((Val & 0xffff) == 0)){
3963         Result = DAG.getTargetConstant(Val, DL, Type);
3964         break;
3965       }
3966     }
3967     return;
3968   case 'N': // immediate in the range of -65535 to -1 (inclusive)
3969     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
3970       EVT Type = Op.getValueType();
3971       int64_t Val = C->getSExtValue();
3972       if ((Val >= -65535) && (Val <= -1)) {
3973         Result = DAG.getTargetConstant(Val, DL, Type);
3974         break;
3975       }
3976     }
3977     return;
3978   case 'O': // signed 15 bit immediate
3979     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
3980       EVT Type = Op.getValueType();
3981       int64_t Val = C->getSExtValue();
3982       if ((isInt<15>(Val))) {
3983         Result = DAG.getTargetConstant(Val, DL, Type);
3984         break;
3985       }
3986     }
3987     return;
3988   case 'P': // immediate in the range of 1 to 65535 (inclusive)
3989     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
3990       EVT Type = Op.getValueType();
3991       int64_t Val = C->getSExtValue();
3992       if ((Val <= 65535) && (Val >= 1)) {
3993         Result = DAG.getTargetConstant(Val, DL, Type);
3994         break;
3995       }
3996     }
3997     return;
3998   }
3999 
4000   if (Result.getNode()) {
4001     Ops.push_back(Result);
4002     return;
4003   }
4004 
4005   TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
4006 }
4007 
4008 bool MipsTargetLowering::isLegalAddressingMode(const DataLayout &DL,
4009                                                const AddrMode &AM, Type *Ty,
4010                                                unsigned AS, Instruction *I) const {
4011   // No global is ever allowed as a base.
4012   if (AM.BaseGV)
4013     return false;
4014 
4015   switch (AM.Scale) {
4016   case 0: // "r+i" or just "i", depending on HasBaseReg.
4017     break;
4018   case 1:
4019     if (!AM.HasBaseReg) // allow "r+i".
4020       break;
4021     return false; // disallow "r+r" or "r+r+i".
4022   default:
4023     return false;
4024   }
4025 
4026   return true;
4027 }
4028 
4029 bool
4030 MipsTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
4031   // The Mips target isn't yet aware of offsets.
4032   return false;
4033 }
4034 
4035 EVT MipsTargetLowering::getOptimalMemOpType(uint64_t Size, unsigned DstAlign,
4036                                             unsigned SrcAlign,
4037                                             bool IsMemset, bool ZeroMemset,
4038                                             bool MemcpyStrSrc,
4039                                             MachineFunction &MF) const {
4040   if (Subtarget.hasMips64())
4041     return MVT::i64;
4042 
4043   return MVT::i32;
4044 }
4045 
4046 bool MipsTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const {
4047   if (VT != MVT::f32 && VT != MVT::f64)
4048     return false;
4049   if (Imm.isNegZero())
4050     return false;
4051   return Imm.isZero();
4052 }
4053 
4054 unsigned MipsTargetLowering::getJumpTableEncoding() const {
4055 
4056   // FIXME: For space reasons this should be: EK_GPRel32BlockAddress.
4057   if (ABI.IsN64() && isPositionIndependent())
4058     return MachineJumpTableInfo::EK_GPRel64BlockAddress;
4059 
4060   return TargetLowering::getJumpTableEncoding();
4061 }
4062 
4063 bool MipsTargetLowering::useSoftFloat() const {
4064   return Subtarget.useSoftFloat();
4065 }
4066 
4067 void MipsTargetLowering::copyByValRegs(
4068     SDValue Chain, const SDLoc &DL, std::vector<SDValue> &OutChains,
4069     SelectionDAG &DAG, const ISD::ArgFlagsTy &Flags,
4070     SmallVectorImpl<SDValue> &InVals, const Argument *FuncArg,
4071     unsigned FirstReg, unsigned LastReg, const CCValAssign &VA,
4072     MipsCCState &State) const {
4073   MachineFunction &MF = DAG.getMachineFunction();
4074   MachineFrameInfo &MFI = MF.getFrameInfo();
4075   unsigned GPRSizeInBytes = Subtarget.getGPRSizeInBytes();
4076   unsigned NumRegs = LastReg - FirstReg;
4077   unsigned RegAreaSize = NumRegs * GPRSizeInBytes;
4078   unsigned FrameObjSize = std::max(Flags.getByValSize(), RegAreaSize);
4079   int FrameObjOffset;
4080   ArrayRef<MCPhysReg> ByValArgRegs = ABI.GetByValArgRegs();
4081 
4082   if (RegAreaSize)
4083     FrameObjOffset =
4084         (int)ABI.GetCalleeAllocdArgSizeInBytes(State.getCallingConv()) -
4085         (int)((ByValArgRegs.size() - FirstReg) * GPRSizeInBytes);
4086   else
4087     FrameObjOffset = VA.getLocMemOffset();
4088 
4089   // Create frame object.
4090   EVT PtrTy = getPointerTy(DAG.getDataLayout());
4091   // Make the fixed object stored to mutable so that the load instructions
4092   // referencing it have their memory dependencies added.
4093   // Set the frame object as isAliased which clears the underlying objects
4094   // vector in ScheduleDAGInstrs::buildSchedGraph() resulting in addition of all
4095   // stores as dependencies for loads referencing this fixed object.
4096   int FI = MFI.CreateFixedObject(FrameObjSize, FrameObjOffset, false, true);
4097   SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
4098   InVals.push_back(FIN);
4099 
4100   if (!NumRegs)
4101     return;
4102 
4103   // Copy arg registers.
4104   MVT RegTy = MVT::getIntegerVT(GPRSizeInBytes * 8);
4105   const TargetRegisterClass *RC = getRegClassFor(RegTy);
4106 
4107   for (unsigned I = 0; I < NumRegs; ++I) {
4108     unsigned ArgReg = ByValArgRegs[FirstReg + I];
4109     unsigned VReg = addLiveIn(MF, ArgReg, RC);
4110     unsigned Offset = I * GPRSizeInBytes;
4111     SDValue StorePtr = DAG.getNode(ISD::ADD, DL, PtrTy, FIN,
4112                                    DAG.getConstant(Offset, DL, PtrTy));
4113     SDValue Store = DAG.getStore(Chain, DL, DAG.getRegister(VReg, RegTy),
4114                                  StorePtr, MachinePointerInfo(FuncArg, Offset));
4115     OutChains.push_back(Store);
4116   }
4117 }
4118 
4119 // Copy byVal arg to registers and stack.
4120 void MipsTargetLowering::passByValArg(
4121     SDValue Chain, const SDLoc &DL,
4122     std::deque<std::pair<unsigned, SDValue>> &RegsToPass,
4123     SmallVectorImpl<SDValue> &MemOpChains, SDValue StackPtr,
4124     MachineFrameInfo &MFI, SelectionDAG &DAG, SDValue Arg, unsigned FirstReg,
4125     unsigned LastReg, const ISD::ArgFlagsTy &Flags, bool isLittle,
4126     const CCValAssign &VA) const {
4127   unsigned ByValSizeInBytes = Flags.getByValSize();
4128   unsigned OffsetInBytes = 0; // From beginning of struct
4129   unsigned RegSizeInBytes = Subtarget.getGPRSizeInBytes();
4130   unsigned Alignment = std::min(Flags.getByValAlign(), RegSizeInBytes);
4131   EVT PtrTy = getPointerTy(DAG.getDataLayout()),
4132       RegTy = MVT::getIntegerVT(RegSizeInBytes * 8);
4133   unsigned NumRegs = LastReg - FirstReg;
4134 
4135   if (NumRegs) {
4136     ArrayRef<MCPhysReg> ArgRegs = ABI.GetByValArgRegs();
4137     bool LeftoverBytes = (NumRegs * RegSizeInBytes > ByValSizeInBytes);
4138     unsigned I = 0;
4139 
4140     // Copy words to registers.
4141     for (; I < NumRegs - LeftoverBytes; ++I, OffsetInBytes += RegSizeInBytes) {
4142       SDValue LoadPtr = DAG.getNode(ISD::ADD, DL, PtrTy, Arg,
4143                                     DAG.getConstant(OffsetInBytes, DL, PtrTy));
4144       SDValue LoadVal = DAG.getLoad(RegTy, DL, Chain, LoadPtr,
4145                                     MachinePointerInfo(), Alignment);
4146       MemOpChains.push_back(LoadVal.getValue(1));
4147       unsigned ArgReg = ArgRegs[FirstReg + I];
4148       RegsToPass.push_back(std::make_pair(ArgReg, LoadVal));
4149     }
4150 
4151     // Return if the struct has been fully copied.
4152     if (ByValSizeInBytes == OffsetInBytes)
4153       return;
4154 
4155     // Copy the remainder of the byval argument with sub-word loads and shifts.
4156     if (LeftoverBytes) {
4157       SDValue Val;
4158 
4159       for (unsigned LoadSizeInBytes = RegSizeInBytes / 2, TotalBytesLoaded = 0;
4160            OffsetInBytes < ByValSizeInBytes; LoadSizeInBytes /= 2) {
4161         unsigned RemainingSizeInBytes = ByValSizeInBytes - OffsetInBytes;
4162 
4163         if (RemainingSizeInBytes < LoadSizeInBytes)
4164           continue;
4165 
4166         // Load subword.
4167         SDValue LoadPtr = DAG.getNode(ISD::ADD, DL, PtrTy, Arg,
4168                                       DAG.getConstant(OffsetInBytes, DL,
4169                                                       PtrTy));
4170         SDValue LoadVal = DAG.getExtLoad(
4171             ISD::ZEXTLOAD, DL, RegTy, Chain, LoadPtr, MachinePointerInfo(),
4172             MVT::getIntegerVT(LoadSizeInBytes * 8), Alignment);
4173         MemOpChains.push_back(LoadVal.getValue(1));
4174 
4175         // Shift the loaded value.
4176         unsigned Shamt;
4177 
4178         if (isLittle)
4179           Shamt = TotalBytesLoaded * 8;
4180         else
4181           Shamt = (RegSizeInBytes - (TotalBytesLoaded + LoadSizeInBytes)) * 8;
4182 
4183         SDValue Shift = DAG.getNode(ISD::SHL, DL, RegTy, LoadVal,
4184                                     DAG.getConstant(Shamt, DL, MVT::i32));
4185 
4186         if (Val.getNode())
4187           Val = DAG.getNode(ISD::OR, DL, RegTy, Val, Shift);
4188         else
4189           Val = Shift;
4190 
4191         OffsetInBytes += LoadSizeInBytes;
4192         TotalBytesLoaded += LoadSizeInBytes;
4193         Alignment = std::min(Alignment, LoadSizeInBytes);
4194       }
4195 
4196       unsigned ArgReg = ArgRegs[FirstReg + I];
4197       RegsToPass.push_back(std::make_pair(ArgReg, Val));
4198       return;
4199     }
4200   }
4201 
4202   // Copy remainder of byval arg to it with memcpy.
4203   unsigned MemCpySize = ByValSizeInBytes - OffsetInBytes;
4204   SDValue Src = DAG.getNode(ISD::ADD, DL, PtrTy, Arg,
4205                             DAG.getConstant(OffsetInBytes, DL, PtrTy));
4206   SDValue Dst = DAG.getNode(ISD::ADD, DL, PtrTy, StackPtr,
4207                             DAG.getIntPtrConstant(VA.getLocMemOffset(), DL));
4208   Chain = DAG.getMemcpy(Chain, DL, Dst, Src,
4209                         DAG.getConstant(MemCpySize, DL, PtrTy),
4210                         Alignment, /*isVolatile=*/false, /*AlwaysInline=*/false,
4211                         /*isTailCall=*/false,
4212                         MachinePointerInfo(), MachinePointerInfo());
4213   MemOpChains.push_back(Chain);
4214 }
4215 
4216 void MipsTargetLowering::writeVarArgRegs(std::vector<SDValue> &OutChains,
4217                                          SDValue Chain, const SDLoc &DL,
4218                                          SelectionDAG &DAG,
4219                                          CCState &State) const {
4220   ArrayRef<MCPhysReg> ArgRegs = ABI.GetVarArgRegs();
4221   unsigned Idx = State.getFirstUnallocated(ArgRegs);
4222   unsigned RegSizeInBytes = Subtarget.getGPRSizeInBytes();
4223   MVT RegTy = MVT::getIntegerVT(RegSizeInBytes * 8);
4224   const TargetRegisterClass *RC = getRegClassFor(RegTy);
4225   MachineFunction &MF = DAG.getMachineFunction();
4226   MachineFrameInfo &MFI = MF.getFrameInfo();
4227   MipsFunctionInfo *MipsFI = MF.getInfo<MipsFunctionInfo>();
4228 
4229   // Offset of the first variable argument from stack pointer.
4230   int VaArgOffset;
4231 
4232   if (ArgRegs.size() == Idx)
4233     VaArgOffset = alignTo(State.getNextStackOffset(), RegSizeInBytes);
4234   else {
4235     VaArgOffset =
4236         (int)ABI.GetCalleeAllocdArgSizeInBytes(State.getCallingConv()) -
4237         (int)(RegSizeInBytes * (ArgRegs.size() - Idx));
4238   }
4239 
4240   // Record the frame index of the first variable argument
4241   // which is a value necessary to VASTART.
4242   int FI = MFI.CreateFixedObject(RegSizeInBytes, VaArgOffset, true);
4243   MipsFI->setVarArgsFrameIndex(FI);
4244 
4245   // Copy the integer registers that have not been used for argument passing
4246   // to the argument register save area. For O32, the save area is allocated
4247   // in the caller's stack frame, while for N32/64, it is allocated in the
4248   // callee's stack frame.
4249   for (unsigned I = Idx; I < ArgRegs.size();
4250        ++I, VaArgOffset += RegSizeInBytes) {
4251     unsigned Reg = addLiveIn(MF, ArgRegs[I], RC);
4252     SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, RegTy);
4253     FI = MFI.CreateFixedObject(RegSizeInBytes, VaArgOffset, true);
4254     SDValue PtrOff = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
4255     SDValue Store =
4256         DAG.getStore(Chain, DL, ArgValue, PtrOff, MachinePointerInfo());
4257     cast<StoreSDNode>(Store.getNode())->getMemOperand()->setValue(
4258         (Value *)nullptr);
4259     OutChains.push_back(Store);
4260   }
4261 }
4262 
4263 void MipsTargetLowering::HandleByVal(CCState *State, unsigned &Size,
4264                                      unsigned Align) const {
4265   const TargetFrameLowering *TFL = Subtarget.getFrameLowering();
4266 
4267   assert(Size && "Byval argument's size shouldn't be 0.");
4268 
4269   Align = std::min(Align, TFL->getStackAlignment());
4270 
4271   unsigned FirstReg = 0;
4272   unsigned NumRegs = 0;
4273 
4274   if (State->getCallingConv() != CallingConv::Fast) {
4275     unsigned RegSizeInBytes = Subtarget.getGPRSizeInBytes();
4276     ArrayRef<MCPhysReg> IntArgRegs = ABI.GetByValArgRegs();
4277     // FIXME: The O32 case actually describes no shadow registers.
4278     const MCPhysReg *ShadowRegs =
4279         ABI.IsO32() ? IntArgRegs.data() : Mips64DPRegs;
4280 
4281     // We used to check the size as well but we can't do that anymore since
4282     // CCState::HandleByVal() rounds up the size after calling this function.
4283     assert(!(Align % RegSizeInBytes) &&
4284            "Byval argument's alignment should be a multiple of"
4285            "RegSizeInBytes.");
4286 
4287     FirstReg = State->getFirstUnallocated(IntArgRegs);
4288 
4289     // If Align > RegSizeInBytes, the first arg register must be even.
4290     // FIXME: This condition happens to do the right thing but it's not the
4291     //        right way to test it. We want to check that the stack frame offset
4292     //        of the register is aligned.
4293     if ((Align > RegSizeInBytes) && (FirstReg % 2)) {
4294       State->AllocateReg(IntArgRegs[FirstReg], ShadowRegs[FirstReg]);
4295       ++FirstReg;
4296     }
4297 
4298     // Mark the registers allocated.
4299     Size = alignTo(Size, RegSizeInBytes);
4300     for (unsigned I = FirstReg; Size > 0 && (I < IntArgRegs.size());
4301          Size -= RegSizeInBytes, ++I, ++NumRegs)
4302       State->AllocateReg(IntArgRegs[I], ShadowRegs[I]);
4303   }
4304 
4305   State->addInRegsParamInfo(FirstReg, FirstReg + NumRegs);
4306 }
4307 
4308 MachineBasicBlock *MipsTargetLowering::emitPseudoSELECT(MachineInstr &MI,
4309                                                         MachineBasicBlock *BB,
4310                                                         bool isFPCmp,
4311                                                         unsigned Opc) const {
4312   assert(!(Subtarget.hasMips4() || Subtarget.hasMips32()) &&
4313          "Subtarget already supports SELECT nodes with the use of"
4314          "conditional-move instructions.");
4315 
4316   const TargetInstrInfo *TII =
4317       Subtarget.getInstrInfo();
4318   DebugLoc DL = MI.getDebugLoc();
4319 
4320   // To "insert" a SELECT instruction, we actually have to insert the
4321   // diamond control-flow pattern.  The incoming instruction knows the
4322   // destination vreg to set, the condition code register to branch on, the
4323   // true/false values to select between, and a branch opcode to use.
4324   const BasicBlock *LLVM_BB = BB->getBasicBlock();
4325   MachineFunction::iterator It = ++BB->getIterator();
4326 
4327   //  thisMBB:
4328   //  ...
4329   //   TrueVal = ...
4330   //   setcc r1, r2, r3
4331   //   bNE   r1, r0, copy1MBB
4332   //   fallthrough --> copy0MBB
4333   MachineBasicBlock *thisMBB  = BB;
4334   MachineFunction *F = BB->getParent();
4335   MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB);
4336   MachineBasicBlock *sinkMBB  = F->CreateMachineBasicBlock(LLVM_BB);
4337   F->insert(It, copy0MBB);
4338   F->insert(It, sinkMBB);
4339 
4340   // Transfer the remainder of BB and its successor edges to sinkMBB.
4341   sinkMBB->splice(sinkMBB->begin(), BB,
4342                   std::next(MachineBasicBlock::iterator(MI)), BB->end());
4343   sinkMBB->transferSuccessorsAndUpdatePHIs(BB);
4344 
4345   // Next, add the true and fallthrough blocks as its successors.
4346   BB->addSuccessor(copy0MBB);
4347   BB->addSuccessor(sinkMBB);
4348 
4349   if (isFPCmp) {
4350     // bc1[tf] cc, sinkMBB
4351     BuildMI(BB, DL, TII->get(Opc))
4352         .addReg(MI.getOperand(1).getReg())
4353         .addMBB(sinkMBB);
4354   } else {
4355     // bne rs, $0, sinkMBB
4356     BuildMI(BB, DL, TII->get(Opc))
4357         .addReg(MI.getOperand(1).getReg())
4358         .addReg(Mips::ZERO)
4359         .addMBB(sinkMBB);
4360   }
4361 
4362   //  copy0MBB:
4363   //   %FalseValue = ...
4364   //   # fallthrough to sinkMBB
4365   BB = copy0MBB;
4366 
4367   // Update machine-CFG edges
4368   BB->addSuccessor(sinkMBB);
4369 
4370   //  sinkMBB:
4371   //   %Result = phi [ %TrueValue, thisMBB ], [ %FalseValue, copy0MBB ]
4372   //  ...
4373   BB = sinkMBB;
4374 
4375   BuildMI(*BB, BB->begin(), DL, TII->get(Mips::PHI), MI.getOperand(0).getReg())
4376       .addReg(MI.getOperand(2).getReg())
4377       .addMBB(thisMBB)
4378       .addReg(MI.getOperand(3).getReg())
4379       .addMBB(copy0MBB);
4380 
4381   MI.eraseFromParent(); // The pseudo instruction is gone now.
4382 
4383   return BB;
4384 }
4385 
4386 // FIXME? Maybe this could be a TableGen attribute on some registers and
4387 // this table could be generated automatically from RegInfo.
4388 unsigned MipsTargetLowering::getRegisterByName(const char* RegName, EVT VT,
4389                                                SelectionDAG &DAG) const {
4390   // Named registers is expected to be fairly rare. For now, just support $28
4391   // since the linux kernel uses it.
4392   if (Subtarget.isGP64bit()) {
4393     unsigned Reg = StringSwitch<unsigned>(RegName)
4394                          .Case("$28", Mips::GP_64)
4395                          .Default(0);
4396     if (Reg)
4397       return Reg;
4398   } else {
4399     unsigned Reg = StringSwitch<unsigned>(RegName)
4400                          .Case("$28", Mips::GP)
4401                          .Default(0);
4402     if (Reg)
4403       return Reg;
4404   }
4405   report_fatal_error("Invalid register name global variable");
4406 }
4407