1 //===- AArch64InstructionSelector.cpp ----------------------------*- C++ -*-==//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 /// \file
10 /// This file implements the targeting of the InstructionSelector class for
11 /// AArch64.
12 /// \todo This should be generated by TableGen.
13 //===----------------------------------------------------------------------===//
14 
15 #include "AArch64InstrInfo.h"
16 #include "AArch64MachineFunctionInfo.h"
17 #include "AArch64RegisterBankInfo.h"
18 #include "AArch64RegisterInfo.h"
19 #include "AArch64Subtarget.h"
20 #include "AArch64TargetMachine.h"
21 #include "MCTargetDesc/AArch64AddressingModes.h"
22 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
23 #include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h"
24 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
25 #include "llvm/CodeGen/GlobalISel/Utils.h"
26 #include "llvm/CodeGen/MachineBasicBlock.h"
27 #include "llvm/CodeGen/MachineFunction.h"
28 #include "llvm/CodeGen/MachineInstr.h"
29 #include "llvm/CodeGen/MachineInstrBuilder.h"
30 #include "llvm/CodeGen/MachineOperand.h"
31 #include "llvm/CodeGen/MachineRegisterInfo.h"
32 #include "llvm/IR/Type.h"
33 #include "llvm/Support/Debug.h"
34 #include "llvm/Support/raw_ostream.h"
35 
36 #define DEBUG_TYPE "aarch64-isel"
37 
38 using namespace llvm;
39 
40 namespace {
41 
42 #define GET_GLOBALISEL_PREDICATE_BITSET
43 #include "AArch64GenGlobalISel.inc"
44 #undef GET_GLOBALISEL_PREDICATE_BITSET
45 
46 class AArch64InstructionSelector : public InstructionSelector {
47 public:
48   AArch64InstructionSelector(const AArch64TargetMachine &TM,
49                              const AArch64Subtarget &STI,
50                              const AArch64RegisterBankInfo &RBI);
51 
52   bool select(MachineInstr &I, CodeGenCoverage &CoverageInfo) const override;
getName()53   static const char *getName() { return DEBUG_TYPE; }
54 
55 private:
56   /// tblgen-erated 'select' implementation, used as the initial selector for
57   /// the patterns that don't require complex C++.
58   bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
59 
60   bool selectVaStartAAPCS(MachineInstr &I, MachineFunction &MF,
61                           MachineRegisterInfo &MRI) const;
62   bool selectVaStartDarwin(MachineInstr &I, MachineFunction &MF,
63                            MachineRegisterInfo &MRI) const;
64 
65   bool selectCompareBranch(MachineInstr &I, MachineFunction &MF,
66                            MachineRegisterInfo &MRI) const;
67 
68   // Helper to generate an equivalent of scalar_to_vector into a new register,
69   // returned via 'Dst'.
70   bool emitScalarToVector(unsigned &Dst, const LLT DstTy,
71                           const TargetRegisterClass *DstRC, unsigned Scalar,
72                           MachineBasicBlock &MBB,
73                           MachineBasicBlock::iterator MBBI,
74                           MachineRegisterInfo &MRI) const;
75   bool selectBuildVector(MachineInstr &I, MachineRegisterInfo &MRI) const;
76   bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI) const;
77 
78   ComplexRendererFns selectArithImmed(MachineOperand &Root) const;
79 
80   ComplexRendererFns selectAddrModeUnscaled(MachineOperand &Root,
81                                             unsigned Size) const;
82 
selectAddrModeUnscaled8(MachineOperand & Root) const83   ComplexRendererFns selectAddrModeUnscaled8(MachineOperand &Root) const {
84     return selectAddrModeUnscaled(Root, 1);
85   }
selectAddrModeUnscaled16(MachineOperand & Root) const86   ComplexRendererFns selectAddrModeUnscaled16(MachineOperand &Root) const {
87     return selectAddrModeUnscaled(Root, 2);
88   }
selectAddrModeUnscaled32(MachineOperand & Root) const89   ComplexRendererFns selectAddrModeUnscaled32(MachineOperand &Root) const {
90     return selectAddrModeUnscaled(Root, 4);
91   }
selectAddrModeUnscaled64(MachineOperand & Root) const92   ComplexRendererFns selectAddrModeUnscaled64(MachineOperand &Root) const {
93     return selectAddrModeUnscaled(Root, 8);
94   }
selectAddrModeUnscaled128(MachineOperand & Root) const95   ComplexRendererFns selectAddrModeUnscaled128(MachineOperand &Root) const {
96     return selectAddrModeUnscaled(Root, 16);
97   }
98 
99   ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root,
100                                            unsigned Size) const;
101   template <int Width>
selectAddrModeIndexed(MachineOperand & Root) const102   ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root) const {
103     return selectAddrModeIndexed(Root, Width / 8);
104   }
105 
106   void renderTruncImm(MachineInstrBuilder &MIB, const MachineInstr &MI) const;
107 
108   // Materialize a GlobalValue or BlockAddress using a movz+movk sequence.
109   void materializeLargeCMVal(MachineInstr &I, const Value *V,
110                              unsigned char OpFlags) const;
111 
112   const AArch64TargetMachine &TM;
113   const AArch64Subtarget &STI;
114   const AArch64InstrInfo &TII;
115   const AArch64RegisterInfo &TRI;
116   const AArch64RegisterBankInfo &RBI;
117 
118 #define GET_GLOBALISEL_PREDICATES_DECL
119 #include "AArch64GenGlobalISel.inc"
120 #undef GET_GLOBALISEL_PREDICATES_DECL
121 
122 // We declare the temporaries used by selectImpl() in the class to minimize the
123 // cost of constructing placeholder values.
124 #define GET_GLOBALISEL_TEMPORARIES_DECL
125 #include "AArch64GenGlobalISel.inc"
126 #undef GET_GLOBALISEL_TEMPORARIES_DECL
127 };
128 
129 } // end anonymous namespace
130 
131 #define GET_GLOBALISEL_IMPL
132 #include "AArch64GenGlobalISel.inc"
133 #undef GET_GLOBALISEL_IMPL
134 
AArch64InstructionSelector(const AArch64TargetMachine & TM,const AArch64Subtarget & STI,const AArch64RegisterBankInfo & RBI)135 AArch64InstructionSelector::AArch64InstructionSelector(
136     const AArch64TargetMachine &TM, const AArch64Subtarget &STI,
137     const AArch64RegisterBankInfo &RBI)
138     : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()),
139       TRI(*STI.getRegisterInfo()), RBI(RBI),
140 #define GET_GLOBALISEL_PREDICATES_INIT
141 #include "AArch64GenGlobalISel.inc"
142 #undef GET_GLOBALISEL_PREDICATES_INIT
143 #define GET_GLOBALISEL_TEMPORARIES_INIT
144 #include "AArch64GenGlobalISel.inc"
145 #undef GET_GLOBALISEL_TEMPORARIES_INIT
146 {
147 }
148 
149 // FIXME: This should be target-independent, inferred from the types declared
150 // for each class in the bank.
151 static const TargetRegisterClass *
getRegClassForTypeOnBank(LLT Ty,const RegisterBank & RB,const RegisterBankInfo & RBI,bool GetAllRegSet=false)152 getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB,
153                          const RegisterBankInfo &RBI,
154                          bool GetAllRegSet = false) {
155   if (RB.getID() == AArch64::GPRRegBankID) {
156     if (Ty.getSizeInBits() <= 32)
157       return GetAllRegSet ? &AArch64::GPR32allRegClass
158                           : &AArch64::GPR32RegClass;
159     if (Ty.getSizeInBits() == 64)
160       return GetAllRegSet ? &AArch64::GPR64allRegClass
161                           : &AArch64::GPR64RegClass;
162     return nullptr;
163   }
164 
165   if (RB.getID() == AArch64::FPRRegBankID) {
166     if (Ty.getSizeInBits() <= 16)
167       return &AArch64::FPR16RegClass;
168     if (Ty.getSizeInBits() == 32)
169       return &AArch64::FPR32RegClass;
170     if (Ty.getSizeInBits() == 64)
171       return &AArch64::FPR64RegClass;
172     if (Ty.getSizeInBits() == 128)
173       return &AArch64::FPR128RegClass;
174     return nullptr;
175   }
176 
177   return nullptr;
178 }
179 
180 /// Check whether \p I is a currently unsupported binary operation:
181 /// - it has an unsized type
182 /// - an operand is not a vreg
183 /// - all operands are not in the same bank
184 /// These are checks that should someday live in the verifier, but right now,
185 /// these are mostly limitations of the aarch64 selector.
unsupportedBinOp(const MachineInstr & I,const AArch64RegisterBankInfo & RBI,const MachineRegisterInfo & MRI,const AArch64RegisterInfo & TRI)186 static bool unsupportedBinOp(const MachineInstr &I,
187                              const AArch64RegisterBankInfo &RBI,
188                              const MachineRegisterInfo &MRI,
189                              const AArch64RegisterInfo &TRI) {
190   LLT Ty = MRI.getType(I.getOperand(0).getReg());
191   if (!Ty.isValid()) {
192     LLVM_DEBUG(dbgs() << "Generic binop register should be typed\n");
193     return true;
194   }
195 
196   const RegisterBank *PrevOpBank = nullptr;
197   for (auto &MO : I.operands()) {
198     // FIXME: Support non-register operands.
199     if (!MO.isReg()) {
200       LLVM_DEBUG(dbgs() << "Generic inst non-reg operands are unsupported\n");
201       return true;
202     }
203 
204     // FIXME: Can generic operations have physical registers operands? If
205     // so, this will need to be taught about that, and we'll need to get the
206     // bank out of the minimal class for the register.
207     // Either way, this needs to be documented (and possibly verified).
208     if (!TargetRegisterInfo::isVirtualRegister(MO.getReg())) {
209       LLVM_DEBUG(dbgs() << "Generic inst has physical register operand\n");
210       return true;
211     }
212 
213     const RegisterBank *OpBank = RBI.getRegBank(MO.getReg(), MRI, TRI);
214     if (!OpBank) {
215       LLVM_DEBUG(dbgs() << "Generic register has no bank or class\n");
216       return true;
217     }
218 
219     if (PrevOpBank && OpBank != PrevOpBank) {
220       LLVM_DEBUG(dbgs() << "Generic inst operands have different banks\n");
221       return true;
222     }
223     PrevOpBank = OpBank;
224   }
225   return false;
226 }
227 
228 /// Select the AArch64 opcode for the basic binary operation \p GenericOpc
229 /// (such as G_OR or G_SDIV), appropriate for the register bank \p RegBankID
230 /// and of size \p OpSize.
231 /// \returns \p GenericOpc if the combination is unsupported.
selectBinaryOp(unsigned GenericOpc,unsigned RegBankID,unsigned OpSize)232 static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID,
233                                unsigned OpSize) {
234   switch (RegBankID) {
235   case AArch64::GPRRegBankID:
236     if (OpSize == 32) {
237       switch (GenericOpc) {
238       case TargetOpcode::G_SHL:
239         return AArch64::LSLVWr;
240       case TargetOpcode::G_LSHR:
241         return AArch64::LSRVWr;
242       case TargetOpcode::G_ASHR:
243         return AArch64::ASRVWr;
244       default:
245         return GenericOpc;
246       }
247     } else if (OpSize == 64) {
248       switch (GenericOpc) {
249       case TargetOpcode::G_GEP:
250         return AArch64::ADDXrr;
251       case TargetOpcode::G_SHL:
252         return AArch64::LSLVXr;
253       case TargetOpcode::G_LSHR:
254         return AArch64::LSRVXr;
255       case TargetOpcode::G_ASHR:
256         return AArch64::ASRVXr;
257       default:
258         return GenericOpc;
259       }
260     }
261     break;
262   case AArch64::FPRRegBankID:
263     switch (OpSize) {
264     case 32:
265       switch (GenericOpc) {
266       case TargetOpcode::G_FADD:
267         return AArch64::FADDSrr;
268       case TargetOpcode::G_FSUB:
269         return AArch64::FSUBSrr;
270       case TargetOpcode::G_FMUL:
271         return AArch64::FMULSrr;
272       case TargetOpcode::G_FDIV:
273         return AArch64::FDIVSrr;
274       default:
275         return GenericOpc;
276       }
277     case 64:
278       switch (GenericOpc) {
279       case TargetOpcode::G_FADD:
280         return AArch64::FADDDrr;
281       case TargetOpcode::G_FSUB:
282         return AArch64::FSUBDrr;
283       case TargetOpcode::G_FMUL:
284         return AArch64::FMULDrr;
285       case TargetOpcode::G_FDIV:
286         return AArch64::FDIVDrr;
287       case TargetOpcode::G_OR:
288         return AArch64::ORRv8i8;
289       default:
290         return GenericOpc;
291       }
292     }
293     break;
294   }
295   return GenericOpc;
296 }
297 
298 /// Select the AArch64 opcode for the G_LOAD or G_STORE operation \p GenericOpc,
299 /// appropriate for the (value) register bank \p RegBankID and of memory access
300 /// size \p OpSize.  This returns the variant with the base+unsigned-immediate
301 /// addressing mode (e.g., LDRXui).
302 /// \returns \p GenericOpc if the combination is unsupported.
selectLoadStoreUIOp(unsigned GenericOpc,unsigned RegBankID,unsigned OpSize)303 static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID,
304                                     unsigned OpSize) {
305   const bool isStore = GenericOpc == TargetOpcode::G_STORE;
306   switch (RegBankID) {
307   case AArch64::GPRRegBankID:
308     switch (OpSize) {
309     case 8:
310       return isStore ? AArch64::STRBBui : AArch64::LDRBBui;
311     case 16:
312       return isStore ? AArch64::STRHHui : AArch64::LDRHHui;
313     case 32:
314       return isStore ? AArch64::STRWui : AArch64::LDRWui;
315     case 64:
316       return isStore ? AArch64::STRXui : AArch64::LDRXui;
317     }
318     break;
319   case AArch64::FPRRegBankID:
320     switch (OpSize) {
321     case 8:
322       return isStore ? AArch64::STRBui : AArch64::LDRBui;
323     case 16:
324       return isStore ? AArch64::STRHui : AArch64::LDRHui;
325     case 32:
326       return isStore ? AArch64::STRSui : AArch64::LDRSui;
327     case 64:
328       return isStore ? AArch64::STRDui : AArch64::LDRDui;
329     }
330     break;
331   }
332   return GenericOpc;
333 }
334 
selectFP16CopyFromGPR32(MachineInstr & I,const TargetInstrInfo & TII,MachineRegisterInfo & MRI,unsigned SrcReg)335 static bool selectFP16CopyFromGPR32(MachineInstr &I, const TargetInstrInfo &TII,
336                                     MachineRegisterInfo &MRI, unsigned SrcReg) {
337   // Copies from gpr32 to fpr16 need to use a sub-register copy.
338   unsigned CopyReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass);
339   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::COPY))
340       .addDef(CopyReg)
341       .addUse(SrcReg);
342   unsigned SubRegCopy = MRI.createVirtualRegister(&AArch64::FPR16RegClass);
343   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY))
344       .addDef(SubRegCopy)
345       .addUse(CopyReg, 0, AArch64::hsub);
346 
347   MachineOperand &RegOp = I.getOperand(1);
348   RegOp.setReg(SubRegCopy);
349   return true;
350 }
351 
selectCopy(MachineInstr & I,const TargetInstrInfo & TII,MachineRegisterInfo & MRI,const TargetRegisterInfo & TRI,const RegisterBankInfo & RBI)352 static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
353                        MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
354                        const RegisterBankInfo &RBI) {
355 
356   unsigned DstReg = I.getOperand(0).getReg();
357   unsigned SrcReg = I.getOperand(1).getReg();
358 
359   if (TargetRegisterInfo::isPhysicalRegister(DstReg)) {
360     if (TRI.getRegClass(AArch64::FPR16RegClassID)->contains(DstReg) &&
361         !TargetRegisterInfo::isPhysicalRegister(SrcReg)) {
362       const RegisterBank &RegBank = *RBI.getRegBank(SrcReg, MRI, TRI);
363       const TargetRegisterClass *SrcRC = getRegClassForTypeOnBank(
364           MRI.getType(SrcReg), RegBank, RBI, /* GetAllRegSet */ true);
365       if (SrcRC == &AArch64::GPR32allRegClass)
366         return selectFP16CopyFromGPR32(I, TII, MRI, SrcReg);
367     }
368     assert(I.isCopy() && "Generic operators do not allow physical registers");
369     return true;
370   }
371 
372   const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
373   const unsigned DstSize = MRI.getType(DstReg).getSizeInBits();
374   (void)DstSize;
375   const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);
376   (void)SrcSize;
377   assert((!TargetRegisterInfo::isPhysicalRegister(SrcReg) || I.isCopy()) &&
378          "No phys reg on generic operators");
379   assert(
380       (DstSize == SrcSize ||
381        // Copies are a mean to setup initial types, the number of
382        // bits may not exactly match.
383        (TargetRegisterInfo::isPhysicalRegister(SrcReg) &&
384         DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI)) ||
385        // Copies are a mean to copy bits around, as long as we are
386        // on the same register class, that's fine. Otherwise, that
387        // means we need some SUBREG_TO_REG or AND & co.
388        (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) &&
389       "Copy with different width?!");
390   assert((DstSize <= 64 || RegBank.getID() == AArch64::FPRRegBankID) &&
391          "GPRs cannot get more than 64-bit width values");
392 
393   const TargetRegisterClass *RC = getRegClassForTypeOnBank(
394       MRI.getType(DstReg), RegBank, RBI, /* GetAllRegSet */ true);
395   if (!RC) {
396     LLVM_DEBUG(dbgs() << "Unexpected bitcast size " << DstSize << '\n');
397     return false;
398   }
399 
400   if (!TargetRegisterInfo::isPhysicalRegister(SrcReg)) {
401     const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(SrcReg);
402     const TargetRegisterClass *SrcRC =
403         RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
404     const RegisterBank *RB = nullptr;
405     if (!SrcRC) {
406       RB = RegClassOrBank.get<const RegisterBank *>();
407       SrcRC = getRegClassForTypeOnBank(MRI.getType(SrcReg), *RB, RBI, true);
408     }
409     // Copies from fpr16 to gpr32 need to use SUBREG_TO_REG.
410     if (RC == &AArch64::GPR32allRegClass && SrcRC == &AArch64::FPR16RegClass) {
411       unsigned PromoteReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass);
412       BuildMI(*I.getParent(), I, I.getDebugLoc(),
413               TII.get(AArch64::SUBREG_TO_REG))
414           .addDef(PromoteReg)
415           .addImm(0)
416           .addUse(SrcReg)
417           .addImm(AArch64::hsub);
418       MachineOperand &RegOp = I.getOperand(1);
419       RegOp.setReg(PromoteReg);
420     } else if (RC == &AArch64::FPR16RegClass &&
421                SrcRC == &AArch64::GPR32allRegClass) {
422       selectFP16CopyFromGPR32(I, TII, MRI, SrcReg);
423     }
424   }
425 
426   // No need to constrain SrcReg. It will get constrained when
427   // we hit another of its use or its defs.
428   // Copies do not have constraints.
429   if (!RBI.constrainGenericRegister(DstReg, *RC, MRI)) {
430     LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
431                       << " operand\n");
432     return false;
433   }
434   I.setDesc(TII.get(AArch64::COPY));
435   return true;
436 }
437 
selectFPConvOpc(unsigned GenericOpc,LLT DstTy,LLT SrcTy)438 static unsigned selectFPConvOpc(unsigned GenericOpc, LLT DstTy, LLT SrcTy) {
439   if (!DstTy.isScalar() || !SrcTy.isScalar())
440     return GenericOpc;
441 
442   const unsigned DstSize = DstTy.getSizeInBits();
443   const unsigned SrcSize = SrcTy.getSizeInBits();
444 
445   switch (DstSize) {
446   case 32:
447     switch (SrcSize) {
448     case 32:
449       switch (GenericOpc) {
450       case TargetOpcode::G_SITOFP:
451         return AArch64::SCVTFUWSri;
452       case TargetOpcode::G_UITOFP:
453         return AArch64::UCVTFUWSri;
454       case TargetOpcode::G_FPTOSI:
455         return AArch64::FCVTZSUWSr;
456       case TargetOpcode::G_FPTOUI:
457         return AArch64::FCVTZUUWSr;
458       default:
459         return GenericOpc;
460       }
461     case 64:
462       switch (GenericOpc) {
463       case TargetOpcode::G_SITOFP:
464         return AArch64::SCVTFUXSri;
465       case TargetOpcode::G_UITOFP:
466         return AArch64::UCVTFUXSri;
467       case TargetOpcode::G_FPTOSI:
468         return AArch64::FCVTZSUWDr;
469       case TargetOpcode::G_FPTOUI:
470         return AArch64::FCVTZUUWDr;
471       default:
472         return GenericOpc;
473       }
474     default:
475       return GenericOpc;
476     }
477   case 64:
478     switch (SrcSize) {
479     case 32:
480       switch (GenericOpc) {
481       case TargetOpcode::G_SITOFP:
482         return AArch64::SCVTFUWDri;
483       case TargetOpcode::G_UITOFP:
484         return AArch64::UCVTFUWDri;
485       case TargetOpcode::G_FPTOSI:
486         return AArch64::FCVTZSUXSr;
487       case TargetOpcode::G_FPTOUI:
488         return AArch64::FCVTZUUXSr;
489       default:
490         return GenericOpc;
491       }
492     case 64:
493       switch (GenericOpc) {
494       case TargetOpcode::G_SITOFP:
495         return AArch64::SCVTFUXDri;
496       case TargetOpcode::G_UITOFP:
497         return AArch64::UCVTFUXDri;
498       case TargetOpcode::G_FPTOSI:
499         return AArch64::FCVTZSUXDr;
500       case TargetOpcode::G_FPTOUI:
501         return AArch64::FCVTZUUXDr;
502       default:
503         return GenericOpc;
504       }
505     default:
506       return GenericOpc;
507     }
508   default:
509     return GenericOpc;
510   };
511   return GenericOpc;
512 }
513 
changeICMPPredToAArch64CC(CmpInst::Predicate P)514 static AArch64CC::CondCode changeICMPPredToAArch64CC(CmpInst::Predicate P) {
515   switch (P) {
516   default:
517     llvm_unreachable("Unknown condition code!");
518   case CmpInst::ICMP_NE:
519     return AArch64CC::NE;
520   case CmpInst::ICMP_EQ:
521     return AArch64CC::EQ;
522   case CmpInst::ICMP_SGT:
523     return AArch64CC::GT;
524   case CmpInst::ICMP_SGE:
525     return AArch64CC::GE;
526   case CmpInst::ICMP_SLT:
527     return AArch64CC::LT;
528   case CmpInst::ICMP_SLE:
529     return AArch64CC::LE;
530   case CmpInst::ICMP_UGT:
531     return AArch64CC::HI;
532   case CmpInst::ICMP_UGE:
533     return AArch64CC::HS;
534   case CmpInst::ICMP_ULT:
535     return AArch64CC::LO;
536   case CmpInst::ICMP_ULE:
537     return AArch64CC::LS;
538   }
539 }
540 
changeFCMPPredToAArch64CC(CmpInst::Predicate P,AArch64CC::CondCode & CondCode,AArch64CC::CondCode & CondCode2)541 static void changeFCMPPredToAArch64CC(CmpInst::Predicate P,
542                                       AArch64CC::CondCode &CondCode,
543                                       AArch64CC::CondCode &CondCode2) {
544   CondCode2 = AArch64CC::AL;
545   switch (P) {
546   default:
547     llvm_unreachable("Unknown FP condition!");
548   case CmpInst::FCMP_OEQ:
549     CondCode = AArch64CC::EQ;
550     break;
551   case CmpInst::FCMP_OGT:
552     CondCode = AArch64CC::GT;
553     break;
554   case CmpInst::FCMP_OGE:
555     CondCode = AArch64CC::GE;
556     break;
557   case CmpInst::FCMP_OLT:
558     CondCode = AArch64CC::MI;
559     break;
560   case CmpInst::FCMP_OLE:
561     CondCode = AArch64CC::LS;
562     break;
563   case CmpInst::FCMP_ONE:
564     CondCode = AArch64CC::MI;
565     CondCode2 = AArch64CC::GT;
566     break;
567   case CmpInst::FCMP_ORD:
568     CondCode = AArch64CC::VC;
569     break;
570   case CmpInst::FCMP_UNO:
571     CondCode = AArch64CC::VS;
572     break;
573   case CmpInst::FCMP_UEQ:
574     CondCode = AArch64CC::EQ;
575     CondCode2 = AArch64CC::VS;
576     break;
577   case CmpInst::FCMP_UGT:
578     CondCode = AArch64CC::HI;
579     break;
580   case CmpInst::FCMP_UGE:
581     CondCode = AArch64CC::PL;
582     break;
583   case CmpInst::FCMP_ULT:
584     CondCode = AArch64CC::LT;
585     break;
586   case CmpInst::FCMP_ULE:
587     CondCode = AArch64CC::LE;
588     break;
589   case CmpInst::FCMP_UNE:
590     CondCode = AArch64CC::NE;
591     break;
592   }
593 }
594 
selectCompareBranch(MachineInstr & I,MachineFunction & MF,MachineRegisterInfo & MRI) const595 bool AArch64InstructionSelector::selectCompareBranch(
596     MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
597 
598   const unsigned CondReg = I.getOperand(0).getReg();
599   MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
600   MachineInstr *CCMI = MRI.getVRegDef(CondReg);
601   if (CCMI->getOpcode() == TargetOpcode::G_TRUNC)
602     CCMI = MRI.getVRegDef(CCMI->getOperand(1).getReg());
603   if (CCMI->getOpcode() != TargetOpcode::G_ICMP)
604     return false;
605 
606   unsigned LHS = CCMI->getOperand(2).getReg();
607   unsigned RHS = CCMI->getOperand(3).getReg();
608   if (!getConstantVRegVal(RHS, MRI))
609     std::swap(RHS, LHS);
610 
611   const auto RHSImm = getConstantVRegVal(RHS, MRI);
612   if (!RHSImm || *RHSImm != 0)
613     return false;
614 
615   const RegisterBank &RB = *RBI.getRegBank(LHS, MRI, TRI);
616   if (RB.getID() != AArch64::GPRRegBankID)
617     return false;
618 
619   const auto Pred = (CmpInst::Predicate)CCMI->getOperand(1).getPredicate();
620   if (Pred != CmpInst::ICMP_NE && Pred != CmpInst::ICMP_EQ)
621     return false;
622 
623   const unsigned CmpWidth = MRI.getType(LHS).getSizeInBits();
624   unsigned CBOpc = 0;
625   if (CmpWidth <= 32)
626     CBOpc = (Pred == CmpInst::ICMP_EQ ? AArch64::CBZW : AArch64::CBNZW);
627   else if (CmpWidth == 64)
628     CBOpc = (Pred == CmpInst::ICMP_EQ ? AArch64::CBZX : AArch64::CBNZX);
629   else
630     return false;
631 
632   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CBOpc))
633       .addUse(LHS)
634       .addMBB(DestMBB)
635       .constrainAllUses(TII, TRI, RBI);
636 
637   I.eraseFromParent();
638   return true;
639 }
640 
selectVaStartAAPCS(MachineInstr & I,MachineFunction & MF,MachineRegisterInfo & MRI) const641 bool AArch64InstructionSelector::selectVaStartAAPCS(
642     MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
643   return false;
644 }
645 
selectVaStartDarwin(MachineInstr & I,MachineFunction & MF,MachineRegisterInfo & MRI) const646 bool AArch64InstructionSelector::selectVaStartDarwin(
647     MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
648   AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
649   unsigned ListReg = I.getOperand(0).getReg();
650 
651   unsigned ArgsAddrReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
652 
653   auto MIB =
654       BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::ADDXri))
655           .addDef(ArgsAddrReg)
656           .addFrameIndex(FuncInfo->getVarArgsStackIndex())
657           .addImm(0)
658           .addImm(0);
659 
660   constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
661 
662   MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::STRXui))
663             .addUse(ArgsAddrReg)
664             .addUse(ListReg)
665             .addImm(0)
666             .addMemOperand(*I.memoperands_begin());
667 
668   constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
669   I.eraseFromParent();
670   return true;
671 }
672 
materializeLargeCMVal(MachineInstr & I,const Value * V,unsigned char OpFlags) const673 void AArch64InstructionSelector::materializeLargeCMVal(
674     MachineInstr &I, const Value *V, unsigned char OpFlags) const {
675   MachineBasicBlock &MBB = *I.getParent();
676   MachineFunction &MF = *MBB.getParent();
677   MachineRegisterInfo &MRI = MF.getRegInfo();
678   MachineIRBuilder MIB(I);
679 
680   auto MovZ = MIB.buildInstr(AArch64::MOVZXi, {&AArch64::GPR64RegClass}, {});
681   MovZ->addOperand(MF, I.getOperand(1));
682   MovZ->getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_G0 |
683                                      AArch64II::MO_NC);
684   MovZ->addOperand(MF, MachineOperand::CreateImm(0));
685   constrainSelectedInstRegOperands(*MovZ, TII, TRI, RBI);
686 
687   auto BuildMovK = [&](unsigned SrcReg, unsigned char Flags, unsigned Offset,
688                        unsigned ForceDstReg) {
689     unsigned DstReg = ForceDstReg
690                           ? ForceDstReg
691                           : MRI.createVirtualRegister(&AArch64::GPR64RegClass);
692     auto MovI = MIB.buildInstr(AArch64::MOVKXi).addDef(DstReg).addUse(SrcReg);
693     if (auto *GV = dyn_cast<GlobalValue>(V)) {
694       MovI->addOperand(MF, MachineOperand::CreateGA(
695                                GV, MovZ->getOperand(1).getOffset(), Flags));
696     } else {
697       MovI->addOperand(
698           MF, MachineOperand::CreateBA(cast<BlockAddress>(V),
699                                        MovZ->getOperand(1).getOffset(), Flags));
700     }
701     MovI->addOperand(MF, MachineOperand::CreateImm(Offset));
702     constrainSelectedInstRegOperands(*MovI, TII, TRI, RBI);
703     return DstReg;
704   };
705   unsigned DstReg = BuildMovK(MovZ->getOperand(0).getReg(),
706                               AArch64II::MO_G1 | AArch64II::MO_NC, 16, 0);
707   DstReg = BuildMovK(DstReg, AArch64II::MO_G2 | AArch64II::MO_NC, 32, 0);
708   BuildMovK(DstReg, AArch64II::MO_G3, 48, I.getOperand(0).getReg());
709   return;
710 }
711 
select(MachineInstr & I,CodeGenCoverage & CoverageInfo) const712 bool AArch64InstructionSelector::select(MachineInstr &I,
713                                         CodeGenCoverage &CoverageInfo) const {
714   assert(I.getParent() && "Instruction should be in a basic block!");
715   assert(I.getParent()->getParent() && "Instruction should be in a function!");
716 
717   MachineBasicBlock &MBB = *I.getParent();
718   MachineFunction &MF = *MBB.getParent();
719   MachineRegisterInfo &MRI = MF.getRegInfo();
720 
721   unsigned Opcode = I.getOpcode();
722   // G_PHI requires same handling as PHI
723   if (!isPreISelGenericOpcode(Opcode) || Opcode == TargetOpcode::G_PHI) {
724     // Certain non-generic instructions also need some special handling.
725 
726     if (Opcode ==  TargetOpcode::LOAD_STACK_GUARD)
727       return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
728 
729     if (Opcode == TargetOpcode::PHI || Opcode == TargetOpcode::G_PHI) {
730       const unsigned DefReg = I.getOperand(0).getReg();
731       const LLT DefTy = MRI.getType(DefReg);
732 
733       const TargetRegisterClass *DefRC = nullptr;
734       if (TargetRegisterInfo::isPhysicalRegister(DefReg)) {
735         DefRC = TRI.getRegClass(DefReg);
736       } else {
737         const RegClassOrRegBank &RegClassOrBank =
738             MRI.getRegClassOrRegBank(DefReg);
739 
740         DefRC = RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
741         if (!DefRC) {
742           if (!DefTy.isValid()) {
743             LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
744             return false;
745           }
746           const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>();
747           DefRC = getRegClassForTypeOnBank(DefTy, RB, RBI);
748           if (!DefRC) {
749             LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
750             return false;
751           }
752         }
753       }
754       I.setDesc(TII.get(TargetOpcode::PHI));
755 
756       return RBI.constrainGenericRegister(DefReg, *DefRC, MRI);
757     }
758 
759     if (I.isCopy())
760       return selectCopy(I, TII, MRI, TRI, RBI);
761 
762     return true;
763   }
764 
765 
766   if (I.getNumOperands() != I.getNumExplicitOperands()) {
767     LLVM_DEBUG(
768         dbgs() << "Generic instruction has unexpected implicit operands\n");
769     return false;
770   }
771 
772   if (selectImpl(I, CoverageInfo))
773     return true;
774 
775   LLT Ty =
776       I.getOperand(0).isReg() ? MRI.getType(I.getOperand(0).getReg()) : LLT{};
777 
778   switch (Opcode) {
779   case TargetOpcode::G_BRCOND: {
780     if (Ty.getSizeInBits() > 32) {
781       // We shouldn't need this on AArch64, but it would be implemented as an
782       // EXTRACT_SUBREG followed by a TBNZW because TBNZX has no encoding if the
783       // bit being tested is < 32.
784       LLVM_DEBUG(dbgs() << "G_BRCOND has type: " << Ty
785                         << ", expected at most 32-bits");
786       return false;
787     }
788 
789     const unsigned CondReg = I.getOperand(0).getReg();
790     MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
791 
792     // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z
793     // instructions will not be produced, as they are conditional branch
794     // instructions that do not set flags.
795     bool ProduceNonFlagSettingCondBr =
796         !MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening);
797     if (ProduceNonFlagSettingCondBr && selectCompareBranch(I, MF, MRI))
798       return true;
799 
800     if (ProduceNonFlagSettingCondBr) {
801       auto MIB = BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::TBNZW))
802                      .addUse(CondReg)
803                      .addImm(/*bit offset=*/0)
804                      .addMBB(DestMBB);
805 
806       I.eraseFromParent();
807       return constrainSelectedInstRegOperands(*MIB.getInstr(), TII, TRI, RBI);
808     } else {
809       auto CMP = BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ANDSWri))
810                      .addDef(AArch64::WZR)
811                      .addUse(CondReg)
812                      .addImm(1);
813       constrainSelectedInstRegOperands(*CMP.getInstr(), TII, TRI, RBI);
814       auto Bcc =
815           BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::Bcc))
816               .addImm(AArch64CC::EQ)
817               .addMBB(DestMBB);
818 
819       I.eraseFromParent();
820       return constrainSelectedInstRegOperands(*Bcc.getInstr(), TII, TRI, RBI);
821     }
822   }
823 
824   case TargetOpcode::G_BRINDIRECT: {
825     I.setDesc(TII.get(AArch64::BR));
826     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
827   }
828 
829   case TargetOpcode::G_FCONSTANT:
830   case TargetOpcode::G_CONSTANT: {
831     const bool isFP = Opcode == TargetOpcode::G_FCONSTANT;
832 
833     const LLT s32 = LLT::scalar(32);
834     const LLT s64 = LLT::scalar(64);
835     const LLT p0 = LLT::pointer(0, 64);
836 
837     const unsigned DefReg = I.getOperand(0).getReg();
838     const LLT DefTy = MRI.getType(DefReg);
839     const unsigned DefSize = DefTy.getSizeInBits();
840     const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
841 
842     // FIXME: Redundant check, but even less readable when factored out.
843     if (isFP) {
844       if (Ty != s32 && Ty != s64) {
845         LLVM_DEBUG(dbgs() << "Unable to materialize FP " << Ty
846                           << " constant, expected: " << s32 << " or " << s64
847                           << '\n');
848         return false;
849       }
850 
851       if (RB.getID() != AArch64::FPRRegBankID) {
852         LLVM_DEBUG(dbgs() << "Unable to materialize FP " << Ty
853                           << " constant on bank: " << RB
854                           << ", expected: FPR\n");
855         return false;
856       }
857 
858       // The case when we have 0.0 is covered by tablegen. Reject it here so we
859       // can be sure tablegen works correctly and isn't rescued by this code.
860       if (I.getOperand(1).getFPImm()->getValueAPF().isExactlyValue(0.0))
861         return false;
862     } else {
863       // s32 and s64 are covered by tablegen.
864       if (Ty != p0) {
865         LLVM_DEBUG(dbgs() << "Unable to materialize integer " << Ty
866                           << " constant, expected: " << s32 << ", " << s64
867                           << ", or " << p0 << '\n');
868         return false;
869       }
870 
871       if (RB.getID() != AArch64::GPRRegBankID) {
872         LLVM_DEBUG(dbgs() << "Unable to materialize integer " << Ty
873                           << " constant on bank: " << RB
874                           << ", expected: GPR\n");
875         return false;
876       }
877     }
878 
879     const unsigned MovOpc =
880         DefSize == 32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;
881 
882     I.setDesc(TII.get(MovOpc));
883 
884     if (isFP) {
885       const TargetRegisterClass &GPRRC =
886           DefSize == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass;
887       const TargetRegisterClass &FPRRC =
888           DefSize == 32 ? AArch64::FPR32RegClass : AArch64::FPR64RegClass;
889 
890       const unsigned DefGPRReg = MRI.createVirtualRegister(&GPRRC);
891       MachineOperand &RegOp = I.getOperand(0);
892       RegOp.setReg(DefGPRReg);
893 
894       BuildMI(MBB, std::next(I.getIterator()), I.getDebugLoc(),
895               TII.get(AArch64::COPY))
896           .addDef(DefReg)
897           .addUse(DefGPRReg);
898 
899       if (!RBI.constrainGenericRegister(DefReg, FPRRC, MRI)) {
900         LLVM_DEBUG(dbgs() << "Failed to constrain G_FCONSTANT def operand\n");
901         return false;
902       }
903 
904       MachineOperand &ImmOp = I.getOperand(1);
905       // FIXME: Is going through int64_t always correct?
906       ImmOp.ChangeToImmediate(
907           ImmOp.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
908     } else if (I.getOperand(1).isCImm()) {
909       uint64_t Val = I.getOperand(1).getCImm()->getZExtValue();
910       I.getOperand(1).ChangeToImmediate(Val);
911     } else if (I.getOperand(1).isImm()) {
912       uint64_t Val = I.getOperand(1).getImm();
913       I.getOperand(1).ChangeToImmediate(Val);
914     }
915 
916     constrainSelectedInstRegOperands(I, TII, TRI, RBI);
917     return true;
918   }
919   case TargetOpcode::G_EXTRACT: {
920     LLT SrcTy = MRI.getType(I.getOperand(1).getReg());
921     LLT DstTy = MRI.getType(I.getOperand(0).getReg());
922     (void)DstTy;
923     unsigned SrcSize = SrcTy.getSizeInBits();
924     // Larger extracts are vectors, same-size extracts should be something else
925     // by now (either split up or simplified to a COPY).
926     if (SrcTy.getSizeInBits() > 64 || Ty.getSizeInBits() > 32)
927       return false;
928 
929     I.setDesc(TII.get(SrcSize == 64 ? AArch64::UBFMXri : AArch64::UBFMWri));
930     MachineInstrBuilder(MF, I).addImm(I.getOperand(2).getImm() +
931                                       Ty.getSizeInBits() - 1);
932 
933     if (SrcSize < 64) {
934       assert(SrcSize == 32 && DstTy.getSizeInBits() == 16 &&
935              "unexpected G_EXTRACT types");
936       return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
937     }
938 
939     unsigned DstReg = MRI.createGenericVirtualRegister(LLT::scalar(64));
940     BuildMI(MBB, std::next(I.getIterator()), I.getDebugLoc(),
941             TII.get(AArch64::COPY))
942         .addDef(I.getOperand(0).getReg())
943         .addUse(DstReg, 0, AArch64::sub_32);
944     RBI.constrainGenericRegister(I.getOperand(0).getReg(),
945                                  AArch64::GPR32RegClass, MRI);
946     I.getOperand(0).setReg(DstReg);
947 
948     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
949   }
950 
951   case TargetOpcode::G_INSERT: {
952     LLT SrcTy = MRI.getType(I.getOperand(2).getReg());
953     LLT DstTy = MRI.getType(I.getOperand(0).getReg());
954     unsigned DstSize = DstTy.getSizeInBits();
955     // Larger inserts are vectors, same-size ones should be something else by
956     // now (split up or turned into COPYs).
957     if (Ty.getSizeInBits() > 64 || SrcTy.getSizeInBits() > 32)
958       return false;
959 
960     I.setDesc(TII.get(DstSize == 64 ? AArch64::BFMXri : AArch64::BFMWri));
961     unsigned LSB = I.getOperand(3).getImm();
962     unsigned Width = MRI.getType(I.getOperand(2).getReg()).getSizeInBits();
963     I.getOperand(3).setImm((DstSize - LSB) % DstSize);
964     MachineInstrBuilder(MF, I).addImm(Width - 1);
965 
966     if (DstSize < 64) {
967       assert(DstSize == 32 && SrcTy.getSizeInBits() == 16 &&
968              "unexpected G_INSERT types");
969       return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
970     }
971 
972     unsigned SrcReg = MRI.createGenericVirtualRegister(LLT::scalar(64));
973     BuildMI(MBB, I.getIterator(), I.getDebugLoc(),
974             TII.get(AArch64::SUBREG_TO_REG))
975         .addDef(SrcReg)
976         .addImm(0)
977         .addUse(I.getOperand(2).getReg())
978         .addImm(AArch64::sub_32);
979     RBI.constrainGenericRegister(I.getOperand(2).getReg(),
980                                  AArch64::GPR32RegClass, MRI);
981     I.getOperand(2).setReg(SrcReg);
982 
983     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
984   }
985   case TargetOpcode::G_FRAME_INDEX: {
986     // allocas and G_FRAME_INDEX are only supported in addrspace(0).
987     if (Ty != LLT::pointer(0, 64)) {
988       LLVM_DEBUG(dbgs() << "G_FRAME_INDEX pointer has type: " << Ty
989                         << ", expected: " << LLT::pointer(0, 64) << '\n');
990       return false;
991     }
992     I.setDesc(TII.get(AArch64::ADDXri));
993 
994     // MOs for a #0 shifted immediate.
995     I.addOperand(MachineOperand::CreateImm(0));
996     I.addOperand(MachineOperand::CreateImm(0));
997 
998     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
999   }
1000 
1001   case TargetOpcode::G_GLOBAL_VALUE: {
1002     auto GV = I.getOperand(1).getGlobal();
1003     if (GV->isThreadLocal()) {
1004       // FIXME: we don't support TLS yet.
1005       return false;
1006     }
1007     unsigned char OpFlags = STI.ClassifyGlobalReference(GV, TM);
1008     if (OpFlags & AArch64II::MO_GOT) {
1009       I.setDesc(TII.get(AArch64::LOADgot));
1010       I.getOperand(1).setTargetFlags(OpFlags);
1011     } else if (TM.getCodeModel() == CodeModel::Large) {
1012       // Materialize the global using movz/movk instructions.
1013       materializeLargeCMVal(I, GV, OpFlags);
1014       I.eraseFromParent();
1015       return true;
1016     } else if (TM.getCodeModel() == CodeModel::Tiny) {
1017       I.setDesc(TII.get(AArch64::ADR));
1018       I.getOperand(1).setTargetFlags(OpFlags);
1019     } else {
1020       I.setDesc(TII.get(AArch64::MOVaddr));
1021       I.getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_PAGE);
1022       MachineInstrBuilder MIB(MF, I);
1023       MIB.addGlobalAddress(GV, I.getOperand(1).getOffset(),
1024                            OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
1025     }
1026     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1027   }
1028 
1029   case TargetOpcode::G_LOAD:
1030   case TargetOpcode::G_STORE: {
1031     LLT PtrTy = MRI.getType(I.getOperand(1).getReg());
1032 
1033     if (PtrTy != LLT::pointer(0, 64)) {
1034       LLVM_DEBUG(dbgs() << "Load/Store pointer has type: " << PtrTy
1035                         << ", expected: " << LLT::pointer(0, 64) << '\n');
1036       return false;
1037     }
1038 
1039     auto &MemOp = **I.memoperands_begin();
1040     if (MemOp.getOrdering() != AtomicOrdering::NotAtomic) {
1041       LLVM_DEBUG(dbgs() << "Atomic load/store not supported yet\n");
1042       return false;
1043     }
1044     unsigned MemSizeInBits = MemOp.getSize() * 8;
1045 
1046     const unsigned PtrReg = I.getOperand(1).getReg();
1047 #ifndef NDEBUG
1048     const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, MRI, TRI);
1049     // Sanity-check the pointer register.
1050     assert(PtrRB.getID() == AArch64::GPRRegBankID &&
1051            "Load/Store pointer operand isn't a GPR");
1052     assert(MRI.getType(PtrReg).isPointer() &&
1053            "Load/Store pointer operand isn't a pointer");
1054 #endif
1055 
1056     const unsigned ValReg = I.getOperand(0).getReg();
1057     const RegisterBank &RB = *RBI.getRegBank(ValReg, MRI, TRI);
1058 
1059     const unsigned NewOpc =
1060         selectLoadStoreUIOp(I.getOpcode(), RB.getID(), MemSizeInBits);
1061     if (NewOpc == I.getOpcode())
1062       return false;
1063 
1064     I.setDesc(TII.get(NewOpc));
1065 
1066     uint64_t Offset = 0;
1067     auto *PtrMI = MRI.getVRegDef(PtrReg);
1068 
1069     // Try to fold a GEP into our unsigned immediate addressing mode.
1070     if (PtrMI->getOpcode() == TargetOpcode::G_GEP) {
1071       if (auto COff = getConstantVRegVal(PtrMI->getOperand(2).getReg(), MRI)) {
1072         int64_t Imm = *COff;
1073         const unsigned Size = MemSizeInBits / 8;
1074         const unsigned Scale = Log2_32(Size);
1075         if ((Imm & (Size - 1)) == 0 && Imm >= 0 && Imm < (0x1000 << Scale)) {
1076           unsigned Ptr2Reg = PtrMI->getOperand(1).getReg();
1077           I.getOperand(1).setReg(Ptr2Reg);
1078           PtrMI = MRI.getVRegDef(Ptr2Reg);
1079           Offset = Imm / Size;
1080         }
1081       }
1082     }
1083 
1084     // If we haven't folded anything into our addressing mode yet, try to fold
1085     // a frame index into the base+offset.
1086     if (!Offset && PtrMI->getOpcode() == TargetOpcode::G_FRAME_INDEX)
1087       I.getOperand(1).ChangeToFrameIndex(PtrMI->getOperand(1).getIndex());
1088 
1089     I.addOperand(MachineOperand::CreateImm(Offset));
1090 
1091     // If we're storing a 0, use WZR/XZR.
1092     if (auto CVal = getConstantVRegVal(ValReg, MRI)) {
1093       if (*CVal == 0 && Opcode == TargetOpcode::G_STORE) {
1094         if (I.getOpcode() == AArch64::STRWui)
1095           I.getOperand(0).setReg(AArch64::WZR);
1096         else if (I.getOpcode() == AArch64::STRXui)
1097           I.getOperand(0).setReg(AArch64::XZR);
1098       }
1099     }
1100 
1101     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1102   }
1103 
1104   case TargetOpcode::G_SMULH:
1105   case TargetOpcode::G_UMULH: {
1106     // Reject the various things we don't support yet.
1107     if (unsupportedBinOp(I, RBI, MRI, TRI))
1108       return false;
1109 
1110     const unsigned DefReg = I.getOperand(0).getReg();
1111     const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
1112 
1113     if (RB.getID() != AArch64::GPRRegBankID) {
1114       LLVM_DEBUG(dbgs() << "G_[SU]MULH on bank: " << RB << ", expected: GPR\n");
1115       return false;
1116     }
1117 
1118     if (Ty != LLT::scalar(64)) {
1119       LLVM_DEBUG(dbgs() << "G_[SU]MULH has type: " << Ty
1120                         << ", expected: " << LLT::scalar(64) << '\n');
1121       return false;
1122     }
1123 
1124     unsigned NewOpc = I.getOpcode() == TargetOpcode::G_SMULH ? AArch64::SMULHrr
1125                                                              : AArch64::UMULHrr;
1126     I.setDesc(TII.get(NewOpc));
1127 
1128     // Now that we selected an opcode, we need to constrain the register
1129     // operands to use appropriate classes.
1130     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1131   }
1132   case TargetOpcode::G_FADD:
1133   case TargetOpcode::G_FSUB:
1134   case TargetOpcode::G_FMUL:
1135   case TargetOpcode::G_FDIV:
1136 
1137   case TargetOpcode::G_OR:
1138   case TargetOpcode::G_SHL:
1139   case TargetOpcode::G_LSHR:
1140   case TargetOpcode::G_ASHR:
1141   case TargetOpcode::G_GEP: {
1142     // Reject the various things we don't support yet.
1143     if (unsupportedBinOp(I, RBI, MRI, TRI))
1144       return false;
1145 
1146     const unsigned OpSize = Ty.getSizeInBits();
1147 
1148     const unsigned DefReg = I.getOperand(0).getReg();
1149     const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
1150 
1151     const unsigned NewOpc = selectBinaryOp(I.getOpcode(), RB.getID(), OpSize);
1152     if (NewOpc == I.getOpcode())
1153       return false;
1154 
1155     I.setDesc(TII.get(NewOpc));
1156     // FIXME: Should the type be always reset in setDesc?
1157 
1158     // Now that we selected an opcode, we need to constrain the register
1159     // operands to use appropriate classes.
1160     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1161   }
1162 
1163   case TargetOpcode::G_PTR_MASK: {
1164     uint64_t Align = I.getOperand(2).getImm();
1165     if (Align >= 64 || Align == 0)
1166       return false;
1167 
1168     uint64_t Mask = ~((1ULL << Align) - 1);
1169     I.setDesc(TII.get(AArch64::ANDXri));
1170     I.getOperand(2).setImm(AArch64_AM::encodeLogicalImmediate(Mask, 64));
1171 
1172     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1173   }
1174   case TargetOpcode::G_PTRTOINT:
1175   case TargetOpcode::G_TRUNC: {
1176     const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
1177     const LLT SrcTy = MRI.getType(I.getOperand(1).getReg());
1178 
1179     const unsigned DstReg = I.getOperand(0).getReg();
1180     const unsigned SrcReg = I.getOperand(1).getReg();
1181 
1182     const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
1183     const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
1184 
1185     if (DstRB.getID() != SrcRB.getID()) {
1186       LLVM_DEBUG(
1187           dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n");
1188       return false;
1189     }
1190 
1191     if (DstRB.getID() == AArch64::GPRRegBankID) {
1192       const TargetRegisterClass *DstRC =
1193           getRegClassForTypeOnBank(DstTy, DstRB, RBI);
1194       if (!DstRC)
1195         return false;
1196 
1197       const TargetRegisterClass *SrcRC =
1198           getRegClassForTypeOnBank(SrcTy, SrcRB, RBI);
1199       if (!SrcRC)
1200         return false;
1201 
1202       if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
1203           !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
1204         LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC/G_PTRTOINT\n");
1205         return false;
1206       }
1207 
1208       if (DstRC == SrcRC) {
1209         // Nothing to be done
1210       } else if (Opcode == TargetOpcode::G_TRUNC && DstTy == LLT::scalar(32) &&
1211                  SrcTy == LLT::scalar(64)) {
1212         llvm_unreachable("TableGen can import this case");
1213         return false;
1214       } else if (DstRC == &AArch64::GPR32RegClass &&
1215                  SrcRC == &AArch64::GPR64RegClass) {
1216         I.getOperand(1).setSubReg(AArch64::sub_32);
1217       } else {
1218         LLVM_DEBUG(
1219             dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n");
1220         return false;
1221       }
1222 
1223       I.setDesc(TII.get(TargetOpcode::COPY));
1224       return true;
1225     } else if (DstRB.getID() == AArch64::FPRRegBankID) {
1226       if (DstTy == LLT::vector(4, 16) && SrcTy == LLT::vector(4, 32)) {
1227         I.setDesc(TII.get(AArch64::XTNv4i16));
1228         constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1229         return true;
1230       }
1231     }
1232 
1233     return false;
1234   }
1235 
1236   case TargetOpcode::G_ANYEXT: {
1237     const unsigned DstReg = I.getOperand(0).getReg();
1238     const unsigned SrcReg = I.getOperand(1).getReg();
1239 
1240     const RegisterBank &RBDst = *RBI.getRegBank(DstReg, MRI, TRI);
1241     if (RBDst.getID() != AArch64::GPRRegBankID) {
1242       LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBDst
1243                         << ", expected: GPR\n");
1244       return false;
1245     }
1246 
1247     const RegisterBank &RBSrc = *RBI.getRegBank(SrcReg, MRI, TRI);
1248     if (RBSrc.getID() != AArch64::GPRRegBankID) {
1249       LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBSrc
1250                         << ", expected: GPR\n");
1251       return false;
1252     }
1253 
1254     const unsigned DstSize = MRI.getType(DstReg).getSizeInBits();
1255 
1256     if (DstSize == 0) {
1257       LLVM_DEBUG(dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n");
1258       return false;
1259     }
1260 
1261     if (DstSize != 64 && DstSize > 32) {
1262       LLVM_DEBUG(dbgs() << "G_ANYEXT to size: " << DstSize
1263                         << ", expected: 32 or 64\n");
1264       return false;
1265     }
1266     // At this point G_ANYEXT is just like a plain COPY, but we need
1267     // to explicitly form the 64-bit value if any.
1268     if (DstSize > 32) {
1269       unsigned ExtSrc = MRI.createVirtualRegister(&AArch64::GPR64allRegClass);
1270       BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG))
1271           .addDef(ExtSrc)
1272           .addImm(0)
1273           .addUse(SrcReg)
1274           .addImm(AArch64::sub_32);
1275       I.getOperand(1).setReg(ExtSrc);
1276     }
1277     return selectCopy(I, TII, MRI, TRI, RBI);
1278   }
1279 
1280   case TargetOpcode::G_ZEXT:
1281   case TargetOpcode::G_SEXT: {
1282     unsigned Opcode = I.getOpcode();
1283     const LLT DstTy = MRI.getType(I.getOperand(0).getReg()),
1284               SrcTy = MRI.getType(I.getOperand(1).getReg());
1285     const bool isSigned = Opcode == TargetOpcode::G_SEXT;
1286     const unsigned DefReg = I.getOperand(0).getReg();
1287     const unsigned SrcReg = I.getOperand(1).getReg();
1288     const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
1289 
1290     if (RB.getID() != AArch64::GPRRegBankID) {
1291       LLVM_DEBUG(dbgs() << TII.getName(I.getOpcode()) << " on bank: " << RB
1292                         << ", expected: GPR\n");
1293       return false;
1294     }
1295 
1296     MachineInstr *ExtI;
1297     if (DstTy == LLT::scalar(64)) {
1298       // FIXME: Can we avoid manually doing this?
1299       if (!RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass, MRI)) {
1300         LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(Opcode)
1301                           << " operand\n");
1302         return false;
1303       }
1304 
1305       const unsigned SrcXReg =
1306           MRI.createVirtualRegister(&AArch64::GPR64RegClass);
1307       BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG))
1308           .addDef(SrcXReg)
1309           .addImm(0)
1310           .addUse(SrcReg)
1311           .addImm(AArch64::sub_32);
1312 
1313       const unsigned NewOpc = isSigned ? AArch64::SBFMXri : AArch64::UBFMXri;
1314       ExtI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(NewOpc))
1315                  .addDef(DefReg)
1316                  .addUse(SrcXReg)
1317                  .addImm(0)
1318                  .addImm(SrcTy.getSizeInBits() - 1);
1319     } else if (DstTy.isScalar() && DstTy.getSizeInBits() <= 32) {
1320       const unsigned NewOpc = isSigned ? AArch64::SBFMWri : AArch64::UBFMWri;
1321       ExtI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(NewOpc))
1322                  .addDef(DefReg)
1323                  .addUse(SrcReg)
1324                  .addImm(0)
1325                  .addImm(SrcTy.getSizeInBits() - 1);
1326     } else {
1327       return false;
1328     }
1329 
1330     constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI);
1331 
1332     I.eraseFromParent();
1333     return true;
1334   }
1335 
1336   case TargetOpcode::G_SITOFP:
1337   case TargetOpcode::G_UITOFP:
1338   case TargetOpcode::G_FPTOSI:
1339   case TargetOpcode::G_FPTOUI: {
1340     const LLT DstTy = MRI.getType(I.getOperand(0).getReg()),
1341               SrcTy = MRI.getType(I.getOperand(1).getReg());
1342     const unsigned NewOpc = selectFPConvOpc(Opcode, DstTy, SrcTy);
1343     if (NewOpc == Opcode)
1344       return false;
1345 
1346     I.setDesc(TII.get(NewOpc));
1347     constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1348 
1349     return true;
1350   }
1351 
1352 
1353   case TargetOpcode::G_INTTOPTR:
1354     // The importer is currently unable to import pointer types since they
1355     // didn't exist in SelectionDAG.
1356     return selectCopy(I, TII, MRI, TRI, RBI);
1357 
1358   case TargetOpcode::G_BITCAST:
1359     // Imported SelectionDAG rules can handle every bitcast except those that
1360     // bitcast from a type to the same type. Ideally, these shouldn't occur
1361     // but we might not run an optimizer that deletes them.
1362     if (MRI.getType(I.getOperand(0).getReg()) ==
1363         MRI.getType(I.getOperand(1).getReg()))
1364       return selectCopy(I, TII, MRI, TRI, RBI);
1365     return false;
1366 
1367   case TargetOpcode::G_SELECT: {
1368     if (MRI.getType(I.getOperand(1).getReg()) != LLT::scalar(1)) {
1369       LLVM_DEBUG(dbgs() << "G_SELECT cond has type: " << Ty
1370                         << ", expected: " << LLT::scalar(1) << '\n');
1371       return false;
1372     }
1373 
1374     const unsigned CondReg = I.getOperand(1).getReg();
1375     const unsigned TReg = I.getOperand(2).getReg();
1376     const unsigned FReg = I.getOperand(3).getReg();
1377 
1378     unsigned CSelOpc = 0;
1379 
1380     if (Ty == LLT::scalar(32)) {
1381       CSelOpc = AArch64::CSELWr;
1382     } else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) {
1383       CSelOpc = AArch64::CSELXr;
1384     } else {
1385       return false;
1386     }
1387 
1388     MachineInstr &TstMI =
1389         *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ANDSWri))
1390              .addDef(AArch64::WZR)
1391              .addUse(CondReg)
1392              .addImm(AArch64_AM::encodeLogicalImmediate(1, 32));
1393 
1394     MachineInstr &CSelMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CSelOpc))
1395                                 .addDef(I.getOperand(0).getReg())
1396                                 .addUse(TReg)
1397                                 .addUse(FReg)
1398                                 .addImm(AArch64CC::NE);
1399 
1400     constrainSelectedInstRegOperands(TstMI, TII, TRI, RBI);
1401     constrainSelectedInstRegOperands(CSelMI, TII, TRI, RBI);
1402 
1403     I.eraseFromParent();
1404     return true;
1405   }
1406   case TargetOpcode::G_ICMP: {
1407     if (Ty != LLT::scalar(32)) {
1408       LLVM_DEBUG(dbgs() << "G_ICMP result has type: " << Ty
1409                         << ", expected: " << LLT::scalar(32) << '\n');
1410       return false;
1411     }
1412 
1413     unsigned CmpOpc = 0;
1414     unsigned ZReg = 0;
1415 
1416     LLT CmpTy = MRI.getType(I.getOperand(2).getReg());
1417     if (CmpTy == LLT::scalar(32)) {
1418       CmpOpc = AArch64::SUBSWrr;
1419       ZReg = AArch64::WZR;
1420     } else if (CmpTy == LLT::scalar(64) || CmpTy.isPointer()) {
1421       CmpOpc = AArch64::SUBSXrr;
1422       ZReg = AArch64::XZR;
1423     } else {
1424       return false;
1425     }
1426 
1427     // CSINC increments the result by one when the condition code is false.
1428     // Therefore, we have to invert the predicate to get an increment by 1 when
1429     // the predicate is true.
1430     const AArch64CC::CondCode invCC =
1431         changeICMPPredToAArch64CC(CmpInst::getInversePredicate(
1432             (CmpInst::Predicate)I.getOperand(1).getPredicate()));
1433 
1434     MachineInstr &CmpMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CmpOpc))
1435                                .addDef(ZReg)
1436                                .addUse(I.getOperand(2).getReg())
1437                                .addUse(I.getOperand(3).getReg());
1438 
1439     MachineInstr &CSetMI =
1440         *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
1441              .addDef(I.getOperand(0).getReg())
1442              .addUse(AArch64::WZR)
1443              .addUse(AArch64::WZR)
1444              .addImm(invCC);
1445 
1446     constrainSelectedInstRegOperands(CmpMI, TII, TRI, RBI);
1447     constrainSelectedInstRegOperands(CSetMI, TII, TRI, RBI);
1448 
1449     I.eraseFromParent();
1450     return true;
1451   }
1452 
1453   case TargetOpcode::G_FCMP: {
1454     if (Ty != LLT::scalar(32)) {
1455       LLVM_DEBUG(dbgs() << "G_FCMP result has type: " << Ty
1456                         << ", expected: " << LLT::scalar(32) << '\n');
1457       return false;
1458     }
1459 
1460     unsigned CmpOpc = 0;
1461     LLT CmpTy = MRI.getType(I.getOperand(2).getReg());
1462     if (CmpTy == LLT::scalar(32)) {
1463       CmpOpc = AArch64::FCMPSrr;
1464     } else if (CmpTy == LLT::scalar(64)) {
1465       CmpOpc = AArch64::FCMPDrr;
1466     } else {
1467       return false;
1468     }
1469 
1470     // FIXME: regbank
1471 
1472     AArch64CC::CondCode CC1, CC2;
1473     changeFCMPPredToAArch64CC(
1474         (CmpInst::Predicate)I.getOperand(1).getPredicate(), CC1, CC2);
1475 
1476     MachineInstr &CmpMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CmpOpc))
1477                                .addUse(I.getOperand(2).getReg())
1478                                .addUse(I.getOperand(3).getReg());
1479 
1480     const unsigned DefReg = I.getOperand(0).getReg();
1481     unsigned Def1Reg = DefReg;
1482     if (CC2 != AArch64CC::AL)
1483       Def1Reg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
1484 
1485     MachineInstr &CSetMI =
1486         *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
1487              .addDef(Def1Reg)
1488              .addUse(AArch64::WZR)
1489              .addUse(AArch64::WZR)
1490              .addImm(getInvertedCondCode(CC1));
1491 
1492     if (CC2 != AArch64CC::AL) {
1493       unsigned Def2Reg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
1494       MachineInstr &CSet2MI =
1495           *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
1496                .addDef(Def2Reg)
1497                .addUse(AArch64::WZR)
1498                .addUse(AArch64::WZR)
1499                .addImm(getInvertedCondCode(CC2));
1500       MachineInstr &OrMI =
1501           *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ORRWrr))
1502                .addDef(DefReg)
1503                .addUse(Def1Reg)
1504                .addUse(Def2Reg);
1505       constrainSelectedInstRegOperands(OrMI, TII, TRI, RBI);
1506       constrainSelectedInstRegOperands(CSet2MI, TII, TRI, RBI);
1507     }
1508 
1509     constrainSelectedInstRegOperands(CmpMI, TII, TRI, RBI);
1510     constrainSelectedInstRegOperands(CSetMI, TII, TRI, RBI);
1511 
1512     I.eraseFromParent();
1513     return true;
1514   }
1515   case TargetOpcode::G_VASTART:
1516     return STI.isTargetDarwin() ? selectVaStartDarwin(I, MF, MRI)
1517                                 : selectVaStartAAPCS(I, MF, MRI);
1518   case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
1519     if (!I.getOperand(0).isIntrinsicID())
1520       return false;
1521     if (I.getOperand(0).getIntrinsicID() != Intrinsic::trap)
1522       return false;
1523     BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::BRK))
1524       .addImm(1);
1525     I.eraseFromParent();
1526     return true;
1527   case TargetOpcode::G_IMPLICIT_DEF: {
1528     I.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
1529     const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
1530     const unsigned DstReg = I.getOperand(0).getReg();
1531     const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
1532     const TargetRegisterClass *DstRC =
1533         getRegClassForTypeOnBank(DstTy, DstRB, RBI);
1534     RBI.constrainGenericRegister(DstReg, *DstRC, MRI);
1535     return true;
1536   }
1537   case TargetOpcode::G_BLOCK_ADDR: {
1538     if (TM.getCodeModel() == CodeModel::Large) {
1539       materializeLargeCMVal(I, I.getOperand(1).getBlockAddress(), 0);
1540       I.eraseFromParent();
1541       return true;
1542     } else {
1543       I.setDesc(TII.get(AArch64::MOVaddrBA));
1544       auto MovMI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::MOVaddrBA),
1545                            I.getOperand(0).getReg())
1546                        .addBlockAddress(I.getOperand(1).getBlockAddress(),
1547                                         /* Offset */ 0, AArch64II::MO_PAGE)
1548                        .addBlockAddress(
1549                            I.getOperand(1).getBlockAddress(), /* Offset */ 0,
1550                            AArch64II::MO_NC | AArch64II::MO_PAGEOFF);
1551       I.eraseFromParent();
1552       return constrainSelectedInstRegOperands(*MovMI, TII, TRI, RBI);
1553     }
1554   }
1555   case TargetOpcode::G_BUILD_VECTOR:
1556     return selectBuildVector(I, MRI);
1557   case TargetOpcode::G_MERGE_VALUES:
1558     return selectMergeValues(I, MRI);
1559   }
1560 
1561   return false;
1562 }
1563 
emitScalarToVector(unsigned & Dst,const LLT DstTy,const TargetRegisterClass * DstRC,unsigned Scalar,MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,MachineRegisterInfo & MRI) const1564 bool AArch64InstructionSelector::emitScalarToVector(
1565     unsigned &Dst, const LLT DstTy, const TargetRegisterClass *DstRC,
1566     unsigned Scalar, MachineBasicBlock &MBB,
1567     MachineBasicBlock::iterator MBBI, MachineRegisterInfo &MRI) const {
1568   Dst = MRI.createVirtualRegister(DstRC);
1569 
1570   unsigned UndefVec = MRI.createVirtualRegister(DstRC);
1571   MachineInstr &UndefMI = *BuildMI(MBB, MBBI, MBBI->getDebugLoc(),
1572                                    TII.get(TargetOpcode::IMPLICIT_DEF))
1573                                .addDef(UndefVec);
1574 
1575   auto BuildFn = [&](unsigned SubregIndex) {
1576     MachineInstr &InsMI = *BuildMI(MBB, MBBI, MBBI->getDebugLoc(),
1577                                    TII.get(TargetOpcode::INSERT_SUBREG))
1578                                .addDef(Dst)
1579                                .addUse(UndefVec)
1580                                .addUse(Scalar)
1581                                .addImm(SubregIndex);
1582     constrainSelectedInstRegOperands(UndefMI, TII, TRI, RBI);
1583     return constrainSelectedInstRegOperands(InsMI, TII, TRI, RBI);
1584   };
1585 
1586   switch (DstTy.getElementType().getSizeInBits()) {
1587   case 32:
1588     return BuildFn(AArch64::ssub);
1589   case 64:
1590     return BuildFn(AArch64::dsub);
1591   default:
1592     return false;
1593   }
1594 }
1595 
selectMergeValues(MachineInstr & I,MachineRegisterInfo & MRI) const1596 bool AArch64InstructionSelector::selectMergeValues(
1597     MachineInstr &I, MachineRegisterInfo &MRI) const {
1598   assert(I.getOpcode() == TargetOpcode::G_MERGE_VALUES && "unexpected opcode");
1599   const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
1600   const LLT SrcTy = MRI.getType(I.getOperand(1).getReg());
1601   assert(!DstTy.isVector() && !SrcTy.isVector() && "invalid merge operation");
1602 
1603   // At the moment we only support merging two s32s into an s64.
1604   if (I.getNumOperands() != 3)
1605     return false;
1606   if (DstTy.getSizeInBits() != 64 || SrcTy.getSizeInBits() != 32)
1607     return false;
1608   const RegisterBank &RB = *RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI);
1609   if (RB.getID() != AArch64::GPRRegBankID)
1610     return false;
1611 
1612   auto *DstRC = &AArch64::GPR64RegClass;
1613   unsigned SubToRegDef = MRI.createVirtualRegister(DstRC);
1614   MachineInstr &SubRegMI = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1615                                     TII.get(TargetOpcode::SUBREG_TO_REG))
1616                                 .addDef(SubToRegDef)
1617                                 .addImm(0)
1618                                 .addUse(I.getOperand(1).getReg())
1619                                 .addImm(AArch64::sub_32);
1620   unsigned SubToRegDef2 = MRI.createVirtualRegister(DstRC);
1621   // Need to anyext the second scalar before we can use bfm
1622   MachineInstr &SubRegMI2 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1623                                     TII.get(TargetOpcode::SUBREG_TO_REG))
1624                                 .addDef(SubToRegDef2)
1625                                 .addImm(0)
1626                                 .addUse(I.getOperand(2).getReg())
1627                                 .addImm(AArch64::sub_32);
1628   MachineInstr &BFM =
1629       *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::BFMXri))
1630            .addDef(I.getOperand(0).getReg())
1631            .addUse(SubToRegDef)
1632            .addUse(SubToRegDef2)
1633            .addImm(32)
1634            .addImm(31);
1635   constrainSelectedInstRegOperands(SubRegMI, TII, TRI, RBI);
1636   constrainSelectedInstRegOperands(SubRegMI2, TII, TRI, RBI);
1637   constrainSelectedInstRegOperands(BFM, TII, TRI, RBI);
1638   I.eraseFromParent();
1639   return true;
1640 }
1641 
selectBuildVector(MachineInstr & I,MachineRegisterInfo & MRI) const1642 bool AArch64InstructionSelector::selectBuildVector(
1643     MachineInstr &I, MachineRegisterInfo &MRI) const {
1644   assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
1645   // Until we port more of the optimized selections, for now just use a vector
1646   // insert sequence.
1647   const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
1648   const LLT EltTy = MRI.getType(I.getOperand(1).getReg());
1649   unsigned EltSize = EltTy.getSizeInBits();
1650   if (EltSize < 32 || EltSize > 64)
1651     return false; // Don't support all element types yet.
1652   const RegisterBank &RB = *RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI);
1653   unsigned Opc;
1654   unsigned SubregIdx;
1655   if (RB.getID() == AArch64::GPRRegBankID) {
1656     if (EltSize == 32) {
1657       Opc = AArch64::INSvi32gpr;
1658       SubregIdx = AArch64::ssub;
1659     } else {
1660       Opc = AArch64::INSvi64gpr;
1661       SubregIdx = AArch64::dsub;
1662     }
1663   } else {
1664     if (EltSize == 32) {
1665       Opc = AArch64::INSvi32lane;
1666       SubregIdx = AArch64::ssub;
1667     } else {
1668       Opc = AArch64::INSvi64lane;
1669       SubregIdx = AArch64::dsub;
1670     }
1671   }
1672 
1673   if (EltSize * DstTy.getNumElements() != 128)
1674     return false; // Don't handle unpacked vectors yet.
1675 
1676   unsigned DstVec = 0;
1677   const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(
1678       DstTy, RBI.getRegBank(AArch64::FPRRegBankID), RBI);
1679   emitScalarToVector(DstVec, DstTy, DstRC, I.getOperand(1).getReg(),
1680                      *I.getParent(), I.getIterator(), MRI);
1681   for (unsigned i = 2, e = DstTy.getSizeInBits() / EltSize + 1; i < e; ++i) {
1682     unsigned InsDef;
1683     // For the last insert re-use the dst reg of the G_BUILD_VECTOR.
1684     if (i + 1 < e)
1685       InsDef = MRI.createVirtualRegister(DstRC);
1686     else
1687       InsDef = I.getOperand(0).getReg();
1688     unsigned LaneIdx = i - 1;
1689     if (RB.getID() == AArch64::FPRRegBankID) {
1690       unsigned ImpDef = MRI.createVirtualRegister(DstRC);
1691       MachineInstr &ImpDefMI = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1692                                         TII.get(TargetOpcode::IMPLICIT_DEF))
1693                                     .addDef(ImpDef);
1694       unsigned InsSubDef = MRI.createVirtualRegister(DstRC);
1695       MachineInstr &InsSubMI = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1696                                         TII.get(TargetOpcode::INSERT_SUBREG))
1697                                     .addDef(InsSubDef)
1698                                     .addUse(ImpDef)
1699                                     .addUse(I.getOperand(i).getReg())
1700                                     .addImm(SubregIdx);
1701       MachineInstr &InsEltMI =
1702           *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opc))
1703                .addDef(InsDef)
1704                .addUse(DstVec)
1705                .addImm(LaneIdx)
1706                .addUse(InsSubDef)
1707                .addImm(0);
1708       constrainSelectedInstRegOperands(ImpDefMI, TII, TRI, RBI);
1709       constrainSelectedInstRegOperands(InsSubMI, TII, TRI, RBI);
1710       constrainSelectedInstRegOperands(InsEltMI, TII, TRI, RBI);
1711       DstVec = InsDef;
1712     } else {
1713       MachineInstr &InsMI =
1714           *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opc))
1715                .addDef(InsDef)
1716                .addUse(DstVec)
1717                .addImm(LaneIdx)
1718                .addUse(I.getOperand(i).getReg());
1719       constrainSelectedInstRegOperands(InsMI, TII, TRI, RBI);
1720       DstVec = InsDef;
1721     }
1722   }
1723   I.eraseFromParent();
1724   return true;
1725 }
1726 
1727 /// SelectArithImmed - Select an immediate value that can be represented as
1728 /// a 12-bit value shifted left by either 0 or 12.  If so, return true with
1729 /// Val set to the 12-bit value and Shift set to the shifter operand.
1730 InstructionSelector::ComplexRendererFns
selectArithImmed(MachineOperand & Root) const1731 AArch64InstructionSelector::selectArithImmed(MachineOperand &Root) const {
1732   MachineInstr &MI = *Root.getParent();
1733   MachineBasicBlock &MBB = *MI.getParent();
1734   MachineFunction &MF = *MBB.getParent();
1735   MachineRegisterInfo &MRI = MF.getRegInfo();
1736 
1737   // This function is called from the addsub_shifted_imm ComplexPattern,
1738   // which lists [imm] as the list of opcode it's interested in, however
1739   // we still need to check whether the operand is actually an immediate
1740   // here because the ComplexPattern opcode list is only used in
1741   // root-level opcode matching.
1742   uint64_t Immed;
1743   if (Root.isImm())
1744     Immed = Root.getImm();
1745   else if (Root.isCImm())
1746     Immed = Root.getCImm()->getZExtValue();
1747   else if (Root.isReg()) {
1748     MachineInstr *Def = MRI.getVRegDef(Root.getReg());
1749     if (Def->getOpcode() != TargetOpcode::G_CONSTANT)
1750       return None;
1751     MachineOperand &Op1 = Def->getOperand(1);
1752     if (!Op1.isCImm() || Op1.getCImm()->getBitWidth() > 64)
1753       return None;
1754     Immed = Op1.getCImm()->getZExtValue();
1755   } else
1756     return None;
1757 
1758   unsigned ShiftAmt;
1759 
1760   if (Immed >> 12 == 0) {
1761     ShiftAmt = 0;
1762   } else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
1763     ShiftAmt = 12;
1764     Immed = Immed >> 12;
1765   } else
1766     return None;
1767 
1768   unsigned ShVal = AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftAmt);
1769   return {{
1770       [=](MachineInstrBuilder &MIB) { MIB.addImm(Immed); },
1771       [=](MachineInstrBuilder &MIB) { MIB.addImm(ShVal); },
1772   }};
1773 }
1774 
1775 /// Select a "register plus unscaled signed 9-bit immediate" address.  This
1776 /// should only match when there is an offset that is not valid for a scaled
1777 /// immediate addressing mode.  The "Size" argument is the size in bytes of the
1778 /// memory reference, which is needed here to know what is valid for a scaled
1779 /// immediate.
1780 InstructionSelector::ComplexRendererFns
selectAddrModeUnscaled(MachineOperand & Root,unsigned Size) const1781 AArch64InstructionSelector::selectAddrModeUnscaled(MachineOperand &Root,
1782                                                    unsigned Size) const {
1783   MachineRegisterInfo &MRI =
1784       Root.getParent()->getParent()->getParent()->getRegInfo();
1785 
1786   if (!Root.isReg())
1787     return None;
1788 
1789   if (!isBaseWithConstantOffset(Root, MRI))
1790     return None;
1791 
1792   MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
1793   if (!RootDef)
1794     return None;
1795 
1796   MachineOperand &OffImm = RootDef->getOperand(2);
1797   if (!OffImm.isReg())
1798     return None;
1799   MachineInstr *RHS = MRI.getVRegDef(OffImm.getReg());
1800   if (!RHS || RHS->getOpcode() != TargetOpcode::G_CONSTANT)
1801     return None;
1802   int64_t RHSC;
1803   MachineOperand &RHSOp1 = RHS->getOperand(1);
1804   if (!RHSOp1.isCImm() || RHSOp1.getCImm()->getBitWidth() > 64)
1805     return None;
1806   RHSC = RHSOp1.getCImm()->getSExtValue();
1807 
1808   // If the offset is valid as a scaled immediate, don't match here.
1809   if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Log2_32(Size)))
1810     return None;
1811   if (RHSC >= -256 && RHSC < 256) {
1812     MachineOperand &Base = RootDef->getOperand(1);
1813     return {{
1814         [=](MachineInstrBuilder &MIB) { MIB.add(Base); },
1815         [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
1816     }};
1817   }
1818   return None;
1819 }
1820 
1821 /// Select a "register plus scaled unsigned 12-bit immediate" address.  The
1822 /// "Size" argument is the size in bytes of the memory reference, which
1823 /// determines the scale.
1824 InstructionSelector::ComplexRendererFns
selectAddrModeIndexed(MachineOperand & Root,unsigned Size) const1825 AArch64InstructionSelector::selectAddrModeIndexed(MachineOperand &Root,
1826                                                   unsigned Size) const {
1827   MachineRegisterInfo &MRI =
1828       Root.getParent()->getParent()->getParent()->getRegInfo();
1829 
1830   if (!Root.isReg())
1831     return None;
1832 
1833   MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
1834   if (!RootDef)
1835     return None;
1836 
1837   if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
1838     return {{
1839         [=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); },
1840         [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
1841     }};
1842   }
1843 
1844   if (isBaseWithConstantOffset(Root, MRI)) {
1845     MachineOperand &LHS = RootDef->getOperand(1);
1846     MachineOperand &RHS = RootDef->getOperand(2);
1847     MachineInstr *LHSDef = MRI.getVRegDef(LHS.getReg());
1848     MachineInstr *RHSDef = MRI.getVRegDef(RHS.getReg());
1849     if (LHSDef && RHSDef) {
1850       int64_t RHSC = (int64_t)RHSDef->getOperand(1).getCImm()->getZExtValue();
1851       unsigned Scale = Log2_32(Size);
1852       if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Scale)) {
1853         if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
1854           return {{
1855               [=](MachineInstrBuilder &MIB) { MIB.add(LHSDef->getOperand(1)); },
1856               [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); },
1857           }};
1858 
1859         return {{
1860             [=](MachineInstrBuilder &MIB) { MIB.add(LHS); },
1861             [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); },
1862         }};
1863       }
1864     }
1865   }
1866 
1867   // Before falling back to our general case, check if the unscaled
1868   // instructions can handle this. If so, that's preferable.
1869   if (selectAddrModeUnscaled(Root, Size).hasValue())
1870     return None;
1871 
1872   return {{
1873       [=](MachineInstrBuilder &MIB) { MIB.add(Root); },
1874       [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
1875   }};
1876 }
1877 
renderTruncImm(MachineInstrBuilder & MIB,const MachineInstr & MI) const1878 void AArch64InstructionSelector::renderTruncImm(MachineInstrBuilder &MIB,
1879                                                 const MachineInstr &MI) const {
1880   const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
1881   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
1882   Optional<int64_t> CstVal = getConstantVRegVal(MI.getOperand(0).getReg(), MRI);
1883   assert(CstVal && "Expected constant value");
1884   MIB.addImm(CstVal.getValue());
1885 }
1886 
1887 namespace llvm {
1888 InstructionSelector *
createAArch64InstructionSelector(const AArch64TargetMachine & TM,AArch64Subtarget & Subtarget,AArch64RegisterBankInfo & RBI)1889 createAArch64InstructionSelector(const AArch64TargetMachine &TM,
1890                                  AArch64Subtarget &Subtarget,
1891                                  AArch64RegisterBankInfo &RBI) {
1892   return new AArch64InstructionSelector(TM, Subtarget, RBI);
1893 }
1894 }
1895