1 //===- AArch64InstructionSelector.cpp ----------------------------*- C++ -*-==//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 /// \file
10 /// This file implements the targeting of the InstructionSelector class for
11 /// AArch64.
12 /// \todo This should be generated by TableGen.
13 //===----------------------------------------------------------------------===//
14 
15 #include "AArch64InstructionSelector.h"
16 #include "AArch64InstrInfo.h"
17 #include "AArch64RegisterBankInfo.h"
18 #include "AArch64RegisterInfo.h"
19 #include "AArch64Subtarget.h"
20 #include "AArch64TargetMachine.h"
21 #include "MCTargetDesc/AArch64AddressingModes.h"
22 #include "llvm/CodeGen/MachineBasicBlock.h"
23 #include "llvm/CodeGen/MachineFunction.h"
24 #include "llvm/CodeGen/MachineInstr.h"
25 #include "llvm/CodeGen/MachineInstrBuilder.h"
26 #include "llvm/CodeGen/MachineRegisterInfo.h"
27 #include "llvm/IR/Type.h"
28 #include "llvm/Support/Debug.h"
29 #include "llvm/Support/raw_ostream.h"
30 
31 #define DEBUG_TYPE "aarch64-isel"
32 
33 using namespace llvm;
34 
35 #ifndef LLVM_BUILD_GLOBAL_ISEL
36 #error "You shouldn't build this"
37 #endif
38 
39 #include "AArch64GenGlobalISel.inc"
40 
41 AArch64InstructionSelector::AArch64InstructionSelector(
42     const AArch64TargetMachine &TM, const AArch64Subtarget &STI,
43     const AArch64RegisterBankInfo &RBI)
44   : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()),
45       TRI(*STI.getRegisterInfo()), RBI(RBI) {}
46 
47 // FIXME: This should be target-independent, inferred from the types declared
48 // for each class in the bank.
49 static const TargetRegisterClass *
50 getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB,
51                          const RegisterBankInfo &RBI) {
52   if (RB.getID() == AArch64::GPRRegBankID) {
53     if (Ty.getSizeInBits() <= 32)
54       return &AArch64::GPR32RegClass;
55     if (Ty.getSizeInBits() == 64)
56       return &AArch64::GPR64RegClass;
57     return nullptr;
58   }
59 
60   if (RB.getID() == AArch64::FPRRegBankID) {
61     if (Ty.getSizeInBits() == 32)
62       return &AArch64::FPR32RegClass;
63     if (Ty.getSizeInBits() == 64)
64       return &AArch64::FPR64RegClass;
65     if (Ty.getSizeInBits() == 128)
66       return &AArch64::FPR128RegClass;
67     return nullptr;
68   }
69 
70   return nullptr;
71 }
72 
73 /// Check whether \p I is a currently unsupported binary operation:
74 /// - it has an unsized type
75 /// - an operand is not a vreg
76 /// - all operands are not in the same bank
77 /// These are checks that should someday live in the verifier, but right now,
78 /// these are mostly limitations of the aarch64 selector.
79 static bool unsupportedBinOp(const MachineInstr &I,
80                              const AArch64RegisterBankInfo &RBI,
81                              const MachineRegisterInfo &MRI,
82                              const AArch64RegisterInfo &TRI) {
83   LLT Ty = MRI.getType(I.getOperand(0).getReg());
84   if (!Ty.isValid()) {
85     DEBUG(dbgs() << "Generic binop register should be typed\n");
86     return true;
87   }
88 
89   const RegisterBank *PrevOpBank = nullptr;
90   for (auto &MO : I.operands()) {
91     // FIXME: Support non-register operands.
92     if (!MO.isReg()) {
93       DEBUG(dbgs() << "Generic inst non-reg operands are unsupported\n");
94       return true;
95     }
96 
97     // FIXME: Can generic operations have physical registers operands? If
98     // so, this will need to be taught about that, and we'll need to get the
99     // bank out of the minimal class for the register.
100     // Either way, this needs to be documented (and possibly verified).
101     if (!TargetRegisterInfo::isVirtualRegister(MO.getReg())) {
102       DEBUG(dbgs() << "Generic inst has physical register operand\n");
103       return true;
104     }
105 
106     const RegisterBank *OpBank = RBI.getRegBank(MO.getReg(), MRI, TRI);
107     if (!OpBank) {
108       DEBUG(dbgs() << "Generic register has no bank or class\n");
109       return true;
110     }
111 
112     if (PrevOpBank && OpBank != PrevOpBank) {
113       DEBUG(dbgs() << "Generic inst operands have different banks\n");
114       return true;
115     }
116     PrevOpBank = OpBank;
117   }
118   return false;
119 }
120 
121 /// Select the AArch64 opcode for the basic binary operation \p GenericOpc
122 /// (such as G_OR or G_ADD), appropriate for the register bank \p RegBankID
123 /// and of size \p OpSize.
124 /// \returns \p GenericOpc if the combination is unsupported.
125 static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID,
126                                unsigned OpSize) {
127   switch (RegBankID) {
128   case AArch64::GPRRegBankID:
129     if (OpSize <= 32) {
130       assert((OpSize == 32 || (GenericOpc != TargetOpcode::G_SDIV &&
131                                GenericOpc != TargetOpcode::G_UDIV &&
132                                GenericOpc != TargetOpcode::G_LSHR &&
133                                GenericOpc != TargetOpcode::G_ASHR)) &&
134              "operation should have been legalized before now");
135 
136       switch (GenericOpc) {
137       case TargetOpcode::G_OR:
138         return AArch64::ORRWrr;
139       case TargetOpcode::G_XOR:
140         return AArch64::EORWrr;
141       case TargetOpcode::G_AND:
142         return AArch64::ANDWrr;
143       case TargetOpcode::G_ADD:
144         assert(OpSize != 32 && "s32 G_ADD should have been selected");
145         return AArch64::ADDWrr;
146       case TargetOpcode::G_SUB:
147         return AArch64::SUBWrr;
148       case TargetOpcode::G_SHL:
149         return AArch64::LSLVWr;
150       case TargetOpcode::G_LSHR:
151         return AArch64::LSRVWr;
152       case TargetOpcode::G_ASHR:
153         return AArch64::ASRVWr;
154       case TargetOpcode::G_SDIV:
155         return AArch64::SDIVWr;
156       case TargetOpcode::G_UDIV:
157         return AArch64::UDIVWr;
158       default:
159         return GenericOpc;
160       }
161     } else if (OpSize == 64) {
162       switch (GenericOpc) {
163       case TargetOpcode::G_OR:
164         return AArch64::ORRXrr;
165       case TargetOpcode::G_XOR:
166         return AArch64::EORXrr;
167       case TargetOpcode::G_AND:
168         return AArch64::ANDXrr;
169       case TargetOpcode::G_GEP:
170         return AArch64::ADDXrr;
171       case TargetOpcode::G_SUB:
172         return AArch64::SUBXrr;
173       case TargetOpcode::G_SHL:
174         return AArch64::LSLVXr;
175       case TargetOpcode::G_LSHR:
176         return AArch64::LSRVXr;
177       case TargetOpcode::G_ASHR:
178         return AArch64::ASRVXr;
179       case TargetOpcode::G_SDIV:
180         return AArch64::SDIVXr;
181       case TargetOpcode::G_UDIV:
182         return AArch64::UDIVXr;
183       default:
184         return GenericOpc;
185       }
186     }
187   case AArch64::FPRRegBankID:
188     switch (OpSize) {
189     case 32:
190       switch (GenericOpc) {
191       case TargetOpcode::G_FADD:
192         return AArch64::FADDSrr;
193       case TargetOpcode::G_FSUB:
194         return AArch64::FSUBSrr;
195       case TargetOpcode::G_FMUL:
196         return AArch64::FMULSrr;
197       case TargetOpcode::G_FDIV:
198         return AArch64::FDIVSrr;
199       default:
200         return GenericOpc;
201       }
202     case 64:
203       switch (GenericOpc) {
204       case TargetOpcode::G_FADD:
205         return AArch64::FADDDrr;
206       case TargetOpcode::G_FSUB:
207         return AArch64::FSUBDrr;
208       case TargetOpcode::G_FMUL:
209         return AArch64::FMULDrr;
210       case TargetOpcode::G_FDIV:
211         return AArch64::FDIVDrr;
212       case TargetOpcode::G_OR:
213         return AArch64::ORRv8i8;
214       default:
215         return GenericOpc;
216       }
217     }
218   };
219   return GenericOpc;
220 }
221 
222 /// Select the AArch64 opcode for the G_LOAD or G_STORE operation \p GenericOpc,
223 /// appropriate for the (value) register bank \p RegBankID and of memory access
224 /// size \p OpSize.  This returns the variant with the base+unsigned-immediate
225 /// addressing mode (e.g., LDRXui).
226 /// \returns \p GenericOpc if the combination is unsupported.
227 static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID,
228                                     unsigned OpSize) {
229   const bool isStore = GenericOpc == TargetOpcode::G_STORE;
230   switch (RegBankID) {
231   case AArch64::GPRRegBankID:
232     switch (OpSize) {
233     case 8:
234       return isStore ? AArch64::STRBBui : AArch64::LDRBBui;
235     case 16:
236       return isStore ? AArch64::STRHHui : AArch64::LDRHHui;
237     case 32:
238       return isStore ? AArch64::STRWui : AArch64::LDRWui;
239     case 64:
240       return isStore ? AArch64::STRXui : AArch64::LDRXui;
241     }
242   case AArch64::FPRRegBankID:
243     switch (OpSize) {
244     case 8:
245       return isStore ? AArch64::STRBui : AArch64::LDRBui;
246     case 16:
247       return isStore ? AArch64::STRHui : AArch64::LDRHui;
248     case 32:
249       return isStore ? AArch64::STRSui : AArch64::LDRSui;
250     case 64:
251       return isStore ? AArch64::STRDui : AArch64::LDRDui;
252     }
253   };
254   return GenericOpc;
255 }
256 
257 static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
258                        MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
259                        const RegisterBankInfo &RBI) {
260 
261   unsigned DstReg = I.getOperand(0).getReg();
262   if (TargetRegisterInfo::isPhysicalRegister(DstReg)) {
263     assert(I.isCopy() && "Generic operators do not allow physical registers");
264     return true;
265   }
266 
267   const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
268   const unsigned DstSize = MRI.getType(DstReg).getSizeInBits();
269   unsigned SrcReg = I.getOperand(1).getReg();
270   const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);
271   (void)SrcSize;
272   assert((!TargetRegisterInfo::isPhysicalRegister(SrcReg) || I.isCopy()) &&
273          "No phys reg on generic operators");
274   assert(
275       (DstSize == SrcSize ||
276        // Copies are a mean to setup initial types, the number of
277        // bits may not exactly match.
278        (TargetRegisterInfo::isPhysicalRegister(SrcReg) &&
279         DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI)) ||
280        // Copies are a mean to copy bits around, as long as we are
281        // on the same register class, that's fine. Otherwise, that
282        // means we need some SUBREG_TO_REG or AND & co.
283        (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) &&
284       "Copy with different width?!");
285   assert((DstSize <= 64 || RegBank.getID() == AArch64::FPRRegBankID) &&
286          "GPRs cannot get more than 64-bit width values");
287   const TargetRegisterClass *RC = nullptr;
288 
289   if (RegBank.getID() == AArch64::FPRRegBankID) {
290     if (DstSize <= 32)
291       RC = &AArch64::FPR32RegClass;
292     else if (DstSize <= 64)
293       RC = &AArch64::FPR64RegClass;
294     else if (DstSize <= 128)
295       RC = &AArch64::FPR128RegClass;
296     else {
297       DEBUG(dbgs() << "Unexpected bitcast size " << DstSize << '\n');
298       return false;
299     }
300   } else {
301     assert(RegBank.getID() == AArch64::GPRRegBankID &&
302            "Bitcast for the flags?");
303     RC =
304         DstSize <= 32 ? &AArch64::GPR32allRegClass : &AArch64::GPR64allRegClass;
305   }
306 
307   // No need to constrain SrcReg. It will get constrained when
308   // we hit another of its use or its defs.
309   // Copies do not have constraints.
310   if (!RBI.constrainGenericRegister(DstReg, *RC, MRI)) {
311     DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
312                  << " operand\n");
313     return false;
314   }
315   I.setDesc(TII.get(AArch64::COPY));
316   return true;
317 }
318 
319 static unsigned selectFPConvOpc(unsigned GenericOpc, LLT DstTy, LLT SrcTy) {
320   if (!DstTy.isScalar() || !SrcTy.isScalar())
321     return GenericOpc;
322 
323   const unsigned DstSize = DstTy.getSizeInBits();
324   const unsigned SrcSize = SrcTy.getSizeInBits();
325 
326   switch (DstSize) {
327   case 32:
328     switch (SrcSize) {
329     case 32:
330       switch (GenericOpc) {
331       case TargetOpcode::G_SITOFP:
332         return AArch64::SCVTFUWSri;
333       case TargetOpcode::G_UITOFP:
334         return AArch64::UCVTFUWSri;
335       case TargetOpcode::G_FPTOSI:
336         return AArch64::FCVTZSUWSr;
337       case TargetOpcode::G_FPTOUI:
338         return AArch64::FCVTZUUWSr;
339       default:
340         return GenericOpc;
341       }
342     case 64:
343       switch (GenericOpc) {
344       case TargetOpcode::G_SITOFP:
345         return AArch64::SCVTFUXSri;
346       case TargetOpcode::G_UITOFP:
347         return AArch64::UCVTFUXSri;
348       case TargetOpcode::G_FPTOSI:
349         return AArch64::FCVTZSUWDr;
350       case TargetOpcode::G_FPTOUI:
351         return AArch64::FCVTZUUWDr;
352       default:
353         return GenericOpc;
354       }
355     default:
356       return GenericOpc;
357     }
358   case 64:
359     switch (SrcSize) {
360     case 32:
361       switch (GenericOpc) {
362       case TargetOpcode::G_SITOFP:
363         return AArch64::SCVTFUWDri;
364       case TargetOpcode::G_UITOFP:
365         return AArch64::UCVTFUWDri;
366       case TargetOpcode::G_FPTOSI:
367         return AArch64::FCVTZSUXSr;
368       case TargetOpcode::G_FPTOUI:
369         return AArch64::FCVTZUUXSr;
370       default:
371         return GenericOpc;
372       }
373     case 64:
374       switch (GenericOpc) {
375       case TargetOpcode::G_SITOFP:
376         return AArch64::SCVTFUXDri;
377       case TargetOpcode::G_UITOFP:
378         return AArch64::UCVTFUXDri;
379       case TargetOpcode::G_FPTOSI:
380         return AArch64::FCVTZSUXDr;
381       case TargetOpcode::G_FPTOUI:
382         return AArch64::FCVTZUUXDr;
383       default:
384         return GenericOpc;
385       }
386     default:
387       return GenericOpc;
388     }
389   default:
390     return GenericOpc;
391   };
392   return GenericOpc;
393 }
394 
395 static AArch64CC::CondCode changeICMPPredToAArch64CC(CmpInst::Predicate P) {
396   switch (P) {
397   default:
398     llvm_unreachable("Unknown condition code!");
399   case CmpInst::ICMP_NE:
400     return AArch64CC::NE;
401   case CmpInst::ICMP_EQ:
402     return AArch64CC::EQ;
403   case CmpInst::ICMP_SGT:
404     return AArch64CC::GT;
405   case CmpInst::ICMP_SGE:
406     return AArch64CC::GE;
407   case CmpInst::ICMP_SLT:
408     return AArch64CC::LT;
409   case CmpInst::ICMP_SLE:
410     return AArch64CC::LE;
411   case CmpInst::ICMP_UGT:
412     return AArch64CC::HI;
413   case CmpInst::ICMP_UGE:
414     return AArch64CC::HS;
415   case CmpInst::ICMP_ULT:
416     return AArch64CC::LO;
417   case CmpInst::ICMP_ULE:
418     return AArch64CC::LS;
419   }
420 }
421 
422 static void changeFCMPPredToAArch64CC(CmpInst::Predicate P,
423                                       AArch64CC::CondCode &CondCode,
424                                       AArch64CC::CondCode &CondCode2) {
425   CondCode2 = AArch64CC::AL;
426   switch (P) {
427   default:
428     llvm_unreachable("Unknown FP condition!");
429   case CmpInst::FCMP_OEQ:
430     CondCode = AArch64CC::EQ;
431     break;
432   case CmpInst::FCMP_OGT:
433     CondCode = AArch64CC::GT;
434     break;
435   case CmpInst::FCMP_OGE:
436     CondCode = AArch64CC::GE;
437     break;
438   case CmpInst::FCMP_OLT:
439     CondCode = AArch64CC::MI;
440     break;
441   case CmpInst::FCMP_OLE:
442     CondCode = AArch64CC::LS;
443     break;
444   case CmpInst::FCMP_ONE:
445     CondCode = AArch64CC::MI;
446     CondCode2 = AArch64CC::GT;
447     break;
448   case CmpInst::FCMP_ORD:
449     CondCode = AArch64CC::VC;
450     break;
451   case CmpInst::FCMP_UNO:
452     CondCode = AArch64CC::VS;
453     break;
454   case CmpInst::FCMP_UEQ:
455     CondCode = AArch64CC::EQ;
456     CondCode2 = AArch64CC::VS;
457     break;
458   case CmpInst::FCMP_UGT:
459     CondCode = AArch64CC::HI;
460     break;
461   case CmpInst::FCMP_UGE:
462     CondCode = AArch64CC::PL;
463     break;
464   case CmpInst::FCMP_ULT:
465     CondCode = AArch64CC::LT;
466     break;
467   case CmpInst::FCMP_ULE:
468     CondCode = AArch64CC::LE;
469     break;
470   case CmpInst::FCMP_UNE:
471     CondCode = AArch64CC::NE;
472     break;
473   }
474 }
475 
476 bool AArch64InstructionSelector::select(MachineInstr &I) const {
477   assert(I.getParent() && "Instruction should be in a basic block!");
478   assert(I.getParent()->getParent() && "Instruction should be in a function!");
479 
480   MachineBasicBlock &MBB = *I.getParent();
481   MachineFunction &MF = *MBB.getParent();
482   MachineRegisterInfo &MRI = MF.getRegInfo();
483 
484   unsigned Opcode = I.getOpcode();
485   if (!isPreISelGenericOpcode(I.getOpcode())) {
486     // Certain non-generic instructions also need some special handling.
487 
488     if (Opcode ==  TargetOpcode::LOAD_STACK_GUARD)
489       return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
490 
491     if (Opcode == TargetOpcode::PHI) {
492       const unsigned DefReg = I.getOperand(0).getReg();
493       const LLT DefTy = MRI.getType(DefReg);
494 
495       const TargetRegisterClass *DefRC = nullptr;
496       if (TargetRegisterInfo::isPhysicalRegister(DefReg)) {
497         DefRC = TRI.getRegClass(DefReg);
498       } else {
499         const RegClassOrRegBank &RegClassOrBank =
500             MRI.getRegClassOrRegBank(DefReg);
501 
502         DefRC = RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
503         if (!DefRC) {
504           if (!DefTy.isValid()) {
505             DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
506             return false;
507           }
508           const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>();
509           DefRC = getRegClassForTypeOnBank(DefTy, RB, RBI);
510           if (!DefRC) {
511             DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
512             return false;
513           }
514         }
515       }
516 
517       return RBI.constrainGenericRegister(DefReg, *DefRC, MRI);
518     }
519 
520     if (I.isCopy())
521       return selectCopy(I, TII, MRI, TRI, RBI);
522 
523     return true;
524   }
525 
526 
527   if (I.getNumOperands() != I.getNumExplicitOperands()) {
528     DEBUG(dbgs() << "Generic instruction has unexpected implicit operands\n");
529     return false;
530   }
531 
532   if (selectImpl(I))
533     return true;
534 
535   LLT Ty =
536       I.getOperand(0).isReg() ? MRI.getType(I.getOperand(0).getReg()) : LLT{};
537 
538   switch (Opcode) {
539   case TargetOpcode::G_BRCOND: {
540     if (Ty.getSizeInBits() > 32) {
541       // We shouldn't need this on AArch64, but it would be implemented as an
542       // EXTRACT_SUBREG followed by a TBNZW because TBNZX has no encoding if the
543       // bit being tested is < 32.
544       DEBUG(dbgs() << "G_BRCOND has type: " << Ty
545                    << ", expected at most 32-bits");
546       return false;
547     }
548 
549     const unsigned CondReg = I.getOperand(0).getReg();
550     MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
551 
552     auto MIB = BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::TBNZW))
553                    .addUse(CondReg)
554                    .addImm(/*bit offset=*/0)
555                    .addMBB(DestMBB);
556 
557     I.eraseFromParent();
558     return constrainSelectedInstRegOperands(*MIB.getInstr(), TII, TRI, RBI);
559   }
560 
561   case TargetOpcode::G_FCONSTANT:
562   case TargetOpcode::G_CONSTANT: {
563     const bool isFP = Opcode == TargetOpcode::G_FCONSTANT;
564 
565     const LLT s32 = LLT::scalar(32);
566     const LLT s64 = LLT::scalar(64);
567     const LLT p0 = LLT::pointer(0, 64);
568 
569     const unsigned DefReg = I.getOperand(0).getReg();
570     const LLT DefTy = MRI.getType(DefReg);
571     const unsigned DefSize = DefTy.getSizeInBits();
572     const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
573 
574     // FIXME: Redundant check, but even less readable when factored out.
575     if (isFP) {
576       if (Ty != s32 && Ty != s64) {
577         DEBUG(dbgs() << "Unable to materialize FP " << Ty
578                      << " constant, expected: " << s32 << " or " << s64
579                      << '\n');
580         return false;
581       }
582 
583       if (RB.getID() != AArch64::FPRRegBankID) {
584         DEBUG(dbgs() << "Unable to materialize FP " << Ty
585                      << " constant on bank: " << RB << ", expected: FPR\n");
586         return false;
587       }
588     } else {
589       if (Ty != s32 && Ty != s64 && Ty != p0) {
590         DEBUG(dbgs() << "Unable to materialize integer " << Ty
591                      << " constant, expected: " << s32 << ", " << s64 << ", or "
592                      << p0 << '\n');
593         return false;
594       }
595 
596       if (RB.getID() != AArch64::GPRRegBankID) {
597         DEBUG(dbgs() << "Unable to materialize integer " << Ty
598                      << " constant on bank: " << RB << ", expected: GPR\n");
599         return false;
600       }
601     }
602 
603     const unsigned MovOpc =
604         DefSize == 32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;
605 
606     I.setDesc(TII.get(MovOpc));
607 
608     if (isFP) {
609       const TargetRegisterClass &GPRRC =
610           DefSize == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass;
611       const TargetRegisterClass &FPRRC =
612           DefSize == 32 ? AArch64::FPR32RegClass : AArch64::FPR64RegClass;
613 
614       const unsigned DefGPRReg = MRI.createVirtualRegister(&GPRRC);
615       MachineOperand &RegOp = I.getOperand(0);
616       RegOp.setReg(DefGPRReg);
617 
618       BuildMI(MBB, std::next(I.getIterator()), I.getDebugLoc(),
619               TII.get(AArch64::COPY))
620           .addDef(DefReg)
621           .addUse(DefGPRReg);
622 
623       if (!RBI.constrainGenericRegister(DefReg, FPRRC, MRI)) {
624         DEBUG(dbgs() << "Failed to constrain G_FCONSTANT def operand\n");
625         return false;
626       }
627 
628       MachineOperand &ImmOp = I.getOperand(1);
629       // FIXME: Is going through int64_t always correct?
630       ImmOp.ChangeToImmediate(
631           ImmOp.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
632     } else {
633       uint64_t Val = I.getOperand(1).getCImm()->getZExtValue();
634       I.getOperand(1).ChangeToImmediate(Val);
635     }
636 
637     constrainSelectedInstRegOperands(I, TII, TRI, RBI);
638     return true;
639   }
640 
641   case TargetOpcode::G_FRAME_INDEX: {
642     // allocas and G_FRAME_INDEX are only supported in addrspace(0).
643     if (Ty != LLT::pointer(0, 64)) {
644       DEBUG(dbgs() << "G_FRAME_INDEX pointer has type: " << Ty
645             << ", expected: " << LLT::pointer(0, 64) << '\n');
646       return false;
647     }
648 
649     I.setDesc(TII.get(AArch64::ADDXri));
650 
651     // MOs for a #0 shifted immediate.
652     I.addOperand(MachineOperand::CreateImm(0));
653     I.addOperand(MachineOperand::CreateImm(0));
654 
655     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
656   }
657 
658   case TargetOpcode::G_GLOBAL_VALUE: {
659     auto GV = I.getOperand(1).getGlobal();
660     if (GV->isThreadLocal()) {
661       // FIXME: we don't support TLS yet.
662       return false;
663     }
664     unsigned char OpFlags = STI.ClassifyGlobalReference(GV, TM);
665     if (OpFlags & AArch64II::MO_GOT) {
666       I.setDesc(TII.get(AArch64::LOADgot));
667       I.getOperand(1).setTargetFlags(OpFlags);
668     } else {
669       I.setDesc(TII.get(AArch64::MOVaddr));
670       I.getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_PAGE);
671       MachineInstrBuilder MIB(MF, I);
672       MIB.addGlobalAddress(GV, I.getOperand(1).getOffset(),
673                            OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
674     }
675     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
676   }
677 
678   case TargetOpcode::G_LOAD:
679   case TargetOpcode::G_STORE: {
680     LLT MemTy = Ty;
681     LLT PtrTy = MRI.getType(I.getOperand(1).getReg());
682 
683     if (PtrTy != LLT::pointer(0, 64)) {
684       DEBUG(dbgs() << "Load/Store pointer has type: " << PtrTy
685                    << ", expected: " << LLT::pointer(0, 64) << '\n');
686       return false;
687     }
688 
689 #ifndef NDEBUG
690     // Sanity-check the pointer register.
691     const unsigned PtrReg = I.getOperand(1).getReg();
692     const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, MRI, TRI);
693     assert(PtrRB.getID() == AArch64::GPRRegBankID &&
694            "Load/Store pointer operand isn't a GPR");
695     assert(MRI.getType(PtrReg).isPointer() &&
696            "Load/Store pointer operand isn't a pointer");
697 #endif
698 
699     const unsigned ValReg = I.getOperand(0).getReg();
700     const RegisterBank &RB = *RBI.getRegBank(ValReg, MRI, TRI);
701 
702     const unsigned NewOpc =
703         selectLoadStoreUIOp(I.getOpcode(), RB.getID(), MemTy.getSizeInBits());
704     if (NewOpc == I.getOpcode())
705       return false;
706 
707     I.setDesc(TII.get(NewOpc));
708 
709     I.addOperand(MachineOperand::CreateImm(0));
710     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
711   }
712 
713   case TargetOpcode::G_MUL: {
714     // Reject the various things we don't support yet.
715     if (unsupportedBinOp(I, RBI, MRI, TRI))
716       return false;
717 
718     const unsigned DefReg = I.getOperand(0).getReg();
719     const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
720 
721     if (RB.getID() != AArch64::GPRRegBankID) {
722       DEBUG(dbgs() << "G_MUL on bank: " << RB << ", expected: GPR\n");
723       return false;
724     }
725 
726     unsigned ZeroReg;
727     unsigned NewOpc;
728     if (Ty.isScalar() && Ty.getSizeInBits() <= 32) {
729       NewOpc = AArch64::MADDWrrr;
730       ZeroReg = AArch64::WZR;
731     } else if (Ty == LLT::scalar(64)) {
732       NewOpc = AArch64::MADDXrrr;
733       ZeroReg = AArch64::XZR;
734     } else {
735       DEBUG(dbgs() << "G_MUL has type: " << Ty << ", expected: "
736                    << LLT::scalar(32) << " or " << LLT::scalar(64) << '\n');
737       return false;
738     }
739 
740     I.setDesc(TII.get(NewOpc));
741 
742     I.addOperand(MachineOperand::CreateReg(ZeroReg, /*isDef=*/false));
743 
744     // Now that we selected an opcode, we need to constrain the register
745     // operands to use appropriate classes.
746     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
747   }
748 
749   case TargetOpcode::G_FADD:
750   case TargetOpcode::G_FSUB:
751   case TargetOpcode::G_FMUL:
752   case TargetOpcode::G_FDIV:
753 
754   case TargetOpcode::G_OR:
755   case TargetOpcode::G_XOR:
756   case TargetOpcode::G_AND:
757   case TargetOpcode::G_SHL:
758   case TargetOpcode::G_LSHR:
759   case TargetOpcode::G_ASHR:
760   case TargetOpcode::G_SDIV:
761   case TargetOpcode::G_UDIV:
762   case TargetOpcode::G_ADD:
763   case TargetOpcode::G_SUB:
764   case TargetOpcode::G_GEP: {
765     // Reject the various things we don't support yet.
766     if (unsupportedBinOp(I, RBI, MRI, TRI))
767       return false;
768 
769     const unsigned OpSize = Ty.getSizeInBits();
770 
771     const unsigned DefReg = I.getOperand(0).getReg();
772     const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
773 
774     const unsigned NewOpc = selectBinaryOp(I.getOpcode(), RB.getID(), OpSize);
775     if (NewOpc == I.getOpcode())
776       return false;
777 
778     I.setDesc(TII.get(NewOpc));
779     // FIXME: Should the type be always reset in setDesc?
780 
781     // Now that we selected an opcode, we need to constrain the register
782     // operands to use appropriate classes.
783     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
784   }
785 
786   case TargetOpcode::G_PTRTOINT:
787   case TargetOpcode::G_TRUNC: {
788     const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
789     const LLT SrcTy = MRI.getType(I.getOperand(1).getReg());
790 
791     const unsigned DstReg = I.getOperand(0).getReg();
792     const unsigned SrcReg = I.getOperand(1).getReg();
793 
794     const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
795     const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
796 
797     if (DstRB.getID() != SrcRB.getID()) {
798       DEBUG(dbgs() << "G_TRUNC input/output on different banks\n");
799       return false;
800     }
801 
802     if (DstRB.getID() == AArch64::GPRRegBankID) {
803       const TargetRegisterClass *DstRC =
804           getRegClassForTypeOnBank(DstTy, DstRB, RBI);
805       if (!DstRC)
806         return false;
807 
808       const TargetRegisterClass *SrcRC =
809           getRegClassForTypeOnBank(SrcTy, SrcRB, RBI);
810       if (!SrcRC)
811         return false;
812 
813       if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
814           !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
815         DEBUG(dbgs() << "Failed to constrain G_TRUNC\n");
816         return false;
817       }
818 
819       if (DstRC == SrcRC) {
820         // Nothing to be done
821       } else if (DstRC == &AArch64::GPR32RegClass &&
822                  SrcRC == &AArch64::GPR64RegClass) {
823         I.getOperand(1).setSubReg(AArch64::sub_32);
824       } else {
825         return false;
826       }
827 
828       I.setDesc(TII.get(TargetOpcode::COPY));
829       return true;
830     } else if (DstRB.getID() == AArch64::FPRRegBankID) {
831       if (DstTy == LLT::vector(4, 16) && SrcTy == LLT::vector(4, 32)) {
832         I.setDesc(TII.get(AArch64::XTNv4i16));
833         constrainSelectedInstRegOperands(I, TII, TRI, RBI);
834         return true;
835       }
836     }
837 
838     return false;
839   }
840 
841   case TargetOpcode::G_ANYEXT: {
842     const unsigned DstReg = I.getOperand(0).getReg();
843     const unsigned SrcReg = I.getOperand(1).getReg();
844 
845     const RegisterBank &RBDst = *RBI.getRegBank(DstReg, MRI, TRI);
846     if (RBDst.getID() != AArch64::GPRRegBankID) {
847       DEBUG(dbgs() << "G_ANYEXT on bank: " << RBDst << ", expected: GPR\n");
848       return false;
849     }
850 
851     const RegisterBank &RBSrc = *RBI.getRegBank(SrcReg, MRI, TRI);
852     if (RBSrc.getID() != AArch64::GPRRegBankID) {
853       DEBUG(dbgs() << "G_ANYEXT on bank: " << RBSrc << ", expected: GPR\n");
854       return false;
855     }
856 
857     const unsigned DstSize = MRI.getType(DstReg).getSizeInBits();
858 
859     if (DstSize == 0) {
860       DEBUG(dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n");
861       return false;
862     }
863 
864     if (DstSize != 64 && DstSize > 32) {
865       DEBUG(dbgs() << "G_ANYEXT to size: " << DstSize
866                    << ", expected: 32 or 64\n");
867       return false;
868     }
869     // At this point G_ANYEXT is just like a plain COPY, but we need
870     // to explicitly form the 64-bit value if any.
871     if (DstSize > 32) {
872       unsigned ExtSrc = MRI.createVirtualRegister(&AArch64::GPR64allRegClass);
873       BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG))
874           .addDef(ExtSrc)
875           .addImm(0)
876           .addUse(SrcReg)
877           .addImm(AArch64::sub_32);
878       I.getOperand(1).setReg(ExtSrc);
879     }
880     return selectCopy(I, TII, MRI, TRI, RBI);
881   }
882 
883   case TargetOpcode::G_ZEXT:
884   case TargetOpcode::G_SEXT: {
885     unsigned Opcode = I.getOpcode();
886     const LLT DstTy = MRI.getType(I.getOperand(0).getReg()),
887               SrcTy = MRI.getType(I.getOperand(1).getReg());
888     const bool isSigned = Opcode == TargetOpcode::G_SEXT;
889     const unsigned DefReg = I.getOperand(0).getReg();
890     const unsigned SrcReg = I.getOperand(1).getReg();
891     const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
892 
893     if (RB.getID() != AArch64::GPRRegBankID) {
894       DEBUG(dbgs() << TII.getName(I.getOpcode()) << " on bank: " << RB
895                    << ", expected: GPR\n");
896       return false;
897     }
898 
899     MachineInstr *ExtI;
900     if (DstTy == LLT::scalar(64)) {
901       // FIXME: Can we avoid manually doing this?
902       if (!RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass, MRI)) {
903         DEBUG(dbgs() << "Failed to constrain " << TII.getName(Opcode)
904                      << " operand\n");
905         return false;
906       }
907 
908       const unsigned SrcXReg =
909           MRI.createVirtualRegister(&AArch64::GPR64RegClass);
910       BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG))
911           .addDef(SrcXReg)
912           .addImm(0)
913           .addUse(SrcReg)
914           .addImm(AArch64::sub_32);
915 
916       const unsigned NewOpc = isSigned ? AArch64::SBFMXri : AArch64::UBFMXri;
917       ExtI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(NewOpc))
918                  .addDef(DefReg)
919                  .addUse(SrcXReg)
920                  .addImm(0)
921                  .addImm(SrcTy.getSizeInBits() - 1);
922     } else if (DstTy.isScalar() && DstTy.getSizeInBits() <= 32) {
923       const unsigned NewOpc = isSigned ? AArch64::SBFMWri : AArch64::UBFMWri;
924       ExtI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(NewOpc))
925                  .addDef(DefReg)
926                  .addUse(SrcReg)
927                  .addImm(0)
928                  .addImm(SrcTy.getSizeInBits() - 1);
929     } else {
930       return false;
931     }
932 
933     constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI);
934 
935     I.eraseFromParent();
936     return true;
937   }
938 
939   case TargetOpcode::G_SITOFP:
940   case TargetOpcode::G_UITOFP:
941   case TargetOpcode::G_FPTOSI:
942   case TargetOpcode::G_FPTOUI: {
943     const LLT DstTy = MRI.getType(I.getOperand(0).getReg()),
944               SrcTy = MRI.getType(I.getOperand(1).getReg());
945     const unsigned NewOpc = selectFPConvOpc(Opcode, DstTy, SrcTy);
946     if (NewOpc == Opcode)
947       return false;
948 
949     I.setDesc(TII.get(NewOpc));
950     constrainSelectedInstRegOperands(I, TII, TRI, RBI);
951 
952     return true;
953   }
954 
955 
956   case TargetOpcode::G_INTTOPTR:
957   case TargetOpcode::G_BITCAST:
958     return selectCopy(I, TII, MRI, TRI, RBI);
959 
960   case TargetOpcode::G_FPEXT: {
961     if (MRI.getType(I.getOperand(0).getReg()) != LLT::scalar(64)) {
962       DEBUG(dbgs() << "G_FPEXT to type " << Ty
963                    << ", expected: " << LLT::scalar(64) << '\n');
964       return false;
965     }
966 
967     if (MRI.getType(I.getOperand(1).getReg()) != LLT::scalar(32)) {
968       DEBUG(dbgs() << "G_FPEXT from type " << Ty
969                    << ", expected: " << LLT::scalar(32) << '\n');
970       return false;
971     }
972 
973     const unsigned DefReg = I.getOperand(0).getReg();
974     const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
975 
976     if (RB.getID() != AArch64::FPRRegBankID) {
977       DEBUG(dbgs() << "G_FPEXT on bank: " << RB << ", expected: FPR\n");
978       return false;
979     }
980 
981     I.setDesc(TII.get(AArch64::FCVTDSr));
982     constrainSelectedInstRegOperands(I, TII, TRI, RBI);
983 
984     return true;
985   }
986 
987   case TargetOpcode::G_FPTRUNC: {
988     if (MRI.getType(I.getOperand(0).getReg()) != LLT::scalar(32)) {
989       DEBUG(dbgs() << "G_FPTRUNC to type " << Ty
990                    << ", expected: " << LLT::scalar(32) << '\n');
991       return false;
992     }
993 
994     if (MRI.getType(I.getOperand(1).getReg()) != LLT::scalar(64)) {
995       DEBUG(dbgs() << "G_FPTRUNC from type " << Ty
996                    << ", expected: " << LLT::scalar(64) << '\n');
997       return false;
998     }
999 
1000     const unsigned DefReg = I.getOperand(0).getReg();
1001     const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
1002 
1003     if (RB.getID() != AArch64::FPRRegBankID) {
1004       DEBUG(dbgs() << "G_FPTRUNC on bank: " << RB << ", expected: FPR\n");
1005       return false;
1006     }
1007 
1008     I.setDesc(TII.get(AArch64::FCVTSDr));
1009     constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1010 
1011     return true;
1012   }
1013 
1014   case TargetOpcode::G_SELECT: {
1015     if (MRI.getType(I.getOperand(1).getReg()) != LLT::scalar(1)) {
1016       DEBUG(dbgs() << "G_SELECT cond has type: " << Ty
1017                    << ", expected: " << LLT::scalar(1) << '\n');
1018       return false;
1019     }
1020 
1021     const unsigned CondReg = I.getOperand(1).getReg();
1022     const unsigned TReg = I.getOperand(2).getReg();
1023     const unsigned FReg = I.getOperand(3).getReg();
1024 
1025     unsigned CSelOpc = 0;
1026 
1027     if (Ty == LLT::scalar(32)) {
1028       CSelOpc = AArch64::CSELWr;
1029     } else if (Ty == LLT::scalar(64)) {
1030       CSelOpc = AArch64::CSELXr;
1031     } else {
1032       return false;
1033     }
1034 
1035     MachineInstr &TstMI =
1036         *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ANDSWri))
1037              .addDef(AArch64::WZR)
1038              .addUse(CondReg)
1039              .addImm(AArch64_AM::encodeLogicalImmediate(1, 32));
1040 
1041     MachineInstr &CSelMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CSelOpc))
1042                                 .addDef(I.getOperand(0).getReg())
1043                                 .addUse(TReg)
1044                                 .addUse(FReg)
1045                                 .addImm(AArch64CC::NE);
1046 
1047     constrainSelectedInstRegOperands(TstMI, TII, TRI, RBI);
1048     constrainSelectedInstRegOperands(CSelMI, TII, TRI, RBI);
1049 
1050     I.eraseFromParent();
1051     return true;
1052   }
1053   case TargetOpcode::G_ICMP: {
1054     if (Ty != LLT::scalar(1)) {
1055       DEBUG(dbgs() << "G_ICMP result has type: " << Ty
1056                    << ", expected: " << LLT::scalar(1) << '\n');
1057       return false;
1058     }
1059 
1060     unsigned CmpOpc = 0;
1061     unsigned ZReg = 0;
1062 
1063     LLT CmpTy = MRI.getType(I.getOperand(2).getReg());
1064     if (CmpTy == LLT::scalar(32)) {
1065       CmpOpc = AArch64::SUBSWrr;
1066       ZReg = AArch64::WZR;
1067     } else if (CmpTy == LLT::scalar(64) || CmpTy.isPointer()) {
1068       CmpOpc = AArch64::SUBSXrr;
1069       ZReg = AArch64::XZR;
1070     } else {
1071       return false;
1072     }
1073 
1074     // CSINC increments the result by one when the condition code is false.
1075     // Therefore, we have to invert the predicate to get an increment by 1 when
1076     // the predicate is true.
1077     const AArch64CC::CondCode invCC =
1078         changeICMPPredToAArch64CC(CmpInst::getInversePredicate(
1079             (CmpInst::Predicate)I.getOperand(1).getPredicate()));
1080 
1081     MachineInstr &CmpMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CmpOpc))
1082                                .addDef(ZReg)
1083                                .addUse(I.getOperand(2).getReg())
1084                                .addUse(I.getOperand(3).getReg());
1085 
1086     MachineInstr &CSetMI =
1087         *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
1088              .addDef(I.getOperand(0).getReg())
1089              .addUse(AArch64::WZR)
1090              .addUse(AArch64::WZR)
1091              .addImm(invCC);
1092 
1093     constrainSelectedInstRegOperands(CmpMI, TII, TRI, RBI);
1094     constrainSelectedInstRegOperands(CSetMI, TII, TRI, RBI);
1095 
1096     I.eraseFromParent();
1097     return true;
1098   }
1099 
1100   case TargetOpcode::G_FCMP: {
1101     if (Ty != LLT::scalar(1)) {
1102       DEBUG(dbgs() << "G_FCMP result has type: " << Ty
1103                    << ", expected: " << LLT::scalar(1) << '\n');
1104       return false;
1105     }
1106 
1107     unsigned CmpOpc = 0;
1108     LLT CmpTy = MRI.getType(I.getOperand(2).getReg());
1109     if (CmpTy == LLT::scalar(32)) {
1110       CmpOpc = AArch64::FCMPSrr;
1111     } else if (CmpTy == LLT::scalar(64)) {
1112       CmpOpc = AArch64::FCMPDrr;
1113     } else {
1114       return false;
1115     }
1116 
1117     // FIXME: regbank
1118 
1119     AArch64CC::CondCode CC1, CC2;
1120     changeFCMPPredToAArch64CC(
1121         (CmpInst::Predicate)I.getOperand(1).getPredicate(), CC1, CC2);
1122 
1123     MachineInstr &CmpMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CmpOpc))
1124                                .addUse(I.getOperand(2).getReg())
1125                                .addUse(I.getOperand(3).getReg());
1126 
1127     const unsigned DefReg = I.getOperand(0).getReg();
1128     unsigned Def1Reg = DefReg;
1129     if (CC2 != AArch64CC::AL)
1130       Def1Reg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
1131 
1132     MachineInstr &CSetMI =
1133         *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
1134              .addDef(Def1Reg)
1135              .addUse(AArch64::WZR)
1136              .addUse(AArch64::WZR)
1137              .addImm(CC1);
1138 
1139     if (CC2 != AArch64CC::AL) {
1140       unsigned Def2Reg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
1141       MachineInstr &CSet2MI =
1142           *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
1143                .addDef(Def2Reg)
1144                .addUse(AArch64::WZR)
1145                .addUse(AArch64::WZR)
1146                .addImm(CC2);
1147       MachineInstr &OrMI =
1148           *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ORRWrr))
1149                .addDef(DefReg)
1150                .addUse(Def1Reg)
1151                .addUse(Def2Reg);
1152       constrainSelectedInstRegOperands(OrMI, TII, TRI, RBI);
1153       constrainSelectedInstRegOperands(CSet2MI, TII, TRI, RBI);
1154     }
1155 
1156     constrainSelectedInstRegOperands(CmpMI, TII, TRI, RBI);
1157     constrainSelectedInstRegOperands(CSetMI, TII, TRI, RBI);
1158 
1159     I.eraseFromParent();
1160     return true;
1161   }
1162   }
1163 
1164   return false;
1165 }
1166