1 //===- X86InstructionSelector.cpp -----------------------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 /// \file
10 /// This file implements the targeting of the InstructionSelector class for
11 /// X86.
12 /// \todo This should be generated by TableGen.
13 //===----------------------------------------------------------------------===//
14 
15 #include "MCTargetDesc/X86BaseInfo.h"
16 #include "X86InstrBuilder.h"
17 #include "X86InstrInfo.h"
18 #include "X86RegisterBankInfo.h"
19 #include "X86RegisterInfo.h"
20 #include "X86Subtarget.h"
21 #include "X86TargetMachine.h"
22 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
23 #include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h"
24 #include "llvm/CodeGen/GlobalISel/RegisterBank.h"
25 #include "llvm/CodeGen/GlobalISel/Utils.h"
26 #include "llvm/CodeGen/MachineBasicBlock.h"
27 #include "llvm/CodeGen/MachineConstantPool.h"
28 #include "llvm/CodeGen/MachineFunction.h"
29 #include "llvm/CodeGen/MachineInstr.h"
30 #include "llvm/CodeGen/MachineInstrBuilder.h"
31 #include "llvm/CodeGen/MachineMemOperand.h"
32 #include "llvm/CodeGen/MachineOperand.h"
33 #include "llvm/CodeGen/MachineRegisterInfo.h"
34 #include "llvm/IR/DataLayout.h"
35 #include "llvm/IR/InstrTypes.h"
36 #include "llvm/Support/AtomicOrdering.h"
37 #include "llvm/Support/CodeGen.h"
38 #include "llvm/Support/Debug.h"
39 #include "llvm/Support/ErrorHandling.h"
40 #include "llvm/Support/LowLevelTypeImpl.h"
41 #include "llvm/Support/MathExtras.h"
42 #include "llvm/Support/raw_ostream.h"
43 #include "llvm/Target/TargetOpcodes.h"
44 #include "llvm/Target/TargetRegisterInfo.h"
45 #include <cassert>
46 #include <cstdint>
47 #include <tuple>
48 
49 #define DEBUG_TYPE "X86-isel"
50 
51 using namespace llvm;
52 
53 namespace {
54 
55 #define GET_GLOBALISEL_PREDICATE_BITSET
56 #include "X86GenGlobalISel.inc"
57 #undef GET_GLOBALISEL_PREDICATE_BITSET
58 
59 class X86InstructionSelector : public InstructionSelector {
60 public:
61   X86InstructionSelector(const X86TargetMachine &TM, const X86Subtarget &STI,
62                          const X86RegisterBankInfo &RBI);
63 
64   bool select(MachineInstr &I) const override;
65   static const char *getName() { return DEBUG_TYPE; }
66 
67 private:
68   /// tblgen-erated 'select' implementation, used as the initial selector for
69   /// the patterns that don't require complex C++.
70   bool selectImpl(MachineInstr &I) const;
71 
72   // TODO: remove after supported by Tablegen-erated instruction selection.
73   unsigned getLoadStoreOp(const LLT &Ty, const RegisterBank &RB, unsigned Opc,
74                           uint64_t Alignment) const;
75 
76   bool selectLoadStoreOp(MachineInstr &I, MachineRegisterInfo &MRI,
77                          MachineFunction &MF) const;
78   bool selectFrameIndexOrGep(MachineInstr &I, MachineRegisterInfo &MRI,
79                              MachineFunction &MF) const;
80   bool selectGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI,
81                          MachineFunction &MF) const;
82   bool selectConstant(MachineInstr &I, MachineRegisterInfo &MRI,
83                       MachineFunction &MF) const;
84   bool selectTrunc(MachineInstr &I, MachineRegisterInfo &MRI,
85                    MachineFunction &MF) const;
86   bool selectZext(MachineInstr &I, MachineRegisterInfo &MRI,
87                   MachineFunction &MF) const;
88   bool selectAnyext(MachineInstr &I, MachineRegisterInfo &MRI,
89                     MachineFunction &MF) const;
90   bool selectCmp(MachineInstr &I, MachineRegisterInfo &MRI,
91                  MachineFunction &MF) const;
92   bool selectUadde(MachineInstr &I, MachineRegisterInfo &MRI,
93                    MachineFunction &MF) const;
94   bool selectCopy(MachineInstr &I, MachineRegisterInfo &MRI) const;
95   bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI,
96                            MachineFunction &MF) const;
97   bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI,
98                          MachineFunction &MF) const;
99   bool selectInsert(MachineInstr &I, MachineRegisterInfo &MRI,
100                     MachineFunction &MF) const;
101   bool selectExtract(MachineInstr &I, MachineRegisterInfo &MRI,
102                      MachineFunction &MF) const;
103   bool selectCondBranch(MachineInstr &I, MachineRegisterInfo &MRI,
104                         MachineFunction &MF) const;
105   bool materializeFP(MachineInstr &I, MachineRegisterInfo &MRI,
106                      MachineFunction &MF) const;
107   bool selectImplicitDefOrPHI(MachineInstr &I, MachineRegisterInfo &MRI) const;
108 
109   // emit insert subreg instruction and insert it before MachineInstr &I
110   bool emitInsertSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I,
111                         MachineRegisterInfo &MRI, MachineFunction &MF) const;
112   // emit extract subreg instruction and insert it before MachineInstr &I
113   bool emitExtractSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I,
114                          MachineRegisterInfo &MRI, MachineFunction &MF) const;
115 
116   const TargetRegisterClass *getRegClass(LLT Ty, const RegisterBank &RB) const;
117   const TargetRegisterClass *getRegClass(LLT Ty, unsigned Reg,
118                                          MachineRegisterInfo &MRI) const;
119 
120   const X86TargetMachine &TM;
121   const X86Subtarget &STI;
122   const X86InstrInfo &TII;
123   const X86RegisterInfo &TRI;
124   const X86RegisterBankInfo &RBI;
125 
126 #define GET_GLOBALISEL_PREDICATES_DECL
127 #include "X86GenGlobalISel.inc"
128 #undef GET_GLOBALISEL_PREDICATES_DECL
129 
130 #define GET_GLOBALISEL_TEMPORARIES_DECL
131 #include "X86GenGlobalISel.inc"
132 #undef GET_GLOBALISEL_TEMPORARIES_DECL
133 };
134 
135 } // end anonymous namespace
136 
137 #define GET_GLOBALISEL_IMPL
138 #include "X86GenGlobalISel.inc"
139 #undef GET_GLOBALISEL_IMPL
140 
141 X86InstructionSelector::X86InstructionSelector(const X86TargetMachine &TM,
142                                                const X86Subtarget &STI,
143                                                const X86RegisterBankInfo &RBI)
144     : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()),
145       TRI(*STI.getRegisterInfo()), RBI(RBI),
146 #define GET_GLOBALISEL_PREDICATES_INIT
147 #include "X86GenGlobalISel.inc"
148 #undef GET_GLOBALISEL_PREDICATES_INIT
149 #define GET_GLOBALISEL_TEMPORARIES_INIT
150 #include "X86GenGlobalISel.inc"
151 #undef GET_GLOBALISEL_TEMPORARIES_INIT
152 {
153 }
154 
155 // FIXME: This should be target-independent, inferred from the types declared
156 // for each class in the bank.
157 const TargetRegisterClass *
158 X86InstructionSelector::getRegClass(LLT Ty, const RegisterBank &RB) const {
159   if (RB.getID() == X86::GPRRegBankID) {
160     if (Ty.getSizeInBits() <= 8)
161       return &X86::GR8RegClass;
162     if (Ty.getSizeInBits() == 16)
163       return &X86::GR16RegClass;
164     if (Ty.getSizeInBits() == 32)
165       return &X86::GR32RegClass;
166     if (Ty.getSizeInBits() == 64)
167       return &X86::GR64RegClass;
168   }
169   if (RB.getID() == X86::VECRRegBankID) {
170     if (Ty.getSizeInBits() == 32)
171       return STI.hasAVX512() ? &X86::FR32XRegClass : &X86::FR32RegClass;
172     if (Ty.getSizeInBits() == 64)
173       return STI.hasAVX512() ? &X86::FR64XRegClass : &X86::FR64RegClass;
174     if (Ty.getSizeInBits() == 128)
175       return STI.hasAVX512() ? &X86::VR128XRegClass : &X86::VR128RegClass;
176     if (Ty.getSizeInBits() == 256)
177       return STI.hasAVX512() ? &X86::VR256XRegClass : &X86::VR256RegClass;
178     if (Ty.getSizeInBits() == 512)
179       return &X86::VR512RegClass;
180   }
181 
182   llvm_unreachable("Unknown RegBank!");
183 }
184 
185 const TargetRegisterClass *
186 X86InstructionSelector::getRegClass(LLT Ty, unsigned Reg,
187                                     MachineRegisterInfo &MRI) const {
188   const RegisterBank &RegBank = *RBI.getRegBank(Reg, MRI, TRI);
189   return getRegClass(Ty, RegBank);
190 }
191 
192 static unsigned getSubRegIndex(const TargetRegisterClass *RC) {
193   unsigned SubIdx = X86::NoSubRegister;
194   if (RC == &X86::GR32RegClass) {
195     SubIdx = X86::sub_32bit;
196   } else if (RC == &X86::GR16RegClass) {
197     SubIdx = X86::sub_16bit;
198   } else if (RC == &X86::GR8RegClass) {
199     SubIdx = X86::sub_8bit;
200   }
201 
202   return SubIdx;
203 }
204 
205 static const TargetRegisterClass *getRegClassFromGRPhysReg(unsigned Reg) {
206   assert(TargetRegisterInfo::isPhysicalRegister(Reg));
207   if (X86::GR64RegClass.contains(Reg))
208     return &X86::GR64RegClass;
209   if (X86::GR32RegClass.contains(Reg))
210     return &X86::GR32RegClass;
211   if (X86::GR16RegClass.contains(Reg))
212     return &X86::GR16RegClass;
213   if (X86::GR8RegClass.contains(Reg))
214     return &X86::GR8RegClass;
215 
216   llvm_unreachable("Unknown RegClass for PhysReg!");
217 }
218 
219 // Set X86 Opcode and constrain DestReg.
220 bool X86InstructionSelector::selectCopy(MachineInstr &I,
221                                         MachineRegisterInfo &MRI) const {
222   unsigned DstReg = I.getOperand(0).getReg();
223   const unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI);
224   const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI);
225 
226   unsigned SrcReg = I.getOperand(1).getReg();
227   const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);
228   const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI);
229 
230   if (TargetRegisterInfo::isPhysicalRegister(DstReg)) {
231     assert(I.isCopy() && "Generic operators do not allow physical registers");
232 
233     if (DstSize > SrcSize && SrcRegBank.getID() == X86::GPRRegBankID &&
234         DstRegBank.getID() == X86::GPRRegBankID) {
235 
236       const TargetRegisterClass *SrcRC =
237           getRegClass(MRI.getType(SrcReg), SrcRegBank);
238       const TargetRegisterClass *DstRC = getRegClassFromGRPhysReg(DstReg);
239 
240       if (SrcRC != DstRC) {
241         // This case can be generated by ABI lowering, performe anyext
242         unsigned ExtSrc = MRI.createVirtualRegister(DstRC);
243         BuildMI(*I.getParent(), I, I.getDebugLoc(),
244                 TII.get(TargetOpcode::SUBREG_TO_REG))
245             .addDef(ExtSrc)
246             .addImm(0)
247             .addReg(SrcReg)
248             .addImm(getSubRegIndex(SrcRC));
249 
250         I.getOperand(1).setReg(ExtSrc);
251       }
252     }
253 
254     return true;
255   }
256 
257   assert((!TargetRegisterInfo::isPhysicalRegister(SrcReg) || I.isCopy()) &&
258          "No phys reg on generic operators");
259   assert((DstSize == SrcSize ||
260           // Copies are a mean to setup initial types, the number of
261           // bits may not exactly match.
262           (TargetRegisterInfo::isPhysicalRegister(SrcReg) &&
263            DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI))) &&
264          "Copy with different width?!");
265 
266   const TargetRegisterClass *DstRC =
267       getRegClass(MRI.getType(DstReg), DstRegBank);
268 
269   if (SrcRegBank.getID() == X86::GPRRegBankID &&
270       DstRegBank.getID() == X86::GPRRegBankID && SrcSize > DstSize &&
271       TargetRegisterInfo::isPhysicalRegister(SrcReg)) {
272     // Change the physical register to performe truncate.
273 
274     const TargetRegisterClass *SrcRC = getRegClassFromGRPhysReg(SrcReg);
275 
276     if (DstRC != SrcRC) {
277       I.getOperand(1).setSubReg(getSubRegIndex(DstRC));
278       I.getOperand(1).substPhysReg(SrcReg, TRI);
279     }
280   }
281 
282   // No need to constrain SrcReg. It will get constrained when
283   // we hit another of its use or its defs.
284   // Copies do not have constraints.
285   const TargetRegisterClass *OldRC = MRI.getRegClassOrNull(DstReg);
286   if (!OldRC || !DstRC->hasSubClassEq(OldRC)) {
287     if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
288       DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
289                    << " operand\n");
290       return false;
291     }
292   }
293   I.setDesc(TII.get(X86::COPY));
294   return true;
295 }
296 
297 bool X86InstructionSelector::select(MachineInstr &I) const {
298   assert(I.getParent() && "Instruction should be in a basic block!");
299   assert(I.getParent()->getParent() && "Instruction should be in a function!");
300 
301   MachineBasicBlock &MBB = *I.getParent();
302   MachineFunction &MF = *MBB.getParent();
303   MachineRegisterInfo &MRI = MF.getRegInfo();
304 
305   unsigned Opcode = I.getOpcode();
306   if (!isPreISelGenericOpcode(Opcode)) {
307     // Certain non-generic instructions also need some special handling.
308 
309     if (Opcode == TargetOpcode::LOAD_STACK_GUARD)
310       return false;
311 
312     if (I.isCopy())
313       return selectCopy(I, MRI);
314 
315     return true;
316   }
317 
318   assert(I.getNumOperands() == I.getNumExplicitOperands() &&
319          "Generic instruction has unexpected implicit operands\n");
320 
321   if (selectImpl(I))
322     return true;
323 
324   DEBUG(dbgs() << " C++ instruction selection: "; I.print(dbgs()));
325 
326   // TODO: This should be implemented by tblgen.
327   switch (I.getOpcode()) {
328   default:
329     return false;
330   case TargetOpcode::G_STORE:
331   case TargetOpcode::G_LOAD:
332     return selectLoadStoreOp(I, MRI, MF);
333   case TargetOpcode::G_GEP:
334   case TargetOpcode::G_FRAME_INDEX:
335     return selectFrameIndexOrGep(I, MRI, MF);
336   case TargetOpcode::G_GLOBAL_VALUE:
337     return selectGlobalValue(I, MRI, MF);
338   case TargetOpcode::G_CONSTANT:
339     return selectConstant(I, MRI, MF);
340   case TargetOpcode::G_FCONSTANT:
341     return materializeFP(I, MRI, MF);
342   case TargetOpcode::G_TRUNC:
343     return selectTrunc(I, MRI, MF);
344   case TargetOpcode::G_ZEXT:
345     return selectZext(I, MRI, MF);
346   case TargetOpcode::G_ANYEXT:
347     return selectAnyext(I, MRI, MF);
348   case TargetOpcode::G_ICMP:
349     return selectCmp(I, MRI, MF);
350   case TargetOpcode::G_UADDE:
351     return selectUadde(I, MRI, MF);
352   case TargetOpcode::G_UNMERGE_VALUES:
353     return selectUnmergeValues(I, MRI, MF);
354   case TargetOpcode::G_MERGE_VALUES:
355     return selectMergeValues(I, MRI, MF);
356   case TargetOpcode::G_EXTRACT:
357     return selectExtract(I, MRI, MF);
358   case TargetOpcode::G_INSERT:
359     return selectInsert(I, MRI, MF);
360   case TargetOpcode::G_BRCOND:
361     return selectCondBranch(I, MRI, MF);
362   case TargetOpcode::G_IMPLICIT_DEF:
363   case TargetOpcode::G_PHI:
364     return selectImplicitDefOrPHI(I, MRI);
365   }
366 
367   return false;
368 }
369 
370 unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
371                                                 const RegisterBank &RB,
372                                                 unsigned Opc,
373                                                 uint64_t Alignment) const {
374   bool Isload = (Opc == TargetOpcode::G_LOAD);
375   bool HasAVX = STI.hasAVX();
376   bool HasAVX512 = STI.hasAVX512();
377   bool HasVLX = STI.hasVLX();
378 
379   if (Ty == LLT::scalar(8)) {
380     if (X86::GPRRegBankID == RB.getID())
381       return Isload ? X86::MOV8rm : X86::MOV8mr;
382   } else if (Ty == LLT::scalar(16)) {
383     if (X86::GPRRegBankID == RB.getID())
384       return Isload ? X86::MOV16rm : X86::MOV16mr;
385   } else if (Ty == LLT::scalar(32) || Ty == LLT::pointer(0, 32)) {
386     if (X86::GPRRegBankID == RB.getID())
387       return Isload ? X86::MOV32rm : X86::MOV32mr;
388     if (X86::VECRRegBankID == RB.getID())
389       return Isload ? (HasAVX512 ? X86::VMOVSSZrm
390                                  : HasAVX ? X86::VMOVSSrm : X86::MOVSSrm)
391                     : (HasAVX512 ? X86::VMOVSSZmr
392                                  : HasAVX ? X86::VMOVSSmr : X86::MOVSSmr);
393   } else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) {
394     if (X86::GPRRegBankID == RB.getID())
395       return Isload ? X86::MOV64rm : X86::MOV64mr;
396     if (X86::VECRRegBankID == RB.getID())
397       return Isload ? (HasAVX512 ? X86::VMOVSDZrm
398                                  : HasAVX ? X86::VMOVSDrm : X86::MOVSDrm)
399                     : (HasAVX512 ? X86::VMOVSDZmr
400                                  : HasAVX ? X86::VMOVSDmr : X86::MOVSDmr);
401   } else if (Ty.isVector() && Ty.getSizeInBits() == 128) {
402     if (Alignment >= 16)
403       return Isload ? (HasVLX ? X86::VMOVAPSZ128rm
404                               : HasAVX512
405                                     ? X86::VMOVAPSZ128rm_NOVLX
406                                     : HasAVX ? X86::VMOVAPSrm : X86::MOVAPSrm)
407                     : (HasVLX ? X86::VMOVAPSZ128mr
408                               : HasAVX512
409                                     ? X86::VMOVAPSZ128mr_NOVLX
410                                     : HasAVX ? X86::VMOVAPSmr : X86::MOVAPSmr);
411     else
412       return Isload ? (HasVLX ? X86::VMOVUPSZ128rm
413                               : HasAVX512
414                                     ? X86::VMOVUPSZ128rm_NOVLX
415                                     : HasAVX ? X86::VMOVUPSrm : X86::MOVUPSrm)
416                     : (HasVLX ? X86::VMOVUPSZ128mr
417                               : HasAVX512
418                                     ? X86::VMOVUPSZ128mr_NOVLX
419                                     : HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr);
420   } else if (Ty.isVector() && Ty.getSizeInBits() == 256) {
421     if (Alignment >= 32)
422       return Isload ? (HasVLX ? X86::VMOVAPSZ256rm
423                               : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX
424                                           : X86::VMOVAPSYrm)
425                     : (HasVLX ? X86::VMOVAPSZ256mr
426                               : HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX
427                                           : X86::VMOVAPSYmr);
428     else
429       return Isload ? (HasVLX ? X86::VMOVUPSZ256rm
430                               : HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX
431                                           : X86::VMOVUPSYrm)
432                     : (HasVLX ? X86::VMOVUPSZ256mr
433                               : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX
434                                           : X86::VMOVUPSYmr);
435   } else if (Ty.isVector() && Ty.getSizeInBits() == 512) {
436     if (Alignment >= 64)
437       return Isload ? X86::VMOVAPSZrm : X86::VMOVAPSZmr;
438     else
439       return Isload ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
440   }
441   return Opc;
442 }
443 
444 // Fill in an address from the given instruction.
445 static void X86SelectAddress(const MachineInstr &I,
446                              const MachineRegisterInfo &MRI,
447                              X86AddressMode &AM) {
448   assert(I.getOperand(0).isReg() && "unsupported opperand.");
449   assert(MRI.getType(I.getOperand(0).getReg()).isPointer() &&
450          "unsupported type.");
451 
452   if (I.getOpcode() == TargetOpcode::G_GEP) {
453     if (auto COff = getConstantVRegVal(I.getOperand(2).getReg(), MRI)) {
454       int64_t Imm = *COff;
455       if (isInt<32>(Imm)) { // Check for displacement overflow.
456         AM.Disp = static_cast<int32_t>(Imm);
457         AM.Base.Reg = I.getOperand(1).getReg();
458         return;
459       }
460     }
461   } else if (I.getOpcode() == TargetOpcode::G_FRAME_INDEX) {
462     AM.Base.FrameIndex = I.getOperand(1).getIndex();
463     AM.BaseType = X86AddressMode::FrameIndexBase;
464     return;
465   }
466 
467   // Default behavior.
468   AM.Base.Reg = I.getOperand(0).getReg();
469 }
470 
471 bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I,
472                                                MachineRegisterInfo &MRI,
473                                                MachineFunction &MF) const {
474   unsigned Opc = I.getOpcode();
475 
476   assert((Opc == TargetOpcode::G_STORE || Opc == TargetOpcode::G_LOAD) &&
477          "unexpected instruction");
478 
479   const unsigned DefReg = I.getOperand(0).getReg();
480   LLT Ty = MRI.getType(DefReg);
481   const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
482 
483   auto &MemOp = **I.memoperands_begin();
484   if (MemOp.getOrdering() != AtomicOrdering::NotAtomic) {
485     DEBUG(dbgs() << "Atomic load/store not supported yet\n");
486     return false;
487   }
488 
489   unsigned NewOpc = getLoadStoreOp(Ty, RB, Opc, MemOp.getAlignment());
490   if (NewOpc == Opc)
491     return false;
492 
493   X86AddressMode AM;
494   X86SelectAddress(*MRI.getVRegDef(I.getOperand(1).getReg()), MRI, AM);
495 
496   I.setDesc(TII.get(NewOpc));
497   MachineInstrBuilder MIB(MF, I);
498   if (Opc == TargetOpcode::G_LOAD) {
499     I.RemoveOperand(1);
500     addFullAddress(MIB, AM);
501   } else {
502     // G_STORE (VAL, Addr), X86Store instruction (Addr, VAL)
503     I.RemoveOperand(1);
504     I.RemoveOperand(0);
505     addFullAddress(MIB, AM).addUse(DefReg);
506   }
507   return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
508 }
509 
510 static unsigned getLeaOP(LLT Ty, const X86Subtarget &STI) {
511   if (Ty == LLT::pointer(0, 64))
512     return X86::LEA64r;
513   else if (Ty == LLT::pointer(0, 32))
514     return STI.isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r;
515   else
516     llvm_unreachable("Can't get LEA opcode. Unsupported type.");
517 }
518 
519 bool X86InstructionSelector::selectFrameIndexOrGep(MachineInstr &I,
520                                                    MachineRegisterInfo &MRI,
521                                                    MachineFunction &MF) const {
522   unsigned Opc = I.getOpcode();
523 
524   assert((Opc == TargetOpcode::G_FRAME_INDEX || Opc == TargetOpcode::G_GEP) &&
525          "unexpected instruction");
526 
527   const unsigned DefReg = I.getOperand(0).getReg();
528   LLT Ty = MRI.getType(DefReg);
529 
530   // Use LEA to calculate frame index and GEP
531   unsigned NewOpc = getLeaOP(Ty, STI);
532   I.setDesc(TII.get(NewOpc));
533   MachineInstrBuilder MIB(MF, I);
534 
535   if (Opc == TargetOpcode::G_FRAME_INDEX) {
536     addOffset(MIB, 0);
537   } else {
538     MachineOperand &InxOp = I.getOperand(2);
539     I.addOperand(InxOp);        // set IndexReg
540     InxOp.ChangeToImmediate(1); // set Scale
541     MIB.addImm(0).addReg(0);
542   }
543 
544   return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
545 }
546 
547 bool X86InstructionSelector::selectGlobalValue(MachineInstr &I,
548                                                MachineRegisterInfo &MRI,
549                                                MachineFunction &MF) const {
550   assert((I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE) &&
551          "unexpected instruction");
552 
553   auto GV = I.getOperand(1).getGlobal();
554   if (GV->isThreadLocal()) {
555     return false; // TODO: we don't support TLS yet.
556   }
557 
558   // Can't handle alternate code models yet.
559   if (TM.getCodeModel() != CodeModel::Small)
560     return false;
561 
562   X86AddressMode AM;
563   AM.GV = GV;
564   AM.GVOpFlags = STI.classifyGlobalReference(GV);
565 
566   // TODO: The ABI requires an extra load. not supported yet.
567   if (isGlobalStubReference(AM.GVOpFlags))
568     return false;
569 
570   // TODO: This reference is relative to the pic base. not supported yet.
571   if (isGlobalRelativeToPICBase(AM.GVOpFlags))
572     return false;
573 
574   if (STI.isPICStyleRIPRel()) {
575     // Use rip-relative addressing.
576     assert(AM.Base.Reg == 0 && AM.IndexReg == 0);
577     AM.Base.Reg = X86::RIP;
578   }
579 
580   const unsigned DefReg = I.getOperand(0).getReg();
581   LLT Ty = MRI.getType(DefReg);
582   unsigned NewOpc = getLeaOP(Ty, STI);
583 
584   I.setDesc(TII.get(NewOpc));
585   MachineInstrBuilder MIB(MF, I);
586 
587   I.RemoveOperand(1);
588   addFullAddress(MIB, AM);
589 
590   return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
591 }
592 
593 bool X86InstructionSelector::selectConstant(MachineInstr &I,
594                                             MachineRegisterInfo &MRI,
595                                             MachineFunction &MF) const {
596   assert((I.getOpcode() == TargetOpcode::G_CONSTANT) &&
597          "unexpected instruction");
598 
599   const unsigned DefReg = I.getOperand(0).getReg();
600   LLT Ty = MRI.getType(DefReg);
601 
602   if (RBI.getRegBank(DefReg, MRI, TRI)->getID() != X86::GPRRegBankID)
603     return false;
604 
605   uint64_t Val = 0;
606   if (I.getOperand(1).isCImm()) {
607     Val = I.getOperand(1).getCImm()->getZExtValue();
608     I.getOperand(1).ChangeToImmediate(Val);
609   } else if (I.getOperand(1).isImm()) {
610     Val = I.getOperand(1).getImm();
611   } else
612     llvm_unreachable("Unsupported operand type.");
613 
614   unsigned NewOpc;
615   switch (Ty.getSizeInBits()) {
616   case 8:
617     NewOpc = X86::MOV8ri;
618     break;
619   case 16:
620     NewOpc = X86::MOV16ri;
621     break;
622   case 32:
623     NewOpc = X86::MOV32ri;
624     break;
625   case 64:
626     // TODO: in case isUInt<32>(Val), X86::MOV32ri can be used
627     if (isInt<32>(Val))
628       NewOpc = X86::MOV64ri32;
629     else
630       NewOpc = X86::MOV64ri;
631     break;
632   default:
633     llvm_unreachable("Can't select G_CONSTANT, unsupported type.");
634   }
635 
636   I.setDesc(TII.get(NewOpc));
637   return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
638 }
639 
640 bool X86InstructionSelector::selectTrunc(MachineInstr &I,
641                                          MachineRegisterInfo &MRI,
642                                          MachineFunction &MF) const {
643   assert((I.getOpcode() == TargetOpcode::G_TRUNC) && "unexpected instruction");
644 
645   const unsigned DstReg = I.getOperand(0).getReg();
646   const unsigned SrcReg = I.getOperand(1).getReg();
647 
648   const LLT DstTy = MRI.getType(DstReg);
649   const LLT SrcTy = MRI.getType(SrcReg);
650 
651   const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
652   const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
653 
654   if (DstRB.getID() != SrcRB.getID()) {
655     DEBUG(dbgs() << "G_TRUNC input/output on different banks\n");
656     return false;
657   }
658 
659   if (DstRB.getID() != X86::GPRRegBankID)
660     return false;
661 
662   const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB);
663   if (!DstRC)
664     return false;
665 
666   const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB);
667   if (!SrcRC)
668     return false;
669 
670   unsigned SubIdx;
671   if (DstRC == SrcRC) {
672     // Nothing to be done
673     SubIdx = X86::NoSubRegister;
674   } else if (DstRC == &X86::GR32RegClass) {
675     SubIdx = X86::sub_32bit;
676   } else if (DstRC == &X86::GR16RegClass) {
677     SubIdx = X86::sub_16bit;
678   } else if (DstRC == &X86::GR8RegClass) {
679     SubIdx = X86::sub_8bit;
680   } else {
681     return false;
682   }
683 
684   SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx);
685 
686   if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
687       !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
688     DEBUG(dbgs() << "Failed to constrain G_TRUNC\n");
689     return false;
690   }
691 
692   I.getOperand(1).setSubReg(SubIdx);
693 
694   I.setDesc(TII.get(X86::COPY));
695   return true;
696 }
697 
698 bool X86InstructionSelector::selectZext(MachineInstr &I,
699                                         MachineRegisterInfo &MRI,
700                                         MachineFunction &MF) const {
701   assert((I.getOpcode() == TargetOpcode::G_ZEXT) && "unexpected instruction");
702 
703   const unsigned DstReg = I.getOperand(0).getReg();
704   const unsigned SrcReg = I.getOperand(1).getReg();
705 
706   const LLT DstTy = MRI.getType(DstReg);
707   const LLT SrcTy = MRI.getType(SrcReg);
708 
709   if (SrcTy != LLT::scalar(1))
710     return false;
711 
712   unsigned AndOpc;
713   if (DstTy == LLT::scalar(8))
714     AndOpc = X86::AND8ri;
715   else if (DstTy == LLT::scalar(16))
716     AndOpc = X86::AND16ri8;
717   else if (DstTy == LLT::scalar(32))
718     AndOpc = X86::AND32ri8;
719   else if (DstTy == LLT::scalar(64))
720     AndOpc = X86::AND64ri8;
721   else
722     return false;
723 
724   unsigned DefReg = SrcReg;
725   if (DstTy != LLT::scalar(8)) {
726     DefReg = MRI.createVirtualRegister(getRegClass(DstTy, DstReg, MRI));
727     BuildMI(*I.getParent(), I, I.getDebugLoc(),
728             TII.get(TargetOpcode::SUBREG_TO_REG), DefReg)
729         .addImm(0)
730         .addReg(SrcReg)
731         .addImm(X86::sub_8bit);
732   }
733 
734   MachineInstr &AndInst =
735       *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AndOpc), DstReg)
736            .addReg(DefReg)
737            .addImm(1);
738 
739   constrainSelectedInstRegOperands(AndInst, TII, TRI, RBI);
740 
741   I.eraseFromParent();
742   return true;
743 }
744 
745 bool X86InstructionSelector::selectAnyext(MachineInstr &I,
746                                           MachineRegisterInfo &MRI,
747                                           MachineFunction &MF) const {
748   assert((I.getOpcode() == TargetOpcode::G_ANYEXT) && "unexpected instruction");
749 
750   const unsigned DstReg = I.getOperand(0).getReg();
751   const unsigned SrcReg = I.getOperand(1).getReg();
752 
753   const LLT DstTy = MRI.getType(DstReg);
754   const LLT SrcTy = MRI.getType(SrcReg);
755 
756   const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
757   const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
758 
759   assert(DstRB.getID() == SrcRB.getID() &&
760          "G_ANYEXT input/output on different banks\n");
761 
762   assert(DstTy.getSizeInBits() > SrcTy.getSizeInBits() &&
763          "G_ANYEXT incorrect operand size");
764 
765   if (DstRB.getID() != X86::GPRRegBankID)
766     return false;
767 
768   const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB);
769   const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB);
770 
771   if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
772       !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
773     DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
774                  << " operand\n");
775     return false;
776   }
777 
778   if (SrcRC == DstRC) {
779     I.setDesc(TII.get(X86::COPY));
780     return true;
781   }
782 
783   BuildMI(*I.getParent(), I, I.getDebugLoc(),
784           TII.get(TargetOpcode::SUBREG_TO_REG))
785       .addDef(DstReg)
786       .addImm(0)
787       .addReg(SrcReg)
788       .addImm(getSubRegIndex(SrcRC));
789 
790   I.eraseFromParent();
791   return true;
792 }
793 
794 bool X86InstructionSelector::selectCmp(MachineInstr &I,
795                                        MachineRegisterInfo &MRI,
796                                        MachineFunction &MF) const {
797   assert((I.getOpcode() == TargetOpcode::G_ICMP) && "unexpected instruction");
798 
799   X86::CondCode CC;
800   bool SwapArgs;
801   std::tie(CC, SwapArgs) = X86::getX86ConditionCode(
802       (CmpInst::Predicate)I.getOperand(1).getPredicate());
803   unsigned OpSet = X86::getSETFromCond(CC);
804 
805   unsigned LHS = I.getOperand(2).getReg();
806   unsigned RHS = I.getOperand(3).getReg();
807 
808   if (SwapArgs)
809     std::swap(LHS, RHS);
810 
811   unsigned OpCmp;
812   LLT Ty = MRI.getType(LHS);
813 
814   switch (Ty.getSizeInBits()) {
815   default:
816     return false;
817   case 8:
818     OpCmp = X86::CMP8rr;
819     break;
820   case 16:
821     OpCmp = X86::CMP16rr;
822     break;
823   case 32:
824     OpCmp = X86::CMP32rr;
825     break;
826   case 64:
827     OpCmp = X86::CMP64rr;
828     break;
829   }
830 
831   MachineInstr &CmpInst =
832       *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp))
833            .addReg(LHS)
834            .addReg(RHS);
835 
836   MachineInstr &SetInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
837                                    TII.get(OpSet), I.getOperand(0).getReg());
838 
839   constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI);
840   constrainSelectedInstRegOperands(SetInst, TII, TRI, RBI);
841 
842   I.eraseFromParent();
843   return true;
844 }
845 
846 bool X86InstructionSelector::selectUadde(MachineInstr &I,
847                                          MachineRegisterInfo &MRI,
848                                          MachineFunction &MF) const {
849   assert((I.getOpcode() == TargetOpcode::G_UADDE) && "unexpected instruction");
850 
851   const unsigned DstReg = I.getOperand(0).getReg();
852   const unsigned CarryOutReg = I.getOperand(1).getReg();
853   const unsigned Op0Reg = I.getOperand(2).getReg();
854   const unsigned Op1Reg = I.getOperand(3).getReg();
855   unsigned CarryInReg = I.getOperand(4).getReg();
856 
857   const LLT DstTy = MRI.getType(DstReg);
858 
859   if (DstTy != LLT::scalar(32))
860     return false;
861 
862   // find CarryIn def instruction.
863   MachineInstr *Def = MRI.getVRegDef(CarryInReg);
864   while (Def->getOpcode() == TargetOpcode::G_TRUNC) {
865     CarryInReg = Def->getOperand(1).getReg();
866     Def = MRI.getVRegDef(CarryInReg);
867   }
868 
869   unsigned Opcode;
870   if (Def->getOpcode() == TargetOpcode::G_UADDE) {
871     // carry set by prev ADD.
872 
873     BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), X86::EFLAGS)
874         .addReg(CarryInReg);
875 
876     if (!RBI.constrainGenericRegister(CarryInReg, X86::GR32RegClass, MRI))
877       return false;
878 
879     Opcode = X86::ADC32rr;
880   } else if (auto val = getConstantVRegVal(CarryInReg, MRI)) {
881     // carry is constant, support only 0.
882     if (*val != 0)
883       return false;
884 
885     Opcode = X86::ADD32rr;
886   } else
887     return false;
888 
889   MachineInstr &AddInst =
890       *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode), DstReg)
891            .addReg(Op0Reg)
892            .addReg(Op1Reg);
893 
894   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), CarryOutReg)
895       .addReg(X86::EFLAGS);
896 
897   if (!constrainSelectedInstRegOperands(AddInst, TII, TRI, RBI) ||
898       !RBI.constrainGenericRegister(CarryOutReg, X86::GR32RegClass, MRI))
899     return false;
900 
901   I.eraseFromParent();
902   return true;
903 }
904 
905 bool X86InstructionSelector::selectExtract(MachineInstr &I,
906                                            MachineRegisterInfo &MRI,
907                                            MachineFunction &MF) const {
908   assert((I.getOpcode() == TargetOpcode::G_EXTRACT) &&
909          "unexpected instruction");
910 
911   const unsigned DstReg = I.getOperand(0).getReg();
912   const unsigned SrcReg = I.getOperand(1).getReg();
913   int64_t Index = I.getOperand(2).getImm();
914 
915   const LLT DstTy = MRI.getType(DstReg);
916   const LLT SrcTy = MRI.getType(SrcReg);
917 
918   // Meanwile handle vector type only.
919   if (!DstTy.isVector())
920     return false;
921 
922   if (Index % DstTy.getSizeInBits() != 0)
923     return false; // Not extract subvector.
924 
925   if (Index == 0) {
926     // Replace by extract subreg copy.
927     if (!emitExtractSubreg(DstReg, SrcReg, I, MRI, MF))
928       return false;
929 
930     I.eraseFromParent();
931     return true;
932   }
933 
934   bool HasAVX = STI.hasAVX();
935   bool HasAVX512 = STI.hasAVX512();
936   bool HasVLX = STI.hasVLX();
937 
938   if (SrcTy.getSizeInBits() == 256 && DstTy.getSizeInBits() == 128) {
939     if (HasVLX)
940       I.setDesc(TII.get(X86::VEXTRACTF32x4Z256rr));
941     else if (HasAVX)
942       I.setDesc(TII.get(X86::VEXTRACTF128rr));
943     else
944       return false;
945   } else if (SrcTy.getSizeInBits() == 512 && HasAVX512) {
946     if (DstTy.getSizeInBits() == 128)
947       I.setDesc(TII.get(X86::VEXTRACTF32x4Zrr));
948     else if (DstTy.getSizeInBits() == 256)
949       I.setDesc(TII.get(X86::VEXTRACTF64x4Zrr));
950     else
951       return false;
952   } else
953     return false;
954 
955   // Convert to X86 VEXTRACT immediate.
956   Index = Index / DstTy.getSizeInBits();
957   I.getOperand(2).setImm(Index);
958 
959   return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
960 }
961 
962 bool X86InstructionSelector::emitExtractSubreg(unsigned DstReg, unsigned SrcReg,
963                                                MachineInstr &I,
964                                                MachineRegisterInfo &MRI,
965                                                MachineFunction &MF) const {
966   const LLT DstTy = MRI.getType(DstReg);
967   const LLT SrcTy = MRI.getType(SrcReg);
968   unsigned SubIdx = X86::NoSubRegister;
969 
970   if (!DstTy.isVector() || !SrcTy.isVector())
971     return false;
972 
973   assert(SrcTy.getSizeInBits() > DstTy.getSizeInBits() &&
974          "Incorrect Src/Dst register size");
975 
976   if (DstTy.getSizeInBits() == 128)
977     SubIdx = X86::sub_xmm;
978   else if (DstTy.getSizeInBits() == 256)
979     SubIdx = X86::sub_ymm;
980   else
981     return false;
982 
983   const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI);
984   const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI);
985 
986   SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx);
987 
988   if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
989       !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
990     DEBUG(dbgs() << "Failed to constrain G_TRUNC\n");
991     return false;
992   }
993 
994   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), DstReg)
995       .addReg(SrcReg, 0, SubIdx);
996 
997   return true;
998 }
999 
1000 bool X86InstructionSelector::emitInsertSubreg(unsigned DstReg, unsigned SrcReg,
1001                                               MachineInstr &I,
1002                                               MachineRegisterInfo &MRI,
1003                                               MachineFunction &MF) const {
1004   const LLT DstTy = MRI.getType(DstReg);
1005   const LLT SrcTy = MRI.getType(SrcReg);
1006   unsigned SubIdx = X86::NoSubRegister;
1007 
1008   // TODO: support scalar types
1009   if (!DstTy.isVector() || !SrcTy.isVector())
1010     return false;
1011 
1012   assert(SrcTy.getSizeInBits() < DstTy.getSizeInBits() &&
1013          "Incorrect Src/Dst register size");
1014 
1015   if (SrcTy.getSizeInBits() == 128)
1016     SubIdx = X86::sub_xmm;
1017   else if (SrcTy.getSizeInBits() == 256)
1018     SubIdx = X86::sub_ymm;
1019   else
1020     return false;
1021 
1022   const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI);
1023   const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI);
1024 
1025   if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
1026       !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
1027     DEBUG(dbgs() << "Failed to constrain INSERT_SUBREG\n");
1028     return false;
1029   }
1030 
1031   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY))
1032       .addReg(DstReg, RegState::DefineNoRead, SubIdx)
1033       .addReg(SrcReg);
1034 
1035   return true;
1036 }
1037 
1038 bool X86InstructionSelector::selectInsert(MachineInstr &I,
1039                                           MachineRegisterInfo &MRI,
1040                                           MachineFunction &MF) const {
1041   assert((I.getOpcode() == TargetOpcode::G_INSERT) && "unexpected instruction");
1042 
1043   const unsigned DstReg = I.getOperand(0).getReg();
1044   const unsigned SrcReg = I.getOperand(1).getReg();
1045   const unsigned InsertReg = I.getOperand(2).getReg();
1046   int64_t Index = I.getOperand(3).getImm();
1047 
1048   const LLT DstTy = MRI.getType(DstReg);
1049   const LLT InsertRegTy = MRI.getType(InsertReg);
1050 
1051   // Meanwile handle vector type only.
1052   if (!DstTy.isVector())
1053     return false;
1054 
1055   if (Index % InsertRegTy.getSizeInBits() != 0)
1056     return false; // Not insert subvector.
1057 
1058   if (Index == 0 && MRI.getVRegDef(SrcReg)->isImplicitDef()) {
1059     // Replace by subreg copy.
1060     if (!emitInsertSubreg(DstReg, InsertReg, I, MRI, MF))
1061       return false;
1062 
1063     I.eraseFromParent();
1064     return true;
1065   }
1066 
1067   bool HasAVX = STI.hasAVX();
1068   bool HasAVX512 = STI.hasAVX512();
1069   bool HasVLX = STI.hasVLX();
1070 
1071   if (DstTy.getSizeInBits() == 256 && InsertRegTy.getSizeInBits() == 128) {
1072     if (HasVLX)
1073       I.setDesc(TII.get(X86::VINSERTF32x4Z256rr));
1074     else if (HasAVX)
1075       I.setDesc(TII.get(X86::VINSERTF128rr));
1076     else
1077       return false;
1078   } else if (DstTy.getSizeInBits() == 512 && HasAVX512) {
1079     if (InsertRegTy.getSizeInBits() == 128)
1080       I.setDesc(TII.get(X86::VINSERTF32x4Zrr));
1081     else if (InsertRegTy.getSizeInBits() == 256)
1082       I.setDesc(TII.get(X86::VINSERTF64x4Zrr));
1083     else
1084       return false;
1085   } else
1086     return false;
1087 
1088   // Convert to X86 VINSERT immediate.
1089   Index = Index / InsertRegTy.getSizeInBits();
1090 
1091   I.getOperand(3).setImm(Index);
1092 
1093   return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1094 }
1095 
1096 bool X86InstructionSelector::selectUnmergeValues(MachineInstr &I,
1097                                                  MachineRegisterInfo &MRI,
1098                                                  MachineFunction &MF) const {
1099   assert((I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES) &&
1100          "unexpected instruction");
1101 
1102   // Split to extracts.
1103   unsigned NumDefs = I.getNumOperands() - 1;
1104   unsigned SrcReg = I.getOperand(NumDefs).getReg();
1105   unsigned DefSize = MRI.getType(I.getOperand(0).getReg()).getSizeInBits();
1106 
1107   for (unsigned Idx = 0; Idx < NumDefs; ++Idx) {
1108     MachineInstr &ExtrInst =
1109         *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1110                  TII.get(TargetOpcode::G_EXTRACT), I.getOperand(Idx).getReg())
1111              .addReg(SrcReg)
1112              .addImm(Idx * DefSize);
1113 
1114     if (!select(ExtrInst))
1115       return false;
1116   }
1117 
1118   I.eraseFromParent();
1119   return true;
1120 }
1121 
1122 bool X86InstructionSelector::selectMergeValues(MachineInstr &I,
1123                                                MachineRegisterInfo &MRI,
1124                                                MachineFunction &MF) const {
1125   assert((I.getOpcode() == TargetOpcode::G_MERGE_VALUES) &&
1126          "unexpected instruction");
1127 
1128   // Split to inserts.
1129   unsigned DstReg = I.getOperand(0).getReg();
1130   unsigned SrcReg0 = I.getOperand(1).getReg();
1131 
1132   const LLT DstTy = MRI.getType(DstReg);
1133   const LLT SrcTy = MRI.getType(SrcReg0);
1134   unsigned SrcSize = SrcTy.getSizeInBits();
1135 
1136   const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
1137 
1138   // For the first src use insertSubReg.
1139   unsigned DefReg = MRI.createGenericVirtualRegister(DstTy);
1140   MRI.setRegBank(DefReg, RegBank);
1141   if (!emitInsertSubreg(DefReg, I.getOperand(1).getReg(), I, MRI, MF))
1142     return false;
1143 
1144   for (unsigned Idx = 2; Idx < I.getNumOperands(); ++Idx) {
1145     unsigned Tmp = MRI.createGenericVirtualRegister(DstTy);
1146     MRI.setRegBank(Tmp, RegBank);
1147 
1148     MachineInstr &InsertInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1149                                         TII.get(TargetOpcode::G_INSERT), Tmp)
1150                                     .addReg(DefReg)
1151                                     .addReg(I.getOperand(Idx).getReg())
1152                                     .addImm((Idx - 1) * SrcSize);
1153 
1154     DefReg = Tmp;
1155 
1156     if (!select(InsertInst))
1157       return false;
1158   }
1159 
1160   MachineInstr &CopyInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1161                                     TII.get(TargetOpcode::COPY), DstReg)
1162                                 .addReg(DefReg);
1163 
1164   if (!select(CopyInst))
1165     return false;
1166 
1167   I.eraseFromParent();
1168   return true;
1169 }
1170 
1171 bool X86InstructionSelector::selectCondBranch(MachineInstr &I,
1172                                               MachineRegisterInfo &MRI,
1173                                               MachineFunction &MF) const {
1174   assert((I.getOpcode() == TargetOpcode::G_BRCOND) && "unexpected instruction");
1175 
1176   const unsigned CondReg = I.getOperand(0).getReg();
1177   MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
1178 
1179   MachineInstr &TestInst =
1180       *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::TEST8ri))
1181            .addReg(CondReg)
1182            .addImm(1);
1183   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::JNE_1))
1184       .addMBB(DestMBB);
1185 
1186   constrainSelectedInstRegOperands(TestInst, TII, TRI, RBI);
1187 
1188   I.eraseFromParent();
1189   return true;
1190 }
1191 
1192 bool X86InstructionSelector::materializeFP(MachineInstr &I,
1193                                            MachineRegisterInfo &MRI,
1194                                            MachineFunction &MF) const {
1195   assert((I.getOpcode() == TargetOpcode::G_FCONSTANT) &&
1196          "unexpected instruction");
1197 
1198   // Can't handle alternate code models yet.
1199   CodeModel::Model CM = TM.getCodeModel();
1200   if (CM != CodeModel::Small && CM != CodeModel::Large)
1201     return false;
1202 
1203   const unsigned DstReg = I.getOperand(0).getReg();
1204   const LLT DstTy = MRI.getType(DstReg);
1205   const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
1206   unsigned Align = DstTy.getSizeInBits();
1207   const DebugLoc &DbgLoc = I.getDebugLoc();
1208 
1209   unsigned Opc = getLoadStoreOp(DstTy, RegBank, TargetOpcode::G_LOAD, Align);
1210 
1211   // Create the load from the constant pool.
1212   const ConstantFP *CFP = I.getOperand(1).getFPImm();
1213   unsigned CPI = MF.getConstantPool()->getConstantPoolIndex(CFP, Align);
1214   MachineInstr *LoadInst = nullptr;
1215   unsigned char OpFlag = STI.classifyLocalReference(nullptr);
1216 
1217   if (CM == CodeModel::Large && STI.is64Bit()) {
1218     // Under X86-64 non-small code model, GV (and friends) are 64-bits, so
1219     // they cannot be folded into immediate fields.
1220 
1221     unsigned AddrReg = MRI.createVirtualRegister(&X86::GR64RegClass);
1222     BuildMI(*I.getParent(), I, DbgLoc, TII.get(X86::MOV64ri), AddrReg)
1223         .addConstantPoolIndex(CPI, 0, OpFlag);
1224 
1225     MachineMemOperand *MMO = MF.getMachineMemOperand(
1226         MachinePointerInfo::getConstantPool(MF), MachineMemOperand::MOLoad,
1227         MF.getDataLayout().getPointerSize(), Align);
1228 
1229     LoadInst =
1230         addDirectMem(BuildMI(*I.getParent(), I, DbgLoc, TII.get(Opc), DstReg),
1231                      AddrReg)
1232             .addMemOperand(MMO);
1233 
1234   } else if (CM == CodeModel::Small || !STI.is64Bit()) {
1235     // Handle the case when globals fit in our immediate field.
1236     // This is true for X86-32 always and X86-64 when in -mcmodel=small mode.
1237 
1238     // x86-32 PIC requires a PIC base register for constant pools.
1239     unsigned PICBase = 0;
1240     if (OpFlag == X86II::MO_PIC_BASE_OFFSET || OpFlag == X86II::MO_GOTOFF) {
1241       // PICBase can be allocated by TII.getGlobalBaseReg(&MF).
1242       // In DAGISEL the code that initialize it generated by the CGBR pass.
1243       return false; // TODO support the mode.
1244     } else if (STI.is64Bit() && TM.getCodeModel() == CodeModel::Small)
1245       PICBase = X86::RIP;
1246 
1247     LoadInst = addConstantPoolReference(
1248         BuildMI(*I.getParent(), I, DbgLoc, TII.get(Opc), DstReg), CPI, PICBase,
1249         OpFlag);
1250   } else
1251     return false;
1252 
1253   constrainSelectedInstRegOperands(*LoadInst, TII, TRI, RBI);
1254   I.eraseFromParent();
1255   return true;
1256 }
1257 
1258 bool X86InstructionSelector::selectImplicitDefOrPHI(
1259     MachineInstr &I, MachineRegisterInfo &MRI) const {
1260   assert((I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF ||
1261           I.getOpcode() == TargetOpcode::G_PHI) &&
1262          "unexpected instruction");
1263 
1264   unsigned DstReg = I.getOperand(0).getReg();
1265 
1266   if (!MRI.getRegClassOrNull(DstReg)) {
1267     const LLT DstTy = MRI.getType(DstReg);
1268     const TargetRegisterClass *RC = getRegClass(DstTy, DstReg, MRI);
1269 
1270     if (!RBI.constrainGenericRegister(DstReg, *RC, MRI)) {
1271       DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1272                    << " operand\n");
1273       return false;
1274     }
1275   }
1276 
1277   if (I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
1278     I.setDesc(TII.get(X86::IMPLICIT_DEF));
1279   else
1280     I.setDesc(TII.get(X86::PHI));
1281 
1282   return true;
1283 }
1284 
1285 InstructionSelector *
1286 llvm::createX86InstructionSelector(const X86TargetMachine &TM,
1287                                    X86Subtarget &Subtarget,
1288                                    X86RegisterBankInfo &RBI) {
1289   return new X86InstructionSelector(TM, Subtarget, RBI);
1290 }
1291