1 //===- X86InstructionSelector.cpp ----------------------------*- C++ -*-==//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 /// \file
10 /// This file implements the targeting of the InstructionSelector class for
11 /// X86.
12 /// \todo This should be generated by TableGen.
13 //===----------------------------------------------------------------------===//
14 
15 #include "X86InstrBuilder.h"
16 #include "X86InstrInfo.h"
17 #include "X86RegisterBankInfo.h"
18 #include "X86RegisterInfo.h"
19 #include "X86Subtarget.h"
20 #include "X86TargetMachine.h"
21 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
22 #include "llvm/CodeGen/GlobalISel/Utils.h"
23 #include "llvm/CodeGen/MachineBasicBlock.h"
24 #include "llvm/CodeGen/MachineConstantPool.h"
25 #include "llvm/CodeGen/MachineFunction.h"
26 #include "llvm/CodeGen/MachineInstr.h"
27 #include "llvm/CodeGen/MachineInstrBuilder.h"
28 #include "llvm/CodeGen/MachineOperand.h"
29 #include "llvm/CodeGen/MachineRegisterInfo.h"
30 #include "llvm/IR/Type.h"
31 #include "llvm/Support/Debug.h"
32 #include "llvm/Support/raw_ostream.h"
33 
34 #define DEBUG_TYPE "X86-isel"
35 
36 #include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h"
37 
38 using namespace llvm;
39 
40 namespace {
41 
42 #define GET_GLOBALISEL_PREDICATE_BITSET
43 #include "X86GenGlobalISel.inc"
44 #undef GET_GLOBALISEL_PREDICATE_BITSET
45 
46 class X86InstructionSelector : public InstructionSelector {
47 public:
48   X86InstructionSelector(const X86TargetMachine &TM, const X86Subtarget &STI,
49                          const X86RegisterBankInfo &RBI);
50 
51   bool select(MachineInstr &I) const override;
52 
53 private:
54   /// tblgen-erated 'select' implementation, used as the initial selector for
55   /// the patterns that don't require complex C++.
56   bool selectImpl(MachineInstr &I) const;
57 
58   // TODO: remove after supported by Tablegen-erated instruction selection.
59   unsigned getLoadStoreOp(const LLT &Ty, const RegisterBank &RB, unsigned Opc,
60                           uint64_t Alignment) const;
61 
62   bool selectLoadStoreOp(MachineInstr &I, MachineRegisterInfo &MRI,
63                          MachineFunction &MF) const;
64   bool selectFrameIndexOrGep(MachineInstr &I, MachineRegisterInfo &MRI,
65                              MachineFunction &MF) const;
66   bool selectGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI,
67                          MachineFunction &MF) const;
68   bool selectConstant(MachineInstr &I, MachineRegisterInfo &MRI,
69                       MachineFunction &MF) const;
70   bool selectTrunc(MachineInstr &I, MachineRegisterInfo &MRI,
71                    MachineFunction &MF) const;
72   bool selectZext(MachineInstr &I, MachineRegisterInfo &MRI,
73                   MachineFunction &MF) const;
74   bool selectAnyext(MachineInstr &I, MachineRegisterInfo &MRI,
75                     MachineFunction &MF) const;
76   bool selectCmp(MachineInstr &I, MachineRegisterInfo &MRI,
77                  MachineFunction &MF) const;
78   bool selectUadde(MachineInstr &I, MachineRegisterInfo &MRI,
79                    MachineFunction &MF) const;
80   bool selectCopy(MachineInstr &I, MachineRegisterInfo &MRI) const;
81   bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI,
82                            MachineFunction &MF) const;
83   bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI,
84                          MachineFunction &MF) const;
85   bool selectInsert(MachineInstr &I, MachineRegisterInfo &MRI,
86                     MachineFunction &MF) const;
87   bool selectExtract(MachineInstr &I, MachineRegisterInfo &MRI,
88                      MachineFunction &MF) const;
89   bool selectCondBranch(MachineInstr &I, MachineRegisterInfo &MRI,
90                         MachineFunction &MF) const;
91   bool materializeFP(MachineInstr &I, MachineRegisterInfo &MRI,
92                      MachineFunction &MF) const;
93   bool selectImplicitDefOrPHI(MachineInstr &I, MachineRegisterInfo &MRI) const;
94 
95   // emit insert subreg instruction and insert it before MachineInstr &I
96   bool emitInsertSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I,
97                         MachineRegisterInfo &MRI, MachineFunction &MF) const;
98   // emit extract subreg instruction and insert it before MachineInstr &I
99   bool emitExtractSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I,
100                          MachineRegisterInfo &MRI, MachineFunction &MF) const;
101 
102   const TargetRegisterClass *getRegClass(LLT Ty, const RegisterBank &RB) const;
103   const TargetRegisterClass *getRegClass(LLT Ty, unsigned Reg,
104                                          MachineRegisterInfo &MRI) const;
105 
106   const X86TargetMachine &TM;
107   const X86Subtarget &STI;
108   const X86InstrInfo &TII;
109   const X86RegisterInfo &TRI;
110   const X86RegisterBankInfo &RBI;
111 
112 #define GET_GLOBALISEL_PREDICATES_DECL
113 #include "X86GenGlobalISel.inc"
114 #undef GET_GLOBALISEL_PREDICATES_DECL
115 
116 #define GET_GLOBALISEL_TEMPORARIES_DECL
117 #include "X86GenGlobalISel.inc"
118 #undef GET_GLOBALISEL_TEMPORARIES_DECL
119 };
120 
121 } // end anonymous namespace
122 
123 #define GET_GLOBALISEL_IMPL
124 #include "X86GenGlobalISel.inc"
125 #undef GET_GLOBALISEL_IMPL
126 
127 X86InstructionSelector::X86InstructionSelector(const X86TargetMachine &TM,
128                                                const X86Subtarget &STI,
129                                                const X86RegisterBankInfo &RBI)
130     : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()),
131       TRI(*STI.getRegisterInfo()), RBI(RBI),
132 #define GET_GLOBALISEL_PREDICATES_INIT
133 #include "X86GenGlobalISel.inc"
134 #undef GET_GLOBALISEL_PREDICATES_INIT
135 #define GET_GLOBALISEL_TEMPORARIES_INIT
136 #include "X86GenGlobalISel.inc"
137 #undef GET_GLOBALISEL_TEMPORARIES_INIT
138 {
139 }
140 
141 // FIXME: This should be target-independent, inferred from the types declared
142 // for each class in the bank.
143 const TargetRegisterClass *
144 X86InstructionSelector::getRegClass(LLT Ty, const RegisterBank &RB) const {
145   if (RB.getID() == X86::GPRRegBankID) {
146     if (Ty.getSizeInBits() <= 8)
147       return &X86::GR8RegClass;
148     if (Ty.getSizeInBits() == 16)
149       return &X86::GR16RegClass;
150     if (Ty.getSizeInBits() == 32)
151       return &X86::GR32RegClass;
152     if (Ty.getSizeInBits() == 64)
153       return &X86::GR64RegClass;
154   }
155   if (RB.getID() == X86::VECRRegBankID) {
156     if (Ty.getSizeInBits() == 32)
157       return STI.hasAVX512() ? &X86::FR32XRegClass : &X86::FR32RegClass;
158     if (Ty.getSizeInBits() == 64)
159       return STI.hasAVX512() ? &X86::FR64XRegClass : &X86::FR64RegClass;
160     if (Ty.getSizeInBits() == 128)
161       return STI.hasAVX512() ? &X86::VR128XRegClass : &X86::VR128RegClass;
162     if (Ty.getSizeInBits() == 256)
163       return STI.hasAVX512() ? &X86::VR256XRegClass : &X86::VR256RegClass;
164     if (Ty.getSizeInBits() == 512)
165       return &X86::VR512RegClass;
166   }
167 
168   llvm_unreachable("Unknown RegBank!");
169 }
170 
171 const TargetRegisterClass *
172 X86InstructionSelector::getRegClass(LLT Ty, unsigned Reg,
173                                     MachineRegisterInfo &MRI) const {
174   const RegisterBank &RegBank = *RBI.getRegBank(Reg, MRI, TRI);
175   return getRegClass(Ty, RegBank);
176 }
177 
178 static unsigned getSubRegIndex(const TargetRegisterClass *RC) {
179   unsigned SubIdx = X86::NoSubRegister;
180   if (RC == &X86::GR32RegClass) {
181     SubIdx = X86::sub_32bit;
182   } else if (RC == &X86::GR16RegClass) {
183     SubIdx = X86::sub_16bit;
184   } else if (RC == &X86::GR8RegClass) {
185     SubIdx = X86::sub_8bit;
186   }
187 
188   return SubIdx;
189 }
190 
191 static const TargetRegisterClass *getRegClassFromGRPhysReg(unsigned Reg) {
192   assert(TargetRegisterInfo::isPhysicalRegister(Reg));
193   if (X86::GR64RegClass.contains(Reg))
194     return &X86::GR64RegClass;
195   if (X86::GR32RegClass.contains(Reg))
196     return &X86::GR32RegClass;
197   if (X86::GR16RegClass.contains(Reg))
198     return &X86::GR16RegClass;
199   if (X86::GR8RegClass.contains(Reg))
200     return &X86::GR8RegClass;
201 
202   llvm_unreachable("Unknown RegClass for PhysReg!");
203 }
204 
205 // Set X86 Opcode and constrain DestReg.
206 bool X86InstructionSelector::selectCopy(MachineInstr &I,
207                                         MachineRegisterInfo &MRI) const {
208 
209   unsigned DstReg = I.getOperand(0).getReg();
210   const unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI);
211   const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI);
212 
213   unsigned SrcReg = I.getOperand(1).getReg();
214   const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);
215   const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI);
216 
217   if (TargetRegisterInfo::isPhysicalRegister(DstReg)) {
218     assert(I.isCopy() && "Generic operators do not allow physical registers");
219 
220     if (DstSize > SrcSize && SrcRegBank.getID() == X86::GPRRegBankID &&
221         DstRegBank.getID() == X86::GPRRegBankID) {
222 
223       const TargetRegisterClass *SrcRC =
224           getRegClass(MRI.getType(SrcReg), SrcRegBank);
225       const TargetRegisterClass *DstRC = getRegClassFromGRPhysReg(DstReg);
226 
227       if (SrcRC != DstRC) {
228         // This case can be generated by ABI lowering, performe anyext
229         unsigned ExtSrc = MRI.createVirtualRegister(DstRC);
230         BuildMI(*I.getParent(), I, I.getDebugLoc(),
231                 TII.get(TargetOpcode::SUBREG_TO_REG))
232             .addDef(ExtSrc)
233             .addImm(0)
234             .addReg(SrcReg)
235             .addImm(getSubRegIndex(SrcRC));
236 
237         I.getOperand(1).setReg(ExtSrc);
238       }
239     }
240 
241     return true;
242   }
243 
244   assert((!TargetRegisterInfo::isPhysicalRegister(SrcReg) || I.isCopy()) &&
245          "No phys reg on generic operators");
246   assert((DstSize == SrcSize ||
247           // Copies are a mean to setup initial types, the number of
248           // bits may not exactly match.
249           (TargetRegisterInfo::isPhysicalRegister(SrcReg) &&
250            DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI))) &&
251          "Copy with different width?!");
252 
253   const TargetRegisterClass *DstRC =
254       getRegClass(MRI.getType(DstReg), DstRegBank);
255 
256   if (SrcRegBank.getID() == X86::GPRRegBankID &&
257       DstRegBank.getID() == X86::GPRRegBankID && SrcSize > DstSize &&
258       TargetRegisterInfo::isPhysicalRegister(SrcReg)) {
259     // Change the physical register to performe truncate.
260 
261     const TargetRegisterClass *SrcRC = getRegClassFromGRPhysReg(SrcReg);
262 
263     if (DstRC != SrcRC) {
264       I.getOperand(1).setSubReg(getSubRegIndex(DstRC));
265       I.getOperand(1).substPhysReg(SrcReg, TRI);
266     }
267   }
268 
269   // No need to constrain SrcReg. It will get constrained when
270   // we hit another of its use or its defs.
271   // Copies do not have constraints.
272   const TargetRegisterClass *OldRC = MRI.getRegClassOrNull(DstReg);
273   if (!OldRC || !DstRC->hasSubClassEq(OldRC)) {
274     if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
275       DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
276                    << " operand\n");
277       return false;
278     }
279   }
280   I.setDesc(TII.get(X86::COPY));
281   return true;
282 }
283 
284 bool X86InstructionSelector::select(MachineInstr &I) const {
285   assert(I.getParent() && "Instruction should be in a basic block!");
286   assert(I.getParent()->getParent() && "Instruction should be in a function!");
287 
288   MachineBasicBlock &MBB = *I.getParent();
289   MachineFunction &MF = *MBB.getParent();
290   MachineRegisterInfo &MRI = MF.getRegInfo();
291 
292   unsigned Opcode = I.getOpcode();
293   if (!isPreISelGenericOpcode(Opcode)) {
294     // Certain non-generic instructions also need some special handling.
295 
296     if (Opcode == TargetOpcode::LOAD_STACK_GUARD)
297       return false;
298 
299     if (I.isCopy())
300       return selectCopy(I, MRI);
301 
302     return true;
303   }
304 
305   assert(I.getNumOperands() == I.getNumExplicitOperands() &&
306          "Generic instruction has unexpected implicit operands\n");
307 
308   if (selectImpl(I))
309     return true;
310 
311   DEBUG(dbgs() << " C++ instruction selection: "; I.print(dbgs()));
312 
313   // TODO: This should be implemented by tblgen.
314   switch (I.getOpcode()) {
315   default:
316     return false;
317   case TargetOpcode::G_STORE:
318   case TargetOpcode::G_LOAD:
319     return selectLoadStoreOp(I, MRI, MF);
320   case TargetOpcode::G_GEP:
321   case TargetOpcode::G_FRAME_INDEX:
322     return selectFrameIndexOrGep(I, MRI, MF);
323   case TargetOpcode::G_GLOBAL_VALUE:
324     return selectGlobalValue(I, MRI, MF);
325   case TargetOpcode::G_CONSTANT:
326     return selectConstant(I, MRI, MF);
327   case TargetOpcode::G_FCONSTANT:
328     return materializeFP(I, MRI, MF);
329   case TargetOpcode::G_TRUNC:
330     return selectTrunc(I, MRI, MF);
331   case TargetOpcode::G_ZEXT:
332     return selectZext(I, MRI, MF);
333   case TargetOpcode::G_ANYEXT:
334     return selectAnyext(I, MRI, MF);
335   case TargetOpcode::G_ICMP:
336     return selectCmp(I, MRI, MF);
337   case TargetOpcode::G_UADDE:
338     return selectUadde(I, MRI, MF);
339   case TargetOpcode::G_UNMERGE_VALUES:
340     return selectUnmergeValues(I, MRI, MF);
341   case TargetOpcode::G_MERGE_VALUES:
342     return selectMergeValues(I, MRI, MF);
343   case TargetOpcode::G_EXTRACT:
344     return selectExtract(I, MRI, MF);
345   case TargetOpcode::G_INSERT:
346     return selectInsert(I, MRI, MF);
347   case TargetOpcode::G_BRCOND:
348     return selectCondBranch(I, MRI, MF);
349   case TargetOpcode::G_IMPLICIT_DEF:
350   case TargetOpcode::G_PHI:
351     return selectImplicitDefOrPHI(I, MRI);
352   }
353 
354   return false;
355 }
356 
357 unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
358                                                 const RegisterBank &RB,
359                                                 unsigned Opc,
360                                                 uint64_t Alignment) const {
361   bool Isload = (Opc == TargetOpcode::G_LOAD);
362   bool HasAVX = STI.hasAVX();
363   bool HasAVX512 = STI.hasAVX512();
364   bool HasVLX = STI.hasVLX();
365 
366   if (Ty == LLT::scalar(8)) {
367     if (X86::GPRRegBankID == RB.getID())
368       return Isload ? X86::MOV8rm : X86::MOV8mr;
369   } else if (Ty == LLT::scalar(16)) {
370     if (X86::GPRRegBankID == RB.getID())
371       return Isload ? X86::MOV16rm : X86::MOV16mr;
372   } else if (Ty == LLT::scalar(32) || Ty == LLT::pointer(0, 32)) {
373     if (X86::GPRRegBankID == RB.getID())
374       return Isload ? X86::MOV32rm : X86::MOV32mr;
375     if (X86::VECRRegBankID == RB.getID())
376       return Isload ? (HasAVX512 ? X86::VMOVSSZrm
377                                  : HasAVX ? X86::VMOVSSrm : X86::MOVSSrm)
378                     : (HasAVX512 ? X86::VMOVSSZmr
379                                  : HasAVX ? X86::VMOVSSmr : X86::MOVSSmr);
380   } else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) {
381     if (X86::GPRRegBankID == RB.getID())
382       return Isload ? X86::MOV64rm : X86::MOV64mr;
383     if (X86::VECRRegBankID == RB.getID())
384       return Isload ? (HasAVX512 ? X86::VMOVSDZrm
385                                  : HasAVX ? X86::VMOVSDrm : X86::MOVSDrm)
386                     : (HasAVX512 ? X86::VMOVSDZmr
387                                  : HasAVX ? X86::VMOVSDmr : X86::MOVSDmr);
388   } else if (Ty.isVector() && Ty.getSizeInBits() == 128) {
389     if (Alignment >= 16)
390       return Isload ? (HasVLX ? X86::VMOVAPSZ128rm
391                               : HasAVX512
392                                     ? X86::VMOVAPSZ128rm_NOVLX
393                                     : HasAVX ? X86::VMOVAPSrm : X86::MOVAPSrm)
394                     : (HasVLX ? X86::VMOVAPSZ128mr
395                               : HasAVX512
396                                     ? X86::VMOVAPSZ128mr_NOVLX
397                                     : HasAVX ? X86::VMOVAPSmr : X86::MOVAPSmr);
398     else
399       return Isload ? (HasVLX ? X86::VMOVUPSZ128rm
400                               : HasAVX512
401                                     ? X86::VMOVUPSZ128rm_NOVLX
402                                     : HasAVX ? X86::VMOVUPSrm : X86::MOVUPSrm)
403                     : (HasVLX ? X86::VMOVUPSZ128mr
404                               : HasAVX512
405                                     ? X86::VMOVUPSZ128mr_NOVLX
406                                     : HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr);
407   } else if (Ty.isVector() && Ty.getSizeInBits() == 256) {
408     if (Alignment >= 32)
409       return Isload ? (HasVLX ? X86::VMOVAPSZ256rm
410                               : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX
411                                           : X86::VMOVAPSYrm)
412                     : (HasVLX ? X86::VMOVAPSZ256mr
413                               : HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX
414                                           : X86::VMOVAPSYmr);
415     else
416       return Isload ? (HasVLX ? X86::VMOVUPSZ256rm
417                               : HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX
418                                           : X86::VMOVUPSYrm)
419                     : (HasVLX ? X86::VMOVUPSZ256mr
420                               : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX
421                                           : X86::VMOVUPSYmr);
422   } else if (Ty.isVector() && Ty.getSizeInBits() == 512) {
423     if (Alignment >= 64)
424       return Isload ? X86::VMOVAPSZrm : X86::VMOVAPSZmr;
425     else
426       return Isload ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
427   }
428   return Opc;
429 }
430 
431 // Fill in an address from the given instruction.
432 static void X86SelectAddress(const MachineInstr &I,
433                              const MachineRegisterInfo &MRI,
434                              X86AddressMode &AM) {
435 
436   assert(I.getOperand(0).isReg() && "unsupported opperand.");
437   assert(MRI.getType(I.getOperand(0).getReg()).isPointer() &&
438          "unsupported type.");
439 
440   if (I.getOpcode() == TargetOpcode::G_GEP) {
441     if (auto COff = getConstantVRegVal(I.getOperand(2).getReg(), MRI)) {
442       int64_t Imm = *COff;
443       if (isInt<32>(Imm)) { // Check for displacement overflow.
444         AM.Disp = static_cast<int32_t>(Imm);
445         AM.Base.Reg = I.getOperand(1).getReg();
446         return;
447       }
448     }
449   } else if (I.getOpcode() == TargetOpcode::G_FRAME_INDEX) {
450     AM.Base.FrameIndex = I.getOperand(1).getIndex();
451     AM.BaseType = X86AddressMode::FrameIndexBase;
452     return;
453   }
454 
455   // Default behavior.
456   AM.Base.Reg = I.getOperand(0).getReg();
457   return;
458 }
459 
460 bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I,
461                                                MachineRegisterInfo &MRI,
462                                                MachineFunction &MF) const {
463 
464   unsigned Opc = I.getOpcode();
465 
466   assert((Opc == TargetOpcode::G_STORE || Opc == TargetOpcode::G_LOAD) &&
467          "unexpected instruction");
468 
469   const unsigned DefReg = I.getOperand(0).getReg();
470   LLT Ty = MRI.getType(DefReg);
471   const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
472 
473   auto &MemOp = **I.memoperands_begin();
474   if (MemOp.getOrdering() != AtomicOrdering::NotAtomic) {
475     DEBUG(dbgs() << "Atomic load/store not supported yet\n");
476     return false;
477   }
478 
479   unsigned NewOpc = getLoadStoreOp(Ty, RB, Opc, MemOp.getAlignment());
480   if (NewOpc == Opc)
481     return false;
482 
483   X86AddressMode AM;
484   X86SelectAddress(*MRI.getVRegDef(I.getOperand(1).getReg()), MRI, AM);
485 
486   I.setDesc(TII.get(NewOpc));
487   MachineInstrBuilder MIB(MF, I);
488   if (Opc == TargetOpcode::G_LOAD) {
489     I.RemoveOperand(1);
490     addFullAddress(MIB, AM);
491   } else {
492     // G_STORE (VAL, Addr), X86Store instruction (Addr, VAL)
493     I.RemoveOperand(1);
494     I.RemoveOperand(0);
495     addFullAddress(MIB, AM).addUse(DefReg);
496   }
497   return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
498 }
499 
500 static unsigned getLeaOP(LLT Ty, const X86Subtarget &STI) {
501   if (Ty == LLT::pointer(0, 64))
502     return X86::LEA64r;
503   else if (Ty == LLT::pointer(0, 32))
504     return STI.isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r;
505   else
506     llvm_unreachable("Can't get LEA opcode. Unsupported type.");
507 }
508 
509 bool X86InstructionSelector::selectFrameIndexOrGep(MachineInstr &I,
510                                                    MachineRegisterInfo &MRI,
511                                                    MachineFunction &MF) const {
512   unsigned Opc = I.getOpcode();
513 
514   assert((Opc == TargetOpcode::G_FRAME_INDEX || Opc == TargetOpcode::G_GEP) &&
515          "unexpected instruction");
516 
517   const unsigned DefReg = I.getOperand(0).getReg();
518   LLT Ty = MRI.getType(DefReg);
519 
520   // Use LEA to calculate frame index and GEP
521   unsigned NewOpc = getLeaOP(Ty, STI);
522   I.setDesc(TII.get(NewOpc));
523   MachineInstrBuilder MIB(MF, I);
524 
525   if (Opc == TargetOpcode::G_FRAME_INDEX) {
526     addOffset(MIB, 0);
527   } else {
528     MachineOperand &InxOp = I.getOperand(2);
529     I.addOperand(InxOp);        // set IndexReg
530     InxOp.ChangeToImmediate(1); // set Scale
531     MIB.addImm(0).addReg(0);
532   }
533 
534   return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
535 }
536 
537 bool X86InstructionSelector::selectGlobalValue(MachineInstr &I,
538                                                MachineRegisterInfo &MRI,
539                                                MachineFunction &MF) const {
540 
541   assert((I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE) &&
542          "unexpected instruction");
543 
544   auto GV = I.getOperand(1).getGlobal();
545   if (GV->isThreadLocal()) {
546     return false; // TODO: we don't support TLS yet.
547   }
548 
549   // Can't handle alternate code models yet.
550   if (TM.getCodeModel() != CodeModel::Small)
551     return 0;
552 
553   X86AddressMode AM;
554   AM.GV = GV;
555   AM.GVOpFlags = STI.classifyGlobalReference(GV);
556 
557   // TODO: The ABI requires an extra load. not supported yet.
558   if (isGlobalStubReference(AM.GVOpFlags))
559     return false;
560 
561   // TODO: This reference is relative to the pic base. not supported yet.
562   if (isGlobalRelativeToPICBase(AM.GVOpFlags))
563     return false;
564 
565   if (STI.isPICStyleRIPRel()) {
566     // Use rip-relative addressing.
567     assert(AM.Base.Reg == 0 && AM.IndexReg == 0);
568     AM.Base.Reg = X86::RIP;
569   }
570 
571   const unsigned DefReg = I.getOperand(0).getReg();
572   LLT Ty = MRI.getType(DefReg);
573   unsigned NewOpc = getLeaOP(Ty, STI);
574 
575   I.setDesc(TII.get(NewOpc));
576   MachineInstrBuilder MIB(MF, I);
577 
578   I.RemoveOperand(1);
579   addFullAddress(MIB, AM);
580 
581   return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
582 }
583 
584 bool X86InstructionSelector::selectConstant(MachineInstr &I,
585                                             MachineRegisterInfo &MRI,
586                                             MachineFunction &MF) const {
587 
588   assert((I.getOpcode() == TargetOpcode::G_CONSTANT) &&
589          "unexpected instruction");
590 
591   const unsigned DefReg = I.getOperand(0).getReg();
592   LLT Ty = MRI.getType(DefReg);
593 
594   if (RBI.getRegBank(DefReg, MRI, TRI)->getID() != X86::GPRRegBankID)
595     return false;
596 
597   uint64_t Val = 0;
598   if (I.getOperand(1).isCImm()) {
599     Val = I.getOperand(1).getCImm()->getZExtValue();
600     I.getOperand(1).ChangeToImmediate(Val);
601   } else if (I.getOperand(1).isImm()) {
602     Val = I.getOperand(1).getImm();
603   } else
604     llvm_unreachable("Unsupported operand type.");
605 
606   unsigned NewOpc;
607   switch (Ty.getSizeInBits()) {
608   case 8:
609     NewOpc = X86::MOV8ri;
610     break;
611   case 16:
612     NewOpc = X86::MOV16ri;
613     break;
614   case 32:
615     NewOpc = X86::MOV32ri;
616     break;
617   case 64: {
618     // TODO: in case isUInt<32>(Val), X86::MOV32ri can be used
619     if (isInt<32>(Val))
620       NewOpc = X86::MOV64ri32;
621     else
622       NewOpc = X86::MOV64ri;
623     break;
624   }
625   default:
626     llvm_unreachable("Can't select G_CONSTANT, unsupported type.");
627   }
628 
629   I.setDesc(TII.get(NewOpc));
630   return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
631 }
632 
633 bool X86InstructionSelector::selectTrunc(MachineInstr &I,
634                                          MachineRegisterInfo &MRI,
635                                          MachineFunction &MF) const {
636 
637   assert((I.getOpcode() == TargetOpcode::G_TRUNC) && "unexpected instruction");
638 
639   const unsigned DstReg = I.getOperand(0).getReg();
640   const unsigned SrcReg = I.getOperand(1).getReg();
641 
642   const LLT DstTy = MRI.getType(DstReg);
643   const LLT SrcTy = MRI.getType(SrcReg);
644 
645   const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
646   const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
647 
648   if (DstRB.getID() != SrcRB.getID()) {
649     DEBUG(dbgs() << "G_TRUNC input/output on different banks\n");
650     return false;
651   }
652 
653   if (DstRB.getID() != X86::GPRRegBankID)
654     return false;
655 
656   const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB);
657   if (!DstRC)
658     return false;
659 
660   const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB);
661   if (!SrcRC)
662     return false;
663 
664   unsigned SubIdx;
665   if (DstRC == SrcRC) {
666     // Nothing to be done
667     SubIdx = X86::NoSubRegister;
668   } else if (DstRC == &X86::GR32RegClass) {
669     SubIdx = X86::sub_32bit;
670   } else if (DstRC == &X86::GR16RegClass) {
671     SubIdx = X86::sub_16bit;
672   } else if (DstRC == &X86::GR8RegClass) {
673     SubIdx = X86::sub_8bit;
674   } else {
675     return false;
676   }
677 
678   SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx);
679 
680   if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
681       !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
682     DEBUG(dbgs() << "Failed to constrain G_TRUNC\n");
683     return false;
684   }
685 
686   I.getOperand(1).setSubReg(SubIdx);
687 
688   I.setDesc(TII.get(X86::COPY));
689   return true;
690 }
691 
692 bool X86InstructionSelector::selectZext(MachineInstr &I,
693                                         MachineRegisterInfo &MRI,
694                                         MachineFunction &MF) const {
695 
696   assert((I.getOpcode() == TargetOpcode::G_ZEXT) && "unexpected instruction");
697 
698   const unsigned DstReg = I.getOperand(0).getReg();
699   const unsigned SrcReg = I.getOperand(1).getReg();
700 
701   const LLT DstTy = MRI.getType(DstReg);
702   const LLT SrcTy = MRI.getType(SrcReg);
703 
704   if (SrcTy != LLT::scalar(1))
705     return false;
706 
707   unsigned AndOpc;
708   if (DstTy == LLT::scalar(8))
709     AndOpc = X86::AND8ri;
710   else if (DstTy == LLT::scalar(16))
711     AndOpc = X86::AND16ri8;
712   else if (DstTy == LLT::scalar(32))
713     AndOpc = X86::AND32ri8;
714   else if (DstTy == LLT::scalar(64))
715     AndOpc = X86::AND64ri8;
716   else
717     return false;
718 
719   unsigned DefReg = SrcReg;
720   if (DstTy != LLT::scalar(8)) {
721     DefReg = MRI.createVirtualRegister(getRegClass(DstTy, DstReg, MRI));
722     BuildMI(*I.getParent(), I, I.getDebugLoc(),
723             TII.get(TargetOpcode::SUBREG_TO_REG), DefReg)
724         .addImm(0)
725         .addReg(SrcReg)
726         .addImm(X86::sub_8bit);
727   }
728 
729   MachineInstr &AndInst =
730       *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AndOpc), DstReg)
731            .addReg(DefReg)
732            .addImm(1);
733 
734   constrainSelectedInstRegOperands(AndInst, TII, TRI, RBI);
735 
736   I.eraseFromParent();
737   return true;
738 }
739 
740 bool X86InstructionSelector::selectAnyext(MachineInstr &I,
741                                           MachineRegisterInfo &MRI,
742                                           MachineFunction &MF) const {
743 
744   assert((I.getOpcode() == TargetOpcode::G_ANYEXT) && "unexpected instruction");
745 
746   const unsigned DstReg = I.getOperand(0).getReg();
747   const unsigned SrcReg = I.getOperand(1).getReg();
748 
749   const LLT DstTy = MRI.getType(DstReg);
750   const LLT SrcTy = MRI.getType(SrcReg);
751 
752   const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
753   const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
754 
755   assert(DstRB.getID() == SrcRB.getID() &&
756          "G_ANYEXT input/output on different banks\n");
757 
758   assert(DstTy.getSizeInBits() > SrcTy.getSizeInBits() &&
759          "G_ANYEXT incorrect operand size");
760 
761   if (DstRB.getID() != X86::GPRRegBankID)
762     return false;
763 
764   const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB);
765   const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB);
766 
767   if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
768       !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
769     DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
770                  << " operand\n");
771     return false;
772   }
773 
774   if (SrcRC == DstRC) {
775     I.setDesc(TII.get(X86::COPY));
776     return true;
777   }
778 
779   BuildMI(*I.getParent(), I, I.getDebugLoc(),
780           TII.get(TargetOpcode::SUBREG_TO_REG))
781       .addDef(DstReg)
782       .addImm(0)
783       .addReg(SrcReg)
784       .addImm(getSubRegIndex(SrcRC));
785 
786   I.eraseFromParent();
787   return true;
788 }
789 
790 bool X86InstructionSelector::selectCmp(MachineInstr &I,
791                                        MachineRegisterInfo &MRI,
792                                        MachineFunction &MF) const {
793 
794   assert((I.getOpcode() == TargetOpcode::G_ICMP) && "unexpected instruction");
795 
796   X86::CondCode CC;
797   bool SwapArgs;
798   std::tie(CC, SwapArgs) = X86::getX86ConditionCode(
799       (CmpInst::Predicate)I.getOperand(1).getPredicate());
800   unsigned OpSet = X86::getSETFromCond(CC);
801 
802   unsigned LHS = I.getOperand(2).getReg();
803   unsigned RHS = I.getOperand(3).getReg();
804 
805   if (SwapArgs)
806     std::swap(LHS, RHS);
807 
808   unsigned OpCmp;
809   LLT Ty = MRI.getType(LHS);
810 
811   switch (Ty.getSizeInBits()) {
812   default:
813     return false;
814   case 8:
815     OpCmp = X86::CMP8rr;
816     break;
817   case 16:
818     OpCmp = X86::CMP16rr;
819     break;
820   case 32:
821     OpCmp = X86::CMP32rr;
822     break;
823   case 64:
824     OpCmp = X86::CMP64rr;
825     break;
826   }
827 
828   MachineInstr &CmpInst =
829       *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp))
830            .addReg(LHS)
831            .addReg(RHS);
832 
833   MachineInstr &SetInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
834                                    TII.get(OpSet), I.getOperand(0).getReg());
835 
836   constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI);
837   constrainSelectedInstRegOperands(SetInst, TII, TRI, RBI);
838 
839   I.eraseFromParent();
840   return true;
841 }
842 
843 bool X86InstructionSelector::selectUadde(MachineInstr &I,
844                                          MachineRegisterInfo &MRI,
845                                          MachineFunction &MF) const {
846 
847   assert((I.getOpcode() == TargetOpcode::G_UADDE) && "unexpected instruction");
848 
849   const unsigned DstReg = I.getOperand(0).getReg();
850   const unsigned CarryOutReg = I.getOperand(1).getReg();
851   const unsigned Op0Reg = I.getOperand(2).getReg();
852   const unsigned Op1Reg = I.getOperand(3).getReg();
853   unsigned CarryInReg = I.getOperand(4).getReg();
854 
855   const LLT DstTy = MRI.getType(DstReg);
856 
857   if (DstTy != LLT::scalar(32))
858     return false;
859 
860   // find CarryIn def instruction.
861   MachineInstr *Def = MRI.getVRegDef(CarryInReg);
862   while (Def->getOpcode() == TargetOpcode::G_TRUNC) {
863     CarryInReg = Def->getOperand(1).getReg();
864     Def = MRI.getVRegDef(CarryInReg);
865   }
866 
867   unsigned Opcode;
868   if (Def->getOpcode() == TargetOpcode::G_UADDE) {
869     // carry set by prev ADD.
870 
871     BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), X86::EFLAGS)
872         .addReg(CarryInReg);
873 
874     if (!RBI.constrainGenericRegister(CarryInReg, X86::GR32RegClass, MRI))
875       return false;
876 
877     Opcode = X86::ADC32rr;
878   } else if (auto val = getConstantVRegVal(CarryInReg, MRI)) {
879     // carry is constant, support only 0.
880     if (*val != 0)
881       return false;
882 
883     Opcode = X86::ADD32rr;
884   } else
885     return false;
886 
887   MachineInstr &AddInst =
888       *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode), DstReg)
889            .addReg(Op0Reg)
890            .addReg(Op1Reg);
891 
892   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), CarryOutReg)
893       .addReg(X86::EFLAGS);
894 
895   if (!constrainSelectedInstRegOperands(AddInst, TII, TRI, RBI) ||
896       !RBI.constrainGenericRegister(CarryOutReg, X86::GR32RegClass, MRI))
897     return false;
898 
899   I.eraseFromParent();
900   return true;
901 }
902 
903 bool X86InstructionSelector::selectExtract(MachineInstr &I,
904                                            MachineRegisterInfo &MRI,
905                                            MachineFunction &MF) const {
906 
907   assert((I.getOpcode() == TargetOpcode::G_EXTRACT) &&
908          "unexpected instruction");
909 
910   const unsigned DstReg = I.getOperand(0).getReg();
911   const unsigned SrcReg = I.getOperand(1).getReg();
912   int64_t Index = I.getOperand(2).getImm();
913 
914   const LLT DstTy = MRI.getType(DstReg);
915   const LLT SrcTy = MRI.getType(SrcReg);
916 
917   // Meanwile handle vector type only.
918   if (!DstTy.isVector())
919     return false;
920 
921   if (Index % DstTy.getSizeInBits() != 0)
922     return false; // Not extract subvector.
923 
924   if (Index == 0) {
925     // Replace by extract subreg copy.
926     if (!emitExtractSubreg(DstReg, SrcReg, I, MRI, MF))
927       return false;
928 
929     I.eraseFromParent();
930     return true;
931   }
932 
933   bool HasAVX = STI.hasAVX();
934   bool HasAVX512 = STI.hasAVX512();
935   bool HasVLX = STI.hasVLX();
936 
937   if (SrcTy.getSizeInBits() == 256 && DstTy.getSizeInBits() == 128) {
938     if (HasVLX)
939       I.setDesc(TII.get(X86::VEXTRACTF32x4Z256rr));
940     else if (HasAVX)
941       I.setDesc(TII.get(X86::VEXTRACTF128rr));
942     else
943       return false;
944   } else if (SrcTy.getSizeInBits() == 512 && HasAVX512) {
945     if (DstTy.getSizeInBits() == 128)
946       I.setDesc(TII.get(X86::VEXTRACTF32x4Zrr));
947     else if (DstTy.getSizeInBits() == 256)
948       I.setDesc(TII.get(X86::VEXTRACTF64x4Zrr));
949     else
950       return false;
951   } else
952     return false;
953 
954   // Convert to X86 VEXTRACT immediate.
955   Index = Index / DstTy.getSizeInBits();
956   I.getOperand(2).setImm(Index);
957 
958   return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
959 }
960 
961 bool X86InstructionSelector::emitExtractSubreg(unsigned DstReg, unsigned SrcReg,
962                                                MachineInstr &I,
963                                                MachineRegisterInfo &MRI,
964                                                MachineFunction &MF) const {
965 
966   const LLT DstTy = MRI.getType(DstReg);
967   const LLT SrcTy = MRI.getType(SrcReg);
968   unsigned SubIdx = X86::NoSubRegister;
969 
970   if (!DstTy.isVector() || !SrcTy.isVector())
971     return false;
972 
973   assert(SrcTy.getSizeInBits() > DstTy.getSizeInBits() &&
974          "Incorrect Src/Dst register size");
975 
976   if (DstTy.getSizeInBits() == 128)
977     SubIdx = X86::sub_xmm;
978   else if (DstTy.getSizeInBits() == 256)
979     SubIdx = X86::sub_ymm;
980   else
981     return false;
982 
983   const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI);
984   const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI);
985 
986   SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx);
987 
988   if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
989       !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
990     DEBUG(dbgs() << "Failed to constrain G_TRUNC\n");
991     return false;
992   }
993 
994   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), DstReg)
995       .addReg(SrcReg, 0, SubIdx);
996 
997   return true;
998 }
999 
1000 bool X86InstructionSelector::emitInsertSubreg(unsigned DstReg, unsigned SrcReg,
1001                                               MachineInstr &I,
1002                                               MachineRegisterInfo &MRI,
1003                                               MachineFunction &MF) const {
1004 
1005   const LLT DstTy = MRI.getType(DstReg);
1006   const LLT SrcTy = MRI.getType(SrcReg);
1007   unsigned SubIdx = X86::NoSubRegister;
1008 
1009   // TODO: support scalar types
1010   if (!DstTy.isVector() || !SrcTy.isVector())
1011     return false;
1012 
1013   assert(SrcTy.getSizeInBits() < DstTy.getSizeInBits() &&
1014          "Incorrect Src/Dst register size");
1015 
1016   if (SrcTy.getSizeInBits() == 128)
1017     SubIdx = X86::sub_xmm;
1018   else if (SrcTy.getSizeInBits() == 256)
1019     SubIdx = X86::sub_ymm;
1020   else
1021     return false;
1022 
1023   const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI);
1024   const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI);
1025 
1026   if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
1027       !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
1028     DEBUG(dbgs() << "Failed to constrain INSERT_SUBREG\n");
1029     return false;
1030   }
1031 
1032   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY))
1033       .addReg(DstReg, RegState::DefineNoRead, SubIdx)
1034       .addReg(SrcReg);
1035 
1036   return true;
1037 }
1038 
1039 bool X86InstructionSelector::selectInsert(MachineInstr &I,
1040                                           MachineRegisterInfo &MRI,
1041                                           MachineFunction &MF) const {
1042 
1043   assert((I.getOpcode() == TargetOpcode::G_INSERT) && "unexpected instruction");
1044 
1045   const unsigned DstReg = I.getOperand(0).getReg();
1046   const unsigned SrcReg = I.getOperand(1).getReg();
1047   const unsigned InsertReg = I.getOperand(2).getReg();
1048   int64_t Index = I.getOperand(3).getImm();
1049 
1050   const LLT DstTy = MRI.getType(DstReg);
1051   const LLT InsertRegTy = MRI.getType(InsertReg);
1052 
1053   // Meanwile handle vector type only.
1054   if (!DstTy.isVector())
1055     return false;
1056 
1057   if (Index % InsertRegTy.getSizeInBits() != 0)
1058     return false; // Not insert subvector.
1059 
1060   if (Index == 0 && MRI.getVRegDef(SrcReg)->isImplicitDef()) {
1061     // Replace by subreg copy.
1062     if (!emitInsertSubreg(DstReg, InsertReg, I, MRI, MF))
1063       return false;
1064 
1065     I.eraseFromParent();
1066     return true;
1067   }
1068 
1069   bool HasAVX = STI.hasAVX();
1070   bool HasAVX512 = STI.hasAVX512();
1071   bool HasVLX = STI.hasVLX();
1072 
1073   if (DstTy.getSizeInBits() == 256 && InsertRegTy.getSizeInBits() == 128) {
1074     if (HasVLX)
1075       I.setDesc(TII.get(X86::VINSERTF32x4Z256rr));
1076     else if (HasAVX)
1077       I.setDesc(TII.get(X86::VINSERTF128rr));
1078     else
1079       return false;
1080   } else if (DstTy.getSizeInBits() == 512 && HasAVX512) {
1081     if (InsertRegTy.getSizeInBits() == 128)
1082       I.setDesc(TII.get(X86::VINSERTF32x4Zrr));
1083     else if (InsertRegTy.getSizeInBits() == 256)
1084       I.setDesc(TII.get(X86::VINSERTF64x4Zrr));
1085     else
1086       return false;
1087   } else
1088     return false;
1089 
1090   // Convert to X86 VINSERT immediate.
1091   Index = Index / InsertRegTy.getSizeInBits();
1092 
1093   I.getOperand(3).setImm(Index);
1094 
1095   return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1096 }
1097 
1098 bool X86InstructionSelector::selectUnmergeValues(MachineInstr &I,
1099                                                  MachineRegisterInfo &MRI,
1100                                                  MachineFunction &MF) const {
1101 
1102   assert((I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES) &&
1103          "unexpected instruction");
1104 
1105   // Split to extracts.
1106   unsigned NumDefs = I.getNumOperands() - 1;
1107   unsigned SrcReg = I.getOperand(NumDefs).getReg();
1108   unsigned DefSize = MRI.getType(I.getOperand(0).getReg()).getSizeInBits();
1109 
1110   for (unsigned Idx = 0; Idx < NumDefs; ++Idx) {
1111 
1112     MachineInstr &ExtrInst =
1113         *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1114                  TII.get(TargetOpcode::G_EXTRACT), I.getOperand(Idx).getReg())
1115              .addReg(SrcReg)
1116              .addImm(Idx * DefSize);
1117 
1118     if (!select(ExtrInst))
1119       return false;
1120   }
1121 
1122   I.eraseFromParent();
1123   return true;
1124 }
1125 
1126 bool X86InstructionSelector::selectMergeValues(MachineInstr &I,
1127                                                MachineRegisterInfo &MRI,
1128                                                MachineFunction &MF) const {
1129 
1130   assert((I.getOpcode() == TargetOpcode::G_MERGE_VALUES) &&
1131          "unexpected instruction");
1132 
1133   // Split to inserts.
1134   unsigned DstReg = I.getOperand(0).getReg();
1135   unsigned SrcReg0 = I.getOperand(1).getReg();
1136 
1137   const LLT DstTy = MRI.getType(DstReg);
1138   const LLT SrcTy = MRI.getType(SrcReg0);
1139   unsigned SrcSize = SrcTy.getSizeInBits();
1140 
1141   const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
1142 
1143   // For the first src use insertSubReg.
1144   unsigned DefReg = MRI.createGenericVirtualRegister(DstTy);
1145   MRI.setRegBank(DefReg, RegBank);
1146   if (!emitInsertSubreg(DefReg, I.getOperand(1).getReg(), I, MRI, MF))
1147     return false;
1148 
1149   for (unsigned Idx = 2; Idx < I.getNumOperands(); ++Idx) {
1150 
1151     unsigned Tmp = MRI.createGenericVirtualRegister(DstTy);
1152     MRI.setRegBank(Tmp, RegBank);
1153 
1154     MachineInstr &InsertInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1155                                         TII.get(TargetOpcode::G_INSERT), Tmp)
1156                                     .addReg(DefReg)
1157                                     .addReg(I.getOperand(Idx).getReg())
1158                                     .addImm((Idx - 1) * SrcSize);
1159 
1160     DefReg = Tmp;
1161 
1162     if (!select(InsertInst))
1163       return false;
1164   }
1165 
1166   MachineInstr &CopyInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1167                                     TII.get(TargetOpcode::COPY), DstReg)
1168                                 .addReg(DefReg);
1169 
1170   if (!select(CopyInst))
1171     return false;
1172 
1173   I.eraseFromParent();
1174   return true;
1175 }
1176 
1177 bool X86InstructionSelector::selectCondBranch(MachineInstr &I,
1178                                               MachineRegisterInfo &MRI,
1179                                               MachineFunction &MF) const {
1180 
1181   assert((I.getOpcode() == TargetOpcode::G_BRCOND) && "unexpected instruction");
1182 
1183   const unsigned CondReg = I.getOperand(0).getReg();
1184   MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
1185 
1186   MachineInstr &TestInst =
1187       *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::TEST8ri))
1188            .addReg(CondReg)
1189            .addImm(1);
1190   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::JNE_1))
1191       .addMBB(DestMBB);
1192 
1193   constrainSelectedInstRegOperands(TestInst, TII, TRI, RBI);
1194 
1195   I.eraseFromParent();
1196   return true;
1197 }
1198 
1199 bool X86InstructionSelector::materializeFP(MachineInstr &I,
1200                                            MachineRegisterInfo &MRI,
1201                                            MachineFunction &MF) const {
1202 
1203   assert((I.getOpcode() == TargetOpcode::G_FCONSTANT) &&
1204          "unexpected instruction");
1205 
1206   // Can't handle alternate code models yet.
1207   CodeModel::Model CM = TM.getCodeModel();
1208   if (CM != CodeModel::Small && CM != CodeModel::Large)
1209     return false;
1210 
1211   const unsigned DstReg = I.getOperand(0).getReg();
1212   const LLT DstTy = MRI.getType(DstReg);
1213   const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
1214   unsigned Align = DstTy.getSizeInBits();
1215   const DebugLoc &DbgLoc = I.getDebugLoc();
1216 
1217   unsigned Opc = getLoadStoreOp(DstTy, RegBank, TargetOpcode::G_LOAD, Align);
1218 
1219   // Create the load from the constant pool.
1220   const ConstantFP *CFP = I.getOperand(1).getFPImm();
1221   unsigned CPI = MF.getConstantPool()->getConstantPoolIndex(CFP, Align);
1222   MachineInstr *LoadInst = nullptr;
1223   unsigned char OpFlag = STI.classifyLocalReference(nullptr);
1224 
1225   if (CM == CodeModel::Large && STI.is64Bit()) {
1226     // Under X86-64 non-small code model, GV (and friends) are 64-bits, so
1227     // they cannot be folded into immediate fields.
1228 
1229     unsigned AddrReg = MRI.createVirtualRegister(&X86::GR64RegClass);
1230     BuildMI(*I.getParent(), I, DbgLoc, TII.get(X86::MOV64ri), AddrReg)
1231         .addConstantPoolIndex(CPI, 0, OpFlag);
1232 
1233     MachineMemOperand *MMO = MF.getMachineMemOperand(
1234         MachinePointerInfo::getConstantPool(MF), MachineMemOperand::MOLoad,
1235         MF.getDataLayout().getPointerSize(), Align);
1236 
1237     LoadInst =
1238         addDirectMem(BuildMI(*I.getParent(), I, DbgLoc, TII.get(Opc), DstReg),
1239                      AddrReg)
1240             .addMemOperand(MMO);
1241 
1242   } else if (CM == CodeModel::Small || !STI.is64Bit()) {
1243     // Handle the case when globals fit in our immediate field.
1244     // This is true for X86-32 always and X86-64 when in -mcmodel=small mode.
1245 
1246     // x86-32 PIC requires a PIC base register for constant pools.
1247     unsigned PICBase = 0;
1248     if (OpFlag == X86II::MO_PIC_BASE_OFFSET || OpFlag == X86II::MO_GOTOFF) {
1249       // PICBase can be allocated by TII.getGlobalBaseReg(&MF).
1250       // In DAGISEL the code that initialize it generated by the CGBR pass.
1251       return false; // TODO support the mode.
1252     } else if (STI.is64Bit() && TM.getCodeModel() == CodeModel::Small)
1253       PICBase = X86::RIP;
1254 
1255     LoadInst = addConstantPoolReference(
1256         BuildMI(*I.getParent(), I, DbgLoc, TII.get(Opc), DstReg), CPI, PICBase,
1257         OpFlag);
1258   } else
1259     return false;
1260 
1261   constrainSelectedInstRegOperands(*LoadInst, TII, TRI, RBI);
1262   I.eraseFromParent();
1263   return true;
1264 }
1265 
1266 bool X86InstructionSelector::selectImplicitDefOrPHI(
1267     MachineInstr &I, MachineRegisterInfo &MRI) const {
1268 
1269   assert((I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF ||
1270           I.getOpcode() == TargetOpcode::G_PHI) &&
1271          "unexpected instruction");
1272 
1273   unsigned DstReg = I.getOperand(0).getReg();
1274 
1275   if (!MRI.getRegClassOrNull(DstReg)) {
1276     const LLT DstTy = MRI.getType(DstReg);
1277     const TargetRegisterClass *RC = getRegClass(DstTy, DstReg, MRI);
1278 
1279     if (!RBI.constrainGenericRegister(DstReg, *RC, MRI)) {
1280       DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1281                    << " operand\n");
1282       return false;
1283     }
1284   }
1285 
1286   if (I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
1287     I.setDesc(TII.get(X86::IMPLICIT_DEF));
1288   else
1289     I.setDesc(TII.get(X86::PHI));
1290 
1291   return true;
1292 }
1293 
1294 InstructionSelector *
1295 llvm::createX86InstructionSelector(const X86TargetMachine &TM,
1296                                    X86Subtarget &Subtarget,
1297                                    X86RegisterBankInfo &RBI) {
1298   return new X86InstructionSelector(TM, Subtarget, RBI);
1299 }
1300