1 //===- bolt/Target/X86/X86MCPlusBuilder.cpp -------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file provides X86-specific MCPlus builder.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "MCTargetDesc/X86BaseInfo.h"
14 #include "MCTargetDesc/X86InstrRelaxTables.h"
15 #include "MCTargetDesc/X86MCTargetDesc.h"
16 #include "bolt/Core/MCPlus.h"
17 #include "bolt/Core/MCPlusBuilder.h"
18 #include "llvm/BinaryFormat/ELF.h"
19 #include "llvm/MC/MCContext.h"
20 #include "llvm/MC/MCFixupKindInfo.h"
21 #include "llvm/MC/MCInst.h"
22 #include "llvm/MC/MCInstBuilder.h"
23 #include "llvm/MC/MCInstrInfo.h"
24 #include "llvm/MC/MCRegister.h"
25 #include "llvm/MC/MCRegisterInfo.h"
26 #include "llvm/Support/CommandLine.h"
27 #include "llvm/Support/DataExtractor.h"
28 #include "llvm/Support/Debug.h"
29 #include "llvm/Support/Errc.h"
30 #include "llvm/Support/ErrorHandling.h"
31 #include "llvm/Support/ErrorOr.h"
32 #include <set>
33 
34 #define DEBUG_TYPE "mcplus"
35 
36 using namespace llvm;
37 using namespace bolt;
38 
39 namespace opts {
40 
41 extern cl::OptionCategory BoltOptCategory;
42 
43 static cl::opt<bool> X86StripRedundantAddressSize(
44     "x86-strip-redundant-address-size",
45     cl::desc("Remove redundant Address-Size override prefix"), cl::init(true),
46     cl::ZeroOrMore, cl::cat(BoltOptCategory));
47 
48 } // namespace opts
49 
50 namespace {
51 
52 unsigned getShortBranchOpcode(unsigned Opcode) {
53   switch (Opcode) {
54   default:
55     return Opcode;
56   case X86::JMP_2: return X86::JMP_1;
57   case X86::JMP_4: return X86::JMP_1;
58   case X86::JCC_2: return X86::JCC_1;
59   case X86::JCC_4: return X86::JCC_1;
60   }
61 }
62 
63 unsigned getShortArithOpcode(unsigned Opcode) {
64   return X86::getShortOpcodeArith(Opcode);
65 }
66 
67 bool isMOVSX64rm32(const MCInst &Inst) {
68   return Inst.getOpcode() == X86::MOVSX64rm32;
69 }
70 
71 bool isADD64rr(const MCInst &Inst) { return Inst.getOpcode() == X86::ADD64rr; }
72 
73 bool isADDri(const MCInst &Inst) {
74   return Inst.getOpcode() == X86::ADD64ri32 ||
75          Inst.getOpcode() == X86::ADD64ri8;
76 }
77 
78 class X86MCPlusBuilder : public MCPlusBuilder {
79 public:
80   X86MCPlusBuilder(const MCInstrAnalysis *Analysis, const MCInstrInfo *Info,
81                    const MCRegisterInfo *RegInfo)
82       : MCPlusBuilder(Analysis, Info, RegInfo) {}
83 
84   bool isBranch(const MCInst &Inst) const override {
85     return Analysis->isBranch(Inst) && !isTailCall(Inst);
86   }
87 
88   bool isNoop(const MCInst &Inst) const override {
89     return X86::isNOP(Inst.getOpcode());
90   }
91 
92   unsigned getCondCode(const MCInst &Inst) const override {
93     unsigned Opcode = Inst.getOpcode();
94     if (X86::isJCC(Opcode))
95       return Inst.getOperand(Info->get(Opcode).NumOperands - 1).getImm();
96     return X86::COND_INVALID;
97   }
98 
99   unsigned getInvertedCondCode(unsigned CC) const override {
100     switch (CC) {
101     default: return X86::COND_INVALID;
102     case X86::COND_E:  return X86::COND_NE;
103     case X86::COND_NE: return X86::COND_E;
104     case X86::COND_L:  return X86::COND_GE;
105     case X86::COND_LE: return X86::COND_G;
106     case X86::COND_G:  return X86::COND_LE;
107     case X86::COND_GE: return X86::COND_L;
108     case X86::COND_B:  return X86::COND_AE;
109     case X86::COND_BE: return X86::COND_A;
110     case X86::COND_A:  return X86::COND_BE;
111     case X86::COND_AE: return X86::COND_B;
112     case X86::COND_S:  return X86::COND_NS;
113     case X86::COND_NS: return X86::COND_S;
114     case X86::COND_P:  return X86::COND_NP;
115     case X86::COND_NP: return X86::COND_P;
116     case X86::COND_O:  return X86::COND_NO;
117     case X86::COND_NO: return X86::COND_O;
118     }
119   }
120 
121   unsigned getCondCodesLogicalOr(unsigned CC1, unsigned CC2) const override {
122     enum DecodedCondCode : uint8_t {
123       DCC_EQUAL = 0x1,
124       DCC_GREATER = 0x2,
125       DCC_LESSER = 0x4,
126       DCC_GREATER_OR_LESSER = 0x6,
127       DCC_UNSIGNED = 0x8,
128       DCC_SIGNED = 0x10,
129       DCC_INVALID = 0x20,
130     };
131 
132     auto decodeCondCode = [&](unsigned CC) -> uint8_t {
133       switch (CC) {
134       default: return DCC_INVALID;
135       case X86::COND_E: return DCC_EQUAL;
136       case X86::COND_NE: return DCC_GREATER | DCC_LESSER;
137       case X86::COND_L: return DCC_LESSER | DCC_SIGNED;
138       case X86::COND_LE: return DCC_EQUAL | DCC_LESSER | DCC_SIGNED;
139       case X86::COND_G: return DCC_GREATER | DCC_SIGNED;
140       case X86::COND_GE: return DCC_GREATER | DCC_EQUAL | DCC_SIGNED;
141       case X86::COND_B: return DCC_LESSER | DCC_UNSIGNED;
142       case X86::COND_BE: return DCC_EQUAL | DCC_LESSER | DCC_UNSIGNED;
143       case X86::COND_A: return DCC_GREATER | DCC_UNSIGNED;
144       case X86::COND_AE: return DCC_GREATER | DCC_EQUAL | DCC_UNSIGNED;
145       }
146     };
147 
148     uint8_t DCC = decodeCondCode(CC1) | decodeCondCode(CC2);
149 
150     if (DCC & DCC_INVALID)
151       return X86::COND_INVALID;
152 
153     if (DCC & DCC_SIGNED && DCC & DCC_UNSIGNED)
154       return X86::COND_INVALID;
155 
156     switch (DCC) {
157     default: return X86::COND_INVALID;
158     case DCC_EQUAL | DCC_LESSER | DCC_SIGNED: return X86::COND_LE;
159     case DCC_EQUAL | DCC_LESSER | DCC_UNSIGNED: return X86::COND_BE;
160     case DCC_EQUAL | DCC_GREATER | DCC_SIGNED: return X86::COND_GE;
161     case DCC_EQUAL | DCC_GREATER | DCC_UNSIGNED: return X86::COND_AE;
162     case DCC_GREATER | DCC_LESSER | DCC_SIGNED: return X86::COND_NE;
163     case DCC_GREATER | DCC_LESSER | DCC_UNSIGNED: return X86::COND_NE;
164     case DCC_GREATER | DCC_LESSER: return X86::COND_NE;
165     case DCC_EQUAL | DCC_SIGNED: return X86::COND_E;
166     case DCC_EQUAL | DCC_UNSIGNED: return X86::COND_E;
167     case DCC_EQUAL: return X86::COND_E;
168     case DCC_LESSER | DCC_SIGNED: return X86::COND_L;
169     case DCC_LESSER | DCC_UNSIGNED: return X86::COND_B;
170     case DCC_GREATER | DCC_SIGNED: return X86::COND_G;
171     case DCC_GREATER | DCC_UNSIGNED: return X86::COND_A;
172     }
173   }
174 
175   bool isValidCondCode(unsigned CC) const override {
176     return (CC != X86::COND_INVALID);
177   }
178 
179   bool isBreakpoint(const MCInst &Inst) const override {
180     return Inst.getOpcode() == X86::INT3;
181   }
182 
183   bool isPrefix(const MCInst &Inst) const override {
184     const MCInstrDesc &Desc = Info->get(Inst.getOpcode());
185     return X86II::isPrefix(Desc.TSFlags);
186   }
187 
188   bool isRep(const MCInst &Inst) const override {
189     return Inst.getFlags() == X86::IP_HAS_REPEAT;
190   }
191 
192   bool deleteREPPrefix(MCInst &Inst) const override {
193     if (Inst.getFlags() == X86::IP_HAS_REPEAT) {
194       Inst.setFlags(0);
195       return true;
196     }
197     return false;
198   }
199 
200   // FIXME: For compatibility with old LLVM only!
201   bool isTerminator(const MCInst &Inst) const override {
202     unsigned Opcode = Inst.getOpcode();
203     return Info->get(Opcode).isTerminator() || X86::isUD1(Opcode) ||
204            X86::isUD2(Opcode);
205   }
206 
207   bool isIndirectCall(const MCInst &Inst) const override {
208     return isCall(Inst) &&
209            ((getMemoryOperandNo(Inst) != -1) || Inst.getOperand(0).isReg());
210   }
211 
212   bool isPop(const MCInst &Inst) const override {
213     return getPopSize(Inst) == 0 ? false : true;
214   }
215 
216   bool isTerminateBranch(const MCInst &Inst) const override {
217     return Inst.getOpcode() == X86::ENDBR32 || Inst.getOpcode() == X86::ENDBR64;
218   }
219 
220   int getPopSize(const MCInst &Inst) const override {
221     switch (Inst.getOpcode()) {
222     case X86::POP16r:
223     case X86::POP16rmm:
224     case X86::POP16rmr:
225     case X86::POPF16:
226     case X86::POPA16:
227     case X86::POPDS16:
228     case X86::POPES16:
229     case X86::POPFS16:
230     case X86::POPGS16:
231     case X86::POPSS16:
232       return 2;
233     case X86::POP32r:
234     case X86::POP32rmm:
235     case X86::POP32rmr:
236     case X86::POPA32:
237     case X86::POPDS32:
238     case X86::POPES32:
239     case X86::POPF32:
240     case X86::POPFS32:
241     case X86::POPGS32:
242     case X86::POPSS32:
243       return 4;
244     case X86::POP64r:
245     case X86::POP64rmm:
246     case X86::POP64rmr:
247     case X86::POPF64:
248     case X86::POPFS64:
249     case X86::POPGS64:
250       return 8;
251     }
252     return 0;
253   }
254 
255   bool isPush(const MCInst &Inst) const override {
256     return getPushSize(Inst) == 0 ? false : true;
257   }
258 
259   int getPushSize(const MCInst &Inst) const override {
260     switch (Inst.getOpcode()) {
261     case X86::PUSH16i8:
262     case X86::PUSH16r:
263     case X86::PUSH16rmm:
264     case X86::PUSH16rmr:
265     case X86::PUSHA16:
266     case X86::PUSHCS16:
267     case X86::PUSHDS16:
268     case X86::PUSHES16:
269     case X86::PUSHF16:
270     case X86::PUSHFS16:
271     case X86::PUSHGS16:
272     case X86::PUSHSS16:
273     case X86::PUSHi16:
274       return 2;
275     case X86::PUSH32i8:
276     case X86::PUSH32r:
277     case X86::PUSH32rmm:
278     case X86::PUSH32rmr:
279     case X86::PUSHA32:
280     case X86::PUSHCS32:
281     case X86::PUSHDS32:
282     case X86::PUSHES32:
283     case X86::PUSHF32:
284     case X86::PUSHFS32:
285     case X86::PUSHGS32:
286     case X86::PUSHSS32:
287     case X86::PUSHi32:
288       return 4;
289     case X86::PUSH64i32:
290     case X86::PUSH64i8:
291     case X86::PUSH64r:
292     case X86::PUSH64rmm:
293     case X86::PUSH64rmr:
294     case X86::PUSHF64:
295     case X86::PUSHFS64:
296     case X86::PUSHGS64:
297       return 8;
298     }
299     return 0;
300   }
301 
302   bool isSUB(const MCInst &Inst) const override {
303     return X86::isSUB(Inst.getOpcode());
304   }
305 
306   bool isLEA64r(const MCInst &Inst) const override {
307     return Inst.getOpcode() == X86::LEA64r;
308   }
309 
310   bool isLeave(const MCInst &Inst) const override {
311     return Inst.getOpcode() == X86::LEAVE || Inst.getOpcode() == X86::LEAVE64;
312   }
313 
314   bool isMoveMem2Reg(const MCInst &Inst) const override {
315     switch (Inst.getOpcode()) {
316     case X86::MOV16rm:
317     case X86::MOV32rm:
318     case X86::MOV64rm:
319       return true;
320     }
321     return false;
322   }
323 
324   bool isUnsupportedBranch(unsigned Opcode) const override {
325     switch (Opcode) {
326     default:
327       return false;
328     case X86::LOOP:
329     case X86::LOOPE:
330     case X86::LOOPNE:
331     case X86::JECXZ:
332     case X86::JRCXZ:
333       return true;
334     }
335   }
336 
337   bool isLoad(const MCInst &Inst) const override {
338     if (isPop(Inst))
339       return true;
340 
341     int MemOpNo = getMemoryOperandNo(Inst);
342     const MCInstrDesc &MCII = Info->get(Inst.getOpcode());
343 
344     if (MemOpNo == -1)
345       return false;
346 
347     return MCII.mayLoad();
348   }
349 
350   bool isStore(const MCInst &Inst) const override {
351     if (isPush(Inst))
352       return true;
353 
354     int MemOpNo = getMemoryOperandNo(Inst);
355     const MCInstrDesc &MCII = Info->get(Inst.getOpcode());
356 
357     if (MemOpNo == -1)
358       return false;
359 
360     return MCII.mayStore();
361   }
362 
363   bool isCleanRegXOR(const MCInst &Inst) const override {
364     switch (Inst.getOpcode()) {
365     case X86::XOR16rr:
366     case X86::XOR32rr:
367     case X86::XOR64rr:
368       break;
369     default:
370       return false;
371     }
372     return (Inst.getOperand(0).getReg() == Inst.getOperand(2).getReg());
373   }
374 
375   bool isPacked(const MCInst &Inst) const override {
376     const MCInstrDesc &Desc = Info->get(Inst.getOpcode());
377     return (Desc.TSFlags & X86II::OpPrefixMask) == X86II::PD;
378   }
379 
380   unsigned getTrapFillValue() const override { return 0xCC; }
381 
382   struct IndJmpMatcherFrag1 : MCInstMatcher {
383     std::unique_ptr<MCInstMatcher> Base;
384     std::unique_ptr<MCInstMatcher> Scale;
385     std::unique_ptr<MCInstMatcher> Index;
386     std::unique_ptr<MCInstMatcher> Offset;
387 
388     IndJmpMatcherFrag1(std::unique_ptr<MCInstMatcher> Base,
389                        std::unique_ptr<MCInstMatcher> Scale,
390                        std::unique_ptr<MCInstMatcher> Index,
391                        std::unique_ptr<MCInstMatcher> Offset)
392         : Base(std::move(Base)), Scale(std::move(Scale)),
393           Index(std::move(Index)), Offset(std::move(Offset)) {}
394 
395     bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
396                MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
397       if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
398         return false;
399 
400       if (CurInst->getOpcode() != X86::JMP64m)
401         return false;
402 
403       int MemOpNo = MIB.getMemoryOperandNo(*CurInst);
404       if (MemOpNo == -1)
405         return false;
406 
407       if (!Base->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrBaseReg))
408         return false;
409       if (!Scale->match(MRI, MIB, this->InstrWindow,
410                         MemOpNo + X86::AddrScaleAmt))
411         return false;
412       if (!Index->match(MRI, MIB, this->InstrWindow,
413                         MemOpNo + X86::AddrIndexReg))
414         return false;
415       if (!Offset->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrDisp))
416         return false;
417       return true;
418     }
419 
420     void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
421       MIB.addAnnotation(*CurInst, Annotation, true);
422       Base->annotate(MIB, Annotation);
423       Scale->annotate(MIB, Annotation);
424       Index->annotate(MIB, Annotation);
425       Offset->annotate(MIB, Annotation);
426     }
427   };
428 
429   std::unique_ptr<MCInstMatcher>
430   matchIndJmp(std::unique_ptr<MCInstMatcher> Base,
431               std::unique_ptr<MCInstMatcher> Scale,
432               std::unique_ptr<MCInstMatcher> Index,
433               std::unique_ptr<MCInstMatcher> Offset) const override {
434     return std::unique_ptr<MCInstMatcher>(
435         new IndJmpMatcherFrag1(std::move(Base), std::move(Scale),
436                                std::move(Index), std::move(Offset)));
437   }
438 
439   struct IndJmpMatcherFrag2 : MCInstMatcher {
440     std::unique_ptr<MCInstMatcher> Reg;
441 
442     IndJmpMatcherFrag2(std::unique_ptr<MCInstMatcher> Reg)
443         : Reg(std::move(Reg)) {}
444 
445     bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
446                MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
447       if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
448         return false;
449 
450       if (CurInst->getOpcode() != X86::JMP64r)
451         return false;
452 
453       return Reg->match(MRI, MIB, this->InstrWindow, 0);
454     }
455 
456     void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
457       MIB.addAnnotation(*CurInst, Annotation, true);
458       Reg->annotate(MIB, Annotation);
459     }
460   };
461 
462   std::unique_ptr<MCInstMatcher>
463   matchIndJmp(std::unique_ptr<MCInstMatcher> Target) const override {
464     return std::unique_ptr<MCInstMatcher>(
465         new IndJmpMatcherFrag2(std::move(Target)));
466   }
467 
468   struct LoadMatcherFrag1 : MCInstMatcher {
469     std::unique_ptr<MCInstMatcher> Base;
470     std::unique_ptr<MCInstMatcher> Scale;
471     std::unique_ptr<MCInstMatcher> Index;
472     std::unique_ptr<MCInstMatcher> Offset;
473 
474     LoadMatcherFrag1(std::unique_ptr<MCInstMatcher> Base,
475                      std::unique_ptr<MCInstMatcher> Scale,
476                      std::unique_ptr<MCInstMatcher> Index,
477                      std::unique_ptr<MCInstMatcher> Offset)
478         : Base(std::move(Base)), Scale(std::move(Scale)),
479           Index(std::move(Index)), Offset(std::move(Offset)) {}
480 
481     bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
482                MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
483       if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
484         return false;
485 
486       if (CurInst->getOpcode() != X86::MOV64rm &&
487           CurInst->getOpcode() != X86::MOVSX64rm32)
488         return false;
489 
490       int MemOpNo = MIB.getMemoryOperandNo(*CurInst);
491       if (MemOpNo == -1)
492         return false;
493 
494       if (!Base->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrBaseReg))
495         return false;
496       if (!Scale->match(MRI, MIB, this->InstrWindow,
497                         MemOpNo + X86::AddrScaleAmt))
498         return false;
499       if (!Index->match(MRI, MIB, this->InstrWindow,
500                         MemOpNo + X86::AddrIndexReg))
501         return false;
502       if (!Offset->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrDisp))
503         return false;
504       return true;
505     }
506 
507     void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
508       MIB.addAnnotation(*CurInst, Annotation, true);
509       Base->annotate(MIB, Annotation);
510       Scale->annotate(MIB, Annotation);
511       Index->annotate(MIB, Annotation);
512       Offset->annotate(MIB, Annotation);
513     }
514   };
515 
516   std::unique_ptr<MCInstMatcher>
517   matchLoad(std::unique_ptr<MCInstMatcher> Base,
518             std::unique_ptr<MCInstMatcher> Scale,
519             std::unique_ptr<MCInstMatcher> Index,
520             std::unique_ptr<MCInstMatcher> Offset) const override {
521     return std::unique_ptr<MCInstMatcher>(
522         new LoadMatcherFrag1(std::move(Base), std::move(Scale),
523                              std::move(Index), std::move(Offset)));
524   }
525 
526   struct AddMatcher : MCInstMatcher {
527     std::unique_ptr<MCInstMatcher> A;
528     std::unique_ptr<MCInstMatcher> B;
529 
530     AddMatcher(std::unique_ptr<MCInstMatcher> A,
531                std::unique_ptr<MCInstMatcher> B)
532         : A(std::move(A)), B(std::move(B)) {}
533 
534     bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
535                MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
536       if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
537         return false;
538 
539       if (CurInst->getOpcode() == X86::ADD64rr ||
540           CurInst->getOpcode() == X86::ADD64rr_DB ||
541           CurInst->getOpcode() == X86::ADD64rr_REV) {
542         if (!A->match(MRI, MIB, this->InstrWindow, 1)) {
543           if (!B->match(MRI, MIB, this->InstrWindow, 1))
544             return false;
545           return A->match(MRI, MIB, this->InstrWindow, 2);
546         }
547 
548         if (B->match(MRI, MIB, this->InstrWindow, 2))
549           return true;
550 
551         if (!B->match(MRI, MIB, this->InstrWindow, 1))
552           return false;
553         return A->match(MRI, MIB, this->InstrWindow, 2);
554       }
555 
556       return false;
557     }
558 
559     void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
560       MIB.addAnnotation(*CurInst, Annotation, true);
561       A->annotate(MIB, Annotation);
562       B->annotate(MIB, Annotation);
563     }
564   };
565 
566   virtual std::unique_ptr<MCInstMatcher>
567   matchAdd(std::unique_ptr<MCInstMatcher> A,
568            std::unique_ptr<MCInstMatcher> B) const override {
569     return std::unique_ptr<MCInstMatcher>(
570         new AddMatcher(std::move(A), std::move(B)));
571   }
572 
573   struct LEAMatcher : MCInstMatcher {
574     std::unique_ptr<MCInstMatcher> Target;
575 
576     LEAMatcher(std::unique_ptr<MCInstMatcher> Target)
577         : Target(std::move(Target)) {}
578 
579     bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
580                MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
581       if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
582         return false;
583 
584       if (CurInst->getOpcode() != X86::LEA64r)
585         return false;
586 
587       if (CurInst->getOperand(1 + X86::AddrScaleAmt).getImm() != 1 ||
588           CurInst->getOperand(1 + X86::AddrIndexReg).getReg() !=
589               X86::NoRegister ||
590           (CurInst->getOperand(1 + X86::AddrBaseReg).getReg() !=
591                X86::NoRegister &&
592            CurInst->getOperand(1 + X86::AddrBaseReg).getReg() != X86::RIP))
593         return false;
594 
595       return Target->match(MRI, MIB, this->InstrWindow, 1 + X86::AddrDisp);
596     }
597 
598     void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
599       MIB.addAnnotation(*CurInst, Annotation, true);
600       Target->annotate(MIB, Annotation);
601     }
602   };
603 
604   virtual std::unique_ptr<MCInstMatcher>
605   matchLoadAddr(std::unique_ptr<MCInstMatcher> Target) const override {
606     return std::unique_ptr<MCInstMatcher>(new LEAMatcher(std::move(Target)));
607   }
608 
609   bool hasPCRelOperand(const MCInst &Inst) const override {
610     for (const MCOperand &Operand : Inst)
611       if (Operand.isReg() && Operand.getReg() == X86::RIP)
612         return true;
613     return false;
614   }
615 
616   int getMemoryOperandNo(const MCInst &Inst) const override {
617     unsigned Opcode = Inst.getOpcode();
618     const MCInstrDesc &Desc = Info->get(Opcode);
619     int MemOpNo = X86II::getMemoryOperandNo(Desc.TSFlags);
620     if (MemOpNo >= 0)
621       MemOpNo += X86II::getOperandBias(Desc);
622     return MemOpNo;
623   }
624 
625   bool hasEVEXEncoding(const MCInst &Inst) const override {
626     const MCInstrDesc &Desc = Info->get(Inst.getOpcode());
627     return (Desc.TSFlags & X86II::EncodingMask) == X86II::EVEX;
628   }
629 
630   bool isMacroOpFusionPair(ArrayRef<MCInst> Insts) const override {
631     const auto *I = Insts.begin();
632     while (I != Insts.end() && isPrefix(*I))
633       ++I;
634     if (I == Insts.end())
635       return false;
636 
637     const MCInst &FirstInst = *I;
638     ++I;
639     while (I != Insts.end() && isPrefix(*I))
640       ++I;
641     if (I == Insts.end())
642       return false;
643     const MCInst &SecondInst = *I;
644 
645     if (!isConditionalBranch(SecondInst))
646       return false;
647     // Cannot fuse if the first instruction uses RIP-relative memory.
648     if (hasPCRelOperand(FirstInst))
649       return false;
650 
651     const X86::FirstMacroFusionInstKind CmpKind =
652         X86::classifyFirstOpcodeInMacroFusion(FirstInst.getOpcode());
653     if (CmpKind == X86::FirstMacroFusionInstKind::Invalid)
654       return false;
655 
656     X86::CondCode CC = static_cast<X86::CondCode>(getCondCode(SecondInst));
657     X86::SecondMacroFusionInstKind BranchKind =
658         X86::classifySecondCondCodeInMacroFusion(CC);
659     if (BranchKind == X86::SecondMacroFusionInstKind::Invalid)
660       return false;
661     return X86::isMacroFused(CmpKind, BranchKind);
662   }
663 
664   bool
665   evaluateX86MemoryOperand(const MCInst &Inst, unsigned *BaseRegNum,
666                            int64_t *ScaleImm, unsigned *IndexRegNum,
667                            int64_t *DispImm, unsigned *SegmentRegNum,
668                            const MCExpr **DispExpr = nullptr) const override {
669     assert(BaseRegNum && ScaleImm && IndexRegNum && SegmentRegNum &&
670            "one of the input pointers is null");
671     int MemOpNo = getMemoryOperandNo(Inst);
672     if (MemOpNo < 0)
673       return false;
674     unsigned MemOpOffset = static_cast<unsigned>(MemOpNo);
675 
676     if (MemOpOffset + X86::AddrSegmentReg >= MCPlus::getNumPrimeOperands(Inst))
677       return false;
678 
679     const MCOperand &Base = Inst.getOperand(MemOpOffset + X86::AddrBaseReg);
680     const MCOperand &Scale = Inst.getOperand(MemOpOffset + X86::AddrScaleAmt);
681     const MCOperand &Index = Inst.getOperand(MemOpOffset + X86::AddrIndexReg);
682     const MCOperand &Disp = Inst.getOperand(MemOpOffset + X86::AddrDisp);
683     const MCOperand &Segment =
684         Inst.getOperand(MemOpOffset + X86::AddrSegmentReg);
685 
686     // Make sure it is a well-formed memory operand.
687     if (!Base.isReg() || !Scale.isImm() || !Index.isReg() ||
688         (!Disp.isImm() && !Disp.isExpr()) || !Segment.isReg())
689       return false;
690 
691     *BaseRegNum = Base.getReg();
692     *ScaleImm = Scale.getImm();
693     *IndexRegNum = Index.getReg();
694     if (Disp.isImm()) {
695       assert(DispImm && "DispImm needs to be set");
696       *DispImm = Disp.getImm();
697       if (DispExpr)
698         *DispExpr = nullptr;
699     } else {
700       assert(DispExpr && "DispExpr needs to be set");
701       *DispExpr = Disp.getExpr();
702       if (DispImm)
703         *DispImm = 0;
704     }
705     *SegmentRegNum = Segment.getReg();
706     return true;
707   }
708 
709   bool evaluateMemOperandTarget(const MCInst &Inst, uint64_t &Target,
710                                 uint64_t Address,
711                                 uint64_t Size) const override {
712     unsigned      BaseRegNum;
713     int64_t       ScaleValue;
714     unsigned      IndexRegNum;
715     int64_t       DispValue;
716     unsigned      SegRegNum;
717     const MCExpr *DispExpr = nullptr;
718     if (!evaluateX86MemoryOperand(Inst, &BaseRegNum, &ScaleValue, &IndexRegNum,
719                                   &DispValue, &SegRegNum, &DispExpr))
720       return false;
721 
722     // Make sure it's a well-formed addressing we can statically evaluate.
723     if ((BaseRegNum != X86::RIP && BaseRegNum != X86::NoRegister) ||
724         IndexRegNum != X86::NoRegister || SegRegNum != X86::NoRegister ||
725         DispExpr)
726       return false;
727 
728     Target = DispValue;
729     if (BaseRegNum == X86::RIP) {
730       assert(Size != 0 && "instruction size required in order to statically "
731                           "evaluate RIP-relative address");
732       Target += Address + Size;
733     }
734     return true;
735   }
736 
737   MCInst::iterator getMemOperandDisp(MCInst &Inst) const override {
738     int MemOpNo = getMemoryOperandNo(Inst);
739     if (MemOpNo < 0)
740       return Inst.end();
741     return Inst.begin() + (MemOpNo + X86::AddrDisp);
742   }
743 
744   bool replaceMemOperandDisp(MCInst &Inst, MCOperand Operand) const override {
745     MCOperand *OI = getMemOperandDisp(Inst);
746     if (OI == Inst.end())
747       return false;
748     *OI = Operand;
749     return true;
750   }
751 
752   /// Get the registers used as function parameters.
753   /// This function is specific to the x86_64 abi on Linux.
754   BitVector getRegsUsedAsParams() const override {
755     BitVector Regs = BitVector(RegInfo->getNumRegs(), false);
756     Regs |= getAliases(X86::RSI);
757     Regs |= getAliases(X86::RDI);
758     Regs |= getAliases(X86::RDX);
759     Regs |= getAliases(X86::RCX);
760     Regs |= getAliases(X86::R8);
761     Regs |= getAliases(X86::R9);
762     return Regs;
763   }
764 
765   void getCalleeSavedRegs(BitVector &Regs) const override {
766     Regs |= getAliases(X86::RBX);
767     Regs |= getAliases(X86::RBP);
768     Regs |= getAliases(X86::R12);
769     Regs |= getAliases(X86::R13);
770     Regs |= getAliases(X86::R14);
771     Regs |= getAliases(X86::R15);
772   }
773 
774   void getDefaultDefIn(BitVector &Regs) const override {
775     assert(Regs.size() >= RegInfo->getNumRegs() &&
776            "The size of BitVector is less than RegInfo->getNumRegs().");
777     Regs.set(X86::RAX);
778     Regs.set(X86::RCX);
779     Regs.set(X86::RDX);
780     Regs.set(X86::RSI);
781     Regs.set(X86::RDI);
782     Regs.set(X86::R8);
783     Regs.set(X86::R9);
784     Regs.set(X86::XMM0);
785     Regs.set(X86::XMM1);
786     Regs.set(X86::XMM2);
787     Regs.set(X86::XMM3);
788     Regs.set(X86::XMM4);
789     Regs.set(X86::XMM5);
790     Regs.set(X86::XMM6);
791     Regs.set(X86::XMM7);
792   }
793 
794   void getDefaultLiveOut(BitVector &Regs) const override {
795     assert(Regs.size() >= RegInfo->getNumRegs() &&
796            "The size of BitVector is less than RegInfo->getNumRegs().");
797     Regs |= getAliases(X86::RAX);
798     Regs |= getAliases(X86::RDX);
799     Regs |= getAliases(X86::RCX);
800     Regs |= getAliases(X86::XMM0);
801     Regs |= getAliases(X86::XMM1);
802   }
803 
804   void getGPRegs(BitVector &Regs, bool IncludeAlias) const override {
805     if (IncludeAlias) {
806       Regs |= getAliases(X86::RAX);
807       Regs |= getAliases(X86::RBX);
808       Regs |= getAliases(X86::RBP);
809       Regs |= getAliases(X86::RSI);
810       Regs |= getAliases(X86::RDI);
811       Regs |= getAliases(X86::RDX);
812       Regs |= getAliases(X86::RCX);
813       Regs |= getAliases(X86::R8);
814       Regs |= getAliases(X86::R9);
815       Regs |= getAliases(X86::R10);
816       Regs |= getAliases(X86::R11);
817       Regs |= getAliases(X86::R12);
818       Regs |= getAliases(X86::R13);
819       Regs |= getAliases(X86::R14);
820       Regs |= getAliases(X86::R15);
821       return;
822     }
823     Regs.set(X86::RAX);
824     Regs.set(X86::RBX);
825     Regs.set(X86::RBP);
826     Regs.set(X86::RSI);
827     Regs.set(X86::RDI);
828     Regs.set(X86::RDX);
829     Regs.set(X86::RCX);
830     Regs.set(X86::R8);
831     Regs.set(X86::R9);
832     Regs.set(X86::R10);
833     Regs.set(X86::R11);
834     Regs.set(X86::R12);
835     Regs.set(X86::R13);
836     Regs.set(X86::R14);
837     Regs.set(X86::R15);
838   }
839 
840   void getClassicGPRegs(BitVector &Regs) const override {
841     Regs |= getAliases(X86::RAX);
842     Regs |= getAliases(X86::RBX);
843     Regs |= getAliases(X86::RBP);
844     Regs |= getAliases(X86::RSI);
845     Regs |= getAliases(X86::RDI);
846     Regs |= getAliases(X86::RDX);
847     Regs |= getAliases(X86::RCX);
848   }
849 
850   void getRepRegs(BitVector &Regs) const override {
851     Regs |= getAliases(X86::RCX);
852   }
853 
854   MCPhysReg getAliasSized(MCPhysReg Reg, uint8_t Size) const override {
855     switch (Reg) {
856     case X86::RAX: case X86::EAX: case X86::AX: case X86::AL: case X86::AH:
857       switch (Size) {
858       case 8: return X86::RAX;       case 4: return X86::EAX;
859       case 2: return X86::AX;        case 1: return X86::AL;
860       default: llvm_unreachable("Unexpected size");
861       }
862     case X86::RBX: case X86::EBX: case X86::BX: case X86::BL: case X86::BH:
863       switch (Size) {
864       case 8: return X86::RBX;       case 4: return X86::EBX;
865       case 2: return X86::BX;        case 1: return X86::BL;
866       default: llvm_unreachable("Unexpected size");
867       }
868     case X86::RDX: case X86::EDX: case X86::DX: case X86::DL: case X86::DH:
869       switch (Size) {
870       case 8: return X86::RDX;       case 4: return X86::EDX;
871       case 2: return X86::DX;        case 1: return X86::DL;
872       default: llvm_unreachable("Unexpected size");
873       }
874     case X86::RDI: case X86::EDI: case X86::DI: case X86::DIL:
875       switch (Size) {
876       case 8: return X86::RDI;       case 4: return X86::EDI;
877       case 2: return X86::DI;        case 1: return X86::DIL;
878       default: llvm_unreachable("Unexpected size");
879       }
880     case X86::RSI: case X86::ESI: case X86::SI: case X86::SIL:
881       switch (Size) {
882       case 8: return X86::RSI;       case 4: return X86::ESI;
883       case 2: return X86::SI;        case 1: return X86::SIL;
884       default: llvm_unreachable("Unexpected size");
885       }
886     case X86::RCX: case X86::ECX: case X86::CX: case X86::CL: case X86::CH:
887       switch (Size) {
888       case 8: return X86::RCX;       case 4: return X86::ECX;
889       case 2: return X86::CX;        case 1: return X86::CL;
890       default: llvm_unreachable("Unexpected size");
891       }
892     case X86::RSP: case X86::ESP: case X86::SP: case X86::SPL:
893       switch (Size) {
894       case 8: return X86::RSP;       case 4: return X86::ESP;
895       case 2: return X86::SP;        case 1: return X86::SPL;
896       default: llvm_unreachable("Unexpected size");
897       }
898     case X86::RBP: case X86::EBP: case X86::BP: case X86::BPL:
899       switch (Size) {
900       case 8: return X86::RBP;       case 4: return X86::EBP;
901       case 2: return X86::BP;        case 1: return X86::BPL;
902       default: llvm_unreachable("Unexpected size");
903       }
904   case X86::R8: case X86::R8D: case X86::R8W: case X86::R8B:
905       switch (Size) {
906       case 8: return X86::R8;        case 4: return X86::R8D;
907       case 2: return X86::R8W;       case 1: return X86::R8B;
908       default: llvm_unreachable("Unexpected size");
909       }
910     case X86::R9: case X86::R9D: case X86::R9W: case X86::R9B:
911       switch (Size) {
912       case 8: return X86::R9;        case 4: return X86::R9D;
913       case 2: return X86::R9W;       case 1: return X86::R9B;
914       default: llvm_unreachable("Unexpected size");
915       }
916     case X86::R10: case X86::R10D: case X86::R10W: case X86::R10B:
917       switch (Size) {
918       case 8: return X86::R10;        case 4: return X86::R10D;
919       case 2: return X86::R10W;       case 1: return X86::R10B;
920       default: llvm_unreachable("Unexpected size");
921       }
922     case X86::R11: case X86::R11D: case X86::R11W: case X86::R11B:
923       switch (Size) {
924       case 8: return X86::R11;        case 4: return X86::R11D;
925       case 2: return X86::R11W;       case 1: return X86::R11B;
926       default: llvm_unreachable("Unexpected size");
927       }
928     case X86::R12: case X86::R12D: case X86::R12W: case X86::R12B:
929       switch (Size) {
930       case 8: return X86::R12;        case 4: return X86::R12D;
931       case 2: return X86::R12W;       case 1: return X86::R12B;
932       default: llvm_unreachable("Unexpected size");
933       }
934     case X86::R13: case X86::R13D: case X86::R13W: case X86::R13B:
935       switch (Size) {
936       case 8: return X86::R13;        case 4: return X86::R13D;
937       case 2: return X86::R13W;       case 1: return X86::R13B;
938       default: llvm_unreachable("Unexpected size");
939       }
940     case X86::R14: case X86::R14D: case X86::R14W: case X86::R14B:
941       switch (Size) {
942       case 8: return X86::R14;        case 4: return X86::R14D;
943       case 2: return X86::R14W;       case 1: return X86::R14B;
944       default: llvm_unreachable("Unexpected size");
945       }
946     case X86::R15: case X86::R15D: case X86::R15W: case X86::R15B:
947       switch (Size) {
948       case 8: return X86::R15;        case 4: return X86::R15D;
949       case 2: return X86::R15W;       case 1: return X86::R15B;
950       default: llvm_unreachable("Unexpected size");
951       }
952     default:
953       dbgs() << Reg << " (get alias sized)\n";
954       llvm_unreachable("Unexpected reg number");
955       break;
956     }
957   }
958 
959   bool isUpper8BitReg(MCPhysReg Reg) const override {
960     switch (Reg) {
961     case X86::AH:
962     case X86::BH:
963     case X86::CH:
964     case X86::DH:
965       return true;
966     default:
967       return false;
968     }
969   }
970 
971   bool cannotUseREX(const MCInst &Inst) const override {
972     switch (Inst.getOpcode()) {
973     case X86::MOV8mr_NOREX:
974     case X86::MOV8rm_NOREX:
975     case X86::MOV8rr_NOREX:
976     case X86::MOVSX32rm8_NOREX:
977     case X86::MOVSX32rr8_NOREX:
978     case X86::MOVZX32rm8_NOREX:
979     case X86::MOVZX32rr8_NOREX:
980     case X86::MOV8mr:
981     case X86::MOV8rm:
982     case X86::MOV8rr:
983     case X86::MOVSX32rm8:
984     case X86::MOVSX32rr8:
985     case X86::MOVZX32rm8:
986     case X86::MOVZX32rr8:
987     case X86::TEST8ri:
988       for (const MCOperand &Operand : MCPlus::primeOperands(Inst)) {
989         if (!Operand.isReg())
990           continue;
991         if (isUpper8BitReg(Operand.getReg()))
992           return true;
993       }
994       LLVM_FALLTHROUGH;
995     default:
996       return false;
997     }
998   }
999 
1000   bool isStackAccess(const MCInst &Inst, bool &IsLoad, bool &IsStore,
1001                      bool &IsStoreFromReg, MCPhysReg &Reg, int32_t &SrcImm,
1002                      uint16_t &StackPtrReg, int64_t &StackOffset, uint8_t &Size,
1003                      bool &IsSimple, bool &IsIndexed) const override {
1004     // Detect simple push/pop cases first
1005     if (int Sz = getPushSize(Inst)) {
1006       IsLoad = false;
1007       IsStore = true;
1008       IsStoreFromReg = true;
1009       StackPtrReg = X86::RSP;
1010       StackOffset = -Sz;
1011       Size = Sz;
1012       IsSimple = true;
1013       if (Inst.getOperand(0).isImm())
1014         SrcImm = Inst.getOperand(0).getImm();
1015       else if (Inst.getOperand(0).isReg())
1016         Reg = Inst.getOperand(0).getReg();
1017       else
1018         IsSimple = false;
1019 
1020       return true;
1021     }
1022     if (int Sz = getPopSize(Inst)) {
1023       IsLoad = true;
1024       IsStore = false;
1025       if (Inst.getNumOperands() == 0 || !Inst.getOperand(0).isReg()) {
1026         IsSimple = false;
1027       } else {
1028         Reg = Inst.getOperand(0).getReg();
1029         IsSimple = true;
1030       }
1031       StackPtrReg = X86::RSP;
1032       StackOffset = 0;
1033       Size = Sz;
1034       return true;
1035     }
1036 
1037     struct InstInfo {
1038       // Size in bytes that Inst loads from memory.
1039       uint8_t DataSize;
1040       bool IsLoad;
1041       bool IsStore;
1042       bool StoreFromReg;
1043       bool Simple;
1044     };
1045 
1046     InstInfo I;
1047     int MemOpNo = getMemoryOperandNo(Inst);
1048     const MCInstrDesc &MCII = Info->get(Inst.getOpcode());
1049     // If it is not dealing with a memory operand, we discard it
1050     if (MemOpNo == -1 || MCII.isCall())
1051       return false;
1052 
1053     switch (Inst.getOpcode()) {
1054     default: {
1055       uint8_t Sz = 0;
1056       bool IsLoad = MCII.mayLoad();
1057       bool IsStore = MCII.mayStore();
1058       // Is it LEA? (deals with memory but is not loading nor storing)
1059       if (!IsLoad && !IsStore)
1060         return false;
1061 
1062       // Try to guess data size involved in the load/store by looking at the
1063       // register size. If there's no reg involved, return 0 as size, meaning
1064       // we don't know.
1065       for (unsigned I = 0, E = MCII.getNumOperands(); I != E; ++I) {
1066         if (MCII.OpInfo[I].OperandType != MCOI::OPERAND_REGISTER)
1067           continue;
1068         if (static_cast<int>(I) >= MemOpNo && I < X86::AddrNumOperands)
1069           continue;
1070         Sz = RegInfo->getRegClass(MCII.OpInfo[I].RegClass).getSizeInBits() / 8;
1071         break;
1072       }
1073       I = {Sz, IsLoad, IsStore, false, false};
1074       break;
1075     }
1076     case X86::MOV16rm: I = {2, true, false, false, true}; break;
1077     case X86::MOV32rm: I = {4, true, false, false, true}; break;
1078     case X86::MOV64rm: I = {8, true, false, false, true}; break;
1079     case X86::MOV16mr: I = {2, false, true, true, true};  break;
1080     case X86::MOV32mr: I = {4, false, true, true, true};  break;
1081     case X86::MOV64mr: I = {8, false, true, true, true};  break;
1082     case X86::MOV16mi: I = {2, false, true, false, true}; break;
1083     case X86::MOV32mi: I = {4, false, true, false, true}; break;
1084     } // end switch (Inst.getOpcode())
1085 
1086     unsigned BaseRegNum;
1087     int64_t ScaleValue;
1088     unsigned IndexRegNum;
1089     int64_t DispValue;
1090     unsigned SegRegNum;
1091     const MCExpr *DispExpr;
1092     if (!evaluateX86MemoryOperand(Inst, &BaseRegNum, &ScaleValue, &IndexRegNum,
1093                                   &DispValue, &SegRegNum, &DispExpr)) {
1094       LLVM_DEBUG(dbgs() << "Evaluate failed on ");
1095       LLVM_DEBUG(Inst.dump());
1096       return false;
1097     }
1098 
1099     // Make sure it's a stack access
1100     if (BaseRegNum != X86::RBP && BaseRegNum != X86::RSP)
1101       return false;
1102 
1103     IsLoad = I.IsLoad;
1104     IsStore = I.IsStore;
1105     IsStoreFromReg = I.StoreFromReg;
1106     Size = I.DataSize;
1107     IsSimple = I.Simple;
1108     StackPtrReg = BaseRegNum;
1109     StackOffset = DispValue;
1110     IsIndexed = IndexRegNum != X86::NoRegister || SegRegNum != X86::NoRegister;
1111 
1112     if (!I.Simple)
1113       return true;
1114 
1115     // Retrieve related register in simple MOV from/to stack operations.
1116     unsigned MemOpOffset = static_cast<unsigned>(MemOpNo);
1117     if (I.IsLoad) {
1118       MCOperand RegOpnd = Inst.getOperand(0);
1119       assert(RegOpnd.isReg() && "unexpected destination operand");
1120       Reg = RegOpnd.getReg();
1121     } else if (I.IsStore) {
1122       MCOperand SrcOpnd =
1123           Inst.getOperand(MemOpOffset + X86::AddrSegmentReg + 1);
1124       if (I.StoreFromReg) {
1125         assert(SrcOpnd.isReg() && "unexpected source operand");
1126         Reg = SrcOpnd.getReg();
1127       } else {
1128         assert(SrcOpnd.isImm() && "unexpected source operand");
1129         SrcImm = SrcOpnd.getImm();
1130       }
1131     }
1132 
1133     return true;
1134   }
1135 
1136   void changeToPushOrPop(MCInst &Inst) const override {
1137     assert(!isPush(Inst) && !isPop(Inst));
1138 
1139     struct InstInfo {
1140       // Size in bytes that Inst loads from memory.
1141       uint8_t DataSize;
1142       bool IsLoad;
1143       bool StoreFromReg;
1144     };
1145 
1146     InstInfo I;
1147     switch (Inst.getOpcode()) {
1148     default: {
1149       llvm_unreachable("Unhandled opcode");
1150       return;
1151     }
1152     case X86::MOV16rm: I = {2, true, false}; break;
1153     case X86::MOV32rm: I = {4, true, false}; break;
1154     case X86::MOV64rm: I = {8, true, false}; break;
1155     case X86::MOV16mr: I = {2, false, true};  break;
1156     case X86::MOV32mr: I = {4, false, true};  break;
1157     case X86::MOV64mr: I = {8, false, true};  break;
1158     case X86::MOV16mi: I = {2, false, false}; break;
1159     case X86::MOV32mi: I = {4, false, false}; break;
1160     } // end switch (Inst.getOpcode())
1161 
1162     unsigned BaseRegNum;
1163     int64_t ScaleValue;
1164     unsigned IndexRegNum;
1165     int64_t DispValue;
1166     unsigned SegRegNum;
1167     const MCExpr *DispExpr;
1168     if (!evaluateX86MemoryOperand(Inst, &BaseRegNum, &ScaleValue, &IndexRegNum,
1169                                   &DispValue, &SegRegNum, &DispExpr)) {
1170       llvm_unreachable("Evaluate failed");
1171       return;
1172     }
1173     // Make sure it's a stack access
1174     if (BaseRegNum != X86::RBP && BaseRegNum != X86::RSP) {
1175       llvm_unreachable("Not a stack access");
1176       return;
1177     }
1178 
1179     unsigned MemOpOffset = getMemoryOperandNo(Inst);
1180     unsigned NewOpcode = 0;
1181     if (I.IsLoad) {
1182       switch (I.DataSize) {
1183       case 2: NewOpcode = X86::POP16r; break;
1184       case 4: NewOpcode = X86::POP32r; break;
1185       case 8: NewOpcode = X86::POP64r; break;
1186       default:
1187         llvm_unreachable("Unexpected size");
1188       }
1189       unsigned RegOpndNum = Inst.getOperand(0).getReg();
1190       Inst.clear();
1191       Inst.setOpcode(NewOpcode);
1192       Inst.addOperand(MCOperand::createReg(RegOpndNum));
1193     } else {
1194       MCOperand SrcOpnd =
1195           Inst.getOperand(MemOpOffset + X86::AddrSegmentReg + 1);
1196       if (I.StoreFromReg) {
1197         switch (I.DataSize) {
1198         case 2: NewOpcode = X86::PUSH16r; break;
1199         case 4: NewOpcode = X86::PUSH32r; break;
1200         case 8: NewOpcode = X86::PUSH64r; break;
1201         default:
1202           llvm_unreachable("Unexpected size");
1203         }
1204         assert(SrcOpnd.isReg() && "Unexpected source operand");
1205         unsigned RegOpndNum = SrcOpnd.getReg();
1206         Inst.clear();
1207         Inst.setOpcode(NewOpcode);
1208         Inst.addOperand(MCOperand::createReg(RegOpndNum));
1209       } else {
1210         switch (I.DataSize) {
1211         case 2: NewOpcode = X86::PUSH16i8; break;
1212         case 4: NewOpcode = X86::PUSH32i8; break;
1213         case 8: NewOpcode = X86::PUSH64i32; break;
1214         default:
1215           llvm_unreachable("Unexpected size");
1216         }
1217         assert(SrcOpnd.isImm() && "Unexpected source operand");
1218         int64_t SrcImm = SrcOpnd.getImm();
1219         Inst.clear();
1220         Inst.setOpcode(NewOpcode);
1221         Inst.addOperand(MCOperand::createImm(SrcImm));
1222       }
1223     }
1224   }
1225 
1226   bool isStackAdjustment(const MCInst &Inst) const override {
1227     switch (Inst.getOpcode()) {
1228     default:
1229       return false;
1230     case X86::SUB64ri32:
1231     case X86::SUB64ri8:
1232     case X86::ADD64ri32:
1233     case X86::ADD64ri8:
1234     case X86::LEA64r:
1235       break;
1236     }
1237 
1238     const MCInstrDesc &MCII = Info->get(Inst.getOpcode());
1239     for (int I = 0, E = MCII.getNumDefs(); I != E; ++I) {
1240       const MCOperand &Operand = Inst.getOperand(I);
1241       if (Operand.isReg() && Operand.getReg() == X86::RSP)
1242         return true;
1243     }
1244     return false;
1245   }
1246 
1247   bool evaluateSimple(const MCInst &Inst, int64_t &Output,
1248                       std::pair<MCPhysReg, int64_t> Input1,
1249                       std::pair<MCPhysReg, int64_t> Input2) const override {
1250 
1251     auto getOperandVal = [&](MCPhysReg Reg) -> ErrorOr<int64_t> {
1252       if (Reg == Input1.first)
1253         return Input1.second;
1254       if (Reg == Input2.first)
1255         return Input2.second;
1256       return make_error_code(errc::result_out_of_range);
1257     };
1258 
1259     switch (Inst.getOpcode()) {
1260     default:
1261       return false;
1262 
1263     case X86::AND64ri32:
1264     case X86::AND64ri8:
1265       if (!Inst.getOperand(2).isImm())
1266         return false;
1267       if (ErrorOr<int64_t> InputVal =
1268               getOperandVal(Inst.getOperand(1).getReg()))
1269         Output = *InputVal & Inst.getOperand(2).getImm();
1270       else
1271         return false;
1272       break;
1273     case X86::SUB64ri32:
1274     case X86::SUB64ri8:
1275       if (!Inst.getOperand(2).isImm())
1276         return false;
1277       if (ErrorOr<int64_t> InputVal =
1278               getOperandVal(Inst.getOperand(1).getReg()))
1279         Output = *InputVal - Inst.getOperand(2).getImm();
1280       else
1281         return false;
1282       break;
1283     case X86::ADD64ri32:
1284     case X86::ADD64ri8:
1285       if (!Inst.getOperand(2).isImm())
1286         return false;
1287       if (ErrorOr<int64_t> InputVal =
1288               getOperandVal(Inst.getOperand(1).getReg()))
1289         Output = *InputVal + Inst.getOperand(2).getImm();
1290       else
1291         return false;
1292       break;
1293     case X86::ADD64i32:
1294       if (!Inst.getOperand(0).isImm())
1295         return false;
1296       if (ErrorOr<int64_t> InputVal = getOperandVal(X86::RAX))
1297         Output = *InputVal + Inst.getOperand(0).getImm();
1298       else
1299         return false;
1300       break;
1301 
1302     case X86::LEA64r: {
1303       unsigned BaseRegNum;
1304       int64_t ScaleValue;
1305       unsigned IndexRegNum;
1306       int64_t DispValue;
1307       unsigned SegRegNum;
1308       const MCExpr *DispExpr = nullptr;
1309       if (!evaluateX86MemoryOperand(Inst, &BaseRegNum, &ScaleValue,
1310                                     &IndexRegNum, &DispValue, &SegRegNum,
1311                                     &DispExpr))
1312         return false;
1313 
1314       if (BaseRegNum == X86::NoRegister || IndexRegNum != X86::NoRegister ||
1315           SegRegNum != X86::NoRegister || DispExpr)
1316         return false;
1317 
1318       if (ErrorOr<int64_t> InputVal = getOperandVal(BaseRegNum))
1319         Output = *InputVal + DispValue;
1320       else
1321         return false;
1322 
1323       break;
1324     }
1325     }
1326     return true;
1327   }
1328 
1329   bool isRegToRegMove(const MCInst &Inst, MCPhysReg &From,
1330                       MCPhysReg &To) const override {
1331     switch (Inst.getOpcode()) {
1332     default:
1333       return false;
1334     case X86::LEAVE:
1335     case X86::LEAVE64:
1336       To = getStackPointer();
1337       From = getFramePointer();
1338       return true;
1339     case X86::MOV64rr:
1340       To = Inst.getOperand(0).getReg();
1341       From = Inst.getOperand(1).getReg();
1342       return true;
1343     }
1344   }
1345 
1346   MCPhysReg getStackPointer() const override { return X86::RSP; }
1347   MCPhysReg getFramePointer() const override { return X86::RBP; }
1348   MCPhysReg getFlagsReg() const override { return X86::EFLAGS; }
1349 
1350   bool escapesVariable(const MCInst &Inst,
1351                        bool HasFramePointer) const override {
1352     int MemOpNo = getMemoryOperandNo(Inst);
1353     const MCInstrDesc &MCII = Info->get(Inst.getOpcode());
1354     const unsigned NumDefs = MCII.getNumDefs();
1355     static BitVector SPBPAliases(BitVector(getAliases(X86::RSP)) |=
1356                                  getAliases(X86::RBP));
1357     static BitVector SPAliases(getAliases(X86::RSP));
1358 
1359     // FIXME: PUSH can be technically a leak, but let's ignore this for now
1360     // because a lot of harmless prologue code will spill SP to the stack.
1361     // Unless push is clearly pushing an object address to the stack as
1362     // demonstrated by having a MemOp.
1363     bool IsPush = isPush(Inst);
1364     if (IsPush && MemOpNo == -1)
1365       return false;
1366 
1367     // We use this to detect LEA (has memop but does not access mem)
1368     bool AccessMem = MCII.mayLoad() || MCII.mayStore();
1369     bool DoesLeak = false;
1370     for (int I = 0, E = MCPlus::getNumPrimeOperands(Inst); I != E; ++I) {
1371       // Ignore if SP/BP is used to dereference memory -- that's fine
1372       if (MemOpNo != -1 && !IsPush && AccessMem && I >= MemOpNo &&
1373           I <= MemOpNo + 5)
1374         continue;
1375       // Ignore if someone is writing to SP/BP
1376       if (I < static_cast<int>(NumDefs))
1377         continue;
1378 
1379       const MCOperand &Operand = Inst.getOperand(I);
1380       if (HasFramePointer && Operand.isReg() && SPBPAliases[Operand.getReg()]) {
1381         DoesLeak = true;
1382         break;
1383       }
1384       if (!HasFramePointer && Operand.isReg() && SPAliases[Operand.getReg()]) {
1385         DoesLeak = true;
1386         break;
1387       }
1388     }
1389 
1390     // If potential leak, check if it is not just writing to itself/sp/bp
1391     if (DoesLeak) {
1392       for (int I = 0, E = NumDefs; I != E; ++I) {
1393         const MCOperand &Operand = Inst.getOperand(I);
1394         if (HasFramePointer && Operand.isReg() &&
1395             SPBPAliases[Operand.getReg()]) {
1396           DoesLeak = false;
1397           break;
1398         }
1399         if (!HasFramePointer && Operand.isReg() &&
1400             SPAliases[Operand.getReg()]) {
1401           DoesLeak = false;
1402           break;
1403         }
1404       }
1405     }
1406     return DoesLeak;
1407   }
1408 
1409   bool addToImm(MCInst &Inst, int64_t &Amt, MCContext *Ctx) const override {
1410     unsigned ImmOpNo = -1U;
1411     int MemOpNo = getMemoryOperandNo(Inst);
1412     if (MemOpNo != -1)
1413       ImmOpNo = MemOpNo + X86::AddrDisp;
1414     else
1415       for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst);
1416            ++Index)
1417         if (Inst.getOperand(Index).isImm())
1418           ImmOpNo = Index;
1419     if (ImmOpNo == -1U)
1420       return false;
1421 
1422     MCOperand &Operand = Inst.getOperand(ImmOpNo);
1423     Amt += Operand.getImm();
1424     Operand.setImm(Amt);
1425     // Check for the need for relaxation
1426     if (int64_t(Amt) == int64_t(int8_t(Amt)))
1427       return true;
1428 
1429     // Relax instruction
1430     switch (Inst.getOpcode()) {
1431     case X86::SUB64ri8:
1432       Inst.setOpcode(X86::SUB64ri32);
1433       break;
1434     case X86::ADD64ri8:
1435       Inst.setOpcode(X86::ADD64ri32);
1436       break;
1437     default:
1438       // No need for relaxation
1439       break;
1440     }
1441     return true;
1442   }
1443 
1444   /// TODO: this implementation currently works for the most common opcodes that
1445   /// load from memory. It can be extended to work with memory store opcodes as
1446   /// well as more memory load opcodes.
1447   bool replaceMemOperandWithImm(MCInst &Inst, StringRef ConstantData,
1448                                 uint64_t Offset) const override {
1449     enum CheckSignExt : uint8_t {
1450       NOCHECK = 0,
1451       CHECK8,
1452       CHECK32,
1453     };
1454 
1455     using CheckList = std::vector<std::pair<CheckSignExt, unsigned>>;
1456     struct InstInfo {
1457       // Size in bytes that Inst loads from memory.
1458       uint8_t DataSize;
1459 
1460       // True when the target operand has to be duplicated because the opcode
1461       // expects a LHS operand.
1462       bool HasLHS;
1463 
1464       // List of checks and corresponding opcodes to be used. We try to use the
1465       // smallest possible immediate value when various sizes are available,
1466       // hence we may need to check whether a larger constant fits in a smaller
1467       // immediate.
1468       CheckList Checks;
1469     };
1470 
1471     InstInfo I;
1472 
1473     switch (Inst.getOpcode()) {
1474     default: {
1475       switch (getPopSize(Inst)) {
1476       case 2:            I = {2, false, {{NOCHECK, X86::MOV16ri}}};  break;
1477       case 4:            I = {4, false, {{NOCHECK, X86::MOV32ri}}};  break;
1478       case 8:            I = {8, false, {{CHECK32, X86::MOV64ri32},
1479                                          {NOCHECK, X86::MOV64rm}}};  break;
1480       default:           return false;
1481       }
1482       break;
1483     }
1484 
1485     // MOV
1486     case X86::MOV8rm:      I = {1, false, {{NOCHECK, X86::MOV8ri}}};   break;
1487     case X86::MOV16rm:     I = {2, false, {{NOCHECK, X86::MOV16ri}}};  break;
1488     case X86::MOV32rm:     I = {4, false, {{NOCHECK, X86::MOV32ri}}};  break;
1489     case X86::MOV64rm:     I = {8, false, {{CHECK32, X86::MOV64ri32},
1490                                            {NOCHECK, X86::MOV64rm}}};  break;
1491 
1492     // MOVZX
1493     case X86::MOVZX16rm8:  I = {1, false, {{NOCHECK, X86::MOV16ri}}};  break;
1494     case X86::MOVZX32rm8:  I = {1, false, {{NOCHECK, X86::MOV32ri}}};  break;
1495     case X86::MOVZX32rm16: I = {2, false, {{NOCHECK, X86::MOV32ri}}};  break;
1496 
1497     // CMP
1498     case X86::CMP8rm:      I = {1, false, {{NOCHECK, X86::CMP8ri}}};   break;
1499     case X86::CMP16rm:     I = {2, false, {{CHECK8,  X86::CMP16ri8},
1500                                            {NOCHECK, X86::CMP16ri}}};  break;
1501     case X86::CMP32rm:     I = {4, false, {{CHECK8,  X86::CMP32ri8},
1502                                            {NOCHECK, X86::CMP32ri}}};  break;
1503     case X86::CMP64rm:     I = {8, false, {{CHECK8,  X86::CMP64ri8},
1504                                            {CHECK32, X86::CMP64ri32},
1505                                            {NOCHECK, X86::CMP64rm}}};  break;
1506 
1507     // TEST
1508     case X86::TEST8mr:     I = {1, false, {{NOCHECK, X86::TEST8ri}}};  break;
1509     case X86::TEST16mr:    I = {2, false, {{NOCHECK, X86::TEST16ri}}}; break;
1510     case X86::TEST32mr:    I = {4, false, {{NOCHECK, X86::TEST32ri}}}; break;
1511     case X86::TEST64mr:    I = {8, false, {{CHECK32, X86::TEST64ri32},
1512                                            {NOCHECK, X86::TEST64mr}}}; break;
1513 
1514     // ADD
1515     case X86::ADD8rm:      I = {1, true,  {{NOCHECK, X86::ADD8ri}}};   break;
1516     case X86::ADD16rm:     I = {2, true,  {{CHECK8,  X86::ADD16ri8},
1517                                            {NOCHECK, X86::ADD16ri}}};  break;
1518     case X86::ADD32rm:     I = {4, true,  {{CHECK8,  X86::ADD32ri8},
1519                                            {NOCHECK, X86::ADD32ri}}};  break;
1520     case X86::ADD64rm:     I = {8, true,  {{CHECK8,  X86::ADD64ri8},
1521                                            {CHECK32, X86::ADD64ri32},
1522                                            {NOCHECK, X86::ADD64rm}}};  break;
1523 
1524     // SUB
1525     case X86::SUB8rm:      I = {1, true,  {{NOCHECK, X86::SUB8ri}}};   break;
1526     case X86::SUB16rm:     I = {2, true,  {{CHECK8,  X86::SUB16ri8},
1527                                            {NOCHECK, X86::SUB16ri}}};  break;
1528     case X86::SUB32rm:     I = {4, true,  {{CHECK8,  X86::SUB32ri8},
1529                                            {NOCHECK, X86::SUB32ri}}};  break;
1530     case X86::SUB64rm:     I = {8, true,  {{CHECK8,  X86::SUB64ri8},
1531                                            {CHECK32, X86::SUB64ri32},
1532                                            {NOCHECK, X86::SUB64rm}}};  break;
1533 
1534     // AND
1535     case X86::AND8rm:      I = {1, true,  {{NOCHECK, X86::AND8ri}}};   break;
1536     case X86::AND16rm:     I = {2, true,  {{CHECK8,  X86::AND16ri8},
1537                                            {NOCHECK, X86::AND16ri}}};  break;
1538     case X86::AND32rm:     I = {4, true,  {{CHECK8,  X86::AND32ri8},
1539                                            {NOCHECK, X86::AND32ri}}};  break;
1540     case X86::AND64rm:     I = {8, true,  {{CHECK8,  X86::AND64ri8},
1541                                            {CHECK32, X86::AND64ri32},
1542                                            {NOCHECK, X86::AND64rm}}};  break;
1543 
1544     // OR
1545     case X86::OR8rm:       I = {1, true,  {{NOCHECK, X86::OR8ri}}};    break;
1546     case X86::OR16rm:      I = {2, true,  {{CHECK8,  X86::OR16ri8},
1547                                            {NOCHECK, X86::OR16ri}}};   break;
1548     case X86::OR32rm:      I = {4, true,  {{CHECK8,  X86::OR32ri8},
1549                                            {NOCHECK, X86::OR32ri}}};   break;
1550     case X86::OR64rm:      I = {8, true,  {{CHECK8,  X86::OR64ri8},
1551                                            {CHECK32, X86::OR64ri32},
1552                                            {NOCHECK, X86::OR64rm}}};   break;
1553 
1554     // XOR
1555     case X86::XOR8rm:      I = {1, true,  {{NOCHECK, X86::XOR8ri}}};   break;
1556     case X86::XOR16rm:     I = {2, true,  {{CHECK8,  X86::XOR16ri8},
1557                                            {NOCHECK, X86::XOR16ri}}};  break;
1558     case X86::XOR32rm:     I = {4, true,  {{CHECK8,  X86::XOR32ri8},
1559                                            {NOCHECK, X86::XOR32ri}}};  break;
1560     case X86::XOR64rm:     I = {8, true,  {{CHECK8,  X86::XOR64ri8},
1561                                            {CHECK32, X86::XOR64ri32},
1562                                            {NOCHECK, X86::XOR64rm}}};  break;
1563     }
1564 
1565     // Compute the immediate value.
1566     assert(Offset + I.DataSize <= ConstantData.size() &&
1567            "invalid offset for given constant data");
1568     int64_t ImmVal =
1569         DataExtractor(ConstantData, true, 8).getSigned(&Offset, I.DataSize);
1570 
1571     // Compute the new opcode.
1572     unsigned NewOpcode = 0;
1573     for (const std::pair<CheckSignExt, unsigned> &Check : I.Checks) {
1574       NewOpcode = Check.second;
1575       if (Check.first == NOCHECK)
1576         break;
1577       if (Check.first == CHECK8 && isInt<8>(ImmVal))
1578         break;
1579       if (Check.first == CHECK32 && isInt<32>(ImmVal))
1580         break;
1581     }
1582     if (NewOpcode == Inst.getOpcode())
1583       return false;
1584 
1585     // Modify the instruction.
1586     MCOperand ImmOp = MCOperand::createImm(ImmVal);
1587     uint32_t TargetOpNum = 0;
1588     // Test instruction does not follow the regular pattern of putting the
1589     // memory reference of a load (5 MCOperands) last in the list of operands.
1590     // Since it is not modifying the register operand, it is not treated as
1591     // a destination operand and it is not the first operand as it is in the
1592     // other instructions we treat here.
1593     if (NewOpcode == X86::TEST8ri || NewOpcode == X86::TEST16ri ||
1594         NewOpcode == X86::TEST32ri || NewOpcode == X86::TEST64ri32)
1595       TargetOpNum = getMemoryOperandNo(Inst) + X86::AddrNumOperands;
1596 
1597     MCOperand TargetOp = Inst.getOperand(TargetOpNum);
1598     Inst.clear();
1599     Inst.setOpcode(NewOpcode);
1600     Inst.addOperand(TargetOp);
1601     if (I.HasLHS)
1602       Inst.addOperand(TargetOp);
1603     Inst.addOperand(ImmOp);
1604 
1605     return true;
1606   }
1607 
1608   /// TODO: this implementation currently works for the most common opcodes that
1609   /// load from memory. It can be extended to work with memory store opcodes as
1610   /// well as more memory load opcodes.
1611   bool replaceMemOperandWithReg(MCInst &Inst, MCPhysReg RegNum) const override {
1612     unsigned NewOpcode;
1613 
1614     switch (Inst.getOpcode()) {
1615     default: {
1616       switch (getPopSize(Inst)) {
1617       case 2:            NewOpcode = X86::MOV16rr; break;
1618       case 4:            NewOpcode = X86::MOV32rr; break;
1619       case 8:            NewOpcode = X86::MOV64rr; break;
1620       default:           return false;
1621       }
1622       break;
1623     }
1624 
1625     // MOV
1626     case X86::MOV8rm:      NewOpcode = X86::MOV8rr;   break;
1627     case X86::MOV16rm:     NewOpcode = X86::MOV16rr;  break;
1628     case X86::MOV32rm:     NewOpcode = X86::MOV32rr;  break;
1629     case X86::MOV64rm:     NewOpcode = X86::MOV64rr;  break;
1630     }
1631 
1632     // Modify the instruction.
1633     MCOperand RegOp = MCOperand::createReg(RegNum);
1634     MCOperand TargetOp = Inst.getOperand(0);
1635     Inst.clear();
1636     Inst.setOpcode(NewOpcode);
1637     Inst.addOperand(TargetOp);
1638     Inst.addOperand(RegOp);
1639 
1640     return true;
1641   }
1642 
1643   bool isRedundantMove(const MCInst &Inst) const override {
1644     switch (Inst.getOpcode()) {
1645     default:
1646       return false;
1647 
1648     // MOV
1649     case X86::MOV8rr:
1650     case X86::MOV16rr:
1651     case X86::MOV32rr:
1652     case X86::MOV64rr:
1653       break;
1654     }
1655 
1656     assert(Inst.getOperand(0).isReg() && Inst.getOperand(1).isReg());
1657     return Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg();
1658   }
1659 
1660   bool requiresAlignedAddress(const MCInst &Inst) const override {
1661     const MCInstrDesc &Desc = Info->get(Inst.getOpcode());
1662     for (unsigned int I = 0; I < Desc.getNumOperands(); ++I) {
1663       const MCOperandInfo &Op = Desc.OpInfo[I];
1664       if (Op.OperandType != MCOI::OPERAND_REGISTER)
1665         continue;
1666       if (Op.RegClass == X86::VR128RegClassID)
1667         return true;
1668     }
1669     return false;
1670   }
1671 
1672   bool convertJmpToTailCall(MCInst &Inst) override {
1673     if (isTailCall(Inst))
1674       return false;
1675 
1676     int NewOpcode;
1677     switch (Inst.getOpcode()) {
1678     default:
1679       return false;
1680     case X86::JMP_1:
1681     case X86::JMP_2:
1682     case X86::JMP_4:
1683       NewOpcode = X86::JMP_4;
1684       break;
1685     case X86::JMP16m:
1686     case X86::JMP32m:
1687     case X86::JMP64m:
1688       NewOpcode = X86::JMP32m;
1689       break;
1690     case X86::JMP16r:
1691     case X86::JMP32r:
1692     case X86::JMP64r:
1693       NewOpcode = X86::JMP32r;
1694       break;
1695     }
1696 
1697     Inst.setOpcode(NewOpcode);
1698     setTailCall(Inst);
1699     return true;
1700   }
1701 
1702   bool convertTailCallToJmp(MCInst &Inst) override {
1703     int NewOpcode;
1704     switch (Inst.getOpcode()) {
1705     default:
1706       return false;
1707     case X86::JMP_4:
1708       NewOpcode = X86::JMP_1;
1709       break;
1710     case X86::JMP32m:
1711       NewOpcode = X86::JMP64m;
1712       break;
1713     case X86::JMP32r:
1714       NewOpcode = X86::JMP64r;
1715       break;
1716     }
1717 
1718     Inst.setOpcode(NewOpcode);
1719     removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall);
1720     clearOffset(Inst);
1721     return true;
1722   }
1723 
1724   bool convertTailCallToCall(MCInst &Inst) override {
1725     int NewOpcode;
1726     switch (Inst.getOpcode()) {
1727     default:
1728       return false;
1729     case X86::JMP_4:
1730       NewOpcode = X86::CALL64pcrel32;
1731       break;
1732     case X86::JMP32m:
1733       NewOpcode = X86::CALL64m;
1734       break;
1735     case X86::JMP32r:
1736       NewOpcode = X86::CALL64r;
1737       break;
1738     }
1739 
1740     Inst.setOpcode(NewOpcode);
1741     removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall);
1742     return true;
1743   }
1744 
1745   bool convertCallToIndirectCall(MCInst &Inst, const MCSymbol *TargetLocation,
1746                                  MCContext *Ctx) override {
1747     assert((Inst.getOpcode() == X86::CALL64pcrel32 ||
1748             (Inst.getOpcode() == X86::JMP_4 && isTailCall(Inst))) &&
1749            "64-bit direct (tail) call instruction expected");
1750     const auto NewOpcode =
1751         (Inst.getOpcode() == X86::CALL64pcrel32) ? X86::CALL64m : X86::JMP32m;
1752     Inst.setOpcode(NewOpcode);
1753 
1754     // Replace the first operand and preserve auxiliary operands of
1755     // the instruction.
1756     Inst.erase(Inst.begin());
1757     Inst.insert(Inst.begin(),
1758                 MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
1759     Inst.insert(Inst.begin(),
1760                 MCOperand::createExpr(                  // Displacement
1761                     MCSymbolRefExpr::create(TargetLocation,
1762                                             MCSymbolRefExpr::VK_None, *Ctx)));
1763     Inst.insert(Inst.begin(),
1764                 MCOperand::createReg(X86::NoRegister)); // IndexReg
1765     Inst.insert(Inst.begin(),
1766                 MCOperand::createImm(1));               // ScaleAmt
1767     Inst.insert(Inst.begin(),
1768                 MCOperand::createReg(X86::RIP));        // BaseReg
1769 
1770     return true;
1771   }
1772 
1773   void convertIndirectCallToLoad(MCInst &Inst, MCPhysReg Reg) override {
1774     bool IsTailCall = isTailCall(Inst);
1775     if (IsTailCall)
1776       removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall);
1777     if (Inst.getOpcode() == X86::CALL64m ||
1778         (Inst.getOpcode() == X86::JMP32m && IsTailCall)) {
1779       Inst.setOpcode(X86::MOV64rm);
1780       Inst.insert(Inst.begin(), MCOperand::createReg(Reg));
1781       return;
1782     }
1783     if (Inst.getOpcode() == X86::CALL64r ||
1784         (Inst.getOpcode() == X86::JMP32r && IsTailCall)) {
1785       Inst.setOpcode(X86::MOV64rr);
1786       Inst.insert(Inst.begin(), MCOperand::createReg(Reg));
1787       return;
1788     }
1789     LLVM_DEBUG(Inst.dump());
1790     llvm_unreachable("not implemented");
1791   }
1792 
1793   bool shortenInstruction(MCInst &Inst,
1794                           const MCSubtargetInfo &STI) const override {
1795     unsigned OldOpcode = Inst.getOpcode();
1796     unsigned NewOpcode = OldOpcode;
1797 
1798     int MemOpNo = getMemoryOperandNo(Inst);
1799 
1800     // Check and remove redundant Address-Size override prefix.
1801     if (opts::X86StripRedundantAddressSize) {
1802       uint64_t TSFlags = Info->get(OldOpcode).TSFlags;
1803       unsigned Flags = Inst.getFlags();
1804 
1805       if (!X86_MC::needsAddressSizeOverride(Inst, STI, MemOpNo, TSFlags) &&
1806           Flags & X86::IP_HAS_AD_SIZE)
1807         Inst.setFlags(Flags ^ X86::IP_HAS_AD_SIZE);
1808     }
1809 
1810     // Check and remove EIZ/RIZ. These cases represent ambiguous cases where
1811     // SIB byte is present, but no index is used and modrm alone should have
1812     // been enough. Converting to NoRegister effectively removes the SIB byte.
1813     if (MemOpNo >= 0) {
1814       MCOperand &IndexOp =
1815           Inst.getOperand(static_cast<unsigned>(MemOpNo) + X86::AddrIndexReg);
1816       if (IndexOp.getReg() == X86::EIZ || IndexOp.getReg() == X86::RIZ)
1817         IndexOp = MCOperand::createReg(X86::NoRegister);
1818     }
1819 
1820     if (isBranch(Inst)) {
1821       NewOpcode = getShortBranchOpcode(OldOpcode);
1822     } else if (OldOpcode == X86::MOV64ri) {
1823       if (Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).isImm()) {
1824         const int64_t Imm =
1825             Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).getImm();
1826         if (int64_t(Imm) == int64_t(int32_t(Imm)))
1827           NewOpcode = X86::MOV64ri32;
1828       }
1829     } else {
1830       // If it's arithmetic instruction check if signed operand fits in 1 byte.
1831       const unsigned ShortOpcode = getShortArithOpcode(OldOpcode);
1832       if (ShortOpcode != OldOpcode &&
1833           Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).isImm()) {
1834         int64_t Imm =
1835             Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).getImm();
1836         if (int64_t(Imm) == int64_t(int8_t(Imm)))
1837           NewOpcode = ShortOpcode;
1838       }
1839     }
1840 
1841     if (NewOpcode == OldOpcode)
1842       return false;
1843 
1844     Inst.setOpcode(NewOpcode);
1845     return true;
1846   }
1847 
1848   bool
1849   convertMoveToConditionalMove(MCInst &Inst, unsigned CC, bool AllowStackMemOp,
1850                                bool AllowBasePtrStackMemOp) const override {
1851     // - Register-register moves are OK
1852     // - Stores are filtered out by opcode (no store CMOV)
1853     // - Non-stack loads are prohibited (generally unsafe)
1854     // - Stack loads are OK if AllowStackMemOp is true
1855     // - Stack loads with RBP are OK if AllowBasePtrStackMemOp is true
1856     if (isLoad(Inst)) {
1857       // If stack memory operands are not allowed, no loads are allowed
1858       if (!AllowStackMemOp)
1859         return false;
1860 
1861       // If stack memory operands are allowed, check if it's a load from stack
1862       bool IsLoad, IsStore, IsStoreFromReg, IsSimple, IsIndexed;
1863       MCPhysReg Reg;
1864       int32_t SrcImm;
1865       uint16_t StackPtrReg;
1866       int64_t StackOffset;
1867       uint8_t Size;
1868       bool IsStackAccess =
1869           isStackAccess(Inst, IsLoad, IsStore, IsStoreFromReg, Reg, SrcImm,
1870                         StackPtrReg, StackOffset, Size, IsSimple, IsIndexed);
1871       // Prohibit non-stack-based loads
1872       if (!IsStackAccess)
1873         return false;
1874       // If stack memory operands are allowed, check if it's RBP-based
1875       if (!AllowBasePtrStackMemOp &&
1876           RegInfo->isSubRegisterEq(X86::RBP, StackPtrReg))
1877         return false;
1878     }
1879 
1880     unsigned NewOpcode = 0;
1881     switch (Inst.getOpcode()) {
1882     case X86::MOV16rr:
1883       NewOpcode = X86::CMOV16rr;
1884       break;
1885     case X86::MOV16rm:
1886       NewOpcode = X86::CMOV16rm;
1887       break;
1888     case X86::MOV32rr:
1889       NewOpcode = X86::CMOV32rr;
1890       break;
1891     case X86::MOV32rm:
1892       NewOpcode = X86::CMOV32rm;
1893       break;
1894     case X86::MOV64rr:
1895       NewOpcode = X86::CMOV64rr;
1896       break;
1897     case X86::MOV64rm:
1898       NewOpcode = X86::CMOV64rm;
1899       break;
1900     default:
1901       return false;
1902     }
1903     Inst.setOpcode(NewOpcode);
1904     // Insert CC at the end of prime operands, before annotations
1905     Inst.insert(Inst.begin() + MCPlus::getNumPrimeOperands(Inst),
1906                 MCOperand::createImm(CC));
1907     // CMOV is a 3-operand MCInst, so duplicate the destination as src1
1908     Inst.insert(Inst.begin(), Inst.getOperand(0));
1909     return true;
1910   }
1911 
1912   bool lowerTailCall(MCInst &Inst) override {
1913     if (Inst.getOpcode() == X86::JMP_4 && isTailCall(Inst)) {
1914       Inst.setOpcode(X86::JMP_1);
1915       removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall);
1916       return true;
1917     }
1918     return false;
1919   }
1920 
1921   const MCSymbol *getTargetSymbol(const MCInst &Inst,
1922                                   unsigned OpNum = 0) const override {
1923     if (OpNum >= MCPlus::getNumPrimeOperands(Inst))
1924       return nullptr;
1925 
1926     const MCOperand &Op = Inst.getOperand(OpNum);
1927     if (!Op.isExpr())
1928       return nullptr;
1929 
1930     auto *SymExpr = dyn_cast<MCSymbolRefExpr>(Op.getExpr());
1931     if (!SymExpr || SymExpr->getKind() != MCSymbolRefExpr::VK_None)
1932       return nullptr;
1933 
1934     return &SymExpr->getSymbol();
1935   }
1936 
1937   // This is the same as the base class, but since we are overriding one of
1938   // getTargetSymbol's signatures above, we need to override all of them.
1939   const MCSymbol *getTargetSymbol(const MCExpr *Expr) const override {
1940     return &cast<const MCSymbolRefExpr>(Expr)->getSymbol();
1941   }
1942 
1943   bool analyzeBranch(InstructionIterator Begin, InstructionIterator End,
1944                      const MCSymbol *&TBB, const MCSymbol *&FBB,
1945                      MCInst *&CondBranch,
1946                      MCInst *&UncondBranch) const override {
1947     auto I = End;
1948 
1949     // Bottom-up analysis
1950     while (I != Begin) {
1951       --I;
1952 
1953       // Ignore nops and CFIs
1954       if (isPseudo(*I))
1955         continue;
1956 
1957       // Stop when we find the first non-terminator
1958       if (!isTerminator(*I))
1959         break;
1960 
1961       if (!isBranch(*I))
1962         break;
1963 
1964       // Handle unconditional branches.
1965       if ((I->getOpcode() == X86::JMP_1 || I->getOpcode() == X86::JMP_2 ||
1966            I->getOpcode() == X86::JMP_4) &&
1967           !isTailCall(*I)) {
1968         // If any code was seen after this unconditional branch, we've seen
1969         // unreachable code. Ignore them.
1970         CondBranch = nullptr;
1971         UncondBranch = &*I;
1972         const MCSymbol *Sym = getTargetSymbol(*I);
1973         assert(Sym != nullptr &&
1974                "Couldn't extract BB symbol from jump operand");
1975         TBB = Sym;
1976         continue;
1977       }
1978 
1979       // Handle conditional branches and ignore indirect branches
1980       if (!isUnsupportedBranch(I->getOpcode()) &&
1981           getCondCode(*I) == X86::COND_INVALID) {
1982         // Indirect branch
1983         return false;
1984       }
1985 
1986       if (CondBranch == nullptr) {
1987         const MCSymbol *TargetBB = getTargetSymbol(*I);
1988         if (TargetBB == nullptr) {
1989           // Unrecognized branch target
1990           return false;
1991         }
1992         FBB = TBB;
1993         TBB = TargetBB;
1994         CondBranch = &*I;
1995         continue;
1996       }
1997 
1998       llvm_unreachable("multiple conditional branches in one BB");
1999     }
2000     return true;
2001   }
2002 
2003   template <typename Itr>
2004   std::pair<IndirectBranchType, MCInst *>
2005   analyzePICJumpTable(Itr II, Itr IE, MCPhysReg R1, MCPhysReg R2) const {
2006     // Analyze PIC-style jump table code template:
2007     //
2008     //    lea PIC_JUMP_TABLE(%rip), {%r1|%r2}     <- MemLocInstr
2009     //    mov ({%r1|%r2}, %index, 4), {%r2|%r1}
2010     //    add %r2, %r1
2011     //    jmp *%r1
2012     //
2013     // (with any irrelevant instructions in-between)
2014     //
2015     // When we call this helper we've already determined %r1 and %r2, and
2016     // reverse instruction iterator \p II is pointing to the ADD instruction.
2017     //
2018     // PIC jump table looks like following:
2019     //
2020     //   JT:  ----------
2021     //    E1:| L1 - JT  |
2022     //       |----------|
2023     //    E2:| L2 - JT  |
2024     //       |----------|
2025     //       |          |
2026     //          ......
2027     //    En:| Ln - JT  |
2028     //        ----------
2029     //
2030     // Where L1, L2, ..., Ln represent labels in the function.
2031     //
2032     // The actual relocations in the table will be of the form:
2033     //
2034     //   Ln - JT
2035     //    = (Ln - En) + (En - JT)
2036     //    = R_X86_64_PC32(Ln) + En - JT
2037     //    = R_X86_64_PC32(Ln + offsetof(En))
2038     //
2039     LLVM_DEBUG(dbgs() << "Checking for PIC jump table\n");
2040     MCInst *MemLocInstr = nullptr;
2041     const MCInst *MovInstr = nullptr;
2042     while (++II != IE) {
2043       MCInst &Instr = *II;
2044       const MCInstrDesc &InstrDesc = Info->get(Instr.getOpcode());
2045       if (!InstrDesc.hasDefOfPhysReg(Instr, R1, *RegInfo) &&
2046           !InstrDesc.hasDefOfPhysReg(Instr, R2, *RegInfo)) {
2047         // Ignore instructions that don't affect R1, R2 registers.
2048         continue;
2049       }
2050       if (!MovInstr) {
2051         // Expect to see MOV instruction.
2052         if (!isMOVSX64rm32(Instr)) {
2053           LLVM_DEBUG(dbgs() << "MOV instruction expected.\n");
2054           break;
2055         }
2056 
2057         // Check if it's setting %r1 or %r2. In canonical form it sets %r2.
2058         // If it sets %r1 - rename the registers so we have to only check
2059         // a single form.
2060         unsigned MovDestReg = Instr.getOperand(0).getReg();
2061         if (MovDestReg != R2)
2062           std::swap(R1, R2);
2063         if (MovDestReg != R2) {
2064           LLVM_DEBUG(dbgs() << "MOV instruction expected to set %r2\n");
2065           break;
2066         }
2067 
2068         // Verify operands for MOV.
2069         unsigned  BaseRegNum;
2070         int64_t   ScaleValue;
2071         unsigned  IndexRegNum;
2072         int64_t   DispValue;
2073         unsigned  SegRegNum;
2074         if (!evaluateX86MemoryOperand(Instr, &BaseRegNum, &ScaleValue,
2075                                       &IndexRegNum, &DispValue, &SegRegNum))
2076           break;
2077         if (BaseRegNum != R1 || ScaleValue != 4 ||
2078             IndexRegNum == X86::NoRegister || DispValue != 0 ||
2079             SegRegNum != X86::NoRegister)
2080           break;
2081         MovInstr = &Instr;
2082       } else {
2083         if (!InstrDesc.hasDefOfPhysReg(Instr, R1, *RegInfo))
2084           continue;
2085         if (!isLEA64r(Instr)) {
2086           LLVM_DEBUG(dbgs() << "LEA instruction expected\n");
2087           break;
2088         }
2089         if (Instr.getOperand(0).getReg() != R1) {
2090           LLVM_DEBUG(dbgs() << "LEA instruction expected to set %r1\n");
2091           break;
2092         }
2093 
2094         // Verify operands for LEA.
2095         unsigned      BaseRegNum;
2096         int64_t       ScaleValue;
2097         unsigned      IndexRegNum;
2098         const MCExpr *DispExpr = nullptr;
2099         int64_t       DispValue;
2100         unsigned      SegRegNum;
2101         if (!evaluateX86MemoryOperand(Instr, &BaseRegNum, &ScaleValue,
2102                                       &IndexRegNum, &DispValue, &SegRegNum,
2103                                       &DispExpr))
2104           break;
2105         if (BaseRegNum != RegInfo->getProgramCounter() ||
2106             IndexRegNum != X86::NoRegister || SegRegNum != X86::NoRegister ||
2107             DispExpr == nullptr)
2108           break;
2109         MemLocInstr = &Instr;
2110         break;
2111       }
2112     }
2113 
2114     if (!MemLocInstr)
2115       return std::make_pair(IndirectBranchType::UNKNOWN, nullptr);
2116 
2117     LLVM_DEBUG(dbgs() << "checking potential PIC jump table\n");
2118     return std::make_pair(IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE,
2119                           MemLocInstr);
2120   }
2121 
2122   IndirectBranchType analyzeIndirectBranch(
2123       MCInst &Instruction, InstructionIterator Begin, InstructionIterator End,
2124       const unsigned PtrSize, MCInst *&MemLocInstrOut, unsigned &BaseRegNumOut,
2125       unsigned &IndexRegNumOut, int64_t &DispValueOut,
2126       const MCExpr *&DispExprOut, MCInst *&PCRelBaseOut) const override {
2127     // Try to find a (base) memory location from where the address for
2128     // the indirect branch is loaded. For X86-64 the memory will be specified
2129     // in the following format:
2130     //
2131     //   {%rip}/{%basereg} + Imm + IndexReg * Scale
2132     //
2133     // We are interested in the cases where Scale == sizeof(uintptr_t) and
2134     // the contents of the memory are presumably an array of pointers to code.
2135     //
2136     // Normal jump table:
2137     //
2138     //    jmp *(JUMP_TABLE, %index, Scale)        <- MemLocInstr
2139     //
2140     //    or
2141     //
2142     //    mov (JUMP_TABLE, %index, Scale), %r1    <- MemLocInstr
2143     //    ...
2144     //    jmp %r1
2145     //
2146     // We handle PIC-style jump tables separately.
2147     //
2148     MemLocInstrOut = nullptr;
2149     BaseRegNumOut = X86::NoRegister;
2150     IndexRegNumOut = X86::NoRegister;
2151     DispValueOut = 0;
2152     DispExprOut = nullptr;
2153 
2154     std::reverse_iterator<InstructionIterator> II(End);
2155     std::reverse_iterator<InstructionIterator> IE(Begin);
2156 
2157     IndirectBranchType Type = IndirectBranchType::UNKNOWN;
2158 
2159     // An instruction referencing memory used by jump instruction (directly or
2160     // via register). This location could be an array of function pointers
2161     // in case of indirect tail call, or a jump table.
2162     MCInst *MemLocInstr = nullptr;
2163 
2164     if (MCPlus::getNumPrimeOperands(Instruction) == 1) {
2165       // If the indirect jump is on register - try to detect if the
2166       // register value is loaded from a memory location.
2167       assert(Instruction.getOperand(0).isReg() && "register operand expected");
2168       const unsigned R1 = Instruction.getOperand(0).getReg();
2169       // Check if one of the previous instructions defines the jump-on register.
2170       for (auto PrevII = II; PrevII != IE; ++PrevII) {
2171         MCInst &PrevInstr = *PrevII;
2172         const MCInstrDesc &PrevInstrDesc = Info->get(PrevInstr.getOpcode());
2173 
2174         if (!PrevInstrDesc.hasDefOfPhysReg(PrevInstr, R1, *RegInfo))
2175           continue;
2176 
2177         if (isMoveMem2Reg(PrevInstr)) {
2178           MemLocInstr = &PrevInstr;
2179           break;
2180         }
2181         if (isADD64rr(PrevInstr)) {
2182           unsigned R2 = PrevInstr.getOperand(2).getReg();
2183           if (R1 == R2)
2184             return IndirectBranchType::UNKNOWN;
2185           std::tie(Type, MemLocInstr) = analyzePICJumpTable(PrevII, IE, R1, R2);
2186           break;
2187         }
2188         return IndirectBranchType::UNKNOWN;
2189       }
2190       if (!MemLocInstr) {
2191         // No definition seen for the register in this function so far. Could be
2192         // an input parameter - which means it is an external code reference.
2193         // It also could be that the definition happens to be in the code that
2194         // we haven't processed yet. Since we have to be conservative, return
2195         // as UNKNOWN case.
2196         return IndirectBranchType::UNKNOWN;
2197       }
2198     } else {
2199       MemLocInstr = &Instruction;
2200     }
2201 
2202     const MCRegister RIPRegister = RegInfo->getProgramCounter();
2203 
2204     // Analyze the memory location.
2205     unsigned BaseRegNum, IndexRegNum, SegRegNum;
2206     int64_t ScaleValue, DispValue;
2207     const MCExpr *DispExpr;
2208 
2209     if (!evaluateX86MemoryOperand(*MemLocInstr, &BaseRegNum, &ScaleValue,
2210                                   &IndexRegNum, &DispValue, &SegRegNum,
2211                                   &DispExpr))
2212       return IndirectBranchType::UNKNOWN;
2213 
2214     BaseRegNumOut = BaseRegNum;
2215     IndexRegNumOut = IndexRegNum;
2216     DispValueOut = DispValue;
2217     DispExprOut = DispExpr;
2218 
2219     if ((BaseRegNum != X86::NoRegister && BaseRegNum != RIPRegister) ||
2220         SegRegNum != X86::NoRegister)
2221       return IndirectBranchType::UNKNOWN;
2222 
2223     if (MemLocInstr == &Instruction &&
2224         (!ScaleValue || IndexRegNum == X86::NoRegister)) {
2225       MemLocInstrOut = MemLocInstr;
2226       return IndirectBranchType::POSSIBLE_FIXED_BRANCH;
2227     }
2228 
2229     if (Type == IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE &&
2230         (ScaleValue != 1 || BaseRegNum != RIPRegister))
2231       return IndirectBranchType::UNKNOWN;
2232 
2233     if (Type != IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE &&
2234         ScaleValue != PtrSize)
2235       return IndirectBranchType::UNKNOWN;
2236 
2237     MemLocInstrOut = MemLocInstr;
2238 
2239     return Type;
2240   }
2241 
2242   /// Analyze a callsite to see if it could be a virtual method call.  This only
2243   /// checks to see if the overall pattern is satisfied, it does not guarantee
2244   /// that the callsite is a true virtual method call.
2245   /// The format of virtual method calls that are recognized is one of the
2246   /// following:
2247   ///
2248   ///  Form 1: (found in debug code)
2249   ///    add METHOD_OFFSET, %VtableReg
2250   ///    mov (%VtableReg), %MethodReg
2251   ///    ...
2252   ///    call or jmp *%MethodReg
2253   ///
2254   ///  Form 2:
2255   ///    mov METHOD_OFFSET(%VtableReg), %MethodReg
2256   ///    ...
2257   ///    call or jmp *%MethodReg
2258   ///
2259   ///  Form 3:
2260   ///    ...
2261   ///    call or jmp *METHOD_OFFSET(%VtableReg)
2262   ///
2263   bool analyzeVirtualMethodCall(InstructionIterator ForwardBegin,
2264                                 InstructionIterator ForwardEnd,
2265                                 std::vector<MCInst *> &MethodFetchInsns,
2266                                 unsigned &VtableRegNum, unsigned &MethodRegNum,
2267                                 uint64_t &MethodOffset) const override {
2268     VtableRegNum = X86::NoRegister;
2269     MethodRegNum = X86::NoRegister;
2270     MethodOffset = 0;
2271 
2272     std::reverse_iterator<InstructionIterator> Itr(ForwardEnd);
2273     std::reverse_iterator<InstructionIterator> End(ForwardBegin);
2274 
2275     MCInst &CallInst = *Itr++;
2276     assert(isIndirectBranch(CallInst) || isCall(CallInst));
2277 
2278     unsigned BaseReg, IndexReg, SegmentReg;
2279     int64_t Scale, Disp;
2280     const MCExpr *DispExpr;
2281 
2282     // The call can just be jmp offset(reg)
2283     if (evaluateX86MemoryOperand(CallInst, &BaseReg, &Scale, &IndexReg, &Disp,
2284                                  &SegmentReg, &DispExpr)) {
2285       if (!DispExpr && BaseReg != X86::RIP && BaseReg != X86::RBP &&
2286           BaseReg != X86::NoRegister) {
2287         MethodRegNum = BaseReg;
2288         if (Scale == 1 && IndexReg == X86::NoRegister &&
2289             SegmentReg == X86::NoRegister) {
2290           VtableRegNum = MethodRegNum;
2291           MethodOffset = Disp;
2292           MethodFetchInsns.push_back(&CallInst);
2293           return true;
2294         }
2295       }
2296       return false;
2297     }
2298     if (CallInst.getOperand(0).isReg())
2299       MethodRegNum = CallInst.getOperand(0).getReg();
2300     else
2301       return false;
2302 
2303     if (MethodRegNum == X86::RIP || MethodRegNum == X86::RBP) {
2304       VtableRegNum = X86::NoRegister;
2305       MethodRegNum = X86::NoRegister;
2306       return false;
2307     }
2308 
2309     // find load from vtable, this may or may not include the method offset
2310     while (Itr != End) {
2311       MCInst &CurInst = *Itr++;
2312       const MCInstrDesc &Desc = Info->get(CurInst.getOpcode());
2313       if (Desc.hasDefOfPhysReg(CurInst, MethodRegNum, *RegInfo)) {
2314         if (isLoad(CurInst) &&
2315             evaluateX86MemoryOperand(CurInst, &BaseReg, &Scale, &IndexReg,
2316                                      &Disp, &SegmentReg, &DispExpr)) {
2317           if (!DispExpr && Scale == 1 && BaseReg != X86::RIP &&
2318               BaseReg != X86::RBP && BaseReg != X86::NoRegister &&
2319               IndexReg == X86::NoRegister && SegmentReg == X86::NoRegister &&
2320               BaseReg != X86::RIP) {
2321             VtableRegNum = BaseReg;
2322             MethodOffset = Disp;
2323             MethodFetchInsns.push_back(&CurInst);
2324             if (MethodOffset != 0)
2325               return true;
2326             break;
2327           }
2328         }
2329         return false;
2330       }
2331     }
2332 
2333     if (!VtableRegNum)
2334       return false;
2335 
2336     // look for any adds affecting the method register.
2337     while (Itr != End) {
2338       MCInst &CurInst = *Itr++;
2339       const MCInstrDesc &Desc = Info->get(CurInst.getOpcode());
2340       if (Desc.hasDefOfPhysReg(CurInst, VtableRegNum, *RegInfo)) {
2341         if (isADDri(CurInst)) {
2342           assert(!MethodOffset);
2343           MethodOffset = CurInst.getOperand(2).getImm();
2344           MethodFetchInsns.insert(MethodFetchInsns.begin(), &CurInst);
2345           break;
2346         }
2347       }
2348     }
2349 
2350     return true;
2351   }
2352 
2353   bool createStackPointerIncrement(MCInst &Inst, int Size,
2354                                    bool NoFlagsClobber) const override {
2355     if (NoFlagsClobber) {
2356       Inst.setOpcode(X86::LEA64r);
2357       Inst.clear();
2358       Inst.addOperand(MCOperand::createReg(X86::RSP));
2359       Inst.addOperand(MCOperand::createReg(X86::RSP));        // BaseReg
2360       Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
2361       Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
2362       Inst.addOperand(MCOperand::createImm(-Size));           // Displacement
2363       Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
2364       return true;
2365     }
2366     Inst.setOpcode(X86::SUB64ri8);
2367     Inst.clear();
2368     Inst.addOperand(MCOperand::createReg(X86::RSP));
2369     Inst.addOperand(MCOperand::createReg(X86::RSP));
2370     Inst.addOperand(MCOperand::createImm(Size));
2371     return true;
2372   }
2373 
2374   bool createStackPointerDecrement(MCInst &Inst, int Size,
2375                                    bool NoFlagsClobber) const override {
2376     if (NoFlagsClobber) {
2377       Inst.setOpcode(X86::LEA64r);
2378       Inst.clear();
2379       Inst.addOperand(MCOperand::createReg(X86::RSP));
2380       Inst.addOperand(MCOperand::createReg(X86::RSP));        // BaseReg
2381       Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
2382       Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
2383       Inst.addOperand(MCOperand::createImm(Size));            // Displacement
2384       Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
2385       return true;
2386     }
2387     Inst.setOpcode(X86::ADD64ri8);
2388     Inst.clear();
2389     Inst.addOperand(MCOperand::createReg(X86::RSP));
2390     Inst.addOperand(MCOperand::createReg(X86::RSP));
2391     Inst.addOperand(MCOperand::createImm(Size));
2392     return true;
2393   }
2394 
2395   bool createSaveToStack(MCInst &Inst, const MCPhysReg &StackReg, int Offset,
2396                          const MCPhysReg &SrcReg, int Size) const override {
2397     unsigned NewOpcode;
2398     switch (Size) {
2399     default:
2400       return false;
2401     case 2:      NewOpcode = X86::MOV16mr; break;
2402     case 4:      NewOpcode = X86::MOV32mr; break;
2403     case 8:      NewOpcode = X86::MOV64mr; break;
2404     }
2405     Inst.setOpcode(NewOpcode);
2406     Inst.clear();
2407     Inst.addOperand(MCOperand::createReg(StackReg));        // BaseReg
2408     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
2409     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
2410     Inst.addOperand(MCOperand::createImm(Offset));          // Displacement
2411     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
2412     Inst.addOperand(MCOperand::createReg(SrcReg));
2413     return true;
2414   }
2415 
2416   bool createRestoreFromStack(MCInst &Inst, const MCPhysReg &StackReg,
2417                               int Offset, const MCPhysReg &DstReg,
2418                               int Size) const override {
2419     return createLoad(Inst, StackReg, /*Scale=*/1, /*IndexReg=*/X86::NoRegister,
2420                       Offset, nullptr, /*AddrSegmentReg=*/X86::NoRegister,
2421                       DstReg, Size);
2422   }
2423 
2424   bool createLoad(MCInst &Inst, const MCPhysReg &BaseReg, int64_t Scale,
2425                   const MCPhysReg &IndexReg, int64_t Offset,
2426                   const MCExpr *OffsetExpr, const MCPhysReg &AddrSegmentReg,
2427                   const MCPhysReg &DstReg, int Size) const override {
2428     unsigned NewOpcode;
2429     switch (Size) {
2430     default:
2431       return false;
2432     case 2:      NewOpcode = X86::MOV16rm; break;
2433     case 4:      NewOpcode = X86::MOV32rm; break;
2434     case 8:      NewOpcode = X86::MOV64rm; break;
2435     }
2436     Inst.setOpcode(NewOpcode);
2437     Inst.clear();
2438     Inst.addOperand(MCOperand::createReg(DstReg));
2439     Inst.addOperand(MCOperand::createReg(BaseReg));
2440     Inst.addOperand(MCOperand::createImm(Scale));
2441     Inst.addOperand(MCOperand::createReg(IndexReg));
2442     if (OffsetExpr)
2443       Inst.addOperand(MCOperand::createExpr(OffsetExpr)); // Displacement
2444     else
2445       Inst.addOperand(MCOperand::createImm(Offset)); // Displacement
2446     Inst.addOperand(MCOperand::createReg(AddrSegmentReg)); // AddrSegmentReg
2447     return true;
2448   }
2449 
2450   void createLoadImmediate(MCInst &Inst, const MCPhysReg Dest,
2451                            uint32_t Imm) const override {
2452     Inst.setOpcode(X86::MOV64ri32);
2453     Inst.clear();
2454     Inst.addOperand(MCOperand::createReg(Dest));
2455     Inst.addOperand(MCOperand::createImm(Imm));
2456   }
2457 
2458   bool createIncMemory(MCInst &Inst, const MCSymbol *Target,
2459                        MCContext *Ctx) const override {
2460 
2461     Inst.setOpcode(X86::LOCK_INC64m);
2462     Inst.clear();
2463     Inst.addOperand(MCOperand::createReg(X86::RIP));        // BaseReg
2464     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
2465     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
2466 
2467     Inst.addOperand(MCOperand::createExpr(
2468         MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None,
2469                                 *Ctx)));                    // Displacement
2470     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
2471     return true;
2472   }
2473 
2474   bool createIJmp32Frag(SmallVectorImpl<MCInst> &Insts,
2475                         const MCOperand &BaseReg, const MCOperand &Scale,
2476                         const MCOperand &IndexReg, const MCOperand &Offset,
2477                         const MCOperand &TmpReg) const override {
2478     // The code fragment we emit here is:
2479     //
2480     //  mov32 (%base, %index, scale), %tmpreg
2481     //  ijmp *(%tmpreg)
2482     //
2483     MCInst IJmp;
2484     IJmp.setOpcode(X86::JMP64r);
2485     IJmp.addOperand(TmpReg);
2486 
2487     MCInst Load;
2488     Load.setOpcode(X86::MOV32rm);
2489     Load.addOperand(TmpReg);
2490     Load.addOperand(BaseReg);
2491     Load.addOperand(Scale);
2492     Load.addOperand(IndexReg);
2493     Load.addOperand(Offset);
2494     Load.addOperand(MCOperand::createReg(X86::NoRegister));
2495 
2496     Insts.push_back(Load);
2497     Insts.push_back(IJmp);
2498     return true;
2499   }
2500 
2501   bool createNoop(MCInst &Inst) const override {
2502     Inst.setOpcode(X86::NOOP);
2503     return true;
2504   }
2505 
2506   bool createReturn(MCInst &Inst) const override {
2507     Inst.setOpcode(X86::RET64);
2508     return true;
2509   }
2510 
2511   InstructionListType createInlineMemcpy(bool ReturnEnd) const override {
2512     InstructionListType Code;
2513     if (ReturnEnd)
2514       Code.emplace_back(MCInstBuilder(X86::LEA64r)
2515                             .addReg(X86::RAX)
2516                             .addReg(X86::RDI)
2517                             .addImm(1)
2518                             .addReg(X86::RDX)
2519                             .addImm(0)
2520                             .addReg(X86::NoRegister));
2521     else
2522       Code.emplace_back(MCInstBuilder(X86::MOV64rr)
2523                             .addReg(X86::RAX)
2524                             .addReg(X86::RDI));
2525 
2526     Code.emplace_back(MCInstBuilder(X86::MOV32rr)
2527                           .addReg(X86::ECX)
2528                           .addReg(X86::EDX));
2529     Code.emplace_back(MCInstBuilder(X86::REP_MOVSB_64));
2530 
2531     return Code;
2532   }
2533 
2534   InstructionListType createOneByteMemcpy() const override {
2535     InstructionListType Code;
2536     Code.emplace_back(MCInstBuilder(X86::MOV8rm)
2537                           .addReg(X86::CL)
2538                           .addReg(X86::RSI)
2539                           .addImm(0)
2540                           .addReg(X86::NoRegister)
2541                           .addImm(0)
2542                           .addReg(X86::NoRegister));
2543     Code.emplace_back(MCInstBuilder(X86::MOV8mr)
2544                           .addReg(X86::RDI)
2545                           .addImm(0)
2546                           .addReg(X86::NoRegister)
2547                           .addImm(0)
2548                           .addReg(X86::NoRegister)
2549                           .addReg(X86::CL));
2550     Code.emplace_back(MCInstBuilder(X86::MOV64rr)
2551                           .addReg(X86::RAX)
2552                           .addReg(X86::RDI));
2553     return Code;
2554   }
2555 
2556   InstructionListType createCmpJE(MCPhysReg RegNo, int64_t Imm,
2557                                   const MCSymbol *Target,
2558                                   MCContext *Ctx) const override {
2559     InstructionListType Code;
2560     Code.emplace_back(MCInstBuilder(X86::CMP64ri8)
2561                           .addReg(RegNo)
2562                           .addImm(Imm));
2563     Code.emplace_back(MCInstBuilder(X86::JCC_1)
2564                           .addExpr(MCSymbolRefExpr::create(
2565                               Target, MCSymbolRefExpr::VK_None, *Ctx))
2566                           .addImm(X86::COND_E));
2567     return Code;
2568   }
2569 
2570   Optional<Relocation>
2571   createRelocation(const MCFixup &Fixup,
2572                    const MCAsmBackend &MAB) const override {
2573     const MCFixupKindInfo &FKI = MAB.getFixupKindInfo(Fixup.getKind());
2574 
2575     assert(FKI.TargetOffset == 0 && "0-bit relocation offset expected");
2576     const uint64_t RelOffset = Fixup.getOffset();
2577 
2578     uint64_t RelType;
2579     if (FKI.Flags & MCFixupKindInfo::FKF_IsPCRel) {
2580       switch (FKI.TargetSize) {
2581       default:
2582         return NoneType();
2583       case  8: RelType = ELF::R_X86_64_PC8; break;
2584       case 16: RelType = ELF::R_X86_64_PC16; break;
2585       case 32: RelType = ELF::R_X86_64_PC32; break;
2586       case 64: RelType = ELF::R_X86_64_PC64; break;
2587       }
2588     } else {
2589       switch (FKI.TargetSize) {
2590       default:
2591         return NoneType();
2592       case  8: RelType = ELF::R_X86_64_8; break;
2593       case 16: RelType = ELF::R_X86_64_16; break;
2594       case 32: RelType = ELF::R_X86_64_32; break;
2595       case 64: RelType = ELF::R_X86_64_64; break;
2596       }
2597     }
2598 
2599     // Extract a symbol and an addend out of the fixup value expression.
2600     //
2601     // Only the following limited expression types are supported:
2602     //   Symbol + Addend
2603     //   Symbol
2604     uint64_t Addend = 0;
2605     MCSymbol *Symbol = nullptr;
2606     const MCExpr *ValueExpr = Fixup.getValue();
2607     if (ValueExpr->getKind() == MCExpr::Binary) {
2608       const auto *BinaryExpr = cast<MCBinaryExpr>(ValueExpr);
2609       assert(BinaryExpr->getOpcode() == MCBinaryExpr::Add &&
2610              "unexpected binary expression");
2611       const MCExpr *LHS = BinaryExpr->getLHS();
2612       assert(LHS->getKind() == MCExpr::SymbolRef && "unexpected LHS");
2613       Symbol = const_cast<MCSymbol *>(this->getTargetSymbol(LHS));
2614       const MCExpr *RHS = BinaryExpr->getRHS();
2615       assert(RHS->getKind() == MCExpr::Constant && "unexpected RHS");
2616       Addend = cast<MCConstantExpr>(RHS)->getValue();
2617     } else {
2618       assert(ValueExpr->getKind() == MCExpr::SymbolRef && "unexpected value");
2619       Symbol = const_cast<MCSymbol *>(this->getTargetSymbol(ValueExpr));
2620     }
2621 
2622     return Relocation({RelOffset, Symbol, RelType, Addend, 0});
2623   }
2624 
2625   bool replaceImmWithSymbolRef(MCInst &Inst, const MCSymbol *Symbol,
2626                                int64_t Addend, MCContext *Ctx, int64_t &Value,
2627                                uint64_t RelType) const override {
2628     unsigned ImmOpNo = -1U;
2629 
2630     for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst);
2631          ++Index) {
2632       if (Inst.getOperand(Index).isImm()) {
2633         ImmOpNo = Index;
2634         // TODO: this is a bit hacky.  It finds the correct operand by
2635         // searching for a specific immediate value.  If no value is
2636         // provided it defaults to the last immediate operand found.
2637         // This could lead to unexpected results if the instruction
2638         // has more than one immediate with the same value.
2639         if (Inst.getOperand(ImmOpNo).getImm() == Value)
2640           break;
2641       }
2642     }
2643 
2644     if (ImmOpNo == -1U)
2645       return false;
2646 
2647     Value = Inst.getOperand(ImmOpNo).getImm();
2648 
2649     setOperandToSymbolRef(Inst, ImmOpNo, Symbol, Addend, Ctx, RelType);
2650 
2651     return true;
2652   }
2653 
2654   bool replaceRegWithImm(MCInst &Inst, unsigned Register,
2655                          int64_t Imm) const override {
2656 
2657     enum CheckSignExt : uint8_t {
2658       NOCHECK = 0,
2659       CHECK8,
2660       CHECK32,
2661     };
2662 
2663     using CheckList = std::vector<std::pair<CheckSignExt, unsigned>>;
2664     struct InstInfo {
2665       // Size in bytes that Inst loads from memory.
2666       uint8_t DataSize;
2667 
2668       // True when the target operand has to be duplicated because the opcode
2669       // expects a LHS operand.
2670       bool HasLHS;
2671 
2672       // List of checks and corresponding opcodes to be used. We try to use the
2673       // smallest possible immediate value when various sizes are available,
2674       // hence we may need to check whether a larger constant fits in a smaller
2675       // immediate.
2676       CheckList Checks;
2677     };
2678 
2679     InstInfo I;
2680 
2681     switch (Inst.getOpcode()) {
2682     default: {
2683       switch (getPushSize(Inst)) {
2684 
2685       case 2: I = {2, false, {{CHECK8, X86::PUSH16i8}, {NOCHECK, X86::PUSHi16}}}; break;
2686       case 4: I = {4, false, {{CHECK8, X86::PUSH32i8}, {NOCHECK, X86::PUSHi32}}}; break;
2687       case 8: I = {8, false, {{CHECK8, X86::PUSH64i8},
2688                               {CHECK32, X86::PUSH64i32},
2689                               {NOCHECK, Inst.getOpcode()}}}; break;
2690       default: return false;
2691       }
2692       break;
2693     }
2694 
2695     // MOV
2696     case X86::MOV8rr:       I = {1, false, {{NOCHECK, X86::MOV8ri}}}; break;
2697     case X86::MOV16rr:      I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break;
2698     case X86::MOV32rr:      I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break;
2699     case X86::MOV64rr:      I = {8, false, {{CHECK32, X86::MOV64ri32},
2700                                             {NOCHECK, X86::MOV64ri}}}; break;
2701 
2702     case X86::MOV8mr:       I = {1, false, {{NOCHECK, X86::MOV8mi}}}; break;
2703     case X86::MOV16mr:      I = {2, false, {{NOCHECK, X86::MOV16mi}}}; break;
2704     case X86::MOV32mr:      I = {4, false, {{NOCHECK, X86::MOV32mi}}}; break;
2705     case X86::MOV64mr:      I = {8, false, {{CHECK32, X86::MOV64mi32},
2706                                             {NOCHECK, X86::MOV64mr}}}; break;
2707 
2708     // MOVZX
2709     case X86::MOVZX16rr8:   I = {1, false, {{NOCHECK, X86::MOV16ri}}}; break;
2710     case X86::MOVZX32rr8:   I = {1, false, {{NOCHECK, X86::MOV32ri}}}; break;
2711     case X86::MOVZX32rr16:  I = {2, false, {{NOCHECK, X86::MOV32ri}}}; break;
2712 
2713     // CMP
2714     case X86::CMP8rr:       I = {1, false, {{NOCHECK, X86::CMP8ri}}}; break;
2715     case X86::CMP16rr:      I = {2, false, {{CHECK8, X86::CMP16ri8},
2716                                             {NOCHECK, X86::CMP16ri}}}; break;
2717     case X86::CMP32rr:      I = {4, false, {{CHECK8, X86::CMP32ri8},
2718                                             {NOCHECK, X86::CMP32ri}}}; break;
2719     case X86::CMP64rr:      I = {8, false, {{CHECK8, X86::CMP64ri8},
2720                                             {CHECK32, X86::CMP64ri32},
2721                                             {NOCHECK, X86::CMP64rr}}}; break;
2722 
2723     // TEST
2724     case X86::TEST8rr:      I = {1, false, {{NOCHECK, X86::TEST8ri}}}; break;
2725     case X86::TEST16rr:     I = {2, false, {{NOCHECK, X86::TEST16ri}}}; break;
2726     case X86::TEST32rr:     I = {4, false, {{NOCHECK, X86::TEST32ri}}}; break;
2727     case X86::TEST64rr:     I = {8, false, {{CHECK32, X86::TEST64ri32},
2728                                             {NOCHECK, X86::TEST64rr}}}; break;
2729 
2730     // ADD
2731     case X86::ADD8rr:       I = {1, true, {{NOCHECK, X86::ADD8ri}}}; break;
2732     case X86::ADD16rr:      I = {2, true, {{CHECK8, X86::ADD16ri8},
2733                                            {NOCHECK, X86::ADD16ri}}}; break;
2734     case X86::ADD32rr:      I = {4, true, {{CHECK8, X86::ADD32ri8},
2735                                            {NOCHECK, X86::ADD32ri}}}; break;
2736     case X86::ADD64rr:      I = {8, true, {{CHECK8, X86::ADD64ri8},
2737                                            {CHECK32, X86::ADD64ri32},
2738                                            {NOCHECK, X86::ADD64rr}}}; break;
2739 
2740     // SUB
2741     case X86::SUB8rr:       I = {1, true, {{NOCHECK, X86::SUB8ri}}}; break;
2742     case X86::SUB16rr:      I = {2, true, {{CHECK8, X86::SUB16ri8},
2743                                            {NOCHECK, X86::SUB16ri}}}; break;
2744     case X86::SUB32rr:      I = {4, true, {{CHECK8, X86::SUB32ri8},
2745                                            {NOCHECK, X86::SUB32ri}}}; break;
2746     case X86::SUB64rr:      I = {8, true, {{CHECK8, X86::SUB64ri8},
2747                                            {CHECK32, X86::SUB64ri32},
2748                                            {NOCHECK, X86::SUB64rr}}}; break;
2749 
2750     // AND
2751     case X86::AND8rr:       I = {1, true, {{NOCHECK, X86::AND8ri}}}; break;
2752     case X86::AND16rr:      I = {2, true, {{CHECK8, X86::AND16ri8},
2753                                            {NOCHECK, X86::AND16ri}}}; break;
2754     case X86::AND32rr:      I = {4, true, {{CHECK8, X86::AND32ri8},
2755                                            {NOCHECK, X86::AND32ri}}}; break;
2756     case X86::AND64rr:      I = {8, true, {{CHECK8, X86::AND64ri8},
2757                                            {CHECK32, X86::AND64ri32},
2758                                            {NOCHECK, X86::AND64rr}}}; break;
2759 
2760     // OR
2761     case X86::OR8rr:        I = {1, true, {{NOCHECK, X86::OR8ri}}}; break;
2762     case X86::OR16rr:       I = {2, true, {{CHECK8, X86::OR16ri8},
2763                                            {NOCHECK, X86::OR16ri}}}; break;
2764     case X86::OR32rr:       I = {4, true, {{CHECK8, X86::OR32ri8},
2765                                            {NOCHECK, X86::OR32ri}}}; break;
2766     case X86::OR64rr:       I = {8, true, {{CHECK8, X86::OR64ri8},
2767                                            {CHECK32, X86::OR64ri32},
2768                                            {NOCHECK, X86::OR64rr}}}; break;
2769 
2770     // XOR
2771     case X86::XOR8rr:       I = {1, true, {{NOCHECK, X86::XOR8ri}}}; break;
2772     case X86::XOR16rr:      I = {2, true, {{CHECK8, X86::XOR16ri8},
2773                                            {NOCHECK, X86::XOR16ri}}}; break;
2774     case X86::XOR32rr:      I = {4, true, {{CHECK8, X86::XOR32ri8},
2775                                            {NOCHECK, X86::XOR32ri}}}; break;
2776     case X86::XOR64rr:      I = {8, true, {{CHECK8, X86::XOR64ri8},
2777                                            {CHECK32, X86::XOR64ri32},
2778                                            {NOCHECK, X86::XOR64rr}}}; break;
2779     }
2780 
2781     // Compute the new opcode.
2782     unsigned NewOpcode = 0;
2783     for (const std::pair<CheckSignExt, unsigned> &Check : I.Checks) {
2784       NewOpcode = Check.second;
2785       if (Check.first == NOCHECK)
2786         break;
2787       if (Check.first == CHECK8 && isInt<8>(Imm))
2788         break;
2789       if (Check.first == CHECK32 && isInt<32>(Imm))
2790         break;
2791     }
2792     if (NewOpcode == Inst.getOpcode())
2793       return false;
2794 
2795     const MCInstrDesc &InstDesc = Info->get(Inst.getOpcode());
2796 
2797     unsigned NumFound = 0;
2798     for (unsigned Index = InstDesc.getNumDefs() + (I.HasLHS ? 1 : 0),
2799                   E = InstDesc.getNumOperands();
2800          Index != E; ++Index)
2801       if (Inst.getOperand(Index).isReg() &&
2802           Inst.getOperand(Index).getReg() == Register)
2803         NumFound++;
2804 
2805     if (NumFound != 1)
2806       return false;
2807 
2808     MCOperand TargetOp = Inst.getOperand(0);
2809     Inst.clear();
2810     Inst.setOpcode(NewOpcode);
2811     Inst.addOperand(TargetOp);
2812     if (I.HasLHS)
2813       Inst.addOperand(TargetOp);
2814     Inst.addOperand(MCOperand::createImm(Imm));
2815 
2816     return true;
2817   }
2818 
2819   bool replaceRegWithReg(MCInst &Inst, unsigned ToReplace,
2820                          unsigned ReplaceWith) const override {
2821 
2822     // Get the HasLHS value so that iteration can be done
2823     bool HasLHS;
2824     if (X86::isAND(Inst.getOpcode()) || X86::isADD(Inst.getOpcode()) ||
2825         X86::isSUB(Inst.getOpcode())) {
2826       HasLHS = true;
2827     } else if (isPop(Inst) || isPush(Inst) || X86::isCMP(Inst.getOpcode()) ||
2828                X86::isTEST(Inst.getOpcode())) {
2829       HasLHS = false;
2830     } else {
2831       switch (Inst.getOpcode()) {
2832       case X86::MOV8rr:
2833       case X86::MOV8rm:
2834       case X86::MOV8mr:
2835       case X86::MOV8ri:
2836       case X86::MOV16rr:
2837       case X86::MOV16rm:
2838       case X86::MOV16mr:
2839       case X86::MOV16ri:
2840       case X86::MOV32rr:
2841       case X86::MOV32rm:
2842       case X86::MOV32mr:
2843       case X86::MOV32ri:
2844       case X86::MOV64rr:
2845       case X86::MOV64rm:
2846       case X86::MOV64mr:
2847       case X86::MOV64ri:
2848       case X86::MOVZX16rr8:
2849       case X86::MOVZX32rr8:
2850       case X86::MOVZX32rr16:
2851       case X86::MOVSX32rm8:
2852       case X86::MOVSX32rr8:
2853       case X86::MOVSX64rm32:
2854       case X86::LEA64r:
2855         HasLHS = false;
2856         break;
2857       default:
2858         return false;
2859       }
2860     }
2861 
2862     const MCInstrDesc &InstDesc = Info->get(Inst.getOpcode());
2863 
2864     bool FoundOne = false;
2865 
2866     // Iterate only through src operands that arent also dest operands
2867     for (unsigned Index = InstDesc.getNumDefs() + (HasLHS ? 1 : 0),
2868                   E = InstDesc.getNumOperands();
2869          Index != E; ++Index) {
2870       BitVector RegAliases = getAliases(ToReplace, true);
2871       if (!Inst.getOperand(Index).isReg() ||
2872           !RegAliases.test(Inst.getOperand(Index).getReg()))
2873         continue;
2874       // Resize register if needed
2875       unsigned SizedReplaceWith = getAliasSized(
2876           ReplaceWith, getRegSize(Inst.getOperand(Index).getReg()));
2877       MCOperand NewOperand = MCOperand::createReg(SizedReplaceWith);
2878       Inst.getOperand(Index) = NewOperand;
2879       FoundOne = true;
2880     }
2881 
2882     // Return true if at least one operand was replaced
2883     return FoundOne;
2884   }
2885 
2886   bool createUncondBranch(MCInst &Inst, const MCSymbol *TBB,
2887                           MCContext *Ctx) const override {
2888     Inst.setOpcode(X86::JMP_1);
2889     Inst.addOperand(MCOperand::createExpr(
2890         MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)));
2891     return true;
2892   }
2893 
2894   bool createCall(MCInst &Inst, const MCSymbol *Target,
2895                   MCContext *Ctx) override {
2896     Inst.setOpcode(X86::CALL64pcrel32);
2897     Inst.addOperand(MCOperand::createExpr(
2898         MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx)));
2899     return true;
2900   }
2901 
2902   bool createTailCall(MCInst &Inst, const MCSymbol *Target,
2903                       MCContext *Ctx) override {
2904     return createDirectCall(Inst, Target, Ctx, /*IsTailCall*/ true);
2905   }
2906 
2907   void createLongTailCall(InstructionListType &Seq, const MCSymbol *Target,
2908                           MCContext *Ctx) override {
2909     Seq.clear();
2910     Seq.emplace_back();
2911     createDirectCall(Seq.back(), Target, Ctx, /*IsTailCall*/ true);
2912   }
2913 
2914   bool createTrap(MCInst &Inst) const override {
2915     Inst.clear();
2916     Inst.setOpcode(X86::TRAP);
2917     return true;
2918   }
2919 
2920   bool reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB,
2921                               MCContext *Ctx) const override {
2922     unsigned InvCC = getInvertedCondCode(getCondCode(Inst));
2923     assert(InvCC != X86::COND_INVALID && "invalid branch instruction");
2924     Inst.getOperand(Info->get(Inst.getOpcode()).NumOperands - 1).setImm(InvCC);
2925     Inst.getOperand(0) = MCOperand::createExpr(
2926         MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx));
2927     return true;
2928   }
2929 
2930   bool replaceBranchCondition(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx,
2931                               unsigned CC) const override {
2932     if (CC == X86::COND_INVALID)
2933       return false;
2934     Inst.getOperand(Info->get(Inst.getOpcode()).NumOperands - 1).setImm(CC);
2935     Inst.getOperand(0) = MCOperand::createExpr(
2936         MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx));
2937     return true;
2938   }
2939 
2940   unsigned getCanonicalBranchCondCode(unsigned CC) const override {
2941     switch (CC) {
2942     default:           return X86::COND_INVALID;
2943 
2944     case X86::COND_E:  return X86::COND_E;
2945     case X86::COND_NE: return X86::COND_E;
2946 
2947     case X86::COND_L:  return X86::COND_L;
2948     case X86::COND_GE: return X86::COND_L;
2949 
2950     case X86::COND_LE: return X86::COND_G;
2951     case X86::COND_G:  return X86::COND_G;
2952 
2953     case X86::COND_B:  return X86::COND_B;
2954     case X86::COND_AE: return X86::COND_B;
2955 
2956     case X86::COND_BE: return X86::COND_A;
2957     case X86::COND_A:  return X86::COND_A;
2958 
2959     case X86::COND_S:  return X86::COND_S;
2960     case X86::COND_NS: return X86::COND_S;
2961 
2962     case X86::COND_P:  return X86::COND_P;
2963     case X86::COND_NP: return X86::COND_P;
2964 
2965     case X86::COND_O:  return X86::COND_O;
2966     case X86::COND_NO: return X86::COND_O;
2967     }
2968   }
2969 
2970   bool replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB,
2971                            MCContext *Ctx) const override {
2972     assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) &&
2973            "Invalid instruction");
2974     Inst.getOperand(0) = MCOperand::createExpr(
2975         MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx));
2976     return true;
2977   }
2978 
2979   MCPhysReg getX86R11() const override { return X86::R11; }
2980 
2981   MCPhysReg getIntArgRegister(unsigned ArgNo) const override {
2982     // FIXME: this should depend on the calling convention.
2983     switch (ArgNo) {
2984     case 0:   return X86::RDI;
2985     case 1:   return X86::RSI;
2986     case 2:   return X86::RDX;
2987     case 3:   return X86::RCX;
2988     case 4:   return X86::R8;
2989     case 5:   return X86::R9;
2990     default:  return getNoRegister();
2991     }
2992   }
2993 
2994   void createPause(MCInst &Inst) const override {
2995     Inst.clear();
2996     Inst.setOpcode(X86::PAUSE);
2997   }
2998 
2999   void createLfence(MCInst &Inst) const override {
3000     Inst.clear();
3001     Inst.setOpcode(X86::LFENCE);
3002   }
3003 
3004   bool createDirectCall(MCInst &Inst, const MCSymbol *Target, MCContext *Ctx,
3005                         bool IsTailCall) override {
3006     Inst.clear();
3007     Inst.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32);
3008     Inst.addOperand(MCOperand::createExpr(
3009         MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx)));
3010     if (IsTailCall)
3011       setTailCall(Inst);
3012     return true;
3013   }
3014 
3015   void createShortJmp(InstructionListType &Seq, const MCSymbol *Target,
3016                       MCContext *Ctx, bool IsTailCall) override {
3017     Seq.clear();
3018     MCInst Inst;
3019     Inst.setOpcode(X86::JMP_1);
3020     Inst.addOperand(MCOperand::createExpr(
3021         MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx)));
3022     if (IsTailCall)
3023       setTailCall(Inst);
3024     Seq.emplace_back(Inst);
3025   }
3026 
3027   bool isConditionalMove(const MCInst &Inst) const override {
3028     unsigned OpCode = Inst.getOpcode();
3029     return (OpCode == X86::CMOV16rr || OpCode == X86::CMOV32rr ||
3030             OpCode == X86::CMOV64rr);
3031   }
3032 
3033   bool isBranchOnMem(const MCInst &Inst) const override {
3034     unsigned OpCode = Inst.getOpcode();
3035     if (OpCode == X86::CALL64m || (OpCode == X86::JMP32m && isTailCall(Inst)) ||
3036         OpCode == X86::JMP64m)
3037       return true;
3038 
3039     return false;
3040   }
3041 
3042   bool isBranchOnReg(const MCInst &Inst) const override {
3043     unsigned OpCode = Inst.getOpcode();
3044     if (OpCode == X86::CALL64r || (OpCode == X86::JMP32r && isTailCall(Inst)) ||
3045         OpCode == X86::JMP64r)
3046       return true;
3047 
3048     return false;
3049   }
3050 
3051   void createPushRegister(MCInst &Inst, MCPhysReg Reg,
3052                           unsigned Size) const override {
3053     Inst.clear();
3054     unsigned NewOpcode = 0;
3055     if (Reg == X86::EFLAGS) {
3056       switch (Size) {
3057       case 2: NewOpcode = X86::PUSHF16;  break;
3058       case 4: NewOpcode = X86::PUSHF32;  break;
3059       case 8: NewOpcode = X86::PUSHF64;  break;
3060       default:
3061         llvm_unreachable("Unexpected size");
3062       }
3063       Inst.setOpcode(NewOpcode);
3064       return;
3065     }
3066     switch (Size) {
3067     case 2: NewOpcode = X86::PUSH16r;  break;
3068     case 4: NewOpcode = X86::PUSH32r;  break;
3069     case 8: NewOpcode = X86::PUSH64r;  break;
3070     default:
3071       llvm_unreachable("Unexpected size");
3072     }
3073     Inst.setOpcode(NewOpcode);
3074     Inst.addOperand(MCOperand::createReg(Reg));
3075   }
3076 
3077   void createPopRegister(MCInst &Inst, MCPhysReg Reg,
3078                          unsigned Size) const override {
3079     Inst.clear();
3080     unsigned NewOpcode = 0;
3081     if (Reg == X86::EFLAGS) {
3082       switch (Size) {
3083       case 2: NewOpcode = X86::POPF16;  break;
3084       case 4: NewOpcode = X86::POPF32;  break;
3085       case 8: NewOpcode = X86::POPF64;  break;
3086       default:
3087         llvm_unreachable("Unexpected size");
3088       }
3089       Inst.setOpcode(NewOpcode);
3090       return;
3091     }
3092     switch (Size) {
3093     case 2: NewOpcode = X86::POP16r;  break;
3094     case 4: NewOpcode = X86::POP32r;  break;
3095     case 8: NewOpcode = X86::POP64r;  break;
3096     default:
3097       llvm_unreachable("Unexpected size");
3098     }
3099     Inst.setOpcode(NewOpcode);
3100     Inst.addOperand(MCOperand::createReg(Reg));
3101   }
3102 
3103   void createPushFlags(MCInst &Inst, unsigned Size) const override {
3104     return createPushRegister(Inst, X86::EFLAGS, Size);
3105   }
3106 
3107   void createPopFlags(MCInst &Inst, unsigned Size) const override {
3108     return createPopRegister(Inst, X86::EFLAGS, Size);
3109   }
3110 
3111   void createAddRegImm(MCInst &Inst, MCPhysReg Reg, int64_t Value,
3112                        unsigned Size) const {
3113     unsigned int Opcode;
3114     switch (Size) {
3115     case 1: Opcode = X86::ADD8ri; break;
3116     case 2: Opcode = X86::ADD16ri; break;
3117     case 4: Opcode = X86::ADD32ri; break;
3118     default:
3119       llvm_unreachable("Unexpected size");
3120     }
3121     Inst.setOpcode(Opcode);
3122     Inst.clear();
3123     Inst.addOperand(MCOperand::createReg(Reg));
3124     Inst.addOperand(MCOperand::createReg(Reg));
3125     Inst.addOperand(MCOperand::createImm(Value));
3126   }
3127 
3128   void createClearRegWithNoEFlagsUpdate(MCInst &Inst, MCPhysReg Reg,
3129                                         unsigned Size) const {
3130     unsigned int Opcode;
3131     switch (Size) {
3132     case 1: Opcode = X86::MOV8ri; break;
3133     case 2: Opcode = X86::MOV16ri; break;
3134     case 4: Opcode = X86::MOV32ri; break;
3135     case 8: Opcode = X86::MOV64ri; break;
3136     default:
3137       llvm_unreachable("Unexpected size");
3138     }
3139     Inst.setOpcode(Opcode);
3140     Inst.clear();
3141     Inst.addOperand(MCOperand::createReg(Reg));
3142     Inst.addOperand(MCOperand::createImm(0));
3143   }
3144 
3145   void createX86SaveOVFlagToRegister(MCInst &Inst, MCPhysReg Reg) const {
3146     Inst.setOpcode(X86::SETCCr);
3147     Inst.clear();
3148     Inst.addOperand(MCOperand::createReg(Reg));
3149     Inst.addOperand(MCOperand::createImm(X86::COND_O));
3150   }
3151 
3152   void createX86Lahf(MCInst &Inst) const {
3153     Inst.setOpcode(X86::LAHF);
3154     Inst.clear();
3155   }
3156 
3157   void createX86Sahf(MCInst &Inst) const {
3158     Inst.setOpcode(X86::SAHF);
3159     Inst.clear();
3160   }
3161 
3162   void createInstrIncMemory(InstructionListType &Instrs, const MCSymbol *Target,
3163                             MCContext *Ctx, bool IsLeaf) const override {
3164     unsigned int I = 0;
3165 
3166     Instrs.resize(IsLeaf ? 13 : 11);
3167     // Don't clobber application red zone (ABI dependent)
3168     if (IsLeaf)
3169       createStackPointerIncrement(Instrs[I++], 128,
3170                                   /*NoFlagsClobber=*/true);
3171 
3172     // Performance improvements based on the optimization discussed at
3173     // https://reviews.llvm.org/D6629
3174     // LAHF/SAHF are used instead of PUSHF/POPF
3175     // PUSHF
3176     createPushRegister(Instrs[I++], X86::RAX, 8);
3177     createClearRegWithNoEFlagsUpdate(Instrs[I++], X86::RAX, 8);
3178     createX86Lahf(Instrs[I++]);
3179     createPushRegister(Instrs[I++], X86::RAX, 8);
3180     createClearRegWithNoEFlagsUpdate(Instrs[I++], X86::RAX, 8);
3181     createX86SaveOVFlagToRegister(Instrs[I++], X86::AL);
3182     // LOCK INC
3183     createIncMemory(Instrs[I++], Target, Ctx);
3184     // POPF
3185     createAddRegImm(Instrs[I++], X86::AL, 127, 1);
3186     createPopRegister(Instrs[I++], X86::RAX, 8);
3187     createX86Sahf(Instrs[I++]);
3188     createPopRegister(Instrs[I++], X86::RAX, 8);
3189 
3190     if (IsLeaf)
3191       createStackPointerDecrement(Instrs[I], 128,
3192                                   /*NoFlagsClobber=*/true);
3193   }
3194 
3195   void createSwap(MCInst &Inst, MCPhysReg Source, MCPhysReg MemBaseReg,
3196                   int64_t Disp) const {
3197     Inst.setOpcode(X86::XCHG64rm);
3198     Inst.addOperand(MCOperand::createReg(Source));
3199     Inst.addOperand(MCOperand::createReg(Source));
3200     Inst.addOperand(MCOperand::createReg(MemBaseReg));      // BaseReg
3201     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
3202     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
3203     Inst.addOperand(MCOperand::createImm(Disp));            // Displacement
3204     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
3205   }
3206 
3207   void createIndirectBranch(MCInst &Inst, MCPhysReg MemBaseReg,
3208                             int64_t Disp) const {
3209     Inst.setOpcode(X86::JMP64m);
3210     Inst.addOperand(MCOperand::createReg(MemBaseReg));      // BaseReg
3211     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
3212     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
3213     Inst.addOperand(MCOperand::createImm(Disp));            // Displacement
3214     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
3215   }
3216 
3217   InstructionListType createInstrumentedIndirectCall(const MCInst &CallInst,
3218                                                      bool TailCall,
3219                                                      MCSymbol *HandlerFuncAddr,
3220                                                      int CallSiteID,
3221                                                      MCContext *Ctx) override {
3222     // Check if the target address expression used in the original indirect call
3223     // uses the stack pointer, which we are going to clobber.
3224     static BitVector SPAliases(getAliases(X86::RSP));
3225     bool UsesSP = false;
3226     // Skip defs.
3227     for (unsigned I = Info->get(CallInst.getOpcode()).getNumDefs(),
3228                   E = MCPlus::getNumPrimeOperands(CallInst);
3229          I != E; ++I) {
3230       const MCOperand &Operand = CallInst.getOperand(I);
3231       if (Operand.isReg() && SPAliases[Operand.getReg()]) {
3232         UsesSP = true;
3233         break;
3234       }
3235     }
3236 
3237     InstructionListType Insts;
3238     MCPhysReg TempReg = getIntArgRegister(0);
3239     // Code sequence used to enter indirect call instrumentation helper:
3240     //   push %rdi
3241     //   add $8, %rsp       ;; $rsp may be used in target, so fix it to prev val
3242     //   movq target, %rdi  ;; via convertIndirectCallTargetToLoad
3243     //   sub $8, %rsp       ;; restore correct stack value
3244     //   push %rdi
3245     //   movq $CallSiteID, %rdi
3246     //   push %rdi
3247     //   callq/jmp HandlerFuncAddr
3248     Insts.emplace_back();
3249     createPushRegister(Insts.back(), TempReg, 8);
3250     if (UsesSP) { // Only adjust SP if we really need to
3251       Insts.emplace_back();
3252       createStackPointerDecrement(Insts.back(), 8, /*NoFlagsClobber=*/false);
3253     }
3254     Insts.emplace_back(CallInst);
3255     // Insts.back() and CallInst now share the same annotation instruction.
3256     // Strip it from Insts.back(), only preserving tail call annotation.
3257     stripAnnotations(Insts.back(), /*KeepTC=*/true);
3258     convertIndirectCallToLoad(Insts.back(), TempReg);
3259     if (UsesSP) {
3260       Insts.emplace_back();
3261       createStackPointerIncrement(Insts.back(), 8, /*NoFlagsClobber=*/false);
3262     }
3263     Insts.emplace_back();
3264     createPushRegister(Insts.back(), TempReg, 8);
3265     Insts.emplace_back();
3266     createLoadImmediate(Insts.back(), TempReg, CallSiteID);
3267     Insts.emplace_back();
3268     createPushRegister(Insts.back(), TempReg, 8);
3269     Insts.emplace_back();
3270     createDirectCall(Insts.back(), HandlerFuncAddr, Ctx,
3271                      /*TailCall=*/TailCall);
3272     // Carry over metadata
3273     for (int I = MCPlus::getNumPrimeOperands(CallInst),
3274              E = CallInst.getNumOperands();
3275          I != E; ++I)
3276       Insts.back().addOperand(CallInst.getOperand(I));
3277 
3278     return Insts;
3279   }
3280 
3281   InstructionListType createInstrumentedIndCallHandlerExitBB() const override {
3282     const MCPhysReg TempReg = getIntArgRegister(0);
3283     // We just need to undo the sequence created for every ind call in
3284     // instrumentIndirectTarget(), which can be accomplished minimally with:
3285     //   popfq
3286     //   pop %rdi
3287     //   add $16, %rsp
3288     //   xchg (%rsp), %rdi
3289     //   jmp *-8(%rsp)
3290     InstructionListType Insts(5);
3291     createPopFlags(Insts[0], 8);
3292     createPopRegister(Insts[1], TempReg, 8);
3293     createStackPointerDecrement(Insts[2], 16, /*NoFlagsClobber=*/false);
3294     createSwap(Insts[3], TempReg, X86::RSP, 0);
3295     createIndirectBranch(Insts[4], X86::RSP, -8);
3296     return Insts;
3297   }
3298 
3299   InstructionListType
3300   createInstrumentedIndTailCallHandlerExitBB() const override {
3301     const MCPhysReg TempReg = getIntArgRegister(0);
3302     // Same thing as above, but for tail calls
3303     //   popfq
3304     //   add $16, %rsp
3305     //   pop %rdi
3306     //   jmp *-16(%rsp)
3307     InstructionListType Insts(4);
3308     createPopFlags(Insts[0], 8);
3309     createStackPointerDecrement(Insts[1], 16, /*NoFlagsClobber=*/false);
3310     createPopRegister(Insts[2], TempReg, 8);
3311     createIndirectBranch(Insts[3], X86::RSP, -16);
3312     return Insts;
3313   }
3314 
3315   InstructionListType
3316   createInstrumentedIndCallHandlerEntryBB(const MCSymbol *InstrTrampoline,
3317                                           const MCSymbol *IndCallHandler,
3318                                           MCContext *Ctx) override {
3319     const MCPhysReg TempReg = getIntArgRegister(0);
3320     // Code sequence used to check whether InstrTampoline was initialized
3321     // and call it if so, returns via IndCallHandler.
3322     //   pushfq
3323     //   mov    InstrTrampoline,%rdi
3324     //   cmp    $0x0,%rdi
3325     //   je     IndCallHandler
3326     //   callq  *%rdi
3327     //   jmpq   IndCallHandler
3328     InstructionListType Insts;
3329     Insts.emplace_back();
3330     createPushFlags(Insts.back(), 8);
3331     Insts.emplace_back();
3332     createMove(Insts.back(), InstrTrampoline, TempReg, Ctx);
3333     InstructionListType cmpJmp = createCmpJE(TempReg, 0, IndCallHandler, Ctx);
3334     Insts.insert(Insts.end(), cmpJmp.begin(), cmpJmp.end());
3335     Insts.emplace_back();
3336     Insts.back().setOpcode(X86::CALL64r);
3337     Insts.back().addOperand(MCOperand::createReg(TempReg));
3338     Insts.emplace_back();
3339     createDirectCall(Insts.back(), IndCallHandler, Ctx, /*IsTailCall*/ true);
3340     return Insts;
3341   }
3342 
3343   InstructionListType createNumCountersGetter(MCContext *Ctx) const override {
3344     InstructionListType Insts(2);
3345     MCSymbol *NumLocs = Ctx->getOrCreateSymbol("__bolt_num_counters");
3346     createMove(Insts[0], NumLocs, X86::EAX, Ctx);
3347     createReturn(Insts[1]);
3348     return Insts;
3349   }
3350 
3351   InstructionListType
3352   createInstrLocationsGetter(MCContext *Ctx) const override {
3353     InstructionListType Insts(2);
3354     MCSymbol *Locs = Ctx->getOrCreateSymbol("__bolt_instr_locations");
3355     createLea(Insts[0], Locs, X86::EAX, Ctx);
3356     createReturn(Insts[1]);
3357     return Insts;
3358   }
3359 
3360   InstructionListType createInstrTablesGetter(MCContext *Ctx) const override {
3361     InstructionListType Insts(2);
3362     MCSymbol *Locs = Ctx->getOrCreateSymbol("__bolt_instr_tables");
3363     createLea(Insts[0], Locs, X86::EAX, Ctx);
3364     createReturn(Insts[1]);
3365     return Insts;
3366   }
3367 
3368   InstructionListType createInstrNumFuncsGetter(MCContext *Ctx) const override {
3369     InstructionListType Insts(2);
3370     MCSymbol *NumFuncs = Ctx->getOrCreateSymbol("__bolt_instr_num_funcs");
3371     createMove(Insts[0], NumFuncs, X86::EAX, Ctx);
3372     createReturn(Insts[1]);
3373     return Insts;
3374   }
3375 
3376   InstructionListType createSymbolTrampoline(const MCSymbol *TgtSym,
3377                                              MCContext *Ctx) const override {
3378     InstructionListType Insts(1);
3379     createUncondBranch(Insts[0], TgtSym, Ctx);
3380     return Insts;
3381   }
3382 
3383   InstructionListType createDummyReturnFunction(MCContext *Ctx) const override {
3384     InstructionListType Insts(1);
3385     createReturn(Insts[0]);
3386     return Insts;
3387   }
3388 
3389   BlocksVectorTy indirectCallPromotion(
3390       const MCInst &CallInst,
3391       const std::vector<std::pair<MCSymbol *, uint64_t>> &Targets,
3392       const std::vector<std::pair<MCSymbol *, uint64_t>> &VtableSyms,
3393       const std::vector<MCInst *> &MethodFetchInsns,
3394       const bool MinimizeCodeSize, MCContext *Ctx) override {
3395     const bool IsTailCall = isTailCall(CallInst);
3396     const bool IsJumpTable = getJumpTable(CallInst) != 0;
3397     BlocksVectorTy Results;
3398 
3399     // Label for the current code block.
3400     MCSymbol *NextTarget = nullptr;
3401 
3402     // The join block which contains all the instructions following CallInst.
3403     // MergeBlock remains null if CallInst is a tail call.
3404     MCSymbol *MergeBlock = nullptr;
3405 
3406     unsigned FuncAddrReg = X86::R10;
3407 
3408     const bool LoadElim = !VtableSyms.empty();
3409     assert((!LoadElim || VtableSyms.size() == Targets.size()) &&
3410            "There must be a vtable entry for every method "
3411            "in the targets vector.");
3412 
3413     if (MinimizeCodeSize && !LoadElim) {
3414       std::set<unsigned> UsedRegs;
3415 
3416       for (unsigned int I = 0; I < MCPlus::getNumPrimeOperands(CallInst); ++I) {
3417         const MCOperand &Op = CallInst.getOperand(I);
3418         if (Op.isReg())
3419           UsedRegs.insert(Op.getReg());
3420       }
3421 
3422       if (UsedRegs.count(X86::R10) == 0)
3423         FuncAddrReg = X86::R10;
3424       else if (UsedRegs.count(X86::R11) == 0)
3425         FuncAddrReg = X86::R11;
3426       else
3427         return Results;
3428     }
3429 
3430     const auto jumpToMergeBlock = [&](InstructionListType &NewCall) {
3431       assert(MergeBlock);
3432       NewCall.push_back(CallInst);
3433       MCInst &Merge = NewCall.back();
3434       Merge.clear();
3435       createUncondBranch(Merge, MergeBlock, Ctx);
3436     };
3437 
3438     for (unsigned int i = 0; i < Targets.size(); ++i) {
3439       Results.emplace_back(NextTarget, InstructionListType());
3440       InstructionListType *NewCall = &Results.back().second;
3441 
3442       if (MinimizeCodeSize && !LoadElim) {
3443         // Load the call target into FuncAddrReg.
3444         NewCall->push_back(CallInst); // Copy CallInst in order to get SMLoc
3445         MCInst &Target = NewCall->back();
3446         Target.clear();
3447         Target.setOpcode(X86::MOV64ri32);
3448         Target.addOperand(MCOperand::createReg(FuncAddrReg));
3449         if (Targets[i].first) {
3450           // Is this OK?
3451           Target.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
3452               Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
3453         } else {
3454           const uint64_t Addr = Targets[i].second;
3455           // Immediate address is out of sign extended 32 bit range.
3456           if (int64_t(Addr) != int64_t(int32_t(Addr)))
3457             return BlocksVectorTy();
3458 
3459           Target.addOperand(MCOperand::createImm(Addr));
3460         }
3461 
3462         // Compare current call target to a specific address.
3463         NewCall->push_back(CallInst);
3464         MCInst &Compare = NewCall->back();
3465         Compare.clear();
3466         if (isBranchOnReg(CallInst))
3467           Compare.setOpcode(X86::CMP64rr);
3468         else if (CallInst.getOpcode() == X86::CALL64pcrel32)
3469           Compare.setOpcode(X86::CMP64ri32);
3470         else
3471           Compare.setOpcode(X86::CMP64rm);
3472 
3473         Compare.addOperand(MCOperand::createReg(FuncAddrReg));
3474 
3475         // TODO: Would be preferable to only load this value once.
3476         for (unsigned i = 0;
3477              i < Info->get(CallInst.getOpcode()).getNumOperands(); ++i)
3478           if (!CallInst.getOperand(i).isInst())
3479             Compare.addOperand(CallInst.getOperand(i));
3480       } else {
3481         // Compare current call target to a specific address.
3482         NewCall->push_back(CallInst);
3483         MCInst &Compare = NewCall->back();
3484         Compare.clear();
3485         if (isBranchOnReg(CallInst))
3486           Compare.setOpcode(X86::CMP64ri32);
3487         else
3488           Compare.setOpcode(X86::CMP64mi32);
3489 
3490         // Original call address.
3491         for (unsigned i = 0;
3492              i < Info->get(CallInst.getOpcode()).getNumOperands(); ++i)
3493           if (!CallInst.getOperand(i).isInst())
3494             Compare.addOperand(CallInst.getOperand(i));
3495 
3496         // Target address.
3497         if (Targets[i].first || LoadElim) {
3498           const MCSymbol *Sym =
3499               LoadElim ? VtableSyms[i].first : Targets[i].first;
3500           const uint64_t Addend = LoadElim ? VtableSyms[i].second : 0;
3501           const MCExpr *Expr = MCSymbolRefExpr::create(Sym, *Ctx);
3502           if (Addend)
3503             Expr = MCBinaryExpr::createAdd(
3504                 Expr, MCConstantExpr::create(Addend, *Ctx), *Ctx);
3505           Compare.addOperand(MCOperand::createExpr(Expr));
3506         } else {
3507           const uint64_t Addr = Targets[i].second;
3508           // Immediate address is out of sign extended 32 bit range.
3509           if (int64_t(Addr) != int64_t(int32_t(Addr)))
3510             return BlocksVectorTy();
3511 
3512           Compare.addOperand(MCOperand::createImm(Addr));
3513         }
3514       }
3515 
3516       // jump to next target compare.
3517       NextTarget =
3518           Ctx->createNamedTempSymbol(); // generate label for the next block
3519       NewCall->push_back(CallInst);
3520 
3521       if (IsJumpTable) {
3522         MCInst &Je = NewCall->back();
3523 
3524         // Jump to next compare if target addresses don't match.
3525         Je.clear();
3526         Je.setOpcode(X86::JCC_1);
3527         if (Targets[i].first)
3528           Je.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
3529               Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
3530         else
3531           Je.addOperand(MCOperand::createImm(Targets[i].second));
3532 
3533         Je.addOperand(MCOperand::createImm(X86::COND_E));
3534         assert(!isInvoke(CallInst));
3535       } else {
3536         MCInst &Jne = NewCall->back();
3537 
3538         // Jump to next compare if target addresses don't match.
3539         Jne.clear();
3540         Jne.setOpcode(X86::JCC_1);
3541         Jne.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
3542             NextTarget, MCSymbolRefExpr::VK_None, *Ctx)));
3543         Jne.addOperand(MCOperand::createImm(X86::COND_NE));
3544 
3545         // Call specific target directly.
3546         Results.emplace_back(Ctx->createNamedTempSymbol(),
3547                              InstructionListType());
3548         NewCall = &Results.back().second;
3549         NewCall->push_back(CallInst);
3550         MCInst &CallOrJmp = NewCall->back();
3551 
3552         CallOrJmp.clear();
3553 
3554         if (MinimizeCodeSize && !LoadElim) {
3555           CallOrJmp.setOpcode(IsTailCall ? X86::JMP32r : X86::CALL64r);
3556           CallOrJmp.addOperand(MCOperand::createReg(FuncAddrReg));
3557         } else {
3558           CallOrJmp.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32);
3559 
3560           if (Targets[i].first)
3561             CallOrJmp.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
3562                 Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
3563           else
3564             CallOrJmp.addOperand(MCOperand::createImm(Targets[i].second));
3565         }
3566         if (IsTailCall)
3567           setTailCall(CallOrJmp);
3568 
3569         if (CallOrJmp.getOpcode() == X86::CALL64r ||
3570             CallOrJmp.getOpcode() == X86::CALL64pcrel32) {
3571           if (Optional<uint32_t> Offset = getOffset(CallInst))
3572             // Annotated as duplicated call
3573             setOffset(CallOrJmp, *Offset);
3574         }
3575 
3576         if (isInvoke(CallInst) && !isInvoke(CallOrJmp)) {
3577           // Copy over any EH or GNU args size information from the original
3578           // call.
3579           Optional<MCPlus::MCLandingPad> EHInfo = getEHInfo(CallInst);
3580           if (EHInfo)
3581             addEHInfo(CallOrJmp, *EHInfo);
3582           int64_t GnuArgsSize = getGnuArgsSize(CallInst);
3583           if (GnuArgsSize >= 0)
3584             addGnuArgsSize(CallOrJmp, GnuArgsSize);
3585         }
3586 
3587         if (!IsTailCall) {
3588           // The fallthrough block for the most common target should be
3589           // the merge block.
3590           if (i == 0) {
3591             // Fallthrough to merge block.
3592             MergeBlock = Ctx->createNamedTempSymbol();
3593           } else {
3594             // Insert jump to the merge block if we are not doing a fallthrough.
3595             jumpToMergeBlock(*NewCall);
3596           }
3597         }
3598       }
3599     }
3600 
3601     // Cold call block.
3602     Results.emplace_back(NextTarget, InstructionListType());
3603     InstructionListType &NewCall = Results.back().second;
3604     for (const MCInst *Inst : MethodFetchInsns)
3605       if (Inst != &CallInst)
3606         NewCall.push_back(*Inst);
3607     NewCall.push_back(CallInst);
3608 
3609     // Jump to merge block from cold call block
3610     if (!IsTailCall && !IsJumpTable) {
3611       jumpToMergeBlock(NewCall);
3612 
3613       // Record merge block
3614       Results.emplace_back(MergeBlock, InstructionListType());
3615     }
3616 
3617     return Results;
3618   }
3619 
3620   BlocksVectorTy jumpTablePromotion(
3621       const MCInst &IJmpInst,
3622       const std::vector<std::pair<MCSymbol *, uint64_t>> &Targets,
3623       const std::vector<MCInst *> &TargetFetchInsns,
3624       MCContext *Ctx) const override {
3625     assert(getJumpTable(IJmpInst) != 0);
3626     uint16_t IndexReg = getAnnotationAs<uint16_t>(IJmpInst, "JTIndexReg");
3627     if (IndexReg == 0)
3628       return BlocksVectorTy();
3629 
3630     BlocksVectorTy Results;
3631 
3632     // Label for the current code block.
3633     MCSymbol *NextTarget = nullptr;
3634 
3635     for (unsigned int i = 0; i < Targets.size(); ++i) {
3636       Results.emplace_back(NextTarget, InstructionListType());
3637       InstructionListType *CurBB = &Results.back().second;
3638 
3639       // Compare current index to a specific index.
3640       CurBB->emplace_back(MCInst());
3641       MCInst &CompareInst = CurBB->back();
3642       CompareInst.setLoc(IJmpInst.getLoc());
3643       CompareInst.setOpcode(X86::CMP64ri32);
3644       CompareInst.addOperand(MCOperand::createReg(IndexReg));
3645 
3646       const uint64_t CaseIdx = Targets[i].second;
3647       // Immediate address is out of sign extended 32 bit range.
3648       if (int64_t(CaseIdx) != int64_t(int32_t(CaseIdx)))
3649         return BlocksVectorTy();
3650 
3651       CompareInst.addOperand(MCOperand::createImm(CaseIdx));
3652       shortenInstruction(CompareInst, *Ctx->getSubtargetInfo());
3653 
3654       // jump to next target compare.
3655       NextTarget =
3656           Ctx->createNamedTempSymbol(); // generate label for the next block
3657       CurBB->push_back(MCInst());
3658 
3659       MCInst &JEInst = CurBB->back();
3660       JEInst.setLoc(IJmpInst.getLoc());
3661 
3662       // Jump to target if indices match
3663       JEInst.setOpcode(X86::JCC_1);
3664       JEInst.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
3665           Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
3666       JEInst.addOperand(MCOperand::createImm(X86::COND_E));
3667     }
3668 
3669     // Cold call block.
3670     Results.emplace_back(NextTarget, InstructionListType());
3671     InstructionListType &CurBB = Results.back().second;
3672     for (const MCInst *Inst : TargetFetchInsns)
3673       if (Inst != &IJmpInst)
3674         CurBB.push_back(*Inst);
3675 
3676     CurBB.push_back(IJmpInst);
3677 
3678     return Results;
3679   }
3680 
3681 private:
3682   bool createMove(MCInst &Inst, const MCSymbol *Src, unsigned Reg,
3683                   MCContext *Ctx) const {
3684     Inst.setOpcode(X86::MOV64rm);
3685     Inst.addOperand(MCOperand::createReg(Reg));
3686     Inst.addOperand(MCOperand::createReg(X86::RIP));        // BaseReg
3687     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
3688     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
3689     Inst.addOperand(MCOperand::createExpr(
3690         MCSymbolRefExpr::create(Src, MCSymbolRefExpr::VK_None,
3691                                 *Ctx)));                    // Displacement
3692     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
3693 
3694     return true;
3695   }
3696 
3697   bool createLea(MCInst &Inst, const MCSymbol *Src, unsigned Reg,
3698                  MCContext *Ctx) const {
3699     Inst.setOpcode(X86::LEA64r);
3700     Inst.addOperand(MCOperand::createReg(Reg));
3701     Inst.addOperand(MCOperand::createReg(X86::RIP));        // BaseReg
3702     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
3703     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
3704     Inst.addOperand(MCOperand::createExpr(
3705         MCSymbolRefExpr::create(Src, MCSymbolRefExpr::VK_None,
3706                                 *Ctx)));                    // Displacement
3707     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
3708     return true;
3709   }
3710 };
3711 
3712 } // namespace
3713 
3714 namespace llvm {
3715 namespace bolt {
3716 
3717 MCPlusBuilder *createX86MCPlusBuilder(const MCInstrAnalysis *Analysis,
3718                                       const MCInstrInfo *Info,
3719                                       const MCRegisterInfo *RegInfo) {
3720   return new X86MCPlusBuilder(Analysis, Info, RegInfo);
3721 }
3722 
3723 } // namespace bolt
3724 } // namespace llvm
3725