1 //===- bolt/Target/X86/X86MCPlusBuilder.cpp -------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file provides X86-specific MCPlus builder.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "MCTargetDesc/X86BaseInfo.h"
14 #include "MCTargetDesc/X86InstrRelaxTables.h"
15 #include "MCTargetDesc/X86MCTargetDesc.h"
16 #include "bolt/Core/MCPlus.h"
17 #include "bolt/Core/MCPlusBuilder.h"
18 #include "llvm/BinaryFormat/ELF.h"
19 #include "llvm/MC/MCContext.h"
20 #include "llvm/MC/MCFixupKindInfo.h"
21 #include "llvm/MC/MCInst.h"
22 #include "llvm/MC/MCInstBuilder.h"
23 #include "llvm/MC/MCInstrInfo.h"
24 #include "llvm/MC/MCRegister.h"
25 #include "llvm/MC/MCRegisterInfo.h"
26 #include "llvm/Support/CommandLine.h"
27 #include "llvm/Support/DataExtractor.h"
28 #include "llvm/Support/Debug.h"
29 #include "llvm/Support/Errc.h"
30 #include "llvm/Support/ErrorHandling.h"
31 #include "llvm/Support/ErrorOr.h"
32 #include <set>
33 
34 #define DEBUG_TYPE "mcplus"
35 
36 using namespace llvm;
37 using namespace bolt;
38 
39 namespace opts {
40 
41 extern cl::OptionCategory BoltOptCategory;
42 
43 static cl::opt<bool> X86StripRedundantAddressSize(
44     "x86-strip-redundant-address-size",
45     cl::desc("Remove redundant Address-Size override prefix"), cl::init(true),
46     cl::ZeroOrMore, cl::cat(BoltOptCategory));
47 
48 } // namespace opts
49 
50 namespace {
51 
52 unsigned getShortBranchOpcode(unsigned Opcode) {
53   switch (Opcode) {
54   default:
55     return Opcode;
56   case X86::JMP_2: return X86::JMP_1;
57   case X86::JMP_4: return X86::JMP_1;
58   case X86::JCC_2: return X86::JCC_1;
59   case X86::JCC_4: return X86::JCC_1;
60   }
61 }
62 
63 unsigned getShortArithOpcode(unsigned Opcode) {
64   return X86::getShortOpcodeArith(Opcode);
65 }
66 
67 bool isMOVSX64rm32(const MCInst &Inst) {
68   return Inst.getOpcode() == X86::MOVSX64rm32;
69 }
70 
71 bool isADD64rr(const MCInst &Inst) { return Inst.getOpcode() == X86::ADD64rr; }
72 
73 bool isADDri(const MCInst &Inst) {
74   return Inst.getOpcode() == X86::ADD64ri32 ||
75          Inst.getOpcode() == X86::ADD64ri8;
76 }
77 
78 class X86MCPlusBuilder : public MCPlusBuilder {
79 public:
80   X86MCPlusBuilder(const MCInstrAnalysis *Analysis, const MCInstrInfo *Info,
81                    const MCRegisterInfo *RegInfo)
82       : MCPlusBuilder(Analysis, Info, RegInfo) {}
83 
84   bool isBranch(const MCInst &Inst) const override {
85     return Analysis->isBranch(Inst) && !isTailCall(Inst);
86   }
87 
88   bool isNoop(const MCInst &Inst) const override {
89     return X86::isNOP(Inst.getOpcode());
90   }
91 
92   unsigned getCondCode(const MCInst &Inst) const override {
93     unsigned Opcode = Inst.getOpcode();
94     if (X86::isJCC(Opcode))
95       return Inst.getOperand(Info->get(Opcode).NumOperands - 1).getImm();
96     return X86::COND_INVALID;
97   }
98 
99   unsigned getInvertedCondCode(unsigned CC) const override {
100     switch (CC) {
101     default: return X86::COND_INVALID;
102     case X86::COND_E:  return X86::COND_NE;
103     case X86::COND_NE: return X86::COND_E;
104     case X86::COND_L:  return X86::COND_GE;
105     case X86::COND_LE: return X86::COND_G;
106     case X86::COND_G:  return X86::COND_LE;
107     case X86::COND_GE: return X86::COND_L;
108     case X86::COND_B:  return X86::COND_AE;
109     case X86::COND_BE: return X86::COND_A;
110     case X86::COND_A:  return X86::COND_BE;
111     case X86::COND_AE: return X86::COND_B;
112     case X86::COND_S:  return X86::COND_NS;
113     case X86::COND_NS: return X86::COND_S;
114     case X86::COND_P:  return X86::COND_NP;
115     case X86::COND_NP: return X86::COND_P;
116     case X86::COND_O:  return X86::COND_NO;
117     case X86::COND_NO: return X86::COND_O;
118     }
119   }
120 
121   unsigned getCondCodesLogicalOr(unsigned CC1, unsigned CC2) const override {
122     enum DecodedCondCode : uint8_t {
123       DCC_EQUAL = 0x1,
124       DCC_GREATER = 0x2,
125       DCC_LESSER = 0x4,
126       DCC_GREATER_OR_LESSER = 0x6,
127       DCC_UNSIGNED = 0x8,
128       DCC_SIGNED = 0x10,
129       DCC_INVALID = 0x20,
130     };
131 
132     auto decodeCondCode = [&](unsigned CC) -> uint8_t {
133       switch (CC) {
134       default: return DCC_INVALID;
135       case X86::COND_E: return DCC_EQUAL;
136       case X86::COND_NE: return DCC_GREATER | DCC_LESSER;
137       case X86::COND_L: return DCC_LESSER | DCC_SIGNED;
138       case X86::COND_LE: return DCC_EQUAL | DCC_LESSER | DCC_SIGNED;
139       case X86::COND_G: return DCC_GREATER | DCC_SIGNED;
140       case X86::COND_GE: return DCC_GREATER | DCC_EQUAL | DCC_SIGNED;
141       case X86::COND_B: return DCC_LESSER | DCC_UNSIGNED;
142       case X86::COND_BE: return DCC_EQUAL | DCC_LESSER | DCC_UNSIGNED;
143       case X86::COND_A: return DCC_GREATER | DCC_UNSIGNED;
144       case X86::COND_AE: return DCC_GREATER | DCC_EQUAL | DCC_UNSIGNED;
145       }
146     };
147 
148     uint8_t DCC = decodeCondCode(CC1) | decodeCondCode(CC2);
149 
150     if (DCC & DCC_INVALID)
151       return X86::COND_INVALID;
152 
153     if (DCC & DCC_SIGNED && DCC & DCC_UNSIGNED)
154       return X86::COND_INVALID;
155 
156     switch (DCC) {
157     default: return X86::COND_INVALID;
158     case DCC_EQUAL | DCC_LESSER | DCC_SIGNED: return X86::COND_LE;
159     case DCC_EQUAL | DCC_LESSER | DCC_UNSIGNED: return X86::COND_BE;
160     case DCC_EQUAL | DCC_GREATER | DCC_SIGNED: return X86::COND_GE;
161     case DCC_EQUAL | DCC_GREATER | DCC_UNSIGNED: return X86::COND_AE;
162     case DCC_GREATER | DCC_LESSER | DCC_SIGNED: return X86::COND_NE;
163     case DCC_GREATER | DCC_LESSER | DCC_UNSIGNED: return X86::COND_NE;
164     case DCC_GREATER | DCC_LESSER: return X86::COND_NE;
165     case DCC_EQUAL | DCC_SIGNED: return X86::COND_E;
166     case DCC_EQUAL | DCC_UNSIGNED: return X86::COND_E;
167     case DCC_EQUAL: return X86::COND_E;
168     case DCC_LESSER | DCC_SIGNED: return X86::COND_L;
169     case DCC_LESSER | DCC_UNSIGNED: return X86::COND_B;
170     case DCC_GREATER | DCC_SIGNED: return X86::COND_G;
171     case DCC_GREATER | DCC_UNSIGNED: return X86::COND_A;
172     }
173   }
174 
175   bool isValidCondCode(unsigned CC) const override {
176     return (CC != X86::COND_INVALID);
177   }
178 
179   bool isBreakpoint(const MCInst &Inst) const override {
180     return Inst.getOpcode() == X86::INT3;
181   }
182 
183   bool isPrefix(const MCInst &Inst) const override {
184     const MCInstrDesc &Desc = Info->get(Inst.getOpcode());
185     return X86II::isPrefix(Desc.TSFlags);
186   }
187 
188   bool isRep(const MCInst &Inst) const override {
189     return Inst.getFlags() == X86::IP_HAS_REPEAT;
190   }
191 
192   bool deleteREPPrefix(MCInst &Inst) const override {
193     if (Inst.getFlags() == X86::IP_HAS_REPEAT) {
194       Inst.setFlags(0);
195       return true;
196     }
197     return false;
198   }
199 
200   // FIXME: For compatibility with old LLVM only!
201   bool isTerminator(const MCInst &Inst) const override {
202     unsigned Opcode = Inst.getOpcode();
203     return Info->get(Opcode).isTerminator() || X86::isUD1(Opcode) ||
204            X86::isUD2(Opcode);
205   }
206 
207   bool isIndirectCall(const MCInst &Inst) const override {
208     return isCall(Inst) &&
209            ((getMemoryOperandNo(Inst) != -1) || Inst.getOperand(0).isReg());
210   }
211 
212   bool isPop(const MCInst &Inst) const override {
213     return getPopSize(Inst) == 0 ? false : true;
214   }
215 
216   bool isTerminateBranch(const MCInst &Inst) const override {
217     return Inst.getOpcode() == X86::ENDBR32 || Inst.getOpcode() == X86::ENDBR64;
218   }
219 
220   int getPopSize(const MCInst &Inst) const override {
221     switch (Inst.getOpcode()) {
222     case X86::POP16r:
223     case X86::POP16rmm:
224     case X86::POP16rmr:
225     case X86::POPF16:
226     case X86::POPA16:
227     case X86::POPDS16:
228     case X86::POPES16:
229     case X86::POPFS16:
230     case X86::POPGS16:
231     case X86::POPSS16:
232       return 2;
233     case X86::POP32r:
234     case X86::POP32rmm:
235     case X86::POP32rmr:
236     case X86::POPA32:
237     case X86::POPDS32:
238     case X86::POPES32:
239     case X86::POPF32:
240     case X86::POPFS32:
241     case X86::POPGS32:
242     case X86::POPSS32:
243       return 4;
244     case X86::POP64r:
245     case X86::POP64rmm:
246     case X86::POP64rmr:
247     case X86::POPF64:
248     case X86::POPFS64:
249     case X86::POPGS64:
250       return 8;
251     }
252     return 0;
253   }
254 
255   bool isPush(const MCInst &Inst) const override {
256     return getPushSize(Inst) == 0 ? false : true;
257   }
258 
259   int getPushSize(const MCInst &Inst) const override {
260     switch (Inst.getOpcode()) {
261     case X86::PUSH16i8:
262     case X86::PUSH16r:
263     case X86::PUSH16rmm:
264     case X86::PUSH16rmr:
265     case X86::PUSHA16:
266     case X86::PUSHCS16:
267     case X86::PUSHDS16:
268     case X86::PUSHES16:
269     case X86::PUSHF16:
270     case X86::PUSHFS16:
271     case X86::PUSHGS16:
272     case X86::PUSHSS16:
273     case X86::PUSHi16:
274       return 2;
275     case X86::PUSH32i8:
276     case X86::PUSH32r:
277     case X86::PUSH32rmm:
278     case X86::PUSH32rmr:
279     case X86::PUSHA32:
280     case X86::PUSHCS32:
281     case X86::PUSHDS32:
282     case X86::PUSHES32:
283     case X86::PUSHF32:
284     case X86::PUSHFS32:
285     case X86::PUSHGS32:
286     case X86::PUSHSS32:
287     case X86::PUSHi32:
288       return 4;
289     case X86::PUSH64i32:
290     case X86::PUSH64i8:
291     case X86::PUSH64r:
292     case X86::PUSH64rmm:
293     case X86::PUSH64rmr:
294     case X86::PUSHF64:
295     case X86::PUSHFS64:
296     case X86::PUSHGS64:
297       return 8;
298     }
299     return 0;
300   }
301 
302   bool isSUB(const MCInst &Inst) const override {
303     return X86::isSUB(Inst.getOpcode());
304   }
305 
306   bool isLEA64r(const MCInst &Inst) const override {
307     return Inst.getOpcode() == X86::LEA64r;
308   }
309 
310   bool isLeave(const MCInst &Inst) const override {
311     return Inst.getOpcode() == X86::LEAVE || Inst.getOpcode() == X86::LEAVE64;
312   }
313 
314   bool isMoveMem2Reg(const MCInst &Inst) const override {
315     switch (Inst.getOpcode()) {
316     case X86::MOV16rm:
317     case X86::MOV32rm:
318     case X86::MOV64rm:
319       return true;
320     }
321     return false;
322   }
323 
324   bool isUnsupportedBranch(unsigned Opcode) const override {
325     switch (Opcode) {
326     default:
327       return false;
328     case X86::LOOP:
329     case X86::LOOPE:
330     case X86::LOOPNE:
331     case X86::JECXZ:
332     case X86::JRCXZ:
333       return true;
334     }
335   }
336 
337   bool isLoad(const MCInst &Inst) const override {
338     if (isPop(Inst))
339       return true;
340 
341     int MemOpNo = getMemoryOperandNo(Inst);
342     const MCInstrDesc &MCII = Info->get(Inst.getOpcode());
343 
344     if (MemOpNo == -1)
345       return false;
346 
347     return MCII.mayLoad();
348   }
349 
350   bool isStore(const MCInst &Inst) const override {
351     if (isPush(Inst))
352       return true;
353 
354     int MemOpNo = getMemoryOperandNo(Inst);
355     const MCInstrDesc &MCII = Info->get(Inst.getOpcode());
356 
357     if (MemOpNo == -1)
358       return false;
359 
360     return MCII.mayStore();
361   }
362 
363   bool isCleanRegXOR(const MCInst &Inst) const override {
364     switch (Inst.getOpcode()) {
365     case X86::XOR16rr:
366     case X86::XOR32rr:
367     case X86::XOR64rr:
368       break;
369     default:
370       return false;
371     }
372     return (Inst.getOperand(0).getReg() == Inst.getOperand(2).getReg());
373   }
374 
375   bool isPacked(const MCInst &Inst) const override {
376     const MCInstrDesc &Desc = Info->get(Inst.getOpcode());
377     return (Desc.TSFlags & X86II::OpPrefixMask) == X86II::PD;
378   }
379 
380   unsigned getTrapFillValue() const override { return 0xCC; }
381 
382   struct IndJmpMatcherFrag1 : MCInstMatcher {
383     std::unique_ptr<MCInstMatcher> Base;
384     std::unique_ptr<MCInstMatcher> Scale;
385     std::unique_ptr<MCInstMatcher> Index;
386     std::unique_ptr<MCInstMatcher> Offset;
387 
388     IndJmpMatcherFrag1(std::unique_ptr<MCInstMatcher> Base,
389                        std::unique_ptr<MCInstMatcher> Scale,
390                        std::unique_ptr<MCInstMatcher> Index,
391                        std::unique_ptr<MCInstMatcher> Offset)
392         : Base(std::move(Base)), Scale(std::move(Scale)),
393           Index(std::move(Index)), Offset(std::move(Offset)) {}
394 
395     bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
396                MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
397       if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
398         return false;
399 
400       if (CurInst->getOpcode() != X86::JMP64m)
401         return false;
402 
403       int MemOpNo = MIB.getMemoryOperandNo(*CurInst);
404       if (MemOpNo == -1)
405         return false;
406 
407       if (!Base->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrBaseReg))
408         return false;
409       if (!Scale->match(MRI, MIB, this->InstrWindow,
410                         MemOpNo + X86::AddrScaleAmt))
411         return false;
412       if (!Index->match(MRI, MIB, this->InstrWindow,
413                         MemOpNo + X86::AddrIndexReg))
414         return false;
415       if (!Offset->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrDisp))
416         return false;
417       return true;
418     }
419 
420     void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
421       MIB.addAnnotation(*CurInst, Annotation, true);
422       Base->annotate(MIB, Annotation);
423       Scale->annotate(MIB, Annotation);
424       Index->annotate(MIB, Annotation);
425       Offset->annotate(MIB, Annotation);
426     }
427   };
428 
429   std::unique_ptr<MCInstMatcher>
430   matchIndJmp(std::unique_ptr<MCInstMatcher> Base,
431               std::unique_ptr<MCInstMatcher> Scale,
432               std::unique_ptr<MCInstMatcher> Index,
433               std::unique_ptr<MCInstMatcher> Offset) const override {
434     return std::unique_ptr<MCInstMatcher>(
435         new IndJmpMatcherFrag1(std::move(Base), std::move(Scale),
436                                std::move(Index), std::move(Offset)));
437   }
438 
439   struct IndJmpMatcherFrag2 : MCInstMatcher {
440     std::unique_ptr<MCInstMatcher> Reg;
441 
442     IndJmpMatcherFrag2(std::unique_ptr<MCInstMatcher> Reg)
443         : Reg(std::move(Reg)) {}
444 
445     bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
446                MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
447       if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
448         return false;
449 
450       if (CurInst->getOpcode() != X86::JMP64r)
451         return false;
452 
453       return Reg->match(MRI, MIB, this->InstrWindow, 0);
454     }
455 
456     void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
457       MIB.addAnnotation(*CurInst, Annotation, true);
458       Reg->annotate(MIB, Annotation);
459     }
460   };
461 
462   std::unique_ptr<MCInstMatcher>
463   matchIndJmp(std::unique_ptr<MCInstMatcher> Target) const override {
464     return std::unique_ptr<MCInstMatcher>(
465         new IndJmpMatcherFrag2(std::move(Target)));
466   }
467 
468   struct LoadMatcherFrag1 : MCInstMatcher {
469     std::unique_ptr<MCInstMatcher> Base;
470     std::unique_ptr<MCInstMatcher> Scale;
471     std::unique_ptr<MCInstMatcher> Index;
472     std::unique_ptr<MCInstMatcher> Offset;
473 
474     LoadMatcherFrag1(std::unique_ptr<MCInstMatcher> Base,
475                      std::unique_ptr<MCInstMatcher> Scale,
476                      std::unique_ptr<MCInstMatcher> Index,
477                      std::unique_ptr<MCInstMatcher> Offset)
478         : Base(std::move(Base)), Scale(std::move(Scale)),
479           Index(std::move(Index)), Offset(std::move(Offset)) {}
480 
481     bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
482                MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
483       if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
484         return false;
485 
486       if (CurInst->getOpcode() != X86::MOV64rm &&
487           CurInst->getOpcode() != X86::MOVSX64rm32)
488         return false;
489 
490       int MemOpNo = MIB.getMemoryOperandNo(*CurInst);
491       if (MemOpNo == -1)
492         return false;
493 
494       if (!Base->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrBaseReg))
495         return false;
496       if (!Scale->match(MRI, MIB, this->InstrWindow,
497                         MemOpNo + X86::AddrScaleAmt))
498         return false;
499       if (!Index->match(MRI, MIB, this->InstrWindow,
500                         MemOpNo + X86::AddrIndexReg))
501         return false;
502       if (!Offset->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrDisp))
503         return false;
504       return true;
505     }
506 
507     void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
508       MIB.addAnnotation(*CurInst, Annotation, true);
509       Base->annotate(MIB, Annotation);
510       Scale->annotate(MIB, Annotation);
511       Index->annotate(MIB, Annotation);
512       Offset->annotate(MIB, Annotation);
513     }
514   };
515 
516   std::unique_ptr<MCInstMatcher>
517   matchLoad(std::unique_ptr<MCInstMatcher> Base,
518             std::unique_ptr<MCInstMatcher> Scale,
519             std::unique_ptr<MCInstMatcher> Index,
520             std::unique_ptr<MCInstMatcher> Offset) const override {
521     return std::unique_ptr<MCInstMatcher>(
522         new LoadMatcherFrag1(std::move(Base), std::move(Scale),
523                              std::move(Index), std::move(Offset)));
524   }
525 
526   struct AddMatcher : MCInstMatcher {
527     std::unique_ptr<MCInstMatcher> A;
528     std::unique_ptr<MCInstMatcher> B;
529 
530     AddMatcher(std::unique_ptr<MCInstMatcher> A,
531                std::unique_ptr<MCInstMatcher> B)
532         : A(std::move(A)), B(std::move(B)) {}
533 
534     bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
535                MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
536       if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
537         return false;
538 
539       if (CurInst->getOpcode() == X86::ADD64rr ||
540           CurInst->getOpcode() == X86::ADD64rr_DB ||
541           CurInst->getOpcode() == X86::ADD64rr_REV) {
542         if (!A->match(MRI, MIB, this->InstrWindow, 1)) {
543           if (!B->match(MRI, MIB, this->InstrWindow, 1))
544             return false;
545           return A->match(MRI, MIB, this->InstrWindow, 2);
546         }
547 
548         if (B->match(MRI, MIB, this->InstrWindow, 2))
549           return true;
550 
551         if (!B->match(MRI, MIB, this->InstrWindow, 1))
552           return false;
553         return A->match(MRI, MIB, this->InstrWindow, 2);
554       }
555 
556       return false;
557     }
558 
559     void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
560       MIB.addAnnotation(*CurInst, Annotation, true);
561       A->annotate(MIB, Annotation);
562       B->annotate(MIB, Annotation);
563     }
564   };
565 
566   virtual std::unique_ptr<MCInstMatcher>
567   matchAdd(std::unique_ptr<MCInstMatcher> A,
568            std::unique_ptr<MCInstMatcher> B) const override {
569     return std::unique_ptr<MCInstMatcher>(
570         new AddMatcher(std::move(A), std::move(B)));
571   }
572 
573   struct LEAMatcher : MCInstMatcher {
574     std::unique_ptr<MCInstMatcher> Target;
575 
576     LEAMatcher(std::unique_ptr<MCInstMatcher> Target)
577         : Target(std::move(Target)) {}
578 
579     bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
580                MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
581       if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
582         return false;
583 
584       if (CurInst->getOpcode() != X86::LEA64r)
585         return false;
586 
587       if (CurInst->getOperand(1 + X86::AddrScaleAmt).getImm() != 1 ||
588           CurInst->getOperand(1 + X86::AddrIndexReg).getReg() !=
589               X86::NoRegister ||
590           (CurInst->getOperand(1 + X86::AddrBaseReg).getReg() !=
591                X86::NoRegister &&
592            CurInst->getOperand(1 + X86::AddrBaseReg).getReg() != X86::RIP))
593         return false;
594 
595       return Target->match(MRI, MIB, this->InstrWindow, 1 + X86::AddrDisp);
596     }
597 
598     void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
599       MIB.addAnnotation(*CurInst, Annotation, true);
600       Target->annotate(MIB, Annotation);
601     }
602   };
603 
604   virtual std::unique_ptr<MCInstMatcher>
605   matchLoadAddr(std::unique_ptr<MCInstMatcher> Target) const override {
606     return std::unique_ptr<MCInstMatcher>(new LEAMatcher(std::move(Target)));
607   }
608 
609   bool hasPCRelOperand(const MCInst &Inst) const override {
610     for (const MCOperand &Operand : Inst)
611       if (Operand.isReg() && Operand.getReg() == X86::RIP)
612         return true;
613     return false;
614   }
615 
616   int getMemoryOperandNo(const MCInst &Inst) const override {
617     unsigned Opcode = Inst.getOpcode();
618     const MCInstrDesc &Desc = Info->get(Opcode);
619     int MemOpNo = X86II::getMemoryOperandNo(Desc.TSFlags);
620     if (MemOpNo >= 0)
621       MemOpNo += X86II::getOperandBias(Desc);
622     return MemOpNo;
623   }
624 
625   bool hasEVEXEncoding(const MCInst &Inst) const override {
626     const MCInstrDesc &Desc = Info->get(Inst.getOpcode());
627     return (Desc.TSFlags & X86II::EncodingMask) == X86II::EVEX;
628   }
629 
630   bool isMacroOpFusionPair(ArrayRef<MCInst> Insts) const override {
631     const auto *I = Insts.begin();
632     while (I != Insts.end() && isPrefix(*I))
633       ++I;
634     if (I == Insts.end())
635       return false;
636 
637     const MCInst &FirstInst = *I;
638     ++I;
639     while (I != Insts.end() && isPrefix(*I))
640       ++I;
641     if (I == Insts.end())
642       return false;
643     const MCInst &SecondInst = *I;
644 
645     if (!isConditionalBranch(SecondInst))
646       return false;
647     // Cannot fuse if the first instruction uses RIP-relative memory.
648     if (hasPCRelOperand(FirstInst))
649       return false;
650 
651     const X86::FirstMacroFusionInstKind CmpKind =
652         X86::classifyFirstOpcodeInMacroFusion(FirstInst.getOpcode());
653     if (CmpKind == X86::FirstMacroFusionInstKind::Invalid)
654       return false;
655 
656     X86::CondCode CC = static_cast<X86::CondCode>(getCondCode(SecondInst));
657     X86::SecondMacroFusionInstKind BranchKind =
658         X86::classifySecondCondCodeInMacroFusion(CC);
659     if (BranchKind == X86::SecondMacroFusionInstKind::Invalid)
660       return false;
661     return X86::isMacroFused(CmpKind, BranchKind);
662   }
663 
664   bool
665   evaluateX86MemoryOperand(const MCInst &Inst, unsigned *BaseRegNum,
666                            int64_t *ScaleImm, unsigned *IndexRegNum,
667                            int64_t *DispImm, unsigned *SegmentRegNum,
668                            const MCExpr **DispExpr = nullptr) const override {
669     assert(BaseRegNum && ScaleImm && IndexRegNum && SegmentRegNum &&
670            "one of the input pointers is null");
671     int MemOpNo = getMemoryOperandNo(Inst);
672     if (MemOpNo < 0)
673       return false;
674     unsigned MemOpOffset = static_cast<unsigned>(MemOpNo);
675 
676     if (MemOpOffset + X86::AddrSegmentReg >= MCPlus::getNumPrimeOperands(Inst))
677       return false;
678 
679     const MCOperand &Base = Inst.getOperand(MemOpOffset + X86::AddrBaseReg);
680     const MCOperand &Scale = Inst.getOperand(MemOpOffset + X86::AddrScaleAmt);
681     const MCOperand &Index = Inst.getOperand(MemOpOffset + X86::AddrIndexReg);
682     const MCOperand &Disp = Inst.getOperand(MemOpOffset + X86::AddrDisp);
683     const MCOperand &Segment =
684         Inst.getOperand(MemOpOffset + X86::AddrSegmentReg);
685 
686     // Make sure it is a well-formed memory operand.
687     if (!Base.isReg() || !Scale.isImm() || !Index.isReg() ||
688         (!Disp.isImm() && !Disp.isExpr()) || !Segment.isReg())
689       return false;
690 
691     *BaseRegNum = Base.getReg();
692     *ScaleImm = Scale.getImm();
693     *IndexRegNum = Index.getReg();
694     if (Disp.isImm()) {
695       assert(DispImm && "DispImm needs to be set");
696       *DispImm = Disp.getImm();
697       if (DispExpr)
698         *DispExpr = nullptr;
699     } else {
700       assert(DispExpr && "DispExpr needs to be set");
701       *DispExpr = Disp.getExpr();
702       if (DispImm)
703         *DispImm = 0;
704     }
705     *SegmentRegNum = Segment.getReg();
706     return true;
707   }
708 
709   bool evaluateMemOperandTarget(const MCInst &Inst, uint64_t &Target,
710                                 uint64_t Address,
711                                 uint64_t Size) const override {
712     unsigned      BaseRegNum;
713     int64_t       ScaleValue;
714     unsigned      IndexRegNum;
715     int64_t       DispValue;
716     unsigned      SegRegNum;
717     const MCExpr *DispExpr = nullptr;
718     if (!evaluateX86MemoryOperand(Inst, &BaseRegNum, &ScaleValue, &IndexRegNum,
719                                   &DispValue, &SegRegNum, &DispExpr))
720       return false;
721 
722     // Make sure it's a well-formed addressing we can statically evaluate.
723     if ((BaseRegNum != X86::RIP && BaseRegNum != X86::NoRegister) ||
724         IndexRegNum != X86::NoRegister || SegRegNum != X86::NoRegister ||
725         DispExpr)
726       return false;
727 
728     Target = DispValue;
729     if (BaseRegNum == X86::RIP) {
730       assert(Size != 0 && "instruction size required in order to statically "
731                           "evaluate RIP-relative address");
732       Target += Address + Size;
733     }
734     return true;
735   }
736 
737   MCInst::iterator getMemOperandDisp(MCInst &Inst) const override {
738     int MemOpNo = getMemoryOperandNo(Inst);
739     if (MemOpNo < 0)
740       return Inst.end();
741     return Inst.begin() + (MemOpNo + X86::AddrDisp);
742   }
743 
744   bool replaceMemOperandDisp(MCInst &Inst, MCOperand Operand) const override {
745     MCOperand *OI = getMemOperandDisp(Inst);
746     if (OI == Inst.end())
747       return false;
748     *OI = Operand;
749     return true;
750   }
751 
752   /// Get the registers used as function parameters.
753   /// This function is specific to the x86_64 abi on Linux.
754   BitVector getRegsUsedAsParams() const override {
755     BitVector Regs = BitVector(RegInfo->getNumRegs(), false);
756     Regs |= getAliases(X86::RSI);
757     Regs |= getAliases(X86::RDI);
758     Regs |= getAliases(X86::RDX);
759     Regs |= getAliases(X86::RCX);
760     Regs |= getAliases(X86::R8);
761     Regs |= getAliases(X86::R9);
762     return Regs;
763   }
764 
765   void getCalleeSavedRegs(BitVector &Regs) const override {
766     Regs |= getAliases(X86::RBX);
767     Regs |= getAliases(X86::RBP);
768     Regs |= getAliases(X86::R12);
769     Regs |= getAliases(X86::R13);
770     Regs |= getAliases(X86::R14);
771     Regs |= getAliases(X86::R15);
772   }
773 
774   void getDefaultDefIn(BitVector &Regs) const override {
775     assert(Regs.size() >= RegInfo->getNumRegs() &&
776            "The size of BitVector is less than RegInfo->getNumRegs().");
777     Regs.set(X86::RAX);
778     Regs.set(X86::RCX);
779     Regs.set(X86::RDX);
780     Regs.set(X86::RSI);
781     Regs.set(X86::RDI);
782     Regs.set(X86::R8);
783     Regs.set(X86::R9);
784     Regs.set(X86::XMM0);
785     Regs.set(X86::XMM1);
786     Regs.set(X86::XMM2);
787     Regs.set(X86::XMM3);
788     Regs.set(X86::XMM4);
789     Regs.set(X86::XMM5);
790     Regs.set(X86::XMM6);
791     Regs.set(X86::XMM7);
792   }
793 
794   void getDefaultLiveOut(BitVector &Regs) const override {
795     assert(Regs.size() >= RegInfo->getNumRegs() &&
796            "The size of BitVector is less than RegInfo->getNumRegs().");
797     Regs |= getAliases(X86::RAX);
798     Regs |= getAliases(X86::RDX);
799     Regs |= getAliases(X86::RCX);
800     Regs |= getAliases(X86::XMM0);
801     Regs |= getAliases(X86::XMM1);
802   }
803 
804   void getGPRegs(BitVector &Regs, bool IncludeAlias) const override {
805     if (IncludeAlias) {
806       Regs |= getAliases(X86::RAX);
807       Regs |= getAliases(X86::RBX);
808       Regs |= getAliases(X86::RBP);
809       Regs |= getAliases(X86::RSI);
810       Regs |= getAliases(X86::RDI);
811       Regs |= getAliases(X86::RDX);
812       Regs |= getAliases(X86::RCX);
813       Regs |= getAliases(X86::R8);
814       Regs |= getAliases(X86::R9);
815       Regs |= getAliases(X86::R10);
816       Regs |= getAliases(X86::R11);
817       Regs |= getAliases(X86::R12);
818       Regs |= getAliases(X86::R13);
819       Regs |= getAliases(X86::R14);
820       Regs |= getAliases(X86::R15);
821       return;
822     }
823     Regs.set(X86::RAX);
824     Regs.set(X86::RBX);
825     Regs.set(X86::RBP);
826     Regs.set(X86::RSI);
827     Regs.set(X86::RDI);
828     Regs.set(X86::RDX);
829     Regs.set(X86::RCX);
830     Regs.set(X86::R8);
831     Regs.set(X86::R9);
832     Regs.set(X86::R10);
833     Regs.set(X86::R11);
834     Regs.set(X86::R12);
835     Regs.set(X86::R13);
836     Regs.set(X86::R14);
837     Regs.set(X86::R15);
838   }
839 
840   void getClassicGPRegs(BitVector &Regs) const override {
841     Regs |= getAliases(X86::RAX);
842     Regs |= getAliases(X86::RBX);
843     Regs |= getAliases(X86::RBP);
844     Regs |= getAliases(X86::RSI);
845     Regs |= getAliases(X86::RDI);
846     Regs |= getAliases(X86::RDX);
847     Regs |= getAliases(X86::RCX);
848   }
849 
850   void getRepRegs(BitVector &Regs) const override {
851     Regs |= getAliases(X86::RCX);
852   }
853 
854   MCPhysReg getAliasSized(MCPhysReg Reg, uint8_t Size) const override {
855     switch (Reg) {
856     case X86::RAX: case X86::EAX: case X86::AX: case X86::AL: case X86::AH:
857       switch (Size) {
858       case 8: return X86::RAX;       case 4: return X86::EAX;
859       case 2: return X86::AX;        case 1: return X86::AL;
860       default: llvm_unreachable("Unexpected size");
861       }
862     case X86::RBX: case X86::EBX: case X86::BX: case X86::BL: case X86::BH:
863       switch (Size) {
864       case 8: return X86::RBX;       case 4: return X86::EBX;
865       case 2: return X86::BX;        case 1: return X86::BL;
866       default: llvm_unreachable("Unexpected size");
867       }
868     case X86::RDX: case X86::EDX: case X86::DX: case X86::DL: case X86::DH:
869       switch (Size) {
870       case 8: return X86::RDX;       case 4: return X86::EDX;
871       case 2: return X86::DX;        case 1: return X86::DL;
872       default: llvm_unreachable("Unexpected size");
873       }
874     case X86::RDI: case X86::EDI: case X86::DI: case X86::DIL:
875       switch (Size) {
876       case 8: return X86::RDI;       case 4: return X86::EDI;
877       case 2: return X86::DI;        case 1: return X86::DIL;
878       default: llvm_unreachable("Unexpected size");
879       }
880     case X86::RSI: case X86::ESI: case X86::SI: case X86::SIL:
881       switch (Size) {
882       case 8: return X86::RSI;       case 4: return X86::ESI;
883       case 2: return X86::SI;        case 1: return X86::SIL;
884       default: llvm_unreachable("Unexpected size");
885       }
886     case X86::RCX: case X86::ECX: case X86::CX: case X86::CL: case X86::CH:
887       switch (Size) {
888       case 8: return X86::RCX;       case 4: return X86::ECX;
889       case 2: return X86::CX;        case 1: return X86::CL;
890       default: llvm_unreachable("Unexpected size");
891       }
892     case X86::RSP: case X86::ESP: case X86::SP: case X86::SPL:
893       switch (Size) {
894       case 8: return X86::RSP;       case 4: return X86::ESP;
895       case 2: return X86::SP;        case 1: return X86::SPL;
896       default: llvm_unreachable("Unexpected size");
897       }
898     case X86::RBP: case X86::EBP: case X86::BP: case X86::BPL:
899       switch (Size) {
900       case 8: return X86::RBP;       case 4: return X86::EBP;
901       case 2: return X86::BP;        case 1: return X86::BPL;
902       default: llvm_unreachable("Unexpected size");
903       }
904   case X86::R8: case X86::R8D: case X86::R8W: case X86::R8B:
905       switch (Size) {
906       case 8: return X86::R8;        case 4: return X86::R8D;
907       case 2: return X86::R8W;       case 1: return X86::R8B;
908       default: llvm_unreachable("Unexpected size");
909       }
910     case X86::R9: case X86::R9D: case X86::R9W: case X86::R9B:
911       switch (Size) {
912       case 8: return X86::R9;        case 4: return X86::R9D;
913       case 2: return X86::R9W;       case 1: return X86::R9B;
914       default: llvm_unreachable("Unexpected size");
915       }
916     case X86::R10: case X86::R10D: case X86::R10W: case X86::R10B:
917       switch (Size) {
918       case 8: return X86::R10;        case 4: return X86::R10D;
919       case 2: return X86::R10W;       case 1: return X86::R10B;
920       default: llvm_unreachable("Unexpected size");
921       }
922     case X86::R11: case X86::R11D: case X86::R11W: case X86::R11B:
923       switch (Size) {
924       case 8: return X86::R11;        case 4: return X86::R11D;
925       case 2: return X86::R11W;       case 1: return X86::R11B;
926       default: llvm_unreachable("Unexpected size");
927       }
928     case X86::R12: case X86::R12D: case X86::R12W: case X86::R12B:
929       switch (Size) {
930       case 8: return X86::R12;        case 4: return X86::R12D;
931       case 2: return X86::R12W;       case 1: return X86::R12B;
932       default: llvm_unreachable("Unexpected size");
933       }
934     case X86::R13: case X86::R13D: case X86::R13W: case X86::R13B:
935       switch (Size) {
936       case 8: return X86::R13;        case 4: return X86::R13D;
937       case 2: return X86::R13W;       case 1: return X86::R13B;
938       default: llvm_unreachable("Unexpected size");
939       }
940     case X86::R14: case X86::R14D: case X86::R14W: case X86::R14B:
941       switch (Size) {
942       case 8: return X86::R14;        case 4: return X86::R14D;
943       case 2: return X86::R14W;       case 1: return X86::R14B;
944       default: llvm_unreachable("Unexpected size");
945       }
946     case X86::R15: case X86::R15D: case X86::R15W: case X86::R15B:
947       switch (Size) {
948       case 8: return X86::R15;        case 4: return X86::R15D;
949       case 2: return X86::R15W;       case 1: return X86::R15B;
950       default: llvm_unreachable("Unexpected size");
951       }
952     default:
953       dbgs() << Reg << " (get alias sized)\n";
954       llvm_unreachable("Unexpected reg number");
955       break;
956     }
957   }
958 
959   bool isUpper8BitReg(MCPhysReg Reg) const override {
960     switch (Reg) {
961     case X86::AH:
962     case X86::BH:
963     case X86::CH:
964     case X86::DH:
965       return true;
966     default:
967       return false;
968     }
969   }
970 
971   bool cannotUseREX(const MCInst &Inst) const override {
972     switch (Inst.getOpcode()) {
973     case X86::MOV8mr_NOREX:
974     case X86::MOV8rm_NOREX:
975     case X86::MOV8rr_NOREX:
976     case X86::MOVSX32rm8_NOREX:
977     case X86::MOVSX32rr8_NOREX:
978     case X86::MOVZX32rm8_NOREX:
979     case X86::MOVZX32rr8_NOREX:
980     case X86::MOV8mr:
981     case X86::MOV8rm:
982     case X86::MOV8rr:
983     case X86::MOVSX32rm8:
984     case X86::MOVSX32rr8:
985     case X86::MOVZX32rm8:
986     case X86::MOVZX32rr8:
987     case X86::TEST8ri:
988       for (int I = 0, E = MCPlus::getNumPrimeOperands(Inst); I != E; ++I) {
989         const MCOperand &Operand = Inst.getOperand(I);
990         if (!Operand.isReg())
991           continue;
992         if (isUpper8BitReg(Operand.getReg()))
993           return true;
994       }
995       LLVM_FALLTHROUGH;
996     default:
997       return false;
998     }
999   }
1000 
1001   bool isStackAccess(const MCInst &Inst, bool &IsLoad, bool &IsStore,
1002                      bool &IsStoreFromReg, MCPhysReg &Reg, int32_t &SrcImm,
1003                      uint16_t &StackPtrReg, int64_t &StackOffset, uint8_t &Size,
1004                      bool &IsSimple, bool &IsIndexed) const override {
1005     // Detect simple push/pop cases first
1006     if (int Sz = getPushSize(Inst)) {
1007       IsLoad = false;
1008       IsStore = true;
1009       IsStoreFromReg = true;
1010       StackPtrReg = X86::RSP;
1011       StackOffset = -Sz;
1012       Size = Sz;
1013       IsSimple = true;
1014       if (Inst.getOperand(0).isImm())
1015         SrcImm = Inst.getOperand(0).getImm();
1016       else if (Inst.getOperand(0).isReg())
1017         Reg = Inst.getOperand(0).getReg();
1018       else
1019         IsSimple = false;
1020 
1021       return true;
1022     }
1023     if (int Sz = getPopSize(Inst)) {
1024       IsLoad = true;
1025       IsStore = false;
1026       if (Inst.getNumOperands() == 0 || !Inst.getOperand(0).isReg()) {
1027         IsSimple = false;
1028       } else {
1029         Reg = Inst.getOperand(0).getReg();
1030         IsSimple = true;
1031       }
1032       StackPtrReg = X86::RSP;
1033       StackOffset = 0;
1034       Size = Sz;
1035       return true;
1036     }
1037 
1038     struct InstInfo {
1039       // Size in bytes that Inst loads from memory.
1040       uint8_t DataSize;
1041       bool IsLoad;
1042       bool IsStore;
1043       bool StoreFromReg;
1044       bool Simple;
1045     };
1046 
1047     InstInfo I;
1048     int MemOpNo = getMemoryOperandNo(Inst);
1049     const MCInstrDesc &MCII = Info->get(Inst.getOpcode());
1050     // If it is not dealing with a memory operand, we discard it
1051     if (MemOpNo == -1 || MCII.isCall())
1052       return false;
1053 
1054     switch (Inst.getOpcode()) {
1055     default: {
1056       uint8_t Sz = 0;
1057       bool IsLoad = MCII.mayLoad();
1058       bool IsStore = MCII.mayStore();
1059       // Is it LEA? (deals with memory but is not loading nor storing)
1060       if (!IsLoad && !IsStore)
1061         return false;
1062 
1063       // Try to guess data size involved in the load/store by looking at the
1064       // register size. If there's no reg involved, return 0 as size, meaning
1065       // we don't know.
1066       for (unsigned I = 0, E = MCII.getNumOperands(); I != E; ++I) {
1067         if (MCII.OpInfo[I].OperandType != MCOI::OPERAND_REGISTER)
1068           continue;
1069         if (static_cast<int>(I) >= MemOpNo && I < X86::AddrNumOperands)
1070           continue;
1071         Sz = RegInfo->getRegClass(MCII.OpInfo[I].RegClass).getSizeInBits() / 8;
1072         break;
1073       }
1074       I = {Sz, IsLoad, IsStore, false, false};
1075       break;
1076     }
1077     case X86::MOV16rm: I = {2, true, false, false, true}; break;
1078     case X86::MOV32rm: I = {4, true, false, false, true}; break;
1079     case X86::MOV64rm: I = {8, true, false, false, true}; break;
1080     case X86::MOV16mr: I = {2, false, true, true, true};  break;
1081     case X86::MOV32mr: I = {4, false, true, true, true};  break;
1082     case X86::MOV64mr: I = {8, false, true, true, true};  break;
1083     case X86::MOV16mi: I = {2, false, true, false, true}; break;
1084     case X86::MOV32mi: I = {4, false, true, false, true}; break;
1085     } // end switch (Inst.getOpcode())
1086 
1087     unsigned BaseRegNum;
1088     int64_t ScaleValue;
1089     unsigned IndexRegNum;
1090     int64_t DispValue;
1091     unsigned SegRegNum;
1092     const MCExpr *DispExpr;
1093     if (!evaluateX86MemoryOperand(Inst, &BaseRegNum, &ScaleValue, &IndexRegNum,
1094                                   &DispValue, &SegRegNum, &DispExpr)) {
1095       LLVM_DEBUG(dbgs() << "Evaluate failed on ");
1096       LLVM_DEBUG(Inst.dump());
1097       return false;
1098     }
1099 
1100     // Make sure it's a stack access
1101     if (BaseRegNum != X86::RBP && BaseRegNum != X86::RSP)
1102       return false;
1103 
1104     IsLoad = I.IsLoad;
1105     IsStore = I.IsStore;
1106     IsStoreFromReg = I.StoreFromReg;
1107     Size = I.DataSize;
1108     IsSimple = I.Simple;
1109     StackPtrReg = BaseRegNum;
1110     StackOffset = DispValue;
1111     IsIndexed = IndexRegNum != X86::NoRegister || SegRegNum != X86::NoRegister;
1112 
1113     if (!I.Simple)
1114       return true;
1115 
1116     // Retrieve related register in simple MOV from/to stack operations.
1117     unsigned MemOpOffset = static_cast<unsigned>(MemOpNo);
1118     if (I.IsLoad) {
1119       MCOperand RegOpnd = Inst.getOperand(0);
1120       assert(RegOpnd.isReg() && "unexpected destination operand");
1121       Reg = RegOpnd.getReg();
1122     } else if (I.IsStore) {
1123       MCOperand SrcOpnd =
1124           Inst.getOperand(MemOpOffset + X86::AddrSegmentReg + 1);
1125       if (I.StoreFromReg) {
1126         assert(SrcOpnd.isReg() && "unexpected source operand");
1127         Reg = SrcOpnd.getReg();
1128       } else {
1129         assert(SrcOpnd.isImm() && "unexpected source operand");
1130         SrcImm = SrcOpnd.getImm();
1131       }
1132     }
1133 
1134     return true;
1135   }
1136 
1137   void changeToPushOrPop(MCInst &Inst) const override {
1138     assert(!isPush(Inst) && !isPop(Inst));
1139 
1140     struct InstInfo {
1141       // Size in bytes that Inst loads from memory.
1142       uint8_t DataSize;
1143       bool IsLoad;
1144       bool StoreFromReg;
1145     };
1146 
1147     InstInfo I;
1148     switch (Inst.getOpcode()) {
1149     default: {
1150       llvm_unreachable("Unhandled opcode");
1151       return;
1152     }
1153     case X86::MOV16rm: I = {2, true, false}; break;
1154     case X86::MOV32rm: I = {4, true, false}; break;
1155     case X86::MOV64rm: I = {8, true, false}; break;
1156     case X86::MOV16mr: I = {2, false, true};  break;
1157     case X86::MOV32mr: I = {4, false, true};  break;
1158     case X86::MOV64mr: I = {8, false, true};  break;
1159     case X86::MOV16mi: I = {2, false, false}; break;
1160     case X86::MOV32mi: I = {4, false, false}; break;
1161     } // end switch (Inst.getOpcode())
1162 
1163     unsigned BaseRegNum;
1164     int64_t ScaleValue;
1165     unsigned IndexRegNum;
1166     int64_t DispValue;
1167     unsigned SegRegNum;
1168     const MCExpr *DispExpr;
1169     if (!evaluateX86MemoryOperand(Inst, &BaseRegNum, &ScaleValue, &IndexRegNum,
1170                                   &DispValue, &SegRegNum, &DispExpr)) {
1171       llvm_unreachable("Evaluate failed");
1172       return;
1173     }
1174     // Make sure it's a stack access
1175     if (BaseRegNum != X86::RBP && BaseRegNum != X86::RSP) {
1176       llvm_unreachable("Not a stack access");
1177       return;
1178     }
1179 
1180     unsigned MemOpOffset = getMemoryOperandNo(Inst);
1181     unsigned NewOpcode = 0;
1182     if (I.IsLoad) {
1183       switch (I.DataSize) {
1184       case 2: NewOpcode = X86::POP16r; break;
1185       case 4: NewOpcode = X86::POP32r; break;
1186       case 8: NewOpcode = X86::POP64r; break;
1187       default:
1188         llvm_unreachable("Unexpected size");
1189       }
1190       unsigned RegOpndNum = Inst.getOperand(0).getReg();
1191       Inst.clear();
1192       Inst.setOpcode(NewOpcode);
1193       Inst.addOperand(MCOperand::createReg(RegOpndNum));
1194     } else {
1195       MCOperand SrcOpnd =
1196           Inst.getOperand(MemOpOffset + X86::AddrSegmentReg + 1);
1197       if (I.StoreFromReg) {
1198         switch (I.DataSize) {
1199         case 2: NewOpcode = X86::PUSH16r; break;
1200         case 4: NewOpcode = X86::PUSH32r; break;
1201         case 8: NewOpcode = X86::PUSH64r; break;
1202         default:
1203           llvm_unreachable("Unexpected size");
1204         }
1205         assert(SrcOpnd.isReg() && "Unexpected source operand");
1206         unsigned RegOpndNum = SrcOpnd.getReg();
1207         Inst.clear();
1208         Inst.setOpcode(NewOpcode);
1209         Inst.addOperand(MCOperand::createReg(RegOpndNum));
1210       } else {
1211         switch (I.DataSize) {
1212         case 2: NewOpcode = X86::PUSH16i8; break;
1213         case 4: NewOpcode = X86::PUSH32i8; break;
1214         case 8: NewOpcode = X86::PUSH64i32; break;
1215         default:
1216           llvm_unreachable("Unexpected size");
1217         }
1218         assert(SrcOpnd.isImm() && "Unexpected source operand");
1219         int64_t SrcImm = SrcOpnd.getImm();
1220         Inst.clear();
1221         Inst.setOpcode(NewOpcode);
1222         Inst.addOperand(MCOperand::createImm(SrcImm));
1223       }
1224     }
1225   }
1226 
1227   bool isStackAdjustment(const MCInst &Inst) const override {
1228     switch (Inst.getOpcode()) {
1229     default:
1230       return false;
1231     case X86::SUB64ri32:
1232     case X86::SUB64ri8:
1233     case X86::ADD64ri32:
1234     case X86::ADD64ri8:
1235     case X86::LEA64r:
1236       break;
1237     }
1238 
1239     const MCInstrDesc &MCII = Info->get(Inst.getOpcode());
1240     for (int I = 0, E = MCII.getNumDefs(); I != E; ++I) {
1241       const MCOperand &Operand = Inst.getOperand(I);
1242       if (Operand.isReg() && Operand.getReg() == X86::RSP)
1243         return true;
1244     }
1245     return false;
1246   }
1247 
1248   bool evaluateSimple(const MCInst &Inst, int64_t &Output,
1249                       std::pair<MCPhysReg, int64_t> Input1,
1250                       std::pair<MCPhysReg, int64_t> Input2) const override {
1251 
1252     auto getOperandVal = [&](MCPhysReg Reg) -> ErrorOr<int64_t> {
1253       if (Reg == Input1.first)
1254         return Input1.second;
1255       if (Reg == Input2.first)
1256         return Input2.second;
1257       return make_error_code(errc::result_out_of_range);
1258     };
1259 
1260     switch (Inst.getOpcode()) {
1261     default:
1262       return false;
1263 
1264     case X86::AND64ri32:
1265     case X86::AND64ri8:
1266       if (!Inst.getOperand(2).isImm())
1267         return false;
1268       if (ErrorOr<int64_t> InputVal =
1269               getOperandVal(Inst.getOperand(1).getReg()))
1270         Output = *InputVal & Inst.getOperand(2).getImm();
1271       else
1272         return false;
1273       break;
1274     case X86::SUB64ri32:
1275     case X86::SUB64ri8:
1276       if (!Inst.getOperand(2).isImm())
1277         return false;
1278       if (ErrorOr<int64_t> InputVal =
1279               getOperandVal(Inst.getOperand(1).getReg()))
1280         Output = *InputVal - Inst.getOperand(2).getImm();
1281       else
1282         return false;
1283       break;
1284     case X86::ADD64ri32:
1285     case X86::ADD64ri8:
1286       if (!Inst.getOperand(2).isImm())
1287         return false;
1288       if (ErrorOr<int64_t> InputVal =
1289               getOperandVal(Inst.getOperand(1).getReg()))
1290         Output = *InputVal + Inst.getOperand(2).getImm();
1291       else
1292         return false;
1293       break;
1294     case X86::ADD64i32:
1295       if (!Inst.getOperand(0).isImm())
1296         return false;
1297       if (ErrorOr<int64_t> InputVal = getOperandVal(X86::RAX))
1298         Output = *InputVal + Inst.getOperand(0).getImm();
1299       else
1300         return false;
1301       break;
1302 
1303     case X86::LEA64r: {
1304       unsigned BaseRegNum;
1305       int64_t ScaleValue;
1306       unsigned IndexRegNum;
1307       int64_t DispValue;
1308       unsigned SegRegNum;
1309       const MCExpr *DispExpr = nullptr;
1310       if (!evaluateX86MemoryOperand(Inst, &BaseRegNum, &ScaleValue,
1311                                     &IndexRegNum, &DispValue, &SegRegNum,
1312                                     &DispExpr))
1313         return false;
1314 
1315       if (BaseRegNum == X86::NoRegister || IndexRegNum != X86::NoRegister ||
1316           SegRegNum != X86::NoRegister || DispExpr)
1317         return false;
1318 
1319       if (ErrorOr<int64_t> InputVal = getOperandVal(BaseRegNum))
1320         Output = *InputVal + DispValue;
1321       else
1322         return false;
1323 
1324       break;
1325     }
1326     }
1327     return true;
1328   }
1329 
1330   bool isRegToRegMove(const MCInst &Inst, MCPhysReg &From,
1331                       MCPhysReg &To) const override {
1332     switch (Inst.getOpcode()) {
1333     default:
1334       return false;
1335     case X86::LEAVE:
1336     case X86::LEAVE64:
1337       To = getStackPointer();
1338       From = getFramePointer();
1339       return true;
1340     case X86::MOV64rr:
1341       To = Inst.getOperand(0).getReg();
1342       From = Inst.getOperand(1).getReg();
1343       return true;
1344     }
1345   }
1346 
1347   MCPhysReg getStackPointer() const override { return X86::RSP; }
1348   MCPhysReg getFramePointer() const override { return X86::RBP; }
1349   MCPhysReg getFlagsReg() const override { return X86::EFLAGS; }
1350 
1351   bool escapesVariable(const MCInst &Inst,
1352                        bool HasFramePointer) const override {
1353     int MemOpNo = getMemoryOperandNo(Inst);
1354     const MCInstrDesc &MCII = Info->get(Inst.getOpcode());
1355     const unsigned NumDefs = MCII.getNumDefs();
1356     static BitVector SPBPAliases(BitVector(getAliases(X86::RSP)) |=
1357                                  getAliases(X86::RBP));
1358     static BitVector SPAliases(getAliases(X86::RSP));
1359 
1360     // FIXME: PUSH can be technically a leak, but let's ignore this for now
1361     // because a lot of harmless prologue code will spill SP to the stack.
1362     // Unless push is clearly pushing an object address to the stack as
1363     // demonstrated by having a MemOp.
1364     bool IsPush = isPush(Inst);
1365     if (IsPush && MemOpNo == -1)
1366       return false;
1367 
1368     // We use this to detect LEA (has memop but does not access mem)
1369     bool AccessMem = MCII.mayLoad() || MCII.mayStore();
1370     bool DoesLeak = false;
1371     for (int I = 0, E = MCPlus::getNumPrimeOperands(Inst); I != E; ++I) {
1372       // Ignore if SP/BP is used to dereference memory -- that's fine
1373       if (MemOpNo != -1 && !IsPush && AccessMem && I >= MemOpNo &&
1374           I <= MemOpNo + 5)
1375         continue;
1376       // Ignore if someone is writing to SP/BP
1377       if (I < static_cast<int>(NumDefs))
1378         continue;
1379 
1380       const MCOperand &Operand = Inst.getOperand(I);
1381       if (HasFramePointer && Operand.isReg() && SPBPAliases[Operand.getReg()]) {
1382         DoesLeak = true;
1383         break;
1384       }
1385       if (!HasFramePointer && Operand.isReg() && SPAliases[Operand.getReg()]) {
1386         DoesLeak = true;
1387         break;
1388       }
1389     }
1390 
1391     // If potential leak, check if it is not just writing to itself/sp/bp
1392     if (DoesLeak) {
1393       for (int I = 0, E = NumDefs; I != E; ++I) {
1394         const MCOperand &Operand = Inst.getOperand(I);
1395         if (HasFramePointer && Operand.isReg() &&
1396             SPBPAliases[Operand.getReg()]) {
1397           DoesLeak = false;
1398           break;
1399         }
1400         if (!HasFramePointer && Operand.isReg() &&
1401             SPAliases[Operand.getReg()]) {
1402           DoesLeak = false;
1403           break;
1404         }
1405       }
1406     }
1407     return DoesLeak;
1408   }
1409 
1410   bool addToImm(MCInst &Inst, int64_t &Amt, MCContext *Ctx) const override {
1411     unsigned ImmOpNo = -1U;
1412     int MemOpNo = getMemoryOperandNo(Inst);
1413     if (MemOpNo != -1)
1414       ImmOpNo = MemOpNo + X86::AddrDisp;
1415     else
1416       for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst);
1417            ++Index)
1418         if (Inst.getOperand(Index).isImm())
1419           ImmOpNo = Index;
1420     if (ImmOpNo == -1U)
1421       return false;
1422 
1423     MCOperand &Operand = Inst.getOperand(ImmOpNo);
1424     Amt += Operand.getImm();
1425     Operand.setImm(Amt);
1426     // Check for the need for relaxation
1427     if (int64_t(Amt) == int64_t(int8_t(Amt)))
1428       return true;
1429 
1430     // Relax instruction
1431     switch (Inst.getOpcode()) {
1432     case X86::SUB64ri8:
1433       Inst.setOpcode(X86::SUB64ri32);
1434       break;
1435     case X86::ADD64ri8:
1436       Inst.setOpcode(X86::ADD64ri32);
1437       break;
1438     default:
1439       // No need for relaxation
1440       break;
1441     }
1442     return true;
1443   }
1444 
1445   /// TODO: this implementation currently works for the most common opcodes that
1446   /// load from memory. It can be extended to work with memory store opcodes as
1447   /// well as more memory load opcodes.
1448   bool replaceMemOperandWithImm(MCInst &Inst, StringRef ConstantData,
1449                                 uint64_t Offset) const override {
1450     enum CheckSignExt : uint8_t {
1451       NOCHECK = 0,
1452       CHECK8,
1453       CHECK32,
1454     };
1455 
1456     using CheckList = std::vector<std::pair<CheckSignExt, unsigned>>;
1457     struct InstInfo {
1458       // Size in bytes that Inst loads from memory.
1459       uint8_t DataSize;
1460 
1461       // True when the target operand has to be duplicated because the opcode
1462       // expects a LHS operand.
1463       bool HasLHS;
1464 
1465       // List of checks and corresponding opcodes to be used. We try to use the
1466       // smallest possible immediate value when various sizes are available,
1467       // hence we may need to check whether a larger constant fits in a smaller
1468       // immediate.
1469       CheckList Checks;
1470     };
1471 
1472     InstInfo I;
1473 
1474     switch (Inst.getOpcode()) {
1475     default: {
1476       switch (getPopSize(Inst)) {
1477       case 2:            I = {2, false, {{NOCHECK, X86::MOV16ri}}};  break;
1478       case 4:            I = {4, false, {{NOCHECK, X86::MOV32ri}}};  break;
1479       case 8:            I = {8, false, {{CHECK32, X86::MOV64ri32},
1480                                          {NOCHECK, X86::MOV64rm}}};  break;
1481       default:           return false;
1482       }
1483       break;
1484     }
1485 
1486     // MOV
1487     case X86::MOV8rm:      I = {1, false, {{NOCHECK, X86::MOV8ri}}};   break;
1488     case X86::MOV16rm:     I = {2, false, {{NOCHECK, X86::MOV16ri}}};  break;
1489     case X86::MOV32rm:     I = {4, false, {{NOCHECK, X86::MOV32ri}}};  break;
1490     case X86::MOV64rm:     I = {8, false, {{CHECK32, X86::MOV64ri32},
1491                                            {NOCHECK, X86::MOV64rm}}};  break;
1492 
1493     // MOVZX
1494     case X86::MOVZX16rm8:  I = {1, false, {{NOCHECK, X86::MOV16ri}}};  break;
1495     case X86::MOVZX32rm8:  I = {1, false, {{NOCHECK, X86::MOV32ri}}};  break;
1496     case X86::MOVZX32rm16: I = {2, false, {{NOCHECK, X86::MOV32ri}}};  break;
1497 
1498     // CMP
1499     case X86::CMP8rm:      I = {1, false, {{NOCHECK, X86::CMP8ri}}};   break;
1500     case X86::CMP16rm:     I = {2, false, {{CHECK8,  X86::CMP16ri8},
1501                                            {NOCHECK, X86::CMP16ri}}};  break;
1502     case X86::CMP32rm:     I = {4, false, {{CHECK8,  X86::CMP32ri8},
1503                                            {NOCHECK, X86::CMP32ri}}};  break;
1504     case X86::CMP64rm:     I = {8, false, {{CHECK8,  X86::CMP64ri8},
1505                                            {CHECK32, X86::CMP64ri32},
1506                                            {NOCHECK, X86::CMP64rm}}};  break;
1507 
1508     // TEST
1509     case X86::TEST8mr:     I = {1, false, {{NOCHECK, X86::TEST8ri}}};  break;
1510     case X86::TEST16mr:    I = {2, false, {{NOCHECK, X86::TEST16ri}}}; break;
1511     case X86::TEST32mr:    I = {4, false, {{NOCHECK, X86::TEST32ri}}}; break;
1512     case X86::TEST64mr:    I = {8, false, {{CHECK32, X86::TEST64ri32},
1513                                            {NOCHECK, X86::TEST64mr}}}; break;
1514 
1515     // ADD
1516     case X86::ADD8rm:      I = {1, true,  {{NOCHECK, X86::ADD8ri}}};   break;
1517     case X86::ADD16rm:     I = {2, true,  {{CHECK8,  X86::ADD16ri8},
1518                                            {NOCHECK, X86::ADD16ri}}};  break;
1519     case X86::ADD32rm:     I = {4, true,  {{CHECK8,  X86::ADD32ri8},
1520                                            {NOCHECK, X86::ADD32ri}}};  break;
1521     case X86::ADD64rm:     I = {8, true,  {{CHECK8,  X86::ADD64ri8},
1522                                            {CHECK32, X86::ADD64ri32},
1523                                            {NOCHECK, X86::ADD64rm}}};  break;
1524 
1525     // SUB
1526     case X86::SUB8rm:      I = {1, true,  {{NOCHECK, X86::SUB8ri}}};   break;
1527     case X86::SUB16rm:     I = {2, true,  {{CHECK8,  X86::SUB16ri8},
1528                                            {NOCHECK, X86::SUB16ri}}};  break;
1529     case X86::SUB32rm:     I = {4, true,  {{CHECK8,  X86::SUB32ri8},
1530                                            {NOCHECK, X86::SUB32ri}}};  break;
1531     case X86::SUB64rm:     I = {8, true,  {{CHECK8,  X86::SUB64ri8},
1532                                            {CHECK32, X86::SUB64ri32},
1533                                            {NOCHECK, X86::SUB64rm}}};  break;
1534 
1535     // AND
1536     case X86::AND8rm:      I = {1, true,  {{NOCHECK, X86::AND8ri}}};   break;
1537     case X86::AND16rm:     I = {2, true,  {{CHECK8,  X86::AND16ri8},
1538                                            {NOCHECK, X86::AND16ri}}};  break;
1539     case X86::AND32rm:     I = {4, true,  {{CHECK8,  X86::AND32ri8},
1540                                            {NOCHECK, X86::AND32ri}}};  break;
1541     case X86::AND64rm:     I = {8, true,  {{CHECK8,  X86::AND64ri8},
1542                                            {CHECK32, X86::AND64ri32},
1543                                            {NOCHECK, X86::AND64rm}}};  break;
1544 
1545     // OR
1546     case X86::OR8rm:       I = {1, true,  {{NOCHECK, X86::OR8ri}}};    break;
1547     case X86::OR16rm:      I = {2, true,  {{CHECK8,  X86::OR16ri8},
1548                                            {NOCHECK, X86::OR16ri}}};   break;
1549     case X86::OR32rm:      I = {4, true,  {{CHECK8,  X86::OR32ri8},
1550                                            {NOCHECK, X86::OR32ri}}};   break;
1551     case X86::OR64rm:      I = {8, true,  {{CHECK8,  X86::OR64ri8},
1552                                            {CHECK32, X86::OR64ri32},
1553                                            {NOCHECK, X86::OR64rm}}};   break;
1554 
1555     // XOR
1556     case X86::XOR8rm:      I = {1, true,  {{NOCHECK, X86::XOR8ri}}};   break;
1557     case X86::XOR16rm:     I = {2, true,  {{CHECK8,  X86::XOR16ri8},
1558                                            {NOCHECK, X86::XOR16ri}}};  break;
1559     case X86::XOR32rm:     I = {4, true,  {{CHECK8,  X86::XOR32ri8},
1560                                            {NOCHECK, X86::XOR32ri}}};  break;
1561     case X86::XOR64rm:     I = {8, true,  {{CHECK8,  X86::XOR64ri8},
1562                                            {CHECK32, X86::XOR64ri32},
1563                                            {NOCHECK, X86::XOR64rm}}};  break;
1564     }
1565 
1566     // Compute the immediate value.
1567     assert(Offset + I.DataSize <= ConstantData.size() &&
1568            "invalid offset for given constant data");
1569     int64_t ImmVal =
1570         DataExtractor(ConstantData, true, 8).getSigned(&Offset, I.DataSize);
1571 
1572     // Compute the new opcode.
1573     unsigned NewOpcode = 0;
1574     for (const std::pair<CheckSignExt, unsigned> &Check : I.Checks) {
1575       NewOpcode = Check.second;
1576       if (Check.first == NOCHECK)
1577         break;
1578       if (Check.first == CHECK8 && isInt<8>(ImmVal))
1579         break;
1580       if (Check.first == CHECK32 && isInt<32>(ImmVal))
1581         break;
1582     }
1583     if (NewOpcode == Inst.getOpcode())
1584       return false;
1585 
1586     // Modify the instruction.
1587     MCOperand ImmOp = MCOperand::createImm(ImmVal);
1588     uint32_t TargetOpNum = 0;
1589     // Test instruction does not follow the regular pattern of putting the
1590     // memory reference of a load (5 MCOperands) last in the list of operands.
1591     // Since it is not modifying the register operand, it is not treated as
1592     // a destination operand and it is not the first operand as it is in the
1593     // other instructions we treat here.
1594     if (NewOpcode == X86::TEST8ri || NewOpcode == X86::TEST16ri ||
1595         NewOpcode == X86::TEST32ri || NewOpcode == X86::TEST64ri32)
1596       TargetOpNum = getMemoryOperandNo(Inst) + X86::AddrNumOperands;
1597 
1598     MCOperand TargetOp = Inst.getOperand(TargetOpNum);
1599     Inst.clear();
1600     Inst.setOpcode(NewOpcode);
1601     Inst.addOperand(TargetOp);
1602     if (I.HasLHS)
1603       Inst.addOperand(TargetOp);
1604     Inst.addOperand(ImmOp);
1605 
1606     return true;
1607   }
1608 
1609   /// TODO: this implementation currently works for the most common opcodes that
1610   /// load from memory. It can be extended to work with memory store opcodes as
1611   /// well as more memory load opcodes.
1612   bool replaceMemOperandWithReg(MCInst &Inst, MCPhysReg RegNum) const override {
1613     unsigned NewOpcode;
1614 
1615     switch (Inst.getOpcode()) {
1616     default: {
1617       switch (getPopSize(Inst)) {
1618       case 2:            NewOpcode = X86::MOV16rr; break;
1619       case 4:            NewOpcode = X86::MOV32rr; break;
1620       case 8:            NewOpcode = X86::MOV64rr; break;
1621       default:           return false;
1622       }
1623       break;
1624     }
1625 
1626     // MOV
1627     case X86::MOV8rm:      NewOpcode = X86::MOV8rr;   break;
1628     case X86::MOV16rm:     NewOpcode = X86::MOV16rr;  break;
1629     case X86::MOV32rm:     NewOpcode = X86::MOV32rr;  break;
1630     case X86::MOV64rm:     NewOpcode = X86::MOV64rr;  break;
1631     }
1632 
1633     // Modify the instruction.
1634     MCOperand RegOp = MCOperand::createReg(RegNum);
1635     MCOperand TargetOp = Inst.getOperand(0);
1636     Inst.clear();
1637     Inst.setOpcode(NewOpcode);
1638     Inst.addOperand(TargetOp);
1639     Inst.addOperand(RegOp);
1640 
1641     return true;
1642   }
1643 
1644   bool isRedundantMove(const MCInst &Inst) const override {
1645     switch (Inst.getOpcode()) {
1646     default:
1647       return false;
1648 
1649     // MOV
1650     case X86::MOV8rr:
1651     case X86::MOV16rr:
1652     case X86::MOV32rr:
1653     case X86::MOV64rr:
1654       break;
1655     }
1656 
1657     assert(Inst.getOperand(0).isReg() && Inst.getOperand(1).isReg());
1658     return Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg();
1659   }
1660 
1661   bool requiresAlignedAddress(const MCInst &Inst) const override {
1662     const MCInstrDesc &Desc = Info->get(Inst.getOpcode());
1663     for (unsigned int I = 0; I < Desc.getNumOperands(); ++I) {
1664       const MCOperandInfo &Op = Desc.OpInfo[I];
1665       if (Op.OperandType != MCOI::OPERAND_REGISTER)
1666         continue;
1667       if (Op.RegClass == X86::VR128RegClassID)
1668         return true;
1669     }
1670     return false;
1671   }
1672 
1673   bool convertJmpToTailCall(MCInst &Inst) override {
1674     if (isTailCall(Inst))
1675       return false;
1676 
1677     int NewOpcode;
1678     switch (Inst.getOpcode()) {
1679     default:
1680       return false;
1681     case X86::JMP_1:
1682     case X86::JMP_2:
1683     case X86::JMP_4:
1684       NewOpcode = X86::JMP_4;
1685       break;
1686     case X86::JMP16m:
1687     case X86::JMP32m:
1688     case X86::JMP64m:
1689       NewOpcode = X86::JMP32m;
1690       break;
1691     case X86::JMP16r:
1692     case X86::JMP32r:
1693     case X86::JMP64r:
1694       NewOpcode = X86::JMP32r;
1695       break;
1696     }
1697 
1698     Inst.setOpcode(NewOpcode);
1699     setTailCall(Inst);
1700     return true;
1701   }
1702 
1703   bool convertTailCallToJmp(MCInst &Inst) override {
1704     int NewOpcode;
1705     switch (Inst.getOpcode()) {
1706     default:
1707       return false;
1708     case X86::JMP_4:
1709       NewOpcode = X86::JMP_1;
1710       break;
1711     case X86::JMP32m:
1712       NewOpcode = X86::JMP64m;
1713       break;
1714     case X86::JMP32r:
1715       NewOpcode = X86::JMP64r;
1716       break;
1717     }
1718 
1719     Inst.setOpcode(NewOpcode);
1720     removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall);
1721     clearOffset(Inst);
1722     return true;
1723   }
1724 
1725   bool convertTailCallToCall(MCInst &Inst) override {
1726     int NewOpcode;
1727     switch (Inst.getOpcode()) {
1728     default:
1729       return false;
1730     case X86::JMP_4:
1731       NewOpcode = X86::CALL64pcrel32;
1732       break;
1733     case X86::JMP32m:
1734       NewOpcode = X86::CALL64m;
1735       break;
1736     case X86::JMP32r:
1737       NewOpcode = X86::CALL64r;
1738       break;
1739     }
1740 
1741     Inst.setOpcode(NewOpcode);
1742     removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall);
1743     return true;
1744   }
1745 
1746   bool convertCallToIndirectCall(MCInst &Inst, const MCSymbol *TargetLocation,
1747                                  MCContext *Ctx) override {
1748     bool IsTailCall = isTailCall(Inst);
1749     assert((Inst.getOpcode() == X86::CALL64pcrel32 ||
1750             (Inst.getOpcode() == X86::JMP_4 && IsTailCall)) &&
1751            "64-bit direct (tail) call instruction expected");
1752     const auto NewOpcode =
1753         (Inst.getOpcode() == X86::CALL64pcrel32) ? X86::CALL64m : X86::JMP32m;
1754     Inst.setOpcode(NewOpcode);
1755 
1756     // Replace the first operand and preserve auxiliary operands of
1757     // the instruction.
1758     Inst.erase(Inst.begin());
1759     Inst.insert(Inst.begin(),
1760                 MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
1761     Inst.insert(Inst.begin(),
1762                 MCOperand::createExpr(                  // Displacement
1763                     MCSymbolRefExpr::create(TargetLocation,
1764                                             MCSymbolRefExpr::VK_None, *Ctx)));
1765     Inst.insert(Inst.begin(),
1766                 MCOperand::createReg(X86::NoRegister)); // IndexReg
1767     Inst.insert(Inst.begin(),
1768                 MCOperand::createImm(1));               // ScaleAmt
1769     Inst.insert(Inst.begin(),
1770                 MCOperand::createReg(X86::RIP));        // BaseReg
1771 
1772     return true;
1773   }
1774 
1775   void convertIndirectCallToLoad(MCInst &Inst, MCPhysReg Reg) override {
1776     bool IsTailCall = isTailCall(Inst);
1777     if (IsTailCall)
1778       removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall);
1779     if (Inst.getOpcode() == X86::CALL64m ||
1780         (Inst.getOpcode() == X86::JMP32m && IsTailCall)) {
1781       Inst.setOpcode(X86::MOV64rm);
1782       Inst.insert(Inst.begin(), MCOperand::createReg(Reg));
1783       return;
1784     }
1785     if (Inst.getOpcode() == X86::CALL64r ||
1786         (Inst.getOpcode() == X86::JMP32r && IsTailCall)) {
1787       Inst.setOpcode(X86::MOV64rr);
1788       Inst.insert(Inst.begin(), MCOperand::createReg(Reg));
1789       return;
1790     }
1791     LLVM_DEBUG(Inst.dump());
1792     llvm_unreachable("not implemented");
1793   }
1794 
1795   bool shortenInstruction(MCInst &Inst,
1796                           const MCSubtargetInfo &STI) const override {
1797     unsigned OldOpcode = Inst.getOpcode();
1798     unsigned NewOpcode = OldOpcode;
1799 
1800     int MemOpNo = getMemoryOperandNo(Inst);
1801 
1802     // Check and remove redundant Address-Size override prefix.
1803     if (opts::X86StripRedundantAddressSize) {
1804       uint64_t TSFlags = Info->get(OldOpcode).TSFlags;
1805       unsigned Flags = Inst.getFlags();
1806 
1807       if (!X86_MC::needsAddressSizeOverride(Inst, STI, MemOpNo, TSFlags) &&
1808           Flags & X86::IP_HAS_AD_SIZE)
1809         Inst.setFlags(Flags ^ X86::IP_HAS_AD_SIZE);
1810     }
1811 
1812     // Check and remove EIZ/RIZ. These cases represent ambiguous cases where
1813     // SIB byte is present, but no index is used and modrm alone should have
1814     // been enough. Converting to NoRegister effectively removes the SIB byte.
1815     if (MemOpNo >= 0) {
1816       MCOperand &IndexOp =
1817           Inst.getOperand(static_cast<unsigned>(MemOpNo) + X86::AddrIndexReg);
1818       if (IndexOp.getReg() == X86::EIZ || IndexOp.getReg() == X86::RIZ)
1819         IndexOp = MCOperand::createReg(X86::NoRegister);
1820     }
1821 
1822     if (isBranch(Inst)) {
1823       NewOpcode = getShortBranchOpcode(OldOpcode);
1824     } else if (OldOpcode == X86::MOV64ri) {
1825       if (Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).isImm()) {
1826         const int64_t Imm =
1827             Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).getImm();
1828         if (int64_t(Imm) == int64_t(int32_t(Imm)))
1829           NewOpcode = X86::MOV64ri32;
1830       }
1831     } else {
1832       // If it's arithmetic instruction check if signed operand fits in 1 byte.
1833       const unsigned ShortOpcode = getShortArithOpcode(OldOpcode);
1834       if (ShortOpcode != OldOpcode &&
1835           Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).isImm()) {
1836         int64_t Imm =
1837             Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).getImm();
1838         if (int64_t(Imm) == int64_t(int8_t(Imm)))
1839           NewOpcode = ShortOpcode;
1840       }
1841     }
1842 
1843     if (NewOpcode == OldOpcode)
1844       return false;
1845 
1846     Inst.setOpcode(NewOpcode);
1847     return true;
1848   }
1849 
1850   bool
1851   convertMoveToConditionalMove(MCInst &Inst, unsigned CC, bool AllowStackMemOp,
1852                                bool AllowBasePtrStackMemOp) const override {
1853     // - Register-register moves are OK
1854     // - Stores are filtered out by opcode (no store CMOV)
1855     // - Non-stack loads are prohibited (generally unsafe)
1856     // - Stack loads are OK if AllowStackMemOp is true
1857     // - Stack loads with RBP are OK if AllowBasePtrStackMemOp is true
1858     if (isLoad(Inst)) {
1859       // If stack memory operands are not allowed, no loads are allowed
1860       if (!AllowStackMemOp)
1861         return false;
1862 
1863       // If stack memory operands are allowed, check if it's a load from stack
1864       bool IsLoad, IsStore, IsStoreFromReg, IsSimple, IsIndexed;
1865       MCPhysReg Reg;
1866       int32_t SrcImm;
1867       uint16_t StackPtrReg;
1868       int64_t StackOffset;
1869       uint8_t Size;
1870       bool IsStackAccess =
1871           isStackAccess(Inst, IsLoad, IsStore, IsStoreFromReg, Reg, SrcImm,
1872                         StackPtrReg, StackOffset, Size, IsSimple, IsIndexed);
1873       // Prohibit non-stack-based loads
1874       if (!IsStackAccess)
1875         return false;
1876       // If stack memory operands are allowed, check if it's RBP-based
1877       if (!AllowBasePtrStackMemOp &&
1878           RegInfo->isSubRegisterEq(X86::RBP, StackPtrReg))
1879         return false;
1880     }
1881 
1882     unsigned NewOpcode = 0;
1883     switch (Inst.getOpcode()) {
1884     case X86::MOV16rr:
1885       NewOpcode = X86::CMOV16rr;
1886       break;
1887     case X86::MOV16rm:
1888       NewOpcode = X86::CMOV16rm;
1889       break;
1890     case X86::MOV32rr:
1891       NewOpcode = X86::CMOV32rr;
1892       break;
1893     case X86::MOV32rm:
1894       NewOpcode = X86::CMOV32rm;
1895       break;
1896     case X86::MOV64rr:
1897       NewOpcode = X86::CMOV64rr;
1898       break;
1899     case X86::MOV64rm:
1900       NewOpcode = X86::CMOV64rm;
1901       break;
1902     default:
1903       return false;
1904     }
1905     Inst.setOpcode(NewOpcode);
1906     // Insert CC at the end of prime operands, before annotations
1907     Inst.insert(Inst.begin() + MCPlus::getNumPrimeOperands(Inst),
1908                 MCOperand::createImm(CC));
1909     // CMOV is a 3-operand MCInst, so duplicate the destination as src1
1910     Inst.insert(Inst.begin(), Inst.getOperand(0));
1911     return true;
1912   }
1913 
1914   bool lowerTailCall(MCInst &Inst) override {
1915     if (Inst.getOpcode() == X86::JMP_4 && isTailCall(Inst)) {
1916       Inst.setOpcode(X86::JMP_1);
1917       removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall);
1918       return true;
1919     }
1920     return false;
1921   }
1922 
1923   const MCSymbol *getTargetSymbol(const MCInst &Inst,
1924                                   unsigned OpNum = 0) const override {
1925     if (OpNum >= MCPlus::getNumPrimeOperands(Inst))
1926       return nullptr;
1927 
1928     const MCOperand &Op = Inst.getOperand(OpNum);
1929     if (!Op.isExpr())
1930       return nullptr;
1931 
1932     auto *SymExpr = dyn_cast<MCSymbolRefExpr>(Op.getExpr());
1933     if (!SymExpr || SymExpr->getKind() != MCSymbolRefExpr::VK_None)
1934       return nullptr;
1935 
1936     return &SymExpr->getSymbol();
1937   }
1938 
1939   // This is the same as the base class, but since we are overriding one of
1940   // getTargetSymbol's signatures above, we need to override all of them.
1941   const MCSymbol *getTargetSymbol(const MCExpr *Expr) const override {
1942     return &cast<const MCSymbolRefExpr>(Expr)->getSymbol();
1943   }
1944 
1945   bool analyzeBranch(InstructionIterator Begin, InstructionIterator End,
1946                      const MCSymbol *&TBB, const MCSymbol *&FBB,
1947                      MCInst *&CondBranch,
1948                      MCInst *&UncondBranch) const override {
1949     auto I = End;
1950 
1951     // Bottom-up analysis
1952     while (I != Begin) {
1953       --I;
1954 
1955       // Ignore nops and CFIs
1956       if (isPseudo(*I))
1957         continue;
1958 
1959       // Stop when we find the first non-terminator
1960       if (!isTerminator(*I))
1961         break;
1962 
1963       if (!isBranch(*I))
1964         break;
1965 
1966       // Handle unconditional branches.
1967       if ((I->getOpcode() == X86::JMP_1 || I->getOpcode() == X86::JMP_2 ||
1968            I->getOpcode() == X86::JMP_4) &&
1969           !isTailCall(*I)) {
1970         // If any code was seen after this unconditional branch, we've seen
1971         // unreachable code. Ignore them.
1972         CondBranch = nullptr;
1973         UncondBranch = &*I;
1974         const MCSymbol *Sym = getTargetSymbol(*I);
1975         assert(Sym != nullptr &&
1976                "Couldn't extract BB symbol from jump operand");
1977         TBB = Sym;
1978         continue;
1979       }
1980 
1981       // Handle conditional branches and ignore indirect branches
1982       if (!isUnsupportedBranch(I->getOpcode()) &&
1983           getCondCode(*I) == X86::COND_INVALID) {
1984         // Indirect branch
1985         return false;
1986       }
1987 
1988       if (CondBranch == nullptr) {
1989         const MCSymbol *TargetBB = getTargetSymbol(*I);
1990         if (TargetBB == nullptr) {
1991           // Unrecognized branch target
1992           return false;
1993         }
1994         FBB = TBB;
1995         TBB = TargetBB;
1996         CondBranch = &*I;
1997         continue;
1998       }
1999 
2000       llvm_unreachable("multiple conditional branches in one BB");
2001     }
2002     return true;
2003   }
2004 
2005   template <typename Itr>
2006   std::pair<IndirectBranchType, MCInst *>
2007   analyzePICJumpTable(Itr II, Itr IE, MCPhysReg R1, MCPhysReg R2) const {
2008     // Analyze PIC-style jump table code template:
2009     //
2010     //    lea PIC_JUMP_TABLE(%rip), {%r1|%r2}     <- MemLocInstr
2011     //    mov ({%r1|%r2}, %index, 4), {%r2|%r1}
2012     //    add %r2, %r1
2013     //    jmp *%r1
2014     //
2015     // (with any irrelevant instructions in-between)
2016     //
2017     // When we call this helper we've already determined %r1 and %r2, and
2018     // reverse instruction iterator \p II is pointing to the ADD instruction.
2019     //
2020     // PIC jump table looks like following:
2021     //
2022     //   JT:  ----------
2023     //    E1:| L1 - JT  |
2024     //       |----------|
2025     //    E2:| L2 - JT  |
2026     //       |----------|
2027     //       |          |
2028     //          ......
2029     //    En:| Ln - JT  |
2030     //        ----------
2031     //
2032     // Where L1, L2, ..., Ln represent labels in the function.
2033     //
2034     // The actual relocations in the table will be of the form:
2035     //
2036     //   Ln - JT
2037     //    = (Ln - En) + (En - JT)
2038     //    = R_X86_64_PC32(Ln) + En - JT
2039     //    = R_X86_64_PC32(Ln + offsetof(En))
2040     //
2041     LLVM_DEBUG(dbgs() << "Checking for PIC jump table\n");
2042     MCInst *MemLocInstr = nullptr;
2043     const MCInst *MovInstr = nullptr;
2044     while (++II != IE) {
2045       MCInst &Instr = *II;
2046       const MCInstrDesc &InstrDesc = Info->get(Instr.getOpcode());
2047       if (!InstrDesc.hasDefOfPhysReg(Instr, R1, *RegInfo) &&
2048           !InstrDesc.hasDefOfPhysReg(Instr, R2, *RegInfo)) {
2049         // Ignore instructions that don't affect R1, R2 registers.
2050         continue;
2051       }
2052       if (!MovInstr) {
2053         // Expect to see MOV instruction.
2054         if (!isMOVSX64rm32(Instr)) {
2055           LLVM_DEBUG(dbgs() << "MOV instruction expected.\n");
2056           break;
2057         }
2058 
2059         // Check if it's setting %r1 or %r2. In canonical form it sets %r2.
2060         // If it sets %r1 - rename the registers so we have to only check
2061         // a single form.
2062         unsigned MovDestReg = Instr.getOperand(0).getReg();
2063         if (MovDestReg != R2)
2064           std::swap(R1, R2);
2065         if (MovDestReg != R2) {
2066           LLVM_DEBUG(dbgs() << "MOV instruction expected to set %r2\n");
2067           break;
2068         }
2069 
2070         // Verify operands for MOV.
2071         unsigned  BaseRegNum;
2072         int64_t   ScaleValue;
2073         unsigned  IndexRegNum;
2074         int64_t   DispValue;
2075         unsigned  SegRegNum;
2076         if (!evaluateX86MemoryOperand(Instr, &BaseRegNum, &ScaleValue,
2077                                       &IndexRegNum, &DispValue, &SegRegNum))
2078           break;
2079         if (BaseRegNum != R1 || ScaleValue != 4 ||
2080             IndexRegNum == X86::NoRegister || DispValue != 0 ||
2081             SegRegNum != X86::NoRegister)
2082           break;
2083         MovInstr = &Instr;
2084       } else {
2085         if (!InstrDesc.hasDefOfPhysReg(Instr, R1, *RegInfo))
2086           continue;
2087         if (!isLEA64r(Instr)) {
2088           LLVM_DEBUG(dbgs() << "LEA instruction expected\n");
2089           break;
2090         }
2091         if (Instr.getOperand(0).getReg() != R1) {
2092           LLVM_DEBUG(dbgs() << "LEA instruction expected to set %r1\n");
2093           break;
2094         }
2095 
2096         // Verify operands for LEA.
2097         unsigned      BaseRegNum;
2098         int64_t       ScaleValue;
2099         unsigned      IndexRegNum;
2100         const MCExpr *DispExpr = nullptr;
2101         int64_t       DispValue;
2102         unsigned      SegRegNum;
2103         if (!evaluateX86MemoryOperand(Instr, &BaseRegNum, &ScaleValue,
2104                                       &IndexRegNum, &DispValue, &SegRegNum,
2105                                       &DispExpr))
2106           break;
2107         if (BaseRegNum != RegInfo->getProgramCounter() ||
2108             IndexRegNum != X86::NoRegister || SegRegNum != X86::NoRegister ||
2109             DispExpr == nullptr)
2110           break;
2111         MemLocInstr = &Instr;
2112         break;
2113       }
2114     }
2115 
2116     if (!MemLocInstr)
2117       return std::make_pair(IndirectBranchType::UNKNOWN, nullptr);
2118 
2119     LLVM_DEBUG(dbgs() << "checking potential PIC jump table\n");
2120     return std::make_pair(IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE,
2121                           MemLocInstr);
2122   }
2123 
2124   IndirectBranchType analyzeIndirectBranch(
2125       MCInst &Instruction, InstructionIterator Begin, InstructionIterator End,
2126       const unsigned PtrSize, MCInst *&MemLocInstrOut, unsigned &BaseRegNumOut,
2127       unsigned &IndexRegNumOut, int64_t &DispValueOut,
2128       const MCExpr *&DispExprOut, MCInst *&PCRelBaseOut) const override {
2129     // Try to find a (base) memory location from where the address for
2130     // the indirect branch is loaded. For X86-64 the memory will be specified
2131     // in the following format:
2132     //
2133     //   {%rip}/{%basereg} + Imm + IndexReg * Scale
2134     //
2135     // We are interested in the cases where Scale == sizeof(uintptr_t) and
2136     // the contents of the memory are presumably an array of pointers to code.
2137     //
2138     // Normal jump table:
2139     //
2140     //    jmp *(JUMP_TABLE, %index, Scale)        <- MemLocInstr
2141     //
2142     //    or
2143     //
2144     //    mov (JUMP_TABLE, %index, Scale), %r1    <- MemLocInstr
2145     //    ...
2146     //    jmp %r1
2147     //
2148     // We handle PIC-style jump tables separately.
2149     //
2150     MemLocInstrOut = nullptr;
2151     BaseRegNumOut = X86::NoRegister;
2152     IndexRegNumOut = X86::NoRegister;
2153     DispValueOut = 0;
2154     DispExprOut = nullptr;
2155 
2156     std::reverse_iterator<InstructionIterator> II(End);
2157     std::reverse_iterator<InstructionIterator> IE(Begin);
2158 
2159     IndirectBranchType Type = IndirectBranchType::UNKNOWN;
2160 
2161     // An instruction referencing memory used by jump instruction (directly or
2162     // via register). This location could be an array of function pointers
2163     // in case of indirect tail call, or a jump table.
2164     MCInst *MemLocInstr = nullptr;
2165 
2166     if (MCPlus::getNumPrimeOperands(Instruction) == 1) {
2167       // If the indirect jump is on register - try to detect if the
2168       // register value is loaded from a memory location.
2169       assert(Instruction.getOperand(0).isReg() && "register operand expected");
2170       const unsigned R1 = Instruction.getOperand(0).getReg();
2171       // Check if one of the previous instructions defines the jump-on register.
2172       for (auto PrevII = II; PrevII != IE; ++PrevII) {
2173         MCInst &PrevInstr = *PrevII;
2174         const MCInstrDesc &PrevInstrDesc = Info->get(PrevInstr.getOpcode());
2175 
2176         if (!PrevInstrDesc.hasDefOfPhysReg(PrevInstr, R1, *RegInfo))
2177           continue;
2178 
2179         if (isMoveMem2Reg(PrevInstr)) {
2180           MemLocInstr = &PrevInstr;
2181           break;
2182         }
2183         if (isADD64rr(PrevInstr)) {
2184           unsigned R2 = PrevInstr.getOperand(2).getReg();
2185           if (R1 == R2)
2186             return IndirectBranchType::UNKNOWN;
2187           std::tie(Type, MemLocInstr) = analyzePICJumpTable(PrevII, IE, R1, R2);
2188           break;
2189         }
2190         return IndirectBranchType::UNKNOWN;
2191       }
2192       if (!MemLocInstr) {
2193         // No definition seen for the register in this function so far. Could be
2194         // an input parameter - which means it is an external code reference.
2195         // It also could be that the definition happens to be in the code that
2196         // we haven't processed yet. Since we have to be conservative, return
2197         // as UNKNOWN case.
2198         return IndirectBranchType::UNKNOWN;
2199       }
2200     } else {
2201       MemLocInstr = &Instruction;
2202     }
2203 
2204     const MCRegister RIPRegister = RegInfo->getProgramCounter();
2205 
2206     // Analyze the memory location.
2207     unsigned BaseRegNum, IndexRegNum, SegRegNum;
2208     int64_t ScaleValue, DispValue;
2209     const MCExpr *DispExpr;
2210 
2211     if (!evaluateX86MemoryOperand(*MemLocInstr, &BaseRegNum, &ScaleValue,
2212                                   &IndexRegNum, &DispValue, &SegRegNum,
2213                                   &DispExpr))
2214       return IndirectBranchType::UNKNOWN;
2215 
2216     BaseRegNumOut = BaseRegNum;
2217     IndexRegNumOut = IndexRegNum;
2218     DispValueOut = DispValue;
2219     DispExprOut = DispExpr;
2220 
2221     if ((BaseRegNum != X86::NoRegister && BaseRegNum != RIPRegister) ||
2222         SegRegNum != X86::NoRegister)
2223       return IndirectBranchType::UNKNOWN;
2224 
2225     if (MemLocInstr == &Instruction &&
2226         (!ScaleValue || IndexRegNum == X86::NoRegister)) {
2227       MemLocInstrOut = MemLocInstr;
2228       return IndirectBranchType::POSSIBLE_FIXED_BRANCH;
2229     }
2230 
2231     if (Type == IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE &&
2232         (ScaleValue != 1 || BaseRegNum != RIPRegister))
2233       return IndirectBranchType::UNKNOWN;
2234 
2235     if (Type != IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE &&
2236         ScaleValue != PtrSize)
2237       return IndirectBranchType::UNKNOWN;
2238 
2239     MemLocInstrOut = MemLocInstr;
2240 
2241     return Type;
2242   }
2243 
2244   /// Analyze a callsite to see if it could be a virtual method call.  This only
2245   /// checks to see if the overall pattern is satisfied, it does not guarantee
2246   /// that the callsite is a true virtual method call.
2247   /// The format of virtual method calls that are recognized is one of the
2248   /// following:
2249   ///
2250   ///  Form 1: (found in debug code)
2251   ///    add METHOD_OFFSET, %VtableReg
2252   ///    mov (%VtableReg), %MethodReg
2253   ///    ...
2254   ///    call or jmp *%MethodReg
2255   ///
2256   ///  Form 2:
2257   ///    mov METHOD_OFFSET(%VtableReg), %MethodReg
2258   ///    ...
2259   ///    call or jmp *%MethodReg
2260   ///
2261   ///  Form 3:
2262   ///    ...
2263   ///    call or jmp *METHOD_OFFSET(%VtableReg)
2264   ///
2265   bool analyzeVirtualMethodCall(InstructionIterator ForwardBegin,
2266                                 InstructionIterator ForwardEnd,
2267                                 std::vector<MCInst *> &MethodFetchInsns,
2268                                 unsigned &VtableRegNum, unsigned &MethodRegNum,
2269                                 uint64_t &MethodOffset) const override {
2270     VtableRegNum = X86::NoRegister;
2271     MethodRegNum = X86::NoRegister;
2272     MethodOffset = 0;
2273 
2274     std::reverse_iterator<InstructionIterator> Itr(ForwardEnd);
2275     std::reverse_iterator<InstructionIterator> End(ForwardBegin);
2276 
2277     MCInst &CallInst = *Itr++;
2278     assert(isIndirectBranch(CallInst) || isCall(CallInst));
2279 
2280     unsigned BaseReg, IndexReg, SegmentReg;
2281     int64_t Scale, Disp;
2282     const MCExpr *DispExpr;
2283 
2284     // The call can just be jmp offset(reg)
2285     if (evaluateX86MemoryOperand(CallInst, &BaseReg, &Scale, &IndexReg, &Disp,
2286                                  &SegmentReg, &DispExpr)) {
2287       if (!DispExpr && BaseReg != X86::RIP && BaseReg != X86::RBP &&
2288           BaseReg != X86::NoRegister) {
2289         MethodRegNum = BaseReg;
2290         if (Scale == 1 && IndexReg == X86::NoRegister &&
2291             SegmentReg == X86::NoRegister) {
2292           VtableRegNum = MethodRegNum;
2293           MethodOffset = Disp;
2294           MethodFetchInsns.push_back(&CallInst);
2295           return true;
2296         }
2297       }
2298       return false;
2299     }
2300     if (CallInst.getOperand(0).isReg())
2301       MethodRegNum = CallInst.getOperand(0).getReg();
2302     else
2303       return false;
2304 
2305     if (MethodRegNum == X86::RIP || MethodRegNum == X86::RBP) {
2306       VtableRegNum = X86::NoRegister;
2307       MethodRegNum = X86::NoRegister;
2308       return false;
2309     }
2310 
2311     // find load from vtable, this may or may not include the method offset
2312     while (Itr != End) {
2313       MCInst &CurInst = *Itr++;
2314       const MCInstrDesc &Desc = Info->get(CurInst.getOpcode());
2315       if (Desc.hasDefOfPhysReg(CurInst, MethodRegNum, *RegInfo)) {
2316         if (isLoad(CurInst) &&
2317             evaluateX86MemoryOperand(CurInst, &BaseReg, &Scale, &IndexReg,
2318                                      &Disp, &SegmentReg, &DispExpr)) {
2319           if (!DispExpr && Scale == 1 && BaseReg != X86::RIP &&
2320               BaseReg != X86::RBP && BaseReg != X86::NoRegister &&
2321               IndexReg == X86::NoRegister && SegmentReg == X86::NoRegister &&
2322               BaseReg != X86::RIP) {
2323             VtableRegNum = BaseReg;
2324             MethodOffset = Disp;
2325             MethodFetchInsns.push_back(&CurInst);
2326             if (MethodOffset != 0)
2327               return true;
2328             break;
2329           }
2330         }
2331         return false;
2332       }
2333     }
2334 
2335     if (!VtableRegNum)
2336       return false;
2337 
2338     // look for any adds affecting the method register.
2339     while (Itr != End) {
2340       MCInst &CurInst = *Itr++;
2341       const MCInstrDesc &Desc = Info->get(CurInst.getOpcode());
2342       if (Desc.hasDefOfPhysReg(CurInst, VtableRegNum, *RegInfo)) {
2343         if (isADDri(CurInst)) {
2344           assert(!MethodOffset);
2345           MethodOffset = CurInst.getOperand(2).getImm();
2346           MethodFetchInsns.insert(MethodFetchInsns.begin(), &CurInst);
2347           break;
2348         }
2349       }
2350     }
2351 
2352     return true;
2353   }
2354 
2355   bool createStackPointerIncrement(MCInst &Inst, int Size,
2356                                    bool NoFlagsClobber) const override {
2357     if (NoFlagsClobber) {
2358       Inst.setOpcode(X86::LEA64r);
2359       Inst.clear();
2360       Inst.addOperand(MCOperand::createReg(X86::RSP));
2361       Inst.addOperand(MCOperand::createReg(X86::RSP));        // BaseReg
2362       Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
2363       Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
2364       Inst.addOperand(MCOperand::createImm(-Size));           // Displacement
2365       Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
2366       return true;
2367     }
2368     Inst.setOpcode(X86::SUB64ri8);
2369     Inst.clear();
2370     Inst.addOperand(MCOperand::createReg(X86::RSP));
2371     Inst.addOperand(MCOperand::createReg(X86::RSP));
2372     Inst.addOperand(MCOperand::createImm(Size));
2373     return true;
2374   }
2375 
2376   bool createStackPointerDecrement(MCInst &Inst, int Size,
2377                                    bool NoFlagsClobber) const override {
2378     if (NoFlagsClobber) {
2379       Inst.setOpcode(X86::LEA64r);
2380       Inst.clear();
2381       Inst.addOperand(MCOperand::createReg(X86::RSP));
2382       Inst.addOperand(MCOperand::createReg(X86::RSP));        // BaseReg
2383       Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
2384       Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
2385       Inst.addOperand(MCOperand::createImm(Size));            // Displacement
2386       Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
2387       return true;
2388     }
2389     Inst.setOpcode(X86::ADD64ri8);
2390     Inst.clear();
2391     Inst.addOperand(MCOperand::createReg(X86::RSP));
2392     Inst.addOperand(MCOperand::createReg(X86::RSP));
2393     Inst.addOperand(MCOperand::createImm(Size));
2394     return true;
2395   }
2396 
2397   bool createSaveToStack(MCInst &Inst, const MCPhysReg &StackReg, int Offset,
2398                          const MCPhysReg &SrcReg, int Size) const override {
2399     unsigned NewOpcode;
2400     switch (Size) {
2401     default:
2402       return false;
2403     case 2:      NewOpcode = X86::MOV16mr; break;
2404     case 4:      NewOpcode = X86::MOV32mr; break;
2405     case 8:      NewOpcode = X86::MOV64mr; break;
2406     }
2407     Inst.setOpcode(NewOpcode);
2408     Inst.clear();
2409     Inst.addOperand(MCOperand::createReg(StackReg));        // BaseReg
2410     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
2411     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
2412     Inst.addOperand(MCOperand::createImm(Offset));          // Displacement
2413     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
2414     Inst.addOperand(MCOperand::createReg(SrcReg));
2415     return true;
2416   }
2417 
2418   bool createRestoreFromStack(MCInst &Inst, const MCPhysReg &StackReg,
2419                               int Offset, const MCPhysReg &DstReg,
2420                               int Size) const override {
2421     return createLoad(Inst, StackReg, /*Scale=*/1, /*IndexReg=*/X86::NoRegister,
2422                       Offset, nullptr, /*AddrSegmentReg=*/X86::NoRegister,
2423                       DstReg, Size);
2424   }
2425 
2426   bool createLoad(MCInst &Inst, const MCPhysReg &BaseReg, int64_t Scale,
2427                   const MCPhysReg &IndexReg, int64_t Offset,
2428                   const MCExpr *OffsetExpr, const MCPhysReg &AddrSegmentReg,
2429                   const MCPhysReg &DstReg, int Size) const override {
2430     unsigned NewOpcode;
2431     switch (Size) {
2432     default:
2433       return false;
2434     case 2:      NewOpcode = X86::MOV16rm; break;
2435     case 4:      NewOpcode = X86::MOV32rm; break;
2436     case 8:      NewOpcode = X86::MOV64rm; break;
2437     }
2438     Inst.setOpcode(NewOpcode);
2439     Inst.clear();
2440     Inst.addOperand(MCOperand::createReg(DstReg));
2441     Inst.addOperand(MCOperand::createReg(BaseReg));
2442     Inst.addOperand(MCOperand::createImm(Scale));
2443     Inst.addOperand(MCOperand::createReg(IndexReg));
2444     if (OffsetExpr)
2445       Inst.addOperand(MCOperand::createExpr(OffsetExpr)); // Displacement
2446     else
2447       Inst.addOperand(MCOperand::createImm(Offset)); // Displacement
2448     Inst.addOperand(MCOperand::createReg(AddrSegmentReg)); // AddrSegmentReg
2449     return true;
2450   }
2451 
2452   void createLoadImmediate(MCInst &Inst, const MCPhysReg Dest,
2453                            uint32_t Imm) const override {
2454     Inst.setOpcode(X86::MOV64ri32);
2455     Inst.clear();
2456     Inst.addOperand(MCOperand::createReg(Dest));
2457     Inst.addOperand(MCOperand::createImm(Imm));
2458   }
2459 
2460   bool createIncMemory(MCInst &Inst, const MCSymbol *Target,
2461                        MCContext *Ctx) const override {
2462 
2463     Inst.setOpcode(X86::LOCK_INC64m);
2464     Inst.clear();
2465     Inst.addOperand(MCOperand::createReg(X86::RIP));        // BaseReg
2466     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
2467     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
2468 
2469     Inst.addOperand(MCOperand::createExpr(
2470         MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None,
2471                                 *Ctx)));                    // Displacement
2472     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
2473     return true;
2474   }
2475 
2476   bool createIJmp32Frag(SmallVectorImpl<MCInst> &Insts,
2477                         const MCOperand &BaseReg, const MCOperand &Scale,
2478                         const MCOperand &IndexReg, const MCOperand &Offset,
2479                         const MCOperand &TmpReg) const override {
2480     // The code fragment we emit here is:
2481     //
2482     //  mov32 (%base, %index, scale), %tmpreg
2483     //  ijmp *(%tmpreg)
2484     //
2485     MCInst IJmp;
2486     IJmp.setOpcode(X86::JMP64r);
2487     IJmp.addOperand(TmpReg);
2488 
2489     MCInst Load;
2490     Load.setOpcode(X86::MOV32rm);
2491     Load.addOperand(TmpReg);
2492     Load.addOperand(BaseReg);
2493     Load.addOperand(Scale);
2494     Load.addOperand(IndexReg);
2495     Load.addOperand(Offset);
2496     Load.addOperand(MCOperand::createReg(X86::NoRegister));
2497 
2498     Insts.push_back(Load);
2499     Insts.push_back(IJmp);
2500     return true;
2501   }
2502 
2503   bool createNoop(MCInst &Inst) const override {
2504     Inst.setOpcode(X86::NOOP);
2505     return true;
2506   }
2507 
2508   bool createReturn(MCInst &Inst) const override {
2509     Inst.setOpcode(X86::RET64);
2510     return true;
2511   }
2512 
2513   InstructionListType createInlineMemcpy(bool ReturnEnd) const override {
2514     InstructionListType Code;
2515     if (ReturnEnd)
2516       Code.emplace_back(MCInstBuilder(X86::LEA64r)
2517                             .addReg(X86::RAX)
2518                             .addReg(X86::RDI)
2519                             .addImm(1)
2520                             .addReg(X86::RDX)
2521                             .addImm(0)
2522                             .addReg(X86::NoRegister));
2523     else
2524       Code.emplace_back(MCInstBuilder(X86::MOV64rr)
2525                             .addReg(X86::RAX)
2526                             .addReg(X86::RDI));
2527 
2528     Code.emplace_back(MCInstBuilder(X86::MOV32rr)
2529                           .addReg(X86::ECX)
2530                           .addReg(X86::EDX));
2531     Code.emplace_back(MCInstBuilder(X86::REP_MOVSB_64));
2532 
2533     return Code;
2534   }
2535 
2536   InstructionListType createOneByteMemcpy() const override {
2537     InstructionListType Code;
2538     Code.emplace_back(MCInstBuilder(X86::MOV8rm)
2539                           .addReg(X86::CL)
2540                           .addReg(X86::RSI)
2541                           .addImm(0)
2542                           .addReg(X86::NoRegister)
2543                           .addImm(0)
2544                           .addReg(X86::NoRegister));
2545     Code.emplace_back(MCInstBuilder(X86::MOV8mr)
2546                           .addReg(X86::RDI)
2547                           .addImm(0)
2548                           .addReg(X86::NoRegister)
2549                           .addImm(0)
2550                           .addReg(X86::NoRegister)
2551                           .addReg(X86::CL));
2552     Code.emplace_back(MCInstBuilder(X86::MOV64rr)
2553                           .addReg(X86::RAX)
2554                           .addReg(X86::RDI));
2555     return Code;
2556   }
2557 
2558   InstructionListType createCmpJE(MCPhysReg RegNo, int64_t Imm,
2559                                   const MCSymbol *Target,
2560                                   MCContext *Ctx) const override {
2561     InstructionListType Code;
2562     Code.emplace_back(MCInstBuilder(X86::CMP64ri8)
2563                           .addReg(RegNo)
2564                           .addImm(Imm));
2565     Code.emplace_back(MCInstBuilder(X86::JCC_1)
2566                           .addExpr(MCSymbolRefExpr::create(
2567                               Target, MCSymbolRefExpr::VK_None, *Ctx))
2568                           .addImm(X86::COND_E));
2569     return Code;
2570   }
2571 
2572   Optional<Relocation>
2573   createRelocation(const MCFixup &Fixup,
2574                    const MCAsmBackend &MAB) const override {
2575     const MCFixupKindInfo &FKI = MAB.getFixupKindInfo(Fixup.getKind());
2576 
2577     assert(FKI.TargetOffset == 0 && "0-bit relocation offset expected");
2578     const uint64_t RelOffset = Fixup.getOffset();
2579 
2580     uint64_t RelType;
2581     if (FKI.Flags & MCFixupKindInfo::FKF_IsPCRel) {
2582       switch (FKI.TargetSize) {
2583       default:
2584         return NoneType();
2585       case  8: RelType = ELF::R_X86_64_PC8; break;
2586       case 16: RelType = ELF::R_X86_64_PC16; break;
2587       case 32: RelType = ELF::R_X86_64_PC32; break;
2588       case 64: RelType = ELF::R_X86_64_PC64; break;
2589       }
2590     } else {
2591       switch (FKI.TargetSize) {
2592       default:
2593         return NoneType();
2594       case  8: RelType = ELF::R_X86_64_8; break;
2595       case 16: RelType = ELF::R_X86_64_16; break;
2596       case 32: RelType = ELF::R_X86_64_32; break;
2597       case 64: RelType = ELF::R_X86_64_64; break;
2598       }
2599     }
2600 
2601     // Extract a symbol and an addend out of the fixup value expression.
2602     //
2603     // Only the following limited expression types are supported:
2604     //   Symbol + Addend
2605     //   Symbol
2606     uint64_t Addend = 0;
2607     MCSymbol *Symbol = nullptr;
2608     const MCExpr *ValueExpr = Fixup.getValue();
2609     if (ValueExpr->getKind() == MCExpr::Binary) {
2610       const auto *BinaryExpr = cast<MCBinaryExpr>(ValueExpr);
2611       assert(BinaryExpr->getOpcode() == MCBinaryExpr::Add &&
2612              "unexpected binary expression");
2613       const MCExpr *LHS = BinaryExpr->getLHS();
2614       assert(LHS->getKind() == MCExpr::SymbolRef && "unexpected LHS");
2615       Symbol = const_cast<MCSymbol *>(this->getTargetSymbol(LHS));
2616       const MCExpr *RHS = BinaryExpr->getRHS();
2617       assert(RHS->getKind() == MCExpr::Constant && "unexpected RHS");
2618       Addend = cast<MCConstantExpr>(RHS)->getValue();
2619     } else {
2620       assert(ValueExpr->getKind() == MCExpr::SymbolRef && "unexpected value");
2621       Symbol = const_cast<MCSymbol *>(this->getTargetSymbol(ValueExpr));
2622     }
2623 
2624     return Relocation({RelOffset, Symbol, RelType, Addend, 0});
2625   }
2626 
2627   bool replaceImmWithSymbolRef(MCInst &Inst, const MCSymbol *Symbol,
2628                                int64_t Addend, MCContext *Ctx, int64_t &Value,
2629                                uint64_t RelType) const override {
2630     unsigned ImmOpNo = -1U;
2631 
2632     for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst);
2633          ++Index) {
2634       if (Inst.getOperand(Index).isImm()) {
2635         ImmOpNo = Index;
2636         // TODO: this is a bit hacky.  It finds the correct operand by
2637         // searching for a specific immediate value.  If no value is
2638         // provided it defaults to the last immediate operand found.
2639         // This could lead to unexpected results if the instruction
2640         // has more than one immediate with the same value.
2641         if (Inst.getOperand(ImmOpNo).getImm() == Value)
2642           break;
2643       }
2644     }
2645 
2646     if (ImmOpNo == -1U)
2647       return false;
2648 
2649     Value = Inst.getOperand(ImmOpNo).getImm();
2650 
2651     setOperandToSymbolRef(Inst, ImmOpNo, Symbol, Addend, Ctx, RelType);
2652 
2653     return true;
2654   }
2655 
2656   bool replaceRegWithImm(MCInst &Inst, unsigned Register,
2657                          int64_t Imm) const override {
2658 
2659     enum CheckSignExt : uint8_t {
2660       NOCHECK = 0,
2661       CHECK8,
2662       CHECK32,
2663     };
2664 
2665     using CheckList = std::vector<std::pair<CheckSignExt, unsigned>>;
2666     struct InstInfo {
2667       // Size in bytes that Inst loads from memory.
2668       uint8_t DataSize;
2669 
2670       // True when the target operand has to be duplicated because the opcode
2671       // expects a LHS operand.
2672       bool HasLHS;
2673 
2674       // List of checks and corresponding opcodes to be used. We try to use the
2675       // smallest possible immediate value when various sizes are available,
2676       // hence we may need to check whether a larger constant fits in a smaller
2677       // immediate.
2678       CheckList Checks;
2679     };
2680 
2681     InstInfo I;
2682 
2683     switch (Inst.getOpcode()) {
2684     default: {
2685       switch (getPushSize(Inst)) {
2686 
2687       case 2: I = {2, false, {{CHECK8, X86::PUSH16i8}, {NOCHECK, X86::PUSHi16}}}; break;
2688       case 4: I = {4, false, {{CHECK8, X86::PUSH32i8}, {NOCHECK, X86::PUSHi32}}}; break;
2689       case 8: I = {8, false, {{CHECK8, X86::PUSH64i8},
2690                               {CHECK32, X86::PUSH64i32},
2691                               {NOCHECK, Inst.getOpcode()}}}; break;
2692       default: return false;
2693       }
2694       break;
2695     }
2696 
2697     // MOV
2698     case X86::MOV8rr:       I = {1, false, {{NOCHECK, X86::MOV8ri}}}; break;
2699     case X86::MOV16rr:      I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break;
2700     case X86::MOV32rr:      I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break;
2701     case X86::MOV64rr:      I = {8, false, {{CHECK32, X86::MOV64ri32},
2702                                             {NOCHECK, X86::MOV64ri}}}; break;
2703 
2704     case X86::MOV8mr:       I = {1, false, {{NOCHECK, X86::MOV8mi}}}; break;
2705     case X86::MOV16mr:      I = {2, false, {{NOCHECK, X86::MOV16mi}}}; break;
2706     case X86::MOV32mr:      I = {4, false, {{NOCHECK, X86::MOV32mi}}}; break;
2707     case X86::MOV64mr:      I = {8, false, {{CHECK32, X86::MOV64mi32},
2708                                             {NOCHECK, X86::MOV64mr}}}; break;
2709 
2710     // MOVZX
2711     case X86::MOVZX16rr8:   I = {1, false, {{NOCHECK, X86::MOV16ri}}}; break;
2712     case X86::MOVZX32rr8:   I = {1, false, {{NOCHECK, X86::MOV32ri}}}; break;
2713     case X86::MOVZX32rr16:  I = {2, false, {{NOCHECK, X86::MOV32ri}}}; break;
2714 
2715     // CMP
2716     case X86::CMP8rr:       I = {1, false, {{NOCHECK, X86::CMP8ri}}}; break;
2717     case X86::CMP16rr:      I = {2, false, {{CHECK8, X86::CMP16ri8},
2718                                             {NOCHECK, X86::CMP16ri}}}; break;
2719     case X86::CMP32rr:      I = {4, false, {{CHECK8, X86::CMP32ri8},
2720                                             {NOCHECK, X86::CMP32ri}}}; break;
2721     case X86::CMP64rr:      I = {8, false, {{CHECK8, X86::CMP64ri8},
2722                                             {CHECK32, X86::CMP64ri32},
2723                                             {NOCHECK, X86::CMP64rr}}}; break;
2724 
2725     // TEST
2726     case X86::TEST8rr:      I = {1, false, {{NOCHECK, X86::TEST8ri}}}; break;
2727     case X86::TEST16rr:     I = {2, false, {{NOCHECK, X86::TEST16ri}}}; break;
2728     case X86::TEST32rr:     I = {4, false, {{NOCHECK, X86::TEST32ri}}}; break;
2729     case X86::TEST64rr:     I = {8, false, {{CHECK32, X86::TEST64ri32},
2730                                             {NOCHECK, X86::TEST64rr}}}; break;
2731 
2732     // ADD
2733     case X86::ADD8rr:       I = {1, true, {{NOCHECK, X86::ADD8ri}}}; break;
2734     case X86::ADD16rr:      I = {2, true, {{CHECK8, X86::ADD16ri8},
2735                                            {NOCHECK, X86::ADD16ri}}}; break;
2736     case X86::ADD32rr:      I = {4, true, {{CHECK8, X86::ADD32ri8},
2737                                            {NOCHECK, X86::ADD32ri}}}; break;
2738     case X86::ADD64rr:      I = {8, true, {{CHECK8, X86::ADD64ri8},
2739                                            {CHECK32, X86::ADD64ri32},
2740                                            {NOCHECK, X86::ADD64rr}}}; break;
2741 
2742     // SUB
2743     case X86::SUB8rr:       I = {1, true, {{NOCHECK, X86::SUB8ri}}}; break;
2744     case X86::SUB16rr:      I = {2, true, {{CHECK8, X86::SUB16ri8},
2745                                            {NOCHECK, X86::SUB16ri}}}; break;
2746     case X86::SUB32rr:      I = {4, true, {{CHECK8, X86::SUB32ri8},
2747                                            {NOCHECK, X86::SUB32ri}}}; break;
2748     case X86::SUB64rr:      I = {8, true, {{CHECK8, X86::SUB64ri8},
2749                                            {CHECK32, X86::SUB64ri32},
2750                                            {NOCHECK, X86::SUB64rr}}}; break;
2751 
2752     // AND
2753     case X86::AND8rr:       I = {1, true, {{NOCHECK, X86::AND8ri}}}; break;
2754     case X86::AND16rr:      I = {2, true, {{CHECK8, X86::AND16ri8},
2755                                            {NOCHECK, X86::AND16ri}}}; break;
2756     case X86::AND32rr:      I = {4, true, {{CHECK8, X86::AND32ri8},
2757                                            {NOCHECK, X86::AND32ri}}}; break;
2758     case X86::AND64rr:      I = {8, true, {{CHECK8, X86::AND64ri8},
2759                                            {CHECK32, X86::AND64ri32},
2760                                            {NOCHECK, X86::AND64rr}}}; break;
2761 
2762     // OR
2763     case X86::OR8rr:        I = {1, true, {{NOCHECK, X86::OR8ri}}}; break;
2764     case X86::OR16rr:       I = {2, true, {{CHECK8, X86::OR16ri8},
2765                                            {NOCHECK, X86::OR16ri}}}; break;
2766     case X86::OR32rr:       I = {4, true, {{CHECK8, X86::OR32ri8},
2767                                            {NOCHECK, X86::OR32ri}}}; break;
2768     case X86::OR64rr:       I = {8, true, {{CHECK8, X86::OR64ri8},
2769                                            {CHECK32, X86::OR64ri32},
2770                                            {NOCHECK, X86::OR64rr}}}; break;
2771 
2772     // XOR
2773     case X86::XOR8rr:       I = {1, true, {{NOCHECK, X86::XOR8ri}}}; break;
2774     case X86::XOR16rr:      I = {2, true, {{CHECK8, X86::XOR16ri8},
2775                                            {NOCHECK, X86::XOR16ri}}}; break;
2776     case X86::XOR32rr:      I = {4, true, {{CHECK8, X86::XOR32ri8},
2777                                            {NOCHECK, X86::XOR32ri}}}; break;
2778     case X86::XOR64rr:      I = {8, true, {{CHECK8, X86::XOR64ri8},
2779                                            {CHECK32, X86::XOR64ri32},
2780                                            {NOCHECK, X86::XOR64rr}}}; break;
2781     }
2782 
2783     // Compute the new opcode.
2784     unsigned NewOpcode = 0;
2785     for (const std::pair<CheckSignExt, unsigned> &Check : I.Checks) {
2786       NewOpcode = Check.second;
2787       if (Check.first == NOCHECK)
2788         break;
2789       if (Check.first == CHECK8 && isInt<8>(Imm))
2790         break;
2791       if (Check.first == CHECK32 && isInt<32>(Imm))
2792         break;
2793     }
2794     if (NewOpcode == Inst.getOpcode())
2795       return false;
2796 
2797     const MCInstrDesc &InstDesc = Info->get(Inst.getOpcode());
2798 
2799     unsigned NumFound = 0;
2800     for (unsigned Index = InstDesc.getNumDefs() + (I.HasLHS ? 1 : 0),
2801                   E = InstDesc.getNumOperands();
2802          Index != E; ++Index)
2803       if (Inst.getOperand(Index).isReg() &&
2804           Inst.getOperand(Index).getReg() == Register)
2805         NumFound++;
2806 
2807     if (NumFound != 1)
2808       return false;
2809 
2810     MCOperand TargetOp = Inst.getOperand(0);
2811     Inst.clear();
2812     Inst.setOpcode(NewOpcode);
2813     Inst.addOperand(TargetOp);
2814     if (I.HasLHS)
2815       Inst.addOperand(TargetOp);
2816     Inst.addOperand(MCOperand::createImm(Imm));
2817 
2818     return true;
2819   }
2820 
2821   bool replaceRegWithReg(MCInst &Inst, unsigned ToReplace,
2822                          unsigned ReplaceWith) const override {
2823 
2824     // Get the HasLHS value so that iteration can be done
2825     bool HasLHS;
2826     if (X86::isAND(Inst.getOpcode()) || X86::isADD(Inst.getOpcode()) ||
2827         X86::isSUB(Inst.getOpcode())) {
2828       HasLHS = true;
2829     } else if (isPop(Inst) || isPush(Inst) || X86::isCMP(Inst.getOpcode()) ||
2830                X86::isTEST(Inst.getOpcode())) {
2831       HasLHS = false;
2832     } else {
2833       switch (Inst.getOpcode()) {
2834       case X86::MOV8rr:
2835       case X86::MOV8rm:
2836       case X86::MOV8mr:
2837       case X86::MOV8ri:
2838       case X86::MOV16rr:
2839       case X86::MOV16rm:
2840       case X86::MOV16mr:
2841       case X86::MOV16ri:
2842       case X86::MOV32rr:
2843       case X86::MOV32rm:
2844       case X86::MOV32mr:
2845       case X86::MOV32ri:
2846       case X86::MOV64rr:
2847       case X86::MOV64rm:
2848       case X86::MOV64mr:
2849       case X86::MOV64ri:
2850       case X86::MOVZX16rr8:
2851       case X86::MOVZX32rr8:
2852       case X86::MOVZX32rr16:
2853       case X86::MOVSX32rm8:
2854       case X86::MOVSX32rr8:
2855       case X86::MOVSX64rm32:
2856       case X86::LEA64r:
2857         HasLHS = false;
2858         break;
2859       default:
2860         return false;
2861       }
2862     }
2863 
2864     const MCInstrDesc &InstDesc = Info->get(Inst.getOpcode());
2865 
2866     bool FoundOne = false;
2867 
2868     // Iterate only through src operands that arent also dest operands
2869     for (unsigned Index = InstDesc.getNumDefs() + (HasLHS ? 1 : 0),
2870                   E = InstDesc.getNumOperands();
2871          Index != E; ++Index) {
2872       BitVector RegAliases = getAliases(ToReplace, true);
2873       if (!Inst.getOperand(Index).isReg() ||
2874           !RegAliases.test(Inst.getOperand(Index).getReg()))
2875         continue;
2876       // Resize register if needed
2877       unsigned SizedReplaceWith = getAliasSized(
2878           ReplaceWith, getRegSize(Inst.getOperand(Index).getReg()));
2879       MCOperand NewOperand = MCOperand::createReg(SizedReplaceWith);
2880       Inst.getOperand(Index) = NewOperand;
2881       FoundOne = true;
2882     }
2883 
2884     // Return true if at least one operand was replaced
2885     return FoundOne;
2886   }
2887 
2888   bool createUncondBranch(MCInst &Inst, const MCSymbol *TBB,
2889                           MCContext *Ctx) const override {
2890     Inst.setOpcode(X86::JMP_1);
2891     Inst.addOperand(MCOperand::createExpr(
2892         MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)));
2893     return true;
2894   }
2895 
2896   bool createCall(MCInst &Inst, const MCSymbol *Target,
2897                   MCContext *Ctx) override {
2898     Inst.setOpcode(X86::CALL64pcrel32);
2899     Inst.addOperand(MCOperand::createExpr(
2900         MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx)));
2901     return true;
2902   }
2903 
2904   bool createTailCall(MCInst &Inst, const MCSymbol *Target,
2905                       MCContext *Ctx) override {
2906     return createDirectCall(Inst, Target, Ctx, /*IsTailCall*/ true);
2907   }
2908 
2909   void createLongTailCall(InstructionListType &Seq, const MCSymbol *Target,
2910                           MCContext *Ctx) override {
2911     Seq.clear();
2912     Seq.emplace_back();
2913     createDirectCall(Seq.back(), Target, Ctx, /*IsTailCall*/ true);
2914   }
2915 
2916   bool createTrap(MCInst &Inst) const override {
2917     Inst.clear();
2918     Inst.setOpcode(X86::TRAP);
2919     return true;
2920   }
2921 
2922   bool reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB,
2923                               MCContext *Ctx) const override {
2924     unsigned InvCC = getInvertedCondCode(getCondCode(Inst));
2925     assert(InvCC != X86::COND_INVALID && "invalid branch instruction");
2926     Inst.getOperand(Info->get(Inst.getOpcode()).NumOperands - 1).setImm(InvCC);
2927     Inst.getOperand(0) = MCOperand::createExpr(
2928         MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx));
2929     return true;
2930   }
2931 
2932   bool replaceBranchCondition(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx,
2933                               unsigned CC) const override {
2934     if (CC == X86::COND_INVALID)
2935       return false;
2936     Inst.getOperand(Info->get(Inst.getOpcode()).NumOperands - 1).setImm(CC);
2937     Inst.getOperand(0) = MCOperand::createExpr(
2938         MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx));
2939     return true;
2940   }
2941 
2942   unsigned getCanonicalBranchCondCode(unsigned CC) const override {
2943     switch (CC) {
2944     default:           return X86::COND_INVALID;
2945 
2946     case X86::COND_E:  return X86::COND_E;
2947     case X86::COND_NE: return X86::COND_E;
2948 
2949     case X86::COND_L:  return X86::COND_L;
2950     case X86::COND_GE: return X86::COND_L;
2951 
2952     case X86::COND_LE: return X86::COND_G;
2953     case X86::COND_G:  return X86::COND_G;
2954 
2955     case X86::COND_B:  return X86::COND_B;
2956     case X86::COND_AE: return X86::COND_B;
2957 
2958     case X86::COND_BE: return X86::COND_A;
2959     case X86::COND_A:  return X86::COND_A;
2960 
2961     case X86::COND_S:  return X86::COND_S;
2962     case X86::COND_NS: return X86::COND_S;
2963 
2964     case X86::COND_P:  return X86::COND_P;
2965     case X86::COND_NP: return X86::COND_P;
2966 
2967     case X86::COND_O:  return X86::COND_O;
2968     case X86::COND_NO: return X86::COND_O;
2969     }
2970   }
2971 
2972   bool replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB,
2973                            MCContext *Ctx) const override {
2974     assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) &&
2975            "Invalid instruction");
2976     Inst.getOperand(0) = MCOperand::createExpr(
2977         MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx));
2978     return true;
2979   }
2980 
2981   MCPhysReg getX86R11() const override { return X86::R11; }
2982 
2983   MCPhysReg getIntArgRegister(unsigned ArgNo) const override {
2984     // FIXME: this should depend on the calling convention.
2985     switch (ArgNo) {
2986     case 0:   return X86::RDI;
2987     case 1:   return X86::RSI;
2988     case 2:   return X86::RDX;
2989     case 3:   return X86::RCX;
2990     case 4:   return X86::R8;
2991     case 5:   return X86::R9;
2992     default:  return getNoRegister();
2993     }
2994   }
2995 
2996   void createPause(MCInst &Inst) const override {
2997     Inst.clear();
2998     Inst.setOpcode(X86::PAUSE);
2999   }
3000 
3001   void createLfence(MCInst &Inst) const override {
3002     Inst.clear();
3003     Inst.setOpcode(X86::LFENCE);
3004   }
3005 
3006   bool createDirectCall(MCInst &Inst, const MCSymbol *Target, MCContext *Ctx,
3007                         bool IsTailCall) override {
3008     Inst.clear();
3009     Inst.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32);
3010     Inst.addOperand(MCOperand::createExpr(
3011         MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx)));
3012     if (IsTailCall)
3013       setTailCall(Inst);
3014     return true;
3015   }
3016 
3017   void createShortJmp(InstructionListType &Seq, const MCSymbol *Target,
3018                       MCContext *Ctx, bool IsTailCall) override {
3019     Seq.clear();
3020     MCInst Inst;
3021     Inst.setOpcode(X86::JMP_1);
3022     Inst.addOperand(MCOperand::createExpr(
3023         MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx)));
3024     if (IsTailCall)
3025       setTailCall(Inst);
3026     Seq.emplace_back(Inst);
3027   }
3028 
3029   bool isConditionalMove(const MCInst &Inst) const override {
3030     unsigned OpCode = Inst.getOpcode();
3031     return (OpCode == X86::CMOV16rr || OpCode == X86::CMOV32rr ||
3032             OpCode == X86::CMOV64rr);
3033   }
3034 
3035   bool isBranchOnMem(const MCInst &Inst) const override {
3036     unsigned OpCode = Inst.getOpcode();
3037     if (OpCode == X86::CALL64m || (OpCode == X86::JMP32m && isTailCall(Inst)) ||
3038         OpCode == X86::JMP64m)
3039       return true;
3040 
3041     return false;
3042   }
3043 
3044   bool isBranchOnReg(const MCInst &Inst) const override {
3045     unsigned OpCode = Inst.getOpcode();
3046     if (OpCode == X86::CALL64r || (OpCode == X86::JMP32r && isTailCall(Inst)) ||
3047         OpCode == X86::JMP64r)
3048       return true;
3049 
3050     return false;
3051   }
3052 
3053   void createPushRegister(MCInst &Inst, MCPhysReg Reg,
3054                           unsigned Size) const override {
3055     Inst.clear();
3056     unsigned NewOpcode = 0;
3057     if (Reg == X86::EFLAGS) {
3058       switch (Size) {
3059       case 2: NewOpcode = X86::PUSHF16;  break;
3060       case 4: NewOpcode = X86::PUSHF32;  break;
3061       case 8: NewOpcode = X86::PUSHF64;  break;
3062       default:
3063         llvm_unreachable("Unexpected size");
3064       }
3065       Inst.setOpcode(NewOpcode);
3066       return;
3067     }
3068     switch (Size) {
3069     case 2: NewOpcode = X86::PUSH16r;  break;
3070     case 4: NewOpcode = X86::PUSH32r;  break;
3071     case 8: NewOpcode = X86::PUSH64r;  break;
3072     default:
3073       llvm_unreachable("Unexpected size");
3074     }
3075     Inst.setOpcode(NewOpcode);
3076     Inst.addOperand(MCOperand::createReg(Reg));
3077   }
3078 
3079   void createPopRegister(MCInst &Inst, MCPhysReg Reg,
3080                          unsigned Size) const override {
3081     Inst.clear();
3082     unsigned NewOpcode = 0;
3083     if (Reg == X86::EFLAGS) {
3084       switch (Size) {
3085       case 2: NewOpcode = X86::POPF16;  break;
3086       case 4: NewOpcode = X86::POPF32;  break;
3087       case 8: NewOpcode = X86::POPF64;  break;
3088       default:
3089         llvm_unreachable("Unexpected size");
3090       }
3091       Inst.setOpcode(NewOpcode);
3092       return;
3093     }
3094     switch (Size) {
3095     case 2: NewOpcode = X86::POP16r;  break;
3096     case 4: NewOpcode = X86::POP32r;  break;
3097     case 8: NewOpcode = X86::POP64r;  break;
3098     default:
3099       llvm_unreachable("Unexpected size");
3100     }
3101     Inst.setOpcode(NewOpcode);
3102     Inst.addOperand(MCOperand::createReg(Reg));
3103   }
3104 
3105   void createPushFlags(MCInst &Inst, unsigned Size) const override {
3106     return createPushRegister(Inst, X86::EFLAGS, Size);
3107   }
3108 
3109   void createPopFlags(MCInst &Inst, unsigned Size) const override {
3110     return createPopRegister(Inst, X86::EFLAGS, Size);
3111   }
3112 
3113   void createAddRegImm(MCInst &Inst, MCPhysReg Reg, int64_t Value,
3114                        unsigned Size) const {
3115     unsigned int Opcode;
3116     switch (Size) {
3117     case 1: Opcode = X86::ADD8ri; break;
3118     case 2: Opcode = X86::ADD16ri; break;
3119     case 4: Opcode = X86::ADD32ri; break;
3120     default:
3121       llvm_unreachable("Unexpected size");
3122     }
3123     Inst.setOpcode(Opcode);
3124     Inst.clear();
3125     Inst.addOperand(MCOperand::createReg(Reg));
3126     Inst.addOperand(MCOperand::createReg(Reg));
3127     Inst.addOperand(MCOperand::createImm(Value));
3128   }
3129 
3130   void createClearRegWithNoEFlagsUpdate(MCInst &Inst, MCPhysReg Reg,
3131                                         unsigned Size) const {
3132     unsigned int Opcode;
3133     switch (Size) {
3134     case 1: Opcode = X86::MOV8ri; break;
3135     case 2: Opcode = X86::MOV16ri; break;
3136     case 4: Opcode = X86::MOV32ri; break;
3137     case 8: Opcode = X86::MOV64ri; break;
3138     default:
3139       llvm_unreachable("Unexpected size");
3140     }
3141     Inst.setOpcode(Opcode);
3142     Inst.clear();
3143     Inst.addOperand(MCOperand::createReg(Reg));
3144     Inst.addOperand(MCOperand::createImm(0));
3145   }
3146 
3147   void createX86SaveOVFlagToRegister(MCInst &Inst, MCPhysReg Reg) const {
3148     Inst.setOpcode(X86::SETCCr);
3149     Inst.clear();
3150     Inst.addOperand(MCOperand::createReg(Reg));
3151     Inst.addOperand(MCOperand::createImm(X86::COND_O));
3152   }
3153 
3154   void createX86Lahf(MCInst &Inst) const {
3155     Inst.setOpcode(X86::LAHF);
3156     Inst.clear();
3157   }
3158 
3159   void createX86Sahf(MCInst &Inst) const {
3160     Inst.setOpcode(X86::SAHF);
3161     Inst.clear();
3162   }
3163 
3164   void createInstrIncMemory(InstructionListType &Instrs, const MCSymbol *Target,
3165                             MCContext *Ctx, bool IsLeaf) const override {
3166     unsigned int I = 0;
3167 
3168     Instrs.resize(IsLeaf ? 13 : 11);
3169     // Don't clobber application red zone (ABI dependent)
3170     if (IsLeaf)
3171       createStackPointerIncrement(Instrs[I++], 128,
3172                                   /*NoFlagsClobber=*/true);
3173 
3174     // Performance improvements based on the optimization discussed at
3175     // https://reviews.llvm.org/D6629
3176     // LAHF/SAHF are used instead of PUSHF/POPF
3177     // PUSHF
3178     createPushRegister(Instrs[I++], X86::RAX, 8);
3179     createClearRegWithNoEFlagsUpdate(Instrs[I++], X86::RAX, 8);
3180     createX86Lahf(Instrs[I++]);
3181     createPushRegister(Instrs[I++], X86::RAX, 8);
3182     createClearRegWithNoEFlagsUpdate(Instrs[I++], X86::RAX, 8);
3183     createX86SaveOVFlagToRegister(Instrs[I++], X86::AL);
3184     // LOCK INC
3185     createIncMemory(Instrs[I++], Target, Ctx);
3186     // POPF
3187     createAddRegImm(Instrs[I++], X86::AL, 127, 1);
3188     createPopRegister(Instrs[I++], X86::RAX, 8);
3189     createX86Sahf(Instrs[I++]);
3190     createPopRegister(Instrs[I++], X86::RAX, 8);
3191 
3192     if (IsLeaf)
3193       createStackPointerDecrement(Instrs[I], 128,
3194                                   /*NoFlagsClobber=*/true);
3195   }
3196 
3197   void createSwap(MCInst &Inst, MCPhysReg Source, MCPhysReg MemBaseReg,
3198                   int64_t Disp) const {
3199     Inst.setOpcode(X86::XCHG64rm);
3200     Inst.addOperand(MCOperand::createReg(Source));
3201     Inst.addOperand(MCOperand::createReg(Source));
3202     Inst.addOperand(MCOperand::createReg(MemBaseReg));      // BaseReg
3203     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
3204     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
3205     Inst.addOperand(MCOperand::createImm(Disp));            // Displacement
3206     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
3207   }
3208 
3209   void createIndirectBranch(MCInst &Inst, MCPhysReg MemBaseReg,
3210                             int64_t Disp) const {
3211     Inst.setOpcode(X86::JMP64m);
3212     Inst.addOperand(MCOperand::createReg(MemBaseReg));      // BaseReg
3213     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
3214     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
3215     Inst.addOperand(MCOperand::createImm(Disp));            // Displacement
3216     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
3217   }
3218 
3219   InstructionListType createInstrumentedIndirectCall(const MCInst &CallInst,
3220                                                      bool TailCall,
3221                                                      MCSymbol *HandlerFuncAddr,
3222                                                      int CallSiteID,
3223                                                      MCContext *Ctx) override {
3224     // Check if the target address expression used in the original indirect call
3225     // uses the stack pointer, which we are going to clobber.
3226     static BitVector SPAliases(getAliases(X86::RSP));
3227     bool UsesSP = false;
3228     // Skip defs.
3229     for (unsigned I = Info->get(CallInst.getOpcode()).getNumDefs(),
3230                   E = MCPlus::getNumPrimeOperands(CallInst);
3231          I != E; ++I) {
3232       const MCOperand &Operand = CallInst.getOperand(I);
3233       if (Operand.isReg() && SPAliases[Operand.getReg()]) {
3234         UsesSP = true;
3235         break;
3236       }
3237     }
3238 
3239     InstructionListType Insts;
3240     MCPhysReg TempReg = getIntArgRegister(0);
3241     // Code sequence used to enter indirect call instrumentation helper:
3242     //   push %rdi
3243     //   add $8, %rsp       ;; $rsp may be used in target, so fix it to prev val
3244     //   movq target, %rdi  ;; via convertIndirectCallTargetToLoad
3245     //   sub $8, %rsp       ;; restore correct stack value
3246     //   push %rdi
3247     //   movq $CallSiteID, %rdi
3248     //   push %rdi
3249     //   callq/jmp HandlerFuncAddr
3250     Insts.emplace_back();
3251     createPushRegister(Insts.back(), TempReg, 8);
3252     if (UsesSP) { // Only adjust SP if we really need to
3253       Insts.emplace_back();
3254       createStackPointerDecrement(Insts.back(), 8, /*NoFlagsClobber=*/false);
3255     }
3256     Insts.emplace_back(CallInst);
3257     // Insts.back() and CallInst now share the same annotation instruction.
3258     // Strip it from Insts.back(), only preserving tail call annotation.
3259     stripAnnotations(Insts.back(), /*KeepTC=*/true);
3260     convertIndirectCallToLoad(Insts.back(), TempReg);
3261     if (UsesSP) {
3262       Insts.emplace_back();
3263       createStackPointerIncrement(Insts.back(), 8, /*NoFlagsClobber=*/false);
3264     }
3265     Insts.emplace_back();
3266     createPushRegister(Insts.back(), TempReg, 8);
3267     Insts.emplace_back();
3268     createLoadImmediate(Insts.back(), TempReg, CallSiteID);
3269     Insts.emplace_back();
3270     createPushRegister(Insts.back(), TempReg, 8);
3271     Insts.emplace_back();
3272     createDirectCall(Insts.back(), HandlerFuncAddr, Ctx,
3273                      /*TailCall=*/TailCall);
3274     // Carry over metadata
3275     for (int I = MCPlus::getNumPrimeOperands(CallInst),
3276              E = CallInst.getNumOperands();
3277          I != E; ++I)
3278       Insts.back().addOperand(CallInst.getOperand(I));
3279 
3280     return Insts;
3281   }
3282 
3283   InstructionListType createInstrumentedIndCallHandlerExitBB() const override {
3284     const MCPhysReg TempReg = getIntArgRegister(0);
3285     // We just need to undo the sequence created for every ind call in
3286     // instrumentIndirectTarget(), which can be accomplished minimally with:
3287     //   popfq
3288     //   pop %rdi
3289     //   add $16, %rsp
3290     //   xchg (%rsp), %rdi
3291     //   jmp *-8(%rsp)
3292     InstructionListType Insts(5);
3293     createPopFlags(Insts[0], 8);
3294     createPopRegister(Insts[1], TempReg, 8);
3295     createStackPointerDecrement(Insts[2], 16, /*NoFlagsClobber=*/false);
3296     createSwap(Insts[3], TempReg, X86::RSP, 0);
3297     createIndirectBranch(Insts[4], X86::RSP, -8);
3298     return Insts;
3299   }
3300 
3301   InstructionListType
3302   createInstrumentedIndTailCallHandlerExitBB() const override {
3303     const MCPhysReg TempReg = getIntArgRegister(0);
3304     // Same thing as above, but for tail calls
3305     //   popfq
3306     //   add $16, %rsp
3307     //   pop %rdi
3308     //   jmp *-16(%rsp)
3309     InstructionListType Insts(4);
3310     createPopFlags(Insts[0], 8);
3311     createStackPointerDecrement(Insts[1], 16, /*NoFlagsClobber=*/false);
3312     createPopRegister(Insts[2], TempReg, 8);
3313     createIndirectBranch(Insts[3], X86::RSP, -16);
3314     return Insts;
3315   }
3316 
3317   InstructionListType
3318   createInstrumentedIndCallHandlerEntryBB(const MCSymbol *InstrTrampoline,
3319                                           const MCSymbol *IndCallHandler,
3320                                           MCContext *Ctx) override {
3321     const MCPhysReg TempReg = getIntArgRegister(0);
3322     // Code sequence used to check whether InstrTampoline was initialized
3323     // and call it if so, returns via IndCallHandler.
3324     //   pushfq
3325     //   mov    InstrTrampoline,%rdi
3326     //   cmp    $0x0,%rdi
3327     //   je     IndCallHandler
3328     //   callq  *%rdi
3329     //   jmpq   IndCallHandler
3330     InstructionListType Insts;
3331     Insts.emplace_back();
3332     createPushFlags(Insts.back(), 8);
3333     Insts.emplace_back();
3334     createMove(Insts.back(), InstrTrampoline, TempReg, Ctx);
3335     InstructionListType cmpJmp = createCmpJE(TempReg, 0, IndCallHandler, Ctx);
3336     Insts.insert(Insts.end(), cmpJmp.begin(), cmpJmp.end());
3337     Insts.emplace_back();
3338     Insts.back().setOpcode(X86::CALL64r);
3339     Insts.back().addOperand(MCOperand::createReg(TempReg));
3340     Insts.emplace_back();
3341     createDirectCall(Insts.back(), IndCallHandler, Ctx, /*IsTailCall*/ true);
3342     return Insts;
3343   }
3344 
3345   InstructionListType createNumCountersGetter(MCContext *Ctx) const override {
3346     InstructionListType Insts(2);
3347     MCSymbol *NumLocs = Ctx->getOrCreateSymbol("__bolt_num_counters");
3348     createMove(Insts[0], NumLocs, X86::EAX, Ctx);
3349     createReturn(Insts[1]);
3350     return Insts;
3351   }
3352 
3353   InstructionListType
3354   createInstrLocationsGetter(MCContext *Ctx) const override {
3355     InstructionListType Insts(2);
3356     MCSymbol *Locs = Ctx->getOrCreateSymbol("__bolt_instr_locations");
3357     createLea(Insts[0], Locs, X86::EAX, Ctx);
3358     createReturn(Insts[1]);
3359     return Insts;
3360   }
3361 
3362   InstructionListType createInstrTablesGetter(MCContext *Ctx) const override {
3363     InstructionListType Insts(2);
3364     MCSymbol *Locs = Ctx->getOrCreateSymbol("__bolt_instr_tables");
3365     createLea(Insts[0], Locs, X86::EAX, Ctx);
3366     createReturn(Insts[1]);
3367     return Insts;
3368   }
3369 
3370   InstructionListType createInstrNumFuncsGetter(MCContext *Ctx) const override {
3371     InstructionListType Insts(2);
3372     MCSymbol *NumFuncs = Ctx->getOrCreateSymbol("__bolt_instr_num_funcs");
3373     createMove(Insts[0], NumFuncs, X86::EAX, Ctx);
3374     createReturn(Insts[1]);
3375     return Insts;
3376   }
3377 
3378   InstructionListType createSymbolTrampoline(const MCSymbol *TgtSym,
3379                                              MCContext *Ctx) const override {
3380     InstructionListType Insts(1);
3381     createUncondBranch(Insts[0], TgtSym, Ctx);
3382     return Insts;
3383   }
3384 
3385   InstructionListType createDummyReturnFunction(MCContext *Ctx) const override {
3386     InstructionListType Insts(1);
3387     createReturn(Insts[0]);
3388     return Insts;
3389   }
3390 
3391   BlocksVectorTy indirectCallPromotion(
3392       const MCInst &CallInst,
3393       const std::vector<std::pair<MCSymbol *, uint64_t>> &Targets,
3394       const std::vector<std::pair<MCSymbol *, uint64_t>> &VtableSyms,
3395       const std::vector<MCInst *> &MethodFetchInsns,
3396       const bool MinimizeCodeSize, MCContext *Ctx) override {
3397     const bool IsTailCall = isTailCall(CallInst);
3398     const bool IsJumpTable = getJumpTable(CallInst) != 0;
3399     BlocksVectorTy Results;
3400 
3401     // Label for the current code block.
3402     MCSymbol *NextTarget = nullptr;
3403 
3404     // The join block which contains all the instructions following CallInst.
3405     // MergeBlock remains null if CallInst is a tail call.
3406     MCSymbol *MergeBlock = nullptr;
3407 
3408     unsigned FuncAddrReg = X86::R10;
3409 
3410     const bool LoadElim = !VtableSyms.empty();
3411     assert((!LoadElim || VtableSyms.size() == Targets.size()) &&
3412            "There must be a vtable entry for every method "
3413            "in the targets vector.");
3414 
3415     if (MinimizeCodeSize && !LoadElim) {
3416       std::set<unsigned> UsedRegs;
3417 
3418       for (unsigned int I = 0; I < MCPlus::getNumPrimeOperands(CallInst); ++I) {
3419         const MCOperand &Op = CallInst.getOperand(I);
3420         if (Op.isReg())
3421           UsedRegs.insert(Op.getReg());
3422       }
3423 
3424       if (UsedRegs.count(X86::R10) == 0)
3425         FuncAddrReg = X86::R10;
3426       else if (UsedRegs.count(X86::R11) == 0)
3427         FuncAddrReg = X86::R11;
3428       else
3429         return Results;
3430     }
3431 
3432     const auto jumpToMergeBlock = [&](InstructionListType &NewCall) {
3433       assert(MergeBlock);
3434       NewCall.push_back(CallInst);
3435       MCInst &Merge = NewCall.back();
3436       Merge.clear();
3437       createUncondBranch(Merge, MergeBlock, Ctx);
3438     };
3439 
3440     for (unsigned int i = 0; i < Targets.size(); ++i) {
3441       Results.emplace_back(NextTarget, InstructionListType());
3442       InstructionListType *NewCall = &Results.back().second;
3443 
3444       if (MinimizeCodeSize && !LoadElim) {
3445         // Load the call target into FuncAddrReg.
3446         NewCall->push_back(CallInst); // Copy CallInst in order to get SMLoc
3447         MCInst &Target = NewCall->back();
3448         Target.clear();
3449         Target.setOpcode(X86::MOV64ri32);
3450         Target.addOperand(MCOperand::createReg(FuncAddrReg));
3451         if (Targets[i].first) {
3452           // Is this OK?
3453           Target.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
3454               Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
3455         } else {
3456           const uint64_t Addr = Targets[i].second;
3457           // Immediate address is out of sign extended 32 bit range.
3458           if (int64_t(Addr) != int64_t(int32_t(Addr)))
3459             return BlocksVectorTy();
3460 
3461           Target.addOperand(MCOperand::createImm(Addr));
3462         }
3463 
3464         // Compare current call target to a specific address.
3465         NewCall->push_back(CallInst);
3466         MCInst &Compare = NewCall->back();
3467         Compare.clear();
3468         if (isBranchOnReg(CallInst))
3469           Compare.setOpcode(X86::CMP64rr);
3470         else if (CallInst.getOpcode() == X86::CALL64pcrel32)
3471           Compare.setOpcode(X86::CMP64ri32);
3472         else
3473           Compare.setOpcode(X86::CMP64rm);
3474 
3475         Compare.addOperand(MCOperand::createReg(FuncAddrReg));
3476 
3477         // TODO: Would be preferable to only load this value once.
3478         for (unsigned i = 0;
3479              i < Info->get(CallInst.getOpcode()).getNumOperands(); ++i)
3480           if (!CallInst.getOperand(i).isInst())
3481             Compare.addOperand(CallInst.getOperand(i));
3482       } else {
3483         // Compare current call target to a specific address.
3484         NewCall->push_back(CallInst);
3485         MCInst &Compare = NewCall->back();
3486         Compare.clear();
3487         if (isBranchOnReg(CallInst))
3488           Compare.setOpcode(X86::CMP64ri32);
3489         else
3490           Compare.setOpcode(X86::CMP64mi32);
3491 
3492         // Original call address.
3493         for (unsigned i = 0;
3494              i < Info->get(CallInst.getOpcode()).getNumOperands(); ++i)
3495           if (!CallInst.getOperand(i).isInst())
3496             Compare.addOperand(CallInst.getOperand(i));
3497 
3498         // Target address.
3499         if (Targets[i].first || LoadElim) {
3500           const MCSymbol *Sym =
3501               LoadElim ? VtableSyms[i].first : Targets[i].first;
3502           const uint64_t Addend = LoadElim ? VtableSyms[i].second : 0;
3503           const MCExpr *Expr = MCSymbolRefExpr::create(Sym, *Ctx);
3504           if (Addend)
3505             Expr = MCBinaryExpr::createAdd(
3506                 Expr, MCConstantExpr::create(Addend, *Ctx), *Ctx);
3507           Compare.addOperand(MCOperand::createExpr(Expr));
3508         } else {
3509           const uint64_t Addr = Targets[i].second;
3510           // Immediate address is out of sign extended 32 bit range.
3511           if (int64_t(Addr) != int64_t(int32_t(Addr)))
3512             return BlocksVectorTy();
3513 
3514           Compare.addOperand(MCOperand::createImm(Addr));
3515         }
3516       }
3517 
3518       // jump to next target compare.
3519       NextTarget =
3520           Ctx->createNamedTempSymbol(); // generate label for the next block
3521       NewCall->push_back(CallInst);
3522 
3523       if (IsJumpTable) {
3524         MCInst &Je = NewCall->back();
3525 
3526         // Jump to next compare if target addresses don't match.
3527         Je.clear();
3528         Je.setOpcode(X86::JCC_1);
3529         if (Targets[i].first)
3530           Je.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
3531               Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
3532         else
3533           Je.addOperand(MCOperand::createImm(Targets[i].second));
3534 
3535         Je.addOperand(MCOperand::createImm(X86::COND_E));
3536         assert(!isInvoke(CallInst));
3537       } else {
3538         MCInst &Jne = NewCall->back();
3539 
3540         // Jump to next compare if target addresses don't match.
3541         Jne.clear();
3542         Jne.setOpcode(X86::JCC_1);
3543         Jne.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
3544             NextTarget, MCSymbolRefExpr::VK_None, *Ctx)));
3545         Jne.addOperand(MCOperand::createImm(X86::COND_NE));
3546 
3547         // Call specific target directly.
3548         Results.emplace_back(Ctx->createNamedTempSymbol(),
3549                              InstructionListType());
3550         NewCall = &Results.back().second;
3551         NewCall->push_back(CallInst);
3552         MCInst &CallOrJmp = NewCall->back();
3553 
3554         CallOrJmp.clear();
3555 
3556         if (MinimizeCodeSize && !LoadElim) {
3557           CallOrJmp.setOpcode(IsTailCall ? X86::JMP32r : X86::CALL64r);
3558           CallOrJmp.addOperand(MCOperand::createReg(FuncAddrReg));
3559         } else {
3560           CallOrJmp.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32);
3561 
3562           if (Targets[i].first)
3563             CallOrJmp.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
3564                 Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
3565           else
3566             CallOrJmp.addOperand(MCOperand::createImm(Targets[i].second));
3567         }
3568         if (IsTailCall)
3569           setTailCall(CallOrJmp);
3570 
3571         if (CallOrJmp.getOpcode() == X86::CALL64r ||
3572             CallOrJmp.getOpcode() == X86::CALL64pcrel32) {
3573           if (Optional<uint32_t> Offset = getOffset(CallInst))
3574             // Annotated as duplicated call
3575             setOffset(CallOrJmp, *Offset);
3576         }
3577 
3578         if (isInvoke(CallInst) && !isInvoke(CallOrJmp)) {
3579           // Copy over any EH or GNU args size information from the original
3580           // call.
3581           Optional<MCPlus::MCLandingPad> EHInfo = getEHInfo(CallInst);
3582           if (EHInfo)
3583             addEHInfo(CallOrJmp, *EHInfo);
3584           int64_t GnuArgsSize = getGnuArgsSize(CallInst);
3585           if (GnuArgsSize >= 0)
3586             addGnuArgsSize(CallOrJmp, GnuArgsSize);
3587         }
3588 
3589         if (!IsTailCall) {
3590           // The fallthrough block for the most common target should be
3591           // the merge block.
3592           if (i == 0) {
3593             // Fallthrough to merge block.
3594             MergeBlock = Ctx->createNamedTempSymbol();
3595           } else {
3596             // Insert jump to the merge block if we are not doing a fallthrough.
3597             jumpToMergeBlock(*NewCall);
3598           }
3599         }
3600       }
3601     }
3602 
3603     // Cold call block.
3604     Results.emplace_back(NextTarget, InstructionListType());
3605     InstructionListType &NewCall = Results.back().second;
3606     for (const MCInst *Inst : MethodFetchInsns)
3607       if (Inst != &CallInst)
3608         NewCall.push_back(*Inst);
3609     NewCall.push_back(CallInst);
3610 
3611     // Jump to merge block from cold call block
3612     if (!IsTailCall && !IsJumpTable) {
3613       jumpToMergeBlock(NewCall);
3614 
3615       // Record merge block
3616       Results.emplace_back(MergeBlock, InstructionListType());
3617     }
3618 
3619     return Results;
3620   }
3621 
3622   BlocksVectorTy jumpTablePromotion(
3623       const MCInst &IJmpInst,
3624       const std::vector<std::pair<MCSymbol *, uint64_t>> &Targets,
3625       const std::vector<MCInst *> &TargetFetchInsns,
3626       MCContext *Ctx) const override {
3627     assert(getJumpTable(IJmpInst) != 0);
3628     uint16_t IndexReg = getAnnotationAs<uint16_t>(IJmpInst, "JTIndexReg");
3629     if (IndexReg == 0)
3630       return BlocksVectorTy();
3631 
3632     BlocksVectorTy Results;
3633 
3634     // Label for the current code block.
3635     MCSymbol *NextTarget = nullptr;
3636 
3637     for (unsigned int i = 0; i < Targets.size(); ++i) {
3638       Results.emplace_back(NextTarget, InstructionListType());
3639       InstructionListType *CurBB = &Results.back().second;
3640 
3641       // Compare current index to a specific index.
3642       CurBB->emplace_back(MCInst());
3643       MCInst &CompareInst = CurBB->back();
3644       CompareInst.setLoc(IJmpInst.getLoc());
3645       CompareInst.setOpcode(X86::CMP64ri32);
3646       CompareInst.addOperand(MCOperand::createReg(IndexReg));
3647 
3648       const uint64_t CaseIdx = Targets[i].second;
3649       // Immediate address is out of sign extended 32 bit range.
3650       if (int64_t(CaseIdx) != int64_t(int32_t(CaseIdx)))
3651         return BlocksVectorTy();
3652 
3653       CompareInst.addOperand(MCOperand::createImm(CaseIdx));
3654       shortenInstruction(CompareInst, *Ctx->getSubtargetInfo());
3655 
3656       // jump to next target compare.
3657       NextTarget =
3658           Ctx->createNamedTempSymbol(); // generate label for the next block
3659       CurBB->push_back(MCInst());
3660 
3661       MCInst &JEInst = CurBB->back();
3662       JEInst.setLoc(IJmpInst.getLoc());
3663 
3664       // Jump to target if indices match
3665       JEInst.setOpcode(X86::JCC_1);
3666       JEInst.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
3667           Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
3668       JEInst.addOperand(MCOperand::createImm(X86::COND_E));
3669     }
3670 
3671     // Cold call block.
3672     Results.emplace_back(NextTarget, InstructionListType());
3673     InstructionListType &CurBB = Results.back().second;
3674     for (const MCInst *Inst : TargetFetchInsns)
3675       if (Inst != &IJmpInst)
3676         CurBB.push_back(*Inst);
3677 
3678     CurBB.push_back(IJmpInst);
3679 
3680     return Results;
3681   }
3682 
3683 private:
3684   bool createMove(MCInst &Inst, const MCSymbol *Src, unsigned Reg,
3685                   MCContext *Ctx) const {
3686     Inst.setOpcode(X86::MOV64rm);
3687     Inst.addOperand(MCOperand::createReg(Reg));
3688     Inst.addOperand(MCOperand::createReg(X86::RIP));        // BaseReg
3689     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
3690     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
3691     Inst.addOperand(MCOperand::createExpr(
3692         MCSymbolRefExpr::create(Src, MCSymbolRefExpr::VK_None,
3693                                 *Ctx)));                    // Displacement
3694     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
3695 
3696     return true;
3697   }
3698 
3699   bool createLea(MCInst &Inst, const MCSymbol *Src, unsigned Reg,
3700                  MCContext *Ctx) const {
3701     Inst.setOpcode(X86::LEA64r);
3702     Inst.addOperand(MCOperand::createReg(Reg));
3703     Inst.addOperand(MCOperand::createReg(X86::RIP));        // BaseReg
3704     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
3705     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
3706     Inst.addOperand(MCOperand::createExpr(
3707         MCSymbolRefExpr::create(Src, MCSymbolRefExpr::VK_None,
3708                                 *Ctx)));                    // Displacement
3709     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
3710     return true;
3711   }
3712 };
3713 
3714 } // namespace
3715 
3716 namespace llvm {
3717 namespace bolt {
3718 
3719 MCPlusBuilder *createX86MCPlusBuilder(const MCInstrAnalysis *Analysis,
3720                                       const MCInstrInfo *Info,
3721                                       const MCRegisterInfo *RegInfo) {
3722   return new X86MCPlusBuilder(Analysis, Info, RegInfo);
3723 }
3724 
3725 } // namespace bolt
3726 } // namespace llvm
3727