1 //===- bolt/Target/X86/X86MCPlusBuilder.cpp -------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file provides X86-specific MCPlus builder.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "MCTargetDesc/X86BaseInfo.h"
14 #include "MCTargetDesc/X86InstrRelaxTables.h"
15 #include "MCTargetDesc/X86MCTargetDesc.h"
16 #include "bolt/Core/MCPlus.h"
17 #include "bolt/Core/MCPlusBuilder.h"
18 #include "llvm/BinaryFormat/ELF.h"
19 #include "llvm/MC/MCContext.h"
20 #include "llvm/MC/MCFixupKindInfo.h"
21 #include "llvm/MC/MCInst.h"
22 #include "llvm/MC/MCInstBuilder.h"
23 #include "llvm/MC/MCInstrInfo.h"
24 #include "llvm/MC/MCRegister.h"
25 #include "llvm/MC/MCRegisterInfo.h"
26 #include "llvm/Support/CommandLine.h"
27 #include "llvm/Support/DataExtractor.h"
28 #include "llvm/Support/Debug.h"
29 #include "llvm/Support/Errc.h"
30 #include "llvm/Support/ErrorHandling.h"
31 #include "llvm/Support/ErrorOr.h"
32 #include <set>
33 
34 #define DEBUG_TYPE "mcplus"
35 
36 using namespace llvm;
37 using namespace bolt;
38 
39 namespace opts {
40 
41 extern cl::OptionCategory BoltOptCategory;
42 
43 static cl::opt<bool> X86StripRedundantAddressSize(
44     "x86-strip-redundant-address-size",
45     cl::desc("Remove redundant Address-Size override prefix"), cl::init(true),
46     cl::ZeroOrMore, cl::cat(BoltOptCategory));
47 
48 } // namespace opts
49 
50 namespace {
51 
52 unsigned getShortBranchOpcode(unsigned Opcode) {
53   switch (Opcode) {
54   default:
55     return Opcode;
56   case X86::JMP_2: return X86::JMP_1;
57   case X86::JMP_4: return X86::JMP_1;
58   case X86::JCC_2: return X86::JCC_1;
59   case X86::JCC_4: return X86::JCC_1;
60   }
61 }
62 
63 unsigned getShortArithOpcode(unsigned Opcode) {
64   return X86::getShortOpcodeArith(Opcode);
65 }
66 
67 bool isMOVSX64rm32(const MCInst &Inst) {
68   return Inst.getOpcode() == X86::MOVSX64rm32;
69 }
70 
71 class X86MCPlusBuilder : public MCPlusBuilder {
72 public:
73   X86MCPlusBuilder(const MCInstrAnalysis *Analysis, const MCInstrInfo *Info,
74                    const MCRegisterInfo *RegInfo)
75       : MCPlusBuilder(Analysis, Info, RegInfo) {}
76 
77   bool isBranch(const MCInst &Inst) const override {
78     return Analysis->isBranch(Inst) && !isTailCall(Inst);
79   }
80 
81   bool isUnconditionalBranch(const MCInst &Inst) const override {
82     return Analysis->isUnconditionalBranch(Inst) && !isTailCall(Inst);
83   }
84 
85   bool isNoop(const MCInst &Inst) const override {
86     return X86::isNOP(Inst.getOpcode());
87   }
88 
89   unsigned getCondCode(const MCInst &Inst) const override {
90     switch (Inst.getOpcode()) {
91     default:
92       return X86::COND_INVALID;
93     case X86::JCC_1:
94     case X86::JCC_2:
95     case X86::JCC_4:
96       return Inst.getOperand(Info->get(Inst.getOpcode()).NumOperands - 1)
97           .getImm();
98     }
99   }
100 
101   unsigned getInvertedCondCode(unsigned CC) const override {
102     switch (CC) {
103     default: return X86::COND_INVALID;
104     case X86::COND_E:  return X86::COND_NE;
105     case X86::COND_NE: return X86::COND_E;
106     case X86::COND_L:  return X86::COND_GE;
107     case X86::COND_LE: return X86::COND_G;
108     case X86::COND_G:  return X86::COND_LE;
109     case X86::COND_GE: return X86::COND_L;
110     case X86::COND_B:  return X86::COND_AE;
111     case X86::COND_BE: return X86::COND_A;
112     case X86::COND_A:  return X86::COND_BE;
113     case X86::COND_AE: return X86::COND_B;
114     case X86::COND_S:  return X86::COND_NS;
115     case X86::COND_NS: return X86::COND_S;
116     case X86::COND_P:  return X86::COND_NP;
117     case X86::COND_NP: return X86::COND_P;
118     case X86::COND_O:  return X86::COND_NO;
119     case X86::COND_NO: return X86::COND_O;
120     }
121   }
122 
123   unsigned getCondCodesLogicalOr(unsigned CC1, unsigned CC2) const override {
124     enum DecodedCondCode : uint8_t {
125       DCC_EQUAL = 0x1,
126       DCC_GREATER = 0x2,
127       DCC_LESSER = 0x4,
128       DCC_GREATER_OR_LESSER = 0x6,
129       DCC_UNSIGNED = 0x8,
130       DCC_SIGNED = 0x10,
131       DCC_INVALID = 0x20,
132     };
133 
134     auto decodeCondCode = [&](unsigned CC) -> uint8_t {
135       switch (CC) {
136       default: return DCC_INVALID;
137       case X86::COND_E: return DCC_EQUAL;
138       case X86::COND_NE: return DCC_GREATER | DCC_LESSER;
139       case X86::COND_L: return DCC_LESSER | DCC_SIGNED;
140       case X86::COND_LE: return DCC_EQUAL | DCC_LESSER | DCC_SIGNED;
141       case X86::COND_G: return DCC_GREATER | DCC_SIGNED;
142       case X86::COND_GE: return DCC_GREATER | DCC_EQUAL | DCC_SIGNED;
143       case X86::COND_B: return DCC_LESSER | DCC_UNSIGNED;
144       case X86::COND_BE: return DCC_EQUAL | DCC_LESSER | DCC_UNSIGNED;
145       case X86::COND_A: return DCC_GREATER | DCC_UNSIGNED;
146       case X86::COND_AE: return DCC_GREATER | DCC_EQUAL | DCC_UNSIGNED;
147       }
148     };
149 
150     uint8_t DCC = decodeCondCode(CC1) | decodeCondCode(CC2);
151 
152     if (DCC & DCC_INVALID)
153       return X86::COND_INVALID;
154 
155     if (DCC & DCC_SIGNED && DCC & DCC_UNSIGNED)
156       return X86::COND_INVALID;
157 
158     switch (DCC) {
159     default: return X86::COND_INVALID;
160     case DCC_EQUAL | DCC_LESSER | DCC_SIGNED: return X86::COND_LE;
161     case DCC_EQUAL | DCC_LESSER | DCC_UNSIGNED: return X86::COND_BE;
162     case DCC_EQUAL | DCC_GREATER | DCC_SIGNED: return X86::COND_GE;
163     case DCC_EQUAL | DCC_GREATER | DCC_UNSIGNED: return X86::COND_AE;
164     case DCC_GREATER | DCC_LESSER | DCC_SIGNED: return X86::COND_NE;
165     case DCC_GREATER | DCC_LESSER | DCC_UNSIGNED: return X86::COND_NE;
166     case DCC_GREATER | DCC_LESSER: return X86::COND_NE;
167     case DCC_EQUAL | DCC_SIGNED: return X86::COND_E;
168     case DCC_EQUAL | DCC_UNSIGNED: return X86::COND_E;
169     case DCC_EQUAL: return X86::COND_E;
170     case DCC_LESSER | DCC_SIGNED: return X86::COND_L;
171     case DCC_LESSER | DCC_UNSIGNED: return X86::COND_B;
172     case DCC_GREATER | DCC_SIGNED: return X86::COND_G;
173     case DCC_GREATER | DCC_UNSIGNED: return X86::COND_A;
174     }
175   }
176 
177   bool isValidCondCode(unsigned CC) const override {
178     return (CC != X86::COND_INVALID);
179   }
180 
181   bool isBreakpoint(const MCInst &Inst) const override {
182     return Inst.getOpcode() == X86::INT3;
183   }
184 
185   bool isPrefix(const MCInst &Inst) const override {
186     switch (Inst.getOpcode()) {
187     case X86::LOCK_PREFIX:
188     case X86::REPNE_PREFIX:
189     case X86::REP_PREFIX:
190       return true;
191     }
192     return false;
193   }
194 
195   bool isRep(const MCInst &Inst) const override {
196     return Inst.getFlags() == X86::IP_HAS_REPEAT;
197   }
198 
199   bool deleteREPPrefix(MCInst &Inst) const override {
200     if (Inst.getFlags() == X86::IP_HAS_REPEAT) {
201       Inst.setFlags(0);
202       return true;
203     }
204     return false;
205   }
206 
207   // FIXME: For compatibility with old LLVM only!
208   bool isTerminator(const MCInst &Inst) const override {
209     if (Info->get(Inst.getOpcode()).isTerminator())
210       return true;
211     switch (Inst.getOpcode()) {
212     default:
213       return false;
214     case X86::TRAP:
215     // Opcodes previously known as X86::UD2B
216     case X86::UD1Wm:
217     case X86::UD1Lm:
218     case X86::UD1Qm:
219     case X86::UD1Wr:
220     case X86::UD1Lr:
221     case X86::UD1Qr:
222       return true;
223     }
224   }
225 
226   bool isIndirectCall(const MCInst &Inst) const override {
227     return isCall(Inst) &&
228            ((getMemoryOperandNo(Inst) != -1) || Inst.getOperand(0).isReg());
229   }
230 
231   bool isPop(const MCInst &Inst) const override {
232     return getPopSize(Inst) == 0 ? false : true;
233   }
234 
235   bool isTerminateBranch(const MCInst &Inst) const override {
236     return Inst.getOpcode() == X86::ENDBR32 || Inst.getOpcode() == X86::ENDBR64;
237   }
238 
239   int getPopSize(const MCInst &Inst) const override {
240     switch (Inst.getOpcode()) {
241     case X86::POP16r:
242     case X86::POP16rmm:
243     case X86::POP16rmr:
244     case X86::POPF16:
245     case X86::POPA16:
246     case X86::POPDS16:
247     case X86::POPES16:
248     case X86::POPFS16:
249     case X86::POPGS16:
250     case X86::POPSS16:
251       return 2;
252     case X86::POP32r:
253     case X86::POP32rmm:
254     case X86::POP32rmr:
255     case X86::POPA32:
256     case X86::POPDS32:
257     case X86::POPES32:
258     case X86::POPF32:
259     case X86::POPFS32:
260     case X86::POPGS32:
261     case X86::POPSS32:
262       return 4;
263     case X86::POP64r:
264     case X86::POP64rmm:
265     case X86::POP64rmr:
266     case X86::POPF64:
267     case X86::POPFS64:
268     case X86::POPGS64:
269       return 8;
270     }
271     return 0;
272   }
273 
274   bool isPush(const MCInst &Inst) const override {
275     return getPushSize(Inst) == 0 ? false : true;
276   }
277 
278   int getPushSize(const MCInst &Inst) const override {
279     switch (Inst.getOpcode()) {
280     case X86::PUSH16i8:
281     case X86::PUSH16r:
282     case X86::PUSH16rmm:
283     case X86::PUSH16rmr:
284     case X86::PUSHA16:
285     case X86::PUSHCS16:
286     case X86::PUSHDS16:
287     case X86::PUSHES16:
288     case X86::PUSHF16:
289     case X86::PUSHFS16:
290     case X86::PUSHGS16:
291     case X86::PUSHSS16:
292     case X86::PUSHi16:
293       return 2;
294     case X86::PUSH32i8:
295     case X86::PUSH32r:
296     case X86::PUSH32rmm:
297     case X86::PUSH32rmr:
298     case X86::PUSHA32:
299     case X86::PUSHCS32:
300     case X86::PUSHDS32:
301     case X86::PUSHES32:
302     case X86::PUSHF32:
303     case X86::PUSHFS32:
304     case X86::PUSHGS32:
305     case X86::PUSHSS32:
306     case X86::PUSHi32:
307       return 4;
308     case X86::PUSH64i32:
309     case X86::PUSH64i8:
310     case X86::PUSH64r:
311     case X86::PUSH64rmm:
312     case X86::PUSH64rmr:
313     case X86::PUSHF64:
314     case X86::PUSHFS64:
315     case X86::PUSHGS64:
316       return 8;
317     }
318     return 0;
319   }
320 
321   bool isADD64rr(const MCInst &Inst) const override {
322     return Inst.getOpcode() == X86::ADD64rr;
323   }
324 
325   bool isSUB(const MCInst &Inst) const override {
326     return X86::isSUB(Inst.getOpcode());
327   }
328 
329   bool isADDri(const MCInst &Inst) const {
330     return Inst.getOpcode() == X86::ADD64ri32 ||
331            Inst.getOpcode() == X86::ADD64ri8;
332   }
333 
334   bool isLEA64r(const MCInst &Inst) const override {
335     return Inst.getOpcode() == X86::LEA64r;
336   }
337 
338   bool isLeave(const MCInst &Inst) const override {
339     return Inst.getOpcode() == X86::LEAVE || Inst.getOpcode() == X86::LEAVE64;
340   }
341 
342   bool isMoveMem2Reg(const MCInst &Inst) const override {
343     switch (Inst.getOpcode()) {
344     case X86::MOV16rm:
345     case X86::MOV32rm:
346     case X86::MOV64rm:
347       return true;
348     }
349     return false;
350   }
351 
352   bool isUnsupportedBranch(unsigned Opcode) const override {
353     switch (Opcode) {
354     default:
355       return false;
356     case X86::LOOP:
357     case X86::LOOPE:
358     case X86::LOOPNE:
359     case X86::JECXZ:
360     case X86::JRCXZ:
361       return true;
362     }
363   }
364 
365   bool isLoad(const MCInst &Inst) const override {
366     if (isPop(Inst))
367       return true;
368 
369     int MemOpNo = getMemoryOperandNo(Inst);
370     const MCInstrDesc &MCII = Info->get(Inst.getOpcode());
371 
372     if (MemOpNo == -1)
373       return false;
374 
375     return MCII.mayLoad();
376   }
377 
378   bool isStore(const MCInst &Inst) const override {
379     if (isPush(Inst))
380       return true;
381 
382     int MemOpNo = getMemoryOperandNo(Inst);
383     const MCInstrDesc &MCII = Info->get(Inst.getOpcode());
384 
385     if (MemOpNo == -1)
386       return false;
387 
388     return MCII.mayStore();
389   }
390 
391   bool isCleanRegXOR(const MCInst &Inst) const override {
392     switch (Inst.getOpcode()) {
393     case X86::XOR16rr:
394     case X86::XOR32rr:
395     case X86::XOR64rr:
396       break;
397     default:
398       return false;
399     }
400     return (Inst.getOperand(0).getReg() == Inst.getOperand(2).getReg());
401   }
402 
403   bool isPacked(const MCInst &Inst) const override {
404     const MCInstrDesc &Desc = Info->get(Inst.getOpcode());
405     return (Desc.TSFlags & X86II::OpPrefixMask) == X86II::PD;
406   }
407 
408   unsigned getTrapFillValue() const override { return 0xCC; }
409 
410   struct IndJmpMatcherFrag1 : MCInstMatcher {
411     std::unique_ptr<MCInstMatcher> Base;
412     std::unique_ptr<MCInstMatcher> Scale;
413     std::unique_ptr<MCInstMatcher> Index;
414     std::unique_ptr<MCInstMatcher> Offset;
415 
416     IndJmpMatcherFrag1(std::unique_ptr<MCInstMatcher> Base,
417                        std::unique_ptr<MCInstMatcher> Scale,
418                        std::unique_ptr<MCInstMatcher> Index,
419                        std::unique_ptr<MCInstMatcher> Offset)
420         : Base(std::move(Base)), Scale(std::move(Scale)),
421           Index(std::move(Index)), Offset(std::move(Offset)) {}
422 
423     bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
424                MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
425       if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
426         return false;
427 
428       if (CurInst->getOpcode() != X86::JMP64m)
429         return false;
430 
431       int MemOpNo = MIB.getMemoryOperandNo(*CurInst);
432       if (MemOpNo == -1)
433         return false;
434 
435       if (!Base->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrBaseReg))
436         return false;
437       if (!Scale->match(MRI, MIB, this->InstrWindow,
438                         MemOpNo + X86::AddrScaleAmt))
439         return false;
440       if (!Index->match(MRI, MIB, this->InstrWindow,
441                         MemOpNo + X86::AddrIndexReg))
442         return false;
443       if (!Offset->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrDisp))
444         return false;
445       return true;
446     }
447 
448     void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
449       MIB.addAnnotation(*CurInst, Annotation, true);
450       Base->annotate(MIB, Annotation);
451       Scale->annotate(MIB, Annotation);
452       Index->annotate(MIB, Annotation);
453       Offset->annotate(MIB, Annotation);
454     }
455   };
456 
457   std::unique_ptr<MCInstMatcher>
458   matchIndJmp(std::unique_ptr<MCInstMatcher> Base,
459               std::unique_ptr<MCInstMatcher> Scale,
460               std::unique_ptr<MCInstMatcher> Index,
461               std::unique_ptr<MCInstMatcher> Offset) const override {
462     return std::unique_ptr<MCInstMatcher>(
463         new IndJmpMatcherFrag1(std::move(Base), std::move(Scale),
464                                std::move(Index), std::move(Offset)));
465   }
466 
467   struct IndJmpMatcherFrag2 : MCInstMatcher {
468     std::unique_ptr<MCInstMatcher> Reg;
469 
470     IndJmpMatcherFrag2(std::unique_ptr<MCInstMatcher> Reg)
471         : Reg(std::move(Reg)) {}
472 
473     bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
474                MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
475       if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
476         return false;
477 
478       if (CurInst->getOpcode() != X86::JMP64r)
479         return false;
480 
481       return Reg->match(MRI, MIB, this->InstrWindow, 0);
482     }
483 
484     void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
485       MIB.addAnnotation(*CurInst, Annotation, true);
486       Reg->annotate(MIB, Annotation);
487     }
488   };
489 
490   std::unique_ptr<MCInstMatcher>
491   matchIndJmp(std::unique_ptr<MCInstMatcher> Target) const override {
492     return std::unique_ptr<MCInstMatcher>(
493         new IndJmpMatcherFrag2(std::move(Target)));
494   }
495 
496   struct LoadMatcherFrag1 : MCInstMatcher {
497     std::unique_ptr<MCInstMatcher> Base;
498     std::unique_ptr<MCInstMatcher> Scale;
499     std::unique_ptr<MCInstMatcher> Index;
500     std::unique_ptr<MCInstMatcher> Offset;
501 
502     LoadMatcherFrag1(std::unique_ptr<MCInstMatcher> Base,
503                      std::unique_ptr<MCInstMatcher> Scale,
504                      std::unique_ptr<MCInstMatcher> Index,
505                      std::unique_ptr<MCInstMatcher> Offset)
506         : Base(std::move(Base)), Scale(std::move(Scale)),
507           Index(std::move(Index)), Offset(std::move(Offset)) {}
508 
509     bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
510                MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
511       if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
512         return false;
513 
514       if (CurInst->getOpcode() != X86::MOV64rm &&
515           CurInst->getOpcode() != X86::MOVSX64rm32)
516         return false;
517 
518       int MemOpNo = MIB.getMemoryOperandNo(*CurInst);
519       if (MemOpNo == -1)
520         return false;
521 
522       if (!Base->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrBaseReg))
523         return false;
524       if (!Scale->match(MRI, MIB, this->InstrWindow,
525                         MemOpNo + X86::AddrScaleAmt))
526         return false;
527       if (!Index->match(MRI, MIB, this->InstrWindow,
528                         MemOpNo + X86::AddrIndexReg))
529         return false;
530       if (!Offset->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrDisp))
531         return false;
532       return true;
533     }
534 
535     void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
536       MIB.addAnnotation(*CurInst, Annotation, true);
537       Base->annotate(MIB, Annotation);
538       Scale->annotate(MIB, Annotation);
539       Index->annotate(MIB, Annotation);
540       Offset->annotate(MIB, Annotation);
541     }
542   };
543 
544   std::unique_ptr<MCInstMatcher>
545   matchLoad(std::unique_ptr<MCInstMatcher> Base,
546             std::unique_ptr<MCInstMatcher> Scale,
547             std::unique_ptr<MCInstMatcher> Index,
548             std::unique_ptr<MCInstMatcher> Offset) const override {
549     return std::unique_ptr<MCInstMatcher>(
550         new LoadMatcherFrag1(std::move(Base), std::move(Scale),
551                              std::move(Index), std::move(Offset)));
552   }
553 
554   struct AddMatcher : MCInstMatcher {
555     std::unique_ptr<MCInstMatcher> A;
556     std::unique_ptr<MCInstMatcher> B;
557 
558     AddMatcher(std::unique_ptr<MCInstMatcher> A,
559                std::unique_ptr<MCInstMatcher> B)
560         : A(std::move(A)), B(std::move(B)) {}
561 
562     bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
563                MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
564       if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
565         return false;
566 
567       if (CurInst->getOpcode() == X86::ADD64rr ||
568           CurInst->getOpcode() == X86::ADD64rr_DB ||
569           CurInst->getOpcode() == X86::ADD64rr_REV) {
570         if (!A->match(MRI, MIB, this->InstrWindow, 1)) {
571           if (!B->match(MRI, MIB, this->InstrWindow, 1))
572             return false;
573           return A->match(MRI, MIB, this->InstrWindow, 2);
574         }
575 
576         if (B->match(MRI, MIB, this->InstrWindow, 2))
577           return true;
578 
579         if (!B->match(MRI, MIB, this->InstrWindow, 1))
580           return false;
581         return A->match(MRI, MIB, this->InstrWindow, 2);
582       }
583 
584       return false;
585     }
586 
587     void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
588       MIB.addAnnotation(*CurInst, Annotation, true);
589       A->annotate(MIB, Annotation);
590       B->annotate(MIB, Annotation);
591     }
592   };
593 
594   virtual std::unique_ptr<MCInstMatcher>
595   matchAdd(std::unique_ptr<MCInstMatcher> A,
596            std::unique_ptr<MCInstMatcher> B) const override {
597     return std::unique_ptr<MCInstMatcher>(
598         new AddMatcher(std::move(A), std::move(B)));
599   }
600 
601   struct LEAMatcher : MCInstMatcher {
602     std::unique_ptr<MCInstMatcher> Target;
603 
604     LEAMatcher(std::unique_ptr<MCInstMatcher> Target)
605         : Target(std::move(Target)) {}
606 
607     bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
608                MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
609       if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
610         return false;
611 
612       if (CurInst->getOpcode() != X86::LEA64r)
613         return false;
614 
615       if (CurInst->getOperand(1 + X86::AddrScaleAmt).getImm() != 1 ||
616           CurInst->getOperand(1 + X86::AddrIndexReg).getReg() !=
617               X86::NoRegister ||
618           (CurInst->getOperand(1 + X86::AddrBaseReg).getReg() !=
619                X86::NoRegister &&
620            CurInst->getOperand(1 + X86::AddrBaseReg).getReg() != X86::RIP))
621         return false;
622 
623       return Target->match(MRI, MIB, this->InstrWindow, 1 + X86::AddrDisp);
624     }
625 
626     void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
627       MIB.addAnnotation(*CurInst, Annotation, true);
628       Target->annotate(MIB, Annotation);
629     }
630   };
631 
632   virtual std::unique_ptr<MCInstMatcher>
633   matchLoadAddr(std::unique_ptr<MCInstMatcher> Target) const override {
634     return std::unique_ptr<MCInstMatcher>(new LEAMatcher(std::move(Target)));
635   }
636 
637   bool hasPCRelOperand(const MCInst &Inst) const override {
638     for (const MCOperand &Operand : Inst)
639       if (Operand.isReg() && Operand.getReg() == X86::RIP)
640         return true;
641     return false;
642   }
643 
644   int getMemoryOperandNo(const MCInst &Inst) const override {
645     unsigned Opcode = Inst.getOpcode();
646     const MCInstrDesc &Desc = Info->get(Opcode);
647     int MemOpNo = X86II::getMemoryOperandNo(Desc.TSFlags);
648     if (MemOpNo >= 0)
649       MemOpNo += X86II::getOperandBias(Desc);
650     return MemOpNo;
651   }
652 
653   bool hasEVEXEncoding(const MCInst &Inst) const override {
654     const MCInstrDesc &Desc = Info->get(Inst.getOpcode());
655     return (Desc.TSFlags & X86II::EncodingMask) == X86II::EVEX;
656   }
657 
658   bool isMacroOpFusionPair(ArrayRef<MCInst> Insts) const override {
659     const auto *I = Insts.begin();
660     while (I != Insts.end() && isPrefix(*I))
661       ++I;
662     if (I == Insts.end())
663       return false;
664 
665     const MCInst &FirstInst = *I;
666     ++I;
667     while (I != Insts.end() && isPrefix(*I))
668       ++I;
669     if (I == Insts.end())
670       return false;
671     const MCInst &SecondInst = *I;
672 
673     if (!isConditionalBranch(SecondInst))
674       return false;
675     // Cannot fuse if the first instruction uses RIP-relative memory.
676     if (hasPCRelOperand(FirstInst))
677       return false;
678 
679     const X86::FirstMacroFusionInstKind CmpKind =
680         X86::classifyFirstOpcodeInMacroFusion(FirstInst.getOpcode());
681     if (CmpKind == X86::FirstMacroFusionInstKind::Invalid)
682       return false;
683 
684     X86::CondCode CC = static_cast<X86::CondCode>(getCondCode(SecondInst));
685     X86::SecondMacroFusionInstKind BranchKind =
686         X86::classifySecondCondCodeInMacroFusion(CC);
687     if (BranchKind == X86::SecondMacroFusionInstKind::Invalid)
688       return false;
689     return X86::isMacroFused(CmpKind, BranchKind);
690   }
691 
692   bool
693   evaluateX86MemoryOperand(const MCInst &Inst, unsigned *BaseRegNum,
694                            int64_t *ScaleImm, unsigned *IndexRegNum,
695                            int64_t *DispImm, unsigned *SegmentRegNum,
696                            const MCExpr **DispExpr = nullptr) const override {
697     assert(BaseRegNum && ScaleImm && IndexRegNum && SegmentRegNum &&
698            "one of the input pointers is null");
699     int MemOpNo = getMemoryOperandNo(Inst);
700     if (MemOpNo < 0)
701       return false;
702     unsigned MemOpOffset = static_cast<unsigned>(MemOpNo);
703 
704     if (MemOpOffset + X86::AddrSegmentReg >= MCPlus::getNumPrimeOperands(Inst))
705       return false;
706 
707     const MCOperand &Base = Inst.getOperand(MemOpOffset + X86::AddrBaseReg);
708     const MCOperand &Scale = Inst.getOperand(MemOpOffset + X86::AddrScaleAmt);
709     const MCOperand &Index = Inst.getOperand(MemOpOffset + X86::AddrIndexReg);
710     const MCOperand &Disp = Inst.getOperand(MemOpOffset + X86::AddrDisp);
711     const MCOperand &Segment =
712         Inst.getOperand(MemOpOffset + X86::AddrSegmentReg);
713 
714     // Make sure it is a well-formed memory operand.
715     if (!Base.isReg() || !Scale.isImm() || !Index.isReg() ||
716         (!Disp.isImm() && !Disp.isExpr()) || !Segment.isReg())
717       return false;
718 
719     *BaseRegNum = Base.getReg();
720     *ScaleImm = Scale.getImm();
721     *IndexRegNum = Index.getReg();
722     if (Disp.isImm()) {
723       assert(DispImm && "DispImm needs to be set");
724       *DispImm = Disp.getImm();
725       if (DispExpr)
726         *DispExpr = nullptr;
727     } else {
728       assert(DispExpr && "DispExpr needs to be set");
729       *DispExpr = Disp.getExpr();
730       if (DispImm)
731         *DispImm = 0;
732     }
733     *SegmentRegNum = Segment.getReg();
734     return true;
735   }
736 
737   bool evaluateMemOperandTarget(const MCInst &Inst, uint64_t &Target,
738                                 uint64_t Address,
739                                 uint64_t Size) const override {
740     unsigned      BaseRegNum;
741     int64_t       ScaleValue;
742     unsigned      IndexRegNum;
743     int64_t       DispValue;
744     unsigned      SegRegNum;
745     const MCExpr *DispExpr = nullptr;
746     if (!evaluateX86MemoryOperand(Inst, &BaseRegNum, &ScaleValue, &IndexRegNum,
747                                   &DispValue, &SegRegNum, &DispExpr))
748       return false;
749 
750     // Make sure it's a well-formed addressing we can statically evaluate.
751     if ((BaseRegNum != X86::RIP && BaseRegNum != X86::NoRegister) ||
752         IndexRegNum != X86::NoRegister || SegRegNum != X86::NoRegister ||
753         DispExpr)
754       return false;
755 
756     Target = DispValue;
757     if (BaseRegNum == X86::RIP) {
758       assert(Size != 0 && "instruction size required in order to statically "
759                           "evaluate RIP-relative address");
760       Target += Address + Size;
761     }
762     return true;
763   }
764 
765   MCInst::iterator getMemOperandDisp(MCInst &Inst) const override {
766     int MemOpNo = getMemoryOperandNo(Inst);
767     if (MemOpNo < 0)
768       return Inst.end();
769     return Inst.begin() + (MemOpNo + X86::AddrDisp);
770   }
771 
772   bool replaceMemOperandDisp(MCInst &Inst, MCOperand Operand) const override {
773     MCOperand *OI = getMemOperandDisp(Inst);
774     if (OI == Inst.end())
775       return false;
776     *OI = Operand;
777     return true;
778   }
779 
780   /// Get the registers used as function parameters.
781   /// This function is specific to the x86_64 abi on Linux.
782   BitVector getRegsUsedAsParams() const override {
783     BitVector Regs = BitVector(RegInfo->getNumRegs(), false);
784     Regs |= getAliases(X86::RSI);
785     Regs |= getAliases(X86::RDI);
786     Regs |= getAliases(X86::RDX);
787     Regs |= getAliases(X86::RCX);
788     Regs |= getAliases(X86::R8);
789     Regs |= getAliases(X86::R9);
790     return Regs;
791   }
792 
793   void getCalleeSavedRegs(BitVector &Regs) const override {
794     Regs |= getAliases(X86::RBX);
795     Regs |= getAliases(X86::RBP);
796     Regs |= getAliases(X86::R12);
797     Regs |= getAliases(X86::R13);
798     Regs |= getAliases(X86::R14);
799     Regs |= getAliases(X86::R15);
800   }
801 
802   void getDefaultDefIn(BitVector &Regs) const override {
803     assert(Regs.size() >= RegInfo->getNumRegs() &&
804            "The size of BitVector is less than RegInfo->getNumRegs().");
805     Regs.set(X86::RAX);
806     Regs.set(X86::RCX);
807     Regs.set(X86::RDX);
808     Regs.set(X86::RSI);
809     Regs.set(X86::RDI);
810     Regs.set(X86::R8);
811     Regs.set(X86::R9);
812     Regs.set(X86::XMM0);
813     Regs.set(X86::XMM1);
814     Regs.set(X86::XMM2);
815     Regs.set(X86::XMM3);
816     Regs.set(X86::XMM4);
817     Regs.set(X86::XMM5);
818     Regs.set(X86::XMM6);
819     Regs.set(X86::XMM7);
820   }
821 
822   void getDefaultLiveOut(BitVector &Regs) const override {
823     assert(Regs.size() >= RegInfo->getNumRegs() &&
824            "The size of BitVector is less than RegInfo->getNumRegs().");
825     Regs |= getAliases(X86::RAX);
826     Regs |= getAliases(X86::RDX);
827     Regs |= getAliases(X86::RCX);
828     Regs |= getAliases(X86::XMM0);
829     Regs |= getAliases(X86::XMM1);
830   }
831 
832   void getGPRegs(BitVector &Regs, bool IncludeAlias) const override {
833     if (IncludeAlias) {
834       Regs |= getAliases(X86::RAX);
835       Regs |= getAliases(X86::RBX);
836       Regs |= getAliases(X86::RBP);
837       Regs |= getAliases(X86::RSI);
838       Regs |= getAliases(X86::RDI);
839       Regs |= getAliases(X86::RDX);
840       Regs |= getAliases(X86::RCX);
841       Regs |= getAliases(X86::R8);
842       Regs |= getAliases(X86::R9);
843       Regs |= getAliases(X86::R10);
844       Regs |= getAliases(X86::R11);
845       Regs |= getAliases(X86::R12);
846       Regs |= getAliases(X86::R13);
847       Regs |= getAliases(X86::R14);
848       Regs |= getAliases(X86::R15);
849       return;
850     }
851     Regs.set(X86::RAX);
852     Regs.set(X86::RBX);
853     Regs.set(X86::RBP);
854     Regs.set(X86::RSI);
855     Regs.set(X86::RDI);
856     Regs.set(X86::RDX);
857     Regs.set(X86::RCX);
858     Regs.set(X86::R8);
859     Regs.set(X86::R9);
860     Regs.set(X86::R10);
861     Regs.set(X86::R11);
862     Regs.set(X86::R12);
863     Regs.set(X86::R13);
864     Regs.set(X86::R14);
865     Regs.set(X86::R15);
866   }
867 
868   void getClassicGPRegs(BitVector &Regs) const override {
869     Regs |= getAliases(X86::RAX);
870     Regs |= getAliases(X86::RBX);
871     Regs |= getAliases(X86::RBP);
872     Regs |= getAliases(X86::RSI);
873     Regs |= getAliases(X86::RDI);
874     Regs |= getAliases(X86::RDX);
875     Regs |= getAliases(X86::RCX);
876   }
877 
878   void getRepRegs(BitVector &Regs) const override {
879     Regs |= getAliases(X86::RCX);
880   }
881 
882   MCPhysReg getAliasSized(MCPhysReg Reg, uint8_t Size) const override {
883     switch (Reg) {
884     case X86::RAX: case X86::EAX: case X86::AX: case X86::AL: case X86::AH:
885       switch (Size) {
886       case 8: return X86::RAX;       case 4: return X86::EAX;
887       case 2: return X86::AX;        case 1: return X86::AL;
888       default: llvm_unreachable("Unexpected size");
889       }
890     case X86::RBX: case X86::EBX: case X86::BX: case X86::BL: case X86::BH:
891       switch (Size) {
892       case 8: return X86::RBX;       case 4: return X86::EBX;
893       case 2: return X86::BX;        case 1: return X86::BL;
894       default: llvm_unreachable("Unexpected size");
895       }
896     case X86::RDX: case X86::EDX: case X86::DX: case X86::DL: case X86::DH:
897       switch (Size) {
898       case 8: return X86::RDX;       case 4: return X86::EDX;
899       case 2: return X86::DX;        case 1: return X86::DL;
900       default: llvm_unreachable("Unexpected size");
901       }
902     case X86::RDI: case X86::EDI: case X86::DI: case X86::DIL:
903       switch (Size) {
904       case 8: return X86::RDI;       case 4: return X86::EDI;
905       case 2: return X86::DI;        case 1: return X86::DIL;
906       default: llvm_unreachable("Unexpected size");
907       }
908     case X86::RSI: case X86::ESI: case X86::SI: case X86::SIL:
909       switch (Size) {
910       case 8: return X86::RSI;       case 4: return X86::ESI;
911       case 2: return X86::SI;        case 1: return X86::SIL;
912       default: llvm_unreachable("Unexpected size");
913       }
914     case X86::RCX: case X86::ECX: case X86::CX: case X86::CL: case X86::CH:
915       switch (Size) {
916       case 8: return X86::RCX;       case 4: return X86::ECX;
917       case 2: return X86::CX;        case 1: return X86::CL;
918       default: llvm_unreachable("Unexpected size");
919       }
920     case X86::RSP: case X86::ESP: case X86::SP: case X86::SPL:
921       switch (Size) {
922       case 8: return X86::RSP;       case 4: return X86::ESP;
923       case 2: return X86::SP;        case 1: return X86::SPL;
924       default: llvm_unreachable("Unexpected size");
925       }
926     case X86::RBP: case X86::EBP: case X86::BP: case X86::BPL:
927       switch (Size) {
928       case 8: return X86::RBP;       case 4: return X86::EBP;
929       case 2: return X86::BP;        case 1: return X86::BPL;
930       default: llvm_unreachable("Unexpected size");
931       }
932   case X86::R8: case X86::R8D: case X86::R8W: case X86::R8B:
933       switch (Size) {
934       case 8: return X86::R8;        case 4: return X86::R8D;
935       case 2: return X86::R8W;       case 1: return X86::R8B;
936       default: llvm_unreachable("Unexpected size");
937       }
938     case X86::R9: case X86::R9D: case X86::R9W: case X86::R9B:
939       switch (Size) {
940       case 8: return X86::R9;        case 4: return X86::R9D;
941       case 2: return X86::R9W;       case 1: return X86::R9B;
942       default: llvm_unreachable("Unexpected size");
943       }
944     case X86::R10: case X86::R10D: case X86::R10W: case X86::R10B:
945       switch (Size) {
946       case 8: return X86::R10;        case 4: return X86::R10D;
947       case 2: return X86::R10W;       case 1: return X86::R10B;
948       default: llvm_unreachable("Unexpected size");
949       }
950     case X86::R11: case X86::R11D: case X86::R11W: case X86::R11B:
951       switch (Size) {
952       case 8: return X86::R11;        case 4: return X86::R11D;
953       case 2: return X86::R11W;       case 1: return X86::R11B;
954       default: llvm_unreachable("Unexpected size");
955       }
956     case X86::R12: case X86::R12D: case X86::R12W: case X86::R12B:
957       switch (Size) {
958       case 8: return X86::R12;        case 4: return X86::R12D;
959       case 2: return X86::R12W;       case 1: return X86::R12B;
960       default: llvm_unreachable("Unexpected size");
961       }
962     case X86::R13: case X86::R13D: case X86::R13W: case X86::R13B:
963       switch (Size) {
964       case 8: return X86::R13;        case 4: return X86::R13D;
965       case 2: return X86::R13W;       case 1: return X86::R13B;
966       default: llvm_unreachable("Unexpected size");
967       }
968     case X86::R14: case X86::R14D: case X86::R14W: case X86::R14B:
969       switch (Size) {
970       case 8: return X86::R14;        case 4: return X86::R14D;
971       case 2: return X86::R14W;       case 1: return X86::R14B;
972       default: llvm_unreachable("Unexpected size");
973       }
974     case X86::R15: case X86::R15D: case X86::R15W: case X86::R15B:
975       switch (Size) {
976       case 8: return X86::R15;        case 4: return X86::R15D;
977       case 2: return X86::R15W;       case 1: return X86::R15B;
978       default: llvm_unreachable("Unexpected size");
979       }
980     default:
981       dbgs() << Reg << " (get alias sized)\n";
982       llvm_unreachable("Unexpected reg number");
983       break;
984     }
985   }
986 
987   bool isUpper8BitReg(MCPhysReg Reg) const override {
988     switch (Reg) {
989     case X86::AH:
990     case X86::BH:
991     case X86::CH:
992     case X86::DH:
993       return true;
994     default:
995       return false;
996     }
997   }
998 
999   bool cannotUseREX(const MCInst &Inst) const override {
1000     switch (Inst.getOpcode()) {
1001     case X86::MOV8mr_NOREX:
1002     case X86::MOV8rm_NOREX:
1003     case X86::MOV8rr_NOREX:
1004     case X86::MOVSX32rm8_NOREX:
1005     case X86::MOVSX32rr8_NOREX:
1006     case X86::MOVZX32rm8_NOREX:
1007     case X86::MOVZX32rr8_NOREX:
1008     case X86::MOV8mr:
1009     case X86::MOV8rm:
1010     case X86::MOV8rr:
1011     case X86::MOVSX32rm8:
1012     case X86::MOVSX32rr8:
1013     case X86::MOVZX32rm8:
1014     case X86::MOVZX32rr8:
1015     case X86::TEST8ri:
1016       for (int I = 0, E = MCPlus::getNumPrimeOperands(Inst); I != E; ++I) {
1017         const MCOperand &Operand = Inst.getOperand(I);
1018         if (!Operand.isReg())
1019           continue;
1020         if (isUpper8BitReg(Operand.getReg()))
1021           return true;
1022       }
1023       LLVM_FALLTHROUGH;
1024     default:
1025       return false;
1026     }
1027   }
1028 
1029   bool isStackAccess(const MCInst &Inst, bool &IsLoad, bool &IsStore,
1030                      bool &IsStoreFromReg, MCPhysReg &Reg, int32_t &SrcImm,
1031                      uint16_t &StackPtrReg, int64_t &StackOffset, uint8_t &Size,
1032                      bool &IsSimple, bool &IsIndexed) const override {
1033     // Detect simple push/pop cases first
1034     if (int Sz = getPushSize(Inst)) {
1035       IsLoad = false;
1036       IsStore = true;
1037       IsStoreFromReg = true;
1038       StackPtrReg = X86::RSP;
1039       StackOffset = -Sz;
1040       Size = Sz;
1041       IsSimple = true;
1042       if (Inst.getOperand(0).isImm())
1043         SrcImm = Inst.getOperand(0).getImm();
1044       else if (Inst.getOperand(0).isReg())
1045         Reg = Inst.getOperand(0).getReg();
1046       else
1047         IsSimple = false;
1048 
1049       return true;
1050     }
1051     if (int Sz = getPopSize(Inst)) {
1052       IsLoad = true;
1053       IsStore = false;
1054       if (Inst.getNumOperands() == 0 || !Inst.getOperand(0).isReg()) {
1055         IsSimple = false;
1056       } else {
1057         Reg = Inst.getOperand(0).getReg();
1058         IsSimple = true;
1059       }
1060       StackPtrReg = X86::RSP;
1061       StackOffset = 0;
1062       Size = Sz;
1063       return true;
1064     }
1065 
1066     struct InstInfo {
1067       // Size in bytes that Inst loads from memory.
1068       uint8_t DataSize;
1069       bool IsLoad;
1070       bool IsStore;
1071       bool StoreFromReg;
1072       bool Simple;
1073     };
1074 
1075     InstInfo I;
1076     int MemOpNo = getMemoryOperandNo(Inst);
1077     const MCInstrDesc &MCII = Info->get(Inst.getOpcode());
1078     // If it is not dealing with a memory operand, we discard it
1079     if (MemOpNo == -1 || MCII.isCall())
1080       return false;
1081 
1082     switch (Inst.getOpcode()) {
1083     default: {
1084       uint8_t Sz = 0;
1085       bool IsLoad = MCII.mayLoad();
1086       bool IsStore = MCII.mayStore();
1087       // Is it LEA? (deals with memory but is not loading nor storing)
1088       if (!IsLoad && !IsStore)
1089         return false;
1090 
1091       // Try to guess data size involved in the load/store by looking at the
1092       // register size. If there's no reg involved, return 0 as size, meaning
1093       // we don't know.
1094       for (unsigned I = 0, E = MCII.getNumOperands(); I != E; ++I) {
1095         if (MCII.OpInfo[I].OperandType != MCOI::OPERAND_REGISTER)
1096           continue;
1097         if (static_cast<int>(I) >= MemOpNo && I < X86::AddrNumOperands)
1098           continue;
1099         Sz = RegInfo->getRegClass(MCII.OpInfo[I].RegClass).getSizeInBits() / 8;
1100         break;
1101       }
1102       I = {Sz, IsLoad, IsStore, false, false};
1103       break;
1104     }
1105     case X86::MOV16rm: I = {2, true, false, false, true}; break;
1106     case X86::MOV32rm: I = {4, true, false, false, true}; break;
1107     case X86::MOV64rm: I = {8, true, false, false, true}; break;
1108     case X86::MOV16mr: I = {2, false, true, true, true};  break;
1109     case X86::MOV32mr: I = {4, false, true, true, true};  break;
1110     case X86::MOV64mr: I = {8, false, true, true, true};  break;
1111     case X86::MOV16mi: I = {2, false, true, false, true}; break;
1112     case X86::MOV32mi: I = {4, false, true, false, true}; break;
1113     } // end switch (Inst.getOpcode())
1114 
1115     unsigned BaseRegNum;
1116     int64_t ScaleValue;
1117     unsigned IndexRegNum;
1118     int64_t DispValue;
1119     unsigned SegRegNum;
1120     const MCExpr *DispExpr;
1121     if (!evaluateX86MemoryOperand(Inst, &BaseRegNum, &ScaleValue, &IndexRegNum,
1122                                   &DispValue, &SegRegNum, &DispExpr)) {
1123       LLVM_DEBUG(dbgs() << "Evaluate failed on ");
1124       LLVM_DEBUG(Inst.dump());
1125       return false;
1126     }
1127 
1128     // Make sure it's a stack access
1129     if (BaseRegNum != X86::RBP && BaseRegNum != X86::RSP)
1130       return false;
1131 
1132     IsLoad = I.IsLoad;
1133     IsStore = I.IsStore;
1134     IsStoreFromReg = I.StoreFromReg;
1135     Size = I.DataSize;
1136     IsSimple = I.Simple;
1137     StackPtrReg = BaseRegNum;
1138     StackOffset = DispValue;
1139     IsIndexed = IndexRegNum != X86::NoRegister || SegRegNum != X86::NoRegister;
1140 
1141     if (!I.Simple)
1142       return true;
1143 
1144     // Retrieve related register in simple MOV from/to stack operations.
1145     unsigned MemOpOffset = static_cast<unsigned>(MemOpNo);
1146     if (I.IsLoad) {
1147       MCOperand RegOpnd = Inst.getOperand(0);
1148       assert(RegOpnd.isReg() && "unexpected destination operand");
1149       Reg = RegOpnd.getReg();
1150     } else if (I.IsStore) {
1151       MCOperand SrcOpnd =
1152           Inst.getOperand(MemOpOffset + X86::AddrSegmentReg + 1);
1153       if (I.StoreFromReg) {
1154         assert(SrcOpnd.isReg() && "unexpected source operand");
1155         Reg = SrcOpnd.getReg();
1156       } else {
1157         assert(SrcOpnd.isImm() && "unexpected source operand");
1158         SrcImm = SrcOpnd.getImm();
1159       }
1160     }
1161 
1162     return true;
1163   }
1164 
1165   void changeToPushOrPop(MCInst &Inst) const override {
1166     assert(!isPush(Inst) && !isPop(Inst));
1167 
1168     struct InstInfo {
1169       // Size in bytes that Inst loads from memory.
1170       uint8_t DataSize;
1171       bool IsLoad;
1172       bool StoreFromReg;
1173     };
1174 
1175     InstInfo I;
1176     switch (Inst.getOpcode()) {
1177     default: {
1178       llvm_unreachable("Unhandled opcode");
1179       return;
1180     }
1181     case X86::MOV16rm: I = {2, true, false}; break;
1182     case X86::MOV32rm: I = {4, true, false}; break;
1183     case X86::MOV64rm: I = {8, true, false}; break;
1184     case X86::MOV16mr: I = {2, false, true};  break;
1185     case X86::MOV32mr: I = {4, false, true};  break;
1186     case X86::MOV64mr: I = {8, false, true};  break;
1187     case X86::MOV16mi: I = {2, false, false}; break;
1188     case X86::MOV32mi: I = {4, false, false}; break;
1189     } // end switch (Inst.getOpcode())
1190 
1191     unsigned BaseRegNum;
1192     int64_t ScaleValue;
1193     unsigned IndexRegNum;
1194     int64_t DispValue;
1195     unsigned SegRegNum;
1196     const MCExpr *DispExpr;
1197     if (!evaluateX86MemoryOperand(Inst, &BaseRegNum, &ScaleValue, &IndexRegNum,
1198                                   &DispValue, &SegRegNum, &DispExpr)) {
1199       llvm_unreachable("Evaluate failed");
1200       return;
1201     }
1202     // Make sure it's a stack access
1203     if (BaseRegNum != X86::RBP && BaseRegNum != X86::RSP) {
1204       llvm_unreachable("Not a stack access");
1205       return;
1206     }
1207 
1208     unsigned MemOpOffset = getMemoryOperandNo(Inst);
1209     unsigned NewOpcode = 0;
1210     if (I.IsLoad) {
1211       switch (I.DataSize) {
1212       case 2: NewOpcode = X86::POP16r; break;
1213       case 4: NewOpcode = X86::POP32r; break;
1214       case 8: NewOpcode = X86::POP64r; break;
1215       default:
1216         llvm_unreachable("Unexpected size");
1217       }
1218       unsigned RegOpndNum = Inst.getOperand(0).getReg();
1219       Inst.clear();
1220       Inst.setOpcode(NewOpcode);
1221       Inst.addOperand(MCOperand::createReg(RegOpndNum));
1222     } else {
1223       MCOperand SrcOpnd =
1224           Inst.getOperand(MemOpOffset + X86::AddrSegmentReg + 1);
1225       if (I.StoreFromReg) {
1226         switch (I.DataSize) {
1227         case 2: NewOpcode = X86::PUSH16r; break;
1228         case 4: NewOpcode = X86::PUSH32r; break;
1229         case 8: NewOpcode = X86::PUSH64r; break;
1230         default:
1231           llvm_unreachable("Unexpected size");
1232         }
1233         assert(SrcOpnd.isReg() && "Unexpected source operand");
1234         unsigned RegOpndNum = SrcOpnd.getReg();
1235         Inst.clear();
1236         Inst.setOpcode(NewOpcode);
1237         Inst.addOperand(MCOperand::createReg(RegOpndNum));
1238       } else {
1239         switch (I.DataSize) {
1240         case 2: NewOpcode = X86::PUSH16i8; break;
1241         case 4: NewOpcode = X86::PUSH32i8; break;
1242         case 8: NewOpcode = X86::PUSH64i32; break;
1243         default:
1244           llvm_unreachable("Unexpected size");
1245         }
1246         assert(SrcOpnd.isImm() && "Unexpected source operand");
1247         int64_t SrcImm = SrcOpnd.getImm();
1248         Inst.clear();
1249         Inst.setOpcode(NewOpcode);
1250         Inst.addOperand(MCOperand::createImm(SrcImm));
1251       }
1252     }
1253   }
1254 
1255   bool isStackAdjustment(const MCInst &Inst) const override {
1256     switch (Inst.getOpcode()) {
1257     default:
1258       return false;
1259     case X86::SUB64ri32:
1260     case X86::SUB64ri8:
1261     case X86::ADD64ri32:
1262     case X86::ADD64ri8:
1263     case X86::LEA64r:
1264       break;
1265     }
1266 
1267     const MCInstrDesc &MCII = Info->get(Inst.getOpcode());
1268     for (int I = 0, E = MCII.getNumDefs(); I != E; ++I) {
1269       const MCOperand &Operand = Inst.getOperand(I);
1270       if (Operand.isReg() && Operand.getReg() == X86::RSP)
1271         return true;
1272     }
1273     return false;
1274   }
1275 
1276   bool evaluateSimple(const MCInst &Inst, int64_t &Output,
1277                       std::pair<MCPhysReg, int64_t> Input1,
1278                       std::pair<MCPhysReg, int64_t> Input2) const override {
1279 
1280     auto getOperandVal = [&](MCPhysReg Reg) -> ErrorOr<int64_t> {
1281       if (Reg == Input1.first)
1282         return Input1.second;
1283       if (Reg == Input2.first)
1284         return Input2.second;
1285       return make_error_code(errc::result_out_of_range);
1286     };
1287 
1288     switch (Inst.getOpcode()) {
1289     default:
1290       return false;
1291 
1292     case X86::AND64ri32:
1293     case X86::AND64ri8:
1294       if (!Inst.getOperand(2).isImm())
1295         return false;
1296       if (ErrorOr<int64_t> InputVal =
1297               getOperandVal(Inst.getOperand(1).getReg()))
1298         Output = *InputVal & Inst.getOperand(2).getImm();
1299       else
1300         return false;
1301       break;
1302     case X86::SUB64ri32:
1303     case X86::SUB64ri8:
1304       if (!Inst.getOperand(2).isImm())
1305         return false;
1306       if (ErrorOr<int64_t> InputVal =
1307               getOperandVal(Inst.getOperand(1).getReg()))
1308         Output = *InputVal - Inst.getOperand(2).getImm();
1309       else
1310         return false;
1311       break;
1312     case X86::ADD64ri32:
1313     case X86::ADD64ri8:
1314       if (!Inst.getOperand(2).isImm())
1315         return false;
1316       if (ErrorOr<int64_t> InputVal =
1317               getOperandVal(Inst.getOperand(1).getReg()))
1318         Output = *InputVal + Inst.getOperand(2).getImm();
1319       else
1320         return false;
1321       break;
1322     case X86::ADD64i32:
1323       if (!Inst.getOperand(0).isImm())
1324         return false;
1325       if (ErrorOr<int64_t> InputVal = getOperandVal(X86::RAX))
1326         Output = *InputVal + Inst.getOperand(0).getImm();
1327       else
1328         return false;
1329       break;
1330 
1331     case X86::LEA64r: {
1332       unsigned BaseRegNum;
1333       int64_t ScaleValue;
1334       unsigned IndexRegNum;
1335       int64_t DispValue;
1336       unsigned SegRegNum;
1337       const MCExpr *DispExpr = nullptr;
1338       if (!evaluateX86MemoryOperand(Inst, &BaseRegNum, &ScaleValue,
1339                                     &IndexRegNum, &DispValue, &SegRegNum,
1340                                     &DispExpr))
1341         return false;
1342 
1343       if (BaseRegNum == X86::NoRegister || IndexRegNum != X86::NoRegister ||
1344           SegRegNum != X86::NoRegister || DispExpr)
1345         return false;
1346 
1347       if (ErrorOr<int64_t> InputVal = getOperandVal(BaseRegNum))
1348         Output = *InputVal + DispValue;
1349       else
1350         return false;
1351 
1352       break;
1353     }
1354     }
1355     return true;
1356   }
1357 
1358   bool isRegToRegMove(const MCInst &Inst, MCPhysReg &From,
1359                       MCPhysReg &To) const override {
1360     switch (Inst.getOpcode()) {
1361     default:
1362       return false;
1363     case X86::LEAVE:
1364     case X86::LEAVE64:
1365       To = getStackPointer();
1366       From = getFramePointer();
1367       return true;
1368     case X86::MOV64rr:
1369       To = Inst.getOperand(0).getReg();
1370       From = Inst.getOperand(1).getReg();
1371       return true;
1372     }
1373   }
1374 
1375   MCPhysReg getStackPointer() const override { return X86::RSP; }
1376   MCPhysReg getFramePointer() const override { return X86::RBP; }
1377   MCPhysReg getFlagsReg() const override { return X86::EFLAGS; }
1378 
1379   bool escapesVariable(const MCInst &Inst,
1380                        bool HasFramePointer) const override {
1381     int MemOpNo = getMemoryOperandNo(Inst);
1382     const MCInstrDesc &MCII = Info->get(Inst.getOpcode());
1383     const unsigned NumDefs = MCII.getNumDefs();
1384     static BitVector SPBPAliases(BitVector(getAliases(X86::RSP)) |=
1385                                  getAliases(X86::RBP));
1386     static BitVector SPAliases(getAliases(X86::RSP));
1387 
1388     // FIXME: PUSH can be technically a leak, but let's ignore this for now
1389     // because a lot of harmless prologue code will spill SP to the stack.
1390     // Unless push is clearly pushing an object address to the stack as
1391     // demonstrated by having a MemOp.
1392     bool IsPush = isPush(Inst);
1393     if (IsPush && MemOpNo == -1)
1394       return false;
1395 
1396     // We use this to detect LEA (has memop but does not access mem)
1397     bool AccessMem = MCII.mayLoad() || MCII.mayStore();
1398     bool DoesLeak = false;
1399     for (int I = 0, E = MCPlus::getNumPrimeOperands(Inst); I != E; ++I) {
1400       // Ignore if SP/BP is used to dereference memory -- that's fine
1401       if (MemOpNo != -1 && !IsPush && AccessMem && I >= MemOpNo &&
1402           I <= MemOpNo + 5)
1403         continue;
1404       // Ignore if someone is writing to SP/BP
1405       if (I < static_cast<int>(NumDefs))
1406         continue;
1407 
1408       const MCOperand &Operand = Inst.getOperand(I);
1409       if (HasFramePointer && Operand.isReg() && SPBPAliases[Operand.getReg()]) {
1410         DoesLeak = true;
1411         break;
1412       }
1413       if (!HasFramePointer && Operand.isReg() && SPAliases[Operand.getReg()]) {
1414         DoesLeak = true;
1415         break;
1416       }
1417     }
1418 
1419     // If potential leak, check if it is not just writing to itself/sp/bp
1420     if (DoesLeak) {
1421       for (int I = 0, E = NumDefs; I != E; ++I) {
1422         const MCOperand &Operand = Inst.getOperand(I);
1423         if (HasFramePointer && Operand.isReg() &&
1424             SPBPAliases[Operand.getReg()]) {
1425           DoesLeak = false;
1426           break;
1427         }
1428         if (!HasFramePointer && Operand.isReg() &&
1429             SPAliases[Operand.getReg()]) {
1430           DoesLeak = false;
1431           break;
1432         }
1433       }
1434     }
1435     return DoesLeak;
1436   }
1437 
1438   bool addToImm(MCInst &Inst, int64_t &Amt, MCContext *Ctx) const override {
1439     unsigned ImmOpNo = -1U;
1440     int MemOpNo = getMemoryOperandNo(Inst);
1441     if (MemOpNo != -1)
1442       ImmOpNo = MemOpNo + X86::AddrDisp;
1443     else
1444       for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst);
1445            ++Index)
1446         if (Inst.getOperand(Index).isImm())
1447           ImmOpNo = Index;
1448     if (ImmOpNo == -1U)
1449       return false;
1450 
1451     MCOperand &Operand = Inst.getOperand(ImmOpNo);
1452     Amt += Operand.getImm();
1453     Operand.setImm(Amt);
1454     // Check for the need for relaxation
1455     if (int64_t(Amt) == int64_t(int8_t(Amt)))
1456       return true;
1457 
1458     // Relax instruction
1459     switch (Inst.getOpcode()) {
1460     case X86::SUB64ri8:
1461       Inst.setOpcode(X86::SUB64ri32);
1462       break;
1463     case X86::ADD64ri8:
1464       Inst.setOpcode(X86::ADD64ri32);
1465       break;
1466     default:
1467       // No need for relaxation
1468       break;
1469     }
1470     return true;
1471   }
1472 
1473   /// TODO: this implementation currently works for the most common opcodes that
1474   /// load from memory. It can be extended to work with memory store opcodes as
1475   /// well as more memory load opcodes.
1476   bool replaceMemOperandWithImm(MCInst &Inst, StringRef ConstantData,
1477                                 uint64_t Offset) const override {
1478     enum CheckSignExt : uint8_t {
1479       NOCHECK = 0,
1480       CHECK8,
1481       CHECK32,
1482     };
1483 
1484     using CheckList = std::vector<std::pair<CheckSignExt, unsigned>>;
1485     struct InstInfo {
1486       // Size in bytes that Inst loads from memory.
1487       uint8_t DataSize;
1488 
1489       // True when the target operand has to be duplicated because the opcode
1490       // expects a LHS operand.
1491       bool HasLHS;
1492 
1493       // List of checks and corresponding opcodes to be used. We try to use the
1494       // smallest possible immediate value when various sizes are available,
1495       // hence we may need to check whether a larger constant fits in a smaller
1496       // immediate.
1497       CheckList Checks;
1498     };
1499 
1500     InstInfo I;
1501 
1502     switch (Inst.getOpcode()) {
1503     default: {
1504       switch (getPopSize(Inst)) {
1505       case 2:            I = {2, false, {{NOCHECK, X86::MOV16ri}}};  break;
1506       case 4:            I = {4, false, {{NOCHECK, X86::MOV32ri}}};  break;
1507       case 8:            I = {8, false, {{CHECK32, X86::MOV64ri32},
1508                                          {NOCHECK, X86::MOV64rm}}};  break;
1509       default:           return false;
1510       }
1511       break;
1512     }
1513 
1514     // MOV
1515     case X86::MOV8rm:      I = {1, false, {{NOCHECK, X86::MOV8ri}}};   break;
1516     case X86::MOV16rm:     I = {2, false, {{NOCHECK, X86::MOV16ri}}};  break;
1517     case X86::MOV32rm:     I = {4, false, {{NOCHECK, X86::MOV32ri}}};  break;
1518     case X86::MOV64rm:     I = {8, false, {{CHECK32, X86::MOV64ri32},
1519                                            {NOCHECK, X86::MOV64rm}}};  break;
1520 
1521     // MOVZX
1522     case X86::MOVZX16rm8:  I = {1, false, {{NOCHECK, X86::MOV16ri}}};  break;
1523     case X86::MOVZX32rm8:  I = {1, false, {{NOCHECK, X86::MOV32ri}}};  break;
1524     case X86::MOVZX32rm16: I = {2, false, {{NOCHECK, X86::MOV32ri}}};  break;
1525 
1526     // CMP
1527     case X86::CMP8rm:      I = {1, false, {{NOCHECK, X86::CMP8ri}}};   break;
1528     case X86::CMP16rm:     I = {2, false, {{CHECK8,  X86::CMP16ri8},
1529                                            {NOCHECK, X86::CMP16ri}}};  break;
1530     case X86::CMP32rm:     I = {4, false, {{CHECK8,  X86::CMP32ri8},
1531                                            {NOCHECK, X86::CMP32ri}}};  break;
1532     case X86::CMP64rm:     I = {8, false, {{CHECK8,  X86::CMP64ri8},
1533                                            {CHECK32, X86::CMP64ri32},
1534                                            {NOCHECK, X86::CMP64rm}}};  break;
1535 
1536     // TEST
1537     case X86::TEST8mr:     I = {1, false, {{NOCHECK, X86::TEST8ri}}};  break;
1538     case X86::TEST16mr:    I = {2, false, {{NOCHECK, X86::TEST16ri}}}; break;
1539     case X86::TEST32mr:    I = {4, false, {{NOCHECK, X86::TEST32ri}}}; break;
1540     case X86::TEST64mr:    I = {8, false, {{CHECK32, X86::TEST64ri32},
1541                                            {NOCHECK, X86::TEST64mr}}}; break;
1542 
1543     // ADD
1544     case X86::ADD8rm:      I = {1, true,  {{NOCHECK, X86::ADD8ri}}};   break;
1545     case X86::ADD16rm:     I = {2, true,  {{CHECK8,  X86::ADD16ri8},
1546                                            {NOCHECK, X86::ADD16ri}}};  break;
1547     case X86::ADD32rm:     I = {4, true,  {{CHECK8,  X86::ADD32ri8},
1548                                            {NOCHECK, X86::ADD32ri}}};  break;
1549     case X86::ADD64rm:     I = {8, true,  {{CHECK8,  X86::ADD64ri8},
1550                                            {CHECK32, X86::ADD64ri32},
1551                                            {NOCHECK, X86::ADD64rm}}};  break;
1552 
1553     // SUB
1554     case X86::SUB8rm:      I = {1, true,  {{NOCHECK, X86::SUB8ri}}};   break;
1555     case X86::SUB16rm:     I = {2, true,  {{CHECK8,  X86::SUB16ri8},
1556                                            {NOCHECK, X86::SUB16ri}}};  break;
1557     case X86::SUB32rm:     I = {4, true,  {{CHECK8,  X86::SUB32ri8},
1558                                            {NOCHECK, X86::SUB32ri}}};  break;
1559     case X86::SUB64rm:     I = {8, true,  {{CHECK8,  X86::SUB64ri8},
1560                                            {CHECK32, X86::SUB64ri32},
1561                                            {NOCHECK, X86::SUB64rm}}};  break;
1562 
1563     // AND
1564     case X86::AND8rm:      I = {1, true,  {{NOCHECK, X86::AND8ri}}};   break;
1565     case X86::AND16rm:     I = {2, true,  {{CHECK8,  X86::AND16ri8},
1566                                            {NOCHECK, X86::AND16ri}}};  break;
1567     case X86::AND32rm:     I = {4, true,  {{CHECK8,  X86::AND32ri8},
1568                                            {NOCHECK, X86::AND32ri}}};  break;
1569     case X86::AND64rm:     I = {8, true,  {{CHECK8,  X86::AND64ri8},
1570                                            {CHECK32, X86::AND64ri32},
1571                                            {NOCHECK, X86::AND64rm}}};  break;
1572 
1573     // OR
1574     case X86::OR8rm:       I = {1, true,  {{NOCHECK, X86::OR8ri}}};    break;
1575     case X86::OR16rm:      I = {2, true,  {{CHECK8,  X86::OR16ri8},
1576                                            {NOCHECK, X86::OR16ri}}};   break;
1577     case X86::OR32rm:      I = {4, true,  {{CHECK8,  X86::OR32ri8},
1578                                            {NOCHECK, X86::OR32ri}}};   break;
1579     case X86::OR64rm:      I = {8, true,  {{CHECK8,  X86::OR64ri8},
1580                                            {CHECK32, X86::OR64ri32},
1581                                            {NOCHECK, X86::OR64rm}}};   break;
1582 
1583     // XOR
1584     case X86::XOR8rm:      I = {1, true,  {{NOCHECK, X86::XOR8ri}}};   break;
1585     case X86::XOR16rm:     I = {2, true,  {{CHECK8,  X86::XOR16ri8},
1586                                            {NOCHECK, X86::XOR16ri}}};  break;
1587     case X86::XOR32rm:     I = {4, true,  {{CHECK8,  X86::XOR32ri8},
1588                                            {NOCHECK, X86::XOR32ri}}};  break;
1589     case X86::XOR64rm:     I = {8, true,  {{CHECK8,  X86::XOR64ri8},
1590                                            {CHECK32, X86::XOR64ri32},
1591                                            {NOCHECK, X86::XOR64rm}}};  break;
1592     }
1593 
1594     // Compute the immediate value.
1595     assert(Offset + I.DataSize <= ConstantData.size() &&
1596            "invalid offset for given constant data");
1597     int64_t ImmVal =
1598         DataExtractor(ConstantData, true, 8).getSigned(&Offset, I.DataSize);
1599 
1600     // Compute the new opcode.
1601     unsigned NewOpcode = 0;
1602     for (const std::pair<CheckSignExt, unsigned> &Check : I.Checks) {
1603       NewOpcode = Check.second;
1604       if (Check.first == NOCHECK)
1605         break;
1606       if (Check.first == CHECK8 && isInt<8>(ImmVal))
1607         break;
1608       if (Check.first == CHECK32 && isInt<32>(ImmVal))
1609         break;
1610     }
1611     if (NewOpcode == Inst.getOpcode())
1612       return false;
1613 
1614     // Modify the instruction.
1615     MCOperand ImmOp = MCOperand::createImm(ImmVal);
1616     uint32_t TargetOpNum = 0;
1617     // Test instruction does not follow the regular pattern of putting the
1618     // memory reference of a load (5 MCOperands) last in the list of operands.
1619     // Since it is not modifying the register operand, it is not treated as
1620     // a destination operand and it is not the first operand as it is in the
1621     // other instructions we treat here.
1622     if (NewOpcode == X86::TEST8ri || NewOpcode == X86::TEST16ri ||
1623         NewOpcode == X86::TEST32ri || NewOpcode == X86::TEST64ri32)
1624       TargetOpNum = getMemoryOperandNo(Inst) + X86::AddrNumOperands;
1625 
1626     MCOperand TargetOp = Inst.getOperand(TargetOpNum);
1627     Inst.clear();
1628     Inst.setOpcode(NewOpcode);
1629     Inst.addOperand(TargetOp);
1630     if (I.HasLHS)
1631       Inst.addOperand(TargetOp);
1632     Inst.addOperand(ImmOp);
1633 
1634     return true;
1635   }
1636 
1637   /// TODO: this implementation currently works for the most common opcodes that
1638   /// load from memory. It can be extended to work with memory store opcodes as
1639   /// well as more memory load opcodes.
1640   bool replaceMemOperandWithReg(MCInst &Inst, MCPhysReg RegNum) const override {
1641     unsigned NewOpcode;
1642 
1643     switch (Inst.getOpcode()) {
1644     default: {
1645       switch (getPopSize(Inst)) {
1646       case 2:            NewOpcode = X86::MOV16rr; break;
1647       case 4:            NewOpcode = X86::MOV32rr; break;
1648       case 8:            NewOpcode = X86::MOV64rr; break;
1649       default:           return false;
1650       }
1651       break;
1652     }
1653 
1654     // MOV
1655     case X86::MOV8rm:      NewOpcode = X86::MOV8rr;   break;
1656     case X86::MOV16rm:     NewOpcode = X86::MOV16rr;  break;
1657     case X86::MOV32rm:     NewOpcode = X86::MOV32rr;  break;
1658     case X86::MOV64rm:     NewOpcode = X86::MOV64rr;  break;
1659     }
1660 
1661     // Modify the instruction.
1662     MCOperand RegOp = MCOperand::createReg(RegNum);
1663     MCOperand TargetOp = Inst.getOperand(0);
1664     Inst.clear();
1665     Inst.setOpcode(NewOpcode);
1666     Inst.addOperand(TargetOp);
1667     Inst.addOperand(RegOp);
1668 
1669     return true;
1670   }
1671 
1672   bool isRedundantMove(const MCInst &Inst) const override {
1673     switch (Inst.getOpcode()) {
1674     default:
1675       return false;
1676 
1677     // MOV
1678     case X86::MOV8rr:
1679     case X86::MOV16rr:
1680     case X86::MOV32rr:
1681     case X86::MOV64rr:
1682       break;
1683     }
1684 
1685     assert(Inst.getOperand(0).isReg() && Inst.getOperand(1).isReg());
1686     return Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg();
1687   }
1688 
1689   bool requiresAlignedAddress(const MCInst &Inst) const override {
1690     const MCInstrDesc &Desc = Info->get(Inst.getOpcode());
1691     for (unsigned int I = 0; I < Desc.getNumOperands(); ++I) {
1692       const MCOperandInfo &Op = Desc.OpInfo[I];
1693       if (Op.OperandType != MCOI::OPERAND_REGISTER)
1694         continue;
1695       if (Op.RegClass == X86::VR128RegClassID)
1696         return true;
1697     }
1698     return false;
1699   }
1700 
1701   bool convertJmpToTailCall(MCInst &Inst) override {
1702     if (isTailCall(Inst))
1703       return false;
1704 
1705     int NewOpcode;
1706     switch (Inst.getOpcode()) {
1707     default:
1708       return false;
1709     case X86::JMP_1:
1710     case X86::JMP_2:
1711     case X86::JMP_4:
1712       NewOpcode = X86::JMP_4;
1713       break;
1714     case X86::JMP16m:
1715     case X86::JMP32m:
1716     case X86::JMP64m:
1717       NewOpcode = X86::JMP32m;
1718       break;
1719     case X86::JMP16r:
1720     case X86::JMP32r:
1721     case X86::JMP64r:
1722       NewOpcode = X86::JMP32r;
1723       break;
1724     }
1725 
1726     Inst.setOpcode(NewOpcode);
1727     setTailCall(Inst);
1728     return true;
1729   }
1730 
1731   bool convertTailCallToJmp(MCInst &Inst) override {
1732     int NewOpcode;
1733     switch (Inst.getOpcode()) {
1734     default:
1735       return false;
1736     case X86::JMP_4:
1737       NewOpcode = X86::JMP_1;
1738       break;
1739     case X86::JMP32m:
1740       NewOpcode = X86::JMP64m;
1741       break;
1742     case X86::JMP32r:
1743       NewOpcode = X86::JMP64r;
1744       break;
1745     }
1746 
1747     Inst.setOpcode(NewOpcode);
1748     removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall);
1749     clearOffset(Inst);
1750     return true;
1751   }
1752 
1753   bool convertTailCallToCall(MCInst &Inst) override {
1754     int NewOpcode;
1755     switch (Inst.getOpcode()) {
1756     default:
1757       return false;
1758     case X86::JMP_4:
1759       NewOpcode = X86::CALL64pcrel32;
1760       break;
1761     case X86::JMP32m:
1762       NewOpcode = X86::CALL64m;
1763       break;
1764     case X86::JMP32r:
1765       NewOpcode = X86::CALL64r;
1766       break;
1767     }
1768 
1769     Inst.setOpcode(NewOpcode);
1770     removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall);
1771     return true;
1772   }
1773 
1774   bool convertCallToIndirectCall(MCInst &Inst, const MCSymbol *TargetLocation,
1775                                  MCContext *Ctx) override {
1776     bool IsTailCall = isTailCall(Inst);
1777     assert((Inst.getOpcode() == X86::CALL64pcrel32 ||
1778             (Inst.getOpcode() == X86::JMP_4 && IsTailCall)) &&
1779            "64-bit direct (tail) call instruction expected");
1780     const auto NewOpcode =
1781         (Inst.getOpcode() == X86::CALL64pcrel32) ? X86::CALL64m : X86::JMP32m;
1782     Inst.setOpcode(NewOpcode);
1783 
1784     // Replace the first operand and preserve auxiliary operands of
1785     // the instruction.
1786     Inst.erase(Inst.begin());
1787     Inst.insert(Inst.begin(),
1788                 MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
1789     Inst.insert(Inst.begin(),
1790                 MCOperand::createExpr(                  // Displacement
1791                     MCSymbolRefExpr::create(TargetLocation,
1792                                             MCSymbolRefExpr::VK_None, *Ctx)));
1793     Inst.insert(Inst.begin(),
1794                 MCOperand::createReg(X86::NoRegister)); // IndexReg
1795     Inst.insert(Inst.begin(),
1796                 MCOperand::createImm(1));               // ScaleAmt
1797     Inst.insert(Inst.begin(),
1798                 MCOperand::createReg(X86::RIP));        // BaseReg
1799 
1800     return true;
1801   }
1802 
1803   void convertIndirectCallToLoad(MCInst &Inst, MCPhysReg Reg) override {
1804     bool IsTailCall = isTailCall(Inst);
1805     if (IsTailCall)
1806       removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall);
1807     if (Inst.getOpcode() == X86::CALL64m ||
1808         (Inst.getOpcode() == X86::JMP32m && IsTailCall)) {
1809       Inst.setOpcode(X86::MOV64rm);
1810       Inst.insert(Inst.begin(), MCOperand::createReg(Reg));
1811       return;
1812     }
1813     if (Inst.getOpcode() == X86::CALL64r ||
1814         (Inst.getOpcode() == X86::JMP32r && IsTailCall)) {
1815       Inst.setOpcode(X86::MOV64rr);
1816       Inst.insert(Inst.begin(), MCOperand::createReg(Reg));
1817       return;
1818     }
1819     LLVM_DEBUG(Inst.dump());
1820     llvm_unreachable("not implemented");
1821   }
1822 
1823   bool shortenInstruction(MCInst &Inst,
1824                           const MCSubtargetInfo &STI) const override {
1825     unsigned OldOpcode = Inst.getOpcode();
1826     unsigned NewOpcode = OldOpcode;
1827 
1828     int MemOpNo = getMemoryOperandNo(Inst);
1829 
1830     // Check and remove redundant Address-Size override prefix.
1831     if (opts::X86StripRedundantAddressSize) {
1832       uint64_t TSFlags = Info->get(OldOpcode).TSFlags;
1833       unsigned Flags = Inst.getFlags();
1834 
1835       if (!X86_MC::needsAddressSizeOverride(Inst, STI, MemOpNo, TSFlags) &&
1836           Flags & X86::IP_HAS_AD_SIZE)
1837         Inst.setFlags(Flags ^ X86::IP_HAS_AD_SIZE);
1838     }
1839 
1840     // Check and remove EIZ/RIZ. These cases represent ambiguous cases where
1841     // SIB byte is present, but no index is used and modrm alone should have
1842     // been enough. Converting to NoRegister effectively removes the SIB byte.
1843     if (MemOpNo >= 0) {
1844       MCOperand &IndexOp =
1845           Inst.getOperand(static_cast<unsigned>(MemOpNo) + X86::AddrIndexReg);
1846       if (IndexOp.getReg() == X86::EIZ || IndexOp.getReg() == X86::RIZ)
1847         IndexOp = MCOperand::createReg(X86::NoRegister);
1848     }
1849 
1850     if (isBranch(Inst)) {
1851       NewOpcode = getShortBranchOpcode(OldOpcode);
1852     } else if (OldOpcode == X86::MOV64ri) {
1853       if (Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).isImm()) {
1854         const int64_t Imm =
1855             Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).getImm();
1856         if (int64_t(Imm) == int64_t(int32_t(Imm)))
1857           NewOpcode = X86::MOV64ri32;
1858       }
1859     } else {
1860       // If it's arithmetic instruction check if signed operand fits in 1 byte.
1861       const unsigned ShortOpcode = getShortArithOpcode(OldOpcode);
1862       if (ShortOpcode != OldOpcode &&
1863           Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).isImm()) {
1864         int64_t Imm =
1865             Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).getImm();
1866         if (int64_t(Imm) == int64_t(int8_t(Imm)))
1867           NewOpcode = ShortOpcode;
1868       }
1869     }
1870 
1871     if (NewOpcode == OldOpcode)
1872       return false;
1873 
1874     Inst.setOpcode(NewOpcode);
1875     return true;
1876   }
1877 
1878   bool
1879   convertMoveToConditionalMove(MCInst &Inst, unsigned CC, bool AllowStackMemOp,
1880                                bool AllowBasePtrStackMemOp) const override {
1881     // - Register-register moves are OK
1882     // - Stores are filtered out by opcode (no store CMOV)
1883     // - Non-stack loads are prohibited (generally unsafe)
1884     // - Stack loads are OK if AllowStackMemOp is true
1885     // - Stack loads with RBP are OK if AllowBasePtrStackMemOp is true
1886     if (isLoad(Inst)) {
1887       // If stack memory operands are not allowed, no loads are allowed
1888       if (!AllowStackMemOp)
1889         return false;
1890 
1891       // If stack memory operands are allowed, check if it's a load from stack
1892       bool IsLoad, IsStore, IsStoreFromReg, IsSimple, IsIndexed;
1893       MCPhysReg Reg;
1894       int32_t SrcImm;
1895       uint16_t StackPtrReg;
1896       int64_t StackOffset;
1897       uint8_t Size;
1898       bool IsStackAccess =
1899           isStackAccess(Inst, IsLoad, IsStore, IsStoreFromReg, Reg, SrcImm,
1900                         StackPtrReg, StackOffset, Size, IsSimple, IsIndexed);
1901       // Prohibit non-stack-based loads
1902       if (!IsStackAccess)
1903         return false;
1904       // If stack memory operands are allowed, check if it's RBP-based
1905       if (!AllowBasePtrStackMemOp &&
1906           RegInfo->isSubRegisterEq(X86::RBP, StackPtrReg))
1907         return false;
1908     }
1909 
1910     unsigned NewOpcode = 0;
1911     switch (Inst.getOpcode()) {
1912     case X86::MOV16rr:
1913       NewOpcode = X86::CMOV16rr;
1914       break;
1915     case X86::MOV16rm:
1916       NewOpcode = X86::CMOV16rm;
1917       break;
1918     case X86::MOV32rr:
1919       NewOpcode = X86::CMOV32rr;
1920       break;
1921     case X86::MOV32rm:
1922       NewOpcode = X86::CMOV32rm;
1923       break;
1924     case X86::MOV64rr:
1925       NewOpcode = X86::CMOV64rr;
1926       break;
1927     case X86::MOV64rm:
1928       NewOpcode = X86::CMOV64rm;
1929       break;
1930     default:
1931       return false;
1932     }
1933     Inst.setOpcode(NewOpcode);
1934     // Insert CC at the end of prime operands, before annotations
1935     Inst.insert(Inst.begin() + MCPlus::getNumPrimeOperands(Inst),
1936                 MCOperand::createImm(CC));
1937     // CMOV is a 3-operand MCInst, so duplicate the destination as src1
1938     Inst.insert(Inst.begin(), Inst.getOperand(0));
1939     return true;
1940   }
1941 
1942   bool lowerTailCall(MCInst &Inst) override {
1943     if (Inst.getOpcode() == X86::JMP_4 && isTailCall(Inst)) {
1944       Inst.setOpcode(X86::JMP_1);
1945       removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall);
1946       return true;
1947     }
1948     return false;
1949   }
1950 
1951   const MCSymbol *getTargetSymbol(const MCInst &Inst,
1952                                   unsigned OpNum = 0) const override {
1953     if (OpNum >= MCPlus::getNumPrimeOperands(Inst))
1954       return nullptr;
1955 
1956     const MCOperand &Op = Inst.getOperand(OpNum);
1957     if (!Op.isExpr())
1958       return nullptr;
1959 
1960     auto *SymExpr = dyn_cast<MCSymbolRefExpr>(Op.getExpr());
1961     if (!SymExpr || SymExpr->getKind() != MCSymbolRefExpr::VK_None)
1962       return nullptr;
1963 
1964     return &SymExpr->getSymbol();
1965   }
1966 
1967   // This is the same as the base class, but since we are overriding one of
1968   // getTargetSymbol's signatures above, we need to override all of them.
1969   const MCSymbol *getTargetSymbol(const MCExpr *Expr) const override {
1970     return &cast<const MCSymbolRefExpr>(Expr)->getSymbol();
1971   }
1972 
1973   bool analyzeBranch(InstructionIterator Begin, InstructionIterator End,
1974                      const MCSymbol *&TBB, const MCSymbol *&FBB,
1975                      MCInst *&CondBranch,
1976                      MCInst *&UncondBranch) const override {
1977     auto I = End;
1978 
1979     // Bottom-up analysis
1980     while (I != Begin) {
1981       --I;
1982 
1983       // Ignore nops and CFIs
1984       if (isPseudo(*I))
1985         continue;
1986 
1987       // Stop when we find the first non-terminator
1988       if (!isTerminator(*I))
1989         break;
1990 
1991       if (!isBranch(*I))
1992         break;
1993 
1994       // Handle unconditional branches.
1995       if ((I->getOpcode() == X86::JMP_1 || I->getOpcode() == X86::JMP_2 ||
1996            I->getOpcode() == X86::JMP_4) &&
1997           !isTailCall(*I)) {
1998         // If any code was seen after this unconditional branch, we've seen
1999         // unreachable code. Ignore them.
2000         CondBranch = nullptr;
2001         UncondBranch = &*I;
2002         const MCSymbol *Sym = getTargetSymbol(*I);
2003         assert(Sym != nullptr &&
2004                "Couldn't extract BB symbol from jump operand");
2005         TBB = Sym;
2006         continue;
2007       }
2008 
2009       // Handle conditional branches and ignore indirect branches
2010       if (!isUnsupportedBranch(I->getOpcode()) &&
2011           getCondCode(*I) == X86::COND_INVALID) {
2012         // Indirect branch
2013         return false;
2014       }
2015 
2016       if (CondBranch == nullptr) {
2017         const MCSymbol *TargetBB = getTargetSymbol(*I);
2018         if (TargetBB == nullptr) {
2019           // Unrecognized branch target
2020           return false;
2021         }
2022         FBB = TBB;
2023         TBB = TargetBB;
2024         CondBranch = &*I;
2025         continue;
2026       }
2027 
2028       llvm_unreachable("multiple conditional branches in one BB");
2029     }
2030     return true;
2031   }
2032 
2033   template <typename Itr>
2034   std::pair<IndirectBranchType, MCInst *>
2035   analyzePICJumpTable(Itr II, Itr IE, MCPhysReg R1, MCPhysReg R2) const {
2036     // Analyze PIC-style jump table code template:
2037     //
2038     //    lea PIC_JUMP_TABLE(%rip), {%r1|%r2}     <- MemLocInstr
2039     //    mov ({%r1|%r2}, %index, 4), {%r2|%r1}
2040     //    add %r2, %r1
2041     //    jmp *%r1
2042     //
2043     // (with any irrelevant instructions in-between)
2044     //
2045     // When we call this helper we've already determined %r1 and %r2, and
2046     // reverse instruction iterator \p II is pointing to the ADD instruction.
2047     //
2048     // PIC jump table looks like following:
2049     //
2050     //   JT:  ----------
2051     //    E1:| L1 - JT  |
2052     //       |----------|
2053     //    E2:| L2 - JT  |
2054     //       |----------|
2055     //       |          |
2056     //          ......
2057     //    En:| Ln - JT  |
2058     //        ----------
2059     //
2060     // Where L1, L2, ..., Ln represent labels in the function.
2061     //
2062     // The actual relocations in the table will be of the form:
2063     //
2064     //   Ln - JT
2065     //    = (Ln - En) + (En - JT)
2066     //    = R_X86_64_PC32(Ln) + En - JT
2067     //    = R_X86_64_PC32(Ln + offsetof(En))
2068     //
2069     LLVM_DEBUG(dbgs() << "Checking for PIC jump table\n");
2070     MCInst *MemLocInstr = nullptr;
2071     const MCInst *MovInstr = nullptr;
2072     while (++II != IE) {
2073       MCInst &Instr = *II;
2074       const MCInstrDesc &InstrDesc = Info->get(Instr.getOpcode());
2075       if (!InstrDesc.hasDefOfPhysReg(Instr, R1, *RegInfo) &&
2076           !InstrDesc.hasDefOfPhysReg(Instr, R2, *RegInfo)) {
2077         // Ignore instructions that don't affect R1, R2 registers.
2078         continue;
2079       }
2080       if (!MovInstr) {
2081         // Expect to see MOV instruction.
2082         if (!isMOVSX64rm32(Instr)) {
2083           LLVM_DEBUG(dbgs() << "MOV instruction expected.\n");
2084           break;
2085         }
2086 
2087         // Check if it's setting %r1 or %r2. In canonical form it sets %r2.
2088         // If it sets %r1 - rename the registers so we have to only check
2089         // a single form.
2090         unsigned MovDestReg = Instr.getOperand(0).getReg();
2091         if (MovDestReg != R2)
2092           std::swap(R1, R2);
2093         if (MovDestReg != R2) {
2094           LLVM_DEBUG(dbgs() << "MOV instruction expected to set %r2\n");
2095           break;
2096         }
2097 
2098         // Verify operands for MOV.
2099         unsigned  BaseRegNum;
2100         int64_t   ScaleValue;
2101         unsigned  IndexRegNum;
2102         int64_t   DispValue;
2103         unsigned  SegRegNum;
2104         if (!evaluateX86MemoryOperand(Instr, &BaseRegNum, &ScaleValue,
2105                                       &IndexRegNum, &DispValue, &SegRegNum))
2106           break;
2107         if (BaseRegNum != R1 || ScaleValue != 4 ||
2108             IndexRegNum == X86::NoRegister || DispValue != 0 ||
2109             SegRegNum != X86::NoRegister)
2110           break;
2111         MovInstr = &Instr;
2112       } else {
2113         if (!InstrDesc.hasDefOfPhysReg(Instr, R1, *RegInfo))
2114           continue;
2115         if (!isLEA64r(Instr)) {
2116           LLVM_DEBUG(dbgs() << "LEA instruction expected\n");
2117           break;
2118         }
2119         if (Instr.getOperand(0).getReg() != R1) {
2120           LLVM_DEBUG(dbgs() << "LEA instruction expected to set %r1\n");
2121           break;
2122         }
2123 
2124         // Verify operands for LEA.
2125         unsigned      BaseRegNum;
2126         int64_t       ScaleValue;
2127         unsigned      IndexRegNum;
2128         const MCExpr *DispExpr = nullptr;
2129         int64_t       DispValue;
2130         unsigned      SegRegNum;
2131         if (!evaluateX86MemoryOperand(Instr, &BaseRegNum, &ScaleValue,
2132                                       &IndexRegNum, &DispValue, &SegRegNum,
2133                                       &DispExpr))
2134           break;
2135         if (BaseRegNum != RegInfo->getProgramCounter() ||
2136             IndexRegNum != X86::NoRegister || SegRegNum != X86::NoRegister ||
2137             DispExpr == nullptr)
2138           break;
2139         MemLocInstr = &Instr;
2140         break;
2141       }
2142     }
2143 
2144     if (!MemLocInstr)
2145       return std::make_pair(IndirectBranchType::UNKNOWN, nullptr);
2146 
2147     LLVM_DEBUG(dbgs() << "checking potential PIC jump table\n");
2148     return std::make_pair(IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE,
2149                           MemLocInstr);
2150   }
2151 
2152   IndirectBranchType analyzeIndirectBranch(
2153       MCInst &Instruction, InstructionIterator Begin, InstructionIterator End,
2154       const unsigned PtrSize, MCInst *&MemLocInstrOut, unsigned &BaseRegNumOut,
2155       unsigned &IndexRegNumOut, int64_t &DispValueOut,
2156       const MCExpr *&DispExprOut, MCInst *&PCRelBaseOut) const override {
2157     // Try to find a (base) memory location from where the address for
2158     // the indirect branch is loaded. For X86-64 the memory will be specified
2159     // in the following format:
2160     //
2161     //   {%rip}/{%basereg} + Imm + IndexReg * Scale
2162     //
2163     // We are interested in the cases where Scale == sizeof(uintptr_t) and
2164     // the contents of the memory are presumably an array of pointers to code.
2165     //
2166     // Normal jump table:
2167     //
2168     //    jmp *(JUMP_TABLE, %index, Scale)        <- MemLocInstr
2169     //
2170     //    or
2171     //
2172     //    mov (JUMP_TABLE, %index, Scale), %r1    <- MemLocInstr
2173     //    ...
2174     //    jmp %r1
2175     //
2176     // We handle PIC-style jump tables separately.
2177     //
2178     MemLocInstrOut = nullptr;
2179     BaseRegNumOut = X86::NoRegister;
2180     IndexRegNumOut = X86::NoRegister;
2181     DispValueOut = 0;
2182     DispExprOut = nullptr;
2183 
2184     std::reverse_iterator<InstructionIterator> II(End);
2185     std::reverse_iterator<InstructionIterator> IE(Begin);
2186 
2187     IndirectBranchType Type = IndirectBranchType::UNKNOWN;
2188 
2189     // An instruction referencing memory used by jump instruction (directly or
2190     // via register). This location could be an array of function pointers
2191     // in case of indirect tail call, or a jump table.
2192     MCInst *MemLocInstr = nullptr;
2193 
2194     if (MCPlus::getNumPrimeOperands(Instruction) == 1) {
2195       // If the indirect jump is on register - try to detect if the
2196       // register value is loaded from a memory location.
2197       assert(Instruction.getOperand(0).isReg() && "register operand expected");
2198       const unsigned R1 = Instruction.getOperand(0).getReg();
2199       // Check if one of the previous instructions defines the jump-on register.
2200       for (auto PrevII = II; PrevII != IE; ++PrevII) {
2201         MCInst &PrevInstr = *PrevII;
2202         const MCInstrDesc &PrevInstrDesc = Info->get(PrevInstr.getOpcode());
2203 
2204         if (!PrevInstrDesc.hasDefOfPhysReg(PrevInstr, R1, *RegInfo))
2205           continue;
2206 
2207         if (isMoveMem2Reg(PrevInstr)) {
2208           MemLocInstr = &PrevInstr;
2209           break;
2210         }
2211         if (isADD64rr(PrevInstr)) {
2212           unsigned R2 = PrevInstr.getOperand(2).getReg();
2213           if (R1 == R2)
2214             return IndirectBranchType::UNKNOWN;
2215           std::tie(Type, MemLocInstr) = analyzePICJumpTable(PrevII, IE, R1, R2);
2216           break;
2217         }
2218         return IndirectBranchType::UNKNOWN;
2219       }
2220       if (!MemLocInstr) {
2221         // No definition seen for the register in this function so far. Could be
2222         // an input parameter - which means it is an external code reference.
2223         // It also could be that the definition happens to be in the code that
2224         // we haven't processed yet. Since we have to be conservative, return
2225         // as UNKNOWN case.
2226         return IndirectBranchType::UNKNOWN;
2227       }
2228     } else {
2229       MemLocInstr = &Instruction;
2230     }
2231 
2232     const MCRegister RIPRegister = RegInfo->getProgramCounter();
2233 
2234     // Analyze the memory location.
2235     unsigned BaseRegNum, IndexRegNum, SegRegNum;
2236     int64_t ScaleValue, DispValue;
2237     const MCExpr *DispExpr;
2238 
2239     if (!evaluateX86MemoryOperand(*MemLocInstr, &BaseRegNum, &ScaleValue,
2240                                   &IndexRegNum, &DispValue, &SegRegNum,
2241                                   &DispExpr))
2242       return IndirectBranchType::UNKNOWN;
2243 
2244     BaseRegNumOut = BaseRegNum;
2245     IndexRegNumOut = IndexRegNum;
2246     DispValueOut = DispValue;
2247     DispExprOut = DispExpr;
2248 
2249     if ((BaseRegNum != X86::NoRegister && BaseRegNum != RIPRegister) ||
2250         SegRegNum != X86::NoRegister)
2251       return IndirectBranchType::UNKNOWN;
2252 
2253     if (MemLocInstr == &Instruction &&
2254         (!ScaleValue || IndexRegNum == X86::NoRegister)) {
2255       MemLocInstrOut = MemLocInstr;
2256       return IndirectBranchType::POSSIBLE_FIXED_BRANCH;
2257     }
2258 
2259     if (Type == IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE &&
2260         (ScaleValue != 1 || BaseRegNum != RIPRegister))
2261       return IndirectBranchType::UNKNOWN;
2262 
2263     if (Type != IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE &&
2264         ScaleValue != PtrSize)
2265       return IndirectBranchType::UNKNOWN;
2266 
2267     MemLocInstrOut = MemLocInstr;
2268 
2269     return Type;
2270   }
2271 
2272   /// Analyze a callsite to see if it could be a virtual method call.  This only
2273   /// checks to see if the overall pattern is satisfied, it does not guarantee
2274   /// that the callsite is a true virtual method call.
2275   /// The format of virtual method calls that are recognized is one of the
2276   /// following:
2277   ///
2278   ///  Form 1: (found in debug code)
2279   ///    add METHOD_OFFSET, %VtableReg
2280   ///    mov (%VtableReg), %MethodReg
2281   ///    ...
2282   ///    call or jmp *%MethodReg
2283   ///
2284   ///  Form 2:
2285   ///    mov METHOD_OFFSET(%VtableReg), %MethodReg
2286   ///    ...
2287   ///    call or jmp *%MethodReg
2288   ///
2289   ///  Form 3:
2290   ///    ...
2291   ///    call or jmp *METHOD_OFFSET(%VtableReg)
2292   ///
2293   bool analyzeVirtualMethodCall(InstructionIterator ForwardBegin,
2294                                 InstructionIterator ForwardEnd,
2295                                 std::vector<MCInst *> &MethodFetchInsns,
2296                                 unsigned &VtableRegNum, unsigned &MethodRegNum,
2297                                 uint64_t &MethodOffset) const override {
2298     VtableRegNum = X86::NoRegister;
2299     MethodRegNum = X86::NoRegister;
2300     MethodOffset = 0;
2301 
2302     std::reverse_iterator<InstructionIterator> Itr(ForwardEnd);
2303     std::reverse_iterator<InstructionIterator> End(ForwardBegin);
2304 
2305     MCInst &CallInst = *Itr++;
2306     assert(isIndirectBranch(CallInst) || isCall(CallInst));
2307 
2308     unsigned BaseReg, IndexReg, SegmentReg;
2309     int64_t Scale, Disp;
2310     const MCExpr *DispExpr;
2311 
2312     // The call can just be jmp offset(reg)
2313     if (evaluateX86MemoryOperand(CallInst, &BaseReg, &Scale, &IndexReg, &Disp,
2314                                  &SegmentReg, &DispExpr)) {
2315       if (!DispExpr && BaseReg != X86::RIP && BaseReg != X86::RBP &&
2316           BaseReg != X86::NoRegister) {
2317         MethodRegNum = BaseReg;
2318         if (Scale == 1 && IndexReg == X86::NoRegister &&
2319             SegmentReg == X86::NoRegister) {
2320           VtableRegNum = MethodRegNum;
2321           MethodOffset = Disp;
2322           MethodFetchInsns.push_back(&CallInst);
2323           return true;
2324         }
2325       }
2326       return false;
2327     }
2328     if (CallInst.getOperand(0).isReg())
2329       MethodRegNum = CallInst.getOperand(0).getReg();
2330     else
2331       return false;
2332 
2333     if (MethodRegNum == X86::RIP || MethodRegNum == X86::RBP) {
2334       VtableRegNum = X86::NoRegister;
2335       MethodRegNum = X86::NoRegister;
2336       return false;
2337     }
2338 
2339     // find load from vtable, this may or may not include the method offset
2340     while (Itr != End) {
2341       MCInst &CurInst = *Itr++;
2342       const MCInstrDesc &Desc = Info->get(CurInst.getOpcode());
2343       if (Desc.hasDefOfPhysReg(CurInst, MethodRegNum, *RegInfo)) {
2344         if (isLoad(CurInst) &&
2345             evaluateX86MemoryOperand(CurInst, &BaseReg, &Scale, &IndexReg,
2346                                      &Disp, &SegmentReg, &DispExpr)) {
2347           if (!DispExpr && Scale == 1 && BaseReg != X86::RIP &&
2348               BaseReg != X86::RBP && BaseReg != X86::NoRegister &&
2349               IndexReg == X86::NoRegister && SegmentReg == X86::NoRegister &&
2350               BaseReg != X86::RIP) {
2351             VtableRegNum = BaseReg;
2352             MethodOffset = Disp;
2353             MethodFetchInsns.push_back(&CurInst);
2354             if (MethodOffset != 0)
2355               return true;
2356             break;
2357           }
2358         }
2359         return false;
2360       }
2361     }
2362 
2363     if (!VtableRegNum)
2364       return false;
2365 
2366     // look for any adds affecting the method register.
2367     while (Itr != End) {
2368       MCInst &CurInst = *Itr++;
2369       const MCInstrDesc &Desc = Info->get(CurInst.getOpcode());
2370       if (Desc.hasDefOfPhysReg(CurInst, VtableRegNum, *RegInfo)) {
2371         if (isADDri(CurInst)) {
2372           assert(!MethodOffset);
2373           MethodOffset = CurInst.getOperand(2).getImm();
2374           MethodFetchInsns.insert(MethodFetchInsns.begin(), &CurInst);
2375           break;
2376         }
2377       }
2378     }
2379 
2380     return true;
2381   }
2382 
2383   bool createStackPointerIncrement(MCInst &Inst, int Size,
2384                                    bool NoFlagsClobber) const override {
2385     if (NoFlagsClobber) {
2386       Inst.setOpcode(X86::LEA64r);
2387       Inst.clear();
2388       Inst.addOperand(MCOperand::createReg(X86::RSP));
2389       Inst.addOperand(MCOperand::createReg(X86::RSP));        // BaseReg
2390       Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
2391       Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
2392       Inst.addOperand(MCOperand::createImm(-Size));           // Displacement
2393       Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
2394       return true;
2395     }
2396     Inst.setOpcode(X86::SUB64ri8);
2397     Inst.clear();
2398     Inst.addOperand(MCOperand::createReg(X86::RSP));
2399     Inst.addOperand(MCOperand::createReg(X86::RSP));
2400     Inst.addOperand(MCOperand::createImm(Size));
2401     return true;
2402   }
2403 
2404   bool createStackPointerDecrement(MCInst &Inst, int Size,
2405                                    bool NoFlagsClobber) const override {
2406     if (NoFlagsClobber) {
2407       Inst.setOpcode(X86::LEA64r);
2408       Inst.clear();
2409       Inst.addOperand(MCOperand::createReg(X86::RSP));
2410       Inst.addOperand(MCOperand::createReg(X86::RSP));        // BaseReg
2411       Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
2412       Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
2413       Inst.addOperand(MCOperand::createImm(Size));            // Displacement
2414       Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
2415       return true;
2416     }
2417     Inst.setOpcode(X86::ADD64ri8);
2418     Inst.clear();
2419     Inst.addOperand(MCOperand::createReg(X86::RSP));
2420     Inst.addOperand(MCOperand::createReg(X86::RSP));
2421     Inst.addOperand(MCOperand::createImm(Size));
2422     return true;
2423   }
2424 
2425   bool createSaveToStack(MCInst &Inst, const MCPhysReg &StackReg, int Offset,
2426                          const MCPhysReg &SrcReg, int Size) const override {
2427     unsigned NewOpcode;
2428     switch (Size) {
2429     default:
2430       return false;
2431     case 2:      NewOpcode = X86::MOV16mr; break;
2432     case 4:      NewOpcode = X86::MOV32mr; break;
2433     case 8:      NewOpcode = X86::MOV64mr; break;
2434     }
2435     Inst.setOpcode(NewOpcode);
2436     Inst.clear();
2437     Inst.addOperand(MCOperand::createReg(StackReg));        // BaseReg
2438     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
2439     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
2440     Inst.addOperand(MCOperand::createImm(Offset));          // Displacement
2441     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
2442     Inst.addOperand(MCOperand::createReg(SrcReg));
2443     return true;
2444   }
2445 
2446   bool createRestoreFromStack(MCInst &Inst, const MCPhysReg &StackReg,
2447                               int Offset, const MCPhysReg &DstReg,
2448                               int Size) const override {
2449     return createLoad(Inst, StackReg, /*Scale=*/1, /*IndexReg=*/X86::NoRegister,
2450                       Offset, nullptr, /*AddrSegmentReg=*/X86::NoRegister,
2451                       DstReg, Size);
2452   }
2453 
2454   bool createLoad(MCInst &Inst, const MCPhysReg &BaseReg, int64_t Scale,
2455                   const MCPhysReg &IndexReg, int64_t Offset,
2456                   const MCExpr *OffsetExpr, const MCPhysReg &AddrSegmentReg,
2457                   const MCPhysReg &DstReg, int Size) const override {
2458     unsigned NewOpcode;
2459     switch (Size) {
2460     default:
2461       return false;
2462     case 2:      NewOpcode = X86::MOV16rm; break;
2463     case 4:      NewOpcode = X86::MOV32rm; break;
2464     case 8:      NewOpcode = X86::MOV64rm; break;
2465     }
2466     Inst.setOpcode(NewOpcode);
2467     Inst.clear();
2468     Inst.addOperand(MCOperand::createReg(DstReg));
2469     Inst.addOperand(MCOperand::createReg(BaseReg));
2470     Inst.addOperand(MCOperand::createImm(Scale));
2471     Inst.addOperand(MCOperand::createReg(IndexReg));
2472     if (OffsetExpr)
2473       Inst.addOperand(MCOperand::createExpr(OffsetExpr)); // Displacement
2474     else
2475       Inst.addOperand(MCOperand::createImm(Offset)); // Displacement
2476     Inst.addOperand(MCOperand::createReg(AddrSegmentReg)); // AddrSegmentReg
2477     return true;
2478   }
2479 
2480   void createLoadImmediate(MCInst &Inst, const MCPhysReg Dest,
2481                            uint32_t Imm) const override {
2482     Inst.setOpcode(X86::MOV64ri32);
2483     Inst.clear();
2484     Inst.addOperand(MCOperand::createReg(Dest));
2485     Inst.addOperand(MCOperand::createImm(Imm));
2486   }
2487 
2488   bool createIncMemory(MCInst &Inst, const MCSymbol *Target,
2489                        MCContext *Ctx) const override {
2490 
2491     Inst.setOpcode(X86::LOCK_INC64m);
2492     Inst.clear();
2493     Inst.addOperand(MCOperand::createReg(X86::RIP));        // BaseReg
2494     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
2495     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
2496 
2497     Inst.addOperand(MCOperand::createExpr(
2498         MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None,
2499                                 *Ctx)));                    // Displacement
2500     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
2501     return true;
2502   }
2503 
2504   bool createIJmp32Frag(SmallVectorImpl<MCInst> &Insts,
2505                         const MCOperand &BaseReg, const MCOperand &Scale,
2506                         const MCOperand &IndexReg, const MCOperand &Offset,
2507                         const MCOperand &TmpReg) const override {
2508     // The code fragment we emit here is:
2509     //
2510     //  mov32 (%base, %index, scale), %tmpreg
2511     //  ijmp *(%tmpreg)
2512     //
2513     MCInst IJmp;
2514     IJmp.setOpcode(X86::JMP64r);
2515     IJmp.addOperand(TmpReg);
2516 
2517     MCInst Load;
2518     Load.setOpcode(X86::MOV32rm);
2519     Load.addOperand(TmpReg);
2520     Load.addOperand(BaseReg);
2521     Load.addOperand(Scale);
2522     Load.addOperand(IndexReg);
2523     Load.addOperand(Offset);
2524     Load.addOperand(MCOperand::createReg(X86::NoRegister));
2525 
2526     Insts.push_back(Load);
2527     Insts.push_back(IJmp);
2528     return true;
2529   }
2530 
2531   bool createNoop(MCInst &Inst) const override {
2532     Inst.setOpcode(X86::NOOP);
2533     return true;
2534   }
2535 
2536   bool createReturn(MCInst &Inst) const override {
2537     Inst.setOpcode(X86::RET64);
2538     return true;
2539   }
2540 
2541   InstructionListType createInlineMemcpy(bool ReturnEnd) const override {
2542     InstructionListType Code;
2543     if (ReturnEnd)
2544       Code.emplace_back(MCInstBuilder(X86::LEA64r)
2545                             .addReg(X86::RAX)
2546                             .addReg(X86::RDI)
2547                             .addImm(1)
2548                             .addReg(X86::RDX)
2549                             .addImm(0)
2550                             .addReg(X86::NoRegister));
2551     else
2552       Code.emplace_back(MCInstBuilder(X86::MOV64rr)
2553                             .addReg(X86::RAX)
2554                             .addReg(X86::RDI));
2555 
2556     Code.emplace_back(MCInstBuilder(X86::MOV32rr)
2557                           .addReg(X86::ECX)
2558                           .addReg(X86::EDX));
2559     Code.emplace_back(MCInstBuilder(X86::REP_MOVSB_64));
2560 
2561     return Code;
2562   }
2563 
2564   InstructionListType createOneByteMemcpy() const override {
2565     InstructionListType Code;
2566     Code.emplace_back(MCInstBuilder(X86::MOV8rm)
2567                           .addReg(X86::CL)
2568                           .addReg(X86::RSI)
2569                           .addImm(0)
2570                           .addReg(X86::NoRegister)
2571                           .addImm(0)
2572                           .addReg(X86::NoRegister));
2573     Code.emplace_back(MCInstBuilder(X86::MOV8mr)
2574                           .addReg(X86::RDI)
2575                           .addImm(0)
2576                           .addReg(X86::NoRegister)
2577                           .addImm(0)
2578                           .addReg(X86::NoRegister)
2579                           .addReg(X86::CL));
2580     Code.emplace_back(MCInstBuilder(X86::MOV64rr)
2581                           .addReg(X86::RAX)
2582                           .addReg(X86::RDI));
2583     return Code;
2584   }
2585 
2586   InstructionListType createCmpJE(MCPhysReg RegNo, int64_t Imm,
2587                                   const MCSymbol *Target,
2588                                   MCContext *Ctx) const override {
2589     InstructionListType Code;
2590     Code.emplace_back(MCInstBuilder(X86::CMP64ri8)
2591                           .addReg(RegNo)
2592                           .addImm(Imm));
2593     Code.emplace_back(MCInstBuilder(X86::JCC_1)
2594                           .addExpr(MCSymbolRefExpr::create(
2595                               Target, MCSymbolRefExpr::VK_None, *Ctx))
2596                           .addImm(X86::COND_E));
2597     return Code;
2598   }
2599 
2600   Optional<Relocation>
2601   createRelocation(const MCFixup &Fixup,
2602                    const MCAsmBackend &MAB) const override {
2603     const MCFixupKindInfo &FKI = MAB.getFixupKindInfo(Fixup.getKind());
2604 
2605     assert(FKI.TargetOffset == 0 && "0-bit relocation offset expected");
2606     const uint64_t RelOffset = Fixup.getOffset();
2607 
2608     uint64_t RelType;
2609     if (FKI.Flags & MCFixupKindInfo::FKF_IsPCRel) {
2610       switch (FKI.TargetSize) {
2611       default:
2612         return NoneType();
2613       case  8: RelType = ELF::R_X86_64_PC8; break;
2614       case 16: RelType = ELF::R_X86_64_PC16; break;
2615       case 32: RelType = ELF::R_X86_64_PC32; break;
2616       case 64: RelType = ELF::R_X86_64_PC64; break;
2617       }
2618     } else {
2619       switch (FKI.TargetSize) {
2620       default:
2621         return NoneType();
2622       case  8: RelType = ELF::R_X86_64_8; break;
2623       case 16: RelType = ELF::R_X86_64_16; break;
2624       case 32: RelType = ELF::R_X86_64_32; break;
2625       case 64: RelType = ELF::R_X86_64_64; break;
2626       }
2627     }
2628 
2629     // Extract a symbol and an addend out of the fixup value expression.
2630     //
2631     // Only the following limited expression types are supported:
2632     //   Symbol + Addend
2633     //   Symbol
2634     uint64_t Addend = 0;
2635     MCSymbol *Symbol = nullptr;
2636     const MCExpr *ValueExpr = Fixup.getValue();
2637     if (ValueExpr->getKind() == MCExpr::Binary) {
2638       const auto *BinaryExpr = cast<MCBinaryExpr>(ValueExpr);
2639       assert(BinaryExpr->getOpcode() == MCBinaryExpr::Add &&
2640              "unexpected binary expression");
2641       const MCExpr *LHS = BinaryExpr->getLHS();
2642       assert(LHS->getKind() == MCExpr::SymbolRef && "unexpected LHS");
2643       Symbol = const_cast<MCSymbol *>(this->getTargetSymbol(LHS));
2644       const MCExpr *RHS = BinaryExpr->getRHS();
2645       assert(RHS->getKind() == MCExpr::Constant && "unexpected RHS");
2646       Addend = cast<MCConstantExpr>(RHS)->getValue();
2647     } else {
2648       assert(ValueExpr->getKind() == MCExpr::SymbolRef && "unexpected value");
2649       Symbol = const_cast<MCSymbol *>(this->getTargetSymbol(ValueExpr));
2650     }
2651 
2652     return Relocation({RelOffset, Symbol, RelType, Addend, 0});
2653   }
2654 
2655   bool replaceImmWithSymbolRef(MCInst &Inst, const MCSymbol *Symbol,
2656                                int64_t Addend, MCContext *Ctx, int64_t &Value,
2657                                uint64_t RelType) const override {
2658     unsigned ImmOpNo = -1U;
2659 
2660     for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst);
2661          ++Index) {
2662       if (Inst.getOperand(Index).isImm()) {
2663         ImmOpNo = Index;
2664         // TODO: this is a bit hacky.  It finds the correct operand by
2665         // searching for a specific immediate value.  If no value is
2666         // provided it defaults to the last immediate operand found.
2667         // This could lead to unexpected results if the instruction
2668         // has more than one immediate with the same value.
2669         if (Inst.getOperand(ImmOpNo).getImm() == Value)
2670           break;
2671       }
2672     }
2673 
2674     if (ImmOpNo == -1U)
2675       return false;
2676 
2677     Value = Inst.getOperand(ImmOpNo).getImm();
2678 
2679     setOperandToSymbolRef(Inst, ImmOpNo, Symbol, Addend, Ctx, RelType);
2680 
2681     return true;
2682   }
2683 
2684   bool replaceRegWithImm(MCInst &Inst, unsigned Register,
2685                          int64_t Imm) const override {
2686 
2687     enum CheckSignExt : uint8_t {
2688       NOCHECK = 0,
2689       CHECK8,
2690       CHECK32,
2691     };
2692 
2693     using CheckList = std::vector<std::pair<CheckSignExt, unsigned>>;
2694     struct InstInfo {
2695       // Size in bytes that Inst loads from memory.
2696       uint8_t DataSize;
2697 
2698       // True when the target operand has to be duplicated because the opcode
2699       // expects a LHS operand.
2700       bool HasLHS;
2701 
2702       // List of checks and corresponding opcodes to be used. We try to use the
2703       // smallest possible immediate value when various sizes are available,
2704       // hence we may need to check whether a larger constant fits in a smaller
2705       // immediate.
2706       CheckList Checks;
2707     };
2708 
2709     InstInfo I;
2710 
2711     switch (Inst.getOpcode()) {
2712     default: {
2713       switch (getPushSize(Inst)) {
2714 
2715       case 2: I = {2, false, {{CHECK8, X86::PUSH16i8}, {NOCHECK, X86::PUSHi16}}}; break;
2716       case 4: I = {4, false, {{CHECK8, X86::PUSH32i8}, {NOCHECK, X86::PUSHi32}}}; break;
2717       case 8: I = {8, false, {{CHECK8, X86::PUSH64i8},
2718                               {CHECK32, X86::PUSH64i32},
2719                               {NOCHECK, Inst.getOpcode()}}}; break;
2720       default: return false;
2721       }
2722       break;
2723     }
2724 
2725     // MOV
2726     case X86::MOV8rr:       I = {1, false, {{NOCHECK, X86::MOV8ri}}}; break;
2727     case X86::MOV16rr:      I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break;
2728     case X86::MOV32rr:      I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break;
2729     case X86::MOV64rr:      I = {8, false, {{CHECK32, X86::MOV64ri32},
2730                                             {NOCHECK, X86::MOV64ri}}}; break;
2731 
2732     case X86::MOV8mr:       I = {1, false, {{NOCHECK, X86::MOV8mi}}}; break;
2733     case X86::MOV16mr:      I = {2, false, {{NOCHECK, X86::MOV16mi}}}; break;
2734     case X86::MOV32mr:      I = {4, false, {{NOCHECK, X86::MOV32mi}}}; break;
2735     case X86::MOV64mr:      I = {8, false, {{CHECK32, X86::MOV64mi32},
2736                                             {NOCHECK, X86::MOV64mr}}}; break;
2737 
2738     // MOVZX
2739     case X86::MOVZX16rr8:   I = {1, false, {{NOCHECK, X86::MOV16ri}}}; break;
2740     case X86::MOVZX32rr8:   I = {1, false, {{NOCHECK, X86::MOV32ri}}}; break;
2741     case X86::MOVZX32rr16:  I = {2, false, {{NOCHECK, X86::MOV32ri}}}; break;
2742 
2743     // CMP
2744     case X86::CMP8rr:       I = {1, false, {{NOCHECK, X86::CMP8ri}}}; break;
2745     case X86::CMP16rr:      I = {2, false, {{CHECK8, X86::CMP16ri8},
2746                                             {NOCHECK, X86::CMP16ri}}}; break;
2747     case X86::CMP32rr:      I = {4, false, {{CHECK8, X86::CMP32ri8},
2748                                             {NOCHECK, X86::CMP32ri}}}; break;
2749     case X86::CMP64rr:      I = {8, false, {{CHECK8, X86::CMP64ri8},
2750                                             {CHECK32, X86::CMP64ri32},
2751                                             {NOCHECK, X86::CMP64rr}}}; break;
2752 
2753     // TEST
2754     case X86::TEST8rr:      I = {1, false, {{NOCHECK, X86::TEST8ri}}}; break;
2755     case X86::TEST16rr:     I = {2, false, {{NOCHECK, X86::TEST16ri}}}; break;
2756     case X86::TEST32rr:     I = {4, false, {{NOCHECK, X86::TEST32ri}}}; break;
2757     case X86::TEST64rr:     I = {8, false, {{CHECK32, X86::TEST64ri32},
2758                                             {NOCHECK, X86::TEST64rr}}}; break;
2759 
2760     // ADD
2761     case X86::ADD8rr:       I = {1, true, {{NOCHECK, X86::ADD8ri}}}; break;
2762     case X86::ADD16rr:      I = {2, true, {{CHECK8, X86::ADD16ri8},
2763                                            {NOCHECK, X86::ADD16ri}}}; break;
2764     case X86::ADD32rr:      I = {4, true, {{CHECK8, X86::ADD32ri8},
2765                                            {NOCHECK, X86::ADD32ri}}}; break;
2766     case X86::ADD64rr:      I = {8, true, {{CHECK8, X86::ADD64ri8},
2767                                            {CHECK32, X86::ADD64ri32},
2768                                            {NOCHECK, X86::ADD64rr}}}; break;
2769 
2770     // SUB
2771     case X86::SUB8rr:       I = {1, true, {{NOCHECK, X86::SUB8ri}}}; break;
2772     case X86::SUB16rr:      I = {2, true, {{CHECK8, X86::SUB16ri8},
2773                                            {NOCHECK, X86::SUB16ri}}}; break;
2774     case X86::SUB32rr:      I = {4, true, {{CHECK8, X86::SUB32ri8},
2775                                            {NOCHECK, X86::SUB32ri}}}; break;
2776     case X86::SUB64rr:      I = {8, true, {{CHECK8, X86::SUB64ri8},
2777                                            {CHECK32, X86::SUB64ri32},
2778                                            {NOCHECK, X86::SUB64rr}}}; break;
2779 
2780     // AND
2781     case X86::AND8rr:       I = {1, true, {{NOCHECK, X86::AND8ri}}}; break;
2782     case X86::AND16rr:      I = {2, true, {{CHECK8, X86::AND16ri8},
2783                                            {NOCHECK, X86::AND16ri}}}; break;
2784     case X86::AND32rr:      I = {4, true, {{CHECK8, X86::AND32ri8},
2785                                            {NOCHECK, X86::AND32ri}}}; break;
2786     case X86::AND64rr:      I = {8, true, {{CHECK8, X86::AND64ri8},
2787                                            {CHECK32, X86::AND64ri32},
2788                                            {NOCHECK, X86::AND64rr}}}; break;
2789 
2790     // OR
2791     case X86::OR8rr:        I = {1, true, {{NOCHECK, X86::OR8ri}}}; break;
2792     case X86::OR16rr:       I = {2, true, {{CHECK8, X86::OR16ri8},
2793                                            {NOCHECK, X86::OR16ri}}}; break;
2794     case X86::OR32rr:       I = {4, true, {{CHECK8, X86::OR32ri8},
2795                                            {NOCHECK, X86::OR32ri}}}; break;
2796     case X86::OR64rr:       I = {8, true, {{CHECK8, X86::OR64ri8},
2797                                            {CHECK32, X86::OR64ri32},
2798                                            {NOCHECK, X86::OR64rr}}}; break;
2799 
2800     // XOR
2801     case X86::XOR8rr:       I = {1, true, {{NOCHECK, X86::XOR8ri}}}; break;
2802     case X86::XOR16rr:      I = {2, true, {{CHECK8, X86::XOR16ri8},
2803                                            {NOCHECK, X86::XOR16ri}}}; break;
2804     case X86::XOR32rr:      I = {4, true, {{CHECK8, X86::XOR32ri8},
2805                                            {NOCHECK, X86::XOR32ri}}}; break;
2806     case X86::XOR64rr:      I = {8, true, {{CHECK8, X86::XOR64ri8},
2807                                            {CHECK32, X86::XOR64ri32},
2808                                            {NOCHECK, X86::XOR64rr}}}; break;
2809     }
2810 
2811     // Compute the new opcode.
2812     unsigned NewOpcode = 0;
2813     for (const std::pair<CheckSignExt, unsigned> &Check : I.Checks) {
2814       NewOpcode = Check.second;
2815       if (Check.first == NOCHECK)
2816         break;
2817       if (Check.first == CHECK8 && isInt<8>(Imm))
2818         break;
2819       if (Check.first == CHECK32 && isInt<32>(Imm))
2820         break;
2821     }
2822     if (NewOpcode == Inst.getOpcode())
2823       return false;
2824 
2825     const MCInstrDesc &InstDesc = Info->get(Inst.getOpcode());
2826 
2827     unsigned NumFound = 0;
2828     for (unsigned Index = InstDesc.getNumDefs() + (I.HasLHS ? 1 : 0),
2829                   E = InstDesc.getNumOperands();
2830          Index != E; ++Index)
2831       if (Inst.getOperand(Index).isReg() &&
2832           Inst.getOperand(Index).getReg() == Register)
2833         NumFound++;
2834 
2835     if (NumFound != 1)
2836       return false;
2837 
2838     MCOperand TargetOp = Inst.getOperand(0);
2839     Inst.clear();
2840     Inst.setOpcode(NewOpcode);
2841     Inst.addOperand(TargetOp);
2842     if (I.HasLHS)
2843       Inst.addOperand(TargetOp);
2844     Inst.addOperand(MCOperand::createImm(Imm));
2845 
2846     return true;
2847   }
2848 
2849   bool replaceRegWithReg(MCInst &Inst, unsigned ToReplace,
2850                          unsigned ReplaceWith) const override {
2851 
2852     // Get the HasLHS value so that iteration can be done
2853     bool HasLHS;
2854     if (X86::isAND(Inst.getOpcode()) || X86::isADD(Inst.getOpcode()) ||
2855         X86::isSUB(Inst.getOpcode())) {
2856       HasLHS = true;
2857     } else if (isPop(Inst) || isPush(Inst) || X86::isCMP(Inst.getOpcode()) ||
2858                X86::isTEST(Inst.getOpcode())) {
2859       HasLHS = false;
2860     } else {
2861       switch (Inst.getOpcode()) {
2862       case X86::MOV8rr:
2863       case X86::MOV8rm:
2864       case X86::MOV8mr:
2865       case X86::MOV8ri:
2866       case X86::MOV16rr:
2867       case X86::MOV16rm:
2868       case X86::MOV16mr:
2869       case X86::MOV16ri:
2870       case X86::MOV32rr:
2871       case X86::MOV32rm:
2872       case X86::MOV32mr:
2873       case X86::MOV32ri:
2874       case X86::MOV64rr:
2875       case X86::MOV64rm:
2876       case X86::MOV64mr:
2877       case X86::MOV64ri:
2878       case X86::MOVZX16rr8:
2879       case X86::MOVZX32rr8:
2880       case X86::MOVZX32rr16:
2881       case X86::MOVSX32rm8:
2882       case X86::MOVSX32rr8:
2883       case X86::MOVSX64rm32:
2884       case X86::LEA64r:
2885         HasLHS = false;
2886         break;
2887       default:
2888         return false;
2889       }
2890     }
2891 
2892     const MCInstrDesc &InstDesc = Info->get(Inst.getOpcode());
2893 
2894     bool FoundOne = false;
2895 
2896     // Iterate only through src operands that arent also dest operands
2897     for (unsigned Index = InstDesc.getNumDefs() + (HasLHS ? 1 : 0),
2898                   E = InstDesc.getNumOperands();
2899          Index != E; ++Index) {
2900       BitVector RegAliases = getAliases(ToReplace, true);
2901       if (!Inst.getOperand(Index).isReg() ||
2902           !RegAliases.test(Inst.getOperand(Index).getReg()))
2903         continue;
2904       // Resize register if needed
2905       unsigned SizedReplaceWith = getAliasSized(
2906           ReplaceWith, getRegSize(Inst.getOperand(Index).getReg()));
2907       MCOperand NewOperand = MCOperand::createReg(SizedReplaceWith);
2908       Inst.getOperand(Index) = NewOperand;
2909       FoundOne = true;
2910     }
2911 
2912     // Return true if at least one operand was replaced
2913     return FoundOne;
2914   }
2915 
2916   bool createUncondBranch(MCInst &Inst, const MCSymbol *TBB,
2917                           MCContext *Ctx) const override {
2918     Inst.setOpcode(X86::JMP_1);
2919     Inst.addOperand(MCOperand::createExpr(
2920         MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)));
2921     return true;
2922   }
2923 
2924   bool createCall(MCInst &Inst, const MCSymbol *Target,
2925                   MCContext *Ctx) override {
2926     Inst.setOpcode(X86::CALL64pcrel32);
2927     Inst.addOperand(MCOperand::createExpr(
2928         MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx)));
2929     return true;
2930   }
2931 
2932   bool createTailCall(MCInst &Inst, const MCSymbol *Target,
2933                       MCContext *Ctx) override {
2934     return createDirectCall(Inst, Target, Ctx, /*IsTailCall*/ true);
2935   }
2936 
2937   void createLongTailCall(InstructionListType &Seq, const MCSymbol *Target,
2938                           MCContext *Ctx) override {
2939     Seq.clear();
2940     Seq.emplace_back();
2941     createDirectCall(Seq.back(), Target, Ctx, /*IsTailCall*/ true);
2942   }
2943 
2944   bool createTrap(MCInst &Inst) const override {
2945     Inst.clear();
2946     Inst.setOpcode(X86::TRAP);
2947     return true;
2948   }
2949 
2950   bool reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB,
2951                               MCContext *Ctx) const override {
2952     unsigned InvCC = getInvertedCondCode(getCondCode(Inst));
2953     assert(InvCC != X86::COND_INVALID && "invalid branch instruction");
2954     Inst.getOperand(Info->get(Inst.getOpcode()).NumOperands - 1).setImm(InvCC);
2955     Inst.getOperand(0) = MCOperand::createExpr(
2956         MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx));
2957     return true;
2958   }
2959 
2960   bool replaceBranchCondition(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx,
2961                               unsigned CC) const override {
2962     if (CC == X86::COND_INVALID)
2963       return false;
2964     Inst.getOperand(Info->get(Inst.getOpcode()).NumOperands - 1).setImm(CC);
2965     Inst.getOperand(0) = MCOperand::createExpr(
2966         MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx));
2967     return true;
2968   }
2969 
2970   unsigned getCanonicalBranchCondCode(unsigned CC) const override {
2971     switch (CC) {
2972     default:           return X86::COND_INVALID;
2973 
2974     case X86::COND_E:  return X86::COND_E;
2975     case X86::COND_NE: return X86::COND_E;
2976 
2977     case X86::COND_L:  return X86::COND_L;
2978     case X86::COND_GE: return X86::COND_L;
2979 
2980     case X86::COND_LE: return X86::COND_G;
2981     case X86::COND_G:  return X86::COND_G;
2982 
2983     case X86::COND_B:  return X86::COND_B;
2984     case X86::COND_AE: return X86::COND_B;
2985 
2986     case X86::COND_BE: return X86::COND_A;
2987     case X86::COND_A:  return X86::COND_A;
2988 
2989     case X86::COND_S:  return X86::COND_S;
2990     case X86::COND_NS: return X86::COND_S;
2991 
2992     case X86::COND_P:  return X86::COND_P;
2993     case X86::COND_NP: return X86::COND_P;
2994 
2995     case X86::COND_O:  return X86::COND_O;
2996     case X86::COND_NO: return X86::COND_O;
2997     }
2998   }
2999 
3000   bool replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB,
3001                            MCContext *Ctx) const override {
3002     assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) &&
3003            "Invalid instruction");
3004     Inst.getOperand(0) = MCOperand::createExpr(
3005         MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx));
3006     return true;
3007   }
3008 
3009   MCPhysReg getX86R11() const override { return X86::R11; }
3010 
3011   MCPhysReg getIntArgRegister(unsigned ArgNo) const override {
3012     // FIXME: this should depend on the calling convention.
3013     switch (ArgNo) {
3014     case 0:   return X86::RDI;
3015     case 1:   return X86::RSI;
3016     case 2:   return X86::RDX;
3017     case 3:   return X86::RCX;
3018     case 4:   return X86::R8;
3019     case 5:   return X86::R9;
3020     default:  return getNoRegister();
3021     }
3022   }
3023 
3024   void createPause(MCInst &Inst) const override {
3025     Inst.clear();
3026     Inst.setOpcode(X86::PAUSE);
3027   }
3028 
3029   void createLfence(MCInst &Inst) const override {
3030     Inst.clear();
3031     Inst.setOpcode(X86::LFENCE);
3032   }
3033 
3034   bool createDirectCall(MCInst &Inst, const MCSymbol *Target, MCContext *Ctx,
3035                         bool IsTailCall) override {
3036     Inst.clear();
3037     Inst.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32);
3038     Inst.addOperand(MCOperand::createExpr(
3039         MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx)));
3040     if (IsTailCall)
3041       setTailCall(Inst);
3042     return true;
3043   }
3044 
3045   void createShortJmp(InstructionListType &Seq, const MCSymbol *Target,
3046                       MCContext *Ctx, bool IsTailCall) override {
3047     Seq.clear();
3048     MCInst Inst;
3049     Inst.setOpcode(X86::JMP_1);
3050     Inst.addOperand(MCOperand::createExpr(
3051         MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx)));
3052     if (IsTailCall)
3053       setTailCall(Inst);
3054     Seq.emplace_back(Inst);
3055   }
3056 
3057   bool isConditionalMove(const MCInst &Inst) const override {
3058     unsigned OpCode = Inst.getOpcode();
3059     return (OpCode == X86::CMOV16rr || OpCode == X86::CMOV32rr ||
3060             OpCode == X86::CMOV64rr);
3061   }
3062 
3063   bool isBranchOnMem(const MCInst &Inst) const override {
3064     unsigned OpCode = Inst.getOpcode();
3065     if (OpCode == X86::CALL64m || (OpCode == X86::JMP32m && isTailCall(Inst)) ||
3066         OpCode == X86::JMP64m)
3067       return true;
3068 
3069     return false;
3070   }
3071 
3072   bool isBranchOnReg(const MCInst &Inst) const override {
3073     unsigned OpCode = Inst.getOpcode();
3074     if (OpCode == X86::CALL64r || (OpCode == X86::JMP32r && isTailCall(Inst)) ||
3075         OpCode == X86::JMP64r)
3076       return true;
3077 
3078     return false;
3079   }
3080 
3081   void createPushRegister(MCInst &Inst, MCPhysReg Reg,
3082                           unsigned Size) const override {
3083     Inst.clear();
3084     unsigned NewOpcode = 0;
3085     if (Reg == X86::EFLAGS) {
3086       switch (Size) {
3087       case 2: NewOpcode = X86::PUSHF16;  break;
3088       case 4: NewOpcode = X86::PUSHF32;  break;
3089       case 8: NewOpcode = X86::PUSHF64;  break;
3090       default:
3091         llvm_unreachable("Unexpected size");
3092       }
3093       Inst.setOpcode(NewOpcode);
3094       return;
3095     }
3096     switch (Size) {
3097     case 2: NewOpcode = X86::PUSH16r;  break;
3098     case 4: NewOpcode = X86::PUSH32r;  break;
3099     case 8: NewOpcode = X86::PUSH64r;  break;
3100     default:
3101       llvm_unreachable("Unexpected size");
3102     }
3103     Inst.setOpcode(NewOpcode);
3104     Inst.addOperand(MCOperand::createReg(Reg));
3105   }
3106 
3107   void createPopRegister(MCInst &Inst, MCPhysReg Reg,
3108                          unsigned Size) const override {
3109     Inst.clear();
3110     unsigned NewOpcode = 0;
3111     if (Reg == X86::EFLAGS) {
3112       switch (Size) {
3113       case 2: NewOpcode = X86::POPF16;  break;
3114       case 4: NewOpcode = X86::POPF32;  break;
3115       case 8: NewOpcode = X86::POPF64;  break;
3116       default:
3117         llvm_unreachable("Unexpected size");
3118       }
3119       Inst.setOpcode(NewOpcode);
3120       return;
3121     }
3122     switch (Size) {
3123     case 2: NewOpcode = X86::POP16r;  break;
3124     case 4: NewOpcode = X86::POP32r;  break;
3125     case 8: NewOpcode = X86::POP64r;  break;
3126     default:
3127       llvm_unreachable("Unexpected size");
3128     }
3129     Inst.setOpcode(NewOpcode);
3130     Inst.addOperand(MCOperand::createReg(Reg));
3131   }
3132 
3133   void createPushFlags(MCInst &Inst, unsigned Size) const override {
3134     return createPushRegister(Inst, X86::EFLAGS, Size);
3135   }
3136 
3137   void createPopFlags(MCInst &Inst, unsigned Size) const override {
3138     return createPopRegister(Inst, X86::EFLAGS, Size);
3139   }
3140 
3141   void createAddRegImm(MCInst &Inst, MCPhysReg Reg, int64_t Value,
3142                        unsigned Size) const {
3143     unsigned int Opcode;
3144     switch (Size) {
3145     case 1: Opcode = X86::ADD8ri; break;
3146     case 2: Opcode = X86::ADD16ri; break;
3147     case 4: Opcode = X86::ADD32ri; break;
3148     default:
3149       llvm_unreachable("Unexpected size");
3150     }
3151     Inst.setOpcode(Opcode);
3152     Inst.clear();
3153     Inst.addOperand(MCOperand::createReg(Reg));
3154     Inst.addOperand(MCOperand::createReg(Reg));
3155     Inst.addOperand(MCOperand::createImm(Value));
3156   }
3157 
3158   void createClearRegWithNoEFlagsUpdate(MCInst &Inst, MCPhysReg Reg,
3159                                         unsigned Size) const {
3160     unsigned int Opcode;
3161     switch (Size) {
3162     case 1: Opcode = X86::MOV8ri; break;
3163     case 2: Opcode = X86::MOV16ri; break;
3164     case 4: Opcode = X86::MOV32ri; break;
3165     case 8: Opcode = X86::MOV64ri; break;
3166     default:
3167       llvm_unreachable("Unexpected size");
3168     }
3169     Inst.setOpcode(Opcode);
3170     Inst.clear();
3171     Inst.addOperand(MCOperand::createReg(Reg));
3172     Inst.addOperand(MCOperand::createImm(0));
3173   }
3174 
3175   void createX86SaveOVFlagToRegister(MCInst &Inst, MCPhysReg Reg) const {
3176     Inst.setOpcode(X86::SETCCr);
3177     Inst.clear();
3178     Inst.addOperand(MCOperand::createReg(Reg));
3179     Inst.addOperand(MCOperand::createImm(X86::COND_O));
3180   }
3181 
3182   void createX86Lahf(MCInst &Inst) const {
3183     Inst.setOpcode(X86::LAHF);
3184     Inst.clear();
3185   }
3186 
3187   void createX86Sahf(MCInst &Inst) const {
3188     Inst.setOpcode(X86::SAHF);
3189     Inst.clear();
3190   }
3191 
3192   void createInstrIncMemory(InstructionListType &Instrs, const MCSymbol *Target,
3193                             MCContext *Ctx, bool IsLeaf) const override {
3194     unsigned int I = 0;
3195 
3196     Instrs.resize(IsLeaf ? 13 : 11);
3197     // Don't clobber application red zone (ABI dependent)
3198     if (IsLeaf)
3199       createStackPointerIncrement(Instrs[I++], 128,
3200                                   /*NoFlagsClobber=*/true);
3201 
3202     // Performance improvements based on the optimization discussed at
3203     // https://reviews.llvm.org/D6629
3204     // LAHF/SAHF are used instead of PUSHF/POPF
3205     // PUSHF
3206     createPushRegister(Instrs[I++], X86::RAX, 8);
3207     createClearRegWithNoEFlagsUpdate(Instrs[I++], X86::RAX, 8);
3208     createX86Lahf(Instrs[I++]);
3209     createPushRegister(Instrs[I++], X86::RAX, 8);
3210     createClearRegWithNoEFlagsUpdate(Instrs[I++], X86::RAX, 8);
3211     createX86SaveOVFlagToRegister(Instrs[I++], X86::AL);
3212     // LOCK INC
3213     createIncMemory(Instrs[I++], Target, Ctx);
3214     // POPF
3215     createAddRegImm(Instrs[I++], X86::AL, 127, 1);
3216     createPopRegister(Instrs[I++], X86::RAX, 8);
3217     createX86Sahf(Instrs[I++]);
3218     createPopRegister(Instrs[I++], X86::RAX, 8);
3219 
3220     if (IsLeaf)
3221       createStackPointerDecrement(Instrs[I], 128,
3222                                   /*NoFlagsClobber=*/true);
3223   }
3224 
3225   void createSwap(MCInst &Inst, MCPhysReg Source, MCPhysReg MemBaseReg,
3226                   int64_t Disp) const {
3227     Inst.setOpcode(X86::XCHG64rm);
3228     Inst.addOperand(MCOperand::createReg(Source));
3229     Inst.addOperand(MCOperand::createReg(Source));
3230     Inst.addOperand(MCOperand::createReg(MemBaseReg));      // BaseReg
3231     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
3232     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
3233     Inst.addOperand(MCOperand::createImm(Disp));            // Displacement
3234     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
3235   }
3236 
3237   void createIndirectBranch(MCInst &Inst, MCPhysReg MemBaseReg,
3238                             int64_t Disp) const {
3239     Inst.setOpcode(X86::JMP64m);
3240     Inst.addOperand(MCOperand::createReg(MemBaseReg));      // BaseReg
3241     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
3242     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
3243     Inst.addOperand(MCOperand::createImm(Disp));            // Displacement
3244     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
3245   }
3246 
3247   InstructionListType createInstrumentedIndirectCall(const MCInst &CallInst,
3248                                                      bool TailCall,
3249                                                      MCSymbol *HandlerFuncAddr,
3250                                                      int CallSiteID,
3251                                                      MCContext *Ctx) override {
3252     // Check if the target address expression used in the original indirect call
3253     // uses the stack pointer, which we are going to clobber.
3254     static BitVector SPAliases(getAliases(X86::RSP));
3255     bool UsesSP = false;
3256     // Skip defs.
3257     for (unsigned I = Info->get(CallInst.getOpcode()).getNumDefs(),
3258                   E = MCPlus::getNumPrimeOperands(CallInst);
3259          I != E; ++I) {
3260       const MCOperand &Operand = CallInst.getOperand(I);
3261       if (Operand.isReg() && SPAliases[Operand.getReg()]) {
3262         UsesSP = true;
3263         break;
3264       }
3265     }
3266 
3267     InstructionListType Insts;
3268     MCPhysReg TempReg = getIntArgRegister(0);
3269     // Code sequence used to enter indirect call instrumentation helper:
3270     //   push %rdi
3271     //   add $8, %rsp       ;; $rsp may be used in target, so fix it to prev val
3272     //   movq target, %rdi  ;; via convertIndirectCallTargetToLoad
3273     //   sub $8, %rsp       ;; restore correct stack value
3274     //   push %rdi
3275     //   movq $CallSiteID, %rdi
3276     //   push %rdi
3277     //   callq/jmp HandlerFuncAddr
3278     Insts.emplace_back();
3279     createPushRegister(Insts.back(), TempReg, 8);
3280     if (UsesSP) { // Only adjust SP if we really need to
3281       Insts.emplace_back();
3282       createStackPointerDecrement(Insts.back(), 8, /*NoFlagsClobber=*/false);
3283     }
3284     Insts.emplace_back(CallInst);
3285     // Insts.back() and CallInst now share the same annotation instruction.
3286     // Strip it from Insts.back(), only preserving tail call annotation.
3287     stripAnnotations(Insts.back(), /*KeepTC=*/true);
3288     convertIndirectCallToLoad(Insts.back(), TempReg);
3289     if (UsesSP) {
3290       Insts.emplace_back();
3291       createStackPointerIncrement(Insts.back(), 8, /*NoFlagsClobber=*/false);
3292     }
3293     Insts.emplace_back();
3294     createPushRegister(Insts.back(), TempReg, 8);
3295     Insts.emplace_back();
3296     createLoadImmediate(Insts.back(), TempReg, CallSiteID);
3297     Insts.emplace_back();
3298     createPushRegister(Insts.back(), TempReg, 8);
3299     Insts.emplace_back();
3300     createDirectCall(Insts.back(), HandlerFuncAddr, Ctx,
3301                      /*TailCall=*/TailCall);
3302     // Carry over metadata
3303     for (int I = MCPlus::getNumPrimeOperands(CallInst),
3304              E = CallInst.getNumOperands();
3305          I != E; ++I)
3306       Insts.back().addOperand(CallInst.getOperand(I));
3307 
3308     return Insts;
3309   }
3310 
3311   InstructionListType createInstrumentedIndCallHandlerExitBB() const override {
3312     const MCPhysReg TempReg = getIntArgRegister(0);
3313     // We just need to undo the sequence created for every ind call in
3314     // instrumentIndirectTarget(), which can be accomplished minimally with:
3315     //   popfq
3316     //   pop %rdi
3317     //   add $16, %rsp
3318     //   xchg (%rsp), %rdi
3319     //   jmp *-8(%rsp)
3320     InstructionListType Insts(5);
3321     createPopFlags(Insts[0], 8);
3322     createPopRegister(Insts[1], TempReg, 8);
3323     createStackPointerDecrement(Insts[2], 16, /*NoFlagsClobber=*/false);
3324     createSwap(Insts[3], TempReg, X86::RSP, 0);
3325     createIndirectBranch(Insts[4], X86::RSP, -8);
3326     return Insts;
3327   }
3328 
3329   InstructionListType
3330   createInstrumentedIndTailCallHandlerExitBB() const override {
3331     const MCPhysReg TempReg = getIntArgRegister(0);
3332     // Same thing as above, but for tail calls
3333     //   popfq
3334     //   add $16, %rsp
3335     //   pop %rdi
3336     //   jmp *-16(%rsp)
3337     InstructionListType Insts(4);
3338     createPopFlags(Insts[0], 8);
3339     createStackPointerDecrement(Insts[1], 16, /*NoFlagsClobber=*/false);
3340     createPopRegister(Insts[2], TempReg, 8);
3341     createIndirectBranch(Insts[3], X86::RSP, -16);
3342     return Insts;
3343   }
3344 
3345   InstructionListType
3346   createInstrumentedIndCallHandlerEntryBB(const MCSymbol *InstrTrampoline,
3347                                           const MCSymbol *IndCallHandler,
3348                                           MCContext *Ctx) override {
3349     const MCPhysReg TempReg = getIntArgRegister(0);
3350     // Code sequence used to check whether InstrTampoline was initialized
3351     // and call it if so, returns via IndCallHandler.
3352     //   pushfq
3353     //   mov    InstrTrampoline,%rdi
3354     //   cmp    $0x0,%rdi
3355     //   je     IndCallHandler
3356     //   callq  *%rdi
3357     //   jmpq   IndCallHandler
3358     InstructionListType Insts;
3359     Insts.emplace_back();
3360     createPushFlags(Insts.back(), 8);
3361     Insts.emplace_back();
3362     createMove(Insts.back(), InstrTrampoline, TempReg, Ctx);
3363     InstructionListType cmpJmp = createCmpJE(TempReg, 0, IndCallHandler, Ctx);
3364     Insts.insert(Insts.end(), cmpJmp.begin(), cmpJmp.end());
3365     Insts.emplace_back();
3366     Insts.back().setOpcode(X86::CALL64r);
3367     Insts.back().addOperand(MCOperand::createReg(TempReg));
3368     Insts.emplace_back();
3369     createDirectCall(Insts.back(), IndCallHandler, Ctx, /*IsTailCall*/ true);
3370     return Insts;
3371   }
3372 
3373   InstructionListType createNumCountersGetter(MCContext *Ctx) const override {
3374     InstructionListType Insts(2);
3375     MCSymbol *NumLocs = Ctx->getOrCreateSymbol("__bolt_num_counters");
3376     createMove(Insts[0], NumLocs, X86::EAX, Ctx);
3377     createReturn(Insts[1]);
3378     return Insts;
3379   }
3380 
3381   InstructionListType
3382   createInstrLocationsGetter(MCContext *Ctx) const override {
3383     InstructionListType Insts(2);
3384     MCSymbol *Locs = Ctx->getOrCreateSymbol("__bolt_instr_locations");
3385     createLea(Insts[0], Locs, X86::EAX, Ctx);
3386     createReturn(Insts[1]);
3387     return Insts;
3388   }
3389 
3390   InstructionListType createInstrTablesGetter(MCContext *Ctx) const override {
3391     InstructionListType Insts(2);
3392     MCSymbol *Locs = Ctx->getOrCreateSymbol("__bolt_instr_tables");
3393     createLea(Insts[0], Locs, X86::EAX, Ctx);
3394     createReturn(Insts[1]);
3395     return Insts;
3396   }
3397 
3398   InstructionListType createInstrNumFuncsGetter(MCContext *Ctx) const override {
3399     InstructionListType Insts(2);
3400     MCSymbol *NumFuncs = Ctx->getOrCreateSymbol("__bolt_instr_num_funcs");
3401     createMove(Insts[0], NumFuncs, X86::EAX, Ctx);
3402     createReturn(Insts[1]);
3403     return Insts;
3404   }
3405 
3406   InstructionListType createSymbolTrampoline(const MCSymbol *TgtSym,
3407                                              MCContext *Ctx) const override {
3408     InstructionListType Insts(1);
3409     createUncondBranch(Insts[0], TgtSym, Ctx);
3410     return Insts;
3411   }
3412 
3413   InstructionListType createDummyReturnFunction(MCContext *Ctx) const override {
3414     InstructionListType Insts(1);
3415     createReturn(Insts[0]);
3416     return Insts;
3417   }
3418 
3419   BlocksVectorTy indirectCallPromotion(
3420       const MCInst &CallInst,
3421       const std::vector<std::pair<MCSymbol *, uint64_t>> &Targets,
3422       const std::vector<std::pair<MCSymbol *, uint64_t>> &VtableSyms,
3423       const std::vector<MCInst *> &MethodFetchInsns,
3424       const bool MinimizeCodeSize, MCContext *Ctx) override {
3425     const bool IsTailCall = isTailCall(CallInst);
3426     const bool IsJumpTable = getJumpTable(CallInst) != 0;
3427     BlocksVectorTy Results;
3428 
3429     // Label for the current code block.
3430     MCSymbol *NextTarget = nullptr;
3431 
3432     // The join block which contains all the instructions following CallInst.
3433     // MergeBlock remains null if CallInst is a tail call.
3434     MCSymbol *MergeBlock = nullptr;
3435 
3436     unsigned FuncAddrReg = X86::R10;
3437 
3438     const bool LoadElim = !VtableSyms.empty();
3439     assert((!LoadElim || VtableSyms.size() == Targets.size()) &&
3440            "There must be a vtable entry for every method "
3441            "in the targets vector.");
3442 
3443     if (MinimizeCodeSize && !LoadElim) {
3444       std::set<unsigned> UsedRegs;
3445 
3446       for (unsigned int I = 0; I < MCPlus::getNumPrimeOperands(CallInst); ++I) {
3447         const MCOperand &Op = CallInst.getOperand(I);
3448         if (Op.isReg())
3449           UsedRegs.insert(Op.getReg());
3450       }
3451 
3452       if (UsedRegs.count(X86::R10) == 0)
3453         FuncAddrReg = X86::R10;
3454       else if (UsedRegs.count(X86::R11) == 0)
3455         FuncAddrReg = X86::R11;
3456       else
3457         return Results;
3458     }
3459 
3460     const auto jumpToMergeBlock = [&](InstructionListType &NewCall) {
3461       assert(MergeBlock);
3462       NewCall.push_back(CallInst);
3463       MCInst &Merge = NewCall.back();
3464       Merge.clear();
3465       createUncondBranch(Merge, MergeBlock, Ctx);
3466     };
3467 
3468     for (unsigned int i = 0; i < Targets.size(); ++i) {
3469       Results.emplace_back(NextTarget, InstructionListType());
3470       InstructionListType *NewCall = &Results.back().second;
3471 
3472       if (MinimizeCodeSize && !LoadElim) {
3473         // Load the call target into FuncAddrReg.
3474         NewCall->push_back(CallInst); // Copy CallInst in order to get SMLoc
3475         MCInst &Target = NewCall->back();
3476         Target.clear();
3477         Target.setOpcode(X86::MOV64ri32);
3478         Target.addOperand(MCOperand::createReg(FuncAddrReg));
3479         if (Targets[i].first) {
3480           // Is this OK?
3481           Target.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
3482               Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
3483         } else {
3484           const uint64_t Addr = Targets[i].second;
3485           // Immediate address is out of sign extended 32 bit range.
3486           if (int64_t(Addr) != int64_t(int32_t(Addr)))
3487             return BlocksVectorTy();
3488 
3489           Target.addOperand(MCOperand::createImm(Addr));
3490         }
3491 
3492         // Compare current call target to a specific address.
3493         NewCall->push_back(CallInst);
3494         MCInst &Compare = NewCall->back();
3495         Compare.clear();
3496         if (isBranchOnReg(CallInst))
3497           Compare.setOpcode(X86::CMP64rr);
3498         else if (CallInst.getOpcode() == X86::CALL64pcrel32)
3499           Compare.setOpcode(X86::CMP64ri32);
3500         else
3501           Compare.setOpcode(X86::CMP64rm);
3502 
3503         Compare.addOperand(MCOperand::createReg(FuncAddrReg));
3504 
3505         // TODO: Would be preferable to only load this value once.
3506         for (unsigned i = 0;
3507              i < Info->get(CallInst.getOpcode()).getNumOperands(); ++i)
3508           if (!CallInst.getOperand(i).isInst())
3509             Compare.addOperand(CallInst.getOperand(i));
3510       } else {
3511         // Compare current call target to a specific address.
3512         NewCall->push_back(CallInst);
3513         MCInst &Compare = NewCall->back();
3514         Compare.clear();
3515         if (isBranchOnReg(CallInst))
3516           Compare.setOpcode(X86::CMP64ri32);
3517         else
3518           Compare.setOpcode(X86::CMP64mi32);
3519 
3520         // Original call address.
3521         for (unsigned i = 0;
3522              i < Info->get(CallInst.getOpcode()).getNumOperands(); ++i)
3523           if (!CallInst.getOperand(i).isInst())
3524             Compare.addOperand(CallInst.getOperand(i));
3525 
3526         // Target address.
3527         if (Targets[i].first || LoadElim) {
3528           const MCSymbol *Sym =
3529               LoadElim ? VtableSyms[i].first : Targets[i].first;
3530           const uint64_t Addend = LoadElim ? VtableSyms[i].second : 0;
3531           const MCExpr *Expr = MCSymbolRefExpr::create(Sym, *Ctx);
3532           if (Addend)
3533             Expr = MCBinaryExpr::createAdd(
3534                 Expr, MCConstantExpr::create(Addend, *Ctx), *Ctx);
3535           Compare.addOperand(MCOperand::createExpr(Expr));
3536         } else {
3537           const uint64_t Addr = Targets[i].second;
3538           // Immediate address is out of sign extended 32 bit range.
3539           if (int64_t(Addr) != int64_t(int32_t(Addr)))
3540             return BlocksVectorTy();
3541 
3542           Compare.addOperand(MCOperand::createImm(Addr));
3543         }
3544       }
3545 
3546       // jump to next target compare.
3547       NextTarget =
3548           Ctx->createNamedTempSymbol(); // generate label for the next block
3549       NewCall->push_back(CallInst);
3550 
3551       if (IsJumpTable) {
3552         MCInst &Je = NewCall->back();
3553 
3554         // Jump to next compare if target addresses don't match.
3555         Je.clear();
3556         Je.setOpcode(X86::JCC_1);
3557         if (Targets[i].first)
3558           Je.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
3559               Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
3560         else
3561           Je.addOperand(MCOperand::createImm(Targets[i].second));
3562 
3563         Je.addOperand(MCOperand::createImm(X86::COND_E));
3564         assert(!isInvoke(CallInst));
3565       } else {
3566         MCInst &Jne = NewCall->back();
3567 
3568         // Jump to next compare if target addresses don't match.
3569         Jne.clear();
3570         Jne.setOpcode(X86::JCC_1);
3571         Jne.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
3572             NextTarget, MCSymbolRefExpr::VK_None, *Ctx)));
3573         Jne.addOperand(MCOperand::createImm(X86::COND_NE));
3574 
3575         // Call specific target directly.
3576         Results.emplace_back(Ctx->createNamedTempSymbol(),
3577                              InstructionListType());
3578         NewCall = &Results.back().second;
3579         NewCall->push_back(CallInst);
3580         MCInst &CallOrJmp = NewCall->back();
3581 
3582         CallOrJmp.clear();
3583 
3584         if (MinimizeCodeSize && !LoadElim) {
3585           CallOrJmp.setOpcode(IsTailCall ? X86::JMP32r : X86::CALL64r);
3586           CallOrJmp.addOperand(MCOperand::createReg(FuncAddrReg));
3587         } else {
3588           CallOrJmp.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32);
3589 
3590           if (Targets[i].first)
3591             CallOrJmp.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
3592                 Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
3593           else
3594             CallOrJmp.addOperand(MCOperand::createImm(Targets[i].second));
3595         }
3596         if (IsTailCall)
3597           setTailCall(CallOrJmp);
3598 
3599         if (CallOrJmp.getOpcode() == X86::CALL64r ||
3600             CallOrJmp.getOpcode() == X86::CALL64pcrel32) {
3601           if (Optional<uint32_t> Offset = getOffset(CallInst))
3602             // Annotated as duplicated call
3603             setOffset(CallOrJmp, *Offset);
3604         }
3605 
3606         if (isInvoke(CallInst) && !isInvoke(CallOrJmp)) {
3607           // Copy over any EH or GNU args size information from the original
3608           // call.
3609           Optional<MCPlus::MCLandingPad> EHInfo = getEHInfo(CallInst);
3610           if (EHInfo)
3611             addEHInfo(CallOrJmp, *EHInfo);
3612           int64_t GnuArgsSize = getGnuArgsSize(CallInst);
3613           if (GnuArgsSize >= 0)
3614             addGnuArgsSize(CallOrJmp, GnuArgsSize);
3615         }
3616 
3617         if (!IsTailCall) {
3618           // The fallthrough block for the most common target should be
3619           // the merge block.
3620           if (i == 0) {
3621             // Fallthrough to merge block.
3622             MergeBlock = Ctx->createNamedTempSymbol();
3623           } else {
3624             // Insert jump to the merge block if we are not doing a fallthrough.
3625             jumpToMergeBlock(*NewCall);
3626           }
3627         }
3628       }
3629     }
3630 
3631     // Cold call block.
3632     Results.emplace_back(NextTarget, InstructionListType());
3633     InstructionListType &NewCall = Results.back().second;
3634     for (const MCInst *Inst : MethodFetchInsns)
3635       if (Inst != &CallInst)
3636         NewCall.push_back(*Inst);
3637     NewCall.push_back(CallInst);
3638 
3639     // Jump to merge block from cold call block
3640     if (!IsTailCall && !IsJumpTable) {
3641       jumpToMergeBlock(NewCall);
3642 
3643       // Record merge block
3644       Results.emplace_back(MergeBlock, InstructionListType());
3645     }
3646 
3647     return Results;
3648   }
3649 
3650   BlocksVectorTy jumpTablePromotion(
3651       const MCInst &IJmpInst,
3652       const std::vector<std::pair<MCSymbol *, uint64_t>> &Targets,
3653       const std::vector<MCInst *> &TargetFetchInsns,
3654       MCContext *Ctx) const override {
3655     assert(getJumpTable(IJmpInst) != 0);
3656     uint16_t IndexReg = getAnnotationAs<uint16_t>(IJmpInst, "JTIndexReg");
3657     if (IndexReg == 0)
3658       return BlocksVectorTy();
3659 
3660     BlocksVectorTy Results;
3661 
3662     // Label for the current code block.
3663     MCSymbol *NextTarget = nullptr;
3664 
3665     for (unsigned int i = 0; i < Targets.size(); ++i) {
3666       Results.emplace_back(NextTarget, InstructionListType());
3667       InstructionListType *CurBB = &Results.back().second;
3668 
3669       // Compare current index to a specific index.
3670       CurBB->emplace_back(MCInst());
3671       MCInst &CompareInst = CurBB->back();
3672       CompareInst.setLoc(IJmpInst.getLoc());
3673       CompareInst.setOpcode(X86::CMP64ri32);
3674       CompareInst.addOperand(MCOperand::createReg(IndexReg));
3675 
3676       const uint64_t CaseIdx = Targets[i].second;
3677       // Immediate address is out of sign extended 32 bit range.
3678       if (int64_t(CaseIdx) != int64_t(int32_t(CaseIdx)))
3679         return BlocksVectorTy();
3680 
3681       CompareInst.addOperand(MCOperand::createImm(CaseIdx));
3682       shortenInstruction(CompareInst, *Ctx->getSubtargetInfo());
3683 
3684       // jump to next target compare.
3685       NextTarget =
3686           Ctx->createNamedTempSymbol(); // generate label for the next block
3687       CurBB->push_back(MCInst());
3688 
3689       MCInst &JEInst = CurBB->back();
3690       JEInst.setLoc(IJmpInst.getLoc());
3691 
3692       // Jump to target if indices match
3693       JEInst.setOpcode(X86::JCC_1);
3694       JEInst.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
3695           Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
3696       JEInst.addOperand(MCOperand::createImm(X86::COND_E));
3697     }
3698 
3699     // Cold call block.
3700     Results.emplace_back(NextTarget, InstructionListType());
3701     InstructionListType &CurBB = Results.back().second;
3702     for (const MCInst *Inst : TargetFetchInsns)
3703       if (Inst != &IJmpInst)
3704         CurBB.push_back(*Inst);
3705 
3706     CurBB.push_back(IJmpInst);
3707 
3708     return Results;
3709   }
3710 
3711 private:
3712   bool createMove(MCInst &Inst, const MCSymbol *Src, unsigned Reg,
3713                   MCContext *Ctx) const {
3714     Inst.setOpcode(X86::MOV64rm);
3715     Inst.addOperand(MCOperand::createReg(Reg));
3716     Inst.addOperand(MCOperand::createReg(X86::RIP));        // BaseReg
3717     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
3718     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
3719     Inst.addOperand(MCOperand::createExpr(
3720         MCSymbolRefExpr::create(Src, MCSymbolRefExpr::VK_None,
3721                                 *Ctx)));                    // Displacement
3722     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
3723 
3724     return true;
3725   }
3726 
3727   bool createLea(MCInst &Inst, const MCSymbol *Src, unsigned Reg,
3728                  MCContext *Ctx) const {
3729     Inst.setOpcode(X86::LEA64r);
3730     Inst.addOperand(MCOperand::createReg(Reg));
3731     Inst.addOperand(MCOperand::createReg(X86::RIP));        // BaseReg
3732     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
3733     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
3734     Inst.addOperand(MCOperand::createExpr(
3735         MCSymbolRefExpr::create(Src, MCSymbolRefExpr::VK_None,
3736                                 *Ctx)));                    // Displacement
3737     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
3738     return true;
3739   }
3740 };
3741 
3742 } // namespace
3743 
3744 namespace llvm {
3745 namespace bolt {
3746 
3747 MCPlusBuilder *createX86MCPlusBuilder(const MCInstrAnalysis *Analysis,
3748                                       const MCInstrInfo *Info,
3749                                       const MCRegisterInfo *RegInfo) {
3750   return new X86MCPlusBuilder(Analysis, Info, RegInfo);
3751 }
3752 
3753 } // namespace bolt
3754 } // namespace llvm
3755