1 //===-- X86MCCodeEmitter.cpp - Convert X86 code to machine code -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the X86MCCodeEmitter class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "MCTargetDesc/X86BaseInfo.h"
14 #include "MCTargetDesc/X86FixupKinds.h"
15 #include "MCTargetDesc/X86MCTargetDesc.h"
16 #include "llvm/ADT/SmallVector.h"
17 #include "llvm/MC/MCCodeEmitter.h"
18 #include "llvm/MC/MCContext.h"
19 #include "llvm/MC/MCExpr.h"
20 #include "llvm/MC/MCFixup.h"
21 #include "llvm/MC/MCInst.h"
22 #include "llvm/MC/MCInstrDesc.h"
23 #include "llvm/MC/MCInstrInfo.h"
24 #include "llvm/MC/MCRegisterInfo.h"
25 #include "llvm/MC/MCSubtargetInfo.h"
26 #include "llvm/MC/MCSymbol.h"
27 #include "llvm/Support/ErrorHandling.h"
28 #include "llvm/Support/raw_ostream.h"
29 #include <cassert>
30 #include <cstdint>
31 #include <cstdlib>
32 
33 using namespace llvm;
34 
35 #define DEBUG_TYPE "mccodeemitter"
36 
37 namespace {
38 
39 class X86MCCodeEmitter : public MCCodeEmitter {
40   const MCInstrInfo &MCII;
41   MCContext &Ctx;
42 
43 public:
44   X86MCCodeEmitter(const MCInstrInfo &mcii, MCContext &ctx)
45       : MCII(mcii), Ctx(ctx) {}
46   X86MCCodeEmitter(const X86MCCodeEmitter &) = delete;
47   X86MCCodeEmitter &operator=(const X86MCCodeEmitter &) = delete;
48   ~X86MCCodeEmitter() override = default;
49 
50   void emitPrefix(const MCInst &MI, raw_ostream &OS,
51                   const MCSubtargetInfo &STI) const override;
52 
53   void encodeInstruction(const MCInst &MI, raw_ostream &OS,
54                          SmallVectorImpl<MCFixup> &Fixups,
55                          const MCSubtargetInfo &STI) const override;
56 
57 private:
58   unsigned getX86RegNum(const MCOperand &MO) const;
59 
60   unsigned getX86RegEncoding(const MCInst &MI, unsigned OpNum) const;
61 
62   /// \param MI a single low-level machine instruction.
63   /// \param OpNum the operand #.
64   /// \returns true if the OpNumth operand of MI  require a bit to be set in
65   /// REX prefix.
66   bool isREXExtendedReg(const MCInst &MI, unsigned OpNum) const;
67 
68   void emitImmediate(const MCOperand &Disp, SMLoc Loc, unsigned ImmSize,
69                      MCFixupKind FixupKind, uint64_t StartByte, raw_ostream &OS,
70                      SmallVectorImpl<MCFixup> &Fixups, int ImmOffset = 0) const;
71 
72   void emitRegModRMByte(const MCOperand &ModRMReg, unsigned RegOpcodeFld,
73                         raw_ostream &OS) const;
74 
75   void emitSIBByte(unsigned SS, unsigned Index, unsigned Base,
76                    raw_ostream &OS) const;
77 
78   void emitMemModRMByte(const MCInst &MI, unsigned Op, unsigned RegOpcodeField,
79                         uint64_t TSFlags, bool HasREX, uint64_t StartByte,
80                         raw_ostream &OS, SmallVectorImpl<MCFixup> &Fixups,
81                         const MCSubtargetInfo &STI,
82                         bool ForceSIB = false) const;
83 
84   bool emitPrefixImpl(unsigned &CurOp, const MCInst &MI,
85                       const MCSubtargetInfo &STI, raw_ostream &OS) const;
86 
87   void emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,
88                            raw_ostream &OS) const;
89 
90   void emitSegmentOverridePrefix(unsigned SegOperand, const MCInst &MI,
91                                  raw_ostream &OS) const;
92 
93   bool emitOpcodePrefix(int MemOperand, const MCInst &MI,
94                         const MCSubtargetInfo &STI, raw_ostream &OS) const;
95 
96   bool emitREXPrefix(int MemOperand, const MCInst &MI, raw_ostream &OS) const;
97 };
98 
99 } // end anonymous namespace
100 
101 static uint8_t modRMByte(unsigned Mod, unsigned RegOpcode, unsigned RM) {
102   assert(Mod < 4 && RegOpcode < 8 && RM < 8 && "ModRM Fields out of range!");
103   return RM | (RegOpcode << 3) | (Mod << 6);
104 }
105 
106 static void emitByte(uint8_t C, raw_ostream &OS) { OS << static_cast<char>(C); }
107 
108 static void emitConstant(uint64_t Val, unsigned Size, raw_ostream &OS) {
109   // Output the constant in little endian byte order.
110   for (unsigned i = 0; i != Size; ++i) {
111     emitByte(Val & 255, OS);
112     Val >>= 8;
113   }
114 }
115 
116 /// Determine if this immediate can fit in a disp8 or a compressed disp8 for
117 /// EVEX instructions. \p will be set to the value to pass to the ImmOffset
118 /// parameter of emitImmediate.
119 static bool isDispOrCDisp8(uint64_t TSFlags, int Value, int &ImmOffset) {
120   bool HasEVEX = (TSFlags & X86II::EncodingMask) == X86II::EVEX;
121 
122   int CD8_Scale =
123       (TSFlags & X86II::CD8_Scale_Mask) >> X86II::CD8_Scale_Shift;
124   if (!HasEVEX || CD8_Scale == 0)
125     return isInt<8>(Value);
126 
127   assert(isPowerOf2_32(CD8_Scale) && "Unexpected CD8 scale!");
128   if (Value & (CD8_Scale - 1)) // Unaligned offset
129     return false;
130 
131   int CDisp8 = Value / CD8_Scale;
132   if (!isInt<8>(CDisp8))
133     return false;
134 
135   // ImmOffset will be added to Value in emitImmediate leaving just CDisp8.
136   ImmOffset = CDisp8 - Value;
137   return true;
138 }
139 
140 /// \returns the appropriate fixup kind to use for an immediate in an
141 /// instruction with the specified TSFlags.
142 static MCFixupKind getImmFixupKind(uint64_t TSFlags) {
143   unsigned Size = X86II::getSizeOfImm(TSFlags);
144   bool isPCRel = X86II::isImmPCRel(TSFlags);
145 
146   if (X86II::isImmSigned(TSFlags)) {
147     switch (Size) {
148     default:
149       llvm_unreachable("Unsupported signed fixup size!");
150     case 4:
151       return MCFixupKind(X86::reloc_signed_4byte);
152     }
153   }
154   return MCFixup::getKindForSize(Size, isPCRel);
155 }
156 
157 /// \param Op operand # of the memory operand.
158 ///
159 /// \returns true if the specified instruction has a 16-bit memory operand.
160 static bool is16BitMemOperand(const MCInst &MI, unsigned Op,
161                               const MCSubtargetInfo &STI) {
162   const MCOperand &Base = MI.getOperand(Op + X86::AddrBaseReg);
163   const MCOperand &Index = MI.getOperand(Op + X86::AddrIndexReg);
164   const MCOperand &Disp = MI.getOperand(Op + X86::AddrDisp);
165 
166   unsigned BaseReg = Base.getReg();
167   unsigned IndexReg = Index.getReg();
168 
169   if (STI.hasFeature(X86::Mode16Bit) && BaseReg == 0 && IndexReg == 0 &&
170       Disp.isImm() && Disp.getImm() < 0x10000)
171     return true;
172   if ((BaseReg != 0 &&
173        X86MCRegisterClasses[X86::GR16RegClassID].contains(BaseReg)) ||
174       (IndexReg != 0 &&
175        X86MCRegisterClasses[X86::GR16RegClassID].contains(IndexReg)))
176     return true;
177   return false;
178 }
179 
180 /// \param Op operand # of the memory operand.
181 ///
182 /// \returns true if the specified instruction has a 32-bit memory operand.
183 static bool is32BitMemOperand(const MCInst &MI, unsigned Op) {
184   const MCOperand &BaseReg = MI.getOperand(Op + X86::AddrBaseReg);
185   const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
186 
187   if ((BaseReg.getReg() != 0 &&
188        X86MCRegisterClasses[X86::GR32RegClassID].contains(BaseReg.getReg())) ||
189       (IndexReg.getReg() != 0 &&
190        X86MCRegisterClasses[X86::GR32RegClassID].contains(IndexReg.getReg())))
191     return true;
192   if (BaseReg.getReg() == X86::EIP) {
193     assert(IndexReg.getReg() == 0 && "Invalid eip-based address.");
194     return true;
195   }
196   if (IndexReg.getReg() == X86::EIZ)
197     return true;
198   return false;
199 }
200 
201 /// \param Op operand # of the memory operand.
202 ///
203 /// \returns true if the specified instruction has a 64-bit memory operand.
204 #ifndef NDEBUG
205 static bool is64BitMemOperand(const MCInst &MI, unsigned Op) {
206   const MCOperand &BaseReg = MI.getOperand(Op + X86::AddrBaseReg);
207   const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
208 
209   if ((BaseReg.getReg() != 0 &&
210        X86MCRegisterClasses[X86::GR64RegClassID].contains(BaseReg.getReg())) ||
211       (IndexReg.getReg() != 0 &&
212        X86MCRegisterClasses[X86::GR64RegClassID].contains(IndexReg.getReg())))
213     return true;
214   return false;
215 }
216 #endif
217 
218 enum GlobalOffsetTableExprKind { GOT_None, GOT_Normal, GOT_SymDiff };
219 
220 /// Check if this expression starts with  _GLOBAL_OFFSET_TABLE_ and if it is
221 /// of the form _GLOBAL_OFFSET_TABLE_-symbol. This is needed to support PIC on
222 /// ELF i386 as _GLOBAL_OFFSET_TABLE_ is magical. We check only simple case that
223 /// are know to be used: _GLOBAL_OFFSET_TABLE_ by itself or at the start of a
224 /// binary expression.
225 static GlobalOffsetTableExprKind
226 startsWithGlobalOffsetTable(const MCExpr *Expr) {
227   const MCExpr *RHS = nullptr;
228   if (Expr->getKind() == MCExpr::Binary) {
229     const MCBinaryExpr *BE = static_cast<const MCBinaryExpr *>(Expr);
230     Expr = BE->getLHS();
231     RHS = BE->getRHS();
232   }
233 
234   if (Expr->getKind() != MCExpr::SymbolRef)
235     return GOT_None;
236 
237   const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr *>(Expr);
238   const MCSymbol &S = Ref->getSymbol();
239   if (S.getName() != "_GLOBAL_OFFSET_TABLE_")
240     return GOT_None;
241   if (RHS && RHS->getKind() == MCExpr::SymbolRef)
242     return GOT_SymDiff;
243   return GOT_Normal;
244 }
245 
246 static bool hasSecRelSymbolRef(const MCExpr *Expr) {
247   if (Expr->getKind() == MCExpr::SymbolRef) {
248     const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr *>(Expr);
249     return Ref->getKind() == MCSymbolRefExpr::VK_SECREL;
250   }
251   return false;
252 }
253 
254 static bool isPCRel32Branch(const MCInst &MI, const MCInstrInfo &MCII) {
255   unsigned Opcode = MI.getOpcode();
256   const MCInstrDesc &Desc = MCII.get(Opcode);
257   if ((Opcode != X86::CALL64pcrel32 && Opcode != X86::JMP_4 &&
258        Opcode != X86::JCC_4) ||
259       getImmFixupKind(Desc.TSFlags) != FK_PCRel_4)
260     return false;
261 
262   unsigned CurOp = X86II::getOperandBias(Desc);
263   const MCOperand &Op = MI.getOperand(CurOp);
264   if (!Op.isExpr())
265     return false;
266 
267   const MCSymbolRefExpr *Ref = dyn_cast<MCSymbolRefExpr>(Op.getExpr());
268   return Ref && Ref->getKind() == MCSymbolRefExpr::VK_None;
269 }
270 
271 unsigned X86MCCodeEmitter::getX86RegNum(const MCOperand &MO) const {
272   return Ctx.getRegisterInfo()->getEncodingValue(MO.getReg()) & 0x7;
273 }
274 
275 unsigned X86MCCodeEmitter::getX86RegEncoding(const MCInst &MI,
276                                              unsigned OpNum) const {
277   return Ctx.getRegisterInfo()->getEncodingValue(MI.getOperand(OpNum).getReg());
278 }
279 
280 /// \param MI a single low-level machine instruction.
281 /// \param OpNum the operand #.
282 /// \returns true if the OpNumth operand of MI  require a bit to be set in
283 /// REX prefix.
284 bool X86MCCodeEmitter::isREXExtendedReg(const MCInst &MI,
285                                         unsigned OpNum) const {
286   return (getX86RegEncoding(MI, OpNum) >> 3) & 1;
287 }
288 
289 void X86MCCodeEmitter::emitImmediate(const MCOperand &DispOp, SMLoc Loc,
290                                      unsigned Size, MCFixupKind FixupKind,
291                                      uint64_t StartByte, raw_ostream &OS,
292                                      SmallVectorImpl<MCFixup> &Fixups,
293                                      int ImmOffset) const {
294   const MCExpr *Expr = nullptr;
295   if (DispOp.isImm()) {
296     // If this is a simple integer displacement that doesn't require a
297     // relocation, emit it now.
298     if (FixupKind != FK_PCRel_1 && FixupKind != FK_PCRel_2 &&
299         FixupKind != FK_PCRel_4) {
300       emitConstant(DispOp.getImm() + ImmOffset, Size, OS);
301       return;
302     }
303     Expr = MCConstantExpr::create(DispOp.getImm(), Ctx);
304   } else {
305     Expr = DispOp.getExpr();
306   }
307 
308   // If we have an immoffset, add it to the expression.
309   if ((FixupKind == FK_Data_4 || FixupKind == FK_Data_8 ||
310        FixupKind == MCFixupKind(X86::reloc_signed_4byte))) {
311     GlobalOffsetTableExprKind Kind = startsWithGlobalOffsetTable(Expr);
312     if (Kind != GOT_None) {
313       assert(ImmOffset == 0);
314 
315       if (Size == 8) {
316         FixupKind = MCFixupKind(X86::reloc_global_offset_table8);
317       } else {
318         assert(Size == 4);
319         FixupKind = MCFixupKind(X86::reloc_global_offset_table);
320       }
321 
322       if (Kind == GOT_Normal)
323         ImmOffset = static_cast<int>(OS.tell() - StartByte);
324     } else if (Expr->getKind() == MCExpr::SymbolRef) {
325       if (hasSecRelSymbolRef(Expr)) {
326         FixupKind = MCFixupKind(FK_SecRel_4);
327       }
328     } else if (Expr->getKind() == MCExpr::Binary) {
329       const MCBinaryExpr *Bin = static_cast<const MCBinaryExpr *>(Expr);
330       if (hasSecRelSymbolRef(Bin->getLHS()) ||
331           hasSecRelSymbolRef(Bin->getRHS())) {
332         FixupKind = MCFixupKind(FK_SecRel_4);
333       }
334     }
335   }
336 
337   // If the fixup is pc-relative, we need to bias the value to be relative to
338   // the start of the field, not the end of the field.
339   if (FixupKind == FK_PCRel_4 ||
340       FixupKind == MCFixupKind(X86::reloc_riprel_4byte) ||
341       FixupKind == MCFixupKind(X86::reloc_riprel_4byte_movq_load) ||
342       FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax) ||
343       FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax_rex) ||
344       FixupKind == MCFixupKind(X86::reloc_branch_4byte_pcrel)) {
345     ImmOffset -= 4;
346     // If this is a pc-relative load off _GLOBAL_OFFSET_TABLE_:
347     // leaq _GLOBAL_OFFSET_TABLE_(%rip), %r15
348     // this needs to be a GOTPC32 relocation.
349     if (startsWithGlobalOffsetTable(Expr) != GOT_None)
350       FixupKind = MCFixupKind(X86::reloc_global_offset_table);
351   }
352   if (FixupKind == FK_PCRel_2)
353     ImmOffset -= 2;
354   if (FixupKind == FK_PCRel_1)
355     ImmOffset -= 1;
356 
357   if (ImmOffset)
358     Expr = MCBinaryExpr::createAdd(Expr, MCConstantExpr::create(ImmOffset, Ctx),
359                                    Ctx);
360 
361   // Emit a symbolic constant as a fixup and 4 zeros.
362   Fixups.push_back(MCFixup::create(static_cast<uint32_t>(OS.tell() - StartByte),
363                                    Expr, FixupKind, Loc));
364   emitConstant(0, Size, OS);
365 }
366 
367 void X86MCCodeEmitter::emitRegModRMByte(const MCOperand &ModRMReg,
368                                         unsigned RegOpcodeFld,
369                                         raw_ostream &OS) const {
370   emitByte(modRMByte(3, RegOpcodeFld, getX86RegNum(ModRMReg)), OS);
371 }
372 
373 void X86MCCodeEmitter::emitSIBByte(unsigned SS, unsigned Index, unsigned Base,
374                                    raw_ostream &OS) const {
375   // SIB byte is in the same format as the modRMByte.
376   emitByte(modRMByte(SS, Index, Base), OS);
377 }
378 
379 void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
380                                         unsigned RegOpcodeField,
381                                         uint64_t TSFlags, bool HasREX,
382                                         uint64_t StartByte, raw_ostream &OS,
383                                         SmallVectorImpl<MCFixup> &Fixups,
384                                         const MCSubtargetInfo &STI,
385                                         bool ForceSIB) const {
386   const MCOperand &Disp = MI.getOperand(Op + X86::AddrDisp);
387   const MCOperand &Base = MI.getOperand(Op + X86::AddrBaseReg);
388   const MCOperand &Scale = MI.getOperand(Op + X86::AddrScaleAmt);
389   const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
390   unsigned BaseReg = Base.getReg();
391 
392   // Handle %rip relative addressing.
393   if (BaseReg == X86::RIP ||
394       BaseReg == X86::EIP) { // [disp32+rIP] in X86-64 mode
395     assert(STI.hasFeature(X86::Mode64Bit) &&
396            "Rip-relative addressing requires 64-bit mode");
397     assert(IndexReg.getReg() == 0 && !ForceSIB &&
398            "Invalid rip-relative address");
399     emitByte(modRMByte(0, RegOpcodeField, 5), OS);
400 
401     unsigned Opcode = MI.getOpcode();
402     // movq loads are handled with a special relocation form which allows the
403     // linker to eliminate some loads for GOT references which end up in the
404     // same linkage unit.
405     unsigned FixupKind = [=]() {
406       switch (Opcode) {
407       default:
408         return X86::reloc_riprel_4byte;
409       case X86::MOV64rm:
410         assert(HasREX);
411         return X86::reloc_riprel_4byte_movq_load;
412       case X86::CALL64m:
413       case X86::JMP64m:
414       case X86::TAILJMPm64:
415       case X86::TEST64mr:
416       case X86::ADC64rm:
417       case X86::ADD64rm:
418       case X86::AND64rm:
419       case X86::CMP64rm:
420       case X86::OR64rm:
421       case X86::SBB64rm:
422       case X86::SUB64rm:
423       case X86::XOR64rm:
424         return HasREX ? X86::reloc_riprel_4byte_relax_rex
425                       : X86::reloc_riprel_4byte_relax;
426       }
427     }();
428 
429     // rip-relative addressing is actually relative to the *next* instruction.
430     // Since an immediate can follow the mod/rm byte for an instruction, this
431     // means that we need to bias the displacement field of the instruction with
432     // the size of the immediate field. If we have this case, add it into the
433     // expression to emit.
434     // Note: rip-relative addressing using immediate displacement values should
435     // not be adjusted, assuming it was the user's intent.
436     int ImmSize = !Disp.isImm() && X86II::hasImm(TSFlags)
437                       ? X86II::getSizeOfImm(TSFlags)
438                       : 0;
439 
440     emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), StartByte, OS,
441                   Fixups, -ImmSize);
442     return;
443   }
444 
445   unsigned BaseRegNo = BaseReg ? getX86RegNum(Base) : -1U;
446 
447   // 16-bit addressing forms of the ModR/M byte have a different encoding for
448   // the R/M field and are far more limited in which registers can be used.
449   if (is16BitMemOperand(MI, Op, STI)) {
450     if (BaseReg) {
451       // For 32-bit addressing, the row and column values in Table 2-2 are
452       // basically the same. It's AX/CX/DX/BX/SP/BP/SI/DI in that order, with
453       // some special cases. And getX86RegNum reflects that numbering.
454       // For 16-bit addressing it's more fun, as shown in the SDM Vol 2A,
455       // Table 2-1 "16-Bit Addressing Forms with the ModR/M byte". We can only
456       // use SI/DI/BP/BX, which have "row" values 4-7 in no particular order,
457       // while values 0-3 indicate the allowed combinations (base+index) of
458       // those: 0 for BX+SI, 1 for BX+DI, 2 for BP+SI, 3 for BP+DI.
459       //
460       // R16Table[] is a lookup from the normal RegNo, to the row values from
461       // Table 2-1 for 16-bit addressing modes. Where zero means disallowed.
462       static const unsigned R16Table[] = {0, 0, 0, 7, 0, 6, 4, 5};
463       unsigned RMfield = R16Table[BaseRegNo];
464 
465       assert(RMfield && "invalid 16-bit base register");
466 
467       if (IndexReg.getReg()) {
468         unsigned IndexReg16 = R16Table[getX86RegNum(IndexReg)];
469 
470         assert(IndexReg16 && "invalid 16-bit index register");
471         // We must have one of SI/DI (4,5), and one of BP/BX (6,7).
472         assert(((IndexReg16 ^ RMfield) & 2) &&
473                "invalid 16-bit base/index register combination");
474         assert(Scale.getImm() == 1 &&
475                "invalid scale for 16-bit memory reference");
476 
477         // Allow base/index to appear in either order (although GAS doesn't).
478         if (IndexReg16 & 2)
479           RMfield = (RMfield & 1) | ((7 - IndexReg16) << 1);
480         else
481           RMfield = (IndexReg16 & 1) | ((7 - RMfield) << 1);
482       }
483 
484       if (Disp.isImm() && isInt<8>(Disp.getImm())) {
485         if (Disp.getImm() == 0 && RMfield != 6) {
486           // There is no displacement; just the register.
487           emitByte(modRMByte(0, RegOpcodeField, RMfield), OS);
488           return;
489         }
490         // Use the [REG]+disp8 form, including for [BP] which cannot be encoded.
491         emitByte(modRMByte(1, RegOpcodeField, RMfield), OS);
492         emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups);
493         return;
494       }
495       // This is the [REG]+disp16 case.
496       emitByte(modRMByte(2, RegOpcodeField, RMfield), OS);
497     } else {
498       assert(IndexReg.getReg() == 0 && "Unexpected index register!");
499       // There is no BaseReg; this is the plain [disp16] case.
500       emitByte(modRMByte(0, RegOpcodeField, 6), OS);
501     }
502 
503     // Emit 16-bit displacement for plain disp16 or [REG]+disp16 cases.
504     emitImmediate(Disp, MI.getLoc(), 2, FK_Data_2, StartByte, OS, Fixups);
505     return;
506   }
507 
508   // Check for presence of {disp8} or {disp32} pseudo prefixes.
509   bool UseDisp8 = MI.getFlags() & X86::IP_USE_DISP8;
510   bool UseDisp32 = MI.getFlags() & X86::IP_USE_DISP32;
511 
512   // We only allow no displacement if no pseudo prefix is present.
513   bool AllowNoDisp = !UseDisp8 && !UseDisp32;
514   // Disp8 is allowed unless the {disp32} prefix is present.
515   bool AllowDisp8 = !UseDisp32;
516 
517   // Determine whether a SIB byte is needed.
518   if (// The SIB byte must be used if there is an index register or the
519       // encoding requires a SIB byte.
520       !ForceSIB && IndexReg.getReg() == 0 &&
521       // The SIB byte must be used if the base is ESP/RSP/R12, all of which
522       // encode to an R/M value of 4, which indicates that a SIB byte is
523       // present.
524       BaseRegNo != N86::ESP &&
525       // If there is no base register and we're in 64-bit mode, we need a SIB
526       // byte to emit an addr that is just 'disp32' (the non-RIP relative form).
527       (!STI.hasFeature(X86::Mode64Bit) || BaseReg != 0)) {
528 
529     if (BaseReg == 0) { // [disp32]     in X86-32 mode
530       emitByte(modRMByte(0, RegOpcodeField, 5), OS);
531       emitImmediate(Disp, MI.getLoc(), 4, FK_Data_4, StartByte, OS, Fixups);
532       return;
533     }
534 
535     // If the base is not EBP/ESP/R12/R13 and there is no displacement, use
536     // simple indirect register encoding, this handles addresses like [EAX].
537     // The encoding for [EBP] or[R13] with no displacement means [disp32] so we
538     // handle it by emitting a displacement of 0 later.
539     if (BaseRegNo != N86::EBP) {
540       if (Disp.isImm() && Disp.getImm() == 0 && AllowNoDisp) {
541         emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), OS);
542         return;
543       }
544 
545       // If the displacement is @tlscall, treat it as a zero.
546       if (Disp.isExpr()) {
547         auto *Sym = dyn_cast<MCSymbolRefExpr>(Disp.getExpr());
548         if (Sym && Sym->getKind() == MCSymbolRefExpr::VK_TLSCALL) {
549           // This is exclusively used by call *a@tlscall(base). The relocation
550           // (R_386_TLSCALL or R_X86_64_TLSCALL) applies to the beginning.
551           Fixups.push_back(MCFixup::create(0, Sym, FK_NONE, MI.getLoc()));
552           emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), OS);
553           return;
554         }
555       }
556     }
557 
558     // Otherwise, if the displacement fits in a byte, encode as [REG+disp8].
559     // Including a compressed disp8 for EVEX instructions that support it.
560     // This also handles the 0 displacement for [EBP] or [R13]. We can't use
561     // disp8 if the {disp32} pseudo prefix is present.
562     if (Disp.isImm() && AllowDisp8) {
563       int ImmOffset = 0;
564       if (isDispOrCDisp8(TSFlags, Disp.getImm(), ImmOffset)) {
565         emitByte(modRMByte(1, RegOpcodeField, BaseRegNo), OS);
566         emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups,
567                       ImmOffset);
568         return;
569       }
570     }
571 
572     // Otherwise, emit the most general non-SIB encoding: [REG+disp32].
573     // Displacement may be 0 for [EBP] or [R13] case if {disp32} pseudo prefix
574     // prevented using disp8 above.
575     emitByte(modRMByte(2, RegOpcodeField, BaseRegNo), OS);
576     unsigned Opcode = MI.getOpcode();
577     unsigned FixupKind = Opcode == X86::MOV32rm ? X86::reloc_signed_4byte_relax
578                                                 : X86::reloc_signed_4byte;
579     emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), StartByte, OS,
580                   Fixups);
581     return;
582   }
583 
584   // We need a SIB byte, so start by outputting the ModR/M byte first
585   assert(IndexReg.getReg() != X86::ESP && IndexReg.getReg() != X86::RSP &&
586          "Cannot use ESP as index reg!");
587 
588   bool ForceDisp32 = false;
589   bool ForceDisp8 = false;
590   int ImmOffset = 0;
591   if (BaseReg == 0) {
592     // If there is no base register, we emit the special case SIB byte with
593     // MOD=0, BASE=5, to JUST get the index, scale, and displacement.
594     BaseRegNo = 5;
595     emitByte(modRMByte(0, RegOpcodeField, 4), OS);
596     ForceDisp32 = true;
597   } else if (Disp.isImm() && Disp.getImm() == 0 && AllowNoDisp &&
598              // Base reg can't be EBP/RBP/R13 as that would end up with '5' as
599              // the base field, but that is the magic [*] nomenclature that
600              // indicates no base when mod=0. For these cases we'll emit a 0
601              // displacement instead.
602              BaseRegNo != N86::EBP) {
603     // Emit no displacement ModR/M byte
604     emitByte(modRMByte(0, RegOpcodeField, 4), OS);
605   } else if (Disp.isImm() && AllowDisp8 &&
606              isDispOrCDisp8(TSFlags, Disp.getImm(), ImmOffset)) {
607     // Displacement fits in a byte or matches an EVEX compressed disp8, use
608     // disp8 encoding. This also handles EBP/R13 base with 0 displacement unless
609     // {disp32} pseudo prefix was used.
610     emitByte(modRMByte(1, RegOpcodeField, 4), OS);
611     ForceDisp8 = true;
612   } else {
613     // Otherwise, emit the normal disp32 encoding.
614     emitByte(modRMByte(2, RegOpcodeField, 4), OS);
615     ForceDisp32 = true;
616   }
617 
618   // Calculate what the SS field value should be...
619   static const unsigned SSTable[] = {~0U, 0, 1, ~0U, 2, ~0U, ~0U, ~0U, 3};
620   unsigned SS = SSTable[Scale.getImm()];
621 
622   unsigned IndexRegNo = IndexReg.getReg() ? getX86RegNum(IndexReg) : 4;
623 
624   emitSIBByte(SS, IndexRegNo, BaseRegNo, OS);
625 
626   // Do we need to output a displacement?
627   if (ForceDisp8)
628     emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups,
629                   ImmOffset);
630   else if (ForceDisp32)
631     emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(X86::reloc_signed_4byte),
632                   StartByte, OS, Fixups);
633 }
634 
635 /// Emit all instruction prefixes.
636 ///
637 /// \returns true if REX prefix is used, otherwise returns false.
638 bool X86MCCodeEmitter::emitPrefixImpl(unsigned &CurOp, const MCInst &MI,
639                                       const MCSubtargetInfo &STI,
640                                       raw_ostream &OS) const {
641   uint64_t TSFlags = MCII.get(MI.getOpcode()).TSFlags;
642   // Determine where the memory operand starts, if present.
643   int MemoryOperand = X86II::getMemoryOperandNo(TSFlags);
644   // Emit segment override opcode prefix as needed.
645   if (MemoryOperand != -1) {
646     MemoryOperand += CurOp;
647     emitSegmentOverridePrefix(MemoryOperand + X86::AddrSegmentReg, MI, OS);
648   }
649 
650   // Emit the repeat opcode prefix as needed.
651   unsigned Flags = MI.getFlags();
652   if (TSFlags & X86II::REP || Flags & X86::IP_HAS_REPEAT)
653     emitByte(0xF3, OS);
654   if (Flags & X86::IP_HAS_REPEAT_NE)
655     emitByte(0xF2, OS);
656 
657   // Emit the address size opcode prefix as needed.
658   bool NeedAddressOverride;
659   uint64_t AdSize = TSFlags & X86II::AdSizeMask;
660   if ((STI.hasFeature(X86::Mode16Bit) && AdSize == X86II::AdSize32) ||
661       (STI.hasFeature(X86::Mode32Bit) && AdSize == X86II::AdSize16) ||
662       (STI.hasFeature(X86::Mode64Bit) && AdSize == X86II::AdSize32)) {
663     NeedAddressOverride = true;
664   } else if (MemoryOperand < 0) {
665     NeedAddressOverride = false;
666   } else if (STI.hasFeature(X86::Mode64Bit)) {
667     assert(!is16BitMemOperand(MI, MemoryOperand, STI));
668     NeedAddressOverride = is32BitMemOperand(MI, MemoryOperand);
669   } else if (STI.hasFeature(X86::Mode32Bit)) {
670     assert(!is64BitMemOperand(MI, MemoryOperand));
671     NeedAddressOverride = is16BitMemOperand(MI, MemoryOperand, STI);
672   } else {
673     assert(STI.hasFeature(X86::Mode16Bit));
674     assert(!is64BitMemOperand(MI, MemoryOperand));
675     NeedAddressOverride = !is16BitMemOperand(MI, MemoryOperand, STI);
676   }
677 
678   if (NeedAddressOverride)
679     emitByte(0x67, OS);
680 
681   // Encoding type for this instruction.
682   uint64_t Encoding = TSFlags & X86II::EncodingMask;
683   bool HasREX = false;
684   if (Encoding)
685     emitVEXOpcodePrefix(MemoryOperand, MI, OS);
686   else
687     HasREX = emitOpcodePrefix(MemoryOperand, MI, STI, OS);
688 
689   uint64_t Form = TSFlags & X86II::FormMask;
690   switch (Form) {
691   default:
692     break;
693   case X86II::RawFrmDstSrc: {
694     unsigned siReg = MI.getOperand(1).getReg();
695     assert(((siReg == X86::SI && MI.getOperand(0).getReg() == X86::DI) ||
696             (siReg == X86::ESI && MI.getOperand(0).getReg() == X86::EDI) ||
697             (siReg == X86::RSI && MI.getOperand(0).getReg() == X86::RDI)) &&
698            "SI and DI register sizes do not match");
699     // Emit segment override opcode prefix as needed (not for %ds).
700     if (MI.getOperand(2).getReg() != X86::DS)
701       emitSegmentOverridePrefix(2, MI, OS);
702     // Emit AdSize prefix as needed.
703     if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::ESI) ||
704         (STI.hasFeature(X86::Mode32Bit) && siReg == X86::SI))
705       emitByte(0x67, OS);
706     CurOp += 3; // Consume operands.
707     break;
708   }
709   case X86II::RawFrmSrc: {
710     unsigned siReg = MI.getOperand(0).getReg();
711     // Emit segment override opcode prefix as needed (not for %ds).
712     if (MI.getOperand(1).getReg() != X86::DS)
713       emitSegmentOverridePrefix(1, MI, OS);
714     // Emit AdSize prefix as needed.
715     if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::ESI) ||
716         (STI.hasFeature(X86::Mode32Bit) && siReg == X86::SI))
717       emitByte(0x67, OS);
718     CurOp += 2; // Consume operands.
719     break;
720   }
721   case X86II::RawFrmDst: {
722     unsigned siReg = MI.getOperand(0).getReg();
723     // Emit AdSize prefix as needed.
724     if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::EDI) ||
725         (STI.hasFeature(X86::Mode32Bit) && siReg == X86::DI))
726       emitByte(0x67, OS);
727     ++CurOp; // Consume operand.
728     break;
729   }
730   case X86II::RawFrmMemOffs: {
731     // Emit segment override opcode prefix as needed.
732     emitSegmentOverridePrefix(1, MI, OS);
733     break;
734   }
735   }
736 
737   return HasREX;
738 }
739 
740 /// AVX instructions are encoded using a opcode prefix called VEX.
741 void X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,
742                                            raw_ostream &OS) const {
743   const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
744   uint64_t TSFlags = Desc.TSFlags;
745 
746   assert(!(TSFlags & X86II::LOCK) && "Can't have LOCK VEX.");
747 
748   uint64_t Encoding = TSFlags & X86II::EncodingMask;
749   bool HasEVEX_K = TSFlags & X86II::EVEX_K;
750   bool HasVEX_4V = TSFlags & X86II::VEX_4V;
751   bool HasEVEX_RC = TSFlags & X86II::EVEX_RC;
752 
753   // VEX_R: opcode externsion equivalent to REX.R in
754   // 1's complement (inverted) form
755   //
756   //  1: Same as REX_R=0 (must be 1 in 32-bit mode)
757   //  0: Same as REX_R=1 (64 bit mode only)
758   //
759   uint8_t VEX_R = 0x1;
760   uint8_t EVEX_R2 = 0x1;
761 
762   // VEX_X: equivalent to REX.X, only used when a
763   // register is used for index in SIB Byte.
764   //
765   //  1: Same as REX.X=0 (must be 1 in 32-bit mode)
766   //  0: Same as REX.X=1 (64-bit mode only)
767   uint8_t VEX_X = 0x1;
768 
769   // VEX_B:
770   //
771   //  1: Same as REX_B=0 (ignored in 32-bit mode)
772   //  0: Same as REX_B=1 (64 bit mode only)
773   //
774   uint8_t VEX_B = 0x1;
775 
776   // VEX_W: opcode specific (use like REX.W, or used for
777   // opcode extension, or ignored, depending on the opcode byte)
778   uint8_t VEX_W = (TSFlags & X86II::VEX_W) ? 1 : 0;
779 
780   // VEX_5M (VEX m-mmmmm field):
781   //
782   //  0b00000: Reserved for future use
783   //  0b00001: implied 0F leading opcode
784   //  0b00010: implied 0F 38 leading opcode bytes
785   //  0b00011: implied 0F 3A leading opcode bytes
786   //  0b00100-0b11111: Reserved for future use
787   //  0b01000: XOP map select - 08h instructions with imm byte
788   //  0b01001: XOP map select - 09h instructions with no imm byte
789   //  0b01010: XOP map select - 0Ah instructions with imm dword
790   uint8_t VEX_5M;
791   switch (TSFlags & X86II::OpMapMask) {
792   default:
793     llvm_unreachable("Invalid prefix!");
794   case X86II::TB:
795     VEX_5M = 0x1;
796     break; // 0F
797   case X86II::T8:
798     VEX_5M = 0x2;
799     break; // 0F 38
800   case X86II::TA:
801     VEX_5M = 0x3;
802     break; // 0F 3A
803   case X86II::XOP8:
804     VEX_5M = 0x8;
805     break;
806   case X86II::XOP9:
807     VEX_5M = 0x9;
808     break;
809   case X86II::XOPA:
810     VEX_5M = 0xA;
811     break;
812   }
813 
814   // VEX_4V (VEX vvvv field): a register specifier
815   // (in 1's complement form) or 1111 if unused.
816   uint8_t VEX_4V = 0xf;
817   uint8_t EVEX_V2 = 0x1;
818 
819   // EVEX_L2/VEX_L (Vector Length):
820   //
821   // L2 L
822   //  0 0: scalar or 128-bit vector
823   //  0 1: 256-bit vector
824   //  1 0: 512-bit vector
825   //
826   uint8_t VEX_L = (TSFlags & X86II::VEX_L) ? 1 : 0;
827   uint8_t EVEX_L2 = (TSFlags & X86II::EVEX_L2) ? 1 : 0;
828 
829   // VEX_PP: opcode extension providing equivalent
830   // functionality of a SIMD prefix
831   //
832   //  0b00: None
833   //  0b01: 66
834   //  0b10: F3
835   //  0b11: F2
836   //
837   uint8_t VEX_PP = 0;
838   switch (TSFlags & X86II::OpPrefixMask) {
839   case X86II::PD:
840     VEX_PP = 0x1;
841     break; // 66
842   case X86II::XS:
843     VEX_PP = 0x2;
844     break; // F3
845   case X86II::XD:
846     VEX_PP = 0x3;
847     break; // F2
848   }
849 
850   // EVEX_U
851   uint8_t EVEX_U = 1; // Always '1' so far
852 
853   // EVEX_z
854   uint8_t EVEX_z = (HasEVEX_K && (TSFlags & X86II::EVEX_Z)) ? 1 : 0;
855 
856   // EVEX_b
857   uint8_t EVEX_b = (TSFlags & X86II::EVEX_B) ? 1 : 0;
858 
859   // EVEX_rc
860   uint8_t EVEX_rc = 0;
861 
862   // EVEX_aaa
863   uint8_t EVEX_aaa = 0;
864 
865   bool EncodeRC = false;
866 
867   // Classify VEX_B, VEX_4V, VEX_R, VEX_X
868   unsigned NumOps = Desc.getNumOperands();
869   unsigned CurOp = X86II::getOperandBias(Desc);
870 
871   switch (TSFlags & X86II::FormMask) {
872   default:
873     llvm_unreachable("Unexpected form in emitVEXOpcodePrefix!");
874   case X86II::MRM_C0:
875   case X86II::RawFrm:
876   case X86II::PrefixByte:
877     break;
878   case X86II::MRMDestMemFSIB:
879   case X86II::MRMDestMem: {
880     // MRMDestMem instructions forms:
881     //  MemAddr, src1(ModR/M)
882     //  MemAddr, src1(VEX_4V), src2(ModR/M)
883     //  MemAddr, src1(ModR/M), imm8
884     //
885     unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
886     VEX_B = ~(BaseRegEnc >> 3) & 1;
887     unsigned IndexRegEnc =
888         getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
889     VEX_X = ~(IndexRegEnc >> 3) & 1;
890     if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV.
891       EVEX_V2 = ~(IndexRegEnc >> 4) & 1;
892 
893     CurOp += X86::AddrNumOperands;
894 
895     if (HasEVEX_K)
896       EVEX_aaa = getX86RegEncoding(MI, CurOp++);
897 
898     if (HasVEX_4V) {
899       unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
900       VEX_4V = ~VRegEnc & 0xf;
901       EVEX_V2 = ~(VRegEnc >> 4) & 1;
902     }
903 
904     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
905     VEX_R = ~(RegEnc >> 3) & 1;
906     EVEX_R2 = ~(RegEnc >> 4) & 1;
907     break;
908   }
909   case X86II::MRMSrcMemFSIB:
910   case X86II::MRMSrcMem: {
911     // MRMSrcMem instructions forms:
912     //  src1(ModR/M), MemAddr
913     //  src1(ModR/M), src2(VEX_4V), MemAddr
914     //  src1(ModR/M), MemAddr, imm8
915     //  src1(ModR/M), MemAddr, src2(Imm[7:4])
916     //
917     //  FMA4:
918     //  dst(ModR/M.reg), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4])
919     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
920     VEX_R = ~(RegEnc >> 3) & 1;
921     EVEX_R2 = ~(RegEnc >> 4) & 1;
922 
923     if (HasEVEX_K)
924       EVEX_aaa = getX86RegEncoding(MI, CurOp++);
925 
926     if (HasVEX_4V) {
927       unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
928       VEX_4V = ~VRegEnc & 0xf;
929       EVEX_V2 = ~(VRegEnc >> 4) & 1;
930     }
931 
932     unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
933     VEX_B = ~(BaseRegEnc >> 3) & 1;
934     unsigned IndexRegEnc =
935         getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
936     VEX_X = ~(IndexRegEnc >> 3) & 1;
937     if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV.
938       EVEX_V2 = ~(IndexRegEnc >> 4) & 1;
939 
940     break;
941   }
942   case X86II::MRMSrcMem4VOp3: {
943     // Instruction format for 4VOp3:
944     //   src1(ModR/M), MemAddr, src3(VEX_4V)
945     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
946     VEX_R = ~(RegEnc >> 3) & 1;
947 
948     unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
949     VEX_B = ~(BaseRegEnc >> 3) & 1;
950     unsigned IndexRegEnc =
951         getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
952     VEX_X = ~(IndexRegEnc >> 3) & 1;
953 
954     VEX_4V = ~getX86RegEncoding(MI, CurOp + X86::AddrNumOperands) & 0xf;
955     break;
956   }
957   case X86II::MRMSrcMemOp4: {
958     //  dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
959     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
960     VEX_R = ~(RegEnc >> 3) & 1;
961 
962     unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
963     VEX_4V = ~VRegEnc & 0xf;
964 
965     unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
966     VEX_B = ~(BaseRegEnc >> 3) & 1;
967     unsigned IndexRegEnc =
968         getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
969     VEX_X = ~(IndexRegEnc >> 3) & 1;
970     break;
971   }
972   case X86II::MRM0m:
973   case X86II::MRM1m:
974   case X86II::MRM2m:
975   case X86II::MRM3m:
976   case X86II::MRM4m:
977   case X86II::MRM5m:
978   case X86II::MRM6m:
979   case X86II::MRM7m: {
980     // MRM[0-9]m instructions forms:
981     //  MemAddr
982     //  src1(VEX_4V), MemAddr
983     if (HasVEX_4V) {
984       unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
985       VEX_4V = ~VRegEnc & 0xf;
986       EVEX_V2 = ~(VRegEnc >> 4) & 1;
987     }
988 
989     if (HasEVEX_K)
990       EVEX_aaa = getX86RegEncoding(MI, CurOp++);
991 
992     unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
993     VEX_B = ~(BaseRegEnc >> 3) & 1;
994     unsigned IndexRegEnc =
995         getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
996     VEX_X = ~(IndexRegEnc >> 3) & 1;
997     if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV.
998       EVEX_V2 = ~(IndexRegEnc >> 4) & 1;
999 
1000     break;
1001   }
1002   case X86II::MRMSrcReg: {
1003     // MRMSrcReg instructions forms:
1004     //  dst(ModR/M), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4])
1005     //  dst(ModR/M), src1(ModR/M)
1006     //  dst(ModR/M), src1(ModR/M), imm8
1007     //
1008     //  FMA4:
1009     //  dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
1010     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1011     VEX_R = ~(RegEnc >> 3) & 1;
1012     EVEX_R2 = ~(RegEnc >> 4) & 1;
1013 
1014     if (HasEVEX_K)
1015       EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1016 
1017     if (HasVEX_4V) {
1018       unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1019       VEX_4V = ~VRegEnc & 0xf;
1020       EVEX_V2 = ~(VRegEnc >> 4) & 1;
1021     }
1022 
1023     RegEnc = getX86RegEncoding(MI, CurOp++);
1024     VEX_B = ~(RegEnc >> 3) & 1;
1025     VEX_X = ~(RegEnc >> 4) & 1;
1026 
1027     if (EVEX_b) {
1028       if (HasEVEX_RC) {
1029         unsigned RcOperand = NumOps - 1;
1030         assert(RcOperand >= CurOp);
1031         EVEX_rc = MI.getOperand(RcOperand).getImm();
1032         assert(EVEX_rc <= 3 && "Invalid rounding control!");
1033       }
1034       EncodeRC = true;
1035     }
1036     break;
1037   }
1038   case X86II::MRMSrcReg4VOp3: {
1039     // Instruction format for 4VOp3:
1040     //   src1(ModR/M), src2(ModR/M), src3(VEX_4V)
1041     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1042     VEX_R = ~(RegEnc >> 3) & 1;
1043 
1044     RegEnc = getX86RegEncoding(MI, CurOp++);
1045     VEX_B = ~(RegEnc >> 3) & 1;
1046 
1047     VEX_4V = ~getX86RegEncoding(MI, CurOp++) & 0xf;
1048     break;
1049   }
1050   case X86II::MRMSrcRegOp4: {
1051     //  dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
1052     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1053     VEX_R = ~(RegEnc >> 3) & 1;
1054 
1055     unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1056     VEX_4V = ~VRegEnc & 0xf;
1057 
1058     // Skip second register source (encoded in Imm[7:4])
1059     ++CurOp;
1060 
1061     RegEnc = getX86RegEncoding(MI, CurOp++);
1062     VEX_B = ~(RegEnc >> 3) & 1;
1063     VEX_X = ~(RegEnc >> 4) & 1;
1064     break;
1065   }
1066   case X86II::MRMDestReg: {
1067     // MRMDestReg instructions forms:
1068     //  dst(ModR/M), src(ModR/M)
1069     //  dst(ModR/M), src(ModR/M), imm8
1070     //  dst(ModR/M), src1(VEX_4V), src2(ModR/M)
1071     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1072     VEX_B = ~(RegEnc >> 3) & 1;
1073     VEX_X = ~(RegEnc >> 4) & 1;
1074 
1075     if (HasEVEX_K)
1076       EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1077 
1078     if (HasVEX_4V) {
1079       unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1080       VEX_4V = ~VRegEnc & 0xf;
1081       EVEX_V2 = ~(VRegEnc >> 4) & 1;
1082     }
1083 
1084     RegEnc = getX86RegEncoding(MI, CurOp++);
1085     VEX_R = ~(RegEnc >> 3) & 1;
1086     EVEX_R2 = ~(RegEnc >> 4) & 1;
1087     if (EVEX_b)
1088       EncodeRC = true;
1089     break;
1090   }
1091   case X86II::MRMr0: {
1092     // MRMr0 instructions forms:
1093     //  11:rrr:000
1094     //  dst(ModR/M)
1095     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1096     VEX_R = ~(RegEnc >> 3) & 1;
1097     EVEX_R2 = ~(RegEnc >> 4) & 1;
1098     break;
1099   }
1100   case X86II::MRM0r:
1101   case X86II::MRM1r:
1102   case X86II::MRM2r:
1103   case X86II::MRM3r:
1104   case X86II::MRM4r:
1105   case X86II::MRM5r:
1106   case X86II::MRM6r:
1107   case X86II::MRM7r: {
1108     // MRM0r-MRM7r instructions forms:
1109     //  dst(VEX_4V), src(ModR/M), imm8
1110     if (HasVEX_4V) {
1111       unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1112       VEX_4V = ~VRegEnc & 0xf;
1113       EVEX_V2 = ~(VRegEnc >> 4) & 1;
1114     }
1115     if (HasEVEX_K)
1116       EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1117 
1118     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1119     VEX_B = ~(RegEnc >> 3) & 1;
1120     VEX_X = ~(RegEnc >> 4) & 1;
1121     break;
1122   }
1123   }
1124 
1125   if (Encoding == X86II::VEX || Encoding == X86II::XOP) {
1126     // VEX opcode prefix can have 2 or 3 bytes
1127     //
1128     //  3 bytes:
1129     //    +-----+ +--------------+ +-------------------+
1130     //    | C4h | | RXB | m-mmmm | | W | vvvv | L | pp |
1131     //    +-----+ +--------------+ +-------------------+
1132     //  2 bytes:
1133     //    +-----+ +-------------------+
1134     //    | C5h | | R | vvvv | L | pp |
1135     //    +-----+ +-------------------+
1136     //
1137     //  XOP uses a similar prefix:
1138     //    +-----+ +--------------+ +-------------------+
1139     //    | 8Fh | | RXB | m-mmmm | | W | vvvv | L | pp |
1140     //    +-----+ +--------------+ +-------------------+
1141     uint8_t LastByte = VEX_PP | (VEX_L << 2) | (VEX_4V << 3);
1142 
1143     // Can we use the 2 byte VEX prefix?
1144     if (!(MI.getFlags() & X86::IP_USE_VEX3) && Encoding == X86II::VEX &&
1145         VEX_B && VEX_X && !VEX_W && (VEX_5M == 1)) {
1146       emitByte(0xC5, OS);
1147       emitByte(LastByte | (VEX_R << 7), OS);
1148       return;
1149     }
1150 
1151     // 3 byte VEX prefix
1152     emitByte(Encoding == X86II::XOP ? 0x8F : 0xC4, OS);
1153     emitByte(VEX_R << 7 | VEX_X << 6 | VEX_B << 5 | VEX_5M, OS);
1154     emitByte(LastByte | (VEX_W << 7), OS);
1155   } else {
1156     assert(Encoding == X86II::EVEX && "unknown encoding!");
1157     // EVEX opcode prefix can have 4 bytes
1158     //
1159     // +-----+ +--------------+ +-------------------+ +------------------------+
1160     // | 62h | | RXBR' | 00mm | | W | vvvv | U | pp | | z | L'L | b | v' | aaa |
1161     // +-----+ +--------------+ +-------------------+ +------------------------+
1162     assert((VEX_5M & 0x3) == VEX_5M &&
1163            "More than 2 significant bits in VEX.m-mmmm fields for EVEX!");
1164 
1165     emitByte(0x62, OS);
1166     emitByte((VEX_R << 7) | (VEX_X << 6) | (VEX_B << 5) | (EVEX_R2 << 4) |
1167                  VEX_5M,
1168              OS);
1169     emitByte((VEX_W << 7) | (VEX_4V << 3) | (EVEX_U << 2) | VEX_PP, OS);
1170     if (EncodeRC)
1171       emitByte((EVEX_z << 7) | (EVEX_rc << 5) | (EVEX_b << 4) | (EVEX_V2 << 3) |
1172                    EVEX_aaa,
1173                OS);
1174     else
1175       emitByte((EVEX_z << 7) | (EVEX_L2 << 6) | (VEX_L << 5) | (EVEX_b << 4) |
1176                    (EVEX_V2 << 3) | EVEX_aaa,
1177                OS);
1178   }
1179 }
1180 
1181 /// Emit REX prefix which specifies
1182 ///   1) 64-bit instructions,
1183 ///   2) non-default operand size, and
1184 ///   3) use of X86-64 extended registers.
1185 ///
1186 /// \returns true if REX prefix is used, otherwise returns false.
1187 bool X86MCCodeEmitter::emitREXPrefix(int MemOperand, const MCInst &MI,
1188                                      raw_ostream &OS) const {
1189   uint8_t REX = [&, MemOperand]() {
1190     uint8_t REX = 0;
1191     bool UsesHighByteReg = false;
1192 
1193     const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
1194     uint64_t TSFlags = Desc.TSFlags;
1195 
1196     if (TSFlags & X86II::REX_W)
1197       REX |= 1 << 3; // set REX.W
1198 
1199     if (MI.getNumOperands() == 0)
1200       return REX;
1201 
1202     unsigned NumOps = MI.getNumOperands();
1203     unsigned CurOp = X86II::getOperandBias(Desc);
1204 
1205     // If it accesses SPL, BPL, SIL, or DIL, then it requires a 0x40 REX prefix.
1206     for (unsigned i = CurOp; i != NumOps; ++i) {
1207       const MCOperand &MO = MI.getOperand(i);
1208       if (!MO.isReg())
1209         continue;
1210       unsigned Reg = MO.getReg();
1211       if (Reg == X86::AH || Reg == X86::BH || Reg == X86::CH || Reg == X86::DH)
1212         UsesHighByteReg = true;
1213       if (X86II::isX86_64NonExtLowByteReg(Reg))
1214         // FIXME: The caller of determineREXPrefix slaps this prefix onto
1215         // anything that returns non-zero.
1216         REX |= 0x40; // REX fixed encoding prefix
1217     }
1218 
1219     switch (TSFlags & X86II::FormMask) {
1220     case X86II::AddRegFrm:
1221       REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1222       break;
1223     case X86II::MRMSrcReg:
1224     case X86II::MRMSrcRegCC:
1225       REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1226       REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1227       break;
1228     case X86II::MRMSrcMem:
1229     case X86II::MRMSrcMemCC:
1230       REX |= isREXExtendedReg(MI, CurOp++) << 2;                        // REX.R
1231       REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0;  // REX.B
1232       REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X
1233       CurOp += X86::AddrNumOperands;
1234       break;
1235     case X86II::MRMDestReg:
1236       REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1237       REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1238       break;
1239     case X86II::MRMDestMem:
1240       REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0;  // REX.B
1241       REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X
1242       CurOp += X86::AddrNumOperands;
1243       REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1244       break;
1245     case X86II::MRMXmCC:
1246     case X86II::MRMXm:
1247     case X86II::MRM0m:
1248     case X86II::MRM1m:
1249     case X86II::MRM2m:
1250     case X86II::MRM3m:
1251     case X86II::MRM4m:
1252     case X86II::MRM5m:
1253     case X86II::MRM6m:
1254     case X86II::MRM7m:
1255       REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0;  // REX.B
1256       REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X
1257       break;
1258     case X86II::MRMXrCC:
1259     case X86II::MRMXr:
1260     case X86II::MRM0r:
1261     case X86II::MRM1r:
1262     case X86II::MRM2r:
1263     case X86II::MRM3r:
1264     case X86II::MRM4r:
1265     case X86II::MRM5r:
1266     case X86II::MRM6r:
1267     case X86II::MRM7r:
1268       REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1269       break;
1270     case X86II::MRMr0:
1271       REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1272       break;
1273     case X86II::MRMDestMemFSIB:
1274       llvm_unreachable("FSIB format never need REX prefix!");
1275     }
1276     if (REX && UsesHighByteReg)
1277       report_fatal_error(
1278           "Cannot encode high byte register in REX-prefixed instruction");
1279     return REX;
1280   }();
1281 
1282   if (!REX)
1283     return false;
1284 
1285   emitByte(0x40 | REX, OS);
1286   return true;
1287 }
1288 
1289 /// Emit segment override opcode prefix as needed.
1290 void X86MCCodeEmitter::emitSegmentOverridePrefix(unsigned SegOperand,
1291                                                  const MCInst &MI,
1292                                                  raw_ostream &OS) const {
1293   // Check for explicit segment override on memory operand.
1294   if (unsigned Reg = MI.getOperand(SegOperand).getReg())
1295     emitByte(X86::getSegmentOverridePrefixForReg(Reg), OS);
1296 }
1297 
1298 /// Emit all instruction prefixes prior to the opcode.
1299 ///
1300 /// \param MemOperand the operand # of the start of a memory operand if present.
1301 /// If not present, it is -1.
1302 ///
1303 /// \returns true if REX prefix is used, otherwise returns false.
1304 bool X86MCCodeEmitter::emitOpcodePrefix(int MemOperand, const MCInst &MI,
1305                                         const MCSubtargetInfo &STI,
1306                                         raw_ostream &OS) const {
1307   const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
1308   uint64_t TSFlags = Desc.TSFlags;
1309 
1310   // Emit the operand size opcode prefix as needed.
1311   if ((TSFlags & X86II::OpSizeMask) ==
1312       (STI.hasFeature(X86::Mode16Bit) ? X86II::OpSize32 : X86II::OpSize16))
1313     emitByte(0x66, OS);
1314 
1315   // Emit the LOCK opcode prefix.
1316   if (TSFlags & X86II::LOCK || MI.getFlags() & X86::IP_HAS_LOCK)
1317     emitByte(0xF0, OS);
1318 
1319   // Emit the NOTRACK opcode prefix.
1320   if (TSFlags & X86II::NOTRACK || MI.getFlags() & X86::IP_HAS_NOTRACK)
1321     emitByte(0x3E, OS);
1322 
1323   switch (TSFlags & X86II::OpPrefixMask) {
1324   case X86II::PD: // 66
1325     emitByte(0x66, OS);
1326     break;
1327   case X86II::XS: // F3
1328     emitByte(0xF3, OS);
1329     break;
1330   case X86II::XD: // F2
1331     emitByte(0xF2, OS);
1332     break;
1333   }
1334 
1335   // Handle REX prefix.
1336   assert((STI.hasFeature(X86::Mode64Bit) || !(TSFlags & X86II::REX_W)) &&
1337          "REX.W requires 64bit mode.");
1338   bool HasREX = STI.hasFeature(X86::Mode64Bit)
1339                     ? emitREXPrefix(MemOperand, MI, OS)
1340                     : false;
1341 
1342   // 0x0F escape code must be emitted just before the opcode.
1343   switch (TSFlags & X86II::OpMapMask) {
1344   case X86II::TB:        // Two-byte opcode map
1345   case X86II::T8:        // 0F 38
1346   case X86II::TA:        // 0F 3A
1347   case X86II::ThreeDNow: // 0F 0F, second 0F emitted by caller.
1348     emitByte(0x0F, OS);
1349     break;
1350   }
1351 
1352   switch (TSFlags & X86II::OpMapMask) {
1353   case X86II::T8: // 0F 38
1354     emitByte(0x38, OS);
1355     break;
1356   case X86II::TA: // 0F 3A
1357     emitByte(0x3A, OS);
1358     break;
1359   }
1360 
1361   return HasREX;
1362 }
1363 
1364 void X86MCCodeEmitter::emitPrefix(const MCInst &MI, raw_ostream &OS,
1365                                   const MCSubtargetInfo &STI) const {
1366   unsigned Opcode = MI.getOpcode();
1367   const MCInstrDesc &Desc = MCII.get(Opcode);
1368   uint64_t TSFlags = Desc.TSFlags;
1369 
1370   // Pseudo instructions don't get encoded.
1371   if (X86II::isPseudo(TSFlags))
1372     return;
1373 
1374   unsigned CurOp = X86II::getOperandBias(Desc);
1375 
1376   emitPrefixImpl(CurOp, MI, STI, OS);
1377 }
1378 
1379 void X86MCCodeEmitter::encodeInstruction(const MCInst &MI, raw_ostream &OS,
1380                                          SmallVectorImpl<MCFixup> &Fixups,
1381                                          const MCSubtargetInfo &STI) const {
1382   unsigned Opcode = MI.getOpcode();
1383   const MCInstrDesc &Desc = MCII.get(Opcode);
1384   uint64_t TSFlags = Desc.TSFlags;
1385 
1386   // Pseudo instructions don't get encoded.
1387   if (X86II::isPseudo(TSFlags))
1388     return;
1389 
1390   unsigned NumOps = Desc.getNumOperands();
1391   unsigned CurOp = X86II::getOperandBias(Desc);
1392 
1393   uint64_t StartByte = OS.tell();
1394 
1395   bool HasREX = emitPrefixImpl(CurOp, MI, STI, OS);
1396 
1397   // It uses the VEX.VVVV field?
1398   bool HasVEX_4V = TSFlags & X86II::VEX_4V;
1399   bool HasVEX_I8Reg = (TSFlags & X86II::ImmMask) == X86II::Imm8Reg;
1400 
1401   // It uses the EVEX.aaa field?
1402   bool HasEVEX_K = TSFlags & X86II::EVEX_K;
1403   bool HasEVEX_RC = TSFlags & X86II::EVEX_RC;
1404 
1405   // Used if a register is encoded in 7:4 of immediate.
1406   unsigned I8RegNum = 0;
1407 
1408   uint8_t BaseOpcode = X86II::getBaseOpcodeFor(TSFlags);
1409 
1410   if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow)
1411     BaseOpcode = 0x0F; // Weird 3DNow! encoding.
1412 
1413   unsigned OpcodeOffset = 0;
1414 
1415   uint64_t Form = TSFlags & X86II::FormMask;
1416   switch (Form) {
1417   default:
1418     errs() << "FORM: " << Form << "\n";
1419     llvm_unreachable("Unknown FormMask value in X86MCCodeEmitter!");
1420   case X86II::Pseudo:
1421     llvm_unreachable("Pseudo instruction shouldn't be emitted");
1422   case X86II::RawFrmDstSrc:
1423   case X86II::RawFrmSrc:
1424   case X86II::RawFrmDst:
1425   case X86II::PrefixByte:
1426     emitByte(BaseOpcode, OS);
1427     break;
1428   case X86II::AddCCFrm: {
1429     // This will be added to the opcode in the fallthrough.
1430     OpcodeOffset = MI.getOperand(NumOps - 1).getImm();
1431     assert(OpcodeOffset < 16 && "Unexpected opcode offset!");
1432     --NumOps; // Drop the operand from the end.
1433     LLVM_FALLTHROUGH;
1434   case X86II::RawFrm:
1435     emitByte(BaseOpcode + OpcodeOffset, OS);
1436 
1437     if (!STI.hasFeature(X86::Mode64Bit) || !isPCRel32Branch(MI, MCII))
1438       break;
1439 
1440     const MCOperand &Op = MI.getOperand(CurOp++);
1441     emitImmediate(Op, MI.getLoc(), X86II::getSizeOfImm(TSFlags),
1442                   MCFixupKind(X86::reloc_branch_4byte_pcrel), StartByte, OS,
1443                   Fixups);
1444     break;
1445   }
1446   case X86II::RawFrmMemOffs:
1447     emitByte(BaseOpcode, OS);
1448     emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1449                   X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1450                   StartByte, OS, Fixups);
1451     ++CurOp; // skip segment operand
1452     break;
1453   case X86II::RawFrmImm8:
1454     emitByte(BaseOpcode, OS);
1455     emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1456                   X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1457                   StartByte, OS, Fixups);
1458     emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1, FK_Data_1, StartByte,
1459                   OS, Fixups);
1460     break;
1461   case X86II::RawFrmImm16:
1462     emitByte(BaseOpcode, OS);
1463     emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1464                   X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1465                   StartByte, OS, Fixups);
1466     emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 2, FK_Data_2, StartByte,
1467                   OS, Fixups);
1468     break;
1469 
1470   case X86II::AddRegFrm:
1471     emitByte(BaseOpcode + getX86RegNum(MI.getOperand(CurOp++)), OS);
1472     break;
1473 
1474   case X86II::MRMDestReg: {
1475     emitByte(BaseOpcode, OS);
1476     unsigned SrcRegNum = CurOp + 1;
1477 
1478     if (HasEVEX_K) // Skip writemask
1479       ++SrcRegNum;
1480 
1481     if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1482       ++SrcRegNum;
1483 
1484     emitRegModRMByte(MI.getOperand(CurOp),
1485                      getX86RegNum(MI.getOperand(SrcRegNum)), OS);
1486     CurOp = SrcRegNum + 1;
1487     break;
1488   }
1489   case X86II::MRMDestMemFSIB:
1490   case X86II::MRMDestMem: {
1491     emitByte(BaseOpcode, OS);
1492     unsigned SrcRegNum = CurOp + X86::AddrNumOperands;
1493 
1494     if (HasEVEX_K) // Skip writemask
1495       ++SrcRegNum;
1496 
1497     if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1498       ++SrcRegNum;
1499 
1500     bool ForceSIB = (Form == X86II::MRMDestMemFSIB);
1501     emitMemModRMByte(MI, CurOp, getX86RegNum(MI.getOperand(SrcRegNum)), TSFlags,
1502                      HasREX, StartByte, OS, Fixups, STI, ForceSIB);
1503     CurOp = SrcRegNum + 1;
1504     break;
1505   }
1506   case X86II::MRMSrcReg: {
1507     emitByte(BaseOpcode, OS);
1508     unsigned SrcRegNum = CurOp + 1;
1509 
1510     if (HasEVEX_K) // Skip writemask
1511       ++SrcRegNum;
1512 
1513     if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1514       ++SrcRegNum;
1515 
1516     emitRegModRMByte(MI.getOperand(SrcRegNum),
1517                      getX86RegNum(MI.getOperand(CurOp)), OS);
1518     CurOp = SrcRegNum + 1;
1519     if (HasVEX_I8Reg)
1520       I8RegNum = getX86RegEncoding(MI, CurOp++);
1521     // do not count the rounding control operand
1522     if (HasEVEX_RC)
1523       --NumOps;
1524     break;
1525   }
1526   case X86II::MRMSrcReg4VOp3: {
1527     emitByte(BaseOpcode, OS);
1528     unsigned SrcRegNum = CurOp + 1;
1529 
1530     emitRegModRMByte(MI.getOperand(SrcRegNum),
1531                      getX86RegNum(MI.getOperand(CurOp)), OS);
1532     CurOp = SrcRegNum + 1;
1533     ++CurOp; // Encoded in VEX.VVVV
1534     break;
1535   }
1536   case X86II::MRMSrcRegOp4: {
1537     emitByte(BaseOpcode, OS);
1538     unsigned SrcRegNum = CurOp + 1;
1539 
1540     // Skip 1st src (which is encoded in VEX_VVVV)
1541     ++SrcRegNum;
1542 
1543     // Capture 2nd src (which is encoded in Imm[7:4])
1544     assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg");
1545     I8RegNum = getX86RegEncoding(MI, SrcRegNum++);
1546 
1547     emitRegModRMByte(MI.getOperand(SrcRegNum),
1548                      getX86RegNum(MI.getOperand(CurOp)), OS);
1549     CurOp = SrcRegNum + 1;
1550     break;
1551   }
1552   case X86II::MRMSrcRegCC: {
1553     unsigned FirstOp = CurOp++;
1554     unsigned SecondOp = CurOp++;
1555 
1556     unsigned CC = MI.getOperand(CurOp++).getImm();
1557     emitByte(BaseOpcode + CC, OS);
1558 
1559     emitRegModRMByte(MI.getOperand(SecondOp),
1560                      getX86RegNum(MI.getOperand(FirstOp)), OS);
1561     break;
1562   }
1563   case X86II::MRMSrcMemFSIB:
1564   case X86II::MRMSrcMem: {
1565     unsigned FirstMemOp = CurOp + 1;
1566 
1567     if (HasEVEX_K) // Skip writemask
1568       ++FirstMemOp;
1569 
1570     if (HasVEX_4V)
1571       ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV).
1572 
1573     emitByte(BaseOpcode, OS);
1574 
1575     bool ForceSIB = (Form == X86II::MRMSrcMemFSIB);
1576     emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1577                      TSFlags, HasREX, StartByte, OS, Fixups, STI, ForceSIB);
1578     CurOp = FirstMemOp + X86::AddrNumOperands;
1579     if (HasVEX_I8Reg)
1580       I8RegNum = getX86RegEncoding(MI, CurOp++);
1581     break;
1582   }
1583   case X86II::MRMSrcMem4VOp3: {
1584     unsigned FirstMemOp = CurOp + 1;
1585 
1586     emitByte(BaseOpcode, OS);
1587 
1588     emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1589                      TSFlags, HasREX, StartByte, OS, Fixups, STI);
1590     CurOp = FirstMemOp + X86::AddrNumOperands;
1591     ++CurOp; // Encoded in VEX.VVVV.
1592     break;
1593   }
1594   case X86II::MRMSrcMemOp4: {
1595     unsigned FirstMemOp = CurOp + 1;
1596 
1597     ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV).
1598 
1599     // Capture second register source (encoded in Imm[7:4])
1600     assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg");
1601     I8RegNum = getX86RegEncoding(MI, FirstMemOp++);
1602 
1603     emitByte(BaseOpcode, OS);
1604 
1605     emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1606                      TSFlags, HasREX, StartByte, OS, Fixups, STI);
1607     CurOp = FirstMemOp + X86::AddrNumOperands;
1608     break;
1609   }
1610   case X86II::MRMSrcMemCC: {
1611     unsigned RegOp = CurOp++;
1612     unsigned FirstMemOp = CurOp;
1613     CurOp = FirstMemOp + X86::AddrNumOperands;
1614 
1615     unsigned CC = MI.getOperand(CurOp++).getImm();
1616     emitByte(BaseOpcode + CC, OS);
1617 
1618     emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(RegOp)),
1619                      TSFlags, HasREX, StartByte, OS, Fixups, STI);
1620     break;
1621   }
1622 
1623   case X86II::MRMXrCC: {
1624     unsigned RegOp = CurOp++;
1625 
1626     unsigned CC = MI.getOperand(CurOp++).getImm();
1627     emitByte(BaseOpcode + CC, OS);
1628     emitRegModRMByte(MI.getOperand(RegOp), 0, OS);
1629     break;
1630   }
1631 
1632   case X86II::MRMXr:
1633   case X86II::MRM0r:
1634   case X86II::MRM1r:
1635   case X86II::MRM2r:
1636   case X86II::MRM3r:
1637   case X86II::MRM4r:
1638   case X86II::MRM5r:
1639   case X86II::MRM6r:
1640   case X86II::MRM7r:
1641     if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV).
1642       ++CurOp;
1643     if (HasEVEX_K) // Skip writemask
1644       ++CurOp;
1645     emitByte(BaseOpcode, OS);
1646     emitRegModRMByte(MI.getOperand(CurOp++),
1647                      (Form == X86II::MRMXr) ? 0 : Form - X86II::MRM0r, OS);
1648     break;
1649   case X86II::MRMr0:
1650     emitByte(BaseOpcode, OS);
1651     emitByte(modRMByte(3, getX86RegNum(MI.getOperand(CurOp++)),0), OS);
1652     break;
1653 
1654   case X86II::MRMXmCC: {
1655     unsigned FirstMemOp = CurOp;
1656     CurOp = FirstMemOp + X86::AddrNumOperands;
1657 
1658     unsigned CC = MI.getOperand(CurOp++).getImm();
1659     emitByte(BaseOpcode + CC, OS);
1660 
1661     emitMemModRMByte(MI, FirstMemOp, 0, TSFlags, HasREX, StartByte, OS, Fixups,
1662                      STI);
1663     break;
1664   }
1665 
1666   case X86II::MRMXm:
1667   case X86II::MRM0m:
1668   case X86II::MRM1m:
1669   case X86II::MRM2m:
1670   case X86II::MRM3m:
1671   case X86II::MRM4m:
1672   case X86II::MRM5m:
1673   case X86II::MRM6m:
1674   case X86II::MRM7m:
1675     if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV).
1676       ++CurOp;
1677     if (HasEVEX_K) // Skip writemask
1678       ++CurOp;
1679     emitByte(BaseOpcode, OS);
1680     emitMemModRMByte(MI, CurOp,
1681                      (Form == X86II::MRMXm) ? 0 : Form - X86II::MRM0m, TSFlags,
1682                      HasREX, StartByte, OS, Fixups, STI);
1683     CurOp += X86::AddrNumOperands;
1684     break;
1685 
1686   case X86II::MRM0X:
1687   case X86II::MRM1X:
1688   case X86II::MRM2X:
1689   case X86II::MRM3X:
1690   case X86II::MRM4X:
1691   case X86II::MRM5X:
1692   case X86II::MRM6X:
1693   case X86II::MRM7X:
1694     emitByte(BaseOpcode, OS);
1695     emitByte(0xC0 + ((Form - X86II::MRM0X) << 3), OS);
1696     break;
1697 
1698   case X86II::MRM_C0:
1699   case X86II::MRM_C1:
1700   case X86II::MRM_C2:
1701   case X86II::MRM_C3:
1702   case X86II::MRM_C4:
1703   case X86II::MRM_C5:
1704   case X86II::MRM_C6:
1705   case X86II::MRM_C7:
1706   case X86II::MRM_C8:
1707   case X86II::MRM_C9:
1708   case X86II::MRM_CA:
1709   case X86II::MRM_CB:
1710   case X86II::MRM_CC:
1711   case X86II::MRM_CD:
1712   case X86II::MRM_CE:
1713   case X86II::MRM_CF:
1714   case X86II::MRM_D0:
1715   case X86II::MRM_D1:
1716   case X86II::MRM_D2:
1717   case X86II::MRM_D3:
1718   case X86II::MRM_D4:
1719   case X86II::MRM_D5:
1720   case X86II::MRM_D6:
1721   case X86II::MRM_D7:
1722   case X86II::MRM_D8:
1723   case X86II::MRM_D9:
1724   case X86II::MRM_DA:
1725   case X86II::MRM_DB:
1726   case X86II::MRM_DC:
1727   case X86II::MRM_DD:
1728   case X86II::MRM_DE:
1729   case X86II::MRM_DF:
1730   case X86II::MRM_E0:
1731   case X86II::MRM_E1:
1732   case X86II::MRM_E2:
1733   case X86II::MRM_E3:
1734   case X86II::MRM_E4:
1735   case X86II::MRM_E5:
1736   case X86II::MRM_E6:
1737   case X86II::MRM_E7:
1738   case X86II::MRM_E8:
1739   case X86II::MRM_E9:
1740   case X86II::MRM_EA:
1741   case X86II::MRM_EB:
1742   case X86II::MRM_EC:
1743   case X86II::MRM_ED:
1744   case X86II::MRM_EE:
1745   case X86II::MRM_EF:
1746   case X86II::MRM_F0:
1747   case X86II::MRM_F1:
1748   case X86II::MRM_F2:
1749   case X86II::MRM_F3:
1750   case X86II::MRM_F4:
1751   case X86II::MRM_F5:
1752   case X86II::MRM_F6:
1753   case X86II::MRM_F7:
1754   case X86II::MRM_F8:
1755   case X86II::MRM_F9:
1756   case X86II::MRM_FA:
1757   case X86II::MRM_FB:
1758   case X86II::MRM_FC:
1759   case X86II::MRM_FD:
1760   case X86II::MRM_FE:
1761   case X86II::MRM_FF:
1762     emitByte(BaseOpcode, OS);
1763     emitByte(0xC0 + Form - X86II::MRM_C0, OS);
1764     break;
1765   }
1766 
1767   if (HasVEX_I8Reg) {
1768     // The last source register of a 4 operand instruction in AVX is encoded
1769     // in bits[7:4] of a immediate byte.
1770     assert(I8RegNum < 16 && "Register encoding out of range");
1771     I8RegNum <<= 4;
1772     if (CurOp != NumOps) {
1773       unsigned Val = MI.getOperand(CurOp++).getImm();
1774       assert(Val < 16 && "Immediate operand value out of range");
1775       I8RegNum |= Val;
1776     }
1777     emitImmediate(MCOperand::createImm(I8RegNum), MI.getLoc(), 1, FK_Data_1,
1778                   StartByte, OS, Fixups);
1779   } else {
1780     // If there is a remaining operand, it must be a trailing immediate. Emit it
1781     // according to the right size for the instruction. Some instructions
1782     // (SSE4a extrq and insertq) have two trailing immediates.
1783     while (CurOp != NumOps && NumOps - CurOp <= 2) {
1784       emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1785                     X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1786                     StartByte, OS, Fixups);
1787     }
1788   }
1789 
1790   if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow)
1791     emitByte(X86II::getBaseOpcodeFor(TSFlags), OS);
1792 
1793   assert(OS.tell() - StartByte <= 15 &&
1794          "The size of instruction must be no longer than 15.");
1795 #ifndef NDEBUG
1796   // FIXME: Verify.
1797   if (/*!Desc.isVariadic() &&*/ CurOp != NumOps) {
1798     errs() << "Cannot encode all operands of: ";
1799     MI.dump();
1800     errs() << '\n';
1801     abort();
1802   }
1803 #endif
1804 }
1805 
1806 MCCodeEmitter *llvm::createX86MCCodeEmitter(const MCInstrInfo &MCII,
1807                                             const MCRegisterInfo &MRI,
1808                                             MCContext &Ctx) {
1809   return new X86MCCodeEmitter(MCII, Ctx);
1810 }
1811