1 //===-- X86MCCodeEmitter.cpp - Convert X86 code to machine code -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the X86MCCodeEmitter class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "MCTargetDesc/X86BaseInfo.h"
14 #include "MCTargetDesc/X86FixupKinds.h"
15 #include "MCTargetDesc/X86MCTargetDesc.h"
16 #include "llvm/ADT/SmallVector.h"
17 #include "llvm/MC/MCCodeEmitter.h"
18 #include "llvm/MC/MCContext.h"
19 #include "llvm/MC/MCExpr.h"
20 #include "llvm/MC/MCFixup.h"
21 #include "llvm/MC/MCInst.h"
22 #include "llvm/MC/MCInstrDesc.h"
23 #include "llvm/MC/MCInstrInfo.h"
24 #include "llvm/MC/MCRegisterInfo.h"
25 #include "llvm/MC/MCSubtargetInfo.h"
26 #include "llvm/MC/MCSymbol.h"
27 #include "llvm/Support/ErrorHandling.h"
28 #include "llvm/Support/raw_ostream.h"
29 #include <cassert>
30 #include <cstdint>
31 #include <cstdlib>
32 
33 using namespace llvm;
34 
35 #define DEBUG_TYPE "mccodeemitter"
36 
37 namespace {
38 
39 class X86MCCodeEmitter : public MCCodeEmitter {
40   const MCInstrInfo &MCII;
41   MCContext &Ctx;
42 
43 public:
44   X86MCCodeEmitter(const MCInstrInfo &mcii, MCContext &ctx)
45       : MCII(mcii), Ctx(ctx) {}
46   X86MCCodeEmitter(const X86MCCodeEmitter &) = delete;
47   X86MCCodeEmitter &operator=(const X86MCCodeEmitter &) = delete;
48   ~X86MCCodeEmitter() override = default;
49 
50   void emitPrefix(const MCInst &MI, raw_ostream &OS,
51                   const MCSubtargetInfo &STI) const override;
52 
53   void encodeInstruction(const MCInst &MI, raw_ostream &OS,
54                          SmallVectorImpl<MCFixup> &Fixups,
55                          const MCSubtargetInfo &STI) const override;
56 
57 private:
58   unsigned getX86RegNum(const MCOperand &MO) const;
59 
60   unsigned getX86RegEncoding(const MCInst &MI, unsigned OpNum) const;
61 
62   /// \param MI a single low-level machine instruction.
63   /// \param OpNum the operand #.
64   /// \returns true if the OpNumth operand of MI  require a bit to be set in
65   /// REX prefix.
66   bool isREXExtendedReg(const MCInst &MI, unsigned OpNum) const;
67 
68   void emitImmediate(const MCOperand &Disp, SMLoc Loc, unsigned ImmSize,
69                      MCFixupKind FixupKind, unsigned &CurByte, raw_ostream &OS,
70                      SmallVectorImpl<MCFixup> &Fixups, int ImmOffset = 0) const;
71 
72   void emitRegModRMByte(const MCOperand &ModRMReg, unsigned RegOpcodeFld,
73                         unsigned &CurByte, raw_ostream &OS) const;
74 
75   void emitSIBByte(unsigned SS, unsigned Index, unsigned Base,
76                    unsigned &CurByte, raw_ostream &OS) const;
77 
78   void emitMemModRMByte(const MCInst &MI, unsigned Op, unsigned RegOpcodeField,
79                         uint64_t TSFlags, bool HasREX, unsigned &CurByte,
80                         raw_ostream &OS, SmallVectorImpl<MCFixup> &Fixups,
81                         const MCSubtargetInfo &STI) const;
82 
83   bool emitPrefixImpl(unsigned &CurOp, unsigned &CurByte, const MCInst &MI,
84                       const MCSubtargetInfo &STI, raw_ostream &OS) const;
85 
86   void emitVEXOpcodePrefix(unsigned &CurByte, int MemOperand, const MCInst &MI,
87                            raw_ostream &OS) const;
88 
89   void emitSegmentOverridePrefix(unsigned &CurByte, unsigned SegOperand,
90                                  const MCInst &MI, raw_ostream &OS) const;
91 
92   bool emitOpcodePrefix(unsigned &CurByte, int MemOperand, const MCInst &MI,
93                         const MCSubtargetInfo &STI, raw_ostream &OS) const;
94 
95   bool emitREXPrefix(unsigned &CurByte, int MemOperand, const MCInst &MI,
96                      raw_ostream &OS) const;
97 };
98 
99 } // end anonymous namespace
100 
101 static uint8_t modRMByte(unsigned Mod, unsigned RegOpcode, unsigned RM) {
102   assert(Mod < 4 && RegOpcode < 8 && RM < 8 && "ModRM Fields out of range!");
103   return RM | (RegOpcode << 3) | (Mod << 6);
104 }
105 
106 static void emitByte(uint8_t C, unsigned &CurByte, raw_ostream &OS) {
107   OS << static_cast<char>(C);
108   ++CurByte;
109 }
110 
111 static void emitConstant(uint64_t Val, unsigned Size, unsigned &CurByte,
112                          raw_ostream &OS) {
113   // Output the constant in little endian byte order.
114   for (unsigned i = 0; i != Size; ++i) {
115     emitByte(Val & 255, CurByte, OS);
116     Val >>= 8;
117   }
118 }
119 
120 /// \returns true if this signed displacement fits in a 8-bit sign-extended
121 /// field.
122 static bool isDisp8(int Value) { return Value == (int8_t)Value; }
123 
124 /// \returns true if this signed displacement fits in a 8-bit compressed
125 /// dispacement field.
126 static bool isCDisp8(uint64_t TSFlags, int Value, int &CValue) {
127   assert(((TSFlags & X86II::EncodingMask) == X86II::EVEX) &&
128          "Compressed 8-bit displacement is only valid for EVEX inst.");
129 
130   unsigned CD8_Scale =
131       (TSFlags & X86II::CD8_Scale_Mask) >> X86II::CD8_Scale_Shift;
132   if (CD8_Scale == 0) {
133     CValue = Value;
134     return isDisp8(Value);
135   }
136 
137   unsigned Mask = CD8_Scale - 1;
138   assert((CD8_Scale & Mask) == 0 && "Invalid memory object size.");
139   if (Value & Mask) // Unaligned offset
140     return false;
141   Value /= (int)CD8_Scale;
142   bool Ret = (Value == (int8_t)Value);
143 
144   if (Ret)
145     CValue = Value;
146   return Ret;
147 }
148 
149 /// \returns the appropriate fixup kind to use for an immediate in an
150 /// instruction with the specified TSFlags.
151 static MCFixupKind getImmFixupKind(uint64_t TSFlags) {
152   unsigned Size = X86II::getSizeOfImm(TSFlags);
153   bool isPCRel = X86II::isImmPCRel(TSFlags);
154 
155   if (X86II::isImmSigned(TSFlags)) {
156     switch (Size) {
157     default:
158       llvm_unreachable("Unsupported signed fixup size!");
159     case 4:
160       return MCFixupKind(X86::reloc_signed_4byte);
161     }
162   }
163   return MCFixup::getKindForSize(Size, isPCRel);
164 }
165 
166 /// \param Op operand # of the memory operand.
167 ///
168 /// \returns true if the specified instruction has a 16-bit memory operand.
169 static bool is16BitMemOperand(const MCInst &MI, unsigned Op,
170                               const MCSubtargetInfo &STI) {
171   const MCOperand &BaseReg = MI.getOperand(Op + X86::AddrBaseReg);
172   const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
173   const MCOperand &Disp = MI.getOperand(Op + X86::AddrDisp);
174 
175   if (STI.hasFeature(X86::Mode16Bit) && BaseReg.getReg() == 0 && Disp.isImm() &&
176       Disp.getImm() < 0x10000)
177     return true;
178   if ((BaseReg.getReg() != 0 &&
179        X86MCRegisterClasses[X86::GR16RegClassID].contains(BaseReg.getReg())) ||
180       (IndexReg.getReg() != 0 &&
181        X86MCRegisterClasses[X86::GR16RegClassID].contains(IndexReg.getReg())))
182     return true;
183   return false;
184 }
185 
186 /// \param Op operand # of the memory operand.
187 ///
188 /// \returns true if the specified instruction has a 32-bit memory operand.
189 static bool is32BitMemOperand(const MCInst &MI, unsigned Op) {
190   const MCOperand &BaseReg = MI.getOperand(Op + X86::AddrBaseReg);
191   const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
192 
193   if ((BaseReg.getReg() != 0 &&
194        X86MCRegisterClasses[X86::GR32RegClassID].contains(BaseReg.getReg())) ||
195       (IndexReg.getReg() != 0 &&
196        X86MCRegisterClasses[X86::GR32RegClassID].contains(IndexReg.getReg())))
197     return true;
198   if (BaseReg.getReg() == X86::EIP) {
199     assert(IndexReg.getReg() == 0 && "Invalid eip-based address.");
200     return true;
201   }
202   if (IndexReg.getReg() == X86::EIZ)
203     return true;
204   return false;
205 }
206 
207 /// \param Op operand # of the memory operand.
208 ///
209 /// \returns true if the specified instruction has a 64-bit memory operand.
210 #ifndef NDEBUG
211 static bool is64BitMemOperand(const MCInst &MI, unsigned Op) {
212   const MCOperand &BaseReg = MI.getOperand(Op + X86::AddrBaseReg);
213   const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
214 
215   if ((BaseReg.getReg() != 0 &&
216        X86MCRegisterClasses[X86::GR64RegClassID].contains(BaseReg.getReg())) ||
217       (IndexReg.getReg() != 0 &&
218        X86MCRegisterClasses[X86::GR64RegClassID].contains(IndexReg.getReg())))
219     return true;
220   return false;
221 }
222 #endif
223 
224 enum GlobalOffsetTableExprKind { GOT_None, GOT_Normal, GOT_SymDiff };
225 
226 /// Check if this expression starts with  _GLOBAL_OFFSET_TABLE_ and if it is
227 /// of the form _GLOBAL_OFFSET_TABLE_-symbol. This is needed to support PIC on
228 /// ELF i386 as _GLOBAL_OFFSET_TABLE_ is magical. We check only simple case that
229 /// are know to be used: _GLOBAL_OFFSET_TABLE_ by itself or at the start of a
230 /// binary expression.
231 static GlobalOffsetTableExprKind
232 startsWithGlobalOffsetTable(const MCExpr *Expr) {
233   const MCExpr *RHS = nullptr;
234   if (Expr->getKind() == MCExpr::Binary) {
235     const MCBinaryExpr *BE = static_cast<const MCBinaryExpr *>(Expr);
236     Expr = BE->getLHS();
237     RHS = BE->getRHS();
238   }
239 
240   if (Expr->getKind() != MCExpr::SymbolRef)
241     return GOT_None;
242 
243   const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr *>(Expr);
244   const MCSymbol &S = Ref->getSymbol();
245   if (S.getName() != "_GLOBAL_OFFSET_TABLE_")
246     return GOT_None;
247   if (RHS && RHS->getKind() == MCExpr::SymbolRef)
248     return GOT_SymDiff;
249   return GOT_Normal;
250 }
251 
252 static bool hasSecRelSymbolRef(const MCExpr *Expr) {
253   if (Expr->getKind() == MCExpr::SymbolRef) {
254     const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr *>(Expr);
255     return Ref->getKind() == MCSymbolRefExpr::VK_SECREL;
256   }
257   return false;
258 }
259 
260 static bool isPCRel32Branch(const MCInst &MI, const MCInstrInfo &MCII) {
261   unsigned Opcode = MI.getOpcode();
262   const MCInstrDesc &Desc = MCII.get(Opcode);
263   if ((Opcode != X86::CALL64pcrel32 && Opcode != X86::JMP_4 &&
264        Opcode != X86::JCC_4) ||
265       getImmFixupKind(Desc.TSFlags) != FK_PCRel_4)
266     return false;
267 
268   unsigned CurOp = X86II::getOperandBias(Desc);
269   const MCOperand &Op = MI.getOperand(CurOp);
270   if (!Op.isExpr())
271     return false;
272 
273   const MCSymbolRefExpr *Ref = dyn_cast<MCSymbolRefExpr>(Op.getExpr());
274   return Ref && Ref->getKind() == MCSymbolRefExpr::VK_None;
275 }
276 
277 unsigned X86MCCodeEmitter::getX86RegNum(const MCOperand &MO) const {
278   return Ctx.getRegisterInfo()->getEncodingValue(MO.getReg()) & 0x7;
279 }
280 
281 unsigned X86MCCodeEmitter::getX86RegEncoding(const MCInst &MI,
282                                              unsigned OpNum) const {
283   return Ctx.getRegisterInfo()->getEncodingValue(MI.getOperand(OpNum).getReg());
284 }
285 
286 /// \param MI a single low-level machine instruction.
287 /// \param OpNum the operand #.
288 /// \returns true if the OpNumth operand of MI  require a bit to be set in
289 /// REX prefix.
290 bool X86MCCodeEmitter::isREXExtendedReg(const MCInst &MI,
291                                         unsigned OpNum) const {
292   return (getX86RegEncoding(MI, OpNum) >> 3) & 1;
293 }
294 
295 void X86MCCodeEmitter::emitImmediate(const MCOperand &DispOp, SMLoc Loc,
296                                      unsigned Size, MCFixupKind FixupKind,
297                                      unsigned &CurByte, raw_ostream &OS,
298                                      SmallVectorImpl<MCFixup> &Fixups,
299                                      int ImmOffset) const {
300   const MCExpr *Expr = nullptr;
301   if (DispOp.isImm()) {
302     // If this is a simple integer displacement that doesn't require a
303     // relocation, emit it now.
304     if (FixupKind != FK_PCRel_1 && FixupKind != FK_PCRel_2 &&
305         FixupKind != FK_PCRel_4) {
306       emitConstant(DispOp.getImm() + ImmOffset, Size, CurByte, OS);
307       return;
308     }
309     Expr = MCConstantExpr::create(DispOp.getImm(), Ctx);
310   } else {
311     Expr = DispOp.getExpr();
312   }
313 
314   // If we have an immoffset, add it to the expression.
315   if ((FixupKind == FK_Data_4 || FixupKind == FK_Data_8 ||
316        FixupKind == MCFixupKind(X86::reloc_signed_4byte))) {
317     GlobalOffsetTableExprKind Kind = startsWithGlobalOffsetTable(Expr);
318     if (Kind != GOT_None) {
319       assert(ImmOffset == 0);
320 
321       if (Size == 8) {
322         FixupKind = MCFixupKind(X86::reloc_global_offset_table8);
323       } else {
324         assert(Size == 4);
325         FixupKind = MCFixupKind(X86::reloc_global_offset_table);
326       }
327 
328       if (Kind == GOT_Normal)
329         ImmOffset = CurByte;
330     } else if (Expr->getKind() == MCExpr::SymbolRef) {
331       if (hasSecRelSymbolRef(Expr)) {
332         FixupKind = MCFixupKind(FK_SecRel_4);
333       }
334     } else if (Expr->getKind() == MCExpr::Binary) {
335       const MCBinaryExpr *Bin = static_cast<const MCBinaryExpr *>(Expr);
336       if (hasSecRelSymbolRef(Bin->getLHS()) ||
337           hasSecRelSymbolRef(Bin->getRHS())) {
338         FixupKind = MCFixupKind(FK_SecRel_4);
339       }
340     }
341   }
342 
343   // If the fixup is pc-relative, we need to bias the value to be relative to
344   // the start of the field, not the end of the field.
345   if (FixupKind == FK_PCRel_4 ||
346       FixupKind == MCFixupKind(X86::reloc_riprel_4byte) ||
347       FixupKind == MCFixupKind(X86::reloc_riprel_4byte_movq_load) ||
348       FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax) ||
349       FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax_rex) ||
350       FixupKind == MCFixupKind(X86::reloc_branch_4byte_pcrel)) {
351     ImmOffset -= 4;
352     // If this is a pc-relative load off _GLOBAL_OFFSET_TABLE_:
353     // leaq _GLOBAL_OFFSET_TABLE_(%rip), %r15
354     // this needs to be a GOTPC32 relocation.
355     if (startsWithGlobalOffsetTable(Expr) != GOT_None)
356       FixupKind = MCFixupKind(X86::reloc_global_offset_table);
357   }
358   if (FixupKind == FK_PCRel_2)
359     ImmOffset -= 2;
360   if (FixupKind == FK_PCRel_1)
361     ImmOffset -= 1;
362 
363   if (ImmOffset)
364     Expr = MCBinaryExpr::createAdd(Expr, MCConstantExpr::create(ImmOffset, Ctx),
365                                    Ctx);
366 
367   // Emit a symbolic constant as a fixup and 4 zeros.
368   Fixups.push_back(MCFixup::create(CurByte, Expr, FixupKind, Loc));
369   emitConstant(0, Size, CurByte, OS);
370 }
371 
372 void X86MCCodeEmitter::emitRegModRMByte(const MCOperand &ModRMReg,
373                                         unsigned RegOpcodeFld,
374                                         unsigned &CurByte,
375                                         raw_ostream &OS) const {
376   emitByte(modRMByte(3, RegOpcodeFld, getX86RegNum(ModRMReg)), CurByte, OS);
377 }
378 
379 void X86MCCodeEmitter::emitSIBByte(unsigned SS, unsigned Index, unsigned Base,
380                                    unsigned &CurByte, raw_ostream &OS) const {
381   // SIB byte is in the same format as the modRMByte.
382   emitByte(modRMByte(SS, Index, Base), CurByte, OS);
383 }
384 
385 void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
386                                         unsigned RegOpcodeField,
387                                         uint64_t TSFlags, bool HasREX,
388                                         unsigned &CurByte, raw_ostream &OS,
389                                         SmallVectorImpl<MCFixup> &Fixups,
390                                         const MCSubtargetInfo &STI) const {
391   const MCOperand &Disp = MI.getOperand(Op + X86::AddrDisp);
392   const MCOperand &Base = MI.getOperand(Op + X86::AddrBaseReg);
393   const MCOperand &Scale = MI.getOperand(Op + X86::AddrScaleAmt);
394   const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
395   unsigned BaseReg = Base.getReg();
396   bool HasEVEX = (TSFlags & X86II::EncodingMask) == X86II::EVEX;
397 
398   // Handle %rip relative addressing.
399   if (BaseReg == X86::RIP ||
400       BaseReg == X86::EIP) { // [disp32+rIP] in X86-64 mode
401     assert(STI.hasFeature(X86::Mode64Bit) &&
402            "Rip-relative addressing requires 64-bit mode");
403     assert(IndexReg.getReg() == 0 && "Invalid rip-relative address");
404     emitByte(modRMByte(0, RegOpcodeField, 5), CurByte, OS);
405 
406     unsigned Opcode = MI.getOpcode();
407     // movq loads are handled with a special relocation form which allows the
408     // linker to eliminate some loads for GOT references which end up in the
409     // same linkage unit.
410     unsigned FixupKind = [=]() {
411       switch (Opcode) {
412       default:
413         return X86::reloc_riprel_4byte;
414       case X86::MOV64rm:
415         assert(HasREX);
416         return X86::reloc_riprel_4byte_movq_load;
417       case X86::CALL64m:
418       case X86::JMP64m:
419       case X86::TAILJMPm64:
420       case X86::TEST64mr:
421       case X86::ADC64rm:
422       case X86::ADD64rm:
423       case X86::AND64rm:
424       case X86::CMP64rm:
425       case X86::OR64rm:
426       case X86::SBB64rm:
427       case X86::SUB64rm:
428       case X86::XOR64rm:
429         return HasREX ? X86::reloc_riprel_4byte_relax_rex
430                       : X86::reloc_riprel_4byte_relax;
431       }
432     }();
433 
434     // rip-relative addressing is actually relative to the *next* instruction.
435     // Since an immediate can follow the mod/rm byte for an instruction, this
436     // means that we need to bias the displacement field of the instruction with
437     // the size of the immediate field. If we have this case, add it into the
438     // expression to emit.
439     // Note: rip-relative addressing using immediate displacement values should
440     // not be adjusted, assuming it was the user's intent.
441     int ImmSize = !Disp.isImm() && X86II::hasImm(TSFlags)
442                       ? X86II::getSizeOfImm(TSFlags)
443                       : 0;
444 
445     emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), CurByte, OS,
446                   Fixups, -ImmSize);
447     return;
448   }
449 
450   unsigned BaseRegNo = BaseReg ? getX86RegNum(Base) : -1U;
451 
452   // 16-bit addressing forms of the ModR/M byte have a different encoding for
453   // the R/M field and are far more limited in which registers can be used.
454   if (is16BitMemOperand(MI, Op, STI)) {
455     if (BaseReg) {
456       // For 32-bit addressing, the row and column values in Table 2-2 are
457       // basically the same. It's AX/CX/DX/BX/SP/BP/SI/DI in that order, with
458       // some special cases. And getX86RegNum reflects that numbering.
459       // For 16-bit addressing it's more fun, as shown in the SDM Vol 2A,
460       // Table 2-1 "16-Bit Addressing Forms with the ModR/M byte". We can only
461       // use SI/DI/BP/BX, which have "row" values 4-7 in no particular order,
462       // while values 0-3 indicate the allowed combinations (base+index) of
463       // those: 0 for BX+SI, 1 for BX+DI, 2 for BP+SI, 3 for BP+DI.
464       //
465       // R16Table[] is a lookup from the normal RegNo, to the row values from
466       // Table 2-1 for 16-bit addressing modes. Where zero means disallowed.
467       static const unsigned R16Table[] = {0, 0, 0, 7, 0, 6, 4, 5};
468       unsigned RMfield = R16Table[BaseRegNo];
469 
470       assert(RMfield && "invalid 16-bit base register");
471 
472       if (IndexReg.getReg()) {
473         unsigned IndexReg16 = R16Table[getX86RegNum(IndexReg)];
474 
475         assert(IndexReg16 && "invalid 16-bit index register");
476         // We must have one of SI/DI (4,5), and one of BP/BX (6,7).
477         assert(((IndexReg16 ^ RMfield) & 2) &&
478                "invalid 16-bit base/index register combination");
479         assert(Scale.getImm() == 1 &&
480                "invalid scale for 16-bit memory reference");
481 
482         // Allow base/index to appear in either order (although GAS doesn't).
483         if (IndexReg16 & 2)
484           RMfield = (RMfield & 1) | ((7 - IndexReg16) << 1);
485         else
486           RMfield = (IndexReg16 & 1) | ((7 - RMfield) << 1);
487       }
488 
489       if (Disp.isImm() && isDisp8(Disp.getImm())) {
490         if (Disp.getImm() == 0 && RMfield != 6) {
491           // There is no displacement; just the register.
492           emitByte(modRMByte(0, RegOpcodeField, RMfield), CurByte, OS);
493           return;
494         }
495         // Use the [REG]+disp8 form, including for [BP] which cannot be encoded.
496         emitByte(modRMByte(1, RegOpcodeField, RMfield), CurByte, OS);
497         emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, CurByte, OS, Fixups);
498         return;
499       }
500       // This is the [REG]+disp16 case.
501       emitByte(modRMByte(2, RegOpcodeField, RMfield), CurByte, OS);
502     } else {
503       // There is no BaseReg; this is the plain [disp16] case.
504       emitByte(modRMByte(0, RegOpcodeField, 6), CurByte, OS);
505     }
506 
507     // Emit 16-bit displacement for plain disp16 or [REG]+disp16 cases.
508     emitImmediate(Disp, MI.getLoc(), 2, FK_Data_2, CurByte, OS, Fixups);
509     return;
510   }
511 
512   // Determine whether a SIB byte is needed.
513   // If no BaseReg, issue a RIP relative instruction only if the MCE can
514   // resolve addresses on-the-fly, otherwise use SIB (Intel Manual 2A, table
515   // 2-7) and absolute references.
516 
517   if ( // The SIB byte must be used if there is an index register.
518       IndexReg.getReg() == 0 &&
519       // The SIB byte must be used if the base is ESP/RSP/R12, all of which
520       // encode to an R/M value of 4, which indicates that a SIB byte is
521       // present.
522       BaseRegNo != N86::ESP &&
523       // If there is no base register and we're in 64-bit mode, we need a SIB
524       // byte to emit an addr that is just 'disp32' (the non-RIP relative form).
525       (!STI.hasFeature(X86::Mode64Bit) || BaseReg != 0)) {
526 
527     if (BaseReg == 0) { // [disp32]     in X86-32 mode
528       emitByte(modRMByte(0, RegOpcodeField, 5), CurByte, OS);
529       emitImmediate(Disp, MI.getLoc(), 4, FK_Data_4, CurByte, OS, Fixups);
530       return;
531     }
532 
533     // If the base is not EBP/ESP and there is no displacement, use simple
534     // indirect register encoding, this handles addresses like [EAX].  The
535     // encoding for [EBP] with no displacement means [disp32] so we handle it
536     // by emitting a displacement of 0 below.
537     if (BaseRegNo != N86::EBP) {
538       if (Disp.isImm() && Disp.getImm() == 0) {
539         emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), CurByte, OS);
540         return;
541       }
542 
543       // If the displacement is @tlscall, treat it as a zero.
544       if (Disp.isExpr()) {
545         auto *Sym = dyn_cast<MCSymbolRefExpr>(Disp.getExpr());
546         if (Sym && Sym->getKind() == MCSymbolRefExpr::VK_TLSCALL) {
547           // This is exclusively used by call *a@tlscall(base). The relocation
548           // (R_386_TLSCALL or R_X86_64_TLSCALL) applies to the beginning.
549           Fixups.push_back(MCFixup::create(0, Sym, FK_NONE, MI.getLoc()));
550           emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), CurByte, OS);
551           return;
552         }
553       }
554     }
555 
556     // Otherwise, if the displacement fits in a byte, encode as [REG+disp8].
557     if (Disp.isImm()) {
558       if (!HasEVEX && isDisp8(Disp.getImm())) {
559         emitByte(modRMByte(1, RegOpcodeField, BaseRegNo), CurByte, OS);
560         emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, CurByte, OS, Fixups);
561         return;
562       }
563       // Try EVEX compressed 8-bit displacement first; if failed, fall back to
564       // 32-bit displacement.
565       int CDisp8 = 0;
566       if (HasEVEX && isCDisp8(TSFlags, Disp.getImm(), CDisp8)) {
567         emitByte(modRMByte(1, RegOpcodeField, BaseRegNo), CurByte, OS);
568         emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, CurByte, OS, Fixups,
569                       CDisp8 - Disp.getImm());
570         return;
571       }
572     }
573 
574     // Otherwise, emit the most general non-SIB encoding: [REG+disp32]
575     emitByte(modRMByte(2, RegOpcodeField, BaseRegNo), CurByte, OS);
576     unsigned Opcode = MI.getOpcode();
577     unsigned FixupKind = Opcode == X86::MOV32rm ? X86::reloc_signed_4byte_relax
578                                                 : X86::reloc_signed_4byte;
579     emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), CurByte, OS,
580                   Fixups);
581     return;
582   }
583 
584   // We need a SIB byte, so start by outputting the ModR/M byte first
585   assert(IndexReg.getReg() != X86::ESP && IndexReg.getReg() != X86::RSP &&
586          "Cannot use ESP as index reg!");
587 
588   bool ForceDisp32 = false;
589   bool ForceDisp8 = false;
590   int CDisp8 = 0;
591   int ImmOffset = 0;
592   if (BaseReg == 0) {
593     // If there is no base register, we emit the special case SIB byte with
594     // MOD=0, BASE=5, to JUST get the index, scale, and displacement.
595     emitByte(modRMByte(0, RegOpcodeField, 4), CurByte, OS);
596     ForceDisp32 = true;
597   } else if (!Disp.isImm()) {
598     // Emit the normal disp32 encoding.
599     emitByte(modRMByte(2, RegOpcodeField, 4), CurByte, OS);
600     ForceDisp32 = true;
601   } else if (Disp.getImm() == 0 &&
602              // Base reg can't be anything that ends up with '5' as the base
603              // reg, it is the magic [*] nomenclature that indicates no base.
604              BaseRegNo != N86::EBP) {
605     // Emit no displacement ModR/M byte
606     emitByte(modRMByte(0, RegOpcodeField, 4), CurByte, OS);
607   } else if (!HasEVEX && isDisp8(Disp.getImm())) {
608     // Emit the disp8 encoding.
609     emitByte(modRMByte(1, RegOpcodeField, 4), CurByte, OS);
610     ForceDisp8 = true; // Make sure to force 8 bit disp if Base=EBP
611   } else if (HasEVEX && isCDisp8(TSFlags, Disp.getImm(), CDisp8)) {
612     // Emit the disp8 encoding.
613     emitByte(modRMByte(1, RegOpcodeField, 4), CurByte, OS);
614     ForceDisp8 = true; // Make sure to force 8 bit disp if Base=EBP
615     ImmOffset = CDisp8 - Disp.getImm();
616   } else {
617     // Emit the normal disp32 encoding.
618     emitByte(modRMByte(2, RegOpcodeField, 4), CurByte, OS);
619   }
620 
621   // Calculate what the SS field value should be...
622   static const unsigned SSTable[] = {~0U, 0, 1, ~0U, 2, ~0U, ~0U, ~0U, 3};
623   unsigned SS = SSTable[Scale.getImm()];
624 
625   if (BaseReg == 0) {
626     // Handle the SIB byte for the case where there is no base, see Intel
627     // Manual 2A, table 2-7. The displacement has already been output.
628     unsigned IndexRegNo;
629     if (IndexReg.getReg())
630       IndexRegNo = getX86RegNum(IndexReg);
631     else // Examples: [ESP+1*<noreg>+4] or [scaled idx]+disp32 (MOD=0,BASE=5)
632       IndexRegNo = 4;
633     emitSIBByte(SS, IndexRegNo, 5, CurByte, OS);
634   } else {
635     unsigned IndexRegNo;
636     if (IndexReg.getReg())
637       IndexRegNo = getX86RegNum(IndexReg);
638     else
639       IndexRegNo = 4; // For example [ESP+1*<noreg>+4]
640     emitSIBByte(SS, IndexRegNo, getX86RegNum(Base), CurByte, OS);
641   }
642 
643   // Do we need to output a displacement?
644   if (ForceDisp8)
645     emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, CurByte, OS, Fixups,
646                   ImmOffset);
647   else if (ForceDisp32 || Disp.getImm() != 0)
648     emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(X86::reloc_signed_4byte),
649                   CurByte, OS, Fixups);
650 }
651 
652 /// Emit all instruction prefixes.
653 ///
654 /// \returns true if REX prefix is used, otherwise returns false.
655 bool X86MCCodeEmitter::emitPrefixImpl(unsigned &CurOp, unsigned &CurByte,
656                                       const MCInst &MI,
657                                       const MCSubtargetInfo &STI,
658                                       raw_ostream &OS) const {
659   uint64_t TSFlags = MCII.get(MI.getOpcode()).TSFlags;
660   // Determine where the memory operand starts, if present.
661   int MemoryOperand = X86II::getMemoryOperandNo(TSFlags);
662   // Emit segment override opcode prefix as needed.
663   if (MemoryOperand != -1) {
664     MemoryOperand += CurOp;
665     emitSegmentOverridePrefix(CurByte, MemoryOperand + X86::AddrSegmentReg, MI,
666                               OS);
667   }
668 
669   // Emit the repeat opcode prefix as needed.
670   unsigned Flags = MI.getFlags();
671   if (TSFlags & X86II::REP || Flags & X86::IP_HAS_REPEAT)
672     emitByte(0xF3, CurByte, OS);
673   if (Flags & X86::IP_HAS_REPEAT_NE)
674     emitByte(0xF2, CurByte, OS);
675 
676   // Emit the address size opcode prefix as needed.
677   bool NeedAddressOverride;
678   uint64_t AdSize = TSFlags & X86II::AdSizeMask;
679   if ((STI.hasFeature(X86::Mode16Bit) && AdSize == X86II::AdSize32) ||
680       (STI.hasFeature(X86::Mode32Bit) && AdSize == X86II::AdSize16) ||
681       (STI.hasFeature(X86::Mode64Bit) && AdSize == X86II::AdSize32)) {
682     NeedAddressOverride = true;
683   } else if (MemoryOperand < 0) {
684     NeedAddressOverride = false;
685   } else if (STI.hasFeature(X86::Mode64Bit)) {
686     assert(!is16BitMemOperand(MI, MemoryOperand, STI));
687     NeedAddressOverride = is32BitMemOperand(MI, MemoryOperand);
688   } else if (STI.hasFeature(X86::Mode32Bit)) {
689     assert(!is64BitMemOperand(MI, MemoryOperand));
690     NeedAddressOverride = is16BitMemOperand(MI, MemoryOperand, STI);
691   } else {
692     assert(STI.hasFeature(X86::Mode16Bit));
693     assert(!is64BitMemOperand(MI, MemoryOperand));
694     NeedAddressOverride = !is16BitMemOperand(MI, MemoryOperand, STI);
695   }
696 
697   if (NeedAddressOverride)
698     emitByte(0x67, CurByte, OS);
699 
700   // Encoding type for this instruction.
701   uint64_t Encoding = TSFlags & X86II::EncodingMask;
702   bool HasREX = false;
703   if (Encoding)
704     emitVEXOpcodePrefix(CurByte, MemoryOperand, MI, OS);
705   else
706     HasREX = emitOpcodePrefix(CurByte, MemoryOperand, MI, STI, OS);
707 
708   uint64_t Form = TSFlags & X86II::FormMask;
709   switch (Form) {
710   default:
711     break;
712   case X86II::RawFrmDstSrc: {
713     unsigned siReg = MI.getOperand(1).getReg();
714     assert(((siReg == X86::SI && MI.getOperand(0).getReg() == X86::DI) ||
715             (siReg == X86::ESI && MI.getOperand(0).getReg() == X86::EDI) ||
716             (siReg == X86::RSI && MI.getOperand(0).getReg() == X86::RDI)) &&
717            "SI and DI register sizes do not match");
718     // Emit segment override opcode prefix as needed (not for %ds).
719     if (MI.getOperand(2).getReg() != X86::DS)
720       emitSegmentOverridePrefix(CurByte, 2, MI, OS);
721     // Emit AdSize prefix as needed.
722     if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::ESI) ||
723         (STI.hasFeature(X86::Mode32Bit) && siReg == X86::SI))
724       emitByte(0x67, CurByte, OS);
725     CurOp += 3; // Consume operands.
726     break;
727   }
728   case X86II::RawFrmSrc: {
729     unsigned siReg = MI.getOperand(0).getReg();
730     // Emit segment override opcode prefix as needed (not for %ds).
731     if (MI.getOperand(1).getReg() != X86::DS)
732       emitSegmentOverridePrefix(CurByte, 1, MI, OS);
733     // Emit AdSize prefix as needed.
734     if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::ESI) ||
735         (STI.hasFeature(X86::Mode32Bit) && siReg == X86::SI))
736       emitByte(0x67, CurByte, OS);
737     CurOp += 2; // Consume operands.
738     break;
739   }
740   case X86II::RawFrmDst: {
741     unsigned siReg = MI.getOperand(0).getReg();
742     // Emit AdSize prefix as needed.
743     if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::EDI) ||
744         (STI.hasFeature(X86::Mode32Bit) && siReg == X86::DI))
745       emitByte(0x67, CurByte, OS);
746     ++CurOp; // Consume operand.
747     break;
748   }
749   case X86II::RawFrmMemOffs: {
750     // Emit segment override opcode prefix as needed.
751     emitSegmentOverridePrefix(CurByte, 1, MI, OS);
752     break;
753   }
754   }
755 
756   return HasREX;
757 }
758 
759 /// AVX instructions are encoded using a opcode prefix called VEX.
760 void X86MCCodeEmitter::emitVEXOpcodePrefix(unsigned &CurByte, int MemOperand,
761                                            const MCInst &MI,
762                                            raw_ostream &OS) const {
763   const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
764   uint64_t TSFlags = Desc.TSFlags;
765 
766   assert(!(TSFlags & X86II::LOCK) && "Can't have LOCK VEX.");
767 
768   uint64_t Encoding = TSFlags & X86II::EncodingMask;
769   bool HasEVEX_K = TSFlags & X86II::EVEX_K;
770   bool HasVEX_4V = TSFlags & X86II::VEX_4V;
771   bool HasEVEX_RC = TSFlags & X86II::EVEX_RC;
772 
773   // VEX_R: opcode externsion equivalent to REX.R in
774   // 1's complement (inverted) form
775   //
776   //  1: Same as REX_R=0 (must be 1 in 32-bit mode)
777   //  0: Same as REX_R=1 (64 bit mode only)
778   //
779   uint8_t VEX_R = 0x1;
780   uint8_t EVEX_R2 = 0x1;
781 
782   // VEX_X: equivalent to REX.X, only used when a
783   // register is used for index in SIB Byte.
784   //
785   //  1: Same as REX.X=0 (must be 1 in 32-bit mode)
786   //  0: Same as REX.X=1 (64-bit mode only)
787   uint8_t VEX_X = 0x1;
788 
789   // VEX_B:
790   //
791   //  1: Same as REX_B=0 (ignored in 32-bit mode)
792   //  0: Same as REX_B=1 (64 bit mode only)
793   //
794   uint8_t VEX_B = 0x1;
795 
796   // VEX_W: opcode specific (use like REX.W, or used for
797   // opcode extension, or ignored, depending on the opcode byte)
798   uint8_t VEX_W = (TSFlags & X86II::VEX_W) ? 1 : 0;
799 
800   // VEX_5M (VEX m-mmmmm field):
801   //
802   //  0b00000: Reserved for future use
803   //  0b00001: implied 0F leading opcode
804   //  0b00010: implied 0F 38 leading opcode bytes
805   //  0b00011: implied 0F 3A leading opcode bytes
806   //  0b00100-0b11111: Reserved for future use
807   //  0b01000: XOP map select - 08h instructions with imm byte
808   //  0b01001: XOP map select - 09h instructions with no imm byte
809   //  0b01010: XOP map select - 0Ah instructions with imm dword
810   uint8_t VEX_5M;
811   switch (TSFlags & X86II::OpMapMask) {
812   default:
813     llvm_unreachable("Invalid prefix!");
814   case X86II::TB:
815     VEX_5M = 0x1;
816     break; // 0F
817   case X86II::T8:
818     VEX_5M = 0x2;
819     break; // 0F 38
820   case X86II::TA:
821     VEX_5M = 0x3;
822     break; // 0F 3A
823   case X86II::XOP8:
824     VEX_5M = 0x8;
825     break;
826   case X86II::XOP9:
827     VEX_5M = 0x9;
828     break;
829   case X86II::XOPA:
830     VEX_5M = 0xA;
831     break;
832   }
833 
834   // VEX_4V (VEX vvvv field): a register specifier
835   // (in 1's complement form) or 1111 if unused.
836   uint8_t VEX_4V = 0xf;
837   uint8_t EVEX_V2 = 0x1;
838 
839   // EVEX_L2/VEX_L (Vector Length):
840   //
841   // L2 L
842   //  0 0: scalar or 128-bit vector
843   //  0 1: 256-bit vector
844   //  1 0: 512-bit vector
845   //
846   uint8_t VEX_L = (TSFlags & X86II::VEX_L) ? 1 : 0;
847   uint8_t EVEX_L2 = (TSFlags & X86II::EVEX_L2) ? 1 : 0;
848 
849   // VEX_PP: opcode extension providing equivalent
850   // functionality of a SIMD prefix
851   //
852   //  0b00: None
853   //  0b01: 66
854   //  0b10: F3
855   //  0b11: F2
856   //
857   uint8_t VEX_PP = 0;
858   switch (TSFlags & X86II::OpPrefixMask) {
859   case X86II::PD:
860     VEX_PP = 0x1;
861     break; // 66
862   case X86II::XS:
863     VEX_PP = 0x2;
864     break; // F3
865   case X86II::XD:
866     VEX_PP = 0x3;
867     break; // F2
868   }
869 
870   // EVEX_U
871   uint8_t EVEX_U = 1; // Always '1' so far
872 
873   // EVEX_z
874   uint8_t EVEX_z = (HasEVEX_K && (TSFlags & X86II::EVEX_Z)) ? 1 : 0;
875 
876   // EVEX_b
877   uint8_t EVEX_b = (TSFlags & X86II::EVEX_B) ? 1 : 0;
878 
879   // EVEX_rc
880   uint8_t EVEX_rc = 0;
881 
882   // EVEX_aaa
883   uint8_t EVEX_aaa = 0;
884 
885   bool EncodeRC = false;
886 
887   // Classify VEX_B, VEX_4V, VEX_R, VEX_X
888   unsigned NumOps = Desc.getNumOperands();
889   unsigned CurOp = X86II::getOperandBias(Desc);
890 
891   switch (TSFlags & X86II::FormMask) {
892   default:
893     llvm_unreachable("Unexpected form in emitVEXOpcodePrefix!");
894   case X86II::RawFrm:
895   case X86II::PrefixByte:
896     break;
897   case X86II::MRMDestMem: {
898     // MRMDestMem instructions forms:
899     //  MemAddr, src1(ModR/M)
900     //  MemAddr, src1(VEX_4V), src2(ModR/M)
901     //  MemAddr, src1(ModR/M), imm8
902     //
903     unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
904     VEX_B = ~(BaseRegEnc >> 3) & 1;
905     unsigned IndexRegEnc =
906         getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
907     VEX_X = ~(IndexRegEnc >> 3) & 1;
908     if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV.
909       EVEX_V2 = ~(IndexRegEnc >> 4) & 1;
910 
911     CurOp += X86::AddrNumOperands;
912 
913     if (HasEVEX_K)
914       EVEX_aaa = getX86RegEncoding(MI, CurOp++);
915 
916     if (HasVEX_4V) {
917       unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
918       VEX_4V = ~VRegEnc & 0xf;
919       EVEX_V2 = ~(VRegEnc >> 4) & 1;
920     }
921 
922     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
923     VEX_R = ~(RegEnc >> 3) & 1;
924     EVEX_R2 = ~(RegEnc >> 4) & 1;
925     break;
926   }
927   case X86II::MRMSrcMem: {
928     // MRMSrcMem instructions forms:
929     //  src1(ModR/M), MemAddr
930     //  src1(ModR/M), src2(VEX_4V), MemAddr
931     //  src1(ModR/M), MemAddr, imm8
932     //  src1(ModR/M), MemAddr, src2(Imm[7:4])
933     //
934     //  FMA4:
935     //  dst(ModR/M.reg), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4])
936     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
937     VEX_R = ~(RegEnc >> 3) & 1;
938     EVEX_R2 = ~(RegEnc >> 4) & 1;
939 
940     if (HasEVEX_K)
941       EVEX_aaa = getX86RegEncoding(MI, CurOp++);
942 
943     if (HasVEX_4V) {
944       unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
945       VEX_4V = ~VRegEnc & 0xf;
946       EVEX_V2 = ~(VRegEnc >> 4) & 1;
947     }
948 
949     unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
950     VEX_B = ~(BaseRegEnc >> 3) & 1;
951     unsigned IndexRegEnc =
952         getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
953     VEX_X = ~(IndexRegEnc >> 3) & 1;
954     if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV.
955       EVEX_V2 = ~(IndexRegEnc >> 4) & 1;
956 
957     break;
958   }
959   case X86II::MRMSrcMem4VOp3: {
960     // Instruction format for 4VOp3:
961     //   src1(ModR/M), MemAddr, src3(VEX_4V)
962     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
963     VEX_R = ~(RegEnc >> 3) & 1;
964 
965     unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
966     VEX_B = ~(BaseRegEnc >> 3) & 1;
967     unsigned IndexRegEnc =
968         getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
969     VEX_X = ~(IndexRegEnc >> 3) & 1;
970 
971     VEX_4V = ~getX86RegEncoding(MI, CurOp + X86::AddrNumOperands) & 0xf;
972     break;
973   }
974   case X86II::MRMSrcMemOp4: {
975     //  dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
976     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
977     VEX_R = ~(RegEnc >> 3) & 1;
978 
979     unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
980     VEX_4V = ~VRegEnc & 0xf;
981 
982     unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
983     VEX_B = ~(BaseRegEnc >> 3) & 1;
984     unsigned IndexRegEnc =
985         getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
986     VEX_X = ~(IndexRegEnc >> 3) & 1;
987     break;
988   }
989   case X86II::MRM0m:
990   case X86II::MRM1m:
991   case X86II::MRM2m:
992   case X86II::MRM3m:
993   case X86II::MRM4m:
994   case X86II::MRM5m:
995   case X86II::MRM6m:
996   case X86II::MRM7m: {
997     // MRM[0-9]m instructions forms:
998     //  MemAddr
999     //  src1(VEX_4V), MemAddr
1000     if (HasVEX_4V) {
1001       unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1002       VEX_4V = ~VRegEnc & 0xf;
1003       EVEX_V2 = ~(VRegEnc >> 4) & 1;
1004     }
1005 
1006     if (HasEVEX_K)
1007       EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1008 
1009     unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
1010     VEX_B = ~(BaseRegEnc >> 3) & 1;
1011     unsigned IndexRegEnc =
1012         getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
1013     VEX_X = ~(IndexRegEnc >> 3) & 1;
1014     if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV.
1015       EVEX_V2 = ~(IndexRegEnc >> 4) & 1;
1016 
1017     break;
1018   }
1019   case X86II::MRMSrcReg: {
1020     // MRMSrcReg instructions forms:
1021     //  dst(ModR/M), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4])
1022     //  dst(ModR/M), src1(ModR/M)
1023     //  dst(ModR/M), src1(ModR/M), imm8
1024     //
1025     //  FMA4:
1026     //  dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
1027     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1028     VEX_R = ~(RegEnc >> 3) & 1;
1029     EVEX_R2 = ~(RegEnc >> 4) & 1;
1030 
1031     if (HasEVEX_K)
1032       EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1033 
1034     if (HasVEX_4V) {
1035       unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1036       VEX_4V = ~VRegEnc & 0xf;
1037       EVEX_V2 = ~(VRegEnc >> 4) & 1;
1038     }
1039 
1040     RegEnc = getX86RegEncoding(MI, CurOp++);
1041     VEX_B = ~(RegEnc >> 3) & 1;
1042     VEX_X = ~(RegEnc >> 4) & 1;
1043 
1044     if (EVEX_b) {
1045       if (HasEVEX_RC) {
1046         unsigned RcOperand = NumOps - 1;
1047         assert(RcOperand >= CurOp);
1048         EVEX_rc = MI.getOperand(RcOperand).getImm();
1049         assert(EVEX_rc <= 3 && "Invalid rounding control!");
1050       }
1051       EncodeRC = true;
1052     }
1053     break;
1054   }
1055   case X86II::MRMSrcReg4VOp3: {
1056     // Instruction format for 4VOp3:
1057     //   src1(ModR/M), src2(ModR/M), src3(VEX_4V)
1058     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1059     VEX_R = ~(RegEnc >> 3) & 1;
1060 
1061     RegEnc = getX86RegEncoding(MI, CurOp++);
1062     VEX_B = ~(RegEnc >> 3) & 1;
1063 
1064     VEX_4V = ~getX86RegEncoding(MI, CurOp++) & 0xf;
1065     break;
1066   }
1067   case X86II::MRMSrcRegOp4: {
1068     //  dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
1069     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1070     VEX_R = ~(RegEnc >> 3) & 1;
1071 
1072     unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1073     VEX_4V = ~VRegEnc & 0xf;
1074 
1075     // Skip second register source (encoded in Imm[7:4])
1076     ++CurOp;
1077 
1078     RegEnc = getX86RegEncoding(MI, CurOp++);
1079     VEX_B = ~(RegEnc >> 3) & 1;
1080     VEX_X = ~(RegEnc >> 4) & 1;
1081     break;
1082   }
1083   case X86II::MRMDestReg: {
1084     // MRMDestReg instructions forms:
1085     //  dst(ModR/M), src(ModR/M)
1086     //  dst(ModR/M), src(ModR/M), imm8
1087     //  dst(ModR/M), src1(VEX_4V), src2(ModR/M)
1088     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1089     VEX_B = ~(RegEnc >> 3) & 1;
1090     VEX_X = ~(RegEnc >> 4) & 1;
1091 
1092     if (HasEVEX_K)
1093       EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1094 
1095     if (HasVEX_4V) {
1096       unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1097       VEX_4V = ~VRegEnc & 0xf;
1098       EVEX_V2 = ~(VRegEnc >> 4) & 1;
1099     }
1100 
1101     RegEnc = getX86RegEncoding(MI, CurOp++);
1102     VEX_R = ~(RegEnc >> 3) & 1;
1103     EVEX_R2 = ~(RegEnc >> 4) & 1;
1104     if (EVEX_b)
1105       EncodeRC = true;
1106     break;
1107   }
1108   case X86II::MRM0r:
1109   case X86II::MRM1r:
1110   case X86II::MRM2r:
1111   case X86II::MRM3r:
1112   case X86II::MRM4r:
1113   case X86II::MRM5r:
1114   case X86II::MRM6r:
1115   case X86II::MRM7r: {
1116     // MRM0r-MRM7r instructions forms:
1117     //  dst(VEX_4V), src(ModR/M), imm8
1118     if (HasVEX_4V) {
1119       unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1120       VEX_4V = ~VRegEnc & 0xf;
1121       EVEX_V2 = ~(VRegEnc >> 4) & 1;
1122     }
1123     if (HasEVEX_K)
1124       EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1125 
1126     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1127     VEX_B = ~(RegEnc >> 3) & 1;
1128     VEX_X = ~(RegEnc >> 4) & 1;
1129     break;
1130   }
1131   }
1132 
1133   if (Encoding == X86II::VEX || Encoding == X86II::XOP) {
1134     // VEX opcode prefix can have 2 or 3 bytes
1135     //
1136     //  3 bytes:
1137     //    +-----+ +--------------+ +-------------------+
1138     //    | C4h | | RXB | m-mmmm | | W | vvvv | L | pp |
1139     //    +-----+ +--------------+ +-------------------+
1140     //  2 bytes:
1141     //    +-----+ +-------------------+
1142     //    | C5h | | R | vvvv | L | pp |
1143     //    +-----+ +-------------------+
1144     //
1145     //  XOP uses a similar prefix:
1146     //    +-----+ +--------------+ +-------------------+
1147     //    | 8Fh | | RXB | m-mmmm | | W | vvvv | L | pp |
1148     //    +-----+ +--------------+ +-------------------+
1149     uint8_t LastByte = VEX_PP | (VEX_L << 2) | (VEX_4V << 3);
1150 
1151     // Can we use the 2 byte VEX prefix?
1152     if (!(MI.getFlags() & X86::IP_USE_VEX3) && Encoding == X86II::VEX &&
1153         VEX_B && VEX_X && !VEX_W && (VEX_5M == 1)) {
1154       emitByte(0xC5, CurByte, OS);
1155       emitByte(LastByte | (VEX_R << 7), CurByte, OS);
1156       return;
1157     }
1158 
1159     // 3 byte VEX prefix
1160     emitByte(Encoding == X86II::XOP ? 0x8F : 0xC4, CurByte, OS);
1161     emitByte(VEX_R << 7 | VEX_X << 6 | VEX_B << 5 | VEX_5M, CurByte, OS);
1162     emitByte(LastByte | (VEX_W << 7), CurByte, OS);
1163   } else {
1164     assert(Encoding == X86II::EVEX && "unknown encoding!");
1165     // EVEX opcode prefix can have 4 bytes
1166     //
1167     // +-----+ +--------------+ +-------------------+ +------------------------+
1168     // | 62h | | RXBR' | 00mm | | W | vvvv | U | pp | | z | L'L | b | v' | aaa |
1169     // +-----+ +--------------+ +-------------------+ +------------------------+
1170     assert((VEX_5M & 0x3) == VEX_5M &&
1171            "More than 2 significant bits in VEX.m-mmmm fields for EVEX!");
1172 
1173     emitByte(0x62, CurByte, OS);
1174     emitByte((VEX_R << 7) | (VEX_X << 6) | (VEX_B << 5) | (EVEX_R2 << 4) |
1175                  VEX_5M,
1176              CurByte, OS);
1177     emitByte((VEX_W << 7) | (VEX_4V << 3) | (EVEX_U << 2) | VEX_PP, CurByte,
1178              OS);
1179     if (EncodeRC)
1180       emitByte((EVEX_z << 7) | (EVEX_rc << 5) | (EVEX_b << 4) | (EVEX_V2 << 3) |
1181                    EVEX_aaa,
1182                CurByte, OS);
1183     else
1184       emitByte((EVEX_z << 7) | (EVEX_L2 << 6) | (VEX_L << 5) | (EVEX_b << 4) |
1185                    (EVEX_V2 << 3) | EVEX_aaa,
1186                CurByte, OS);
1187   }
1188 }
1189 
1190 /// Emit REX prefix which specifies
1191 ///   1) 64-bit instructions,
1192 ///   2) non-default operand size, and
1193 ///   3) use of X86-64 extended registers.
1194 ///
1195 /// \returns true if REX prefix is used, otherwise returns false.
1196 bool X86MCCodeEmitter::emitREXPrefix(unsigned &CurByte, int MemOperand,
1197                                      const MCInst &MI, raw_ostream &OS) const {
1198   uint8_t REX = [&, MemOperand]() {
1199     uint8_t REX = 0;
1200     bool UsesHighByteReg = false;
1201 
1202     const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
1203     uint64_t TSFlags = Desc.TSFlags;
1204 
1205     if (TSFlags & X86II::REX_W)
1206       REX |= 1 << 3; // set REX.W
1207 
1208     if (MI.getNumOperands() == 0)
1209       return REX;
1210 
1211     unsigned NumOps = MI.getNumOperands();
1212     unsigned CurOp = X86II::getOperandBias(Desc);
1213 
1214     // If it accesses SPL, BPL, SIL, or DIL, then it requires a 0x40 REX prefix.
1215     for (unsigned i = CurOp; i != NumOps; ++i) {
1216       const MCOperand &MO = MI.getOperand(i);
1217       if (!MO.isReg())
1218         continue;
1219       unsigned Reg = MO.getReg();
1220       if (Reg == X86::AH || Reg == X86::BH || Reg == X86::CH || Reg == X86::DH)
1221         UsesHighByteReg = true;
1222       if (X86II::isX86_64NonExtLowByteReg(Reg))
1223         // FIXME: The caller of determineREXPrefix slaps this prefix onto
1224         // anything that returns non-zero.
1225         REX |= 0x40; // REX fixed encoding prefix
1226     }
1227 
1228     switch (TSFlags & X86II::FormMask) {
1229     case X86II::AddRegFrm:
1230       REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1231       break;
1232     case X86II::MRMSrcReg:
1233     case X86II::MRMSrcRegCC:
1234       REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1235       REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1236       break;
1237     case X86II::MRMSrcMem:
1238     case X86II::MRMSrcMemCC:
1239       REX |= isREXExtendedReg(MI, CurOp++) << 2;                        // REX.R
1240       REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0;  // REX.B
1241       REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X
1242       CurOp += X86::AddrNumOperands;
1243       break;
1244     case X86II::MRMDestReg:
1245       REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1246       REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1247       break;
1248     case X86II::MRMDestMem:
1249       REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0;  // REX.B
1250       REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X
1251       CurOp += X86::AddrNumOperands;
1252       REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1253       break;
1254     case X86II::MRMXmCC:
1255     case X86II::MRMXm:
1256     case X86II::MRM0m:
1257     case X86II::MRM1m:
1258     case X86II::MRM2m:
1259     case X86II::MRM3m:
1260     case X86II::MRM4m:
1261     case X86II::MRM5m:
1262     case X86II::MRM6m:
1263     case X86II::MRM7m:
1264       REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0;  // REX.B
1265       REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X
1266       break;
1267     case X86II::MRMXrCC:
1268     case X86II::MRMXr:
1269     case X86II::MRM0r:
1270     case X86II::MRM1r:
1271     case X86II::MRM2r:
1272     case X86II::MRM3r:
1273     case X86II::MRM4r:
1274     case X86II::MRM5r:
1275     case X86II::MRM6r:
1276     case X86II::MRM7r:
1277       REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1278       break;
1279     }
1280     if (REX && UsesHighByteReg)
1281       report_fatal_error(
1282           "Cannot encode high byte register in REX-prefixed instruction");
1283     return REX;
1284   }();
1285 
1286   if (!REX)
1287     return false;
1288 
1289   emitByte(0x40 | REX, CurByte, OS);
1290   return true;
1291 }
1292 
1293 /// Emit segment override opcode prefix as needed.
1294 void X86MCCodeEmitter::emitSegmentOverridePrefix(unsigned &CurByte,
1295                                                  unsigned SegOperand,
1296                                                  const MCInst &MI,
1297                                                  raw_ostream &OS) const {
1298   // Check for explicit segment override on memory operand.
1299   if (unsigned Reg = MI.getOperand(SegOperand).getReg())
1300     emitByte(X86::getSegmentOverridePrefixForReg(Reg), CurByte, OS);
1301 }
1302 
1303 /// Emit all instruction prefixes prior to the opcode.
1304 ///
1305 /// \param MemOperand the operand # of the start of a memory operand if present.
1306 /// If not present, it is -1.
1307 ///
1308 /// \returns true if REX prefix is used, otherwise returns false.
1309 bool X86MCCodeEmitter::emitOpcodePrefix(unsigned &CurByte, int MemOperand,
1310                                         const MCInst &MI,
1311                                         const MCSubtargetInfo &STI,
1312                                         raw_ostream &OS) const {
1313   const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
1314   uint64_t TSFlags = Desc.TSFlags;
1315 
1316   // Emit the operand size opcode prefix as needed.
1317   if ((TSFlags & X86II::OpSizeMask) ==
1318       (STI.hasFeature(X86::Mode16Bit) ? X86II::OpSize32 : X86II::OpSize16))
1319     emitByte(0x66, CurByte, OS);
1320 
1321   // Emit the LOCK opcode prefix.
1322   if (TSFlags & X86II::LOCK || MI.getFlags() & X86::IP_HAS_LOCK)
1323     emitByte(0xF0, CurByte, OS);
1324 
1325   // Emit the NOTRACK opcode prefix.
1326   if (TSFlags & X86II::NOTRACK || MI.getFlags() & X86::IP_HAS_NOTRACK)
1327     emitByte(0x3E, CurByte, OS);
1328 
1329   switch (TSFlags & X86II::OpPrefixMask) {
1330   case X86II::PD: // 66
1331     emitByte(0x66, CurByte, OS);
1332     break;
1333   case X86II::XS: // F3
1334     emitByte(0xF3, CurByte, OS);
1335     break;
1336   case X86II::XD: // F2
1337     emitByte(0xF2, CurByte, OS);
1338     break;
1339   }
1340 
1341   // Handle REX prefix.
1342   assert((STI.hasFeature(X86::Mode64Bit) || !(TSFlags & X86II::REX_W)) &&
1343          "REX.W requires 64bit mode.");
1344   bool HasREX = STI.hasFeature(X86::Mode64Bit)
1345                     ? emitREXPrefix(CurByte, MemOperand, MI, OS)
1346                     : false;
1347 
1348   // 0x0F escape code must be emitted just before the opcode.
1349   switch (TSFlags & X86II::OpMapMask) {
1350   case X86II::TB:        // Two-byte opcode map
1351   case X86II::T8:        // 0F 38
1352   case X86II::TA:        // 0F 3A
1353   case X86II::ThreeDNow: // 0F 0F, second 0F emitted by caller.
1354     emitByte(0x0F, CurByte, OS);
1355     break;
1356   }
1357 
1358   switch (TSFlags & X86II::OpMapMask) {
1359   case X86II::T8: // 0F 38
1360     emitByte(0x38, CurByte, OS);
1361     break;
1362   case X86II::TA: // 0F 3A
1363     emitByte(0x3A, CurByte, OS);
1364     break;
1365   }
1366 
1367   return HasREX;
1368 }
1369 
1370 void X86MCCodeEmitter::emitPrefix(const MCInst &MI, raw_ostream &OS,
1371                                   const MCSubtargetInfo &STI) const {
1372   unsigned Opcode = MI.getOpcode();
1373   const MCInstrDesc &Desc = MCII.get(Opcode);
1374   uint64_t TSFlags = Desc.TSFlags;
1375 
1376   // Pseudo instructions don't get encoded.
1377   if (X86II::isPseudo(TSFlags))
1378     return;
1379 
1380   unsigned CurOp = X86II::getOperandBias(Desc);
1381 
1382   // Keep track of the current byte being emitted.
1383   unsigned CurByte = 0;
1384 
1385   emitPrefixImpl(CurOp, CurByte, MI, STI, OS);
1386 }
1387 
1388 void X86MCCodeEmitter::encodeInstruction(const MCInst &MI, raw_ostream &OS,
1389                                          SmallVectorImpl<MCFixup> &Fixups,
1390                                          const MCSubtargetInfo &STI) const {
1391   unsigned Opcode = MI.getOpcode();
1392   const MCInstrDesc &Desc = MCII.get(Opcode);
1393   uint64_t TSFlags = Desc.TSFlags;
1394 
1395   // Pseudo instructions don't get encoded.
1396   if (X86II::isPseudo(TSFlags))
1397     return;
1398 
1399   unsigned NumOps = Desc.getNumOperands();
1400   unsigned CurOp = X86II::getOperandBias(Desc);
1401 
1402   // Keep track of the current byte being emitted.
1403   unsigned CurByte = 0;
1404 
1405   bool HasREX = emitPrefixImpl(CurOp, CurByte, MI, STI, OS);
1406 
1407   // It uses the VEX.VVVV field?
1408   bool HasVEX_4V = TSFlags & X86II::VEX_4V;
1409   bool HasVEX_I8Reg = (TSFlags & X86II::ImmMask) == X86II::Imm8Reg;
1410 
1411   // It uses the EVEX.aaa field?
1412   bool HasEVEX_K = TSFlags & X86II::EVEX_K;
1413   bool HasEVEX_RC = TSFlags & X86II::EVEX_RC;
1414 
1415   // Used if a register is encoded in 7:4 of immediate.
1416   unsigned I8RegNum = 0;
1417 
1418   uint8_t BaseOpcode = X86II::getBaseOpcodeFor(TSFlags);
1419 
1420   if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow)
1421     BaseOpcode = 0x0F; // Weird 3DNow! encoding.
1422 
1423   unsigned OpcodeOffset = 0;
1424 
1425   uint64_t Form = TSFlags & X86II::FormMask;
1426   switch (Form) {
1427   default:
1428     errs() << "FORM: " << Form << "\n";
1429     llvm_unreachable("Unknown FormMask value in X86MCCodeEmitter!");
1430   case X86II::Pseudo:
1431     llvm_unreachable("Pseudo instruction shouldn't be emitted");
1432   case X86II::RawFrmDstSrc:
1433   case X86II::RawFrmSrc:
1434   case X86II::RawFrmDst:
1435   case X86II::PrefixByte:
1436     emitByte(BaseOpcode, CurByte, OS);
1437     break;
1438   case X86II::AddCCFrm: {
1439     // This will be added to the opcode in the fallthrough.
1440     OpcodeOffset = MI.getOperand(NumOps - 1).getImm();
1441     assert(OpcodeOffset < 16 && "Unexpected opcode offset!");
1442     --NumOps; // Drop the operand from the end.
1443     LLVM_FALLTHROUGH;
1444   case X86II::RawFrm:
1445     emitByte(BaseOpcode + OpcodeOffset, CurByte, OS);
1446 
1447     if (!STI.hasFeature(X86::Mode64Bit) || !isPCRel32Branch(MI, MCII))
1448       break;
1449 
1450     const MCOperand &Op = MI.getOperand(CurOp++);
1451     emitImmediate(Op, MI.getLoc(), X86II::getSizeOfImm(TSFlags),
1452                   MCFixupKind(X86::reloc_branch_4byte_pcrel), CurByte, OS,
1453                   Fixups);
1454     break;
1455   }
1456   case X86II::RawFrmMemOffs:
1457     emitByte(BaseOpcode, CurByte, OS);
1458     emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1459                   X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1460                   CurByte, OS, Fixups);
1461     ++CurOp; // skip segment operand
1462     break;
1463   case X86II::RawFrmImm8:
1464     emitByte(BaseOpcode, CurByte, OS);
1465     emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1466                   X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1467                   CurByte, OS, Fixups);
1468     emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1, FK_Data_1, CurByte,
1469                   OS, Fixups);
1470     break;
1471   case X86II::RawFrmImm16:
1472     emitByte(BaseOpcode, CurByte, OS);
1473     emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1474                   X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1475                   CurByte, OS, Fixups);
1476     emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 2, FK_Data_2, CurByte,
1477                   OS, Fixups);
1478     break;
1479 
1480   case X86II::AddRegFrm:
1481     emitByte(BaseOpcode + getX86RegNum(MI.getOperand(CurOp++)), CurByte, OS);
1482     break;
1483 
1484   case X86II::MRMDestReg: {
1485     emitByte(BaseOpcode, CurByte, OS);
1486     unsigned SrcRegNum = CurOp + 1;
1487 
1488     if (HasEVEX_K) // Skip writemask
1489       ++SrcRegNum;
1490 
1491     if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1492       ++SrcRegNum;
1493 
1494     emitRegModRMByte(MI.getOperand(CurOp),
1495                      getX86RegNum(MI.getOperand(SrcRegNum)), CurByte, OS);
1496     CurOp = SrcRegNum + 1;
1497     break;
1498   }
1499   case X86II::MRMDestMem: {
1500     emitByte(BaseOpcode, CurByte, OS);
1501     unsigned SrcRegNum = CurOp + X86::AddrNumOperands;
1502 
1503     if (HasEVEX_K) // Skip writemask
1504       ++SrcRegNum;
1505 
1506     if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1507       ++SrcRegNum;
1508 
1509     emitMemModRMByte(MI, CurOp, getX86RegNum(MI.getOperand(SrcRegNum)), TSFlags,
1510                      HasREX, CurByte, OS, Fixups, STI);
1511     CurOp = SrcRegNum + 1;
1512     break;
1513   }
1514   case X86II::MRMSrcReg: {
1515     emitByte(BaseOpcode, CurByte, OS);
1516     unsigned SrcRegNum = CurOp + 1;
1517 
1518     if (HasEVEX_K) // Skip writemask
1519       ++SrcRegNum;
1520 
1521     if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1522       ++SrcRegNum;
1523 
1524     emitRegModRMByte(MI.getOperand(SrcRegNum),
1525                      getX86RegNum(MI.getOperand(CurOp)), CurByte, OS);
1526     CurOp = SrcRegNum + 1;
1527     if (HasVEX_I8Reg)
1528       I8RegNum = getX86RegEncoding(MI, CurOp++);
1529     // do not count the rounding control operand
1530     if (HasEVEX_RC)
1531       --NumOps;
1532     break;
1533   }
1534   case X86II::MRMSrcReg4VOp3: {
1535     emitByte(BaseOpcode, CurByte, OS);
1536     unsigned SrcRegNum = CurOp + 1;
1537 
1538     emitRegModRMByte(MI.getOperand(SrcRegNum),
1539                      getX86RegNum(MI.getOperand(CurOp)), CurByte, OS);
1540     CurOp = SrcRegNum + 1;
1541     ++CurOp; // Encoded in VEX.VVVV
1542     break;
1543   }
1544   case X86II::MRMSrcRegOp4: {
1545     emitByte(BaseOpcode, CurByte, OS);
1546     unsigned SrcRegNum = CurOp + 1;
1547 
1548     // Skip 1st src (which is encoded in VEX_VVVV)
1549     ++SrcRegNum;
1550 
1551     // Capture 2nd src (which is encoded in Imm[7:4])
1552     assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg");
1553     I8RegNum = getX86RegEncoding(MI, SrcRegNum++);
1554 
1555     emitRegModRMByte(MI.getOperand(SrcRegNum),
1556                      getX86RegNum(MI.getOperand(CurOp)), CurByte, OS);
1557     CurOp = SrcRegNum + 1;
1558     break;
1559   }
1560   case X86II::MRMSrcRegCC: {
1561     unsigned FirstOp = CurOp++;
1562     unsigned SecondOp = CurOp++;
1563 
1564     unsigned CC = MI.getOperand(CurOp++).getImm();
1565     emitByte(BaseOpcode + CC, CurByte, OS);
1566 
1567     emitRegModRMByte(MI.getOperand(SecondOp),
1568                      getX86RegNum(MI.getOperand(FirstOp)), CurByte, OS);
1569     break;
1570   }
1571   case X86II::MRMSrcMem: {
1572     unsigned FirstMemOp = CurOp + 1;
1573 
1574     if (HasEVEX_K) // Skip writemask
1575       ++FirstMemOp;
1576 
1577     if (HasVEX_4V)
1578       ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV).
1579 
1580     emitByte(BaseOpcode, CurByte, OS);
1581 
1582     emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1583                      TSFlags, HasREX, CurByte, OS, Fixups, STI);
1584     CurOp = FirstMemOp + X86::AddrNumOperands;
1585     if (HasVEX_I8Reg)
1586       I8RegNum = getX86RegEncoding(MI, CurOp++);
1587     break;
1588   }
1589   case X86II::MRMSrcMem4VOp3: {
1590     unsigned FirstMemOp = CurOp + 1;
1591 
1592     emitByte(BaseOpcode, CurByte, OS);
1593 
1594     emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1595                      TSFlags, HasREX, CurByte, OS, Fixups, STI);
1596     CurOp = FirstMemOp + X86::AddrNumOperands;
1597     ++CurOp; // Encoded in VEX.VVVV.
1598     break;
1599   }
1600   case X86II::MRMSrcMemOp4: {
1601     unsigned FirstMemOp = CurOp + 1;
1602 
1603     ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV).
1604 
1605     // Capture second register source (encoded in Imm[7:4])
1606     assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg");
1607     I8RegNum = getX86RegEncoding(MI, FirstMemOp++);
1608 
1609     emitByte(BaseOpcode, CurByte, OS);
1610 
1611     emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1612                      TSFlags, HasREX, CurByte, OS, Fixups, STI);
1613     CurOp = FirstMemOp + X86::AddrNumOperands;
1614     break;
1615   }
1616   case X86II::MRMSrcMemCC: {
1617     unsigned RegOp = CurOp++;
1618     unsigned FirstMemOp = CurOp;
1619     CurOp = FirstMemOp + X86::AddrNumOperands;
1620 
1621     unsigned CC = MI.getOperand(CurOp++).getImm();
1622     emitByte(BaseOpcode + CC, CurByte, OS);
1623 
1624     emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(RegOp)),
1625                      TSFlags, HasREX, CurByte, OS, Fixups, STI);
1626     break;
1627   }
1628 
1629   case X86II::MRMXrCC: {
1630     unsigned RegOp = CurOp++;
1631 
1632     unsigned CC = MI.getOperand(CurOp++).getImm();
1633     emitByte(BaseOpcode + CC, CurByte, OS);
1634     emitRegModRMByte(MI.getOperand(RegOp), 0, CurByte, OS);
1635     break;
1636   }
1637 
1638   case X86II::MRMXr:
1639   case X86II::MRM0r:
1640   case X86II::MRM1r:
1641   case X86II::MRM2r:
1642   case X86II::MRM3r:
1643   case X86II::MRM4r:
1644   case X86II::MRM5r:
1645   case X86II::MRM6r:
1646   case X86II::MRM7r:
1647     if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV).
1648       ++CurOp;
1649     if (HasEVEX_K) // Skip writemask
1650       ++CurOp;
1651     emitByte(BaseOpcode, CurByte, OS);
1652     emitRegModRMByte(MI.getOperand(CurOp++),
1653                      (Form == X86II::MRMXr) ? 0 : Form - X86II::MRM0r, CurByte,
1654                      OS);
1655     break;
1656 
1657   case X86II::MRMXmCC: {
1658     unsigned FirstMemOp = CurOp;
1659     CurOp = FirstMemOp + X86::AddrNumOperands;
1660 
1661     unsigned CC = MI.getOperand(CurOp++).getImm();
1662     emitByte(BaseOpcode + CC, CurByte, OS);
1663 
1664     emitMemModRMByte(MI, FirstMemOp, 0, TSFlags, HasREX, CurByte, OS, Fixups,
1665                      STI);
1666     break;
1667   }
1668 
1669   case X86II::MRMXm:
1670   case X86II::MRM0m:
1671   case X86II::MRM1m:
1672   case X86II::MRM2m:
1673   case X86II::MRM3m:
1674   case X86II::MRM4m:
1675   case X86II::MRM5m:
1676   case X86II::MRM6m:
1677   case X86II::MRM7m:
1678     if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV).
1679       ++CurOp;
1680     if (HasEVEX_K) // Skip writemask
1681       ++CurOp;
1682     emitByte(BaseOpcode, CurByte, OS);
1683     emitMemModRMByte(MI, CurOp,
1684                      (Form == X86II::MRMXm) ? 0 : Form - X86II::MRM0m, TSFlags,
1685                      HasREX, CurByte, OS, Fixups, STI);
1686     CurOp += X86::AddrNumOperands;
1687     break;
1688 
1689   case X86II::MRM_C0:
1690   case X86II::MRM_C1:
1691   case X86II::MRM_C2:
1692   case X86II::MRM_C3:
1693   case X86II::MRM_C4:
1694   case X86II::MRM_C5:
1695   case X86II::MRM_C6:
1696   case X86II::MRM_C7:
1697   case X86II::MRM_C8:
1698   case X86II::MRM_C9:
1699   case X86II::MRM_CA:
1700   case X86II::MRM_CB:
1701   case X86II::MRM_CC:
1702   case X86II::MRM_CD:
1703   case X86II::MRM_CE:
1704   case X86II::MRM_CF:
1705   case X86II::MRM_D0:
1706   case X86II::MRM_D1:
1707   case X86II::MRM_D2:
1708   case X86II::MRM_D3:
1709   case X86II::MRM_D4:
1710   case X86II::MRM_D5:
1711   case X86II::MRM_D6:
1712   case X86II::MRM_D7:
1713   case X86II::MRM_D8:
1714   case X86II::MRM_D9:
1715   case X86II::MRM_DA:
1716   case X86II::MRM_DB:
1717   case X86II::MRM_DC:
1718   case X86II::MRM_DD:
1719   case X86II::MRM_DE:
1720   case X86II::MRM_DF:
1721   case X86II::MRM_E0:
1722   case X86II::MRM_E1:
1723   case X86II::MRM_E2:
1724   case X86II::MRM_E3:
1725   case X86II::MRM_E4:
1726   case X86II::MRM_E5:
1727   case X86II::MRM_E6:
1728   case X86II::MRM_E7:
1729   case X86II::MRM_E8:
1730   case X86II::MRM_E9:
1731   case X86II::MRM_EA:
1732   case X86II::MRM_EB:
1733   case X86II::MRM_EC:
1734   case X86II::MRM_ED:
1735   case X86II::MRM_EE:
1736   case X86II::MRM_EF:
1737   case X86II::MRM_F0:
1738   case X86II::MRM_F1:
1739   case X86II::MRM_F2:
1740   case X86II::MRM_F3:
1741   case X86II::MRM_F4:
1742   case X86II::MRM_F5:
1743   case X86II::MRM_F6:
1744   case X86II::MRM_F7:
1745   case X86II::MRM_F8:
1746   case X86II::MRM_F9:
1747   case X86II::MRM_FA:
1748   case X86II::MRM_FB:
1749   case X86II::MRM_FC:
1750   case X86II::MRM_FD:
1751   case X86II::MRM_FE:
1752   case X86II::MRM_FF:
1753     emitByte(BaseOpcode, CurByte, OS);
1754     emitByte(0xC0 + Form - X86II::MRM_C0, CurByte, OS);
1755     break;
1756   }
1757 
1758   if (HasVEX_I8Reg) {
1759     // The last source register of a 4 operand instruction in AVX is encoded
1760     // in bits[7:4] of a immediate byte.
1761     assert(I8RegNum < 16 && "Register encoding out of range");
1762     I8RegNum <<= 4;
1763     if (CurOp != NumOps) {
1764       unsigned Val = MI.getOperand(CurOp++).getImm();
1765       assert(Val < 16 && "Immediate operand value out of range");
1766       I8RegNum |= Val;
1767     }
1768     emitImmediate(MCOperand::createImm(I8RegNum), MI.getLoc(), 1, FK_Data_1,
1769                   CurByte, OS, Fixups);
1770   } else {
1771     // If there is a remaining operand, it must be a trailing immediate. Emit it
1772     // according to the right size for the instruction. Some instructions
1773     // (SSE4a extrq and insertq) have two trailing immediates.
1774     while (CurOp != NumOps && NumOps - CurOp <= 2) {
1775       emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1776                     X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1777                     CurByte, OS, Fixups);
1778     }
1779   }
1780 
1781   if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow)
1782     emitByte(X86II::getBaseOpcodeFor(TSFlags), CurByte, OS);
1783 
1784 #ifndef NDEBUG
1785   // FIXME: Verify.
1786   if (/*!Desc.isVariadic() &&*/ CurOp != NumOps) {
1787     errs() << "Cannot encode all operands of: ";
1788     MI.dump();
1789     errs() << '\n';
1790     abort();
1791   }
1792 #endif
1793 }
1794 
1795 MCCodeEmitter *llvm::createX86MCCodeEmitter(const MCInstrInfo &MCII,
1796                                             const MCRegisterInfo &MRI,
1797                                             MCContext &Ctx) {
1798   return new X86MCCodeEmitter(MCII, Ctx);
1799 }
1800