1 //===-- X86MCCodeEmitter.cpp - Convert X86 code to machine code -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the X86MCCodeEmitter class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "MCTargetDesc/X86BaseInfo.h"
14 #include "MCTargetDesc/X86FixupKinds.h"
15 #include "MCTargetDesc/X86MCTargetDesc.h"
16 #include "llvm/ADT/SmallVector.h"
17 #include "llvm/MC/MCCodeEmitter.h"
18 #include "llvm/MC/MCContext.h"
19 #include "llvm/MC/MCExpr.h"
20 #include "llvm/MC/MCFixup.h"
21 #include "llvm/MC/MCInst.h"
22 #include "llvm/MC/MCInstrDesc.h"
23 #include "llvm/MC/MCInstrInfo.h"
24 #include "llvm/MC/MCRegisterInfo.h"
25 #include "llvm/MC/MCSubtargetInfo.h"
26 #include "llvm/MC/MCSymbol.h"
27 #include "llvm/Support/ErrorHandling.h"
28 #include "llvm/Support/raw_ostream.h"
29 #include <cassert>
30 #include <cstdint>
31 #include <cstdlib>
32 
33 using namespace llvm;
34 
35 #define DEBUG_TYPE "mccodeemitter"
36 
37 namespace {
38 
39 class X86MCCodeEmitter : public MCCodeEmitter {
40   const MCInstrInfo &MCII;
41   MCContext &Ctx;
42 
43 public:
44   X86MCCodeEmitter(const MCInstrInfo &mcii, MCContext &ctx)
45       : MCII(mcii), Ctx(ctx) {}
46   X86MCCodeEmitter(const X86MCCodeEmitter &) = delete;
47   X86MCCodeEmitter &operator=(const X86MCCodeEmitter &) = delete;
48   ~X86MCCodeEmitter() override = default;
49 
50   void emitPrefix(const MCInst &MI, raw_ostream &OS,
51                   const MCSubtargetInfo &STI) const override;
52 
53   void encodeInstruction(const MCInst &MI, raw_ostream &OS,
54                          SmallVectorImpl<MCFixup> &Fixups,
55                          const MCSubtargetInfo &STI) const override;
56 
57 private:
58   unsigned getX86RegNum(const MCOperand &MO) const {
59     return Ctx.getRegisterInfo()->getEncodingValue(MO.getReg()) & 0x7;
60   }
61 
62   unsigned getX86RegEncoding(const MCInst &MI, unsigned OpNum) const {
63     return Ctx.getRegisterInfo()->getEncodingValue(
64         MI.getOperand(OpNum).getReg());
65   }
66 
67   /// \param MI a single low-level machine instruction.
68   /// \param OpNum the operand #.
69   /// \returns true if the OpNumth operand of MI  require a bit to be set in
70   /// REX prefix.
71   bool isREXExtendedReg(const MCInst &MI, unsigned OpNum) const {
72     return (getX86RegEncoding(MI, OpNum) >> 3) & 1;
73   }
74 
75   void emitByte(uint8_t C, unsigned &CurByte, raw_ostream &OS) const {
76     OS << (char)C;
77     ++CurByte;
78   }
79 
80   void emitConstant(uint64_t Val, unsigned Size, unsigned &CurByte,
81                     raw_ostream &OS) const {
82     // Output the constant in little endian byte order.
83     for (unsigned i = 0; i != Size; ++i) {
84       emitByte(Val & 255, CurByte, OS);
85       Val >>= 8;
86     }
87   }
88 
89   void emitImmediate(const MCOperand &Disp, SMLoc Loc, unsigned ImmSize,
90                      MCFixupKind FixupKind, unsigned &CurByte, raw_ostream &OS,
91                      SmallVectorImpl<MCFixup> &Fixups, int ImmOffset = 0) const;
92 
93   static uint8_t modRMByte(unsigned Mod, unsigned RegOpcode, unsigned RM) {
94     assert(Mod < 4 && RegOpcode < 8 && RM < 8 && "ModRM Fields out of range!");
95     return RM | (RegOpcode << 3) | (Mod << 6);
96   }
97 
98   void emitRegModRMByte(const MCOperand &ModRMReg, unsigned RegOpcodeFld,
99                         unsigned &CurByte, raw_ostream &OS) const {
100     emitByte(modRMByte(3, RegOpcodeFld, getX86RegNum(ModRMReg)), CurByte, OS);
101   }
102 
103   void emitSIBByte(unsigned SS, unsigned Index, unsigned Base,
104                    unsigned &CurByte, raw_ostream &OS) const {
105     // SIB byte is in the same format as the modRMByte.
106     emitByte(modRMByte(SS, Index, Base), CurByte, OS);
107   }
108 
109   void emitMemModRMByte(const MCInst &MI, unsigned Op, unsigned RegOpcodeField,
110                         uint64_t TSFlags, bool Rex, unsigned &CurByte,
111                         raw_ostream &OS, SmallVectorImpl<MCFixup> &Fixups,
112                         const MCSubtargetInfo &STI) const;
113 
114   void emitPrefixImpl(uint64_t TSFlags, unsigned &CurOp, unsigned &CurByte,
115                   bool &Rex, const MCInst &MI, const MCInstrDesc &Desc,
116                   const MCSubtargetInfo &STI, raw_ostream &OS) const;
117 
118   void emitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, int MemOperand,
119                            const MCInst &MI, const MCInstrDesc &Desc,
120                            raw_ostream &OS) const;
121 
122   void emitSegmentOverridePrefix(unsigned &CurByte, unsigned SegOperand,
123                                  const MCInst &MI, raw_ostream &OS) const;
124 
125   bool emitOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, int MemOperand,
126                         const MCInst &MI, const MCInstrDesc &Desc,
127                         const MCSubtargetInfo &STI, raw_ostream &OS) const;
128 
129   uint8_t determineREXPrefix(const MCInst &MI, uint64_t TSFlags, int MemOperand,
130                              const MCInstrDesc &Desc) const;
131 };
132 
133 } // end anonymous namespace
134 
135 /// \returns true if this signed displacement fits in a 8-bit sign-extended
136 /// field.
137 static bool isDisp8(int Value) { return Value == (int8_t)Value; }
138 
139 /// \returns true if this signed displacement fits in a 8-bit compressed
140 /// dispacement field.
141 static bool isCDisp8(uint64_t TSFlags, int Value, int &CValue) {
142   assert(((TSFlags & X86II::EncodingMask) == X86II::EVEX) &&
143          "Compressed 8-bit displacement is only valid for EVEX inst.");
144 
145   unsigned CD8_Scale =
146       (TSFlags & X86II::CD8_Scale_Mask) >> X86II::CD8_Scale_Shift;
147   if (CD8_Scale == 0) {
148     CValue = Value;
149     return isDisp8(Value);
150   }
151 
152   unsigned Mask = CD8_Scale - 1;
153   assert((CD8_Scale & Mask) == 0 && "Invalid memory object size.");
154   if (Value & Mask) // Unaligned offset
155     return false;
156   Value /= (int)CD8_Scale;
157   bool Ret = (Value == (int8_t)Value);
158 
159   if (Ret)
160     CValue = Value;
161   return Ret;
162 }
163 
164 /// \returns the appropriate fixup kind to use for an immediate in an
165 /// instruction with the specified TSFlags.
166 static MCFixupKind getImmFixupKind(uint64_t TSFlags) {
167   unsigned Size = X86II::getSizeOfImm(TSFlags);
168   bool isPCRel = X86II::isImmPCRel(TSFlags);
169 
170   if (X86II::isImmSigned(TSFlags)) {
171     switch (Size) {
172     default:
173       llvm_unreachable("Unsupported signed fixup size!");
174     case 4:
175       return MCFixupKind(X86::reloc_signed_4byte);
176     }
177   }
178   return MCFixup::getKindForSize(Size, isPCRel);
179 }
180 
181 /// \param Op operand # of the memory operand.
182 ///
183 /// \returns true if the specified instruction has a 16-bit memory operand.
184 static bool is16BitMemOperand(const MCInst &MI, unsigned Op,
185                               const MCSubtargetInfo &STI) {
186   const MCOperand &BaseReg = MI.getOperand(Op + X86::AddrBaseReg);
187   const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
188   const MCOperand &Disp = MI.getOperand(Op + X86::AddrDisp);
189 
190   if (STI.hasFeature(X86::Mode16Bit) && BaseReg.getReg() == 0 && Disp.isImm() &&
191       Disp.getImm() < 0x10000)
192     return true;
193   if ((BaseReg.getReg() != 0 &&
194        X86MCRegisterClasses[X86::GR16RegClassID].contains(BaseReg.getReg())) ||
195       (IndexReg.getReg() != 0 &&
196        X86MCRegisterClasses[X86::GR16RegClassID].contains(IndexReg.getReg())))
197     return true;
198   return false;
199 }
200 
201 /// \param Op operand # of the memory operand.
202 ///
203 /// \returns true if the specified instruction has a 32-bit memory operand.
204 static bool is32BitMemOperand(const MCInst &MI, unsigned Op) {
205   const MCOperand &BaseReg = MI.getOperand(Op + X86::AddrBaseReg);
206   const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
207 
208   if ((BaseReg.getReg() != 0 &&
209        X86MCRegisterClasses[X86::GR32RegClassID].contains(BaseReg.getReg())) ||
210       (IndexReg.getReg() != 0 &&
211        X86MCRegisterClasses[X86::GR32RegClassID].contains(IndexReg.getReg())))
212     return true;
213   if (BaseReg.getReg() == X86::EIP) {
214     assert(IndexReg.getReg() == 0 && "Invalid eip-based address.");
215     return true;
216   }
217   if (IndexReg.getReg() == X86::EIZ)
218     return true;
219   return false;
220 }
221 
222 /// \param Op operand # of the memory operand.
223 ///
224 /// \returns true if the specified instruction has a 64-bit memory operand.
225 #ifndef NDEBUG
226 static bool is64BitMemOperand(const MCInst &MI, unsigned Op) {
227   const MCOperand &BaseReg = MI.getOperand(Op + X86::AddrBaseReg);
228   const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
229 
230   if ((BaseReg.getReg() != 0 &&
231        X86MCRegisterClasses[X86::GR64RegClassID].contains(BaseReg.getReg())) ||
232       (IndexReg.getReg() != 0 &&
233        X86MCRegisterClasses[X86::GR64RegClassID].contains(IndexReg.getReg())))
234     return true;
235   return false;
236 }
237 #endif
238 
239 enum GlobalOffsetTableExprKind { GOT_None, GOT_Normal, GOT_SymDiff };
240 
241 /// Check if this expression starts with  _GLOBAL_OFFSET_TABLE_ and if it is
242 /// of the form _GLOBAL_OFFSET_TABLE_-symbol. This is needed to support PIC on
243 /// ELF i386 as _GLOBAL_OFFSET_TABLE_ is magical. We check only simple case that
244 /// are know to be used: _GLOBAL_OFFSET_TABLE_ by itself or at the start of a
245 /// binary expression.
246 static GlobalOffsetTableExprKind
247 startsWithGlobalOffsetTable(const MCExpr *Expr) {
248   const MCExpr *RHS = nullptr;
249   if (Expr->getKind() == MCExpr::Binary) {
250     const MCBinaryExpr *BE = static_cast<const MCBinaryExpr *>(Expr);
251     Expr = BE->getLHS();
252     RHS = BE->getRHS();
253   }
254 
255   if (Expr->getKind() != MCExpr::SymbolRef)
256     return GOT_None;
257 
258   const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr *>(Expr);
259   const MCSymbol &S = Ref->getSymbol();
260   if (S.getName() != "_GLOBAL_OFFSET_TABLE_")
261     return GOT_None;
262   if (RHS && RHS->getKind() == MCExpr::SymbolRef)
263     return GOT_SymDiff;
264   return GOT_Normal;
265 }
266 
267 static bool hasSecRelSymbolRef(const MCExpr *Expr) {
268   if (Expr->getKind() == MCExpr::SymbolRef) {
269     const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr *>(Expr);
270     return Ref->getKind() == MCSymbolRefExpr::VK_SECREL;
271   }
272   return false;
273 }
274 
275 static bool isPCRel32Branch(const MCInst &MI, const MCInstrInfo &MCII) {
276   unsigned Opcode = MI.getOpcode();
277   const MCInstrDesc &Desc = MCII.get(Opcode);
278   if ((Opcode != X86::CALL64pcrel32 && Opcode != X86::JMP_4 &&
279        Opcode != X86::JCC_4) ||
280       getImmFixupKind(Desc.TSFlags) != FK_PCRel_4)
281     return false;
282 
283   unsigned CurOp = X86II::getOperandBias(Desc);
284   const MCOperand &Op = MI.getOperand(CurOp);
285   if (!Op.isExpr())
286     return false;
287 
288   const MCSymbolRefExpr *Ref = dyn_cast<MCSymbolRefExpr>(Op.getExpr());
289   return Ref && Ref->getKind() == MCSymbolRefExpr::VK_None;
290 }
291 
292 void X86MCCodeEmitter::emitImmediate(const MCOperand &DispOp, SMLoc Loc,
293                                      unsigned Size, MCFixupKind FixupKind,
294                                      unsigned &CurByte, raw_ostream &OS,
295                                      SmallVectorImpl<MCFixup> &Fixups,
296                                      int ImmOffset) const {
297   const MCExpr *Expr = nullptr;
298   if (DispOp.isImm()) {
299     // If this is a simple integer displacement that doesn't require a
300     // relocation, emit it now.
301     if (FixupKind != FK_PCRel_1 && FixupKind != FK_PCRel_2 &&
302         FixupKind != FK_PCRel_4) {
303       emitConstant(DispOp.getImm() + ImmOffset, Size, CurByte, OS);
304       return;
305     }
306     Expr = MCConstantExpr::create(DispOp.getImm(), Ctx);
307   } else {
308     Expr = DispOp.getExpr();
309   }
310 
311   // If we have an immoffset, add it to the expression.
312   if ((FixupKind == FK_Data_4 || FixupKind == FK_Data_8 ||
313        FixupKind == MCFixupKind(X86::reloc_signed_4byte))) {
314     GlobalOffsetTableExprKind Kind = startsWithGlobalOffsetTable(Expr);
315     if (Kind != GOT_None) {
316       assert(ImmOffset == 0);
317 
318       if (Size == 8) {
319         FixupKind = MCFixupKind(X86::reloc_global_offset_table8);
320       } else {
321         assert(Size == 4);
322         FixupKind = MCFixupKind(X86::reloc_global_offset_table);
323       }
324 
325       if (Kind == GOT_Normal)
326         ImmOffset = CurByte;
327     } else if (Expr->getKind() == MCExpr::SymbolRef) {
328       if (hasSecRelSymbolRef(Expr)) {
329         FixupKind = MCFixupKind(FK_SecRel_4);
330       }
331     } else if (Expr->getKind() == MCExpr::Binary) {
332       const MCBinaryExpr *Bin = static_cast<const MCBinaryExpr *>(Expr);
333       if (hasSecRelSymbolRef(Bin->getLHS()) ||
334           hasSecRelSymbolRef(Bin->getRHS())) {
335         FixupKind = MCFixupKind(FK_SecRel_4);
336       }
337     }
338   }
339 
340   // If the fixup is pc-relative, we need to bias the value to be relative to
341   // the start of the field, not the end of the field.
342   if (FixupKind == FK_PCRel_4 ||
343       FixupKind == MCFixupKind(X86::reloc_riprel_4byte) ||
344       FixupKind == MCFixupKind(X86::reloc_riprel_4byte_movq_load) ||
345       FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax) ||
346       FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax_rex) ||
347       FixupKind == MCFixupKind(X86::reloc_branch_4byte_pcrel)) {
348     ImmOffset -= 4;
349     // If this is a pc-relative load off _GLOBAL_OFFSET_TABLE_:
350     // leaq _GLOBAL_OFFSET_TABLE_(%rip), %r15
351     // this needs to be a GOTPC32 relocation.
352     if (startsWithGlobalOffsetTable(Expr) != GOT_None)
353       FixupKind = MCFixupKind(X86::reloc_global_offset_table);
354   }
355   if (FixupKind == FK_PCRel_2)
356     ImmOffset -= 2;
357   if (FixupKind == FK_PCRel_1)
358     ImmOffset -= 1;
359 
360   if (ImmOffset)
361     Expr = MCBinaryExpr::createAdd(Expr, MCConstantExpr::create(ImmOffset, Ctx),
362                                    Ctx);
363 
364   // Emit a symbolic constant as a fixup and 4 zeros.
365   Fixups.push_back(MCFixup::create(CurByte, Expr, FixupKind, Loc));
366   emitConstant(0, Size, CurByte, OS);
367 }
368 
369 void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
370                                         unsigned RegOpcodeField,
371                                         uint64_t TSFlags, bool Rex,
372                                         unsigned &CurByte, raw_ostream &OS,
373                                         SmallVectorImpl<MCFixup> &Fixups,
374                                         const MCSubtargetInfo &STI) const {
375   const MCOperand &Disp = MI.getOperand(Op + X86::AddrDisp);
376   const MCOperand &Base = MI.getOperand(Op + X86::AddrBaseReg);
377   const MCOperand &Scale = MI.getOperand(Op + X86::AddrScaleAmt);
378   const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
379   unsigned BaseReg = Base.getReg();
380   bool HasEVEX = (TSFlags & X86II::EncodingMask) == X86II::EVEX;
381 
382   // Handle %rip relative addressing.
383   if (BaseReg == X86::RIP ||
384       BaseReg == X86::EIP) { // [disp32+rIP] in X86-64 mode
385     assert(STI.hasFeature(X86::Mode64Bit) &&
386            "Rip-relative addressing requires 64-bit mode");
387     assert(IndexReg.getReg() == 0 && "Invalid rip-relative address");
388     emitByte(modRMByte(0, RegOpcodeField, 5), CurByte, OS);
389 
390     unsigned Opcode = MI.getOpcode();
391     // movq loads are handled with a special relocation form which allows the
392     // linker to eliminate some loads for GOT references which end up in the
393     // same linkage unit.
394     unsigned FixupKind = [=]() {
395       switch (Opcode) {
396       default:
397         return X86::reloc_riprel_4byte;
398       case X86::MOV64rm:
399         assert(Rex);
400         return X86::reloc_riprel_4byte_movq_load;
401       case X86::CALL64m:
402       case X86::JMP64m:
403       case X86::TAILJMPm64:
404       case X86::TEST64mr:
405       case X86::ADC64rm:
406       case X86::ADD64rm:
407       case X86::AND64rm:
408       case X86::CMP64rm:
409       case X86::OR64rm:
410       case X86::SBB64rm:
411       case X86::SUB64rm:
412       case X86::XOR64rm:
413         return Rex ? X86::reloc_riprel_4byte_relax_rex
414                    : X86::reloc_riprel_4byte_relax;
415       }
416     }();
417 
418     // rip-relative addressing is actually relative to the *next* instruction.
419     // Since an immediate can follow the mod/rm byte for an instruction, this
420     // means that we need to bias the displacement field of the instruction with
421     // the size of the immediate field. If we have this case, add it into the
422     // expression to emit.
423     // Note: rip-relative addressing using immediate displacement values should
424     // not be adjusted, assuming it was the user's intent.
425     int ImmSize = !Disp.isImm() && X86II::hasImm(TSFlags)
426                       ? X86II::getSizeOfImm(TSFlags)
427                       : 0;
428 
429     emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), CurByte, OS,
430                   Fixups, -ImmSize);
431     return;
432   }
433 
434   unsigned BaseRegNo = BaseReg ? getX86RegNum(Base) : -1U;
435 
436   // 16-bit addressing forms of the ModR/M byte have a different encoding for
437   // the R/M field and are far more limited in which registers can be used.
438   if (is16BitMemOperand(MI, Op, STI)) {
439     if (BaseReg) {
440       // For 32-bit addressing, the row and column values in Table 2-2 are
441       // basically the same. It's AX/CX/DX/BX/SP/BP/SI/DI in that order, with
442       // some special cases. And getX86RegNum reflects that numbering.
443       // For 16-bit addressing it's more fun, as shown in the SDM Vol 2A,
444       // Table 2-1 "16-Bit Addressing Forms with the ModR/M byte". We can only
445       // use SI/DI/BP/BX, which have "row" values 4-7 in no particular order,
446       // while values 0-3 indicate the allowed combinations (base+index) of
447       // those: 0 for BX+SI, 1 for BX+DI, 2 for BP+SI, 3 for BP+DI.
448       //
449       // R16Table[] is a lookup from the normal RegNo, to the row values from
450       // Table 2-1 for 16-bit addressing modes. Where zero means disallowed.
451       static const unsigned R16Table[] = {0, 0, 0, 7, 0, 6, 4, 5};
452       unsigned RMfield = R16Table[BaseRegNo];
453 
454       assert(RMfield && "invalid 16-bit base register");
455 
456       if (IndexReg.getReg()) {
457         unsigned IndexReg16 = R16Table[getX86RegNum(IndexReg)];
458 
459         assert(IndexReg16 && "invalid 16-bit index register");
460         // We must have one of SI/DI (4,5), and one of BP/BX (6,7).
461         assert(((IndexReg16 ^ RMfield) & 2) &&
462                "invalid 16-bit base/index register combination");
463         assert(Scale.getImm() == 1 &&
464                "invalid scale for 16-bit memory reference");
465 
466         // Allow base/index to appear in either order (although GAS doesn't).
467         if (IndexReg16 & 2)
468           RMfield = (RMfield & 1) | ((7 - IndexReg16) << 1);
469         else
470           RMfield = (IndexReg16 & 1) | ((7 - RMfield) << 1);
471       }
472 
473       if (Disp.isImm() && isDisp8(Disp.getImm())) {
474         if (Disp.getImm() == 0 && RMfield != 6) {
475           // There is no displacement; just the register.
476           emitByte(modRMByte(0, RegOpcodeField, RMfield), CurByte, OS);
477           return;
478         }
479         // Use the [REG]+disp8 form, including for [BP] which cannot be encoded.
480         emitByte(modRMByte(1, RegOpcodeField, RMfield), CurByte, OS);
481         emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, CurByte, OS, Fixups);
482         return;
483       }
484       // This is the [REG]+disp16 case.
485       emitByte(modRMByte(2, RegOpcodeField, RMfield), CurByte, OS);
486     } else {
487       // There is no BaseReg; this is the plain [disp16] case.
488       emitByte(modRMByte(0, RegOpcodeField, 6), CurByte, OS);
489     }
490 
491     // Emit 16-bit displacement for plain disp16 or [REG]+disp16 cases.
492     emitImmediate(Disp, MI.getLoc(), 2, FK_Data_2, CurByte, OS, Fixups);
493     return;
494   }
495 
496   // Determine whether a SIB byte is needed.
497   // If no BaseReg, issue a RIP relative instruction only if the MCE can
498   // resolve addresses on-the-fly, otherwise use SIB (Intel Manual 2A, table
499   // 2-7) and absolute references.
500 
501   if ( // The SIB byte must be used if there is an index register.
502       IndexReg.getReg() == 0 &&
503       // The SIB byte must be used if the base is ESP/RSP/R12, all of which
504       // encode to an R/M value of 4, which indicates that a SIB byte is
505       // present.
506       BaseRegNo != N86::ESP &&
507       // If there is no base register and we're in 64-bit mode, we need a SIB
508       // byte to emit an addr that is just 'disp32' (the non-RIP relative form).
509       (!STI.hasFeature(X86::Mode64Bit) || BaseReg != 0)) {
510 
511     if (BaseReg == 0) { // [disp32]     in X86-32 mode
512       emitByte(modRMByte(0, RegOpcodeField, 5), CurByte, OS);
513       emitImmediate(Disp, MI.getLoc(), 4, FK_Data_4, CurByte, OS, Fixups);
514       return;
515     }
516 
517     // If the base is not EBP/ESP and there is no displacement, use simple
518     // indirect register encoding, this handles addresses like [EAX].  The
519     // encoding for [EBP] with no displacement means [disp32] so we handle it
520     // by emitting a displacement of 0 below.
521     if (BaseRegNo != N86::EBP) {
522       if (Disp.isImm() && Disp.getImm() == 0) {
523         emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), CurByte, OS);
524         return;
525       }
526 
527       // If the displacement is @tlscall, treat it as a zero.
528       if (Disp.isExpr()) {
529         auto *Sym = dyn_cast<MCSymbolRefExpr>(Disp.getExpr());
530         if (Sym && Sym->getKind() == MCSymbolRefExpr::VK_TLSCALL) {
531           // This is exclusively used by call *a@tlscall(base). The relocation
532           // (R_386_TLSCALL or R_X86_64_TLSCALL) applies to the beginning.
533           Fixups.push_back(MCFixup::create(0, Sym, FK_NONE, MI.getLoc()));
534           emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), CurByte, OS);
535           return;
536         }
537       }
538     }
539 
540     // Otherwise, if the displacement fits in a byte, encode as [REG+disp8].
541     if (Disp.isImm()) {
542       if (!HasEVEX && isDisp8(Disp.getImm())) {
543         emitByte(modRMByte(1, RegOpcodeField, BaseRegNo), CurByte, OS);
544         emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, CurByte, OS, Fixups);
545         return;
546       }
547       // Try EVEX compressed 8-bit displacement first; if failed, fall back to
548       // 32-bit displacement.
549       int CDisp8 = 0;
550       if (HasEVEX && isCDisp8(TSFlags, Disp.getImm(), CDisp8)) {
551         emitByte(modRMByte(1, RegOpcodeField, BaseRegNo), CurByte, OS);
552         emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, CurByte, OS, Fixups,
553                       CDisp8 - Disp.getImm());
554         return;
555       }
556     }
557 
558     // Otherwise, emit the most general non-SIB encoding: [REG+disp32]
559     emitByte(modRMByte(2, RegOpcodeField, BaseRegNo), CurByte, OS);
560     unsigned Opcode = MI.getOpcode();
561     unsigned FixupKind = Opcode == X86::MOV32rm ? X86::reloc_signed_4byte_relax
562                                                 : X86::reloc_signed_4byte;
563     emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), CurByte, OS,
564                   Fixups);
565     return;
566   }
567 
568   // We need a SIB byte, so start by outputting the ModR/M byte first
569   assert(IndexReg.getReg() != X86::ESP && IndexReg.getReg() != X86::RSP &&
570          "Cannot use ESP as index reg!");
571 
572   bool ForceDisp32 = false;
573   bool ForceDisp8 = false;
574   int CDisp8 = 0;
575   int ImmOffset = 0;
576   if (BaseReg == 0) {
577     // If there is no base register, we emit the special case SIB byte with
578     // MOD=0, BASE=5, to JUST get the index, scale, and displacement.
579     emitByte(modRMByte(0, RegOpcodeField, 4), CurByte, OS);
580     ForceDisp32 = true;
581   } else if (!Disp.isImm()) {
582     // Emit the normal disp32 encoding.
583     emitByte(modRMByte(2, RegOpcodeField, 4), CurByte, OS);
584     ForceDisp32 = true;
585   } else if (Disp.getImm() == 0 &&
586              // Base reg can't be anything that ends up with '5' as the base
587              // reg, it is the magic [*] nomenclature that indicates no base.
588              BaseRegNo != N86::EBP) {
589     // Emit no displacement ModR/M byte
590     emitByte(modRMByte(0, RegOpcodeField, 4), CurByte, OS);
591   } else if (!HasEVEX && isDisp8(Disp.getImm())) {
592     // Emit the disp8 encoding.
593     emitByte(modRMByte(1, RegOpcodeField, 4), CurByte, OS);
594     ForceDisp8 = true; // Make sure to force 8 bit disp if Base=EBP
595   } else if (HasEVEX && isCDisp8(TSFlags, Disp.getImm(), CDisp8)) {
596     // Emit the disp8 encoding.
597     emitByte(modRMByte(1, RegOpcodeField, 4), CurByte, OS);
598     ForceDisp8 = true; // Make sure to force 8 bit disp if Base=EBP
599     ImmOffset = CDisp8 - Disp.getImm();
600   } else {
601     // Emit the normal disp32 encoding.
602     emitByte(modRMByte(2, RegOpcodeField, 4), CurByte, OS);
603   }
604 
605   // Calculate what the SS field value should be...
606   static const unsigned SSTable[] = {~0U, 0, 1, ~0U, 2, ~0U, ~0U, ~0U, 3};
607   unsigned SS = SSTable[Scale.getImm()];
608 
609   if (BaseReg == 0) {
610     // Handle the SIB byte for the case where there is no base, see Intel
611     // Manual 2A, table 2-7. The displacement has already been output.
612     unsigned IndexRegNo;
613     if (IndexReg.getReg())
614       IndexRegNo = getX86RegNum(IndexReg);
615     else // Examples: [ESP+1*<noreg>+4] or [scaled idx]+disp32 (MOD=0,BASE=5)
616       IndexRegNo = 4;
617     emitSIBByte(SS, IndexRegNo, 5, CurByte, OS);
618   } else {
619     unsigned IndexRegNo;
620     if (IndexReg.getReg())
621       IndexRegNo = getX86RegNum(IndexReg);
622     else
623       IndexRegNo = 4; // For example [ESP+1*<noreg>+4]
624     emitSIBByte(SS, IndexRegNo, getX86RegNum(Base), CurByte, OS);
625   }
626 
627   // Do we need to output a displacement?
628   if (ForceDisp8)
629     emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, CurByte, OS, Fixups,
630                   ImmOffset);
631   else if (ForceDisp32 || Disp.getImm() != 0)
632     emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(X86::reloc_signed_4byte),
633                   CurByte, OS, Fixups);
634 }
635 
636 void X86MCCodeEmitter::emitPrefixImpl(uint64_t TSFlags, unsigned &CurOp,
637                                   unsigned &CurByte, bool &Rex,
638                                   const MCInst &MI, const MCInstrDesc &Desc,
639                                   const MCSubtargetInfo &STI,
640                                   raw_ostream &OS) const {
641   // Determine where the memory operand starts, if present.
642   int MemoryOperand = X86II::getMemoryOperandNo(TSFlags);
643   if (MemoryOperand != -1)
644     MemoryOperand += CurOp;
645 
646   // Emit segment override opcode prefix as needed.
647   if (MemoryOperand >= 0)
648     emitSegmentOverridePrefix(CurByte, MemoryOperand + X86::AddrSegmentReg, MI,
649                               OS);
650 
651   // Emit the repeat opcode prefix as needed.
652   unsigned Flags = MI.getFlags();
653   if (TSFlags & X86II::REP || Flags & X86::IP_HAS_REPEAT)
654     emitByte(0xF3, CurByte, OS);
655   if (Flags & X86::IP_HAS_REPEAT_NE)
656     emitByte(0xF2, CurByte, OS);
657 
658   // Emit the address size opcode prefix as needed.
659   bool need_address_override;
660   uint64_t AdSize = TSFlags & X86II::AdSizeMask;
661   if ((STI.hasFeature(X86::Mode16Bit) && AdSize == X86II::AdSize32) ||
662       (STI.hasFeature(X86::Mode32Bit) && AdSize == X86II::AdSize16) ||
663       (STI.hasFeature(X86::Mode64Bit) && AdSize == X86II::AdSize32)) {
664     need_address_override = true;
665   } else if (MemoryOperand < 0) {
666     need_address_override = false;
667   } else if (STI.hasFeature(X86::Mode64Bit)) {
668     assert(!is16BitMemOperand(MI, MemoryOperand, STI));
669     need_address_override = is32BitMemOperand(MI, MemoryOperand);
670   } else if (STI.hasFeature(X86::Mode32Bit)) {
671     assert(!is64BitMemOperand(MI, MemoryOperand));
672     need_address_override = is16BitMemOperand(MI, MemoryOperand, STI);
673   } else {
674     assert(STI.hasFeature(X86::Mode16Bit));
675     assert(!is64BitMemOperand(MI, MemoryOperand));
676     need_address_override = !is16BitMemOperand(MI, MemoryOperand, STI);
677   }
678 
679   if (need_address_override)
680     emitByte(0x67, CurByte, OS);
681 
682   // Encoding type for this instruction.
683   uint64_t Encoding = TSFlags & X86II::EncodingMask;
684   if (Encoding == 0)
685     Rex = emitOpcodePrefix(TSFlags, CurByte, MemoryOperand, MI, Desc, STI, OS);
686   else
687     emitVEXOpcodePrefix(TSFlags, CurByte, MemoryOperand, MI, Desc, OS);
688 
689   uint64_t Form = TSFlags & X86II::FormMask;
690   switch (Form) {
691   default:
692     break;
693   case X86II::RawFrmDstSrc: {
694     unsigned siReg = MI.getOperand(1).getReg();
695     assert(((siReg == X86::SI && MI.getOperand(0).getReg() == X86::DI) ||
696             (siReg == X86::ESI && MI.getOperand(0).getReg() == X86::EDI) ||
697             (siReg == X86::RSI && MI.getOperand(0).getReg() == X86::RDI)) &&
698            "SI and DI register sizes do not match");
699     // Emit segment override opcode prefix as needed (not for %ds).
700     if (MI.getOperand(2).getReg() != X86::DS)
701       emitSegmentOverridePrefix(CurByte, 2, MI, OS);
702     // Emit AdSize prefix as needed.
703     if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::ESI) ||
704         (STI.hasFeature(X86::Mode32Bit) && siReg == X86::SI))
705       emitByte(0x67, CurByte, OS);
706     CurOp += 3; // Consume operands.
707     break;
708   }
709   case X86II::RawFrmSrc: {
710     unsigned siReg = MI.getOperand(0).getReg();
711     // Emit segment override opcode prefix as needed (not for %ds).
712     if (MI.getOperand(1).getReg() != X86::DS)
713       emitSegmentOverridePrefix(CurByte, 1, MI, OS);
714     // Emit AdSize prefix as needed.
715     if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::ESI) ||
716         (STI.hasFeature(X86::Mode32Bit) && siReg == X86::SI))
717       emitByte(0x67, CurByte, OS);
718     CurOp += 2; // Consume operands.
719     break;
720   }
721   case X86II::RawFrmDst: {
722     unsigned siReg = MI.getOperand(0).getReg();
723     // Emit AdSize prefix as needed.
724     if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::EDI) ||
725         (STI.hasFeature(X86::Mode32Bit) && siReg == X86::DI))
726       emitByte(0x67, CurByte, OS);
727     ++CurOp; // Consume operand.
728     break;
729   }
730   case X86II::RawFrmMemOffs: {
731     // Emit segment override opcode prefix as needed.
732     emitSegmentOverridePrefix(CurByte, 1, MI, OS);
733     break;
734   }
735   }
736 }
737 
738 /// emitVEXOpcodePrefix - AVX instructions are encoded using a opcode prefix
739 /// called VEX.
740 void X86MCCodeEmitter::emitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte,
741                                            int MemOperand, const MCInst &MI,
742                                            const MCInstrDesc &Desc,
743                                            raw_ostream &OS) const {
744   assert(!(TSFlags & X86II::LOCK) && "Can't have LOCK VEX.");
745 
746   uint64_t Encoding = TSFlags & X86II::EncodingMask;
747   bool HasEVEX_K = TSFlags & X86II::EVEX_K;
748   bool HasVEX_4V = TSFlags & X86II::VEX_4V;
749   bool HasEVEX_RC = TSFlags & X86II::EVEX_RC;
750 
751   // VEX_R: opcode externsion equivalent to REX.R in
752   // 1's complement (inverted) form
753   //
754   //  1: Same as REX_R=0 (must be 1 in 32-bit mode)
755   //  0: Same as REX_R=1 (64 bit mode only)
756   //
757   uint8_t VEX_R = 0x1;
758   uint8_t EVEX_R2 = 0x1;
759 
760   // VEX_X: equivalent to REX.X, only used when a
761   // register is used for index in SIB Byte.
762   //
763   //  1: Same as REX.X=0 (must be 1 in 32-bit mode)
764   //  0: Same as REX.X=1 (64-bit mode only)
765   uint8_t VEX_X = 0x1;
766 
767   // VEX_B:
768   //
769   //  1: Same as REX_B=0 (ignored in 32-bit mode)
770   //  0: Same as REX_B=1 (64 bit mode only)
771   //
772   uint8_t VEX_B = 0x1;
773 
774   // VEX_W: opcode specific (use like REX.W, or used for
775   // opcode extension, or ignored, depending on the opcode byte)
776   uint8_t VEX_W = (TSFlags & X86II::VEX_W) ? 1 : 0;
777 
778   // VEX_5M (VEX m-mmmmm field):
779   //
780   //  0b00000: Reserved for future use
781   //  0b00001: implied 0F leading opcode
782   //  0b00010: implied 0F 38 leading opcode bytes
783   //  0b00011: implied 0F 3A leading opcode bytes
784   //  0b00100-0b11111: Reserved for future use
785   //  0b01000: XOP map select - 08h instructions with imm byte
786   //  0b01001: XOP map select - 09h instructions with no imm byte
787   //  0b01010: XOP map select - 0Ah instructions with imm dword
788   uint8_t VEX_5M;
789   switch (TSFlags & X86II::OpMapMask) {
790   default:
791     llvm_unreachable("Invalid prefix!");
792   case X86II::TB:
793     VEX_5M = 0x1;
794     break; // 0F
795   case X86II::T8:
796     VEX_5M = 0x2;
797     break; // 0F 38
798   case X86II::TA:
799     VEX_5M = 0x3;
800     break; // 0F 3A
801   case X86II::XOP8:
802     VEX_5M = 0x8;
803     break;
804   case X86II::XOP9:
805     VEX_5M = 0x9;
806     break;
807   case X86II::XOPA:
808     VEX_5M = 0xA;
809     break;
810   }
811 
812   // VEX_4V (VEX vvvv field): a register specifier
813   // (in 1's complement form) or 1111 if unused.
814   uint8_t VEX_4V = 0xf;
815   uint8_t EVEX_V2 = 0x1;
816 
817   // EVEX_L2/VEX_L (Vector Length):
818   //
819   // L2 L
820   //  0 0: scalar or 128-bit vector
821   //  0 1: 256-bit vector
822   //  1 0: 512-bit vector
823   //
824   uint8_t VEX_L = (TSFlags & X86II::VEX_L) ? 1 : 0;
825   uint8_t EVEX_L2 = (TSFlags & X86II::EVEX_L2) ? 1 : 0;
826 
827   // VEX_PP: opcode extension providing equivalent
828   // functionality of a SIMD prefix
829   //
830   //  0b00: None
831   //  0b01: 66
832   //  0b10: F3
833   //  0b11: F2
834   //
835   uint8_t VEX_PP = 0;
836   switch (TSFlags & X86II::OpPrefixMask) {
837   case X86II::PD:
838     VEX_PP = 0x1;
839     break; // 66
840   case X86II::XS:
841     VEX_PP = 0x2;
842     break; // F3
843   case X86II::XD:
844     VEX_PP = 0x3;
845     break; // F2
846   }
847 
848   // EVEX_U
849   uint8_t EVEX_U = 1; // Always '1' so far
850 
851   // EVEX_z
852   uint8_t EVEX_z = (HasEVEX_K && (TSFlags & X86II::EVEX_Z)) ? 1 : 0;
853 
854   // EVEX_b
855   uint8_t EVEX_b = (TSFlags & X86II::EVEX_B) ? 1 : 0;
856 
857   // EVEX_rc
858   uint8_t EVEX_rc = 0;
859 
860   // EVEX_aaa
861   uint8_t EVEX_aaa = 0;
862 
863   bool EncodeRC = false;
864 
865   // Classify VEX_B, VEX_4V, VEX_R, VEX_X
866   unsigned NumOps = Desc.getNumOperands();
867   unsigned CurOp = X86II::getOperandBias(Desc);
868 
869   switch (TSFlags & X86II::FormMask) {
870   default:
871     llvm_unreachable("Unexpected form in emitVEXOpcodePrefix!");
872   case X86II::RawFrm:
873     break;
874   case X86II::MRMDestMem: {
875     // MRMDestMem instructions forms:
876     //  MemAddr, src1(ModR/M)
877     //  MemAddr, src1(VEX_4V), src2(ModR/M)
878     //  MemAddr, src1(ModR/M), imm8
879     //
880     unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
881     VEX_B = ~(BaseRegEnc >> 3) & 1;
882     unsigned IndexRegEnc =
883         getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
884     VEX_X = ~(IndexRegEnc >> 3) & 1;
885     if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV.
886       EVEX_V2 = ~(IndexRegEnc >> 4) & 1;
887 
888     CurOp += X86::AddrNumOperands;
889 
890     if (HasEVEX_K)
891       EVEX_aaa = getX86RegEncoding(MI, CurOp++);
892 
893     if (HasVEX_4V) {
894       unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
895       VEX_4V = ~VRegEnc & 0xf;
896       EVEX_V2 = ~(VRegEnc >> 4) & 1;
897     }
898 
899     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
900     VEX_R = ~(RegEnc >> 3) & 1;
901     EVEX_R2 = ~(RegEnc >> 4) & 1;
902     break;
903   }
904   case X86II::MRMSrcMem: {
905     // MRMSrcMem instructions forms:
906     //  src1(ModR/M), MemAddr
907     //  src1(ModR/M), src2(VEX_4V), MemAddr
908     //  src1(ModR/M), MemAddr, imm8
909     //  src1(ModR/M), MemAddr, src2(Imm[7:4])
910     //
911     //  FMA4:
912     //  dst(ModR/M.reg), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4])
913     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
914     VEX_R = ~(RegEnc >> 3) & 1;
915     EVEX_R2 = ~(RegEnc >> 4) & 1;
916 
917     if (HasEVEX_K)
918       EVEX_aaa = getX86RegEncoding(MI, CurOp++);
919 
920     if (HasVEX_4V) {
921       unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
922       VEX_4V = ~VRegEnc & 0xf;
923       EVEX_V2 = ~(VRegEnc >> 4) & 1;
924     }
925 
926     unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
927     VEX_B = ~(BaseRegEnc >> 3) & 1;
928     unsigned IndexRegEnc =
929         getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
930     VEX_X = ~(IndexRegEnc >> 3) & 1;
931     if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV.
932       EVEX_V2 = ~(IndexRegEnc >> 4) & 1;
933 
934     break;
935   }
936   case X86II::MRMSrcMem4VOp3: {
937     // Instruction format for 4VOp3:
938     //   src1(ModR/M), MemAddr, src3(VEX_4V)
939     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
940     VEX_R = ~(RegEnc >> 3) & 1;
941 
942     unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
943     VEX_B = ~(BaseRegEnc >> 3) & 1;
944     unsigned IndexRegEnc =
945         getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
946     VEX_X = ~(IndexRegEnc >> 3) & 1;
947 
948     VEX_4V = ~getX86RegEncoding(MI, CurOp + X86::AddrNumOperands) & 0xf;
949     break;
950   }
951   case X86II::MRMSrcMemOp4: {
952     //  dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
953     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
954     VEX_R = ~(RegEnc >> 3) & 1;
955 
956     unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
957     VEX_4V = ~VRegEnc & 0xf;
958 
959     unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
960     VEX_B = ~(BaseRegEnc >> 3) & 1;
961     unsigned IndexRegEnc =
962         getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
963     VEX_X = ~(IndexRegEnc >> 3) & 1;
964     break;
965   }
966   case X86II::MRM0m:
967   case X86II::MRM1m:
968   case X86II::MRM2m:
969   case X86II::MRM3m:
970   case X86II::MRM4m:
971   case X86II::MRM5m:
972   case X86II::MRM6m:
973   case X86II::MRM7m: {
974     // MRM[0-9]m instructions forms:
975     //  MemAddr
976     //  src1(VEX_4V), MemAddr
977     if (HasVEX_4V) {
978       unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
979       VEX_4V = ~VRegEnc & 0xf;
980       EVEX_V2 = ~(VRegEnc >> 4) & 1;
981     }
982 
983     if (HasEVEX_K)
984       EVEX_aaa = getX86RegEncoding(MI, CurOp++);
985 
986     unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
987     VEX_B = ~(BaseRegEnc >> 3) & 1;
988     unsigned IndexRegEnc =
989         getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
990     VEX_X = ~(IndexRegEnc >> 3) & 1;
991     if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV.
992       EVEX_V2 = ~(IndexRegEnc >> 4) & 1;
993 
994     break;
995   }
996   case X86II::MRMSrcReg: {
997     // MRMSrcReg instructions forms:
998     //  dst(ModR/M), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4])
999     //  dst(ModR/M), src1(ModR/M)
1000     //  dst(ModR/M), src1(ModR/M), imm8
1001     //
1002     //  FMA4:
1003     //  dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
1004     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1005     VEX_R = ~(RegEnc >> 3) & 1;
1006     EVEX_R2 = ~(RegEnc >> 4) & 1;
1007 
1008     if (HasEVEX_K)
1009       EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1010 
1011     if (HasVEX_4V) {
1012       unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1013       VEX_4V = ~VRegEnc & 0xf;
1014       EVEX_V2 = ~(VRegEnc >> 4) & 1;
1015     }
1016 
1017     RegEnc = getX86RegEncoding(MI, CurOp++);
1018     VEX_B = ~(RegEnc >> 3) & 1;
1019     VEX_X = ~(RegEnc >> 4) & 1;
1020 
1021     if (EVEX_b) {
1022       if (HasEVEX_RC) {
1023         unsigned RcOperand = NumOps - 1;
1024         assert(RcOperand >= CurOp);
1025         EVEX_rc = MI.getOperand(RcOperand).getImm();
1026         assert(EVEX_rc <= 3 && "Invalid rounding control!");
1027       }
1028       EncodeRC = true;
1029     }
1030     break;
1031   }
1032   case X86II::MRMSrcReg4VOp3: {
1033     // Instruction format for 4VOp3:
1034     //   src1(ModR/M), src2(ModR/M), src3(VEX_4V)
1035     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1036     VEX_R = ~(RegEnc >> 3) & 1;
1037 
1038     RegEnc = getX86RegEncoding(MI, CurOp++);
1039     VEX_B = ~(RegEnc >> 3) & 1;
1040 
1041     VEX_4V = ~getX86RegEncoding(MI, CurOp++) & 0xf;
1042     break;
1043   }
1044   case X86II::MRMSrcRegOp4: {
1045     //  dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
1046     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1047     VEX_R = ~(RegEnc >> 3) & 1;
1048 
1049     unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1050     VEX_4V = ~VRegEnc & 0xf;
1051 
1052     // Skip second register source (encoded in Imm[7:4])
1053     ++CurOp;
1054 
1055     RegEnc = getX86RegEncoding(MI, CurOp++);
1056     VEX_B = ~(RegEnc >> 3) & 1;
1057     VEX_X = ~(RegEnc >> 4) & 1;
1058     break;
1059   }
1060   case X86II::MRMDestReg: {
1061     // MRMDestReg instructions forms:
1062     //  dst(ModR/M), src(ModR/M)
1063     //  dst(ModR/M), src(ModR/M), imm8
1064     //  dst(ModR/M), src1(VEX_4V), src2(ModR/M)
1065     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1066     VEX_B = ~(RegEnc >> 3) & 1;
1067     VEX_X = ~(RegEnc >> 4) & 1;
1068 
1069     if (HasEVEX_K)
1070       EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1071 
1072     if (HasVEX_4V) {
1073       unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1074       VEX_4V = ~VRegEnc & 0xf;
1075       EVEX_V2 = ~(VRegEnc >> 4) & 1;
1076     }
1077 
1078     RegEnc = getX86RegEncoding(MI, CurOp++);
1079     VEX_R = ~(RegEnc >> 3) & 1;
1080     EVEX_R2 = ~(RegEnc >> 4) & 1;
1081     if (EVEX_b)
1082       EncodeRC = true;
1083     break;
1084   }
1085   case X86II::MRM0r:
1086   case X86II::MRM1r:
1087   case X86II::MRM2r:
1088   case X86II::MRM3r:
1089   case X86II::MRM4r:
1090   case X86II::MRM5r:
1091   case X86II::MRM6r:
1092   case X86II::MRM7r: {
1093     // MRM0r-MRM7r instructions forms:
1094     //  dst(VEX_4V), src(ModR/M), imm8
1095     if (HasVEX_4V) {
1096       unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1097       VEX_4V = ~VRegEnc & 0xf;
1098       EVEX_V2 = ~(VRegEnc >> 4) & 1;
1099     }
1100     if (HasEVEX_K)
1101       EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1102 
1103     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1104     VEX_B = ~(RegEnc >> 3) & 1;
1105     VEX_X = ~(RegEnc >> 4) & 1;
1106     break;
1107   }
1108   }
1109 
1110   if (Encoding == X86II::VEX || Encoding == X86II::XOP) {
1111     // VEX opcode prefix can have 2 or 3 bytes
1112     //
1113     //  3 bytes:
1114     //    +-----+ +--------------+ +-------------------+
1115     //    | C4h | | RXB | m-mmmm | | W | vvvv | L | pp |
1116     //    +-----+ +--------------+ +-------------------+
1117     //  2 bytes:
1118     //    +-----+ +-------------------+
1119     //    | C5h | | R | vvvv | L | pp |
1120     //    +-----+ +-------------------+
1121     //
1122     //  XOP uses a similar prefix:
1123     //    +-----+ +--------------+ +-------------------+
1124     //    | 8Fh | | RXB | m-mmmm | | W | vvvv | L | pp |
1125     //    +-----+ +--------------+ +-------------------+
1126     uint8_t LastByte = VEX_PP | (VEX_L << 2) | (VEX_4V << 3);
1127 
1128     // Can we use the 2 byte VEX prefix?
1129     if (!(MI.getFlags() & X86::IP_USE_VEX3) && Encoding == X86II::VEX &&
1130         VEX_B && VEX_X && !VEX_W && (VEX_5M == 1)) {
1131       emitByte(0xC5, CurByte, OS);
1132       emitByte(LastByte | (VEX_R << 7), CurByte, OS);
1133       return;
1134     }
1135 
1136     // 3 byte VEX prefix
1137     emitByte(Encoding == X86II::XOP ? 0x8F : 0xC4, CurByte, OS);
1138     emitByte(VEX_R << 7 | VEX_X << 6 | VEX_B << 5 | VEX_5M, CurByte, OS);
1139     emitByte(LastByte | (VEX_W << 7), CurByte, OS);
1140   } else {
1141     assert(Encoding == X86II::EVEX && "unknown encoding!");
1142     // EVEX opcode prefix can have 4 bytes
1143     //
1144     // +-----+ +--------------+ +-------------------+ +------------------------+
1145     // | 62h | | RXBR' | 00mm | | W | vvvv | U | pp | | z | L'L | b | v' | aaa |
1146     // +-----+ +--------------+ +-------------------+ +------------------------+
1147     assert((VEX_5M & 0x3) == VEX_5M &&
1148            "More than 2 significant bits in VEX.m-mmmm fields for EVEX!");
1149 
1150     emitByte(0x62, CurByte, OS);
1151     emitByte((VEX_R << 7) | (VEX_X << 6) | (VEX_B << 5) | (EVEX_R2 << 4) |
1152                  VEX_5M,
1153              CurByte, OS);
1154     emitByte((VEX_W << 7) | (VEX_4V << 3) | (EVEX_U << 2) | VEX_PP, CurByte,
1155              OS);
1156     if (EncodeRC)
1157       emitByte((EVEX_z << 7) | (EVEX_rc << 5) | (EVEX_b << 4) | (EVEX_V2 << 3) |
1158                    EVEX_aaa,
1159                CurByte, OS);
1160     else
1161       emitByte((EVEX_z << 7) | (EVEX_L2 << 6) | (VEX_L << 5) | (EVEX_b << 4) |
1162                    (EVEX_V2 << 3) | EVEX_aaa,
1163                CurByte, OS);
1164   }
1165 }
1166 
1167 /// Determine if the MCInst has to be encoded with a X86-64 REX prefix which
1168 /// specifies 1) 64-bit instructions, 2) non-default operand size, and 3) use
1169 /// of X86-64 extended registers.
1170 uint8_t X86MCCodeEmitter::determineREXPrefix(const MCInst &MI, uint64_t TSFlags,
1171                                              int MemOperand,
1172                                              const MCInstrDesc &Desc) const {
1173   uint8_t REX = 0;
1174   bool UsesHighByteReg = false;
1175 
1176   if (TSFlags & X86II::REX_W)
1177     REX |= 1 << 3; // set REX.W
1178 
1179   if (MI.getNumOperands() == 0)
1180     return REX;
1181 
1182   unsigned NumOps = MI.getNumOperands();
1183   unsigned CurOp = X86II::getOperandBias(Desc);
1184 
1185   // If it accesses SPL, BPL, SIL, or DIL, then it requires a 0x40 REX prefix.
1186   for (unsigned i = CurOp; i != NumOps; ++i) {
1187     const MCOperand &MO = MI.getOperand(i);
1188     if (!MO.isReg())
1189       continue;
1190     unsigned Reg = MO.getReg();
1191     if (Reg == X86::AH || Reg == X86::BH || Reg == X86::CH || Reg == X86::DH)
1192       UsesHighByteReg = true;
1193     if (X86II::isX86_64NonExtLowByteReg(Reg))
1194       // FIXME: The caller of determineREXPrefix slaps this prefix onto anything
1195       // that returns non-zero.
1196       REX |= 0x40; // REX fixed encoding prefix
1197   }
1198 
1199   switch (TSFlags & X86II::FormMask) {
1200   case X86II::AddRegFrm:
1201     REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1202     break;
1203   case X86II::MRMSrcReg:
1204   case X86II::MRMSrcRegCC:
1205     REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1206     REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1207     break;
1208   case X86II::MRMSrcMem:
1209   case X86II::MRMSrcMemCC:
1210     REX |= isREXExtendedReg(MI, CurOp++) << 2;                        // REX.R
1211     REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0;  // REX.B
1212     REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X
1213     CurOp += X86::AddrNumOperands;
1214     break;
1215   case X86II::MRMDestReg:
1216     REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1217     REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1218     break;
1219   case X86II::MRMDestMem:
1220     REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0;  // REX.B
1221     REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X
1222     CurOp += X86::AddrNumOperands;
1223     REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1224     break;
1225   case X86II::MRMXmCC:
1226   case X86II::MRMXm:
1227   case X86II::MRM0m:
1228   case X86II::MRM1m:
1229   case X86II::MRM2m:
1230   case X86II::MRM3m:
1231   case X86II::MRM4m:
1232   case X86II::MRM5m:
1233   case X86II::MRM6m:
1234   case X86II::MRM7m:
1235     REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0;  // REX.B
1236     REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X
1237     break;
1238   case X86II::MRMXrCC:
1239   case X86II::MRMXr:
1240   case X86II::MRM0r:
1241   case X86II::MRM1r:
1242   case X86II::MRM2r:
1243   case X86II::MRM3r:
1244   case X86II::MRM4r:
1245   case X86II::MRM5r:
1246   case X86II::MRM6r:
1247   case X86II::MRM7r:
1248     REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1249     break;
1250   }
1251   if (REX && UsesHighByteReg)
1252     report_fatal_error(
1253         "Cannot encode high byte register in REX-prefixed instruction");
1254 
1255   return REX;
1256 }
1257 
1258 /// Emit segment override opcode prefix as needed.
1259 void X86MCCodeEmitter::emitSegmentOverridePrefix(unsigned &CurByte,
1260                                                  unsigned SegOperand,
1261                                                  const MCInst &MI,
1262                                                  raw_ostream &OS) const {
1263   // Check for explicit segment override on memory operand.
1264   switch (MI.getOperand(SegOperand).getReg()) {
1265   default:
1266     llvm_unreachable("Unknown segment register!");
1267   case 0:
1268     break;
1269   case X86::CS:
1270     emitByte(0x2E, CurByte, OS);
1271     break;
1272   case X86::SS:
1273     emitByte(0x36, CurByte, OS);
1274     break;
1275   case X86::DS:
1276     emitByte(0x3E, CurByte, OS);
1277     break;
1278   case X86::ES:
1279     emitByte(0x26, CurByte, OS);
1280     break;
1281   case X86::FS:
1282     emitByte(0x64, CurByte, OS);
1283     break;
1284   case X86::GS:
1285     emitByte(0x65, CurByte, OS);
1286     break;
1287   }
1288 }
1289 
1290 /// Emit all instruction prefixes prior to the opcode.
1291 ///
1292 /// \param MemOperand the operand # of the start of a memory operand if present.
1293 /// If not present, it is -1.
1294 ///
1295 /// \returns true if a REX prefix was used.
1296 bool X86MCCodeEmitter::emitOpcodePrefix(uint64_t TSFlags, unsigned &CurByte,
1297                                         int MemOperand, const MCInst &MI,
1298                                         const MCInstrDesc &Desc,
1299                                         const MCSubtargetInfo &STI,
1300                                         raw_ostream &OS) const {
1301   bool Ret = false;
1302   // Emit the operand size opcode prefix as needed.
1303   if ((TSFlags & X86II::OpSizeMask) ==
1304       (STI.hasFeature(X86::Mode16Bit) ? X86II::OpSize32 : X86II::OpSize16))
1305     emitByte(0x66, CurByte, OS);
1306 
1307   // Emit the LOCK opcode prefix.
1308   if (TSFlags & X86II::LOCK || MI.getFlags() & X86::IP_HAS_LOCK)
1309     emitByte(0xF0, CurByte, OS);
1310 
1311   // Emit the NOTRACK opcode prefix.
1312   if (TSFlags & X86II::NOTRACK || MI.getFlags() & X86::IP_HAS_NOTRACK)
1313     emitByte(0x3E, CurByte, OS);
1314 
1315   switch (TSFlags & X86II::OpPrefixMask) {
1316   case X86II::PD: // 66
1317     emitByte(0x66, CurByte, OS);
1318     break;
1319   case X86II::XS: // F3
1320     emitByte(0xF3, CurByte, OS);
1321     break;
1322   case X86II::XD: // F2
1323     emitByte(0xF2, CurByte, OS);
1324     break;
1325   }
1326 
1327   // Handle REX prefix.
1328   // FIXME: Can this come before F2 etc to simplify emission?
1329   if (STI.hasFeature(X86::Mode64Bit)) {
1330     if (uint8_t REX = determineREXPrefix(MI, TSFlags, MemOperand, Desc)) {
1331       emitByte(0x40 | REX, CurByte, OS);
1332       Ret = true;
1333     }
1334   } else {
1335     assert(!(TSFlags & X86II::REX_W) && "REX.W requires 64bit mode.");
1336   }
1337 
1338   // 0x0F escape code must be emitted just before the opcode.
1339   switch (TSFlags & X86II::OpMapMask) {
1340   case X86II::TB:        // Two-byte opcode map
1341   case X86II::T8:        // 0F 38
1342   case X86II::TA:        // 0F 3A
1343   case X86II::ThreeDNow: // 0F 0F, second 0F emitted by caller.
1344     emitByte(0x0F, CurByte, OS);
1345     break;
1346   }
1347 
1348   switch (TSFlags & X86II::OpMapMask) {
1349   case X86II::T8: // 0F 38
1350     emitByte(0x38, CurByte, OS);
1351     break;
1352   case X86II::TA: // 0F 3A
1353     emitByte(0x3A, CurByte, OS);
1354     break;
1355   }
1356   return Ret;
1357 }
1358 
1359 void X86MCCodeEmitter::emitPrefix(const MCInst &MI, raw_ostream &OS,
1360                                   const MCSubtargetInfo &STI) const {
1361   unsigned Opcode = MI.getOpcode();
1362   const MCInstrDesc &Desc = MCII.get(Opcode);
1363   uint64_t TSFlags = Desc.TSFlags;
1364 
1365   // Pseudo instructions don't get encoded.
1366   if ((TSFlags & X86II::FormMask) == X86II::Pseudo)
1367     return;
1368 
1369   unsigned CurOp = X86II::getOperandBias(Desc);
1370 
1371   // Keep track of the current byte being emitted.
1372   unsigned CurByte = 0;
1373 
1374   bool Rex = false;
1375   emitPrefixImpl(TSFlags, CurOp, CurByte, Rex, MI, Desc, STI, OS);
1376 }
1377 
1378 void X86MCCodeEmitter::encodeInstruction(const MCInst &MI, raw_ostream &OS,
1379                                          SmallVectorImpl<MCFixup> &Fixups,
1380                                          const MCSubtargetInfo &STI) const {
1381   unsigned Opcode = MI.getOpcode();
1382   const MCInstrDesc &Desc = MCII.get(Opcode);
1383   uint64_t TSFlags = Desc.TSFlags;
1384 
1385   // Pseudo instructions don't get encoded.
1386   if ((TSFlags & X86II::FormMask) == X86II::Pseudo)
1387     return;
1388 
1389   unsigned NumOps = Desc.getNumOperands();
1390   unsigned CurOp = X86II::getOperandBias(Desc);
1391 
1392   // Keep track of the current byte being emitted.
1393   unsigned CurByte = 0;
1394 
1395   bool Rex = false;
1396   emitPrefixImpl(TSFlags, CurOp, CurByte, Rex, MI, Desc, STI, OS);
1397 
1398   // It uses the VEX.VVVV field?
1399   bool HasVEX_4V = TSFlags & X86II::VEX_4V;
1400   bool HasVEX_I8Reg = (TSFlags & X86II::ImmMask) == X86II::Imm8Reg;
1401 
1402   // It uses the EVEX.aaa field?
1403   bool HasEVEX_K = TSFlags & X86II::EVEX_K;
1404   bool HasEVEX_RC = TSFlags & X86II::EVEX_RC;
1405 
1406   // Used if a register is encoded in 7:4 of immediate.
1407   unsigned I8RegNum = 0;
1408 
1409   uint8_t BaseOpcode = X86II::getBaseOpcodeFor(TSFlags);
1410 
1411   if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow)
1412     BaseOpcode = 0x0F; // Weird 3DNow! encoding.
1413 
1414   unsigned OpcodeOffset = 0;
1415 
1416   uint64_t Form = TSFlags & X86II::FormMask;
1417   switch (Form) {
1418   default:
1419     errs() << "FORM: " << Form << "\n";
1420     llvm_unreachable("Unknown FormMask value in X86MCCodeEmitter!");
1421   case X86II::Pseudo:
1422     llvm_unreachable("Pseudo instruction shouldn't be emitted");
1423   case X86II::RawFrmDstSrc:
1424   case X86II::RawFrmSrc:
1425   case X86II::RawFrmDst:
1426     emitByte(BaseOpcode, CurByte, OS);
1427     break;
1428   case X86II::AddCCFrm: {
1429     // This will be added to the opcode in the fallthrough.
1430     OpcodeOffset = MI.getOperand(NumOps - 1).getImm();
1431     assert(OpcodeOffset < 16 && "Unexpected opcode offset!");
1432     --NumOps; // Drop the operand from the end.
1433     LLVM_FALLTHROUGH;
1434   case X86II::RawFrm:
1435     emitByte(BaseOpcode + OpcodeOffset, CurByte, OS);
1436 
1437     if (!STI.hasFeature(X86::Mode64Bit) || !isPCRel32Branch(MI, MCII))
1438       break;
1439 
1440     const MCOperand &Op = MI.getOperand(CurOp++);
1441     emitImmediate(Op, MI.getLoc(), X86II::getSizeOfImm(TSFlags),
1442                   MCFixupKind(X86::reloc_branch_4byte_pcrel), CurByte, OS,
1443                   Fixups);
1444     break;
1445   }
1446   case X86II::RawFrmMemOffs:
1447     emitByte(BaseOpcode, CurByte, OS);
1448     emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1449                   X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1450                   CurByte, OS, Fixups);
1451     ++CurOp; // skip segment operand
1452     break;
1453   case X86II::RawFrmImm8:
1454     emitByte(BaseOpcode, CurByte, OS);
1455     emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1456                   X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1457                   CurByte, OS, Fixups);
1458     emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1, FK_Data_1, CurByte,
1459                   OS, Fixups);
1460     break;
1461   case X86II::RawFrmImm16:
1462     emitByte(BaseOpcode, CurByte, OS);
1463     emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1464                   X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1465                   CurByte, OS, Fixups);
1466     emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 2, FK_Data_2, CurByte,
1467                   OS, Fixups);
1468     break;
1469 
1470   case X86II::AddRegFrm:
1471     emitByte(BaseOpcode + getX86RegNum(MI.getOperand(CurOp++)), CurByte, OS);
1472     break;
1473 
1474   case X86II::MRMDestReg: {
1475     emitByte(BaseOpcode, CurByte, OS);
1476     unsigned SrcRegNum = CurOp + 1;
1477 
1478     if (HasEVEX_K) // Skip writemask
1479       ++SrcRegNum;
1480 
1481     if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1482       ++SrcRegNum;
1483 
1484     emitRegModRMByte(MI.getOperand(CurOp),
1485                      getX86RegNum(MI.getOperand(SrcRegNum)), CurByte, OS);
1486     CurOp = SrcRegNum + 1;
1487     break;
1488   }
1489   case X86II::MRMDestMem: {
1490     emitByte(BaseOpcode, CurByte, OS);
1491     unsigned SrcRegNum = CurOp + X86::AddrNumOperands;
1492 
1493     if (HasEVEX_K) // Skip writemask
1494       ++SrcRegNum;
1495 
1496     if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1497       ++SrcRegNum;
1498 
1499     emitMemModRMByte(MI, CurOp, getX86RegNum(MI.getOperand(SrcRegNum)), TSFlags,
1500                      Rex, CurByte, OS, Fixups, STI);
1501     CurOp = SrcRegNum + 1;
1502     break;
1503   }
1504   case X86II::MRMSrcReg: {
1505     emitByte(BaseOpcode, CurByte, OS);
1506     unsigned SrcRegNum = CurOp + 1;
1507 
1508     if (HasEVEX_K) // Skip writemask
1509       ++SrcRegNum;
1510 
1511     if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1512       ++SrcRegNum;
1513 
1514     emitRegModRMByte(MI.getOperand(SrcRegNum),
1515                      getX86RegNum(MI.getOperand(CurOp)), CurByte, OS);
1516     CurOp = SrcRegNum + 1;
1517     if (HasVEX_I8Reg)
1518       I8RegNum = getX86RegEncoding(MI, CurOp++);
1519     // do not count the rounding control operand
1520     if (HasEVEX_RC)
1521       --NumOps;
1522     break;
1523   }
1524   case X86II::MRMSrcReg4VOp3: {
1525     emitByte(BaseOpcode, CurByte, OS);
1526     unsigned SrcRegNum = CurOp + 1;
1527 
1528     emitRegModRMByte(MI.getOperand(SrcRegNum),
1529                      getX86RegNum(MI.getOperand(CurOp)), CurByte, OS);
1530     CurOp = SrcRegNum + 1;
1531     ++CurOp; // Encoded in VEX.VVVV
1532     break;
1533   }
1534   case X86II::MRMSrcRegOp4: {
1535     emitByte(BaseOpcode, CurByte, OS);
1536     unsigned SrcRegNum = CurOp + 1;
1537 
1538     // Skip 1st src (which is encoded in VEX_VVVV)
1539     ++SrcRegNum;
1540 
1541     // Capture 2nd src (which is encoded in Imm[7:4])
1542     assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg");
1543     I8RegNum = getX86RegEncoding(MI, SrcRegNum++);
1544 
1545     emitRegModRMByte(MI.getOperand(SrcRegNum),
1546                      getX86RegNum(MI.getOperand(CurOp)), CurByte, OS);
1547     CurOp = SrcRegNum + 1;
1548     break;
1549   }
1550   case X86II::MRMSrcRegCC: {
1551     unsigned FirstOp = CurOp++;
1552     unsigned SecondOp = CurOp++;
1553 
1554     unsigned CC = MI.getOperand(CurOp++).getImm();
1555     emitByte(BaseOpcode + CC, CurByte, OS);
1556 
1557     emitRegModRMByte(MI.getOperand(SecondOp),
1558                      getX86RegNum(MI.getOperand(FirstOp)), CurByte, OS);
1559     break;
1560   }
1561   case X86II::MRMSrcMem: {
1562     unsigned FirstMemOp = CurOp + 1;
1563 
1564     if (HasEVEX_K) // Skip writemask
1565       ++FirstMemOp;
1566 
1567     if (HasVEX_4V)
1568       ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV).
1569 
1570     emitByte(BaseOpcode, CurByte, OS);
1571 
1572     emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1573                      TSFlags, Rex, CurByte, OS, Fixups, STI);
1574     CurOp = FirstMemOp + X86::AddrNumOperands;
1575     if (HasVEX_I8Reg)
1576       I8RegNum = getX86RegEncoding(MI, CurOp++);
1577     break;
1578   }
1579   case X86II::MRMSrcMem4VOp3: {
1580     unsigned FirstMemOp = CurOp + 1;
1581 
1582     emitByte(BaseOpcode, CurByte, OS);
1583 
1584     emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1585                      TSFlags, Rex, CurByte, OS, Fixups, STI);
1586     CurOp = FirstMemOp + X86::AddrNumOperands;
1587     ++CurOp; // Encoded in VEX.VVVV.
1588     break;
1589   }
1590   case X86II::MRMSrcMemOp4: {
1591     unsigned FirstMemOp = CurOp + 1;
1592 
1593     ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV).
1594 
1595     // Capture second register source (encoded in Imm[7:4])
1596     assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg");
1597     I8RegNum = getX86RegEncoding(MI, FirstMemOp++);
1598 
1599     emitByte(BaseOpcode, CurByte, OS);
1600 
1601     emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1602                      TSFlags, Rex, CurByte, OS, Fixups, STI);
1603     CurOp = FirstMemOp + X86::AddrNumOperands;
1604     break;
1605   }
1606   case X86II::MRMSrcMemCC: {
1607     unsigned RegOp = CurOp++;
1608     unsigned FirstMemOp = CurOp;
1609     CurOp = FirstMemOp + X86::AddrNumOperands;
1610 
1611     unsigned CC = MI.getOperand(CurOp++).getImm();
1612     emitByte(BaseOpcode + CC, CurByte, OS);
1613 
1614     emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(RegOp)),
1615                      TSFlags, Rex, CurByte, OS, Fixups, STI);
1616     break;
1617   }
1618 
1619   case X86II::MRMXrCC: {
1620     unsigned RegOp = CurOp++;
1621 
1622     unsigned CC = MI.getOperand(CurOp++).getImm();
1623     emitByte(BaseOpcode + CC, CurByte, OS);
1624     emitRegModRMByte(MI.getOperand(RegOp), 0, CurByte, OS);
1625     break;
1626   }
1627 
1628   case X86II::MRMXr:
1629   case X86II::MRM0r:
1630   case X86II::MRM1r:
1631   case X86II::MRM2r:
1632   case X86II::MRM3r:
1633   case X86II::MRM4r:
1634   case X86II::MRM5r:
1635   case X86II::MRM6r:
1636   case X86II::MRM7r:
1637     if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV).
1638       ++CurOp;
1639     if (HasEVEX_K) // Skip writemask
1640       ++CurOp;
1641     emitByte(BaseOpcode, CurByte, OS);
1642     emitRegModRMByte(MI.getOperand(CurOp++),
1643                      (Form == X86II::MRMXr) ? 0 : Form - X86II::MRM0r, CurByte,
1644                      OS);
1645     break;
1646 
1647   case X86II::MRMXmCC: {
1648     unsigned FirstMemOp = CurOp;
1649     CurOp = FirstMemOp + X86::AddrNumOperands;
1650 
1651     unsigned CC = MI.getOperand(CurOp++).getImm();
1652     emitByte(BaseOpcode + CC, CurByte, OS);
1653 
1654     emitMemModRMByte(MI, FirstMemOp, 0, TSFlags, Rex, CurByte, OS, Fixups, STI);
1655     break;
1656   }
1657 
1658   case X86II::MRMXm:
1659   case X86II::MRM0m:
1660   case X86II::MRM1m:
1661   case X86II::MRM2m:
1662   case X86II::MRM3m:
1663   case X86II::MRM4m:
1664   case X86II::MRM5m:
1665   case X86II::MRM6m:
1666   case X86II::MRM7m:
1667     if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV).
1668       ++CurOp;
1669     if (HasEVEX_K) // Skip writemask
1670       ++CurOp;
1671     emitByte(BaseOpcode, CurByte, OS);
1672     emitMemModRMByte(MI, CurOp,
1673                      (Form == X86II::MRMXm) ? 0 : Form - X86II::MRM0m, TSFlags,
1674                      Rex, CurByte, OS, Fixups, STI);
1675     CurOp += X86::AddrNumOperands;
1676     break;
1677 
1678   case X86II::MRM_C0:
1679   case X86II::MRM_C1:
1680   case X86II::MRM_C2:
1681   case X86II::MRM_C3:
1682   case X86II::MRM_C4:
1683   case X86II::MRM_C5:
1684   case X86II::MRM_C6:
1685   case X86II::MRM_C7:
1686   case X86II::MRM_C8:
1687   case X86II::MRM_C9:
1688   case X86II::MRM_CA:
1689   case X86II::MRM_CB:
1690   case X86II::MRM_CC:
1691   case X86II::MRM_CD:
1692   case X86II::MRM_CE:
1693   case X86II::MRM_CF:
1694   case X86II::MRM_D0:
1695   case X86II::MRM_D1:
1696   case X86II::MRM_D2:
1697   case X86II::MRM_D3:
1698   case X86II::MRM_D4:
1699   case X86II::MRM_D5:
1700   case X86II::MRM_D6:
1701   case X86II::MRM_D7:
1702   case X86II::MRM_D8:
1703   case X86II::MRM_D9:
1704   case X86II::MRM_DA:
1705   case X86II::MRM_DB:
1706   case X86II::MRM_DC:
1707   case X86II::MRM_DD:
1708   case X86II::MRM_DE:
1709   case X86II::MRM_DF:
1710   case X86II::MRM_E0:
1711   case X86II::MRM_E1:
1712   case X86II::MRM_E2:
1713   case X86II::MRM_E3:
1714   case X86II::MRM_E4:
1715   case X86II::MRM_E5:
1716   case X86II::MRM_E6:
1717   case X86II::MRM_E7:
1718   case X86II::MRM_E8:
1719   case X86II::MRM_E9:
1720   case X86II::MRM_EA:
1721   case X86II::MRM_EB:
1722   case X86II::MRM_EC:
1723   case X86II::MRM_ED:
1724   case X86II::MRM_EE:
1725   case X86II::MRM_EF:
1726   case X86II::MRM_F0:
1727   case X86II::MRM_F1:
1728   case X86II::MRM_F2:
1729   case X86II::MRM_F3:
1730   case X86II::MRM_F4:
1731   case X86II::MRM_F5:
1732   case X86II::MRM_F6:
1733   case X86II::MRM_F7:
1734   case X86II::MRM_F8:
1735   case X86II::MRM_F9:
1736   case X86II::MRM_FA:
1737   case X86II::MRM_FB:
1738   case X86II::MRM_FC:
1739   case X86II::MRM_FD:
1740   case X86II::MRM_FE:
1741   case X86II::MRM_FF:
1742     emitByte(BaseOpcode, CurByte, OS);
1743     emitByte(0xC0 + Form - X86II::MRM_C0, CurByte, OS);
1744     break;
1745   }
1746 
1747   if (HasVEX_I8Reg) {
1748     // The last source register of a 4 operand instruction in AVX is encoded
1749     // in bits[7:4] of a immediate byte.
1750     assert(I8RegNum < 16 && "Register encoding out of range");
1751     I8RegNum <<= 4;
1752     if (CurOp != NumOps) {
1753       unsigned Val = MI.getOperand(CurOp++).getImm();
1754       assert(Val < 16 && "Immediate operand value out of range");
1755       I8RegNum |= Val;
1756     }
1757     emitImmediate(MCOperand::createImm(I8RegNum), MI.getLoc(), 1, FK_Data_1,
1758                   CurByte, OS, Fixups);
1759   } else {
1760     // If there is a remaining operand, it must be a trailing immediate. Emit it
1761     // according to the right size for the instruction. Some instructions
1762     // (SSE4a extrq and insertq) have two trailing immediates.
1763     while (CurOp != NumOps && NumOps - CurOp <= 2) {
1764       emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1765                     X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1766                     CurByte, OS, Fixups);
1767     }
1768   }
1769 
1770   if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow)
1771     emitByte(X86II::getBaseOpcodeFor(TSFlags), CurByte, OS);
1772 
1773 #ifndef NDEBUG
1774   // FIXME: Verify.
1775   if (/*!Desc.isVariadic() &&*/ CurOp != NumOps) {
1776     errs() << "Cannot encode all operands of: ";
1777     MI.dump();
1778     errs() << '\n';
1779     abort();
1780   }
1781 #endif
1782 }
1783 
1784 MCCodeEmitter *llvm::createX86MCCodeEmitter(const MCInstrInfo &MCII,
1785                                             const MCRegisterInfo &MRI,
1786                                             MCContext &Ctx) {
1787   return new X86MCCodeEmitter(MCII, Ctx);
1788 }
1789