1 //===-- X86MCInstLower.cpp - Convert X86 MachineInstr to an MCInst --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains code to lower X86 MachineInstrs to their corresponding
10 // MCInst records.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "MCTargetDesc/X86ATTInstPrinter.h"
15 #include "MCTargetDesc/X86BaseInfo.h"
16 #include "MCTargetDesc/X86InstComments.h"
17 #include "MCTargetDesc/X86ShuffleDecode.h"
18 #include "MCTargetDesc/X86TargetStreamer.h"
19 #include "X86AsmPrinter.h"
20 #include "X86RegisterInfo.h"
21 #include "X86ShuffleDecodeConstantPool.h"
22 #include "X86Subtarget.h"
23 #include "llvm/ADT/Optional.h"
24 #include "llvm/ADT/SmallString.h"
25 #include "llvm/ADT/iterator_range.h"
26 #include "llvm/CodeGen/MachineConstantPool.h"
27 #include "llvm/CodeGen/MachineFunction.h"
28 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
29 #include "llvm/CodeGen/MachineOperand.h"
30 #include "llvm/CodeGen/StackMaps.h"
31 #include "llvm/IR/DataLayout.h"
32 #include "llvm/IR/GlobalValue.h"
33 #include "llvm/IR/Mangler.h"
34 #include "llvm/MC/MCAsmInfo.h"
35 #include "llvm/MC/MCCodeEmitter.h"
36 #include "llvm/MC/MCContext.h"
37 #include "llvm/MC/MCExpr.h"
38 #include "llvm/MC/MCFixup.h"
39 #include "llvm/MC/MCInst.h"
40 #include "llvm/MC/MCInstBuilder.h"
41 #include "llvm/MC/MCSection.h"
42 #include "llvm/MC/MCSectionELF.h"
43 #include "llvm/MC/MCStreamer.h"
44 #include "llvm/MC/MCSymbol.h"
45 #include "llvm/MC/MCSymbolELF.h"
46 #include "llvm/Support/TargetRegistry.h"
47 #include "llvm/Target/TargetLoweringObjectFile.h"
48 #include "llvm/Target/TargetMachine.h"
49 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
50 #include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
51 
52 using namespace llvm;
53 
54 namespace {
55 
56 /// X86MCInstLower - This class is used to lower an MachineInstr into an MCInst.
57 class X86MCInstLower {
58   MCContext &Ctx;
59   const MachineFunction &MF;
60   const TargetMachine &TM;
61   const MCAsmInfo &MAI;
62   X86AsmPrinter &AsmPrinter;
63 
64 public:
65   X86MCInstLower(const MachineFunction &MF, X86AsmPrinter &asmprinter);
66 
67   Optional<MCOperand> LowerMachineOperand(const MachineInstr *MI,
68                                           const MachineOperand &MO) const;
69   void Lower(const MachineInstr *MI, MCInst &OutMI) const;
70 
71   MCSymbol *GetSymbolFromOperand(const MachineOperand &MO) const;
72   MCOperand LowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym) const;
73 
74 private:
75   MachineModuleInfoMachO &getMachOMMI() const;
76 };
77 
78 } // end anonymous namespace
79 
80 /// A RAII helper which defines a region of instructions which can't have
81 /// padding added between them for correctness.
82 struct NoAutoPaddingScope {
83   MCStreamer &OS;
84   const bool OldAllowAutoPadding;
85   NoAutoPaddingScope(MCStreamer &OS)
86       : OS(OS), OldAllowAutoPadding(OS.getAllowAutoPadding()) {
87     changeAndComment(false);
88   }
89   ~NoAutoPaddingScope() { changeAndComment(OldAllowAutoPadding); }
90   void changeAndComment(bool b) {
91     if (b == OS.getAllowAutoPadding())
92       return;
93     OS.setAllowAutoPadding(b);
94     if (b)
95       OS.emitRawComment("autopadding");
96     else
97       OS.emitRawComment("noautopadding");
98   }
99 };
100 
101 // Emit a minimal sequence of nops spanning NumBytes bytes.
102 static void emitX86Nops(MCStreamer &OS, unsigned NumBytes,
103                         const X86Subtarget *Subtarget);
104 
105 void X86AsmPrinter::StackMapShadowTracker::count(MCInst &Inst,
106                                                  const MCSubtargetInfo &STI,
107                                                  MCCodeEmitter *CodeEmitter) {
108   if (InShadow) {
109     SmallString<256> Code;
110     SmallVector<MCFixup, 4> Fixups;
111     raw_svector_ostream VecOS(Code);
112     CodeEmitter->encodeInstruction(Inst, VecOS, Fixups, STI);
113     CurrentShadowSize += Code.size();
114     if (CurrentShadowSize >= RequiredShadowSize)
115       InShadow = false; // The shadow is big enough. Stop counting.
116   }
117 }
118 
119 void X86AsmPrinter::StackMapShadowTracker::emitShadowPadding(
120     MCStreamer &OutStreamer, const MCSubtargetInfo &STI) {
121   if (InShadow && CurrentShadowSize < RequiredShadowSize) {
122     InShadow = false;
123     emitX86Nops(OutStreamer, RequiredShadowSize - CurrentShadowSize,
124                 &MF->getSubtarget<X86Subtarget>());
125   }
126 }
127 
128 void X86AsmPrinter::EmitAndCountInstruction(MCInst &Inst) {
129   OutStreamer->emitInstruction(Inst, getSubtargetInfo());
130   SMShadowTracker.count(Inst, getSubtargetInfo(), CodeEmitter.get());
131 }
132 
133 X86MCInstLower::X86MCInstLower(const MachineFunction &mf,
134                                X86AsmPrinter &asmprinter)
135     : Ctx(mf.getContext()), MF(mf), TM(mf.getTarget()), MAI(*TM.getMCAsmInfo()),
136       AsmPrinter(asmprinter) {}
137 
138 MachineModuleInfoMachO &X86MCInstLower::getMachOMMI() const {
139   return MF.getMMI().getObjFileInfo<MachineModuleInfoMachO>();
140 }
141 
142 /// GetSymbolFromOperand - Lower an MO_GlobalAddress or MO_ExternalSymbol
143 /// operand to an MCSymbol.
144 MCSymbol *X86MCInstLower::GetSymbolFromOperand(const MachineOperand &MO) const {
145   const Triple &TT = TM.getTargetTriple();
146   if (MO.isGlobal() && TT.isOSBinFormatELF())
147     return AsmPrinter.getSymbolPreferLocal(*MO.getGlobal());
148 
149   const DataLayout &DL = MF.getDataLayout();
150   assert((MO.isGlobal() || MO.isSymbol() || MO.isMBB()) &&
151          "Isn't a symbol reference");
152 
153   MCSymbol *Sym = nullptr;
154   SmallString<128> Name;
155   StringRef Suffix;
156 
157   switch (MO.getTargetFlags()) {
158   case X86II::MO_DLLIMPORT:
159     // Handle dllimport linkage.
160     Name += "__imp_";
161     break;
162   case X86II::MO_COFFSTUB:
163     Name += ".refptr.";
164     break;
165   case X86II::MO_DARWIN_NONLAZY:
166   case X86II::MO_DARWIN_NONLAZY_PIC_BASE:
167     Suffix = "$non_lazy_ptr";
168     break;
169   }
170 
171   if (!Suffix.empty())
172     Name += DL.getPrivateGlobalPrefix();
173 
174   if (MO.isGlobal()) {
175     const GlobalValue *GV = MO.getGlobal();
176     AsmPrinter.getNameWithPrefix(Name, GV);
177   } else if (MO.isSymbol()) {
178     Mangler::getNameWithPrefix(Name, MO.getSymbolName(), DL);
179   } else if (MO.isMBB()) {
180     assert(Suffix.empty());
181     Sym = MO.getMBB()->getSymbol();
182   }
183 
184   Name += Suffix;
185   if (!Sym)
186     Sym = Ctx.getOrCreateSymbol(Name);
187 
188   // If the target flags on the operand changes the name of the symbol, do that
189   // before we return the symbol.
190   switch (MO.getTargetFlags()) {
191   default:
192     break;
193   case X86II::MO_COFFSTUB: {
194     MachineModuleInfoCOFF &MMICOFF =
195         MF.getMMI().getObjFileInfo<MachineModuleInfoCOFF>();
196     MachineModuleInfoImpl::StubValueTy &StubSym = MMICOFF.getGVStubEntry(Sym);
197     if (!StubSym.getPointer()) {
198       assert(MO.isGlobal() && "Extern symbol not handled yet");
199       StubSym = MachineModuleInfoImpl::StubValueTy(
200           AsmPrinter.getSymbol(MO.getGlobal()), true);
201     }
202     break;
203   }
204   case X86II::MO_DARWIN_NONLAZY:
205   case X86II::MO_DARWIN_NONLAZY_PIC_BASE: {
206     MachineModuleInfoImpl::StubValueTy &StubSym =
207         getMachOMMI().getGVStubEntry(Sym);
208     if (!StubSym.getPointer()) {
209       assert(MO.isGlobal() && "Extern symbol not handled yet");
210       StubSym = MachineModuleInfoImpl::StubValueTy(
211           AsmPrinter.getSymbol(MO.getGlobal()),
212           !MO.getGlobal()->hasInternalLinkage());
213     }
214     break;
215   }
216   }
217 
218   return Sym;
219 }
220 
221 MCOperand X86MCInstLower::LowerSymbolOperand(const MachineOperand &MO,
222                                              MCSymbol *Sym) const {
223   // FIXME: We would like an efficient form for this, so we don't have to do a
224   // lot of extra uniquing.
225   const MCExpr *Expr = nullptr;
226   MCSymbolRefExpr::VariantKind RefKind = MCSymbolRefExpr::VK_None;
227 
228   switch (MO.getTargetFlags()) {
229   default:
230     llvm_unreachable("Unknown target flag on GV operand");
231   case X86II::MO_NO_FLAG: // No flag.
232   // These affect the name of the symbol, not any suffix.
233   case X86II::MO_DARWIN_NONLAZY:
234   case X86II::MO_DLLIMPORT:
235   case X86II::MO_COFFSTUB:
236     break;
237 
238   case X86II::MO_TLVP:
239     RefKind = MCSymbolRefExpr::VK_TLVP;
240     break;
241   case X86II::MO_TLVP_PIC_BASE:
242     Expr = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_TLVP, Ctx);
243     // Subtract the pic base.
244     Expr = MCBinaryExpr::createSub(
245         Expr, MCSymbolRefExpr::create(MF.getPICBaseSymbol(), Ctx), Ctx);
246     break;
247   case X86II::MO_SECREL:
248     RefKind = MCSymbolRefExpr::VK_SECREL;
249     break;
250   case X86II::MO_TLSGD:
251     RefKind = MCSymbolRefExpr::VK_TLSGD;
252     break;
253   case X86II::MO_TLSLD:
254     RefKind = MCSymbolRefExpr::VK_TLSLD;
255     break;
256   case X86II::MO_TLSLDM:
257     RefKind = MCSymbolRefExpr::VK_TLSLDM;
258     break;
259   case X86II::MO_GOTTPOFF:
260     RefKind = MCSymbolRefExpr::VK_GOTTPOFF;
261     break;
262   case X86II::MO_INDNTPOFF:
263     RefKind = MCSymbolRefExpr::VK_INDNTPOFF;
264     break;
265   case X86II::MO_TPOFF:
266     RefKind = MCSymbolRefExpr::VK_TPOFF;
267     break;
268   case X86II::MO_DTPOFF:
269     RefKind = MCSymbolRefExpr::VK_DTPOFF;
270     break;
271   case X86II::MO_NTPOFF:
272     RefKind = MCSymbolRefExpr::VK_NTPOFF;
273     break;
274   case X86II::MO_GOTNTPOFF:
275     RefKind = MCSymbolRefExpr::VK_GOTNTPOFF;
276     break;
277   case X86II::MO_GOTPCREL:
278     RefKind = MCSymbolRefExpr::VK_GOTPCREL;
279     break;
280   case X86II::MO_GOT:
281     RefKind = MCSymbolRefExpr::VK_GOT;
282     break;
283   case X86II::MO_GOTOFF:
284     RefKind = MCSymbolRefExpr::VK_GOTOFF;
285     break;
286   case X86II::MO_PLT:
287     RefKind = MCSymbolRefExpr::VK_PLT;
288     break;
289   case X86II::MO_ABS8:
290     RefKind = MCSymbolRefExpr::VK_X86_ABS8;
291     break;
292   case X86II::MO_PIC_BASE_OFFSET:
293   case X86II::MO_DARWIN_NONLAZY_PIC_BASE:
294     Expr = MCSymbolRefExpr::create(Sym, Ctx);
295     // Subtract the pic base.
296     Expr = MCBinaryExpr::createSub(
297         Expr, MCSymbolRefExpr::create(MF.getPICBaseSymbol(), Ctx), Ctx);
298     if (MO.isJTI()) {
299       assert(MAI.doesSetDirectiveSuppressReloc());
300       // If .set directive is supported, use it to reduce the number of
301       // relocations the assembler will generate for differences between
302       // local labels. This is only safe when the symbols are in the same
303       // section so we are restricting it to jumptable references.
304       MCSymbol *Label = Ctx.createTempSymbol();
305       AsmPrinter.OutStreamer->emitAssignment(Label, Expr);
306       Expr = MCSymbolRefExpr::create(Label, Ctx);
307     }
308     break;
309   }
310 
311   if (!Expr)
312     Expr = MCSymbolRefExpr::create(Sym, RefKind, Ctx);
313 
314   if (!MO.isJTI() && !MO.isMBB() && MO.getOffset())
315     Expr = MCBinaryExpr::createAdd(
316         Expr, MCConstantExpr::create(MO.getOffset(), Ctx), Ctx);
317   return MCOperand::createExpr(Expr);
318 }
319 
320 /// Simplify FOO $imm, %{al,ax,eax,rax} to FOO $imm, for instruction with
321 /// a short fixed-register form.
322 static void SimplifyShortImmForm(MCInst &Inst, unsigned Opcode) {
323   unsigned ImmOp = Inst.getNumOperands() - 1;
324   assert(Inst.getOperand(0).isReg() &&
325          (Inst.getOperand(ImmOp).isImm() || Inst.getOperand(ImmOp).isExpr()) &&
326          ((Inst.getNumOperands() == 3 && Inst.getOperand(1).isReg() &&
327            Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg()) ||
328           Inst.getNumOperands() == 2) &&
329          "Unexpected instruction!");
330 
331   // Check whether the destination register can be fixed.
332   unsigned Reg = Inst.getOperand(0).getReg();
333   if (Reg != X86::AL && Reg != X86::AX && Reg != X86::EAX && Reg != X86::RAX)
334     return;
335 
336   // If so, rewrite the instruction.
337   MCOperand Saved = Inst.getOperand(ImmOp);
338   Inst = MCInst();
339   Inst.setOpcode(Opcode);
340   Inst.addOperand(Saved);
341 }
342 
343 /// If a movsx instruction has a shorter encoding for the used register
344 /// simplify the instruction to use it instead.
345 static void SimplifyMOVSX(MCInst &Inst) {
346   unsigned NewOpcode = 0;
347   unsigned Op0 = Inst.getOperand(0).getReg(), Op1 = Inst.getOperand(1).getReg();
348   switch (Inst.getOpcode()) {
349   default:
350     llvm_unreachable("Unexpected instruction!");
351   case X86::MOVSX16rr8: // movsbw %al, %ax   --> cbtw
352     if (Op0 == X86::AX && Op1 == X86::AL)
353       NewOpcode = X86::CBW;
354     break;
355   case X86::MOVSX32rr16: // movswl %ax, %eax  --> cwtl
356     if (Op0 == X86::EAX && Op1 == X86::AX)
357       NewOpcode = X86::CWDE;
358     break;
359   case X86::MOVSX64rr32: // movslq %eax, %rax --> cltq
360     if (Op0 == X86::RAX && Op1 == X86::EAX)
361       NewOpcode = X86::CDQE;
362     break;
363   }
364 
365   if (NewOpcode != 0) {
366     Inst = MCInst();
367     Inst.setOpcode(NewOpcode);
368   }
369 }
370 
371 /// Simplify things like MOV32rm to MOV32o32a.
372 static void SimplifyShortMoveForm(X86AsmPrinter &Printer, MCInst &Inst,
373                                   unsigned Opcode) {
374   // Don't make these simplifications in 64-bit mode; other assemblers don't
375   // perform them because they make the code larger.
376   if (Printer.getSubtarget().is64Bit())
377     return;
378 
379   bool IsStore = Inst.getOperand(0).isReg() && Inst.getOperand(1).isReg();
380   unsigned AddrBase = IsStore;
381   unsigned RegOp = IsStore ? 0 : 5;
382   unsigned AddrOp = AddrBase + 3;
383   assert(
384       Inst.getNumOperands() == 6 && Inst.getOperand(RegOp).isReg() &&
385       Inst.getOperand(AddrBase + X86::AddrBaseReg).isReg() &&
386       Inst.getOperand(AddrBase + X86::AddrScaleAmt).isImm() &&
387       Inst.getOperand(AddrBase + X86::AddrIndexReg).isReg() &&
388       Inst.getOperand(AddrBase + X86::AddrSegmentReg).isReg() &&
389       (Inst.getOperand(AddrOp).isExpr() || Inst.getOperand(AddrOp).isImm()) &&
390       "Unexpected instruction!");
391 
392   // Check whether the destination register can be fixed.
393   unsigned Reg = Inst.getOperand(RegOp).getReg();
394   if (Reg != X86::AL && Reg != X86::AX && Reg != X86::EAX && Reg != X86::RAX)
395     return;
396 
397   // Check whether this is an absolute address.
398   // FIXME: We know TLVP symbol refs aren't, but there should be a better way
399   // to do this here.
400   bool Absolute = true;
401   if (Inst.getOperand(AddrOp).isExpr()) {
402     const MCExpr *MCE = Inst.getOperand(AddrOp).getExpr();
403     if (const MCSymbolRefExpr *SRE = dyn_cast<MCSymbolRefExpr>(MCE))
404       if (SRE->getKind() == MCSymbolRefExpr::VK_TLVP)
405         Absolute = false;
406   }
407 
408   if (Absolute &&
409       (Inst.getOperand(AddrBase + X86::AddrBaseReg).getReg() != 0 ||
410        Inst.getOperand(AddrBase + X86::AddrScaleAmt).getImm() != 1 ||
411        Inst.getOperand(AddrBase + X86::AddrIndexReg).getReg() != 0))
412     return;
413 
414   // If so, rewrite the instruction.
415   MCOperand Saved = Inst.getOperand(AddrOp);
416   MCOperand Seg = Inst.getOperand(AddrBase + X86::AddrSegmentReg);
417   Inst = MCInst();
418   Inst.setOpcode(Opcode);
419   Inst.addOperand(Saved);
420   Inst.addOperand(Seg);
421 }
422 
423 static unsigned getRetOpcode(const X86Subtarget &Subtarget) {
424   return Subtarget.is64Bit() ? X86::RETQ : X86::RETL;
425 }
426 
427 Optional<MCOperand>
428 X86MCInstLower::LowerMachineOperand(const MachineInstr *MI,
429                                     const MachineOperand &MO) const {
430   switch (MO.getType()) {
431   default:
432     MI->print(errs());
433     llvm_unreachable("unknown operand type");
434   case MachineOperand::MO_Register:
435     // Ignore all implicit register operands.
436     if (MO.isImplicit())
437       return None;
438     return MCOperand::createReg(MO.getReg());
439   case MachineOperand::MO_Immediate:
440     return MCOperand::createImm(MO.getImm());
441   case MachineOperand::MO_MachineBasicBlock:
442   case MachineOperand::MO_GlobalAddress:
443   case MachineOperand::MO_ExternalSymbol:
444     return LowerSymbolOperand(MO, GetSymbolFromOperand(MO));
445   case MachineOperand::MO_MCSymbol:
446     return LowerSymbolOperand(MO, MO.getMCSymbol());
447   case MachineOperand::MO_JumpTableIndex:
448     return LowerSymbolOperand(MO, AsmPrinter.GetJTISymbol(MO.getIndex()));
449   case MachineOperand::MO_ConstantPoolIndex:
450     return LowerSymbolOperand(MO, AsmPrinter.GetCPISymbol(MO.getIndex()));
451   case MachineOperand::MO_BlockAddress:
452     return LowerSymbolOperand(
453         MO, AsmPrinter.GetBlockAddressSymbol(MO.getBlockAddress()));
454   case MachineOperand::MO_RegisterMask:
455     // Ignore call clobbers.
456     return None;
457   }
458 }
459 
460 // Replace TAILJMP opcodes with their equivalent opcodes that have encoding
461 // information.
462 static unsigned convertTailJumpOpcode(unsigned Opcode) {
463   switch (Opcode) {
464   case X86::TAILJMPr:
465     Opcode = X86::JMP32r;
466     break;
467   case X86::TAILJMPm:
468     Opcode = X86::JMP32m;
469     break;
470   case X86::TAILJMPr64:
471     Opcode = X86::JMP64r;
472     break;
473   case X86::TAILJMPm64:
474     Opcode = X86::JMP64m;
475     break;
476   case X86::TAILJMPr64_REX:
477     Opcode = X86::JMP64r_REX;
478     break;
479   case X86::TAILJMPm64_REX:
480     Opcode = X86::JMP64m_REX;
481     break;
482   case X86::TAILJMPd:
483   case X86::TAILJMPd64:
484     Opcode = X86::JMP_1;
485     break;
486   case X86::TAILJMPd_CC:
487   case X86::TAILJMPd64_CC:
488     Opcode = X86::JCC_1;
489     break;
490   }
491 
492   return Opcode;
493 }
494 
495 void X86MCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
496   OutMI.setOpcode(MI->getOpcode());
497 
498   for (const MachineOperand &MO : MI->operands())
499     if (auto MaybeMCOp = LowerMachineOperand(MI, MO))
500       OutMI.addOperand(MaybeMCOp.getValue());
501 
502   // Handle a few special cases to eliminate operand modifiers.
503   switch (OutMI.getOpcode()) {
504   case X86::LEA64_32r:
505   case X86::LEA64r:
506   case X86::LEA16r:
507   case X86::LEA32r:
508     // LEA should have a segment register, but it must be empty.
509     assert(OutMI.getNumOperands() == 1 + X86::AddrNumOperands &&
510            "Unexpected # of LEA operands");
511     assert(OutMI.getOperand(1 + X86::AddrSegmentReg).getReg() == 0 &&
512            "LEA has segment specified!");
513     break;
514 
515   case X86::MULX32Hrr:
516   case X86::MULX32Hrm:
517   case X86::MULX64Hrr:
518   case X86::MULX64Hrm: {
519     // Turn into regular MULX by duplicating the destination.
520     unsigned NewOpc;
521     switch (OutMI.getOpcode()) {
522     default: llvm_unreachable("Invalid opcode");
523     case X86::MULX32Hrr: NewOpc = X86::MULX32rr; break;
524     case X86::MULX32Hrm: NewOpc = X86::MULX32rm; break;
525     case X86::MULX64Hrr: NewOpc = X86::MULX64rr; break;
526     case X86::MULX64Hrm: NewOpc = X86::MULX64rm; break;
527     }
528     OutMI.setOpcode(NewOpc);
529     // Duplicate the destination.
530     unsigned DestReg = OutMI.getOperand(0).getReg();
531     OutMI.insert(OutMI.begin(), MCOperand::createReg(DestReg));
532     break;
533   }
534 
535   // Commute operands to get a smaller encoding by using VEX.R instead of VEX.B
536   // if one of the registers is extended, but other isn't.
537   case X86::VMOVZPQILo2PQIrr:
538   case X86::VMOVAPDrr:
539   case X86::VMOVAPDYrr:
540   case X86::VMOVAPSrr:
541   case X86::VMOVAPSYrr:
542   case X86::VMOVDQArr:
543   case X86::VMOVDQAYrr:
544   case X86::VMOVDQUrr:
545   case X86::VMOVDQUYrr:
546   case X86::VMOVUPDrr:
547   case X86::VMOVUPDYrr:
548   case X86::VMOVUPSrr:
549   case X86::VMOVUPSYrr: {
550     if (!X86II::isX86_64ExtendedReg(OutMI.getOperand(0).getReg()) &&
551         X86II::isX86_64ExtendedReg(OutMI.getOperand(1).getReg())) {
552       unsigned NewOpc;
553       switch (OutMI.getOpcode()) {
554       default: llvm_unreachable("Invalid opcode");
555       case X86::VMOVZPQILo2PQIrr: NewOpc = X86::VMOVPQI2QIrr;   break;
556       case X86::VMOVAPDrr:        NewOpc = X86::VMOVAPDrr_REV;  break;
557       case X86::VMOVAPDYrr:       NewOpc = X86::VMOVAPDYrr_REV; break;
558       case X86::VMOVAPSrr:        NewOpc = X86::VMOVAPSrr_REV;  break;
559       case X86::VMOVAPSYrr:       NewOpc = X86::VMOVAPSYrr_REV; break;
560       case X86::VMOVDQArr:        NewOpc = X86::VMOVDQArr_REV;  break;
561       case X86::VMOVDQAYrr:       NewOpc = X86::VMOVDQAYrr_REV; break;
562       case X86::VMOVDQUrr:        NewOpc = X86::VMOVDQUrr_REV;  break;
563       case X86::VMOVDQUYrr:       NewOpc = X86::VMOVDQUYrr_REV; break;
564       case X86::VMOVUPDrr:        NewOpc = X86::VMOVUPDrr_REV;  break;
565       case X86::VMOVUPDYrr:       NewOpc = X86::VMOVUPDYrr_REV; break;
566       case X86::VMOVUPSrr:        NewOpc = X86::VMOVUPSrr_REV;  break;
567       case X86::VMOVUPSYrr:       NewOpc = X86::VMOVUPSYrr_REV; break;
568       }
569       OutMI.setOpcode(NewOpc);
570     }
571     break;
572   }
573   case X86::VMOVSDrr:
574   case X86::VMOVSSrr: {
575     if (!X86II::isX86_64ExtendedReg(OutMI.getOperand(0).getReg()) &&
576         X86II::isX86_64ExtendedReg(OutMI.getOperand(2).getReg())) {
577       unsigned NewOpc;
578       switch (OutMI.getOpcode()) {
579       default: llvm_unreachable("Invalid opcode");
580       case X86::VMOVSDrr: NewOpc = X86::VMOVSDrr_REV; break;
581       case X86::VMOVSSrr: NewOpc = X86::VMOVSSrr_REV; break;
582       }
583       OutMI.setOpcode(NewOpc);
584     }
585     break;
586   }
587 
588   case X86::VPCMPBZ128rmi:  case X86::VPCMPBZ128rmik:
589   case X86::VPCMPBZ128rri:  case X86::VPCMPBZ128rrik:
590   case X86::VPCMPBZ256rmi:  case X86::VPCMPBZ256rmik:
591   case X86::VPCMPBZ256rri:  case X86::VPCMPBZ256rrik:
592   case X86::VPCMPBZrmi:     case X86::VPCMPBZrmik:
593   case X86::VPCMPBZrri:     case X86::VPCMPBZrrik:
594   case X86::VPCMPDZ128rmi:  case X86::VPCMPDZ128rmik:
595   case X86::VPCMPDZ128rmib: case X86::VPCMPDZ128rmibk:
596   case X86::VPCMPDZ128rri:  case X86::VPCMPDZ128rrik:
597   case X86::VPCMPDZ256rmi:  case X86::VPCMPDZ256rmik:
598   case X86::VPCMPDZ256rmib: case X86::VPCMPDZ256rmibk:
599   case X86::VPCMPDZ256rri:  case X86::VPCMPDZ256rrik:
600   case X86::VPCMPDZrmi:     case X86::VPCMPDZrmik:
601   case X86::VPCMPDZrmib:    case X86::VPCMPDZrmibk:
602   case X86::VPCMPDZrri:     case X86::VPCMPDZrrik:
603   case X86::VPCMPQZ128rmi:  case X86::VPCMPQZ128rmik:
604   case X86::VPCMPQZ128rmib: case X86::VPCMPQZ128rmibk:
605   case X86::VPCMPQZ128rri:  case X86::VPCMPQZ128rrik:
606   case X86::VPCMPQZ256rmi:  case X86::VPCMPQZ256rmik:
607   case X86::VPCMPQZ256rmib: case X86::VPCMPQZ256rmibk:
608   case X86::VPCMPQZ256rri:  case X86::VPCMPQZ256rrik:
609   case X86::VPCMPQZrmi:     case X86::VPCMPQZrmik:
610   case X86::VPCMPQZrmib:    case X86::VPCMPQZrmibk:
611   case X86::VPCMPQZrri:     case X86::VPCMPQZrrik:
612   case X86::VPCMPWZ128rmi:  case X86::VPCMPWZ128rmik:
613   case X86::VPCMPWZ128rri:  case X86::VPCMPWZ128rrik:
614   case X86::VPCMPWZ256rmi:  case X86::VPCMPWZ256rmik:
615   case X86::VPCMPWZ256rri:  case X86::VPCMPWZ256rrik:
616   case X86::VPCMPWZrmi:     case X86::VPCMPWZrmik:
617   case X86::VPCMPWZrri:     case X86::VPCMPWZrrik: {
618     // Turn immediate 0 into the VPCMPEQ instruction.
619     if (OutMI.getOperand(OutMI.getNumOperands() - 1).getImm() == 0) {
620       unsigned NewOpc;
621       switch (OutMI.getOpcode()) {
622       default: llvm_unreachable("Invalid opcode");
623       case X86::VPCMPBZ128rmi:   NewOpc = X86::VPCMPEQBZ128rm;   break;
624       case X86::VPCMPBZ128rmik:  NewOpc = X86::VPCMPEQBZ128rmk;  break;
625       case X86::VPCMPBZ128rri:   NewOpc = X86::VPCMPEQBZ128rr;   break;
626       case X86::VPCMPBZ128rrik:  NewOpc = X86::VPCMPEQBZ128rrk;  break;
627       case X86::VPCMPBZ256rmi:   NewOpc = X86::VPCMPEQBZ256rm;   break;
628       case X86::VPCMPBZ256rmik:  NewOpc = X86::VPCMPEQBZ256rmk;  break;
629       case X86::VPCMPBZ256rri:   NewOpc = X86::VPCMPEQBZ256rr;   break;
630       case X86::VPCMPBZ256rrik:  NewOpc = X86::VPCMPEQBZ256rrk;  break;
631       case X86::VPCMPBZrmi:      NewOpc = X86::VPCMPEQBZrm;      break;
632       case X86::VPCMPBZrmik:     NewOpc = X86::VPCMPEQBZrmk;     break;
633       case X86::VPCMPBZrri:      NewOpc = X86::VPCMPEQBZrr;      break;
634       case X86::VPCMPBZrrik:     NewOpc = X86::VPCMPEQBZrrk;     break;
635       case X86::VPCMPDZ128rmi:   NewOpc = X86::VPCMPEQDZ128rm;   break;
636       case X86::VPCMPDZ128rmib:  NewOpc = X86::VPCMPEQDZ128rmb;  break;
637       case X86::VPCMPDZ128rmibk: NewOpc = X86::VPCMPEQDZ128rmbk; break;
638       case X86::VPCMPDZ128rmik:  NewOpc = X86::VPCMPEQDZ128rmk;  break;
639       case X86::VPCMPDZ128rri:   NewOpc = X86::VPCMPEQDZ128rr;   break;
640       case X86::VPCMPDZ128rrik:  NewOpc = X86::VPCMPEQDZ128rrk;  break;
641       case X86::VPCMPDZ256rmi:   NewOpc = X86::VPCMPEQDZ256rm;   break;
642       case X86::VPCMPDZ256rmib:  NewOpc = X86::VPCMPEQDZ256rmb;  break;
643       case X86::VPCMPDZ256rmibk: NewOpc = X86::VPCMPEQDZ256rmbk; break;
644       case X86::VPCMPDZ256rmik:  NewOpc = X86::VPCMPEQDZ256rmk;  break;
645       case X86::VPCMPDZ256rri:   NewOpc = X86::VPCMPEQDZ256rr;   break;
646       case X86::VPCMPDZ256rrik:  NewOpc = X86::VPCMPEQDZ256rrk;  break;
647       case X86::VPCMPDZrmi:      NewOpc = X86::VPCMPEQDZrm;      break;
648       case X86::VPCMPDZrmib:     NewOpc = X86::VPCMPEQDZrmb;     break;
649       case X86::VPCMPDZrmibk:    NewOpc = X86::VPCMPEQDZrmbk;    break;
650       case X86::VPCMPDZrmik:     NewOpc = X86::VPCMPEQDZrmk;     break;
651       case X86::VPCMPDZrri:      NewOpc = X86::VPCMPEQDZrr;      break;
652       case X86::VPCMPDZrrik:     NewOpc = X86::VPCMPEQDZrrk;     break;
653       case X86::VPCMPQZ128rmi:   NewOpc = X86::VPCMPEQQZ128rm;   break;
654       case X86::VPCMPQZ128rmib:  NewOpc = X86::VPCMPEQQZ128rmb;  break;
655       case X86::VPCMPQZ128rmibk: NewOpc = X86::VPCMPEQQZ128rmbk; break;
656       case X86::VPCMPQZ128rmik:  NewOpc = X86::VPCMPEQQZ128rmk;  break;
657       case X86::VPCMPQZ128rri:   NewOpc = X86::VPCMPEQQZ128rr;   break;
658       case X86::VPCMPQZ128rrik:  NewOpc = X86::VPCMPEQQZ128rrk;  break;
659       case X86::VPCMPQZ256rmi:   NewOpc = X86::VPCMPEQQZ256rm;   break;
660       case X86::VPCMPQZ256rmib:  NewOpc = X86::VPCMPEQQZ256rmb;  break;
661       case X86::VPCMPQZ256rmibk: NewOpc = X86::VPCMPEQQZ256rmbk; break;
662       case X86::VPCMPQZ256rmik:  NewOpc = X86::VPCMPEQQZ256rmk;  break;
663       case X86::VPCMPQZ256rri:   NewOpc = X86::VPCMPEQQZ256rr;   break;
664       case X86::VPCMPQZ256rrik:  NewOpc = X86::VPCMPEQQZ256rrk;  break;
665       case X86::VPCMPQZrmi:      NewOpc = X86::VPCMPEQQZrm;      break;
666       case X86::VPCMPQZrmib:     NewOpc = X86::VPCMPEQQZrmb;     break;
667       case X86::VPCMPQZrmibk:    NewOpc = X86::VPCMPEQQZrmbk;    break;
668       case X86::VPCMPQZrmik:     NewOpc = X86::VPCMPEQQZrmk;     break;
669       case X86::VPCMPQZrri:      NewOpc = X86::VPCMPEQQZrr;      break;
670       case X86::VPCMPQZrrik:     NewOpc = X86::VPCMPEQQZrrk;     break;
671       case X86::VPCMPWZ128rmi:   NewOpc = X86::VPCMPEQWZ128rm;   break;
672       case X86::VPCMPWZ128rmik:  NewOpc = X86::VPCMPEQWZ128rmk;  break;
673       case X86::VPCMPWZ128rri:   NewOpc = X86::VPCMPEQWZ128rr;   break;
674       case X86::VPCMPWZ128rrik:  NewOpc = X86::VPCMPEQWZ128rrk;  break;
675       case X86::VPCMPWZ256rmi:   NewOpc = X86::VPCMPEQWZ256rm;   break;
676       case X86::VPCMPWZ256rmik:  NewOpc = X86::VPCMPEQWZ256rmk;  break;
677       case X86::VPCMPWZ256rri:   NewOpc = X86::VPCMPEQWZ256rr;   break;
678       case X86::VPCMPWZ256rrik:  NewOpc = X86::VPCMPEQWZ256rrk;  break;
679       case X86::VPCMPWZrmi:      NewOpc = X86::VPCMPEQWZrm;      break;
680       case X86::VPCMPWZrmik:     NewOpc = X86::VPCMPEQWZrmk;     break;
681       case X86::VPCMPWZrri:      NewOpc = X86::VPCMPEQWZrr;      break;
682       case X86::VPCMPWZrrik:     NewOpc = X86::VPCMPEQWZrrk;     break;
683       }
684 
685       OutMI.setOpcode(NewOpc);
686       OutMI.erase(&OutMI.getOperand(OutMI.getNumOperands() - 1));
687       break;
688     }
689 
690     // Turn immediate 6 into the VPCMPGT instruction.
691     if (OutMI.getOperand(OutMI.getNumOperands() - 1).getImm() == 6) {
692       unsigned NewOpc;
693       switch (OutMI.getOpcode()) {
694       default: llvm_unreachable("Invalid opcode");
695       case X86::VPCMPBZ128rmi:   NewOpc = X86::VPCMPGTBZ128rm;   break;
696       case X86::VPCMPBZ128rmik:  NewOpc = X86::VPCMPGTBZ128rmk;  break;
697       case X86::VPCMPBZ128rri:   NewOpc = X86::VPCMPGTBZ128rr;   break;
698       case X86::VPCMPBZ128rrik:  NewOpc = X86::VPCMPGTBZ128rrk;  break;
699       case X86::VPCMPBZ256rmi:   NewOpc = X86::VPCMPGTBZ256rm;   break;
700       case X86::VPCMPBZ256rmik:  NewOpc = X86::VPCMPGTBZ256rmk;  break;
701       case X86::VPCMPBZ256rri:   NewOpc = X86::VPCMPGTBZ256rr;   break;
702       case X86::VPCMPBZ256rrik:  NewOpc = X86::VPCMPGTBZ256rrk;  break;
703       case X86::VPCMPBZrmi:      NewOpc = X86::VPCMPGTBZrm;      break;
704       case X86::VPCMPBZrmik:     NewOpc = X86::VPCMPGTBZrmk;     break;
705       case X86::VPCMPBZrri:      NewOpc = X86::VPCMPGTBZrr;      break;
706       case X86::VPCMPBZrrik:     NewOpc = X86::VPCMPGTBZrrk;     break;
707       case X86::VPCMPDZ128rmi:   NewOpc = X86::VPCMPGTDZ128rm;   break;
708       case X86::VPCMPDZ128rmib:  NewOpc = X86::VPCMPGTDZ128rmb;  break;
709       case X86::VPCMPDZ128rmibk: NewOpc = X86::VPCMPGTDZ128rmbk; break;
710       case X86::VPCMPDZ128rmik:  NewOpc = X86::VPCMPGTDZ128rmk;  break;
711       case X86::VPCMPDZ128rri:   NewOpc = X86::VPCMPGTDZ128rr;   break;
712       case X86::VPCMPDZ128rrik:  NewOpc = X86::VPCMPGTDZ128rrk;  break;
713       case X86::VPCMPDZ256rmi:   NewOpc = X86::VPCMPGTDZ256rm;   break;
714       case X86::VPCMPDZ256rmib:  NewOpc = X86::VPCMPGTDZ256rmb;  break;
715       case X86::VPCMPDZ256rmibk: NewOpc = X86::VPCMPGTDZ256rmbk; break;
716       case X86::VPCMPDZ256rmik:  NewOpc = X86::VPCMPGTDZ256rmk;  break;
717       case X86::VPCMPDZ256rri:   NewOpc = X86::VPCMPGTDZ256rr;   break;
718       case X86::VPCMPDZ256rrik:  NewOpc = X86::VPCMPGTDZ256rrk;  break;
719       case X86::VPCMPDZrmi:      NewOpc = X86::VPCMPGTDZrm;      break;
720       case X86::VPCMPDZrmib:     NewOpc = X86::VPCMPGTDZrmb;     break;
721       case X86::VPCMPDZrmibk:    NewOpc = X86::VPCMPGTDZrmbk;    break;
722       case X86::VPCMPDZrmik:     NewOpc = X86::VPCMPGTDZrmk;     break;
723       case X86::VPCMPDZrri:      NewOpc = X86::VPCMPGTDZrr;      break;
724       case X86::VPCMPDZrrik:     NewOpc = X86::VPCMPGTDZrrk;     break;
725       case X86::VPCMPQZ128rmi:   NewOpc = X86::VPCMPGTQZ128rm;   break;
726       case X86::VPCMPQZ128rmib:  NewOpc = X86::VPCMPGTQZ128rmb;  break;
727       case X86::VPCMPQZ128rmibk: NewOpc = X86::VPCMPGTQZ128rmbk; break;
728       case X86::VPCMPQZ128rmik:  NewOpc = X86::VPCMPGTQZ128rmk;  break;
729       case X86::VPCMPQZ128rri:   NewOpc = X86::VPCMPGTQZ128rr;   break;
730       case X86::VPCMPQZ128rrik:  NewOpc = X86::VPCMPGTQZ128rrk;  break;
731       case X86::VPCMPQZ256rmi:   NewOpc = X86::VPCMPGTQZ256rm;   break;
732       case X86::VPCMPQZ256rmib:  NewOpc = X86::VPCMPGTQZ256rmb;  break;
733       case X86::VPCMPQZ256rmibk: NewOpc = X86::VPCMPGTQZ256rmbk; break;
734       case X86::VPCMPQZ256rmik:  NewOpc = X86::VPCMPGTQZ256rmk;  break;
735       case X86::VPCMPQZ256rri:   NewOpc = X86::VPCMPGTQZ256rr;   break;
736       case X86::VPCMPQZ256rrik:  NewOpc = X86::VPCMPGTQZ256rrk;  break;
737       case X86::VPCMPQZrmi:      NewOpc = X86::VPCMPGTQZrm;      break;
738       case X86::VPCMPQZrmib:     NewOpc = X86::VPCMPGTQZrmb;     break;
739       case X86::VPCMPQZrmibk:    NewOpc = X86::VPCMPGTQZrmbk;    break;
740       case X86::VPCMPQZrmik:     NewOpc = X86::VPCMPGTQZrmk;     break;
741       case X86::VPCMPQZrri:      NewOpc = X86::VPCMPGTQZrr;      break;
742       case X86::VPCMPQZrrik:     NewOpc = X86::VPCMPGTQZrrk;     break;
743       case X86::VPCMPWZ128rmi:   NewOpc = X86::VPCMPGTWZ128rm;   break;
744       case X86::VPCMPWZ128rmik:  NewOpc = X86::VPCMPGTWZ128rmk;  break;
745       case X86::VPCMPWZ128rri:   NewOpc = X86::VPCMPGTWZ128rr;   break;
746       case X86::VPCMPWZ128rrik:  NewOpc = X86::VPCMPGTWZ128rrk;  break;
747       case X86::VPCMPWZ256rmi:   NewOpc = X86::VPCMPGTWZ256rm;   break;
748       case X86::VPCMPWZ256rmik:  NewOpc = X86::VPCMPGTWZ256rmk;  break;
749       case X86::VPCMPWZ256rri:   NewOpc = X86::VPCMPGTWZ256rr;   break;
750       case X86::VPCMPWZ256rrik:  NewOpc = X86::VPCMPGTWZ256rrk;  break;
751       case X86::VPCMPWZrmi:      NewOpc = X86::VPCMPGTWZrm;      break;
752       case X86::VPCMPWZrmik:     NewOpc = X86::VPCMPGTWZrmk;     break;
753       case X86::VPCMPWZrri:      NewOpc = X86::VPCMPGTWZrr;      break;
754       case X86::VPCMPWZrrik:     NewOpc = X86::VPCMPGTWZrrk;     break;
755       }
756 
757       OutMI.setOpcode(NewOpc);
758       OutMI.erase(&OutMI.getOperand(OutMI.getNumOperands() - 1));
759       break;
760     }
761 
762     break;
763   }
764 
765   // CALL64r, CALL64pcrel32 - These instructions used to have
766   // register inputs modeled as normal uses instead of implicit uses.  As such,
767   // they we used to truncate off all but the first operand (the callee). This
768   // issue seems to have been fixed at some point. This assert verifies that.
769   case X86::CALL64r:
770   case X86::CALL64pcrel32:
771     assert(OutMI.getNumOperands() == 1 && "Unexpected number of operands!");
772     break;
773 
774   case X86::EH_RETURN:
775   case X86::EH_RETURN64: {
776     OutMI = MCInst();
777     OutMI.setOpcode(getRetOpcode(AsmPrinter.getSubtarget()));
778     break;
779   }
780 
781   case X86::CLEANUPRET: {
782     // Replace CLEANUPRET with the appropriate RET.
783     OutMI = MCInst();
784     OutMI.setOpcode(getRetOpcode(AsmPrinter.getSubtarget()));
785     break;
786   }
787 
788   case X86::CATCHRET: {
789     // Replace CATCHRET with the appropriate RET.
790     const X86Subtarget &Subtarget = AsmPrinter.getSubtarget();
791     unsigned ReturnReg = Subtarget.is64Bit() ? X86::RAX : X86::EAX;
792     OutMI = MCInst();
793     OutMI.setOpcode(getRetOpcode(Subtarget));
794     OutMI.addOperand(MCOperand::createReg(ReturnReg));
795     break;
796   }
797 
798   // TAILJMPd, TAILJMPd64, TailJMPd_cc - Lower to the correct jump
799   // instruction.
800   case X86::TAILJMPr:
801   case X86::TAILJMPr64:
802   case X86::TAILJMPr64_REX:
803   case X86::TAILJMPd:
804   case X86::TAILJMPd64:
805     assert(OutMI.getNumOperands() == 1 && "Unexpected number of operands!");
806     OutMI.setOpcode(convertTailJumpOpcode(OutMI.getOpcode()));
807     break;
808 
809   case X86::TAILJMPd_CC:
810   case X86::TAILJMPd64_CC:
811     assert(OutMI.getNumOperands() == 2 && "Unexpected number of operands!");
812     OutMI.setOpcode(convertTailJumpOpcode(OutMI.getOpcode()));
813     break;
814 
815   case X86::TAILJMPm:
816   case X86::TAILJMPm64:
817   case X86::TAILJMPm64_REX:
818     assert(OutMI.getNumOperands() == X86::AddrNumOperands &&
819            "Unexpected number of operands!");
820     OutMI.setOpcode(convertTailJumpOpcode(OutMI.getOpcode()));
821     break;
822 
823   case X86::DEC16r:
824   case X86::DEC32r:
825   case X86::INC16r:
826   case X86::INC32r:
827     // If we aren't in 64-bit mode we can use the 1-byte inc/dec instructions.
828     if (!AsmPrinter.getSubtarget().is64Bit()) {
829       unsigned Opcode;
830       switch (OutMI.getOpcode()) {
831       default: llvm_unreachable("Invalid opcode");
832       case X86::DEC16r: Opcode = X86::DEC16r_alt; break;
833       case X86::DEC32r: Opcode = X86::DEC32r_alt; break;
834       case X86::INC16r: Opcode = X86::INC16r_alt; break;
835       case X86::INC32r: Opcode = X86::INC32r_alt; break;
836       }
837       OutMI.setOpcode(Opcode);
838     }
839     break;
840 
841   // We don't currently select the correct instruction form for instructions
842   // which have a short %eax, etc. form. Handle this by custom lowering, for
843   // now.
844   //
845   // Note, we are currently not handling the following instructions:
846   // MOV64ao8, MOV64o8a
847   // XCHG16ar, XCHG32ar, XCHG64ar
848   case X86::MOV8mr_NOREX:
849   case X86::MOV8mr:
850   case X86::MOV8rm_NOREX:
851   case X86::MOV8rm:
852   case X86::MOV16mr:
853   case X86::MOV16rm:
854   case X86::MOV32mr:
855   case X86::MOV32rm: {
856     unsigned NewOpc;
857     switch (OutMI.getOpcode()) {
858     default: llvm_unreachable("Invalid opcode");
859     case X86::MOV8mr_NOREX:
860     case X86::MOV8mr:  NewOpc = X86::MOV8o32a; break;
861     case X86::MOV8rm_NOREX:
862     case X86::MOV8rm:  NewOpc = X86::MOV8ao32; break;
863     case X86::MOV16mr: NewOpc = X86::MOV16o32a; break;
864     case X86::MOV16rm: NewOpc = X86::MOV16ao32; break;
865     case X86::MOV32mr: NewOpc = X86::MOV32o32a; break;
866     case X86::MOV32rm: NewOpc = X86::MOV32ao32; break;
867     }
868     SimplifyShortMoveForm(AsmPrinter, OutMI, NewOpc);
869     break;
870   }
871 
872   case X86::ADC8ri: case X86::ADC16ri: case X86::ADC32ri: case X86::ADC64ri32:
873   case X86::ADD8ri: case X86::ADD16ri: case X86::ADD32ri: case X86::ADD64ri32:
874   case X86::AND8ri: case X86::AND16ri: case X86::AND32ri: case X86::AND64ri32:
875   case X86::CMP8ri: case X86::CMP16ri: case X86::CMP32ri: case X86::CMP64ri32:
876   case X86::OR8ri:  case X86::OR16ri:  case X86::OR32ri:  case X86::OR64ri32:
877   case X86::SBB8ri: case X86::SBB16ri: case X86::SBB32ri: case X86::SBB64ri32:
878   case X86::SUB8ri: case X86::SUB16ri: case X86::SUB32ri: case X86::SUB64ri32:
879   case X86::TEST8ri:case X86::TEST16ri:case X86::TEST32ri:case X86::TEST64ri32:
880   case X86::XOR8ri: case X86::XOR16ri: case X86::XOR32ri: case X86::XOR64ri32: {
881     unsigned NewOpc;
882     switch (OutMI.getOpcode()) {
883     default: llvm_unreachable("Invalid opcode");
884     case X86::ADC8ri:     NewOpc = X86::ADC8i8;    break;
885     case X86::ADC16ri:    NewOpc = X86::ADC16i16;  break;
886     case X86::ADC32ri:    NewOpc = X86::ADC32i32;  break;
887     case X86::ADC64ri32:  NewOpc = X86::ADC64i32;  break;
888     case X86::ADD8ri:     NewOpc = X86::ADD8i8;    break;
889     case X86::ADD16ri:    NewOpc = X86::ADD16i16;  break;
890     case X86::ADD32ri:    NewOpc = X86::ADD32i32;  break;
891     case X86::ADD64ri32:  NewOpc = X86::ADD64i32;  break;
892     case X86::AND8ri:     NewOpc = X86::AND8i8;    break;
893     case X86::AND16ri:    NewOpc = X86::AND16i16;  break;
894     case X86::AND32ri:    NewOpc = X86::AND32i32;  break;
895     case X86::AND64ri32:  NewOpc = X86::AND64i32;  break;
896     case X86::CMP8ri:     NewOpc = X86::CMP8i8;    break;
897     case X86::CMP16ri:    NewOpc = X86::CMP16i16;  break;
898     case X86::CMP32ri:    NewOpc = X86::CMP32i32;  break;
899     case X86::CMP64ri32:  NewOpc = X86::CMP64i32;  break;
900     case X86::OR8ri:      NewOpc = X86::OR8i8;     break;
901     case X86::OR16ri:     NewOpc = X86::OR16i16;   break;
902     case X86::OR32ri:     NewOpc = X86::OR32i32;   break;
903     case X86::OR64ri32:   NewOpc = X86::OR64i32;   break;
904     case X86::SBB8ri:     NewOpc = X86::SBB8i8;    break;
905     case X86::SBB16ri:    NewOpc = X86::SBB16i16;  break;
906     case X86::SBB32ri:    NewOpc = X86::SBB32i32;  break;
907     case X86::SBB64ri32:  NewOpc = X86::SBB64i32;  break;
908     case X86::SUB8ri:     NewOpc = X86::SUB8i8;    break;
909     case X86::SUB16ri:    NewOpc = X86::SUB16i16;  break;
910     case X86::SUB32ri:    NewOpc = X86::SUB32i32;  break;
911     case X86::SUB64ri32:  NewOpc = X86::SUB64i32;  break;
912     case X86::TEST8ri:    NewOpc = X86::TEST8i8;   break;
913     case X86::TEST16ri:   NewOpc = X86::TEST16i16; break;
914     case X86::TEST32ri:   NewOpc = X86::TEST32i32; break;
915     case X86::TEST64ri32: NewOpc = X86::TEST64i32; break;
916     case X86::XOR8ri:     NewOpc = X86::XOR8i8;    break;
917     case X86::XOR16ri:    NewOpc = X86::XOR16i16;  break;
918     case X86::XOR32ri:    NewOpc = X86::XOR32i32;  break;
919     case X86::XOR64ri32:  NewOpc = X86::XOR64i32;  break;
920     }
921     SimplifyShortImmForm(OutMI, NewOpc);
922     break;
923   }
924 
925   // Try to shrink some forms of movsx.
926   case X86::MOVSX16rr8:
927   case X86::MOVSX32rr16:
928   case X86::MOVSX64rr32:
929     SimplifyMOVSX(OutMI);
930     break;
931 
932   case X86::VCMPPDrri:
933   case X86::VCMPPDYrri:
934   case X86::VCMPPSrri:
935   case X86::VCMPPSYrri:
936   case X86::VCMPSDrr:
937   case X86::VCMPSSrr: {
938     // Swap the operands if it will enable a 2 byte VEX encoding.
939     // FIXME: Change the immediate to improve opportunities?
940     if (!X86II::isX86_64ExtendedReg(OutMI.getOperand(1).getReg()) &&
941         X86II::isX86_64ExtendedReg(OutMI.getOperand(2).getReg())) {
942       unsigned Imm = MI->getOperand(3).getImm() & 0x7;
943       switch (Imm) {
944       default: break;
945       case 0x00: // EQUAL
946       case 0x03: // UNORDERED
947       case 0x04: // NOT EQUAL
948       case 0x07: // ORDERED
949         std::swap(OutMI.getOperand(1), OutMI.getOperand(2));
950         break;
951       }
952     }
953     break;
954   }
955 
956   case X86::VMOVHLPSrr:
957   case X86::VUNPCKHPDrr:
958     // These are not truly commutable so hide them from the default case.
959     break;
960 
961   default: {
962     // If the instruction is a commutable arithmetic instruction we might be
963     // able to commute the operands to get a 2 byte VEX prefix.
964     uint64_t TSFlags = MI->getDesc().TSFlags;
965     if (MI->getDesc().isCommutable() &&
966         (TSFlags & X86II::EncodingMask) == X86II::VEX &&
967         (TSFlags & X86II::OpMapMask) == X86II::TB &&
968         (TSFlags & X86II::FormMask) == X86II::MRMSrcReg &&
969         !(TSFlags & X86II::VEX_W) && (TSFlags & X86II::VEX_4V) &&
970         OutMI.getNumOperands() == 3) {
971       if (!X86II::isX86_64ExtendedReg(OutMI.getOperand(1).getReg()) &&
972           X86II::isX86_64ExtendedReg(OutMI.getOperand(2).getReg()))
973         std::swap(OutMI.getOperand(1), OutMI.getOperand(2));
974     }
975     break;
976   }
977   }
978 }
979 
980 void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
981                                  const MachineInstr &MI) {
982   NoAutoPaddingScope NoPadScope(*OutStreamer);
983   bool Is64Bits = MI.getOpcode() != X86::TLS_addr32 &&
984                   MI.getOpcode() != X86::TLS_base_addr32;
985   bool Is64BitsLP64 = MI.getOpcode() == X86::TLS_addr64 ||
986                       MI.getOpcode() == X86::TLS_base_addr64;
987   MCContext &Ctx = OutStreamer->getContext();
988 
989   MCSymbolRefExpr::VariantKind SRVK;
990   switch (MI.getOpcode()) {
991   case X86::TLS_addr32:
992   case X86::TLS_addr64:
993   case X86::TLS_addrX32:
994     SRVK = MCSymbolRefExpr::VK_TLSGD;
995     break;
996   case X86::TLS_base_addr32:
997     SRVK = MCSymbolRefExpr::VK_TLSLDM;
998     break;
999   case X86::TLS_base_addr64:
1000   case X86::TLS_base_addrX32:
1001     SRVK = MCSymbolRefExpr::VK_TLSLD;
1002     break;
1003   default:
1004     llvm_unreachable("unexpected opcode");
1005   }
1006 
1007   const MCSymbolRefExpr *Sym = MCSymbolRefExpr::create(
1008       MCInstLowering.GetSymbolFromOperand(MI.getOperand(3)), SRVK, Ctx);
1009 
1010   // As of binutils 2.32, ld has a bogus TLS relaxation error when the GD/LD
1011   // code sequence using R_X86_64_GOTPCREL (instead of R_X86_64_GOTPCRELX) is
1012   // attempted to be relaxed to IE/LE (binutils PR24784). Work around the bug by
1013   // only using GOT when GOTPCRELX is enabled.
1014   // TODO Delete the workaround when GOTPCRELX becomes commonplace.
1015   bool UseGot = MMI->getModule()->getRtLibUseGOT() &&
1016                 Ctx.getAsmInfo()->canRelaxRelocations();
1017 
1018   if (Is64Bits) {
1019     bool NeedsPadding = SRVK == MCSymbolRefExpr::VK_TLSGD;
1020     if (NeedsPadding && Is64BitsLP64)
1021       EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX));
1022     EmitAndCountInstruction(MCInstBuilder(X86::LEA64r)
1023                                 .addReg(X86::RDI)
1024                                 .addReg(X86::RIP)
1025                                 .addImm(1)
1026                                 .addReg(0)
1027                                 .addExpr(Sym)
1028                                 .addReg(0));
1029     const MCSymbol *TlsGetAddr = Ctx.getOrCreateSymbol("__tls_get_addr");
1030     if (NeedsPadding) {
1031       if (!UseGot)
1032         EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX));
1033       EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX));
1034       EmitAndCountInstruction(MCInstBuilder(X86::REX64_PREFIX));
1035     }
1036     if (UseGot) {
1037       const MCExpr *Expr = MCSymbolRefExpr::create(
1038           TlsGetAddr, MCSymbolRefExpr::VK_GOTPCREL, Ctx);
1039       EmitAndCountInstruction(MCInstBuilder(X86::CALL64m)
1040                                   .addReg(X86::RIP)
1041                                   .addImm(1)
1042                                   .addReg(0)
1043                                   .addExpr(Expr)
1044                                   .addReg(0));
1045     } else {
1046       EmitAndCountInstruction(
1047           MCInstBuilder(X86::CALL64pcrel32)
1048               .addExpr(MCSymbolRefExpr::create(TlsGetAddr,
1049                                                MCSymbolRefExpr::VK_PLT, Ctx)));
1050     }
1051   } else {
1052     if (SRVK == MCSymbolRefExpr::VK_TLSGD && !UseGot) {
1053       EmitAndCountInstruction(MCInstBuilder(X86::LEA32r)
1054                                   .addReg(X86::EAX)
1055                                   .addReg(0)
1056                                   .addImm(1)
1057                                   .addReg(X86::EBX)
1058                                   .addExpr(Sym)
1059                                   .addReg(0));
1060     } else {
1061       EmitAndCountInstruction(MCInstBuilder(X86::LEA32r)
1062                                   .addReg(X86::EAX)
1063                                   .addReg(X86::EBX)
1064                                   .addImm(1)
1065                                   .addReg(0)
1066                                   .addExpr(Sym)
1067                                   .addReg(0));
1068     }
1069 
1070     const MCSymbol *TlsGetAddr = Ctx.getOrCreateSymbol("___tls_get_addr");
1071     if (UseGot) {
1072       const MCExpr *Expr =
1073           MCSymbolRefExpr::create(TlsGetAddr, MCSymbolRefExpr::VK_GOT, Ctx);
1074       EmitAndCountInstruction(MCInstBuilder(X86::CALL32m)
1075                                   .addReg(X86::EBX)
1076                                   .addImm(1)
1077                                   .addReg(0)
1078                                   .addExpr(Expr)
1079                                   .addReg(0));
1080     } else {
1081       EmitAndCountInstruction(
1082           MCInstBuilder(X86::CALLpcrel32)
1083               .addExpr(MCSymbolRefExpr::create(TlsGetAddr,
1084                                                MCSymbolRefExpr::VK_PLT, Ctx)));
1085     }
1086   }
1087 }
1088 
1089 /// Emit the largest nop instruction smaller than or equal to \p NumBytes
1090 /// bytes.  Return the size of nop emitted.
1091 static unsigned emitNop(MCStreamer &OS, unsigned NumBytes,
1092                         const X86Subtarget *Subtarget) {
1093   // Determine the longest nop which can be efficiently decoded for the given
1094   // target cpu.  15-bytes is the longest single NOP instruction, but some
1095   // platforms can't decode the longest forms efficiently.
1096   unsigned MaxNopLength = 1;
1097   if (Subtarget->is64Bit()) {
1098     // FIXME: We can use NOOPL on 32-bit targets with FeatureNOPL, but the
1099     // IndexReg/BaseReg below need to be updated.
1100     if (Subtarget->hasFeature(X86::TuningFast7ByteNOP))
1101       MaxNopLength = 7;
1102     else if (Subtarget->hasFeature(X86::TuningFast15ByteNOP))
1103       MaxNopLength = 15;
1104     else if (Subtarget->hasFeature(X86::TuningFast11ByteNOP))
1105       MaxNopLength = 11;
1106     else
1107       MaxNopLength = 10;
1108   } if (Subtarget->is32Bit())
1109     MaxNopLength = 2;
1110 
1111   // Cap a single nop emission at the profitable value for the target
1112   NumBytes = std::min(NumBytes, MaxNopLength);
1113 
1114   unsigned NopSize;
1115   unsigned Opc, BaseReg, ScaleVal, IndexReg, Displacement, SegmentReg;
1116   IndexReg = Displacement = SegmentReg = 0;
1117   BaseReg = X86::RAX;
1118   ScaleVal = 1;
1119   switch (NumBytes) {
1120   case 0:
1121     llvm_unreachable("Zero nops?");
1122     break;
1123   case 1:
1124     NopSize = 1;
1125     Opc = X86::NOOP;
1126     break;
1127   case 2:
1128     NopSize = 2;
1129     Opc = X86::XCHG16ar;
1130     break;
1131   case 3:
1132     NopSize = 3;
1133     Opc = X86::NOOPL;
1134     break;
1135   case 4:
1136     NopSize = 4;
1137     Opc = X86::NOOPL;
1138     Displacement = 8;
1139     break;
1140   case 5:
1141     NopSize = 5;
1142     Opc = X86::NOOPL;
1143     Displacement = 8;
1144     IndexReg = X86::RAX;
1145     break;
1146   case 6:
1147     NopSize = 6;
1148     Opc = X86::NOOPW;
1149     Displacement = 8;
1150     IndexReg = X86::RAX;
1151     break;
1152   case 7:
1153     NopSize = 7;
1154     Opc = X86::NOOPL;
1155     Displacement = 512;
1156     break;
1157   case 8:
1158     NopSize = 8;
1159     Opc = X86::NOOPL;
1160     Displacement = 512;
1161     IndexReg = X86::RAX;
1162     break;
1163   case 9:
1164     NopSize = 9;
1165     Opc = X86::NOOPW;
1166     Displacement = 512;
1167     IndexReg = X86::RAX;
1168     break;
1169   default:
1170     NopSize = 10;
1171     Opc = X86::NOOPW;
1172     Displacement = 512;
1173     IndexReg = X86::RAX;
1174     SegmentReg = X86::CS;
1175     break;
1176   }
1177 
1178   unsigned NumPrefixes = std::min(NumBytes - NopSize, 5U);
1179   NopSize += NumPrefixes;
1180   for (unsigned i = 0; i != NumPrefixes; ++i)
1181     OS.emitBytes("\x66");
1182 
1183   switch (Opc) {
1184   default: llvm_unreachable("Unexpected opcode");
1185   case X86::NOOP:
1186     OS.emitInstruction(MCInstBuilder(Opc), *Subtarget);
1187     break;
1188   case X86::XCHG16ar:
1189     OS.emitInstruction(MCInstBuilder(Opc).addReg(X86::AX).addReg(X86::AX),
1190                        *Subtarget);
1191     break;
1192   case X86::NOOPL:
1193   case X86::NOOPW:
1194     OS.emitInstruction(MCInstBuilder(Opc)
1195                            .addReg(BaseReg)
1196                            .addImm(ScaleVal)
1197                            .addReg(IndexReg)
1198                            .addImm(Displacement)
1199                            .addReg(SegmentReg),
1200                        *Subtarget);
1201     break;
1202   }
1203   assert(NopSize <= NumBytes && "We overemitted?");
1204   return NopSize;
1205 }
1206 
1207 /// Emit the optimal amount of multi-byte nops on X86.
1208 static void emitX86Nops(MCStreamer &OS, unsigned NumBytes,
1209                         const X86Subtarget *Subtarget) {
1210   unsigned NopsToEmit = NumBytes;
1211   (void)NopsToEmit;
1212   while (NumBytes) {
1213     NumBytes -= emitNop(OS, NumBytes, Subtarget);
1214     assert(NopsToEmit >= NumBytes && "Emitted more than I asked for!");
1215   }
1216 }
1217 
1218 void X86AsmPrinter::LowerSTATEPOINT(const MachineInstr &MI,
1219                                     X86MCInstLower &MCIL) {
1220   assert(Subtarget->is64Bit() && "Statepoint currently only supports X86-64");
1221 
1222   NoAutoPaddingScope NoPadScope(*OutStreamer);
1223 
1224   StatepointOpers SOpers(&MI);
1225   if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
1226     emitX86Nops(*OutStreamer, PatchBytes, Subtarget);
1227   } else {
1228     // Lower call target and choose correct opcode
1229     const MachineOperand &CallTarget = SOpers.getCallTarget();
1230     MCOperand CallTargetMCOp;
1231     unsigned CallOpcode;
1232     switch (CallTarget.getType()) {
1233     case MachineOperand::MO_GlobalAddress:
1234     case MachineOperand::MO_ExternalSymbol:
1235       CallTargetMCOp = MCIL.LowerSymbolOperand(
1236           CallTarget, MCIL.GetSymbolFromOperand(CallTarget));
1237       CallOpcode = X86::CALL64pcrel32;
1238       // Currently, we only support relative addressing with statepoints.
1239       // Otherwise, we'll need a scratch register to hold the target
1240       // address.  You'll fail asserts during load & relocation if this
1241       // symbol is to far away. (TODO: support non-relative addressing)
1242       break;
1243     case MachineOperand::MO_Immediate:
1244       CallTargetMCOp = MCOperand::createImm(CallTarget.getImm());
1245       CallOpcode = X86::CALL64pcrel32;
1246       // Currently, we only support relative addressing with statepoints.
1247       // Otherwise, we'll need a scratch register to hold the target
1248       // immediate.  You'll fail asserts during load & relocation if this
1249       // address is to far away. (TODO: support non-relative addressing)
1250       break;
1251     case MachineOperand::MO_Register:
1252       // FIXME: Add retpoline support and remove this.
1253       if (Subtarget->useIndirectThunkCalls())
1254         report_fatal_error("Lowering register statepoints with thunks not "
1255                            "yet implemented.");
1256       CallTargetMCOp = MCOperand::createReg(CallTarget.getReg());
1257       CallOpcode = X86::CALL64r;
1258       break;
1259     default:
1260       llvm_unreachable("Unsupported operand type in statepoint call target");
1261       break;
1262     }
1263 
1264     // Emit call
1265     MCInst CallInst;
1266     CallInst.setOpcode(CallOpcode);
1267     CallInst.addOperand(CallTargetMCOp);
1268     OutStreamer->emitInstruction(CallInst, getSubtargetInfo());
1269   }
1270 
1271   // Record our statepoint node in the same section used by STACKMAP
1272   // and PATCHPOINT
1273   auto &Ctx = OutStreamer->getContext();
1274   MCSymbol *MILabel = Ctx.createTempSymbol();
1275   OutStreamer->emitLabel(MILabel);
1276   SM.recordStatepoint(*MILabel, MI);
1277 }
1278 
1279 void X86AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI,
1280                                      X86MCInstLower &MCIL) {
1281   // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
1282   //                  <opcode>, <operands>
1283 
1284   NoAutoPaddingScope NoPadScope(*OutStreamer);
1285 
1286   Register DefRegister = FaultingMI.getOperand(0).getReg();
1287   FaultMaps::FaultKind FK =
1288       static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(1).getImm());
1289   MCSymbol *HandlerLabel = FaultingMI.getOperand(2).getMBB()->getSymbol();
1290   unsigned Opcode = FaultingMI.getOperand(3).getImm();
1291   unsigned OperandsBeginIdx = 4;
1292 
1293   auto &Ctx = OutStreamer->getContext();
1294   MCSymbol *FaultingLabel = Ctx.createTempSymbol();
1295   OutStreamer->emitLabel(FaultingLabel);
1296 
1297   assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
1298   FM.recordFaultingOp(FK, FaultingLabel, HandlerLabel);
1299 
1300   MCInst MI;
1301   MI.setOpcode(Opcode);
1302 
1303   if (DefRegister != X86::NoRegister)
1304     MI.addOperand(MCOperand::createReg(DefRegister));
1305 
1306   for (auto I = FaultingMI.operands_begin() + OperandsBeginIdx,
1307             E = FaultingMI.operands_end();
1308        I != E; ++I)
1309     if (auto MaybeOperand = MCIL.LowerMachineOperand(&FaultingMI, *I))
1310       MI.addOperand(MaybeOperand.getValue());
1311 
1312   OutStreamer->AddComment("on-fault: " + HandlerLabel->getName());
1313   OutStreamer->emitInstruction(MI, getSubtargetInfo());
1314 }
1315 
1316 void X86AsmPrinter::LowerFENTRY_CALL(const MachineInstr &MI,
1317                                      X86MCInstLower &MCIL) {
1318   bool Is64Bits = Subtarget->is64Bit();
1319   MCContext &Ctx = OutStreamer->getContext();
1320   MCSymbol *fentry = Ctx.getOrCreateSymbol("__fentry__");
1321   const MCSymbolRefExpr *Op =
1322       MCSymbolRefExpr::create(fentry, MCSymbolRefExpr::VK_None, Ctx);
1323 
1324   EmitAndCountInstruction(
1325       MCInstBuilder(Is64Bits ? X86::CALL64pcrel32 : X86::CALLpcrel32)
1326           .addExpr(Op));
1327 }
1328 
1329 void X86AsmPrinter::LowerASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
1330   // FIXME: Make this work on non-ELF.
1331   if (!TM.getTargetTriple().isOSBinFormatELF()) {
1332     report_fatal_error("llvm.asan.check.memaccess only supported on ELF");
1333     return;
1334   }
1335 
1336   unsigned Reg = MI.getOperand(0).getReg().id();
1337   ASanAccessInfo AccessInfo(MI.getOperand(1).getImm());
1338 
1339   MCSymbol *&Sym = AsanMemaccessSymbols[{Reg, AccessInfo.Packed}];
1340   if (!Sym) {
1341     std::string Name = AccessInfo.IsWrite ? "store" : "load";
1342     std::string SymName = "__asan_check_" + Name +
1343                           utostr(1 << AccessInfo.AccessSizeIndex) + "_rn" +
1344                           utostr(Reg);
1345     Sym = OutContext.getOrCreateSymbol(SymName);
1346   }
1347 
1348   EmitAndCountInstruction(
1349       MCInstBuilder(X86::CALL64pcrel32)
1350           .addExpr(MCSymbolRefExpr::create(Sym, OutContext)));
1351 }
1352 
1353 void X86AsmPrinter::emitAsanMemaccessPartial(Module &M, unsigned Reg,
1354                                              const ASanAccessInfo &AccessInfo,
1355                                              MCSubtargetInfo &STI) {
1356   assert(AccessInfo.AccessSizeIndex == 0 || AccessInfo.AccessSizeIndex == 1 ||
1357          AccessInfo.AccessSizeIndex == 2);
1358   assert(Reg != X86::R8);
1359 
1360   uint64_t ShadowBase;
1361   int MappingScale;
1362   bool OrShadowOffset;
1363   getAddressSanitizerParams(
1364       Triple(M.getTargetTriple()), M.getDataLayout().getPointerSizeInBits(),
1365       AccessInfo.CompileKernel, &ShadowBase, &MappingScale, &OrShadowOffset);
1366 
1367   OutStreamer->emitInstruction(
1368       MCInstBuilder(X86::MOV64rr).addReg(X86::R8).addReg(X86::NoRegister + Reg),
1369       STI);
1370   OutStreamer->emitInstruction(MCInstBuilder(X86::SHR64ri)
1371                                    .addReg(X86::R8)
1372                                    .addReg(X86::NoRegister)
1373                                    .addImm(MappingScale),
1374                                STI);
1375   if (OrShadowOffset) {
1376     OutStreamer->emitInstruction(MCInstBuilder(X86::OR64ri32)
1377                                      .addReg(X86::NoRegister)
1378                                      .addReg(X86::R8)
1379                                      .addImm(ShadowBase),
1380                                  STI);
1381     OutStreamer->emitInstruction(MCInstBuilder(X86::MOV8rm)
1382                                      .addReg(X86::R8B)
1383                                      .addReg(X86::R8)
1384                                      .addImm(1)
1385                                      .addReg(X86::NoRegister)
1386                                      .addImm(0)
1387                                      .addReg(X86::NoRegister),
1388                                  STI);
1389     OutStreamer->emitInstruction(
1390         MCInstBuilder(X86::TEST8rr).addReg(X86::R8B).addReg(X86::R8B), STI);
1391   } else {
1392     OutStreamer->emitInstruction(MCInstBuilder(X86::MOVSX32rm8)
1393                                      .addReg(X86::R8D)
1394                                      .addReg(X86::R8)
1395                                      .addImm(1)
1396                                      .addReg(X86::NoRegister)
1397                                      .addImm(ShadowBase)
1398                                      .addReg(X86::NoRegister),
1399                                  STI);
1400     OutStreamer->emitInstruction(
1401         MCInstBuilder(X86::TEST32rr).addReg(X86::R8D).addReg(X86::R8D), STI);
1402   }
1403   MCSymbol *AdditionalCheck = OutContext.createTempSymbol();
1404   OutStreamer->emitInstruction(
1405       MCInstBuilder(X86::JCC_1)
1406           .addExpr(MCSymbolRefExpr::create(AdditionalCheck, OutContext))
1407           .addImm(X86::COND_NE),
1408       STI);
1409   MCSymbol *ReturnSym = OutContext.createTempSymbol();
1410   OutStreamer->emitLabel(ReturnSym);
1411   OutStreamer->emitInstruction(MCInstBuilder(getRetOpcode(*Subtarget)), STI);
1412 
1413   // Shadow byte is non-zero so we need to perform additional checks.
1414   OutStreamer->emitLabel(AdditionalCheck);
1415   OutStreamer->emitInstruction(MCInstBuilder(X86::PUSH64r).addReg(X86::RCX),
1416                                STI);
1417   OutStreamer->emitInstruction(MCInstBuilder(X86::MOV64rr)
1418                                    .addReg(X86::RCX)
1419                                    .addReg(X86::NoRegister + Reg),
1420                                STI);
1421   const size_t Granularity = 1ULL << MappingScale;
1422   OutStreamer->emitInstruction(MCInstBuilder(X86::AND32ri8)
1423                                    .addReg(X86::NoRegister)
1424                                    .addReg(X86::ECX)
1425                                    .addImm(Granularity - 1),
1426                                STI);
1427   if (AccessInfo.AccessSizeIndex == 1) {
1428     OutStreamer->emitInstruction(MCInstBuilder(X86::ADD32ri8)
1429                                      .addReg(X86::NoRegister)
1430                                      .addReg(X86::ECX)
1431                                      .addImm(1),
1432                                  STI);
1433   } else if (AccessInfo.AccessSizeIndex == 2) {
1434     OutStreamer->emitInstruction(MCInstBuilder(X86::ADD32ri8)
1435                                      .addReg(X86::NoRegister)
1436                                      .addReg(X86::ECX)
1437                                      .addImm(3),
1438                                  STI);
1439   }
1440 
1441   OutStreamer->emitInstruction(
1442       MCInstBuilder(X86::CMP32rr).addReg(X86::ECX).addReg(X86::R8D).addImm(1),
1443       STI);
1444   OutStreamer->emitInstruction(MCInstBuilder(X86::POP64r).addReg(X86::RCX),
1445                                STI);
1446   OutStreamer->emitInstruction(
1447       MCInstBuilder(X86::JCC_1)
1448           .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext))
1449           .addImm(X86::COND_L),
1450       STI);
1451 
1452   emitAsanReportError(M, Reg, AccessInfo, STI);
1453 }
1454 
1455 void X86AsmPrinter::emitAsanMemaccessFull(Module &M, unsigned Reg,
1456                                           const ASanAccessInfo &AccessInfo,
1457                                           MCSubtargetInfo &STI) {
1458   assert(AccessInfo.AccessSizeIndex == 3 || AccessInfo.AccessSizeIndex == 4);
1459   assert(Reg != X86::R8);
1460 
1461   uint64_t ShadowBase;
1462   int MappingScale;
1463   bool OrShadowOffset;
1464   getAddressSanitizerParams(
1465       Triple(M.getTargetTriple()), M.getDataLayout().getPointerSizeInBits(),
1466       AccessInfo.CompileKernel, &ShadowBase, &MappingScale, &OrShadowOffset);
1467 
1468   OutStreamer->emitInstruction(
1469       MCInstBuilder(X86::MOV64rr).addReg(X86::R8).addReg(X86::NoRegister + Reg),
1470       STI);
1471   OutStreamer->emitInstruction(MCInstBuilder(X86::SHR64ri)
1472                                    .addReg(X86::R8)
1473                                    .addReg(X86::R8)
1474                                    .addImm(MappingScale),
1475                                STI);
1476   if (OrShadowOffset) {
1477     OutStreamer->emitInstruction(MCInstBuilder(X86::OR64ri32)
1478                                      .addReg(X86::R8)
1479                                      .addReg(X86::R8)
1480                                      .addImm(ShadowBase),
1481                                  STI);
1482     auto OpCode = AccessInfo.AccessSizeIndex == 3 ? X86::CMP8mi : X86::CMP16mi8;
1483     OutStreamer->emitInstruction(MCInstBuilder(OpCode)
1484                                      .addReg(X86::R8)
1485                                      .addImm(1)
1486                                      .addReg(X86::NoRegister)
1487                                      .addImm(0)
1488                                      .addReg(X86::NoRegister)
1489                                      .addImm(0),
1490                                  STI);
1491   } else {
1492     auto OpCode = AccessInfo.AccessSizeIndex == 3 ? X86::CMP8mi : X86::CMP16mi8;
1493     OutStreamer->emitInstruction(MCInstBuilder(OpCode)
1494                                      .addReg(X86::R8)
1495                                      .addImm(1)
1496                                      .addReg(X86::NoRegister)
1497                                      .addImm(ShadowBase)
1498                                      .addReg(X86::NoRegister)
1499                                      .addImm(0),
1500                                  STI);
1501   }
1502   MCSymbol *ReportCode = OutContext.createTempSymbol();
1503   OutStreamer->emitInstruction(
1504       MCInstBuilder(X86::JCC_1)
1505           .addExpr(MCSymbolRefExpr::create(ReportCode, OutContext))
1506           .addImm(X86::COND_NE),
1507       STI);
1508   MCSymbol *ReturnSym = OutContext.createTempSymbol();
1509   OutStreamer->emitLabel(ReturnSym);
1510   OutStreamer->emitInstruction(MCInstBuilder(getRetOpcode(*Subtarget)), STI);
1511 
1512   OutStreamer->emitLabel(ReportCode);
1513   emitAsanReportError(M, Reg, AccessInfo, STI);
1514 }
1515 
1516 void X86AsmPrinter::emitAsanReportError(Module &M, unsigned Reg,
1517                                         const ASanAccessInfo &AccessInfo,
1518                                         MCSubtargetInfo &STI) {
1519   std::string Name = AccessInfo.IsWrite ? "store" : "load";
1520   MCSymbol *ReportError = OutContext.getOrCreateSymbol(
1521       "__asan_report_" + Name + utostr(1 << AccessInfo.AccessSizeIndex));
1522   OutStreamer->emitInstruction(MCInstBuilder(X86::MOV64rr)
1523                                    .addReg(X86::RDI)
1524                                    .addReg(X86::NoRegister + Reg),
1525                                STI);
1526   OutStreamer->emitInstruction(
1527       MCInstBuilder(X86::JMP_1)
1528           .addExpr(MCSymbolRefExpr::create(ReportError, OutContext)),
1529       STI);
1530 }
1531 
1532 void X86AsmPrinter::emitAsanMemaccessSymbols(Module &M) {
1533   if (AsanMemaccessSymbols.empty())
1534     return;
1535 
1536   const Triple &TT = TM.getTargetTriple();
1537   assert(TT.isOSBinFormatELF());
1538   std::unique_ptr<MCSubtargetInfo> STI(
1539       TM.getTarget().createMCSubtargetInfo(TT.str(), "", ""));
1540   assert(STI && "Unable to create subtarget info");
1541 
1542   for (auto &P : AsanMemaccessSymbols) {
1543     MCSymbol *Sym = P.second;
1544     OutStreamer->SwitchSection(OutContext.getELFSection(
1545         ".text.hot", ELF::SHT_PROGBITS,
1546         ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, Sym->getName(),
1547         /*IsComdat=*/true));
1548 
1549     OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
1550     OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
1551     OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden);
1552     OutStreamer->emitLabel(Sym);
1553 
1554     unsigned Reg = std::get<0>(P.first);
1555     ASanAccessInfo AccessInfo(std::get<1>(P.first));
1556 
1557     if (AccessInfo.AccessSizeIndex < 3) {
1558       emitAsanMemaccessPartial(M, Reg, AccessInfo, *STI);
1559     } else {
1560       emitAsanMemaccessFull(M, Reg, AccessInfo, *STI);
1561     }
1562   }
1563 }
1564 
1565 void X86AsmPrinter::LowerPATCHABLE_OP(const MachineInstr &MI,
1566                                       X86MCInstLower &MCIL) {
1567   // PATCHABLE_OP minsize, opcode, operands
1568 
1569   NoAutoPaddingScope NoPadScope(*OutStreamer);
1570 
1571   unsigned MinSize = MI.getOperand(0).getImm();
1572   unsigned Opcode = MI.getOperand(1).getImm();
1573 
1574   MCInst MCI;
1575   MCI.setOpcode(Opcode);
1576   for (auto &MO : drop_begin(MI.operands(), 2))
1577     if (auto MaybeOperand = MCIL.LowerMachineOperand(&MI, MO))
1578       MCI.addOperand(MaybeOperand.getValue());
1579 
1580   SmallString<256> Code;
1581   SmallVector<MCFixup, 4> Fixups;
1582   raw_svector_ostream VecOS(Code);
1583   CodeEmitter->encodeInstruction(MCI, VecOS, Fixups, getSubtargetInfo());
1584 
1585   if (Code.size() < MinSize) {
1586     if (MinSize == 2 && Subtarget->is32Bit() &&
1587         Subtarget->isTargetWindowsMSVC() &&
1588         (Subtarget->getCPU().empty() || Subtarget->getCPU() == "pentium3")) {
1589       // For compatibilty reasons, when targetting MSVC, is is important to
1590       // generate a 'legacy' NOP in the form of a 8B FF MOV EDI, EDI. Some tools
1591       // rely specifically on this pattern to be able to patch a function.
1592       // This is only for 32-bit targets, when using /arch:IA32 or /arch:SSE.
1593       OutStreamer->emitInstruction(
1594           MCInstBuilder(X86::MOV32rr_REV).addReg(X86::EDI).addReg(X86::EDI),
1595           *Subtarget);
1596     } else if (MinSize == 2 && Opcode == X86::PUSH64r) {
1597       // This is an optimization that lets us get away without emitting a nop in
1598       // many cases.
1599       //
1600       // NB! In some cases the encoding for PUSH64r (e.g. PUSH64r %r9) takes two
1601       // bytes too, so the check on MinSize is important.
1602       MCI.setOpcode(X86::PUSH64rmr);
1603     } else {
1604       unsigned NopSize = emitNop(*OutStreamer, MinSize, Subtarget);
1605       assert(NopSize == MinSize && "Could not implement MinSize!");
1606       (void)NopSize;
1607     }
1608   }
1609 
1610   OutStreamer->emitInstruction(MCI, getSubtargetInfo());
1611 }
1612 
1613 // Lower a stackmap of the form:
1614 // <id>, <shadowBytes>, ...
1615 void X86AsmPrinter::LowerSTACKMAP(const MachineInstr &MI) {
1616   SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo());
1617 
1618   auto &Ctx = OutStreamer->getContext();
1619   MCSymbol *MILabel = Ctx.createTempSymbol();
1620   OutStreamer->emitLabel(MILabel);
1621 
1622   SM.recordStackMap(*MILabel, MI);
1623   unsigned NumShadowBytes = MI.getOperand(1).getImm();
1624   SMShadowTracker.reset(NumShadowBytes);
1625 }
1626 
1627 // Lower a patchpoint of the form:
1628 // [<def>], <id>, <numBytes>, <target>, <numArgs>, <cc>, ...
1629 void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI,
1630                                     X86MCInstLower &MCIL) {
1631   assert(Subtarget->is64Bit() && "Patchpoint currently only supports X86-64");
1632 
1633   SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo());
1634 
1635   NoAutoPaddingScope NoPadScope(*OutStreamer);
1636 
1637   auto &Ctx = OutStreamer->getContext();
1638   MCSymbol *MILabel = Ctx.createTempSymbol();
1639   OutStreamer->emitLabel(MILabel);
1640   SM.recordPatchPoint(*MILabel, MI);
1641 
1642   PatchPointOpers opers(&MI);
1643   unsigned ScratchIdx = opers.getNextScratchIdx();
1644   unsigned EncodedBytes = 0;
1645   const MachineOperand &CalleeMO = opers.getCallTarget();
1646 
1647   // Check for null target. If target is non-null (i.e. is non-zero or is
1648   // symbolic) then emit a call.
1649   if (!(CalleeMO.isImm() && !CalleeMO.getImm())) {
1650     MCOperand CalleeMCOp;
1651     switch (CalleeMO.getType()) {
1652     default:
1653       /// FIXME: Add a verifier check for bad callee types.
1654       llvm_unreachable("Unrecognized callee operand type.");
1655     case MachineOperand::MO_Immediate:
1656       if (CalleeMO.getImm())
1657         CalleeMCOp = MCOperand::createImm(CalleeMO.getImm());
1658       break;
1659     case MachineOperand::MO_ExternalSymbol:
1660     case MachineOperand::MO_GlobalAddress:
1661       CalleeMCOp = MCIL.LowerSymbolOperand(CalleeMO,
1662                                            MCIL.GetSymbolFromOperand(CalleeMO));
1663       break;
1664     }
1665 
1666     // Emit MOV to materialize the target address and the CALL to target.
1667     // This is encoded with 12-13 bytes, depending on which register is used.
1668     Register ScratchReg = MI.getOperand(ScratchIdx).getReg();
1669     if (X86II::isX86_64ExtendedReg(ScratchReg))
1670       EncodedBytes = 13;
1671     else
1672       EncodedBytes = 12;
1673 
1674     EmitAndCountInstruction(
1675         MCInstBuilder(X86::MOV64ri).addReg(ScratchReg).addOperand(CalleeMCOp));
1676     // FIXME: Add retpoline support and remove this.
1677     if (Subtarget->useIndirectThunkCalls())
1678       report_fatal_error(
1679           "Lowering patchpoint with thunks not yet implemented.");
1680     EmitAndCountInstruction(MCInstBuilder(X86::CALL64r).addReg(ScratchReg));
1681   }
1682 
1683   // Emit padding.
1684   unsigned NumBytes = opers.getNumPatchBytes();
1685   assert(NumBytes >= EncodedBytes &&
1686          "Patchpoint can't request size less than the length of a call.");
1687 
1688   emitX86Nops(*OutStreamer, NumBytes - EncodedBytes, Subtarget);
1689 }
1690 
1691 void X86AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI,
1692                                               X86MCInstLower &MCIL) {
1693   assert(Subtarget->is64Bit() && "XRay custom events only supports X86-64");
1694 
1695   NoAutoPaddingScope NoPadScope(*OutStreamer);
1696 
1697   // We want to emit the following pattern, which follows the x86 calling
1698   // convention to prepare for the trampoline call to be patched in.
1699   //
1700   //   .p2align 1, ...
1701   // .Lxray_event_sled_N:
1702   //   jmp +N                        // jump across the instrumentation sled
1703   //   ...                           // set up arguments in register
1704   //   callq __xray_CustomEvent@plt  // force dependency to symbol
1705   //   ...
1706   //   <jump here>
1707   //
1708   // After patching, it would look something like:
1709   //
1710   //   nopw (2-byte nop)
1711   //   ...
1712   //   callq __xrayCustomEvent  // already lowered
1713   //   ...
1714   //
1715   // ---
1716   // First we emit the label and the jump.
1717   auto CurSled = OutContext.createTempSymbol("xray_event_sled_", true);
1718   OutStreamer->AddComment("# XRay Custom Event Log");
1719   OutStreamer->emitCodeAlignment(2);
1720   OutStreamer->emitLabel(CurSled);
1721 
1722   // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1723   // an operand (computed as an offset from the jmp instruction).
1724   // FIXME: Find another less hacky way do force the relative jump.
1725   OutStreamer->emitBinaryData("\xeb\x0f");
1726 
1727   // The default C calling convention will place two arguments into %rcx and
1728   // %rdx -- so we only work with those.
1729   const Register DestRegs[] = {X86::RDI, X86::RSI};
1730   bool UsedMask[] = {false, false};
1731   // Filled out in loop.
1732   Register SrcRegs[] = {0, 0};
1733 
1734   // Then we put the operands in the %rdi and %rsi registers. We spill the
1735   // values in the register before we clobber them, and mark them as used in
1736   // UsedMask. In case the arguments are already in the correct register, we use
1737   // emit nops appropriately sized to keep the sled the same size in every
1738   // situation.
1739   for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1740     if (auto Op = MCIL.LowerMachineOperand(&MI, MI.getOperand(I))) {
1741       assert(Op->isReg() && "Only support arguments in registers");
1742       SrcRegs[I] = getX86SubSuperRegister(Op->getReg(), 64);
1743       if (SrcRegs[I] != DestRegs[I]) {
1744         UsedMask[I] = true;
1745         EmitAndCountInstruction(
1746             MCInstBuilder(X86::PUSH64r).addReg(DestRegs[I]));
1747       } else {
1748         emitX86Nops(*OutStreamer, 4, Subtarget);
1749       }
1750     }
1751 
1752   // Now that the register values are stashed, mov arguments into place.
1753   // FIXME: This doesn't work if one of the later SrcRegs is equal to an
1754   // earlier DestReg. We will have already overwritten over the register before
1755   // we can copy from it.
1756   for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1757     if (SrcRegs[I] != DestRegs[I])
1758       EmitAndCountInstruction(
1759           MCInstBuilder(X86::MOV64rr).addReg(DestRegs[I]).addReg(SrcRegs[I]));
1760 
1761   // We emit a hard dependency on the __xray_CustomEvent symbol, which is the
1762   // name of the trampoline to be implemented by the XRay runtime.
1763   auto TSym = OutContext.getOrCreateSymbol("__xray_CustomEvent");
1764   MachineOperand TOp = MachineOperand::CreateMCSymbol(TSym);
1765   if (isPositionIndependent())
1766     TOp.setTargetFlags(X86II::MO_PLT);
1767 
1768   // Emit the call instruction.
1769   EmitAndCountInstruction(MCInstBuilder(X86::CALL64pcrel32)
1770                               .addOperand(MCIL.LowerSymbolOperand(TOp, TSym)));
1771 
1772   // Restore caller-saved and used registers.
1773   for (unsigned I = sizeof UsedMask; I-- > 0;)
1774     if (UsedMask[I])
1775       EmitAndCountInstruction(MCInstBuilder(X86::POP64r).addReg(DestRegs[I]));
1776     else
1777       emitX86Nops(*OutStreamer, 1, Subtarget);
1778 
1779   OutStreamer->AddComment("xray custom event end.");
1780 
1781   // Record the sled version. Version 0 of this sled was spelled differently, so
1782   // we let the runtime handle the different offsets we're using. Version 2
1783   // changed the absolute address to a PC-relative address.
1784   recordSled(CurSled, MI, SledKind::CUSTOM_EVENT, 2);
1785 }
1786 
1787 void X86AsmPrinter::LowerPATCHABLE_TYPED_EVENT_CALL(const MachineInstr &MI,
1788                                                     X86MCInstLower &MCIL) {
1789   assert(Subtarget->is64Bit() && "XRay typed events only supports X86-64");
1790 
1791   NoAutoPaddingScope NoPadScope(*OutStreamer);
1792 
1793   // We want to emit the following pattern, which follows the x86 calling
1794   // convention to prepare for the trampoline call to be patched in.
1795   //
1796   //   .p2align 1, ...
1797   // .Lxray_event_sled_N:
1798   //   jmp +N                        // jump across the instrumentation sled
1799   //   ...                           // set up arguments in register
1800   //   callq __xray_TypedEvent@plt  // force dependency to symbol
1801   //   ...
1802   //   <jump here>
1803   //
1804   // After patching, it would look something like:
1805   //
1806   //   nopw (2-byte nop)
1807   //   ...
1808   //   callq __xrayTypedEvent  // already lowered
1809   //   ...
1810   //
1811   // ---
1812   // First we emit the label and the jump.
1813   auto CurSled = OutContext.createTempSymbol("xray_typed_event_sled_", true);
1814   OutStreamer->AddComment("# XRay Typed Event Log");
1815   OutStreamer->emitCodeAlignment(2);
1816   OutStreamer->emitLabel(CurSled);
1817 
1818   // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1819   // an operand (computed as an offset from the jmp instruction).
1820   // FIXME: Find another less hacky way do force the relative jump.
1821   OutStreamer->emitBinaryData("\xeb\x14");
1822 
1823   // An x86-64 convention may place three arguments into %rcx, %rdx, and R8,
1824   // so we'll work with those. Or we may be called via SystemV, in which case
1825   // we don't have to do any translation.
1826   const Register DestRegs[] = {X86::RDI, X86::RSI, X86::RDX};
1827   bool UsedMask[] = {false, false, false};
1828 
1829   // Will fill out src regs in the loop.
1830   Register SrcRegs[] = {0, 0, 0};
1831 
1832   // Then we put the operands in the SystemV registers. We spill the values in
1833   // the registers before we clobber them, and mark them as used in UsedMask.
1834   // In case the arguments are already in the correct register, we emit nops
1835   // appropriately sized to keep the sled the same size in every situation.
1836   for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1837     if (auto Op = MCIL.LowerMachineOperand(&MI, MI.getOperand(I))) {
1838       // TODO: Is register only support adequate?
1839       assert(Op->isReg() && "Only supports arguments in registers");
1840       SrcRegs[I] = getX86SubSuperRegister(Op->getReg(), 64);
1841       if (SrcRegs[I] != DestRegs[I]) {
1842         UsedMask[I] = true;
1843         EmitAndCountInstruction(
1844             MCInstBuilder(X86::PUSH64r).addReg(DestRegs[I]));
1845       } else {
1846         emitX86Nops(*OutStreamer, 4, Subtarget);
1847       }
1848     }
1849 
1850   // In the above loop we only stash all of the destination registers or emit
1851   // nops if the arguments are already in the right place. Doing the actually
1852   // moving is postponed until after all the registers are stashed so nothing
1853   // is clobbers. We've already added nops to account for the size of mov and
1854   // push if the register is in the right place, so we only have to worry about
1855   // emitting movs.
1856   // FIXME: This doesn't work if one of the later SrcRegs is equal to an
1857   // earlier DestReg. We will have already overwritten over the register before
1858   // we can copy from it.
1859   for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1860     if (UsedMask[I])
1861       EmitAndCountInstruction(
1862           MCInstBuilder(X86::MOV64rr).addReg(DestRegs[I]).addReg(SrcRegs[I]));
1863 
1864   // We emit a hard dependency on the __xray_TypedEvent symbol, which is the
1865   // name of the trampoline to be implemented by the XRay runtime.
1866   auto TSym = OutContext.getOrCreateSymbol("__xray_TypedEvent");
1867   MachineOperand TOp = MachineOperand::CreateMCSymbol(TSym);
1868   if (isPositionIndependent())
1869     TOp.setTargetFlags(X86II::MO_PLT);
1870 
1871   // Emit the call instruction.
1872   EmitAndCountInstruction(MCInstBuilder(X86::CALL64pcrel32)
1873                               .addOperand(MCIL.LowerSymbolOperand(TOp, TSym)));
1874 
1875   // Restore caller-saved and used registers.
1876   for (unsigned I = sizeof UsedMask; I-- > 0;)
1877     if (UsedMask[I])
1878       EmitAndCountInstruction(MCInstBuilder(X86::POP64r).addReg(DestRegs[I]));
1879     else
1880       emitX86Nops(*OutStreamer, 1, Subtarget);
1881 
1882   OutStreamer->AddComment("xray typed event end.");
1883 
1884   // Record the sled version.
1885   recordSled(CurSled, MI, SledKind::TYPED_EVENT, 2);
1886 }
1887 
1888 void X86AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI,
1889                                                   X86MCInstLower &MCIL) {
1890 
1891   NoAutoPaddingScope NoPadScope(*OutStreamer);
1892 
1893   const Function &F = MF->getFunction();
1894   if (F.hasFnAttribute("patchable-function-entry")) {
1895     unsigned Num;
1896     if (F.getFnAttribute("patchable-function-entry")
1897             .getValueAsString()
1898             .getAsInteger(10, Num))
1899       return;
1900     emitX86Nops(*OutStreamer, Num, Subtarget);
1901     return;
1902   }
1903   // We want to emit the following pattern:
1904   //
1905   //   .p2align 1, ...
1906   // .Lxray_sled_N:
1907   //   jmp .tmpN
1908   //   # 9 bytes worth of noops
1909   //
1910   // We need the 9 bytes because at runtime, we'd be patching over the full 11
1911   // bytes with the following pattern:
1912   //
1913   //   mov %r10, <function id, 32-bit>   // 6 bytes
1914   //   call <relative offset, 32-bits>   // 5 bytes
1915   //
1916   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
1917   OutStreamer->emitCodeAlignment(2);
1918   OutStreamer->emitLabel(CurSled);
1919 
1920   // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1921   // an operand (computed as an offset from the jmp instruction).
1922   // FIXME: Find another less hacky way do force the relative jump.
1923   OutStreamer->emitBytes("\xeb\x09");
1924   emitX86Nops(*OutStreamer, 9, Subtarget);
1925   recordSled(CurSled, MI, SledKind::FUNCTION_ENTER, 2);
1926 }
1927 
1928 void X86AsmPrinter::LowerPATCHABLE_RET(const MachineInstr &MI,
1929                                        X86MCInstLower &MCIL) {
1930   NoAutoPaddingScope NoPadScope(*OutStreamer);
1931 
1932   // Since PATCHABLE_RET takes the opcode of the return statement as an
1933   // argument, we use that to emit the correct form of the RET that we want.
1934   // i.e. when we see this:
1935   //
1936   //   PATCHABLE_RET X86::RET ...
1937   //
1938   // We should emit the RET followed by sleds.
1939   //
1940   //   .p2align 1, ...
1941   // .Lxray_sled_N:
1942   //   ret  # or equivalent instruction
1943   //   # 10 bytes worth of noops
1944   //
1945   // This just makes sure that the alignment for the next instruction is 2.
1946   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
1947   OutStreamer->emitCodeAlignment(2);
1948   OutStreamer->emitLabel(CurSled);
1949   unsigned OpCode = MI.getOperand(0).getImm();
1950   MCInst Ret;
1951   Ret.setOpcode(OpCode);
1952   for (auto &MO : drop_begin(MI.operands()))
1953     if (auto MaybeOperand = MCIL.LowerMachineOperand(&MI, MO))
1954       Ret.addOperand(MaybeOperand.getValue());
1955   OutStreamer->emitInstruction(Ret, getSubtargetInfo());
1956   emitX86Nops(*OutStreamer, 10, Subtarget);
1957   recordSled(CurSled, MI, SledKind::FUNCTION_EXIT, 2);
1958 }
1959 
1960 void X86AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI,
1961                                              X86MCInstLower &MCIL) {
1962   NoAutoPaddingScope NoPadScope(*OutStreamer);
1963 
1964   // Like PATCHABLE_RET, we have the actual instruction in the operands to this
1965   // instruction so we lower that particular instruction and its operands.
1966   // Unlike PATCHABLE_RET though, we put the sled before the JMP, much like how
1967   // we do it for PATCHABLE_FUNCTION_ENTER. The sled should be very similar to
1968   // the PATCHABLE_FUNCTION_ENTER case, followed by the lowering of the actual
1969   // tail call much like how we have it in PATCHABLE_RET.
1970   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
1971   OutStreamer->emitCodeAlignment(2);
1972   OutStreamer->emitLabel(CurSled);
1973   auto Target = OutContext.createTempSymbol();
1974 
1975   // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1976   // an operand (computed as an offset from the jmp instruction).
1977   // FIXME: Find another less hacky way do force the relative jump.
1978   OutStreamer->emitBytes("\xeb\x09");
1979   emitX86Nops(*OutStreamer, 9, Subtarget);
1980   OutStreamer->emitLabel(Target);
1981   recordSled(CurSled, MI, SledKind::TAIL_CALL, 2);
1982 
1983   unsigned OpCode = MI.getOperand(0).getImm();
1984   OpCode = convertTailJumpOpcode(OpCode);
1985   MCInst TC;
1986   TC.setOpcode(OpCode);
1987 
1988   // Before emitting the instruction, add a comment to indicate that this is
1989   // indeed a tail call.
1990   OutStreamer->AddComment("TAILCALL");
1991   for (auto &MO : drop_begin(MI.operands()))
1992     if (auto MaybeOperand = MCIL.LowerMachineOperand(&MI, MO))
1993       TC.addOperand(MaybeOperand.getValue());
1994   OutStreamer->emitInstruction(TC, getSubtargetInfo());
1995 }
1996 
1997 // Returns instruction preceding MBBI in MachineFunction.
1998 // If MBBI is the first instruction of the first basic block, returns null.
1999 static MachineBasicBlock::const_iterator
2000 PrevCrossBBInst(MachineBasicBlock::const_iterator MBBI) {
2001   const MachineBasicBlock *MBB = MBBI->getParent();
2002   while (MBBI == MBB->begin()) {
2003     if (MBB == &MBB->getParent()->front())
2004       return MachineBasicBlock::const_iterator();
2005     MBB = MBB->getPrevNode();
2006     MBBI = MBB->end();
2007   }
2008   --MBBI;
2009   return MBBI;
2010 }
2011 
2012 static const Constant *getConstantFromPool(const MachineInstr &MI,
2013                                            const MachineOperand &Op) {
2014   if (!Op.isCPI() || Op.getOffset() != 0)
2015     return nullptr;
2016 
2017   ArrayRef<MachineConstantPoolEntry> Constants =
2018       MI.getParent()->getParent()->getConstantPool()->getConstants();
2019   const MachineConstantPoolEntry &ConstantEntry = Constants[Op.getIndex()];
2020 
2021   // Bail if this is a machine constant pool entry, we won't be able to dig out
2022   // anything useful.
2023   if (ConstantEntry.isMachineConstantPoolEntry())
2024     return nullptr;
2025 
2026   return ConstantEntry.Val.ConstVal;
2027 }
2028 
2029 static std::string getShuffleComment(const MachineInstr *MI, unsigned SrcOp1Idx,
2030                                      unsigned SrcOp2Idx, ArrayRef<int> Mask) {
2031   std::string Comment;
2032 
2033   // Compute the name for a register. This is really goofy because we have
2034   // multiple instruction printers that could (in theory) use different
2035   // names. Fortunately most people use the ATT style (outside of Windows)
2036   // and they actually agree on register naming here. Ultimately, this is
2037   // a comment, and so its OK if it isn't perfect.
2038   auto GetRegisterName = [](unsigned RegNum) -> StringRef {
2039     return X86ATTInstPrinter::getRegisterName(RegNum);
2040   };
2041 
2042   const MachineOperand &DstOp = MI->getOperand(0);
2043   const MachineOperand &SrcOp1 = MI->getOperand(SrcOp1Idx);
2044   const MachineOperand &SrcOp2 = MI->getOperand(SrcOp2Idx);
2045 
2046   StringRef DstName = DstOp.isReg() ? GetRegisterName(DstOp.getReg()) : "mem";
2047   StringRef Src1Name =
2048       SrcOp1.isReg() ? GetRegisterName(SrcOp1.getReg()) : "mem";
2049   StringRef Src2Name =
2050       SrcOp2.isReg() ? GetRegisterName(SrcOp2.getReg()) : "mem";
2051 
2052   // One source operand, fix the mask to print all elements in one span.
2053   SmallVector<int, 8> ShuffleMask(Mask.begin(), Mask.end());
2054   if (Src1Name == Src2Name)
2055     for (int i = 0, e = ShuffleMask.size(); i != e; ++i)
2056       if (ShuffleMask[i] >= e)
2057         ShuffleMask[i] -= e;
2058 
2059   raw_string_ostream CS(Comment);
2060   CS << DstName;
2061 
2062   // Handle AVX512 MASK/MASXZ write mask comments.
2063   // MASK: zmmX {%kY}
2064   // MASKZ: zmmX {%kY} {z}
2065   if (SrcOp1Idx > 1) {
2066     assert((SrcOp1Idx == 2 || SrcOp1Idx == 3) && "Unexpected writemask");
2067 
2068     const MachineOperand &WriteMaskOp = MI->getOperand(SrcOp1Idx - 1);
2069     if (WriteMaskOp.isReg()) {
2070       CS << " {%" << GetRegisterName(WriteMaskOp.getReg()) << "}";
2071 
2072       if (SrcOp1Idx == 2) {
2073         CS << " {z}";
2074       }
2075     }
2076   }
2077 
2078   CS << " = ";
2079 
2080   for (int i = 0, e = ShuffleMask.size(); i != e; ++i) {
2081     if (i != 0)
2082       CS << ",";
2083     if (ShuffleMask[i] == SM_SentinelZero) {
2084       CS << "zero";
2085       continue;
2086     }
2087 
2088     // Otherwise, it must come from src1 or src2.  Print the span of elements
2089     // that comes from this src.
2090     bool isSrc1 = ShuffleMask[i] < (int)e;
2091     CS << (isSrc1 ? Src1Name : Src2Name) << '[';
2092 
2093     bool IsFirst = true;
2094     while (i != e && ShuffleMask[i] != SM_SentinelZero &&
2095            (ShuffleMask[i] < (int)e) == isSrc1) {
2096       if (!IsFirst)
2097         CS << ',';
2098       else
2099         IsFirst = false;
2100       if (ShuffleMask[i] == SM_SentinelUndef)
2101         CS << "u";
2102       else
2103         CS << ShuffleMask[i] % (int)e;
2104       ++i;
2105     }
2106     CS << ']';
2107     --i; // For loop increments element #.
2108   }
2109   CS.flush();
2110 
2111   return Comment;
2112 }
2113 
2114 static void printConstant(const APInt &Val, raw_ostream &CS) {
2115   if (Val.getBitWidth() <= 64) {
2116     CS << Val.getZExtValue();
2117   } else {
2118     // print multi-word constant as (w0,w1)
2119     CS << "(";
2120     for (int i = 0, N = Val.getNumWords(); i < N; ++i) {
2121       if (i > 0)
2122         CS << ",";
2123       CS << Val.getRawData()[i];
2124     }
2125     CS << ")";
2126   }
2127 }
2128 
2129 static void printConstant(const APFloat &Flt, raw_ostream &CS) {
2130   SmallString<32> Str;
2131   // Force scientific notation to distinquish from integers.
2132   Flt.toString(Str, 0, 0);
2133   CS << Str;
2134 }
2135 
2136 static void printConstant(const Constant *COp, raw_ostream &CS) {
2137   if (isa<UndefValue>(COp)) {
2138     CS << "u";
2139   } else if (auto *CI = dyn_cast<ConstantInt>(COp)) {
2140     printConstant(CI->getValue(), CS);
2141   } else if (auto *CF = dyn_cast<ConstantFP>(COp)) {
2142     printConstant(CF->getValueAPF(), CS);
2143   } else {
2144     CS << "?";
2145   }
2146 }
2147 
2148 void X86AsmPrinter::EmitSEHInstruction(const MachineInstr *MI) {
2149   assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
2150   assert(getSubtarget().isOSWindows() && "SEH_ instruction Windows only");
2151 
2152   // Use the .cv_fpo directives if we're emitting CodeView on 32-bit x86.
2153   if (EmitFPOData) {
2154     X86TargetStreamer *XTS =
2155         static_cast<X86TargetStreamer *>(OutStreamer->getTargetStreamer());
2156     switch (MI->getOpcode()) {
2157     case X86::SEH_PushReg:
2158       XTS->emitFPOPushReg(MI->getOperand(0).getImm());
2159       break;
2160     case X86::SEH_StackAlloc:
2161       XTS->emitFPOStackAlloc(MI->getOperand(0).getImm());
2162       break;
2163     case X86::SEH_StackAlign:
2164       XTS->emitFPOStackAlign(MI->getOperand(0).getImm());
2165       break;
2166     case X86::SEH_SetFrame:
2167       assert(MI->getOperand(1).getImm() == 0 &&
2168              ".cv_fpo_setframe takes no offset");
2169       XTS->emitFPOSetFrame(MI->getOperand(0).getImm());
2170       break;
2171     case X86::SEH_EndPrologue:
2172       XTS->emitFPOEndPrologue();
2173       break;
2174     case X86::SEH_SaveReg:
2175     case X86::SEH_SaveXMM:
2176     case X86::SEH_PushFrame:
2177       llvm_unreachable("SEH_ directive incompatible with FPO");
2178       break;
2179     default:
2180       llvm_unreachable("expected SEH_ instruction");
2181     }
2182     return;
2183   }
2184 
2185   // Otherwise, use the .seh_ directives for all other Windows platforms.
2186   switch (MI->getOpcode()) {
2187   case X86::SEH_PushReg:
2188     OutStreamer->EmitWinCFIPushReg(MI->getOperand(0).getImm());
2189     break;
2190 
2191   case X86::SEH_SaveReg:
2192     OutStreamer->EmitWinCFISaveReg(MI->getOperand(0).getImm(),
2193                                    MI->getOperand(1).getImm());
2194     break;
2195 
2196   case X86::SEH_SaveXMM:
2197     OutStreamer->EmitWinCFISaveXMM(MI->getOperand(0).getImm(),
2198                                    MI->getOperand(1).getImm());
2199     break;
2200 
2201   case X86::SEH_StackAlloc:
2202     OutStreamer->EmitWinCFIAllocStack(MI->getOperand(0).getImm());
2203     break;
2204 
2205   case X86::SEH_SetFrame:
2206     OutStreamer->EmitWinCFISetFrame(MI->getOperand(0).getImm(),
2207                                     MI->getOperand(1).getImm());
2208     break;
2209 
2210   case X86::SEH_PushFrame:
2211     OutStreamer->EmitWinCFIPushFrame(MI->getOperand(0).getImm());
2212     break;
2213 
2214   case X86::SEH_EndPrologue:
2215     OutStreamer->EmitWinCFIEndProlog();
2216     break;
2217 
2218   default:
2219     llvm_unreachable("expected SEH_ instruction");
2220   }
2221 }
2222 
2223 static unsigned getRegisterWidth(const MCOperandInfo &Info) {
2224   if (Info.RegClass == X86::VR128RegClassID ||
2225       Info.RegClass == X86::VR128XRegClassID)
2226     return 128;
2227   if (Info.RegClass == X86::VR256RegClassID ||
2228       Info.RegClass == X86::VR256XRegClassID)
2229     return 256;
2230   if (Info.RegClass == X86::VR512RegClassID)
2231     return 512;
2232   llvm_unreachable("Unknown register class!");
2233 }
2234 
2235 static void addConstantComments(const MachineInstr *MI,
2236                                 MCStreamer &OutStreamer) {
2237   switch (MI->getOpcode()) {
2238   // Lower PSHUFB and VPERMILP normally but add a comment if we can find
2239   // a constant shuffle mask. We won't be able to do this at the MC layer
2240   // because the mask isn't an immediate.
2241   case X86::PSHUFBrm:
2242   case X86::VPSHUFBrm:
2243   case X86::VPSHUFBYrm:
2244   case X86::VPSHUFBZ128rm:
2245   case X86::VPSHUFBZ128rmk:
2246   case X86::VPSHUFBZ128rmkz:
2247   case X86::VPSHUFBZ256rm:
2248   case X86::VPSHUFBZ256rmk:
2249   case X86::VPSHUFBZ256rmkz:
2250   case X86::VPSHUFBZrm:
2251   case X86::VPSHUFBZrmk:
2252   case X86::VPSHUFBZrmkz: {
2253     unsigned SrcIdx = 1;
2254     if (X86II::isKMasked(MI->getDesc().TSFlags)) {
2255       // Skip mask operand.
2256       ++SrcIdx;
2257       if (X86II::isKMergeMasked(MI->getDesc().TSFlags)) {
2258         // Skip passthru operand.
2259         ++SrcIdx;
2260       }
2261     }
2262     unsigned MaskIdx = SrcIdx + 1 + X86::AddrDisp;
2263 
2264     assert(MI->getNumOperands() >= (SrcIdx + 1 + X86::AddrNumOperands) &&
2265            "Unexpected number of operands!");
2266 
2267     const MachineOperand &MaskOp = MI->getOperand(MaskIdx);
2268     if (auto *C = getConstantFromPool(*MI, MaskOp)) {
2269       unsigned Width = getRegisterWidth(MI->getDesc().OpInfo[0]);
2270       SmallVector<int, 64> Mask;
2271       DecodePSHUFBMask(C, Width, Mask);
2272       if (!Mask.empty())
2273         OutStreamer.AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask));
2274     }
2275     break;
2276   }
2277 
2278   case X86::VPERMILPSrm:
2279   case X86::VPERMILPSYrm:
2280   case X86::VPERMILPSZ128rm:
2281   case X86::VPERMILPSZ128rmk:
2282   case X86::VPERMILPSZ128rmkz:
2283   case X86::VPERMILPSZ256rm:
2284   case X86::VPERMILPSZ256rmk:
2285   case X86::VPERMILPSZ256rmkz:
2286   case X86::VPERMILPSZrm:
2287   case X86::VPERMILPSZrmk:
2288   case X86::VPERMILPSZrmkz:
2289   case X86::VPERMILPDrm:
2290   case X86::VPERMILPDYrm:
2291   case X86::VPERMILPDZ128rm:
2292   case X86::VPERMILPDZ128rmk:
2293   case X86::VPERMILPDZ128rmkz:
2294   case X86::VPERMILPDZ256rm:
2295   case X86::VPERMILPDZ256rmk:
2296   case X86::VPERMILPDZ256rmkz:
2297   case X86::VPERMILPDZrm:
2298   case X86::VPERMILPDZrmk:
2299   case X86::VPERMILPDZrmkz: {
2300     unsigned ElSize;
2301     switch (MI->getOpcode()) {
2302     default: llvm_unreachable("Invalid opcode");
2303     case X86::VPERMILPSrm:
2304     case X86::VPERMILPSYrm:
2305     case X86::VPERMILPSZ128rm:
2306     case X86::VPERMILPSZ256rm:
2307     case X86::VPERMILPSZrm:
2308     case X86::VPERMILPSZ128rmkz:
2309     case X86::VPERMILPSZ256rmkz:
2310     case X86::VPERMILPSZrmkz:
2311     case X86::VPERMILPSZ128rmk:
2312     case X86::VPERMILPSZ256rmk:
2313     case X86::VPERMILPSZrmk:
2314       ElSize = 32;
2315       break;
2316     case X86::VPERMILPDrm:
2317     case X86::VPERMILPDYrm:
2318     case X86::VPERMILPDZ128rm:
2319     case X86::VPERMILPDZ256rm:
2320     case X86::VPERMILPDZrm:
2321     case X86::VPERMILPDZ128rmkz:
2322     case X86::VPERMILPDZ256rmkz:
2323     case X86::VPERMILPDZrmkz:
2324     case X86::VPERMILPDZ128rmk:
2325     case X86::VPERMILPDZ256rmk:
2326     case X86::VPERMILPDZrmk:
2327       ElSize = 64;
2328       break;
2329     }
2330 
2331     unsigned SrcIdx = 1;
2332     if (X86II::isKMasked(MI->getDesc().TSFlags)) {
2333       // Skip mask operand.
2334       ++SrcIdx;
2335       if (X86II::isKMergeMasked(MI->getDesc().TSFlags)) {
2336         // Skip passthru operand.
2337         ++SrcIdx;
2338       }
2339     }
2340     unsigned MaskIdx = SrcIdx + 1 + X86::AddrDisp;
2341 
2342     assert(MI->getNumOperands() >= (SrcIdx + 1 + X86::AddrNumOperands) &&
2343            "Unexpected number of operands!");
2344 
2345     const MachineOperand &MaskOp = MI->getOperand(MaskIdx);
2346     if (auto *C = getConstantFromPool(*MI, MaskOp)) {
2347       unsigned Width = getRegisterWidth(MI->getDesc().OpInfo[0]);
2348       SmallVector<int, 16> Mask;
2349       DecodeVPERMILPMask(C, ElSize, Width, Mask);
2350       if (!Mask.empty())
2351         OutStreamer.AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask));
2352     }
2353     break;
2354   }
2355 
2356   case X86::VPERMIL2PDrm:
2357   case X86::VPERMIL2PSrm:
2358   case X86::VPERMIL2PDYrm:
2359   case X86::VPERMIL2PSYrm: {
2360     assert(MI->getNumOperands() >= (3 + X86::AddrNumOperands + 1) &&
2361            "Unexpected number of operands!");
2362 
2363     const MachineOperand &CtrlOp = MI->getOperand(MI->getNumOperands() - 1);
2364     if (!CtrlOp.isImm())
2365       break;
2366 
2367     unsigned ElSize;
2368     switch (MI->getOpcode()) {
2369     default: llvm_unreachable("Invalid opcode");
2370     case X86::VPERMIL2PSrm: case X86::VPERMIL2PSYrm: ElSize = 32; break;
2371     case X86::VPERMIL2PDrm: case X86::VPERMIL2PDYrm: ElSize = 64; break;
2372     }
2373 
2374     const MachineOperand &MaskOp = MI->getOperand(3 + X86::AddrDisp);
2375     if (auto *C = getConstantFromPool(*MI, MaskOp)) {
2376       unsigned Width = getRegisterWidth(MI->getDesc().OpInfo[0]);
2377       SmallVector<int, 16> Mask;
2378       DecodeVPERMIL2PMask(C, (unsigned)CtrlOp.getImm(), ElSize, Width, Mask);
2379       if (!Mask.empty())
2380         OutStreamer.AddComment(getShuffleComment(MI, 1, 2, Mask));
2381     }
2382     break;
2383   }
2384 
2385   case X86::VPPERMrrm: {
2386     assert(MI->getNumOperands() >= (3 + X86::AddrNumOperands) &&
2387            "Unexpected number of operands!");
2388 
2389     const MachineOperand &MaskOp = MI->getOperand(3 + X86::AddrDisp);
2390     if (auto *C = getConstantFromPool(*MI, MaskOp)) {
2391       unsigned Width = getRegisterWidth(MI->getDesc().OpInfo[0]);
2392       SmallVector<int, 16> Mask;
2393       DecodeVPPERMMask(C, Width, Mask);
2394       if (!Mask.empty())
2395         OutStreamer.AddComment(getShuffleComment(MI, 1, 2, Mask));
2396     }
2397     break;
2398   }
2399 
2400   case X86::MMX_MOVQ64rm: {
2401     assert(MI->getNumOperands() == (1 + X86::AddrNumOperands) &&
2402            "Unexpected number of operands!");
2403     if (auto *C = getConstantFromPool(*MI, MI->getOperand(1 + X86::AddrDisp))) {
2404       std::string Comment;
2405       raw_string_ostream CS(Comment);
2406       const MachineOperand &DstOp = MI->getOperand(0);
2407       CS << X86ATTInstPrinter::getRegisterName(DstOp.getReg()) << " = ";
2408       if (auto *CF = dyn_cast<ConstantFP>(C)) {
2409         CS << "0x" << toString(CF->getValueAPF().bitcastToAPInt(), 16, false);
2410         OutStreamer.AddComment(CS.str());
2411       }
2412     }
2413     break;
2414   }
2415 
2416 #define MOV_CASE(Prefix, Suffix)                                               \
2417   case X86::Prefix##MOVAPD##Suffix##rm:                                        \
2418   case X86::Prefix##MOVAPS##Suffix##rm:                                        \
2419   case X86::Prefix##MOVUPD##Suffix##rm:                                        \
2420   case X86::Prefix##MOVUPS##Suffix##rm:                                        \
2421   case X86::Prefix##MOVDQA##Suffix##rm:                                        \
2422   case X86::Prefix##MOVDQU##Suffix##rm:
2423 
2424 #define MOV_AVX512_CASE(Suffix)                                                \
2425   case X86::VMOVDQA64##Suffix##rm:                                             \
2426   case X86::VMOVDQA32##Suffix##rm:                                             \
2427   case X86::VMOVDQU64##Suffix##rm:                                             \
2428   case X86::VMOVDQU32##Suffix##rm:                                             \
2429   case X86::VMOVDQU16##Suffix##rm:                                             \
2430   case X86::VMOVDQU8##Suffix##rm:                                              \
2431   case X86::VMOVAPS##Suffix##rm:                                               \
2432   case X86::VMOVAPD##Suffix##rm:                                               \
2433   case X86::VMOVUPS##Suffix##rm:                                               \
2434   case X86::VMOVUPD##Suffix##rm:
2435 
2436 #define CASE_ALL_MOV_RM()                                                      \
2437   MOV_CASE(, )   /* SSE */                                                     \
2438   MOV_CASE(V, )  /* AVX-128 */                                                 \
2439   MOV_CASE(V, Y) /* AVX-256 */                                                 \
2440   MOV_AVX512_CASE(Z)                                                           \
2441   MOV_AVX512_CASE(Z256)                                                        \
2442   MOV_AVX512_CASE(Z128)
2443 
2444     // For loads from a constant pool to a vector register, print the constant
2445     // loaded.
2446     CASE_ALL_MOV_RM()
2447   case X86::VBROADCASTF128:
2448   case X86::VBROADCASTI128:
2449   case X86::VBROADCASTF32X4Z256rm:
2450   case X86::VBROADCASTF32X4rm:
2451   case X86::VBROADCASTF32X8rm:
2452   case X86::VBROADCASTF64X2Z128rm:
2453   case X86::VBROADCASTF64X2rm:
2454   case X86::VBROADCASTF64X4rm:
2455   case X86::VBROADCASTI32X4Z256rm:
2456   case X86::VBROADCASTI32X4rm:
2457   case X86::VBROADCASTI32X8rm:
2458   case X86::VBROADCASTI64X2Z128rm:
2459   case X86::VBROADCASTI64X2rm:
2460   case X86::VBROADCASTI64X4rm:
2461     assert(MI->getNumOperands() >= (1 + X86::AddrNumOperands) &&
2462            "Unexpected number of operands!");
2463     if (auto *C = getConstantFromPool(*MI, MI->getOperand(1 + X86::AddrDisp))) {
2464       int NumLanes = 1;
2465       // Override NumLanes for the broadcast instructions.
2466       switch (MI->getOpcode()) {
2467       case X86::VBROADCASTF128:        NumLanes = 2; break;
2468       case X86::VBROADCASTI128:        NumLanes = 2; break;
2469       case X86::VBROADCASTF32X4Z256rm: NumLanes = 2; break;
2470       case X86::VBROADCASTF32X4rm:     NumLanes = 4; break;
2471       case X86::VBROADCASTF32X8rm:     NumLanes = 2; break;
2472       case X86::VBROADCASTF64X2Z128rm: NumLanes = 2; break;
2473       case X86::VBROADCASTF64X2rm:     NumLanes = 4; break;
2474       case X86::VBROADCASTF64X4rm:     NumLanes = 2; break;
2475       case X86::VBROADCASTI32X4Z256rm: NumLanes = 2; break;
2476       case X86::VBROADCASTI32X4rm:     NumLanes = 4; break;
2477       case X86::VBROADCASTI32X8rm:     NumLanes = 2; break;
2478       case X86::VBROADCASTI64X2Z128rm: NumLanes = 2; break;
2479       case X86::VBROADCASTI64X2rm:     NumLanes = 4; break;
2480       case X86::VBROADCASTI64X4rm:     NumLanes = 2; break;
2481       }
2482 
2483       std::string Comment;
2484       raw_string_ostream CS(Comment);
2485       const MachineOperand &DstOp = MI->getOperand(0);
2486       CS << X86ATTInstPrinter::getRegisterName(DstOp.getReg()) << " = ";
2487       if (auto *CDS = dyn_cast<ConstantDataSequential>(C)) {
2488         CS << "[";
2489         for (int l = 0; l != NumLanes; ++l) {
2490           for (int i = 0, NumElements = CDS->getNumElements(); i < NumElements;
2491                ++i) {
2492             if (i != 0 || l != 0)
2493               CS << ",";
2494             if (CDS->getElementType()->isIntegerTy())
2495               printConstant(CDS->getElementAsAPInt(i), CS);
2496             else if (CDS->getElementType()->isHalfTy() ||
2497                      CDS->getElementType()->isFloatTy() ||
2498                      CDS->getElementType()->isDoubleTy())
2499               printConstant(CDS->getElementAsAPFloat(i), CS);
2500             else
2501               CS << "?";
2502           }
2503         }
2504         CS << "]";
2505         OutStreamer.AddComment(CS.str());
2506       } else if (auto *CV = dyn_cast<ConstantVector>(C)) {
2507         CS << "<";
2508         for (int l = 0; l != NumLanes; ++l) {
2509           for (int i = 0, NumOperands = CV->getNumOperands(); i < NumOperands;
2510                ++i) {
2511             if (i != 0 || l != 0)
2512               CS << ",";
2513             printConstant(CV->getOperand(i), CS);
2514           }
2515         }
2516         CS << ">";
2517         OutStreamer.AddComment(CS.str());
2518       }
2519     }
2520     break;
2521 
2522   case X86::MOVDDUPrm:
2523   case X86::VMOVDDUPrm:
2524   case X86::VMOVDDUPZ128rm:
2525   case X86::VBROADCASTSSrm:
2526   case X86::VBROADCASTSSYrm:
2527   case X86::VBROADCASTSSZ128rm:
2528   case X86::VBROADCASTSSZ256rm:
2529   case X86::VBROADCASTSSZrm:
2530   case X86::VBROADCASTSDYrm:
2531   case X86::VBROADCASTSDZ256rm:
2532   case X86::VBROADCASTSDZrm:
2533   case X86::VPBROADCASTBrm:
2534   case X86::VPBROADCASTBYrm:
2535   case X86::VPBROADCASTBZ128rm:
2536   case X86::VPBROADCASTBZ256rm:
2537   case X86::VPBROADCASTBZrm:
2538   case X86::VPBROADCASTDrm:
2539   case X86::VPBROADCASTDYrm:
2540   case X86::VPBROADCASTDZ128rm:
2541   case X86::VPBROADCASTDZ256rm:
2542   case X86::VPBROADCASTDZrm:
2543   case X86::VPBROADCASTQrm:
2544   case X86::VPBROADCASTQYrm:
2545   case X86::VPBROADCASTQZ128rm:
2546   case X86::VPBROADCASTQZ256rm:
2547   case X86::VPBROADCASTQZrm:
2548   case X86::VPBROADCASTWrm:
2549   case X86::VPBROADCASTWYrm:
2550   case X86::VPBROADCASTWZ128rm:
2551   case X86::VPBROADCASTWZ256rm:
2552   case X86::VPBROADCASTWZrm:
2553     assert(MI->getNumOperands() >= (1 + X86::AddrNumOperands) &&
2554            "Unexpected number of operands!");
2555     if (auto *C = getConstantFromPool(*MI, MI->getOperand(1 + X86::AddrDisp))) {
2556       int NumElts;
2557       switch (MI->getOpcode()) {
2558       default: llvm_unreachable("Invalid opcode");
2559       case X86::MOVDDUPrm:          NumElts = 2;  break;
2560       case X86::VMOVDDUPrm:         NumElts = 2;  break;
2561       case X86::VMOVDDUPZ128rm:     NumElts = 2;  break;
2562       case X86::VBROADCASTSSrm:     NumElts = 4;  break;
2563       case X86::VBROADCASTSSYrm:    NumElts = 8;  break;
2564       case X86::VBROADCASTSSZ128rm: NumElts = 4;  break;
2565       case X86::VBROADCASTSSZ256rm: NumElts = 8;  break;
2566       case X86::VBROADCASTSSZrm:    NumElts = 16; break;
2567       case X86::VBROADCASTSDYrm:    NumElts = 4;  break;
2568       case X86::VBROADCASTSDZ256rm: NumElts = 4;  break;
2569       case X86::VBROADCASTSDZrm:    NumElts = 8;  break;
2570       case X86::VPBROADCASTBrm:     NumElts = 16; break;
2571       case X86::VPBROADCASTBYrm:    NumElts = 32; break;
2572       case X86::VPBROADCASTBZ128rm: NumElts = 16; break;
2573       case X86::VPBROADCASTBZ256rm: NumElts = 32; break;
2574       case X86::VPBROADCASTBZrm:    NumElts = 64; break;
2575       case X86::VPBROADCASTDrm:     NumElts = 4;  break;
2576       case X86::VPBROADCASTDYrm:    NumElts = 8;  break;
2577       case X86::VPBROADCASTDZ128rm: NumElts = 4;  break;
2578       case X86::VPBROADCASTDZ256rm: NumElts = 8;  break;
2579       case X86::VPBROADCASTDZrm:    NumElts = 16; break;
2580       case X86::VPBROADCASTQrm:     NumElts = 2;  break;
2581       case X86::VPBROADCASTQYrm:    NumElts = 4;  break;
2582       case X86::VPBROADCASTQZ128rm: NumElts = 2;  break;
2583       case X86::VPBROADCASTQZ256rm: NumElts = 4;  break;
2584       case X86::VPBROADCASTQZrm:    NumElts = 8;  break;
2585       case X86::VPBROADCASTWrm:     NumElts = 8;  break;
2586       case X86::VPBROADCASTWYrm:    NumElts = 16; break;
2587       case X86::VPBROADCASTWZ128rm: NumElts = 8;  break;
2588       case X86::VPBROADCASTWZ256rm: NumElts = 16; break;
2589       case X86::VPBROADCASTWZrm:    NumElts = 32; break;
2590       }
2591 
2592       std::string Comment;
2593       raw_string_ostream CS(Comment);
2594       const MachineOperand &DstOp = MI->getOperand(0);
2595       CS << X86ATTInstPrinter::getRegisterName(DstOp.getReg()) << " = ";
2596       CS << "[";
2597       for (int i = 0; i != NumElts; ++i) {
2598         if (i != 0)
2599           CS << ",";
2600         printConstant(C, CS);
2601       }
2602       CS << "]";
2603       OutStreamer.AddComment(CS.str());
2604     }
2605   }
2606 }
2607 
2608 void X86AsmPrinter::emitInstruction(const MachineInstr *MI) {
2609   X86MCInstLower MCInstLowering(*MF, *this);
2610   const X86RegisterInfo *RI =
2611       MF->getSubtarget<X86Subtarget>().getRegisterInfo();
2612 
2613   // Add a comment about EVEX-2-VEX compression for AVX-512 instrs that
2614   // are compressed from EVEX encoding to VEX encoding.
2615   if (TM.Options.MCOptions.ShowMCEncoding) {
2616     if (MI->getAsmPrinterFlags() & X86::AC_EVEX_2_VEX)
2617       OutStreamer->AddComment("EVEX TO VEX Compression ", false);
2618   }
2619 
2620   // Add comments for values loaded from constant pool.
2621   if (OutStreamer->isVerboseAsm())
2622     addConstantComments(MI, *OutStreamer);
2623 
2624   switch (MI->getOpcode()) {
2625   case TargetOpcode::DBG_VALUE:
2626     llvm_unreachable("Should be handled target independently");
2627 
2628   // Emit nothing here but a comment if we can.
2629   case X86::Int_MemBarrier:
2630     OutStreamer->emitRawComment("MEMBARRIER");
2631     return;
2632 
2633   case X86::EH_RETURN:
2634   case X86::EH_RETURN64: {
2635     // Lower these as normal, but add some comments.
2636     Register Reg = MI->getOperand(0).getReg();
2637     OutStreamer->AddComment(StringRef("eh_return, addr: %") +
2638                             X86ATTInstPrinter::getRegisterName(Reg));
2639     break;
2640   }
2641   case X86::CLEANUPRET: {
2642     // Lower these as normal, but add some comments.
2643     OutStreamer->AddComment("CLEANUPRET");
2644     break;
2645   }
2646 
2647   case X86::CATCHRET: {
2648     // Lower these as normal, but add some comments.
2649     OutStreamer->AddComment("CATCHRET");
2650     break;
2651   }
2652 
2653   case X86::ENDBR32:
2654   case X86::ENDBR64: {
2655     // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
2656     // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
2657     // non-empty. If MI is the initial ENDBR, place the
2658     // __patchable_function_entries label after ENDBR.
2659     if (CurrentPatchableFunctionEntrySym &&
2660         CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
2661         MI == &MF->front().front()) {
2662       MCInst Inst;
2663       MCInstLowering.Lower(MI, Inst);
2664       EmitAndCountInstruction(Inst);
2665       CurrentPatchableFunctionEntrySym = createTempSymbol("patch");
2666       OutStreamer->emitLabel(CurrentPatchableFunctionEntrySym);
2667       return;
2668     }
2669     break;
2670   }
2671 
2672   case X86::TAILJMPr:
2673   case X86::TAILJMPm:
2674   case X86::TAILJMPd:
2675   case X86::TAILJMPd_CC:
2676   case X86::TAILJMPr64:
2677   case X86::TAILJMPm64:
2678   case X86::TAILJMPd64:
2679   case X86::TAILJMPd64_CC:
2680   case X86::TAILJMPr64_REX:
2681   case X86::TAILJMPm64_REX:
2682     // Lower these as normal, but add some comments.
2683     OutStreamer->AddComment("TAILCALL");
2684     break;
2685 
2686   case X86::TLS_addr32:
2687   case X86::TLS_addr64:
2688   case X86::TLS_addrX32:
2689   case X86::TLS_base_addr32:
2690   case X86::TLS_base_addr64:
2691   case X86::TLS_base_addrX32:
2692     return LowerTlsAddr(MCInstLowering, *MI);
2693 
2694   case X86::MOVPC32r: {
2695     // This is a pseudo op for a two instruction sequence with a label, which
2696     // looks like:
2697     //     call "L1$pb"
2698     // "L1$pb":
2699     //     popl %esi
2700 
2701     // Emit the call.
2702     MCSymbol *PICBase = MF->getPICBaseSymbol();
2703     // FIXME: We would like an efficient form for this, so we don't have to do a
2704     // lot of extra uniquing.
2705     EmitAndCountInstruction(
2706         MCInstBuilder(X86::CALLpcrel32)
2707             .addExpr(MCSymbolRefExpr::create(PICBase, OutContext)));
2708 
2709     const X86FrameLowering *FrameLowering =
2710         MF->getSubtarget<X86Subtarget>().getFrameLowering();
2711     bool hasFP = FrameLowering->hasFP(*MF);
2712 
2713     // TODO: This is needed only if we require precise CFA.
2714     bool HasActiveDwarfFrame = OutStreamer->getNumFrameInfos() &&
2715                                !OutStreamer->getDwarfFrameInfos().back().End;
2716 
2717     int stackGrowth = -RI->getSlotSize();
2718 
2719     if (HasActiveDwarfFrame && !hasFP) {
2720       OutStreamer->emitCFIAdjustCfaOffset(-stackGrowth);
2721     }
2722 
2723     // Emit the label.
2724     OutStreamer->emitLabel(PICBase);
2725 
2726     // popl $reg
2727     EmitAndCountInstruction(
2728         MCInstBuilder(X86::POP32r).addReg(MI->getOperand(0).getReg()));
2729 
2730     if (HasActiveDwarfFrame && !hasFP) {
2731       OutStreamer->emitCFIAdjustCfaOffset(stackGrowth);
2732     }
2733     return;
2734   }
2735 
2736   case X86::ADD32ri: {
2737     // Lower the MO_GOT_ABSOLUTE_ADDRESS form of ADD32ri.
2738     if (MI->getOperand(2).getTargetFlags() != X86II::MO_GOT_ABSOLUTE_ADDRESS)
2739       break;
2740 
2741     // Okay, we have something like:
2742     //  EAX = ADD32ri EAX, MO_GOT_ABSOLUTE_ADDRESS(@MYGLOBAL)
2743 
2744     // For this, we want to print something like:
2745     //   MYGLOBAL + (. - PICBASE)
2746     // However, we can't generate a ".", so just emit a new label here and refer
2747     // to it.
2748     MCSymbol *DotSym = OutContext.createTempSymbol();
2749     OutStreamer->emitLabel(DotSym);
2750 
2751     // Now that we have emitted the label, lower the complex operand expression.
2752     MCSymbol *OpSym = MCInstLowering.GetSymbolFromOperand(MI->getOperand(2));
2753 
2754     const MCExpr *DotExpr = MCSymbolRefExpr::create(DotSym, OutContext);
2755     const MCExpr *PICBase =
2756         MCSymbolRefExpr::create(MF->getPICBaseSymbol(), OutContext);
2757     DotExpr = MCBinaryExpr::createSub(DotExpr, PICBase, OutContext);
2758 
2759     DotExpr = MCBinaryExpr::createAdd(
2760         MCSymbolRefExpr::create(OpSym, OutContext), DotExpr, OutContext);
2761 
2762     EmitAndCountInstruction(MCInstBuilder(X86::ADD32ri)
2763                                 .addReg(MI->getOperand(0).getReg())
2764                                 .addReg(MI->getOperand(1).getReg())
2765                                 .addExpr(DotExpr));
2766     return;
2767   }
2768   case TargetOpcode::STATEPOINT:
2769     return LowerSTATEPOINT(*MI, MCInstLowering);
2770 
2771   case TargetOpcode::FAULTING_OP:
2772     return LowerFAULTING_OP(*MI, MCInstLowering);
2773 
2774   case TargetOpcode::FENTRY_CALL:
2775     return LowerFENTRY_CALL(*MI, MCInstLowering);
2776 
2777   case TargetOpcode::PATCHABLE_OP:
2778     return LowerPATCHABLE_OP(*MI, MCInstLowering);
2779 
2780   case TargetOpcode::STACKMAP:
2781     return LowerSTACKMAP(*MI);
2782 
2783   case TargetOpcode::PATCHPOINT:
2784     return LowerPATCHPOINT(*MI, MCInstLowering);
2785 
2786   case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
2787     return LowerPATCHABLE_FUNCTION_ENTER(*MI, MCInstLowering);
2788 
2789   case TargetOpcode::PATCHABLE_RET:
2790     return LowerPATCHABLE_RET(*MI, MCInstLowering);
2791 
2792   case TargetOpcode::PATCHABLE_TAIL_CALL:
2793     return LowerPATCHABLE_TAIL_CALL(*MI, MCInstLowering);
2794 
2795   case TargetOpcode::PATCHABLE_EVENT_CALL:
2796     return LowerPATCHABLE_EVENT_CALL(*MI, MCInstLowering);
2797 
2798   case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
2799     return LowerPATCHABLE_TYPED_EVENT_CALL(*MI, MCInstLowering);
2800 
2801   case X86::MORESTACK_RET:
2802     EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget)));
2803     return;
2804 
2805   case X86::ASAN_CHECK_MEMACCESS:
2806     return LowerASAN_CHECK_MEMACCESS(*MI);
2807 
2808   case X86::MORESTACK_RET_RESTORE_R10:
2809     // Return, then restore R10.
2810     EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget)));
2811     EmitAndCountInstruction(
2812         MCInstBuilder(X86::MOV64rr).addReg(X86::R10).addReg(X86::RAX));
2813     return;
2814 
2815   case X86::SEH_PushReg:
2816   case X86::SEH_SaveReg:
2817   case X86::SEH_SaveXMM:
2818   case X86::SEH_StackAlloc:
2819   case X86::SEH_StackAlign:
2820   case X86::SEH_SetFrame:
2821   case X86::SEH_PushFrame:
2822   case X86::SEH_EndPrologue:
2823     EmitSEHInstruction(MI);
2824     return;
2825 
2826   case X86::SEH_Epilogue: {
2827     assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
2828     MachineBasicBlock::const_iterator MBBI(MI);
2829     // Check if preceded by a call and emit nop if so.
2830     for (MBBI = PrevCrossBBInst(MBBI);
2831          MBBI != MachineBasicBlock::const_iterator();
2832          MBBI = PrevCrossBBInst(MBBI)) {
2833       // Conservatively assume that pseudo instructions don't emit code and keep
2834       // looking for a call. We may emit an unnecessary nop in some cases.
2835       if (!MBBI->isPseudo()) {
2836         if (MBBI->isCall())
2837           EmitAndCountInstruction(MCInstBuilder(X86::NOOP));
2838         break;
2839       }
2840     }
2841     return;
2842   }
2843   case X86::UBSAN_UD1:
2844     EmitAndCountInstruction(MCInstBuilder(X86::UD1Lm)
2845                                 .addReg(X86::EAX)
2846                                 .addReg(X86::EAX)
2847                                 .addImm(1)
2848                                 .addReg(X86::NoRegister)
2849                                 .addImm(MI->getOperand(0).getImm())
2850                                 .addReg(X86::NoRegister));
2851     return;
2852   }
2853 
2854   MCInst TmpInst;
2855   MCInstLowering.Lower(MI, TmpInst);
2856 
2857   // Stackmap shadows cannot include branch targets, so we can count the bytes
2858   // in a call towards the shadow, but must ensure that the no thread returns
2859   // in to the stackmap shadow.  The only way to achieve this is if the call
2860   // is at the end of the shadow.
2861   if (MI->isCall()) {
2862     // Count then size of the call towards the shadow
2863     SMShadowTracker.count(TmpInst, getSubtargetInfo(), CodeEmitter.get());
2864     // Then flush the shadow so that we fill with nops before the call, not
2865     // after it.
2866     SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo());
2867     // Then emit the call
2868     OutStreamer->emitInstruction(TmpInst, getSubtargetInfo());
2869     return;
2870   }
2871 
2872   EmitAndCountInstruction(TmpInst);
2873 }
2874