1 //===-- X86MCInstLower.cpp - Convert X86 MachineInstr to an MCInst --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains code to lower X86 MachineInstrs to their corresponding
10 // MCInst records.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "MCTargetDesc/X86ATTInstPrinter.h"
15 #include "MCTargetDesc/X86BaseInfo.h"
16 #include "MCTargetDesc/X86InstComments.h"
17 #include "MCTargetDesc/X86ShuffleDecode.h"
18 #include "MCTargetDesc/X86TargetStreamer.h"
19 #include "X86AsmPrinter.h"
20 #include "X86RegisterInfo.h"
21 #include "X86ShuffleDecodeConstantPool.h"
22 #include "X86Subtarget.h"
23 #include "llvm/ADT/Optional.h"
24 #include "llvm/ADT/SmallString.h"
25 #include "llvm/ADT/iterator_range.h"
26 #include "llvm/CodeGen/MachineConstantPool.h"
27 #include "llvm/CodeGen/MachineFunction.h"
28 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
29 #include "llvm/CodeGen/MachineOperand.h"
30 #include "llvm/CodeGen/StackMaps.h"
31 #include "llvm/IR/DataLayout.h"
32 #include "llvm/IR/GlobalValue.h"
33 #include "llvm/IR/Mangler.h"
34 #include "llvm/MC/MCAsmInfo.h"
35 #include "llvm/MC/MCCodeEmitter.h"
36 #include "llvm/MC/MCContext.h"
37 #include "llvm/MC/MCExpr.h"
38 #include "llvm/MC/MCFixup.h"
39 #include "llvm/MC/MCInst.h"
40 #include "llvm/MC/MCInstBuilder.h"
41 #include "llvm/MC/MCSection.h"
42 #include "llvm/MC/MCSectionELF.h"
43 #include "llvm/MC/MCStreamer.h"
44 #include "llvm/MC/MCSymbol.h"
45 #include "llvm/MC/MCSymbolELF.h"
46 #include "llvm/Support/TargetRegistry.h"
47 #include "llvm/Target/TargetLoweringObjectFile.h"
48 #include "llvm/Target/TargetMachine.h"
49 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
50 #include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
51 
52 using namespace llvm;
53 
54 namespace {
55 
56 /// X86MCInstLower - This class is used to lower an MachineInstr into an MCInst.
57 class X86MCInstLower {
58   MCContext &Ctx;
59   const MachineFunction &MF;
60   const TargetMachine &TM;
61   const MCAsmInfo &MAI;
62   X86AsmPrinter &AsmPrinter;
63 
64 public:
65   X86MCInstLower(const MachineFunction &MF, X86AsmPrinter &asmprinter);
66 
67   Optional<MCOperand> LowerMachineOperand(const MachineInstr *MI,
68                                           const MachineOperand &MO) const;
69   void Lower(const MachineInstr *MI, MCInst &OutMI) const;
70 
71   MCSymbol *GetSymbolFromOperand(const MachineOperand &MO) const;
72   MCOperand LowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym) const;
73 
74 private:
75   MachineModuleInfoMachO &getMachOMMI() const;
76 };
77 
78 } // end anonymous namespace
79 
80 /// A RAII helper which defines a region of instructions which can't have
81 /// padding added between them for correctness.
82 struct NoAutoPaddingScope {
83   MCStreamer &OS;
84   const bool OldAllowAutoPadding;
85   NoAutoPaddingScope(MCStreamer &OS)
86       : OS(OS), OldAllowAutoPadding(OS.getAllowAutoPadding()) {
87     changeAndComment(false);
88   }
89   ~NoAutoPaddingScope() { changeAndComment(OldAllowAutoPadding); }
90   void changeAndComment(bool b) {
91     if (b == OS.getAllowAutoPadding())
92       return;
93     OS.setAllowAutoPadding(b);
94     if (b)
95       OS.emitRawComment("autopadding");
96     else
97       OS.emitRawComment("noautopadding");
98   }
99 };
100 
101 // Emit a minimal sequence of nops spanning NumBytes bytes.
102 static void emitX86Nops(MCStreamer &OS, unsigned NumBytes,
103                         const X86Subtarget *Subtarget);
104 
105 void X86AsmPrinter::StackMapShadowTracker::count(MCInst &Inst,
106                                                  const MCSubtargetInfo &STI,
107                                                  MCCodeEmitter *CodeEmitter) {
108   if (InShadow) {
109     SmallString<256> Code;
110     SmallVector<MCFixup, 4> Fixups;
111     raw_svector_ostream VecOS(Code);
112     CodeEmitter->encodeInstruction(Inst, VecOS, Fixups, STI);
113     CurrentShadowSize += Code.size();
114     if (CurrentShadowSize >= RequiredShadowSize)
115       InShadow = false; // The shadow is big enough. Stop counting.
116   }
117 }
118 
119 void X86AsmPrinter::StackMapShadowTracker::emitShadowPadding(
120     MCStreamer &OutStreamer, const MCSubtargetInfo &STI) {
121   if (InShadow && CurrentShadowSize < RequiredShadowSize) {
122     InShadow = false;
123     emitX86Nops(OutStreamer, RequiredShadowSize - CurrentShadowSize,
124                 &MF->getSubtarget<X86Subtarget>());
125   }
126 }
127 
128 void X86AsmPrinter::EmitAndCountInstruction(MCInst &Inst) {
129   OutStreamer->emitInstruction(Inst, getSubtargetInfo());
130   SMShadowTracker.count(Inst, getSubtargetInfo(), CodeEmitter.get());
131 }
132 
133 X86MCInstLower::X86MCInstLower(const MachineFunction &mf,
134                                X86AsmPrinter &asmprinter)
135     : Ctx(mf.getContext()), MF(mf), TM(mf.getTarget()), MAI(*TM.getMCAsmInfo()),
136       AsmPrinter(asmprinter) {}
137 
138 MachineModuleInfoMachO &X86MCInstLower::getMachOMMI() const {
139   return MF.getMMI().getObjFileInfo<MachineModuleInfoMachO>();
140 }
141 
142 /// GetSymbolFromOperand - Lower an MO_GlobalAddress or MO_ExternalSymbol
143 /// operand to an MCSymbol.
144 MCSymbol *X86MCInstLower::GetSymbolFromOperand(const MachineOperand &MO) const {
145   const Triple &TT = TM.getTargetTriple();
146   if (MO.isGlobal() && TT.isOSBinFormatELF())
147     return AsmPrinter.getSymbolPreferLocal(*MO.getGlobal());
148 
149   const DataLayout &DL = MF.getDataLayout();
150   assert((MO.isGlobal() || MO.isSymbol() || MO.isMBB()) &&
151          "Isn't a symbol reference");
152 
153   MCSymbol *Sym = nullptr;
154   SmallString<128> Name;
155   StringRef Suffix;
156 
157   switch (MO.getTargetFlags()) {
158   case X86II::MO_DLLIMPORT:
159     // Handle dllimport linkage.
160     Name += "__imp_";
161     break;
162   case X86II::MO_COFFSTUB:
163     Name += ".refptr.";
164     break;
165   case X86II::MO_DARWIN_NONLAZY:
166   case X86II::MO_DARWIN_NONLAZY_PIC_BASE:
167     Suffix = "$non_lazy_ptr";
168     break;
169   }
170 
171   if (!Suffix.empty())
172     Name += DL.getPrivateGlobalPrefix();
173 
174   if (MO.isGlobal()) {
175     const GlobalValue *GV = MO.getGlobal();
176     AsmPrinter.getNameWithPrefix(Name, GV);
177   } else if (MO.isSymbol()) {
178     Mangler::getNameWithPrefix(Name, MO.getSymbolName(), DL);
179   } else if (MO.isMBB()) {
180     assert(Suffix.empty());
181     Sym = MO.getMBB()->getSymbol();
182   }
183 
184   Name += Suffix;
185   if (!Sym)
186     Sym = Ctx.getOrCreateSymbol(Name);
187 
188   // If the target flags on the operand changes the name of the symbol, do that
189   // before we return the symbol.
190   switch (MO.getTargetFlags()) {
191   default:
192     break;
193   case X86II::MO_COFFSTUB: {
194     MachineModuleInfoCOFF &MMICOFF =
195         MF.getMMI().getObjFileInfo<MachineModuleInfoCOFF>();
196     MachineModuleInfoImpl::StubValueTy &StubSym = MMICOFF.getGVStubEntry(Sym);
197     if (!StubSym.getPointer()) {
198       assert(MO.isGlobal() && "Extern symbol not handled yet");
199       StubSym = MachineModuleInfoImpl::StubValueTy(
200           AsmPrinter.getSymbol(MO.getGlobal()), true);
201     }
202     break;
203   }
204   case X86II::MO_DARWIN_NONLAZY:
205   case X86II::MO_DARWIN_NONLAZY_PIC_BASE: {
206     MachineModuleInfoImpl::StubValueTy &StubSym =
207         getMachOMMI().getGVStubEntry(Sym);
208     if (!StubSym.getPointer()) {
209       assert(MO.isGlobal() && "Extern symbol not handled yet");
210       StubSym = MachineModuleInfoImpl::StubValueTy(
211           AsmPrinter.getSymbol(MO.getGlobal()),
212           !MO.getGlobal()->hasInternalLinkage());
213     }
214     break;
215   }
216   }
217 
218   return Sym;
219 }
220 
221 MCOperand X86MCInstLower::LowerSymbolOperand(const MachineOperand &MO,
222                                              MCSymbol *Sym) const {
223   // FIXME: We would like an efficient form for this, so we don't have to do a
224   // lot of extra uniquing.
225   const MCExpr *Expr = nullptr;
226   MCSymbolRefExpr::VariantKind RefKind = MCSymbolRefExpr::VK_None;
227 
228   switch (MO.getTargetFlags()) {
229   default:
230     llvm_unreachable("Unknown target flag on GV operand");
231   case X86II::MO_NO_FLAG: // No flag.
232   // These affect the name of the symbol, not any suffix.
233   case X86II::MO_DARWIN_NONLAZY:
234   case X86II::MO_DLLIMPORT:
235   case X86II::MO_COFFSTUB:
236     break;
237 
238   case X86II::MO_TLVP:
239     RefKind = MCSymbolRefExpr::VK_TLVP;
240     break;
241   case X86II::MO_TLVP_PIC_BASE:
242     Expr = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_TLVP, Ctx);
243     // Subtract the pic base.
244     Expr = MCBinaryExpr::createSub(
245         Expr, MCSymbolRefExpr::create(MF.getPICBaseSymbol(), Ctx), Ctx);
246     break;
247   case X86II::MO_SECREL:
248     RefKind = MCSymbolRefExpr::VK_SECREL;
249     break;
250   case X86II::MO_TLSGD:
251     RefKind = MCSymbolRefExpr::VK_TLSGD;
252     break;
253   case X86II::MO_TLSLD:
254     RefKind = MCSymbolRefExpr::VK_TLSLD;
255     break;
256   case X86II::MO_TLSLDM:
257     RefKind = MCSymbolRefExpr::VK_TLSLDM;
258     break;
259   case X86II::MO_GOTTPOFF:
260     RefKind = MCSymbolRefExpr::VK_GOTTPOFF;
261     break;
262   case X86II::MO_INDNTPOFF:
263     RefKind = MCSymbolRefExpr::VK_INDNTPOFF;
264     break;
265   case X86II::MO_TPOFF:
266     RefKind = MCSymbolRefExpr::VK_TPOFF;
267     break;
268   case X86II::MO_DTPOFF:
269     RefKind = MCSymbolRefExpr::VK_DTPOFF;
270     break;
271   case X86II::MO_NTPOFF:
272     RefKind = MCSymbolRefExpr::VK_NTPOFF;
273     break;
274   case X86II::MO_GOTNTPOFF:
275     RefKind = MCSymbolRefExpr::VK_GOTNTPOFF;
276     break;
277   case X86II::MO_GOTPCREL:
278     RefKind = MCSymbolRefExpr::VK_GOTPCREL;
279     break;
280   case X86II::MO_GOT:
281     RefKind = MCSymbolRefExpr::VK_GOT;
282     break;
283   case X86II::MO_GOTOFF:
284     RefKind = MCSymbolRefExpr::VK_GOTOFF;
285     break;
286   case X86II::MO_PLT:
287     RefKind = MCSymbolRefExpr::VK_PLT;
288     break;
289   case X86II::MO_ABS8:
290     RefKind = MCSymbolRefExpr::VK_X86_ABS8;
291     break;
292   case X86II::MO_PIC_BASE_OFFSET:
293   case X86II::MO_DARWIN_NONLAZY_PIC_BASE:
294     Expr = MCSymbolRefExpr::create(Sym, Ctx);
295     // Subtract the pic base.
296     Expr = MCBinaryExpr::createSub(
297         Expr, MCSymbolRefExpr::create(MF.getPICBaseSymbol(), Ctx), Ctx);
298     if (MO.isJTI()) {
299       assert(MAI.doesSetDirectiveSuppressReloc());
300       // If .set directive is supported, use it to reduce the number of
301       // relocations the assembler will generate for differences between
302       // local labels. This is only safe when the symbols are in the same
303       // section so we are restricting it to jumptable references.
304       MCSymbol *Label = Ctx.createTempSymbol();
305       AsmPrinter.OutStreamer->emitAssignment(Label, Expr);
306       Expr = MCSymbolRefExpr::create(Label, Ctx);
307     }
308     break;
309   }
310 
311   if (!Expr)
312     Expr = MCSymbolRefExpr::create(Sym, RefKind, Ctx);
313 
314   if (!MO.isJTI() && !MO.isMBB() && MO.getOffset())
315     Expr = MCBinaryExpr::createAdd(
316         Expr, MCConstantExpr::create(MO.getOffset(), Ctx), Ctx);
317   return MCOperand::createExpr(Expr);
318 }
319 
320 /// Simplify FOO $imm, %{al,ax,eax,rax} to FOO $imm, for instruction with
321 /// a short fixed-register form.
322 static void SimplifyShortImmForm(MCInst &Inst, unsigned Opcode) {
323   unsigned ImmOp = Inst.getNumOperands() - 1;
324   assert(Inst.getOperand(0).isReg() &&
325          (Inst.getOperand(ImmOp).isImm() || Inst.getOperand(ImmOp).isExpr()) &&
326          ((Inst.getNumOperands() == 3 && Inst.getOperand(1).isReg() &&
327            Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg()) ||
328           Inst.getNumOperands() == 2) &&
329          "Unexpected instruction!");
330 
331   // Check whether the destination register can be fixed.
332   unsigned Reg = Inst.getOperand(0).getReg();
333   if (Reg != X86::AL && Reg != X86::AX && Reg != X86::EAX && Reg != X86::RAX)
334     return;
335 
336   // If so, rewrite the instruction.
337   MCOperand Saved = Inst.getOperand(ImmOp);
338   Inst = MCInst();
339   Inst.setOpcode(Opcode);
340   Inst.addOperand(Saved);
341 }
342 
343 /// If a movsx instruction has a shorter encoding for the used register
344 /// simplify the instruction to use it instead.
345 static void SimplifyMOVSX(MCInst &Inst) {
346   unsigned NewOpcode = 0;
347   unsigned Op0 = Inst.getOperand(0).getReg(), Op1 = Inst.getOperand(1).getReg();
348   switch (Inst.getOpcode()) {
349   default:
350     llvm_unreachable("Unexpected instruction!");
351   case X86::MOVSX16rr8: // movsbw %al, %ax   --> cbtw
352     if (Op0 == X86::AX && Op1 == X86::AL)
353       NewOpcode = X86::CBW;
354     break;
355   case X86::MOVSX32rr16: // movswl %ax, %eax  --> cwtl
356     if (Op0 == X86::EAX && Op1 == X86::AX)
357       NewOpcode = X86::CWDE;
358     break;
359   case X86::MOVSX64rr32: // movslq %eax, %rax --> cltq
360     if (Op0 == X86::RAX && Op1 == X86::EAX)
361       NewOpcode = X86::CDQE;
362     break;
363   }
364 
365   if (NewOpcode != 0) {
366     Inst = MCInst();
367     Inst.setOpcode(NewOpcode);
368   }
369 }
370 
371 /// Simplify things like MOV32rm to MOV32o32a.
372 static void SimplifyShortMoveForm(X86AsmPrinter &Printer, MCInst &Inst,
373                                   unsigned Opcode) {
374   // Don't make these simplifications in 64-bit mode; other assemblers don't
375   // perform them because they make the code larger.
376   if (Printer.getSubtarget().is64Bit())
377     return;
378 
379   bool IsStore = Inst.getOperand(0).isReg() && Inst.getOperand(1).isReg();
380   unsigned AddrBase = IsStore;
381   unsigned RegOp = IsStore ? 0 : 5;
382   unsigned AddrOp = AddrBase + 3;
383   assert(
384       Inst.getNumOperands() == 6 && Inst.getOperand(RegOp).isReg() &&
385       Inst.getOperand(AddrBase + X86::AddrBaseReg).isReg() &&
386       Inst.getOperand(AddrBase + X86::AddrScaleAmt).isImm() &&
387       Inst.getOperand(AddrBase + X86::AddrIndexReg).isReg() &&
388       Inst.getOperand(AddrBase + X86::AddrSegmentReg).isReg() &&
389       (Inst.getOperand(AddrOp).isExpr() || Inst.getOperand(AddrOp).isImm()) &&
390       "Unexpected instruction!");
391 
392   // Check whether the destination register can be fixed.
393   unsigned Reg = Inst.getOperand(RegOp).getReg();
394   if (Reg != X86::AL && Reg != X86::AX && Reg != X86::EAX && Reg != X86::RAX)
395     return;
396 
397   // Check whether this is an absolute address.
398   // FIXME: We know TLVP symbol refs aren't, but there should be a better way
399   // to do this here.
400   bool Absolute = true;
401   if (Inst.getOperand(AddrOp).isExpr()) {
402     const MCExpr *MCE = Inst.getOperand(AddrOp).getExpr();
403     if (const MCSymbolRefExpr *SRE = dyn_cast<MCSymbolRefExpr>(MCE))
404       if (SRE->getKind() == MCSymbolRefExpr::VK_TLVP)
405         Absolute = false;
406   }
407 
408   if (Absolute &&
409       (Inst.getOperand(AddrBase + X86::AddrBaseReg).getReg() != 0 ||
410        Inst.getOperand(AddrBase + X86::AddrScaleAmt).getImm() != 1 ||
411        Inst.getOperand(AddrBase + X86::AddrIndexReg).getReg() != 0))
412     return;
413 
414   // If so, rewrite the instruction.
415   MCOperand Saved = Inst.getOperand(AddrOp);
416   MCOperand Seg = Inst.getOperand(AddrBase + X86::AddrSegmentReg);
417   Inst = MCInst();
418   Inst.setOpcode(Opcode);
419   Inst.addOperand(Saved);
420   Inst.addOperand(Seg);
421 }
422 
423 static unsigned getRetOpcode(const X86Subtarget &Subtarget) {
424   return Subtarget.is64Bit() ? X86::RETQ : X86::RETL;
425 }
426 
427 Optional<MCOperand>
428 X86MCInstLower::LowerMachineOperand(const MachineInstr *MI,
429                                     const MachineOperand &MO) const {
430   switch (MO.getType()) {
431   default:
432     MI->print(errs());
433     llvm_unreachable("unknown operand type");
434   case MachineOperand::MO_Register:
435     // Ignore all implicit register operands.
436     if (MO.isImplicit())
437       return None;
438     return MCOperand::createReg(MO.getReg());
439   case MachineOperand::MO_Immediate:
440     return MCOperand::createImm(MO.getImm());
441   case MachineOperand::MO_MachineBasicBlock:
442   case MachineOperand::MO_GlobalAddress:
443   case MachineOperand::MO_ExternalSymbol:
444     return LowerSymbolOperand(MO, GetSymbolFromOperand(MO));
445   case MachineOperand::MO_MCSymbol:
446     return LowerSymbolOperand(MO, MO.getMCSymbol());
447   case MachineOperand::MO_JumpTableIndex:
448     return LowerSymbolOperand(MO, AsmPrinter.GetJTISymbol(MO.getIndex()));
449   case MachineOperand::MO_ConstantPoolIndex:
450     return LowerSymbolOperand(MO, AsmPrinter.GetCPISymbol(MO.getIndex()));
451   case MachineOperand::MO_BlockAddress:
452     return LowerSymbolOperand(
453         MO, AsmPrinter.GetBlockAddressSymbol(MO.getBlockAddress()));
454   case MachineOperand::MO_RegisterMask:
455     // Ignore call clobbers.
456     return None;
457   }
458 }
459 
460 // Replace TAILJMP opcodes with their equivalent opcodes that have encoding
461 // information.
462 static unsigned convertTailJumpOpcode(unsigned Opcode) {
463   switch (Opcode) {
464   case X86::TAILJMPr:
465     Opcode = X86::JMP32r;
466     break;
467   case X86::TAILJMPm:
468     Opcode = X86::JMP32m;
469     break;
470   case X86::TAILJMPr64:
471     Opcode = X86::JMP64r;
472     break;
473   case X86::TAILJMPm64:
474     Opcode = X86::JMP64m;
475     break;
476   case X86::TAILJMPr64_REX:
477     Opcode = X86::JMP64r_REX;
478     break;
479   case X86::TAILJMPm64_REX:
480     Opcode = X86::JMP64m_REX;
481     break;
482   case X86::TAILJMPd:
483   case X86::TAILJMPd64:
484     Opcode = X86::JMP_1;
485     break;
486   case X86::TAILJMPd_CC:
487   case X86::TAILJMPd64_CC:
488     Opcode = X86::JCC_1;
489     break;
490   }
491 
492   return Opcode;
493 }
494 
495 void X86MCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
496   OutMI.setOpcode(MI->getOpcode());
497 
498   for (const MachineOperand &MO : MI->operands())
499     if (auto MaybeMCOp = LowerMachineOperand(MI, MO))
500       OutMI.addOperand(MaybeMCOp.getValue());
501 
502   // Handle a few special cases to eliminate operand modifiers.
503   switch (OutMI.getOpcode()) {
504   case X86::LEA64_32r:
505   case X86::LEA64r:
506   case X86::LEA16r:
507   case X86::LEA32r:
508     // LEA should have a segment register, but it must be empty.
509     assert(OutMI.getNumOperands() == 1 + X86::AddrNumOperands &&
510            "Unexpected # of LEA operands");
511     assert(OutMI.getOperand(1 + X86::AddrSegmentReg).getReg() == 0 &&
512            "LEA has segment specified!");
513     break;
514 
515   case X86::MULX32Hrr:
516   case X86::MULX32Hrm:
517   case X86::MULX64Hrr:
518   case X86::MULX64Hrm: {
519     // Turn into regular MULX by duplicating the destination.
520     unsigned NewOpc;
521     switch (OutMI.getOpcode()) {
522     default: llvm_unreachable("Invalid opcode");
523     case X86::MULX32Hrr: NewOpc = X86::MULX32rr; break;
524     case X86::MULX32Hrm: NewOpc = X86::MULX32rm; break;
525     case X86::MULX64Hrr: NewOpc = X86::MULX64rr; break;
526     case X86::MULX64Hrm: NewOpc = X86::MULX64rm; break;
527     }
528     OutMI.setOpcode(NewOpc);
529     // Duplicate the destination.
530     unsigned DestReg = OutMI.getOperand(0).getReg();
531     OutMI.insert(OutMI.begin(), MCOperand::createReg(DestReg));
532     break;
533   }
534 
535   // Commute operands to get a smaller encoding by using VEX.R instead of VEX.B
536   // if one of the registers is extended, but other isn't.
537   case X86::VMOVZPQILo2PQIrr:
538   case X86::VMOVAPDrr:
539   case X86::VMOVAPDYrr:
540   case X86::VMOVAPSrr:
541   case X86::VMOVAPSYrr:
542   case X86::VMOVDQArr:
543   case X86::VMOVDQAYrr:
544   case X86::VMOVDQUrr:
545   case X86::VMOVDQUYrr:
546   case X86::VMOVUPDrr:
547   case X86::VMOVUPDYrr:
548   case X86::VMOVUPSrr:
549   case X86::VMOVUPSYrr: {
550     if (!X86II::isX86_64ExtendedReg(OutMI.getOperand(0).getReg()) &&
551         X86II::isX86_64ExtendedReg(OutMI.getOperand(1).getReg())) {
552       unsigned NewOpc;
553       switch (OutMI.getOpcode()) {
554       default: llvm_unreachable("Invalid opcode");
555       case X86::VMOVZPQILo2PQIrr: NewOpc = X86::VMOVPQI2QIrr;   break;
556       case X86::VMOVAPDrr:        NewOpc = X86::VMOVAPDrr_REV;  break;
557       case X86::VMOVAPDYrr:       NewOpc = X86::VMOVAPDYrr_REV; break;
558       case X86::VMOVAPSrr:        NewOpc = X86::VMOVAPSrr_REV;  break;
559       case X86::VMOVAPSYrr:       NewOpc = X86::VMOVAPSYrr_REV; break;
560       case X86::VMOVDQArr:        NewOpc = X86::VMOVDQArr_REV;  break;
561       case X86::VMOVDQAYrr:       NewOpc = X86::VMOVDQAYrr_REV; break;
562       case X86::VMOVDQUrr:        NewOpc = X86::VMOVDQUrr_REV;  break;
563       case X86::VMOVDQUYrr:       NewOpc = X86::VMOVDQUYrr_REV; break;
564       case X86::VMOVUPDrr:        NewOpc = X86::VMOVUPDrr_REV;  break;
565       case X86::VMOVUPDYrr:       NewOpc = X86::VMOVUPDYrr_REV; break;
566       case X86::VMOVUPSrr:        NewOpc = X86::VMOVUPSrr_REV;  break;
567       case X86::VMOVUPSYrr:       NewOpc = X86::VMOVUPSYrr_REV; break;
568       }
569       OutMI.setOpcode(NewOpc);
570     }
571     break;
572   }
573   case X86::VMOVSDrr:
574   case X86::VMOVSSrr: {
575     if (!X86II::isX86_64ExtendedReg(OutMI.getOperand(0).getReg()) &&
576         X86II::isX86_64ExtendedReg(OutMI.getOperand(2).getReg())) {
577       unsigned NewOpc;
578       switch (OutMI.getOpcode()) {
579       default: llvm_unreachable("Invalid opcode");
580       case X86::VMOVSDrr: NewOpc = X86::VMOVSDrr_REV; break;
581       case X86::VMOVSSrr: NewOpc = X86::VMOVSSrr_REV; break;
582       }
583       OutMI.setOpcode(NewOpc);
584     }
585     break;
586   }
587 
588   case X86::VPCMPBZ128rmi:  case X86::VPCMPBZ128rmik:
589   case X86::VPCMPBZ128rri:  case X86::VPCMPBZ128rrik:
590   case X86::VPCMPBZ256rmi:  case X86::VPCMPBZ256rmik:
591   case X86::VPCMPBZ256rri:  case X86::VPCMPBZ256rrik:
592   case X86::VPCMPBZrmi:     case X86::VPCMPBZrmik:
593   case X86::VPCMPBZrri:     case X86::VPCMPBZrrik:
594   case X86::VPCMPDZ128rmi:  case X86::VPCMPDZ128rmik:
595   case X86::VPCMPDZ128rmib: case X86::VPCMPDZ128rmibk:
596   case X86::VPCMPDZ128rri:  case X86::VPCMPDZ128rrik:
597   case X86::VPCMPDZ256rmi:  case X86::VPCMPDZ256rmik:
598   case X86::VPCMPDZ256rmib: case X86::VPCMPDZ256rmibk:
599   case X86::VPCMPDZ256rri:  case X86::VPCMPDZ256rrik:
600   case X86::VPCMPDZrmi:     case X86::VPCMPDZrmik:
601   case X86::VPCMPDZrmib:    case X86::VPCMPDZrmibk:
602   case X86::VPCMPDZrri:     case X86::VPCMPDZrrik:
603   case X86::VPCMPQZ128rmi:  case X86::VPCMPQZ128rmik:
604   case X86::VPCMPQZ128rmib: case X86::VPCMPQZ128rmibk:
605   case X86::VPCMPQZ128rri:  case X86::VPCMPQZ128rrik:
606   case X86::VPCMPQZ256rmi:  case X86::VPCMPQZ256rmik:
607   case X86::VPCMPQZ256rmib: case X86::VPCMPQZ256rmibk:
608   case X86::VPCMPQZ256rri:  case X86::VPCMPQZ256rrik:
609   case X86::VPCMPQZrmi:     case X86::VPCMPQZrmik:
610   case X86::VPCMPQZrmib:    case X86::VPCMPQZrmibk:
611   case X86::VPCMPQZrri:     case X86::VPCMPQZrrik:
612   case X86::VPCMPWZ128rmi:  case X86::VPCMPWZ128rmik:
613   case X86::VPCMPWZ128rri:  case X86::VPCMPWZ128rrik:
614   case X86::VPCMPWZ256rmi:  case X86::VPCMPWZ256rmik:
615   case X86::VPCMPWZ256rri:  case X86::VPCMPWZ256rrik:
616   case X86::VPCMPWZrmi:     case X86::VPCMPWZrmik:
617   case X86::VPCMPWZrri:     case X86::VPCMPWZrrik: {
618     // Turn immediate 0 into the VPCMPEQ instruction.
619     if (OutMI.getOperand(OutMI.getNumOperands() - 1).getImm() == 0) {
620       unsigned NewOpc;
621       switch (OutMI.getOpcode()) {
622       default: llvm_unreachable("Invalid opcode");
623       case X86::VPCMPBZ128rmi:   NewOpc = X86::VPCMPEQBZ128rm;   break;
624       case X86::VPCMPBZ128rmik:  NewOpc = X86::VPCMPEQBZ128rmk;  break;
625       case X86::VPCMPBZ128rri:   NewOpc = X86::VPCMPEQBZ128rr;   break;
626       case X86::VPCMPBZ128rrik:  NewOpc = X86::VPCMPEQBZ128rrk;  break;
627       case X86::VPCMPBZ256rmi:   NewOpc = X86::VPCMPEQBZ256rm;   break;
628       case X86::VPCMPBZ256rmik:  NewOpc = X86::VPCMPEQBZ256rmk;  break;
629       case X86::VPCMPBZ256rri:   NewOpc = X86::VPCMPEQBZ256rr;   break;
630       case X86::VPCMPBZ256rrik:  NewOpc = X86::VPCMPEQBZ256rrk;  break;
631       case X86::VPCMPBZrmi:      NewOpc = X86::VPCMPEQBZrm;      break;
632       case X86::VPCMPBZrmik:     NewOpc = X86::VPCMPEQBZrmk;     break;
633       case X86::VPCMPBZrri:      NewOpc = X86::VPCMPEQBZrr;      break;
634       case X86::VPCMPBZrrik:     NewOpc = X86::VPCMPEQBZrrk;     break;
635       case X86::VPCMPDZ128rmi:   NewOpc = X86::VPCMPEQDZ128rm;   break;
636       case X86::VPCMPDZ128rmib:  NewOpc = X86::VPCMPEQDZ128rmb;  break;
637       case X86::VPCMPDZ128rmibk: NewOpc = X86::VPCMPEQDZ128rmbk; break;
638       case X86::VPCMPDZ128rmik:  NewOpc = X86::VPCMPEQDZ128rmk;  break;
639       case X86::VPCMPDZ128rri:   NewOpc = X86::VPCMPEQDZ128rr;   break;
640       case X86::VPCMPDZ128rrik:  NewOpc = X86::VPCMPEQDZ128rrk;  break;
641       case X86::VPCMPDZ256rmi:   NewOpc = X86::VPCMPEQDZ256rm;   break;
642       case X86::VPCMPDZ256rmib:  NewOpc = X86::VPCMPEQDZ256rmb;  break;
643       case X86::VPCMPDZ256rmibk: NewOpc = X86::VPCMPEQDZ256rmbk; break;
644       case X86::VPCMPDZ256rmik:  NewOpc = X86::VPCMPEQDZ256rmk;  break;
645       case X86::VPCMPDZ256rri:   NewOpc = X86::VPCMPEQDZ256rr;   break;
646       case X86::VPCMPDZ256rrik:  NewOpc = X86::VPCMPEQDZ256rrk;  break;
647       case X86::VPCMPDZrmi:      NewOpc = X86::VPCMPEQDZrm;      break;
648       case X86::VPCMPDZrmib:     NewOpc = X86::VPCMPEQDZrmb;     break;
649       case X86::VPCMPDZrmibk:    NewOpc = X86::VPCMPEQDZrmbk;    break;
650       case X86::VPCMPDZrmik:     NewOpc = X86::VPCMPEQDZrmk;     break;
651       case X86::VPCMPDZrri:      NewOpc = X86::VPCMPEQDZrr;      break;
652       case X86::VPCMPDZrrik:     NewOpc = X86::VPCMPEQDZrrk;     break;
653       case X86::VPCMPQZ128rmi:   NewOpc = X86::VPCMPEQQZ128rm;   break;
654       case X86::VPCMPQZ128rmib:  NewOpc = X86::VPCMPEQQZ128rmb;  break;
655       case X86::VPCMPQZ128rmibk: NewOpc = X86::VPCMPEQQZ128rmbk; break;
656       case X86::VPCMPQZ128rmik:  NewOpc = X86::VPCMPEQQZ128rmk;  break;
657       case X86::VPCMPQZ128rri:   NewOpc = X86::VPCMPEQQZ128rr;   break;
658       case X86::VPCMPQZ128rrik:  NewOpc = X86::VPCMPEQQZ128rrk;  break;
659       case X86::VPCMPQZ256rmi:   NewOpc = X86::VPCMPEQQZ256rm;   break;
660       case X86::VPCMPQZ256rmib:  NewOpc = X86::VPCMPEQQZ256rmb;  break;
661       case X86::VPCMPQZ256rmibk: NewOpc = X86::VPCMPEQQZ256rmbk; break;
662       case X86::VPCMPQZ256rmik:  NewOpc = X86::VPCMPEQQZ256rmk;  break;
663       case X86::VPCMPQZ256rri:   NewOpc = X86::VPCMPEQQZ256rr;   break;
664       case X86::VPCMPQZ256rrik:  NewOpc = X86::VPCMPEQQZ256rrk;  break;
665       case X86::VPCMPQZrmi:      NewOpc = X86::VPCMPEQQZrm;      break;
666       case X86::VPCMPQZrmib:     NewOpc = X86::VPCMPEQQZrmb;     break;
667       case X86::VPCMPQZrmibk:    NewOpc = X86::VPCMPEQQZrmbk;    break;
668       case X86::VPCMPQZrmik:     NewOpc = X86::VPCMPEQQZrmk;     break;
669       case X86::VPCMPQZrri:      NewOpc = X86::VPCMPEQQZrr;      break;
670       case X86::VPCMPQZrrik:     NewOpc = X86::VPCMPEQQZrrk;     break;
671       case X86::VPCMPWZ128rmi:   NewOpc = X86::VPCMPEQWZ128rm;   break;
672       case X86::VPCMPWZ128rmik:  NewOpc = X86::VPCMPEQWZ128rmk;  break;
673       case X86::VPCMPWZ128rri:   NewOpc = X86::VPCMPEQWZ128rr;   break;
674       case X86::VPCMPWZ128rrik:  NewOpc = X86::VPCMPEQWZ128rrk;  break;
675       case X86::VPCMPWZ256rmi:   NewOpc = X86::VPCMPEQWZ256rm;   break;
676       case X86::VPCMPWZ256rmik:  NewOpc = X86::VPCMPEQWZ256rmk;  break;
677       case X86::VPCMPWZ256rri:   NewOpc = X86::VPCMPEQWZ256rr;   break;
678       case X86::VPCMPWZ256rrik:  NewOpc = X86::VPCMPEQWZ256rrk;  break;
679       case X86::VPCMPWZrmi:      NewOpc = X86::VPCMPEQWZrm;      break;
680       case X86::VPCMPWZrmik:     NewOpc = X86::VPCMPEQWZrmk;     break;
681       case X86::VPCMPWZrri:      NewOpc = X86::VPCMPEQWZrr;      break;
682       case X86::VPCMPWZrrik:     NewOpc = X86::VPCMPEQWZrrk;     break;
683       }
684 
685       OutMI.setOpcode(NewOpc);
686       OutMI.erase(&OutMI.getOperand(OutMI.getNumOperands() - 1));
687       break;
688     }
689 
690     // Turn immediate 6 into the VPCMPGT instruction.
691     if (OutMI.getOperand(OutMI.getNumOperands() - 1).getImm() == 6) {
692       unsigned NewOpc;
693       switch (OutMI.getOpcode()) {
694       default: llvm_unreachable("Invalid opcode");
695       case X86::VPCMPBZ128rmi:   NewOpc = X86::VPCMPGTBZ128rm;   break;
696       case X86::VPCMPBZ128rmik:  NewOpc = X86::VPCMPGTBZ128rmk;  break;
697       case X86::VPCMPBZ128rri:   NewOpc = X86::VPCMPGTBZ128rr;   break;
698       case X86::VPCMPBZ128rrik:  NewOpc = X86::VPCMPGTBZ128rrk;  break;
699       case X86::VPCMPBZ256rmi:   NewOpc = X86::VPCMPGTBZ256rm;   break;
700       case X86::VPCMPBZ256rmik:  NewOpc = X86::VPCMPGTBZ256rmk;  break;
701       case X86::VPCMPBZ256rri:   NewOpc = X86::VPCMPGTBZ256rr;   break;
702       case X86::VPCMPBZ256rrik:  NewOpc = X86::VPCMPGTBZ256rrk;  break;
703       case X86::VPCMPBZrmi:      NewOpc = X86::VPCMPGTBZrm;      break;
704       case X86::VPCMPBZrmik:     NewOpc = X86::VPCMPGTBZrmk;     break;
705       case X86::VPCMPBZrri:      NewOpc = X86::VPCMPGTBZrr;      break;
706       case X86::VPCMPBZrrik:     NewOpc = X86::VPCMPGTBZrrk;     break;
707       case X86::VPCMPDZ128rmi:   NewOpc = X86::VPCMPGTDZ128rm;   break;
708       case X86::VPCMPDZ128rmib:  NewOpc = X86::VPCMPGTDZ128rmb;  break;
709       case X86::VPCMPDZ128rmibk: NewOpc = X86::VPCMPGTDZ128rmbk; break;
710       case X86::VPCMPDZ128rmik:  NewOpc = X86::VPCMPGTDZ128rmk;  break;
711       case X86::VPCMPDZ128rri:   NewOpc = X86::VPCMPGTDZ128rr;   break;
712       case X86::VPCMPDZ128rrik:  NewOpc = X86::VPCMPGTDZ128rrk;  break;
713       case X86::VPCMPDZ256rmi:   NewOpc = X86::VPCMPGTDZ256rm;   break;
714       case X86::VPCMPDZ256rmib:  NewOpc = X86::VPCMPGTDZ256rmb;  break;
715       case X86::VPCMPDZ256rmibk: NewOpc = X86::VPCMPGTDZ256rmbk; break;
716       case X86::VPCMPDZ256rmik:  NewOpc = X86::VPCMPGTDZ256rmk;  break;
717       case X86::VPCMPDZ256rri:   NewOpc = X86::VPCMPGTDZ256rr;   break;
718       case X86::VPCMPDZ256rrik:  NewOpc = X86::VPCMPGTDZ256rrk;  break;
719       case X86::VPCMPDZrmi:      NewOpc = X86::VPCMPGTDZrm;      break;
720       case X86::VPCMPDZrmib:     NewOpc = X86::VPCMPGTDZrmb;     break;
721       case X86::VPCMPDZrmibk:    NewOpc = X86::VPCMPGTDZrmbk;    break;
722       case X86::VPCMPDZrmik:     NewOpc = X86::VPCMPGTDZrmk;     break;
723       case X86::VPCMPDZrri:      NewOpc = X86::VPCMPGTDZrr;      break;
724       case X86::VPCMPDZrrik:     NewOpc = X86::VPCMPGTDZrrk;     break;
725       case X86::VPCMPQZ128rmi:   NewOpc = X86::VPCMPGTQZ128rm;   break;
726       case X86::VPCMPQZ128rmib:  NewOpc = X86::VPCMPGTQZ128rmb;  break;
727       case X86::VPCMPQZ128rmibk: NewOpc = X86::VPCMPGTQZ128rmbk; break;
728       case X86::VPCMPQZ128rmik:  NewOpc = X86::VPCMPGTQZ128rmk;  break;
729       case X86::VPCMPQZ128rri:   NewOpc = X86::VPCMPGTQZ128rr;   break;
730       case X86::VPCMPQZ128rrik:  NewOpc = X86::VPCMPGTQZ128rrk;  break;
731       case X86::VPCMPQZ256rmi:   NewOpc = X86::VPCMPGTQZ256rm;   break;
732       case X86::VPCMPQZ256rmib:  NewOpc = X86::VPCMPGTQZ256rmb;  break;
733       case X86::VPCMPQZ256rmibk: NewOpc = X86::VPCMPGTQZ256rmbk; break;
734       case X86::VPCMPQZ256rmik:  NewOpc = X86::VPCMPGTQZ256rmk;  break;
735       case X86::VPCMPQZ256rri:   NewOpc = X86::VPCMPGTQZ256rr;   break;
736       case X86::VPCMPQZ256rrik:  NewOpc = X86::VPCMPGTQZ256rrk;  break;
737       case X86::VPCMPQZrmi:      NewOpc = X86::VPCMPGTQZrm;      break;
738       case X86::VPCMPQZrmib:     NewOpc = X86::VPCMPGTQZrmb;     break;
739       case X86::VPCMPQZrmibk:    NewOpc = X86::VPCMPGTQZrmbk;    break;
740       case X86::VPCMPQZrmik:     NewOpc = X86::VPCMPGTQZrmk;     break;
741       case X86::VPCMPQZrri:      NewOpc = X86::VPCMPGTQZrr;      break;
742       case X86::VPCMPQZrrik:     NewOpc = X86::VPCMPGTQZrrk;     break;
743       case X86::VPCMPWZ128rmi:   NewOpc = X86::VPCMPGTWZ128rm;   break;
744       case X86::VPCMPWZ128rmik:  NewOpc = X86::VPCMPGTWZ128rmk;  break;
745       case X86::VPCMPWZ128rri:   NewOpc = X86::VPCMPGTWZ128rr;   break;
746       case X86::VPCMPWZ128rrik:  NewOpc = X86::VPCMPGTWZ128rrk;  break;
747       case X86::VPCMPWZ256rmi:   NewOpc = X86::VPCMPGTWZ256rm;   break;
748       case X86::VPCMPWZ256rmik:  NewOpc = X86::VPCMPGTWZ256rmk;  break;
749       case X86::VPCMPWZ256rri:   NewOpc = X86::VPCMPGTWZ256rr;   break;
750       case X86::VPCMPWZ256rrik:  NewOpc = X86::VPCMPGTWZ256rrk;  break;
751       case X86::VPCMPWZrmi:      NewOpc = X86::VPCMPGTWZrm;      break;
752       case X86::VPCMPWZrmik:     NewOpc = X86::VPCMPGTWZrmk;     break;
753       case X86::VPCMPWZrri:      NewOpc = X86::VPCMPGTWZrr;      break;
754       case X86::VPCMPWZrrik:     NewOpc = X86::VPCMPGTWZrrk;     break;
755       }
756 
757       OutMI.setOpcode(NewOpc);
758       OutMI.erase(&OutMI.getOperand(OutMI.getNumOperands() - 1));
759       break;
760     }
761 
762     break;
763   }
764 
765   // CALL64r, CALL64pcrel32 - These instructions used to have
766   // register inputs modeled as normal uses instead of implicit uses.  As such,
767   // they we used to truncate off all but the first operand (the callee). This
768   // issue seems to have been fixed at some point. This assert verifies that.
769   case X86::CALL64r:
770   case X86::CALL64pcrel32:
771     assert(OutMI.getNumOperands() == 1 && "Unexpected number of operands!");
772     break;
773 
774   case X86::EH_RETURN:
775   case X86::EH_RETURN64: {
776     OutMI = MCInst();
777     OutMI.setOpcode(getRetOpcode(AsmPrinter.getSubtarget()));
778     break;
779   }
780 
781   case X86::CLEANUPRET: {
782     // Replace CLEANUPRET with the appropriate RET.
783     OutMI = MCInst();
784     OutMI.setOpcode(getRetOpcode(AsmPrinter.getSubtarget()));
785     break;
786   }
787 
788   case X86::CATCHRET: {
789     // Replace CATCHRET with the appropriate RET.
790     const X86Subtarget &Subtarget = AsmPrinter.getSubtarget();
791     unsigned ReturnReg = Subtarget.is64Bit() ? X86::RAX : X86::EAX;
792     OutMI = MCInst();
793     OutMI.setOpcode(getRetOpcode(Subtarget));
794     OutMI.addOperand(MCOperand::createReg(ReturnReg));
795     break;
796   }
797 
798   // TAILJMPd, TAILJMPd64, TailJMPd_cc - Lower to the correct jump
799   // instruction.
800   case X86::TAILJMPr:
801   case X86::TAILJMPr64:
802   case X86::TAILJMPr64_REX:
803   case X86::TAILJMPd:
804   case X86::TAILJMPd64:
805     assert(OutMI.getNumOperands() == 1 && "Unexpected number of operands!");
806     OutMI.setOpcode(convertTailJumpOpcode(OutMI.getOpcode()));
807     break;
808 
809   case X86::TAILJMPd_CC:
810   case X86::TAILJMPd64_CC:
811     assert(OutMI.getNumOperands() == 2 && "Unexpected number of operands!");
812     OutMI.setOpcode(convertTailJumpOpcode(OutMI.getOpcode()));
813     break;
814 
815   case X86::TAILJMPm:
816   case X86::TAILJMPm64:
817   case X86::TAILJMPm64_REX:
818     assert(OutMI.getNumOperands() == X86::AddrNumOperands &&
819            "Unexpected number of operands!");
820     OutMI.setOpcode(convertTailJumpOpcode(OutMI.getOpcode()));
821     break;
822 
823   case X86::DEC16r:
824   case X86::DEC32r:
825   case X86::INC16r:
826   case X86::INC32r:
827     // If we aren't in 64-bit mode we can use the 1-byte inc/dec instructions.
828     if (!AsmPrinter.getSubtarget().is64Bit()) {
829       unsigned Opcode;
830       switch (OutMI.getOpcode()) {
831       default: llvm_unreachable("Invalid opcode");
832       case X86::DEC16r: Opcode = X86::DEC16r_alt; break;
833       case X86::DEC32r: Opcode = X86::DEC32r_alt; break;
834       case X86::INC16r: Opcode = X86::INC16r_alt; break;
835       case X86::INC32r: Opcode = X86::INC32r_alt; break;
836       }
837       OutMI.setOpcode(Opcode);
838     }
839     break;
840 
841   // We don't currently select the correct instruction form for instructions
842   // which have a short %eax, etc. form. Handle this by custom lowering, for
843   // now.
844   //
845   // Note, we are currently not handling the following instructions:
846   // MOV64ao8, MOV64o8a
847   // XCHG16ar, XCHG32ar, XCHG64ar
848   case X86::MOV8mr_NOREX:
849   case X86::MOV8mr:
850   case X86::MOV8rm_NOREX:
851   case X86::MOV8rm:
852   case X86::MOV16mr:
853   case X86::MOV16rm:
854   case X86::MOV32mr:
855   case X86::MOV32rm: {
856     unsigned NewOpc;
857     switch (OutMI.getOpcode()) {
858     default: llvm_unreachable("Invalid opcode");
859     case X86::MOV8mr_NOREX:
860     case X86::MOV8mr:  NewOpc = X86::MOV8o32a; break;
861     case X86::MOV8rm_NOREX:
862     case X86::MOV8rm:  NewOpc = X86::MOV8ao32; break;
863     case X86::MOV16mr: NewOpc = X86::MOV16o32a; break;
864     case X86::MOV16rm: NewOpc = X86::MOV16ao32; break;
865     case X86::MOV32mr: NewOpc = X86::MOV32o32a; break;
866     case X86::MOV32rm: NewOpc = X86::MOV32ao32; break;
867     }
868     SimplifyShortMoveForm(AsmPrinter, OutMI, NewOpc);
869     break;
870   }
871 
872   case X86::ADC8ri: case X86::ADC16ri: case X86::ADC32ri: case X86::ADC64ri32:
873   case X86::ADD8ri: case X86::ADD16ri: case X86::ADD32ri: case X86::ADD64ri32:
874   case X86::AND8ri: case X86::AND16ri: case X86::AND32ri: case X86::AND64ri32:
875   case X86::CMP8ri: case X86::CMP16ri: case X86::CMP32ri: case X86::CMP64ri32:
876   case X86::OR8ri:  case X86::OR16ri:  case X86::OR32ri:  case X86::OR64ri32:
877   case X86::SBB8ri: case X86::SBB16ri: case X86::SBB32ri: case X86::SBB64ri32:
878   case X86::SUB8ri: case X86::SUB16ri: case X86::SUB32ri: case X86::SUB64ri32:
879   case X86::TEST8ri:case X86::TEST16ri:case X86::TEST32ri:case X86::TEST64ri32:
880   case X86::XOR8ri: case X86::XOR16ri: case X86::XOR32ri: case X86::XOR64ri32: {
881     unsigned NewOpc;
882     switch (OutMI.getOpcode()) {
883     default: llvm_unreachable("Invalid opcode");
884     case X86::ADC8ri:     NewOpc = X86::ADC8i8;    break;
885     case X86::ADC16ri:    NewOpc = X86::ADC16i16;  break;
886     case X86::ADC32ri:    NewOpc = X86::ADC32i32;  break;
887     case X86::ADC64ri32:  NewOpc = X86::ADC64i32;  break;
888     case X86::ADD8ri:     NewOpc = X86::ADD8i8;    break;
889     case X86::ADD16ri:    NewOpc = X86::ADD16i16;  break;
890     case X86::ADD32ri:    NewOpc = X86::ADD32i32;  break;
891     case X86::ADD64ri32:  NewOpc = X86::ADD64i32;  break;
892     case X86::AND8ri:     NewOpc = X86::AND8i8;    break;
893     case X86::AND16ri:    NewOpc = X86::AND16i16;  break;
894     case X86::AND32ri:    NewOpc = X86::AND32i32;  break;
895     case X86::AND64ri32:  NewOpc = X86::AND64i32;  break;
896     case X86::CMP8ri:     NewOpc = X86::CMP8i8;    break;
897     case X86::CMP16ri:    NewOpc = X86::CMP16i16;  break;
898     case X86::CMP32ri:    NewOpc = X86::CMP32i32;  break;
899     case X86::CMP64ri32:  NewOpc = X86::CMP64i32;  break;
900     case X86::OR8ri:      NewOpc = X86::OR8i8;     break;
901     case X86::OR16ri:     NewOpc = X86::OR16i16;   break;
902     case X86::OR32ri:     NewOpc = X86::OR32i32;   break;
903     case X86::OR64ri32:   NewOpc = X86::OR64i32;   break;
904     case X86::SBB8ri:     NewOpc = X86::SBB8i8;    break;
905     case X86::SBB16ri:    NewOpc = X86::SBB16i16;  break;
906     case X86::SBB32ri:    NewOpc = X86::SBB32i32;  break;
907     case X86::SBB64ri32:  NewOpc = X86::SBB64i32;  break;
908     case X86::SUB8ri:     NewOpc = X86::SUB8i8;    break;
909     case X86::SUB16ri:    NewOpc = X86::SUB16i16;  break;
910     case X86::SUB32ri:    NewOpc = X86::SUB32i32;  break;
911     case X86::SUB64ri32:  NewOpc = X86::SUB64i32;  break;
912     case X86::TEST8ri:    NewOpc = X86::TEST8i8;   break;
913     case X86::TEST16ri:   NewOpc = X86::TEST16i16; break;
914     case X86::TEST32ri:   NewOpc = X86::TEST32i32; break;
915     case X86::TEST64ri32: NewOpc = X86::TEST64i32; break;
916     case X86::XOR8ri:     NewOpc = X86::XOR8i8;    break;
917     case X86::XOR16ri:    NewOpc = X86::XOR16i16;  break;
918     case X86::XOR32ri:    NewOpc = X86::XOR32i32;  break;
919     case X86::XOR64ri32:  NewOpc = X86::XOR64i32;  break;
920     }
921     SimplifyShortImmForm(OutMI, NewOpc);
922     break;
923   }
924 
925   // Try to shrink some forms of movsx.
926   case X86::MOVSX16rr8:
927   case X86::MOVSX32rr16:
928   case X86::MOVSX64rr32:
929     SimplifyMOVSX(OutMI);
930     break;
931 
932   case X86::VCMPPDrri:
933   case X86::VCMPPDYrri:
934   case X86::VCMPPSrri:
935   case X86::VCMPPSYrri:
936   case X86::VCMPSDrr:
937   case X86::VCMPSSrr: {
938     // Swap the operands if it will enable a 2 byte VEX encoding.
939     // FIXME: Change the immediate to improve opportunities?
940     if (!X86II::isX86_64ExtendedReg(OutMI.getOperand(1).getReg()) &&
941         X86II::isX86_64ExtendedReg(OutMI.getOperand(2).getReg())) {
942       unsigned Imm = MI->getOperand(3).getImm() & 0x7;
943       switch (Imm) {
944       default: break;
945       case 0x00: // EQUAL
946       case 0x03: // UNORDERED
947       case 0x04: // NOT EQUAL
948       case 0x07: // ORDERED
949         std::swap(OutMI.getOperand(1), OutMI.getOperand(2));
950         break;
951       }
952     }
953     break;
954   }
955 
956   case X86::VMOVHLPSrr:
957   case X86::VUNPCKHPDrr:
958     // These are not truly commutable so hide them from the default case.
959     break;
960 
961   default: {
962     // If the instruction is a commutable arithmetic instruction we might be
963     // able to commute the operands to get a 2 byte VEX prefix.
964     uint64_t TSFlags = MI->getDesc().TSFlags;
965     if (MI->getDesc().isCommutable() &&
966         (TSFlags & X86II::EncodingMask) == X86II::VEX &&
967         (TSFlags & X86II::OpMapMask) == X86II::TB &&
968         (TSFlags & X86II::FormMask) == X86II::MRMSrcReg &&
969         !(TSFlags & X86II::VEX_W) && (TSFlags & X86II::VEX_4V) &&
970         OutMI.getNumOperands() == 3) {
971       if (!X86II::isX86_64ExtendedReg(OutMI.getOperand(1).getReg()) &&
972           X86II::isX86_64ExtendedReg(OutMI.getOperand(2).getReg()))
973         std::swap(OutMI.getOperand(1), OutMI.getOperand(2));
974     }
975     break;
976   }
977   }
978 }
979 
980 void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
981                                  const MachineInstr &MI) {
982   NoAutoPaddingScope NoPadScope(*OutStreamer);
983   bool Is64Bits = MI.getOpcode() != X86::TLS_addr32 &&
984                   MI.getOpcode() != X86::TLS_base_addr32;
985   bool Is64BitsLP64 = MI.getOpcode() == X86::TLS_addr64 ||
986                       MI.getOpcode() == X86::TLS_base_addr64;
987   MCContext &Ctx = OutStreamer->getContext();
988 
989   MCSymbolRefExpr::VariantKind SRVK;
990   switch (MI.getOpcode()) {
991   case X86::TLS_addr32:
992   case X86::TLS_addr64:
993   case X86::TLS_addrX32:
994     SRVK = MCSymbolRefExpr::VK_TLSGD;
995     break;
996   case X86::TLS_base_addr32:
997     SRVK = MCSymbolRefExpr::VK_TLSLDM;
998     break;
999   case X86::TLS_base_addr64:
1000   case X86::TLS_base_addrX32:
1001     SRVK = MCSymbolRefExpr::VK_TLSLD;
1002     break;
1003   default:
1004     llvm_unreachable("unexpected opcode");
1005   }
1006 
1007   const MCSymbolRefExpr *Sym = MCSymbolRefExpr::create(
1008       MCInstLowering.GetSymbolFromOperand(MI.getOperand(3)), SRVK, Ctx);
1009 
1010   // As of binutils 2.32, ld has a bogus TLS relaxation error when the GD/LD
1011   // code sequence using R_X86_64_GOTPCREL (instead of R_X86_64_GOTPCRELX) is
1012   // attempted to be relaxed to IE/LE (binutils PR24784). Work around the bug by
1013   // only using GOT when GOTPCRELX is enabled.
1014   // TODO Delete the workaround when GOTPCRELX becomes commonplace.
1015   bool UseGot = MMI->getModule()->getRtLibUseGOT() &&
1016                 Ctx.getAsmInfo()->canRelaxRelocations();
1017 
1018   if (Is64Bits) {
1019     bool NeedsPadding = SRVK == MCSymbolRefExpr::VK_TLSGD;
1020     if (NeedsPadding && Is64BitsLP64)
1021       EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX));
1022     EmitAndCountInstruction(MCInstBuilder(X86::LEA64r)
1023                                 .addReg(X86::RDI)
1024                                 .addReg(X86::RIP)
1025                                 .addImm(1)
1026                                 .addReg(0)
1027                                 .addExpr(Sym)
1028                                 .addReg(0));
1029     const MCSymbol *TlsGetAddr = Ctx.getOrCreateSymbol("__tls_get_addr");
1030     if (NeedsPadding) {
1031       if (!UseGot)
1032         EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX));
1033       EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX));
1034       EmitAndCountInstruction(MCInstBuilder(X86::REX64_PREFIX));
1035     }
1036     if (UseGot) {
1037       const MCExpr *Expr = MCSymbolRefExpr::create(
1038           TlsGetAddr, MCSymbolRefExpr::VK_GOTPCREL, Ctx);
1039       EmitAndCountInstruction(MCInstBuilder(X86::CALL64m)
1040                                   .addReg(X86::RIP)
1041                                   .addImm(1)
1042                                   .addReg(0)
1043                                   .addExpr(Expr)
1044                                   .addReg(0));
1045     } else {
1046       EmitAndCountInstruction(
1047           MCInstBuilder(X86::CALL64pcrel32)
1048               .addExpr(MCSymbolRefExpr::create(TlsGetAddr,
1049                                                MCSymbolRefExpr::VK_PLT, Ctx)));
1050     }
1051   } else {
1052     if (SRVK == MCSymbolRefExpr::VK_TLSGD && !UseGot) {
1053       EmitAndCountInstruction(MCInstBuilder(X86::LEA32r)
1054                                   .addReg(X86::EAX)
1055                                   .addReg(0)
1056                                   .addImm(1)
1057                                   .addReg(X86::EBX)
1058                                   .addExpr(Sym)
1059                                   .addReg(0));
1060     } else {
1061       EmitAndCountInstruction(MCInstBuilder(X86::LEA32r)
1062                                   .addReg(X86::EAX)
1063                                   .addReg(X86::EBX)
1064                                   .addImm(1)
1065                                   .addReg(0)
1066                                   .addExpr(Sym)
1067                                   .addReg(0));
1068     }
1069 
1070     const MCSymbol *TlsGetAddr = Ctx.getOrCreateSymbol("___tls_get_addr");
1071     if (UseGot) {
1072       const MCExpr *Expr =
1073           MCSymbolRefExpr::create(TlsGetAddr, MCSymbolRefExpr::VK_GOT, Ctx);
1074       EmitAndCountInstruction(MCInstBuilder(X86::CALL32m)
1075                                   .addReg(X86::EBX)
1076                                   .addImm(1)
1077                                   .addReg(0)
1078                                   .addExpr(Expr)
1079                                   .addReg(0));
1080     } else {
1081       EmitAndCountInstruction(
1082           MCInstBuilder(X86::CALLpcrel32)
1083               .addExpr(MCSymbolRefExpr::create(TlsGetAddr,
1084                                                MCSymbolRefExpr::VK_PLT, Ctx)));
1085     }
1086   }
1087 }
1088 
1089 /// Emit the largest nop instruction smaller than or equal to \p NumBytes
1090 /// bytes.  Return the size of nop emitted.
1091 static unsigned emitNop(MCStreamer &OS, unsigned NumBytes,
1092                         const X86Subtarget *Subtarget) {
1093   // Determine the longest nop which can be efficiently decoded for the given
1094   // target cpu.  15-bytes is the longest single NOP instruction, but some
1095   // platforms can't decode the longest forms efficiently.
1096   unsigned MaxNopLength = 1;
1097   if (Subtarget->is64Bit()) {
1098     // FIXME: We can use NOOPL on 32-bit targets with FeatureNOPL, but the
1099     // IndexReg/BaseReg below need to be updated.
1100     if (Subtarget->hasFeature(X86::TuningFast7ByteNOP))
1101       MaxNopLength = 7;
1102     else if (Subtarget->hasFeature(X86::TuningFast15ByteNOP))
1103       MaxNopLength = 15;
1104     else if (Subtarget->hasFeature(X86::TuningFast11ByteNOP))
1105       MaxNopLength = 11;
1106     else
1107       MaxNopLength = 10;
1108   } if (Subtarget->is32Bit())
1109     MaxNopLength = 2;
1110 
1111   // Cap a single nop emission at the profitable value for the target
1112   NumBytes = std::min(NumBytes, MaxNopLength);
1113 
1114   unsigned NopSize;
1115   unsigned Opc, BaseReg, ScaleVal, IndexReg, Displacement, SegmentReg;
1116   IndexReg = Displacement = SegmentReg = 0;
1117   BaseReg = X86::RAX;
1118   ScaleVal = 1;
1119   switch (NumBytes) {
1120   case 0:
1121     llvm_unreachable("Zero nops?");
1122     break;
1123   case 1:
1124     NopSize = 1;
1125     Opc = X86::NOOP;
1126     break;
1127   case 2:
1128     NopSize = 2;
1129     Opc = X86::XCHG16ar;
1130     break;
1131   case 3:
1132     NopSize = 3;
1133     Opc = X86::NOOPL;
1134     break;
1135   case 4:
1136     NopSize = 4;
1137     Opc = X86::NOOPL;
1138     Displacement = 8;
1139     break;
1140   case 5:
1141     NopSize = 5;
1142     Opc = X86::NOOPL;
1143     Displacement = 8;
1144     IndexReg = X86::RAX;
1145     break;
1146   case 6:
1147     NopSize = 6;
1148     Opc = X86::NOOPW;
1149     Displacement = 8;
1150     IndexReg = X86::RAX;
1151     break;
1152   case 7:
1153     NopSize = 7;
1154     Opc = X86::NOOPL;
1155     Displacement = 512;
1156     break;
1157   case 8:
1158     NopSize = 8;
1159     Opc = X86::NOOPL;
1160     Displacement = 512;
1161     IndexReg = X86::RAX;
1162     break;
1163   case 9:
1164     NopSize = 9;
1165     Opc = X86::NOOPW;
1166     Displacement = 512;
1167     IndexReg = X86::RAX;
1168     break;
1169   default:
1170     NopSize = 10;
1171     Opc = X86::NOOPW;
1172     Displacement = 512;
1173     IndexReg = X86::RAX;
1174     SegmentReg = X86::CS;
1175     break;
1176   }
1177 
1178   unsigned NumPrefixes = std::min(NumBytes - NopSize, 5U);
1179   NopSize += NumPrefixes;
1180   for (unsigned i = 0; i != NumPrefixes; ++i)
1181     OS.emitBytes("\x66");
1182 
1183   switch (Opc) {
1184   default: llvm_unreachable("Unexpected opcode");
1185   case X86::NOOP:
1186     OS.emitInstruction(MCInstBuilder(Opc), *Subtarget);
1187     break;
1188   case X86::XCHG16ar:
1189     OS.emitInstruction(MCInstBuilder(Opc).addReg(X86::AX).addReg(X86::AX),
1190                        *Subtarget);
1191     break;
1192   case X86::NOOPL:
1193   case X86::NOOPW:
1194     OS.emitInstruction(MCInstBuilder(Opc)
1195                            .addReg(BaseReg)
1196                            .addImm(ScaleVal)
1197                            .addReg(IndexReg)
1198                            .addImm(Displacement)
1199                            .addReg(SegmentReg),
1200                        *Subtarget);
1201     break;
1202   }
1203   assert(NopSize <= NumBytes && "We overemitted?");
1204   return NopSize;
1205 }
1206 
1207 /// Emit the optimal amount of multi-byte nops on X86.
1208 static void emitX86Nops(MCStreamer &OS, unsigned NumBytes,
1209                         const X86Subtarget *Subtarget) {
1210   unsigned NopsToEmit = NumBytes;
1211   (void)NopsToEmit;
1212   while (NumBytes) {
1213     NumBytes -= emitNop(OS, NumBytes, Subtarget);
1214     assert(NopsToEmit >= NumBytes && "Emitted more than I asked for!");
1215   }
1216 }
1217 
1218 void X86AsmPrinter::LowerSTATEPOINT(const MachineInstr &MI,
1219                                     X86MCInstLower &MCIL) {
1220   assert(Subtarget->is64Bit() && "Statepoint currently only supports X86-64");
1221 
1222   NoAutoPaddingScope NoPadScope(*OutStreamer);
1223 
1224   StatepointOpers SOpers(&MI);
1225   if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
1226     emitX86Nops(*OutStreamer, PatchBytes, Subtarget);
1227   } else {
1228     // Lower call target and choose correct opcode
1229     const MachineOperand &CallTarget = SOpers.getCallTarget();
1230     MCOperand CallTargetMCOp;
1231     unsigned CallOpcode;
1232     switch (CallTarget.getType()) {
1233     case MachineOperand::MO_GlobalAddress:
1234     case MachineOperand::MO_ExternalSymbol:
1235       CallTargetMCOp = MCIL.LowerSymbolOperand(
1236           CallTarget, MCIL.GetSymbolFromOperand(CallTarget));
1237       CallOpcode = X86::CALL64pcrel32;
1238       // Currently, we only support relative addressing with statepoints.
1239       // Otherwise, we'll need a scratch register to hold the target
1240       // address.  You'll fail asserts during load & relocation if this
1241       // symbol is to far away. (TODO: support non-relative addressing)
1242       break;
1243     case MachineOperand::MO_Immediate:
1244       CallTargetMCOp = MCOperand::createImm(CallTarget.getImm());
1245       CallOpcode = X86::CALL64pcrel32;
1246       // Currently, we only support relative addressing with statepoints.
1247       // Otherwise, we'll need a scratch register to hold the target
1248       // immediate.  You'll fail asserts during load & relocation if this
1249       // address is to far away. (TODO: support non-relative addressing)
1250       break;
1251     case MachineOperand::MO_Register:
1252       // FIXME: Add retpoline support and remove this.
1253       if (Subtarget->useIndirectThunkCalls())
1254         report_fatal_error("Lowering register statepoints with thunks not "
1255                            "yet implemented.");
1256       CallTargetMCOp = MCOperand::createReg(CallTarget.getReg());
1257       CallOpcode = X86::CALL64r;
1258       break;
1259     default:
1260       llvm_unreachable("Unsupported operand type in statepoint call target");
1261       break;
1262     }
1263 
1264     // Emit call
1265     MCInst CallInst;
1266     CallInst.setOpcode(CallOpcode);
1267     CallInst.addOperand(CallTargetMCOp);
1268     OutStreamer->emitInstruction(CallInst, getSubtargetInfo());
1269   }
1270 
1271   // Record our statepoint node in the same section used by STACKMAP
1272   // and PATCHPOINT
1273   auto &Ctx = OutStreamer->getContext();
1274   MCSymbol *MILabel = Ctx.createTempSymbol();
1275   OutStreamer->emitLabel(MILabel);
1276   SM.recordStatepoint(*MILabel, MI);
1277 }
1278 
1279 void X86AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI,
1280                                      X86MCInstLower &MCIL) {
1281   // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
1282   //                  <opcode>, <operands>
1283 
1284   NoAutoPaddingScope NoPadScope(*OutStreamer);
1285 
1286   Register DefRegister = FaultingMI.getOperand(0).getReg();
1287   FaultMaps::FaultKind FK =
1288       static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(1).getImm());
1289   MCSymbol *HandlerLabel = FaultingMI.getOperand(2).getMBB()->getSymbol();
1290   unsigned Opcode = FaultingMI.getOperand(3).getImm();
1291   unsigned OperandsBeginIdx = 4;
1292 
1293   auto &Ctx = OutStreamer->getContext();
1294   MCSymbol *FaultingLabel = Ctx.createTempSymbol();
1295   OutStreamer->emitLabel(FaultingLabel);
1296 
1297   assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
1298   FM.recordFaultingOp(FK, FaultingLabel, HandlerLabel);
1299 
1300   MCInst MI;
1301   MI.setOpcode(Opcode);
1302 
1303   if (DefRegister != X86::NoRegister)
1304     MI.addOperand(MCOperand::createReg(DefRegister));
1305 
1306   for (auto I = FaultingMI.operands_begin() + OperandsBeginIdx,
1307             E = FaultingMI.operands_end();
1308        I != E; ++I)
1309     if (auto MaybeOperand = MCIL.LowerMachineOperand(&FaultingMI, *I))
1310       MI.addOperand(MaybeOperand.getValue());
1311 
1312   OutStreamer->AddComment("on-fault: " + HandlerLabel->getName());
1313   OutStreamer->emitInstruction(MI, getSubtargetInfo());
1314 }
1315 
1316 void X86AsmPrinter::LowerFENTRY_CALL(const MachineInstr &MI,
1317                                      X86MCInstLower &MCIL) {
1318   bool Is64Bits = Subtarget->is64Bit();
1319   MCContext &Ctx = OutStreamer->getContext();
1320   MCSymbol *fentry = Ctx.getOrCreateSymbol("__fentry__");
1321   const MCSymbolRefExpr *Op =
1322       MCSymbolRefExpr::create(fentry, MCSymbolRefExpr::VK_None, Ctx);
1323 
1324   EmitAndCountInstruction(
1325       MCInstBuilder(Is64Bits ? X86::CALL64pcrel32 : X86::CALLpcrel32)
1326           .addExpr(Op));
1327 }
1328 
1329 void X86AsmPrinter::LowerASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
1330   // FIXME: Make this work on non-ELF.
1331   if (!TM.getTargetTriple().isOSBinFormatELF()) {
1332     report_fatal_error("llvm.asan.check.memaccess only supported on ELF");
1333     return;
1334   }
1335 
1336   unsigned Reg = MI.getOperand(0).getReg().id();
1337   ASanAccessInfo AccessInfo(MI.getOperand(1).getImm());
1338 
1339   MCSymbol *&Sym =
1340       AsanMemaccessSymbols[AsanMemaccessTuple(Reg, AccessInfo.Packed)];
1341   if (!Sym) {
1342     std::string Name = AccessInfo.IsWrite ? "store" : "load";
1343     std::string SymName = "__asan_check_" + Name +
1344                           utostr(1ULL << AccessInfo.AccessSizeIndex) + "_rn" +
1345                           utostr(Reg);
1346     Sym = OutContext.getOrCreateSymbol(SymName);
1347   }
1348 
1349   EmitAndCountInstruction(
1350       MCInstBuilder(X86::CALL64pcrel32)
1351           .addExpr(MCSymbolRefExpr::create(Sym, OutContext)));
1352 }
1353 
1354 void X86AsmPrinter::emitAsanMemaccessPartial(Module &M, unsigned Reg,
1355                                              const ASanAccessInfo &AccessInfo,
1356                                              MCSubtargetInfo &STI) {
1357   assert(AccessInfo.AccessSizeIndex == 0 || AccessInfo.AccessSizeIndex == 1 ||
1358          AccessInfo.AccessSizeIndex == 2);
1359   assert(Reg != X86::R8);
1360 
1361   uint64_t ShadowBase;
1362   int MappingScale;
1363   bool OrShadowOffset;
1364   getAddressSanitizerParams(
1365       Triple(M.getTargetTriple()), M.getDataLayout().getPointerSizeInBits(),
1366       AccessInfo.CompileKernel, &ShadowBase, &MappingScale, &OrShadowOffset);
1367 
1368   OutStreamer->emitInstruction(
1369       MCInstBuilder(X86::MOV64rr).addReg(X86::R8).addReg(X86::NoRegister + Reg),
1370       STI);
1371   OutStreamer->emitInstruction(MCInstBuilder(X86::SHR64ri)
1372                                    .addReg(X86::R8)
1373                                    .addReg(X86::R8)
1374                                    .addImm(MappingScale),
1375                                STI);
1376   if (OrShadowOffset) {
1377     OutStreamer->emitInstruction(MCInstBuilder(X86::OR64ri32)
1378                                      .addReg(X86::R8)
1379                                      .addReg(X86::R8)
1380                                      .addImm(ShadowBase),
1381                                  STI);
1382     OutStreamer->emitInstruction(MCInstBuilder(X86::MOV8rm)
1383                                      .addReg(X86::R8B)
1384                                      .addReg(X86::R8)
1385                                      .addImm(1)
1386                                      .addReg(X86::NoRegister)
1387                                      .addImm(0)
1388                                      .addReg(X86::NoRegister),
1389                                  STI);
1390     OutStreamer->emitInstruction(
1391         MCInstBuilder(X86::TEST8rr).addReg(X86::R8B).addReg(X86::R8B), STI);
1392   } else {
1393     OutStreamer->emitInstruction(MCInstBuilder(X86::MOVSX32rm8)
1394                                      .addReg(X86::R8D)
1395                                      .addReg(X86::R8)
1396                                      .addImm(1)
1397                                      .addReg(X86::NoRegister)
1398                                      .addImm(ShadowBase)
1399                                      .addReg(X86::NoRegister),
1400                                  STI);
1401     OutStreamer->emitInstruction(
1402         MCInstBuilder(X86::TEST32rr).addReg(X86::R8D).addReg(X86::R8D), STI);
1403   }
1404   MCSymbol *AdditionalCheck = OutContext.createTempSymbol();
1405   OutStreamer->emitInstruction(
1406       MCInstBuilder(X86::JCC_1)
1407           .addExpr(MCSymbolRefExpr::create(AdditionalCheck, OutContext))
1408           .addImm(X86::COND_NE),
1409       STI);
1410   MCSymbol *ReturnSym = OutContext.createTempSymbol();
1411   OutStreamer->emitLabel(ReturnSym);
1412   OutStreamer->emitInstruction(MCInstBuilder(getRetOpcode(*Subtarget)), STI);
1413 
1414   // Shadow byte is non-zero so we need to perform additional checks.
1415   OutStreamer->emitLabel(AdditionalCheck);
1416   OutStreamer->emitInstruction(MCInstBuilder(X86::PUSH64r).addReg(X86::RCX),
1417                                STI);
1418   OutStreamer->emitInstruction(MCInstBuilder(X86::MOV64rr)
1419                                    .addReg(X86::RCX)
1420                                    .addReg(X86::NoRegister + Reg),
1421                                STI);
1422   const size_t Granularity = 1ULL << MappingScale;
1423   OutStreamer->emitInstruction(MCInstBuilder(X86::AND32ri8)
1424                                    .addReg(X86::NoRegister)
1425                                    .addReg(X86::ECX)
1426                                    .addImm(Granularity - 1),
1427                                STI);
1428   if (AccessInfo.AccessSizeIndex == 1) {
1429     OutStreamer->emitInstruction(MCInstBuilder(X86::ADD32ri8)
1430                                      .addReg(X86::NoRegister)
1431                                      .addReg(X86::ECX)
1432                                      .addImm(1),
1433                                  STI);
1434   } else if (AccessInfo.AccessSizeIndex == 2) {
1435     OutStreamer->emitInstruction(MCInstBuilder(X86::ADD32ri8)
1436                                      .addReg(X86::NoRegister)
1437                                      .addReg(X86::ECX)
1438                                      .addImm(3),
1439                                  STI);
1440   }
1441 
1442   OutStreamer->emitInstruction(
1443       MCInstBuilder(X86::CMP32rr).addReg(X86::ECX).addReg(X86::R8D).addImm(1),
1444       STI);
1445   OutStreamer->emitInstruction(MCInstBuilder(X86::POP64r).addReg(X86::RCX),
1446                                STI);
1447   OutStreamer->emitInstruction(
1448       MCInstBuilder(X86::JCC_1)
1449           .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext))
1450           .addImm(X86::COND_L),
1451       STI);
1452 
1453   emitAsanReportError(M, Reg, AccessInfo, STI);
1454 }
1455 
1456 void X86AsmPrinter::emitAsanMemaccessFull(Module &M, unsigned Reg,
1457                                           const ASanAccessInfo &AccessInfo,
1458                                           MCSubtargetInfo &STI) {
1459   assert(AccessInfo.AccessSizeIndex == 3 || AccessInfo.AccessSizeIndex == 4);
1460   assert(Reg != X86::R8);
1461 
1462   uint64_t ShadowBase;
1463   int MappingScale;
1464   bool OrShadowOffset;
1465   getAddressSanitizerParams(
1466       Triple(M.getTargetTriple()), M.getDataLayout().getPointerSizeInBits(),
1467       AccessInfo.CompileKernel, &ShadowBase, &MappingScale, &OrShadowOffset);
1468 
1469   OutStreamer->emitInstruction(
1470       MCInstBuilder(X86::MOV64rr).addReg(X86::R8).addReg(X86::NoRegister + Reg),
1471       STI);
1472   OutStreamer->emitInstruction(MCInstBuilder(X86::SHR64ri)
1473                                    .addReg(X86::R8)
1474                                    .addReg(X86::R8)
1475                                    .addImm(MappingScale),
1476                                STI);
1477   if (OrShadowOffset) {
1478     OutStreamer->emitInstruction(MCInstBuilder(X86::OR64ri32)
1479                                      .addReg(X86::R8)
1480                                      .addReg(X86::R8)
1481                                      .addImm(ShadowBase),
1482                                  STI);
1483     auto OpCode = AccessInfo.AccessSizeIndex == 3 ? X86::CMP8mi : X86::CMP16mi8;
1484     OutStreamer->emitInstruction(MCInstBuilder(OpCode)
1485                                      .addReg(X86::R8)
1486                                      .addImm(1)
1487                                      .addReg(X86::NoRegister)
1488                                      .addImm(0)
1489                                      .addReg(X86::NoRegister)
1490                                      .addImm(0),
1491                                  STI);
1492   } else {
1493     auto OpCode = AccessInfo.AccessSizeIndex == 3 ? X86::CMP8mi : X86::CMP16mi8;
1494     OutStreamer->emitInstruction(MCInstBuilder(OpCode)
1495                                      .addReg(X86::R8)
1496                                      .addImm(1)
1497                                      .addReg(X86::NoRegister)
1498                                      .addImm(ShadowBase)
1499                                      .addReg(X86::NoRegister)
1500                                      .addImm(0),
1501                                  STI);
1502   }
1503   MCSymbol *ReportCode = OutContext.createTempSymbol();
1504   OutStreamer->emitInstruction(
1505       MCInstBuilder(X86::JCC_1)
1506           .addExpr(MCSymbolRefExpr::create(ReportCode, OutContext))
1507           .addImm(X86::COND_NE),
1508       STI);
1509   MCSymbol *ReturnSym = OutContext.createTempSymbol();
1510   OutStreamer->emitLabel(ReturnSym);
1511   OutStreamer->emitInstruction(MCInstBuilder(getRetOpcode(*Subtarget)), STI);
1512 
1513   OutStreamer->emitLabel(ReportCode);
1514   emitAsanReportError(M, Reg, AccessInfo, STI);
1515 }
1516 
1517 void X86AsmPrinter::emitAsanReportError(Module &M, unsigned Reg,
1518                                         const ASanAccessInfo &AccessInfo,
1519                                         MCSubtargetInfo &STI) {
1520   std::string Name = AccessInfo.IsWrite ? "store" : "load";
1521   MCSymbol *ReportError = OutContext.getOrCreateSymbol(
1522       "__asan_report_" + Name + utostr(1ULL << AccessInfo.AccessSizeIndex));
1523   OutStreamer->emitInstruction(MCInstBuilder(X86::MOV64rr)
1524                                    .addReg(X86::RDI)
1525                                    .addReg(X86::NoRegister + Reg),
1526                                STI);
1527   OutStreamer->emitInstruction(
1528       MCInstBuilder(X86::JMP_4)
1529           .addExpr(MCSymbolRefExpr::create(ReportError, MCSymbolRefExpr::VK_PLT,
1530                                            OutContext)),
1531       STI);
1532 }
1533 
1534 void X86AsmPrinter::emitAsanMemaccessSymbols(Module &M) {
1535   if (AsanMemaccessSymbols.empty())
1536     return;
1537 
1538   const Triple &TT = TM.getTargetTriple();
1539   assert(TT.isOSBinFormatELF());
1540   std::unique_ptr<MCSubtargetInfo> STI(
1541       TM.getTarget().createMCSubtargetInfo(TT.str(), "", ""));
1542   assert(STI && "Unable to create subtarget info");
1543 
1544   for (auto &P : AsanMemaccessSymbols) {
1545     MCSymbol *Sym = P.second;
1546     OutStreamer->SwitchSection(OutContext.getELFSection(
1547         ".text.hot", ELF::SHT_PROGBITS,
1548         ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, Sym->getName(),
1549         /*IsComdat=*/true));
1550 
1551     OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
1552     OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
1553     OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden);
1554     OutStreamer->emitLabel(Sym);
1555 
1556     unsigned Reg = std::get<0>(P.first);
1557     ASanAccessInfo AccessInfo(std::get<1>(P.first));
1558 
1559     if (AccessInfo.AccessSizeIndex < 3) {
1560       emitAsanMemaccessPartial(M, Reg, AccessInfo, *STI);
1561     } else {
1562       emitAsanMemaccessFull(M, Reg, AccessInfo, *STI);
1563     }
1564   }
1565 }
1566 
1567 void X86AsmPrinter::LowerPATCHABLE_OP(const MachineInstr &MI,
1568                                       X86MCInstLower &MCIL) {
1569   // PATCHABLE_OP minsize, opcode, operands
1570 
1571   NoAutoPaddingScope NoPadScope(*OutStreamer);
1572 
1573   unsigned MinSize = MI.getOperand(0).getImm();
1574   unsigned Opcode = MI.getOperand(1).getImm();
1575 
1576   MCInst MCI;
1577   MCI.setOpcode(Opcode);
1578   for (auto &MO : drop_begin(MI.operands(), 2))
1579     if (auto MaybeOperand = MCIL.LowerMachineOperand(&MI, MO))
1580       MCI.addOperand(MaybeOperand.getValue());
1581 
1582   SmallString<256> Code;
1583   SmallVector<MCFixup, 4> Fixups;
1584   raw_svector_ostream VecOS(Code);
1585   CodeEmitter->encodeInstruction(MCI, VecOS, Fixups, getSubtargetInfo());
1586 
1587   if (Code.size() < MinSize) {
1588     if (MinSize == 2 && Subtarget->is32Bit() &&
1589         Subtarget->isTargetWindowsMSVC() &&
1590         (Subtarget->getCPU().empty() || Subtarget->getCPU() == "pentium3")) {
1591       // For compatibilty reasons, when targetting MSVC, is is important to
1592       // generate a 'legacy' NOP in the form of a 8B FF MOV EDI, EDI. Some tools
1593       // rely specifically on this pattern to be able to patch a function.
1594       // This is only for 32-bit targets, when using /arch:IA32 or /arch:SSE.
1595       OutStreamer->emitInstruction(
1596           MCInstBuilder(X86::MOV32rr_REV).addReg(X86::EDI).addReg(X86::EDI),
1597           *Subtarget);
1598     } else if (MinSize == 2 && Opcode == X86::PUSH64r) {
1599       // This is an optimization that lets us get away without emitting a nop in
1600       // many cases.
1601       //
1602       // NB! In some cases the encoding for PUSH64r (e.g. PUSH64r %r9) takes two
1603       // bytes too, so the check on MinSize is important.
1604       MCI.setOpcode(X86::PUSH64rmr);
1605     } else {
1606       unsigned NopSize = emitNop(*OutStreamer, MinSize, Subtarget);
1607       assert(NopSize == MinSize && "Could not implement MinSize!");
1608       (void)NopSize;
1609     }
1610   }
1611 
1612   OutStreamer->emitInstruction(MCI, getSubtargetInfo());
1613 }
1614 
1615 // Lower a stackmap of the form:
1616 // <id>, <shadowBytes>, ...
1617 void X86AsmPrinter::LowerSTACKMAP(const MachineInstr &MI) {
1618   SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo());
1619 
1620   auto &Ctx = OutStreamer->getContext();
1621   MCSymbol *MILabel = Ctx.createTempSymbol();
1622   OutStreamer->emitLabel(MILabel);
1623 
1624   SM.recordStackMap(*MILabel, MI);
1625   unsigned NumShadowBytes = MI.getOperand(1).getImm();
1626   SMShadowTracker.reset(NumShadowBytes);
1627 }
1628 
1629 // Lower a patchpoint of the form:
1630 // [<def>], <id>, <numBytes>, <target>, <numArgs>, <cc>, ...
1631 void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI,
1632                                     X86MCInstLower &MCIL) {
1633   assert(Subtarget->is64Bit() && "Patchpoint currently only supports X86-64");
1634 
1635   SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo());
1636 
1637   NoAutoPaddingScope NoPadScope(*OutStreamer);
1638 
1639   auto &Ctx = OutStreamer->getContext();
1640   MCSymbol *MILabel = Ctx.createTempSymbol();
1641   OutStreamer->emitLabel(MILabel);
1642   SM.recordPatchPoint(*MILabel, MI);
1643 
1644   PatchPointOpers opers(&MI);
1645   unsigned ScratchIdx = opers.getNextScratchIdx();
1646   unsigned EncodedBytes = 0;
1647   const MachineOperand &CalleeMO = opers.getCallTarget();
1648 
1649   // Check for null target. If target is non-null (i.e. is non-zero or is
1650   // symbolic) then emit a call.
1651   if (!(CalleeMO.isImm() && !CalleeMO.getImm())) {
1652     MCOperand CalleeMCOp;
1653     switch (CalleeMO.getType()) {
1654     default:
1655       /// FIXME: Add a verifier check for bad callee types.
1656       llvm_unreachable("Unrecognized callee operand type.");
1657     case MachineOperand::MO_Immediate:
1658       if (CalleeMO.getImm())
1659         CalleeMCOp = MCOperand::createImm(CalleeMO.getImm());
1660       break;
1661     case MachineOperand::MO_ExternalSymbol:
1662     case MachineOperand::MO_GlobalAddress:
1663       CalleeMCOp = MCIL.LowerSymbolOperand(CalleeMO,
1664                                            MCIL.GetSymbolFromOperand(CalleeMO));
1665       break;
1666     }
1667 
1668     // Emit MOV to materialize the target address and the CALL to target.
1669     // This is encoded with 12-13 bytes, depending on which register is used.
1670     Register ScratchReg = MI.getOperand(ScratchIdx).getReg();
1671     if (X86II::isX86_64ExtendedReg(ScratchReg))
1672       EncodedBytes = 13;
1673     else
1674       EncodedBytes = 12;
1675 
1676     EmitAndCountInstruction(
1677         MCInstBuilder(X86::MOV64ri).addReg(ScratchReg).addOperand(CalleeMCOp));
1678     // FIXME: Add retpoline support and remove this.
1679     if (Subtarget->useIndirectThunkCalls())
1680       report_fatal_error(
1681           "Lowering patchpoint with thunks not yet implemented.");
1682     EmitAndCountInstruction(MCInstBuilder(X86::CALL64r).addReg(ScratchReg));
1683   }
1684 
1685   // Emit padding.
1686   unsigned NumBytes = opers.getNumPatchBytes();
1687   assert(NumBytes >= EncodedBytes &&
1688          "Patchpoint can't request size less than the length of a call.");
1689 
1690   emitX86Nops(*OutStreamer, NumBytes - EncodedBytes, Subtarget);
1691 }
1692 
1693 void X86AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI,
1694                                               X86MCInstLower &MCIL) {
1695   assert(Subtarget->is64Bit() && "XRay custom events only supports X86-64");
1696 
1697   NoAutoPaddingScope NoPadScope(*OutStreamer);
1698 
1699   // We want to emit the following pattern, which follows the x86 calling
1700   // convention to prepare for the trampoline call to be patched in.
1701   //
1702   //   .p2align 1, ...
1703   // .Lxray_event_sled_N:
1704   //   jmp +N                        // jump across the instrumentation sled
1705   //   ...                           // set up arguments in register
1706   //   callq __xray_CustomEvent@plt  // force dependency to symbol
1707   //   ...
1708   //   <jump here>
1709   //
1710   // After patching, it would look something like:
1711   //
1712   //   nopw (2-byte nop)
1713   //   ...
1714   //   callq __xrayCustomEvent  // already lowered
1715   //   ...
1716   //
1717   // ---
1718   // First we emit the label and the jump.
1719   auto CurSled = OutContext.createTempSymbol("xray_event_sled_", true);
1720   OutStreamer->AddComment("# XRay Custom Event Log");
1721   OutStreamer->emitCodeAlignment(2, &getSubtargetInfo());
1722   OutStreamer->emitLabel(CurSled);
1723 
1724   // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1725   // an operand (computed as an offset from the jmp instruction).
1726   // FIXME: Find another less hacky way do force the relative jump.
1727   OutStreamer->emitBinaryData("\xeb\x0f");
1728 
1729   // The default C calling convention will place two arguments into %rcx and
1730   // %rdx -- so we only work with those.
1731   const Register DestRegs[] = {X86::RDI, X86::RSI};
1732   bool UsedMask[] = {false, false};
1733   // Filled out in loop.
1734   Register SrcRegs[] = {0, 0};
1735 
1736   // Then we put the operands in the %rdi and %rsi registers. We spill the
1737   // values in the register before we clobber them, and mark them as used in
1738   // UsedMask. In case the arguments are already in the correct register, we use
1739   // emit nops appropriately sized to keep the sled the same size in every
1740   // situation.
1741   for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1742     if (auto Op = MCIL.LowerMachineOperand(&MI, MI.getOperand(I))) {
1743       assert(Op->isReg() && "Only support arguments in registers");
1744       SrcRegs[I] = getX86SubSuperRegister(Op->getReg(), 64);
1745       if (SrcRegs[I] != DestRegs[I]) {
1746         UsedMask[I] = true;
1747         EmitAndCountInstruction(
1748             MCInstBuilder(X86::PUSH64r).addReg(DestRegs[I]));
1749       } else {
1750         emitX86Nops(*OutStreamer, 4, Subtarget);
1751       }
1752     }
1753 
1754   // Now that the register values are stashed, mov arguments into place.
1755   // FIXME: This doesn't work if one of the later SrcRegs is equal to an
1756   // earlier DestReg. We will have already overwritten over the register before
1757   // we can copy from it.
1758   for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1759     if (SrcRegs[I] != DestRegs[I])
1760       EmitAndCountInstruction(
1761           MCInstBuilder(X86::MOV64rr).addReg(DestRegs[I]).addReg(SrcRegs[I]));
1762 
1763   // We emit a hard dependency on the __xray_CustomEvent symbol, which is the
1764   // name of the trampoline to be implemented by the XRay runtime.
1765   auto TSym = OutContext.getOrCreateSymbol("__xray_CustomEvent");
1766   MachineOperand TOp = MachineOperand::CreateMCSymbol(TSym);
1767   if (isPositionIndependent())
1768     TOp.setTargetFlags(X86II::MO_PLT);
1769 
1770   // Emit the call instruction.
1771   EmitAndCountInstruction(MCInstBuilder(X86::CALL64pcrel32)
1772                               .addOperand(MCIL.LowerSymbolOperand(TOp, TSym)));
1773 
1774   // Restore caller-saved and used registers.
1775   for (unsigned I = sizeof UsedMask; I-- > 0;)
1776     if (UsedMask[I])
1777       EmitAndCountInstruction(MCInstBuilder(X86::POP64r).addReg(DestRegs[I]));
1778     else
1779       emitX86Nops(*OutStreamer, 1, Subtarget);
1780 
1781   OutStreamer->AddComment("xray custom event end.");
1782 
1783   // Record the sled version. Version 0 of this sled was spelled differently, so
1784   // we let the runtime handle the different offsets we're using. Version 2
1785   // changed the absolute address to a PC-relative address.
1786   recordSled(CurSled, MI, SledKind::CUSTOM_EVENT, 2);
1787 }
1788 
1789 void X86AsmPrinter::LowerPATCHABLE_TYPED_EVENT_CALL(const MachineInstr &MI,
1790                                                     X86MCInstLower &MCIL) {
1791   assert(Subtarget->is64Bit() && "XRay typed events only supports X86-64");
1792 
1793   NoAutoPaddingScope NoPadScope(*OutStreamer);
1794 
1795   // We want to emit the following pattern, which follows the x86 calling
1796   // convention to prepare for the trampoline call to be patched in.
1797   //
1798   //   .p2align 1, ...
1799   // .Lxray_event_sled_N:
1800   //   jmp +N                        // jump across the instrumentation sled
1801   //   ...                           // set up arguments in register
1802   //   callq __xray_TypedEvent@plt  // force dependency to symbol
1803   //   ...
1804   //   <jump here>
1805   //
1806   // After patching, it would look something like:
1807   //
1808   //   nopw (2-byte nop)
1809   //   ...
1810   //   callq __xrayTypedEvent  // already lowered
1811   //   ...
1812   //
1813   // ---
1814   // First we emit the label and the jump.
1815   auto CurSled = OutContext.createTempSymbol("xray_typed_event_sled_", true);
1816   OutStreamer->AddComment("# XRay Typed Event Log");
1817   OutStreamer->emitCodeAlignment(2, &getSubtargetInfo());
1818   OutStreamer->emitLabel(CurSled);
1819 
1820   // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1821   // an operand (computed as an offset from the jmp instruction).
1822   // FIXME: Find another less hacky way do force the relative jump.
1823   OutStreamer->emitBinaryData("\xeb\x14");
1824 
1825   // An x86-64 convention may place three arguments into %rcx, %rdx, and R8,
1826   // so we'll work with those. Or we may be called via SystemV, in which case
1827   // we don't have to do any translation.
1828   const Register DestRegs[] = {X86::RDI, X86::RSI, X86::RDX};
1829   bool UsedMask[] = {false, false, false};
1830 
1831   // Will fill out src regs in the loop.
1832   Register SrcRegs[] = {0, 0, 0};
1833 
1834   // Then we put the operands in the SystemV registers. We spill the values in
1835   // the registers before we clobber them, and mark them as used in UsedMask.
1836   // In case the arguments are already in the correct register, we emit nops
1837   // appropriately sized to keep the sled the same size in every situation.
1838   for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1839     if (auto Op = MCIL.LowerMachineOperand(&MI, MI.getOperand(I))) {
1840       // TODO: Is register only support adequate?
1841       assert(Op->isReg() && "Only supports arguments in registers");
1842       SrcRegs[I] = getX86SubSuperRegister(Op->getReg(), 64);
1843       if (SrcRegs[I] != DestRegs[I]) {
1844         UsedMask[I] = true;
1845         EmitAndCountInstruction(
1846             MCInstBuilder(X86::PUSH64r).addReg(DestRegs[I]));
1847       } else {
1848         emitX86Nops(*OutStreamer, 4, Subtarget);
1849       }
1850     }
1851 
1852   // In the above loop we only stash all of the destination registers or emit
1853   // nops if the arguments are already in the right place. Doing the actually
1854   // moving is postponed until after all the registers are stashed so nothing
1855   // is clobbers. We've already added nops to account for the size of mov and
1856   // push if the register is in the right place, so we only have to worry about
1857   // emitting movs.
1858   // FIXME: This doesn't work if one of the later SrcRegs is equal to an
1859   // earlier DestReg. We will have already overwritten over the register before
1860   // we can copy from it.
1861   for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1862     if (UsedMask[I])
1863       EmitAndCountInstruction(
1864           MCInstBuilder(X86::MOV64rr).addReg(DestRegs[I]).addReg(SrcRegs[I]));
1865 
1866   // We emit a hard dependency on the __xray_TypedEvent symbol, which is the
1867   // name of the trampoline to be implemented by the XRay runtime.
1868   auto TSym = OutContext.getOrCreateSymbol("__xray_TypedEvent");
1869   MachineOperand TOp = MachineOperand::CreateMCSymbol(TSym);
1870   if (isPositionIndependent())
1871     TOp.setTargetFlags(X86II::MO_PLT);
1872 
1873   // Emit the call instruction.
1874   EmitAndCountInstruction(MCInstBuilder(X86::CALL64pcrel32)
1875                               .addOperand(MCIL.LowerSymbolOperand(TOp, TSym)));
1876 
1877   // Restore caller-saved and used registers.
1878   for (unsigned I = sizeof UsedMask; I-- > 0;)
1879     if (UsedMask[I])
1880       EmitAndCountInstruction(MCInstBuilder(X86::POP64r).addReg(DestRegs[I]));
1881     else
1882       emitX86Nops(*OutStreamer, 1, Subtarget);
1883 
1884   OutStreamer->AddComment("xray typed event end.");
1885 
1886   // Record the sled version.
1887   recordSled(CurSled, MI, SledKind::TYPED_EVENT, 2);
1888 }
1889 
1890 void X86AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI,
1891                                                   X86MCInstLower &MCIL) {
1892 
1893   NoAutoPaddingScope NoPadScope(*OutStreamer);
1894 
1895   const Function &F = MF->getFunction();
1896   if (F.hasFnAttribute("patchable-function-entry")) {
1897     unsigned Num;
1898     if (F.getFnAttribute("patchable-function-entry")
1899             .getValueAsString()
1900             .getAsInteger(10, Num))
1901       return;
1902     emitX86Nops(*OutStreamer, Num, Subtarget);
1903     return;
1904   }
1905   // We want to emit the following pattern:
1906   //
1907   //   .p2align 1, ...
1908   // .Lxray_sled_N:
1909   //   jmp .tmpN
1910   //   # 9 bytes worth of noops
1911   //
1912   // We need the 9 bytes because at runtime, we'd be patching over the full 11
1913   // bytes with the following pattern:
1914   //
1915   //   mov %r10, <function id, 32-bit>   // 6 bytes
1916   //   call <relative offset, 32-bits>   // 5 bytes
1917   //
1918   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
1919   OutStreamer->emitCodeAlignment(2, &getSubtargetInfo());
1920   OutStreamer->emitLabel(CurSled);
1921 
1922   // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1923   // an operand (computed as an offset from the jmp instruction).
1924   // FIXME: Find another less hacky way do force the relative jump.
1925   OutStreamer->emitBytes("\xeb\x09");
1926   emitX86Nops(*OutStreamer, 9, Subtarget);
1927   recordSled(CurSled, MI, SledKind::FUNCTION_ENTER, 2);
1928 }
1929 
1930 void X86AsmPrinter::LowerPATCHABLE_RET(const MachineInstr &MI,
1931                                        X86MCInstLower &MCIL) {
1932   NoAutoPaddingScope NoPadScope(*OutStreamer);
1933 
1934   // Since PATCHABLE_RET takes the opcode of the return statement as an
1935   // argument, we use that to emit the correct form of the RET that we want.
1936   // i.e. when we see this:
1937   //
1938   //   PATCHABLE_RET X86::RET ...
1939   //
1940   // We should emit the RET followed by sleds.
1941   //
1942   //   .p2align 1, ...
1943   // .Lxray_sled_N:
1944   //   ret  # or equivalent instruction
1945   //   # 10 bytes worth of noops
1946   //
1947   // This just makes sure that the alignment for the next instruction is 2.
1948   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
1949   OutStreamer->emitCodeAlignment(2, &getSubtargetInfo());
1950   OutStreamer->emitLabel(CurSled);
1951   unsigned OpCode = MI.getOperand(0).getImm();
1952   MCInst Ret;
1953   Ret.setOpcode(OpCode);
1954   for (auto &MO : drop_begin(MI.operands()))
1955     if (auto MaybeOperand = MCIL.LowerMachineOperand(&MI, MO))
1956       Ret.addOperand(MaybeOperand.getValue());
1957   OutStreamer->emitInstruction(Ret, getSubtargetInfo());
1958   emitX86Nops(*OutStreamer, 10, Subtarget);
1959   recordSled(CurSled, MI, SledKind::FUNCTION_EXIT, 2);
1960 }
1961 
1962 void X86AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI,
1963                                              X86MCInstLower &MCIL) {
1964   NoAutoPaddingScope NoPadScope(*OutStreamer);
1965 
1966   // Like PATCHABLE_RET, we have the actual instruction in the operands to this
1967   // instruction so we lower that particular instruction and its operands.
1968   // Unlike PATCHABLE_RET though, we put the sled before the JMP, much like how
1969   // we do it for PATCHABLE_FUNCTION_ENTER. The sled should be very similar to
1970   // the PATCHABLE_FUNCTION_ENTER case, followed by the lowering of the actual
1971   // tail call much like how we have it in PATCHABLE_RET.
1972   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
1973   OutStreamer->emitCodeAlignment(2, &getSubtargetInfo());
1974   OutStreamer->emitLabel(CurSled);
1975   auto Target = OutContext.createTempSymbol();
1976 
1977   // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1978   // an operand (computed as an offset from the jmp instruction).
1979   // FIXME: Find another less hacky way do force the relative jump.
1980   OutStreamer->emitBytes("\xeb\x09");
1981   emitX86Nops(*OutStreamer, 9, Subtarget);
1982   OutStreamer->emitLabel(Target);
1983   recordSled(CurSled, MI, SledKind::TAIL_CALL, 2);
1984 
1985   unsigned OpCode = MI.getOperand(0).getImm();
1986   OpCode = convertTailJumpOpcode(OpCode);
1987   MCInst TC;
1988   TC.setOpcode(OpCode);
1989 
1990   // Before emitting the instruction, add a comment to indicate that this is
1991   // indeed a tail call.
1992   OutStreamer->AddComment("TAILCALL");
1993   for (auto &MO : drop_begin(MI.operands()))
1994     if (auto MaybeOperand = MCIL.LowerMachineOperand(&MI, MO))
1995       TC.addOperand(MaybeOperand.getValue());
1996   OutStreamer->emitInstruction(TC, getSubtargetInfo());
1997 }
1998 
1999 // Returns instruction preceding MBBI in MachineFunction.
2000 // If MBBI is the first instruction of the first basic block, returns null.
2001 static MachineBasicBlock::const_iterator
2002 PrevCrossBBInst(MachineBasicBlock::const_iterator MBBI) {
2003   const MachineBasicBlock *MBB = MBBI->getParent();
2004   while (MBBI == MBB->begin()) {
2005     if (MBB == &MBB->getParent()->front())
2006       return MachineBasicBlock::const_iterator();
2007     MBB = MBB->getPrevNode();
2008     MBBI = MBB->end();
2009   }
2010   --MBBI;
2011   return MBBI;
2012 }
2013 
2014 static const Constant *getConstantFromPool(const MachineInstr &MI,
2015                                            const MachineOperand &Op) {
2016   if (!Op.isCPI() || Op.getOffset() != 0)
2017     return nullptr;
2018 
2019   ArrayRef<MachineConstantPoolEntry> Constants =
2020       MI.getParent()->getParent()->getConstantPool()->getConstants();
2021   const MachineConstantPoolEntry &ConstantEntry = Constants[Op.getIndex()];
2022 
2023   // Bail if this is a machine constant pool entry, we won't be able to dig out
2024   // anything useful.
2025   if (ConstantEntry.isMachineConstantPoolEntry())
2026     return nullptr;
2027 
2028   return ConstantEntry.Val.ConstVal;
2029 }
2030 
2031 static std::string getShuffleComment(const MachineInstr *MI, unsigned SrcOp1Idx,
2032                                      unsigned SrcOp2Idx, ArrayRef<int> Mask) {
2033   std::string Comment;
2034 
2035   // Compute the name for a register. This is really goofy because we have
2036   // multiple instruction printers that could (in theory) use different
2037   // names. Fortunately most people use the ATT style (outside of Windows)
2038   // and they actually agree on register naming here. Ultimately, this is
2039   // a comment, and so its OK if it isn't perfect.
2040   auto GetRegisterName = [](unsigned RegNum) -> StringRef {
2041     return X86ATTInstPrinter::getRegisterName(RegNum);
2042   };
2043 
2044   const MachineOperand &DstOp = MI->getOperand(0);
2045   const MachineOperand &SrcOp1 = MI->getOperand(SrcOp1Idx);
2046   const MachineOperand &SrcOp2 = MI->getOperand(SrcOp2Idx);
2047 
2048   StringRef DstName = DstOp.isReg() ? GetRegisterName(DstOp.getReg()) : "mem";
2049   StringRef Src1Name =
2050       SrcOp1.isReg() ? GetRegisterName(SrcOp1.getReg()) : "mem";
2051   StringRef Src2Name =
2052       SrcOp2.isReg() ? GetRegisterName(SrcOp2.getReg()) : "mem";
2053 
2054   // One source operand, fix the mask to print all elements in one span.
2055   SmallVector<int, 8> ShuffleMask(Mask.begin(), Mask.end());
2056   if (Src1Name == Src2Name)
2057     for (int i = 0, e = ShuffleMask.size(); i != e; ++i)
2058       if (ShuffleMask[i] >= e)
2059         ShuffleMask[i] -= e;
2060 
2061   raw_string_ostream CS(Comment);
2062   CS << DstName;
2063 
2064   // Handle AVX512 MASK/MASXZ write mask comments.
2065   // MASK: zmmX {%kY}
2066   // MASKZ: zmmX {%kY} {z}
2067   if (SrcOp1Idx > 1) {
2068     assert((SrcOp1Idx == 2 || SrcOp1Idx == 3) && "Unexpected writemask");
2069 
2070     const MachineOperand &WriteMaskOp = MI->getOperand(SrcOp1Idx - 1);
2071     if (WriteMaskOp.isReg()) {
2072       CS << " {%" << GetRegisterName(WriteMaskOp.getReg()) << "}";
2073 
2074       if (SrcOp1Idx == 2) {
2075         CS << " {z}";
2076       }
2077     }
2078   }
2079 
2080   CS << " = ";
2081 
2082   for (int i = 0, e = ShuffleMask.size(); i != e; ++i) {
2083     if (i != 0)
2084       CS << ",";
2085     if (ShuffleMask[i] == SM_SentinelZero) {
2086       CS << "zero";
2087       continue;
2088     }
2089 
2090     // Otherwise, it must come from src1 or src2.  Print the span of elements
2091     // that comes from this src.
2092     bool isSrc1 = ShuffleMask[i] < (int)e;
2093     CS << (isSrc1 ? Src1Name : Src2Name) << '[';
2094 
2095     bool IsFirst = true;
2096     while (i != e && ShuffleMask[i] != SM_SentinelZero &&
2097            (ShuffleMask[i] < (int)e) == isSrc1) {
2098       if (!IsFirst)
2099         CS << ',';
2100       else
2101         IsFirst = false;
2102       if (ShuffleMask[i] == SM_SentinelUndef)
2103         CS << "u";
2104       else
2105         CS << ShuffleMask[i] % (int)e;
2106       ++i;
2107     }
2108     CS << ']';
2109     --i; // For loop increments element #.
2110   }
2111   CS.flush();
2112 
2113   return Comment;
2114 }
2115 
2116 static void printConstant(const APInt &Val, raw_ostream &CS) {
2117   if (Val.getBitWidth() <= 64) {
2118     CS << Val.getZExtValue();
2119   } else {
2120     // print multi-word constant as (w0,w1)
2121     CS << "(";
2122     for (int i = 0, N = Val.getNumWords(); i < N; ++i) {
2123       if (i > 0)
2124         CS << ",";
2125       CS << Val.getRawData()[i];
2126     }
2127     CS << ")";
2128   }
2129 }
2130 
2131 static void printConstant(const APFloat &Flt, raw_ostream &CS) {
2132   SmallString<32> Str;
2133   // Force scientific notation to distinquish from integers.
2134   Flt.toString(Str, 0, 0);
2135   CS << Str;
2136 }
2137 
2138 static void printConstant(const Constant *COp, raw_ostream &CS) {
2139   if (isa<UndefValue>(COp)) {
2140     CS << "u";
2141   } else if (auto *CI = dyn_cast<ConstantInt>(COp)) {
2142     printConstant(CI->getValue(), CS);
2143   } else if (auto *CF = dyn_cast<ConstantFP>(COp)) {
2144     printConstant(CF->getValueAPF(), CS);
2145   } else {
2146     CS << "?";
2147   }
2148 }
2149 
2150 void X86AsmPrinter::EmitSEHInstruction(const MachineInstr *MI) {
2151   assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
2152   assert(getSubtarget().isOSWindows() && "SEH_ instruction Windows only");
2153 
2154   // Use the .cv_fpo directives if we're emitting CodeView on 32-bit x86.
2155   if (EmitFPOData) {
2156     X86TargetStreamer *XTS =
2157         static_cast<X86TargetStreamer *>(OutStreamer->getTargetStreamer());
2158     switch (MI->getOpcode()) {
2159     case X86::SEH_PushReg:
2160       XTS->emitFPOPushReg(MI->getOperand(0).getImm());
2161       break;
2162     case X86::SEH_StackAlloc:
2163       XTS->emitFPOStackAlloc(MI->getOperand(0).getImm());
2164       break;
2165     case X86::SEH_StackAlign:
2166       XTS->emitFPOStackAlign(MI->getOperand(0).getImm());
2167       break;
2168     case X86::SEH_SetFrame:
2169       assert(MI->getOperand(1).getImm() == 0 &&
2170              ".cv_fpo_setframe takes no offset");
2171       XTS->emitFPOSetFrame(MI->getOperand(0).getImm());
2172       break;
2173     case X86::SEH_EndPrologue:
2174       XTS->emitFPOEndPrologue();
2175       break;
2176     case X86::SEH_SaveReg:
2177     case X86::SEH_SaveXMM:
2178     case X86::SEH_PushFrame:
2179       llvm_unreachable("SEH_ directive incompatible with FPO");
2180       break;
2181     default:
2182       llvm_unreachable("expected SEH_ instruction");
2183     }
2184     return;
2185   }
2186 
2187   // Otherwise, use the .seh_ directives for all other Windows platforms.
2188   switch (MI->getOpcode()) {
2189   case X86::SEH_PushReg:
2190     OutStreamer->EmitWinCFIPushReg(MI->getOperand(0).getImm());
2191     break;
2192 
2193   case X86::SEH_SaveReg:
2194     OutStreamer->EmitWinCFISaveReg(MI->getOperand(0).getImm(),
2195                                    MI->getOperand(1).getImm());
2196     break;
2197 
2198   case X86::SEH_SaveXMM:
2199     OutStreamer->EmitWinCFISaveXMM(MI->getOperand(0).getImm(),
2200                                    MI->getOperand(1).getImm());
2201     break;
2202 
2203   case X86::SEH_StackAlloc:
2204     OutStreamer->EmitWinCFIAllocStack(MI->getOperand(0).getImm());
2205     break;
2206 
2207   case X86::SEH_SetFrame:
2208     OutStreamer->EmitWinCFISetFrame(MI->getOperand(0).getImm(),
2209                                     MI->getOperand(1).getImm());
2210     break;
2211 
2212   case X86::SEH_PushFrame:
2213     OutStreamer->EmitWinCFIPushFrame(MI->getOperand(0).getImm());
2214     break;
2215 
2216   case X86::SEH_EndPrologue:
2217     OutStreamer->EmitWinCFIEndProlog();
2218     break;
2219 
2220   default:
2221     llvm_unreachable("expected SEH_ instruction");
2222   }
2223 }
2224 
2225 static unsigned getRegisterWidth(const MCOperandInfo &Info) {
2226   if (Info.RegClass == X86::VR128RegClassID ||
2227       Info.RegClass == X86::VR128XRegClassID)
2228     return 128;
2229   if (Info.RegClass == X86::VR256RegClassID ||
2230       Info.RegClass == X86::VR256XRegClassID)
2231     return 256;
2232   if (Info.RegClass == X86::VR512RegClassID)
2233     return 512;
2234   llvm_unreachable("Unknown register class!");
2235 }
2236 
2237 static void addConstantComments(const MachineInstr *MI,
2238                                 MCStreamer &OutStreamer) {
2239   switch (MI->getOpcode()) {
2240   // Lower PSHUFB and VPERMILP normally but add a comment if we can find
2241   // a constant shuffle mask. We won't be able to do this at the MC layer
2242   // because the mask isn't an immediate.
2243   case X86::PSHUFBrm:
2244   case X86::VPSHUFBrm:
2245   case X86::VPSHUFBYrm:
2246   case X86::VPSHUFBZ128rm:
2247   case X86::VPSHUFBZ128rmk:
2248   case X86::VPSHUFBZ128rmkz:
2249   case X86::VPSHUFBZ256rm:
2250   case X86::VPSHUFBZ256rmk:
2251   case X86::VPSHUFBZ256rmkz:
2252   case X86::VPSHUFBZrm:
2253   case X86::VPSHUFBZrmk:
2254   case X86::VPSHUFBZrmkz: {
2255     unsigned SrcIdx = 1;
2256     if (X86II::isKMasked(MI->getDesc().TSFlags)) {
2257       // Skip mask operand.
2258       ++SrcIdx;
2259       if (X86II::isKMergeMasked(MI->getDesc().TSFlags)) {
2260         // Skip passthru operand.
2261         ++SrcIdx;
2262       }
2263     }
2264     unsigned MaskIdx = SrcIdx + 1 + X86::AddrDisp;
2265 
2266     assert(MI->getNumOperands() >= (SrcIdx + 1 + X86::AddrNumOperands) &&
2267            "Unexpected number of operands!");
2268 
2269     const MachineOperand &MaskOp = MI->getOperand(MaskIdx);
2270     if (auto *C = getConstantFromPool(*MI, MaskOp)) {
2271       unsigned Width = getRegisterWidth(MI->getDesc().OpInfo[0]);
2272       SmallVector<int, 64> Mask;
2273       DecodePSHUFBMask(C, Width, Mask);
2274       if (!Mask.empty())
2275         OutStreamer.AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask));
2276     }
2277     break;
2278   }
2279 
2280   case X86::VPERMILPSrm:
2281   case X86::VPERMILPSYrm:
2282   case X86::VPERMILPSZ128rm:
2283   case X86::VPERMILPSZ128rmk:
2284   case X86::VPERMILPSZ128rmkz:
2285   case X86::VPERMILPSZ256rm:
2286   case X86::VPERMILPSZ256rmk:
2287   case X86::VPERMILPSZ256rmkz:
2288   case X86::VPERMILPSZrm:
2289   case X86::VPERMILPSZrmk:
2290   case X86::VPERMILPSZrmkz:
2291   case X86::VPERMILPDrm:
2292   case X86::VPERMILPDYrm:
2293   case X86::VPERMILPDZ128rm:
2294   case X86::VPERMILPDZ128rmk:
2295   case X86::VPERMILPDZ128rmkz:
2296   case X86::VPERMILPDZ256rm:
2297   case X86::VPERMILPDZ256rmk:
2298   case X86::VPERMILPDZ256rmkz:
2299   case X86::VPERMILPDZrm:
2300   case X86::VPERMILPDZrmk:
2301   case X86::VPERMILPDZrmkz: {
2302     unsigned ElSize;
2303     switch (MI->getOpcode()) {
2304     default: llvm_unreachable("Invalid opcode");
2305     case X86::VPERMILPSrm:
2306     case X86::VPERMILPSYrm:
2307     case X86::VPERMILPSZ128rm:
2308     case X86::VPERMILPSZ256rm:
2309     case X86::VPERMILPSZrm:
2310     case X86::VPERMILPSZ128rmkz:
2311     case X86::VPERMILPSZ256rmkz:
2312     case X86::VPERMILPSZrmkz:
2313     case X86::VPERMILPSZ128rmk:
2314     case X86::VPERMILPSZ256rmk:
2315     case X86::VPERMILPSZrmk:
2316       ElSize = 32;
2317       break;
2318     case X86::VPERMILPDrm:
2319     case X86::VPERMILPDYrm:
2320     case X86::VPERMILPDZ128rm:
2321     case X86::VPERMILPDZ256rm:
2322     case X86::VPERMILPDZrm:
2323     case X86::VPERMILPDZ128rmkz:
2324     case X86::VPERMILPDZ256rmkz:
2325     case X86::VPERMILPDZrmkz:
2326     case X86::VPERMILPDZ128rmk:
2327     case X86::VPERMILPDZ256rmk:
2328     case X86::VPERMILPDZrmk:
2329       ElSize = 64;
2330       break;
2331     }
2332 
2333     unsigned SrcIdx = 1;
2334     if (X86II::isKMasked(MI->getDesc().TSFlags)) {
2335       // Skip mask operand.
2336       ++SrcIdx;
2337       if (X86II::isKMergeMasked(MI->getDesc().TSFlags)) {
2338         // Skip passthru operand.
2339         ++SrcIdx;
2340       }
2341     }
2342     unsigned MaskIdx = SrcIdx + 1 + X86::AddrDisp;
2343 
2344     assert(MI->getNumOperands() >= (SrcIdx + 1 + X86::AddrNumOperands) &&
2345            "Unexpected number of operands!");
2346 
2347     const MachineOperand &MaskOp = MI->getOperand(MaskIdx);
2348     if (auto *C = getConstantFromPool(*MI, MaskOp)) {
2349       unsigned Width = getRegisterWidth(MI->getDesc().OpInfo[0]);
2350       SmallVector<int, 16> Mask;
2351       DecodeVPERMILPMask(C, ElSize, Width, Mask);
2352       if (!Mask.empty())
2353         OutStreamer.AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask));
2354     }
2355     break;
2356   }
2357 
2358   case X86::VPERMIL2PDrm:
2359   case X86::VPERMIL2PSrm:
2360   case X86::VPERMIL2PDYrm:
2361   case X86::VPERMIL2PSYrm: {
2362     assert(MI->getNumOperands() >= (3 + X86::AddrNumOperands + 1) &&
2363            "Unexpected number of operands!");
2364 
2365     const MachineOperand &CtrlOp = MI->getOperand(MI->getNumOperands() - 1);
2366     if (!CtrlOp.isImm())
2367       break;
2368 
2369     unsigned ElSize;
2370     switch (MI->getOpcode()) {
2371     default: llvm_unreachable("Invalid opcode");
2372     case X86::VPERMIL2PSrm: case X86::VPERMIL2PSYrm: ElSize = 32; break;
2373     case X86::VPERMIL2PDrm: case X86::VPERMIL2PDYrm: ElSize = 64; break;
2374     }
2375 
2376     const MachineOperand &MaskOp = MI->getOperand(3 + X86::AddrDisp);
2377     if (auto *C = getConstantFromPool(*MI, MaskOp)) {
2378       unsigned Width = getRegisterWidth(MI->getDesc().OpInfo[0]);
2379       SmallVector<int, 16> Mask;
2380       DecodeVPERMIL2PMask(C, (unsigned)CtrlOp.getImm(), ElSize, Width, Mask);
2381       if (!Mask.empty())
2382         OutStreamer.AddComment(getShuffleComment(MI, 1, 2, Mask));
2383     }
2384     break;
2385   }
2386 
2387   case X86::VPPERMrrm: {
2388     assert(MI->getNumOperands() >= (3 + X86::AddrNumOperands) &&
2389            "Unexpected number of operands!");
2390 
2391     const MachineOperand &MaskOp = MI->getOperand(3 + X86::AddrDisp);
2392     if (auto *C = getConstantFromPool(*MI, MaskOp)) {
2393       unsigned Width = getRegisterWidth(MI->getDesc().OpInfo[0]);
2394       SmallVector<int, 16> Mask;
2395       DecodeVPPERMMask(C, Width, Mask);
2396       if (!Mask.empty())
2397         OutStreamer.AddComment(getShuffleComment(MI, 1, 2, Mask));
2398     }
2399     break;
2400   }
2401 
2402   case X86::MMX_MOVQ64rm: {
2403     assert(MI->getNumOperands() == (1 + X86::AddrNumOperands) &&
2404            "Unexpected number of operands!");
2405     if (auto *C = getConstantFromPool(*MI, MI->getOperand(1 + X86::AddrDisp))) {
2406       std::string Comment;
2407       raw_string_ostream CS(Comment);
2408       const MachineOperand &DstOp = MI->getOperand(0);
2409       CS << X86ATTInstPrinter::getRegisterName(DstOp.getReg()) << " = ";
2410       if (auto *CF = dyn_cast<ConstantFP>(C)) {
2411         CS << "0x" << toString(CF->getValueAPF().bitcastToAPInt(), 16, false);
2412         OutStreamer.AddComment(CS.str());
2413       }
2414     }
2415     break;
2416   }
2417 
2418 #define MOV_CASE(Prefix, Suffix)                                               \
2419   case X86::Prefix##MOVAPD##Suffix##rm:                                        \
2420   case X86::Prefix##MOVAPS##Suffix##rm:                                        \
2421   case X86::Prefix##MOVUPD##Suffix##rm:                                        \
2422   case X86::Prefix##MOVUPS##Suffix##rm:                                        \
2423   case X86::Prefix##MOVDQA##Suffix##rm:                                        \
2424   case X86::Prefix##MOVDQU##Suffix##rm:
2425 
2426 #define MOV_AVX512_CASE(Suffix)                                                \
2427   case X86::VMOVDQA64##Suffix##rm:                                             \
2428   case X86::VMOVDQA32##Suffix##rm:                                             \
2429   case X86::VMOVDQU64##Suffix##rm:                                             \
2430   case X86::VMOVDQU32##Suffix##rm:                                             \
2431   case X86::VMOVDQU16##Suffix##rm:                                             \
2432   case X86::VMOVDQU8##Suffix##rm:                                              \
2433   case X86::VMOVAPS##Suffix##rm:                                               \
2434   case X86::VMOVAPD##Suffix##rm:                                               \
2435   case X86::VMOVUPS##Suffix##rm:                                               \
2436   case X86::VMOVUPD##Suffix##rm:
2437 
2438 #define CASE_ALL_MOV_RM()                                                      \
2439   MOV_CASE(, )   /* SSE */                                                     \
2440   MOV_CASE(V, )  /* AVX-128 */                                                 \
2441   MOV_CASE(V, Y) /* AVX-256 */                                                 \
2442   MOV_AVX512_CASE(Z)                                                           \
2443   MOV_AVX512_CASE(Z256)                                                        \
2444   MOV_AVX512_CASE(Z128)
2445 
2446     // For loads from a constant pool to a vector register, print the constant
2447     // loaded.
2448     CASE_ALL_MOV_RM()
2449   case X86::VBROADCASTF128:
2450   case X86::VBROADCASTI128:
2451   case X86::VBROADCASTF32X4Z256rm:
2452   case X86::VBROADCASTF32X4rm:
2453   case X86::VBROADCASTF32X8rm:
2454   case X86::VBROADCASTF64X2Z128rm:
2455   case X86::VBROADCASTF64X2rm:
2456   case X86::VBROADCASTF64X4rm:
2457   case X86::VBROADCASTI32X4Z256rm:
2458   case X86::VBROADCASTI32X4rm:
2459   case X86::VBROADCASTI32X8rm:
2460   case X86::VBROADCASTI64X2Z128rm:
2461   case X86::VBROADCASTI64X2rm:
2462   case X86::VBROADCASTI64X4rm:
2463     assert(MI->getNumOperands() >= (1 + X86::AddrNumOperands) &&
2464            "Unexpected number of operands!");
2465     if (auto *C = getConstantFromPool(*MI, MI->getOperand(1 + X86::AddrDisp))) {
2466       int NumLanes = 1;
2467       // Override NumLanes for the broadcast instructions.
2468       switch (MI->getOpcode()) {
2469       case X86::VBROADCASTF128:        NumLanes = 2; break;
2470       case X86::VBROADCASTI128:        NumLanes = 2; break;
2471       case X86::VBROADCASTF32X4Z256rm: NumLanes = 2; break;
2472       case X86::VBROADCASTF32X4rm:     NumLanes = 4; break;
2473       case X86::VBROADCASTF32X8rm:     NumLanes = 2; break;
2474       case X86::VBROADCASTF64X2Z128rm: NumLanes = 2; break;
2475       case X86::VBROADCASTF64X2rm:     NumLanes = 4; break;
2476       case X86::VBROADCASTF64X4rm:     NumLanes = 2; break;
2477       case X86::VBROADCASTI32X4Z256rm: NumLanes = 2; break;
2478       case X86::VBROADCASTI32X4rm:     NumLanes = 4; break;
2479       case X86::VBROADCASTI32X8rm:     NumLanes = 2; break;
2480       case X86::VBROADCASTI64X2Z128rm: NumLanes = 2; break;
2481       case X86::VBROADCASTI64X2rm:     NumLanes = 4; break;
2482       case X86::VBROADCASTI64X4rm:     NumLanes = 2; break;
2483       }
2484 
2485       std::string Comment;
2486       raw_string_ostream CS(Comment);
2487       const MachineOperand &DstOp = MI->getOperand(0);
2488       CS << X86ATTInstPrinter::getRegisterName(DstOp.getReg()) << " = ";
2489       if (auto *CDS = dyn_cast<ConstantDataSequential>(C)) {
2490         CS << "[";
2491         for (int l = 0; l != NumLanes; ++l) {
2492           for (int i = 0, NumElements = CDS->getNumElements(); i < NumElements;
2493                ++i) {
2494             if (i != 0 || l != 0)
2495               CS << ",";
2496             if (CDS->getElementType()->isIntegerTy())
2497               printConstant(CDS->getElementAsAPInt(i), CS);
2498             else if (CDS->getElementType()->isHalfTy() ||
2499                      CDS->getElementType()->isFloatTy() ||
2500                      CDS->getElementType()->isDoubleTy())
2501               printConstant(CDS->getElementAsAPFloat(i), CS);
2502             else
2503               CS << "?";
2504           }
2505         }
2506         CS << "]";
2507         OutStreamer.AddComment(CS.str());
2508       } else if (auto *CV = dyn_cast<ConstantVector>(C)) {
2509         CS << "<";
2510         for (int l = 0; l != NumLanes; ++l) {
2511           for (int i = 0, NumOperands = CV->getNumOperands(); i < NumOperands;
2512                ++i) {
2513             if (i != 0 || l != 0)
2514               CS << ",";
2515             printConstant(CV->getOperand(i), CS);
2516           }
2517         }
2518         CS << ">";
2519         OutStreamer.AddComment(CS.str());
2520       }
2521     }
2522     break;
2523 
2524   case X86::MOVDDUPrm:
2525   case X86::VMOVDDUPrm:
2526   case X86::VMOVDDUPZ128rm:
2527   case X86::VBROADCASTSSrm:
2528   case X86::VBROADCASTSSYrm:
2529   case X86::VBROADCASTSSZ128rm:
2530   case X86::VBROADCASTSSZ256rm:
2531   case X86::VBROADCASTSSZrm:
2532   case X86::VBROADCASTSDYrm:
2533   case X86::VBROADCASTSDZ256rm:
2534   case X86::VBROADCASTSDZrm:
2535   case X86::VPBROADCASTBrm:
2536   case X86::VPBROADCASTBYrm:
2537   case X86::VPBROADCASTBZ128rm:
2538   case X86::VPBROADCASTBZ256rm:
2539   case X86::VPBROADCASTBZrm:
2540   case X86::VPBROADCASTDrm:
2541   case X86::VPBROADCASTDYrm:
2542   case X86::VPBROADCASTDZ128rm:
2543   case X86::VPBROADCASTDZ256rm:
2544   case X86::VPBROADCASTDZrm:
2545   case X86::VPBROADCASTQrm:
2546   case X86::VPBROADCASTQYrm:
2547   case X86::VPBROADCASTQZ128rm:
2548   case X86::VPBROADCASTQZ256rm:
2549   case X86::VPBROADCASTQZrm:
2550   case X86::VPBROADCASTWrm:
2551   case X86::VPBROADCASTWYrm:
2552   case X86::VPBROADCASTWZ128rm:
2553   case X86::VPBROADCASTWZ256rm:
2554   case X86::VPBROADCASTWZrm:
2555     assert(MI->getNumOperands() >= (1 + X86::AddrNumOperands) &&
2556            "Unexpected number of operands!");
2557     if (auto *C = getConstantFromPool(*MI, MI->getOperand(1 + X86::AddrDisp))) {
2558       int NumElts;
2559       switch (MI->getOpcode()) {
2560       default: llvm_unreachable("Invalid opcode");
2561       case X86::MOVDDUPrm:          NumElts = 2;  break;
2562       case X86::VMOVDDUPrm:         NumElts = 2;  break;
2563       case X86::VMOVDDUPZ128rm:     NumElts = 2;  break;
2564       case X86::VBROADCASTSSrm:     NumElts = 4;  break;
2565       case X86::VBROADCASTSSYrm:    NumElts = 8;  break;
2566       case X86::VBROADCASTSSZ128rm: NumElts = 4;  break;
2567       case X86::VBROADCASTSSZ256rm: NumElts = 8;  break;
2568       case X86::VBROADCASTSSZrm:    NumElts = 16; break;
2569       case X86::VBROADCASTSDYrm:    NumElts = 4;  break;
2570       case X86::VBROADCASTSDZ256rm: NumElts = 4;  break;
2571       case X86::VBROADCASTSDZrm:    NumElts = 8;  break;
2572       case X86::VPBROADCASTBrm:     NumElts = 16; break;
2573       case X86::VPBROADCASTBYrm:    NumElts = 32; break;
2574       case X86::VPBROADCASTBZ128rm: NumElts = 16; break;
2575       case X86::VPBROADCASTBZ256rm: NumElts = 32; break;
2576       case X86::VPBROADCASTBZrm:    NumElts = 64; break;
2577       case X86::VPBROADCASTDrm:     NumElts = 4;  break;
2578       case X86::VPBROADCASTDYrm:    NumElts = 8;  break;
2579       case X86::VPBROADCASTDZ128rm: NumElts = 4;  break;
2580       case X86::VPBROADCASTDZ256rm: NumElts = 8;  break;
2581       case X86::VPBROADCASTDZrm:    NumElts = 16; break;
2582       case X86::VPBROADCASTQrm:     NumElts = 2;  break;
2583       case X86::VPBROADCASTQYrm:    NumElts = 4;  break;
2584       case X86::VPBROADCASTQZ128rm: NumElts = 2;  break;
2585       case X86::VPBROADCASTQZ256rm: NumElts = 4;  break;
2586       case X86::VPBROADCASTQZrm:    NumElts = 8;  break;
2587       case X86::VPBROADCASTWrm:     NumElts = 8;  break;
2588       case X86::VPBROADCASTWYrm:    NumElts = 16; break;
2589       case X86::VPBROADCASTWZ128rm: NumElts = 8;  break;
2590       case X86::VPBROADCASTWZ256rm: NumElts = 16; break;
2591       case X86::VPBROADCASTWZrm:    NumElts = 32; break;
2592       }
2593 
2594       std::string Comment;
2595       raw_string_ostream CS(Comment);
2596       const MachineOperand &DstOp = MI->getOperand(0);
2597       CS << X86ATTInstPrinter::getRegisterName(DstOp.getReg()) << " = ";
2598       CS << "[";
2599       for (int i = 0; i != NumElts; ++i) {
2600         if (i != 0)
2601           CS << ",";
2602         printConstant(C, CS);
2603       }
2604       CS << "]";
2605       OutStreamer.AddComment(CS.str());
2606     }
2607   }
2608 }
2609 
2610 void X86AsmPrinter::emitInstruction(const MachineInstr *MI) {
2611   X86MCInstLower MCInstLowering(*MF, *this);
2612   const X86RegisterInfo *RI =
2613       MF->getSubtarget<X86Subtarget>().getRegisterInfo();
2614 
2615   // Add a comment about EVEX-2-VEX compression for AVX-512 instrs that
2616   // are compressed from EVEX encoding to VEX encoding.
2617   if (TM.Options.MCOptions.ShowMCEncoding) {
2618     if (MI->getAsmPrinterFlags() & X86::AC_EVEX_2_VEX)
2619       OutStreamer->AddComment("EVEX TO VEX Compression ", false);
2620   }
2621 
2622   // Add comments for values loaded from constant pool.
2623   if (OutStreamer->isVerboseAsm())
2624     addConstantComments(MI, *OutStreamer);
2625 
2626   switch (MI->getOpcode()) {
2627   case TargetOpcode::DBG_VALUE:
2628     llvm_unreachable("Should be handled target independently");
2629 
2630   // Emit nothing here but a comment if we can.
2631   case X86::Int_MemBarrier:
2632     OutStreamer->emitRawComment("MEMBARRIER");
2633     return;
2634 
2635   case X86::EH_RETURN:
2636   case X86::EH_RETURN64: {
2637     // Lower these as normal, but add some comments.
2638     Register Reg = MI->getOperand(0).getReg();
2639     OutStreamer->AddComment(StringRef("eh_return, addr: %") +
2640                             X86ATTInstPrinter::getRegisterName(Reg));
2641     break;
2642   }
2643   case X86::CLEANUPRET: {
2644     // Lower these as normal, but add some comments.
2645     OutStreamer->AddComment("CLEANUPRET");
2646     break;
2647   }
2648 
2649   case X86::CATCHRET: {
2650     // Lower these as normal, but add some comments.
2651     OutStreamer->AddComment("CATCHRET");
2652     break;
2653   }
2654 
2655   case X86::ENDBR32:
2656   case X86::ENDBR64: {
2657     // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
2658     // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
2659     // non-empty. If MI is the initial ENDBR, place the
2660     // __patchable_function_entries label after ENDBR.
2661     if (CurrentPatchableFunctionEntrySym &&
2662         CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
2663         MI == &MF->front().front()) {
2664       MCInst Inst;
2665       MCInstLowering.Lower(MI, Inst);
2666       EmitAndCountInstruction(Inst);
2667       CurrentPatchableFunctionEntrySym = createTempSymbol("patch");
2668       OutStreamer->emitLabel(CurrentPatchableFunctionEntrySym);
2669       return;
2670     }
2671     break;
2672   }
2673 
2674   case X86::TAILJMPr:
2675   case X86::TAILJMPm:
2676   case X86::TAILJMPd:
2677   case X86::TAILJMPd_CC:
2678   case X86::TAILJMPr64:
2679   case X86::TAILJMPm64:
2680   case X86::TAILJMPd64:
2681   case X86::TAILJMPd64_CC:
2682   case X86::TAILJMPr64_REX:
2683   case X86::TAILJMPm64_REX:
2684     // Lower these as normal, but add some comments.
2685     OutStreamer->AddComment("TAILCALL");
2686     break;
2687 
2688   case X86::TLS_addr32:
2689   case X86::TLS_addr64:
2690   case X86::TLS_addrX32:
2691   case X86::TLS_base_addr32:
2692   case X86::TLS_base_addr64:
2693   case X86::TLS_base_addrX32:
2694     return LowerTlsAddr(MCInstLowering, *MI);
2695 
2696   case X86::MOVPC32r: {
2697     // This is a pseudo op for a two instruction sequence with a label, which
2698     // looks like:
2699     //     call "L1$pb"
2700     // "L1$pb":
2701     //     popl %esi
2702 
2703     // Emit the call.
2704     MCSymbol *PICBase = MF->getPICBaseSymbol();
2705     // FIXME: We would like an efficient form for this, so we don't have to do a
2706     // lot of extra uniquing.
2707     EmitAndCountInstruction(
2708         MCInstBuilder(X86::CALLpcrel32)
2709             .addExpr(MCSymbolRefExpr::create(PICBase, OutContext)));
2710 
2711     const X86FrameLowering *FrameLowering =
2712         MF->getSubtarget<X86Subtarget>().getFrameLowering();
2713     bool hasFP = FrameLowering->hasFP(*MF);
2714 
2715     // TODO: This is needed only if we require precise CFA.
2716     bool HasActiveDwarfFrame = OutStreamer->getNumFrameInfos() &&
2717                                !OutStreamer->getDwarfFrameInfos().back().End;
2718 
2719     int stackGrowth = -RI->getSlotSize();
2720 
2721     if (HasActiveDwarfFrame && !hasFP) {
2722       OutStreamer->emitCFIAdjustCfaOffset(-stackGrowth);
2723     }
2724 
2725     // Emit the label.
2726     OutStreamer->emitLabel(PICBase);
2727 
2728     // popl $reg
2729     EmitAndCountInstruction(
2730         MCInstBuilder(X86::POP32r).addReg(MI->getOperand(0).getReg()));
2731 
2732     if (HasActiveDwarfFrame && !hasFP) {
2733       OutStreamer->emitCFIAdjustCfaOffset(stackGrowth);
2734     }
2735     return;
2736   }
2737 
2738   case X86::ADD32ri: {
2739     // Lower the MO_GOT_ABSOLUTE_ADDRESS form of ADD32ri.
2740     if (MI->getOperand(2).getTargetFlags() != X86II::MO_GOT_ABSOLUTE_ADDRESS)
2741       break;
2742 
2743     // Okay, we have something like:
2744     //  EAX = ADD32ri EAX, MO_GOT_ABSOLUTE_ADDRESS(@MYGLOBAL)
2745 
2746     // For this, we want to print something like:
2747     //   MYGLOBAL + (. - PICBASE)
2748     // However, we can't generate a ".", so just emit a new label here and refer
2749     // to it.
2750     MCSymbol *DotSym = OutContext.createTempSymbol();
2751     OutStreamer->emitLabel(DotSym);
2752 
2753     // Now that we have emitted the label, lower the complex operand expression.
2754     MCSymbol *OpSym = MCInstLowering.GetSymbolFromOperand(MI->getOperand(2));
2755 
2756     const MCExpr *DotExpr = MCSymbolRefExpr::create(DotSym, OutContext);
2757     const MCExpr *PICBase =
2758         MCSymbolRefExpr::create(MF->getPICBaseSymbol(), OutContext);
2759     DotExpr = MCBinaryExpr::createSub(DotExpr, PICBase, OutContext);
2760 
2761     DotExpr = MCBinaryExpr::createAdd(
2762         MCSymbolRefExpr::create(OpSym, OutContext), DotExpr, OutContext);
2763 
2764     EmitAndCountInstruction(MCInstBuilder(X86::ADD32ri)
2765                                 .addReg(MI->getOperand(0).getReg())
2766                                 .addReg(MI->getOperand(1).getReg())
2767                                 .addExpr(DotExpr));
2768     return;
2769   }
2770   case TargetOpcode::STATEPOINT:
2771     return LowerSTATEPOINT(*MI, MCInstLowering);
2772 
2773   case TargetOpcode::FAULTING_OP:
2774     return LowerFAULTING_OP(*MI, MCInstLowering);
2775 
2776   case TargetOpcode::FENTRY_CALL:
2777     return LowerFENTRY_CALL(*MI, MCInstLowering);
2778 
2779   case TargetOpcode::PATCHABLE_OP:
2780     return LowerPATCHABLE_OP(*MI, MCInstLowering);
2781 
2782   case TargetOpcode::STACKMAP:
2783     return LowerSTACKMAP(*MI);
2784 
2785   case TargetOpcode::PATCHPOINT:
2786     return LowerPATCHPOINT(*MI, MCInstLowering);
2787 
2788   case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
2789     return LowerPATCHABLE_FUNCTION_ENTER(*MI, MCInstLowering);
2790 
2791   case TargetOpcode::PATCHABLE_RET:
2792     return LowerPATCHABLE_RET(*MI, MCInstLowering);
2793 
2794   case TargetOpcode::PATCHABLE_TAIL_CALL:
2795     return LowerPATCHABLE_TAIL_CALL(*MI, MCInstLowering);
2796 
2797   case TargetOpcode::PATCHABLE_EVENT_CALL:
2798     return LowerPATCHABLE_EVENT_CALL(*MI, MCInstLowering);
2799 
2800   case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
2801     return LowerPATCHABLE_TYPED_EVENT_CALL(*MI, MCInstLowering);
2802 
2803   case X86::MORESTACK_RET:
2804     EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget)));
2805     return;
2806 
2807   case X86::ASAN_CHECK_MEMACCESS:
2808     return LowerASAN_CHECK_MEMACCESS(*MI);
2809 
2810   case X86::MORESTACK_RET_RESTORE_R10:
2811     // Return, then restore R10.
2812     EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget)));
2813     EmitAndCountInstruction(
2814         MCInstBuilder(X86::MOV64rr).addReg(X86::R10).addReg(X86::RAX));
2815     return;
2816 
2817   case X86::SEH_PushReg:
2818   case X86::SEH_SaveReg:
2819   case X86::SEH_SaveXMM:
2820   case X86::SEH_StackAlloc:
2821   case X86::SEH_StackAlign:
2822   case X86::SEH_SetFrame:
2823   case X86::SEH_PushFrame:
2824   case X86::SEH_EndPrologue:
2825     EmitSEHInstruction(MI);
2826     return;
2827 
2828   case X86::SEH_Epilogue: {
2829     assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
2830     MachineBasicBlock::const_iterator MBBI(MI);
2831     // Check if preceded by a call and emit nop if so.
2832     for (MBBI = PrevCrossBBInst(MBBI);
2833          MBBI != MachineBasicBlock::const_iterator();
2834          MBBI = PrevCrossBBInst(MBBI)) {
2835       // Conservatively assume that pseudo instructions don't emit code and keep
2836       // looking for a call. We may emit an unnecessary nop in some cases.
2837       if (!MBBI->isPseudo()) {
2838         if (MBBI->isCall())
2839           EmitAndCountInstruction(MCInstBuilder(X86::NOOP));
2840         break;
2841       }
2842     }
2843     return;
2844   }
2845   case X86::UBSAN_UD1:
2846     EmitAndCountInstruction(MCInstBuilder(X86::UD1Lm)
2847                                 .addReg(X86::EAX)
2848                                 .addReg(X86::EAX)
2849                                 .addImm(1)
2850                                 .addReg(X86::NoRegister)
2851                                 .addImm(MI->getOperand(0).getImm())
2852                                 .addReg(X86::NoRegister));
2853     return;
2854   }
2855 
2856   MCInst TmpInst;
2857   MCInstLowering.Lower(MI, TmpInst);
2858 
2859   // Stackmap shadows cannot include branch targets, so we can count the bytes
2860   // in a call towards the shadow, but must ensure that the no thread returns
2861   // in to the stackmap shadow.  The only way to achieve this is if the call
2862   // is at the end of the shadow.
2863   if (MI->isCall()) {
2864     // Count then size of the call towards the shadow
2865     SMShadowTracker.count(TmpInst, getSubtargetInfo(), CodeEmitter.get());
2866     // Then flush the shadow so that we fill with nops before the call, not
2867     // after it.
2868     SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo());
2869     // Then emit the call
2870     OutStreamer->emitInstruction(TmpInst, getSubtargetInfo());
2871     return;
2872   }
2873 
2874   EmitAndCountInstruction(TmpInst);
2875 }
2876