1 //===-- X86MCInstLower.cpp - Convert X86 MachineInstr to an MCInst --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains code to lower X86 MachineInstrs to their corresponding
10 // MCInst records.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "MCTargetDesc/X86ATTInstPrinter.h"
15 #include "MCTargetDesc/X86BaseInfo.h"
16 #include "MCTargetDesc/X86InstComments.h"
17 #include "MCTargetDesc/X86ShuffleDecode.h"
18 #include "MCTargetDesc/X86TargetStreamer.h"
19 #include "X86AsmPrinter.h"
20 #include "X86RegisterInfo.h"
21 #include "X86ShuffleDecodeConstantPool.h"
22 #include "X86Subtarget.h"
23 #include "llvm/ADT/Optional.h"
24 #include "llvm/ADT/SmallString.h"
25 #include "llvm/ADT/iterator_range.h"
26 #include "llvm/CodeGen/MachineConstantPool.h"
27 #include "llvm/CodeGen/MachineFunction.h"
28 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
29 #include "llvm/CodeGen/MachineOperand.h"
30 #include "llvm/CodeGen/StackMaps.h"
31 #include "llvm/IR/DataLayout.h"
32 #include "llvm/IR/GlobalValue.h"
33 #include "llvm/IR/Mangler.h"
34 #include "llvm/MC/MCAsmInfo.h"
35 #include "llvm/MC/MCCodeEmitter.h"
36 #include "llvm/MC/MCContext.h"
37 #include "llvm/MC/MCExpr.h"
38 #include "llvm/MC/MCFixup.h"
39 #include "llvm/MC/MCInst.h"
40 #include "llvm/MC/MCInstBuilder.h"
41 #include "llvm/MC/MCSection.h"
42 #include "llvm/MC/MCSectionELF.h"
43 #include "llvm/MC/MCStreamer.h"
44 #include "llvm/MC/MCSymbol.h"
45 #include "llvm/MC/MCSymbolELF.h"
46 #include "llvm/Support/TargetRegistry.h"
47 #include "llvm/Target/TargetLoweringObjectFile.h"
48 #include "llvm/Target/TargetMachine.h"
49 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
50 #include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
51 
52 using namespace llvm;
53 
54 namespace {
55 
56 /// X86MCInstLower - This class is used to lower an MachineInstr into an MCInst.
57 class X86MCInstLower {
58   MCContext &Ctx;
59   const MachineFunction &MF;
60   const TargetMachine &TM;
61   const MCAsmInfo &MAI;
62   X86AsmPrinter &AsmPrinter;
63 
64 public:
65   X86MCInstLower(const MachineFunction &MF, X86AsmPrinter &asmprinter);
66 
67   Optional<MCOperand> LowerMachineOperand(const MachineInstr *MI,
68                                           const MachineOperand &MO) const;
69   void Lower(const MachineInstr *MI, MCInst &OutMI) const;
70 
71   MCSymbol *GetSymbolFromOperand(const MachineOperand &MO) const;
72   MCOperand LowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym) const;
73 
74 private:
75   MachineModuleInfoMachO &getMachOMMI() const;
76 };
77 
78 } // end anonymous namespace
79 
80 /// A RAII helper which defines a region of instructions which can't have
81 /// padding added between them for correctness.
82 struct NoAutoPaddingScope {
83   MCStreamer &OS;
84   const bool OldAllowAutoPadding;
85   NoAutoPaddingScope(MCStreamer &OS)
86       : OS(OS), OldAllowAutoPadding(OS.getAllowAutoPadding()) {
87     changeAndComment(false);
88   }
89   ~NoAutoPaddingScope() { changeAndComment(OldAllowAutoPadding); }
90   void changeAndComment(bool b) {
91     if (b == OS.getAllowAutoPadding())
92       return;
93     OS.setAllowAutoPadding(b);
94     if (b)
95       OS.emitRawComment("autopadding");
96     else
97       OS.emitRawComment("noautopadding");
98   }
99 };
100 
101 // Emit a minimal sequence of nops spanning NumBytes bytes.
102 static void emitX86Nops(MCStreamer &OS, unsigned NumBytes,
103                         const X86Subtarget *Subtarget);
104 
105 void X86AsmPrinter::StackMapShadowTracker::count(MCInst &Inst,
106                                                  const MCSubtargetInfo &STI,
107                                                  MCCodeEmitter *CodeEmitter) {
108   if (InShadow) {
109     SmallString<256> Code;
110     SmallVector<MCFixup, 4> Fixups;
111     raw_svector_ostream VecOS(Code);
112     CodeEmitter->encodeInstruction(Inst, VecOS, Fixups, STI);
113     CurrentShadowSize += Code.size();
114     if (CurrentShadowSize >= RequiredShadowSize)
115       InShadow = false; // The shadow is big enough. Stop counting.
116   }
117 }
118 
119 void X86AsmPrinter::StackMapShadowTracker::emitShadowPadding(
120     MCStreamer &OutStreamer, const MCSubtargetInfo &STI) {
121   if (InShadow && CurrentShadowSize < RequiredShadowSize) {
122     InShadow = false;
123     emitX86Nops(OutStreamer, RequiredShadowSize - CurrentShadowSize,
124                 &MF->getSubtarget<X86Subtarget>());
125   }
126 }
127 
128 void X86AsmPrinter::EmitAndCountInstruction(MCInst &Inst) {
129   OutStreamer->emitInstruction(Inst, getSubtargetInfo());
130   SMShadowTracker.count(Inst, getSubtargetInfo(), CodeEmitter.get());
131 }
132 
133 X86MCInstLower::X86MCInstLower(const MachineFunction &mf,
134                                X86AsmPrinter &asmprinter)
135     : Ctx(mf.getContext()), MF(mf), TM(mf.getTarget()), MAI(*TM.getMCAsmInfo()),
136       AsmPrinter(asmprinter) {}
137 
138 MachineModuleInfoMachO &X86MCInstLower::getMachOMMI() const {
139   return MF.getMMI().getObjFileInfo<MachineModuleInfoMachO>();
140 }
141 
142 /// GetSymbolFromOperand - Lower an MO_GlobalAddress or MO_ExternalSymbol
143 /// operand to an MCSymbol.
144 MCSymbol *X86MCInstLower::GetSymbolFromOperand(const MachineOperand &MO) const {
145   const Triple &TT = TM.getTargetTriple();
146   if (MO.isGlobal() && TT.isOSBinFormatELF())
147     return AsmPrinter.getSymbolPreferLocal(*MO.getGlobal());
148 
149   const DataLayout &DL = MF.getDataLayout();
150   assert((MO.isGlobal() || MO.isSymbol() || MO.isMBB()) &&
151          "Isn't a symbol reference");
152 
153   MCSymbol *Sym = nullptr;
154   SmallString<128> Name;
155   StringRef Suffix;
156 
157   switch (MO.getTargetFlags()) {
158   case X86II::MO_DLLIMPORT:
159     // Handle dllimport linkage.
160     Name += "__imp_";
161     break;
162   case X86II::MO_COFFSTUB:
163     Name += ".refptr.";
164     break;
165   case X86II::MO_DARWIN_NONLAZY:
166   case X86II::MO_DARWIN_NONLAZY_PIC_BASE:
167     Suffix = "$non_lazy_ptr";
168     break;
169   }
170 
171   if (!Suffix.empty())
172     Name += DL.getPrivateGlobalPrefix();
173 
174   if (MO.isGlobal()) {
175     const GlobalValue *GV = MO.getGlobal();
176     AsmPrinter.getNameWithPrefix(Name, GV);
177   } else if (MO.isSymbol()) {
178     Mangler::getNameWithPrefix(Name, MO.getSymbolName(), DL);
179   } else if (MO.isMBB()) {
180     assert(Suffix.empty());
181     Sym = MO.getMBB()->getSymbol();
182   }
183 
184   Name += Suffix;
185   if (!Sym)
186     Sym = Ctx.getOrCreateSymbol(Name);
187 
188   // If the target flags on the operand changes the name of the symbol, do that
189   // before we return the symbol.
190   switch (MO.getTargetFlags()) {
191   default:
192     break;
193   case X86II::MO_COFFSTUB: {
194     MachineModuleInfoCOFF &MMICOFF =
195         MF.getMMI().getObjFileInfo<MachineModuleInfoCOFF>();
196     MachineModuleInfoImpl::StubValueTy &StubSym = MMICOFF.getGVStubEntry(Sym);
197     if (!StubSym.getPointer()) {
198       assert(MO.isGlobal() && "Extern symbol not handled yet");
199       StubSym = MachineModuleInfoImpl::StubValueTy(
200           AsmPrinter.getSymbol(MO.getGlobal()), true);
201     }
202     break;
203   }
204   case X86II::MO_DARWIN_NONLAZY:
205   case X86II::MO_DARWIN_NONLAZY_PIC_BASE: {
206     MachineModuleInfoImpl::StubValueTy &StubSym =
207         getMachOMMI().getGVStubEntry(Sym);
208     if (!StubSym.getPointer()) {
209       assert(MO.isGlobal() && "Extern symbol not handled yet");
210       StubSym = MachineModuleInfoImpl::StubValueTy(
211           AsmPrinter.getSymbol(MO.getGlobal()),
212           !MO.getGlobal()->hasInternalLinkage());
213     }
214     break;
215   }
216   }
217 
218   return Sym;
219 }
220 
221 MCOperand X86MCInstLower::LowerSymbolOperand(const MachineOperand &MO,
222                                              MCSymbol *Sym) const {
223   // FIXME: We would like an efficient form for this, so we don't have to do a
224   // lot of extra uniquing.
225   const MCExpr *Expr = nullptr;
226   MCSymbolRefExpr::VariantKind RefKind = MCSymbolRefExpr::VK_None;
227 
228   switch (MO.getTargetFlags()) {
229   default:
230     llvm_unreachable("Unknown target flag on GV operand");
231   case X86II::MO_NO_FLAG: // No flag.
232   // These affect the name of the symbol, not any suffix.
233   case X86II::MO_DARWIN_NONLAZY:
234   case X86II::MO_DLLIMPORT:
235   case X86II::MO_COFFSTUB:
236     break;
237 
238   case X86II::MO_TLVP:
239     RefKind = MCSymbolRefExpr::VK_TLVP;
240     break;
241   case X86II::MO_TLVP_PIC_BASE:
242     Expr = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_TLVP, Ctx);
243     // Subtract the pic base.
244     Expr = MCBinaryExpr::createSub(
245         Expr, MCSymbolRefExpr::create(MF.getPICBaseSymbol(), Ctx), Ctx);
246     break;
247   case X86II::MO_SECREL:
248     RefKind = MCSymbolRefExpr::VK_SECREL;
249     break;
250   case X86II::MO_TLSGD:
251     RefKind = MCSymbolRefExpr::VK_TLSGD;
252     break;
253   case X86II::MO_TLSLD:
254     RefKind = MCSymbolRefExpr::VK_TLSLD;
255     break;
256   case X86II::MO_TLSLDM:
257     RefKind = MCSymbolRefExpr::VK_TLSLDM;
258     break;
259   case X86II::MO_GOTTPOFF:
260     RefKind = MCSymbolRefExpr::VK_GOTTPOFF;
261     break;
262   case X86II::MO_INDNTPOFF:
263     RefKind = MCSymbolRefExpr::VK_INDNTPOFF;
264     break;
265   case X86II::MO_TPOFF:
266     RefKind = MCSymbolRefExpr::VK_TPOFF;
267     break;
268   case X86II::MO_DTPOFF:
269     RefKind = MCSymbolRefExpr::VK_DTPOFF;
270     break;
271   case X86II::MO_NTPOFF:
272     RefKind = MCSymbolRefExpr::VK_NTPOFF;
273     break;
274   case X86II::MO_GOTNTPOFF:
275     RefKind = MCSymbolRefExpr::VK_GOTNTPOFF;
276     break;
277   case X86II::MO_GOTPCREL:
278     RefKind = MCSymbolRefExpr::VK_GOTPCREL;
279     break;
280   case X86II::MO_GOT:
281     RefKind = MCSymbolRefExpr::VK_GOT;
282     break;
283   case X86II::MO_GOTOFF:
284     RefKind = MCSymbolRefExpr::VK_GOTOFF;
285     break;
286   case X86II::MO_PLT:
287     RefKind = MCSymbolRefExpr::VK_PLT;
288     break;
289   case X86II::MO_ABS8:
290     RefKind = MCSymbolRefExpr::VK_X86_ABS8;
291     break;
292   case X86II::MO_PIC_BASE_OFFSET:
293   case X86II::MO_DARWIN_NONLAZY_PIC_BASE:
294     Expr = MCSymbolRefExpr::create(Sym, Ctx);
295     // Subtract the pic base.
296     Expr = MCBinaryExpr::createSub(
297         Expr, MCSymbolRefExpr::create(MF.getPICBaseSymbol(), Ctx), Ctx);
298     if (MO.isJTI()) {
299       assert(MAI.doesSetDirectiveSuppressReloc());
300       // If .set directive is supported, use it to reduce the number of
301       // relocations the assembler will generate for differences between
302       // local labels. This is only safe when the symbols are in the same
303       // section so we are restricting it to jumptable references.
304       MCSymbol *Label = Ctx.createTempSymbol();
305       AsmPrinter.OutStreamer->emitAssignment(Label, Expr);
306       Expr = MCSymbolRefExpr::create(Label, Ctx);
307     }
308     break;
309   }
310 
311   if (!Expr)
312     Expr = MCSymbolRefExpr::create(Sym, RefKind, Ctx);
313 
314   if (!MO.isJTI() && !MO.isMBB() && MO.getOffset())
315     Expr = MCBinaryExpr::createAdd(
316         Expr, MCConstantExpr::create(MO.getOffset(), Ctx), Ctx);
317   return MCOperand::createExpr(Expr);
318 }
319 
320 /// Simplify FOO $imm, %{al,ax,eax,rax} to FOO $imm, for instruction with
321 /// a short fixed-register form.
322 static void SimplifyShortImmForm(MCInst &Inst, unsigned Opcode) {
323   unsigned ImmOp = Inst.getNumOperands() - 1;
324   assert(Inst.getOperand(0).isReg() &&
325          (Inst.getOperand(ImmOp).isImm() || Inst.getOperand(ImmOp).isExpr()) &&
326          ((Inst.getNumOperands() == 3 && Inst.getOperand(1).isReg() &&
327            Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg()) ||
328           Inst.getNumOperands() == 2) &&
329          "Unexpected instruction!");
330 
331   // Check whether the destination register can be fixed.
332   unsigned Reg = Inst.getOperand(0).getReg();
333   if (Reg != X86::AL && Reg != X86::AX && Reg != X86::EAX && Reg != X86::RAX)
334     return;
335 
336   // If so, rewrite the instruction.
337   MCOperand Saved = Inst.getOperand(ImmOp);
338   Inst = MCInst();
339   Inst.setOpcode(Opcode);
340   Inst.addOperand(Saved);
341 }
342 
343 /// If a movsx instruction has a shorter encoding for the used register
344 /// simplify the instruction to use it instead.
345 static void SimplifyMOVSX(MCInst &Inst) {
346   unsigned NewOpcode = 0;
347   unsigned Op0 = Inst.getOperand(0).getReg(), Op1 = Inst.getOperand(1).getReg();
348   switch (Inst.getOpcode()) {
349   default:
350     llvm_unreachable("Unexpected instruction!");
351   case X86::MOVSX16rr8: // movsbw %al, %ax   --> cbtw
352     if (Op0 == X86::AX && Op1 == X86::AL)
353       NewOpcode = X86::CBW;
354     break;
355   case X86::MOVSX32rr16: // movswl %ax, %eax  --> cwtl
356     if (Op0 == X86::EAX && Op1 == X86::AX)
357       NewOpcode = X86::CWDE;
358     break;
359   case X86::MOVSX64rr32: // movslq %eax, %rax --> cltq
360     if (Op0 == X86::RAX && Op1 == X86::EAX)
361       NewOpcode = X86::CDQE;
362     break;
363   }
364 
365   if (NewOpcode != 0) {
366     Inst = MCInst();
367     Inst.setOpcode(NewOpcode);
368   }
369 }
370 
371 /// Simplify things like MOV32rm to MOV32o32a.
372 static void SimplifyShortMoveForm(X86AsmPrinter &Printer, MCInst &Inst,
373                                   unsigned Opcode) {
374   // Don't make these simplifications in 64-bit mode; other assemblers don't
375   // perform them because they make the code larger.
376   if (Printer.getSubtarget().is64Bit())
377     return;
378 
379   bool IsStore = Inst.getOperand(0).isReg() && Inst.getOperand(1).isReg();
380   unsigned AddrBase = IsStore;
381   unsigned RegOp = IsStore ? 0 : 5;
382   unsigned AddrOp = AddrBase + 3;
383   assert(
384       Inst.getNumOperands() == 6 && Inst.getOperand(RegOp).isReg() &&
385       Inst.getOperand(AddrBase + X86::AddrBaseReg).isReg() &&
386       Inst.getOperand(AddrBase + X86::AddrScaleAmt).isImm() &&
387       Inst.getOperand(AddrBase + X86::AddrIndexReg).isReg() &&
388       Inst.getOperand(AddrBase + X86::AddrSegmentReg).isReg() &&
389       (Inst.getOperand(AddrOp).isExpr() || Inst.getOperand(AddrOp).isImm()) &&
390       "Unexpected instruction!");
391 
392   // Check whether the destination register can be fixed.
393   unsigned Reg = Inst.getOperand(RegOp).getReg();
394   if (Reg != X86::AL && Reg != X86::AX && Reg != X86::EAX && Reg != X86::RAX)
395     return;
396 
397   // Check whether this is an absolute address.
398   // FIXME: We know TLVP symbol refs aren't, but there should be a better way
399   // to do this here.
400   bool Absolute = true;
401   if (Inst.getOperand(AddrOp).isExpr()) {
402     const MCExpr *MCE = Inst.getOperand(AddrOp).getExpr();
403     if (const MCSymbolRefExpr *SRE = dyn_cast<MCSymbolRefExpr>(MCE))
404       if (SRE->getKind() == MCSymbolRefExpr::VK_TLVP)
405         Absolute = false;
406   }
407 
408   if (Absolute &&
409       (Inst.getOperand(AddrBase + X86::AddrBaseReg).getReg() != 0 ||
410        Inst.getOperand(AddrBase + X86::AddrScaleAmt).getImm() != 1 ||
411        Inst.getOperand(AddrBase + X86::AddrIndexReg).getReg() != 0))
412     return;
413 
414   // If so, rewrite the instruction.
415   MCOperand Saved = Inst.getOperand(AddrOp);
416   MCOperand Seg = Inst.getOperand(AddrBase + X86::AddrSegmentReg);
417   Inst = MCInst();
418   Inst.setOpcode(Opcode);
419   Inst.addOperand(Saved);
420   Inst.addOperand(Seg);
421 }
422 
423 static unsigned getRetOpcode(const X86Subtarget &Subtarget) {
424   return Subtarget.is64Bit() ? X86::RETQ : X86::RETL;
425 }
426 
427 Optional<MCOperand>
428 X86MCInstLower::LowerMachineOperand(const MachineInstr *MI,
429                                     const MachineOperand &MO) const {
430   switch (MO.getType()) {
431   default:
432     MI->print(errs());
433     llvm_unreachable("unknown operand type");
434   case MachineOperand::MO_Register:
435     // Ignore all implicit register operands.
436     if (MO.isImplicit())
437       return None;
438     return MCOperand::createReg(MO.getReg());
439   case MachineOperand::MO_Immediate:
440     return MCOperand::createImm(MO.getImm());
441   case MachineOperand::MO_MachineBasicBlock:
442   case MachineOperand::MO_GlobalAddress:
443   case MachineOperand::MO_ExternalSymbol:
444     return LowerSymbolOperand(MO, GetSymbolFromOperand(MO));
445   case MachineOperand::MO_MCSymbol:
446     return LowerSymbolOperand(MO, MO.getMCSymbol());
447   case MachineOperand::MO_JumpTableIndex:
448     return LowerSymbolOperand(MO, AsmPrinter.GetJTISymbol(MO.getIndex()));
449   case MachineOperand::MO_ConstantPoolIndex:
450     return LowerSymbolOperand(MO, AsmPrinter.GetCPISymbol(MO.getIndex()));
451   case MachineOperand::MO_BlockAddress:
452     return LowerSymbolOperand(
453         MO, AsmPrinter.GetBlockAddressSymbol(MO.getBlockAddress()));
454   case MachineOperand::MO_RegisterMask:
455     // Ignore call clobbers.
456     return None;
457   }
458 }
459 
460 // Replace TAILJMP opcodes with their equivalent opcodes that have encoding
461 // information.
462 static unsigned convertTailJumpOpcode(unsigned Opcode) {
463   switch (Opcode) {
464   case X86::TAILJMPr:
465     Opcode = X86::JMP32r;
466     break;
467   case X86::TAILJMPm:
468     Opcode = X86::JMP32m;
469     break;
470   case X86::TAILJMPr64:
471     Opcode = X86::JMP64r;
472     break;
473   case X86::TAILJMPm64:
474     Opcode = X86::JMP64m;
475     break;
476   case X86::TAILJMPr64_REX:
477     Opcode = X86::JMP64r_REX;
478     break;
479   case X86::TAILJMPm64_REX:
480     Opcode = X86::JMP64m_REX;
481     break;
482   case X86::TAILJMPd:
483   case X86::TAILJMPd64:
484     Opcode = X86::JMP_1;
485     break;
486   case X86::TAILJMPd_CC:
487   case X86::TAILJMPd64_CC:
488     Opcode = X86::JCC_1;
489     break;
490   }
491 
492   return Opcode;
493 }
494 
495 void X86MCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
496   OutMI.setOpcode(MI->getOpcode());
497 
498   for (const MachineOperand &MO : MI->operands())
499     if (auto MaybeMCOp = LowerMachineOperand(MI, MO))
500       OutMI.addOperand(MaybeMCOp.getValue());
501 
502   // Handle a few special cases to eliminate operand modifiers.
503   switch (OutMI.getOpcode()) {
504   case X86::LEA64_32r:
505   case X86::LEA64r:
506   case X86::LEA16r:
507   case X86::LEA32r:
508     // LEA should have a segment register, but it must be empty.
509     assert(OutMI.getNumOperands() == 1 + X86::AddrNumOperands &&
510            "Unexpected # of LEA operands");
511     assert(OutMI.getOperand(1 + X86::AddrSegmentReg).getReg() == 0 &&
512            "LEA has segment specified!");
513     break;
514 
515   case X86::MULX32Hrr:
516   case X86::MULX32Hrm:
517   case X86::MULX64Hrr:
518   case X86::MULX64Hrm: {
519     // Turn into regular MULX by duplicating the destination.
520     unsigned NewOpc;
521     switch (OutMI.getOpcode()) {
522     default: llvm_unreachable("Invalid opcode");
523     case X86::MULX32Hrr: NewOpc = X86::MULX32rr; break;
524     case X86::MULX32Hrm: NewOpc = X86::MULX32rm; break;
525     case X86::MULX64Hrr: NewOpc = X86::MULX64rr; break;
526     case X86::MULX64Hrm: NewOpc = X86::MULX64rm; break;
527     }
528     OutMI.setOpcode(NewOpc);
529     // Duplicate the destination.
530     unsigned DestReg = OutMI.getOperand(0).getReg();
531     OutMI.insert(OutMI.begin(), MCOperand::createReg(DestReg));
532     break;
533   }
534 
535   // Commute operands to get a smaller encoding by using VEX.R instead of VEX.B
536   // if one of the registers is extended, but other isn't.
537   case X86::VMOVZPQILo2PQIrr:
538   case X86::VMOVAPDrr:
539   case X86::VMOVAPDYrr:
540   case X86::VMOVAPSrr:
541   case X86::VMOVAPSYrr:
542   case X86::VMOVDQArr:
543   case X86::VMOVDQAYrr:
544   case X86::VMOVDQUrr:
545   case X86::VMOVDQUYrr:
546   case X86::VMOVUPDrr:
547   case X86::VMOVUPDYrr:
548   case X86::VMOVUPSrr:
549   case X86::VMOVUPSYrr: {
550     if (!X86II::isX86_64ExtendedReg(OutMI.getOperand(0).getReg()) &&
551         X86II::isX86_64ExtendedReg(OutMI.getOperand(1).getReg())) {
552       unsigned NewOpc;
553       switch (OutMI.getOpcode()) {
554       default: llvm_unreachable("Invalid opcode");
555       case X86::VMOVZPQILo2PQIrr: NewOpc = X86::VMOVPQI2QIrr;   break;
556       case X86::VMOVAPDrr:        NewOpc = X86::VMOVAPDrr_REV;  break;
557       case X86::VMOVAPDYrr:       NewOpc = X86::VMOVAPDYrr_REV; break;
558       case X86::VMOVAPSrr:        NewOpc = X86::VMOVAPSrr_REV;  break;
559       case X86::VMOVAPSYrr:       NewOpc = X86::VMOVAPSYrr_REV; break;
560       case X86::VMOVDQArr:        NewOpc = X86::VMOVDQArr_REV;  break;
561       case X86::VMOVDQAYrr:       NewOpc = X86::VMOVDQAYrr_REV; break;
562       case X86::VMOVDQUrr:        NewOpc = X86::VMOVDQUrr_REV;  break;
563       case X86::VMOVDQUYrr:       NewOpc = X86::VMOVDQUYrr_REV; break;
564       case X86::VMOVUPDrr:        NewOpc = X86::VMOVUPDrr_REV;  break;
565       case X86::VMOVUPDYrr:       NewOpc = X86::VMOVUPDYrr_REV; break;
566       case X86::VMOVUPSrr:        NewOpc = X86::VMOVUPSrr_REV;  break;
567       case X86::VMOVUPSYrr:       NewOpc = X86::VMOVUPSYrr_REV; break;
568       }
569       OutMI.setOpcode(NewOpc);
570     }
571     break;
572   }
573   case X86::VMOVSDrr:
574   case X86::VMOVSSrr: {
575     if (!X86II::isX86_64ExtendedReg(OutMI.getOperand(0).getReg()) &&
576         X86II::isX86_64ExtendedReg(OutMI.getOperand(2).getReg())) {
577       unsigned NewOpc;
578       switch (OutMI.getOpcode()) {
579       default: llvm_unreachable("Invalid opcode");
580       case X86::VMOVSDrr: NewOpc = X86::VMOVSDrr_REV; break;
581       case X86::VMOVSSrr: NewOpc = X86::VMOVSSrr_REV; break;
582       }
583       OutMI.setOpcode(NewOpc);
584     }
585     break;
586   }
587 
588   case X86::VPCMPBZ128rmi:  case X86::VPCMPBZ128rmik:
589   case X86::VPCMPBZ128rri:  case X86::VPCMPBZ128rrik:
590   case X86::VPCMPBZ256rmi:  case X86::VPCMPBZ256rmik:
591   case X86::VPCMPBZ256rri:  case X86::VPCMPBZ256rrik:
592   case X86::VPCMPBZrmi:     case X86::VPCMPBZrmik:
593   case X86::VPCMPBZrri:     case X86::VPCMPBZrrik:
594   case X86::VPCMPDZ128rmi:  case X86::VPCMPDZ128rmik:
595   case X86::VPCMPDZ128rmib: case X86::VPCMPDZ128rmibk:
596   case X86::VPCMPDZ128rri:  case X86::VPCMPDZ128rrik:
597   case X86::VPCMPDZ256rmi:  case X86::VPCMPDZ256rmik:
598   case X86::VPCMPDZ256rmib: case X86::VPCMPDZ256rmibk:
599   case X86::VPCMPDZ256rri:  case X86::VPCMPDZ256rrik:
600   case X86::VPCMPDZrmi:     case X86::VPCMPDZrmik:
601   case X86::VPCMPDZrmib:    case X86::VPCMPDZrmibk:
602   case X86::VPCMPDZrri:     case X86::VPCMPDZrrik:
603   case X86::VPCMPQZ128rmi:  case X86::VPCMPQZ128rmik:
604   case X86::VPCMPQZ128rmib: case X86::VPCMPQZ128rmibk:
605   case X86::VPCMPQZ128rri:  case X86::VPCMPQZ128rrik:
606   case X86::VPCMPQZ256rmi:  case X86::VPCMPQZ256rmik:
607   case X86::VPCMPQZ256rmib: case X86::VPCMPQZ256rmibk:
608   case X86::VPCMPQZ256rri:  case X86::VPCMPQZ256rrik:
609   case X86::VPCMPQZrmi:     case X86::VPCMPQZrmik:
610   case X86::VPCMPQZrmib:    case X86::VPCMPQZrmibk:
611   case X86::VPCMPQZrri:     case X86::VPCMPQZrrik:
612   case X86::VPCMPWZ128rmi:  case X86::VPCMPWZ128rmik:
613   case X86::VPCMPWZ128rri:  case X86::VPCMPWZ128rrik:
614   case X86::VPCMPWZ256rmi:  case X86::VPCMPWZ256rmik:
615   case X86::VPCMPWZ256rri:  case X86::VPCMPWZ256rrik:
616   case X86::VPCMPWZrmi:     case X86::VPCMPWZrmik:
617   case X86::VPCMPWZrri:     case X86::VPCMPWZrrik: {
618     // Turn immediate 0 into the VPCMPEQ instruction.
619     if (OutMI.getOperand(OutMI.getNumOperands() - 1).getImm() == 0) {
620       unsigned NewOpc;
621       switch (OutMI.getOpcode()) {
622       default: llvm_unreachable("Invalid opcode");
623       case X86::VPCMPBZ128rmi:   NewOpc = X86::VPCMPEQBZ128rm;   break;
624       case X86::VPCMPBZ128rmik:  NewOpc = X86::VPCMPEQBZ128rmk;  break;
625       case X86::VPCMPBZ128rri:   NewOpc = X86::VPCMPEQBZ128rr;   break;
626       case X86::VPCMPBZ128rrik:  NewOpc = X86::VPCMPEQBZ128rrk;  break;
627       case X86::VPCMPBZ256rmi:   NewOpc = X86::VPCMPEQBZ256rm;   break;
628       case X86::VPCMPBZ256rmik:  NewOpc = X86::VPCMPEQBZ256rmk;  break;
629       case X86::VPCMPBZ256rri:   NewOpc = X86::VPCMPEQBZ256rr;   break;
630       case X86::VPCMPBZ256rrik:  NewOpc = X86::VPCMPEQBZ256rrk;  break;
631       case X86::VPCMPBZrmi:      NewOpc = X86::VPCMPEQBZrm;      break;
632       case X86::VPCMPBZrmik:     NewOpc = X86::VPCMPEQBZrmk;     break;
633       case X86::VPCMPBZrri:      NewOpc = X86::VPCMPEQBZrr;      break;
634       case X86::VPCMPBZrrik:     NewOpc = X86::VPCMPEQBZrrk;     break;
635       case X86::VPCMPDZ128rmi:   NewOpc = X86::VPCMPEQDZ128rm;   break;
636       case X86::VPCMPDZ128rmib:  NewOpc = X86::VPCMPEQDZ128rmb;  break;
637       case X86::VPCMPDZ128rmibk: NewOpc = X86::VPCMPEQDZ128rmbk; break;
638       case X86::VPCMPDZ128rmik:  NewOpc = X86::VPCMPEQDZ128rmk;  break;
639       case X86::VPCMPDZ128rri:   NewOpc = X86::VPCMPEQDZ128rr;   break;
640       case X86::VPCMPDZ128rrik:  NewOpc = X86::VPCMPEQDZ128rrk;  break;
641       case X86::VPCMPDZ256rmi:   NewOpc = X86::VPCMPEQDZ256rm;   break;
642       case X86::VPCMPDZ256rmib:  NewOpc = X86::VPCMPEQDZ256rmb;  break;
643       case X86::VPCMPDZ256rmibk: NewOpc = X86::VPCMPEQDZ256rmbk; break;
644       case X86::VPCMPDZ256rmik:  NewOpc = X86::VPCMPEQDZ256rmk;  break;
645       case X86::VPCMPDZ256rri:   NewOpc = X86::VPCMPEQDZ256rr;   break;
646       case X86::VPCMPDZ256rrik:  NewOpc = X86::VPCMPEQDZ256rrk;  break;
647       case X86::VPCMPDZrmi:      NewOpc = X86::VPCMPEQDZrm;      break;
648       case X86::VPCMPDZrmib:     NewOpc = X86::VPCMPEQDZrmb;     break;
649       case X86::VPCMPDZrmibk:    NewOpc = X86::VPCMPEQDZrmbk;    break;
650       case X86::VPCMPDZrmik:     NewOpc = X86::VPCMPEQDZrmk;     break;
651       case X86::VPCMPDZrri:      NewOpc = X86::VPCMPEQDZrr;      break;
652       case X86::VPCMPDZrrik:     NewOpc = X86::VPCMPEQDZrrk;     break;
653       case X86::VPCMPQZ128rmi:   NewOpc = X86::VPCMPEQQZ128rm;   break;
654       case X86::VPCMPQZ128rmib:  NewOpc = X86::VPCMPEQQZ128rmb;  break;
655       case X86::VPCMPQZ128rmibk: NewOpc = X86::VPCMPEQQZ128rmbk; break;
656       case X86::VPCMPQZ128rmik:  NewOpc = X86::VPCMPEQQZ128rmk;  break;
657       case X86::VPCMPQZ128rri:   NewOpc = X86::VPCMPEQQZ128rr;   break;
658       case X86::VPCMPQZ128rrik:  NewOpc = X86::VPCMPEQQZ128rrk;  break;
659       case X86::VPCMPQZ256rmi:   NewOpc = X86::VPCMPEQQZ256rm;   break;
660       case X86::VPCMPQZ256rmib:  NewOpc = X86::VPCMPEQQZ256rmb;  break;
661       case X86::VPCMPQZ256rmibk: NewOpc = X86::VPCMPEQQZ256rmbk; break;
662       case X86::VPCMPQZ256rmik:  NewOpc = X86::VPCMPEQQZ256rmk;  break;
663       case X86::VPCMPQZ256rri:   NewOpc = X86::VPCMPEQQZ256rr;   break;
664       case X86::VPCMPQZ256rrik:  NewOpc = X86::VPCMPEQQZ256rrk;  break;
665       case X86::VPCMPQZrmi:      NewOpc = X86::VPCMPEQQZrm;      break;
666       case X86::VPCMPQZrmib:     NewOpc = X86::VPCMPEQQZrmb;     break;
667       case X86::VPCMPQZrmibk:    NewOpc = X86::VPCMPEQQZrmbk;    break;
668       case X86::VPCMPQZrmik:     NewOpc = X86::VPCMPEQQZrmk;     break;
669       case X86::VPCMPQZrri:      NewOpc = X86::VPCMPEQQZrr;      break;
670       case X86::VPCMPQZrrik:     NewOpc = X86::VPCMPEQQZrrk;     break;
671       case X86::VPCMPWZ128rmi:   NewOpc = X86::VPCMPEQWZ128rm;   break;
672       case X86::VPCMPWZ128rmik:  NewOpc = X86::VPCMPEQWZ128rmk;  break;
673       case X86::VPCMPWZ128rri:   NewOpc = X86::VPCMPEQWZ128rr;   break;
674       case X86::VPCMPWZ128rrik:  NewOpc = X86::VPCMPEQWZ128rrk;  break;
675       case X86::VPCMPWZ256rmi:   NewOpc = X86::VPCMPEQWZ256rm;   break;
676       case X86::VPCMPWZ256rmik:  NewOpc = X86::VPCMPEQWZ256rmk;  break;
677       case X86::VPCMPWZ256rri:   NewOpc = X86::VPCMPEQWZ256rr;   break;
678       case X86::VPCMPWZ256rrik:  NewOpc = X86::VPCMPEQWZ256rrk;  break;
679       case X86::VPCMPWZrmi:      NewOpc = X86::VPCMPEQWZrm;      break;
680       case X86::VPCMPWZrmik:     NewOpc = X86::VPCMPEQWZrmk;     break;
681       case X86::VPCMPWZrri:      NewOpc = X86::VPCMPEQWZrr;      break;
682       case X86::VPCMPWZrrik:     NewOpc = X86::VPCMPEQWZrrk;     break;
683       }
684 
685       OutMI.setOpcode(NewOpc);
686       OutMI.erase(&OutMI.getOperand(OutMI.getNumOperands() - 1));
687       break;
688     }
689 
690     // Turn immediate 6 into the VPCMPGT instruction.
691     if (OutMI.getOperand(OutMI.getNumOperands() - 1).getImm() == 6) {
692       unsigned NewOpc;
693       switch (OutMI.getOpcode()) {
694       default: llvm_unreachable("Invalid opcode");
695       case X86::VPCMPBZ128rmi:   NewOpc = X86::VPCMPGTBZ128rm;   break;
696       case X86::VPCMPBZ128rmik:  NewOpc = X86::VPCMPGTBZ128rmk;  break;
697       case X86::VPCMPBZ128rri:   NewOpc = X86::VPCMPGTBZ128rr;   break;
698       case X86::VPCMPBZ128rrik:  NewOpc = X86::VPCMPGTBZ128rrk;  break;
699       case X86::VPCMPBZ256rmi:   NewOpc = X86::VPCMPGTBZ256rm;   break;
700       case X86::VPCMPBZ256rmik:  NewOpc = X86::VPCMPGTBZ256rmk;  break;
701       case X86::VPCMPBZ256rri:   NewOpc = X86::VPCMPGTBZ256rr;   break;
702       case X86::VPCMPBZ256rrik:  NewOpc = X86::VPCMPGTBZ256rrk;  break;
703       case X86::VPCMPBZrmi:      NewOpc = X86::VPCMPGTBZrm;      break;
704       case X86::VPCMPBZrmik:     NewOpc = X86::VPCMPGTBZrmk;     break;
705       case X86::VPCMPBZrri:      NewOpc = X86::VPCMPGTBZrr;      break;
706       case X86::VPCMPBZrrik:     NewOpc = X86::VPCMPGTBZrrk;     break;
707       case X86::VPCMPDZ128rmi:   NewOpc = X86::VPCMPGTDZ128rm;   break;
708       case X86::VPCMPDZ128rmib:  NewOpc = X86::VPCMPGTDZ128rmb;  break;
709       case X86::VPCMPDZ128rmibk: NewOpc = X86::VPCMPGTDZ128rmbk; break;
710       case X86::VPCMPDZ128rmik:  NewOpc = X86::VPCMPGTDZ128rmk;  break;
711       case X86::VPCMPDZ128rri:   NewOpc = X86::VPCMPGTDZ128rr;   break;
712       case X86::VPCMPDZ128rrik:  NewOpc = X86::VPCMPGTDZ128rrk;  break;
713       case X86::VPCMPDZ256rmi:   NewOpc = X86::VPCMPGTDZ256rm;   break;
714       case X86::VPCMPDZ256rmib:  NewOpc = X86::VPCMPGTDZ256rmb;  break;
715       case X86::VPCMPDZ256rmibk: NewOpc = X86::VPCMPGTDZ256rmbk; break;
716       case X86::VPCMPDZ256rmik:  NewOpc = X86::VPCMPGTDZ256rmk;  break;
717       case X86::VPCMPDZ256rri:   NewOpc = X86::VPCMPGTDZ256rr;   break;
718       case X86::VPCMPDZ256rrik:  NewOpc = X86::VPCMPGTDZ256rrk;  break;
719       case X86::VPCMPDZrmi:      NewOpc = X86::VPCMPGTDZrm;      break;
720       case X86::VPCMPDZrmib:     NewOpc = X86::VPCMPGTDZrmb;     break;
721       case X86::VPCMPDZrmibk:    NewOpc = X86::VPCMPGTDZrmbk;    break;
722       case X86::VPCMPDZrmik:     NewOpc = X86::VPCMPGTDZrmk;     break;
723       case X86::VPCMPDZrri:      NewOpc = X86::VPCMPGTDZrr;      break;
724       case X86::VPCMPDZrrik:     NewOpc = X86::VPCMPGTDZrrk;     break;
725       case X86::VPCMPQZ128rmi:   NewOpc = X86::VPCMPGTQZ128rm;   break;
726       case X86::VPCMPQZ128rmib:  NewOpc = X86::VPCMPGTQZ128rmb;  break;
727       case X86::VPCMPQZ128rmibk: NewOpc = X86::VPCMPGTQZ128rmbk; break;
728       case X86::VPCMPQZ128rmik:  NewOpc = X86::VPCMPGTQZ128rmk;  break;
729       case X86::VPCMPQZ128rri:   NewOpc = X86::VPCMPGTQZ128rr;   break;
730       case X86::VPCMPQZ128rrik:  NewOpc = X86::VPCMPGTQZ128rrk;  break;
731       case X86::VPCMPQZ256rmi:   NewOpc = X86::VPCMPGTQZ256rm;   break;
732       case X86::VPCMPQZ256rmib:  NewOpc = X86::VPCMPGTQZ256rmb;  break;
733       case X86::VPCMPQZ256rmibk: NewOpc = X86::VPCMPGTQZ256rmbk; break;
734       case X86::VPCMPQZ256rmik:  NewOpc = X86::VPCMPGTQZ256rmk;  break;
735       case X86::VPCMPQZ256rri:   NewOpc = X86::VPCMPGTQZ256rr;   break;
736       case X86::VPCMPQZ256rrik:  NewOpc = X86::VPCMPGTQZ256rrk;  break;
737       case X86::VPCMPQZrmi:      NewOpc = X86::VPCMPGTQZrm;      break;
738       case X86::VPCMPQZrmib:     NewOpc = X86::VPCMPGTQZrmb;     break;
739       case X86::VPCMPQZrmibk:    NewOpc = X86::VPCMPGTQZrmbk;    break;
740       case X86::VPCMPQZrmik:     NewOpc = X86::VPCMPGTQZrmk;     break;
741       case X86::VPCMPQZrri:      NewOpc = X86::VPCMPGTQZrr;      break;
742       case X86::VPCMPQZrrik:     NewOpc = X86::VPCMPGTQZrrk;     break;
743       case X86::VPCMPWZ128rmi:   NewOpc = X86::VPCMPGTWZ128rm;   break;
744       case X86::VPCMPWZ128rmik:  NewOpc = X86::VPCMPGTWZ128rmk;  break;
745       case X86::VPCMPWZ128rri:   NewOpc = X86::VPCMPGTWZ128rr;   break;
746       case X86::VPCMPWZ128rrik:  NewOpc = X86::VPCMPGTWZ128rrk;  break;
747       case X86::VPCMPWZ256rmi:   NewOpc = X86::VPCMPGTWZ256rm;   break;
748       case X86::VPCMPWZ256rmik:  NewOpc = X86::VPCMPGTWZ256rmk;  break;
749       case X86::VPCMPWZ256rri:   NewOpc = X86::VPCMPGTWZ256rr;   break;
750       case X86::VPCMPWZ256rrik:  NewOpc = X86::VPCMPGTWZ256rrk;  break;
751       case X86::VPCMPWZrmi:      NewOpc = X86::VPCMPGTWZrm;      break;
752       case X86::VPCMPWZrmik:     NewOpc = X86::VPCMPGTWZrmk;     break;
753       case X86::VPCMPWZrri:      NewOpc = X86::VPCMPGTWZrr;      break;
754       case X86::VPCMPWZrrik:     NewOpc = X86::VPCMPGTWZrrk;     break;
755       }
756 
757       OutMI.setOpcode(NewOpc);
758       OutMI.erase(&OutMI.getOperand(OutMI.getNumOperands() - 1));
759       break;
760     }
761 
762     break;
763   }
764 
765   // CALL64r, CALL64pcrel32 - These instructions used to have
766   // register inputs modeled as normal uses instead of implicit uses.  As such,
767   // they we used to truncate off all but the first operand (the callee). This
768   // issue seems to have been fixed at some point. This assert verifies that.
769   case X86::CALL64r:
770   case X86::CALL64pcrel32:
771     assert(OutMI.getNumOperands() == 1 && "Unexpected number of operands!");
772     break;
773 
774   case X86::EH_RETURN:
775   case X86::EH_RETURN64: {
776     OutMI = MCInst();
777     OutMI.setOpcode(getRetOpcode(AsmPrinter.getSubtarget()));
778     break;
779   }
780 
781   case X86::CLEANUPRET: {
782     // Replace CLEANUPRET with the appropriate RET.
783     OutMI = MCInst();
784     OutMI.setOpcode(getRetOpcode(AsmPrinter.getSubtarget()));
785     break;
786   }
787 
788   case X86::CATCHRET: {
789     // Replace CATCHRET with the appropriate RET.
790     const X86Subtarget &Subtarget = AsmPrinter.getSubtarget();
791     unsigned ReturnReg = Subtarget.is64Bit() ? X86::RAX : X86::EAX;
792     OutMI = MCInst();
793     OutMI.setOpcode(getRetOpcode(Subtarget));
794     OutMI.addOperand(MCOperand::createReg(ReturnReg));
795     break;
796   }
797 
798   // TAILJMPd, TAILJMPd64, TailJMPd_cc - Lower to the correct jump
799   // instruction.
800   case X86::TAILJMPr:
801   case X86::TAILJMPr64:
802   case X86::TAILJMPr64_REX:
803   case X86::TAILJMPd:
804   case X86::TAILJMPd64:
805     assert(OutMI.getNumOperands() == 1 && "Unexpected number of operands!");
806     OutMI.setOpcode(convertTailJumpOpcode(OutMI.getOpcode()));
807     break;
808 
809   case X86::TAILJMPd_CC:
810   case X86::TAILJMPd64_CC:
811     assert(OutMI.getNumOperands() == 2 && "Unexpected number of operands!");
812     OutMI.setOpcode(convertTailJumpOpcode(OutMI.getOpcode()));
813     break;
814 
815   case X86::TAILJMPm:
816   case X86::TAILJMPm64:
817   case X86::TAILJMPm64_REX:
818     assert(OutMI.getNumOperands() == X86::AddrNumOperands &&
819            "Unexpected number of operands!");
820     OutMI.setOpcode(convertTailJumpOpcode(OutMI.getOpcode()));
821     break;
822 
823   case X86::DEC16r:
824   case X86::DEC32r:
825   case X86::INC16r:
826   case X86::INC32r:
827     // If we aren't in 64-bit mode we can use the 1-byte inc/dec instructions.
828     if (!AsmPrinter.getSubtarget().is64Bit()) {
829       unsigned Opcode;
830       switch (OutMI.getOpcode()) {
831       default: llvm_unreachable("Invalid opcode");
832       case X86::DEC16r: Opcode = X86::DEC16r_alt; break;
833       case X86::DEC32r: Opcode = X86::DEC32r_alt; break;
834       case X86::INC16r: Opcode = X86::INC16r_alt; break;
835       case X86::INC32r: Opcode = X86::INC32r_alt; break;
836       }
837       OutMI.setOpcode(Opcode);
838     }
839     break;
840 
841   // We don't currently select the correct instruction form for instructions
842   // which have a short %eax, etc. form. Handle this by custom lowering, for
843   // now.
844   //
845   // Note, we are currently not handling the following instructions:
846   // MOV64ao8, MOV64o8a
847   // XCHG16ar, XCHG32ar, XCHG64ar
848   case X86::MOV8mr_NOREX:
849   case X86::MOV8mr:
850   case X86::MOV8rm_NOREX:
851   case X86::MOV8rm:
852   case X86::MOV16mr:
853   case X86::MOV16rm:
854   case X86::MOV32mr:
855   case X86::MOV32rm: {
856     unsigned NewOpc;
857     switch (OutMI.getOpcode()) {
858     default: llvm_unreachable("Invalid opcode");
859     case X86::MOV8mr_NOREX:
860     case X86::MOV8mr:  NewOpc = X86::MOV8o32a; break;
861     case X86::MOV8rm_NOREX:
862     case X86::MOV8rm:  NewOpc = X86::MOV8ao32; break;
863     case X86::MOV16mr: NewOpc = X86::MOV16o32a; break;
864     case X86::MOV16rm: NewOpc = X86::MOV16ao32; break;
865     case X86::MOV32mr: NewOpc = X86::MOV32o32a; break;
866     case X86::MOV32rm: NewOpc = X86::MOV32ao32; break;
867     }
868     SimplifyShortMoveForm(AsmPrinter, OutMI, NewOpc);
869     break;
870   }
871 
872   case X86::ADC8ri: case X86::ADC16ri: case X86::ADC32ri: case X86::ADC64ri32:
873   case X86::ADD8ri: case X86::ADD16ri: case X86::ADD32ri: case X86::ADD64ri32:
874   case X86::AND8ri: case X86::AND16ri: case X86::AND32ri: case X86::AND64ri32:
875   case X86::CMP8ri: case X86::CMP16ri: case X86::CMP32ri: case X86::CMP64ri32:
876   case X86::OR8ri:  case X86::OR16ri:  case X86::OR32ri:  case X86::OR64ri32:
877   case X86::SBB8ri: case X86::SBB16ri: case X86::SBB32ri: case X86::SBB64ri32:
878   case X86::SUB8ri: case X86::SUB16ri: case X86::SUB32ri: case X86::SUB64ri32:
879   case X86::TEST8ri:case X86::TEST16ri:case X86::TEST32ri:case X86::TEST64ri32:
880   case X86::XOR8ri: case X86::XOR16ri: case X86::XOR32ri: case X86::XOR64ri32: {
881     unsigned NewOpc;
882     switch (OutMI.getOpcode()) {
883     default: llvm_unreachable("Invalid opcode");
884     case X86::ADC8ri:     NewOpc = X86::ADC8i8;    break;
885     case X86::ADC16ri:    NewOpc = X86::ADC16i16;  break;
886     case X86::ADC32ri:    NewOpc = X86::ADC32i32;  break;
887     case X86::ADC64ri32:  NewOpc = X86::ADC64i32;  break;
888     case X86::ADD8ri:     NewOpc = X86::ADD8i8;    break;
889     case X86::ADD16ri:    NewOpc = X86::ADD16i16;  break;
890     case X86::ADD32ri:    NewOpc = X86::ADD32i32;  break;
891     case X86::ADD64ri32:  NewOpc = X86::ADD64i32;  break;
892     case X86::AND8ri:     NewOpc = X86::AND8i8;    break;
893     case X86::AND16ri:    NewOpc = X86::AND16i16;  break;
894     case X86::AND32ri:    NewOpc = X86::AND32i32;  break;
895     case X86::AND64ri32:  NewOpc = X86::AND64i32;  break;
896     case X86::CMP8ri:     NewOpc = X86::CMP8i8;    break;
897     case X86::CMP16ri:    NewOpc = X86::CMP16i16;  break;
898     case X86::CMP32ri:    NewOpc = X86::CMP32i32;  break;
899     case X86::CMP64ri32:  NewOpc = X86::CMP64i32;  break;
900     case X86::OR8ri:      NewOpc = X86::OR8i8;     break;
901     case X86::OR16ri:     NewOpc = X86::OR16i16;   break;
902     case X86::OR32ri:     NewOpc = X86::OR32i32;   break;
903     case X86::OR64ri32:   NewOpc = X86::OR64i32;   break;
904     case X86::SBB8ri:     NewOpc = X86::SBB8i8;    break;
905     case X86::SBB16ri:    NewOpc = X86::SBB16i16;  break;
906     case X86::SBB32ri:    NewOpc = X86::SBB32i32;  break;
907     case X86::SBB64ri32:  NewOpc = X86::SBB64i32;  break;
908     case X86::SUB8ri:     NewOpc = X86::SUB8i8;    break;
909     case X86::SUB16ri:    NewOpc = X86::SUB16i16;  break;
910     case X86::SUB32ri:    NewOpc = X86::SUB32i32;  break;
911     case X86::SUB64ri32:  NewOpc = X86::SUB64i32;  break;
912     case X86::TEST8ri:    NewOpc = X86::TEST8i8;   break;
913     case X86::TEST16ri:   NewOpc = X86::TEST16i16; break;
914     case X86::TEST32ri:   NewOpc = X86::TEST32i32; break;
915     case X86::TEST64ri32: NewOpc = X86::TEST64i32; break;
916     case X86::XOR8ri:     NewOpc = X86::XOR8i8;    break;
917     case X86::XOR16ri:    NewOpc = X86::XOR16i16;  break;
918     case X86::XOR32ri:    NewOpc = X86::XOR32i32;  break;
919     case X86::XOR64ri32:  NewOpc = X86::XOR64i32;  break;
920     }
921     SimplifyShortImmForm(OutMI, NewOpc);
922     break;
923   }
924 
925   // Try to shrink some forms of movsx.
926   case X86::MOVSX16rr8:
927   case X86::MOVSX32rr16:
928   case X86::MOVSX64rr32:
929     SimplifyMOVSX(OutMI);
930     break;
931 
932   case X86::VCMPPDrri:
933   case X86::VCMPPDYrri:
934   case X86::VCMPPSrri:
935   case X86::VCMPPSYrri:
936   case X86::VCMPSDrr:
937   case X86::VCMPSSrr: {
938     // Swap the operands if it will enable a 2 byte VEX encoding.
939     // FIXME: Change the immediate to improve opportunities?
940     if (!X86II::isX86_64ExtendedReg(OutMI.getOperand(1).getReg()) &&
941         X86II::isX86_64ExtendedReg(OutMI.getOperand(2).getReg())) {
942       unsigned Imm = MI->getOperand(3).getImm() & 0x7;
943       switch (Imm) {
944       default: break;
945       case 0x00: // EQUAL
946       case 0x03: // UNORDERED
947       case 0x04: // NOT EQUAL
948       case 0x07: // ORDERED
949         std::swap(OutMI.getOperand(1), OutMI.getOperand(2));
950         break;
951       }
952     }
953     break;
954   }
955 
956   case X86::VMOVHLPSrr:
957   case X86::VUNPCKHPDrr:
958     // These are not truly commutable so hide them from the default case.
959     break;
960 
961   default: {
962     // If the instruction is a commutable arithmetic instruction we might be
963     // able to commute the operands to get a 2 byte VEX prefix.
964     uint64_t TSFlags = MI->getDesc().TSFlags;
965     if (MI->getDesc().isCommutable() &&
966         (TSFlags & X86II::EncodingMask) == X86II::VEX &&
967         (TSFlags & X86II::OpMapMask) == X86II::TB &&
968         (TSFlags & X86II::FormMask) == X86II::MRMSrcReg &&
969         !(TSFlags & X86II::VEX_W) && (TSFlags & X86II::VEX_4V) &&
970         OutMI.getNumOperands() == 3) {
971       if (!X86II::isX86_64ExtendedReg(OutMI.getOperand(1).getReg()) &&
972           X86II::isX86_64ExtendedReg(OutMI.getOperand(2).getReg()))
973         std::swap(OutMI.getOperand(1), OutMI.getOperand(2));
974     }
975     break;
976   }
977   }
978 }
979 
980 void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
981                                  const MachineInstr &MI) {
982   NoAutoPaddingScope NoPadScope(*OutStreamer);
983   bool Is64Bits = MI.getOpcode() != X86::TLS_addr32 &&
984                   MI.getOpcode() != X86::TLS_base_addr32;
985   bool Is64BitsLP64 = MI.getOpcode() == X86::TLS_addr64 ||
986                       MI.getOpcode() == X86::TLS_base_addr64;
987   MCContext &Ctx = OutStreamer->getContext();
988 
989   MCSymbolRefExpr::VariantKind SRVK;
990   switch (MI.getOpcode()) {
991   case X86::TLS_addr32:
992   case X86::TLS_addr64:
993   case X86::TLS_addrX32:
994     SRVK = MCSymbolRefExpr::VK_TLSGD;
995     break;
996   case X86::TLS_base_addr32:
997     SRVK = MCSymbolRefExpr::VK_TLSLDM;
998     break;
999   case X86::TLS_base_addr64:
1000   case X86::TLS_base_addrX32:
1001     SRVK = MCSymbolRefExpr::VK_TLSLD;
1002     break;
1003   default:
1004     llvm_unreachable("unexpected opcode");
1005   }
1006 
1007   const MCSymbolRefExpr *Sym = MCSymbolRefExpr::create(
1008       MCInstLowering.GetSymbolFromOperand(MI.getOperand(3)), SRVK, Ctx);
1009 
1010   // As of binutils 2.32, ld has a bogus TLS relaxation error when the GD/LD
1011   // code sequence using R_X86_64_GOTPCREL (instead of R_X86_64_GOTPCRELX) is
1012   // attempted to be relaxed to IE/LE (binutils PR24784). Work around the bug by
1013   // only using GOT when GOTPCRELX is enabled.
1014   // TODO Delete the workaround when GOTPCRELX becomes commonplace.
1015   bool UseGot = MMI->getModule()->getRtLibUseGOT() &&
1016                 Ctx.getAsmInfo()->canRelaxRelocations();
1017 
1018   if (Is64Bits) {
1019     bool NeedsPadding = SRVK == MCSymbolRefExpr::VK_TLSGD;
1020     if (NeedsPadding && Is64BitsLP64)
1021       EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX));
1022     EmitAndCountInstruction(MCInstBuilder(X86::LEA64r)
1023                                 .addReg(X86::RDI)
1024                                 .addReg(X86::RIP)
1025                                 .addImm(1)
1026                                 .addReg(0)
1027                                 .addExpr(Sym)
1028                                 .addReg(0));
1029     const MCSymbol *TlsGetAddr = Ctx.getOrCreateSymbol("__tls_get_addr");
1030     if (NeedsPadding) {
1031       if (!UseGot)
1032         EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX));
1033       EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX));
1034       EmitAndCountInstruction(MCInstBuilder(X86::REX64_PREFIX));
1035     }
1036     if (UseGot) {
1037       const MCExpr *Expr = MCSymbolRefExpr::create(
1038           TlsGetAddr, MCSymbolRefExpr::VK_GOTPCREL, Ctx);
1039       EmitAndCountInstruction(MCInstBuilder(X86::CALL64m)
1040                                   .addReg(X86::RIP)
1041                                   .addImm(1)
1042                                   .addReg(0)
1043                                   .addExpr(Expr)
1044                                   .addReg(0));
1045     } else {
1046       EmitAndCountInstruction(
1047           MCInstBuilder(X86::CALL64pcrel32)
1048               .addExpr(MCSymbolRefExpr::create(TlsGetAddr,
1049                                                MCSymbolRefExpr::VK_PLT, Ctx)));
1050     }
1051   } else {
1052     if (SRVK == MCSymbolRefExpr::VK_TLSGD && !UseGot) {
1053       EmitAndCountInstruction(MCInstBuilder(X86::LEA32r)
1054                                   .addReg(X86::EAX)
1055                                   .addReg(0)
1056                                   .addImm(1)
1057                                   .addReg(X86::EBX)
1058                                   .addExpr(Sym)
1059                                   .addReg(0));
1060     } else {
1061       EmitAndCountInstruction(MCInstBuilder(X86::LEA32r)
1062                                   .addReg(X86::EAX)
1063                                   .addReg(X86::EBX)
1064                                   .addImm(1)
1065                                   .addReg(0)
1066                                   .addExpr(Sym)
1067                                   .addReg(0));
1068     }
1069 
1070     const MCSymbol *TlsGetAddr = Ctx.getOrCreateSymbol("___tls_get_addr");
1071     if (UseGot) {
1072       const MCExpr *Expr =
1073           MCSymbolRefExpr::create(TlsGetAddr, MCSymbolRefExpr::VK_GOT, Ctx);
1074       EmitAndCountInstruction(MCInstBuilder(X86::CALL32m)
1075                                   .addReg(X86::EBX)
1076                                   .addImm(1)
1077                                   .addReg(0)
1078                                   .addExpr(Expr)
1079                                   .addReg(0));
1080     } else {
1081       EmitAndCountInstruction(
1082           MCInstBuilder(X86::CALLpcrel32)
1083               .addExpr(MCSymbolRefExpr::create(TlsGetAddr,
1084                                                MCSymbolRefExpr::VK_PLT, Ctx)));
1085     }
1086   }
1087 }
1088 
1089 /// Emit the largest nop instruction smaller than or equal to \p NumBytes
1090 /// bytes.  Return the size of nop emitted.
1091 static unsigned emitNop(MCStreamer &OS, unsigned NumBytes,
1092                         const X86Subtarget *Subtarget) {
1093   // Determine the longest nop which can be efficiently decoded for the given
1094   // target cpu.  15-bytes is the longest single NOP instruction, but some
1095   // platforms can't decode the longest forms efficiently.
1096   unsigned MaxNopLength = 1;
1097   if (Subtarget->is64Bit()) {
1098     // FIXME: We can use NOOPL on 32-bit targets with FeatureNOPL, but the
1099     // IndexReg/BaseReg below need to be updated.
1100     if (Subtarget->hasFeature(X86::TuningFast7ByteNOP))
1101       MaxNopLength = 7;
1102     else if (Subtarget->hasFeature(X86::TuningFast15ByteNOP))
1103       MaxNopLength = 15;
1104     else if (Subtarget->hasFeature(X86::TuningFast11ByteNOP))
1105       MaxNopLength = 11;
1106     else
1107       MaxNopLength = 10;
1108   } if (Subtarget->is32Bit())
1109     MaxNopLength = 2;
1110 
1111   // Cap a single nop emission at the profitable value for the target
1112   NumBytes = std::min(NumBytes, MaxNopLength);
1113 
1114   unsigned NopSize;
1115   unsigned Opc, BaseReg, ScaleVal, IndexReg, Displacement, SegmentReg;
1116   IndexReg = Displacement = SegmentReg = 0;
1117   BaseReg = X86::RAX;
1118   ScaleVal = 1;
1119   switch (NumBytes) {
1120   case 0:
1121     llvm_unreachable("Zero nops?");
1122     break;
1123   case 1:
1124     NopSize = 1;
1125     Opc = X86::NOOP;
1126     break;
1127   case 2:
1128     NopSize = 2;
1129     Opc = X86::XCHG16ar;
1130     break;
1131   case 3:
1132     NopSize = 3;
1133     Opc = X86::NOOPL;
1134     break;
1135   case 4:
1136     NopSize = 4;
1137     Opc = X86::NOOPL;
1138     Displacement = 8;
1139     break;
1140   case 5:
1141     NopSize = 5;
1142     Opc = X86::NOOPL;
1143     Displacement = 8;
1144     IndexReg = X86::RAX;
1145     break;
1146   case 6:
1147     NopSize = 6;
1148     Opc = X86::NOOPW;
1149     Displacement = 8;
1150     IndexReg = X86::RAX;
1151     break;
1152   case 7:
1153     NopSize = 7;
1154     Opc = X86::NOOPL;
1155     Displacement = 512;
1156     break;
1157   case 8:
1158     NopSize = 8;
1159     Opc = X86::NOOPL;
1160     Displacement = 512;
1161     IndexReg = X86::RAX;
1162     break;
1163   case 9:
1164     NopSize = 9;
1165     Opc = X86::NOOPW;
1166     Displacement = 512;
1167     IndexReg = X86::RAX;
1168     break;
1169   default:
1170     NopSize = 10;
1171     Opc = X86::NOOPW;
1172     Displacement = 512;
1173     IndexReg = X86::RAX;
1174     SegmentReg = X86::CS;
1175     break;
1176   }
1177 
1178   unsigned NumPrefixes = std::min(NumBytes - NopSize, 5U);
1179   NopSize += NumPrefixes;
1180   for (unsigned i = 0; i != NumPrefixes; ++i)
1181     OS.emitBytes("\x66");
1182 
1183   switch (Opc) {
1184   default: llvm_unreachable("Unexpected opcode");
1185   case X86::NOOP:
1186     OS.emitInstruction(MCInstBuilder(Opc), *Subtarget);
1187     break;
1188   case X86::XCHG16ar:
1189     OS.emitInstruction(MCInstBuilder(Opc).addReg(X86::AX).addReg(X86::AX),
1190                        *Subtarget);
1191     break;
1192   case X86::NOOPL:
1193   case X86::NOOPW:
1194     OS.emitInstruction(MCInstBuilder(Opc)
1195                            .addReg(BaseReg)
1196                            .addImm(ScaleVal)
1197                            .addReg(IndexReg)
1198                            .addImm(Displacement)
1199                            .addReg(SegmentReg),
1200                        *Subtarget);
1201     break;
1202   }
1203   assert(NopSize <= NumBytes && "We overemitted?");
1204   return NopSize;
1205 }
1206 
1207 /// Emit the optimal amount of multi-byte nops on X86.
1208 static void emitX86Nops(MCStreamer &OS, unsigned NumBytes,
1209                         const X86Subtarget *Subtarget) {
1210   unsigned NopsToEmit = NumBytes;
1211   (void)NopsToEmit;
1212   while (NumBytes) {
1213     NumBytes -= emitNop(OS, NumBytes, Subtarget);
1214     assert(NopsToEmit >= NumBytes && "Emitted more than I asked for!");
1215   }
1216 }
1217 
1218 void X86AsmPrinter::LowerSTATEPOINT(const MachineInstr &MI,
1219                                     X86MCInstLower &MCIL) {
1220   assert(Subtarget->is64Bit() && "Statepoint currently only supports X86-64");
1221 
1222   NoAutoPaddingScope NoPadScope(*OutStreamer);
1223 
1224   StatepointOpers SOpers(&MI);
1225   if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
1226     emitX86Nops(*OutStreamer, PatchBytes, Subtarget);
1227   } else {
1228     // Lower call target and choose correct opcode
1229     const MachineOperand &CallTarget = SOpers.getCallTarget();
1230     MCOperand CallTargetMCOp;
1231     unsigned CallOpcode;
1232     switch (CallTarget.getType()) {
1233     case MachineOperand::MO_GlobalAddress:
1234     case MachineOperand::MO_ExternalSymbol:
1235       CallTargetMCOp = MCIL.LowerSymbolOperand(
1236           CallTarget, MCIL.GetSymbolFromOperand(CallTarget));
1237       CallOpcode = X86::CALL64pcrel32;
1238       // Currently, we only support relative addressing with statepoints.
1239       // Otherwise, we'll need a scratch register to hold the target
1240       // address.  You'll fail asserts during load & relocation if this
1241       // symbol is to far away. (TODO: support non-relative addressing)
1242       break;
1243     case MachineOperand::MO_Immediate:
1244       CallTargetMCOp = MCOperand::createImm(CallTarget.getImm());
1245       CallOpcode = X86::CALL64pcrel32;
1246       // Currently, we only support relative addressing with statepoints.
1247       // Otherwise, we'll need a scratch register to hold the target
1248       // immediate.  You'll fail asserts during load & relocation if this
1249       // address is to far away. (TODO: support non-relative addressing)
1250       break;
1251     case MachineOperand::MO_Register:
1252       // FIXME: Add retpoline support and remove this.
1253       if (Subtarget->useIndirectThunkCalls())
1254         report_fatal_error("Lowering register statepoints with thunks not "
1255                            "yet implemented.");
1256       CallTargetMCOp = MCOperand::createReg(CallTarget.getReg());
1257       CallOpcode = X86::CALL64r;
1258       break;
1259     default:
1260       llvm_unreachable("Unsupported operand type in statepoint call target");
1261       break;
1262     }
1263 
1264     // Emit call
1265     MCInst CallInst;
1266     CallInst.setOpcode(CallOpcode);
1267     CallInst.addOperand(CallTargetMCOp);
1268     OutStreamer->emitInstruction(CallInst, getSubtargetInfo());
1269   }
1270 
1271   // Record our statepoint node in the same section used by STACKMAP
1272   // and PATCHPOINT
1273   auto &Ctx = OutStreamer->getContext();
1274   MCSymbol *MILabel = Ctx.createTempSymbol();
1275   OutStreamer->emitLabel(MILabel);
1276   SM.recordStatepoint(*MILabel, MI);
1277 }
1278 
1279 void X86AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI,
1280                                      X86MCInstLower &MCIL) {
1281   // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
1282   //                  <opcode>, <operands>
1283 
1284   NoAutoPaddingScope NoPadScope(*OutStreamer);
1285 
1286   Register DefRegister = FaultingMI.getOperand(0).getReg();
1287   FaultMaps::FaultKind FK =
1288       static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(1).getImm());
1289   MCSymbol *HandlerLabel = FaultingMI.getOperand(2).getMBB()->getSymbol();
1290   unsigned Opcode = FaultingMI.getOperand(3).getImm();
1291   unsigned OperandsBeginIdx = 4;
1292 
1293   auto &Ctx = OutStreamer->getContext();
1294   MCSymbol *FaultingLabel = Ctx.createTempSymbol();
1295   OutStreamer->emitLabel(FaultingLabel);
1296 
1297   assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
1298   FM.recordFaultingOp(FK, FaultingLabel, HandlerLabel);
1299 
1300   MCInst MI;
1301   MI.setOpcode(Opcode);
1302 
1303   if (DefRegister != X86::NoRegister)
1304     MI.addOperand(MCOperand::createReg(DefRegister));
1305 
1306   for (auto I = FaultingMI.operands_begin() + OperandsBeginIdx,
1307             E = FaultingMI.operands_end();
1308        I != E; ++I)
1309     if (auto MaybeOperand = MCIL.LowerMachineOperand(&FaultingMI, *I))
1310       MI.addOperand(MaybeOperand.getValue());
1311 
1312   OutStreamer->AddComment("on-fault: " + HandlerLabel->getName());
1313   OutStreamer->emitInstruction(MI, getSubtargetInfo());
1314 }
1315 
1316 void X86AsmPrinter::LowerFENTRY_CALL(const MachineInstr &MI,
1317                                      X86MCInstLower &MCIL) {
1318   bool Is64Bits = Subtarget->is64Bit();
1319   MCContext &Ctx = OutStreamer->getContext();
1320   MCSymbol *fentry = Ctx.getOrCreateSymbol("__fentry__");
1321   const MCSymbolRefExpr *Op =
1322       MCSymbolRefExpr::create(fentry, MCSymbolRefExpr::VK_None, Ctx);
1323 
1324   EmitAndCountInstruction(
1325       MCInstBuilder(Is64Bits ? X86::CALL64pcrel32 : X86::CALLpcrel32)
1326           .addExpr(Op));
1327 }
1328 
1329 void X86AsmPrinter::LowerASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
1330   // FIXME: Make this work on non-ELF.
1331   if (!TM.getTargetTriple().isOSBinFormatELF()) {
1332     report_fatal_error("llvm.asan.check.memaccess only supported on ELF");
1333     return;
1334   }
1335 
1336   unsigned Reg = MI.getOperand(0).getReg().id();
1337   ASanAccessInfo AccessInfo(MI.getOperand(1).getImm());
1338 
1339   MCSymbol *&Sym =
1340       AsanMemaccessSymbols[AsanMemaccessTuple(Reg, AccessInfo.Packed)];
1341   if (!Sym) {
1342     std::string Name = AccessInfo.IsWrite ? "store" : "load";
1343     std::string SymName = "__asan_check_" + Name +
1344                           utostr(1ULL << AccessInfo.AccessSizeIndex) + "_rn" +
1345                           utostr(Reg);
1346     Sym = OutContext.getOrCreateSymbol(SymName);
1347   }
1348 
1349   EmitAndCountInstruction(
1350       MCInstBuilder(X86::CALL64pcrel32)
1351           .addExpr(MCSymbolRefExpr::create(Sym, OutContext)));
1352 }
1353 
1354 void X86AsmPrinter::emitAsanMemaccessPartial(Module &M, unsigned Reg,
1355                                              const ASanAccessInfo &AccessInfo,
1356                                              MCSubtargetInfo &STI) {
1357   assert(AccessInfo.AccessSizeIndex == 0 || AccessInfo.AccessSizeIndex == 1 ||
1358          AccessInfo.AccessSizeIndex == 2);
1359   assert(Reg != X86::R8);
1360 
1361   uint64_t ShadowBase;
1362   int MappingScale;
1363   bool OrShadowOffset;
1364   getAddressSanitizerParams(
1365       Triple(M.getTargetTriple()), M.getDataLayout().getPointerSizeInBits(),
1366       AccessInfo.CompileKernel, &ShadowBase, &MappingScale, &OrShadowOffset);
1367 
1368   OutStreamer->emitInstruction(
1369       MCInstBuilder(X86::MOV64rr).addReg(X86::R8).addReg(X86::NoRegister + Reg),
1370       STI);
1371   OutStreamer->emitInstruction(MCInstBuilder(X86::SHR64ri)
1372                                    .addReg(X86::R8)
1373                                    .addReg(X86::R8)
1374                                    .addImm(MappingScale),
1375                                STI);
1376   if (OrShadowOffset) {
1377     OutStreamer->emitInstruction(MCInstBuilder(X86::OR64ri32)
1378                                      .addReg(X86::R8)
1379                                      .addReg(X86::R8)
1380                                      .addImm(ShadowBase),
1381                                  STI);
1382     OutStreamer->emitInstruction(MCInstBuilder(X86::MOV8rm)
1383                                      .addReg(X86::R8B)
1384                                      .addReg(X86::R8)
1385                                      .addImm(1)
1386                                      .addReg(X86::NoRegister)
1387                                      .addImm(0)
1388                                      .addReg(X86::NoRegister),
1389                                  STI);
1390     OutStreamer->emitInstruction(
1391         MCInstBuilder(X86::TEST8rr).addReg(X86::R8B).addReg(X86::R8B), STI);
1392   } else {
1393     OutStreamer->emitInstruction(MCInstBuilder(X86::MOVSX32rm8)
1394                                      .addReg(X86::R8D)
1395                                      .addReg(X86::R8)
1396                                      .addImm(1)
1397                                      .addReg(X86::NoRegister)
1398                                      .addImm(ShadowBase)
1399                                      .addReg(X86::NoRegister),
1400                                  STI);
1401     OutStreamer->emitInstruction(
1402         MCInstBuilder(X86::TEST32rr).addReg(X86::R8D).addReg(X86::R8D), STI);
1403   }
1404   MCSymbol *AdditionalCheck = OutContext.createTempSymbol();
1405   OutStreamer->emitInstruction(
1406       MCInstBuilder(X86::JCC_1)
1407           .addExpr(MCSymbolRefExpr::create(AdditionalCheck, OutContext))
1408           .addImm(X86::COND_NE),
1409       STI);
1410   MCSymbol *ReturnSym = OutContext.createTempSymbol();
1411   OutStreamer->emitLabel(ReturnSym);
1412   OutStreamer->emitInstruction(MCInstBuilder(getRetOpcode(*Subtarget)), STI);
1413 
1414   // Shadow byte is non-zero so we need to perform additional checks.
1415   OutStreamer->emitLabel(AdditionalCheck);
1416   OutStreamer->emitInstruction(MCInstBuilder(X86::PUSH64r).addReg(X86::RCX),
1417                                STI);
1418   OutStreamer->emitInstruction(MCInstBuilder(X86::MOV64rr)
1419                                    .addReg(X86::RCX)
1420                                    .addReg(X86::NoRegister + Reg),
1421                                STI);
1422   const size_t Granularity = 1ULL << MappingScale;
1423   OutStreamer->emitInstruction(MCInstBuilder(X86::AND32ri8)
1424                                    .addReg(X86::NoRegister)
1425                                    .addReg(X86::ECX)
1426                                    .addImm(Granularity - 1),
1427                                STI);
1428   if (AccessInfo.AccessSizeIndex == 1) {
1429     OutStreamer->emitInstruction(MCInstBuilder(X86::ADD32ri8)
1430                                      .addReg(X86::NoRegister)
1431                                      .addReg(X86::ECX)
1432                                      .addImm(1),
1433                                  STI);
1434   } else if (AccessInfo.AccessSizeIndex == 2) {
1435     OutStreamer->emitInstruction(MCInstBuilder(X86::ADD32ri8)
1436                                      .addReg(X86::NoRegister)
1437                                      .addReg(X86::ECX)
1438                                      .addImm(3),
1439                                  STI);
1440   }
1441 
1442   OutStreamer->emitInstruction(
1443       MCInstBuilder(X86::CMP32rr).addReg(X86::ECX).addReg(X86::R8D).addImm(1),
1444       STI);
1445   OutStreamer->emitInstruction(MCInstBuilder(X86::POP64r).addReg(X86::RCX),
1446                                STI);
1447   OutStreamer->emitInstruction(
1448       MCInstBuilder(X86::JCC_1)
1449           .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext))
1450           .addImm(X86::COND_L),
1451       STI);
1452 
1453   emitAsanReportError(M, Reg, AccessInfo, STI);
1454 }
1455 
1456 void X86AsmPrinter::emitAsanMemaccessFull(Module &M, unsigned Reg,
1457                                           const ASanAccessInfo &AccessInfo,
1458                                           MCSubtargetInfo &STI) {
1459   assert(AccessInfo.AccessSizeIndex == 3 || AccessInfo.AccessSizeIndex == 4);
1460   assert(Reg != X86::R8);
1461 
1462   uint64_t ShadowBase;
1463   int MappingScale;
1464   bool OrShadowOffset;
1465   getAddressSanitizerParams(
1466       Triple(M.getTargetTriple()), M.getDataLayout().getPointerSizeInBits(),
1467       AccessInfo.CompileKernel, &ShadowBase, &MappingScale, &OrShadowOffset);
1468 
1469   OutStreamer->emitInstruction(
1470       MCInstBuilder(X86::MOV64rr).addReg(X86::R8).addReg(X86::NoRegister + Reg),
1471       STI);
1472   OutStreamer->emitInstruction(MCInstBuilder(X86::SHR64ri)
1473                                    .addReg(X86::R8)
1474                                    .addReg(X86::R8)
1475                                    .addImm(MappingScale),
1476                                STI);
1477   if (OrShadowOffset) {
1478     OutStreamer->emitInstruction(MCInstBuilder(X86::OR64ri32)
1479                                      .addReg(X86::R8)
1480                                      .addReg(X86::R8)
1481                                      .addImm(ShadowBase),
1482                                  STI);
1483     auto OpCode = AccessInfo.AccessSizeIndex == 3 ? X86::CMP8mi : X86::CMP16mi8;
1484     OutStreamer->emitInstruction(MCInstBuilder(OpCode)
1485                                      .addReg(X86::R8)
1486                                      .addImm(1)
1487                                      .addReg(X86::NoRegister)
1488                                      .addImm(0)
1489                                      .addReg(X86::NoRegister)
1490                                      .addImm(0),
1491                                  STI);
1492   } else {
1493     auto OpCode = AccessInfo.AccessSizeIndex == 3 ? X86::CMP8mi : X86::CMP16mi8;
1494     OutStreamer->emitInstruction(MCInstBuilder(OpCode)
1495                                      .addReg(X86::R8)
1496                                      .addImm(1)
1497                                      .addReg(X86::NoRegister)
1498                                      .addImm(ShadowBase)
1499                                      .addReg(X86::NoRegister)
1500                                      .addImm(0),
1501                                  STI);
1502   }
1503   MCSymbol *ReportCode = OutContext.createTempSymbol();
1504   OutStreamer->emitInstruction(
1505       MCInstBuilder(X86::JCC_1)
1506           .addExpr(MCSymbolRefExpr::create(ReportCode, OutContext))
1507           .addImm(X86::COND_NE),
1508       STI);
1509   MCSymbol *ReturnSym = OutContext.createTempSymbol();
1510   OutStreamer->emitLabel(ReturnSym);
1511   OutStreamer->emitInstruction(MCInstBuilder(getRetOpcode(*Subtarget)), STI);
1512 
1513   OutStreamer->emitLabel(ReportCode);
1514   emitAsanReportError(M, Reg, AccessInfo, STI);
1515 }
1516 
1517 void X86AsmPrinter::emitAsanReportError(Module &M, unsigned Reg,
1518                                         const ASanAccessInfo &AccessInfo,
1519                                         MCSubtargetInfo &STI) {
1520   std::string Name = AccessInfo.IsWrite ? "store" : "load";
1521   MCSymbol *ReportError = OutContext.getOrCreateSymbol(
1522       "__asan_report_" + Name + utostr(1ULL << AccessInfo.AccessSizeIndex));
1523   OutStreamer->emitInstruction(MCInstBuilder(X86::MOV64rr)
1524                                    .addReg(X86::RDI)
1525                                    .addReg(X86::NoRegister + Reg),
1526                                STI);
1527   OutStreamer->emitInstruction(
1528       MCInstBuilder(X86::JMP_1)
1529           .addExpr(MCSymbolRefExpr::create(ReportError, OutContext)),
1530       STI);
1531 }
1532 
1533 void X86AsmPrinter::emitAsanMemaccessSymbols(Module &M) {
1534   if (AsanMemaccessSymbols.empty())
1535     return;
1536 
1537   const Triple &TT = TM.getTargetTriple();
1538   assert(TT.isOSBinFormatELF());
1539   std::unique_ptr<MCSubtargetInfo> STI(
1540       TM.getTarget().createMCSubtargetInfo(TT.str(), "", ""));
1541   assert(STI && "Unable to create subtarget info");
1542 
1543   for (auto &P : AsanMemaccessSymbols) {
1544     MCSymbol *Sym = P.second;
1545     OutStreamer->SwitchSection(OutContext.getELFSection(
1546         ".text.hot", ELF::SHT_PROGBITS,
1547         ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, Sym->getName(),
1548         /*IsComdat=*/true));
1549 
1550     OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
1551     OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
1552     OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden);
1553     OutStreamer->emitLabel(Sym);
1554 
1555     unsigned Reg = std::get<0>(P.first);
1556     ASanAccessInfo AccessInfo(std::get<1>(P.first));
1557 
1558     if (AccessInfo.AccessSizeIndex < 3) {
1559       emitAsanMemaccessPartial(M, Reg, AccessInfo, *STI);
1560     } else {
1561       emitAsanMemaccessFull(M, Reg, AccessInfo, *STI);
1562     }
1563   }
1564 }
1565 
1566 void X86AsmPrinter::LowerPATCHABLE_OP(const MachineInstr &MI,
1567                                       X86MCInstLower &MCIL) {
1568   // PATCHABLE_OP minsize, opcode, operands
1569 
1570   NoAutoPaddingScope NoPadScope(*OutStreamer);
1571 
1572   unsigned MinSize = MI.getOperand(0).getImm();
1573   unsigned Opcode = MI.getOperand(1).getImm();
1574 
1575   MCInst MCI;
1576   MCI.setOpcode(Opcode);
1577   for (auto &MO : drop_begin(MI.operands(), 2))
1578     if (auto MaybeOperand = MCIL.LowerMachineOperand(&MI, MO))
1579       MCI.addOperand(MaybeOperand.getValue());
1580 
1581   SmallString<256> Code;
1582   SmallVector<MCFixup, 4> Fixups;
1583   raw_svector_ostream VecOS(Code);
1584   CodeEmitter->encodeInstruction(MCI, VecOS, Fixups, getSubtargetInfo());
1585 
1586   if (Code.size() < MinSize) {
1587     if (MinSize == 2 && Subtarget->is32Bit() &&
1588         Subtarget->isTargetWindowsMSVC() &&
1589         (Subtarget->getCPU().empty() || Subtarget->getCPU() == "pentium3")) {
1590       // For compatibilty reasons, when targetting MSVC, is is important to
1591       // generate a 'legacy' NOP in the form of a 8B FF MOV EDI, EDI. Some tools
1592       // rely specifically on this pattern to be able to patch a function.
1593       // This is only for 32-bit targets, when using /arch:IA32 or /arch:SSE.
1594       OutStreamer->emitInstruction(
1595           MCInstBuilder(X86::MOV32rr_REV).addReg(X86::EDI).addReg(X86::EDI),
1596           *Subtarget);
1597     } else if (MinSize == 2 && Opcode == X86::PUSH64r) {
1598       // This is an optimization that lets us get away without emitting a nop in
1599       // many cases.
1600       //
1601       // NB! In some cases the encoding for PUSH64r (e.g. PUSH64r %r9) takes two
1602       // bytes too, so the check on MinSize is important.
1603       MCI.setOpcode(X86::PUSH64rmr);
1604     } else {
1605       unsigned NopSize = emitNop(*OutStreamer, MinSize, Subtarget);
1606       assert(NopSize == MinSize && "Could not implement MinSize!");
1607       (void)NopSize;
1608     }
1609   }
1610 
1611   OutStreamer->emitInstruction(MCI, getSubtargetInfo());
1612 }
1613 
1614 // Lower a stackmap of the form:
1615 // <id>, <shadowBytes>, ...
1616 void X86AsmPrinter::LowerSTACKMAP(const MachineInstr &MI) {
1617   SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo());
1618 
1619   auto &Ctx = OutStreamer->getContext();
1620   MCSymbol *MILabel = Ctx.createTempSymbol();
1621   OutStreamer->emitLabel(MILabel);
1622 
1623   SM.recordStackMap(*MILabel, MI);
1624   unsigned NumShadowBytes = MI.getOperand(1).getImm();
1625   SMShadowTracker.reset(NumShadowBytes);
1626 }
1627 
1628 // Lower a patchpoint of the form:
1629 // [<def>], <id>, <numBytes>, <target>, <numArgs>, <cc>, ...
1630 void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI,
1631                                     X86MCInstLower &MCIL) {
1632   assert(Subtarget->is64Bit() && "Patchpoint currently only supports X86-64");
1633 
1634   SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo());
1635 
1636   NoAutoPaddingScope NoPadScope(*OutStreamer);
1637 
1638   auto &Ctx = OutStreamer->getContext();
1639   MCSymbol *MILabel = Ctx.createTempSymbol();
1640   OutStreamer->emitLabel(MILabel);
1641   SM.recordPatchPoint(*MILabel, MI);
1642 
1643   PatchPointOpers opers(&MI);
1644   unsigned ScratchIdx = opers.getNextScratchIdx();
1645   unsigned EncodedBytes = 0;
1646   const MachineOperand &CalleeMO = opers.getCallTarget();
1647 
1648   // Check for null target. If target is non-null (i.e. is non-zero or is
1649   // symbolic) then emit a call.
1650   if (!(CalleeMO.isImm() && !CalleeMO.getImm())) {
1651     MCOperand CalleeMCOp;
1652     switch (CalleeMO.getType()) {
1653     default:
1654       /// FIXME: Add a verifier check for bad callee types.
1655       llvm_unreachable("Unrecognized callee operand type.");
1656     case MachineOperand::MO_Immediate:
1657       if (CalleeMO.getImm())
1658         CalleeMCOp = MCOperand::createImm(CalleeMO.getImm());
1659       break;
1660     case MachineOperand::MO_ExternalSymbol:
1661     case MachineOperand::MO_GlobalAddress:
1662       CalleeMCOp = MCIL.LowerSymbolOperand(CalleeMO,
1663                                            MCIL.GetSymbolFromOperand(CalleeMO));
1664       break;
1665     }
1666 
1667     // Emit MOV to materialize the target address and the CALL to target.
1668     // This is encoded with 12-13 bytes, depending on which register is used.
1669     Register ScratchReg = MI.getOperand(ScratchIdx).getReg();
1670     if (X86II::isX86_64ExtendedReg(ScratchReg))
1671       EncodedBytes = 13;
1672     else
1673       EncodedBytes = 12;
1674 
1675     EmitAndCountInstruction(
1676         MCInstBuilder(X86::MOV64ri).addReg(ScratchReg).addOperand(CalleeMCOp));
1677     // FIXME: Add retpoline support and remove this.
1678     if (Subtarget->useIndirectThunkCalls())
1679       report_fatal_error(
1680           "Lowering patchpoint with thunks not yet implemented.");
1681     EmitAndCountInstruction(MCInstBuilder(X86::CALL64r).addReg(ScratchReg));
1682   }
1683 
1684   // Emit padding.
1685   unsigned NumBytes = opers.getNumPatchBytes();
1686   assert(NumBytes >= EncodedBytes &&
1687          "Patchpoint can't request size less than the length of a call.");
1688 
1689   emitX86Nops(*OutStreamer, NumBytes - EncodedBytes, Subtarget);
1690 }
1691 
1692 void X86AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI,
1693                                               X86MCInstLower &MCIL) {
1694   assert(Subtarget->is64Bit() && "XRay custom events only supports X86-64");
1695 
1696   NoAutoPaddingScope NoPadScope(*OutStreamer);
1697 
1698   // We want to emit the following pattern, which follows the x86 calling
1699   // convention to prepare for the trampoline call to be patched in.
1700   //
1701   //   .p2align 1, ...
1702   // .Lxray_event_sled_N:
1703   //   jmp +N                        // jump across the instrumentation sled
1704   //   ...                           // set up arguments in register
1705   //   callq __xray_CustomEvent@plt  // force dependency to symbol
1706   //   ...
1707   //   <jump here>
1708   //
1709   // After patching, it would look something like:
1710   //
1711   //   nopw (2-byte nop)
1712   //   ...
1713   //   callq __xrayCustomEvent  // already lowered
1714   //   ...
1715   //
1716   // ---
1717   // First we emit the label and the jump.
1718   auto CurSled = OutContext.createTempSymbol("xray_event_sled_", true);
1719   OutStreamer->AddComment("# XRay Custom Event Log");
1720   OutStreamer->emitCodeAlignment(2);
1721   OutStreamer->emitLabel(CurSled);
1722 
1723   // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1724   // an operand (computed as an offset from the jmp instruction).
1725   // FIXME: Find another less hacky way do force the relative jump.
1726   OutStreamer->emitBinaryData("\xeb\x0f");
1727 
1728   // The default C calling convention will place two arguments into %rcx and
1729   // %rdx -- so we only work with those.
1730   const Register DestRegs[] = {X86::RDI, X86::RSI};
1731   bool UsedMask[] = {false, false};
1732   // Filled out in loop.
1733   Register SrcRegs[] = {0, 0};
1734 
1735   // Then we put the operands in the %rdi and %rsi registers. We spill the
1736   // values in the register before we clobber them, and mark them as used in
1737   // UsedMask. In case the arguments are already in the correct register, we use
1738   // emit nops appropriately sized to keep the sled the same size in every
1739   // situation.
1740   for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1741     if (auto Op = MCIL.LowerMachineOperand(&MI, MI.getOperand(I))) {
1742       assert(Op->isReg() && "Only support arguments in registers");
1743       SrcRegs[I] = getX86SubSuperRegister(Op->getReg(), 64);
1744       if (SrcRegs[I] != DestRegs[I]) {
1745         UsedMask[I] = true;
1746         EmitAndCountInstruction(
1747             MCInstBuilder(X86::PUSH64r).addReg(DestRegs[I]));
1748       } else {
1749         emitX86Nops(*OutStreamer, 4, Subtarget);
1750       }
1751     }
1752 
1753   // Now that the register values are stashed, mov arguments into place.
1754   // FIXME: This doesn't work if one of the later SrcRegs is equal to an
1755   // earlier DestReg. We will have already overwritten over the register before
1756   // we can copy from it.
1757   for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1758     if (SrcRegs[I] != DestRegs[I])
1759       EmitAndCountInstruction(
1760           MCInstBuilder(X86::MOV64rr).addReg(DestRegs[I]).addReg(SrcRegs[I]));
1761 
1762   // We emit a hard dependency on the __xray_CustomEvent symbol, which is the
1763   // name of the trampoline to be implemented by the XRay runtime.
1764   auto TSym = OutContext.getOrCreateSymbol("__xray_CustomEvent");
1765   MachineOperand TOp = MachineOperand::CreateMCSymbol(TSym);
1766   if (isPositionIndependent())
1767     TOp.setTargetFlags(X86II::MO_PLT);
1768 
1769   // Emit the call instruction.
1770   EmitAndCountInstruction(MCInstBuilder(X86::CALL64pcrel32)
1771                               .addOperand(MCIL.LowerSymbolOperand(TOp, TSym)));
1772 
1773   // Restore caller-saved and used registers.
1774   for (unsigned I = sizeof UsedMask; I-- > 0;)
1775     if (UsedMask[I])
1776       EmitAndCountInstruction(MCInstBuilder(X86::POP64r).addReg(DestRegs[I]));
1777     else
1778       emitX86Nops(*OutStreamer, 1, Subtarget);
1779 
1780   OutStreamer->AddComment("xray custom event end.");
1781 
1782   // Record the sled version. Version 0 of this sled was spelled differently, so
1783   // we let the runtime handle the different offsets we're using. Version 2
1784   // changed the absolute address to a PC-relative address.
1785   recordSled(CurSled, MI, SledKind::CUSTOM_EVENT, 2);
1786 }
1787 
1788 void X86AsmPrinter::LowerPATCHABLE_TYPED_EVENT_CALL(const MachineInstr &MI,
1789                                                     X86MCInstLower &MCIL) {
1790   assert(Subtarget->is64Bit() && "XRay typed events only supports X86-64");
1791 
1792   NoAutoPaddingScope NoPadScope(*OutStreamer);
1793 
1794   // We want to emit the following pattern, which follows the x86 calling
1795   // convention to prepare for the trampoline call to be patched in.
1796   //
1797   //   .p2align 1, ...
1798   // .Lxray_event_sled_N:
1799   //   jmp +N                        // jump across the instrumentation sled
1800   //   ...                           // set up arguments in register
1801   //   callq __xray_TypedEvent@plt  // force dependency to symbol
1802   //   ...
1803   //   <jump here>
1804   //
1805   // After patching, it would look something like:
1806   //
1807   //   nopw (2-byte nop)
1808   //   ...
1809   //   callq __xrayTypedEvent  // already lowered
1810   //   ...
1811   //
1812   // ---
1813   // First we emit the label and the jump.
1814   auto CurSled = OutContext.createTempSymbol("xray_typed_event_sled_", true);
1815   OutStreamer->AddComment("# XRay Typed Event Log");
1816   OutStreamer->emitCodeAlignment(2);
1817   OutStreamer->emitLabel(CurSled);
1818 
1819   // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1820   // an operand (computed as an offset from the jmp instruction).
1821   // FIXME: Find another less hacky way do force the relative jump.
1822   OutStreamer->emitBinaryData("\xeb\x14");
1823 
1824   // An x86-64 convention may place three arguments into %rcx, %rdx, and R8,
1825   // so we'll work with those. Or we may be called via SystemV, in which case
1826   // we don't have to do any translation.
1827   const Register DestRegs[] = {X86::RDI, X86::RSI, X86::RDX};
1828   bool UsedMask[] = {false, false, false};
1829 
1830   // Will fill out src regs in the loop.
1831   Register SrcRegs[] = {0, 0, 0};
1832 
1833   // Then we put the operands in the SystemV registers. We spill the values in
1834   // the registers before we clobber them, and mark them as used in UsedMask.
1835   // In case the arguments are already in the correct register, we emit nops
1836   // appropriately sized to keep the sled the same size in every situation.
1837   for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1838     if (auto Op = MCIL.LowerMachineOperand(&MI, MI.getOperand(I))) {
1839       // TODO: Is register only support adequate?
1840       assert(Op->isReg() && "Only supports arguments in registers");
1841       SrcRegs[I] = getX86SubSuperRegister(Op->getReg(), 64);
1842       if (SrcRegs[I] != DestRegs[I]) {
1843         UsedMask[I] = true;
1844         EmitAndCountInstruction(
1845             MCInstBuilder(X86::PUSH64r).addReg(DestRegs[I]));
1846       } else {
1847         emitX86Nops(*OutStreamer, 4, Subtarget);
1848       }
1849     }
1850 
1851   // In the above loop we only stash all of the destination registers or emit
1852   // nops if the arguments are already in the right place. Doing the actually
1853   // moving is postponed until after all the registers are stashed so nothing
1854   // is clobbers. We've already added nops to account for the size of mov and
1855   // push if the register is in the right place, so we only have to worry about
1856   // emitting movs.
1857   // FIXME: This doesn't work if one of the later SrcRegs is equal to an
1858   // earlier DestReg. We will have already overwritten over the register before
1859   // we can copy from it.
1860   for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1861     if (UsedMask[I])
1862       EmitAndCountInstruction(
1863           MCInstBuilder(X86::MOV64rr).addReg(DestRegs[I]).addReg(SrcRegs[I]));
1864 
1865   // We emit a hard dependency on the __xray_TypedEvent symbol, which is the
1866   // name of the trampoline to be implemented by the XRay runtime.
1867   auto TSym = OutContext.getOrCreateSymbol("__xray_TypedEvent");
1868   MachineOperand TOp = MachineOperand::CreateMCSymbol(TSym);
1869   if (isPositionIndependent())
1870     TOp.setTargetFlags(X86II::MO_PLT);
1871 
1872   // Emit the call instruction.
1873   EmitAndCountInstruction(MCInstBuilder(X86::CALL64pcrel32)
1874                               .addOperand(MCIL.LowerSymbolOperand(TOp, TSym)));
1875 
1876   // Restore caller-saved and used registers.
1877   for (unsigned I = sizeof UsedMask; I-- > 0;)
1878     if (UsedMask[I])
1879       EmitAndCountInstruction(MCInstBuilder(X86::POP64r).addReg(DestRegs[I]));
1880     else
1881       emitX86Nops(*OutStreamer, 1, Subtarget);
1882 
1883   OutStreamer->AddComment("xray typed event end.");
1884 
1885   // Record the sled version.
1886   recordSled(CurSled, MI, SledKind::TYPED_EVENT, 2);
1887 }
1888 
1889 void X86AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI,
1890                                                   X86MCInstLower &MCIL) {
1891 
1892   NoAutoPaddingScope NoPadScope(*OutStreamer);
1893 
1894   const Function &F = MF->getFunction();
1895   if (F.hasFnAttribute("patchable-function-entry")) {
1896     unsigned Num;
1897     if (F.getFnAttribute("patchable-function-entry")
1898             .getValueAsString()
1899             .getAsInteger(10, Num))
1900       return;
1901     emitX86Nops(*OutStreamer, Num, Subtarget);
1902     return;
1903   }
1904   // We want to emit the following pattern:
1905   //
1906   //   .p2align 1, ...
1907   // .Lxray_sled_N:
1908   //   jmp .tmpN
1909   //   # 9 bytes worth of noops
1910   //
1911   // We need the 9 bytes because at runtime, we'd be patching over the full 11
1912   // bytes with the following pattern:
1913   //
1914   //   mov %r10, <function id, 32-bit>   // 6 bytes
1915   //   call <relative offset, 32-bits>   // 5 bytes
1916   //
1917   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
1918   OutStreamer->emitCodeAlignment(2);
1919   OutStreamer->emitLabel(CurSled);
1920 
1921   // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1922   // an operand (computed as an offset from the jmp instruction).
1923   // FIXME: Find another less hacky way do force the relative jump.
1924   OutStreamer->emitBytes("\xeb\x09");
1925   emitX86Nops(*OutStreamer, 9, Subtarget);
1926   recordSled(CurSled, MI, SledKind::FUNCTION_ENTER, 2);
1927 }
1928 
1929 void X86AsmPrinter::LowerPATCHABLE_RET(const MachineInstr &MI,
1930                                        X86MCInstLower &MCIL) {
1931   NoAutoPaddingScope NoPadScope(*OutStreamer);
1932 
1933   // Since PATCHABLE_RET takes the opcode of the return statement as an
1934   // argument, we use that to emit the correct form of the RET that we want.
1935   // i.e. when we see this:
1936   //
1937   //   PATCHABLE_RET X86::RET ...
1938   //
1939   // We should emit the RET followed by sleds.
1940   //
1941   //   .p2align 1, ...
1942   // .Lxray_sled_N:
1943   //   ret  # or equivalent instruction
1944   //   # 10 bytes worth of noops
1945   //
1946   // This just makes sure that the alignment for the next instruction is 2.
1947   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
1948   OutStreamer->emitCodeAlignment(2);
1949   OutStreamer->emitLabel(CurSled);
1950   unsigned OpCode = MI.getOperand(0).getImm();
1951   MCInst Ret;
1952   Ret.setOpcode(OpCode);
1953   for (auto &MO : drop_begin(MI.operands()))
1954     if (auto MaybeOperand = MCIL.LowerMachineOperand(&MI, MO))
1955       Ret.addOperand(MaybeOperand.getValue());
1956   OutStreamer->emitInstruction(Ret, getSubtargetInfo());
1957   emitX86Nops(*OutStreamer, 10, Subtarget);
1958   recordSled(CurSled, MI, SledKind::FUNCTION_EXIT, 2);
1959 }
1960 
1961 void X86AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI,
1962                                              X86MCInstLower &MCIL) {
1963   NoAutoPaddingScope NoPadScope(*OutStreamer);
1964 
1965   // Like PATCHABLE_RET, we have the actual instruction in the operands to this
1966   // instruction so we lower that particular instruction and its operands.
1967   // Unlike PATCHABLE_RET though, we put the sled before the JMP, much like how
1968   // we do it for PATCHABLE_FUNCTION_ENTER. The sled should be very similar to
1969   // the PATCHABLE_FUNCTION_ENTER case, followed by the lowering of the actual
1970   // tail call much like how we have it in PATCHABLE_RET.
1971   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
1972   OutStreamer->emitCodeAlignment(2);
1973   OutStreamer->emitLabel(CurSled);
1974   auto Target = OutContext.createTempSymbol();
1975 
1976   // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1977   // an operand (computed as an offset from the jmp instruction).
1978   // FIXME: Find another less hacky way do force the relative jump.
1979   OutStreamer->emitBytes("\xeb\x09");
1980   emitX86Nops(*OutStreamer, 9, Subtarget);
1981   OutStreamer->emitLabel(Target);
1982   recordSled(CurSled, MI, SledKind::TAIL_CALL, 2);
1983 
1984   unsigned OpCode = MI.getOperand(0).getImm();
1985   OpCode = convertTailJumpOpcode(OpCode);
1986   MCInst TC;
1987   TC.setOpcode(OpCode);
1988 
1989   // Before emitting the instruction, add a comment to indicate that this is
1990   // indeed a tail call.
1991   OutStreamer->AddComment("TAILCALL");
1992   for (auto &MO : drop_begin(MI.operands()))
1993     if (auto MaybeOperand = MCIL.LowerMachineOperand(&MI, MO))
1994       TC.addOperand(MaybeOperand.getValue());
1995   OutStreamer->emitInstruction(TC, getSubtargetInfo());
1996 }
1997 
1998 // Returns instruction preceding MBBI in MachineFunction.
1999 // If MBBI is the first instruction of the first basic block, returns null.
2000 static MachineBasicBlock::const_iterator
2001 PrevCrossBBInst(MachineBasicBlock::const_iterator MBBI) {
2002   const MachineBasicBlock *MBB = MBBI->getParent();
2003   while (MBBI == MBB->begin()) {
2004     if (MBB == &MBB->getParent()->front())
2005       return MachineBasicBlock::const_iterator();
2006     MBB = MBB->getPrevNode();
2007     MBBI = MBB->end();
2008   }
2009   --MBBI;
2010   return MBBI;
2011 }
2012 
2013 static const Constant *getConstantFromPool(const MachineInstr &MI,
2014                                            const MachineOperand &Op) {
2015   if (!Op.isCPI() || Op.getOffset() != 0)
2016     return nullptr;
2017 
2018   ArrayRef<MachineConstantPoolEntry> Constants =
2019       MI.getParent()->getParent()->getConstantPool()->getConstants();
2020   const MachineConstantPoolEntry &ConstantEntry = Constants[Op.getIndex()];
2021 
2022   // Bail if this is a machine constant pool entry, we won't be able to dig out
2023   // anything useful.
2024   if (ConstantEntry.isMachineConstantPoolEntry())
2025     return nullptr;
2026 
2027   return ConstantEntry.Val.ConstVal;
2028 }
2029 
2030 static std::string getShuffleComment(const MachineInstr *MI, unsigned SrcOp1Idx,
2031                                      unsigned SrcOp2Idx, ArrayRef<int> Mask) {
2032   std::string Comment;
2033 
2034   // Compute the name for a register. This is really goofy because we have
2035   // multiple instruction printers that could (in theory) use different
2036   // names. Fortunately most people use the ATT style (outside of Windows)
2037   // and they actually agree on register naming here. Ultimately, this is
2038   // a comment, and so its OK if it isn't perfect.
2039   auto GetRegisterName = [](unsigned RegNum) -> StringRef {
2040     return X86ATTInstPrinter::getRegisterName(RegNum);
2041   };
2042 
2043   const MachineOperand &DstOp = MI->getOperand(0);
2044   const MachineOperand &SrcOp1 = MI->getOperand(SrcOp1Idx);
2045   const MachineOperand &SrcOp2 = MI->getOperand(SrcOp2Idx);
2046 
2047   StringRef DstName = DstOp.isReg() ? GetRegisterName(DstOp.getReg()) : "mem";
2048   StringRef Src1Name =
2049       SrcOp1.isReg() ? GetRegisterName(SrcOp1.getReg()) : "mem";
2050   StringRef Src2Name =
2051       SrcOp2.isReg() ? GetRegisterName(SrcOp2.getReg()) : "mem";
2052 
2053   // One source operand, fix the mask to print all elements in one span.
2054   SmallVector<int, 8> ShuffleMask(Mask.begin(), Mask.end());
2055   if (Src1Name == Src2Name)
2056     for (int i = 0, e = ShuffleMask.size(); i != e; ++i)
2057       if (ShuffleMask[i] >= e)
2058         ShuffleMask[i] -= e;
2059 
2060   raw_string_ostream CS(Comment);
2061   CS << DstName;
2062 
2063   // Handle AVX512 MASK/MASXZ write mask comments.
2064   // MASK: zmmX {%kY}
2065   // MASKZ: zmmX {%kY} {z}
2066   if (SrcOp1Idx > 1) {
2067     assert((SrcOp1Idx == 2 || SrcOp1Idx == 3) && "Unexpected writemask");
2068 
2069     const MachineOperand &WriteMaskOp = MI->getOperand(SrcOp1Idx - 1);
2070     if (WriteMaskOp.isReg()) {
2071       CS << " {%" << GetRegisterName(WriteMaskOp.getReg()) << "}";
2072 
2073       if (SrcOp1Idx == 2) {
2074         CS << " {z}";
2075       }
2076     }
2077   }
2078 
2079   CS << " = ";
2080 
2081   for (int i = 0, e = ShuffleMask.size(); i != e; ++i) {
2082     if (i != 0)
2083       CS << ",";
2084     if (ShuffleMask[i] == SM_SentinelZero) {
2085       CS << "zero";
2086       continue;
2087     }
2088 
2089     // Otherwise, it must come from src1 or src2.  Print the span of elements
2090     // that comes from this src.
2091     bool isSrc1 = ShuffleMask[i] < (int)e;
2092     CS << (isSrc1 ? Src1Name : Src2Name) << '[';
2093 
2094     bool IsFirst = true;
2095     while (i != e && ShuffleMask[i] != SM_SentinelZero &&
2096            (ShuffleMask[i] < (int)e) == isSrc1) {
2097       if (!IsFirst)
2098         CS << ',';
2099       else
2100         IsFirst = false;
2101       if (ShuffleMask[i] == SM_SentinelUndef)
2102         CS << "u";
2103       else
2104         CS << ShuffleMask[i] % (int)e;
2105       ++i;
2106     }
2107     CS << ']';
2108     --i; // For loop increments element #.
2109   }
2110   CS.flush();
2111 
2112   return Comment;
2113 }
2114 
2115 static void printConstant(const APInt &Val, raw_ostream &CS) {
2116   if (Val.getBitWidth() <= 64) {
2117     CS << Val.getZExtValue();
2118   } else {
2119     // print multi-word constant as (w0,w1)
2120     CS << "(";
2121     for (int i = 0, N = Val.getNumWords(); i < N; ++i) {
2122       if (i > 0)
2123         CS << ",";
2124       CS << Val.getRawData()[i];
2125     }
2126     CS << ")";
2127   }
2128 }
2129 
2130 static void printConstant(const APFloat &Flt, raw_ostream &CS) {
2131   SmallString<32> Str;
2132   // Force scientific notation to distinquish from integers.
2133   Flt.toString(Str, 0, 0);
2134   CS << Str;
2135 }
2136 
2137 static void printConstant(const Constant *COp, raw_ostream &CS) {
2138   if (isa<UndefValue>(COp)) {
2139     CS << "u";
2140   } else if (auto *CI = dyn_cast<ConstantInt>(COp)) {
2141     printConstant(CI->getValue(), CS);
2142   } else if (auto *CF = dyn_cast<ConstantFP>(COp)) {
2143     printConstant(CF->getValueAPF(), CS);
2144   } else {
2145     CS << "?";
2146   }
2147 }
2148 
2149 void X86AsmPrinter::EmitSEHInstruction(const MachineInstr *MI) {
2150   assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
2151   assert(getSubtarget().isOSWindows() && "SEH_ instruction Windows only");
2152 
2153   // Use the .cv_fpo directives if we're emitting CodeView on 32-bit x86.
2154   if (EmitFPOData) {
2155     X86TargetStreamer *XTS =
2156         static_cast<X86TargetStreamer *>(OutStreamer->getTargetStreamer());
2157     switch (MI->getOpcode()) {
2158     case X86::SEH_PushReg:
2159       XTS->emitFPOPushReg(MI->getOperand(0).getImm());
2160       break;
2161     case X86::SEH_StackAlloc:
2162       XTS->emitFPOStackAlloc(MI->getOperand(0).getImm());
2163       break;
2164     case X86::SEH_StackAlign:
2165       XTS->emitFPOStackAlign(MI->getOperand(0).getImm());
2166       break;
2167     case X86::SEH_SetFrame:
2168       assert(MI->getOperand(1).getImm() == 0 &&
2169              ".cv_fpo_setframe takes no offset");
2170       XTS->emitFPOSetFrame(MI->getOperand(0).getImm());
2171       break;
2172     case X86::SEH_EndPrologue:
2173       XTS->emitFPOEndPrologue();
2174       break;
2175     case X86::SEH_SaveReg:
2176     case X86::SEH_SaveXMM:
2177     case X86::SEH_PushFrame:
2178       llvm_unreachable("SEH_ directive incompatible with FPO");
2179       break;
2180     default:
2181       llvm_unreachable("expected SEH_ instruction");
2182     }
2183     return;
2184   }
2185 
2186   // Otherwise, use the .seh_ directives for all other Windows platforms.
2187   switch (MI->getOpcode()) {
2188   case X86::SEH_PushReg:
2189     OutStreamer->EmitWinCFIPushReg(MI->getOperand(0).getImm());
2190     break;
2191 
2192   case X86::SEH_SaveReg:
2193     OutStreamer->EmitWinCFISaveReg(MI->getOperand(0).getImm(),
2194                                    MI->getOperand(1).getImm());
2195     break;
2196 
2197   case X86::SEH_SaveXMM:
2198     OutStreamer->EmitWinCFISaveXMM(MI->getOperand(0).getImm(),
2199                                    MI->getOperand(1).getImm());
2200     break;
2201 
2202   case X86::SEH_StackAlloc:
2203     OutStreamer->EmitWinCFIAllocStack(MI->getOperand(0).getImm());
2204     break;
2205 
2206   case X86::SEH_SetFrame:
2207     OutStreamer->EmitWinCFISetFrame(MI->getOperand(0).getImm(),
2208                                     MI->getOperand(1).getImm());
2209     break;
2210 
2211   case X86::SEH_PushFrame:
2212     OutStreamer->EmitWinCFIPushFrame(MI->getOperand(0).getImm());
2213     break;
2214 
2215   case X86::SEH_EndPrologue:
2216     OutStreamer->EmitWinCFIEndProlog();
2217     break;
2218 
2219   default:
2220     llvm_unreachable("expected SEH_ instruction");
2221   }
2222 }
2223 
2224 static unsigned getRegisterWidth(const MCOperandInfo &Info) {
2225   if (Info.RegClass == X86::VR128RegClassID ||
2226       Info.RegClass == X86::VR128XRegClassID)
2227     return 128;
2228   if (Info.RegClass == X86::VR256RegClassID ||
2229       Info.RegClass == X86::VR256XRegClassID)
2230     return 256;
2231   if (Info.RegClass == X86::VR512RegClassID)
2232     return 512;
2233   llvm_unreachable("Unknown register class!");
2234 }
2235 
2236 static void addConstantComments(const MachineInstr *MI,
2237                                 MCStreamer &OutStreamer) {
2238   switch (MI->getOpcode()) {
2239   // Lower PSHUFB and VPERMILP normally but add a comment if we can find
2240   // a constant shuffle mask. We won't be able to do this at the MC layer
2241   // because the mask isn't an immediate.
2242   case X86::PSHUFBrm:
2243   case X86::VPSHUFBrm:
2244   case X86::VPSHUFBYrm:
2245   case X86::VPSHUFBZ128rm:
2246   case X86::VPSHUFBZ128rmk:
2247   case X86::VPSHUFBZ128rmkz:
2248   case X86::VPSHUFBZ256rm:
2249   case X86::VPSHUFBZ256rmk:
2250   case X86::VPSHUFBZ256rmkz:
2251   case X86::VPSHUFBZrm:
2252   case X86::VPSHUFBZrmk:
2253   case X86::VPSHUFBZrmkz: {
2254     unsigned SrcIdx = 1;
2255     if (X86II::isKMasked(MI->getDesc().TSFlags)) {
2256       // Skip mask operand.
2257       ++SrcIdx;
2258       if (X86II::isKMergeMasked(MI->getDesc().TSFlags)) {
2259         // Skip passthru operand.
2260         ++SrcIdx;
2261       }
2262     }
2263     unsigned MaskIdx = SrcIdx + 1 + X86::AddrDisp;
2264 
2265     assert(MI->getNumOperands() >= (SrcIdx + 1 + X86::AddrNumOperands) &&
2266            "Unexpected number of operands!");
2267 
2268     const MachineOperand &MaskOp = MI->getOperand(MaskIdx);
2269     if (auto *C = getConstantFromPool(*MI, MaskOp)) {
2270       unsigned Width = getRegisterWidth(MI->getDesc().OpInfo[0]);
2271       SmallVector<int, 64> Mask;
2272       DecodePSHUFBMask(C, Width, Mask);
2273       if (!Mask.empty())
2274         OutStreamer.AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask));
2275     }
2276     break;
2277   }
2278 
2279   case X86::VPERMILPSrm:
2280   case X86::VPERMILPSYrm:
2281   case X86::VPERMILPSZ128rm:
2282   case X86::VPERMILPSZ128rmk:
2283   case X86::VPERMILPSZ128rmkz:
2284   case X86::VPERMILPSZ256rm:
2285   case X86::VPERMILPSZ256rmk:
2286   case X86::VPERMILPSZ256rmkz:
2287   case X86::VPERMILPSZrm:
2288   case X86::VPERMILPSZrmk:
2289   case X86::VPERMILPSZrmkz:
2290   case X86::VPERMILPDrm:
2291   case X86::VPERMILPDYrm:
2292   case X86::VPERMILPDZ128rm:
2293   case X86::VPERMILPDZ128rmk:
2294   case X86::VPERMILPDZ128rmkz:
2295   case X86::VPERMILPDZ256rm:
2296   case X86::VPERMILPDZ256rmk:
2297   case X86::VPERMILPDZ256rmkz:
2298   case X86::VPERMILPDZrm:
2299   case X86::VPERMILPDZrmk:
2300   case X86::VPERMILPDZrmkz: {
2301     unsigned ElSize;
2302     switch (MI->getOpcode()) {
2303     default: llvm_unreachable("Invalid opcode");
2304     case X86::VPERMILPSrm:
2305     case X86::VPERMILPSYrm:
2306     case X86::VPERMILPSZ128rm:
2307     case X86::VPERMILPSZ256rm:
2308     case X86::VPERMILPSZrm:
2309     case X86::VPERMILPSZ128rmkz:
2310     case X86::VPERMILPSZ256rmkz:
2311     case X86::VPERMILPSZrmkz:
2312     case X86::VPERMILPSZ128rmk:
2313     case X86::VPERMILPSZ256rmk:
2314     case X86::VPERMILPSZrmk:
2315       ElSize = 32;
2316       break;
2317     case X86::VPERMILPDrm:
2318     case X86::VPERMILPDYrm:
2319     case X86::VPERMILPDZ128rm:
2320     case X86::VPERMILPDZ256rm:
2321     case X86::VPERMILPDZrm:
2322     case X86::VPERMILPDZ128rmkz:
2323     case X86::VPERMILPDZ256rmkz:
2324     case X86::VPERMILPDZrmkz:
2325     case X86::VPERMILPDZ128rmk:
2326     case X86::VPERMILPDZ256rmk:
2327     case X86::VPERMILPDZrmk:
2328       ElSize = 64;
2329       break;
2330     }
2331 
2332     unsigned SrcIdx = 1;
2333     if (X86II::isKMasked(MI->getDesc().TSFlags)) {
2334       // Skip mask operand.
2335       ++SrcIdx;
2336       if (X86II::isKMergeMasked(MI->getDesc().TSFlags)) {
2337         // Skip passthru operand.
2338         ++SrcIdx;
2339       }
2340     }
2341     unsigned MaskIdx = SrcIdx + 1 + X86::AddrDisp;
2342 
2343     assert(MI->getNumOperands() >= (SrcIdx + 1 + X86::AddrNumOperands) &&
2344            "Unexpected number of operands!");
2345 
2346     const MachineOperand &MaskOp = MI->getOperand(MaskIdx);
2347     if (auto *C = getConstantFromPool(*MI, MaskOp)) {
2348       unsigned Width = getRegisterWidth(MI->getDesc().OpInfo[0]);
2349       SmallVector<int, 16> Mask;
2350       DecodeVPERMILPMask(C, ElSize, Width, Mask);
2351       if (!Mask.empty())
2352         OutStreamer.AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask));
2353     }
2354     break;
2355   }
2356 
2357   case X86::VPERMIL2PDrm:
2358   case X86::VPERMIL2PSrm:
2359   case X86::VPERMIL2PDYrm:
2360   case X86::VPERMIL2PSYrm: {
2361     assert(MI->getNumOperands() >= (3 + X86::AddrNumOperands + 1) &&
2362            "Unexpected number of operands!");
2363 
2364     const MachineOperand &CtrlOp = MI->getOperand(MI->getNumOperands() - 1);
2365     if (!CtrlOp.isImm())
2366       break;
2367 
2368     unsigned ElSize;
2369     switch (MI->getOpcode()) {
2370     default: llvm_unreachable("Invalid opcode");
2371     case X86::VPERMIL2PSrm: case X86::VPERMIL2PSYrm: ElSize = 32; break;
2372     case X86::VPERMIL2PDrm: case X86::VPERMIL2PDYrm: ElSize = 64; break;
2373     }
2374 
2375     const MachineOperand &MaskOp = MI->getOperand(3 + X86::AddrDisp);
2376     if (auto *C = getConstantFromPool(*MI, MaskOp)) {
2377       unsigned Width = getRegisterWidth(MI->getDesc().OpInfo[0]);
2378       SmallVector<int, 16> Mask;
2379       DecodeVPERMIL2PMask(C, (unsigned)CtrlOp.getImm(), ElSize, Width, Mask);
2380       if (!Mask.empty())
2381         OutStreamer.AddComment(getShuffleComment(MI, 1, 2, Mask));
2382     }
2383     break;
2384   }
2385 
2386   case X86::VPPERMrrm: {
2387     assert(MI->getNumOperands() >= (3 + X86::AddrNumOperands) &&
2388            "Unexpected number of operands!");
2389 
2390     const MachineOperand &MaskOp = MI->getOperand(3 + X86::AddrDisp);
2391     if (auto *C = getConstantFromPool(*MI, MaskOp)) {
2392       unsigned Width = getRegisterWidth(MI->getDesc().OpInfo[0]);
2393       SmallVector<int, 16> Mask;
2394       DecodeVPPERMMask(C, Width, Mask);
2395       if (!Mask.empty())
2396         OutStreamer.AddComment(getShuffleComment(MI, 1, 2, Mask));
2397     }
2398     break;
2399   }
2400 
2401   case X86::MMX_MOVQ64rm: {
2402     assert(MI->getNumOperands() == (1 + X86::AddrNumOperands) &&
2403            "Unexpected number of operands!");
2404     if (auto *C = getConstantFromPool(*MI, MI->getOperand(1 + X86::AddrDisp))) {
2405       std::string Comment;
2406       raw_string_ostream CS(Comment);
2407       const MachineOperand &DstOp = MI->getOperand(0);
2408       CS << X86ATTInstPrinter::getRegisterName(DstOp.getReg()) << " = ";
2409       if (auto *CF = dyn_cast<ConstantFP>(C)) {
2410         CS << "0x" << toString(CF->getValueAPF().bitcastToAPInt(), 16, false);
2411         OutStreamer.AddComment(CS.str());
2412       }
2413     }
2414     break;
2415   }
2416 
2417 #define MOV_CASE(Prefix, Suffix)                                               \
2418   case X86::Prefix##MOVAPD##Suffix##rm:                                        \
2419   case X86::Prefix##MOVAPS##Suffix##rm:                                        \
2420   case X86::Prefix##MOVUPD##Suffix##rm:                                        \
2421   case X86::Prefix##MOVUPS##Suffix##rm:                                        \
2422   case X86::Prefix##MOVDQA##Suffix##rm:                                        \
2423   case X86::Prefix##MOVDQU##Suffix##rm:
2424 
2425 #define MOV_AVX512_CASE(Suffix)                                                \
2426   case X86::VMOVDQA64##Suffix##rm:                                             \
2427   case X86::VMOVDQA32##Suffix##rm:                                             \
2428   case X86::VMOVDQU64##Suffix##rm:                                             \
2429   case X86::VMOVDQU32##Suffix##rm:                                             \
2430   case X86::VMOVDQU16##Suffix##rm:                                             \
2431   case X86::VMOVDQU8##Suffix##rm:                                              \
2432   case X86::VMOVAPS##Suffix##rm:                                               \
2433   case X86::VMOVAPD##Suffix##rm:                                               \
2434   case X86::VMOVUPS##Suffix##rm:                                               \
2435   case X86::VMOVUPD##Suffix##rm:
2436 
2437 #define CASE_ALL_MOV_RM()                                                      \
2438   MOV_CASE(, )   /* SSE */                                                     \
2439   MOV_CASE(V, )  /* AVX-128 */                                                 \
2440   MOV_CASE(V, Y) /* AVX-256 */                                                 \
2441   MOV_AVX512_CASE(Z)                                                           \
2442   MOV_AVX512_CASE(Z256)                                                        \
2443   MOV_AVX512_CASE(Z128)
2444 
2445     // For loads from a constant pool to a vector register, print the constant
2446     // loaded.
2447     CASE_ALL_MOV_RM()
2448   case X86::VBROADCASTF128:
2449   case X86::VBROADCASTI128:
2450   case X86::VBROADCASTF32X4Z256rm:
2451   case X86::VBROADCASTF32X4rm:
2452   case X86::VBROADCASTF32X8rm:
2453   case X86::VBROADCASTF64X2Z128rm:
2454   case X86::VBROADCASTF64X2rm:
2455   case X86::VBROADCASTF64X4rm:
2456   case X86::VBROADCASTI32X4Z256rm:
2457   case X86::VBROADCASTI32X4rm:
2458   case X86::VBROADCASTI32X8rm:
2459   case X86::VBROADCASTI64X2Z128rm:
2460   case X86::VBROADCASTI64X2rm:
2461   case X86::VBROADCASTI64X4rm:
2462     assert(MI->getNumOperands() >= (1 + X86::AddrNumOperands) &&
2463            "Unexpected number of operands!");
2464     if (auto *C = getConstantFromPool(*MI, MI->getOperand(1 + X86::AddrDisp))) {
2465       int NumLanes = 1;
2466       // Override NumLanes for the broadcast instructions.
2467       switch (MI->getOpcode()) {
2468       case X86::VBROADCASTF128:        NumLanes = 2; break;
2469       case X86::VBROADCASTI128:        NumLanes = 2; break;
2470       case X86::VBROADCASTF32X4Z256rm: NumLanes = 2; break;
2471       case X86::VBROADCASTF32X4rm:     NumLanes = 4; break;
2472       case X86::VBROADCASTF32X8rm:     NumLanes = 2; break;
2473       case X86::VBROADCASTF64X2Z128rm: NumLanes = 2; break;
2474       case X86::VBROADCASTF64X2rm:     NumLanes = 4; break;
2475       case X86::VBROADCASTF64X4rm:     NumLanes = 2; break;
2476       case X86::VBROADCASTI32X4Z256rm: NumLanes = 2; break;
2477       case X86::VBROADCASTI32X4rm:     NumLanes = 4; break;
2478       case X86::VBROADCASTI32X8rm:     NumLanes = 2; break;
2479       case X86::VBROADCASTI64X2Z128rm: NumLanes = 2; break;
2480       case X86::VBROADCASTI64X2rm:     NumLanes = 4; break;
2481       case X86::VBROADCASTI64X4rm:     NumLanes = 2; break;
2482       }
2483 
2484       std::string Comment;
2485       raw_string_ostream CS(Comment);
2486       const MachineOperand &DstOp = MI->getOperand(0);
2487       CS << X86ATTInstPrinter::getRegisterName(DstOp.getReg()) << " = ";
2488       if (auto *CDS = dyn_cast<ConstantDataSequential>(C)) {
2489         CS << "[";
2490         for (int l = 0; l != NumLanes; ++l) {
2491           for (int i = 0, NumElements = CDS->getNumElements(); i < NumElements;
2492                ++i) {
2493             if (i != 0 || l != 0)
2494               CS << ",";
2495             if (CDS->getElementType()->isIntegerTy())
2496               printConstant(CDS->getElementAsAPInt(i), CS);
2497             else if (CDS->getElementType()->isHalfTy() ||
2498                      CDS->getElementType()->isFloatTy() ||
2499                      CDS->getElementType()->isDoubleTy())
2500               printConstant(CDS->getElementAsAPFloat(i), CS);
2501             else
2502               CS << "?";
2503           }
2504         }
2505         CS << "]";
2506         OutStreamer.AddComment(CS.str());
2507       } else if (auto *CV = dyn_cast<ConstantVector>(C)) {
2508         CS << "<";
2509         for (int l = 0; l != NumLanes; ++l) {
2510           for (int i = 0, NumOperands = CV->getNumOperands(); i < NumOperands;
2511                ++i) {
2512             if (i != 0 || l != 0)
2513               CS << ",";
2514             printConstant(CV->getOperand(i), CS);
2515           }
2516         }
2517         CS << ">";
2518         OutStreamer.AddComment(CS.str());
2519       }
2520     }
2521     break;
2522 
2523   case X86::MOVDDUPrm:
2524   case X86::VMOVDDUPrm:
2525   case X86::VMOVDDUPZ128rm:
2526   case X86::VBROADCASTSSrm:
2527   case X86::VBROADCASTSSYrm:
2528   case X86::VBROADCASTSSZ128rm:
2529   case X86::VBROADCASTSSZ256rm:
2530   case X86::VBROADCASTSSZrm:
2531   case X86::VBROADCASTSDYrm:
2532   case X86::VBROADCASTSDZ256rm:
2533   case X86::VBROADCASTSDZrm:
2534   case X86::VPBROADCASTBrm:
2535   case X86::VPBROADCASTBYrm:
2536   case X86::VPBROADCASTBZ128rm:
2537   case X86::VPBROADCASTBZ256rm:
2538   case X86::VPBROADCASTBZrm:
2539   case X86::VPBROADCASTDrm:
2540   case X86::VPBROADCASTDYrm:
2541   case X86::VPBROADCASTDZ128rm:
2542   case X86::VPBROADCASTDZ256rm:
2543   case X86::VPBROADCASTDZrm:
2544   case X86::VPBROADCASTQrm:
2545   case X86::VPBROADCASTQYrm:
2546   case X86::VPBROADCASTQZ128rm:
2547   case X86::VPBROADCASTQZ256rm:
2548   case X86::VPBROADCASTQZrm:
2549   case X86::VPBROADCASTWrm:
2550   case X86::VPBROADCASTWYrm:
2551   case X86::VPBROADCASTWZ128rm:
2552   case X86::VPBROADCASTWZ256rm:
2553   case X86::VPBROADCASTWZrm:
2554     assert(MI->getNumOperands() >= (1 + X86::AddrNumOperands) &&
2555            "Unexpected number of operands!");
2556     if (auto *C = getConstantFromPool(*MI, MI->getOperand(1 + X86::AddrDisp))) {
2557       int NumElts;
2558       switch (MI->getOpcode()) {
2559       default: llvm_unreachable("Invalid opcode");
2560       case X86::MOVDDUPrm:          NumElts = 2;  break;
2561       case X86::VMOVDDUPrm:         NumElts = 2;  break;
2562       case X86::VMOVDDUPZ128rm:     NumElts = 2;  break;
2563       case X86::VBROADCASTSSrm:     NumElts = 4;  break;
2564       case X86::VBROADCASTSSYrm:    NumElts = 8;  break;
2565       case X86::VBROADCASTSSZ128rm: NumElts = 4;  break;
2566       case X86::VBROADCASTSSZ256rm: NumElts = 8;  break;
2567       case X86::VBROADCASTSSZrm:    NumElts = 16; break;
2568       case X86::VBROADCASTSDYrm:    NumElts = 4;  break;
2569       case X86::VBROADCASTSDZ256rm: NumElts = 4;  break;
2570       case X86::VBROADCASTSDZrm:    NumElts = 8;  break;
2571       case X86::VPBROADCASTBrm:     NumElts = 16; break;
2572       case X86::VPBROADCASTBYrm:    NumElts = 32; break;
2573       case X86::VPBROADCASTBZ128rm: NumElts = 16; break;
2574       case X86::VPBROADCASTBZ256rm: NumElts = 32; break;
2575       case X86::VPBROADCASTBZrm:    NumElts = 64; break;
2576       case X86::VPBROADCASTDrm:     NumElts = 4;  break;
2577       case X86::VPBROADCASTDYrm:    NumElts = 8;  break;
2578       case X86::VPBROADCASTDZ128rm: NumElts = 4;  break;
2579       case X86::VPBROADCASTDZ256rm: NumElts = 8;  break;
2580       case X86::VPBROADCASTDZrm:    NumElts = 16; break;
2581       case X86::VPBROADCASTQrm:     NumElts = 2;  break;
2582       case X86::VPBROADCASTQYrm:    NumElts = 4;  break;
2583       case X86::VPBROADCASTQZ128rm: NumElts = 2;  break;
2584       case X86::VPBROADCASTQZ256rm: NumElts = 4;  break;
2585       case X86::VPBROADCASTQZrm:    NumElts = 8;  break;
2586       case X86::VPBROADCASTWrm:     NumElts = 8;  break;
2587       case X86::VPBROADCASTWYrm:    NumElts = 16; break;
2588       case X86::VPBROADCASTWZ128rm: NumElts = 8;  break;
2589       case X86::VPBROADCASTWZ256rm: NumElts = 16; break;
2590       case X86::VPBROADCASTWZrm:    NumElts = 32; break;
2591       }
2592 
2593       std::string Comment;
2594       raw_string_ostream CS(Comment);
2595       const MachineOperand &DstOp = MI->getOperand(0);
2596       CS << X86ATTInstPrinter::getRegisterName(DstOp.getReg()) << " = ";
2597       CS << "[";
2598       for (int i = 0; i != NumElts; ++i) {
2599         if (i != 0)
2600           CS << ",";
2601         printConstant(C, CS);
2602       }
2603       CS << "]";
2604       OutStreamer.AddComment(CS.str());
2605     }
2606   }
2607 }
2608 
2609 void X86AsmPrinter::emitInstruction(const MachineInstr *MI) {
2610   X86MCInstLower MCInstLowering(*MF, *this);
2611   const X86RegisterInfo *RI =
2612       MF->getSubtarget<X86Subtarget>().getRegisterInfo();
2613 
2614   // Add a comment about EVEX-2-VEX compression for AVX-512 instrs that
2615   // are compressed from EVEX encoding to VEX encoding.
2616   if (TM.Options.MCOptions.ShowMCEncoding) {
2617     if (MI->getAsmPrinterFlags() & X86::AC_EVEX_2_VEX)
2618       OutStreamer->AddComment("EVEX TO VEX Compression ", false);
2619   }
2620 
2621   // Add comments for values loaded from constant pool.
2622   if (OutStreamer->isVerboseAsm())
2623     addConstantComments(MI, *OutStreamer);
2624 
2625   switch (MI->getOpcode()) {
2626   case TargetOpcode::DBG_VALUE:
2627     llvm_unreachable("Should be handled target independently");
2628 
2629   // Emit nothing here but a comment if we can.
2630   case X86::Int_MemBarrier:
2631     OutStreamer->emitRawComment("MEMBARRIER");
2632     return;
2633 
2634   case X86::EH_RETURN:
2635   case X86::EH_RETURN64: {
2636     // Lower these as normal, but add some comments.
2637     Register Reg = MI->getOperand(0).getReg();
2638     OutStreamer->AddComment(StringRef("eh_return, addr: %") +
2639                             X86ATTInstPrinter::getRegisterName(Reg));
2640     break;
2641   }
2642   case X86::CLEANUPRET: {
2643     // Lower these as normal, but add some comments.
2644     OutStreamer->AddComment("CLEANUPRET");
2645     break;
2646   }
2647 
2648   case X86::CATCHRET: {
2649     // Lower these as normal, but add some comments.
2650     OutStreamer->AddComment("CATCHRET");
2651     break;
2652   }
2653 
2654   case X86::ENDBR32:
2655   case X86::ENDBR64: {
2656     // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
2657     // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
2658     // non-empty. If MI is the initial ENDBR, place the
2659     // __patchable_function_entries label after ENDBR.
2660     if (CurrentPatchableFunctionEntrySym &&
2661         CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
2662         MI == &MF->front().front()) {
2663       MCInst Inst;
2664       MCInstLowering.Lower(MI, Inst);
2665       EmitAndCountInstruction(Inst);
2666       CurrentPatchableFunctionEntrySym = createTempSymbol("patch");
2667       OutStreamer->emitLabel(CurrentPatchableFunctionEntrySym);
2668       return;
2669     }
2670     break;
2671   }
2672 
2673   case X86::TAILJMPr:
2674   case X86::TAILJMPm:
2675   case X86::TAILJMPd:
2676   case X86::TAILJMPd_CC:
2677   case X86::TAILJMPr64:
2678   case X86::TAILJMPm64:
2679   case X86::TAILJMPd64:
2680   case X86::TAILJMPd64_CC:
2681   case X86::TAILJMPr64_REX:
2682   case X86::TAILJMPm64_REX:
2683     // Lower these as normal, but add some comments.
2684     OutStreamer->AddComment("TAILCALL");
2685     break;
2686 
2687   case X86::TLS_addr32:
2688   case X86::TLS_addr64:
2689   case X86::TLS_addrX32:
2690   case X86::TLS_base_addr32:
2691   case X86::TLS_base_addr64:
2692   case X86::TLS_base_addrX32:
2693     return LowerTlsAddr(MCInstLowering, *MI);
2694 
2695   case X86::MOVPC32r: {
2696     // This is a pseudo op for a two instruction sequence with a label, which
2697     // looks like:
2698     //     call "L1$pb"
2699     // "L1$pb":
2700     //     popl %esi
2701 
2702     // Emit the call.
2703     MCSymbol *PICBase = MF->getPICBaseSymbol();
2704     // FIXME: We would like an efficient form for this, so we don't have to do a
2705     // lot of extra uniquing.
2706     EmitAndCountInstruction(
2707         MCInstBuilder(X86::CALLpcrel32)
2708             .addExpr(MCSymbolRefExpr::create(PICBase, OutContext)));
2709 
2710     const X86FrameLowering *FrameLowering =
2711         MF->getSubtarget<X86Subtarget>().getFrameLowering();
2712     bool hasFP = FrameLowering->hasFP(*MF);
2713 
2714     // TODO: This is needed only if we require precise CFA.
2715     bool HasActiveDwarfFrame = OutStreamer->getNumFrameInfos() &&
2716                                !OutStreamer->getDwarfFrameInfos().back().End;
2717 
2718     int stackGrowth = -RI->getSlotSize();
2719 
2720     if (HasActiveDwarfFrame && !hasFP) {
2721       OutStreamer->emitCFIAdjustCfaOffset(-stackGrowth);
2722     }
2723 
2724     // Emit the label.
2725     OutStreamer->emitLabel(PICBase);
2726 
2727     // popl $reg
2728     EmitAndCountInstruction(
2729         MCInstBuilder(X86::POP32r).addReg(MI->getOperand(0).getReg()));
2730 
2731     if (HasActiveDwarfFrame && !hasFP) {
2732       OutStreamer->emitCFIAdjustCfaOffset(stackGrowth);
2733     }
2734     return;
2735   }
2736 
2737   case X86::ADD32ri: {
2738     // Lower the MO_GOT_ABSOLUTE_ADDRESS form of ADD32ri.
2739     if (MI->getOperand(2).getTargetFlags() != X86II::MO_GOT_ABSOLUTE_ADDRESS)
2740       break;
2741 
2742     // Okay, we have something like:
2743     //  EAX = ADD32ri EAX, MO_GOT_ABSOLUTE_ADDRESS(@MYGLOBAL)
2744 
2745     // For this, we want to print something like:
2746     //   MYGLOBAL + (. - PICBASE)
2747     // However, we can't generate a ".", so just emit a new label here and refer
2748     // to it.
2749     MCSymbol *DotSym = OutContext.createTempSymbol();
2750     OutStreamer->emitLabel(DotSym);
2751 
2752     // Now that we have emitted the label, lower the complex operand expression.
2753     MCSymbol *OpSym = MCInstLowering.GetSymbolFromOperand(MI->getOperand(2));
2754 
2755     const MCExpr *DotExpr = MCSymbolRefExpr::create(DotSym, OutContext);
2756     const MCExpr *PICBase =
2757         MCSymbolRefExpr::create(MF->getPICBaseSymbol(), OutContext);
2758     DotExpr = MCBinaryExpr::createSub(DotExpr, PICBase, OutContext);
2759 
2760     DotExpr = MCBinaryExpr::createAdd(
2761         MCSymbolRefExpr::create(OpSym, OutContext), DotExpr, OutContext);
2762 
2763     EmitAndCountInstruction(MCInstBuilder(X86::ADD32ri)
2764                                 .addReg(MI->getOperand(0).getReg())
2765                                 .addReg(MI->getOperand(1).getReg())
2766                                 .addExpr(DotExpr));
2767     return;
2768   }
2769   case TargetOpcode::STATEPOINT:
2770     return LowerSTATEPOINT(*MI, MCInstLowering);
2771 
2772   case TargetOpcode::FAULTING_OP:
2773     return LowerFAULTING_OP(*MI, MCInstLowering);
2774 
2775   case TargetOpcode::FENTRY_CALL:
2776     return LowerFENTRY_CALL(*MI, MCInstLowering);
2777 
2778   case TargetOpcode::PATCHABLE_OP:
2779     return LowerPATCHABLE_OP(*MI, MCInstLowering);
2780 
2781   case TargetOpcode::STACKMAP:
2782     return LowerSTACKMAP(*MI);
2783 
2784   case TargetOpcode::PATCHPOINT:
2785     return LowerPATCHPOINT(*MI, MCInstLowering);
2786 
2787   case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
2788     return LowerPATCHABLE_FUNCTION_ENTER(*MI, MCInstLowering);
2789 
2790   case TargetOpcode::PATCHABLE_RET:
2791     return LowerPATCHABLE_RET(*MI, MCInstLowering);
2792 
2793   case TargetOpcode::PATCHABLE_TAIL_CALL:
2794     return LowerPATCHABLE_TAIL_CALL(*MI, MCInstLowering);
2795 
2796   case TargetOpcode::PATCHABLE_EVENT_CALL:
2797     return LowerPATCHABLE_EVENT_CALL(*MI, MCInstLowering);
2798 
2799   case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
2800     return LowerPATCHABLE_TYPED_EVENT_CALL(*MI, MCInstLowering);
2801 
2802   case X86::MORESTACK_RET:
2803     EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget)));
2804     return;
2805 
2806   case X86::ASAN_CHECK_MEMACCESS:
2807     return LowerASAN_CHECK_MEMACCESS(*MI);
2808 
2809   case X86::MORESTACK_RET_RESTORE_R10:
2810     // Return, then restore R10.
2811     EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget)));
2812     EmitAndCountInstruction(
2813         MCInstBuilder(X86::MOV64rr).addReg(X86::R10).addReg(X86::RAX));
2814     return;
2815 
2816   case X86::SEH_PushReg:
2817   case X86::SEH_SaveReg:
2818   case X86::SEH_SaveXMM:
2819   case X86::SEH_StackAlloc:
2820   case X86::SEH_StackAlign:
2821   case X86::SEH_SetFrame:
2822   case X86::SEH_PushFrame:
2823   case X86::SEH_EndPrologue:
2824     EmitSEHInstruction(MI);
2825     return;
2826 
2827   case X86::SEH_Epilogue: {
2828     assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
2829     MachineBasicBlock::const_iterator MBBI(MI);
2830     // Check if preceded by a call and emit nop if so.
2831     for (MBBI = PrevCrossBBInst(MBBI);
2832          MBBI != MachineBasicBlock::const_iterator();
2833          MBBI = PrevCrossBBInst(MBBI)) {
2834       // Conservatively assume that pseudo instructions don't emit code and keep
2835       // looking for a call. We may emit an unnecessary nop in some cases.
2836       if (!MBBI->isPseudo()) {
2837         if (MBBI->isCall())
2838           EmitAndCountInstruction(MCInstBuilder(X86::NOOP));
2839         break;
2840       }
2841     }
2842     return;
2843   }
2844   case X86::UBSAN_UD1:
2845     EmitAndCountInstruction(MCInstBuilder(X86::UD1Lm)
2846                                 .addReg(X86::EAX)
2847                                 .addReg(X86::EAX)
2848                                 .addImm(1)
2849                                 .addReg(X86::NoRegister)
2850                                 .addImm(MI->getOperand(0).getImm())
2851                                 .addReg(X86::NoRegister));
2852     return;
2853   }
2854 
2855   MCInst TmpInst;
2856   MCInstLowering.Lower(MI, TmpInst);
2857 
2858   // Stackmap shadows cannot include branch targets, so we can count the bytes
2859   // in a call towards the shadow, but must ensure that the no thread returns
2860   // in to the stackmap shadow.  The only way to achieve this is if the call
2861   // is at the end of the shadow.
2862   if (MI->isCall()) {
2863     // Count then size of the call towards the shadow
2864     SMShadowTracker.count(TmpInst, getSubtargetInfo(), CodeEmitter.get());
2865     // Then flush the shadow so that we fill with nops before the call, not
2866     // after it.
2867     SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo());
2868     // Then emit the call
2869     OutStreamer->emitInstruction(TmpInst, getSubtargetInfo());
2870     return;
2871   }
2872 
2873   EmitAndCountInstruction(TmpInst);
2874 }
2875