1 //===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains a printer that converts from our internal representation
11 // of machine-dependent LLVM code to the AArch64 assembly language.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "AArch64.h"
16 #include "AArch64MCInstLower.h"
17 #include "AArch64MachineFunctionInfo.h"
18 #include "AArch64RegisterInfo.h"
19 #include "AArch64Subtarget.h"
20 #include "AArch64TargetObjectFile.h"
21 #include "InstPrinter/AArch64InstPrinter.h"
22 #include "MCTargetDesc/AArch64AddressingModes.h"
23 #include "MCTargetDesc/AArch64MCTargetDesc.h"
24 #include "MCTargetDesc/AArch64TargetStreamer.h"
25 #include "Utils/AArch64BaseInfo.h"
26 #include "llvm/ADT/SmallString.h"
27 #include "llvm/ADT/SmallVector.h"
28 #include "llvm/ADT/StringRef.h"
29 #include "llvm/ADT/Triple.h"
30 #include "llvm/ADT/Twine.h"
31 #include "llvm/CodeGen/AsmPrinter.h"
32 #include "llvm/CodeGen/MachineBasicBlock.h"
33 #include "llvm/CodeGen/MachineFunction.h"
34 #include "llvm/CodeGen/MachineInstr.h"
35 #include "llvm/CodeGen/MachineJumpTableInfo.h"
36 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
37 #include "llvm/CodeGen/MachineOperand.h"
38 #include "llvm/CodeGen/StackMaps.h"
39 #include "llvm/CodeGen/TargetRegisterInfo.h"
40 #include "llvm/IR/DataLayout.h"
41 #include "llvm/IR/DebugInfoMetadata.h"
42 #include "llvm/MC/MCAsmInfo.h"
43 #include "llvm/MC/MCContext.h"
44 #include "llvm/MC/MCInst.h"
45 #include "llvm/MC/MCInstBuilder.h"
46 #include "llvm/MC/MCStreamer.h"
47 #include "llvm/MC/MCSymbol.h"
48 #include "llvm/Support/Casting.h"
49 #include "llvm/Support/ErrorHandling.h"
50 #include "llvm/Support/TargetRegistry.h"
51 #include "llvm/Support/raw_ostream.h"
52 #include "llvm/Target/TargetMachine.h"
53 #include <algorithm>
54 #include <cassert>
55 #include <cstdint>
56 #include <map>
57 #include <memory>
58 
59 using namespace llvm;
60 
61 #define DEBUG_TYPE "asm-printer"
62 
63 namespace {
64 
65 class AArch64AsmPrinter : public AsmPrinter {
66   AArch64MCInstLower MCInstLowering;
67   StackMaps SM;
68   const AArch64Subtarget *STI;
69 
70 public:
71   AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
72       : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this),
73         SM(*this) {}
74 
75   StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
76 
77   /// Wrapper for MCInstLowering.lowerOperand() for the
78   /// tblgen'erated pseudo lowering.
79   bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
80     return MCInstLowering.lowerOperand(MO, MCOp);
81   }
82 
83   void EmitJumpTableInfo() override;
84   void emitJumpTableEntry(const MachineJumpTableInfo *MJTI,
85                           const MachineBasicBlock *MBB, unsigned JTI);
86 
87   void LowerJumpTableDestSmall(MCStreamer &OutStreamer, const MachineInstr &MI);
88 
89   void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
90                      const MachineInstr &MI);
91   void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
92                        const MachineInstr &MI);
93 
94   void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
95   void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
96   void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
97 
98   void EmitSled(const MachineInstr &MI, SledKind Kind);
99 
100   /// tblgen'erated driver function for lowering simple MI->MC
101   /// pseudo instructions.
102   bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
103                                    const MachineInstr *MI);
104 
105   void EmitInstruction(const MachineInstr *MI) override;
106 
107   void getAnalysisUsage(AnalysisUsage &AU) const override {
108     AsmPrinter::getAnalysisUsage(AU);
109     AU.setPreservesAll();
110   }
111 
112   bool runOnMachineFunction(MachineFunction &F) override {
113     AArch64FI = F.getInfo<AArch64FunctionInfo>();
114     STI = static_cast<const AArch64Subtarget*>(&F.getSubtarget());
115     bool Result = AsmPrinter::runOnMachineFunction(F);
116     emitXRayTable();
117     return Result;
118   }
119 
120 private:
121   void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
122   bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
123   bool printAsmRegInClass(const MachineOperand &MO,
124                           const TargetRegisterClass *RC, bool isVector,
125                           raw_ostream &O);
126 
127   bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
128                        unsigned AsmVariant, const char *ExtraCode,
129                        raw_ostream &O) override;
130   bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
131                              unsigned AsmVariant, const char *ExtraCode,
132                              raw_ostream &O) override;
133 
134   void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
135 
136   void EmitFunctionBodyEnd() override;
137 
138   MCSymbol *GetCPISymbol(unsigned CPID) const override;
139   void EmitEndOfAsmFile(Module &M) override;
140 
141   AArch64FunctionInfo *AArch64FI = nullptr;
142 
143   /// Emit the LOHs contained in AArch64FI.
144   void EmitLOHs();
145 
146   /// Emit instruction to set float register to zero.
147   void EmitFMov0(const MachineInstr &MI);
148 
149   using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
150 
151   MInstToMCSymbol LOHInstToLabel;
152 };
153 
154 } // end anonymous namespace
155 
156 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
157 {
158   EmitSled(MI, SledKind::FUNCTION_ENTER);
159 }
160 
161 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI)
162 {
163   EmitSled(MI, SledKind::FUNCTION_EXIT);
164 }
165 
166 void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI)
167 {
168   EmitSled(MI, SledKind::TAIL_CALL);
169 }
170 
171 void AArch64AsmPrinter::EmitSled(const MachineInstr &MI, SledKind Kind)
172 {
173   static const int8_t NoopsInSledCount = 7;
174   // We want to emit the following pattern:
175   //
176   // .Lxray_sled_N:
177   //   ALIGN
178   //   B #32
179   //   ; 7 NOP instructions (28 bytes)
180   // .tmpN
181   //
182   // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
183   // over the full 32 bytes (8 instructions) with the following pattern:
184   //
185   //   STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
186   //   LDR W0, #12 ; W0 := function ID
187   //   LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
188   //   BLR X16 ; call the tracing trampoline
189   //   ;DATA: 32 bits of function ID
190   //   ;DATA: lower 32 bits of the address of the trampoline
191   //   ;DATA: higher 32 bits of the address of the trampoline
192   //   LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
193   //
194   OutStreamer->EmitCodeAlignment(4);
195   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
196   OutStreamer->EmitLabel(CurSled);
197   auto Target = OutContext.createTempSymbol();
198 
199   // Emit "B #32" instruction, which jumps over the next 28 bytes.
200   // The operand has to be the number of 4-byte instructions to jump over,
201   // including the current instruction.
202   EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8));
203 
204   for (int8_t I = 0; I < NoopsInSledCount; I++)
205     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
206 
207   OutStreamer->EmitLabel(Target);
208   recordSled(CurSled, MI, Kind);
209 }
210 
211 void AArch64AsmPrinter::EmitEndOfAsmFile(Module &M) {
212   const Triple &TT = TM.getTargetTriple();
213   if (TT.isOSBinFormatMachO()) {
214     // Funny Darwin hack: This flag tells the linker that no global symbols
215     // contain code that falls through to other global symbols (e.g. the obvious
216     // implementation of multiple entry points).  If this doesn't occur, the
217     // linker can safely perform dead code stripping.  Since LLVM never
218     // generates code that does this, it is always safe to set.
219     OutStreamer->EmitAssemblerFlag(MCAF_SubsectionsViaSymbols);
220     SM.serializeToStackMapSection();
221   }
222 }
223 
224 void AArch64AsmPrinter::EmitLOHs() {
225   SmallVector<MCSymbol *, 3> MCArgs;
226 
227   for (const auto &D : AArch64FI->getLOHContainer()) {
228     for (const MachineInstr *MI : D.getArgs()) {
229       MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI);
230       assert(LabelIt != LOHInstToLabel.end() &&
231              "Label hasn't been inserted for LOH related instruction");
232       MCArgs.push_back(LabelIt->second);
233     }
234     OutStreamer->EmitLOHDirective(D.getKind(), MCArgs);
235     MCArgs.clear();
236   }
237 }
238 
239 void AArch64AsmPrinter::EmitFunctionBodyEnd() {
240   if (!AArch64FI->getLOHRelated().empty())
241     EmitLOHs();
242 }
243 
244 /// GetCPISymbol - Return the symbol for the specified constant pool entry.
245 MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
246   // Darwin uses a linker-private symbol name for constant-pools (to
247   // avoid addends on the relocation?), ELF has no such concept and
248   // uses a normal private symbol.
249   if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
250     return OutContext.getOrCreateSymbol(
251         Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
252         Twine(getFunctionNumber()) + "_" + Twine(CPID));
253 
254   return AsmPrinter::GetCPISymbol(CPID);
255 }
256 
257 void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
258                                      raw_ostream &O) {
259   const MachineOperand &MO = MI->getOperand(OpNum);
260   switch (MO.getType()) {
261   default:
262     llvm_unreachable("<unknown operand type>");
263   case MachineOperand::MO_Register: {
264     unsigned Reg = MO.getReg();
265     assert(TargetRegisterInfo::isPhysicalRegister(Reg));
266     assert(!MO.getSubReg() && "Subregs should be eliminated!");
267     O << AArch64InstPrinter::getRegisterName(Reg);
268     break;
269   }
270   case MachineOperand::MO_Immediate: {
271     int64_t Imm = MO.getImm();
272     O << '#' << Imm;
273     break;
274   }
275   case MachineOperand::MO_GlobalAddress: {
276     const GlobalValue *GV = MO.getGlobal();
277     MCSymbol *Sym = getSymbol(GV);
278 
279     // FIXME: Can we get anything other than a plain symbol here?
280     assert(!MO.getTargetFlags() && "Unknown operand target flag!");
281 
282     Sym->print(O, MAI);
283     printOffset(MO.getOffset(), O);
284     break;
285   }
286   case MachineOperand::MO_BlockAddress: {
287     MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
288     Sym->print(O, MAI);
289     break;
290   }
291   }
292 }
293 
294 bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
295                                           raw_ostream &O) {
296   unsigned Reg = MO.getReg();
297   switch (Mode) {
298   default:
299     return true; // Unknown mode.
300   case 'w':
301     Reg = getWRegFromXReg(Reg);
302     break;
303   case 'x':
304     Reg = getXRegFromWReg(Reg);
305     break;
306   }
307 
308   O << AArch64InstPrinter::getRegisterName(Reg);
309   return false;
310 }
311 
312 // Prints the register in MO using class RC using the offset in the
313 // new register class. This should not be used for cross class
314 // printing.
315 bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
316                                            const TargetRegisterClass *RC,
317                                            bool isVector, raw_ostream &O) {
318   assert(MO.isReg() && "Should only get here with a register!");
319   const TargetRegisterInfo *RI = STI->getRegisterInfo();
320   unsigned Reg = MO.getReg();
321   unsigned RegToPrint = RC->getRegister(RI->getEncodingValue(Reg));
322   assert(RI->regsOverlap(RegToPrint, Reg));
323   O << AArch64InstPrinter::getRegisterName(
324            RegToPrint, isVector ? AArch64::vreg : AArch64::NoRegAltName);
325   return false;
326 }
327 
328 bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
329                                         unsigned AsmVariant,
330                                         const char *ExtraCode, raw_ostream &O) {
331   const MachineOperand &MO = MI->getOperand(OpNum);
332 
333   // First try the generic code, which knows about modifiers like 'c' and 'n'.
334   if (!AsmPrinter::PrintAsmOperand(MI, OpNum, AsmVariant, ExtraCode, O))
335     return false;
336 
337   // Does this asm operand have a single letter operand modifier?
338   if (ExtraCode && ExtraCode[0]) {
339     if (ExtraCode[1] != 0)
340       return true; // Unknown modifier.
341 
342     switch (ExtraCode[0]) {
343     default:
344       return true; // Unknown modifier.
345     case 'a':      // Print 'a' modifier
346       PrintAsmMemoryOperand(MI, OpNum, AsmVariant, ExtraCode, O);
347       return false;
348     case 'w':      // Print W register
349     case 'x':      // Print X register
350       if (MO.isReg())
351         return printAsmMRegister(MO, ExtraCode[0], O);
352       if (MO.isImm() && MO.getImm() == 0) {
353         unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
354         O << AArch64InstPrinter::getRegisterName(Reg);
355         return false;
356       }
357       printOperand(MI, OpNum, O);
358       return false;
359     case 'b': // Print B register.
360     case 'h': // Print H register.
361     case 's': // Print S register.
362     case 'd': // Print D register.
363     case 'q': // Print Q register.
364       if (MO.isReg()) {
365         const TargetRegisterClass *RC;
366         switch (ExtraCode[0]) {
367         case 'b':
368           RC = &AArch64::FPR8RegClass;
369           break;
370         case 'h':
371           RC = &AArch64::FPR16RegClass;
372           break;
373         case 's':
374           RC = &AArch64::FPR32RegClass;
375           break;
376         case 'd':
377           RC = &AArch64::FPR64RegClass;
378           break;
379         case 'q':
380           RC = &AArch64::FPR128RegClass;
381           break;
382         default:
383           return true;
384         }
385         return printAsmRegInClass(MO, RC, false /* vector */, O);
386       }
387       printOperand(MI, OpNum, O);
388       return false;
389     }
390   }
391 
392   // According to ARM, we should emit x and v registers unless we have a
393   // modifier.
394   if (MO.isReg()) {
395     unsigned Reg = MO.getReg();
396 
397     // If this is a w or x register, print an x register.
398     if (AArch64::GPR32allRegClass.contains(Reg) ||
399         AArch64::GPR64allRegClass.contains(Reg))
400       return printAsmMRegister(MO, 'x', O);
401 
402     // If this is a b, h, s, d, or q register, print it as a v register.
403     return printAsmRegInClass(MO, &AArch64::FPR128RegClass, true /* vector */,
404                               O);
405   }
406 
407   printOperand(MI, OpNum, O);
408   return false;
409 }
410 
411 bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
412                                               unsigned OpNum,
413                                               unsigned AsmVariant,
414                                               const char *ExtraCode,
415                                               raw_ostream &O) {
416   if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
417     return true; // Unknown modifier.
418 
419   const MachineOperand &MO = MI->getOperand(OpNum);
420   assert(MO.isReg() && "unexpected inline asm memory operand");
421   O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]";
422   return false;
423 }
424 
425 void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
426                                                raw_ostream &OS) {
427   unsigned NOps = MI->getNumOperands();
428   assert(NOps == 4);
429   OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
430   // cast away const; DIetc do not take const operands for some reason.
431   OS << cast<DILocalVariable>(MI->getOperand(NOps - 2).getMetadata())
432             ->getName();
433   OS << " <- ";
434   // Frame address.  Currently handles register +- offset only.
435   assert(MI->getOperand(0).isReg() && MI->getOperand(1).isImm());
436   OS << '[';
437   printOperand(MI, 0, OS);
438   OS << '+';
439   printOperand(MI, 1, OS);
440   OS << ']';
441   OS << "+";
442   printOperand(MI, NOps - 2, OS);
443 }
444 
445 void AArch64AsmPrinter::EmitJumpTableInfo() {
446   const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
447   if (!MJTI) return;
448 
449   const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
450   if (JT.empty()) return;
451 
452   const TargetLoweringObjectFile &TLOF = getObjFileLowering();
453   MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(MF->getFunction(), TM);
454   OutStreamer->SwitchSection(ReadOnlySec);
455 
456   auto AFI = MF->getInfo<AArch64FunctionInfo>();
457   for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
458     const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
459 
460     // If this jump table was deleted, ignore it.
461     if (JTBBs.empty()) continue;
462 
463     unsigned Size = AFI->getJumpTableEntrySize(JTI);
464     EmitAlignment(Log2_32(Size));
465     OutStreamer->EmitLabel(GetJTISymbol(JTI));
466 
467     for (auto *JTBB : JTBBs)
468       emitJumpTableEntry(MJTI, JTBB, JTI);
469   }
470 }
471 
472 void AArch64AsmPrinter::emitJumpTableEntry(const MachineJumpTableInfo *MJTI,
473                                            const MachineBasicBlock *MBB,
474                                            unsigned JTI) {
475   const MCExpr *Value = MCSymbolRefExpr::create(MBB->getSymbol(), OutContext);
476   auto AFI = MF->getInfo<AArch64FunctionInfo>();
477   unsigned Size = AFI->getJumpTableEntrySize(JTI);
478 
479   if (Size == 4) {
480     // .word LBB - LJTI
481     const TargetLowering *TLI = MF->getSubtarget().getTargetLowering();
482     const MCExpr *Base = TLI->getPICJumpTableRelocBaseExpr(MF, JTI, OutContext);
483     Value = MCBinaryExpr::createSub(Value, Base, OutContext);
484   } else {
485     // .byte (LBB - LBB) >> 2 (or .hword)
486     const MCSymbol *BaseSym = AFI->getJumpTableEntryPCRelSymbol(JTI);
487     const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext);
488     Value = MCBinaryExpr::createSub(Value, Base, OutContext);
489     Value = MCBinaryExpr::createLShr(
490         Value, MCConstantExpr::create(2, OutContext), OutContext);
491   }
492 
493   OutStreamer->EmitValue(Value, Size);
494 }
495 
496 /// Small jump tables contain an unsigned byte or half, representing the offset
497 /// from the lowest-addressed possible destination to the desired basic
498 /// block. Since all instructions are 4-byte aligned, this is further compressed
499 /// by counting in instructions rather than bytes (i.e. divided by 4). So, to
500 /// materialize the correct destination we need:
501 ///
502 ///             adr xDest, .LBB0_0
503 ///             ldrb wScratch, [xTable, xEntry]   (with "lsl #1" for ldrh).
504 ///             add xDest, xDest, xScratch, lsl #2
505 void AArch64AsmPrinter::LowerJumpTableDestSmall(llvm::MCStreamer &OutStreamer,
506                                                 const llvm::MachineInstr &MI) {
507   unsigned DestReg = MI.getOperand(0).getReg();
508   unsigned ScratchReg = MI.getOperand(1).getReg();
509   unsigned ScratchRegW =
510       STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32);
511   unsigned TableReg = MI.getOperand(2).getReg();
512   unsigned EntryReg = MI.getOperand(3).getReg();
513   int JTIdx = MI.getOperand(4).getIndex();
514   bool IsByteEntry = MI.getOpcode() == AArch64::JumpTableDest8;
515 
516   // This has to be first because the compression pass based its reachability
517   // calculations on the start of the JumpTableDest instruction.
518   auto Label =
519       MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx);
520   EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR)
521                                   .addReg(DestReg)
522                                   .addExpr(MCSymbolRefExpr::create(
523                                       Label, MF->getContext())));
524 
525   // Load the number of instruction-steps to offset from the label.
526   unsigned LdrOpcode = IsByteEntry ? AArch64::LDRBBroX : AArch64::LDRHHroX;
527   EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode)
528                                   .addReg(ScratchRegW)
529                                   .addReg(TableReg)
530                                   .addReg(EntryReg)
531                                   .addImm(0)
532                                   .addImm(IsByteEntry ? 0 : 1));
533 
534   // Multiply the steps by 4 and add to the already materialized base label
535   // address.
536   EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs)
537                                   .addReg(DestReg)
538                                   .addReg(DestReg)
539                                   .addReg(ScratchReg)
540                                   .addImm(2));
541 }
542 
543 void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
544                                       const MachineInstr &MI) {
545   unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
546 
547   SM.recordStackMap(MI);
548   assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
549 
550   // Scan ahead to trim the shadow.
551   const MachineBasicBlock &MBB = *MI.getParent();
552   MachineBasicBlock::const_iterator MII(MI);
553   ++MII;
554   while (NumNOPBytes > 0) {
555     if (MII == MBB.end() || MII->isCall() ||
556         MII->getOpcode() == AArch64::DBG_VALUE ||
557         MII->getOpcode() == TargetOpcode::PATCHPOINT ||
558         MII->getOpcode() == TargetOpcode::STACKMAP)
559       break;
560     ++MII;
561     NumNOPBytes -= 4;
562   }
563 
564   // Emit nops.
565   for (unsigned i = 0; i < NumNOPBytes; i += 4)
566     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
567 }
568 
569 // Lower a patchpoint of the form:
570 // [<def>], <id>, <numBytes>, <target>, <numArgs>
571 void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
572                                         const MachineInstr &MI) {
573   SM.recordPatchPoint(MI);
574 
575   PatchPointOpers Opers(&MI);
576 
577   int64_t CallTarget = Opers.getCallTarget().getImm();
578   unsigned EncodedBytes = 0;
579   if (CallTarget) {
580     assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
581            "High 16 bits of call target should be zero.");
582     unsigned ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
583     EncodedBytes = 16;
584     // Materialize the jump address:
585     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi)
586                                     .addReg(ScratchReg)
587                                     .addImm((CallTarget >> 32) & 0xFFFF)
588                                     .addImm(32));
589     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
590                                     .addReg(ScratchReg)
591                                     .addReg(ScratchReg)
592                                     .addImm((CallTarget >> 16) & 0xFFFF)
593                                     .addImm(16));
594     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
595                                     .addReg(ScratchReg)
596                                     .addReg(ScratchReg)
597                                     .addImm(CallTarget & 0xFFFF)
598                                     .addImm(0));
599     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
600   }
601   // Emit padding.
602   unsigned NumBytes = Opers.getNumPatchBytes();
603   assert(NumBytes >= EncodedBytes &&
604          "Patchpoint can't request size less than the length of a call.");
605   assert((NumBytes - EncodedBytes) % 4 == 0 &&
606          "Invalid number of NOP bytes requested!");
607   for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
608     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
609 }
610 
611 void AArch64AsmPrinter::EmitFMov0(const MachineInstr &MI) {
612   unsigned DestReg = MI.getOperand(0).getReg();
613   if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround()) {
614     // Convert H/S/D register to corresponding Q register
615     if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31)
616       DestReg = AArch64::Q0 + (DestReg - AArch64::H0);
617     else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31)
618       DestReg = AArch64::Q0 + (DestReg - AArch64::S0);
619     else {
620       assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31);
621       DestReg = AArch64::Q0 + (DestReg - AArch64::D0);
622     }
623     MCInst MOVI;
624     MOVI.setOpcode(AArch64::MOVIv2d_ns);
625     MOVI.addOperand(MCOperand::createReg(DestReg));
626     MOVI.addOperand(MCOperand::createImm(0));
627     EmitToStreamer(*OutStreamer, MOVI);
628   } else {
629     MCInst FMov;
630     switch (MI.getOpcode()) {
631     default: llvm_unreachable("Unexpected opcode");
632     case AArch64::FMOVH0:
633       FMov.setOpcode(AArch64::FMOVWHr);
634       FMov.addOperand(MCOperand::createReg(DestReg));
635       FMov.addOperand(MCOperand::createReg(AArch64::WZR));
636       break;
637     case AArch64::FMOVS0:
638       FMov.setOpcode(AArch64::FMOVWSr);
639       FMov.addOperand(MCOperand::createReg(DestReg));
640       FMov.addOperand(MCOperand::createReg(AArch64::WZR));
641       break;
642     case AArch64::FMOVD0:
643       FMov.setOpcode(AArch64::FMOVXDr);
644       FMov.addOperand(MCOperand::createReg(DestReg));
645       FMov.addOperand(MCOperand::createReg(AArch64::XZR));
646       break;
647     }
648     EmitToStreamer(*OutStreamer, FMov);
649   }
650 }
651 
652 // Simple pseudo-instructions have their lowering (with expansion to real
653 // instructions) auto-generated.
654 #include "AArch64GenMCPseudoLowering.inc"
655 
656 void AArch64AsmPrinter::EmitInstruction(const MachineInstr *MI) {
657   // Do any auto-generated pseudo lowerings.
658   if (emitPseudoExpansionLowering(*OutStreamer, MI))
659     return;
660 
661   if (AArch64FI->getLOHRelated().count(MI)) {
662     // Generate a label for LOH related instruction
663     MCSymbol *LOHLabel = createTempSymbol("loh");
664     // Associate the instruction with the label
665     LOHInstToLabel[MI] = LOHLabel;
666     OutStreamer->EmitLabel(LOHLabel);
667   }
668 
669   AArch64TargetStreamer *TS =
670     static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
671   // Do any manual lowerings.
672   switch (MI->getOpcode()) {
673   default:
674     break;
675   case AArch64::MOVIv2d_ns:
676     // If the target has <rdar://problem/16473581>, lower this
677     // instruction to movi.16b instead.
678     if (STI->hasZeroCycleZeroingFPWorkaround() &&
679         MI->getOperand(1).getImm() == 0) {
680       MCInst TmpInst;
681       TmpInst.setOpcode(AArch64::MOVIv16b_ns);
682       TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
683       TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm()));
684       EmitToStreamer(*OutStreamer, TmpInst);
685       return;
686     }
687     break;
688 
689   case AArch64::DBG_VALUE: {
690     if (isVerbose() && OutStreamer->hasRawTextSupport()) {
691       SmallString<128> TmpStr;
692       raw_svector_ostream OS(TmpStr);
693       PrintDebugValueComment(MI, OS);
694       OutStreamer->EmitRawText(StringRef(OS.str()));
695     }
696     return;
697   }
698 
699   // Tail calls use pseudo instructions so they have the proper code-gen
700   // attributes (isCall, isReturn, etc.). We lower them to the real
701   // instruction here.
702   case AArch64::TCRETURNri:
703   case AArch64::TCRETURNriBTI:
704   case AArch64::TCRETURNriALL: {
705     MCInst TmpInst;
706     TmpInst.setOpcode(AArch64::BR);
707     TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
708     EmitToStreamer(*OutStreamer, TmpInst);
709     return;
710   }
711   case AArch64::TCRETURNdi: {
712     MCOperand Dest;
713     MCInstLowering.lowerOperand(MI->getOperand(0), Dest);
714     MCInst TmpInst;
715     TmpInst.setOpcode(AArch64::B);
716     TmpInst.addOperand(Dest);
717     EmitToStreamer(*OutStreamer, TmpInst);
718     return;
719   }
720   case AArch64::TLSDESC_CALLSEQ: {
721     /// lower this to:
722     ///    adrp  x0, :tlsdesc:var
723     ///    ldr   x1, [x0, #:tlsdesc_lo12:var]
724     ///    add   x0, x0, #:tlsdesc_lo12:var
725     ///    .tlsdesccall var
726     ///    blr   x1
727     ///    (TPIDR_EL0 offset now in x0)
728     const MachineOperand &MO_Sym = MI->getOperand(0);
729     MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
730     MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
731     MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
732     MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
733     MCInstLowering.lowerOperand(MO_Sym, Sym);
734     MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12);
735     MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc);
736 
737     MCInst Adrp;
738     Adrp.setOpcode(AArch64::ADRP);
739     Adrp.addOperand(MCOperand::createReg(AArch64::X0));
740     Adrp.addOperand(SymTLSDesc);
741     EmitToStreamer(*OutStreamer, Adrp);
742 
743     MCInst Ldr;
744     Ldr.setOpcode(AArch64::LDRXui);
745     Ldr.addOperand(MCOperand::createReg(AArch64::X1));
746     Ldr.addOperand(MCOperand::createReg(AArch64::X0));
747     Ldr.addOperand(SymTLSDescLo12);
748     Ldr.addOperand(MCOperand::createImm(0));
749     EmitToStreamer(*OutStreamer, Ldr);
750 
751     MCInst Add;
752     Add.setOpcode(AArch64::ADDXri);
753     Add.addOperand(MCOperand::createReg(AArch64::X0));
754     Add.addOperand(MCOperand::createReg(AArch64::X0));
755     Add.addOperand(SymTLSDescLo12);
756     Add.addOperand(MCOperand::createImm(AArch64_AM::getShiftValue(0)));
757     EmitToStreamer(*OutStreamer, Add);
758 
759     // Emit a relocation-annotation. This expands to no code, but requests
760     // the following instruction gets an R_AARCH64_TLSDESC_CALL.
761     MCInst TLSDescCall;
762     TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
763     TLSDescCall.addOperand(Sym);
764     EmitToStreamer(*OutStreamer, TLSDescCall);
765 
766     MCInst Blr;
767     Blr.setOpcode(AArch64::BLR);
768     Blr.addOperand(MCOperand::createReg(AArch64::X1));
769     EmitToStreamer(*OutStreamer, Blr);
770 
771     return;
772   }
773 
774   case AArch64::JumpTableDest32: {
775     // We want:
776     //     ldrsw xScratch, [xTable, xEntry, lsl #2]
777     //     add xDest, xTable, xScratch
778     unsigned DestReg = MI->getOperand(0).getReg(),
779              ScratchReg = MI->getOperand(1).getReg(),
780              TableReg = MI->getOperand(2).getReg(),
781              EntryReg = MI->getOperand(3).getReg();
782     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDRSWroX)
783                                      .addReg(ScratchReg)
784                                      .addReg(TableReg)
785                                      .addReg(EntryReg)
786                                      .addImm(0)
787                                      .addImm(1));
788     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADDXrs)
789                                      .addReg(DestReg)
790                                      .addReg(TableReg)
791                                      .addReg(ScratchReg)
792                                      .addImm(0));
793     return;
794   }
795   case AArch64::JumpTableDest16:
796   case AArch64::JumpTableDest8:
797     LowerJumpTableDestSmall(*OutStreamer, *MI);
798     return;
799 
800   case AArch64::FMOVH0:
801   case AArch64::FMOVS0:
802   case AArch64::FMOVD0:
803     EmitFMov0(*MI);
804     return;
805 
806   case TargetOpcode::STACKMAP:
807     return LowerSTACKMAP(*OutStreamer, SM, *MI);
808 
809   case TargetOpcode::PATCHPOINT:
810     return LowerPATCHPOINT(*OutStreamer, SM, *MI);
811 
812   case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
813     LowerPATCHABLE_FUNCTION_ENTER(*MI);
814     return;
815 
816   case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
817     LowerPATCHABLE_FUNCTION_EXIT(*MI);
818     return;
819 
820   case TargetOpcode::PATCHABLE_TAIL_CALL:
821     LowerPATCHABLE_TAIL_CALL(*MI);
822     return;
823 
824   case AArch64::SEH_StackAlloc:
825     TS->EmitARM64WinCFIAllocStack(MI->getOperand(0).getImm());
826     return;
827 
828   case AArch64::SEH_SaveFPLR:
829     TS->EmitARM64WinCFISaveFPLR(MI->getOperand(0).getImm());
830     return;
831 
832   case AArch64::SEH_SaveFPLR_X:
833     assert(MI->getOperand(0).getImm() < 0 &&
834            "Pre increment SEH opcode must have a negative offset");
835     TS->EmitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm());
836     return;
837 
838   case AArch64::SEH_SaveReg:
839     TS->EmitARM64WinCFISaveReg(MI->getOperand(0).getImm(),
840                                MI->getOperand(1).getImm());
841     return;
842 
843   case AArch64::SEH_SaveReg_X:
844     assert(MI->getOperand(1).getImm() < 0 &&
845            "Pre increment SEH opcode must have a negative offset");
846     TS->EmitARM64WinCFISaveRegX(MI->getOperand(0).getImm(),
847 		                -MI->getOperand(1).getImm());
848     return;
849 
850   case AArch64::SEH_SaveRegP:
851     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
852             "Non-consecutive registers not allowed for save_regp");
853     TS->EmitARM64WinCFISaveRegP(MI->getOperand(0).getImm(),
854                                 MI->getOperand(2).getImm());
855     return;
856 
857   case AArch64::SEH_SaveRegP_X:
858     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
859             "Non-consecutive registers not allowed for save_regp_x");
860     assert(MI->getOperand(2).getImm() < 0 &&
861            "Pre increment SEH opcode must have a negative offset");
862     TS->EmitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(),
863                                  -MI->getOperand(2).getImm());
864     return;
865 
866   case AArch64::SEH_SaveFReg:
867     TS->EmitARM64WinCFISaveFReg(MI->getOperand(0).getImm(),
868                                 MI->getOperand(1).getImm());
869     return;
870 
871   case AArch64::SEH_SaveFReg_X:
872     assert(MI->getOperand(1).getImm() < 0 &&
873            "Pre increment SEH opcode must have a negative offset");
874     TS->EmitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(),
875                                  -MI->getOperand(1).getImm());
876     return;
877 
878   case AArch64::SEH_SaveFRegP:
879     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
880             "Non-consecutive registers not allowed for save_regp");
881     TS->EmitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(),
882                                  MI->getOperand(2).getImm());
883     return;
884 
885   case AArch64::SEH_SaveFRegP_X:
886     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
887             "Non-consecutive registers not allowed for save_regp_x");
888     assert(MI->getOperand(2).getImm() < 0 &&
889            "Pre increment SEH opcode must have a negative offset");
890     TS->EmitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(),
891                                   -MI->getOperand(2).getImm());
892     return;
893 
894   case AArch64::SEH_SetFP:
895     TS->EmitARM64WinCFISetFP();
896     return;
897 
898   case AArch64::SEH_AddFP:
899     TS->EmitARM64WinCFIAddFP(MI->getOperand(0).getImm());
900     return;
901 
902   case AArch64::SEH_Nop:
903     TS->EmitARM64WinCFINop();
904     return;
905 
906   case AArch64::SEH_PrologEnd:
907     TS->EmitARM64WinCFIPrologEnd();
908     return;
909 
910   case AArch64::SEH_EpilogStart:
911     TS->EmitARM64WinCFIEpilogStart();
912     return;
913 
914   case AArch64::SEH_EpilogEnd:
915     TS->EmitARM64WinCFIEpilogEnd();
916     return;
917   }
918 
919   // Finally, do the automated lowerings for everything else.
920   MCInst TmpInst;
921   MCInstLowering.Lower(MI, TmpInst);
922   EmitToStreamer(*OutStreamer, TmpInst);
923 }
924 
925 // Force static initialization.
926 extern "C" void LLVMInitializeAArch64AsmPrinter() {
927   RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
928   RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget());
929   RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target());
930 }
931