1 //===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains a printer that converts from our internal representation
11 // of machine-dependent LLVM code to the AArch64 assembly language.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "AArch64.h"
16 #include "AArch64MCInstLower.h"
17 #include "AArch64MachineFunctionInfo.h"
18 #include "AArch64RegisterInfo.h"
19 #include "AArch64Subtarget.h"
20 #include "AArch64TargetObjectFile.h"
21 #include "InstPrinter/AArch64InstPrinter.h"
22 #include "MCTargetDesc/AArch64AddressingModes.h"
23 #include "MCTargetDesc/AArch64MCTargetDesc.h"
24 #include "MCTargetDesc/AArch64TargetStreamer.h"
25 #include "Utils/AArch64BaseInfo.h"
26 #include "llvm/ADT/SmallString.h"
27 #include "llvm/ADT/SmallVector.h"
28 #include "llvm/ADT/StringRef.h"
29 #include "llvm/ADT/Triple.h"
30 #include "llvm/ADT/Twine.h"
31 #include "llvm/BinaryFormat/COFF.h"
32 #include "llvm/CodeGen/AsmPrinter.h"
33 #include "llvm/CodeGen/MachineBasicBlock.h"
34 #include "llvm/CodeGen/MachineFunction.h"
35 #include "llvm/CodeGen/MachineInstr.h"
36 #include "llvm/CodeGen/MachineJumpTableInfo.h"
37 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
38 #include "llvm/CodeGen/MachineOperand.h"
39 #include "llvm/CodeGen/StackMaps.h"
40 #include "llvm/CodeGen/TargetRegisterInfo.h"
41 #include "llvm/IR/DataLayout.h"
42 #include "llvm/IR/DebugInfoMetadata.h"
43 #include "llvm/MC/MCAsmInfo.h"
44 #include "llvm/MC/MCContext.h"
45 #include "llvm/MC/MCInst.h"
46 #include "llvm/MC/MCInstBuilder.h"
47 #include "llvm/MC/MCStreamer.h"
48 #include "llvm/MC/MCSymbol.h"
49 #include "llvm/Support/Casting.h"
50 #include "llvm/Support/ErrorHandling.h"
51 #include "llvm/Support/TargetRegistry.h"
52 #include "llvm/Support/raw_ostream.h"
53 #include "llvm/Target/TargetMachine.h"
54 #include <algorithm>
55 #include <cassert>
56 #include <cstdint>
57 #include <map>
58 #include <memory>
59 
60 using namespace llvm;
61 
62 #define DEBUG_TYPE "asm-printer"
63 
64 namespace {
65 
66 class AArch64AsmPrinter : public AsmPrinter {
67   AArch64MCInstLower MCInstLowering;
68   StackMaps SM;
69   const AArch64Subtarget *STI;
70 
71 public:
72   AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
73       : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this),
74         SM(*this) {}
75 
76   StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
77 
78   /// Wrapper for MCInstLowering.lowerOperand() for the
79   /// tblgen'erated pseudo lowering.
80   bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
81     return MCInstLowering.lowerOperand(MO, MCOp);
82   }
83 
84   void EmitJumpTableInfo() override;
85   void emitJumpTableEntry(const MachineJumpTableInfo *MJTI,
86                           const MachineBasicBlock *MBB, unsigned JTI);
87 
88   void LowerJumpTableDestSmall(MCStreamer &OutStreamer, const MachineInstr &MI);
89 
90   void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
91                      const MachineInstr &MI);
92   void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
93                        const MachineInstr &MI);
94 
95   void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
96   void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
97   void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
98 
99   void EmitSled(const MachineInstr &MI, SledKind Kind);
100 
101   /// tblgen'erated driver function for lowering simple MI->MC
102   /// pseudo instructions.
103   bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
104                                    const MachineInstr *MI);
105 
106   void EmitInstruction(const MachineInstr *MI) override;
107 
108   void getAnalysisUsage(AnalysisUsage &AU) const override {
109     AsmPrinter::getAnalysisUsage(AU);
110     AU.setPreservesAll();
111   }
112 
113   bool runOnMachineFunction(MachineFunction &MF) override {
114     AArch64FI = MF.getInfo<AArch64FunctionInfo>();
115     STI = static_cast<const AArch64Subtarget*>(&MF.getSubtarget());
116 
117     SetupMachineFunction(MF);
118 
119     if (STI->isTargetCOFF()) {
120       bool Internal = MF.getFunction().hasInternalLinkage();
121       COFF::SymbolStorageClass Scl = Internal ? COFF::IMAGE_SYM_CLASS_STATIC
122                                               : COFF::IMAGE_SYM_CLASS_EXTERNAL;
123       int Type =
124         COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT;
125 
126       OutStreamer->BeginCOFFSymbolDef(CurrentFnSym);
127       OutStreamer->EmitCOFFSymbolStorageClass(Scl);
128       OutStreamer->EmitCOFFSymbolType(Type);
129       OutStreamer->EndCOFFSymbolDef();
130     }
131 
132     // Emit the rest of the function body.
133     EmitFunctionBody();
134 
135     // Emit the XRay table for this function.
136     emitXRayTable();
137 
138     // We didn't modify anything.
139     return false;
140   }
141 
142 private:
143   void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
144   bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
145   bool printAsmRegInClass(const MachineOperand &MO,
146                           const TargetRegisterClass *RC, bool isVector,
147                           raw_ostream &O);
148 
149   bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
150                        unsigned AsmVariant, const char *ExtraCode,
151                        raw_ostream &O) override;
152   bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
153                              unsigned AsmVariant, const char *ExtraCode,
154                              raw_ostream &O) override;
155 
156   void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
157 
158   void EmitFunctionBodyEnd() override;
159 
160   MCSymbol *GetCPISymbol(unsigned CPID) const override;
161   void EmitEndOfAsmFile(Module &M) override;
162 
163   AArch64FunctionInfo *AArch64FI = nullptr;
164 
165   /// Emit the LOHs contained in AArch64FI.
166   void EmitLOHs();
167 
168   /// Emit instruction to set float register to zero.
169   void EmitFMov0(const MachineInstr &MI);
170 
171   using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
172 
173   MInstToMCSymbol LOHInstToLabel;
174 };
175 
176 } // end anonymous namespace
177 
178 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
179 {
180   EmitSled(MI, SledKind::FUNCTION_ENTER);
181 }
182 
183 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI)
184 {
185   EmitSled(MI, SledKind::FUNCTION_EXIT);
186 }
187 
188 void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI)
189 {
190   EmitSled(MI, SledKind::TAIL_CALL);
191 }
192 
193 void AArch64AsmPrinter::EmitSled(const MachineInstr &MI, SledKind Kind)
194 {
195   static const int8_t NoopsInSledCount = 7;
196   // We want to emit the following pattern:
197   //
198   // .Lxray_sled_N:
199   //   ALIGN
200   //   B #32
201   //   ; 7 NOP instructions (28 bytes)
202   // .tmpN
203   //
204   // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
205   // over the full 32 bytes (8 instructions) with the following pattern:
206   //
207   //   STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
208   //   LDR W0, #12 ; W0 := function ID
209   //   LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
210   //   BLR X16 ; call the tracing trampoline
211   //   ;DATA: 32 bits of function ID
212   //   ;DATA: lower 32 bits of the address of the trampoline
213   //   ;DATA: higher 32 bits of the address of the trampoline
214   //   LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
215   //
216   OutStreamer->EmitCodeAlignment(4);
217   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
218   OutStreamer->EmitLabel(CurSled);
219   auto Target = OutContext.createTempSymbol();
220 
221   // Emit "B #32" instruction, which jumps over the next 28 bytes.
222   // The operand has to be the number of 4-byte instructions to jump over,
223   // including the current instruction.
224   EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8));
225 
226   for (int8_t I = 0; I < NoopsInSledCount; I++)
227     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
228 
229   OutStreamer->EmitLabel(Target);
230   recordSled(CurSled, MI, Kind);
231 }
232 
233 void AArch64AsmPrinter::EmitEndOfAsmFile(Module &M) {
234   const Triple &TT = TM.getTargetTriple();
235   if (TT.isOSBinFormatMachO()) {
236     // Funny Darwin hack: This flag tells the linker that no global symbols
237     // contain code that falls through to other global symbols (e.g. the obvious
238     // implementation of multiple entry points).  If this doesn't occur, the
239     // linker can safely perform dead code stripping.  Since LLVM never
240     // generates code that does this, it is always safe to set.
241     OutStreamer->EmitAssemblerFlag(MCAF_SubsectionsViaSymbols);
242     emitStackMaps(SM);
243   }
244 }
245 
246 void AArch64AsmPrinter::EmitLOHs() {
247   SmallVector<MCSymbol *, 3> MCArgs;
248 
249   for (const auto &D : AArch64FI->getLOHContainer()) {
250     for (const MachineInstr *MI : D.getArgs()) {
251       MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI);
252       assert(LabelIt != LOHInstToLabel.end() &&
253              "Label hasn't been inserted for LOH related instruction");
254       MCArgs.push_back(LabelIt->second);
255     }
256     OutStreamer->EmitLOHDirective(D.getKind(), MCArgs);
257     MCArgs.clear();
258   }
259 }
260 
261 void AArch64AsmPrinter::EmitFunctionBodyEnd() {
262   if (!AArch64FI->getLOHRelated().empty())
263     EmitLOHs();
264 }
265 
266 /// GetCPISymbol - Return the symbol for the specified constant pool entry.
267 MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
268   // Darwin uses a linker-private symbol name for constant-pools (to
269   // avoid addends on the relocation?), ELF has no such concept and
270   // uses a normal private symbol.
271   if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
272     return OutContext.getOrCreateSymbol(
273         Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
274         Twine(getFunctionNumber()) + "_" + Twine(CPID));
275 
276   return AsmPrinter::GetCPISymbol(CPID);
277 }
278 
279 void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
280                                      raw_ostream &O) {
281   const MachineOperand &MO = MI->getOperand(OpNum);
282   switch (MO.getType()) {
283   default:
284     llvm_unreachable("<unknown operand type>");
285   case MachineOperand::MO_Register: {
286     unsigned Reg = MO.getReg();
287     assert(TargetRegisterInfo::isPhysicalRegister(Reg));
288     assert(!MO.getSubReg() && "Subregs should be eliminated!");
289     O << AArch64InstPrinter::getRegisterName(Reg);
290     break;
291   }
292   case MachineOperand::MO_Immediate: {
293     int64_t Imm = MO.getImm();
294     O << '#' << Imm;
295     break;
296   }
297   case MachineOperand::MO_GlobalAddress: {
298     const GlobalValue *GV = MO.getGlobal();
299     MCSymbol *Sym = getSymbol(GV);
300 
301     // FIXME: Can we get anything other than a plain symbol here?
302     assert(!MO.getTargetFlags() && "Unknown operand target flag!");
303 
304     Sym->print(O, MAI);
305     printOffset(MO.getOffset(), O);
306     break;
307   }
308   case MachineOperand::MO_BlockAddress: {
309     MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
310     Sym->print(O, MAI);
311     break;
312   }
313   }
314 }
315 
316 bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
317                                           raw_ostream &O) {
318   unsigned Reg = MO.getReg();
319   switch (Mode) {
320   default:
321     return true; // Unknown mode.
322   case 'w':
323     Reg = getWRegFromXReg(Reg);
324     break;
325   case 'x':
326     Reg = getXRegFromWReg(Reg);
327     break;
328   }
329 
330   O << AArch64InstPrinter::getRegisterName(Reg);
331   return false;
332 }
333 
334 // Prints the register in MO using class RC using the offset in the
335 // new register class. This should not be used for cross class
336 // printing.
337 bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
338                                            const TargetRegisterClass *RC,
339                                            bool isVector, raw_ostream &O) {
340   assert(MO.isReg() && "Should only get here with a register!");
341   const TargetRegisterInfo *RI = STI->getRegisterInfo();
342   unsigned Reg = MO.getReg();
343   unsigned RegToPrint = RC->getRegister(RI->getEncodingValue(Reg));
344   assert(RI->regsOverlap(RegToPrint, Reg));
345   O << AArch64InstPrinter::getRegisterName(
346            RegToPrint, isVector ? AArch64::vreg : AArch64::NoRegAltName);
347   return false;
348 }
349 
350 bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
351                                         unsigned AsmVariant,
352                                         const char *ExtraCode, raw_ostream &O) {
353   const MachineOperand &MO = MI->getOperand(OpNum);
354 
355   // First try the generic code, which knows about modifiers like 'c' and 'n'.
356   if (!AsmPrinter::PrintAsmOperand(MI, OpNum, AsmVariant, ExtraCode, O))
357     return false;
358 
359   // Does this asm operand have a single letter operand modifier?
360   if (ExtraCode && ExtraCode[0]) {
361     if (ExtraCode[1] != 0)
362       return true; // Unknown modifier.
363 
364     switch (ExtraCode[0]) {
365     default:
366       return true; // Unknown modifier.
367     case 'a':      // Print 'a' modifier
368       PrintAsmMemoryOperand(MI, OpNum, AsmVariant, ExtraCode, O);
369       return false;
370     case 'w':      // Print W register
371     case 'x':      // Print X register
372       if (MO.isReg())
373         return printAsmMRegister(MO, ExtraCode[0], O);
374       if (MO.isImm() && MO.getImm() == 0) {
375         unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
376         O << AArch64InstPrinter::getRegisterName(Reg);
377         return false;
378       }
379       printOperand(MI, OpNum, O);
380       return false;
381     case 'b': // Print B register.
382     case 'h': // Print H register.
383     case 's': // Print S register.
384     case 'd': // Print D register.
385     case 'q': // Print Q register.
386       if (MO.isReg()) {
387         const TargetRegisterClass *RC;
388         switch (ExtraCode[0]) {
389         case 'b':
390           RC = &AArch64::FPR8RegClass;
391           break;
392         case 'h':
393           RC = &AArch64::FPR16RegClass;
394           break;
395         case 's':
396           RC = &AArch64::FPR32RegClass;
397           break;
398         case 'd':
399           RC = &AArch64::FPR64RegClass;
400           break;
401         case 'q':
402           RC = &AArch64::FPR128RegClass;
403           break;
404         default:
405           return true;
406         }
407         return printAsmRegInClass(MO, RC, false /* vector */, O);
408       }
409       printOperand(MI, OpNum, O);
410       return false;
411     }
412   }
413 
414   // According to ARM, we should emit x and v registers unless we have a
415   // modifier.
416   if (MO.isReg()) {
417     unsigned Reg = MO.getReg();
418 
419     // If this is a w or x register, print an x register.
420     if (AArch64::GPR32allRegClass.contains(Reg) ||
421         AArch64::GPR64allRegClass.contains(Reg))
422       return printAsmMRegister(MO, 'x', O);
423 
424     // If this is a b, h, s, d, or q register, print it as a v register.
425     return printAsmRegInClass(MO, &AArch64::FPR128RegClass, true /* vector */,
426                               O);
427   }
428 
429   printOperand(MI, OpNum, O);
430   return false;
431 }
432 
433 bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
434                                               unsigned OpNum,
435                                               unsigned AsmVariant,
436                                               const char *ExtraCode,
437                                               raw_ostream &O) {
438   if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
439     return true; // Unknown modifier.
440 
441   const MachineOperand &MO = MI->getOperand(OpNum);
442   assert(MO.isReg() && "unexpected inline asm memory operand");
443   O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]";
444   return false;
445 }
446 
447 void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
448                                                raw_ostream &OS) {
449   unsigned NOps = MI->getNumOperands();
450   assert(NOps == 4);
451   OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
452   // cast away const; DIetc do not take const operands for some reason.
453   OS << cast<DILocalVariable>(MI->getOperand(NOps - 2).getMetadata())
454             ->getName();
455   OS << " <- ";
456   // Frame address.  Currently handles register +- offset only.
457   assert(MI->getOperand(0).isReg() && MI->getOperand(1).isImm());
458   OS << '[';
459   printOperand(MI, 0, OS);
460   OS << '+';
461   printOperand(MI, 1, OS);
462   OS << ']';
463   OS << "+";
464   printOperand(MI, NOps - 2, OS);
465 }
466 
467 void AArch64AsmPrinter::EmitJumpTableInfo() {
468   const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
469   if (!MJTI) return;
470 
471   const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
472   if (JT.empty()) return;
473 
474   const Function &F = MF->getFunction();
475   const TargetLoweringObjectFile &TLOF = getObjFileLowering();
476   bool JTInDiffSection =
477       !STI->isTargetCOFF() ||
478       !TLOF.shouldPutJumpTableInFunctionSection(
479           MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32,
480           F);
481   if (JTInDiffSection) {
482       // Drop it in the readonly section.
483       MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(F, TM);
484       OutStreamer->SwitchSection(ReadOnlySec);
485   }
486 
487   auto AFI = MF->getInfo<AArch64FunctionInfo>();
488   for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
489     const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
490 
491     // If this jump table was deleted, ignore it.
492     if (JTBBs.empty()) continue;
493 
494     unsigned Size = AFI->getJumpTableEntrySize(JTI);
495     EmitAlignment(Log2_32(Size));
496     OutStreamer->EmitLabel(GetJTISymbol(JTI));
497 
498     for (auto *JTBB : JTBBs)
499       emitJumpTableEntry(MJTI, JTBB, JTI);
500   }
501 }
502 
503 void AArch64AsmPrinter::emitJumpTableEntry(const MachineJumpTableInfo *MJTI,
504                                            const MachineBasicBlock *MBB,
505                                            unsigned JTI) {
506   const MCExpr *Value = MCSymbolRefExpr::create(MBB->getSymbol(), OutContext);
507   auto AFI = MF->getInfo<AArch64FunctionInfo>();
508   unsigned Size = AFI->getJumpTableEntrySize(JTI);
509 
510   if (Size == 4) {
511     // .word LBB - LJTI
512     const TargetLowering *TLI = MF->getSubtarget().getTargetLowering();
513     const MCExpr *Base = TLI->getPICJumpTableRelocBaseExpr(MF, JTI, OutContext);
514     Value = MCBinaryExpr::createSub(Value, Base, OutContext);
515   } else {
516     // .byte (LBB - LBB) >> 2 (or .hword)
517     const MCSymbol *BaseSym = AFI->getJumpTableEntryPCRelSymbol(JTI);
518     const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext);
519     Value = MCBinaryExpr::createSub(Value, Base, OutContext);
520     Value = MCBinaryExpr::createLShr(
521         Value, MCConstantExpr::create(2, OutContext), OutContext);
522   }
523 
524   OutStreamer->EmitValue(Value, Size);
525 }
526 
527 /// Small jump tables contain an unsigned byte or half, representing the offset
528 /// from the lowest-addressed possible destination to the desired basic
529 /// block. Since all instructions are 4-byte aligned, this is further compressed
530 /// by counting in instructions rather than bytes (i.e. divided by 4). So, to
531 /// materialize the correct destination we need:
532 ///
533 ///             adr xDest, .LBB0_0
534 ///             ldrb wScratch, [xTable, xEntry]   (with "lsl #1" for ldrh).
535 ///             add xDest, xDest, xScratch, lsl #2
536 void AArch64AsmPrinter::LowerJumpTableDestSmall(llvm::MCStreamer &OutStreamer,
537                                                 const llvm::MachineInstr &MI) {
538   unsigned DestReg = MI.getOperand(0).getReg();
539   unsigned ScratchReg = MI.getOperand(1).getReg();
540   unsigned ScratchRegW =
541       STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32);
542   unsigned TableReg = MI.getOperand(2).getReg();
543   unsigned EntryReg = MI.getOperand(3).getReg();
544   int JTIdx = MI.getOperand(4).getIndex();
545   bool IsByteEntry = MI.getOpcode() == AArch64::JumpTableDest8;
546 
547   // This has to be first because the compression pass based its reachability
548   // calculations on the start of the JumpTableDest instruction.
549   auto Label =
550       MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx);
551   EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR)
552                                   .addReg(DestReg)
553                                   .addExpr(MCSymbolRefExpr::create(
554                                       Label, MF->getContext())));
555 
556   // Load the number of instruction-steps to offset from the label.
557   unsigned LdrOpcode = IsByteEntry ? AArch64::LDRBBroX : AArch64::LDRHHroX;
558   EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode)
559                                   .addReg(ScratchRegW)
560                                   .addReg(TableReg)
561                                   .addReg(EntryReg)
562                                   .addImm(0)
563                                   .addImm(IsByteEntry ? 0 : 1));
564 
565   // Multiply the steps by 4 and add to the already materialized base label
566   // address.
567   EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs)
568                                   .addReg(DestReg)
569                                   .addReg(DestReg)
570                                   .addReg(ScratchReg)
571                                   .addImm(2));
572 }
573 
574 void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
575                                       const MachineInstr &MI) {
576   unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
577 
578   SM.recordStackMap(MI);
579   assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
580 
581   // Scan ahead to trim the shadow.
582   const MachineBasicBlock &MBB = *MI.getParent();
583   MachineBasicBlock::const_iterator MII(MI);
584   ++MII;
585   while (NumNOPBytes > 0) {
586     if (MII == MBB.end() || MII->isCall() ||
587         MII->getOpcode() == AArch64::DBG_VALUE ||
588         MII->getOpcode() == TargetOpcode::PATCHPOINT ||
589         MII->getOpcode() == TargetOpcode::STACKMAP)
590       break;
591     ++MII;
592     NumNOPBytes -= 4;
593   }
594 
595   // Emit nops.
596   for (unsigned i = 0; i < NumNOPBytes; i += 4)
597     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
598 }
599 
600 // Lower a patchpoint of the form:
601 // [<def>], <id>, <numBytes>, <target>, <numArgs>
602 void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
603                                         const MachineInstr &MI) {
604   SM.recordPatchPoint(MI);
605 
606   PatchPointOpers Opers(&MI);
607 
608   int64_t CallTarget = Opers.getCallTarget().getImm();
609   unsigned EncodedBytes = 0;
610   if (CallTarget) {
611     assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
612            "High 16 bits of call target should be zero.");
613     unsigned ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
614     EncodedBytes = 16;
615     // Materialize the jump address:
616     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi)
617                                     .addReg(ScratchReg)
618                                     .addImm((CallTarget >> 32) & 0xFFFF)
619                                     .addImm(32));
620     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
621                                     .addReg(ScratchReg)
622                                     .addReg(ScratchReg)
623                                     .addImm((CallTarget >> 16) & 0xFFFF)
624                                     .addImm(16));
625     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
626                                     .addReg(ScratchReg)
627                                     .addReg(ScratchReg)
628                                     .addImm(CallTarget & 0xFFFF)
629                                     .addImm(0));
630     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
631   }
632   // Emit padding.
633   unsigned NumBytes = Opers.getNumPatchBytes();
634   assert(NumBytes >= EncodedBytes &&
635          "Patchpoint can't request size less than the length of a call.");
636   assert((NumBytes - EncodedBytes) % 4 == 0 &&
637          "Invalid number of NOP bytes requested!");
638   for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
639     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
640 }
641 
642 void AArch64AsmPrinter::EmitFMov0(const MachineInstr &MI) {
643   unsigned DestReg = MI.getOperand(0).getReg();
644   if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround()) {
645     // Convert H/S/D register to corresponding Q register
646     if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31)
647       DestReg = AArch64::Q0 + (DestReg - AArch64::H0);
648     else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31)
649       DestReg = AArch64::Q0 + (DestReg - AArch64::S0);
650     else {
651       assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31);
652       DestReg = AArch64::Q0 + (DestReg - AArch64::D0);
653     }
654     MCInst MOVI;
655     MOVI.setOpcode(AArch64::MOVIv2d_ns);
656     MOVI.addOperand(MCOperand::createReg(DestReg));
657     MOVI.addOperand(MCOperand::createImm(0));
658     EmitToStreamer(*OutStreamer, MOVI);
659   } else {
660     MCInst FMov;
661     switch (MI.getOpcode()) {
662     default: llvm_unreachable("Unexpected opcode");
663     case AArch64::FMOVH0:
664       FMov.setOpcode(AArch64::FMOVWHr);
665       FMov.addOperand(MCOperand::createReg(DestReg));
666       FMov.addOperand(MCOperand::createReg(AArch64::WZR));
667       break;
668     case AArch64::FMOVS0:
669       FMov.setOpcode(AArch64::FMOVWSr);
670       FMov.addOperand(MCOperand::createReg(DestReg));
671       FMov.addOperand(MCOperand::createReg(AArch64::WZR));
672       break;
673     case AArch64::FMOVD0:
674       FMov.setOpcode(AArch64::FMOVXDr);
675       FMov.addOperand(MCOperand::createReg(DestReg));
676       FMov.addOperand(MCOperand::createReg(AArch64::XZR));
677       break;
678     }
679     EmitToStreamer(*OutStreamer, FMov);
680   }
681 }
682 
683 // Simple pseudo-instructions have their lowering (with expansion to real
684 // instructions) auto-generated.
685 #include "AArch64GenMCPseudoLowering.inc"
686 
687 void AArch64AsmPrinter::EmitInstruction(const MachineInstr *MI) {
688   // Do any auto-generated pseudo lowerings.
689   if (emitPseudoExpansionLowering(*OutStreamer, MI))
690     return;
691 
692   if (AArch64FI->getLOHRelated().count(MI)) {
693     // Generate a label for LOH related instruction
694     MCSymbol *LOHLabel = createTempSymbol("loh");
695     // Associate the instruction with the label
696     LOHInstToLabel[MI] = LOHLabel;
697     OutStreamer->EmitLabel(LOHLabel);
698   }
699 
700   AArch64TargetStreamer *TS =
701     static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
702   // Do any manual lowerings.
703   switch (MI->getOpcode()) {
704   default:
705     break;
706     case AArch64::MOVMCSym: {
707     unsigned DestReg = MI->getOperand(0).getReg();
708     const MachineOperand &MO_Sym = MI->getOperand(1);
709     MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
710     MCOperand Hi_MCSym, Lo_MCSym;
711 
712     Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
713     Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
714 
715     MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym);
716     MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym);
717 
718     MCInst MovZ;
719     MovZ.setOpcode(AArch64::MOVZXi);
720     MovZ.addOperand(MCOperand::createReg(DestReg));
721     MovZ.addOperand(Hi_MCSym);
722     MovZ.addOperand(MCOperand::createImm(16));
723     EmitToStreamer(*OutStreamer, MovZ);
724 
725     MCInst MovK;
726     MovK.setOpcode(AArch64::MOVKXi);
727     MovK.addOperand(MCOperand::createReg(DestReg));
728     MovK.addOperand(MCOperand::createReg(DestReg));
729     MovK.addOperand(Lo_MCSym);
730     MovK.addOperand(MCOperand::createImm(0));
731     EmitToStreamer(*OutStreamer, MovK);
732     return;
733   }
734   case AArch64::MOVIv2d_ns:
735     // If the target has <rdar://problem/16473581>, lower this
736     // instruction to movi.16b instead.
737     if (STI->hasZeroCycleZeroingFPWorkaround() &&
738         MI->getOperand(1).getImm() == 0) {
739       MCInst TmpInst;
740       TmpInst.setOpcode(AArch64::MOVIv16b_ns);
741       TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
742       TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm()));
743       EmitToStreamer(*OutStreamer, TmpInst);
744       return;
745     }
746     break;
747 
748   case AArch64::DBG_VALUE: {
749     if (isVerbose() && OutStreamer->hasRawTextSupport()) {
750       SmallString<128> TmpStr;
751       raw_svector_ostream OS(TmpStr);
752       PrintDebugValueComment(MI, OS);
753       OutStreamer->EmitRawText(StringRef(OS.str()));
754     }
755     return;
756 
757   case AArch64::EMITBKEY: {
758       ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
759       if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
760           ExceptionHandlingType != ExceptionHandling::ARM)
761         return;
762 
763       if (needsCFIMoves() == CFI_M_None)
764         return;
765 
766       OutStreamer->EmitCFIBKeyFrame();
767       return;
768     }
769   }
770 
771   // Tail calls use pseudo instructions so they have the proper code-gen
772   // attributes (isCall, isReturn, etc.). We lower them to the real
773   // instruction here.
774   case AArch64::TCRETURNri:
775   case AArch64::TCRETURNriBTI:
776   case AArch64::TCRETURNriALL: {
777     MCInst TmpInst;
778     TmpInst.setOpcode(AArch64::BR);
779     TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
780     EmitToStreamer(*OutStreamer, TmpInst);
781     return;
782   }
783   case AArch64::TCRETURNdi: {
784     MCOperand Dest;
785     MCInstLowering.lowerOperand(MI->getOperand(0), Dest);
786     MCInst TmpInst;
787     TmpInst.setOpcode(AArch64::B);
788     TmpInst.addOperand(Dest);
789     EmitToStreamer(*OutStreamer, TmpInst);
790     return;
791   }
792   case AArch64::TLSDESC_CALLSEQ: {
793     /// lower this to:
794     ///    adrp  x0, :tlsdesc:var
795     ///    ldr   x1, [x0, #:tlsdesc_lo12:var]
796     ///    add   x0, x0, #:tlsdesc_lo12:var
797     ///    .tlsdesccall var
798     ///    blr   x1
799     ///    (TPIDR_EL0 offset now in x0)
800     const MachineOperand &MO_Sym = MI->getOperand(0);
801     MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
802     MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
803     MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
804     MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
805     MCInstLowering.lowerOperand(MO_Sym, Sym);
806     MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12);
807     MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc);
808 
809     MCInst Adrp;
810     Adrp.setOpcode(AArch64::ADRP);
811     Adrp.addOperand(MCOperand::createReg(AArch64::X0));
812     Adrp.addOperand(SymTLSDesc);
813     EmitToStreamer(*OutStreamer, Adrp);
814 
815     MCInst Ldr;
816     Ldr.setOpcode(AArch64::LDRXui);
817     Ldr.addOperand(MCOperand::createReg(AArch64::X1));
818     Ldr.addOperand(MCOperand::createReg(AArch64::X0));
819     Ldr.addOperand(SymTLSDescLo12);
820     Ldr.addOperand(MCOperand::createImm(0));
821     EmitToStreamer(*OutStreamer, Ldr);
822 
823     MCInst Add;
824     Add.setOpcode(AArch64::ADDXri);
825     Add.addOperand(MCOperand::createReg(AArch64::X0));
826     Add.addOperand(MCOperand::createReg(AArch64::X0));
827     Add.addOperand(SymTLSDescLo12);
828     Add.addOperand(MCOperand::createImm(AArch64_AM::getShiftValue(0)));
829     EmitToStreamer(*OutStreamer, Add);
830 
831     // Emit a relocation-annotation. This expands to no code, but requests
832     // the following instruction gets an R_AARCH64_TLSDESC_CALL.
833     MCInst TLSDescCall;
834     TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
835     TLSDescCall.addOperand(Sym);
836     EmitToStreamer(*OutStreamer, TLSDescCall);
837 
838     MCInst Blr;
839     Blr.setOpcode(AArch64::BLR);
840     Blr.addOperand(MCOperand::createReg(AArch64::X1));
841     EmitToStreamer(*OutStreamer, Blr);
842 
843     return;
844   }
845 
846   case AArch64::JumpTableDest32: {
847     // We want:
848     //     ldrsw xScratch, [xTable, xEntry, lsl #2]
849     //     add xDest, xTable, xScratch
850     unsigned DestReg = MI->getOperand(0).getReg(),
851              ScratchReg = MI->getOperand(1).getReg(),
852              TableReg = MI->getOperand(2).getReg(),
853              EntryReg = MI->getOperand(3).getReg();
854     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDRSWroX)
855                                      .addReg(ScratchReg)
856                                      .addReg(TableReg)
857                                      .addReg(EntryReg)
858                                      .addImm(0)
859                                      .addImm(1));
860     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADDXrs)
861                                      .addReg(DestReg)
862                                      .addReg(TableReg)
863                                      .addReg(ScratchReg)
864                                      .addImm(0));
865     return;
866   }
867   case AArch64::JumpTableDest16:
868   case AArch64::JumpTableDest8:
869     LowerJumpTableDestSmall(*OutStreamer, *MI);
870     return;
871 
872   case AArch64::FMOVH0:
873   case AArch64::FMOVS0:
874   case AArch64::FMOVD0:
875     EmitFMov0(*MI);
876     return;
877 
878   case TargetOpcode::STACKMAP:
879     return LowerSTACKMAP(*OutStreamer, SM, *MI);
880 
881   case TargetOpcode::PATCHPOINT:
882     return LowerPATCHPOINT(*OutStreamer, SM, *MI);
883 
884   case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
885     LowerPATCHABLE_FUNCTION_ENTER(*MI);
886     return;
887 
888   case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
889     LowerPATCHABLE_FUNCTION_EXIT(*MI);
890     return;
891 
892   case TargetOpcode::PATCHABLE_TAIL_CALL:
893     LowerPATCHABLE_TAIL_CALL(*MI);
894     return;
895 
896   case AArch64::SEH_StackAlloc:
897     TS->EmitARM64WinCFIAllocStack(MI->getOperand(0).getImm());
898     return;
899 
900   case AArch64::SEH_SaveFPLR:
901     TS->EmitARM64WinCFISaveFPLR(MI->getOperand(0).getImm());
902     return;
903 
904   case AArch64::SEH_SaveFPLR_X:
905     assert(MI->getOperand(0).getImm() < 0 &&
906            "Pre increment SEH opcode must have a negative offset");
907     TS->EmitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm());
908     return;
909 
910   case AArch64::SEH_SaveReg:
911     TS->EmitARM64WinCFISaveReg(MI->getOperand(0).getImm(),
912                                MI->getOperand(1).getImm());
913     return;
914 
915   case AArch64::SEH_SaveReg_X:
916     assert(MI->getOperand(1).getImm() < 0 &&
917            "Pre increment SEH opcode must have a negative offset");
918     TS->EmitARM64WinCFISaveRegX(MI->getOperand(0).getImm(),
919 		                -MI->getOperand(1).getImm());
920     return;
921 
922   case AArch64::SEH_SaveRegP:
923     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
924             "Non-consecutive registers not allowed for save_regp");
925     TS->EmitARM64WinCFISaveRegP(MI->getOperand(0).getImm(),
926                                 MI->getOperand(2).getImm());
927     return;
928 
929   case AArch64::SEH_SaveRegP_X:
930     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
931             "Non-consecutive registers not allowed for save_regp_x");
932     assert(MI->getOperand(2).getImm() < 0 &&
933            "Pre increment SEH opcode must have a negative offset");
934     TS->EmitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(),
935                                  -MI->getOperand(2).getImm());
936     return;
937 
938   case AArch64::SEH_SaveFReg:
939     TS->EmitARM64WinCFISaveFReg(MI->getOperand(0).getImm(),
940                                 MI->getOperand(1).getImm());
941     return;
942 
943   case AArch64::SEH_SaveFReg_X:
944     assert(MI->getOperand(1).getImm() < 0 &&
945            "Pre increment SEH opcode must have a negative offset");
946     TS->EmitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(),
947                                  -MI->getOperand(1).getImm());
948     return;
949 
950   case AArch64::SEH_SaveFRegP:
951     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
952             "Non-consecutive registers not allowed for save_regp");
953     TS->EmitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(),
954                                  MI->getOperand(2).getImm());
955     return;
956 
957   case AArch64::SEH_SaveFRegP_X:
958     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
959             "Non-consecutive registers not allowed for save_regp_x");
960     assert(MI->getOperand(2).getImm() < 0 &&
961            "Pre increment SEH opcode must have a negative offset");
962     TS->EmitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(),
963                                   -MI->getOperand(2).getImm());
964     return;
965 
966   case AArch64::SEH_SetFP:
967     TS->EmitARM64WinCFISetFP();
968     return;
969 
970   case AArch64::SEH_AddFP:
971     TS->EmitARM64WinCFIAddFP(MI->getOperand(0).getImm());
972     return;
973 
974   case AArch64::SEH_Nop:
975     TS->EmitARM64WinCFINop();
976     return;
977 
978   case AArch64::SEH_PrologEnd:
979     TS->EmitARM64WinCFIPrologEnd();
980     return;
981 
982   case AArch64::SEH_EpilogStart:
983     TS->EmitARM64WinCFIEpilogStart();
984     return;
985 
986   case AArch64::SEH_EpilogEnd:
987     TS->EmitARM64WinCFIEpilogEnd();
988     return;
989   }
990 
991   // Finally, do the automated lowerings for everything else.
992   MCInst TmpInst;
993   MCInstLowering.Lower(MI, TmpInst);
994   EmitToStreamer(*OutStreamer, TmpInst);
995 }
996 
997 // Force static initialization.
998 extern "C" void LLVMInitializeAArch64AsmPrinter() {
999   RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
1000   RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget());
1001   RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target());
1002 }
1003