1 //===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains a printer that converts from our internal representation
10 // of machine-dependent LLVM code to the AArch64 assembly language.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "AArch64.h"
15 #include "AArch64MCInstLower.h"
16 #include "AArch64MachineFunctionInfo.h"
17 #include "AArch64RegisterInfo.h"
18 #include "AArch64Subtarget.h"
19 #include "AArch64TargetObjectFile.h"
20 #include "MCTargetDesc/AArch64AddressingModes.h"
21 #include "MCTargetDesc/AArch64InstPrinter.h"
22 #include "MCTargetDesc/AArch64MCExpr.h"
23 #include "MCTargetDesc/AArch64MCTargetDesc.h"
24 #include "MCTargetDesc/AArch64TargetStreamer.h"
25 #include "TargetInfo/AArch64TargetInfo.h"
26 #include "Utils/AArch64BaseInfo.h"
27 #include "llvm/ADT/SmallString.h"
28 #include "llvm/ADT/SmallVector.h"
29 #include "llvm/ADT/StringRef.h"
30 #include "llvm/ADT/Triple.h"
31 #include "llvm/ADT/Twine.h"
32 #include "llvm/BinaryFormat/COFF.h"
33 #include "llvm/BinaryFormat/ELF.h"
34 #include "llvm/CodeGen/AsmPrinter.h"
35 #include "llvm/CodeGen/FaultMaps.h"
36 #include "llvm/CodeGen/MachineBasicBlock.h"
37 #include "llvm/CodeGen/MachineFunction.h"
38 #include "llvm/CodeGen/MachineInstr.h"
39 #include "llvm/CodeGen/MachineJumpTableInfo.h"
40 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
41 #include "llvm/CodeGen/MachineOperand.h"
42 #include "llvm/CodeGen/StackMaps.h"
43 #include "llvm/CodeGen/TargetRegisterInfo.h"
44 #include "llvm/IR/DataLayout.h"
45 #include "llvm/IR/DebugInfoMetadata.h"
46 #include "llvm/MC/MCAsmInfo.h"
47 #include "llvm/MC/MCContext.h"
48 #include "llvm/MC/MCInst.h"
49 #include "llvm/MC/MCInstBuilder.h"
50 #include "llvm/MC/MCSectionELF.h"
51 #include "llvm/MC/MCStreamer.h"
52 #include "llvm/MC/MCSymbol.h"
53 #include "llvm/Support/Casting.h"
54 #include "llvm/Support/ErrorHandling.h"
55 #include "llvm/Support/TargetRegistry.h"
56 #include "llvm/Support/raw_ostream.h"
57 #include "llvm/Target/TargetMachine.h"
58 #include <algorithm>
59 #include <cassert>
60 #include <cstdint>
61 #include <map>
62 #include <memory>
63 
64 using namespace llvm;
65 
66 #define DEBUG_TYPE "asm-printer"
67 
68 namespace {
69 
70 class AArch64AsmPrinter : public AsmPrinter {
71   AArch64MCInstLower MCInstLowering;
72   StackMaps SM;
73   FaultMaps FM;
74   const AArch64Subtarget *STI;
75 
76 public:
77   AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
78       : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this),
79         SM(*this), FM(*this) {}
80 
81   StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
82 
83   /// Wrapper for MCInstLowering.lowerOperand() for the
84   /// tblgen'erated pseudo lowering.
85   bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
86     return MCInstLowering.lowerOperand(MO, MCOp);
87   }
88 
89   void emitStartOfAsmFile(Module &M) override;
90   void emitJumpTableInfo() override;
91 
92   void LowerJumpTableDest(MCStreamer &OutStreamer, const MachineInstr &MI);
93 
94   void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
95                      const MachineInstr &MI);
96   void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
97                        const MachineInstr &MI);
98   void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
99                        const MachineInstr &MI);
100   void LowerFAULTING_OP(const MachineInstr &MI);
101 
102   void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
103   void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
104   void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
105 
106   typedef std::tuple<unsigned, bool, uint32_t> HwasanMemaccessTuple;
107   std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
108   void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
109   void EmitHwasanMemaccessSymbols(Module &M);
110 
111   void EmitSled(const MachineInstr &MI, SledKind Kind);
112 
113   /// tblgen'erated driver function for lowering simple MI->MC
114   /// pseudo instructions.
115   bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
116                                    const MachineInstr *MI);
117 
118   void emitInstruction(const MachineInstr *MI) override;
119 
120   void emitFunctionHeaderComment() override;
121 
122   void getAnalysisUsage(AnalysisUsage &AU) const override {
123     AsmPrinter::getAnalysisUsage(AU);
124     AU.setPreservesAll();
125   }
126 
127   bool runOnMachineFunction(MachineFunction &MF) override {
128     AArch64FI = MF.getInfo<AArch64FunctionInfo>();
129     STI = static_cast<const AArch64Subtarget*>(&MF.getSubtarget());
130 
131     SetupMachineFunction(MF);
132 
133     if (STI->isTargetCOFF()) {
134       bool Internal = MF.getFunction().hasInternalLinkage();
135       COFF::SymbolStorageClass Scl = Internal ? COFF::IMAGE_SYM_CLASS_STATIC
136                                               : COFF::IMAGE_SYM_CLASS_EXTERNAL;
137       int Type =
138         COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT;
139 
140       OutStreamer->BeginCOFFSymbolDef(CurrentFnSym);
141       OutStreamer->EmitCOFFSymbolStorageClass(Scl);
142       OutStreamer->EmitCOFFSymbolType(Type);
143       OutStreamer->EndCOFFSymbolDef();
144     }
145 
146     // Emit the rest of the function body.
147     emitFunctionBody();
148 
149     // Emit the XRay table for this function.
150     emitXRayTable();
151 
152     // We didn't modify anything.
153     return false;
154   }
155 
156 private:
157   void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
158   bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
159   bool printAsmRegInClass(const MachineOperand &MO,
160                           const TargetRegisterClass *RC, unsigned AltName,
161                           raw_ostream &O);
162 
163   bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
164                        const char *ExtraCode, raw_ostream &O) override;
165   bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
166                              const char *ExtraCode, raw_ostream &O) override;
167 
168   void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
169 
170   void emitFunctionBodyEnd() override;
171 
172   MCSymbol *GetCPISymbol(unsigned CPID) const override;
173   void emitEndOfAsmFile(Module &M) override;
174 
175   AArch64FunctionInfo *AArch64FI = nullptr;
176 
177   /// Emit the LOHs contained in AArch64FI.
178   void EmitLOHs();
179 
180   /// Emit instruction to set float register to zero.
181   void EmitFMov0(const MachineInstr &MI);
182 
183   using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
184 
185   MInstToMCSymbol LOHInstToLabel;
186 };
187 
188 } // end anonymous namespace
189 
190 void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
191   if (!TM.getTargetTriple().isOSBinFormatELF())
192     return;
193 
194   // Assemble feature flags that may require creation of a note section.
195   unsigned Flags = 0;
196   if (const auto *BTE = mdconst::extract_or_null<ConstantInt>(
197           M.getModuleFlag("branch-target-enforcement")))
198     if (BTE->getZExtValue())
199       Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI;
200 
201   if (const auto *Sign = mdconst::extract_or_null<ConstantInt>(
202           M.getModuleFlag("sign-return-address")))
203     if (Sign->getZExtValue())
204       Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
205 
206   if (Flags == 0)
207     return;
208 
209   // Emit a .note.gnu.property section with the flags.
210   if (auto *TS = static_cast<AArch64TargetStreamer *>(
211           OutStreamer->getTargetStreamer()))
212     TS->emitNoteSection(Flags);
213 }
214 
215 void AArch64AsmPrinter::emitFunctionHeaderComment() {
216   const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>();
217   Optional<std::string> OutlinerString = FI->getOutliningStyle();
218   if (OutlinerString != None)
219     OutStreamer->GetCommentOS() << ' ' << OutlinerString;
220 }
221 
222 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
223 {
224   const Function &F = MF->getFunction();
225   if (F.hasFnAttribute("patchable-function-entry")) {
226     unsigned Num;
227     if (F.getFnAttribute("patchable-function-entry")
228             .getValueAsString()
229             .getAsInteger(10, Num))
230       return;
231     emitNops(Num);
232     return;
233   }
234 
235   EmitSled(MI, SledKind::FUNCTION_ENTER);
236 }
237 
238 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI)
239 {
240   EmitSled(MI, SledKind::FUNCTION_EXIT);
241 }
242 
243 void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI)
244 {
245   EmitSled(MI, SledKind::TAIL_CALL);
246 }
247 
248 void AArch64AsmPrinter::EmitSled(const MachineInstr &MI, SledKind Kind)
249 {
250   static const int8_t NoopsInSledCount = 7;
251   // We want to emit the following pattern:
252   //
253   // .Lxray_sled_N:
254   //   ALIGN
255   //   B #32
256   //   ; 7 NOP instructions (28 bytes)
257   // .tmpN
258   //
259   // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
260   // over the full 32 bytes (8 instructions) with the following pattern:
261   //
262   //   STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
263   //   LDR W0, #12 ; W0 := function ID
264   //   LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
265   //   BLR X16 ; call the tracing trampoline
266   //   ;DATA: 32 bits of function ID
267   //   ;DATA: lower 32 bits of the address of the trampoline
268   //   ;DATA: higher 32 bits of the address of the trampoline
269   //   LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
270   //
271   OutStreamer->emitCodeAlignment(4);
272   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
273   OutStreamer->emitLabel(CurSled);
274   auto Target = OutContext.createTempSymbol();
275 
276   // Emit "B #32" instruction, which jumps over the next 28 bytes.
277   // The operand has to be the number of 4-byte instructions to jump over,
278   // including the current instruction.
279   EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8));
280 
281   for (int8_t I = 0; I < NoopsInSledCount; I++)
282     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
283 
284   OutStreamer->emitLabel(Target);
285   recordSled(CurSled, MI, Kind, 2);
286 }
287 
288 void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
289   Register Reg = MI.getOperand(0).getReg();
290   bool IsShort =
291       MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES;
292   uint32_t AccessInfo = MI.getOperand(1).getImm();
293   MCSymbol *&Sym =
294       HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, IsShort, AccessInfo)];
295   if (!Sym) {
296     // FIXME: Make this work on non-ELF.
297     if (!TM.getTargetTriple().isOSBinFormatELF())
298       report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF");
299 
300     std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" +
301                           utostr(AccessInfo);
302     if (IsShort)
303       SymName += "_short";
304     Sym = OutContext.getOrCreateSymbol(SymName);
305   }
306 
307   EmitToStreamer(*OutStreamer,
308                  MCInstBuilder(AArch64::BL)
309                      .addExpr(MCSymbolRefExpr::create(Sym, OutContext)));
310 }
311 
312 void AArch64AsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
313   if (HwasanMemaccessSymbols.empty())
314     return;
315 
316   const Triple &TT = TM.getTargetTriple();
317   assert(TT.isOSBinFormatELF());
318   std::unique_ptr<MCSubtargetInfo> STI(
319       TM.getTarget().createMCSubtargetInfo(TT.str(), "", ""));
320 
321   MCSymbol *HwasanTagMismatchV1Sym =
322       OutContext.getOrCreateSymbol("__hwasan_tag_mismatch");
323   MCSymbol *HwasanTagMismatchV2Sym =
324       OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2");
325 
326   const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
327       MCSymbolRefExpr::create(HwasanTagMismatchV1Sym, OutContext);
328   const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
329       MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext);
330 
331   for (auto &P : HwasanMemaccessSymbols) {
332     unsigned Reg = std::get<0>(P.first);
333     bool IsShort = std::get<1>(P.first);
334     uint32_t AccessInfo = std::get<2>(P.first);
335     const MCSymbolRefExpr *HwasanTagMismatchRef =
336         IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
337     MCSymbol *Sym = P.second;
338 
339     OutStreamer->SwitchSection(OutContext.getELFSection(
340         ".text.hot", ELF::SHT_PROGBITS,
341         ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0,
342         Sym->getName()));
343 
344     OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
345     OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
346     OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden);
347     OutStreamer->emitLabel(Sym);
348 
349     OutStreamer->emitInstruction(MCInstBuilder(AArch64::UBFMXri)
350                                      .addReg(AArch64::X16)
351                                      .addReg(Reg)
352                                      .addImm(4)
353                                      .addImm(55),
354                                  *STI);
355     OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRBBroX)
356                                      .addReg(AArch64::W16)
357                                      .addReg(AArch64::X9)
358                                      .addReg(AArch64::X16)
359                                      .addImm(0)
360                                      .addImm(0),
361                                  *STI);
362     OutStreamer->emitInstruction(
363         MCInstBuilder(AArch64::SUBSXrs)
364             .addReg(AArch64::XZR)
365             .addReg(AArch64::X16)
366             .addReg(Reg)
367             .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
368         *STI);
369     MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
370     OutStreamer->emitInstruction(
371         MCInstBuilder(AArch64::Bcc)
372             .addImm(AArch64CC::NE)
373             .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym,
374                                              OutContext)),
375         *STI);
376     MCSymbol *ReturnSym = OutContext.createTempSymbol();
377     OutStreamer->emitLabel(ReturnSym);
378     OutStreamer->emitInstruction(
379         MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI);
380     OutStreamer->emitLabel(HandleMismatchOrPartialSym);
381 
382     if (IsShort) {
383       OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWri)
384                                        .addReg(AArch64::WZR)
385                                        .addReg(AArch64::W16)
386                                        .addImm(15)
387                                        .addImm(0),
388                                    *STI);
389       MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
390       OutStreamer->emitInstruction(
391           MCInstBuilder(AArch64::Bcc)
392               .addImm(AArch64CC::HI)
393               .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
394           *STI);
395 
396       OutStreamer->emitInstruction(
397           MCInstBuilder(AArch64::ANDXri)
398               .addReg(AArch64::X17)
399               .addReg(Reg)
400               .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
401           *STI);
402       unsigned Size = 1 << (AccessInfo & 0xf);
403       if (Size != 1)
404         OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
405                                          .addReg(AArch64::X17)
406                                          .addReg(AArch64::X17)
407                                          .addImm(Size - 1)
408                                          .addImm(0),
409                                      *STI);
410       OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWrs)
411                                        .addReg(AArch64::WZR)
412                                        .addReg(AArch64::W16)
413                                        .addReg(AArch64::W17)
414                                        .addImm(0),
415                                    *STI);
416       OutStreamer->emitInstruction(
417           MCInstBuilder(AArch64::Bcc)
418               .addImm(AArch64CC::LS)
419               .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
420           *STI);
421 
422       OutStreamer->emitInstruction(
423           MCInstBuilder(AArch64::ORRXri)
424               .addReg(AArch64::X16)
425               .addReg(Reg)
426               .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
427           *STI);
428       OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRBBui)
429                                        .addReg(AArch64::W16)
430                                        .addReg(AArch64::X16)
431                                        .addImm(0),
432                                    *STI);
433       OutStreamer->emitInstruction(
434           MCInstBuilder(AArch64::SUBSXrs)
435               .addReg(AArch64::XZR)
436               .addReg(AArch64::X16)
437               .addReg(Reg)
438               .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
439           *STI);
440       OutStreamer->emitInstruction(
441           MCInstBuilder(AArch64::Bcc)
442               .addImm(AArch64CC::EQ)
443               .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
444           *STI);
445 
446       OutStreamer->emitLabel(HandleMismatchSym);
447     }
448 
449     OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre)
450                                      .addReg(AArch64::SP)
451                                      .addReg(AArch64::X0)
452                                      .addReg(AArch64::X1)
453                                      .addReg(AArch64::SP)
454                                      .addImm(-32),
455                                  *STI);
456     OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi)
457                                      .addReg(AArch64::FP)
458                                      .addReg(AArch64::LR)
459                                      .addReg(AArch64::SP)
460                                      .addImm(29),
461                                  *STI);
462 
463     if (Reg != AArch64::X0)
464       OutStreamer->emitInstruction(MCInstBuilder(AArch64::ORRXrs)
465                                        .addReg(AArch64::X0)
466                                        .addReg(AArch64::XZR)
467                                        .addReg(Reg)
468                                        .addImm(0),
469                                    *STI);
470     OutStreamer->emitInstruction(MCInstBuilder(AArch64::MOVZXi)
471                                      .addReg(AArch64::X1)
472                                      .addImm(AccessInfo)
473                                      .addImm(0),
474                                  *STI);
475 
476     // Intentionally load the GOT entry and branch to it, rather than possibly
477     // late binding the function, which may clobber the registers before we have
478     // a chance to save them.
479     OutStreamer->emitInstruction(
480         MCInstBuilder(AArch64::ADRP)
481             .addReg(AArch64::X16)
482             .addExpr(AArch64MCExpr::create(
483                 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_PAGE,
484                 OutContext)),
485         *STI);
486     OutStreamer->emitInstruction(
487         MCInstBuilder(AArch64::LDRXui)
488             .addReg(AArch64::X16)
489             .addReg(AArch64::X16)
490             .addExpr(AArch64MCExpr::create(
491                 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_LO12,
492                 OutContext)),
493         *STI);
494     OutStreamer->emitInstruction(
495         MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI);
496   }
497 }
498 
499 void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
500   EmitHwasanMemaccessSymbols(M);
501 
502   const Triple &TT = TM.getTargetTriple();
503   if (TT.isOSBinFormatMachO()) {
504     // Funny Darwin hack: This flag tells the linker that no global symbols
505     // contain code that falls through to other global symbols (e.g. the obvious
506     // implementation of multiple entry points).  If this doesn't occur, the
507     // linker can safely perform dead code stripping.  Since LLVM never
508     // generates code that does this, it is always safe to set.
509     OutStreamer->emitAssemblerFlag(MCAF_SubsectionsViaSymbols);
510   }
511 
512   // Emit stack and fault map information.
513   emitStackMaps(SM);
514   FM.serializeToFaultMapSection();
515 
516 }
517 
518 void AArch64AsmPrinter::EmitLOHs() {
519   SmallVector<MCSymbol *, 3> MCArgs;
520 
521   for (const auto &D : AArch64FI->getLOHContainer()) {
522     for (const MachineInstr *MI : D.getArgs()) {
523       MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI);
524       assert(LabelIt != LOHInstToLabel.end() &&
525              "Label hasn't been inserted for LOH related instruction");
526       MCArgs.push_back(LabelIt->second);
527     }
528     OutStreamer->emitLOHDirective(D.getKind(), MCArgs);
529     MCArgs.clear();
530   }
531 }
532 
533 void AArch64AsmPrinter::emitFunctionBodyEnd() {
534   if (!AArch64FI->getLOHRelated().empty())
535     EmitLOHs();
536 }
537 
538 /// GetCPISymbol - Return the symbol for the specified constant pool entry.
539 MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
540   // Darwin uses a linker-private symbol name for constant-pools (to
541   // avoid addends on the relocation?), ELF has no such concept and
542   // uses a normal private symbol.
543   if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
544     return OutContext.getOrCreateSymbol(
545         Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
546         Twine(getFunctionNumber()) + "_" + Twine(CPID));
547 
548   return AsmPrinter::GetCPISymbol(CPID);
549 }
550 
551 void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
552                                      raw_ostream &O) {
553   const MachineOperand &MO = MI->getOperand(OpNum);
554   switch (MO.getType()) {
555   default:
556     llvm_unreachable("<unknown operand type>");
557   case MachineOperand::MO_Register: {
558     Register Reg = MO.getReg();
559     assert(Register::isPhysicalRegister(Reg));
560     assert(!MO.getSubReg() && "Subregs should be eliminated!");
561     O << AArch64InstPrinter::getRegisterName(Reg);
562     break;
563   }
564   case MachineOperand::MO_Immediate: {
565     O << MO.getImm();
566     break;
567   }
568   case MachineOperand::MO_GlobalAddress: {
569     PrintSymbolOperand(MO, O);
570     break;
571   }
572   case MachineOperand::MO_BlockAddress: {
573     MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
574     Sym->print(O, MAI);
575     break;
576   }
577   }
578 }
579 
580 bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
581                                           raw_ostream &O) {
582   Register Reg = MO.getReg();
583   switch (Mode) {
584   default:
585     return true; // Unknown mode.
586   case 'w':
587     Reg = getWRegFromXReg(Reg);
588     break;
589   case 'x':
590     Reg = getXRegFromWReg(Reg);
591     break;
592   }
593 
594   O << AArch64InstPrinter::getRegisterName(Reg);
595   return false;
596 }
597 
598 // Prints the register in MO using class RC using the offset in the
599 // new register class. This should not be used for cross class
600 // printing.
601 bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
602                                            const TargetRegisterClass *RC,
603                                            unsigned AltName, raw_ostream &O) {
604   assert(MO.isReg() && "Should only get here with a register!");
605   const TargetRegisterInfo *RI = STI->getRegisterInfo();
606   Register Reg = MO.getReg();
607   unsigned RegToPrint = RC->getRegister(RI->getEncodingValue(Reg));
608   assert(RI->regsOverlap(RegToPrint, Reg));
609   O << AArch64InstPrinter::getRegisterName(RegToPrint, AltName);
610   return false;
611 }
612 
613 bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
614                                         const char *ExtraCode, raw_ostream &O) {
615   const MachineOperand &MO = MI->getOperand(OpNum);
616 
617   // First try the generic code, which knows about modifiers like 'c' and 'n'.
618   if (!AsmPrinter::PrintAsmOperand(MI, OpNum, ExtraCode, O))
619     return false;
620 
621   // Does this asm operand have a single letter operand modifier?
622   if (ExtraCode && ExtraCode[0]) {
623     if (ExtraCode[1] != 0)
624       return true; // Unknown modifier.
625 
626     switch (ExtraCode[0]) {
627     default:
628       return true; // Unknown modifier.
629     case 'w':      // Print W register
630     case 'x':      // Print X register
631       if (MO.isReg())
632         return printAsmMRegister(MO, ExtraCode[0], O);
633       if (MO.isImm() && MO.getImm() == 0) {
634         unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
635         O << AArch64InstPrinter::getRegisterName(Reg);
636         return false;
637       }
638       printOperand(MI, OpNum, O);
639       return false;
640     case 'b': // Print B register.
641     case 'h': // Print H register.
642     case 's': // Print S register.
643     case 'd': // Print D register.
644     case 'q': // Print Q register.
645     case 'z': // Print Z register.
646       if (MO.isReg()) {
647         const TargetRegisterClass *RC;
648         switch (ExtraCode[0]) {
649         case 'b':
650           RC = &AArch64::FPR8RegClass;
651           break;
652         case 'h':
653           RC = &AArch64::FPR16RegClass;
654           break;
655         case 's':
656           RC = &AArch64::FPR32RegClass;
657           break;
658         case 'd':
659           RC = &AArch64::FPR64RegClass;
660           break;
661         case 'q':
662           RC = &AArch64::FPR128RegClass;
663           break;
664         case 'z':
665           RC = &AArch64::ZPRRegClass;
666           break;
667         default:
668           return true;
669         }
670         return printAsmRegInClass(MO, RC, AArch64::NoRegAltName, O);
671       }
672       printOperand(MI, OpNum, O);
673       return false;
674     }
675   }
676 
677   // According to ARM, we should emit x and v registers unless we have a
678   // modifier.
679   if (MO.isReg()) {
680     Register Reg = MO.getReg();
681 
682     // If this is a w or x register, print an x register.
683     if (AArch64::GPR32allRegClass.contains(Reg) ||
684         AArch64::GPR64allRegClass.contains(Reg))
685       return printAsmMRegister(MO, 'x', O);
686 
687     unsigned AltName = AArch64::NoRegAltName;
688     const TargetRegisterClass *RegClass;
689     if (AArch64::ZPRRegClass.contains(Reg)) {
690       RegClass = &AArch64::ZPRRegClass;
691     } else if (AArch64::PPRRegClass.contains(Reg)) {
692       RegClass = &AArch64::PPRRegClass;
693     } else {
694       RegClass = &AArch64::FPR128RegClass;
695       AltName = AArch64::vreg;
696     }
697 
698     // If this is a b, h, s, d, or q register, print it as a v register.
699     return printAsmRegInClass(MO, RegClass, AltName, O);
700   }
701 
702   printOperand(MI, OpNum, O);
703   return false;
704 }
705 
706 bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
707                                               unsigned OpNum,
708                                               const char *ExtraCode,
709                                               raw_ostream &O) {
710   if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
711     return true; // Unknown modifier.
712 
713   const MachineOperand &MO = MI->getOperand(OpNum);
714   assert(MO.isReg() && "unexpected inline asm memory operand");
715   O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]";
716   return false;
717 }
718 
719 void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
720                                                raw_ostream &OS) {
721   unsigned NOps = MI->getNumOperands();
722   assert(NOps == 4);
723   OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
724   // cast away const; DIetc do not take const operands for some reason.
725   OS << MI->getDebugVariable()->getName();
726   OS << " <- ";
727   // Frame address.  Currently handles register +- offset only.
728   assert(MI->getDebugOperand(0).isReg() && MI->isDebugOffsetImm());
729   OS << '[';
730   printOperand(MI, 0, OS);
731   OS << '+';
732   printOperand(MI, 1, OS);
733   OS << ']';
734   OS << "+";
735   printOperand(MI, NOps - 2, OS);
736 }
737 
738 void AArch64AsmPrinter::emitJumpTableInfo() {
739   const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
740   if (!MJTI) return;
741 
742   const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
743   if (JT.empty()) return;
744 
745   const Function &F = MF->getFunction();
746   const TargetLoweringObjectFile &TLOF = getObjFileLowering();
747   bool JTInDiffSection =
748       !STI->isTargetCOFF() ||
749       !TLOF.shouldPutJumpTableInFunctionSection(
750           MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32,
751           F);
752   if (JTInDiffSection) {
753       // Drop it in the readonly section.
754       MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(F, TM);
755       OutStreamer->SwitchSection(ReadOnlySec);
756   }
757 
758   auto AFI = MF->getInfo<AArch64FunctionInfo>();
759   for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
760     const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
761 
762     // If this jump table was deleted, ignore it.
763     if (JTBBs.empty()) continue;
764 
765     unsigned Size = AFI->getJumpTableEntrySize(JTI);
766     emitAlignment(Align(Size));
767     OutStreamer->emitLabel(GetJTISymbol(JTI));
768 
769     const MCSymbol *BaseSym = AArch64FI->getJumpTableEntryPCRelSymbol(JTI);
770     const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext);
771 
772     for (auto *JTBB : JTBBs) {
773       const MCExpr *Value =
774           MCSymbolRefExpr::create(JTBB->getSymbol(), OutContext);
775 
776       // Each entry is:
777       //     .byte/.hword (LBB - Lbase)>>2
778       // or plain:
779       //     .word LBB - Lbase
780       Value = MCBinaryExpr::createSub(Value, Base, OutContext);
781       if (Size != 4)
782         Value = MCBinaryExpr::createLShr(
783             Value, MCConstantExpr::create(2, OutContext), OutContext);
784 
785       OutStreamer->emitValue(Value, Size);
786     }
787   }
788 }
789 
790 /// Small jump tables contain an unsigned byte or half, representing the offset
791 /// from the lowest-addressed possible destination to the desired basic
792 /// block. Since all instructions are 4-byte aligned, this is further compressed
793 /// by counting in instructions rather than bytes (i.e. divided by 4). So, to
794 /// materialize the correct destination we need:
795 ///
796 ///             adr xDest, .LBB0_0
797 ///             ldrb wScratch, [xTable, xEntry]   (with "lsl #1" for ldrh).
798 ///             add xDest, xDest, xScratch (with "lsl #2" for smaller entries)
799 void AArch64AsmPrinter::LowerJumpTableDest(llvm::MCStreamer &OutStreamer,
800                                            const llvm::MachineInstr &MI) {
801   Register DestReg = MI.getOperand(0).getReg();
802   Register ScratchReg = MI.getOperand(1).getReg();
803   Register ScratchRegW =
804       STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32);
805   Register TableReg = MI.getOperand(2).getReg();
806   Register EntryReg = MI.getOperand(3).getReg();
807   int JTIdx = MI.getOperand(4).getIndex();
808   int Size = AArch64FI->getJumpTableEntrySize(JTIdx);
809 
810   // This has to be first because the compression pass based its reachability
811   // calculations on the start of the JumpTableDest instruction.
812   auto Label =
813       MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx);
814 
815   // If we don't already have a symbol to use as the base, use the ADR
816   // instruction itself.
817   if (!Label) {
818     Label = MF->getContext().createTempSymbol();
819     AArch64FI->setJumpTableEntryInfo(JTIdx, Size, Label);
820     OutStreamer.emitLabel(Label);
821   }
822 
823   auto LabelExpr = MCSymbolRefExpr::create(Label, MF->getContext());
824   EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR)
825                                   .addReg(DestReg)
826                                   .addExpr(LabelExpr));
827 
828   // Load the number of instruction-steps to offset from the label.
829   unsigned LdrOpcode;
830   switch (Size) {
831   case 1: LdrOpcode = AArch64::LDRBBroX; break;
832   case 2: LdrOpcode = AArch64::LDRHHroX; break;
833   case 4: LdrOpcode = AArch64::LDRSWroX; break;
834   default:
835     llvm_unreachable("Unknown jump table size");
836   }
837 
838   EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode)
839                                   .addReg(Size == 4 ? ScratchReg : ScratchRegW)
840                                   .addReg(TableReg)
841                                   .addReg(EntryReg)
842                                   .addImm(0)
843                                   .addImm(Size == 1 ? 0 : 1));
844 
845   // Add to the already materialized base label address, multiplying by 4 if
846   // compressed.
847   EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs)
848                                   .addReg(DestReg)
849                                   .addReg(DestReg)
850                                   .addReg(ScratchReg)
851                                   .addImm(Size == 4 ? 0 : 2));
852 }
853 
854 void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
855                                       const MachineInstr &MI) {
856   unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
857 
858   auto &Ctx = OutStreamer.getContext();
859   MCSymbol *MILabel = Ctx.createTempSymbol();
860   OutStreamer.emitLabel(MILabel);
861 
862   SM.recordStackMap(*MILabel, MI);
863   assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
864 
865   // Scan ahead to trim the shadow.
866   const MachineBasicBlock &MBB = *MI.getParent();
867   MachineBasicBlock::const_iterator MII(MI);
868   ++MII;
869   while (NumNOPBytes > 0) {
870     if (MII == MBB.end() || MII->isCall() ||
871         MII->getOpcode() == AArch64::DBG_VALUE ||
872         MII->getOpcode() == TargetOpcode::PATCHPOINT ||
873         MII->getOpcode() == TargetOpcode::STACKMAP)
874       break;
875     ++MII;
876     NumNOPBytes -= 4;
877   }
878 
879   // Emit nops.
880   for (unsigned i = 0; i < NumNOPBytes; i += 4)
881     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
882 }
883 
884 // Lower a patchpoint of the form:
885 // [<def>], <id>, <numBytes>, <target>, <numArgs>
886 void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
887                                         const MachineInstr &MI) {
888   auto &Ctx = OutStreamer.getContext();
889   MCSymbol *MILabel = Ctx.createTempSymbol();
890   OutStreamer.emitLabel(MILabel);
891   SM.recordPatchPoint(*MILabel, MI);
892 
893   PatchPointOpers Opers(&MI);
894 
895   int64_t CallTarget = Opers.getCallTarget().getImm();
896   unsigned EncodedBytes = 0;
897   if (CallTarget) {
898     assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
899            "High 16 bits of call target should be zero.");
900     Register ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
901     EncodedBytes = 16;
902     // Materialize the jump address:
903     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi)
904                                     .addReg(ScratchReg)
905                                     .addImm((CallTarget >> 32) & 0xFFFF)
906                                     .addImm(32));
907     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
908                                     .addReg(ScratchReg)
909                                     .addReg(ScratchReg)
910                                     .addImm((CallTarget >> 16) & 0xFFFF)
911                                     .addImm(16));
912     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
913                                     .addReg(ScratchReg)
914                                     .addReg(ScratchReg)
915                                     .addImm(CallTarget & 0xFFFF)
916                                     .addImm(0));
917     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
918   }
919   // Emit padding.
920   unsigned NumBytes = Opers.getNumPatchBytes();
921   assert(NumBytes >= EncodedBytes &&
922          "Patchpoint can't request size less than the length of a call.");
923   assert((NumBytes - EncodedBytes) % 4 == 0 &&
924          "Invalid number of NOP bytes requested!");
925   for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
926     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
927 }
928 
929 void AArch64AsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
930                                         const MachineInstr &MI) {
931   StatepointOpers SOpers(&MI);
932   if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
933     assert(PatchBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
934     for (unsigned i = 0; i < PatchBytes; i += 4)
935       EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
936   } else {
937     // Lower call target and choose correct opcode
938     const MachineOperand &CallTarget = SOpers.getCallTarget();
939     MCOperand CallTargetMCOp;
940     unsigned CallOpcode;
941     switch (CallTarget.getType()) {
942     case MachineOperand::MO_GlobalAddress:
943     case MachineOperand::MO_ExternalSymbol:
944       MCInstLowering.lowerOperand(CallTarget, CallTargetMCOp);
945       CallOpcode = AArch64::BL;
946       break;
947     case MachineOperand::MO_Immediate:
948       CallTargetMCOp = MCOperand::createImm(CallTarget.getImm());
949       CallOpcode = AArch64::BL;
950       break;
951     case MachineOperand::MO_Register:
952       CallTargetMCOp = MCOperand::createReg(CallTarget.getReg());
953       CallOpcode = AArch64::BLR;
954       break;
955     default:
956       llvm_unreachable("Unsupported operand type in statepoint call target");
957       break;
958     }
959 
960     EmitToStreamer(OutStreamer,
961                    MCInstBuilder(CallOpcode).addOperand(CallTargetMCOp));
962   }
963 
964   auto &Ctx = OutStreamer.getContext();
965   MCSymbol *MILabel = Ctx.createTempSymbol();
966   OutStreamer.emitLabel(MILabel);
967   SM.recordStatepoint(*MILabel, MI);
968 }
969 
970 void AArch64AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI) {
971   // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
972   //                  <opcode>, <operands>
973 
974   Register DefRegister = FaultingMI.getOperand(0).getReg();
975   FaultMaps::FaultKind FK =
976       static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(1).getImm());
977   MCSymbol *HandlerLabel = FaultingMI.getOperand(2).getMBB()->getSymbol();
978   unsigned Opcode = FaultingMI.getOperand(3).getImm();
979   unsigned OperandsBeginIdx = 4;
980 
981   auto &Ctx = OutStreamer->getContext();
982   MCSymbol *FaultingLabel = Ctx.createTempSymbol();
983   OutStreamer->emitLabel(FaultingLabel);
984 
985   assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
986   FM.recordFaultingOp(FK, FaultingLabel, HandlerLabel);
987 
988   MCInst MI;
989   MI.setOpcode(Opcode);
990 
991   if (DefRegister != (Register)0)
992     MI.addOperand(MCOperand::createReg(DefRegister));
993 
994   for (auto I = FaultingMI.operands_begin() + OperandsBeginIdx,
995             E = FaultingMI.operands_end();
996        I != E; ++I) {
997     MCOperand Dest;
998     lowerOperand(*I, Dest);
999     MI.addOperand(Dest);
1000   }
1001 
1002   OutStreamer->AddComment("on-fault: " + HandlerLabel->getName());
1003   OutStreamer->emitInstruction(MI, getSubtargetInfo());
1004 }
1005 
1006 void AArch64AsmPrinter::EmitFMov0(const MachineInstr &MI) {
1007   Register DestReg = MI.getOperand(0).getReg();
1008   if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround()) {
1009     // Convert H/S/D register to corresponding Q register
1010     if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31)
1011       DestReg = AArch64::Q0 + (DestReg - AArch64::H0);
1012     else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31)
1013       DestReg = AArch64::Q0 + (DestReg - AArch64::S0);
1014     else {
1015       assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31);
1016       DestReg = AArch64::Q0 + (DestReg - AArch64::D0);
1017     }
1018     MCInst MOVI;
1019     MOVI.setOpcode(AArch64::MOVIv2d_ns);
1020     MOVI.addOperand(MCOperand::createReg(DestReg));
1021     MOVI.addOperand(MCOperand::createImm(0));
1022     EmitToStreamer(*OutStreamer, MOVI);
1023   } else {
1024     MCInst FMov;
1025     switch (MI.getOpcode()) {
1026     default: llvm_unreachable("Unexpected opcode");
1027     case AArch64::FMOVH0:
1028       FMov.setOpcode(AArch64::FMOVWHr);
1029       FMov.addOperand(MCOperand::createReg(DestReg));
1030       FMov.addOperand(MCOperand::createReg(AArch64::WZR));
1031       break;
1032     case AArch64::FMOVS0:
1033       FMov.setOpcode(AArch64::FMOVWSr);
1034       FMov.addOperand(MCOperand::createReg(DestReg));
1035       FMov.addOperand(MCOperand::createReg(AArch64::WZR));
1036       break;
1037     case AArch64::FMOVD0:
1038       FMov.setOpcode(AArch64::FMOVXDr);
1039       FMov.addOperand(MCOperand::createReg(DestReg));
1040       FMov.addOperand(MCOperand::createReg(AArch64::XZR));
1041       break;
1042     }
1043     EmitToStreamer(*OutStreamer, FMov);
1044   }
1045 }
1046 
1047 // Simple pseudo-instructions have their lowering (with expansion to real
1048 // instructions) auto-generated.
1049 #include "AArch64GenMCPseudoLowering.inc"
1050 
1051 void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
1052   // Do any auto-generated pseudo lowerings.
1053   if (emitPseudoExpansionLowering(*OutStreamer, MI))
1054     return;
1055 
1056   if (AArch64FI->getLOHRelated().count(MI)) {
1057     // Generate a label for LOH related instruction
1058     MCSymbol *LOHLabel = createTempSymbol("loh");
1059     // Associate the instruction with the label
1060     LOHInstToLabel[MI] = LOHLabel;
1061     OutStreamer->emitLabel(LOHLabel);
1062   }
1063 
1064   AArch64TargetStreamer *TS =
1065     static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
1066   // Do any manual lowerings.
1067   switch (MI->getOpcode()) {
1068   default:
1069     break;
1070   case AArch64::HINT: {
1071     // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
1072     // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
1073     // non-empty. If MI is the initial BTI, place the
1074     // __patchable_function_entries label after BTI.
1075     if (CurrentPatchableFunctionEntrySym &&
1076         CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
1077         MI == &MF->front().front()) {
1078       int64_t Imm = MI->getOperand(0).getImm();
1079       if ((Imm & 32) && (Imm & 6)) {
1080         MCInst Inst;
1081         MCInstLowering.Lower(MI, Inst);
1082         EmitToStreamer(*OutStreamer, Inst);
1083         CurrentPatchableFunctionEntrySym = createTempSymbol("patch");
1084         OutStreamer->emitLabel(CurrentPatchableFunctionEntrySym);
1085         return;
1086       }
1087     }
1088     break;
1089   }
1090     case AArch64::MOVMCSym: {
1091       Register DestReg = MI->getOperand(0).getReg();
1092       const MachineOperand &MO_Sym = MI->getOperand(1);
1093       MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
1094       MCOperand Hi_MCSym, Lo_MCSym;
1095 
1096       Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
1097       Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
1098 
1099       MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym);
1100       MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym);
1101 
1102       MCInst MovZ;
1103       MovZ.setOpcode(AArch64::MOVZXi);
1104       MovZ.addOperand(MCOperand::createReg(DestReg));
1105       MovZ.addOperand(Hi_MCSym);
1106       MovZ.addOperand(MCOperand::createImm(16));
1107       EmitToStreamer(*OutStreamer, MovZ);
1108 
1109       MCInst MovK;
1110       MovK.setOpcode(AArch64::MOVKXi);
1111       MovK.addOperand(MCOperand::createReg(DestReg));
1112       MovK.addOperand(MCOperand::createReg(DestReg));
1113       MovK.addOperand(Lo_MCSym);
1114       MovK.addOperand(MCOperand::createImm(0));
1115       EmitToStreamer(*OutStreamer, MovK);
1116       return;
1117   }
1118   case AArch64::MOVIv2d_ns:
1119     // If the target has <rdar://problem/16473581>, lower this
1120     // instruction to movi.16b instead.
1121     if (STI->hasZeroCycleZeroingFPWorkaround() &&
1122         MI->getOperand(1).getImm() == 0) {
1123       MCInst TmpInst;
1124       TmpInst.setOpcode(AArch64::MOVIv16b_ns);
1125       TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1126       TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm()));
1127       EmitToStreamer(*OutStreamer, TmpInst);
1128       return;
1129     }
1130     break;
1131 
1132   case AArch64::DBG_VALUE: {
1133     if (isVerbose() && OutStreamer->hasRawTextSupport()) {
1134       SmallString<128> TmpStr;
1135       raw_svector_ostream OS(TmpStr);
1136       PrintDebugValueComment(MI, OS);
1137       OutStreamer->emitRawText(StringRef(OS.str()));
1138     }
1139     return;
1140 
1141   case AArch64::EMITBKEY: {
1142       ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
1143       if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
1144           ExceptionHandlingType != ExceptionHandling::ARM)
1145         return;
1146 
1147       if (needsCFIMoves() == CFI_M_None)
1148         return;
1149 
1150       OutStreamer->emitCFIBKeyFrame();
1151       return;
1152     }
1153   }
1154 
1155   // Tail calls use pseudo instructions so they have the proper code-gen
1156   // attributes (isCall, isReturn, etc.). We lower them to the real
1157   // instruction here.
1158   case AArch64::TCRETURNri:
1159   case AArch64::TCRETURNriBTI:
1160   case AArch64::TCRETURNriALL: {
1161     MCInst TmpInst;
1162     TmpInst.setOpcode(AArch64::BR);
1163     TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1164     EmitToStreamer(*OutStreamer, TmpInst);
1165     return;
1166   }
1167   case AArch64::TCRETURNdi: {
1168     MCOperand Dest;
1169     MCInstLowering.lowerOperand(MI->getOperand(0), Dest);
1170     MCInst TmpInst;
1171     TmpInst.setOpcode(AArch64::B);
1172     TmpInst.addOperand(Dest);
1173     EmitToStreamer(*OutStreamer, TmpInst);
1174     return;
1175   }
1176   case AArch64::SpeculationBarrierISBDSBEndBB: {
1177     // Print DSB SYS + ISB
1178     MCInst TmpInstDSB;
1179     TmpInstDSB.setOpcode(AArch64::DSB);
1180     TmpInstDSB.addOperand(MCOperand::createImm(0xf));
1181     EmitToStreamer(*OutStreamer, TmpInstDSB);
1182     MCInst TmpInstISB;
1183     TmpInstISB.setOpcode(AArch64::ISB);
1184     TmpInstISB.addOperand(MCOperand::createImm(0xf));
1185     EmitToStreamer(*OutStreamer, TmpInstISB);
1186     return;
1187   }
1188   case AArch64::SpeculationBarrierSBEndBB: {
1189     // Print SB
1190     MCInst TmpInstSB;
1191     TmpInstSB.setOpcode(AArch64::SB);
1192     EmitToStreamer(*OutStreamer, TmpInstSB);
1193     return;
1194   }
1195   case AArch64::TLSDESC_CALLSEQ: {
1196     /// lower this to:
1197     ///    adrp  x0, :tlsdesc:var
1198     ///    ldr   x1, [x0, #:tlsdesc_lo12:var]
1199     ///    add   x0, x0, #:tlsdesc_lo12:var
1200     ///    .tlsdesccall var
1201     ///    blr   x1
1202     ///    (TPIDR_EL0 offset now in x0)
1203     const MachineOperand &MO_Sym = MI->getOperand(0);
1204     MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
1205     MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
1206     MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
1207     MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
1208     MCInstLowering.lowerOperand(MO_Sym, Sym);
1209     MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12);
1210     MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc);
1211 
1212     MCInst Adrp;
1213     Adrp.setOpcode(AArch64::ADRP);
1214     Adrp.addOperand(MCOperand::createReg(AArch64::X0));
1215     Adrp.addOperand(SymTLSDesc);
1216     EmitToStreamer(*OutStreamer, Adrp);
1217 
1218     MCInst Ldr;
1219     Ldr.setOpcode(AArch64::LDRXui);
1220     Ldr.addOperand(MCOperand::createReg(AArch64::X1));
1221     Ldr.addOperand(MCOperand::createReg(AArch64::X0));
1222     Ldr.addOperand(SymTLSDescLo12);
1223     Ldr.addOperand(MCOperand::createImm(0));
1224     EmitToStreamer(*OutStreamer, Ldr);
1225 
1226     MCInst Add;
1227     Add.setOpcode(AArch64::ADDXri);
1228     Add.addOperand(MCOperand::createReg(AArch64::X0));
1229     Add.addOperand(MCOperand::createReg(AArch64::X0));
1230     Add.addOperand(SymTLSDescLo12);
1231     Add.addOperand(MCOperand::createImm(AArch64_AM::getShiftValue(0)));
1232     EmitToStreamer(*OutStreamer, Add);
1233 
1234     // Emit a relocation-annotation. This expands to no code, but requests
1235     // the following instruction gets an R_AARCH64_TLSDESC_CALL.
1236     MCInst TLSDescCall;
1237     TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
1238     TLSDescCall.addOperand(Sym);
1239     EmitToStreamer(*OutStreamer, TLSDescCall);
1240 
1241     MCInst Blr;
1242     Blr.setOpcode(AArch64::BLR);
1243     Blr.addOperand(MCOperand::createReg(AArch64::X1));
1244     EmitToStreamer(*OutStreamer, Blr);
1245 
1246     return;
1247   }
1248 
1249   case AArch64::JumpTableDest32:
1250   case AArch64::JumpTableDest16:
1251   case AArch64::JumpTableDest8:
1252     LowerJumpTableDest(*OutStreamer, *MI);
1253     return;
1254 
1255   case AArch64::FMOVH0:
1256   case AArch64::FMOVS0:
1257   case AArch64::FMOVD0:
1258     EmitFMov0(*MI);
1259     return;
1260 
1261   case TargetOpcode::STACKMAP:
1262     return LowerSTACKMAP(*OutStreamer, SM, *MI);
1263 
1264   case TargetOpcode::PATCHPOINT:
1265     return LowerPATCHPOINT(*OutStreamer, SM, *MI);
1266 
1267   case TargetOpcode::STATEPOINT:
1268     return LowerSTATEPOINT(*OutStreamer, SM, *MI);
1269 
1270   case TargetOpcode::FAULTING_OP:
1271     return LowerFAULTING_OP(*MI);
1272 
1273   case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
1274     LowerPATCHABLE_FUNCTION_ENTER(*MI);
1275     return;
1276 
1277   case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
1278     LowerPATCHABLE_FUNCTION_EXIT(*MI);
1279     return;
1280 
1281   case TargetOpcode::PATCHABLE_TAIL_CALL:
1282     LowerPATCHABLE_TAIL_CALL(*MI);
1283     return;
1284 
1285   case AArch64::HWASAN_CHECK_MEMACCESS:
1286   case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
1287     LowerHWASAN_CHECK_MEMACCESS(*MI);
1288     return;
1289 
1290   case AArch64::SEH_StackAlloc:
1291     TS->EmitARM64WinCFIAllocStack(MI->getOperand(0).getImm());
1292     return;
1293 
1294   case AArch64::SEH_SaveFPLR:
1295     TS->EmitARM64WinCFISaveFPLR(MI->getOperand(0).getImm());
1296     return;
1297 
1298   case AArch64::SEH_SaveFPLR_X:
1299     assert(MI->getOperand(0).getImm() < 0 &&
1300            "Pre increment SEH opcode must have a negative offset");
1301     TS->EmitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm());
1302     return;
1303 
1304   case AArch64::SEH_SaveReg:
1305     TS->EmitARM64WinCFISaveReg(MI->getOperand(0).getImm(),
1306                                MI->getOperand(1).getImm());
1307     return;
1308 
1309   case AArch64::SEH_SaveReg_X:
1310     assert(MI->getOperand(1).getImm() < 0 &&
1311            "Pre increment SEH opcode must have a negative offset");
1312     TS->EmitARM64WinCFISaveRegX(MI->getOperand(0).getImm(),
1313 		                -MI->getOperand(1).getImm());
1314     return;
1315 
1316   case AArch64::SEH_SaveRegP:
1317     if (MI->getOperand(1).getImm() == 30 && MI->getOperand(0).getImm() >= 19 &&
1318         MI->getOperand(0).getImm() <= 28) {
1319       assert((MI->getOperand(0).getImm() - 19) % 2 == 0 &&
1320              "Register paired with LR must be odd");
1321       TS->EmitARM64WinCFISaveLRPair(MI->getOperand(0).getImm(),
1322                                     MI->getOperand(2).getImm());
1323       return;
1324     }
1325     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1326             "Non-consecutive registers not allowed for save_regp");
1327     TS->EmitARM64WinCFISaveRegP(MI->getOperand(0).getImm(),
1328                                 MI->getOperand(2).getImm());
1329     return;
1330 
1331   case AArch64::SEH_SaveRegP_X:
1332     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1333             "Non-consecutive registers not allowed for save_regp_x");
1334     assert(MI->getOperand(2).getImm() < 0 &&
1335            "Pre increment SEH opcode must have a negative offset");
1336     TS->EmitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(),
1337                                  -MI->getOperand(2).getImm());
1338     return;
1339 
1340   case AArch64::SEH_SaveFReg:
1341     TS->EmitARM64WinCFISaveFReg(MI->getOperand(0).getImm(),
1342                                 MI->getOperand(1).getImm());
1343     return;
1344 
1345   case AArch64::SEH_SaveFReg_X:
1346     assert(MI->getOperand(1).getImm() < 0 &&
1347            "Pre increment SEH opcode must have a negative offset");
1348     TS->EmitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(),
1349                                  -MI->getOperand(1).getImm());
1350     return;
1351 
1352   case AArch64::SEH_SaveFRegP:
1353     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1354             "Non-consecutive registers not allowed for save_regp");
1355     TS->EmitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(),
1356                                  MI->getOperand(2).getImm());
1357     return;
1358 
1359   case AArch64::SEH_SaveFRegP_X:
1360     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1361             "Non-consecutive registers not allowed for save_regp_x");
1362     assert(MI->getOperand(2).getImm() < 0 &&
1363            "Pre increment SEH opcode must have a negative offset");
1364     TS->EmitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(),
1365                                   -MI->getOperand(2).getImm());
1366     return;
1367 
1368   case AArch64::SEH_SetFP:
1369     TS->EmitARM64WinCFISetFP();
1370     return;
1371 
1372   case AArch64::SEH_AddFP:
1373     TS->EmitARM64WinCFIAddFP(MI->getOperand(0).getImm());
1374     return;
1375 
1376   case AArch64::SEH_Nop:
1377     TS->EmitARM64WinCFINop();
1378     return;
1379 
1380   case AArch64::SEH_PrologEnd:
1381     TS->EmitARM64WinCFIPrologEnd();
1382     return;
1383 
1384   case AArch64::SEH_EpilogStart:
1385     TS->EmitARM64WinCFIEpilogStart();
1386     return;
1387 
1388   case AArch64::SEH_EpilogEnd:
1389     TS->EmitARM64WinCFIEpilogEnd();
1390     return;
1391   }
1392 
1393   // Finally, do the automated lowerings for everything else.
1394   MCInst TmpInst;
1395   MCInstLowering.Lower(MI, TmpInst);
1396   EmitToStreamer(*OutStreamer, TmpInst);
1397 }
1398 
1399 // Force static initialization.
1400 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter() {
1401   RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
1402   RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget());
1403   RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target());
1404   RegisterAsmPrinter<AArch64AsmPrinter> W(getTheARM64_32Target());
1405   RegisterAsmPrinter<AArch64AsmPrinter> V(getTheAArch64_32Target());
1406 }
1407