1 //===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains a printer that converts from our internal representation
10 // of machine-dependent LLVM code to the AArch64 assembly language.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "AArch64.h"
15 #include "AArch64MCInstLower.h"
16 #include "AArch64MachineFunctionInfo.h"
17 #include "AArch64RegisterInfo.h"
18 #include "AArch64Subtarget.h"
19 #include "AArch64TargetObjectFile.h"
20 #include "MCTargetDesc/AArch64AddressingModes.h"
21 #include "MCTargetDesc/AArch64InstPrinter.h"
22 #include "MCTargetDesc/AArch64MCExpr.h"
23 #include "MCTargetDesc/AArch64MCTargetDesc.h"
24 #include "MCTargetDesc/AArch64TargetStreamer.h"
25 #include "TargetInfo/AArch64TargetInfo.h"
26 #include "Utils/AArch64BaseInfo.h"
27 #include "llvm/ADT/SmallString.h"
28 #include "llvm/ADT/SmallVector.h"
29 #include "llvm/ADT/StringRef.h"
30 #include "llvm/ADT/Triple.h"
31 #include "llvm/ADT/Twine.h"
32 #include "llvm/BinaryFormat/COFF.h"
33 #include "llvm/BinaryFormat/ELF.h"
34 #include "llvm/CodeGen/AsmPrinter.h"
35 #include "llvm/CodeGen/MachineBasicBlock.h"
36 #include "llvm/CodeGen/MachineFunction.h"
37 #include "llvm/CodeGen/MachineInstr.h"
38 #include "llvm/CodeGen/MachineJumpTableInfo.h"
39 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
40 #include "llvm/CodeGen/MachineOperand.h"
41 #include "llvm/CodeGen/StackMaps.h"
42 #include "llvm/CodeGen/TargetRegisterInfo.h"
43 #include "llvm/IR/DataLayout.h"
44 #include "llvm/IR/DebugInfoMetadata.h"
45 #include "llvm/MC/MCAsmInfo.h"
46 #include "llvm/MC/MCContext.h"
47 #include "llvm/MC/MCInst.h"
48 #include "llvm/MC/MCInstBuilder.h"
49 #include "llvm/MC/MCSectionELF.h"
50 #include "llvm/MC/MCStreamer.h"
51 #include "llvm/MC/MCSymbol.h"
52 #include "llvm/Support/Casting.h"
53 #include "llvm/Support/ErrorHandling.h"
54 #include "llvm/Support/TargetRegistry.h"
55 #include "llvm/Support/raw_ostream.h"
56 #include "llvm/Target/TargetMachine.h"
57 #include <algorithm>
58 #include <cassert>
59 #include <cstdint>
60 #include <map>
61 #include <memory>
62 
63 using namespace llvm;
64 
65 #define DEBUG_TYPE "asm-printer"
66 
67 namespace {
68 
69 class AArch64AsmPrinter : public AsmPrinter {
70   AArch64MCInstLower MCInstLowering;
71   StackMaps SM;
72   const AArch64Subtarget *STI;
73 
74 public:
75   AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
76       : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this),
77         SM(*this) {}
78 
79   StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
80 
81   /// Wrapper for MCInstLowering.lowerOperand() for the
82   /// tblgen'erated pseudo lowering.
83   bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
84     return MCInstLowering.lowerOperand(MO, MCOp);
85   }
86 
87   void emitStartOfAsmFile(Module &M) override;
88   void emitJumpTableInfo() override;
89   void emitJumpTableEntry(const MachineJumpTableInfo *MJTI,
90                           const MachineBasicBlock *MBB, unsigned JTI);
91 
92   void LowerJumpTableDestSmall(MCStreamer &OutStreamer, const MachineInstr &MI);
93 
94   void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
95                      const MachineInstr &MI);
96   void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
97                        const MachineInstr &MI);
98 
99   void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
100   void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
101   void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
102 
103   typedef std::tuple<unsigned, bool, uint32_t> HwasanMemaccessTuple;
104   std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
105   void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
106   void EmitHwasanMemaccessSymbols(Module &M);
107 
108   void EmitSled(const MachineInstr &MI, SledKind Kind);
109 
110   /// tblgen'erated driver function for lowering simple MI->MC
111   /// pseudo instructions.
112   bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
113                                    const MachineInstr *MI);
114 
115   void emitInstruction(const MachineInstr *MI) override;
116 
117   void emitFunctionHeaderComment() override;
118 
119   void getAnalysisUsage(AnalysisUsage &AU) const override {
120     AsmPrinter::getAnalysisUsage(AU);
121     AU.setPreservesAll();
122   }
123 
124   bool runOnMachineFunction(MachineFunction &MF) override {
125     AArch64FI = MF.getInfo<AArch64FunctionInfo>();
126     STI = static_cast<const AArch64Subtarget*>(&MF.getSubtarget());
127 
128     SetupMachineFunction(MF);
129 
130     if (STI->isTargetCOFF()) {
131       bool Internal = MF.getFunction().hasInternalLinkage();
132       COFF::SymbolStorageClass Scl = Internal ? COFF::IMAGE_SYM_CLASS_STATIC
133                                               : COFF::IMAGE_SYM_CLASS_EXTERNAL;
134       int Type =
135         COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT;
136 
137       OutStreamer->BeginCOFFSymbolDef(CurrentFnSym);
138       OutStreamer->EmitCOFFSymbolStorageClass(Scl);
139       OutStreamer->EmitCOFFSymbolType(Type);
140       OutStreamer->EndCOFFSymbolDef();
141     }
142 
143     // Emit the rest of the function body.
144     emitFunctionBody();
145 
146     // Emit the XRay table for this function.
147     emitXRayTable();
148 
149     // We didn't modify anything.
150     return false;
151   }
152 
153 private:
154   void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
155   bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
156   bool printAsmRegInClass(const MachineOperand &MO,
157                           const TargetRegisterClass *RC, unsigned AltName,
158                           raw_ostream &O);
159 
160   bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
161                        const char *ExtraCode, raw_ostream &O) override;
162   bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
163                              const char *ExtraCode, raw_ostream &O) override;
164 
165   void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
166 
167   void emitFunctionBodyEnd() override;
168 
169   MCSymbol *GetCPISymbol(unsigned CPID) const override;
170   void emitEndOfAsmFile(Module &M) override;
171 
172   AArch64FunctionInfo *AArch64FI = nullptr;
173 
174   /// Emit the LOHs contained in AArch64FI.
175   void EmitLOHs();
176 
177   /// Emit instruction to set float register to zero.
178   void EmitFMov0(const MachineInstr &MI);
179 
180   using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
181 
182   MInstToMCSymbol LOHInstToLabel;
183 };
184 
185 } // end anonymous namespace
186 
187 void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
188   if (!TM.getTargetTriple().isOSBinFormatELF())
189     return;
190 
191   // Assemble feature flags that may require creation of a note section.
192   unsigned Flags = ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI |
193                    ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
194 
195   if (any_of(M, [](const Function &F) {
196         return !F.isDeclaration() &&
197                !F.hasFnAttribute("branch-target-enforcement");
198       })) {
199     Flags &= ~ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI;
200   }
201 
202   if ((Flags & ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI) == 0 &&
203       any_of(M, [](const Function &F) {
204         return F.hasFnAttribute("branch-target-enforcement");
205       })) {
206     errs() << "warning: some functions compiled with BTI and some compiled "
207               "without BTI\n"
208            << "warning: not setting BTI in feature flags\n";
209   }
210 
211   if (any_of(M, [](const Function &F) {
212         if (F.isDeclaration())
213           return false;
214         Attribute A = F.getFnAttribute("sign-return-address");
215         return !A.isStringAttribute() || A.getValueAsString() == "none";
216       })) {
217     Flags &= ~ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
218   }
219 
220   if (Flags == 0)
221     return;
222 
223   // Emit a .note.gnu.property section with the flags.
224   MCSection *Cur = OutStreamer->getCurrentSectionOnly();
225   MCSection *Nt = MMI->getContext().getELFSection(
226       ".note.gnu.property", ELF::SHT_NOTE, ELF::SHF_ALLOC);
227   OutStreamer->SwitchSection(Nt);
228 
229   // Emit the note header.
230   emitAlignment(Align(8));
231   OutStreamer->emitInt32(4);     // data size for "GNU\0"
232   OutStreamer->emitInt32(4 * 4); // Elf_Prop size
233   OutStreamer->emitInt32(ELF::NT_GNU_PROPERTY_TYPE_0);
234   OutStreamer->emitBytes(StringRef("GNU", 4)); // note name
235 
236   // Emit the PAC/BTI properties.
237   OutStreamer->emitInt32(ELF::GNU_PROPERTY_AARCH64_FEATURE_1_AND);
238   OutStreamer->emitInt32(4);     // data size
239   OutStreamer->emitInt32(Flags); // data
240   OutStreamer->emitInt32(0);     // pad
241 
242   OutStreamer->endSection(Nt);
243   OutStreamer->SwitchSection(Cur);
244 }
245 
246 void AArch64AsmPrinter::emitFunctionHeaderComment() {
247   const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>();
248   Optional<std::string> OutlinerString = FI->getOutliningStyle();
249   if (OutlinerString != None)
250     OutStreamer->GetCommentOS() << ' ' << OutlinerString;
251 }
252 
253 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
254 {
255   const Function &F = MF->getFunction();
256   if (F.hasFnAttribute("patchable-function-entry")) {
257     unsigned Num;
258     if (F.getFnAttribute("patchable-function-entry")
259             .getValueAsString()
260             .getAsInteger(10, Num))
261       return;
262     emitNops(Num);
263     return;
264   }
265 
266   EmitSled(MI, SledKind::FUNCTION_ENTER);
267 }
268 
269 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI)
270 {
271   EmitSled(MI, SledKind::FUNCTION_EXIT);
272 }
273 
274 void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI)
275 {
276   EmitSled(MI, SledKind::TAIL_CALL);
277 }
278 
279 void AArch64AsmPrinter::EmitSled(const MachineInstr &MI, SledKind Kind)
280 {
281   static const int8_t NoopsInSledCount = 7;
282   // We want to emit the following pattern:
283   //
284   // .Lxray_sled_N:
285   //   ALIGN
286   //   B #32
287   //   ; 7 NOP instructions (28 bytes)
288   // .tmpN
289   //
290   // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
291   // over the full 32 bytes (8 instructions) with the following pattern:
292   //
293   //   STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
294   //   LDR W0, #12 ; W0 := function ID
295   //   LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
296   //   BLR X16 ; call the tracing trampoline
297   //   ;DATA: 32 bits of function ID
298   //   ;DATA: lower 32 bits of the address of the trampoline
299   //   ;DATA: higher 32 bits of the address of the trampoline
300   //   LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
301   //
302   OutStreamer->emitCodeAlignment(4);
303   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
304   OutStreamer->emitLabel(CurSled);
305   auto Target = OutContext.createTempSymbol();
306 
307   // Emit "B #32" instruction, which jumps over the next 28 bytes.
308   // The operand has to be the number of 4-byte instructions to jump over,
309   // including the current instruction.
310   EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8));
311 
312   for (int8_t I = 0; I < NoopsInSledCount; I++)
313     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
314 
315   OutStreamer->emitLabel(Target);
316   recordSled(CurSled, MI, Kind, 2);
317 }
318 
319 void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
320   Register Reg = MI.getOperand(0).getReg();
321   bool IsShort =
322       MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES;
323   uint32_t AccessInfo = MI.getOperand(1).getImm();
324   MCSymbol *&Sym =
325       HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, IsShort, AccessInfo)];
326   if (!Sym) {
327     // FIXME: Make this work on non-ELF.
328     if (!TM.getTargetTriple().isOSBinFormatELF())
329       report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF");
330 
331     std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" +
332                           utostr(AccessInfo);
333     if (IsShort)
334       SymName += "_short";
335     Sym = OutContext.getOrCreateSymbol(SymName);
336   }
337 
338   EmitToStreamer(*OutStreamer,
339                  MCInstBuilder(AArch64::BL)
340                      .addExpr(MCSymbolRefExpr::create(Sym, OutContext)));
341 }
342 
343 void AArch64AsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
344   if (HwasanMemaccessSymbols.empty())
345     return;
346 
347   const Triple &TT = TM.getTargetTriple();
348   assert(TT.isOSBinFormatELF());
349   std::unique_ptr<MCSubtargetInfo> STI(
350       TM.getTarget().createMCSubtargetInfo(TT.str(), "", ""));
351 
352   MCSymbol *HwasanTagMismatchV1Sym =
353       OutContext.getOrCreateSymbol("__hwasan_tag_mismatch");
354   MCSymbol *HwasanTagMismatchV2Sym =
355       OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2");
356 
357   const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
358       MCSymbolRefExpr::create(HwasanTagMismatchV1Sym, OutContext);
359   const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
360       MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext);
361 
362   for (auto &P : HwasanMemaccessSymbols) {
363     unsigned Reg = std::get<0>(P.first);
364     bool IsShort = std::get<1>(P.first);
365     uint32_t AccessInfo = std::get<2>(P.first);
366     const MCSymbolRefExpr *HwasanTagMismatchRef =
367         IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
368     MCSymbol *Sym = P.second;
369 
370     OutStreamer->SwitchSection(OutContext.getELFSection(
371         ".text.hot", ELF::SHT_PROGBITS,
372         ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0,
373         Sym->getName()));
374 
375     OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
376     OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
377     OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden);
378     OutStreamer->emitLabel(Sym);
379 
380     OutStreamer->emitInstruction(MCInstBuilder(AArch64::UBFMXri)
381                                      .addReg(AArch64::X16)
382                                      .addReg(Reg)
383                                      .addImm(4)
384                                      .addImm(55),
385                                  *STI);
386     OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRBBroX)
387                                      .addReg(AArch64::W16)
388                                      .addReg(AArch64::X9)
389                                      .addReg(AArch64::X16)
390                                      .addImm(0)
391                                      .addImm(0),
392                                  *STI);
393     OutStreamer->emitInstruction(
394         MCInstBuilder(AArch64::SUBSXrs)
395             .addReg(AArch64::XZR)
396             .addReg(AArch64::X16)
397             .addReg(Reg)
398             .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
399         *STI);
400     MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
401     OutStreamer->emitInstruction(
402         MCInstBuilder(AArch64::Bcc)
403             .addImm(AArch64CC::NE)
404             .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym,
405                                              OutContext)),
406         *STI);
407     MCSymbol *ReturnSym = OutContext.createTempSymbol();
408     OutStreamer->emitLabel(ReturnSym);
409     OutStreamer->emitInstruction(
410         MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI);
411     OutStreamer->emitLabel(HandleMismatchOrPartialSym);
412 
413     if (IsShort) {
414       OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWri)
415                                        .addReg(AArch64::WZR)
416                                        .addReg(AArch64::W16)
417                                        .addImm(15)
418                                        .addImm(0),
419                                    *STI);
420       MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
421       OutStreamer->emitInstruction(
422           MCInstBuilder(AArch64::Bcc)
423               .addImm(AArch64CC::HI)
424               .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
425           *STI);
426 
427       OutStreamer->emitInstruction(
428           MCInstBuilder(AArch64::ANDXri)
429               .addReg(AArch64::X17)
430               .addReg(Reg)
431               .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
432           *STI);
433       unsigned Size = 1 << (AccessInfo & 0xf);
434       if (Size != 1)
435         OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
436                                          .addReg(AArch64::X17)
437                                          .addReg(AArch64::X17)
438                                          .addImm(Size - 1)
439                                          .addImm(0),
440                                      *STI);
441       OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWrs)
442                                        .addReg(AArch64::WZR)
443                                        .addReg(AArch64::W16)
444                                        .addReg(AArch64::W17)
445                                        .addImm(0),
446                                    *STI);
447       OutStreamer->emitInstruction(
448           MCInstBuilder(AArch64::Bcc)
449               .addImm(AArch64CC::LS)
450               .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
451           *STI);
452 
453       OutStreamer->emitInstruction(
454           MCInstBuilder(AArch64::ORRXri)
455               .addReg(AArch64::X16)
456               .addReg(Reg)
457               .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
458           *STI);
459       OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRBBui)
460                                        .addReg(AArch64::W16)
461                                        .addReg(AArch64::X16)
462                                        .addImm(0),
463                                    *STI);
464       OutStreamer->emitInstruction(
465           MCInstBuilder(AArch64::SUBSXrs)
466               .addReg(AArch64::XZR)
467               .addReg(AArch64::X16)
468               .addReg(Reg)
469               .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
470           *STI);
471       OutStreamer->emitInstruction(
472           MCInstBuilder(AArch64::Bcc)
473               .addImm(AArch64CC::EQ)
474               .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
475           *STI);
476 
477       OutStreamer->emitLabel(HandleMismatchSym);
478     }
479 
480     OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre)
481                                      .addReg(AArch64::SP)
482                                      .addReg(AArch64::X0)
483                                      .addReg(AArch64::X1)
484                                      .addReg(AArch64::SP)
485                                      .addImm(-32),
486                                  *STI);
487     OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi)
488                                      .addReg(AArch64::FP)
489                                      .addReg(AArch64::LR)
490                                      .addReg(AArch64::SP)
491                                      .addImm(29),
492                                  *STI);
493 
494     if (Reg != AArch64::X0)
495       OutStreamer->emitInstruction(MCInstBuilder(AArch64::ORRXrs)
496                                        .addReg(AArch64::X0)
497                                        .addReg(AArch64::XZR)
498                                        .addReg(Reg)
499                                        .addImm(0),
500                                    *STI);
501     OutStreamer->emitInstruction(MCInstBuilder(AArch64::MOVZXi)
502                                      .addReg(AArch64::X1)
503                                      .addImm(AccessInfo)
504                                      .addImm(0),
505                                  *STI);
506 
507     // Intentionally load the GOT entry and branch to it, rather than possibly
508     // late binding the function, which may clobber the registers before we have
509     // a chance to save them.
510     OutStreamer->emitInstruction(
511         MCInstBuilder(AArch64::ADRP)
512             .addReg(AArch64::X16)
513             .addExpr(AArch64MCExpr::create(
514                 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_PAGE,
515                 OutContext)),
516         *STI);
517     OutStreamer->emitInstruction(
518         MCInstBuilder(AArch64::LDRXui)
519             .addReg(AArch64::X16)
520             .addReg(AArch64::X16)
521             .addExpr(AArch64MCExpr::create(
522                 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_LO12,
523                 OutContext)),
524         *STI);
525     OutStreamer->emitInstruction(
526         MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI);
527   }
528 }
529 
530 void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
531   EmitHwasanMemaccessSymbols(M);
532 
533   const Triple &TT = TM.getTargetTriple();
534   if (TT.isOSBinFormatMachO()) {
535     // Funny Darwin hack: This flag tells the linker that no global symbols
536     // contain code that falls through to other global symbols (e.g. the obvious
537     // implementation of multiple entry points).  If this doesn't occur, the
538     // linker can safely perform dead code stripping.  Since LLVM never
539     // generates code that does this, it is always safe to set.
540     OutStreamer->emitAssemblerFlag(MCAF_SubsectionsViaSymbols);
541   }
542   emitStackMaps(SM);
543 }
544 
545 void AArch64AsmPrinter::EmitLOHs() {
546   SmallVector<MCSymbol *, 3> MCArgs;
547 
548   for (const auto &D : AArch64FI->getLOHContainer()) {
549     for (const MachineInstr *MI : D.getArgs()) {
550       MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI);
551       assert(LabelIt != LOHInstToLabel.end() &&
552              "Label hasn't been inserted for LOH related instruction");
553       MCArgs.push_back(LabelIt->second);
554     }
555     OutStreamer->emitLOHDirective(D.getKind(), MCArgs);
556     MCArgs.clear();
557   }
558 }
559 
560 void AArch64AsmPrinter::emitFunctionBodyEnd() {
561   if (!AArch64FI->getLOHRelated().empty())
562     EmitLOHs();
563 }
564 
565 /// GetCPISymbol - Return the symbol for the specified constant pool entry.
566 MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
567   // Darwin uses a linker-private symbol name for constant-pools (to
568   // avoid addends on the relocation?), ELF has no such concept and
569   // uses a normal private symbol.
570   if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
571     return OutContext.getOrCreateSymbol(
572         Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
573         Twine(getFunctionNumber()) + "_" + Twine(CPID));
574 
575   return AsmPrinter::GetCPISymbol(CPID);
576 }
577 
578 void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
579                                      raw_ostream &O) {
580   const MachineOperand &MO = MI->getOperand(OpNum);
581   switch (MO.getType()) {
582   default:
583     llvm_unreachable("<unknown operand type>");
584   case MachineOperand::MO_Register: {
585     Register Reg = MO.getReg();
586     assert(Register::isPhysicalRegister(Reg));
587     assert(!MO.getSubReg() && "Subregs should be eliminated!");
588     O << AArch64InstPrinter::getRegisterName(Reg);
589     break;
590   }
591   case MachineOperand::MO_Immediate: {
592     O << MO.getImm();
593     break;
594   }
595   case MachineOperand::MO_GlobalAddress: {
596     PrintSymbolOperand(MO, O);
597     break;
598   }
599   case MachineOperand::MO_BlockAddress: {
600     MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
601     Sym->print(O, MAI);
602     break;
603   }
604   }
605 }
606 
607 bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
608                                           raw_ostream &O) {
609   Register Reg = MO.getReg();
610   switch (Mode) {
611   default:
612     return true; // Unknown mode.
613   case 'w':
614     Reg = getWRegFromXReg(Reg);
615     break;
616   case 'x':
617     Reg = getXRegFromWReg(Reg);
618     break;
619   }
620 
621   O << AArch64InstPrinter::getRegisterName(Reg);
622   return false;
623 }
624 
625 // Prints the register in MO using class RC using the offset in the
626 // new register class. This should not be used for cross class
627 // printing.
628 bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
629                                            const TargetRegisterClass *RC,
630                                            unsigned AltName, raw_ostream &O) {
631   assert(MO.isReg() && "Should only get here with a register!");
632   const TargetRegisterInfo *RI = STI->getRegisterInfo();
633   Register Reg = MO.getReg();
634   unsigned RegToPrint = RC->getRegister(RI->getEncodingValue(Reg));
635   assert(RI->regsOverlap(RegToPrint, Reg));
636   O << AArch64InstPrinter::getRegisterName(RegToPrint, AltName);
637   return false;
638 }
639 
640 bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
641                                         const char *ExtraCode, raw_ostream &O) {
642   const MachineOperand &MO = MI->getOperand(OpNum);
643 
644   // First try the generic code, which knows about modifiers like 'c' and 'n'.
645   if (!AsmPrinter::PrintAsmOperand(MI, OpNum, ExtraCode, O))
646     return false;
647 
648   // Does this asm operand have a single letter operand modifier?
649   if (ExtraCode && ExtraCode[0]) {
650     if (ExtraCode[1] != 0)
651       return true; // Unknown modifier.
652 
653     switch (ExtraCode[0]) {
654     default:
655       return true; // Unknown modifier.
656     case 'w':      // Print W register
657     case 'x':      // Print X register
658       if (MO.isReg())
659         return printAsmMRegister(MO, ExtraCode[0], O);
660       if (MO.isImm() && MO.getImm() == 0) {
661         unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
662         O << AArch64InstPrinter::getRegisterName(Reg);
663         return false;
664       }
665       printOperand(MI, OpNum, O);
666       return false;
667     case 'b': // Print B register.
668     case 'h': // Print H register.
669     case 's': // Print S register.
670     case 'd': // Print D register.
671     case 'q': // Print Q register.
672     case 'z': // Print Z register.
673       if (MO.isReg()) {
674         const TargetRegisterClass *RC;
675         switch (ExtraCode[0]) {
676         case 'b':
677           RC = &AArch64::FPR8RegClass;
678           break;
679         case 'h':
680           RC = &AArch64::FPR16RegClass;
681           break;
682         case 's':
683           RC = &AArch64::FPR32RegClass;
684           break;
685         case 'd':
686           RC = &AArch64::FPR64RegClass;
687           break;
688         case 'q':
689           RC = &AArch64::FPR128RegClass;
690           break;
691         case 'z':
692           RC = &AArch64::ZPRRegClass;
693           break;
694         default:
695           return true;
696         }
697         return printAsmRegInClass(MO, RC, AArch64::NoRegAltName, O);
698       }
699       printOperand(MI, OpNum, O);
700       return false;
701     }
702   }
703 
704   // According to ARM, we should emit x and v registers unless we have a
705   // modifier.
706   if (MO.isReg()) {
707     Register Reg = MO.getReg();
708 
709     // If this is a w or x register, print an x register.
710     if (AArch64::GPR32allRegClass.contains(Reg) ||
711         AArch64::GPR64allRegClass.contains(Reg))
712       return printAsmMRegister(MO, 'x', O);
713 
714     unsigned AltName = AArch64::NoRegAltName;
715     const TargetRegisterClass *RegClass;
716     if (AArch64::ZPRRegClass.contains(Reg)) {
717       RegClass = &AArch64::ZPRRegClass;
718     } else if (AArch64::PPRRegClass.contains(Reg)) {
719       RegClass = &AArch64::PPRRegClass;
720     } else {
721       RegClass = &AArch64::FPR128RegClass;
722       AltName = AArch64::vreg;
723     }
724 
725     // If this is a b, h, s, d, or q register, print it as a v register.
726     return printAsmRegInClass(MO, RegClass, AltName, O);
727   }
728 
729   printOperand(MI, OpNum, O);
730   return false;
731 }
732 
733 bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
734                                               unsigned OpNum,
735                                               const char *ExtraCode,
736                                               raw_ostream &O) {
737   if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
738     return true; // Unknown modifier.
739 
740   const MachineOperand &MO = MI->getOperand(OpNum);
741   assert(MO.isReg() && "unexpected inline asm memory operand");
742   O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]";
743   return false;
744 }
745 
746 void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
747                                                raw_ostream &OS) {
748   unsigned NOps = MI->getNumOperands();
749   assert(NOps == 4);
750   OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
751   // cast away const; DIetc do not take const operands for some reason.
752   OS << cast<DILocalVariable>(MI->getOperand(NOps - 2).getMetadata())
753             ->getName();
754   OS << " <- ";
755   // Frame address.  Currently handles register +- offset only.
756   assert(MI->getOperand(0).isReg() && MI->getOperand(1).isImm());
757   OS << '[';
758   printOperand(MI, 0, OS);
759   OS << '+';
760   printOperand(MI, 1, OS);
761   OS << ']';
762   OS << "+";
763   printOperand(MI, NOps - 2, OS);
764 }
765 
766 void AArch64AsmPrinter::emitJumpTableInfo() {
767   const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
768   if (!MJTI) return;
769 
770   const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
771   if (JT.empty()) return;
772 
773   const Function &F = MF->getFunction();
774   const TargetLoweringObjectFile &TLOF = getObjFileLowering();
775   bool JTInDiffSection =
776       !STI->isTargetCOFF() ||
777       !TLOF.shouldPutJumpTableInFunctionSection(
778           MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32,
779           F);
780   if (JTInDiffSection) {
781       // Drop it in the readonly section.
782       MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(F, TM);
783       OutStreamer->SwitchSection(ReadOnlySec);
784   }
785 
786   auto AFI = MF->getInfo<AArch64FunctionInfo>();
787   for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
788     const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
789 
790     // If this jump table was deleted, ignore it.
791     if (JTBBs.empty()) continue;
792 
793     unsigned Size = AFI->getJumpTableEntrySize(JTI);
794     emitAlignment(Align(Size));
795     OutStreamer->emitLabel(GetJTISymbol(JTI));
796 
797     for (auto *JTBB : JTBBs)
798       emitJumpTableEntry(MJTI, JTBB, JTI);
799   }
800 }
801 
802 void AArch64AsmPrinter::emitJumpTableEntry(const MachineJumpTableInfo *MJTI,
803                                            const MachineBasicBlock *MBB,
804                                            unsigned JTI) {
805   const MCExpr *Value = MCSymbolRefExpr::create(MBB->getSymbol(), OutContext);
806   auto AFI = MF->getInfo<AArch64FunctionInfo>();
807   unsigned Size = AFI->getJumpTableEntrySize(JTI);
808 
809   if (Size == 4) {
810     // .word LBB - LJTI
811     const TargetLowering *TLI = MF->getSubtarget().getTargetLowering();
812     const MCExpr *Base = TLI->getPICJumpTableRelocBaseExpr(MF, JTI, OutContext);
813     Value = MCBinaryExpr::createSub(Value, Base, OutContext);
814   } else {
815     // .byte (LBB - LBB) >> 2 (or .hword)
816     const MCSymbol *BaseSym = AFI->getJumpTableEntryPCRelSymbol(JTI);
817     const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext);
818     Value = MCBinaryExpr::createSub(Value, Base, OutContext);
819     Value = MCBinaryExpr::createLShr(
820         Value, MCConstantExpr::create(2, OutContext), OutContext);
821   }
822 
823   OutStreamer->emitValue(Value, Size);
824 }
825 
826 /// Small jump tables contain an unsigned byte or half, representing the offset
827 /// from the lowest-addressed possible destination to the desired basic
828 /// block. Since all instructions are 4-byte aligned, this is further compressed
829 /// by counting in instructions rather than bytes (i.e. divided by 4). So, to
830 /// materialize the correct destination we need:
831 ///
832 ///             adr xDest, .LBB0_0
833 ///             ldrb wScratch, [xTable, xEntry]   (with "lsl #1" for ldrh).
834 ///             add xDest, xDest, xScratch, lsl #2
835 void AArch64AsmPrinter::LowerJumpTableDestSmall(llvm::MCStreamer &OutStreamer,
836                                                 const llvm::MachineInstr &MI) {
837   Register DestReg = MI.getOperand(0).getReg();
838   Register ScratchReg = MI.getOperand(1).getReg();
839   Register ScratchRegW =
840       STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32);
841   Register TableReg = MI.getOperand(2).getReg();
842   Register EntryReg = MI.getOperand(3).getReg();
843   int JTIdx = MI.getOperand(4).getIndex();
844   bool IsByteEntry = MI.getOpcode() == AArch64::JumpTableDest8;
845 
846   // This has to be first because the compression pass based its reachability
847   // calculations on the start of the JumpTableDest instruction.
848   auto Label =
849       MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx);
850   EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR)
851                                   .addReg(DestReg)
852                                   .addExpr(MCSymbolRefExpr::create(
853                                       Label, MF->getContext())));
854 
855   // Load the number of instruction-steps to offset from the label.
856   unsigned LdrOpcode = IsByteEntry ? AArch64::LDRBBroX : AArch64::LDRHHroX;
857   EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode)
858                                   .addReg(ScratchRegW)
859                                   .addReg(TableReg)
860                                   .addReg(EntryReg)
861                                   .addImm(0)
862                                   .addImm(IsByteEntry ? 0 : 1));
863 
864   // Multiply the steps by 4 and add to the already materialized base label
865   // address.
866   EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs)
867                                   .addReg(DestReg)
868                                   .addReg(DestReg)
869                                   .addReg(ScratchReg)
870                                   .addImm(2));
871 }
872 
873 void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
874                                       const MachineInstr &MI) {
875   unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
876 
877   auto &Ctx = OutStreamer.getContext();
878   MCSymbol *MILabel = Ctx.createTempSymbol();
879   OutStreamer.emitLabel(MILabel);
880 
881   SM.recordStackMap(*MILabel, MI);
882   assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
883 
884   // Scan ahead to trim the shadow.
885   const MachineBasicBlock &MBB = *MI.getParent();
886   MachineBasicBlock::const_iterator MII(MI);
887   ++MII;
888   while (NumNOPBytes > 0) {
889     if (MII == MBB.end() || MII->isCall() ||
890         MII->getOpcode() == AArch64::DBG_VALUE ||
891         MII->getOpcode() == TargetOpcode::PATCHPOINT ||
892         MII->getOpcode() == TargetOpcode::STACKMAP)
893       break;
894     ++MII;
895     NumNOPBytes -= 4;
896   }
897 
898   // Emit nops.
899   for (unsigned i = 0; i < NumNOPBytes; i += 4)
900     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
901 }
902 
903 // Lower a patchpoint of the form:
904 // [<def>], <id>, <numBytes>, <target>, <numArgs>
905 void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
906                                         const MachineInstr &MI) {
907   auto &Ctx = OutStreamer.getContext();
908   MCSymbol *MILabel = Ctx.createTempSymbol();
909   OutStreamer.emitLabel(MILabel);
910   SM.recordPatchPoint(*MILabel, MI);
911 
912   PatchPointOpers Opers(&MI);
913 
914   int64_t CallTarget = Opers.getCallTarget().getImm();
915   unsigned EncodedBytes = 0;
916   if (CallTarget) {
917     assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
918            "High 16 bits of call target should be zero.");
919     Register ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
920     EncodedBytes = 16;
921     // Materialize the jump address:
922     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi)
923                                     .addReg(ScratchReg)
924                                     .addImm((CallTarget >> 32) & 0xFFFF)
925                                     .addImm(32));
926     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
927                                     .addReg(ScratchReg)
928                                     .addReg(ScratchReg)
929                                     .addImm((CallTarget >> 16) & 0xFFFF)
930                                     .addImm(16));
931     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
932                                     .addReg(ScratchReg)
933                                     .addReg(ScratchReg)
934                                     .addImm(CallTarget & 0xFFFF)
935                                     .addImm(0));
936     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
937   }
938   // Emit padding.
939   unsigned NumBytes = Opers.getNumPatchBytes();
940   assert(NumBytes >= EncodedBytes &&
941          "Patchpoint can't request size less than the length of a call.");
942   assert((NumBytes - EncodedBytes) % 4 == 0 &&
943          "Invalid number of NOP bytes requested!");
944   for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
945     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
946 }
947 
948 void AArch64AsmPrinter::EmitFMov0(const MachineInstr &MI) {
949   Register DestReg = MI.getOperand(0).getReg();
950   if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround()) {
951     // Convert H/S/D register to corresponding Q register
952     if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31)
953       DestReg = AArch64::Q0 + (DestReg - AArch64::H0);
954     else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31)
955       DestReg = AArch64::Q0 + (DestReg - AArch64::S0);
956     else {
957       assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31);
958       DestReg = AArch64::Q0 + (DestReg - AArch64::D0);
959     }
960     MCInst MOVI;
961     MOVI.setOpcode(AArch64::MOVIv2d_ns);
962     MOVI.addOperand(MCOperand::createReg(DestReg));
963     MOVI.addOperand(MCOperand::createImm(0));
964     EmitToStreamer(*OutStreamer, MOVI);
965   } else {
966     MCInst FMov;
967     switch (MI.getOpcode()) {
968     default: llvm_unreachable("Unexpected opcode");
969     case AArch64::FMOVH0:
970       FMov.setOpcode(AArch64::FMOVWHr);
971       FMov.addOperand(MCOperand::createReg(DestReg));
972       FMov.addOperand(MCOperand::createReg(AArch64::WZR));
973       break;
974     case AArch64::FMOVS0:
975       FMov.setOpcode(AArch64::FMOVWSr);
976       FMov.addOperand(MCOperand::createReg(DestReg));
977       FMov.addOperand(MCOperand::createReg(AArch64::WZR));
978       break;
979     case AArch64::FMOVD0:
980       FMov.setOpcode(AArch64::FMOVXDr);
981       FMov.addOperand(MCOperand::createReg(DestReg));
982       FMov.addOperand(MCOperand::createReg(AArch64::XZR));
983       break;
984     }
985     EmitToStreamer(*OutStreamer, FMov);
986   }
987 }
988 
989 // Simple pseudo-instructions have their lowering (with expansion to real
990 // instructions) auto-generated.
991 #include "AArch64GenMCPseudoLowering.inc"
992 
993 void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
994   // Do any auto-generated pseudo lowerings.
995   if (emitPseudoExpansionLowering(*OutStreamer, MI))
996     return;
997 
998   if (AArch64FI->getLOHRelated().count(MI)) {
999     // Generate a label for LOH related instruction
1000     MCSymbol *LOHLabel = createTempSymbol("loh");
1001     // Associate the instruction with the label
1002     LOHInstToLabel[MI] = LOHLabel;
1003     OutStreamer->emitLabel(LOHLabel);
1004   }
1005 
1006   AArch64TargetStreamer *TS =
1007     static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
1008   // Do any manual lowerings.
1009   switch (MI->getOpcode()) {
1010   default:
1011     break;
1012   case AArch64::HINT: {
1013     // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
1014     // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
1015     // non-empty. If MI is the initial BTI, place the
1016     // __patchable_function_entries label after BTI.
1017     if (CurrentPatchableFunctionEntrySym &&
1018         CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
1019         MI == &MF->front().front()) {
1020       int64_t Imm = MI->getOperand(0).getImm();
1021       if ((Imm & 32) && (Imm & 6)) {
1022         MCInst Inst;
1023         MCInstLowering.Lower(MI, Inst);
1024         EmitToStreamer(*OutStreamer, Inst);
1025         CurrentPatchableFunctionEntrySym = createTempSymbol("patch");
1026         OutStreamer->emitLabel(CurrentPatchableFunctionEntrySym);
1027         return;
1028       }
1029     }
1030     break;
1031   }
1032     case AArch64::MOVMCSym: {
1033       Register DestReg = MI->getOperand(0).getReg();
1034       const MachineOperand &MO_Sym = MI->getOperand(1);
1035       MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
1036       MCOperand Hi_MCSym, Lo_MCSym;
1037 
1038       Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
1039       Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
1040 
1041       MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym);
1042       MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym);
1043 
1044       MCInst MovZ;
1045       MovZ.setOpcode(AArch64::MOVZXi);
1046       MovZ.addOperand(MCOperand::createReg(DestReg));
1047       MovZ.addOperand(Hi_MCSym);
1048       MovZ.addOperand(MCOperand::createImm(16));
1049       EmitToStreamer(*OutStreamer, MovZ);
1050 
1051       MCInst MovK;
1052       MovK.setOpcode(AArch64::MOVKXi);
1053       MovK.addOperand(MCOperand::createReg(DestReg));
1054       MovK.addOperand(MCOperand::createReg(DestReg));
1055       MovK.addOperand(Lo_MCSym);
1056       MovK.addOperand(MCOperand::createImm(0));
1057       EmitToStreamer(*OutStreamer, MovK);
1058       return;
1059   }
1060   case AArch64::MOVIv2d_ns:
1061     // If the target has <rdar://problem/16473581>, lower this
1062     // instruction to movi.16b instead.
1063     if (STI->hasZeroCycleZeroingFPWorkaround() &&
1064         MI->getOperand(1).getImm() == 0) {
1065       MCInst TmpInst;
1066       TmpInst.setOpcode(AArch64::MOVIv16b_ns);
1067       TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1068       TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm()));
1069       EmitToStreamer(*OutStreamer, TmpInst);
1070       return;
1071     }
1072     break;
1073 
1074   case AArch64::DBG_VALUE: {
1075     if (isVerbose() && OutStreamer->hasRawTextSupport()) {
1076       SmallString<128> TmpStr;
1077       raw_svector_ostream OS(TmpStr);
1078       PrintDebugValueComment(MI, OS);
1079       OutStreamer->emitRawText(StringRef(OS.str()));
1080     }
1081     return;
1082 
1083   case AArch64::EMITBKEY: {
1084       ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
1085       if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
1086           ExceptionHandlingType != ExceptionHandling::ARM)
1087         return;
1088 
1089       if (needsCFIMoves() == CFI_M_None)
1090         return;
1091 
1092       OutStreamer->emitCFIBKeyFrame();
1093       return;
1094     }
1095   }
1096 
1097   // Tail calls use pseudo instructions so they have the proper code-gen
1098   // attributes (isCall, isReturn, etc.). We lower them to the real
1099   // instruction here.
1100   case AArch64::TCRETURNri:
1101   case AArch64::TCRETURNriBTI:
1102   case AArch64::TCRETURNriALL: {
1103     MCInst TmpInst;
1104     TmpInst.setOpcode(AArch64::BR);
1105     TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1106     EmitToStreamer(*OutStreamer, TmpInst);
1107     return;
1108   }
1109   case AArch64::TCRETURNdi: {
1110     MCOperand Dest;
1111     MCInstLowering.lowerOperand(MI->getOperand(0), Dest);
1112     MCInst TmpInst;
1113     TmpInst.setOpcode(AArch64::B);
1114     TmpInst.addOperand(Dest);
1115     EmitToStreamer(*OutStreamer, TmpInst);
1116     return;
1117   }
1118   case AArch64::SpeculationBarrierISBDSBEndBB: {
1119     // Print DSB SYS + ISB
1120     MCInst TmpInstDSB;
1121     TmpInstDSB.setOpcode(AArch64::DSB);
1122     TmpInstDSB.addOperand(MCOperand::createImm(0xf));
1123     EmitToStreamer(*OutStreamer, TmpInstDSB);
1124     MCInst TmpInstISB;
1125     TmpInstISB.setOpcode(AArch64::ISB);
1126     TmpInstISB.addOperand(MCOperand::createImm(0xf));
1127     EmitToStreamer(*OutStreamer, TmpInstISB);
1128     return;
1129   }
1130   case AArch64::SpeculationBarrierSBEndBB: {
1131     // Print SB
1132     MCInst TmpInstSB;
1133     TmpInstSB.setOpcode(AArch64::SB);
1134     EmitToStreamer(*OutStreamer, TmpInstSB);
1135     return;
1136   }
1137   case AArch64::TLSDESC_CALLSEQ: {
1138     /// lower this to:
1139     ///    adrp  x0, :tlsdesc:var
1140     ///    ldr   x1, [x0, #:tlsdesc_lo12:var]
1141     ///    add   x0, x0, #:tlsdesc_lo12:var
1142     ///    .tlsdesccall var
1143     ///    blr   x1
1144     ///    (TPIDR_EL0 offset now in x0)
1145     const MachineOperand &MO_Sym = MI->getOperand(0);
1146     MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
1147     MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
1148     MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
1149     MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
1150     MCInstLowering.lowerOperand(MO_Sym, Sym);
1151     MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12);
1152     MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc);
1153 
1154     MCInst Adrp;
1155     Adrp.setOpcode(AArch64::ADRP);
1156     Adrp.addOperand(MCOperand::createReg(AArch64::X0));
1157     Adrp.addOperand(SymTLSDesc);
1158     EmitToStreamer(*OutStreamer, Adrp);
1159 
1160     MCInst Ldr;
1161     Ldr.setOpcode(AArch64::LDRXui);
1162     Ldr.addOperand(MCOperand::createReg(AArch64::X1));
1163     Ldr.addOperand(MCOperand::createReg(AArch64::X0));
1164     Ldr.addOperand(SymTLSDescLo12);
1165     Ldr.addOperand(MCOperand::createImm(0));
1166     EmitToStreamer(*OutStreamer, Ldr);
1167 
1168     MCInst Add;
1169     Add.setOpcode(AArch64::ADDXri);
1170     Add.addOperand(MCOperand::createReg(AArch64::X0));
1171     Add.addOperand(MCOperand::createReg(AArch64::X0));
1172     Add.addOperand(SymTLSDescLo12);
1173     Add.addOperand(MCOperand::createImm(AArch64_AM::getShiftValue(0)));
1174     EmitToStreamer(*OutStreamer, Add);
1175 
1176     // Emit a relocation-annotation. This expands to no code, but requests
1177     // the following instruction gets an R_AARCH64_TLSDESC_CALL.
1178     MCInst TLSDescCall;
1179     TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
1180     TLSDescCall.addOperand(Sym);
1181     EmitToStreamer(*OutStreamer, TLSDescCall);
1182 
1183     MCInst Blr;
1184     Blr.setOpcode(AArch64::BLR);
1185     Blr.addOperand(MCOperand::createReg(AArch64::X1));
1186     EmitToStreamer(*OutStreamer, Blr);
1187 
1188     return;
1189   }
1190 
1191   case AArch64::JumpTableDest32: {
1192     // We want:
1193     //     ldrsw xScratch, [xTable, xEntry, lsl #2]
1194     //     add xDest, xTable, xScratch
1195     unsigned DestReg = MI->getOperand(0).getReg(),
1196              ScratchReg = MI->getOperand(1).getReg(),
1197              TableReg = MI->getOperand(2).getReg(),
1198              EntryReg = MI->getOperand(3).getReg();
1199     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDRSWroX)
1200                                      .addReg(ScratchReg)
1201                                      .addReg(TableReg)
1202                                      .addReg(EntryReg)
1203                                      .addImm(0)
1204                                      .addImm(1));
1205     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADDXrs)
1206                                      .addReg(DestReg)
1207                                      .addReg(TableReg)
1208                                      .addReg(ScratchReg)
1209                                      .addImm(0));
1210     return;
1211   }
1212   case AArch64::JumpTableDest16:
1213   case AArch64::JumpTableDest8:
1214     LowerJumpTableDestSmall(*OutStreamer, *MI);
1215     return;
1216 
1217   case AArch64::FMOVH0:
1218   case AArch64::FMOVS0:
1219   case AArch64::FMOVD0:
1220     EmitFMov0(*MI);
1221     return;
1222 
1223   case TargetOpcode::STACKMAP:
1224     return LowerSTACKMAP(*OutStreamer, SM, *MI);
1225 
1226   case TargetOpcode::PATCHPOINT:
1227     return LowerPATCHPOINT(*OutStreamer, SM, *MI);
1228 
1229   case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
1230     LowerPATCHABLE_FUNCTION_ENTER(*MI);
1231     return;
1232 
1233   case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
1234     LowerPATCHABLE_FUNCTION_EXIT(*MI);
1235     return;
1236 
1237   case TargetOpcode::PATCHABLE_TAIL_CALL:
1238     LowerPATCHABLE_TAIL_CALL(*MI);
1239     return;
1240 
1241   case AArch64::HWASAN_CHECK_MEMACCESS:
1242   case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
1243     LowerHWASAN_CHECK_MEMACCESS(*MI);
1244     return;
1245 
1246   case AArch64::SEH_StackAlloc:
1247     TS->EmitARM64WinCFIAllocStack(MI->getOperand(0).getImm());
1248     return;
1249 
1250   case AArch64::SEH_SaveFPLR:
1251     TS->EmitARM64WinCFISaveFPLR(MI->getOperand(0).getImm());
1252     return;
1253 
1254   case AArch64::SEH_SaveFPLR_X:
1255     assert(MI->getOperand(0).getImm() < 0 &&
1256            "Pre increment SEH opcode must have a negative offset");
1257     TS->EmitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm());
1258     return;
1259 
1260   case AArch64::SEH_SaveReg:
1261     TS->EmitARM64WinCFISaveReg(MI->getOperand(0).getImm(),
1262                                MI->getOperand(1).getImm());
1263     return;
1264 
1265   case AArch64::SEH_SaveReg_X:
1266     assert(MI->getOperand(1).getImm() < 0 &&
1267            "Pre increment SEH opcode must have a negative offset");
1268     TS->EmitARM64WinCFISaveRegX(MI->getOperand(0).getImm(),
1269 		                -MI->getOperand(1).getImm());
1270     return;
1271 
1272   case AArch64::SEH_SaveRegP:
1273     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1274             "Non-consecutive registers not allowed for save_regp");
1275     TS->EmitARM64WinCFISaveRegP(MI->getOperand(0).getImm(),
1276                                 MI->getOperand(2).getImm());
1277     return;
1278 
1279   case AArch64::SEH_SaveRegP_X:
1280     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1281             "Non-consecutive registers not allowed for save_regp_x");
1282     assert(MI->getOperand(2).getImm() < 0 &&
1283            "Pre increment SEH opcode must have a negative offset");
1284     TS->EmitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(),
1285                                  -MI->getOperand(2).getImm());
1286     return;
1287 
1288   case AArch64::SEH_SaveFReg:
1289     TS->EmitARM64WinCFISaveFReg(MI->getOperand(0).getImm(),
1290                                 MI->getOperand(1).getImm());
1291     return;
1292 
1293   case AArch64::SEH_SaveFReg_X:
1294     assert(MI->getOperand(1).getImm() < 0 &&
1295            "Pre increment SEH opcode must have a negative offset");
1296     TS->EmitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(),
1297                                  -MI->getOperand(1).getImm());
1298     return;
1299 
1300   case AArch64::SEH_SaveFRegP:
1301     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1302             "Non-consecutive registers not allowed for save_regp");
1303     TS->EmitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(),
1304                                  MI->getOperand(2).getImm());
1305     return;
1306 
1307   case AArch64::SEH_SaveFRegP_X:
1308     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1309             "Non-consecutive registers not allowed for save_regp_x");
1310     assert(MI->getOperand(2).getImm() < 0 &&
1311            "Pre increment SEH opcode must have a negative offset");
1312     TS->EmitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(),
1313                                   -MI->getOperand(2).getImm());
1314     return;
1315 
1316   case AArch64::SEH_SetFP:
1317     TS->EmitARM64WinCFISetFP();
1318     return;
1319 
1320   case AArch64::SEH_AddFP:
1321     TS->EmitARM64WinCFIAddFP(MI->getOperand(0).getImm());
1322     return;
1323 
1324   case AArch64::SEH_Nop:
1325     TS->EmitARM64WinCFINop();
1326     return;
1327 
1328   case AArch64::SEH_PrologEnd:
1329     TS->EmitARM64WinCFIPrologEnd();
1330     return;
1331 
1332   case AArch64::SEH_EpilogStart:
1333     TS->EmitARM64WinCFIEpilogStart();
1334     return;
1335 
1336   case AArch64::SEH_EpilogEnd:
1337     TS->EmitARM64WinCFIEpilogEnd();
1338     return;
1339   }
1340 
1341   // Finally, do the automated lowerings for everything else.
1342   MCInst TmpInst;
1343   MCInstLowering.Lower(MI, TmpInst);
1344   EmitToStreamer(*OutStreamer, TmpInst);
1345 }
1346 
1347 // Force static initialization.
1348 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter() {
1349   RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
1350   RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget());
1351   RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target());
1352   RegisterAsmPrinter<AArch64AsmPrinter> W(getTheARM64_32Target());
1353   RegisterAsmPrinter<AArch64AsmPrinter> V(getTheAArch64_32Target());
1354 }
1355