1 //===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains a printer that converts from our internal representation
10 // of machine-dependent LLVM code to the AArch64 assembly language.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "AArch64.h"
15 #include "AArch64MCInstLower.h"
16 #include "AArch64MachineFunctionInfo.h"
17 #include "AArch64RegisterInfo.h"
18 #include "AArch64Subtarget.h"
19 #include "AArch64TargetObjectFile.h"
20 #include "MCTargetDesc/AArch64AddressingModes.h"
21 #include "MCTargetDesc/AArch64InstPrinter.h"
22 #include "MCTargetDesc/AArch64MCExpr.h"
23 #include "MCTargetDesc/AArch64MCTargetDesc.h"
24 #include "MCTargetDesc/AArch64TargetStreamer.h"
25 #include "TargetInfo/AArch64TargetInfo.h"
26 #include "Utils/AArch64BaseInfo.h"
27 #include "llvm/ADT/SmallString.h"
28 #include "llvm/ADT/SmallVector.h"
29 #include "llvm/ADT/StringRef.h"
30 #include "llvm/ADT/Triple.h"
31 #include "llvm/ADT/Twine.h"
32 #include "llvm/BinaryFormat/COFF.h"
33 #include "llvm/BinaryFormat/ELF.h"
34 #include "llvm/CodeGen/AsmPrinter.h"
35 #include "llvm/CodeGen/FaultMaps.h"
36 #include "llvm/CodeGen/MachineBasicBlock.h"
37 #include "llvm/CodeGen/MachineFunction.h"
38 #include "llvm/CodeGen/MachineInstr.h"
39 #include "llvm/CodeGen/MachineJumpTableInfo.h"
40 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
41 #include "llvm/CodeGen/MachineOperand.h"
42 #include "llvm/CodeGen/StackMaps.h"
43 #include "llvm/CodeGen/TargetRegisterInfo.h"
44 #include "llvm/IR/DataLayout.h"
45 #include "llvm/IR/DebugInfoMetadata.h"
46 #include "llvm/MC/MCAsmInfo.h"
47 #include "llvm/MC/MCContext.h"
48 #include "llvm/MC/MCInst.h"
49 #include "llvm/MC/MCInstBuilder.h"
50 #include "llvm/MC/MCSectionELF.h"
51 #include "llvm/MC/MCStreamer.h"
52 #include "llvm/MC/MCSymbol.h"
53 #include "llvm/Support/Casting.h"
54 #include "llvm/Support/ErrorHandling.h"
55 #include "llvm/Support/TargetRegistry.h"
56 #include "llvm/Support/raw_ostream.h"
57 #include "llvm/Target/TargetMachine.h"
58 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
59 #include <algorithm>
60 #include <cassert>
61 #include <cstdint>
62 #include <map>
63 #include <memory>
64 
65 using namespace llvm;
66 
67 #define DEBUG_TYPE "asm-printer"
68 
69 namespace {
70 
71 class AArch64AsmPrinter : public AsmPrinter {
72   AArch64MCInstLower MCInstLowering;
73   StackMaps SM;
74   FaultMaps FM;
75   const AArch64Subtarget *STI;
76 
77 public:
78   AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
79       : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this),
80         SM(*this), FM(*this) {}
81 
82   StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
83 
84   /// Wrapper for MCInstLowering.lowerOperand() for the
85   /// tblgen'erated pseudo lowering.
86   bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
87     return MCInstLowering.lowerOperand(MO, MCOp);
88   }
89 
90   void emitStartOfAsmFile(Module &M) override;
91   void emitJumpTableInfo() override;
92 
93   void emitFunctionEntryLabel() override;
94 
95   void LowerJumpTableDest(MCStreamer &OutStreamer, const MachineInstr &MI);
96 
97   void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
98                      const MachineInstr &MI);
99   void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
100                        const MachineInstr &MI);
101   void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
102                        const MachineInstr &MI);
103   void LowerFAULTING_OP(const MachineInstr &MI);
104 
105   void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
106   void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
107   void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
108 
109   typedef std::tuple<unsigned, bool, uint32_t> HwasanMemaccessTuple;
110   std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
111   void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
112   void EmitHwasanMemaccessSymbols(Module &M);
113 
114   void EmitSled(const MachineInstr &MI, SledKind Kind);
115 
116   /// tblgen'erated driver function for lowering simple MI->MC
117   /// pseudo instructions.
118   bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
119                                    const MachineInstr *MI);
120 
121   void emitInstruction(const MachineInstr *MI) override;
122 
123   void emitFunctionHeaderComment() override;
124 
125   void getAnalysisUsage(AnalysisUsage &AU) const override {
126     AsmPrinter::getAnalysisUsage(AU);
127     AU.setPreservesAll();
128   }
129 
130   bool runOnMachineFunction(MachineFunction &MF) override {
131     AArch64FI = MF.getInfo<AArch64FunctionInfo>();
132     STI = static_cast<const AArch64Subtarget*>(&MF.getSubtarget());
133 
134     SetupMachineFunction(MF);
135 
136     if (STI->isTargetCOFF()) {
137       bool Internal = MF.getFunction().hasInternalLinkage();
138       COFF::SymbolStorageClass Scl = Internal ? COFF::IMAGE_SYM_CLASS_STATIC
139                                               : COFF::IMAGE_SYM_CLASS_EXTERNAL;
140       int Type =
141         COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT;
142 
143       OutStreamer->BeginCOFFSymbolDef(CurrentFnSym);
144       OutStreamer->EmitCOFFSymbolStorageClass(Scl);
145       OutStreamer->EmitCOFFSymbolType(Type);
146       OutStreamer->EndCOFFSymbolDef();
147     }
148 
149     // Emit the rest of the function body.
150     emitFunctionBody();
151 
152     // Emit the XRay table for this function.
153     emitXRayTable();
154 
155     // We didn't modify anything.
156     return false;
157   }
158 
159 private:
160   void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
161   bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
162   bool printAsmRegInClass(const MachineOperand &MO,
163                           const TargetRegisterClass *RC, unsigned AltName,
164                           raw_ostream &O);
165 
166   bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
167                        const char *ExtraCode, raw_ostream &O) override;
168   bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
169                              const char *ExtraCode, raw_ostream &O) override;
170 
171   void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
172 
173   void emitFunctionBodyEnd() override;
174 
175   MCSymbol *GetCPISymbol(unsigned CPID) const override;
176   void emitEndOfAsmFile(Module &M) override;
177 
178   AArch64FunctionInfo *AArch64FI = nullptr;
179 
180   /// Emit the LOHs contained in AArch64FI.
181   void EmitLOHs();
182 
183   /// Emit instruction to set float register to zero.
184   void EmitFMov0(const MachineInstr &MI);
185 
186   using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
187 
188   MInstToMCSymbol LOHInstToLabel;
189 };
190 
191 } // end anonymous namespace
192 
193 void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
194   if (!TM.getTargetTriple().isOSBinFormatELF())
195     return;
196 
197   // Assemble feature flags that may require creation of a note section.
198   unsigned Flags = 0;
199   if (const auto *BTE = mdconst::extract_or_null<ConstantInt>(
200           M.getModuleFlag("branch-target-enforcement")))
201     if (BTE->getZExtValue())
202       Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI;
203 
204   if (const auto *Sign = mdconst::extract_or_null<ConstantInt>(
205           M.getModuleFlag("sign-return-address")))
206     if (Sign->getZExtValue())
207       Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
208 
209   if (Flags == 0)
210     return;
211 
212   // Emit a .note.gnu.property section with the flags.
213   if (auto *TS = static_cast<AArch64TargetStreamer *>(
214           OutStreamer->getTargetStreamer()))
215     TS->emitNoteSection(Flags);
216 }
217 
218 void AArch64AsmPrinter::emitFunctionHeaderComment() {
219   const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>();
220   Optional<std::string> OutlinerString = FI->getOutliningStyle();
221   if (OutlinerString != None)
222     OutStreamer->GetCommentOS() << ' ' << OutlinerString;
223 }
224 
225 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
226 {
227   const Function &F = MF->getFunction();
228   if (F.hasFnAttribute("patchable-function-entry")) {
229     unsigned Num;
230     if (F.getFnAttribute("patchable-function-entry")
231             .getValueAsString()
232             .getAsInteger(10, Num))
233       return;
234     emitNops(Num);
235     return;
236   }
237 
238   EmitSled(MI, SledKind::FUNCTION_ENTER);
239 }
240 
241 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI)
242 {
243   EmitSled(MI, SledKind::FUNCTION_EXIT);
244 }
245 
246 void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI)
247 {
248   EmitSled(MI, SledKind::TAIL_CALL);
249 }
250 
251 void AArch64AsmPrinter::EmitSled(const MachineInstr &MI, SledKind Kind)
252 {
253   static const int8_t NoopsInSledCount = 7;
254   // We want to emit the following pattern:
255   //
256   // .Lxray_sled_N:
257   //   ALIGN
258   //   B #32
259   //   ; 7 NOP instructions (28 bytes)
260   // .tmpN
261   //
262   // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
263   // over the full 32 bytes (8 instructions) with the following pattern:
264   //
265   //   STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
266   //   LDR W0, #12 ; W0 := function ID
267   //   LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
268   //   BLR X16 ; call the tracing trampoline
269   //   ;DATA: 32 bits of function ID
270   //   ;DATA: lower 32 bits of the address of the trampoline
271   //   ;DATA: higher 32 bits of the address of the trampoline
272   //   LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
273   //
274   OutStreamer->emitCodeAlignment(4);
275   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
276   OutStreamer->emitLabel(CurSled);
277   auto Target = OutContext.createTempSymbol();
278 
279   // Emit "B #32" instruction, which jumps over the next 28 bytes.
280   // The operand has to be the number of 4-byte instructions to jump over,
281   // including the current instruction.
282   EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8));
283 
284   for (int8_t I = 0; I < NoopsInSledCount; I++)
285     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
286 
287   OutStreamer->emitLabel(Target);
288   recordSled(CurSled, MI, Kind, 2);
289 }
290 
291 void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
292   Register Reg = MI.getOperand(0).getReg();
293   bool IsShort =
294       MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES;
295   uint32_t AccessInfo = MI.getOperand(1).getImm();
296   MCSymbol *&Sym =
297       HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, IsShort, AccessInfo)];
298   if (!Sym) {
299     // FIXME: Make this work on non-ELF.
300     if (!TM.getTargetTriple().isOSBinFormatELF())
301       report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF");
302 
303     std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" +
304                           utostr(AccessInfo);
305     if (IsShort)
306       SymName += "_short_v2";
307     Sym = OutContext.getOrCreateSymbol(SymName);
308   }
309 
310   EmitToStreamer(*OutStreamer,
311                  MCInstBuilder(AArch64::BL)
312                      .addExpr(MCSymbolRefExpr::create(Sym, OutContext)));
313 }
314 
315 void AArch64AsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
316   if (HwasanMemaccessSymbols.empty())
317     return;
318 
319   const Triple &TT = TM.getTargetTriple();
320   assert(TT.isOSBinFormatELF());
321   std::unique_ptr<MCSubtargetInfo> STI(
322       TM.getTarget().createMCSubtargetInfo(TT.str(), "", ""));
323   assert(STI && "Unable to create subtarget info");
324 
325   MCSymbol *HwasanTagMismatchV1Sym =
326       OutContext.getOrCreateSymbol("__hwasan_tag_mismatch");
327   MCSymbol *HwasanTagMismatchV2Sym =
328       OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2");
329 
330   const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
331       MCSymbolRefExpr::create(HwasanTagMismatchV1Sym, OutContext);
332   const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
333       MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext);
334 
335   for (auto &P : HwasanMemaccessSymbols) {
336     unsigned Reg = std::get<0>(P.first);
337     bool IsShort = std::get<1>(P.first);
338     uint32_t AccessInfo = std::get<2>(P.first);
339     const MCSymbolRefExpr *HwasanTagMismatchRef =
340         IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
341     MCSymbol *Sym = P.second;
342 
343     bool HasMatchAllTag =
344         (AccessInfo >> HWASanAccessInfo::HasMatchAllShift) & 1;
345     uint8_t MatchAllTag =
346         (AccessInfo >> HWASanAccessInfo::MatchAllShift) & 0xff;
347     unsigned Size =
348         1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf);
349     bool CompileKernel =
350         (AccessInfo >> HWASanAccessInfo::CompileKernelShift) & 1;
351 
352     OutStreamer->SwitchSection(OutContext.getELFSection(
353         ".text.hot", ELF::SHT_PROGBITS,
354         ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0,
355         Sym->getName()));
356 
357     OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
358     OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
359     OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden);
360     OutStreamer->emitLabel(Sym);
361 
362     OutStreamer->emitInstruction(MCInstBuilder(AArch64::SBFMXri)
363                                      .addReg(AArch64::X16)
364                                      .addReg(Reg)
365                                      .addImm(4)
366                                      .addImm(55),
367                                  *STI);
368     OutStreamer->emitInstruction(
369         MCInstBuilder(AArch64::LDRBBroX)
370             .addReg(AArch64::W16)
371             .addReg(IsShort ? AArch64::X20 : AArch64::X9)
372             .addReg(AArch64::X16)
373             .addImm(0)
374             .addImm(0),
375         *STI);
376     OutStreamer->emitInstruction(
377         MCInstBuilder(AArch64::SUBSXrs)
378             .addReg(AArch64::XZR)
379             .addReg(AArch64::X16)
380             .addReg(Reg)
381             .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
382         *STI);
383     MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
384     OutStreamer->emitInstruction(
385         MCInstBuilder(AArch64::Bcc)
386             .addImm(AArch64CC::NE)
387             .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym,
388                                              OutContext)),
389         *STI);
390     MCSymbol *ReturnSym = OutContext.createTempSymbol();
391     OutStreamer->emitLabel(ReturnSym);
392     OutStreamer->emitInstruction(
393         MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI);
394     OutStreamer->emitLabel(HandleMismatchOrPartialSym);
395 
396     if (HasMatchAllTag) {
397       OutStreamer->emitInstruction(MCInstBuilder(AArch64::UBFMXri)
398                                        .addReg(AArch64::X16)
399                                        .addReg(Reg)
400                                        .addImm(56)
401                                        .addImm(63),
402                                    *STI);
403       OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSXri)
404                                        .addReg(AArch64::XZR)
405                                        .addReg(AArch64::X16)
406                                        .addImm(MatchAllTag)
407                                        .addImm(0),
408                                    *STI);
409       OutStreamer->emitInstruction(
410           MCInstBuilder(AArch64::Bcc)
411               .addImm(AArch64CC::EQ)
412               .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
413           *STI);
414     }
415 
416     if (IsShort) {
417       OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWri)
418                                        .addReg(AArch64::WZR)
419                                        .addReg(AArch64::W16)
420                                        .addImm(15)
421                                        .addImm(0),
422                                    *STI);
423       MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
424       OutStreamer->emitInstruction(
425           MCInstBuilder(AArch64::Bcc)
426               .addImm(AArch64CC::HI)
427               .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
428           *STI);
429 
430       OutStreamer->emitInstruction(
431           MCInstBuilder(AArch64::ANDXri)
432               .addReg(AArch64::X17)
433               .addReg(Reg)
434               .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
435           *STI);
436       if (Size != 1)
437         OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
438                                          .addReg(AArch64::X17)
439                                          .addReg(AArch64::X17)
440                                          .addImm(Size - 1)
441                                          .addImm(0),
442                                      *STI);
443       OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWrs)
444                                        .addReg(AArch64::WZR)
445                                        .addReg(AArch64::W16)
446                                        .addReg(AArch64::W17)
447                                        .addImm(0),
448                                    *STI);
449       OutStreamer->emitInstruction(
450           MCInstBuilder(AArch64::Bcc)
451               .addImm(AArch64CC::LS)
452               .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
453           *STI);
454 
455       OutStreamer->emitInstruction(
456           MCInstBuilder(AArch64::ORRXri)
457               .addReg(AArch64::X16)
458               .addReg(Reg)
459               .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
460           *STI);
461       OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRBBui)
462                                        .addReg(AArch64::W16)
463                                        .addReg(AArch64::X16)
464                                        .addImm(0),
465                                    *STI);
466       OutStreamer->emitInstruction(
467           MCInstBuilder(AArch64::SUBSXrs)
468               .addReg(AArch64::XZR)
469               .addReg(AArch64::X16)
470               .addReg(Reg)
471               .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
472           *STI);
473       OutStreamer->emitInstruction(
474           MCInstBuilder(AArch64::Bcc)
475               .addImm(AArch64CC::EQ)
476               .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
477           *STI);
478 
479       OutStreamer->emitLabel(HandleMismatchSym);
480     }
481 
482     OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre)
483                                      .addReg(AArch64::SP)
484                                      .addReg(AArch64::X0)
485                                      .addReg(AArch64::X1)
486                                      .addReg(AArch64::SP)
487                                      .addImm(-32),
488                                  *STI);
489     OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi)
490                                      .addReg(AArch64::FP)
491                                      .addReg(AArch64::LR)
492                                      .addReg(AArch64::SP)
493                                      .addImm(29),
494                                  *STI);
495 
496     if (Reg != AArch64::X0)
497       OutStreamer->emitInstruction(MCInstBuilder(AArch64::ORRXrs)
498                                        .addReg(AArch64::X0)
499                                        .addReg(AArch64::XZR)
500                                        .addReg(Reg)
501                                        .addImm(0),
502                                    *STI);
503     OutStreamer->emitInstruction(
504         MCInstBuilder(AArch64::MOVZXi)
505             .addReg(AArch64::X1)
506             .addImm(AccessInfo & HWASanAccessInfo::RuntimeMask)
507             .addImm(0),
508         *STI);
509 
510     if (CompileKernel) {
511       // The Linux kernel's dynamic loader doesn't support GOT relative
512       // relocations, but it doesn't support late binding either, so just call
513       // the function directly.
514       OutStreamer->emitInstruction(
515           MCInstBuilder(AArch64::B).addExpr(HwasanTagMismatchRef), *STI);
516     } else {
517       // Intentionally load the GOT entry and branch to it, rather than possibly
518       // late binding the function, which may clobber the registers before we
519       // have a chance to save them.
520       OutStreamer->emitInstruction(
521           MCInstBuilder(AArch64::ADRP)
522               .addReg(AArch64::X16)
523               .addExpr(AArch64MCExpr::create(
524                   HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_PAGE,
525                   OutContext)),
526           *STI);
527       OutStreamer->emitInstruction(
528           MCInstBuilder(AArch64::LDRXui)
529               .addReg(AArch64::X16)
530               .addReg(AArch64::X16)
531               .addExpr(AArch64MCExpr::create(
532                   HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_LO12,
533                   OutContext)),
534           *STI);
535       OutStreamer->emitInstruction(
536           MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI);
537     }
538   }
539 }
540 
541 void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
542   EmitHwasanMemaccessSymbols(M);
543 
544   const Triple &TT = TM.getTargetTriple();
545   if (TT.isOSBinFormatMachO()) {
546     // Funny Darwin hack: This flag tells the linker that no global symbols
547     // contain code that falls through to other global symbols (e.g. the obvious
548     // implementation of multiple entry points).  If this doesn't occur, the
549     // linker can safely perform dead code stripping.  Since LLVM never
550     // generates code that does this, it is always safe to set.
551     OutStreamer->emitAssemblerFlag(MCAF_SubsectionsViaSymbols);
552   }
553 
554   // Emit stack and fault map information.
555   emitStackMaps(SM);
556   FM.serializeToFaultMapSection();
557 
558 }
559 
560 void AArch64AsmPrinter::EmitLOHs() {
561   SmallVector<MCSymbol *, 3> MCArgs;
562 
563   for (const auto &D : AArch64FI->getLOHContainer()) {
564     for (const MachineInstr *MI : D.getArgs()) {
565       MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI);
566       assert(LabelIt != LOHInstToLabel.end() &&
567              "Label hasn't been inserted for LOH related instruction");
568       MCArgs.push_back(LabelIt->second);
569     }
570     OutStreamer->emitLOHDirective(D.getKind(), MCArgs);
571     MCArgs.clear();
572   }
573 }
574 
575 void AArch64AsmPrinter::emitFunctionBodyEnd() {
576   if (!AArch64FI->getLOHRelated().empty())
577     EmitLOHs();
578 }
579 
580 /// GetCPISymbol - Return the symbol for the specified constant pool entry.
581 MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
582   // Darwin uses a linker-private symbol name for constant-pools (to
583   // avoid addends on the relocation?), ELF has no such concept and
584   // uses a normal private symbol.
585   if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
586     return OutContext.getOrCreateSymbol(
587         Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
588         Twine(getFunctionNumber()) + "_" + Twine(CPID));
589 
590   return AsmPrinter::GetCPISymbol(CPID);
591 }
592 
593 void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
594                                      raw_ostream &O) {
595   const MachineOperand &MO = MI->getOperand(OpNum);
596   switch (MO.getType()) {
597   default:
598     llvm_unreachable("<unknown operand type>");
599   case MachineOperand::MO_Register: {
600     Register Reg = MO.getReg();
601     assert(Register::isPhysicalRegister(Reg));
602     assert(!MO.getSubReg() && "Subregs should be eliminated!");
603     O << AArch64InstPrinter::getRegisterName(Reg);
604     break;
605   }
606   case MachineOperand::MO_Immediate: {
607     O << MO.getImm();
608     break;
609   }
610   case MachineOperand::MO_GlobalAddress: {
611     PrintSymbolOperand(MO, O);
612     break;
613   }
614   case MachineOperand::MO_BlockAddress: {
615     MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
616     Sym->print(O, MAI);
617     break;
618   }
619   }
620 }
621 
622 bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
623                                           raw_ostream &O) {
624   Register Reg = MO.getReg();
625   switch (Mode) {
626   default:
627     return true; // Unknown mode.
628   case 'w':
629     Reg = getWRegFromXReg(Reg);
630     break;
631   case 'x':
632     Reg = getXRegFromWReg(Reg);
633     break;
634   }
635 
636   O << AArch64InstPrinter::getRegisterName(Reg);
637   return false;
638 }
639 
640 // Prints the register in MO using class RC using the offset in the
641 // new register class. This should not be used for cross class
642 // printing.
643 bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
644                                            const TargetRegisterClass *RC,
645                                            unsigned AltName, raw_ostream &O) {
646   assert(MO.isReg() && "Should only get here with a register!");
647   const TargetRegisterInfo *RI = STI->getRegisterInfo();
648   Register Reg = MO.getReg();
649   unsigned RegToPrint = RC->getRegister(RI->getEncodingValue(Reg));
650   assert(RI->regsOverlap(RegToPrint, Reg));
651   O << AArch64InstPrinter::getRegisterName(RegToPrint, AltName);
652   return false;
653 }
654 
655 bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
656                                         const char *ExtraCode, raw_ostream &O) {
657   const MachineOperand &MO = MI->getOperand(OpNum);
658 
659   // First try the generic code, which knows about modifiers like 'c' and 'n'.
660   if (!AsmPrinter::PrintAsmOperand(MI, OpNum, ExtraCode, O))
661     return false;
662 
663   // Does this asm operand have a single letter operand modifier?
664   if (ExtraCode && ExtraCode[0]) {
665     if (ExtraCode[1] != 0)
666       return true; // Unknown modifier.
667 
668     switch (ExtraCode[0]) {
669     default:
670       return true; // Unknown modifier.
671     case 'w':      // Print W register
672     case 'x':      // Print X register
673       if (MO.isReg())
674         return printAsmMRegister(MO, ExtraCode[0], O);
675       if (MO.isImm() && MO.getImm() == 0) {
676         unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
677         O << AArch64InstPrinter::getRegisterName(Reg);
678         return false;
679       }
680       printOperand(MI, OpNum, O);
681       return false;
682     case 'b': // Print B register.
683     case 'h': // Print H register.
684     case 's': // Print S register.
685     case 'd': // Print D register.
686     case 'q': // Print Q register.
687     case 'z': // Print Z register.
688       if (MO.isReg()) {
689         const TargetRegisterClass *RC;
690         switch (ExtraCode[0]) {
691         case 'b':
692           RC = &AArch64::FPR8RegClass;
693           break;
694         case 'h':
695           RC = &AArch64::FPR16RegClass;
696           break;
697         case 's':
698           RC = &AArch64::FPR32RegClass;
699           break;
700         case 'd':
701           RC = &AArch64::FPR64RegClass;
702           break;
703         case 'q':
704           RC = &AArch64::FPR128RegClass;
705           break;
706         case 'z':
707           RC = &AArch64::ZPRRegClass;
708           break;
709         default:
710           return true;
711         }
712         return printAsmRegInClass(MO, RC, AArch64::NoRegAltName, O);
713       }
714       printOperand(MI, OpNum, O);
715       return false;
716     }
717   }
718 
719   // According to ARM, we should emit x and v registers unless we have a
720   // modifier.
721   if (MO.isReg()) {
722     Register Reg = MO.getReg();
723 
724     // If this is a w or x register, print an x register.
725     if (AArch64::GPR32allRegClass.contains(Reg) ||
726         AArch64::GPR64allRegClass.contains(Reg))
727       return printAsmMRegister(MO, 'x', O);
728 
729     unsigned AltName = AArch64::NoRegAltName;
730     const TargetRegisterClass *RegClass;
731     if (AArch64::ZPRRegClass.contains(Reg)) {
732       RegClass = &AArch64::ZPRRegClass;
733     } else if (AArch64::PPRRegClass.contains(Reg)) {
734       RegClass = &AArch64::PPRRegClass;
735     } else {
736       RegClass = &AArch64::FPR128RegClass;
737       AltName = AArch64::vreg;
738     }
739 
740     // If this is a b, h, s, d, or q register, print it as a v register.
741     return printAsmRegInClass(MO, RegClass, AltName, O);
742   }
743 
744   printOperand(MI, OpNum, O);
745   return false;
746 }
747 
748 bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
749                                               unsigned OpNum,
750                                               const char *ExtraCode,
751                                               raw_ostream &O) {
752   if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
753     return true; // Unknown modifier.
754 
755   const MachineOperand &MO = MI->getOperand(OpNum);
756   assert(MO.isReg() && "unexpected inline asm memory operand");
757   O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]";
758   return false;
759 }
760 
761 void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
762                                                raw_ostream &OS) {
763   unsigned NOps = MI->getNumOperands();
764   assert(NOps == 4);
765   OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
766   // cast away const; DIetc do not take const operands for some reason.
767   OS << MI->getDebugVariable()->getName();
768   OS << " <- ";
769   // Frame address.  Currently handles register +- offset only.
770   assert(MI->getDebugOperand(0).isReg() && MI->isDebugOffsetImm());
771   OS << '[';
772   printOperand(MI, 0, OS);
773   OS << '+';
774   printOperand(MI, 1, OS);
775   OS << ']';
776   OS << "+";
777   printOperand(MI, NOps - 2, OS);
778 }
779 
780 void AArch64AsmPrinter::emitJumpTableInfo() {
781   const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
782   if (!MJTI) return;
783 
784   const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
785   if (JT.empty()) return;
786 
787   const Function &F = MF->getFunction();
788   const TargetLoweringObjectFile &TLOF = getObjFileLowering();
789   bool JTInDiffSection =
790       !STI->isTargetCOFF() ||
791       !TLOF.shouldPutJumpTableInFunctionSection(
792           MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32,
793           F);
794   if (JTInDiffSection) {
795       // Drop it in the readonly section.
796       MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(F, TM);
797       OutStreamer->SwitchSection(ReadOnlySec);
798   }
799 
800   auto AFI = MF->getInfo<AArch64FunctionInfo>();
801   for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
802     const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
803 
804     // If this jump table was deleted, ignore it.
805     if (JTBBs.empty()) continue;
806 
807     unsigned Size = AFI->getJumpTableEntrySize(JTI);
808     emitAlignment(Align(Size));
809     OutStreamer->emitLabel(GetJTISymbol(JTI));
810 
811     const MCSymbol *BaseSym = AArch64FI->getJumpTableEntryPCRelSymbol(JTI);
812     const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext);
813 
814     for (auto *JTBB : JTBBs) {
815       const MCExpr *Value =
816           MCSymbolRefExpr::create(JTBB->getSymbol(), OutContext);
817 
818       // Each entry is:
819       //     .byte/.hword (LBB - Lbase)>>2
820       // or plain:
821       //     .word LBB - Lbase
822       Value = MCBinaryExpr::createSub(Value, Base, OutContext);
823       if (Size != 4)
824         Value = MCBinaryExpr::createLShr(
825             Value, MCConstantExpr::create(2, OutContext), OutContext);
826 
827       OutStreamer->emitValue(Value, Size);
828     }
829   }
830 }
831 
832 void AArch64AsmPrinter::emitFunctionEntryLabel() {
833   if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall ||
834       MF->getFunction().getCallingConv() ==
835           CallingConv::AArch64_SVE_VectorCall ||
836       STI->getRegisterInfo()->hasSVEArgsOrReturn(MF)) {
837     auto *TS =
838         static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
839     TS->emitDirectiveVariantPCS(CurrentFnSym);
840   }
841 
842   return AsmPrinter::emitFunctionEntryLabel();
843 }
844 
845 /// Small jump tables contain an unsigned byte or half, representing the offset
846 /// from the lowest-addressed possible destination to the desired basic
847 /// block. Since all instructions are 4-byte aligned, this is further compressed
848 /// by counting in instructions rather than bytes (i.e. divided by 4). So, to
849 /// materialize the correct destination we need:
850 ///
851 ///             adr xDest, .LBB0_0
852 ///             ldrb wScratch, [xTable, xEntry]   (with "lsl #1" for ldrh).
853 ///             add xDest, xDest, xScratch (with "lsl #2" for smaller entries)
854 void AArch64AsmPrinter::LowerJumpTableDest(llvm::MCStreamer &OutStreamer,
855                                            const llvm::MachineInstr &MI) {
856   Register DestReg = MI.getOperand(0).getReg();
857   Register ScratchReg = MI.getOperand(1).getReg();
858   Register ScratchRegW =
859       STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32);
860   Register TableReg = MI.getOperand(2).getReg();
861   Register EntryReg = MI.getOperand(3).getReg();
862   int JTIdx = MI.getOperand(4).getIndex();
863   int Size = AArch64FI->getJumpTableEntrySize(JTIdx);
864 
865   // This has to be first because the compression pass based its reachability
866   // calculations on the start of the JumpTableDest instruction.
867   auto Label =
868       MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx);
869 
870   // If we don't already have a symbol to use as the base, use the ADR
871   // instruction itself.
872   if (!Label) {
873     Label = MF->getContext().createTempSymbol();
874     AArch64FI->setJumpTableEntryInfo(JTIdx, Size, Label);
875     OutStreamer.emitLabel(Label);
876   }
877 
878   auto LabelExpr = MCSymbolRefExpr::create(Label, MF->getContext());
879   EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR)
880                                   .addReg(DestReg)
881                                   .addExpr(LabelExpr));
882 
883   // Load the number of instruction-steps to offset from the label.
884   unsigned LdrOpcode;
885   switch (Size) {
886   case 1: LdrOpcode = AArch64::LDRBBroX; break;
887   case 2: LdrOpcode = AArch64::LDRHHroX; break;
888   case 4: LdrOpcode = AArch64::LDRSWroX; break;
889   default:
890     llvm_unreachable("Unknown jump table size");
891   }
892 
893   EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode)
894                                   .addReg(Size == 4 ? ScratchReg : ScratchRegW)
895                                   .addReg(TableReg)
896                                   .addReg(EntryReg)
897                                   .addImm(0)
898                                   .addImm(Size == 1 ? 0 : 1));
899 
900   // Add to the already materialized base label address, multiplying by 4 if
901   // compressed.
902   EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs)
903                                   .addReg(DestReg)
904                                   .addReg(DestReg)
905                                   .addReg(ScratchReg)
906                                   .addImm(Size == 4 ? 0 : 2));
907 }
908 
909 void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
910                                       const MachineInstr &MI) {
911   unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
912 
913   auto &Ctx = OutStreamer.getContext();
914   MCSymbol *MILabel = Ctx.createTempSymbol();
915   OutStreamer.emitLabel(MILabel);
916 
917   SM.recordStackMap(*MILabel, MI);
918   assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
919 
920   // Scan ahead to trim the shadow.
921   const MachineBasicBlock &MBB = *MI.getParent();
922   MachineBasicBlock::const_iterator MII(MI);
923   ++MII;
924   while (NumNOPBytes > 0) {
925     if (MII == MBB.end() || MII->isCall() ||
926         MII->getOpcode() == AArch64::DBG_VALUE ||
927         MII->getOpcode() == TargetOpcode::PATCHPOINT ||
928         MII->getOpcode() == TargetOpcode::STACKMAP)
929       break;
930     ++MII;
931     NumNOPBytes -= 4;
932   }
933 
934   // Emit nops.
935   for (unsigned i = 0; i < NumNOPBytes; i += 4)
936     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
937 }
938 
939 // Lower a patchpoint of the form:
940 // [<def>], <id>, <numBytes>, <target>, <numArgs>
941 void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
942                                         const MachineInstr &MI) {
943   auto &Ctx = OutStreamer.getContext();
944   MCSymbol *MILabel = Ctx.createTempSymbol();
945   OutStreamer.emitLabel(MILabel);
946   SM.recordPatchPoint(*MILabel, MI);
947 
948   PatchPointOpers Opers(&MI);
949 
950   int64_t CallTarget = Opers.getCallTarget().getImm();
951   unsigned EncodedBytes = 0;
952   if (CallTarget) {
953     assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
954            "High 16 bits of call target should be zero.");
955     Register ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
956     EncodedBytes = 16;
957     // Materialize the jump address:
958     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi)
959                                     .addReg(ScratchReg)
960                                     .addImm((CallTarget >> 32) & 0xFFFF)
961                                     .addImm(32));
962     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
963                                     .addReg(ScratchReg)
964                                     .addReg(ScratchReg)
965                                     .addImm((CallTarget >> 16) & 0xFFFF)
966                                     .addImm(16));
967     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
968                                     .addReg(ScratchReg)
969                                     .addReg(ScratchReg)
970                                     .addImm(CallTarget & 0xFFFF)
971                                     .addImm(0));
972     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
973   }
974   // Emit padding.
975   unsigned NumBytes = Opers.getNumPatchBytes();
976   assert(NumBytes >= EncodedBytes &&
977          "Patchpoint can't request size less than the length of a call.");
978   assert((NumBytes - EncodedBytes) % 4 == 0 &&
979          "Invalid number of NOP bytes requested!");
980   for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
981     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
982 }
983 
984 void AArch64AsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
985                                         const MachineInstr &MI) {
986   StatepointOpers SOpers(&MI);
987   if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
988     assert(PatchBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
989     for (unsigned i = 0; i < PatchBytes; i += 4)
990       EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
991   } else {
992     // Lower call target and choose correct opcode
993     const MachineOperand &CallTarget = SOpers.getCallTarget();
994     MCOperand CallTargetMCOp;
995     unsigned CallOpcode;
996     switch (CallTarget.getType()) {
997     case MachineOperand::MO_GlobalAddress:
998     case MachineOperand::MO_ExternalSymbol:
999       MCInstLowering.lowerOperand(CallTarget, CallTargetMCOp);
1000       CallOpcode = AArch64::BL;
1001       break;
1002     case MachineOperand::MO_Immediate:
1003       CallTargetMCOp = MCOperand::createImm(CallTarget.getImm());
1004       CallOpcode = AArch64::BL;
1005       break;
1006     case MachineOperand::MO_Register:
1007       CallTargetMCOp = MCOperand::createReg(CallTarget.getReg());
1008       CallOpcode = AArch64::BLR;
1009       break;
1010     default:
1011       llvm_unreachable("Unsupported operand type in statepoint call target");
1012       break;
1013     }
1014 
1015     EmitToStreamer(OutStreamer,
1016                    MCInstBuilder(CallOpcode).addOperand(CallTargetMCOp));
1017   }
1018 
1019   auto &Ctx = OutStreamer.getContext();
1020   MCSymbol *MILabel = Ctx.createTempSymbol();
1021   OutStreamer.emitLabel(MILabel);
1022   SM.recordStatepoint(*MILabel, MI);
1023 }
1024 
1025 void AArch64AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI) {
1026   // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
1027   //                  <opcode>, <operands>
1028 
1029   Register DefRegister = FaultingMI.getOperand(0).getReg();
1030   FaultMaps::FaultKind FK =
1031       static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(1).getImm());
1032   MCSymbol *HandlerLabel = FaultingMI.getOperand(2).getMBB()->getSymbol();
1033   unsigned Opcode = FaultingMI.getOperand(3).getImm();
1034   unsigned OperandsBeginIdx = 4;
1035 
1036   auto &Ctx = OutStreamer->getContext();
1037   MCSymbol *FaultingLabel = Ctx.createTempSymbol();
1038   OutStreamer->emitLabel(FaultingLabel);
1039 
1040   assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
1041   FM.recordFaultingOp(FK, FaultingLabel, HandlerLabel);
1042 
1043   MCInst MI;
1044   MI.setOpcode(Opcode);
1045 
1046   if (DefRegister != (Register)0)
1047     MI.addOperand(MCOperand::createReg(DefRegister));
1048 
1049   for (auto I = FaultingMI.operands_begin() + OperandsBeginIdx,
1050             E = FaultingMI.operands_end();
1051        I != E; ++I) {
1052     MCOperand Dest;
1053     lowerOperand(*I, Dest);
1054     MI.addOperand(Dest);
1055   }
1056 
1057   OutStreamer->AddComment("on-fault: " + HandlerLabel->getName());
1058   OutStreamer->emitInstruction(MI, getSubtargetInfo());
1059 }
1060 
1061 void AArch64AsmPrinter::EmitFMov0(const MachineInstr &MI) {
1062   Register DestReg = MI.getOperand(0).getReg();
1063   if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround()) {
1064     // Convert H/S/D register to corresponding Q register
1065     if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31)
1066       DestReg = AArch64::Q0 + (DestReg - AArch64::H0);
1067     else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31)
1068       DestReg = AArch64::Q0 + (DestReg - AArch64::S0);
1069     else {
1070       assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31);
1071       DestReg = AArch64::Q0 + (DestReg - AArch64::D0);
1072     }
1073     MCInst MOVI;
1074     MOVI.setOpcode(AArch64::MOVIv2d_ns);
1075     MOVI.addOperand(MCOperand::createReg(DestReg));
1076     MOVI.addOperand(MCOperand::createImm(0));
1077     EmitToStreamer(*OutStreamer, MOVI);
1078   } else {
1079     MCInst FMov;
1080     switch (MI.getOpcode()) {
1081     default: llvm_unreachable("Unexpected opcode");
1082     case AArch64::FMOVH0:
1083       FMov.setOpcode(AArch64::FMOVWHr);
1084       FMov.addOperand(MCOperand::createReg(DestReg));
1085       FMov.addOperand(MCOperand::createReg(AArch64::WZR));
1086       break;
1087     case AArch64::FMOVS0:
1088       FMov.setOpcode(AArch64::FMOVWSr);
1089       FMov.addOperand(MCOperand::createReg(DestReg));
1090       FMov.addOperand(MCOperand::createReg(AArch64::WZR));
1091       break;
1092     case AArch64::FMOVD0:
1093       FMov.setOpcode(AArch64::FMOVXDr);
1094       FMov.addOperand(MCOperand::createReg(DestReg));
1095       FMov.addOperand(MCOperand::createReg(AArch64::XZR));
1096       break;
1097     }
1098     EmitToStreamer(*OutStreamer, FMov);
1099   }
1100 }
1101 
1102 // Simple pseudo-instructions have their lowering (with expansion to real
1103 // instructions) auto-generated.
1104 #include "AArch64GenMCPseudoLowering.inc"
1105 
1106 void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
1107   // Do any auto-generated pseudo lowerings.
1108   if (emitPseudoExpansionLowering(*OutStreamer, MI))
1109     return;
1110 
1111   if (AArch64FI->getLOHRelated().count(MI)) {
1112     // Generate a label for LOH related instruction
1113     MCSymbol *LOHLabel = createTempSymbol("loh");
1114     // Associate the instruction with the label
1115     LOHInstToLabel[MI] = LOHLabel;
1116     OutStreamer->emitLabel(LOHLabel);
1117   }
1118 
1119   AArch64TargetStreamer *TS =
1120     static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
1121   // Do any manual lowerings.
1122   switch (MI->getOpcode()) {
1123   default:
1124     break;
1125   case AArch64::HINT: {
1126     // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
1127     // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
1128     // non-empty. If MI is the initial BTI, place the
1129     // __patchable_function_entries label after BTI.
1130     if (CurrentPatchableFunctionEntrySym &&
1131         CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
1132         MI == &MF->front().front()) {
1133       int64_t Imm = MI->getOperand(0).getImm();
1134       if ((Imm & 32) && (Imm & 6)) {
1135         MCInst Inst;
1136         MCInstLowering.Lower(MI, Inst);
1137         EmitToStreamer(*OutStreamer, Inst);
1138         CurrentPatchableFunctionEntrySym = createTempSymbol("patch");
1139         OutStreamer->emitLabel(CurrentPatchableFunctionEntrySym);
1140         return;
1141       }
1142     }
1143     break;
1144   }
1145     case AArch64::MOVMCSym: {
1146       Register DestReg = MI->getOperand(0).getReg();
1147       const MachineOperand &MO_Sym = MI->getOperand(1);
1148       MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
1149       MCOperand Hi_MCSym, Lo_MCSym;
1150 
1151       Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
1152       Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
1153 
1154       MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym);
1155       MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym);
1156 
1157       MCInst MovZ;
1158       MovZ.setOpcode(AArch64::MOVZXi);
1159       MovZ.addOperand(MCOperand::createReg(DestReg));
1160       MovZ.addOperand(Hi_MCSym);
1161       MovZ.addOperand(MCOperand::createImm(16));
1162       EmitToStreamer(*OutStreamer, MovZ);
1163 
1164       MCInst MovK;
1165       MovK.setOpcode(AArch64::MOVKXi);
1166       MovK.addOperand(MCOperand::createReg(DestReg));
1167       MovK.addOperand(MCOperand::createReg(DestReg));
1168       MovK.addOperand(Lo_MCSym);
1169       MovK.addOperand(MCOperand::createImm(0));
1170       EmitToStreamer(*OutStreamer, MovK);
1171       return;
1172   }
1173   case AArch64::MOVIv2d_ns:
1174     // If the target has <rdar://problem/16473581>, lower this
1175     // instruction to movi.16b instead.
1176     if (STI->hasZeroCycleZeroingFPWorkaround() &&
1177         MI->getOperand(1).getImm() == 0) {
1178       MCInst TmpInst;
1179       TmpInst.setOpcode(AArch64::MOVIv16b_ns);
1180       TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1181       TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm()));
1182       EmitToStreamer(*OutStreamer, TmpInst);
1183       return;
1184     }
1185     break;
1186 
1187   case AArch64::DBG_VALUE: {
1188     if (isVerbose() && OutStreamer->hasRawTextSupport()) {
1189       SmallString<128> TmpStr;
1190       raw_svector_ostream OS(TmpStr);
1191       PrintDebugValueComment(MI, OS);
1192       OutStreamer->emitRawText(StringRef(OS.str()));
1193     }
1194     return;
1195 
1196   case AArch64::EMITBKEY: {
1197       ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
1198       if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
1199           ExceptionHandlingType != ExceptionHandling::ARM)
1200         return;
1201 
1202       if (needsCFIMoves() == CFI_M_None)
1203         return;
1204 
1205       OutStreamer->emitCFIBKeyFrame();
1206       return;
1207     }
1208   }
1209 
1210   // Tail calls use pseudo instructions so they have the proper code-gen
1211   // attributes (isCall, isReturn, etc.). We lower them to the real
1212   // instruction here.
1213   case AArch64::TCRETURNri:
1214   case AArch64::TCRETURNriBTI:
1215   case AArch64::TCRETURNriALL: {
1216     MCInst TmpInst;
1217     TmpInst.setOpcode(AArch64::BR);
1218     TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1219     EmitToStreamer(*OutStreamer, TmpInst);
1220     return;
1221   }
1222   case AArch64::TCRETURNdi: {
1223     MCOperand Dest;
1224     MCInstLowering.lowerOperand(MI->getOperand(0), Dest);
1225     MCInst TmpInst;
1226     TmpInst.setOpcode(AArch64::B);
1227     TmpInst.addOperand(Dest);
1228     EmitToStreamer(*OutStreamer, TmpInst);
1229     return;
1230   }
1231   case AArch64::SpeculationBarrierISBDSBEndBB: {
1232     // Print DSB SYS + ISB
1233     MCInst TmpInstDSB;
1234     TmpInstDSB.setOpcode(AArch64::DSB);
1235     TmpInstDSB.addOperand(MCOperand::createImm(0xf));
1236     EmitToStreamer(*OutStreamer, TmpInstDSB);
1237     MCInst TmpInstISB;
1238     TmpInstISB.setOpcode(AArch64::ISB);
1239     TmpInstISB.addOperand(MCOperand::createImm(0xf));
1240     EmitToStreamer(*OutStreamer, TmpInstISB);
1241     return;
1242   }
1243   case AArch64::SpeculationBarrierSBEndBB: {
1244     // Print SB
1245     MCInst TmpInstSB;
1246     TmpInstSB.setOpcode(AArch64::SB);
1247     EmitToStreamer(*OutStreamer, TmpInstSB);
1248     return;
1249   }
1250   case AArch64::TLSDESC_CALLSEQ: {
1251     /// lower this to:
1252     ///    adrp  x0, :tlsdesc:var
1253     ///    ldr   x1, [x0, #:tlsdesc_lo12:var]
1254     ///    add   x0, x0, #:tlsdesc_lo12:var
1255     ///    .tlsdesccall var
1256     ///    blr   x1
1257     ///    (TPIDR_EL0 offset now in x0)
1258     const MachineOperand &MO_Sym = MI->getOperand(0);
1259     MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
1260     MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
1261     MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
1262     MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
1263     MCInstLowering.lowerOperand(MO_Sym, Sym);
1264     MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12);
1265     MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc);
1266 
1267     MCInst Adrp;
1268     Adrp.setOpcode(AArch64::ADRP);
1269     Adrp.addOperand(MCOperand::createReg(AArch64::X0));
1270     Adrp.addOperand(SymTLSDesc);
1271     EmitToStreamer(*OutStreamer, Adrp);
1272 
1273     MCInst Ldr;
1274     Ldr.setOpcode(AArch64::LDRXui);
1275     Ldr.addOperand(MCOperand::createReg(AArch64::X1));
1276     Ldr.addOperand(MCOperand::createReg(AArch64::X0));
1277     Ldr.addOperand(SymTLSDescLo12);
1278     Ldr.addOperand(MCOperand::createImm(0));
1279     EmitToStreamer(*OutStreamer, Ldr);
1280 
1281     MCInst Add;
1282     Add.setOpcode(AArch64::ADDXri);
1283     Add.addOperand(MCOperand::createReg(AArch64::X0));
1284     Add.addOperand(MCOperand::createReg(AArch64::X0));
1285     Add.addOperand(SymTLSDescLo12);
1286     Add.addOperand(MCOperand::createImm(AArch64_AM::getShiftValue(0)));
1287     EmitToStreamer(*OutStreamer, Add);
1288 
1289     // Emit a relocation-annotation. This expands to no code, but requests
1290     // the following instruction gets an R_AARCH64_TLSDESC_CALL.
1291     MCInst TLSDescCall;
1292     TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
1293     TLSDescCall.addOperand(Sym);
1294     EmitToStreamer(*OutStreamer, TLSDescCall);
1295 
1296     MCInst Blr;
1297     Blr.setOpcode(AArch64::BLR);
1298     Blr.addOperand(MCOperand::createReg(AArch64::X1));
1299     EmitToStreamer(*OutStreamer, Blr);
1300 
1301     return;
1302   }
1303 
1304   case AArch64::JumpTableDest32:
1305   case AArch64::JumpTableDest16:
1306   case AArch64::JumpTableDest8:
1307     LowerJumpTableDest(*OutStreamer, *MI);
1308     return;
1309 
1310   case AArch64::FMOVH0:
1311   case AArch64::FMOVS0:
1312   case AArch64::FMOVD0:
1313     EmitFMov0(*MI);
1314     return;
1315 
1316   case TargetOpcode::STACKMAP:
1317     return LowerSTACKMAP(*OutStreamer, SM, *MI);
1318 
1319   case TargetOpcode::PATCHPOINT:
1320     return LowerPATCHPOINT(*OutStreamer, SM, *MI);
1321 
1322   case TargetOpcode::STATEPOINT:
1323     return LowerSTATEPOINT(*OutStreamer, SM, *MI);
1324 
1325   case TargetOpcode::FAULTING_OP:
1326     return LowerFAULTING_OP(*MI);
1327 
1328   case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
1329     LowerPATCHABLE_FUNCTION_ENTER(*MI);
1330     return;
1331 
1332   case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
1333     LowerPATCHABLE_FUNCTION_EXIT(*MI);
1334     return;
1335 
1336   case TargetOpcode::PATCHABLE_TAIL_CALL:
1337     LowerPATCHABLE_TAIL_CALL(*MI);
1338     return;
1339 
1340   case AArch64::HWASAN_CHECK_MEMACCESS:
1341   case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
1342     LowerHWASAN_CHECK_MEMACCESS(*MI);
1343     return;
1344 
1345   case AArch64::SEH_StackAlloc:
1346     TS->EmitARM64WinCFIAllocStack(MI->getOperand(0).getImm());
1347     return;
1348 
1349   case AArch64::SEH_SaveFPLR:
1350     TS->EmitARM64WinCFISaveFPLR(MI->getOperand(0).getImm());
1351     return;
1352 
1353   case AArch64::SEH_SaveFPLR_X:
1354     assert(MI->getOperand(0).getImm() < 0 &&
1355            "Pre increment SEH opcode must have a negative offset");
1356     TS->EmitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm());
1357     return;
1358 
1359   case AArch64::SEH_SaveReg:
1360     TS->EmitARM64WinCFISaveReg(MI->getOperand(0).getImm(),
1361                                MI->getOperand(1).getImm());
1362     return;
1363 
1364   case AArch64::SEH_SaveReg_X:
1365     assert(MI->getOperand(1).getImm() < 0 &&
1366            "Pre increment SEH opcode must have a negative offset");
1367     TS->EmitARM64WinCFISaveRegX(MI->getOperand(0).getImm(),
1368 		                -MI->getOperand(1).getImm());
1369     return;
1370 
1371   case AArch64::SEH_SaveRegP:
1372     if (MI->getOperand(1).getImm() == 30 && MI->getOperand(0).getImm() >= 19 &&
1373         MI->getOperand(0).getImm() <= 28) {
1374       assert((MI->getOperand(0).getImm() - 19) % 2 == 0 &&
1375              "Register paired with LR must be odd");
1376       TS->EmitARM64WinCFISaveLRPair(MI->getOperand(0).getImm(),
1377                                     MI->getOperand(2).getImm());
1378       return;
1379     }
1380     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1381             "Non-consecutive registers not allowed for save_regp");
1382     TS->EmitARM64WinCFISaveRegP(MI->getOperand(0).getImm(),
1383                                 MI->getOperand(2).getImm());
1384     return;
1385 
1386   case AArch64::SEH_SaveRegP_X:
1387     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1388             "Non-consecutive registers not allowed for save_regp_x");
1389     assert(MI->getOperand(2).getImm() < 0 &&
1390            "Pre increment SEH opcode must have a negative offset");
1391     TS->EmitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(),
1392                                  -MI->getOperand(2).getImm());
1393     return;
1394 
1395   case AArch64::SEH_SaveFReg:
1396     TS->EmitARM64WinCFISaveFReg(MI->getOperand(0).getImm(),
1397                                 MI->getOperand(1).getImm());
1398     return;
1399 
1400   case AArch64::SEH_SaveFReg_X:
1401     assert(MI->getOperand(1).getImm() < 0 &&
1402            "Pre increment SEH opcode must have a negative offset");
1403     TS->EmitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(),
1404                                  -MI->getOperand(1).getImm());
1405     return;
1406 
1407   case AArch64::SEH_SaveFRegP:
1408     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1409             "Non-consecutive registers not allowed for save_regp");
1410     TS->EmitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(),
1411                                  MI->getOperand(2).getImm());
1412     return;
1413 
1414   case AArch64::SEH_SaveFRegP_X:
1415     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1416             "Non-consecutive registers not allowed for save_regp_x");
1417     assert(MI->getOperand(2).getImm() < 0 &&
1418            "Pre increment SEH opcode must have a negative offset");
1419     TS->EmitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(),
1420                                   -MI->getOperand(2).getImm());
1421     return;
1422 
1423   case AArch64::SEH_SetFP:
1424     TS->EmitARM64WinCFISetFP();
1425     return;
1426 
1427   case AArch64::SEH_AddFP:
1428     TS->EmitARM64WinCFIAddFP(MI->getOperand(0).getImm());
1429     return;
1430 
1431   case AArch64::SEH_Nop:
1432     TS->EmitARM64WinCFINop();
1433     return;
1434 
1435   case AArch64::SEH_PrologEnd:
1436     TS->EmitARM64WinCFIPrologEnd();
1437     return;
1438 
1439   case AArch64::SEH_EpilogStart:
1440     TS->EmitARM64WinCFIEpilogStart();
1441     return;
1442 
1443   case AArch64::SEH_EpilogEnd:
1444     TS->EmitARM64WinCFIEpilogEnd();
1445     return;
1446   }
1447 
1448   // Finally, do the automated lowerings for everything else.
1449   MCInst TmpInst;
1450   MCInstLowering.Lower(MI, TmpInst);
1451   EmitToStreamer(*OutStreamer, TmpInst);
1452 }
1453 
1454 // Force static initialization.
1455 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter() {
1456   RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
1457   RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget());
1458   RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target());
1459   RegisterAsmPrinter<AArch64AsmPrinter> W(getTheARM64_32Target());
1460   RegisterAsmPrinter<AArch64AsmPrinter> V(getTheAArch64_32Target());
1461 }
1462