1 //===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains a printer that converts from our internal representation
10 // of machine-dependent LLVM code to the AArch64 assembly language.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "AArch64.h"
15 #include "AArch64MCInstLower.h"
16 #include "AArch64MachineFunctionInfo.h"
17 #include "AArch64RegisterInfo.h"
18 #include "AArch64Subtarget.h"
19 #include "AArch64TargetObjectFile.h"
20 #include "MCTargetDesc/AArch64AddressingModes.h"
21 #include "MCTargetDesc/AArch64InstPrinter.h"
22 #include "MCTargetDesc/AArch64MCExpr.h"
23 #include "MCTargetDesc/AArch64MCTargetDesc.h"
24 #include "MCTargetDesc/AArch64TargetStreamer.h"
25 #include "TargetInfo/AArch64TargetInfo.h"
26 #include "Utils/AArch64BaseInfo.h"
27 #include "llvm/ADT/SmallString.h"
28 #include "llvm/ADT/SmallVector.h"
29 #include "llvm/ADT/StringRef.h"
30 #include "llvm/ADT/Triple.h"
31 #include "llvm/ADT/Twine.h"
32 #include "llvm/BinaryFormat/COFF.h"
33 #include "llvm/BinaryFormat/ELF.h"
34 #include "llvm/CodeGen/AsmPrinter.h"
35 #include "llvm/CodeGen/FaultMaps.h"
36 #include "llvm/CodeGen/MachineBasicBlock.h"
37 #include "llvm/CodeGen/MachineFunction.h"
38 #include "llvm/CodeGen/MachineInstr.h"
39 #include "llvm/CodeGen/MachineJumpTableInfo.h"
40 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
41 #include "llvm/CodeGen/MachineOperand.h"
42 #include "llvm/CodeGen/StackMaps.h"
43 #include "llvm/CodeGen/TargetRegisterInfo.h"
44 #include "llvm/IR/DataLayout.h"
45 #include "llvm/IR/DebugInfoMetadata.h"
46 #include "llvm/MC/MCAsmInfo.h"
47 #include "llvm/MC/MCContext.h"
48 #include "llvm/MC/MCInst.h"
49 #include "llvm/MC/MCInstBuilder.h"
50 #include "llvm/MC/MCSectionELF.h"
51 #include "llvm/MC/MCStreamer.h"
52 #include "llvm/MC/MCSymbol.h"
53 #include "llvm/MC/TargetRegistry.h"
54 #include "llvm/Support/Casting.h"
55 #include "llvm/Support/ErrorHandling.h"
56 #include "llvm/Support/raw_ostream.h"
57 #include "llvm/Target/TargetMachine.h"
58 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
59 #include <algorithm>
60 #include <cassert>
61 #include <cstdint>
62 #include <map>
63 #include <memory>
64 
65 using namespace llvm;
66 
67 #define DEBUG_TYPE "asm-printer"
68 
69 namespace {
70 
71 class AArch64AsmPrinter : public AsmPrinter {
72   AArch64MCInstLower MCInstLowering;
73   StackMaps SM;
74   FaultMaps FM;
75   const AArch64Subtarget *STI;
76   bool ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = false;
77 
78 public:
79   AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
80       : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this),
81         SM(*this), FM(*this) {}
82 
83   StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
84 
85   /// Wrapper for MCInstLowering.lowerOperand() for the
86   /// tblgen'erated pseudo lowering.
87   bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
88     return MCInstLowering.lowerOperand(MO, MCOp);
89   }
90 
91   void emitStartOfAsmFile(Module &M) override;
92   void emitJumpTableInfo() override;
93 
94   void emitFunctionEntryLabel() override;
95 
96   void LowerJumpTableDest(MCStreamer &OutStreamer, const MachineInstr &MI);
97 
98   void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
99                      const MachineInstr &MI);
100   void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
101                        const MachineInstr &MI);
102   void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
103                        const MachineInstr &MI);
104   void LowerFAULTING_OP(const MachineInstr &MI);
105 
106   void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
107   void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
108   void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
109 
110   typedef std::tuple<unsigned, bool, uint32_t> HwasanMemaccessTuple;
111   std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
112   void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
113   void emitHwasanMemaccessSymbols(Module &M);
114 
115   void emitSled(const MachineInstr &MI, SledKind Kind);
116 
117   /// tblgen'erated driver function for lowering simple MI->MC
118   /// pseudo instructions.
119   bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
120                                    const MachineInstr *MI);
121 
122   void emitInstruction(const MachineInstr *MI) override;
123 
124   void emitFunctionHeaderComment() override;
125 
126   void getAnalysisUsage(AnalysisUsage &AU) const override {
127     AsmPrinter::getAnalysisUsage(AU);
128     AU.setPreservesAll();
129   }
130 
131   bool runOnMachineFunction(MachineFunction &MF) override {
132     AArch64FI = MF.getInfo<AArch64FunctionInfo>();
133     STI = static_cast<const AArch64Subtarget*>(&MF.getSubtarget());
134 
135     SetupMachineFunction(MF);
136 
137     if (STI->isTargetCOFF()) {
138       bool Internal = MF.getFunction().hasInternalLinkage();
139       COFF::SymbolStorageClass Scl = Internal ? COFF::IMAGE_SYM_CLASS_STATIC
140                                               : COFF::IMAGE_SYM_CLASS_EXTERNAL;
141       int Type =
142         COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT;
143 
144       OutStreamer->BeginCOFFSymbolDef(CurrentFnSym);
145       OutStreamer->EmitCOFFSymbolStorageClass(Scl);
146       OutStreamer->EmitCOFFSymbolType(Type);
147       OutStreamer->EndCOFFSymbolDef();
148     }
149 
150     // Emit the rest of the function body.
151     emitFunctionBody();
152 
153     // Emit the XRay table for this function.
154     emitXRayTable();
155 
156     // We didn't modify anything.
157     return false;
158   }
159 
160 private:
161   void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
162   bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
163   bool printAsmRegInClass(const MachineOperand &MO,
164                           const TargetRegisterClass *RC, unsigned AltName,
165                           raw_ostream &O);
166 
167   bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
168                        const char *ExtraCode, raw_ostream &O) override;
169   bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
170                              const char *ExtraCode, raw_ostream &O) override;
171 
172   void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
173 
174   void emitFunctionBodyEnd() override;
175 
176   MCSymbol *GetCPISymbol(unsigned CPID) const override;
177   void emitEndOfAsmFile(Module &M) override;
178 
179   AArch64FunctionInfo *AArch64FI = nullptr;
180 
181   /// Emit the LOHs contained in AArch64FI.
182   void emitLOHs();
183 
184   /// Emit instruction to set float register to zero.
185   void emitFMov0(const MachineInstr &MI);
186 
187   using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
188 
189   MInstToMCSymbol LOHInstToLabel;
190 
191   bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const override {
192     return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags;
193   }
194 };
195 
196 } // end anonymous namespace
197 
198 void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
199   const Triple &TT = TM.getTargetTriple();
200 
201   if (TT.isOSBinFormatCOFF()) {
202     // Emit an absolute @feat.00 symbol.  This appears to be some kind of
203     // compiler features bitfield read by link.exe.
204     MCSymbol *S = MMI->getContext().getOrCreateSymbol(StringRef("@feat.00"));
205     OutStreamer->BeginCOFFSymbolDef(S);
206     OutStreamer->EmitCOFFSymbolStorageClass(COFF::IMAGE_SYM_CLASS_STATIC);
207     OutStreamer->EmitCOFFSymbolType(COFF::IMAGE_SYM_DTYPE_NULL);
208     OutStreamer->EndCOFFSymbolDef();
209     int64_t Feat00Flags = 0;
210 
211     if (M.getModuleFlag("cfguard")) {
212       Feat00Flags |= 0x800; // Object is CFG-aware.
213     }
214 
215     if (M.getModuleFlag("ehcontguard")) {
216       Feat00Flags |= 0x4000; // Object also has EHCont.
217     }
218 
219     OutStreamer->emitSymbolAttribute(S, MCSA_Global);
220     OutStreamer->emitAssignment(
221         S, MCConstantExpr::create(Feat00Flags, MMI->getContext()));
222   }
223 
224   if (!TT.isOSBinFormatELF())
225     return;
226 
227   // Assemble feature flags that may require creation of a note section.
228   unsigned Flags = 0;
229   if (const auto *BTE = mdconst::extract_or_null<ConstantInt>(
230           M.getModuleFlag("branch-target-enforcement")))
231     if (BTE->getZExtValue())
232       Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI;
233 
234   if (const auto *Sign = mdconst::extract_or_null<ConstantInt>(
235           M.getModuleFlag("sign-return-address")))
236     if (Sign->getZExtValue())
237       Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
238 
239   if (Flags == 0)
240     return;
241 
242   // Emit a .note.gnu.property section with the flags.
243   if (auto *TS = static_cast<AArch64TargetStreamer *>(
244           OutStreamer->getTargetStreamer()))
245     TS->emitNoteSection(Flags);
246 }
247 
248 void AArch64AsmPrinter::emitFunctionHeaderComment() {
249   const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>();
250   Optional<std::string> OutlinerString = FI->getOutliningStyle();
251   if (OutlinerString != None)
252     OutStreamer->GetCommentOS() << ' ' << OutlinerString;
253 }
254 
255 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
256 {
257   const Function &F = MF->getFunction();
258   if (F.hasFnAttribute("patchable-function-entry")) {
259     unsigned Num;
260     if (F.getFnAttribute("patchable-function-entry")
261             .getValueAsString()
262             .getAsInteger(10, Num))
263       return;
264     emitNops(Num);
265     return;
266   }
267 
268   emitSled(MI, SledKind::FUNCTION_ENTER);
269 }
270 
271 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI) {
272   emitSled(MI, SledKind::FUNCTION_EXIT);
273 }
274 
275 void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI) {
276   emitSled(MI, SledKind::TAIL_CALL);
277 }
278 
279 void AArch64AsmPrinter::emitSled(const MachineInstr &MI, SledKind Kind) {
280   static const int8_t NoopsInSledCount = 7;
281   // We want to emit the following pattern:
282   //
283   // .Lxray_sled_N:
284   //   ALIGN
285   //   B #32
286   //   ; 7 NOP instructions (28 bytes)
287   // .tmpN
288   //
289   // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
290   // over the full 32 bytes (8 instructions) with the following pattern:
291   //
292   //   STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
293   //   LDR W0, #12 ; W0 := function ID
294   //   LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
295   //   BLR X16 ; call the tracing trampoline
296   //   ;DATA: 32 bits of function ID
297   //   ;DATA: lower 32 bits of the address of the trampoline
298   //   ;DATA: higher 32 bits of the address of the trampoline
299   //   LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
300   //
301   OutStreamer->emitCodeAlignment(4, &getSubtargetInfo());
302   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
303   OutStreamer->emitLabel(CurSled);
304   auto Target = OutContext.createTempSymbol();
305 
306   // Emit "B #32" instruction, which jumps over the next 28 bytes.
307   // The operand has to be the number of 4-byte instructions to jump over,
308   // including the current instruction.
309   EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8));
310 
311   for (int8_t I = 0; I < NoopsInSledCount; I++)
312     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
313 
314   OutStreamer->emitLabel(Target);
315   recordSled(CurSled, MI, Kind, 2);
316 }
317 
318 void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
319   Register Reg = MI.getOperand(0).getReg();
320   bool IsShort =
321       MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES;
322   uint32_t AccessInfo = MI.getOperand(1).getImm();
323   MCSymbol *&Sym =
324       HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, IsShort, AccessInfo)];
325   if (!Sym) {
326     // FIXME: Make this work on non-ELF.
327     if (!TM.getTargetTriple().isOSBinFormatELF())
328       report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF");
329 
330     std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" +
331                           utostr(AccessInfo);
332     if (IsShort)
333       SymName += "_short_v2";
334     Sym = OutContext.getOrCreateSymbol(SymName);
335   }
336 
337   EmitToStreamer(*OutStreamer,
338                  MCInstBuilder(AArch64::BL)
339                      .addExpr(MCSymbolRefExpr::create(Sym, OutContext)));
340 }
341 
342 void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) {
343   if (HwasanMemaccessSymbols.empty())
344     return;
345 
346   const Triple &TT = TM.getTargetTriple();
347   assert(TT.isOSBinFormatELF());
348   std::unique_ptr<MCSubtargetInfo> STI(
349       TM.getTarget().createMCSubtargetInfo(TT.str(), "", ""));
350   assert(STI && "Unable to create subtarget info");
351 
352   MCSymbol *HwasanTagMismatchV1Sym =
353       OutContext.getOrCreateSymbol("__hwasan_tag_mismatch");
354   MCSymbol *HwasanTagMismatchV2Sym =
355       OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2");
356 
357   const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
358       MCSymbolRefExpr::create(HwasanTagMismatchV1Sym, OutContext);
359   const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
360       MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext);
361 
362   for (auto &P : HwasanMemaccessSymbols) {
363     unsigned Reg = std::get<0>(P.first);
364     bool IsShort = std::get<1>(P.first);
365     uint32_t AccessInfo = std::get<2>(P.first);
366     const MCSymbolRefExpr *HwasanTagMismatchRef =
367         IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
368     MCSymbol *Sym = P.second;
369 
370     bool HasMatchAllTag =
371         (AccessInfo >> HWASanAccessInfo::HasMatchAllShift) & 1;
372     uint8_t MatchAllTag =
373         (AccessInfo >> HWASanAccessInfo::MatchAllShift) & 0xff;
374     unsigned Size =
375         1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf);
376     bool CompileKernel =
377         (AccessInfo >> HWASanAccessInfo::CompileKernelShift) & 1;
378 
379     OutStreamer->SwitchSection(OutContext.getELFSection(
380         ".text.hot", ELF::SHT_PROGBITS,
381         ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0,
382         Sym->getName(), /*IsComdat=*/true));
383 
384     OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
385     OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
386     OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden);
387     OutStreamer->emitLabel(Sym);
388 
389     OutStreamer->emitInstruction(MCInstBuilder(AArch64::SBFMXri)
390                                      .addReg(AArch64::X16)
391                                      .addReg(Reg)
392                                      .addImm(4)
393                                      .addImm(55),
394                                  *STI);
395     OutStreamer->emitInstruction(
396         MCInstBuilder(AArch64::LDRBBroX)
397             .addReg(AArch64::W16)
398             .addReg(IsShort ? AArch64::X20 : AArch64::X9)
399             .addReg(AArch64::X16)
400             .addImm(0)
401             .addImm(0),
402         *STI);
403     OutStreamer->emitInstruction(
404         MCInstBuilder(AArch64::SUBSXrs)
405             .addReg(AArch64::XZR)
406             .addReg(AArch64::X16)
407             .addReg(Reg)
408             .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
409         *STI);
410     MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
411     OutStreamer->emitInstruction(
412         MCInstBuilder(AArch64::Bcc)
413             .addImm(AArch64CC::NE)
414             .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym,
415                                              OutContext)),
416         *STI);
417     MCSymbol *ReturnSym = OutContext.createTempSymbol();
418     OutStreamer->emitLabel(ReturnSym);
419     OutStreamer->emitInstruction(
420         MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI);
421     OutStreamer->emitLabel(HandleMismatchOrPartialSym);
422 
423     if (HasMatchAllTag) {
424       OutStreamer->emitInstruction(MCInstBuilder(AArch64::UBFMXri)
425                                        .addReg(AArch64::X16)
426                                        .addReg(Reg)
427                                        .addImm(56)
428                                        .addImm(63),
429                                    *STI);
430       OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSXri)
431                                        .addReg(AArch64::XZR)
432                                        .addReg(AArch64::X16)
433                                        .addImm(MatchAllTag)
434                                        .addImm(0),
435                                    *STI);
436       OutStreamer->emitInstruction(
437           MCInstBuilder(AArch64::Bcc)
438               .addImm(AArch64CC::EQ)
439               .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
440           *STI);
441     }
442 
443     if (IsShort) {
444       OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWri)
445                                        .addReg(AArch64::WZR)
446                                        .addReg(AArch64::W16)
447                                        .addImm(15)
448                                        .addImm(0),
449                                    *STI);
450       MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
451       OutStreamer->emitInstruction(
452           MCInstBuilder(AArch64::Bcc)
453               .addImm(AArch64CC::HI)
454               .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
455           *STI);
456 
457       OutStreamer->emitInstruction(
458           MCInstBuilder(AArch64::ANDXri)
459               .addReg(AArch64::X17)
460               .addReg(Reg)
461               .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
462           *STI);
463       if (Size != 1)
464         OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
465                                          .addReg(AArch64::X17)
466                                          .addReg(AArch64::X17)
467                                          .addImm(Size - 1)
468                                          .addImm(0),
469                                      *STI);
470       OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWrs)
471                                        .addReg(AArch64::WZR)
472                                        .addReg(AArch64::W16)
473                                        .addReg(AArch64::W17)
474                                        .addImm(0),
475                                    *STI);
476       OutStreamer->emitInstruction(
477           MCInstBuilder(AArch64::Bcc)
478               .addImm(AArch64CC::LS)
479               .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
480           *STI);
481 
482       OutStreamer->emitInstruction(
483           MCInstBuilder(AArch64::ORRXri)
484               .addReg(AArch64::X16)
485               .addReg(Reg)
486               .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
487           *STI);
488       OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRBBui)
489                                        .addReg(AArch64::W16)
490                                        .addReg(AArch64::X16)
491                                        .addImm(0),
492                                    *STI);
493       OutStreamer->emitInstruction(
494           MCInstBuilder(AArch64::SUBSXrs)
495               .addReg(AArch64::XZR)
496               .addReg(AArch64::X16)
497               .addReg(Reg)
498               .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
499           *STI);
500       OutStreamer->emitInstruction(
501           MCInstBuilder(AArch64::Bcc)
502               .addImm(AArch64CC::EQ)
503               .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
504           *STI);
505 
506       OutStreamer->emitLabel(HandleMismatchSym);
507     }
508 
509     OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre)
510                                      .addReg(AArch64::SP)
511                                      .addReg(AArch64::X0)
512                                      .addReg(AArch64::X1)
513                                      .addReg(AArch64::SP)
514                                      .addImm(-32),
515                                  *STI);
516     OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi)
517                                      .addReg(AArch64::FP)
518                                      .addReg(AArch64::LR)
519                                      .addReg(AArch64::SP)
520                                      .addImm(29),
521                                  *STI);
522 
523     if (Reg != AArch64::X0)
524       OutStreamer->emitInstruction(MCInstBuilder(AArch64::ORRXrs)
525                                        .addReg(AArch64::X0)
526                                        .addReg(AArch64::XZR)
527                                        .addReg(Reg)
528                                        .addImm(0),
529                                    *STI);
530     OutStreamer->emitInstruction(
531         MCInstBuilder(AArch64::MOVZXi)
532             .addReg(AArch64::X1)
533             .addImm(AccessInfo & HWASanAccessInfo::RuntimeMask)
534             .addImm(0),
535         *STI);
536 
537     if (CompileKernel) {
538       // The Linux kernel's dynamic loader doesn't support GOT relative
539       // relocations, but it doesn't support late binding either, so just call
540       // the function directly.
541       OutStreamer->emitInstruction(
542           MCInstBuilder(AArch64::B).addExpr(HwasanTagMismatchRef), *STI);
543     } else {
544       // Intentionally load the GOT entry and branch to it, rather than possibly
545       // late binding the function, which may clobber the registers before we
546       // have a chance to save them.
547       OutStreamer->emitInstruction(
548           MCInstBuilder(AArch64::ADRP)
549               .addReg(AArch64::X16)
550               .addExpr(AArch64MCExpr::create(
551                   HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_PAGE,
552                   OutContext)),
553           *STI);
554       OutStreamer->emitInstruction(
555           MCInstBuilder(AArch64::LDRXui)
556               .addReg(AArch64::X16)
557               .addReg(AArch64::X16)
558               .addExpr(AArch64MCExpr::create(
559                   HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_LO12,
560                   OutContext)),
561           *STI);
562       OutStreamer->emitInstruction(
563           MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI);
564     }
565   }
566 }
567 
568 void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
569   emitHwasanMemaccessSymbols(M);
570 
571   const Triple &TT = TM.getTargetTriple();
572   if (TT.isOSBinFormatMachO()) {
573     // Funny Darwin hack: This flag tells the linker that no global symbols
574     // contain code that falls through to other global symbols (e.g. the obvious
575     // implementation of multiple entry points).  If this doesn't occur, the
576     // linker can safely perform dead code stripping.  Since LLVM never
577     // generates code that does this, it is always safe to set.
578     OutStreamer->emitAssemblerFlag(MCAF_SubsectionsViaSymbols);
579   }
580 
581   // Emit stack and fault map information.
582   emitStackMaps(SM);
583   FM.serializeToFaultMapSection();
584 
585 }
586 
587 void AArch64AsmPrinter::emitLOHs() {
588   SmallVector<MCSymbol *, 3> MCArgs;
589 
590   for (const auto &D : AArch64FI->getLOHContainer()) {
591     for (const MachineInstr *MI : D.getArgs()) {
592       MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI);
593       assert(LabelIt != LOHInstToLabel.end() &&
594              "Label hasn't been inserted for LOH related instruction");
595       MCArgs.push_back(LabelIt->second);
596     }
597     OutStreamer->emitLOHDirective(D.getKind(), MCArgs);
598     MCArgs.clear();
599   }
600 }
601 
602 void AArch64AsmPrinter::emitFunctionBodyEnd() {
603   if (!AArch64FI->getLOHRelated().empty())
604     emitLOHs();
605 }
606 
607 /// GetCPISymbol - Return the symbol for the specified constant pool entry.
608 MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
609   // Darwin uses a linker-private symbol name for constant-pools (to
610   // avoid addends on the relocation?), ELF has no such concept and
611   // uses a normal private symbol.
612   if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
613     return OutContext.getOrCreateSymbol(
614         Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
615         Twine(getFunctionNumber()) + "_" + Twine(CPID));
616 
617   return AsmPrinter::GetCPISymbol(CPID);
618 }
619 
620 void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
621                                      raw_ostream &O) {
622   const MachineOperand &MO = MI->getOperand(OpNum);
623   switch (MO.getType()) {
624   default:
625     llvm_unreachable("<unknown operand type>");
626   case MachineOperand::MO_Register: {
627     Register Reg = MO.getReg();
628     assert(Register::isPhysicalRegister(Reg));
629     assert(!MO.getSubReg() && "Subregs should be eliminated!");
630     O << AArch64InstPrinter::getRegisterName(Reg);
631     break;
632   }
633   case MachineOperand::MO_Immediate: {
634     O << MO.getImm();
635     break;
636   }
637   case MachineOperand::MO_GlobalAddress: {
638     PrintSymbolOperand(MO, O);
639     break;
640   }
641   case MachineOperand::MO_BlockAddress: {
642     MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
643     Sym->print(O, MAI);
644     break;
645   }
646   }
647 }
648 
649 bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
650                                           raw_ostream &O) {
651   Register Reg = MO.getReg();
652   switch (Mode) {
653   default:
654     return true; // Unknown mode.
655   case 'w':
656     Reg = getWRegFromXReg(Reg);
657     break;
658   case 'x':
659     Reg = getXRegFromWReg(Reg);
660     break;
661   case 't':
662     Reg = getXRegFromXRegTuple(Reg);
663     break;
664   }
665 
666   O << AArch64InstPrinter::getRegisterName(Reg);
667   return false;
668 }
669 
670 // Prints the register in MO using class RC using the offset in the
671 // new register class. This should not be used for cross class
672 // printing.
673 bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
674                                            const TargetRegisterClass *RC,
675                                            unsigned AltName, raw_ostream &O) {
676   assert(MO.isReg() && "Should only get here with a register!");
677   const TargetRegisterInfo *RI = STI->getRegisterInfo();
678   Register Reg = MO.getReg();
679   unsigned RegToPrint = RC->getRegister(RI->getEncodingValue(Reg));
680   if (!RI->regsOverlap(RegToPrint, Reg))
681     return true;
682   O << AArch64InstPrinter::getRegisterName(RegToPrint, AltName);
683   return false;
684 }
685 
686 bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
687                                         const char *ExtraCode, raw_ostream &O) {
688   const MachineOperand &MO = MI->getOperand(OpNum);
689 
690   // First try the generic code, which knows about modifiers like 'c' and 'n'.
691   if (!AsmPrinter::PrintAsmOperand(MI, OpNum, ExtraCode, O))
692     return false;
693 
694   // Does this asm operand have a single letter operand modifier?
695   if (ExtraCode && ExtraCode[0]) {
696     if (ExtraCode[1] != 0)
697       return true; // Unknown modifier.
698 
699     switch (ExtraCode[0]) {
700     default:
701       return true; // Unknown modifier.
702     case 'w':      // Print W register
703     case 'x':      // Print X register
704       if (MO.isReg())
705         return printAsmMRegister(MO, ExtraCode[0], O);
706       if (MO.isImm() && MO.getImm() == 0) {
707         unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
708         O << AArch64InstPrinter::getRegisterName(Reg);
709         return false;
710       }
711       printOperand(MI, OpNum, O);
712       return false;
713     case 'b': // Print B register.
714     case 'h': // Print H register.
715     case 's': // Print S register.
716     case 'd': // Print D register.
717     case 'q': // Print Q register.
718     case 'z': // Print Z register.
719       if (MO.isReg()) {
720         const TargetRegisterClass *RC;
721         switch (ExtraCode[0]) {
722         case 'b':
723           RC = &AArch64::FPR8RegClass;
724           break;
725         case 'h':
726           RC = &AArch64::FPR16RegClass;
727           break;
728         case 's':
729           RC = &AArch64::FPR32RegClass;
730           break;
731         case 'd':
732           RC = &AArch64::FPR64RegClass;
733           break;
734         case 'q':
735           RC = &AArch64::FPR128RegClass;
736           break;
737         case 'z':
738           RC = &AArch64::ZPRRegClass;
739           break;
740         default:
741           return true;
742         }
743         return printAsmRegInClass(MO, RC, AArch64::NoRegAltName, O);
744       }
745       printOperand(MI, OpNum, O);
746       return false;
747     }
748   }
749 
750   // According to ARM, we should emit x and v registers unless we have a
751   // modifier.
752   if (MO.isReg()) {
753     Register Reg = MO.getReg();
754 
755     // If this is a w or x register, print an x register.
756     if (AArch64::GPR32allRegClass.contains(Reg) ||
757         AArch64::GPR64allRegClass.contains(Reg))
758       return printAsmMRegister(MO, 'x', O);
759 
760     // If this is an x register tuple, print an x register.
761     if (AArch64::GPR64x8ClassRegClass.contains(Reg))
762       return printAsmMRegister(MO, 't', O);
763 
764     unsigned AltName = AArch64::NoRegAltName;
765     const TargetRegisterClass *RegClass;
766     if (AArch64::ZPRRegClass.contains(Reg)) {
767       RegClass = &AArch64::ZPRRegClass;
768     } else if (AArch64::PPRRegClass.contains(Reg)) {
769       RegClass = &AArch64::PPRRegClass;
770     } else {
771       RegClass = &AArch64::FPR128RegClass;
772       AltName = AArch64::vreg;
773     }
774 
775     // If this is a b, h, s, d, or q register, print it as a v register.
776     return printAsmRegInClass(MO, RegClass, AltName, O);
777   }
778 
779   printOperand(MI, OpNum, O);
780   return false;
781 }
782 
783 bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
784                                               unsigned OpNum,
785                                               const char *ExtraCode,
786                                               raw_ostream &O) {
787   if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
788     return true; // Unknown modifier.
789 
790   const MachineOperand &MO = MI->getOperand(OpNum);
791   assert(MO.isReg() && "unexpected inline asm memory operand");
792   O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]";
793   return false;
794 }
795 
796 void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
797                                                raw_ostream &OS) {
798   unsigned NOps = MI->getNumOperands();
799   assert(NOps == 4);
800   OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
801   // cast away const; DIetc do not take const operands for some reason.
802   OS << MI->getDebugVariable()->getName();
803   OS << " <- ";
804   // Frame address.  Currently handles register +- offset only.
805   assert(MI->isIndirectDebugValue());
806   OS << '[';
807   for (unsigned I = 0, E = std::distance(MI->debug_operands().begin(),
808                                          MI->debug_operands().end());
809        I < E; ++I) {
810     if (I != 0)
811       OS << ", ";
812     printOperand(MI, I, OS);
813   }
814   OS << ']';
815   OS << "+";
816   printOperand(MI, NOps - 2, OS);
817 }
818 
819 void AArch64AsmPrinter::emitJumpTableInfo() {
820   const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
821   if (!MJTI) return;
822 
823   const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
824   if (JT.empty()) return;
825 
826   const TargetLoweringObjectFile &TLOF = getObjFileLowering();
827   MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(MF->getFunction(), TM);
828   OutStreamer->SwitchSection(ReadOnlySec);
829 
830   auto AFI = MF->getInfo<AArch64FunctionInfo>();
831   for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
832     const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
833 
834     // If this jump table was deleted, ignore it.
835     if (JTBBs.empty()) continue;
836 
837     unsigned Size = AFI->getJumpTableEntrySize(JTI);
838     emitAlignment(Align(Size));
839     OutStreamer->emitLabel(GetJTISymbol(JTI));
840 
841     const MCSymbol *BaseSym = AArch64FI->getJumpTableEntryPCRelSymbol(JTI);
842     const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext);
843 
844     for (auto *JTBB : JTBBs) {
845       const MCExpr *Value =
846           MCSymbolRefExpr::create(JTBB->getSymbol(), OutContext);
847 
848       // Each entry is:
849       //     .byte/.hword (LBB - Lbase)>>2
850       // or plain:
851       //     .word LBB - Lbase
852       Value = MCBinaryExpr::createSub(Value, Base, OutContext);
853       if (Size != 4)
854         Value = MCBinaryExpr::createLShr(
855             Value, MCConstantExpr::create(2, OutContext), OutContext);
856 
857       OutStreamer->emitValue(Value, Size);
858     }
859   }
860 }
861 
862 void AArch64AsmPrinter::emitFunctionEntryLabel() {
863   if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall ||
864       MF->getFunction().getCallingConv() ==
865           CallingConv::AArch64_SVE_VectorCall ||
866       STI->getRegisterInfo()->hasSVEArgsOrReturn(MF)) {
867     auto *TS =
868         static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
869     TS->emitDirectiveVariantPCS(CurrentFnSym);
870   }
871 
872   return AsmPrinter::emitFunctionEntryLabel();
873 }
874 
875 /// Small jump tables contain an unsigned byte or half, representing the offset
876 /// from the lowest-addressed possible destination to the desired basic
877 /// block. Since all instructions are 4-byte aligned, this is further compressed
878 /// by counting in instructions rather than bytes (i.e. divided by 4). So, to
879 /// materialize the correct destination we need:
880 ///
881 ///             adr xDest, .LBB0_0
882 ///             ldrb wScratch, [xTable, xEntry]   (with "lsl #1" for ldrh).
883 ///             add xDest, xDest, xScratch (with "lsl #2" for smaller entries)
884 void AArch64AsmPrinter::LowerJumpTableDest(llvm::MCStreamer &OutStreamer,
885                                            const llvm::MachineInstr &MI) {
886   Register DestReg = MI.getOperand(0).getReg();
887   Register ScratchReg = MI.getOperand(1).getReg();
888   Register ScratchRegW =
889       STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32);
890   Register TableReg = MI.getOperand(2).getReg();
891   Register EntryReg = MI.getOperand(3).getReg();
892   int JTIdx = MI.getOperand(4).getIndex();
893   int Size = AArch64FI->getJumpTableEntrySize(JTIdx);
894 
895   // This has to be first because the compression pass based its reachability
896   // calculations on the start of the JumpTableDest instruction.
897   auto Label =
898       MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx);
899 
900   // If we don't already have a symbol to use as the base, use the ADR
901   // instruction itself.
902   if (!Label) {
903     Label = MF->getContext().createTempSymbol();
904     AArch64FI->setJumpTableEntryInfo(JTIdx, Size, Label);
905     OutStreamer.emitLabel(Label);
906   }
907 
908   auto LabelExpr = MCSymbolRefExpr::create(Label, MF->getContext());
909   EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR)
910                                   .addReg(DestReg)
911                                   .addExpr(LabelExpr));
912 
913   // Load the number of instruction-steps to offset from the label.
914   unsigned LdrOpcode;
915   switch (Size) {
916   case 1: LdrOpcode = AArch64::LDRBBroX; break;
917   case 2: LdrOpcode = AArch64::LDRHHroX; break;
918   case 4: LdrOpcode = AArch64::LDRSWroX; break;
919   default:
920     llvm_unreachable("Unknown jump table size");
921   }
922 
923   EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode)
924                                   .addReg(Size == 4 ? ScratchReg : ScratchRegW)
925                                   .addReg(TableReg)
926                                   .addReg(EntryReg)
927                                   .addImm(0)
928                                   .addImm(Size == 1 ? 0 : 1));
929 
930   // Add to the already materialized base label address, multiplying by 4 if
931   // compressed.
932   EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs)
933                                   .addReg(DestReg)
934                                   .addReg(DestReg)
935                                   .addReg(ScratchReg)
936                                   .addImm(Size == 4 ? 0 : 2));
937 }
938 
939 void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
940                                       const MachineInstr &MI) {
941   unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
942 
943   auto &Ctx = OutStreamer.getContext();
944   MCSymbol *MILabel = Ctx.createTempSymbol();
945   OutStreamer.emitLabel(MILabel);
946 
947   SM.recordStackMap(*MILabel, MI);
948   assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
949 
950   // Scan ahead to trim the shadow.
951   const MachineBasicBlock &MBB = *MI.getParent();
952   MachineBasicBlock::const_iterator MII(MI);
953   ++MII;
954   while (NumNOPBytes > 0) {
955     if (MII == MBB.end() || MII->isCall() ||
956         MII->getOpcode() == AArch64::DBG_VALUE ||
957         MII->getOpcode() == TargetOpcode::PATCHPOINT ||
958         MII->getOpcode() == TargetOpcode::STACKMAP)
959       break;
960     ++MII;
961     NumNOPBytes -= 4;
962   }
963 
964   // Emit nops.
965   for (unsigned i = 0; i < NumNOPBytes; i += 4)
966     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
967 }
968 
969 // Lower a patchpoint of the form:
970 // [<def>], <id>, <numBytes>, <target>, <numArgs>
971 void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
972                                         const MachineInstr &MI) {
973   auto &Ctx = OutStreamer.getContext();
974   MCSymbol *MILabel = Ctx.createTempSymbol();
975   OutStreamer.emitLabel(MILabel);
976   SM.recordPatchPoint(*MILabel, MI);
977 
978   PatchPointOpers Opers(&MI);
979 
980   int64_t CallTarget = Opers.getCallTarget().getImm();
981   unsigned EncodedBytes = 0;
982   if (CallTarget) {
983     assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
984            "High 16 bits of call target should be zero.");
985     Register ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
986     EncodedBytes = 16;
987     // Materialize the jump address:
988     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi)
989                                     .addReg(ScratchReg)
990                                     .addImm((CallTarget >> 32) & 0xFFFF)
991                                     .addImm(32));
992     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
993                                     .addReg(ScratchReg)
994                                     .addReg(ScratchReg)
995                                     .addImm((CallTarget >> 16) & 0xFFFF)
996                                     .addImm(16));
997     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
998                                     .addReg(ScratchReg)
999                                     .addReg(ScratchReg)
1000                                     .addImm(CallTarget & 0xFFFF)
1001                                     .addImm(0));
1002     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
1003   }
1004   // Emit padding.
1005   unsigned NumBytes = Opers.getNumPatchBytes();
1006   assert(NumBytes >= EncodedBytes &&
1007          "Patchpoint can't request size less than the length of a call.");
1008   assert((NumBytes - EncodedBytes) % 4 == 0 &&
1009          "Invalid number of NOP bytes requested!");
1010   for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
1011     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
1012 }
1013 
1014 void AArch64AsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1015                                         const MachineInstr &MI) {
1016   StatepointOpers SOpers(&MI);
1017   if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
1018     assert(PatchBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1019     for (unsigned i = 0; i < PatchBytes; i += 4)
1020       EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
1021   } else {
1022     // Lower call target and choose correct opcode
1023     const MachineOperand &CallTarget = SOpers.getCallTarget();
1024     MCOperand CallTargetMCOp;
1025     unsigned CallOpcode;
1026     switch (CallTarget.getType()) {
1027     case MachineOperand::MO_GlobalAddress:
1028     case MachineOperand::MO_ExternalSymbol:
1029       MCInstLowering.lowerOperand(CallTarget, CallTargetMCOp);
1030       CallOpcode = AArch64::BL;
1031       break;
1032     case MachineOperand::MO_Immediate:
1033       CallTargetMCOp = MCOperand::createImm(CallTarget.getImm());
1034       CallOpcode = AArch64::BL;
1035       break;
1036     case MachineOperand::MO_Register:
1037       CallTargetMCOp = MCOperand::createReg(CallTarget.getReg());
1038       CallOpcode = AArch64::BLR;
1039       break;
1040     default:
1041       llvm_unreachable("Unsupported operand type in statepoint call target");
1042       break;
1043     }
1044 
1045     EmitToStreamer(OutStreamer,
1046                    MCInstBuilder(CallOpcode).addOperand(CallTargetMCOp));
1047   }
1048 
1049   auto &Ctx = OutStreamer.getContext();
1050   MCSymbol *MILabel = Ctx.createTempSymbol();
1051   OutStreamer.emitLabel(MILabel);
1052   SM.recordStatepoint(*MILabel, MI);
1053 }
1054 
1055 void AArch64AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI) {
1056   // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
1057   //                  <opcode>, <operands>
1058 
1059   Register DefRegister = FaultingMI.getOperand(0).getReg();
1060   FaultMaps::FaultKind FK =
1061       static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(1).getImm());
1062   MCSymbol *HandlerLabel = FaultingMI.getOperand(2).getMBB()->getSymbol();
1063   unsigned Opcode = FaultingMI.getOperand(3).getImm();
1064   unsigned OperandsBeginIdx = 4;
1065 
1066   auto &Ctx = OutStreamer->getContext();
1067   MCSymbol *FaultingLabel = Ctx.createTempSymbol();
1068   OutStreamer->emitLabel(FaultingLabel);
1069 
1070   assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
1071   FM.recordFaultingOp(FK, FaultingLabel, HandlerLabel);
1072 
1073   MCInst MI;
1074   MI.setOpcode(Opcode);
1075 
1076   if (DefRegister != (Register)0)
1077     MI.addOperand(MCOperand::createReg(DefRegister));
1078 
1079   for (auto I = FaultingMI.operands_begin() + OperandsBeginIdx,
1080             E = FaultingMI.operands_end();
1081        I != E; ++I) {
1082     MCOperand Dest;
1083     lowerOperand(*I, Dest);
1084     MI.addOperand(Dest);
1085   }
1086 
1087   OutStreamer->AddComment("on-fault: " + HandlerLabel->getName());
1088   OutStreamer->emitInstruction(MI, getSubtargetInfo());
1089 }
1090 
1091 void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) {
1092   Register DestReg = MI.getOperand(0).getReg();
1093   if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround()) {
1094     // Convert H/S register to corresponding D register
1095     if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31)
1096       DestReg = AArch64::D0 + (DestReg - AArch64::H0);
1097     else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31)
1098       DestReg = AArch64::D0 + (DestReg - AArch64::S0);
1099     else
1100       assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31);
1101 
1102     MCInst MOVI;
1103     MOVI.setOpcode(AArch64::MOVID);
1104     MOVI.addOperand(MCOperand::createReg(DestReg));
1105     MOVI.addOperand(MCOperand::createImm(0));
1106     EmitToStreamer(*OutStreamer, MOVI);
1107   } else {
1108     MCInst FMov;
1109     switch (MI.getOpcode()) {
1110     default: llvm_unreachable("Unexpected opcode");
1111     case AArch64::FMOVH0:
1112       FMov.setOpcode(AArch64::FMOVWHr);
1113       FMov.addOperand(MCOperand::createReg(DestReg));
1114       FMov.addOperand(MCOperand::createReg(AArch64::WZR));
1115       break;
1116     case AArch64::FMOVS0:
1117       FMov.setOpcode(AArch64::FMOVWSr);
1118       FMov.addOperand(MCOperand::createReg(DestReg));
1119       FMov.addOperand(MCOperand::createReg(AArch64::WZR));
1120       break;
1121     case AArch64::FMOVD0:
1122       FMov.setOpcode(AArch64::FMOVXDr);
1123       FMov.addOperand(MCOperand::createReg(DestReg));
1124       FMov.addOperand(MCOperand::createReg(AArch64::XZR));
1125       break;
1126     }
1127     EmitToStreamer(*OutStreamer, FMov);
1128   }
1129 }
1130 
1131 // Simple pseudo-instructions have their lowering (with expansion to real
1132 // instructions) auto-generated.
1133 #include "AArch64GenMCPseudoLowering.inc"
1134 
1135 void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
1136   // Do any auto-generated pseudo lowerings.
1137   if (emitPseudoExpansionLowering(*OutStreamer, MI))
1138     return;
1139 
1140   if (MI->getOpcode() == AArch64::ADRP) {
1141     for (auto &Opd : MI->operands()) {
1142       if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) ==
1143                                 "swift_async_extendedFramePointerFlags") {
1144         ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true;
1145       }
1146     }
1147   }
1148 
1149   if (AArch64FI->getLOHRelated().count(MI)) {
1150     // Generate a label for LOH related instruction
1151     MCSymbol *LOHLabel = createTempSymbol("loh");
1152     // Associate the instruction with the label
1153     LOHInstToLabel[MI] = LOHLabel;
1154     OutStreamer->emitLabel(LOHLabel);
1155   }
1156 
1157   AArch64TargetStreamer *TS =
1158     static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
1159   // Do any manual lowerings.
1160   switch (MI->getOpcode()) {
1161   default:
1162     break;
1163   case AArch64::HINT: {
1164     // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
1165     // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
1166     // non-empty. If MI is the initial BTI, place the
1167     // __patchable_function_entries label after BTI.
1168     if (CurrentPatchableFunctionEntrySym &&
1169         CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
1170         MI == &MF->front().front()) {
1171       int64_t Imm = MI->getOperand(0).getImm();
1172       if ((Imm & 32) && (Imm & 6)) {
1173         MCInst Inst;
1174         MCInstLowering.Lower(MI, Inst);
1175         EmitToStreamer(*OutStreamer, Inst);
1176         CurrentPatchableFunctionEntrySym = createTempSymbol("patch");
1177         OutStreamer->emitLabel(CurrentPatchableFunctionEntrySym);
1178         return;
1179       }
1180     }
1181     break;
1182   }
1183     case AArch64::MOVMCSym: {
1184       Register DestReg = MI->getOperand(0).getReg();
1185       const MachineOperand &MO_Sym = MI->getOperand(1);
1186       MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
1187       MCOperand Hi_MCSym, Lo_MCSym;
1188 
1189       Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
1190       Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
1191 
1192       MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym);
1193       MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym);
1194 
1195       MCInst MovZ;
1196       MovZ.setOpcode(AArch64::MOVZXi);
1197       MovZ.addOperand(MCOperand::createReg(DestReg));
1198       MovZ.addOperand(Hi_MCSym);
1199       MovZ.addOperand(MCOperand::createImm(16));
1200       EmitToStreamer(*OutStreamer, MovZ);
1201 
1202       MCInst MovK;
1203       MovK.setOpcode(AArch64::MOVKXi);
1204       MovK.addOperand(MCOperand::createReg(DestReg));
1205       MovK.addOperand(MCOperand::createReg(DestReg));
1206       MovK.addOperand(Lo_MCSym);
1207       MovK.addOperand(MCOperand::createImm(0));
1208       EmitToStreamer(*OutStreamer, MovK);
1209       return;
1210   }
1211   case AArch64::MOVIv2d_ns:
1212     // If the target has <rdar://problem/16473581>, lower this
1213     // instruction to movi.16b instead.
1214     if (STI->hasZeroCycleZeroingFPWorkaround() &&
1215         MI->getOperand(1).getImm() == 0) {
1216       MCInst TmpInst;
1217       TmpInst.setOpcode(AArch64::MOVIv16b_ns);
1218       TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1219       TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm()));
1220       EmitToStreamer(*OutStreamer, TmpInst);
1221       return;
1222     }
1223     break;
1224 
1225   case AArch64::DBG_VALUE:
1226   case AArch64::DBG_VALUE_LIST: {
1227     if (isVerbose() && OutStreamer->hasRawTextSupport()) {
1228       SmallString<128> TmpStr;
1229       raw_svector_ostream OS(TmpStr);
1230       PrintDebugValueComment(MI, OS);
1231       OutStreamer->emitRawText(StringRef(OS.str()));
1232     }
1233     return;
1234 
1235   case AArch64::EMITBKEY: {
1236       ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
1237       if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
1238           ExceptionHandlingType != ExceptionHandling::ARM)
1239         return;
1240 
1241       if (getFunctionCFISectionType(*MF) == CFISection::None)
1242         return;
1243 
1244       OutStreamer->emitCFIBKeyFrame();
1245       return;
1246     }
1247     }
1248 
1249   // Tail calls use pseudo instructions so they have the proper code-gen
1250   // attributes (isCall, isReturn, etc.). We lower them to the real
1251   // instruction here.
1252   case AArch64::TCRETURNri:
1253   case AArch64::TCRETURNriBTI:
1254   case AArch64::TCRETURNriALL: {
1255     MCInst TmpInst;
1256     TmpInst.setOpcode(AArch64::BR);
1257     TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1258     EmitToStreamer(*OutStreamer, TmpInst);
1259     return;
1260   }
1261   case AArch64::TCRETURNdi: {
1262     MCOperand Dest;
1263     MCInstLowering.lowerOperand(MI->getOperand(0), Dest);
1264     MCInst TmpInst;
1265     TmpInst.setOpcode(AArch64::B);
1266     TmpInst.addOperand(Dest);
1267     EmitToStreamer(*OutStreamer, TmpInst);
1268     return;
1269   }
1270   case AArch64::SpeculationBarrierISBDSBEndBB: {
1271     // Print DSB SYS + ISB
1272     MCInst TmpInstDSB;
1273     TmpInstDSB.setOpcode(AArch64::DSB);
1274     TmpInstDSB.addOperand(MCOperand::createImm(0xf));
1275     EmitToStreamer(*OutStreamer, TmpInstDSB);
1276     MCInst TmpInstISB;
1277     TmpInstISB.setOpcode(AArch64::ISB);
1278     TmpInstISB.addOperand(MCOperand::createImm(0xf));
1279     EmitToStreamer(*OutStreamer, TmpInstISB);
1280     return;
1281   }
1282   case AArch64::SpeculationBarrierSBEndBB: {
1283     // Print SB
1284     MCInst TmpInstSB;
1285     TmpInstSB.setOpcode(AArch64::SB);
1286     EmitToStreamer(*OutStreamer, TmpInstSB);
1287     return;
1288   }
1289   case AArch64::TLSDESC_CALLSEQ: {
1290     /// lower this to:
1291     ///    adrp  x0, :tlsdesc:var
1292     ///    ldr   x1, [x0, #:tlsdesc_lo12:var]
1293     ///    add   x0, x0, #:tlsdesc_lo12:var
1294     ///    .tlsdesccall var
1295     ///    blr   x1
1296     ///    (TPIDR_EL0 offset now in x0)
1297     const MachineOperand &MO_Sym = MI->getOperand(0);
1298     MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
1299     MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
1300     MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
1301     MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
1302     MCInstLowering.lowerOperand(MO_Sym, Sym);
1303     MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12);
1304     MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc);
1305 
1306     MCInst Adrp;
1307     Adrp.setOpcode(AArch64::ADRP);
1308     Adrp.addOperand(MCOperand::createReg(AArch64::X0));
1309     Adrp.addOperand(SymTLSDesc);
1310     EmitToStreamer(*OutStreamer, Adrp);
1311 
1312     MCInst Ldr;
1313     if (STI->isTargetILP32()) {
1314       Ldr.setOpcode(AArch64::LDRWui);
1315       Ldr.addOperand(MCOperand::createReg(AArch64::W1));
1316     } else {
1317       Ldr.setOpcode(AArch64::LDRXui);
1318       Ldr.addOperand(MCOperand::createReg(AArch64::X1));
1319     }
1320     Ldr.addOperand(MCOperand::createReg(AArch64::X0));
1321     Ldr.addOperand(SymTLSDescLo12);
1322     Ldr.addOperand(MCOperand::createImm(0));
1323     EmitToStreamer(*OutStreamer, Ldr);
1324 
1325     MCInst Add;
1326     if (STI->isTargetILP32()) {
1327       Add.setOpcode(AArch64::ADDWri);
1328       Add.addOperand(MCOperand::createReg(AArch64::W0));
1329       Add.addOperand(MCOperand::createReg(AArch64::W0));
1330     } else {
1331       Add.setOpcode(AArch64::ADDXri);
1332       Add.addOperand(MCOperand::createReg(AArch64::X0));
1333       Add.addOperand(MCOperand::createReg(AArch64::X0));
1334     }
1335     Add.addOperand(SymTLSDescLo12);
1336     Add.addOperand(MCOperand::createImm(AArch64_AM::getShiftValue(0)));
1337     EmitToStreamer(*OutStreamer, Add);
1338 
1339     // Emit a relocation-annotation. This expands to no code, but requests
1340     // the following instruction gets an R_AARCH64_TLSDESC_CALL.
1341     MCInst TLSDescCall;
1342     TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
1343     TLSDescCall.addOperand(Sym);
1344     EmitToStreamer(*OutStreamer, TLSDescCall);
1345 
1346     MCInst Blr;
1347     Blr.setOpcode(AArch64::BLR);
1348     Blr.addOperand(MCOperand::createReg(AArch64::X1));
1349     EmitToStreamer(*OutStreamer, Blr);
1350 
1351     return;
1352   }
1353 
1354   case AArch64::JumpTableDest32:
1355   case AArch64::JumpTableDest16:
1356   case AArch64::JumpTableDest8:
1357     LowerJumpTableDest(*OutStreamer, *MI);
1358     return;
1359 
1360   case AArch64::FMOVH0:
1361   case AArch64::FMOVS0:
1362   case AArch64::FMOVD0:
1363     emitFMov0(*MI);
1364     return;
1365 
1366   case TargetOpcode::STACKMAP:
1367     return LowerSTACKMAP(*OutStreamer, SM, *MI);
1368 
1369   case TargetOpcode::PATCHPOINT:
1370     return LowerPATCHPOINT(*OutStreamer, SM, *MI);
1371 
1372   case TargetOpcode::STATEPOINT:
1373     return LowerSTATEPOINT(*OutStreamer, SM, *MI);
1374 
1375   case TargetOpcode::FAULTING_OP:
1376     return LowerFAULTING_OP(*MI);
1377 
1378   case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
1379     LowerPATCHABLE_FUNCTION_ENTER(*MI);
1380     return;
1381 
1382   case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
1383     LowerPATCHABLE_FUNCTION_EXIT(*MI);
1384     return;
1385 
1386   case TargetOpcode::PATCHABLE_TAIL_CALL:
1387     LowerPATCHABLE_TAIL_CALL(*MI);
1388     return;
1389 
1390   case AArch64::HWASAN_CHECK_MEMACCESS:
1391   case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
1392     LowerHWASAN_CHECK_MEMACCESS(*MI);
1393     return;
1394 
1395   case AArch64::SEH_StackAlloc:
1396     TS->emitARM64WinCFIAllocStack(MI->getOperand(0).getImm());
1397     return;
1398 
1399   case AArch64::SEH_SaveFPLR:
1400     TS->emitARM64WinCFISaveFPLR(MI->getOperand(0).getImm());
1401     return;
1402 
1403   case AArch64::SEH_SaveFPLR_X:
1404     assert(MI->getOperand(0).getImm() < 0 &&
1405            "Pre increment SEH opcode must have a negative offset");
1406     TS->emitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm());
1407     return;
1408 
1409   case AArch64::SEH_SaveReg:
1410     TS->emitARM64WinCFISaveReg(MI->getOperand(0).getImm(),
1411                                MI->getOperand(1).getImm());
1412     return;
1413 
1414   case AArch64::SEH_SaveReg_X:
1415     assert(MI->getOperand(1).getImm() < 0 &&
1416            "Pre increment SEH opcode must have a negative offset");
1417     TS->emitARM64WinCFISaveRegX(MI->getOperand(0).getImm(),
1418                                 -MI->getOperand(1).getImm());
1419     return;
1420 
1421   case AArch64::SEH_SaveRegP:
1422     if (MI->getOperand(1).getImm() == 30 && MI->getOperand(0).getImm() >= 19 &&
1423         MI->getOperand(0).getImm() <= 28) {
1424       assert((MI->getOperand(0).getImm() - 19) % 2 == 0 &&
1425              "Register paired with LR must be odd");
1426       TS->emitARM64WinCFISaveLRPair(MI->getOperand(0).getImm(),
1427                                     MI->getOperand(2).getImm());
1428       return;
1429     }
1430     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1431             "Non-consecutive registers not allowed for save_regp");
1432     TS->emitARM64WinCFISaveRegP(MI->getOperand(0).getImm(),
1433                                 MI->getOperand(2).getImm());
1434     return;
1435 
1436   case AArch64::SEH_SaveRegP_X:
1437     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1438             "Non-consecutive registers not allowed for save_regp_x");
1439     assert(MI->getOperand(2).getImm() < 0 &&
1440            "Pre increment SEH opcode must have a negative offset");
1441     TS->emitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(),
1442                                  -MI->getOperand(2).getImm());
1443     return;
1444 
1445   case AArch64::SEH_SaveFReg:
1446     TS->emitARM64WinCFISaveFReg(MI->getOperand(0).getImm(),
1447                                 MI->getOperand(1).getImm());
1448     return;
1449 
1450   case AArch64::SEH_SaveFReg_X:
1451     assert(MI->getOperand(1).getImm() < 0 &&
1452            "Pre increment SEH opcode must have a negative offset");
1453     TS->emitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(),
1454                                  -MI->getOperand(1).getImm());
1455     return;
1456 
1457   case AArch64::SEH_SaveFRegP:
1458     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1459             "Non-consecutive registers not allowed for save_regp");
1460     TS->emitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(),
1461                                  MI->getOperand(2).getImm());
1462     return;
1463 
1464   case AArch64::SEH_SaveFRegP_X:
1465     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1466             "Non-consecutive registers not allowed for save_regp_x");
1467     assert(MI->getOperand(2).getImm() < 0 &&
1468            "Pre increment SEH opcode must have a negative offset");
1469     TS->emitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(),
1470                                   -MI->getOperand(2).getImm());
1471     return;
1472 
1473   case AArch64::SEH_SetFP:
1474     TS->emitARM64WinCFISetFP();
1475     return;
1476 
1477   case AArch64::SEH_AddFP:
1478     TS->emitARM64WinCFIAddFP(MI->getOperand(0).getImm());
1479     return;
1480 
1481   case AArch64::SEH_Nop:
1482     TS->emitARM64WinCFINop();
1483     return;
1484 
1485   case AArch64::SEH_PrologEnd:
1486     TS->emitARM64WinCFIPrologEnd();
1487     return;
1488 
1489   case AArch64::SEH_EpilogStart:
1490     TS->emitARM64WinCFIEpilogStart();
1491     return;
1492 
1493   case AArch64::SEH_EpilogEnd:
1494     TS->emitARM64WinCFIEpilogEnd();
1495     return;
1496   }
1497 
1498   // Finally, do the automated lowerings for everything else.
1499   MCInst TmpInst;
1500   MCInstLowering.Lower(MI, TmpInst);
1501   EmitToStreamer(*OutStreamer, TmpInst);
1502 }
1503 
1504 // Force static initialization.
1505 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter() {
1506   RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
1507   RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget());
1508   RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target());
1509   RegisterAsmPrinter<AArch64AsmPrinter> W(getTheARM64_32Target());
1510   RegisterAsmPrinter<AArch64AsmPrinter> V(getTheAArch64_32Target());
1511 }
1512