1 //===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains a printer that converts from our internal representation
10 // of machine-dependent LLVM code to the AArch64 assembly language.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "AArch64.h"
15 #include "AArch64MCInstLower.h"
16 #include "AArch64MachineFunctionInfo.h"
17 #include "AArch64RegisterInfo.h"
18 #include "AArch64Subtarget.h"
19 #include "AArch64TargetObjectFile.h"
20 #include "MCTargetDesc/AArch64AddressingModes.h"
21 #include "MCTargetDesc/AArch64InstPrinter.h"
22 #include "MCTargetDesc/AArch64MCExpr.h"
23 #include "MCTargetDesc/AArch64MCTargetDesc.h"
24 #include "MCTargetDesc/AArch64TargetStreamer.h"
25 #include "TargetInfo/AArch64TargetInfo.h"
26 #include "Utils/AArch64BaseInfo.h"
27 #include "llvm/ADT/SmallString.h"
28 #include "llvm/ADT/SmallVector.h"
29 #include "llvm/ADT/StringRef.h"
30 #include "llvm/ADT/Triple.h"
31 #include "llvm/ADT/Twine.h"
32 #include "llvm/BinaryFormat/COFF.h"
33 #include "llvm/BinaryFormat/ELF.h"
34 #include "llvm/CodeGen/AsmPrinter.h"
35 #include "llvm/CodeGen/FaultMaps.h"
36 #include "llvm/CodeGen/MachineBasicBlock.h"
37 #include "llvm/CodeGen/MachineFunction.h"
38 #include "llvm/CodeGen/MachineInstr.h"
39 #include "llvm/CodeGen/MachineJumpTableInfo.h"
40 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
41 #include "llvm/CodeGen/MachineOperand.h"
42 #include "llvm/CodeGen/StackMaps.h"
43 #include "llvm/CodeGen/TargetRegisterInfo.h"
44 #include "llvm/IR/DataLayout.h"
45 #include "llvm/IR/DebugInfoMetadata.h"
46 #include "llvm/MC/MCAsmInfo.h"
47 #include "llvm/MC/MCContext.h"
48 #include "llvm/MC/MCInst.h"
49 #include "llvm/MC/MCInstBuilder.h"
50 #include "llvm/MC/MCSectionELF.h"
51 #include "llvm/MC/MCStreamer.h"
52 #include "llvm/MC/MCSymbol.h"
53 #include "llvm/Support/Casting.h"
54 #include "llvm/Support/ErrorHandling.h"
55 #include "llvm/Support/TargetRegistry.h"
56 #include "llvm/Support/raw_ostream.h"
57 #include "llvm/Target/TargetMachine.h"
58 #include <algorithm>
59 #include <cassert>
60 #include <cstdint>
61 #include <map>
62 #include <memory>
63 
64 using namespace llvm;
65 
66 #define DEBUG_TYPE "asm-printer"
67 
68 namespace {
69 
70 class AArch64AsmPrinter : public AsmPrinter {
71   AArch64MCInstLower MCInstLowering;
72   StackMaps SM;
73   FaultMaps FM;
74   const AArch64Subtarget *STI;
75 
76 public:
77   AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
78       : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this),
79         SM(*this), FM(*this) {}
80 
81   StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
82 
83   /// Wrapper for MCInstLowering.lowerOperand() for the
84   /// tblgen'erated pseudo lowering.
85   bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
86     return MCInstLowering.lowerOperand(MO, MCOp);
87   }
88 
89   void emitStartOfAsmFile(Module &M) override;
90   void emitJumpTableInfo() override;
91 
92   void emitFunctionEntryLabel() override;
93 
94   void LowerJumpTableDest(MCStreamer &OutStreamer, const MachineInstr &MI);
95 
96   void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
97                      const MachineInstr &MI);
98   void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
99                        const MachineInstr &MI);
100   void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
101                        const MachineInstr &MI);
102   void LowerFAULTING_OP(const MachineInstr &MI);
103 
104   void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
105   void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
106   void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
107 
108   typedef std::tuple<unsigned, bool, uint32_t> HwasanMemaccessTuple;
109   std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
110   void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
111   void EmitHwasanMemaccessSymbols(Module &M);
112 
113   void EmitSled(const MachineInstr &MI, SledKind Kind);
114 
115   /// tblgen'erated driver function for lowering simple MI->MC
116   /// pseudo instructions.
117   bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
118                                    const MachineInstr *MI);
119 
120   void emitInstruction(const MachineInstr *MI) override;
121 
122   void emitFunctionHeaderComment() override;
123 
124   void getAnalysisUsage(AnalysisUsage &AU) const override {
125     AsmPrinter::getAnalysisUsage(AU);
126     AU.setPreservesAll();
127   }
128 
129   bool runOnMachineFunction(MachineFunction &MF) override {
130     AArch64FI = MF.getInfo<AArch64FunctionInfo>();
131     STI = static_cast<const AArch64Subtarget*>(&MF.getSubtarget());
132 
133     SetupMachineFunction(MF);
134 
135     if (STI->isTargetCOFF()) {
136       bool Internal = MF.getFunction().hasInternalLinkage();
137       COFF::SymbolStorageClass Scl = Internal ? COFF::IMAGE_SYM_CLASS_STATIC
138                                               : COFF::IMAGE_SYM_CLASS_EXTERNAL;
139       int Type =
140         COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT;
141 
142       OutStreamer->BeginCOFFSymbolDef(CurrentFnSym);
143       OutStreamer->EmitCOFFSymbolStorageClass(Scl);
144       OutStreamer->EmitCOFFSymbolType(Type);
145       OutStreamer->EndCOFFSymbolDef();
146     }
147 
148     // Emit the rest of the function body.
149     emitFunctionBody();
150 
151     // Emit the XRay table for this function.
152     emitXRayTable();
153 
154     // We didn't modify anything.
155     return false;
156   }
157 
158 private:
159   void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
160   bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
161   bool printAsmRegInClass(const MachineOperand &MO,
162                           const TargetRegisterClass *RC, unsigned AltName,
163                           raw_ostream &O);
164 
165   bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
166                        const char *ExtraCode, raw_ostream &O) override;
167   bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
168                              const char *ExtraCode, raw_ostream &O) override;
169 
170   void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
171 
172   void emitFunctionBodyEnd() override;
173 
174   MCSymbol *GetCPISymbol(unsigned CPID) const override;
175   void emitEndOfAsmFile(Module &M) override;
176 
177   AArch64FunctionInfo *AArch64FI = nullptr;
178 
179   /// Emit the LOHs contained in AArch64FI.
180   void EmitLOHs();
181 
182   /// Emit instruction to set float register to zero.
183   void EmitFMov0(const MachineInstr &MI);
184 
185   using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
186 
187   MInstToMCSymbol LOHInstToLabel;
188 };
189 
190 } // end anonymous namespace
191 
192 void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
193   if (!TM.getTargetTriple().isOSBinFormatELF())
194     return;
195 
196   // Assemble feature flags that may require creation of a note section.
197   unsigned Flags = 0;
198   if (const auto *BTE = mdconst::extract_or_null<ConstantInt>(
199           M.getModuleFlag("branch-target-enforcement")))
200     if (BTE->getZExtValue())
201       Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI;
202 
203   if (const auto *Sign = mdconst::extract_or_null<ConstantInt>(
204           M.getModuleFlag("sign-return-address")))
205     if (Sign->getZExtValue())
206       Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
207 
208   if (Flags == 0)
209     return;
210 
211   // Emit a .note.gnu.property section with the flags.
212   if (auto *TS = static_cast<AArch64TargetStreamer *>(
213           OutStreamer->getTargetStreamer()))
214     TS->emitNoteSection(Flags);
215 }
216 
217 void AArch64AsmPrinter::emitFunctionHeaderComment() {
218   const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>();
219   Optional<std::string> OutlinerString = FI->getOutliningStyle();
220   if (OutlinerString != None)
221     OutStreamer->GetCommentOS() << ' ' << OutlinerString;
222 }
223 
224 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
225 {
226   const Function &F = MF->getFunction();
227   if (F.hasFnAttribute("patchable-function-entry")) {
228     unsigned Num;
229     if (F.getFnAttribute("patchable-function-entry")
230             .getValueAsString()
231             .getAsInteger(10, Num))
232       return;
233     emitNops(Num);
234     return;
235   }
236 
237   EmitSled(MI, SledKind::FUNCTION_ENTER);
238 }
239 
240 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI)
241 {
242   EmitSled(MI, SledKind::FUNCTION_EXIT);
243 }
244 
245 void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI)
246 {
247   EmitSled(MI, SledKind::TAIL_CALL);
248 }
249 
250 void AArch64AsmPrinter::EmitSled(const MachineInstr &MI, SledKind Kind)
251 {
252   static const int8_t NoopsInSledCount = 7;
253   // We want to emit the following pattern:
254   //
255   // .Lxray_sled_N:
256   //   ALIGN
257   //   B #32
258   //   ; 7 NOP instructions (28 bytes)
259   // .tmpN
260   //
261   // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
262   // over the full 32 bytes (8 instructions) with the following pattern:
263   //
264   //   STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
265   //   LDR W0, #12 ; W0 := function ID
266   //   LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
267   //   BLR X16 ; call the tracing trampoline
268   //   ;DATA: 32 bits of function ID
269   //   ;DATA: lower 32 bits of the address of the trampoline
270   //   ;DATA: higher 32 bits of the address of the trampoline
271   //   LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
272   //
273   OutStreamer->emitCodeAlignment(4);
274   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
275   OutStreamer->emitLabel(CurSled);
276   auto Target = OutContext.createTempSymbol();
277 
278   // Emit "B #32" instruction, which jumps over the next 28 bytes.
279   // The operand has to be the number of 4-byte instructions to jump over,
280   // including the current instruction.
281   EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8));
282 
283   for (int8_t I = 0; I < NoopsInSledCount; I++)
284     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
285 
286   OutStreamer->emitLabel(Target);
287   recordSled(CurSled, MI, Kind, 2);
288 }
289 
290 void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
291   Register Reg = MI.getOperand(0).getReg();
292   bool IsShort =
293       MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES;
294   uint32_t AccessInfo = MI.getOperand(1).getImm();
295   MCSymbol *&Sym =
296       HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, IsShort, AccessInfo)];
297   if (!Sym) {
298     // FIXME: Make this work on non-ELF.
299     if (!TM.getTargetTriple().isOSBinFormatELF())
300       report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF");
301 
302     std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" +
303                           utostr(AccessInfo);
304     if (IsShort)
305       SymName += "_short";
306     Sym = OutContext.getOrCreateSymbol(SymName);
307   }
308 
309   EmitToStreamer(*OutStreamer,
310                  MCInstBuilder(AArch64::BL)
311                      .addExpr(MCSymbolRefExpr::create(Sym, OutContext)));
312 }
313 
314 void AArch64AsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
315   if (HwasanMemaccessSymbols.empty())
316     return;
317 
318   const Triple &TT = TM.getTargetTriple();
319   assert(TT.isOSBinFormatELF());
320   std::unique_ptr<MCSubtargetInfo> STI(
321       TM.getTarget().createMCSubtargetInfo(TT.str(), "", ""));
322 
323   MCSymbol *HwasanTagMismatchV1Sym =
324       OutContext.getOrCreateSymbol("__hwasan_tag_mismatch");
325   MCSymbol *HwasanTagMismatchV2Sym =
326       OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2");
327 
328   const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
329       MCSymbolRefExpr::create(HwasanTagMismatchV1Sym, OutContext);
330   const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
331       MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext);
332 
333   for (auto &P : HwasanMemaccessSymbols) {
334     unsigned Reg = std::get<0>(P.first);
335     bool IsShort = std::get<1>(P.first);
336     uint32_t AccessInfo = std::get<2>(P.first);
337     const MCSymbolRefExpr *HwasanTagMismatchRef =
338         IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
339     MCSymbol *Sym = P.second;
340 
341     OutStreamer->SwitchSection(OutContext.getELFSection(
342         ".text.hot", ELF::SHT_PROGBITS,
343         ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0,
344         Sym->getName()));
345 
346     OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
347     OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
348     OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden);
349     OutStreamer->emitLabel(Sym);
350 
351     OutStreamer->emitInstruction(MCInstBuilder(AArch64::UBFMXri)
352                                      .addReg(AArch64::X16)
353                                      .addReg(Reg)
354                                      .addImm(4)
355                                      .addImm(55),
356                                  *STI);
357     OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRBBroX)
358                                      .addReg(AArch64::W16)
359                                      .addReg(AArch64::X9)
360                                      .addReg(AArch64::X16)
361                                      .addImm(0)
362                                      .addImm(0),
363                                  *STI);
364     OutStreamer->emitInstruction(
365         MCInstBuilder(AArch64::SUBSXrs)
366             .addReg(AArch64::XZR)
367             .addReg(AArch64::X16)
368             .addReg(Reg)
369             .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
370         *STI);
371     MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
372     OutStreamer->emitInstruction(
373         MCInstBuilder(AArch64::Bcc)
374             .addImm(AArch64CC::NE)
375             .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym,
376                                              OutContext)),
377         *STI);
378     MCSymbol *ReturnSym = OutContext.createTempSymbol();
379     OutStreamer->emitLabel(ReturnSym);
380     OutStreamer->emitInstruction(
381         MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI);
382     OutStreamer->emitLabel(HandleMismatchOrPartialSym);
383 
384     if (IsShort) {
385       OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWri)
386                                        .addReg(AArch64::WZR)
387                                        .addReg(AArch64::W16)
388                                        .addImm(15)
389                                        .addImm(0),
390                                    *STI);
391       MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
392       OutStreamer->emitInstruction(
393           MCInstBuilder(AArch64::Bcc)
394               .addImm(AArch64CC::HI)
395               .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
396           *STI);
397 
398       OutStreamer->emitInstruction(
399           MCInstBuilder(AArch64::ANDXri)
400               .addReg(AArch64::X17)
401               .addReg(Reg)
402               .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
403           *STI);
404       unsigned Size = 1 << (AccessInfo & 0xf);
405       if (Size != 1)
406         OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
407                                          .addReg(AArch64::X17)
408                                          .addReg(AArch64::X17)
409                                          .addImm(Size - 1)
410                                          .addImm(0),
411                                      *STI);
412       OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWrs)
413                                        .addReg(AArch64::WZR)
414                                        .addReg(AArch64::W16)
415                                        .addReg(AArch64::W17)
416                                        .addImm(0),
417                                    *STI);
418       OutStreamer->emitInstruction(
419           MCInstBuilder(AArch64::Bcc)
420               .addImm(AArch64CC::LS)
421               .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
422           *STI);
423 
424       OutStreamer->emitInstruction(
425           MCInstBuilder(AArch64::ORRXri)
426               .addReg(AArch64::X16)
427               .addReg(Reg)
428               .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
429           *STI);
430       OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRBBui)
431                                        .addReg(AArch64::W16)
432                                        .addReg(AArch64::X16)
433                                        .addImm(0),
434                                    *STI);
435       OutStreamer->emitInstruction(
436           MCInstBuilder(AArch64::SUBSXrs)
437               .addReg(AArch64::XZR)
438               .addReg(AArch64::X16)
439               .addReg(Reg)
440               .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
441           *STI);
442       OutStreamer->emitInstruction(
443           MCInstBuilder(AArch64::Bcc)
444               .addImm(AArch64CC::EQ)
445               .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
446           *STI);
447 
448       OutStreamer->emitLabel(HandleMismatchSym);
449     }
450 
451     OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre)
452                                      .addReg(AArch64::SP)
453                                      .addReg(AArch64::X0)
454                                      .addReg(AArch64::X1)
455                                      .addReg(AArch64::SP)
456                                      .addImm(-32),
457                                  *STI);
458     OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi)
459                                      .addReg(AArch64::FP)
460                                      .addReg(AArch64::LR)
461                                      .addReg(AArch64::SP)
462                                      .addImm(29),
463                                  *STI);
464 
465     if (Reg != AArch64::X0)
466       OutStreamer->emitInstruction(MCInstBuilder(AArch64::ORRXrs)
467                                        .addReg(AArch64::X0)
468                                        .addReg(AArch64::XZR)
469                                        .addReg(Reg)
470                                        .addImm(0),
471                                    *STI);
472     OutStreamer->emitInstruction(MCInstBuilder(AArch64::MOVZXi)
473                                      .addReg(AArch64::X1)
474                                      .addImm(AccessInfo)
475                                      .addImm(0),
476                                  *STI);
477 
478     // Intentionally load the GOT entry and branch to it, rather than possibly
479     // late binding the function, which may clobber the registers before we have
480     // a chance to save them.
481     OutStreamer->emitInstruction(
482         MCInstBuilder(AArch64::ADRP)
483             .addReg(AArch64::X16)
484             .addExpr(AArch64MCExpr::create(
485                 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_PAGE,
486                 OutContext)),
487         *STI);
488     OutStreamer->emitInstruction(
489         MCInstBuilder(AArch64::LDRXui)
490             .addReg(AArch64::X16)
491             .addReg(AArch64::X16)
492             .addExpr(AArch64MCExpr::create(
493                 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_LO12,
494                 OutContext)),
495         *STI);
496     OutStreamer->emitInstruction(
497         MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI);
498   }
499 }
500 
501 void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
502   EmitHwasanMemaccessSymbols(M);
503 
504   const Triple &TT = TM.getTargetTriple();
505   if (TT.isOSBinFormatMachO()) {
506     // Funny Darwin hack: This flag tells the linker that no global symbols
507     // contain code that falls through to other global symbols (e.g. the obvious
508     // implementation of multiple entry points).  If this doesn't occur, the
509     // linker can safely perform dead code stripping.  Since LLVM never
510     // generates code that does this, it is always safe to set.
511     OutStreamer->emitAssemblerFlag(MCAF_SubsectionsViaSymbols);
512   }
513 
514   // Emit stack and fault map information.
515   emitStackMaps(SM);
516   FM.serializeToFaultMapSection();
517 
518 }
519 
520 void AArch64AsmPrinter::EmitLOHs() {
521   SmallVector<MCSymbol *, 3> MCArgs;
522 
523   for (const auto &D : AArch64FI->getLOHContainer()) {
524     for (const MachineInstr *MI : D.getArgs()) {
525       MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI);
526       assert(LabelIt != LOHInstToLabel.end() &&
527              "Label hasn't been inserted for LOH related instruction");
528       MCArgs.push_back(LabelIt->second);
529     }
530     OutStreamer->emitLOHDirective(D.getKind(), MCArgs);
531     MCArgs.clear();
532   }
533 }
534 
535 void AArch64AsmPrinter::emitFunctionBodyEnd() {
536   if (!AArch64FI->getLOHRelated().empty())
537     EmitLOHs();
538 }
539 
540 /// GetCPISymbol - Return the symbol for the specified constant pool entry.
541 MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
542   // Darwin uses a linker-private symbol name for constant-pools (to
543   // avoid addends on the relocation?), ELF has no such concept and
544   // uses a normal private symbol.
545   if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
546     return OutContext.getOrCreateSymbol(
547         Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
548         Twine(getFunctionNumber()) + "_" + Twine(CPID));
549 
550   return AsmPrinter::GetCPISymbol(CPID);
551 }
552 
553 void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
554                                      raw_ostream &O) {
555   const MachineOperand &MO = MI->getOperand(OpNum);
556   switch (MO.getType()) {
557   default:
558     llvm_unreachable("<unknown operand type>");
559   case MachineOperand::MO_Register: {
560     Register Reg = MO.getReg();
561     assert(Register::isPhysicalRegister(Reg));
562     assert(!MO.getSubReg() && "Subregs should be eliminated!");
563     O << AArch64InstPrinter::getRegisterName(Reg);
564     break;
565   }
566   case MachineOperand::MO_Immediate: {
567     O << MO.getImm();
568     break;
569   }
570   case MachineOperand::MO_GlobalAddress: {
571     PrintSymbolOperand(MO, O);
572     break;
573   }
574   case MachineOperand::MO_BlockAddress: {
575     MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
576     Sym->print(O, MAI);
577     break;
578   }
579   }
580 }
581 
582 bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
583                                           raw_ostream &O) {
584   Register Reg = MO.getReg();
585   switch (Mode) {
586   default:
587     return true; // Unknown mode.
588   case 'w':
589     Reg = getWRegFromXReg(Reg);
590     break;
591   case 'x':
592     Reg = getXRegFromWReg(Reg);
593     break;
594   }
595 
596   O << AArch64InstPrinter::getRegisterName(Reg);
597   return false;
598 }
599 
600 // Prints the register in MO using class RC using the offset in the
601 // new register class. This should not be used for cross class
602 // printing.
603 bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
604                                            const TargetRegisterClass *RC,
605                                            unsigned AltName, raw_ostream &O) {
606   assert(MO.isReg() && "Should only get here with a register!");
607   const TargetRegisterInfo *RI = STI->getRegisterInfo();
608   Register Reg = MO.getReg();
609   unsigned RegToPrint = RC->getRegister(RI->getEncodingValue(Reg));
610   assert(RI->regsOverlap(RegToPrint, Reg));
611   O << AArch64InstPrinter::getRegisterName(RegToPrint, AltName);
612   return false;
613 }
614 
615 bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
616                                         const char *ExtraCode, raw_ostream &O) {
617   const MachineOperand &MO = MI->getOperand(OpNum);
618 
619   // First try the generic code, which knows about modifiers like 'c' and 'n'.
620   if (!AsmPrinter::PrintAsmOperand(MI, OpNum, ExtraCode, O))
621     return false;
622 
623   // Does this asm operand have a single letter operand modifier?
624   if (ExtraCode && ExtraCode[0]) {
625     if (ExtraCode[1] != 0)
626       return true; // Unknown modifier.
627 
628     switch (ExtraCode[0]) {
629     default:
630       return true; // Unknown modifier.
631     case 'w':      // Print W register
632     case 'x':      // Print X register
633       if (MO.isReg())
634         return printAsmMRegister(MO, ExtraCode[0], O);
635       if (MO.isImm() && MO.getImm() == 0) {
636         unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
637         O << AArch64InstPrinter::getRegisterName(Reg);
638         return false;
639       }
640       printOperand(MI, OpNum, O);
641       return false;
642     case 'b': // Print B register.
643     case 'h': // Print H register.
644     case 's': // Print S register.
645     case 'd': // Print D register.
646     case 'q': // Print Q register.
647     case 'z': // Print Z register.
648       if (MO.isReg()) {
649         const TargetRegisterClass *RC;
650         switch (ExtraCode[0]) {
651         case 'b':
652           RC = &AArch64::FPR8RegClass;
653           break;
654         case 'h':
655           RC = &AArch64::FPR16RegClass;
656           break;
657         case 's':
658           RC = &AArch64::FPR32RegClass;
659           break;
660         case 'd':
661           RC = &AArch64::FPR64RegClass;
662           break;
663         case 'q':
664           RC = &AArch64::FPR128RegClass;
665           break;
666         case 'z':
667           RC = &AArch64::ZPRRegClass;
668           break;
669         default:
670           return true;
671         }
672         return printAsmRegInClass(MO, RC, AArch64::NoRegAltName, O);
673       }
674       printOperand(MI, OpNum, O);
675       return false;
676     }
677   }
678 
679   // According to ARM, we should emit x and v registers unless we have a
680   // modifier.
681   if (MO.isReg()) {
682     Register Reg = MO.getReg();
683 
684     // If this is a w or x register, print an x register.
685     if (AArch64::GPR32allRegClass.contains(Reg) ||
686         AArch64::GPR64allRegClass.contains(Reg))
687       return printAsmMRegister(MO, 'x', O);
688 
689     unsigned AltName = AArch64::NoRegAltName;
690     const TargetRegisterClass *RegClass;
691     if (AArch64::ZPRRegClass.contains(Reg)) {
692       RegClass = &AArch64::ZPRRegClass;
693     } else if (AArch64::PPRRegClass.contains(Reg)) {
694       RegClass = &AArch64::PPRRegClass;
695     } else {
696       RegClass = &AArch64::FPR128RegClass;
697       AltName = AArch64::vreg;
698     }
699 
700     // If this is a b, h, s, d, or q register, print it as a v register.
701     return printAsmRegInClass(MO, RegClass, AltName, O);
702   }
703 
704   printOperand(MI, OpNum, O);
705   return false;
706 }
707 
708 bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
709                                               unsigned OpNum,
710                                               const char *ExtraCode,
711                                               raw_ostream &O) {
712   if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
713     return true; // Unknown modifier.
714 
715   const MachineOperand &MO = MI->getOperand(OpNum);
716   assert(MO.isReg() && "unexpected inline asm memory operand");
717   O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]";
718   return false;
719 }
720 
721 void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
722                                                raw_ostream &OS) {
723   unsigned NOps = MI->getNumOperands();
724   assert(NOps == 4);
725   OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
726   // cast away const; DIetc do not take const operands for some reason.
727   OS << MI->getDebugVariable()->getName();
728   OS << " <- ";
729   // Frame address.  Currently handles register +- offset only.
730   assert(MI->getDebugOperand(0).isReg() && MI->isDebugOffsetImm());
731   OS << '[';
732   printOperand(MI, 0, OS);
733   OS << '+';
734   printOperand(MI, 1, OS);
735   OS << ']';
736   OS << "+";
737   printOperand(MI, NOps - 2, OS);
738 }
739 
740 void AArch64AsmPrinter::emitJumpTableInfo() {
741   const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
742   if (!MJTI) return;
743 
744   const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
745   if (JT.empty()) return;
746 
747   const Function &F = MF->getFunction();
748   const TargetLoweringObjectFile &TLOF = getObjFileLowering();
749   bool JTInDiffSection =
750       !STI->isTargetCOFF() ||
751       !TLOF.shouldPutJumpTableInFunctionSection(
752           MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32,
753           F);
754   if (JTInDiffSection) {
755       // Drop it in the readonly section.
756       MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(F, TM);
757       OutStreamer->SwitchSection(ReadOnlySec);
758   }
759 
760   auto AFI = MF->getInfo<AArch64FunctionInfo>();
761   for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
762     const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
763 
764     // If this jump table was deleted, ignore it.
765     if (JTBBs.empty()) continue;
766 
767     unsigned Size = AFI->getJumpTableEntrySize(JTI);
768     emitAlignment(Align(Size));
769     OutStreamer->emitLabel(GetJTISymbol(JTI));
770 
771     const MCSymbol *BaseSym = AArch64FI->getJumpTableEntryPCRelSymbol(JTI);
772     const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext);
773 
774     for (auto *JTBB : JTBBs) {
775       const MCExpr *Value =
776           MCSymbolRefExpr::create(JTBB->getSymbol(), OutContext);
777 
778       // Each entry is:
779       //     .byte/.hword (LBB - Lbase)>>2
780       // or plain:
781       //     .word LBB - Lbase
782       Value = MCBinaryExpr::createSub(Value, Base, OutContext);
783       if (Size != 4)
784         Value = MCBinaryExpr::createLShr(
785             Value, MCConstantExpr::create(2, OutContext), OutContext);
786 
787       OutStreamer->emitValue(Value, Size);
788     }
789   }
790 }
791 
792 void AArch64AsmPrinter::emitFunctionEntryLabel() {
793   if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall ||
794       MF->getFunction().getCallingConv() ==
795           CallingConv::AArch64_SVE_VectorCall ||
796       STI->getRegisterInfo()->hasSVEArgsOrReturn(MF)) {
797     auto *TS =
798         static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
799     TS->emitDirectiveVariantPCS(CurrentFnSym);
800   }
801 
802   return AsmPrinter::emitFunctionEntryLabel();
803 }
804 
805 /// Small jump tables contain an unsigned byte or half, representing the offset
806 /// from the lowest-addressed possible destination to the desired basic
807 /// block. Since all instructions are 4-byte aligned, this is further compressed
808 /// by counting in instructions rather than bytes (i.e. divided by 4). So, to
809 /// materialize the correct destination we need:
810 ///
811 ///             adr xDest, .LBB0_0
812 ///             ldrb wScratch, [xTable, xEntry]   (with "lsl #1" for ldrh).
813 ///             add xDest, xDest, xScratch (with "lsl #2" for smaller entries)
814 void AArch64AsmPrinter::LowerJumpTableDest(llvm::MCStreamer &OutStreamer,
815                                            const llvm::MachineInstr &MI) {
816   Register DestReg = MI.getOperand(0).getReg();
817   Register ScratchReg = MI.getOperand(1).getReg();
818   Register ScratchRegW =
819       STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32);
820   Register TableReg = MI.getOperand(2).getReg();
821   Register EntryReg = MI.getOperand(3).getReg();
822   int JTIdx = MI.getOperand(4).getIndex();
823   int Size = AArch64FI->getJumpTableEntrySize(JTIdx);
824 
825   // This has to be first because the compression pass based its reachability
826   // calculations on the start of the JumpTableDest instruction.
827   auto Label =
828       MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx);
829 
830   // If we don't already have a symbol to use as the base, use the ADR
831   // instruction itself.
832   if (!Label) {
833     Label = MF->getContext().createTempSymbol();
834     AArch64FI->setJumpTableEntryInfo(JTIdx, Size, Label);
835     OutStreamer.emitLabel(Label);
836   }
837 
838   auto LabelExpr = MCSymbolRefExpr::create(Label, MF->getContext());
839   EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR)
840                                   .addReg(DestReg)
841                                   .addExpr(LabelExpr));
842 
843   // Load the number of instruction-steps to offset from the label.
844   unsigned LdrOpcode;
845   switch (Size) {
846   case 1: LdrOpcode = AArch64::LDRBBroX; break;
847   case 2: LdrOpcode = AArch64::LDRHHroX; break;
848   case 4: LdrOpcode = AArch64::LDRSWroX; break;
849   default:
850     llvm_unreachable("Unknown jump table size");
851   }
852 
853   EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode)
854                                   .addReg(Size == 4 ? ScratchReg : ScratchRegW)
855                                   .addReg(TableReg)
856                                   .addReg(EntryReg)
857                                   .addImm(0)
858                                   .addImm(Size == 1 ? 0 : 1));
859 
860   // Add to the already materialized base label address, multiplying by 4 if
861   // compressed.
862   EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs)
863                                   .addReg(DestReg)
864                                   .addReg(DestReg)
865                                   .addReg(ScratchReg)
866                                   .addImm(Size == 4 ? 0 : 2));
867 }
868 
869 void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
870                                       const MachineInstr &MI) {
871   unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
872 
873   auto &Ctx = OutStreamer.getContext();
874   MCSymbol *MILabel = Ctx.createTempSymbol();
875   OutStreamer.emitLabel(MILabel);
876 
877   SM.recordStackMap(*MILabel, MI);
878   assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
879 
880   // Scan ahead to trim the shadow.
881   const MachineBasicBlock &MBB = *MI.getParent();
882   MachineBasicBlock::const_iterator MII(MI);
883   ++MII;
884   while (NumNOPBytes > 0) {
885     if (MII == MBB.end() || MII->isCall() ||
886         MII->getOpcode() == AArch64::DBG_VALUE ||
887         MII->getOpcode() == TargetOpcode::PATCHPOINT ||
888         MII->getOpcode() == TargetOpcode::STACKMAP)
889       break;
890     ++MII;
891     NumNOPBytes -= 4;
892   }
893 
894   // Emit nops.
895   for (unsigned i = 0; i < NumNOPBytes; i += 4)
896     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
897 }
898 
899 // Lower a patchpoint of the form:
900 // [<def>], <id>, <numBytes>, <target>, <numArgs>
901 void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
902                                         const MachineInstr &MI) {
903   auto &Ctx = OutStreamer.getContext();
904   MCSymbol *MILabel = Ctx.createTempSymbol();
905   OutStreamer.emitLabel(MILabel);
906   SM.recordPatchPoint(*MILabel, MI);
907 
908   PatchPointOpers Opers(&MI);
909 
910   int64_t CallTarget = Opers.getCallTarget().getImm();
911   unsigned EncodedBytes = 0;
912   if (CallTarget) {
913     assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
914            "High 16 bits of call target should be zero.");
915     Register ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
916     EncodedBytes = 16;
917     // Materialize the jump address:
918     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi)
919                                     .addReg(ScratchReg)
920                                     .addImm((CallTarget >> 32) & 0xFFFF)
921                                     .addImm(32));
922     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
923                                     .addReg(ScratchReg)
924                                     .addReg(ScratchReg)
925                                     .addImm((CallTarget >> 16) & 0xFFFF)
926                                     .addImm(16));
927     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
928                                     .addReg(ScratchReg)
929                                     .addReg(ScratchReg)
930                                     .addImm(CallTarget & 0xFFFF)
931                                     .addImm(0));
932     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
933   }
934   // Emit padding.
935   unsigned NumBytes = Opers.getNumPatchBytes();
936   assert(NumBytes >= EncodedBytes &&
937          "Patchpoint can't request size less than the length of a call.");
938   assert((NumBytes - EncodedBytes) % 4 == 0 &&
939          "Invalid number of NOP bytes requested!");
940   for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
941     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
942 }
943 
944 void AArch64AsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
945                                         const MachineInstr &MI) {
946   StatepointOpers SOpers(&MI);
947   if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
948     assert(PatchBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
949     for (unsigned i = 0; i < PatchBytes; i += 4)
950       EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
951   } else {
952     // Lower call target and choose correct opcode
953     const MachineOperand &CallTarget = SOpers.getCallTarget();
954     MCOperand CallTargetMCOp;
955     unsigned CallOpcode;
956     switch (CallTarget.getType()) {
957     case MachineOperand::MO_GlobalAddress:
958     case MachineOperand::MO_ExternalSymbol:
959       MCInstLowering.lowerOperand(CallTarget, CallTargetMCOp);
960       CallOpcode = AArch64::BL;
961       break;
962     case MachineOperand::MO_Immediate:
963       CallTargetMCOp = MCOperand::createImm(CallTarget.getImm());
964       CallOpcode = AArch64::BL;
965       break;
966     case MachineOperand::MO_Register:
967       CallTargetMCOp = MCOperand::createReg(CallTarget.getReg());
968       CallOpcode = AArch64::BLR;
969       break;
970     default:
971       llvm_unreachable("Unsupported operand type in statepoint call target");
972       break;
973     }
974 
975     EmitToStreamer(OutStreamer,
976                    MCInstBuilder(CallOpcode).addOperand(CallTargetMCOp));
977   }
978 
979   auto &Ctx = OutStreamer.getContext();
980   MCSymbol *MILabel = Ctx.createTempSymbol();
981   OutStreamer.emitLabel(MILabel);
982   SM.recordStatepoint(*MILabel, MI);
983 }
984 
985 void AArch64AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI) {
986   // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
987   //                  <opcode>, <operands>
988 
989   Register DefRegister = FaultingMI.getOperand(0).getReg();
990   FaultMaps::FaultKind FK =
991       static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(1).getImm());
992   MCSymbol *HandlerLabel = FaultingMI.getOperand(2).getMBB()->getSymbol();
993   unsigned Opcode = FaultingMI.getOperand(3).getImm();
994   unsigned OperandsBeginIdx = 4;
995 
996   auto &Ctx = OutStreamer->getContext();
997   MCSymbol *FaultingLabel = Ctx.createTempSymbol();
998   OutStreamer->emitLabel(FaultingLabel);
999 
1000   assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
1001   FM.recordFaultingOp(FK, FaultingLabel, HandlerLabel);
1002 
1003   MCInst MI;
1004   MI.setOpcode(Opcode);
1005 
1006   if (DefRegister != (Register)0)
1007     MI.addOperand(MCOperand::createReg(DefRegister));
1008 
1009   for (auto I = FaultingMI.operands_begin() + OperandsBeginIdx,
1010             E = FaultingMI.operands_end();
1011        I != E; ++I) {
1012     MCOperand Dest;
1013     lowerOperand(*I, Dest);
1014     MI.addOperand(Dest);
1015   }
1016 
1017   OutStreamer->AddComment("on-fault: " + HandlerLabel->getName());
1018   OutStreamer->emitInstruction(MI, getSubtargetInfo());
1019 }
1020 
1021 void AArch64AsmPrinter::EmitFMov0(const MachineInstr &MI) {
1022   Register DestReg = MI.getOperand(0).getReg();
1023   if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround()) {
1024     // Convert H/S/D register to corresponding Q register
1025     if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31)
1026       DestReg = AArch64::Q0 + (DestReg - AArch64::H0);
1027     else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31)
1028       DestReg = AArch64::Q0 + (DestReg - AArch64::S0);
1029     else {
1030       assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31);
1031       DestReg = AArch64::Q0 + (DestReg - AArch64::D0);
1032     }
1033     MCInst MOVI;
1034     MOVI.setOpcode(AArch64::MOVIv2d_ns);
1035     MOVI.addOperand(MCOperand::createReg(DestReg));
1036     MOVI.addOperand(MCOperand::createImm(0));
1037     EmitToStreamer(*OutStreamer, MOVI);
1038   } else {
1039     MCInst FMov;
1040     switch (MI.getOpcode()) {
1041     default: llvm_unreachable("Unexpected opcode");
1042     case AArch64::FMOVH0:
1043       FMov.setOpcode(AArch64::FMOVWHr);
1044       FMov.addOperand(MCOperand::createReg(DestReg));
1045       FMov.addOperand(MCOperand::createReg(AArch64::WZR));
1046       break;
1047     case AArch64::FMOVS0:
1048       FMov.setOpcode(AArch64::FMOVWSr);
1049       FMov.addOperand(MCOperand::createReg(DestReg));
1050       FMov.addOperand(MCOperand::createReg(AArch64::WZR));
1051       break;
1052     case AArch64::FMOVD0:
1053       FMov.setOpcode(AArch64::FMOVXDr);
1054       FMov.addOperand(MCOperand::createReg(DestReg));
1055       FMov.addOperand(MCOperand::createReg(AArch64::XZR));
1056       break;
1057     }
1058     EmitToStreamer(*OutStreamer, FMov);
1059   }
1060 }
1061 
1062 // Simple pseudo-instructions have their lowering (with expansion to real
1063 // instructions) auto-generated.
1064 #include "AArch64GenMCPseudoLowering.inc"
1065 
1066 void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
1067   // Do any auto-generated pseudo lowerings.
1068   if (emitPseudoExpansionLowering(*OutStreamer, MI))
1069     return;
1070 
1071   if (AArch64FI->getLOHRelated().count(MI)) {
1072     // Generate a label for LOH related instruction
1073     MCSymbol *LOHLabel = createTempSymbol("loh");
1074     // Associate the instruction with the label
1075     LOHInstToLabel[MI] = LOHLabel;
1076     OutStreamer->emitLabel(LOHLabel);
1077   }
1078 
1079   AArch64TargetStreamer *TS =
1080     static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
1081   // Do any manual lowerings.
1082   switch (MI->getOpcode()) {
1083   default:
1084     break;
1085   case AArch64::HINT: {
1086     // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
1087     // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
1088     // non-empty. If MI is the initial BTI, place the
1089     // __patchable_function_entries label after BTI.
1090     if (CurrentPatchableFunctionEntrySym &&
1091         CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
1092         MI == &MF->front().front()) {
1093       int64_t Imm = MI->getOperand(0).getImm();
1094       if ((Imm & 32) && (Imm & 6)) {
1095         MCInst Inst;
1096         MCInstLowering.Lower(MI, Inst);
1097         EmitToStreamer(*OutStreamer, Inst);
1098         CurrentPatchableFunctionEntrySym = createTempSymbol("patch");
1099         OutStreamer->emitLabel(CurrentPatchableFunctionEntrySym);
1100         return;
1101       }
1102     }
1103     break;
1104   }
1105     case AArch64::MOVMCSym: {
1106       Register DestReg = MI->getOperand(0).getReg();
1107       const MachineOperand &MO_Sym = MI->getOperand(1);
1108       MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
1109       MCOperand Hi_MCSym, Lo_MCSym;
1110 
1111       Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
1112       Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
1113 
1114       MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym);
1115       MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym);
1116 
1117       MCInst MovZ;
1118       MovZ.setOpcode(AArch64::MOVZXi);
1119       MovZ.addOperand(MCOperand::createReg(DestReg));
1120       MovZ.addOperand(Hi_MCSym);
1121       MovZ.addOperand(MCOperand::createImm(16));
1122       EmitToStreamer(*OutStreamer, MovZ);
1123 
1124       MCInst MovK;
1125       MovK.setOpcode(AArch64::MOVKXi);
1126       MovK.addOperand(MCOperand::createReg(DestReg));
1127       MovK.addOperand(MCOperand::createReg(DestReg));
1128       MovK.addOperand(Lo_MCSym);
1129       MovK.addOperand(MCOperand::createImm(0));
1130       EmitToStreamer(*OutStreamer, MovK);
1131       return;
1132   }
1133   case AArch64::MOVIv2d_ns:
1134     // If the target has <rdar://problem/16473581>, lower this
1135     // instruction to movi.16b instead.
1136     if (STI->hasZeroCycleZeroingFPWorkaround() &&
1137         MI->getOperand(1).getImm() == 0) {
1138       MCInst TmpInst;
1139       TmpInst.setOpcode(AArch64::MOVIv16b_ns);
1140       TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1141       TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm()));
1142       EmitToStreamer(*OutStreamer, TmpInst);
1143       return;
1144     }
1145     break;
1146 
1147   case AArch64::DBG_VALUE: {
1148     if (isVerbose() && OutStreamer->hasRawTextSupport()) {
1149       SmallString<128> TmpStr;
1150       raw_svector_ostream OS(TmpStr);
1151       PrintDebugValueComment(MI, OS);
1152       OutStreamer->emitRawText(StringRef(OS.str()));
1153     }
1154     return;
1155 
1156   case AArch64::EMITBKEY: {
1157       ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
1158       if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
1159           ExceptionHandlingType != ExceptionHandling::ARM)
1160         return;
1161 
1162       if (needsCFIMoves() == CFI_M_None)
1163         return;
1164 
1165       OutStreamer->emitCFIBKeyFrame();
1166       return;
1167     }
1168   }
1169 
1170   // Tail calls use pseudo instructions so they have the proper code-gen
1171   // attributes (isCall, isReturn, etc.). We lower them to the real
1172   // instruction here.
1173   case AArch64::TCRETURNri:
1174   case AArch64::TCRETURNriBTI:
1175   case AArch64::TCRETURNriALL: {
1176     MCInst TmpInst;
1177     TmpInst.setOpcode(AArch64::BR);
1178     TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1179     EmitToStreamer(*OutStreamer, TmpInst);
1180     return;
1181   }
1182   case AArch64::TCRETURNdi: {
1183     MCOperand Dest;
1184     MCInstLowering.lowerOperand(MI->getOperand(0), Dest);
1185     MCInst TmpInst;
1186     TmpInst.setOpcode(AArch64::B);
1187     TmpInst.addOperand(Dest);
1188     EmitToStreamer(*OutStreamer, TmpInst);
1189     return;
1190   }
1191   case AArch64::SpeculationBarrierISBDSBEndBB: {
1192     // Print DSB SYS + ISB
1193     MCInst TmpInstDSB;
1194     TmpInstDSB.setOpcode(AArch64::DSB);
1195     TmpInstDSB.addOperand(MCOperand::createImm(0xf));
1196     EmitToStreamer(*OutStreamer, TmpInstDSB);
1197     MCInst TmpInstISB;
1198     TmpInstISB.setOpcode(AArch64::ISB);
1199     TmpInstISB.addOperand(MCOperand::createImm(0xf));
1200     EmitToStreamer(*OutStreamer, TmpInstISB);
1201     return;
1202   }
1203   case AArch64::SpeculationBarrierSBEndBB: {
1204     // Print SB
1205     MCInst TmpInstSB;
1206     TmpInstSB.setOpcode(AArch64::SB);
1207     EmitToStreamer(*OutStreamer, TmpInstSB);
1208     return;
1209   }
1210   case AArch64::TLSDESC_CALLSEQ: {
1211     /// lower this to:
1212     ///    adrp  x0, :tlsdesc:var
1213     ///    ldr   x1, [x0, #:tlsdesc_lo12:var]
1214     ///    add   x0, x0, #:tlsdesc_lo12:var
1215     ///    .tlsdesccall var
1216     ///    blr   x1
1217     ///    (TPIDR_EL0 offset now in x0)
1218     const MachineOperand &MO_Sym = MI->getOperand(0);
1219     MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
1220     MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
1221     MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
1222     MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
1223     MCInstLowering.lowerOperand(MO_Sym, Sym);
1224     MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12);
1225     MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc);
1226 
1227     MCInst Adrp;
1228     Adrp.setOpcode(AArch64::ADRP);
1229     Adrp.addOperand(MCOperand::createReg(AArch64::X0));
1230     Adrp.addOperand(SymTLSDesc);
1231     EmitToStreamer(*OutStreamer, Adrp);
1232 
1233     MCInst Ldr;
1234     Ldr.setOpcode(AArch64::LDRXui);
1235     Ldr.addOperand(MCOperand::createReg(AArch64::X1));
1236     Ldr.addOperand(MCOperand::createReg(AArch64::X0));
1237     Ldr.addOperand(SymTLSDescLo12);
1238     Ldr.addOperand(MCOperand::createImm(0));
1239     EmitToStreamer(*OutStreamer, Ldr);
1240 
1241     MCInst Add;
1242     Add.setOpcode(AArch64::ADDXri);
1243     Add.addOperand(MCOperand::createReg(AArch64::X0));
1244     Add.addOperand(MCOperand::createReg(AArch64::X0));
1245     Add.addOperand(SymTLSDescLo12);
1246     Add.addOperand(MCOperand::createImm(AArch64_AM::getShiftValue(0)));
1247     EmitToStreamer(*OutStreamer, Add);
1248 
1249     // Emit a relocation-annotation. This expands to no code, but requests
1250     // the following instruction gets an R_AARCH64_TLSDESC_CALL.
1251     MCInst TLSDescCall;
1252     TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
1253     TLSDescCall.addOperand(Sym);
1254     EmitToStreamer(*OutStreamer, TLSDescCall);
1255 
1256     MCInst Blr;
1257     Blr.setOpcode(AArch64::BLR);
1258     Blr.addOperand(MCOperand::createReg(AArch64::X1));
1259     EmitToStreamer(*OutStreamer, Blr);
1260 
1261     return;
1262   }
1263 
1264   case AArch64::JumpTableDest32:
1265   case AArch64::JumpTableDest16:
1266   case AArch64::JumpTableDest8:
1267     LowerJumpTableDest(*OutStreamer, *MI);
1268     return;
1269 
1270   case AArch64::FMOVH0:
1271   case AArch64::FMOVS0:
1272   case AArch64::FMOVD0:
1273     EmitFMov0(*MI);
1274     return;
1275 
1276   case TargetOpcode::STACKMAP:
1277     return LowerSTACKMAP(*OutStreamer, SM, *MI);
1278 
1279   case TargetOpcode::PATCHPOINT:
1280     return LowerPATCHPOINT(*OutStreamer, SM, *MI);
1281 
1282   case TargetOpcode::STATEPOINT:
1283     return LowerSTATEPOINT(*OutStreamer, SM, *MI);
1284 
1285   case TargetOpcode::FAULTING_OP:
1286     return LowerFAULTING_OP(*MI);
1287 
1288   case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
1289     LowerPATCHABLE_FUNCTION_ENTER(*MI);
1290     return;
1291 
1292   case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
1293     LowerPATCHABLE_FUNCTION_EXIT(*MI);
1294     return;
1295 
1296   case TargetOpcode::PATCHABLE_TAIL_CALL:
1297     LowerPATCHABLE_TAIL_CALL(*MI);
1298     return;
1299 
1300   case AArch64::HWASAN_CHECK_MEMACCESS:
1301   case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
1302     LowerHWASAN_CHECK_MEMACCESS(*MI);
1303     return;
1304 
1305   case AArch64::SEH_StackAlloc:
1306     TS->EmitARM64WinCFIAllocStack(MI->getOperand(0).getImm());
1307     return;
1308 
1309   case AArch64::SEH_SaveFPLR:
1310     TS->EmitARM64WinCFISaveFPLR(MI->getOperand(0).getImm());
1311     return;
1312 
1313   case AArch64::SEH_SaveFPLR_X:
1314     assert(MI->getOperand(0).getImm() < 0 &&
1315            "Pre increment SEH opcode must have a negative offset");
1316     TS->EmitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm());
1317     return;
1318 
1319   case AArch64::SEH_SaveReg:
1320     TS->EmitARM64WinCFISaveReg(MI->getOperand(0).getImm(),
1321                                MI->getOperand(1).getImm());
1322     return;
1323 
1324   case AArch64::SEH_SaveReg_X:
1325     assert(MI->getOperand(1).getImm() < 0 &&
1326            "Pre increment SEH opcode must have a negative offset");
1327     TS->EmitARM64WinCFISaveRegX(MI->getOperand(0).getImm(),
1328 		                -MI->getOperand(1).getImm());
1329     return;
1330 
1331   case AArch64::SEH_SaveRegP:
1332     if (MI->getOperand(1).getImm() == 30 && MI->getOperand(0).getImm() >= 19 &&
1333         MI->getOperand(0).getImm() <= 28) {
1334       assert((MI->getOperand(0).getImm() - 19) % 2 == 0 &&
1335              "Register paired with LR must be odd");
1336       TS->EmitARM64WinCFISaveLRPair(MI->getOperand(0).getImm(),
1337                                     MI->getOperand(2).getImm());
1338       return;
1339     }
1340     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1341             "Non-consecutive registers not allowed for save_regp");
1342     TS->EmitARM64WinCFISaveRegP(MI->getOperand(0).getImm(),
1343                                 MI->getOperand(2).getImm());
1344     return;
1345 
1346   case AArch64::SEH_SaveRegP_X:
1347     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1348             "Non-consecutive registers not allowed for save_regp_x");
1349     assert(MI->getOperand(2).getImm() < 0 &&
1350            "Pre increment SEH opcode must have a negative offset");
1351     TS->EmitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(),
1352                                  -MI->getOperand(2).getImm());
1353     return;
1354 
1355   case AArch64::SEH_SaveFReg:
1356     TS->EmitARM64WinCFISaveFReg(MI->getOperand(0).getImm(),
1357                                 MI->getOperand(1).getImm());
1358     return;
1359 
1360   case AArch64::SEH_SaveFReg_X:
1361     assert(MI->getOperand(1).getImm() < 0 &&
1362            "Pre increment SEH opcode must have a negative offset");
1363     TS->EmitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(),
1364                                  -MI->getOperand(1).getImm());
1365     return;
1366 
1367   case AArch64::SEH_SaveFRegP:
1368     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1369             "Non-consecutive registers not allowed for save_regp");
1370     TS->EmitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(),
1371                                  MI->getOperand(2).getImm());
1372     return;
1373 
1374   case AArch64::SEH_SaveFRegP_X:
1375     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1376             "Non-consecutive registers not allowed for save_regp_x");
1377     assert(MI->getOperand(2).getImm() < 0 &&
1378            "Pre increment SEH opcode must have a negative offset");
1379     TS->EmitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(),
1380                                   -MI->getOperand(2).getImm());
1381     return;
1382 
1383   case AArch64::SEH_SetFP:
1384     TS->EmitARM64WinCFISetFP();
1385     return;
1386 
1387   case AArch64::SEH_AddFP:
1388     TS->EmitARM64WinCFIAddFP(MI->getOperand(0).getImm());
1389     return;
1390 
1391   case AArch64::SEH_Nop:
1392     TS->EmitARM64WinCFINop();
1393     return;
1394 
1395   case AArch64::SEH_PrologEnd:
1396     TS->EmitARM64WinCFIPrologEnd();
1397     return;
1398 
1399   case AArch64::SEH_EpilogStart:
1400     TS->EmitARM64WinCFIEpilogStart();
1401     return;
1402 
1403   case AArch64::SEH_EpilogEnd:
1404     TS->EmitARM64WinCFIEpilogEnd();
1405     return;
1406   }
1407 
1408   // Finally, do the automated lowerings for everything else.
1409   MCInst TmpInst;
1410   MCInstLowering.Lower(MI, TmpInst);
1411   EmitToStreamer(*OutStreamer, TmpInst);
1412 }
1413 
1414 // Force static initialization.
1415 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter() {
1416   RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
1417   RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget());
1418   RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target());
1419   RegisterAsmPrinter<AArch64AsmPrinter> W(getTheARM64_32Target());
1420   RegisterAsmPrinter<AArch64AsmPrinter> V(getTheAArch64_32Target());
1421 }
1422