1 //===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains a printer that converts from our internal representation
11 // of machine-dependent LLVM code to the AArch64 assembly language.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #include "AArch64.h"
16 #include "AArch64MCInstLower.h"
17 #include "AArch64MachineFunctionInfo.h"
18 #include "AArch64RegisterInfo.h"
19 #include "AArch64Subtarget.h"
20 #include "AArch64TargetObjectFile.h"
21 #include "InstPrinter/AArch64InstPrinter.h"
22 #include "MCTargetDesc/AArch64AddressingModes.h"
23 #include "MCTargetDesc/AArch64MCTargetDesc.h"
24 #include "MCTargetDesc/AArch64TargetStreamer.h"
25 #include "Utils/AArch64BaseInfo.h"
26 #include "llvm/ADT/SmallString.h"
27 #include "llvm/ADT/SmallVector.h"
28 #include "llvm/ADT/StringRef.h"
29 #include "llvm/ADT/Triple.h"
30 #include "llvm/ADT/Twine.h"
31 #include "llvm/BinaryFormat/COFF.h"
32 #include "llvm/CodeGen/AsmPrinter.h"
33 #include "llvm/CodeGen/MachineBasicBlock.h"
34 #include "llvm/CodeGen/MachineFunction.h"
35 #include "llvm/CodeGen/MachineInstr.h"
36 #include "llvm/CodeGen/MachineJumpTableInfo.h"
37 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
38 #include "llvm/CodeGen/MachineOperand.h"
39 #include "llvm/CodeGen/StackMaps.h"
40 #include "llvm/CodeGen/TargetRegisterInfo.h"
41 #include "llvm/IR/DataLayout.h"
42 #include "llvm/IR/DebugInfoMetadata.h"
43 #include "llvm/MC/MCAsmInfo.h"
44 #include "llvm/MC/MCContext.h"
45 #include "llvm/MC/MCInst.h"
46 #include "llvm/MC/MCInstBuilder.h"
47 #include "llvm/MC/MCStreamer.h"
48 #include "llvm/MC/MCSymbol.h"
49 #include "llvm/Support/Casting.h"
50 #include "llvm/Support/ErrorHandling.h"
51 #include "llvm/Support/TargetRegistry.h"
52 #include "llvm/Support/raw_ostream.h"
53 #include "llvm/Target/TargetMachine.h"
54 #include <algorithm>
55 #include <cassert>
56 #include <cstdint>
57 #include <map>
58 #include <memory>
59
60 using namespace llvm;
61
62 #define DEBUG_TYPE "asm-printer"
63
64 namespace {
65
66 class AArch64AsmPrinter : public AsmPrinter {
67 AArch64MCInstLower MCInstLowering;
68 StackMaps SM;
69 const AArch64Subtarget *STI;
70
71 public:
AArch64AsmPrinter(TargetMachine & TM,std::unique_ptr<MCStreamer> Streamer)72 AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
73 : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this),
74 SM(*this) {}
75
getPassName() const76 StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
77
78 /// Wrapper for MCInstLowering.lowerOperand() for the
79 /// tblgen'erated pseudo lowering.
lowerOperand(const MachineOperand & MO,MCOperand & MCOp) const80 bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
81 return MCInstLowering.lowerOperand(MO, MCOp);
82 }
83
84 void EmitJumpTableInfo() override;
85 void emitJumpTableEntry(const MachineJumpTableInfo *MJTI,
86 const MachineBasicBlock *MBB, unsigned JTI);
87
88 void LowerJumpTableDestSmall(MCStreamer &OutStreamer, const MachineInstr &MI);
89
90 void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
91 const MachineInstr &MI);
92 void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
93 const MachineInstr &MI);
94
95 void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
96 void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
97 void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
98
99 void EmitSled(const MachineInstr &MI, SledKind Kind);
100
101 /// tblgen'erated driver function for lowering simple MI->MC
102 /// pseudo instructions.
103 bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
104 const MachineInstr *MI);
105
106 void EmitInstruction(const MachineInstr *MI) override;
107
getAnalysisUsage(AnalysisUsage & AU) const108 void getAnalysisUsage(AnalysisUsage &AU) const override {
109 AsmPrinter::getAnalysisUsage(AU);
110 AU.setPreservesAll();
111 }
112
runOnMachineFunction(MachineFunction & MF)113 bool runOnMachineFunction(MachineFunction &MF) override {
114 AArch64FI = MF.getInfo<AArch64FunctionInfo>();
115 STI = static_cast<const AArch64Subtarget*>(&MF.getSubtarget());
116
117 SetupMachineFunction(MF);
118
119 if (STI->isTargetCOFF()) {
120 bool Internal = MF.getFunction().hasInternalLinkage();
121 COFF::SymbolStorageClass Scl = Internal ? COFF::IMAGE_SYM_CLASS_STATIC
122 : COFF::IMAGE_SYM_CLASS_EXTERNAL;
123 int Type =
124 COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT;
125
126 OutStreamer->BeginCOFFSymbolDef(CurrentFnSym);
127 OutStreamer->EmitCOFFSymbolStorageClass(Scl);
128 OutStreamer->EmitCOFFSymbolType(Type);
129 OutStreamer->EndCOFFSymbolDef();
130 }
131
132 // Emit the rest of the function body.
133 EmitFunctionBody();
134
135 // Emit the XRay table for this function.
136 emitXRayTable();
137
138 // We didn't modify anything.
139 return false;
140 }
141
142 private:
143 void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
144 bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
145 bool printAsmRegInClass(const MachineOperand &MO,
146 const TargetRegisterClass *RC, bool isVector,
147 raw_ostream &O);
148
149 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
150 unsigned AsmVariant, const char *ExtraCode,
151 raw_ostream &O) override;
152 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
153 unsigned AsmVariant, const char *ExtraCode,
154 raw_ostream &O) override;
155
156 void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
157
158 void EmitFunctionBodyEnd() override;
159
160 MCSymbol *GetCPISymbol(unsigned CPID) const override;
161 void EmitEndOfAsmFile(Module &M) override;
162
163 AArch64FunctionInfo *AArch64FI = nullptr;
164
165 /// Emit the LOHs contained in AArch64FI.
166 void EmitLOHs();
167
168 /// Emit instruction to set float register to zero.
169 void EmitFMov0(const MachineInstr &MI);
170
171 using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
172
173 MInstToMCSymbol LOHInstToLabel;
174 };
175
176 } // end anonymous namespace
177
LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr & MI)178 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
179 {
180 EmitSled(MI, SledKind::FUNCTION_ENTER);
181 }
182
LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr & MI)183 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI)
184 {
185 EmitSled(MI, SledKind::FUNCTION_EXIT);
186 }
187
LowerPATCHABLE_TAIL_CALL(const MachineInstr & MI)188 void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI)
189 {
190 EmitSled(MI, SledKind::TAIL_CALL);
191 }
192
EmitSled(const MachineInstr & MI,SledKind Kind)193 void AArch64AsmPrinter::EmitSled(const MachineInstr &MI, SledKind Kind)
194 {
195 static const int8_t NoopsInSledCount = 7;
196 // We want to emit the following pattern:
197 //
198 // .Lxray_sled_N:
199 // ALIGN
200 // B #32
201 // ; 7 NOP instructions (28 bytes)
202 // .tmpN
203 //
204 // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
205 // over the full 32 bytes (8 instructions) with the following pattern:
206 //
207 // STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
208 // LDR W0, #12 ; W0 := function ID
209 // LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
210 // BLR X16 ; call the tracing trampoline
211 // ;DATA: 32 bits of function ID
212 // ;DATA: lower 32 bits of the address of the trampoline
213 // ;DATA: higher 32 bits of the address of the trampoline
214 // LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
215 //
216 OutStreamer->EmitCodeAlignment(4);
217 auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
218 OutStreamer->EmitLabel(CurSled);
219 auto Target = OutContext.createTempSymbol();
220
221 // Emit "B #32" instruction, which jumps over the next 28 bytes.
222 // The operand has to be the number of 4-byte instructions to jump over,
223 // including the current instruction.
224 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8));
225
226 for (int8_t I = 0; I < NoopsInSledCount; I++)
227 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
228
229 OutStreamer->EmitLabel(Target);
230 recordSled(CurSled, MI, Kind);
231 }
232
EmitEndOfAsmFile(Module & M)233 void AArch64AsmPrinter::EmitEndOfAsmFile(Module &M) {
234 const Triple &TT = TM.getTargetTriple();
235 if (TT.isOSBinFormatMachO()) {
236 // Funny Darwin hack: This flag tells the linker that no global symbols
237 // contain code that falls through to other global symbols (e.g. the obvious
238 // implementation of multiple entry points). If this doesn't occur, the
239 // linker can safely perform dead code stripping. Since LLVM never
240 // generates code that does this, it is always safe to set.
241 OutStreamer->EmitAssemblerFlag(MCAF_SubsectionsViaSymbols);
242 emitStackMaps(SM);
243 }
244 }
245
EmitLOHs()246 void AArch64AsmPrinter::EmitLOHs() {
247 SmallVector<MCSymbol *, 3> MCArgs;
248
249 for (const auto &D : AArch64FI->getLOHContainer()) {
250 for (const MachineInstr *MI : D.getArgs()) {
251 MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI);
252 assert(LabelIt != LOHInstToLabel.end() &&
253 "Label hasn't been inserted for LOH related instruction");
254 MCArgs.push_back(LabelIt->second);
255 }
256 OutStreamer->EmitLOHDirective(D.getKind(), MCArgs);
257 MCArgs.clear();
258 }
259 }
260
EmitFunctionBodyEnd()261 void AArch64AsmPrinter::EmitFunctionBodyEnd() {
262 if (!AArch64FI->getLOHRelated().empty())
263 EmitLOHs();
264 }
265
266 /// GetCPISymbol - Return the symbol for the specified constant pool entry.
GetCPISymbol(unsigned CPID) const267 MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
268 // Darwin uses a linker-private symbol name for constant-pools (to
269 // avoid addends on the relocation?), ELF has no such concept and
270 // uses a normal private symbol.
271 if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
272 return OutContext.getOrCreateSymbol(
273 Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
274 Twine(getFunctionNumber()) + "_" + Twine(CPID));
275
276 return AsmPrinter::GetCPISymbol(CPID);
277 }
278
printOperand(const MachineInstr * MI,unsigned OpNum,raw_ostream & O)279 void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
280 raw_ostream &O) {
281 const MachineOperand &MO = MI->getOperand(OpNum);
282 switch (MO.getType()) {
283 default:
284 llvm_unreachable("<unknown operand type>");
285 case MachineOperand::MO_Register: {
286 unsigned Reg = MO.getReg();
287 assert(TargetRegisterInfo::isPhysicalRegister(Reg));
288 assert(!MO.getSubReg() && "Subregs should be eliminated!");
289 O << AArch64InstPrinter::getRegisterName(Reg);
290 break;
291 }
292 case MachineOperand::MO_Immediate: {
293 int64_t Imm = MO.getImm();
294 O << '#' << Imm;
295 break;
296 }
297 case MachineOperand::MO_GlobalAddress: {
298 const GlobalValue *GV = MO.getGlobal();
299 MCSymbol *Sym = getSymbol(GV);
300
301 // FIXME: Can we get anything other than a plain symbol here?
302 assert(!MO.getTargetFlags() && "Unknown operand target flag!");
303
304 Sym->print(O, MAI);
305 printOffset(MO.getOffset(), O);
306 break;
307 }
308 case MachineOperand::MO_BlockAddress: {
309 MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
310 Sym->print(O, MAI);
311 break;
312 }
313 }
314 }
315
printAsmMRegister(const MachineOperand & MO,char Mode,raw_ostream & O)316 bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
317 raw_ostream &O) {
318 unsigned Reg = MO.getReg();
319 switch (Mode) {
320 default:
321 return true; // Unknown mode.
322 case 'w':
323 Reg = getWRegFromXReg(Reg);
324 break;
325 case 'x':
326 Reg = getXRegFromWReg(Reg);
327 break;
328 }
329
330 O << AArch64InstPrinter::getRegisterName(Reg);
331 return false;
332 }
333
334 // Prints the register in MO using class RC using the offset in the
335 // new register class. This should not be used for cross class
336 // printing.
printAsmRegInClass(const MachineOperand & MO,const TargetRegisterClass * RC,bool isVector,raw_ostream & O)337 bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
338 const TargetRegisterClass *RC,
339 bool isVector, raw_ostream &O) {
340 assert(MO.isReg() && "Should only get here with a register!");
341 const TargetRegisterInfo *RI = STI->getRegisterInfo();
342 unsigned Reg = MO.getReg();
343 unsigned RegToPrint = RC->getRegister(RI->getEncodingValue(Reg));
344 assert(RI->regsOverlap(RegToPrint, Reg));
345 O << AArch64InstPrinter::getRegisterName(
346 RegToPrint, isVector ? AArch64::vreg : AArch64::NoRegAltName);
347 return false;
348 }
349
PrintAsmOperand(const MachineInstr * MI,unsigned OpNum,unsigned AsmVariant,const char * ExtraCode,raw_ostream & O)350 bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
351 unsigned AsmVariant,
352 const char *ExtraCode, raw_ostream &O) {
353 const MachineOperand &MO = MI->getOperand(OpNum);
354
355 // First try the generic code, which knows about modifiers like 'c' and 'n'.
356 if (!AsmPrinter::PrintAsmOperand(MI, OpNum, AsmVariant, ExtraCode, O))
357 return false;
358
359 // Does this asm operand have a single letter operand modifier?
360 if (ExtraCode && ExtraCode[0]) {
361 if (ExtraCode[1] != 0)
362 return true; // Unknown modifier.
363
364 switch (ExtraCode[0]) {
365 default:
366 return true; // Unknown modifier.
367 case 'a': // Print 'a' modifier
368 PrintAsmMemoryOperand(MI, OpNum, AsmVariant, ExtraCode, O);
369 return false;
370 case 'w': // Print W register
371 case 'x': // Print X register
372 if (MO.isReg())
373 return printAsmMRegister(MO, ExtraCode[0], O);
374 if (MO.isImm() && MO.getImm() == 0) {
375 unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
376 O << AArch64InstPrinter::getRegisterName(Reg);
377 return false;
378 }
379 printOperand(MI, OpNum, O);
380 return false;
381 case 'b': // Print B register.
382 case 'h': // Print H register.
383 case 's': // Print S register.
384 case 'd': // Print D register.
385 case 'q': // Print Q register.
386 if (MO.isReg()) {
387 const TargetRegisterClass *RC;
388 switch (ExtraCode[0]) {
389 case 'b':
390 RC = &AArch64::FPR8RegClass;
391 break;
392 case 'h':
393 RC = &AArch64::FPR16RegClass;
394 break;
395 case 's':
396 RC = &AArch64::FPR32RegClass;
397 break;
398 case 'd':
399 RC = &AArch64::FPR64RegClass;
400 break;
401 case 'q':
402 RC = &AArch64::FPR128RegClass;
403 break;
404 default:
405 return true;
406 }
407 return printAsmRegInClass(MO, RC, false /* vector */, O);
408 }
409 printOperand(MI, OpNum, O);
410 return false;
411 }
412 }
413
414 // According to ARM, we should emit x and v registers unless we have a
415 // modifier.
416 if (MO.isReg()) {
417 unsigned Reg = MO.getReg();
418
419 // If this is a w or x register, print an x register.
420 if (AArch64::GPR32allRegClass.contains(Reg) ||
421 AArch64::GPR64allRegClass.contains(Reg))
422 return printAsmMRegister(MO, 'x', O);
423
424 // If this is a b, h, s, d, or q register, print it as a v register.
425 return printAsmRegInClass(MO, &AArch64::FPR128RegClass, true /* vector */,
426 O);
427 }
428
429 printOperand(MI, OpNum, O);
430 return false;
431 }
432
PrintAsmMemoryOperand(const MachineInstr * MI,unsigned OpNum,unsigned AsmVariant,const char * ExtraCode,raw_ostream & O)433 bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
434 unsigned OpNum,
435 unsigned AsmVariant,
436 const char *ExtraCode,
437 raw_ostream &O) {
438 if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
439 return true; // Unknown modifier.
440
441 const MachineOperand &MO = MI->getOperand(OpNum);
442 assert(MO.isReg() && "unexpected inline asm memory operand");
443 O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]";
444 return false;
445 }
446
PrintDebugValueComment(const MachineInstr * MI,raw_ostream & OS)447 void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
448 raw_ostream &OS) {
449 unsigned NOps = MI->getNumOperands();
450 assert(NOps == 4);
451 OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
452 // cast away const; DIetc do not take const operands for some reason.
453 OS << cast<DILocalVariable>(MI->getOperand(NOps - 2).getMetadata())
454 ->getName();
455 OS << " <- ";
456 // Frame address. Currently handles register +- offset only.
457 assert(MI->getOperand(0).isReg() && MI->getOperand(1).isImm());
458 OS << '[';
459 printOperand(MI, 0, OS);
460 OS << '+';
461 printOperand(MI, 1, OS);
462 OS << ']';
463 OS << "+";
464 printOperand(MI, NOps - 2, OS);
465 }
466
EmitJumpTableInfo()467 void AArch64AsmPrinter::EmitJumpTableInfo() {
468 const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
469 if (!MJTI) return;
470
471 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
472 if (JT.empty()) return;
473
474 const Function &F = MF->getFunction();
475 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
476 bool JTInDiffSection =
477 !STI->isTargetCOFF() ||
478 !TLOF.shouldPutJumpTableInFunctionSection(
479 MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32,
480 F);
481 if (JTInDiffSection) {
482 // Drop it in the readonly section.
483 MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(F, TM);
484 OutStreamer->SwitchSection(ReadOnlySec);
485 }
486
487 auto AFI = MF->getInfo<AArch64FunctionInfo>();
488 for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
489 const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
490
491 // If this jump table was deleted, ignore it.
492 if (JTBBs.empty()) continue;
493
494 unsigned Size = AFI->getJumpTableEntrySize(JTI);
495 EmitAlignment(Log2_32(Size));
496 OutStreamer->EmitLabel(GetJTISymbol(JTI));
497
498 for (auto *JTBB : JTBBs)
499 emitJumpTableEntry(MJTI, JTBB, JTI);
500 }
501 }
502
emitJumpTableEntry(const MachineJumpTableInfo * MJTI,const MachineBasicBlock * MBB,unsigned JTI)503 void AArch64AsmPrinter::emitJumpTableEntry(const MachineJumpTableInfo *MJTI,
504 const MachineBasicBlock *MBB,
505 unsigned JTI) {
506 const MCExpr *Value = MCSymbolRefExpr::create(MBB->getSymbol(), OutContext);
507 auto AFI = MF->getInfo<AArch64FunctionInfo>();
508 unsigned Size = AFI->getJumpTableEntrySize(JTI);
509
510 if (Size == 4) {
511 // .word LBB - LJTI
512 const TargetLowering *TLI = MF->getSubtarget().getTargetLowering();
513 const MCExpr *Base = TLI->getPICJumpTableRelocBaseExpr(MF, JTI, OutContext);
514 Value = MCBinaryExpr::createSub(Value, Base, OutContext);
515 } else {
516 // .byte (LBB - LBB) >> 2 (or .hword)
517 const MCSymbol *BaseSym = AFI->getJumpTableEntryPCRelSymbol(JTI);
518 const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext);
519 Value = MCBinaryExpr::createSub(Value, Base, OutContext);
520 Value = MCBinaryExpr::createLShr(
521 Value, MCConstantExpr::create(2, OutContext), OutContext);
522 }
523
524 OutStreamer->EmitValue(Value, Size);
525 }
526
527 /// Small jump tables contain an unsigned byte or half, representing the offset
528 /// from the lowest-addressed possible destination to the desired basic
529 /// block. Since all instructions are 4-byte aligned, this is further compressed
530 /// by counting in instructions rather than bytes (i.e. divided by 4). So, to
531 /// materialize the correct destination we need:
532 ///
533 /// adr xDest, .LBB0_0
534 /// ldrb wScratch, [xTable, xEntry] (with "lsl #1" for ldrh).
535 /// add xDest, xDest, xScratch, lsl #2
LowerJumpTableDestSmall(llvm::MCStreamer & OutStreamer,const llvm::MachineInstr & MI)536 void AArch64AsmPrinter::LowerJumpTableDestSmall(llvm::MCStreamer &OutStreamer,
537 const llvm::MachineInstr &MI) {
538 unsigned DestReg = MI.getOperand(0).getReg();
539 unsigned ScratchReg = MI.getOperand(1).getReg();
540 unsigned ScratchRegW =
541 STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32);
542 unsigned TableReg = MI.getOperand(2).getReg();
543 unsigned EntryReg = MI.getOperand(3).getReg();
544 int JTIdx = MI.getOperand(4).getIndex();
545 bool IsByteEntry = MI.getOpcode() == AArch64::JumpTableDest8;
546
547 // This has to be first because the compression pass based its reachability
548 // calculations on the start of the JumpTableDest instruction.
549 auto Label =
550 MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx);
551 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR)
552 .addReg(DestReg)
553 .addExpr(MCSymbolRefExpr::create(
554 Label, MF->getContext())));
555
556 // Load the number of instruction-steps to offset from the label.
557 unsigned LdrOpcode = IsByteEntry ? AArch64::LDRBBroX : AArch64::LDRHHroX;
558 EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode)
559 .addReg(ScratchRegW)
560 .addReg(TableReg)
561 .addReg(EntryReg)
562 .addImm(0)
563 .addImm(IsByteEntry ? 0 : 1));
564
565 // Multiply the steps by 4 and add to the already materialized base label
566 // address.
567 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs)
568 .addReg(DestReg)
569 .addReg(DestReg)
570 .addReg(ScratchReg)
571 .addImm(2));
572 }
573
LowerSTACKMAP(MCStreamer & OutStreamer,StackMaps & SM,const MachineInstr & MI)574 void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
575 const MachineInstr &MI) {
576 unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
577
578 SM.recordStackMap(MI);
579 assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
580
581 // Scan ahead to trim the shadow.
582 const MachineBasicBlock &MBB = *MI.getParent();
583 MachineBasicBlock::const_iterator MII(MI);
584 ++MII;
585 while (NumNOPBytes > 0) {
586 if (MII == MBB.end() || MII->isCall() ||
587 MII->getOpcode() == AArch64::DBG_VALUE ||
588 MII->getOpcode() == TargetOpcode::PATCHPOINT ||
589 MII->getOpcode() == TargetOpcode::STACKMAP)
590 break;
591 ++MII;
592 NumNOPBytes -= 4;
593 }
594
595 // Emit nops.
596 for (unsigned i = 0; i < NumNOPBytes; i += 4)
597 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
598 }
599
600 // Lower a patchpoint of the form:
601 // [<def>], <id>, <numBytes>, <target>, <numArgs>
LowerPATCHPOINT(MCStreamer & OutStreamer,StackMaps & SM,const MachineInstr & MI)602 void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
603 const MachineInstr &MI) {
604 SM.recordPatchPoint(MI);
605
606 PatchPointOpers Opers(&MI);
607
608 int64_t CallTarget = Opers.getCallTarget().getImm();
609 unsigned EncodedBytes = 0;
610 if (CallTarget) {
611 assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
612 "High 16 bits of call target should be zero.");
613 unsigned ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
614 EncodedBytes = 16;
615 // Materialize the jump address:
616 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi)
617 .addReg(ScratchReg)
618 .addImm((CallTarget >> 32) & 0xFFFF)
619 .addImm(32));
620 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
621 .addReg(ScratchReg)
622 .addReg(ScratchReg)
623 .addImm((CallTarget >> 16) & 0xFFFF)
624 .addImm(16));
625 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
626 .addReg(ScratchReg)
627 .addReg(ScratchReg)
628 .addImm(CallTarget & 0xFFFF)
629 .addImm(0));
630 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
631 }
632 // Emit padding.
633 unsigned NumBytes = Opers.getNumPatchBytes();
634 assert(NumBytes >= EncodedBytes &&
635 "Patchpoint can't request size less than the length of a call.");
636 assert((NumBytes - EncodedBytes) % 4 == 0 &&
637 "Invalid number of NOP bytes requested!");
638 for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
639 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
640 }
641
EmitFMov0(const MachineInstr & MI)642 void AArch64AsmPrinter::EmitFMov0(const MachineInstr &MI) {
643 unsigned DestReg = MI.getOperand(0).getReg();
644 if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround()) {
645 // Convert H/S/D register to corresponding Q register
646 if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31)
647 DestReg = AArch64::Q0 + (DestReg - AArch64::H0);
648 else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31)
649 DestReg = AArch64::Q0 + (DestReg - AArch64::S0);
650 else {
651 assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31);
652 DestReg = AArch64::Q0 + (DestReg - AArch64::D0);
653 }
654 MCInst MOVI;
655 MOVI.setOpcode(AArch64::MOVIv2d_ns);
656 MOVI.addOperand(MCOperand::createReg(DestReg));
657 MOVI.addOperand(MCOperand::createImm(0));
658 EmitToStreamer(*OutStreamer, MOVI);
659 } else {
660 MCInst FMov;
661 switch (MI.getOpcode()) {
662 default: llvm_unreachable("Unexpected opcode");
663 case AArch64::FMOVH0:
664 FMov.setOpcode(AArch64::FMOVWHr);
665 FMov.addOperand(MCOperand::createReg(DestReg));
666 FMov.addOperand(MCOperand::createReg(AArch64::WZR));
667 break;
668 case AArch64::FMOVS0:
669 FMov.setOpcode(AArch64::FMOVWSr);
670 FMov.addOperand(MCOperand::createReg(DestReg));
671 FMov.addOperand(MCOperand::createReg(AArch64::WZR));
672 break;
673 case AArch64::FMOVD0:
674 FMov.setOpcode(AArch64::FMOVXDr);
675 FMov.addOperand(MCOperand::createReg(DestReg));
676 FMov.addOperand(MCOperand::createReg(AArch64::XZR));
677 break;
678 }
679 EmitToStreamer(*OutStreamer, FMov);
680 }
681 }
682
683 // Simple pseudo-instructions have their lowering (with expansion to real
684 // instructions) auto-generated.
685 #include "AArch64GenMCPseudoLowering.inc"
686
EmitInstruction(const MachineInstr * MI)687 void AArch64AsmPrinter::EmitInstruction(const MachineInstr *MI) {
688 // Do any auto-generated pseudo lowerings.
689 if (emitPseudoExpansionLowering(*OutStreamer, MI))
690 return;
691
692 if (AArch64FI->getLOHRelated().count(MI)) {
693 // Generate a label for LOH related instruction
694 MCSymbol *LOHLabel = createTempSymbol("loh");
695 // Associate the instruction with the label
696 LOHInstToLabel[MI] = LOHLabel;
697 OutStreamer->EmitLabel(LOHLabel);
698 }
699
700 AArch64TargetStreamer *TS =
701 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
702 // Do any manual lowerings.
703 switch (MI->getOpcode()) {
704 default:
705 break;
706 case AArch64::MOVMCSym: {
707 unsigned DestReg = MI->getOperand(0).getReg();
708 const MachineOperand &MO_Sym = MI->getOperand(1);
709 MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
710 MCOperand Hi_MCSym, Lo_MCSym;
711
712 Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
713 Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
714
715 MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym);
716 MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym);
717
718 MCInst MovZ;
719 MovZ.setOpcode(AArch64::MOVZXi);
720 MovZ.addOperand(MCOperand::createReg(DestReg));
721 MovZ.addOperand(Hi_MCSym);
722 MovZ.addOperand(MCOperand::createImm(16));
723 EmitToStreamer(*OutStreamer, MovZ);
724
725 MCInst MovK;
726 MovK.setOpcode(AArch64::MOVKXi);
727 MovK.addOperand(MCOperand::createReg(DestReg));
728 MovK.addOperand(MCOperand::createReg(DestReg));
729 MovK.addOperand(Lo_MCSym);
730 MovK.addOperand(MCOperand::createImm(0));
731 EmitToStreamer(*OutStreamer, MovK);
732 return;
733 }
734 case AArch64::MOVIv2d_ns:
735 // If the target has <rdar://problem/16473581>, lower this
736 // instruction to movi.16b instead.
737 if (STI->hasZeroCycleZeroingFPWorkaround() &&
738 MI->getOperand(1).getImm() == 0) {
739 MCInst TmpInst;
740 TmpInst.setOpcode(AArch64::MOVIv16b_ns);
741 TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
742 TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm()));
743 EmitToStreamer(*OutStreamer, TmpInst);
744 return;
745 }
746 break;
747
748 case AArch64::DBG_VALUE: {
749 if (isVerbose() && OutStreamer->hasRawTextSupport()) {
750 SmallString<128> TmpStr;
751 raw_svector_ostream OS(TmpStr);
752 PrintDebugValueComment(MI, OS);
753 OutStreamer->EmitRawText(StringRef(OS.str()));
754 }
755 return;
756
757 case AArch64::EMITBKEY: {
758 ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
759 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
760 ExceptionHandlingType != ExceptionHandling::ARM)
761 return;
762
763 if (needsCFIMoves() == CFI_M_None)
764 return;
765
766 OutStreamer->EmitCFIBKeyFrame();
767 return;
768 }
769 }
770
771 // Tail calls use pseudo instructions so they have the proper code-gen
772 // attributes (isCall, isReturn, etc.). We lower them to the real
773 // instruction here.
774 case AArch64::TCRETURNri:
775 case AArch64::TCRETURNriBTI:
776 case AArch64::TCRETURNriALL: {
777 MCInst TmpInst;
778 TmpInst.setOpcode(AArch64::BR);
779 TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
780 EmitToStreamer(*OutStreamer, TmpInst);
781 return;
782 }
783 case AArch64::TCRETURNdi: {
784 MCOperand Dest;
785 MCInstLowering.lowerOperand(MI->getOperand(0), Dest);
786 MCInst TmpInst;
787 TmpInst.setOpcode(AArch64::B);
788 TmpInst.addOperand(Dest);
789 EmitToStreamer(*OutStreamer, TmpInst);
790 return;
791 }
792 case AArch64::TLSDESC_CALLSEQ: {
793 /// lower this to:
794 /// adrp x0, :tlsdesc:var
795 /// ldr x1, [x0, #:tlsdesc_lo12:var]
796 /// add x0, x0, #:tlsdesc_lo12:var
797 /// .tlsdesccall var
798 /// blr x1
799 /// (TPIDR_EL0 offset now in x0)
800 const MachineOperand &MO_Sym = MI->getOperand(0);
801 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
802 MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
803 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
804 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
805 MCInstLowering.lowerOperand(MO_Sym, Sym);
806 MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12);
807 MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc);
808
809 MCInst Adrp;
810 Adrp.setOpcode(AArch64::ADRP);
811 Adrp.addOperand(MCOperand::createReg(AArch64::X0));
812 Adrp.addOperand(SymTLSDesc);
813 EmitToStreamer(*OutStreamer, Adrp);
814
815 MCInst Ldr;
816 Ldr.setOpcode(AArch64::LDRXui);
817 Ldr.addOperand(MCOperand::createReg(AArch64::X1));
818 Ldr.addOperand(MCOperand::createReg(AArch64::X0));
819 Ldr.addOperand(SymTLSDescLo12);
820 Ldr.addOperand(MCOperand::createImm(0));
821 EmitToStreamer(*OutStreamer, Ldr);
822
823 MCInst Add;
824 Add.setOpcode(AArch64::ADDXri);
825 Add.addOperand(MCOperand::createReg(AArch64::X0));
826 Add.addOperand(MCOperand::createReg(AArch64::X0));
827 Add.addOperand(SymTLSDescLo12);
828 Add.addOperand(MCOperand::createImm(AArch64_AM::getShiftValue(0)));
829 EmitToStreamer(*OutStreamer, Add);
830
831 // Emit a relocation-annotation. This expands to no code, but requests
832 // the following instruction gets an R_AARCH64_TLSDESC_CALL.
833 MCInst TLSDescCall;
834 TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
835 TLSDescCall.addOperand(Sym);
836 EmitToStreamer(*OutStreamer, TLSDescCall);
837
838 MCInst Blr;
839 Blr.setOpcode(AArch64::BLR);
840 Blr.addOperand(MCOperand::createReg(AArch64::X1));
841 EmitToStreamer(*OutStreamer, Blr);
842
843 return;
844 }
845
846 case AArch64::JumpTableDest32: {
847 // We want:
848 // ldrsw xScratch, [xTable, xEntry, lsl #2]
849 // add xDest, xTable, xScratch
850 unsigned DestReg = MI->getOperand(0).getReg(),
851 ScratchReg = MI->getOperand(1).getReg(),
852 TableReg = MI->getOperand(2).getReg(),
853 EntryReg = MI->getOperand(3).getReg();
854 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDRSWroX)
855 .addReg(ScratchReg)
856 .addReg(TableReg)
857 .addReg(EntryReg)
858 .addImm(0)
859 .addImm(1));
860 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADDXrs)
861 .addReg(DestReg)
862 .addReg(TableReg)
863 .addReg(ScratchReg)
864 .addImm(0));
865 return;
866 }
867 case AArch64::JumpTableDest16:
868 case AArch64::JumpTableDest8:
869 LowerJumpTableDestSmall(*OutStreamer, *MI);
870 return;
871
872 case AArch64::FMOVH0:
873 case AArch64::FMOVS0:
874 case AArch64::FMOVD0:
875 EmitFMov0(*MI);
876 return;
877
878 case TargetOpcode::STACKMAP:
879 return LowerSTACKMAP(*OutStreamer, SM, *MI);
880
881 case TargetOpcode::PATCHPOINT:
882 return LowerPATCHPOINT(*OutStreamer, SM, *MI);
883
884 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
885 LowerPATCHABLE_FUNCTION_ENTER(*MI);
886 return;
887
888 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
889 LowerPATCHABLE_FUNCTION_EXIT(*MI);
890 return;
891
892 case TargetOpcode::PATCHABLE_TAIL_CALL:
893 LowerPATCHABLE_TAIL_CALL(*MI);
894 return;
895
896 case AArch64::SEH_StackAlloc:
897 TS->EmitARM64WinCFIAllocStack(MI->getOperand(0).getImm());
898 return;
899
900 case AArch64::SEH_SaveFPLR:
901 TS->EmitARM64WinCFISaveFPLR(MI->getOperand(0).getImm());
902 return;
903
904 case AArch64::SEH_SaveFPLR_X:
905 assert(MI->getOperand(0).getImm() < 0 &&
906 "Pre increment SEH opcode must have a negative offset");
907 TS->EmitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm());
908 return;
909
910 case AArch64::SEH_SaveReg:
911 TS->EmitARM64WinCFISaveReg(MI->getOperand(0).getImm(),
912 MI->getOperand(1).getImm());
913 return;
914
915 case AArch64::SEH_SaveReg_X:
916 assert(MI->getOperand(1).getImm() < 0 &&
917 "Pre increment SEH opcode must have a negative offset");
918 TS->EmitARM64WinCFISaveRegX(MI->getOperand(0).getImm(),
919 -MI->getOperand(1).getImm());
920 return;
921
922 case AArch64::SEH_SaveRegP:
923 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
924 "Non-consecutive registers not allowed for save_regp");
925 TS->EmitARM64WinCFISaveRegP(MI->getOperand(0).getImm(),
926 MI->getOperand(2).getImm());
927 return;
928
929 case AArch64::SEH_SaveRegP_X:
930 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
931 "Non-consecutive registers not allowed for save_regp_x");
932 assert(MI->getOperand(2).getImm() < 0 &&
933 "Pre increment SEH opcode must have a negative offset");
934 TS->EmitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(),
935 -MI->getOperand(2).getImm());
936 return;
937
938 case AArch64::SEH_SaveFReg:
939 TS->EmitARM64WinCFISaveFReg(MI->getOperand(0).getImm(),
940 MI->getOperand(1).getImm());
941 return;
942
943 case AArch64::SEH_SaveFReg_X:
944 assert(MI->getOperand(1).getImm() < 0 &&
945 "Pre increment SEH opcode must have a negative offset");
946 TS->EmitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(),
947 -MI->getOperand(1).getImm());
948 return;
949
950 case AArch64::SEH_SaveFRegP:
951 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
952 "Non-consecutive registers not allowed for save_regp");
953 TS->EmitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(),
954 MI->getOperand(2).getImm());
955 return;
956
957 case AArch64::SEH_SaveFRegP_X:
958 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
959 "Non-consecutive registers not allowed for save_regp_x");
960 assert(MI->getOperand(2).getImm() < 0 &&
961 "Pre increment SEH opcode must have a negative offset");
962 TS->EmitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(),
963 -MI->getOperand(2).getImm());
964 return;
965
966 case AArch64::SEH_SetFP:
967 TS->EmitARM64WinCFISetFP();
968 return;
969
970 case AArch64::SEH_AddFP:
971 TS->EmitARM64WinCFIAddFP(MI->getOperand(0).getImm());
972 return;
973
974 case AArch64::SEH_Nop:
975 TS->EmitARM64WinCFINop();
976 return;
977
978 case AArch64::SEH_PrologEnd:
979 TS->EmitARM64WinCFIPrologEnd();
980 return;
981
982 case AArch64::SEH_EpilogStart:
983 TS->EmitARM64WinCFIEpilogStart();
984 return;
985
986 case AArch64::SEH_EpilogEnd:
987 TS->EmitARM64WinCFIEpilogEnd();
988 return;
989 }
990
991 // Finally, do the automated lowerings for everything else.
992 MCInst TmpInst;
993 MCInstLowering.Lower(MI, TmpInst);
994 EmitToStreamer(*OutStreamer, TmpInst);
995 }
996
997 // Force static initialization.
LLVMInitializeAArch64AsmPrinter()998 extern "C" void LLVMInitializeAArch64AsmPrinter() {
999 RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
1000 RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget());
1001 RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target());
1002 }
1003