1 //===----- ELF_aarch64.cpp - JIT linker implementation for ELF/aarch64 ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // ELF/aarch64 jit-link implementation.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/ExecutionEngine/JITLink/ELF_aarch64.h"
14 #include "EHFrameSupportImpl.h"
15 #include "ELFLinkGraphBuilder.h"
16 #include "JITLinkGeneric.h"
17 #include "llvm/BinaryFormat/ELF.h"
18 #include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h"
19 #include "llvm/ExecutionEngine/JITLink/aarch64.h"
20 #include "llvm/Object/ELFObjectFile.h"
21 #include "llvm/Support/Endian.h"
22 #include "llvm/Support/MathExtras.h"
23 
24 #define DEBUG_TYPE "jitlink"
25 
26 using namespace llvm;
27 using namespace llvm::jitlink;
28 
29 namespace {
30 
31 class ELFJITLinker_aarch64 : public JITLinker<ELFJITLinker_aarch64> {
32   friend class JITLinker<ELFJITLinker_aarch64>;
33 
34 public:
35   ELFJITLinker_aarch64(std::unique_ptr<JITLinkContext> Ctx,
36                        std::unique_ptr<LinkGraph> G,
37                        PassConfiguration PassConfig)
38       : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {}
39 
40 private:
41   Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const {
42     return aarch64::applyFixup(G, B, E);
43   }
44 };
45 
46 template <typename ELFT>
47 class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> {
48 private:
49   enum ELFAArch64RelocationKind : Edge::Kind {
50     ELFCall26 = Edge::FirstRelocation,
51     ELFAdrPage21,
52     ELFAddAbs12,
53     ELFLdSt8Abs12,
54     ELFLdSt16Abs12,
55     ELFLdSt32Abs12,
56     ELFLdSt64Abs12,
57     ELFLdSt128Abs12,
58     ELFMovwAbsG0,
59     ELFMovwAbsG1,
60     ELFMovwAbsG2,
61     ELFMovwAbsG3,
62     ELFAbs64,
63     ELFPrel32,
64     ELFPrel64,
65     ELFAdrGOTPage21,
66     ELFLd64GOTLo12,
67   };
68 
69   static Expected<ELFAArch64RelocationKind>
70   getRelocationKind(const uint32_t Type) {
71     using namespace aarch64;
72     switch (Type) {
73     case ELF::R_AARCH64_CALL26:
74     case ELF::R_AARCH64_JUMP26:
75       return ELFCall26;
76     case ELF::R_AARCH64_ADR_PREL_PG_HI21:
77       return ELFAdrPage21;
78     case ELF::R_AARCH64_ADD_ABS_LO12_NC:
79       return ELFAddAbs12;
80     case ELF::R_AARCH64_LDST8_ABS_LO12_NC:
81       return ELFLdSt8Abs12;
82     case ELF::R_AARCH64_LDST16_ABS_LO12_NC:
83       return ELFLdSt16Abs12;
84     case ELF::R_AARCH64_LDST32_ABS_LO12_NC:
85       return ELFLdSt32Abs12;
86     case ELF::R_AARCH64_LDST64_ABS_LO12_NC:
87       return ELFLdSt64Abs12;
88     case ELF::R_AARCH64_LDST128_ABS_LO12_NC:
89       return ELFLdSt128Abs12;
90     case ELF::R_AARCH64_MOVW_UABS_G0_NC:
91       return ELFMovwAbsG0;
92     case ELF::R_AARCH64_MOVW_UABS_G1_NC:
93       return ELFMovwAbsG1;
94     case ELF::R_AARCH64_MOVW_UABS_G2_NC:
95       return ELFMovwAbsG2;
96     case ELF::R_AARCH64_MOVW_UABS_G3:
97       return ELFMovwAbsG3;
98     case ELF::R_AARCH64_ABS64:
99       return ELFAbs64;
100     case ELF::R_AARCH64_PREL32:
101       return ELFPrel32;
102     case ELF::R_AARCH64_PREL64:
103       return ELFPrel64;
104     case ELF::R_AARCH64_ADR_GOT_PAGE:
105       return ELFAdrGOTPage21;
106     case ELF::R_AARCH64_LD64_GOT_LO12_NC:
107       return ELFLd64GOTLo12;
108     }
109 
110     return make_error<JITLinkError>(
111         "Unsupported aarch64 relocation:" + formatv("{0:d}: ", Type) +
112         object::getELFRelocationTypeName(ELF::EM_AARCH64, Type));
113   }
114 
115   Error addRelocations() override {
116     LLVM_DEBUG(dbgs() << "Processing relocations:\n");
117 
118     using Base = ELFLinkGraphBuilder<ELFT>;
119     using Self = ELFLinkGraphBuilder_aarch64<ELFT>;
120     for (const auto &RelSect : Base::Sections)
121       if (Error Err = Base::forEachRelocation(RelSect, this,
122                                               &Self::addSingleRelocation))
123         return Err;
124 
125     return Error::success();
126   }
127 
128   Error addSingleRelocation(const typename ELFT::Rela &Rel,
129                             const typename ELFT::Shdr &FixupSect,
130                             Block &BlockToFix) {
131     using support::ulittle32_t;
132     using Base = ELFLinkGraphBuilder<ELFT>;
133 
134     uint32_t SymbolIndex = Rel.getSymbol(false);
135     auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec);
136     if (!ObjSymbol)
137       return ObjSymbol.takeError();
138 
139     Symbol *GraphSymbol = Base::getGraphSymbol(SymbolIndex);
140     if (!GraphSymbol)
141       return make_error<StringError>(
142           formatv("Could not find symbol at given index, did you add it to "
143                   "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}",
144                   SymbolIndex, (*ObjSymbol)->st_shndx,
145                   Base::GraphSymbols.size()),
146           inconvertibleErrorCode());
147 
148     uint32_t Type = Rel.getType(false);
149     Expected<ELFAArch64RelocationKind> RelocKind = getRelocationKind(Type);
150     if (!RelocKind)
151       return RelocKind.takeError();
152 
153     int64_t Addend = Rel.r_addend;
154     orc::ExecutorAddr FixupAddress =
155         orc::ExecutorAddr(FixupSect.sh_addr) + Rel.r_offset;
156     Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress();
157 
158     // Get a pointer to the fixup content.
159     const void *FixupContent = BlockToFix.getContent().data() +
160                                (FixupAddress - BlockToFix.getAddress());
161 
162     Edge::Kind Kind = Edge::Invalid;
163 
164     switch (*RelocKind) {
165     case ELFCall26: {
166       Kind = aarch64::Branch26;
167       break;
168     }
169     case ELFAdrPage21: {
170       Kind = aarch64::Page21;
171       break;
172     }
173     case ELFAddAbs12: {
174       Kind = aarch64::PageOffset12;
175       break;
176     }
177     case ELFLdSt8Abs12: {
178       uint32_t Instr = *(const ulittle32_t *)FixupContent;
179       if (!aarch64::isLoadStoreImm12(Instr) ||
180           aarch64::getPageOffset12Shift(Instr) != 0)
181         return make_error<JITLinkError>(
182             "R_AARCH64_LDST8_ABS_LO12_NC target is not a "
183             "LDRB/STRB (imm12) instruction");
184 
185       Kind = aarch64::PageOffset12;
186       break;
187     }
188     case ELFLdSt16Abs12: {
189       uint32_t Instr = *(const ulittle32_t *)FixupContent;
190       if (!aarch64::isLoadStoreImm12(Instr) ||
191           aarch64::getPageOffset12Shift(Instr) != 1)
192         return make_error<JITLinkError>(
193             "R_AARCH64_LDST16_ABS_LO12_NC target is not a "
194             "LDRH/STRH (imm12) instruction");
195 
196       Kind = aarch64::PageOffset12;
197       break;
198     }
199     case ELFLdSt32Abs12: {
200       uint32_t Instr = *(const ulittle32_t *)FixupContent;
201       if (!aarch64::isLoadStoreImm12(Instr) ||
202           aarch64::getPageOffset12Shift(Instr) != 2)
203         return make_error<JITLinkError>(
204             "R_AARCH64_LDST32_ABS_LO12_NC target is not a "
205             "LDR/STR (imm12, 32 bit) instruction");
206 
207       Kind = aarch64::PageOffset12;
208       break;
209     }
210     case ELFLdSt64Abs12: {
211       uint32_t Instr = *(const ulittle32_t *)FixupContent;
212       if (!aarch64::isLoadStoreImm12(Instr) ||
213           aarch64::getPageOffset12Shift(Instr) != 3)
214         return make_error<JITLinkError>(
215             "R_AARCH64_LDST64_ABS_LO12_NC target is not a "
216             "LDR/STR (imm12, 64 bit) instruction");
217 
218       Kind = aarch64::PageOffset12;
219       break;
220     }
221     case ELFLdSt128Abs12: {
222       uint32_t Instr = *(const ulittle32_t *)FixupContent;
223       if (!aarch64::isLoadStoreImm12(Instr) ||
224           aarch64::getPageOffset12Shift(Instr) != 4)
225         return make_error<JITLinkError>(
226             "R_AARCH64_LDST128_ABS_LO12_NC target is not a "
227             "LDR/STR (imm12, 128 bit) instruction");
228 
229       Kind = aarch64::PageOffset12;
230       break;
231     }
232     case ELFMovwAbsG0: {
233       uint32_t Instr = *(const ulittle32_t *)FixupContent;
234       if (!aarch64::isMoveWideImm16(Instr) ||
235           aarch64::getMoveWide16Shift(Instr) != 0)
236         return make_error<JITLinkError>(
237             "R_AARCH64_MOVW_UABS_G0_NC target is not a "
238             "MOVK/MOVZ (imm16, LSL #0) instruction");
239 
240       Kind = aarch64::MoveWide16;
241       break;
242     }
243     case ELFMovwAbsG1: {
244       uint32_t Instr = *(const ulittle32_t *)FixupContent;
245       if (!aarch64::isMoveWideImm16(Instr) ||
246           aarch64::getMoveWide16Shift(Instr) != 16)
247         return make_error<JITLinkError>(
248             "R_AARCH64_MOVW_UABS_G1_NC target is not a "
249             "MOVK/MOVZ (imm16, LSL #16) instruction");
250 
251       Kind = aarch64::MoveWide16;
252       break;
253     }
254     case ELFMovwAbsG2: {
255       uint32_t Instr = *(const ulittle32_t *)FixupContent;
256       if (!aarch64::isMoveWideImm16(Instr) ||
257           aarch64::getMoveWide16Shift(Instr) != 32)
258         return make_error<JITLinkError>(
259             "R_AARCH64_MOVW_UABS_G2_NC target is not a "
260             "MOVK/MOVZ (imm16, LSL #32) instruction");
261 
262       Kind = aarch64::MoveWide16;
263       break;
264     }
265     case ELFMovwAbsG3: {
266       uint32_t Instr = *(const ulittle32_t *)FixupContent;
267       if (!aarch64::isMoveWideImm16(Instr) ||
268           aarch64::getMoveWide16Shift(Instr) != 48)
269         return make_error<JITLinkError>(
270             "R_AARCH64_MOVW_UABS_G3 target is not a "
271             "MOVK/MOVZ (imm16, LSL #48) instruction");
272 
273       Kind = aarch64::MoveWide16;
274       break;
275     }
276     case ELFAbs64: {
277       Kind = aarch64::Pointer64;
278       break;
279     }
280     case ELFPrel32: {
281       Kind = aarch64::Delta32;
282       break;
283     }
284     case ELFPrel64: {
285       Kind = aarch64::Delta64;
286       break;
287     }
288     case ELFAdrGOTPage21: {
289       Kind = aarch64::GOTPage21;
290       break;
291     }
292     case ELFLd64GOTLo12: {
293       Kind = aarch64::GOTPageOffset12;
294       break;
295     }
296     };
297 
298     Edge GE(Kind, Offset, *GraphSymbol, Addend);
299     LLVM_DEBUG({
300       dbgs() << "    ";
301       printEdge(dbgs(), BlockToFix, GE, aarch64::getEdgeKindName(Kind));
302       dbgs() << "\n";
303     });
304 
305     BlockToFix.addEdge(std::move(GE));
306     return Error::success();
307   }
308 
309   /// Return the string name of the given ELF aarch64 edge kind.
310   const char *getELFAArch64RelocationKindName(Edge::Kind R) {
311     switch (R) {
312     case ELFCall26:
313       return "ELFCall26";
314     case ELFAdrPage21:
315       return "ELFAdrPage21";
316     case ELFAddAbs12:
317       return "ELFAddAbs12";
318     case ELFLdSt8Abs12:
319       return "ELFLdSt8Abs12";
320     case ELFLdSt16Abs12:
321       return "ELFLdSt16Abs12";
322     case ELFLdSt32Abs12:
323       return "ELFLdSt32Abs12";
324     case ELFLdSt64Abs12:
325       return "ELFLdSt64Abs12";
326     case ELFLdSt128Abs12:
327       return "ELFLdSt128Abs12";
328     case ELFMovwAbsG0:
329       return "ELFMovwAbsG0";
330     case ELFMovwAbsG1:
331       return "ELFMovwAbsG1";
332     case ELFMovwAbsG2:
333       return "ELFMovwAbsG2";
334     case ELFMovwAbsG3:
335       return "ELFMovwAbsG3";
336     case ELFAbs64:
337       return "ELFAbs64";
338     case ELFPrel32:
339       return "ELFPrel32";
340     case ELFPrel64:
341       return "ELFPrel64";
342     case ELFAdrGOTPage21:
343       return "ELFAdrGOTPage21";
344     case ELFLd64GOTLo12:
345       return "ELFLd64GOTLo12";
346     default:
347       return getGenericEdgeKindName(static_cast<Edge::Kind>(R));
348     }
349   }
350 
351 public:
352   ELFLinkGraphBuilder_aarch64(StringRef FileName,
353                               const object::ELFFile<ELFT> &Obj, const Triple T)
354       : ELFLinkGraphBuilder<ELFT>(Obj, std::move(T), FileName,
355                                   aarch64::getEdgeKindName) {}
356 };
357 
358 Error buildTables_ELF_aarch64(LinkGraph &G) {
359   LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n");
360 
361   aarch64::GOTTableManager GOT;
362   aarch64::PLTTableManager PLT(GOT);
363   visitExistingEdges(G, GOT, PLT);
364   return Error::success();
365 }
366 
367 } // namespace
368 
369 namespace llvm {
370 namespace jitlink {
371 
372 Expected<std::unique_ptr<LinkGraph>>
373 createLinkGraphFromELFObject_aarch64(MemoryBufferRef ObjectBuffer) {
374   LLVM_DEBUG({
375     dbgs() << "Building jitlink graph for new input "
376            << ObjectBuffer.getBufferIdentifier() << "...\n";
377   });
378 
379   auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer);
380   if (!ELFObj)
381     return ELFObj.takeError();
382 
383   assert((*ELFObj)->getArch() == Triple::aarch64 &&
384          "Only AArch64 (little endian) is supported for now");
385 
386   auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj);
387   return ELFLinkGraphBuilder_aarch64<object::ELF64LE>((*ELFObj)->getFileName(),
388                                                       ELFObjFile.getELFFile(),
389                                                       (*ELFObj)->makeTriple())
390       .buildGraph();
391 }
392 
393 void link_ELF_aarch64(std::unique_ptr<LinkGraph> G,
394                       std::unique_ptr<JITLinkContext> Ctx) {
395   PassConfiguration Config;
396   const Triple &TT = G->getTargetTriple();
397   if (Ctx->shouldAddDefaultTargetPasses(TT)) {
398     // Add eh-frame passses.
399     Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame"));
400     Config.PrePrunePasses.push_back(EHFrameEdgeFixer(
401         ".eh_frame", 8, aarch64::Pointer32, aarch64::Pointer64,
402         aarch64::Delta32, aarch64::Delta64, aarch64::NegDelta32));
403 
404     // Add a mark-live pass.
405     if (auto MarkLive = Ctx->getMarkLivePass(TT))
406       Config.PrePrunePasses.push_back(std::move(MarkLive));
407     else
408       Config.PrePrunePasses.push_back(markAllSymbolsLive);
409 
410     // Add an in-place GOT/Stubs build pass.
411     Config.PostPrunePasses.push_back(buildTables_ELF_aarch64);
412   }
413 
414   if (auto Err = Ctx->modifyPassConfig(*G, Config))
415     return Ctx->notifyFailed(std::move(Err));
416 
417   ELFJITLinker_aarch64::link(std::move(Ctx), std::move(G), std::move(Config));
418 }
419 
420 } // namespace jitlink
421 } // namespace llvm
422