1 //===----- ELF_aarch64.cpp - JIT linker implementation for ELF/aarch64 ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // ELF/aarch64 jit-link implementation.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/ExecutionEngine/JITLink/ELF_aarch64.h"
14 #include "EHFrameSupportImpl.h"
15 #include "ELFLinkGraphBuilder.h"
16 #include "JITLinkGeneric.h"
17 #include "llvm/BinaryFormat/ELF.h"
18 #include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h"
19 #include "llvm/ExecutionEngine/JITLink/aarch64.h"
20 #include "llvm/Object/ELFObjectFile.h"
21 #include "llvm/Support/Endian.h"
22 #include "llvm/Support/MathExtras.h"
23 
24 #include "PerGraphGOTAndPLTStubsBuilder.h"
25 
26 #define DEBUG_TYPE "jitlink"
27 
28 using namespace llvm;
29 using namespace llvm::jitlink;
30 
31 namespace llvm {
32 namespace jitlink {
33 
34 class ELFJITLinker_aarch64 : public JITLinker<ELFJITLinker_aarch64> {
35   friend class JITLinker<ELFJITLinker_aarch64>;
36 
37 public:
38   ELFJITLinker_aarch64(std::unique_ptr<JITLinkContext> Ctx,
39                        std::unique_ptr<LinkGraph> G,
40                        PassConfiguration PassConfig)
41       : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {}
42 
43 private:
44   Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const {
45     return aarch64::applyFixup(G, B, E);
46   }
47 };
48 
49 template <typename ELFT>
50 class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> {
51 private:
52   enum ELFAArch64RelocationKind : Edge::Kind {
53     ELFCall26 = Edge::FirstRelocation,
54     ELFAdrPage21,
55     ELFAddAbs12,
56     ELFLdSt8Abs12,
57     ELFLdSt16Abs12,
58     ELFLdSt32Abs12,
59     ELFLdSt64Abs12,
60     ELFLdSt128Abs12,
61     ELFAbs64,
62     ELFPrel32,
63     ELFPrel64,
64     ELFAdrGOTPage21,
65     ELFLd64GOTLo12,
66   };
67 
68   static Expected<ELFAArch64RelocationKind>
69   getRelocationKind(const uint32_t Type) {
70     using namespace aarch64;
71     switch (Type) {
72     case ELF::R_AARCH64_CALL26:
73     case ELF::R_AARCH64_JUMP26:
74       return ELFCall26;
75     case ELF::R_AARCH64_ADR_PREL_PG_HI21:
76       return ELFAdrPage21;
77     case ELF::R_AARCH64_ADD_ABS_LO12_NC:
78       return ELFAddAbs12;
79     case ELF::R_AARCH64_LDST8_ABS_LO12_NC:
80       return ELFLdSt8Abs12;
81     case ELF::R_AARCH64_LDST16_ABS_LO12_NC:
82       return ELFLdSt16Abs12;
83     case ELF::R_AARCH64_LDST32_ABS_LO12_NC:
84       return ELFLdSt32Abs12;
85     case ELF::R_AARCH64_LDST64_ABS_LO12_NC:
86       return ELFLdSt64Abs12;
87     case ELF::R_AARCH64_LDST128_ABS_LO12_NC:
88       return ELFLdSt128Abs12;
89     case ELF::R_AARCH64_ABS64:
90       return ELFAbs64;
91     case ELF::R_AARCH64_PREL32:
92       return ELFPrel32;
93     case ELF::R_AARCH64_PREL64:
94       return ELFPrel64;
95     case ELF::R_AARCH64_ADR_GOT_PAGE:
96       return ELFAdrGOTPage21;
97     case ELF::R_AARCH64_LD64_GOT_LO12_NC:
98       return ELFLd64GOTLo12;
99     }
100 
101     return make_error<JITLinkError>("Unsupported aarch64 relocation:" +
102                                     formatv("{0:d}", Type));
103   }
104 
105   Error addRelocations() override {
106     LLVM_DEBUG(dbgs() << "Processing relocations:\n");
107 
108     using Base = ELFLinkGraphBuilder<ELFT>;
109     using Self = ELFLinkGraphBuilder_aarch64<ELFT>;
110     for (const auto &RelSect : Base::Sections)
111       if (Error Err = Base::forEachRelocation(RelSect, this,
112                                               &Self::addSingleRelocation))
113         return Err;
114 
115     return Error::success();
116   }
117 
118   Error addSingleRelocation(const typename ELFT::Rela &Rel,
119                             const typename ELFT::Shdr &FixupSect,
120                             Block &BlockToFix) {
121     using support::ulittle32_t;
122     using Base = ELFLinkGraphBuilder<ELFT>;
123 
124     uint32_t SymbolIndex = Rel.getSymbol(false);
125     auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec);
126     if (!ObjSymbol)
127       return ObjSymbol.takeError();
128 
129     Symbol *GraphSymbol = Base::getGraphSymbol(SymbolIndex);
130     if (!GraphSymbol)
131       return make_error<StringError>(
132           formatv("Could not find symbol at given index, did you add it to "
133                   "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}",
134                   SymbolIndex, (*ObjSymbol)->st_shndx,
135                   Base::GraphSymbols.size()),
136           inconvertibleErrorCode());
137 
138     uint32_t Type = Rel.getType(false);
139     Expected<ELFAArch64RelocationKind> RelocKind = getRelocationKind(Type);
140     if (!RelocKind)
141       return RelocKind.takeError();
142 
143     int64_t Addend = Rel.r_addend;
144     orc::ExecutorAddr FixupAddress =
145         orc::ExecutorAddr(FixupSect.sh_addr) + Rel.r_offset;
146     Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress();
147 
148     // Get a pointer to the fixup content.
149     const void *FixupContent = BlockToFix.getContent().data() +
150                                (FixupAddress - BlockToFix.getAddress());
151 
152     Edge::Kind Kind = Edge::Invalid;
153 
154     switch (*RelocKind) {
155     case ELFCall26: {
156       Kind = aarch64::Branch26;
157       break;
158     }
159     case ELFAdrPage21: {
160       Kind = aarch64::Page21;
161       break;
162     }
163     case ELFAddAbs12: {
164       Kind = aarch64::PageOffset12;
165       break;
166     }
167     case ELFLdSt8Abs12: {
168       uint32_t Instr = *(const ulittle32_t *)FixupContent;
169       if (!aarch64::isLoadStoreImm12(Instr) ||
170           aarch64::getPageOffset12Shift(Instr) != 0)
171         return make_error<JITLinkError>(
172             "R_AARCH64_LDST8_ABS_LO12_NC target is not a "
173             "LDRB/STRB (imm12) instruction");
174 
175       Kind = aarch64::PageOffset12;
176       break;
177     }
178     case ELFLdSt16Abs12: {
179       uint32_t Instr = *(const ulittle32_t *)FixupContent;
180       if (!aarch64::isLoadStoreImm12(Instr) ||
181           aarch64::getPageOffset12Shift(Instr) != 1)
182         return make_error<JITLinkError>(
183             "R_AARCH64_LDST16_ABS_LO12_NC target is not a "
184             "LDRH/STRH (imm12) instruction");
185 
186       Kind = aarch64::PageOffset12;
187       break;
188     }
189     case ELFLdSt32Abs12: {
190       uint32_t Instr = *(const ulittle32_t *)FixupContent;
191       if (!aarch64::isLoadStoreImm12(Instr) ||
192           aarch64::getPageOffset12Shift(Instr) != 2)
193         return make_error<JITLinkError>(
194             "R_AARCH64_LDST32_ABS_LO12_NC target is not a "
195             "LDR/STR (imm12, 32 bit) instruction");
196 
197       Kind = aarch64::PageOffset12;
198       break;
199     }
200     case ELFLdSt64Abs12: {
201       uint32_t Instr = *(const ulittle32_t *)FixupContent;
202       if (!aarch64::isLoadStoreImm12(Instr) ||
203           aarch64::getPageOffset12Shift(Instr) != 3)
204         return make_error<JITLinkError>(
205             "R_AARCH64_LDST64_ABS_LO12_NC target is not a "
206             "LDR/STR (imm12, 64 bit) instruction");
207 
208       Kind = aarch64::PageOffset12;
209       break;
210     }
211     case ELFLdSt128Abs12: {
212       uint32_t Instr = *(const ulittle32_t *)FixupContent;
213       if (!aarch64::isLoadStoreImm12(Instr) ||
214           aarch64::getPageOffset12Shift(Instr) != 4)
215         return make_error<JITLinkError>(
216             "R_AARCH64_LDST128_ABS_LO12_NC target is not a "
217             "LDR/STR (imm12, 128 bit) instruction");
218 
219       Kind = aarch64::PageOffset12;
220       break;
221     }
222     case ELFAbs64: {
223       Kind = aarch64::Pointer64;
224       break;
225     }
226     case ELFPrel32: {
227       Kind = aarch64::Delta32;
228       break;
229     }
230     case ELFPrel64: {
231       Kind = aarch64::Delta64;
232       break;
233     }
234     case ELFAdrGOTPage21: {
235       Kind = aarch64::GOTPage21;
236       break;
237     }
238     case ELFLd64GOTLo12: {
239       Kind = aarch64::GOTPageOffset12;
240       break;
241     }
242     };
243 
244     Edge GE(Kind, Offset, *GraphSymbol, Addend);
245     LLVM_DEBUG({
246       dbgs() << "    ";
247       printEdge(dbgs(), BlockToFix, GE, aarch64::getEdgeKindName(Kind));
248       dbgs() << "\n";
249     });
250 
251     BlockToFix.addEdge(std::move(GE));
252     return Error::success();
253   }
254 
255   /// Return the string name of the given ELF aarch64 edge kind.
256   const char *getELFAArch64RelocationKindName(Edge::Kind R) {
257     switch (R) {
258     case ELFCall26:
259       return "ELFCall26";
260     case ELFAdrPage21:
261       return "ELFAdrPage21";
262     case ELFAddAbs12:
263       return "ELFAddAbs12";
264     case ELFLdSt8Abs12:
265       return "ELFLdSt8Abs12";
266     case ELFLdSt16Abs12:
267       return "ELFLdSt16Abs12";
268     case ELFLdSt32Abs12:
269       return "ELFLdSt32Abs12";
270     case ELFLdSt64Abs12:
271       return "ELFLdSt64Abs12";
272     case ELFLdSt128Abs12:
273       return "ELFLdSt128Abs12";
274     case ELFAbs64:
275       return "ELFAbs64";
276     case ELFPrel32:
277       return "ELFPrel32";
278     case ELFPrel64:
279       return "ELFPrel64";
280     case ELFAdrGOTPage21:
281       return "ELFAdrGOTPage21";
282     case ELFLd64GOTLo12:
283       return "ELFLd64GOTLo12";
284     default:
285       return getGenericEdgeKindName(static_cast<Edge::Kind>(R));
286     }
287   }
288 
289 public:
290   ELFLinkGraphBuilder_aarch64(StringRef FileName,
291                               const object::ELFFile<ELFT> &Obj, const Triple T)
292       : ELFLinkGraphBuilder<ELFT>(Obj, std::move(T), FileName,
293                                   aarch64::getEdgeKindName) {}
294 };
295 
296 class PerGraphGOTAndPLTStubsBuilder_ELF_arm64
297     : public PerGraphGOTAndPLTStubsBuilder<
298           PerGraphGOTAndPLTStubsBuilder_ELF_arm64> {
299 public:
300   using PerGraphGOTAndPLTStubsBuilder<
301       PerGraphGOTAndPLTStubsBuilder_ELF_arm64>::PerGraphGOTAndPLTStubsBuilder;
302 
303   bool isGOTEdgeToFix(Edge &E) const {
304     return E.getKind() == aarch64::GOTPage21 ||
305            E.getKind() == aarch64::GOTPageOffset12;
306   }
307 
308   Symbol &createGOTEntry(Symbol &Target) {
309     auto &GOTEntryBlock = G.createContentBlock(
310         getGOTSection(), getGOTEntryBlockContent(), orc::ExecutorAddr(), 8, 0);
311     GOTEntryBlock.addEdge(aarch64::Pointer64, 0, Target, 0);
312     return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false);
313   }
314 
315   void fixGOTEdge(Edge &E, Symbol &GOTEntry) {
316     if (E.getKind() == aarch64::GOTPage21) {
317       E.setKind(aarch64::Page21);
318       E.setTarget(GOTEntry);
319     } else if (E.getKind() == aarch64::GOTPageOffset12) {
320       E.setKind(aarch64::PageOffset12);
321       E.setTarget(GOTEntry);
322     } else
323       llvm_unreachable("Not a GOT edge?");
324   }
325 
326   bool isExternalBranchEdge(Edge &E) {
327     return E.getKind() == aarch64::Branch26 && !E.getTarget().isDefined();
328   }
329 
330   Symbol &createPLTStub(Symbol &Target) {
331     auto &StubContentBlock = G.createContentBlock(
332         getStubsSection(), getStubBlockContent(), orc::ExecutorAddr(), 1, 0);
333     // Re-use GOT entries for stub targets.
334     auto &GOTEntrySymbol = getGOTEntry(Target);
335     StubContentBlock.addEdge(aarch64::LDRLiteral19, 0, GOTEntrySymbol, 0);
336     return G.addAnonymousSymbol(StubContentBlock, 0, 8, true, false);
337   }
338 
339   void fixPLTEdge(Edge &E, Symbol &Stub) {
340     assert(E.getKind() == aarch64::Branch26 && "Not a Branch26 edge?");
341     assert(E.getAddend() == 0 && "Branch26 edge has non-zero addend?");
342     E.setTarget(Stub);
343   }
344 
345 private:
346   Section &getGOTSection() {
347     if (!GOTSection)
348       GOTSection = &G.createSection("$__GOT", MemProt::Read | MemProt::Exec);
349     return *GOTSection;
350   }
351 
352   Section &getStubsSection() {
353     if (!StubsSection)
354       StubsSection =
355           &G.createSection("$__STUBS", MemProt::Read | MemProt::Exec);
356     return *StubsSection;
357   }
358 
359   ArrayRef<char> getGOTEntryBlockContent() {
360     return {reinterpret_cast<const char *>(NullGOTEntryContent),
361             sizeof(NullGOTEntryContent)};
362   }
363 
364   ArrayRef<char> getStubBlockContent() {
365     return {reinterpret_cast<const char *>(StubContent), sizeof(StubContent)};
366   }
367 
368   static const uint8_t NullGOTEntryContent[8];
369   static const uint8_t StubContent[8];
370   Section *GOTSection = nullptr;
371   Section *StubsSection = nullptr;
372 };
373 
374 const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_arm64::NullGOTEntryContent[8] =
375     {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
376 const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_arm64::StubContent[8] = {
377     0x10, 0x00, 0x00, 0x58, // LDR x16, <literal>
378     0x00, 0x02, 0x1f, 0xd6  // BR  x16
379 };
380 
381 Expected<std::unique_ptr<LinkGraph>>
382 createLinkGraphFromELFObject_aarch64(MemoryBufferRef ObjectBuffer) {
383   LLVM_DEBUG({
384     dbgs() << "Building jitlink graph for new input "
385            << ObjectBuffer.getBufferIdentifier() << "...\n";
386   });
387 
388   auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer);
389   if (!ELFObj)
390     return ELFObj.takeError();
391 
392   assert((*ELFObj)->getArch() == Triple::aarch64 &&
393          "Only AArch64 (little endian) is supported for now");
394 
395   auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj);
396   return ELFLinkGraphBuilder_aarch64<object::ELF64LE>((*ELFObj)->getFileName(),
397                                                       ELFObjFile.getELFFile(),
398                                                       (*ELFObj)->makeTriple())
399       .buildGraph();
400 }
401 
402 void link_ELF_aarch64(std::unique_ptr<LinkGraph> G,
403                       std::unique_ptr<JITLinkContext> Ctx) {
404   PassConfiguration Config;
405   const Triple &TT = G->getTargetTriple();
406   if (Ctx->shouldAddDefaultTargetPasses(TT)) {
407     Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame"));
408     Config.PrePrunePasses.push_back(EHFrameEdgeFixer(
409         ".eh_frame", 8, aarch64::Pointer32, aarch64::Pointer64,
410         aarch64::Delta32, aarch64::Delta64, aarch64::NegDelta32));
411     if (auto MarkLive = Ctx->getMarkLivePass(TT))
412       Config.PrePrunePasses.push_back(std::move(MarkLive));
413     else
414       Config.PrePrunePasses.push_back(markAllSymbolsLive);
415   }
416 
417   Config.PostPrunePasses.push_back(
418       PerGraphGOTAndPLTStubsBuilder_ELF_arm64::asPass);
419 
420   if (auto Err = Ctx->modifyPassConfig(*G, Config))
421     return Ctx->notifyFailed(std::move(Err));
422 
423   ELFJITLinker_aarch64::link(std::move(Ctx), std::move(G), std::move(Config));
424 }
425 
426 } // namespace jitlink
427 } // namespace llvm
428