1 //===---- ELF_x86_64.cpp -JIT linker implementation for ELF/x86-64 ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // ELF/x86-64 jit-link implementation.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
14 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
15 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
16 #include "llvm/Object/ELFObjectFile.h"
17 #include "llvm/Support/Endian.h"
18 
19 #include "DefineExternalSectionStartAndEndSymbols.h"
20 #include "EHFrameSupportImpl.h"
21 #include "ELFLinkGraphBuilder.h"
22 #include "JITLinkGeneric.h"
23 #include "PerGraphGOTAndPLTStubsBuilder.h"
24 #include "PerGraphTLSInfoEntryBuilder.h"
25 
26 #define DEBUG_TYPE "jitlink"
27 
28 using namespace llvm;
29 using namespace llvm::jitlink;
30 using namespace llvm::jitlink::ELF_x86_64_Edges;
31 
32 namespace {
33 
34 constexpr StringRef ELFGOTSectionName = "$__GOT";
35 constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_";
36 constexpr StringRef ELFTLSInfoSectionName = "$__TLSINFO";
37 
38 class PerGraphTLSInfoBuilder_ELF_x86_64
39     : public PerGraphTLSInfoEntryBuilder<PerGraphTLSInfoBuilder_ELF_x86_64> {
40 public:
41   static const uint8_t TLSInfoEntryContent[16];
42   using PerGraphTLSInfoEntryBuilder<
43       PerGraphTLSInfoBuilder_ELF_x86_64>::PerGraphTLSInfoEntryBuilder;
44 
45   bool isTLSEdgeToFix(Edge &E) {
46     return E.getKind() == x86_64::RequestTLSDescInGOTAndTransformToDelta32;
47   }
48 
49   Symbol &createTLSInfoEntry(Symbol &Target) {
50     // the TLS Info entry's key value will be written by the fixTLVSectionByName
51     // pass, so create mutable content.
52     auto &TLSInfoEntry = G.createMutableContentBlock(
53         getTLSInfoSection(), G.allocateContent(getTLSInfoEntryContent()), 0, 8,
54         0);
55     TLSInfoEntry.addEdge(x86_64::Pointer64, 8, Target, 0);
56     return G.addAnonymousSymbol(TLSInfoEntry, 0, 16, false, false);
57   }
58 
59   void fixTLSEdge(Edge &E, Symbol &Target) {
60     if (E.getKind() == x86_64::RequestTLSDescInGOTAndTransformToDelta32) {
61       E.setTarget(Target);
62       E.setKind(x86_64::Delta32);
63     }
64   }
65 
66   Section &getTLSInfoSection() const {
67     if (!TLSInfoSection)
68       TLSInfoSection =
69           &G.createSection(ELFTLSInfoSectionName, sys::Memory::MF_READ);
70     return *TLSInfoSection;
71   }
72 
73 private:
74   ArrayRef<char> getTLSInfoEntryContent() {
75     return {reinterpret_cast<const char *>(TLSInfoEntryContent),
76             sizeof(TLSInfoEntryContent)};
77   }
78 
79   mutable Section *TLSInfoSection = nullptr;
80 };
81 
82 const uint8_t PerGraphTLSInfoBuilder_ELF_x86_64::TLSInfoEntryContent[16] = {
83     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*pthread key */
84     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00  /*data address*/
85 };
86 
87 class PerGraphGOTAndPLTStubsBuilder_ELF_x86_64
88     : public PerGraphGOTAndPLTStubsBuilder<
89           PerGraphGOTAndPLTStubsBuilder_ELF_x86_64> {
90 public:
91   static const uint8_t NullGOTEntryContent[8];
92   static const uint8_t StubContent[6];
93 
94   using PerGraphGOTAndPLTStubsBuilder<
95       PerGraphGOTAndPLTStubsBuilder_ELF_x86_64>::PerGraphGOTAndPLTStubsBuilder;
96 
97   bool isGOTEdgeToFix(Edge &E) const {
98     if (E.getKind() == x86_64::Delta64FromGOT) {
99       // We need to make sure that the GOT section exists, but don't otherwise
100       // need to fix up this edge.
101       getGOTSection();
102       return false;
103     }
104     return E.getKind() == x86_64::RequestGOTAndTransformToDelta32 ||
105            E.getKind() == x86_64::RequestGOTAndTransformToDelta64 ||
106            E.getKind() ==
107                x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable ||
108            E.getKind() == x86_64::RequestGOTAndTransformToDelta64FromGOT ||
109            E.getKind() ==
110                x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable;
111   }
112 
113   Symbol &createGOTEntry(Symbol &Target) {
114     auto &GOTEntryBlock = G.createContentBlock(
115         getGOTSection(), getGOTEntryBlockContent(), 0, 8, 0);
116     GOTEntryBlock.addEdge(x86_64::Pointer64, 0, Target, 0);
117     return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false);
118   }
119 
120   void fixGOTEdge(Edge &E, Symbol &GOTEntry) {
121     // If this is a PCRel32GOT/PCRel64GOT then change it to an ordinary
122     // PCRel32/PCRel64. If it is a PCRel32GOTLoad then leave it as-is for now:
123     // We will use the kind to check for GOT optimization opportunities in the
124     // optimizeMachO_x86_64_GOTAndStubs pass below.
125     // If it's a GOT64 leave it as is.
126     switch (E.getKind()) {
127     case x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable:
128       E.setKind(x86_64::PCRel32GOTLoadREXRelaxable);
129       break;
130     case x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable:
131       E.setKind(x86_64::PCRel32GOTLoadRelaxable);
132       break;
133     case x86_64::RequestGOTAndTransformToDelta64:
134       E.setKind(x86_64::Delta64);
135       break;
136     case x86_64::RequestGOTAndTransformToDelta64FromGOT:
137       E.setKind(x86_64::Delta64FromGOT);
138       break;
139     case x86_64::RequestGOTAndTransformToDelta32:
140       E.setKind(x86_64::Delta32);
141       break;
142     default:
143       llvm_unreachable("Unexpected GOT edge kind");
144     }
145 
146     E.setTarget(GOTEntry);
147     // Leave the edge addend as-is.
148   }
149 
150   bool isExternalBranchEdge(Edge &E) {
151     return E.getKind() == x86_64::BranchPCRel32 && !E.getTarget().isDefined();
152   }
153 
154   Symbol &createPLTStub(Symbol &Target) {
155     auto &StubContentBlock =
156         G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0);
157     // Re-use GOT entries for stub targets.
158     auto &GOTEntrySymbol = getGOTEntry(Target);
159     StubContentBlock.addEdge(x86_64::Delta32, 2, GOTEntrySymbol, -4);
160     return G.addAnonymousSymbol(StubContentBlock, 0, 6, true, false);
161   }
162 
163   void fixPLTEdge(Edge &E, Symbol &Stub) {
164     assert(E.getKind() == x86_64::BranchPCRel32 && "Not a Branch32 edge?");
165 
166     // Set the edge kind to Branch32ToPtrJumpStubBypassable to enable it to be
167     // optimized when the target is in-range.
168     E.setKind(x86_64::BranchPCRel32ToPtrJumpStubBypassable);
169     E.setTarget(Stub);
170   }
171 
172 private:
173   Section &getGOTSection() const {
174     if (!GOTSection)
175       GOTSection = &G.createSection(ELFGOTSectionName, sys::Memory::MF_READ);
176     return *GOTSection;
177   }
178 
179   Section &getStubsSection() const {
180     if (!StubsSection) {
181       auto StubsProt = static_cast<sys::Memory::ProtectionFlags>(
182           sys::Memory::MF_READ | sys::Memory::MF_EXEC);
183       StubsSection = &G.createSection("$__STUBS", StubsProt);
184     }
185     return *StubsSection;
186   }
187 
188   ArrayRef<char> getGOTEntryBlockContent() {
189     return {reinterpret_cast<const char *>(NullGOTEntryContent),
190             sizeof(NullGOTEntryContent)};
191   }
192 
193   ArrayRef<char> getStubBlockContent() {
194     return {reinterpret_cast<const char *>(StubContent), sizeof(StubContent)};
195   }
196 
197   mutable Section *GOTSection = nullptr;
198   mutable Section *StubsSection = nullptr;
199 };
200 
201 } // namespace
202 
203 const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::NullGOTEntryContent[8] =
204     {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
205 const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::StubContent[6] = {
206     0xFF, 0x25, 0x00, 0x00, 0x00, 0x00};
207 
208 static const char *getELFX86_64RelocName(uint32_t Type) {
209   switch (Type) {
210 #define ELF_RELOC(Name, Number)                                                \
211   case Number:                                                                 \
212     return #Name;
213 #include "llvm/BinaryFormat/ELFRelocs/x86_64.def"
214 #undef ELF_RELOC
215   }
216   return "Unrecognized ELF/x86-64 relocation type";
217 }
218 
219 namespace llvm {
220 namespace jitlink {
221 
222 // This should become a template as the ELFFile is so a lot of this could become
223 // generic
224 class ELFLinkGraphBuilder_x86_64 : public ELFLinkGraphBuilder<object::ELF64LE> {
225 private:
226   using ELFT = object::ELF64LE;
227 
228   static Expected<ELF_x86_64_Edges::ELFX86RelocationKind>
229   getRelocationKind(const uint32_t Type) {
230     switch (Type) {
231     case ELF::R_X86_64_32S:
232       return ELF_x86_64_Edges::ELFX86RelocationKind::Pointer32Signed;
233     case ELF::R_X86_64_PC32:
234       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32;
235     case ELF::R_X86_64_PC64:
236     case ELF::R_X86_64_GOTPC64:
237       return ELF_x86_64_Edges::ELFX86RelocationKind::Delta64;
238     case ELF::R_X86_64_64:
239       return ELF_x86_64_Edges::ELFX86RelocationKind::Pointer64;
240     case ELF::R_X86_64_GOTPCREL:
241       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32GOTLoad;
242     case ELF::R_X86_64_GOTPCRELX:
243       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32GOTLoadRelaxable;
244     case ELF::R_X86_64_REX_GOTPCRELX:
245       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32REXGOTLoadRelaxable;
246     case ELF::R_X86_64_GOTPCREL64:
247       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel64GOT;
248     case ELF::R_X86_64_GOT64:
249       return ELF_x86_64_Edges::ELFX86RelocationKind::GOT64;
250     case ELF::R_X86_64_GOTOFF64:
251       return ELF_x86_64_Edges::ELFX86RelocationKind::GOTOFF64;
252     case ELF::R_X86_64_PLT32:
253       return ELF_x86_64_Edges::ELFX86RelocationKind::Branch32;
254     case ELF::R_X86_64_TLSGD:
255       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32TLV;
256     }
257     return make_error<JITLinkError>("Unsupported x86-64 relocation type " +
258                                     formatv("{0:d}: ", Type) +
259                                     getELFX86_64RelocName(Type));
260   }
261 
262   Error addRelocations() override {
263     LLVM_DEBUG(dbgs() << "Processing relocations:\n");
264 
265     using Base = ELFLinkGraphBuilder<ELFT>;
266     using Self = ELFLinkGraphBuilder_x86_64;
267     for (const auto &RelSect : Base::Sections) {
268       // Sanity check the section to read relocation entries from.
269       if (RelSect.sh_type == ELF::SHT_REL)
270         return make_error<StringError>(
271             "No SHT_REL in valid x64 ELF object files",
272             inconvertibleErrorCode());
273 
274       if (Error Err = Base::forEachRelocation(RelSect, this,
275                                               &Self::addSingleRelocation))
276         return Err;
277     }
278 
279     return Error::success();
280   }
281 
282   Error addSingleRelocation(const typename ELFT::Rela &Rel,
283                             const typename ELFT::Shdr &FixupSection,
284                             Section &GraphSection) {
285     using Base = ELFLinkGraphBuilder<ELFT>;
286 
287     uint32_t SymbolIndex = Rel.getSymbol(false);
288     auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec);
289     if (!ObjSymbol)
290       return ObjSymbol.takeError();
291 
292     Symbol *GraphSymbol = Base::getGraphSymbol(SymbolIndex);
293     if (!GraphSymbol)
294       return make_error<StringError>(
295           formatv("Could not find symbol at given index, did you add it to "
296                   "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}",
297                   SymbolIndex, (*ObjSymbol)->st_shndx,
298                   Base::GraphSymbols.size()),
299           inconvertibleErrorCode());
300 
301     // Sanity check the relocation kind.
302     auto ELFRelocKind = getRelocationKind(Rel.getType(false));
303     if (!ELFRelocKind)
304       return ELFRelocKind.takeError();
305 
306     int64_t Addend = Rel.r_addend;
307     Edge::Kind Kind = Edge::Invalid;
308     switch (*ELFRelocKind) {
309     case PCRel32:
310       Kind = x86_64::Delta32;
311       break;
312     case Delta64:
313       Kind = x86_64::Delta64;
314       break;
315     case Pointer32Signed:
316       Kind = x86_64::Pointer32Signed;
317       break;
318     case Pointer64:
319       Kind = x86_64::Pointer64;
320       break;
321     case PCRel32GOTLoad: {
322       Kind = x86_64::RequestGOTAndTransformToDelta32;
323       break;
324     }
325     case PCRel32REXGOTLoadRelaxable: {
326       Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable;
327       Addend = 0;
328       break;
329     }
330     case PCRel32GOTLoadRelaxable: {
331       Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable;
332       Addend = 0;
333       break;
334     }
335     case PCRel64GOT: {
336       Kind = x86_64::RequestGOTAndTransformToDelta64;
337       break;
338     }
339     case GOT64: {
340       Kind = x86_64::RequestGOTAndTransformToDelta64FromGOT;
341       break;
342     }
343     case GOTOFF64: {
344       Kind = x86_64::Delta64FromGOT;
345       break;
346     }
347     case Branch32: {
348       Kind = x86_64::BranchPCRel32;
349       Addend = 0;
350       break;
351     }
352     }
353 
354     Block *BlockToFix = *(GraphSection.blocks().begin());
355     JITTargetAddress FixupAddress = FixupSection.sh_addr + Rel.r_offset;
356     Edge::OffsetT Offset = FixupAddress - BlockToFix->getAddress();
357     Edge GE(Kind, Offset, *GraphSymbol, Addend);
358     LLVM_DEBUG({
359       dbgs() << "    ";
360       printEdge(dbgs(), *BlockToFix, GE, getELFX86RelocationKindName(Kind));
361       dbgs() << "\n";
362     });
363 
364     BlockToFix->addEdge(std::move(GE));
365     return Error::success();
366   }
367 
368 public:
369   ELFLinkGraphBuilder_x86_64(StringRef FileName,
370                              const object::ELFFile<object::ELF64LE> &Obj)
371       : ELFLinkGraphBuilder(Obj, Triple("x86_64-unknown-linux"), FileName,
372                             x86_64::getEdgeKindName) {}
373 };
374 
375 class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> {
376   friend class JITLinker<ELFJITLinker_x86_64>;
377 
378 public:
379   ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx,
380                       std::unique_ptr<LinkGraph> G,
381                       PassConfiguration PassConfig)
382       : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {
383     getPassConfig().PostAllocationPasses.push_back(
384         [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); });
385   }
386 
387 private:
388   Symbol *GOTSymbol = nullptr;
389 
390   Error getOrCreateGOTSymbol(LinkGraph &G) {
391     auto DefineExternalGOTSymbolIfPresent =
392         createDefineExternalSectionStartAndEndSymbolsPass(
393             [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc {
394               if (Sym.getName() == ELFGOTSymbolName)
395                 if (auto *GOTSection = G.findSectionByName(ELFGOTSectionName)) {
396                   GOTSymbol = &Sym;
397                   return {*GOTSection, true};
398                 }
399               return {};
400             });
401 
402     // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an
403     // external.
404     if (auto Err = DefineExternalGOTSymbolIfPresent(G))
405       return Err;
406 
407     // If we succeeded then we're done.
408     if (GOTSymbol)
409       return Error::success();
410 
411     // Otherwise look for a GOT section: If it already has a start symbol we'll
412     // record it, otherwise we'll create our own.
413     // If there's a GOT section but we didn't find an external GOT symbol...
414     if (auto *GOTSection = G.findSectionByName(ELFGOTSectionName)) {
415 
416       // Check for an existing defined symbol.
417       for (auto *Sym : GOTSection->symbols())
418         if (Sym->getName() == ELFGOTSymbolName) {
419           GOTSymbol = Sym;
420           return Error::success();
421         }
422 
423       // If there's no defined symbol then create one.
424       SectionRange SR(*GOTSection);
425       if (SR.empty())
426         GOTSymbol = &G.addAbsoluteSymbol(ELFGOTSymbolName, 0, 0,
427                                          Linkage::Strong, Scope::Local, true);
428       else
429         GOTSymbol =
430             &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0,
431                                 Linkage::Strong, Scope::Local, false, true);
432     }
433 
434     return Error::success();
435   }
436 
437   Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const {
438     return x86_64::applyFixup(G, B, E, GOTSymbol);
439   }
440 };
441 
442 Expected<std::unique_ptr<LinkGraph>>
443 createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) {
444   LLVM_DEBUG({
445     dbgs() << "Building jitlink graph for new input "
446            << ObjectBuffer.getBufferIdentifier() << "...\n";
447   });
448 
449   auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer);
450   if (!ELFObj)
451     return ELFObj.takeError();
452 
453   auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj);
454   return ELFLinkGraphBuilder_x86_64((*ELFObj)->getFileName(),
455                                     ELFObjFile.getELFFile())
456       .buildGraph();
457 }
458 
459 static SectionRangeSymbolDesc
460 identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) {
461   constexpr StringRef StartSymbolPrefix = "__start";
462   constexpr StringRef EndSymbolPrefix = "__end";
463 
464   auto SymName = Sym.getName();
465   if (SymName.startswith(StartSymbolPrefix)) {
466     if (auto *Sec =
467             G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size())))
468       return {*Sec, true};
469   } else if (SymName.startswith(EndSymbolPrefix)) {
470     if (auto *Sec =
471             G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size())))
472       return {*Sec, false};
473   }
474   return {};
475 }
476 
477 void link_ELF_x86_64(std::unique_ptr<LinkGraph> G,
478                      std::unique_ptr<JITLinkContext> Ctx) {
479   PassConfiguration Config;
480 
481   if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) {
482 
483     Config.PrePrunePasses.push_back(EHFrameSplitter(".eh_frame"));
484     Config.PrePrunePasses.push_back(
485         EHFrameEdgeFixer(".eh_frame", x86_64::PointerSize, x86_64::Delta64,
486                          x86_64::Delta32, x86_64::NegDelta32));
487     Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame"));
488 
489     // Construct a JITLinker and run the link function.
490     // Add a mark-live pass.
491     if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple()))
492       Config.PrePrunePasses.push_back(std::move(MarkLive));
493     else
494       Config.PrePrunePasses.push_back(markAllSymbolsLive);
495 
496     // Add an in-place GOT/Stubs pass.
497 
498     Config.PostPrunePasses.push_back(PerGraphTLSInfoBuilder_ELF_x86_64::asPass);
499     Config.PostPrunePasses.push_back(
500         PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::asPass);
501 
502     // Resolve any external section start / end symbols.
503     Config.PostAllocationPasses.push_back(
504         createDefineExternalSectionStartAndEndSymbolsPass(
505             identifyELFSectionStartAndEndSymbols));
506 
507     // Add GOT/Stubs optimizer pass.
508     Config.PreFixupPasses.push_back(x86_64::optimize_x86_64_GOTAndStubs);
509   }
510 
511   if (auto Err = Ctx->modifyPassConfig(*G, Config))
512     return Ctx->notifyFailed(std::move(Err));
513 
514   ELFJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config));
515 }
516 const char *getELFX86RelocationKindName(Edge::Kind R) {
517   switch (R) {
518   case Branch32:
519     return "Branch32";
520   case Pointer32Signed:
521     return "Pointer32Signed";
522   case Pointer64:
523     return "Pointer64";
524   case PCRel32:
525     return "PCRel32";
526   case PCRel32GOTLoad:
527     return "PCRel32GOTLoad";
528   case PCRel32GOTLoadRelaxable:
529     return "PCRel32GOTLoadRelaxable";
530   case PCRel32REXGOTLoadRelaxable:
531     return "PCRel32REXGOTLoad";
532   case PCRel64GOT:
533     return "PCRel64GOT";
534   case Delta64:
535     return "Delta64";
536   case GOT64:
537     return "GOT64";
538   case GOTOFF64:
539     return "GOTOFF64";
540   }
541   return getGenericEdgeKindName(static_cast<Edge::Kind>(R));
542 }
543 } // end namespace jitlink
544 } // end namespace llvm
545