1 //===---- ELF_x86_64.cpp -JIT linker implementation for ELF/x86-64 ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // ELF/x86-64 jit-link implementation.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
14 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
15 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
16 #include "llvm/Object/ELFObjectFile.h"
17 #include "llvm/Support/Endian.h"
18 
19 #include "DefineExternalSectionStartAndEndSymbols.h"
20 #include "EHFrameSupportImpl.h"
21 #include "ELFLinkGraphBuilder.h"
22 #include "JITLinkGeneric.h"
23 #include "PerGraphGOTAndPLTStubsBuilder.h"
24 #include "TableManager.h"
25 
26 #define DEBUG_TYPE "jitlink"
27 
28 using namespace llvm;
29 using namespace llvm::jitlink;
30 using namespace llvm::jitlink::ELF_x86_64_Edges;
31 
32 namespace {
33 
34 constexpr StringRef ELFGOTSectionName = "$__GOT";
35 constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_";
36 constexpr StringRef ELFTLSInfoSectionName = "$__TLSINFO";
37 
38 class GOTTableManager_ELF_x86_64
39     : public TableManager<GOTTableManager_ELF_x86_64> {
40 public:
41   static const uint8_t NullGOTEntryContent[8];
42 
43   // Nice name for table
44   StringRef getTableName() { return "GOT"; }
45 
46   bool fixEdgeKind(LinkGraph &G, Block *B, Edge &E) {
47     Edge::Kind KindToSet = E.getKind();
48     switch (E.getKind()) {
49     case x86_64::Delta64FromGOT: {
50       // we need to make sure that the GOT section exists, but don't otherwise
51       // need to fix up this edge
52       getGOTSection(G);
53       return false;
54     }
55     case x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable:
56       KindToSet = x86_64::PCRel32GOTLoadREXRelaxable;
57       break;
58     case x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable:
59       KindToSet = x86_64::PCRel32GOTLoadRelaxable;
60       break;
61     case x86_64::RequestGOTAndTransformToDelta64:
62       KindToSet = x86_64::Delta64;
63       break;
64     case x86_64::RequestGOTAndTransformToDelta64FromGOT:
65       KindToSet = x86_64::Delta64FromGOT;
66       break;
67     case x86_64::RequestGOTAndTransformToDelta32:
68       KindToSet = x86_64::Delta32;
69       break;
70     default:
71       return false;
72     }
73     LLVM_DEBUG({
74       dbgs() << "  Fixing " << G.getEdgeKindName(E.getKind()) << " edge at "
75              << formatv("{0:x}", B->getFixupAddress(E)) << " ("
76              << formatv("{0:x}", B->getAddress()) << " + "
77              << formatv("{0:x}", E.getOffset()) << ")\n";
78     });
79     E.setKind(KindToSet);
80     return true;
81   }
82 
83   Symbol &createEntry(LinkGraph &G, Symbol &Target) {
84     auto &GOTEntryBlock = G.createContentBlock(
85         getGOTSection(G), getGOTEntryBlockContent(), 0, 8, 0);
86     GOTEntryBlock.addEdge(x86_64::Pointer64, 0, Target, 0);
87     return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false);
88   }
89 
90 private:
91   Section &getGOTSection(LinkGraph &G) {
92     if (!GOTSection)
93       GOTSection = &G.createSection(ELFGOTSectionName, sys::Memory::MF_READ);
94     return *GOTSection;
95   }
96   ArrayRef<char> getGOTEntryBlockContent() const {
97     return {reinterpret_cast<const char *>(NullGOTEntryContent),
98             sizeof(NullGOTEntryContent)};
99   }
100   Section *GOTSection = nullptr;
101 };
102 const uint8_t GOTTableManager_ELF_x86_64::NullGOTEntryContent[8] = {
103     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
104 
105 class PLTTableManager_ELF_x86_64
106     : public TableManager<PLTTableManager_ELF_x86_64> {
107 public:
108   PLTTableManager_ELF_x86_64(GOTTableManager_ELF_x86_64 &GOTTable)
109       : GOTTable(GOTTable) {}
110 
111   StringRef getTableName() { return "PLT"; }
112 
113   static const uint8_t StubContent[6];
114   bool fixEdgeKind(LinkGraph &G, Block *B, Edge &E) {
115     if (E.getKind() == x86_64::BranchPCRel32 && !E.getTarget().isDefined()) {
116       LLVM_DEBUG({
117         dbgs() << "  Fixing " << G.getEdgeKindName(E.getKind()) << " edge at "
118                << formatv("{0:x}", B->getFixupAddress(E)) << " ("
119                << formatv("{0:x}", B->getAddress()) << " + "
120                << formatv("{0:x}", E.getOffset()) << ")\n";
121       });
122       // Set the edge kind to Branch32ToPtrJumpStubBypassable to enable it to
123       // be optimized when the target is in-range.
124       E.setKind(x86_64::BranchPCRel32ToPtrJumpStubBypassable);
125       return true;
126     }
127     return false;
128   }
129 
130   Symbol &createEntry(LinkGraph &G, Symbol &Target) {
131     auto &StubContentBlock = G.createContentBlock(
132         getStubsSection(G), getStubBlockContent(), 0, 1, 0);
133     // Re-use GOT entries for stub targets.
134     auto &GOTEntrySymbol = GOTTable.getEntryForTarget(G, Target);
135     StubContentBlock.addEdge(x86_64::Delta32, 2, GOTEntrySymbol, -4);
136     return G.addAnonymousSymbol(StubContentBlock, 0, 6, true, false);
137   }
138 
139 private:
140   Section &getStubsSection(LinkGraph &G) {
141     if (!StubsSection) {
142       auto StubsProt = static_cast<sys::Memory::ProtectionFlags>(
143           sys::Memory::MF_READ | sys::Memory::MF_EXEC);
144       StubsSection = &G.createSection("$__STUBS", StubsProt);
145     }
146     return *StubsSection;
147   }
148 
149   ArrayRef<char> getStubBlockContent() {
150     return {reinterpret_cast<const char *>(StubContent), sizeof(StubContent)};
151   }
152 
153   Section *StubsSection = nullptr;
154   GOTTableManager_ELF_x86_64 &GOTTable;
155 };
156 const uint8_t PLTTableManager_ELF_x86_64::StubContent[6] = {0xFF, 0x25, 0x00,
157                                                             0x00, 0x00, 0x00};
158 
159 class TLSInfoTableManager_ELF_x86_64
160     : public TableManager<TLSInfoTableManager_ELF_x86_64> {
161 public:
162   static const uint8_t TLSInfoEntryContent[16];
163 
164   StringRef getTableName() { return "TLSInfo"; }
165 
166   bool fixEdgeKind(LinkGraph &G, Block *B, Edge &E) {
167     if (E.getKind() == x86_64::RequestTLSDescInGOTAndTransformToDelta32) {
168       LLVM_DEBUG({
169         dbgs() << "  Fixing " << G.getEdgeKindName(E.getKind()) << " edge at "
170                << formatv("{0:x}", B->getFixupAddress(E)) << " ("
171                << formatv("{0:x}", B->getAddress()) << " + "
172                << formatv("{0:x}", E.getOffset()) << ")\n";
173       });
174       E.setKind(x86_64::Delta32);
175       return true;
176     }
177     return false;
178   }
179 
180   Symbol &createEntry(LinkGraph &G, Symbol &Target) {
181     // the TLS Info entry's key value will be written by the fixTLVSectionByName
182     // pass, so create mutable content.
183     auto &TLSInfoEntry = G.createMutableContentBlock(
184         getTLSInfoSection(G), G.allocateContent(getTLSInfoEntryContent()), 0, 8,
185         0);
186     TLSInfoEntry.addEdge(x86_64::Pointer64, 8, Target, 0);
187     return G.addAnonymousSymbol(TLSInfoEntry, 0, 16, false, false);
188   }
189 
190 private:
191   Section &getTLSInfoSection(LinkGraph &G) {
192     if (!TLSInfoTable)
193       TLSInfoTable =
194           &G.createSection(ELFTLSInfoSectionName, sys::Memory::MF_READ);
195     return *TLSInfoTable;
196   }
197 
198   ArrayRef<char> getTLSInfoEntryContent() const {
199     return {reinterpret_cast<const char *>(TLSInfoEntryContent),
200             sizeof(TLSInfoEntryContent)};
201   }
202 
203   Section *TLSInfoTable = nullptr;
204 };
205 
206 const uint8_t TLSInfoTableManager_ELF_x86_64::TLSInfoEntryContent[16] = {
207     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*pthread key */
208     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00  /*data address*/
209 };
210 
211 Error buildTables_ELF_x86_64(LinkGraph &G) {
212   LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n");
213 
214   GOTTableManager_ELF_x86_64 GOT;
215   PLTTableManager_ELF_x86_64 PLT(GOT);
216   TLSInfoTableManager_ELF_x86_64 TLSInfo;
217   visitExistingEdges(G, GOT, PLT, TLSInfo);
218   return Error::success();
219 }
220 } // namespace
221 
222 static const char *getELFX86_64RelocName(uint32_t Type) {
223   switch (Type) {
224 #define ELF_RELOC(Name, Number)                                                \
225   case Number:                                                                 \
226     return #Name;
227 #include "llvm/BinaryFormat/ELFRelocs/x86_64.def"
228 #undef ELF_RELOC
229   }
230   return "Unrecognized ELF/x86-64 relocation type";
231 }
232 
233 namespace llvm {
234 namespace jitlink {
235 
236 // This should become a template as the ELFFile is so a lot of this could become
237 // generic
238 class ELFLinkGraphBuilder_x86_64 : public ELFLinkGraphBuilder<object::ELF64LE> {
239 private:
240   using ELFT = object::ELF64LE;
241 
242   static Expected<ELF_x86_64_Edges::ELFX86RelocationKind>
243   getRelocationKind(const uint32_t Type) {
244     switch (Type) {
245     case ELF::R_X86_64_32S:
246       return ELF_x86_64_Edges::ELFX86RelocationKind::Pointer32Signed;
247     case ELF::R_X86_64_PC32:
248       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32;
249     case ELF::R_X86_64_PC64:
250     case ELF::R_X86_64_GOTPC64:
251       return ELF_x86_64_Edges::ELFX86RelocationKind::Delta64;
252     case ELF::R_X86_64_64:
253       return ELF_x86_64_Edges::ELFX86RelocationKind::Pointer64;
254     case ELF::R_X86_64_GOTPCREL:
255       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32GOTLoad;
256     case ELF::R_X86_64_GOTPCRELX:
257       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32GOTLoadRelaxable;
258     case ELF::R_X86_64_REX_GOTPCRELX:
259       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32REXGOTLoadRelaxable;
260     case ELF::R_X86_64_GOTPCREL64:
261       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel64GOT;
262     case ELF::R_X86_64_GOT64:
263       return ELF_x86_64_Edges::ELFX86RelocationKind::GOT64;
264     case ELF::R_X86_64_GOTOFF64:
265       return ELF_x86_64_Edges::ELFX86RelocationKind::GOTOFF64;
266     case ELF::R_X86_64_PLT32:
267       return ELF_x86_64_Edges::ELFX86RelocationKind::Branch32;
268     case ELF::R_X86_64_TLSGD:
269       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32TLV;
270     }
271     return make_error<JITLinkError>("Unsupported x86-64 relocation type " +
272                                     formatv("{0:d}: ", Type) +
273                                     getELFX86_64RelocName(Type));
274   }
275 
276   Error addRelocations() override {
277     LLVM_DEBUG(dbgs() << "Processing relocations:\n");
278 
279     using Base = ELFLinkGraphBuilder<ELFT>;
280     using Self = ELFLinkGraphBuilder_x86_64;
281     for (const auto &RelSect : Base::Sections) {
282       // Sanity check the section to read relocation entries from.
283       if (RelSect.sh_type == ELF::SHT_REL)
284         return make_error<StringError>(
285             "No SHT_REL in valid x64 ELF object files",
286             inconvertibleErrorCode());
287 
288       if (Error Err = Base::forEachRelocation(RelSect, this,
289                                               &Self::addSingleRelocation))
290         return Err;
291     }
292 
293     return Error::success();
294   }
295 
296   Error addSingleRelocation(const typename ELFT::Rela &Rel,
297                             const typename ELFT::Shdr &FixupSection,
298                             Section &GraphSection) {
299     using Base = ELFLinkGraphBuilder<ELFT>;
300 
301     uint32_t SymbolIndex = Rel.getSymbol(false);
302     auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec);
303     if (!ObjSymbol)
304       return ObjSymbol.takeError();
305 
306     Symbol *GraphSymbol = Base::getGraphSymbol(SymbolIndex);
307     if (!GraphSymbol)
308       return make_error<StringError>(
309           formatv("Could not find symbol at given index, did you add it to "
310                   "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}",
311                   SymbolIndex, (*ObjSymbol)->st_shndx,
312                   Base::GraphSymbols.size()),
313           inconvertibleErrorCode());
314 
315     // Sanity check the relocation kind.
316     auto ELFRelocKind = getRelocationKind(Rel.getType(false));
317     if (!ELFRelocKind)
318       return ELFRelocKind.takeError();
319 
320     int64_t Addend = Rel.r_addend;
321     Edge::Kind Kind = Edge::Invalid;
322     switch (*ELFRelocKind) {
323     case PCRel32:
324       Kind = x86_64::Delta32;
325       break;
326     case Delta64:
327       Kind = x86_64::Delta64;
328       break;
329     case Pointer32Signed:
330       Kind = x86_64::Pointer32Signed;
331       break;
332     case Pointer64:
333       Kind = x86_64::Pointer64;
334       break;
335     case PCRel32GOTLoad: {
336       Kind = x86_64::RequestGOTAndTransformToDelta32;
337       break;
338     }
339     case PCRel32REXGOTLoadRelaxable: {
340       Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable;
341       Addend = 0;
342       break;
343     }
344     case PCRel32TLV: {
345       Kind = x86_64::RequestTLSDescInGOTAndTransformToDelta32;
346       break;
347     }
348     case PCRel32GOTLoadRelaxable: {
349       Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable;
350       Addend = 0;
351       break;
352     }
353     case PCRel64GOT: {
354       Kind = x86_64::RequestGOTAndTransformToDelta64;
355       break;
356     }
357     case GOT64: {
358       Kind = x86_64::RequestGOTAndTransformToDelta64FromGOT;
359       break;
360     }
361     case GOTOFF64: {
362       Kind = x86_64::Delta64FromGOT;
363       break;
364     }
365     case Branch32: {
366       Kind = x86_64::BranchPCRel32;
367       Addend = 0;
368       break;
369     }
370     }
371 
372     Block *BlockToFix = *(GraphSection.blocks().begin());
373     JITTargetAddress FixupAddress = FixupSection.sh_addr + Rel.r_offset;
374     Edge::OffsetT Offset = FixupAddress - BlockToFix->getAddress();
375     Edge GE(Kind, Offset, *GraphSymbol, Addend);
376     LLVM_DEBUG({
377       dbgs() << "    ";
378       printEdge(dbgs(), *BlockToFix, GE, getELFX86RelocationKindName(Kind));
379       dbgs() << "\n";
380     });
381 
382     BlockToFix->addEdge(std::move(GE));
383     return Error::success();
384   }
385 
386 public:
387   ELFLinkGraphBuilder_x86_64(StringRef FileName,
388                              const object::ELFFile<object::ELF64LE> &Obj)
389       : ELFLinkGraphBuilder(Obj, Triple("x86_64-unknown-linux"), FileName,
390                             x86_64::getEdgeKindName) {}
391 };
392 
393 class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> {
394   friend class JITLinker<ELFJITLinker_x86_64>;
395 
396 public:
397   ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx,
398                       std::unique_ptr<LinkGraph> G,
399                       PassConfiguration PassConfig)
400       : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {
401     getPassConfig().PostAllocationPasses.push_back(
402         [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); });
403   }
404 
405 private:
406   Symbol *GOTSymbol = nullptr;
407 
408   Error getOrCreateGOTSymbol(LinkGraph &G) {
409     auto DefineExternalGOTSymbolIfPresent =
410         createDefineExternalSectionStartAndEndSymbolsPass(
411             [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc {
412               if (Sym.getName() == ELFGOTSymbolName)
413                 if (auto *GOTSection = G.findSectionByName(ELFGOTSectionName)) {
414                   GOTSymbol = &Sym;
415                   return {*GOTSection, true};
416                 }
417               return {};
418             });
419 
420     // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an
421     // external.
422     if (auto Err = DefineExternalGOTSymbolIfPresent(G))
423       return Err;
424 
425     // If we succeeded then we're done.
426     if (GOTSymbol)
427       return Error::success();
428 
429     // Otherwise look for a GOT section: If it already has a start symbol we'll
430     // record it, otherwise we'll create our own.
431     // If there's a GOT section but we didn't find an external GOT symbol...
432     if (auto *GOTSection = G.findSectionByName(ELFGOTSectionName)) {
433 
434       // Check for an existing defined symbol.
435       for (auto *Sym : GOTSection->symbols())
436         if (Sym->getName() == ELFGOTSymbolName) {
437           GOTSymbol = Sym;
438           return Error::success();
439         }
440 
441       // If there's no defined symbol then create one.
442       SectionRange SR(*GOTSection);
443       if (SR.empty())
444         GOTSymbol = &G.addAbsoluteSymbol(ELFGOTSymbolName, 0, 0,
445                                          Linkage::Strong, Scope::Local, true);
446       else
447         GOTSymbol =
448             &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0,
449                                 Linkage::Strong, Scope::Local, false, true);
450     }
451 
452     return Error::success();
453   }
454 
455   Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const {
456     return x86_64::applyFixup(G, B, E, GOTSymbol);
457   }
458 };
459 
460 Expected<std::unique_ptr<LinkGraph>>
461 createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) {
462   LLVM_DEBUG({
463     dbgs() << "Building jitlink graph for new input "
464            << ObjectBuffer.getBufferIdentifier() << "...\n";
465   });
466 
467   auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer);
468   if (!ELFObj)
469     return ELFObj.takeError();
470 
471   auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj);
472   return ELFLinkGraphBuilder_x86_64((*ELFObj)->getFileName(),
473                                     ELFObjFile.getELFFile())
474       .buildGraph();
475 }
476 
477 static SectionRangeSymbolDesc
478 identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) {
479   constexpr StringRef StartSymbolPrefix = "__start";
480   constexpr StringRef EndSymbolPrefix = "__end";
481 
482   auto SymName = Sym.getName();
483   if (SymName.startswith(StartSymbolPrefix)) {
484     if (auto *Sec =
485             G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size())))
486       return {*Sec, true};
487   } else if (SymName.startswith(EndSymbolPrefix)) {
488     if (auto *Sec =
489             G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size())))
490       return {*Sec, false};
491   }
492   return {};
493 }
494 
495 void link_ELF_x86_64(std::unique_ptr<LinkGraph> G,
496                      std::unique_ptr<JITLinkContext> Ctx) {
497   PassConfiguration Config;
498 
499   if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) {
500 
501     Config.PrePrunePasses.push_back(EHFrameSplitter(".eh_frame"));
502     Config.PrePrunePasses.push_back(
503         EHFrameEdgeFixer(".eh_frame", x86_64::PointerSize, x86_64::Delta64,
504                          x86_64::Delta32, x86_64::NegDelta32));
505     Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame"));
506 
507     // Construct a JITLinker and run the link function.
508     // Add a mark-live pass.
509     if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple()))
510       Config.PrePrunePasses.push_back(std::move(MarkLive));
511     else
512       Config.PrePrunePasses.push_back(markAllSymbolsLive);
513 
514     // Add an in-place GOT/Stubs/TLSInfoEntry build pass.
515     Config.PostPrunePasses.push_back(buildTables_ELF_x86_64);
516 
517     // Resolve any external section start / end symbols.
518     Config.PostAllocationPasses.push_back(
519         createDefineExternalSectionStartAndEndSymbolsPass(
520             identifyELFSectionStartAndEndSymbols));
521 
522     // Add GOT/Stubs optimizer pass.
523     Config.PreFixupPasses.push_back(x86_64::optimize_x86_64_GOTAndStubs);
524   }
525 
526   if (auto Err = Ctx->modifyPassConfig(*G, Config))
527     return Ctx->notifyFailed(std::move(Err));
528 
529   ELFJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config));
530 }
531 const char *getELFX86RelocationKindName(Edge::Kind R) {
532   switch (R) {
533   case Branch32:
534     return "Branch32";
535   case Pointer32Signed:
536     return "Pointer32Signed";
537   case Pointer64:
538     return "Pointer64";
539   case PCRel32:
540     return "PCRel32";
541   case PCRel32GOTLoad:
542     return "PCRel32GOTLoad";
543   case PCRel32GOTLoadRelaxable:
544     return "PCRel32GOTLoadRelaxable";
545   case PCRel32REXGOTLoadRelaxable:
546     return "PCRel32REXGOTLoad";
547   case PCRel64GOT:
548     return "PCRel64GOT";
549   case Delta64:
550     return "Delta64";
551   case GOT64:
552     return "GOT64";
553   case GOTOFF64:
554     return "GOTOFF64";
555   }
556   return getGenericEdgeKindName(static_cast<Edge::Kind>(R));
557 }
558 } // end namespace jitlink
559 } // end namespace llvm
560