1 //===---- ELF_x86_64.cpp -JIT linker implementation for ELF/x86-64 ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // ELF/x86-64 jit-link implementation.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
14 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
15 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
16 #include "llvm/Object/ELFObjectFile.h"
17 #include "llvm/Support/Endian.h"
18 
19 #include "DefineExternalSectionStartAndEndSymbols.h"
20 #include "EHFrameSupportImpl.h"
21 #include "ELFLinkGraphBuilder.h"
22 #include "JITLinkGeneric.h"
23 #include "PerGraphGOTAndPLTStubsBuilder.h"
24 #include "PerGraphTLSInfoEntryBuilder.h"
25 
26 #define DEBUG_TYPE "jitlink"
27 
28 using namespace llvm;
29 using namespace llvm::jitlink;
30 using namespace llvm::jitlink::ELF_x86_64_Edges;
31 
32 namespace {
33 
34 constexpr StringRef ELFGOTSectionName = "$__GOT";
35 constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_";
36 constexpr StringRef ELFTLSInfoSectionName = "$__TLSINFO";
37 
38 class PerGraphTLSInfoBuilder_ELF_x86_64
39     : public PerGraphTLSInfoEntryBuilder<PerGraphTLSInfoBuilder_ELF_x86_64> {
40 public:
41   static const uint8_t TLSInfoEntryContent[16];
42   using PerGraphTLSInfoEntryBuilder<
43       PerGraphTLSInfoBuilder_ELF_x86_64>::PerGraphTLSInfoEntryBuilder;
44 
45   bool isTLSEdgeToFix(Edge &E) {
46     return E.getKind() == x86_64::RequestTLSDescInGOTAndTransformToDelta32;
47   }
48 
49   Symbol &createTLSInfoEntry(Symbol &Target) {
50     // the TLS Info entry's key value will be written by the fixTLVSectionByName
51     // pass, so create mutable content.
52     auto &TLSInfoEntry = G.createMutableContentBlock(
53         getTLSInfoSection(), G.allocateContent(getTLSInfoEntryContent()), 0, 8,
54         0);
55     TLSInfoEntry.addEdge(x86_64::Pointer64, 8, Target, 0);
56     return G.addAnonymousSymbol(TLSInfoEntry, 0, 16, false, false);
57   }
58 
59   void fixTLSEdge(Edge &E, Symbol &Target) {
60     if (E.getKind() == x86_64::RequestTLSDescInGOTAndTransformToDelta32) {
61       E.setTarget(Target);
62       E.setKind(x86_64::Delta32);
63     }
64   }
65 
66   Section &getTLSInfoSection() const {
67     if (!TLSInfoSection)
68       TLSInfoSection =
69           &G.createSection(ELFTLSInfoSectionName, sys::Memory::MF_READ);
70     return *TLSInfoSection;
71   }
72 
73 private:
74   ArrayRef<char> getTLSInfoEntryContent() {
75     return {reinterpret_cast<const char *>(TLSInfoEntryContent),
76             sizeof(TLSInfoEntryContent)};
77   }
78 
79   mutable Section *TLSInfoSection = nullptr;
80 };
81 
82 const uint8_t PerGraphTLSInfoBuilder_ELF_x86_64::TLSInfoEntryContent[16] = {
83     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*pthread key */
84     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00  /*data address*/
85 };
86 
87 class PerGraphGOTAndPLTStubsBuilder_ELF_x86_64
88     : public PerGraphGOTAndPLTStubsBuilder<
89           PerGraphGOTAndPLTStubsBuilder_ELF_x86_64> {
90 public:
91   static const uint8_t NullGOTEntryContent[8];
92   static const uint8_t StubContent[6];
93 
94   using PerGraphGOTAndPLTStubsBuilder<
95       PerGraphGOTAndPLTStubsBuilder_ELF_x86_64>::PerGraphGOTAndPLTStubsBuilder;
96 
97   bool isGOTEdgeToFix(Edge &E) const {
98     if (E.getKind() == x86_64::Delta64FromGOT) {
99       // We need to make sure that the GOT section exists, but don't otherwise
100       // need to fix up this edge.
101       getGOTSection();
102       return false;
103     }
104     return E.getKind() == x86_64::RequestGOTAndTransformToDelta32 ||
105            E.getKind() == x86_64::RequestGOTAndTransformToDelta64 ||
106            E.getKind() ==
107                x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable ||
108            E.getKind() == x86_64::RequestGOTAndTransformToDelta64FromGOT ||
109            E.getKind() ==
110                x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable;
111   }
112 
113   Symbol &createGOTEntry(Symbol &Target) {
114     auto &GOTEntryBlock = G.createContentBlock(
115         getGOTSection(), getGOTEntryBlockContent(), 0, 8, 0);
116     GOTEntryBlock.addEdge(x86_64::Pointer64, 0, Target, 0);
117     return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false);
118   }
119 
120   void fixGOTEdge(Edge &E, Symbol &GOTEntry) {
121     // If this is a PCRel32GOT/PCRel64GOT then change it to an ordinary
122     // PCRel32/PCRel64. If it is a PCRel32GOTLoad then leave it as-is for now:
123     // We will use the kind to check for GOT optimization opportunities in the
124     // optimizeMachO_x86_64_GOTAndStubs pass below.
125     // If it's a GOT64 leave it as is.
126     switch (E.getKind()) {
127     case x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable:
128       E.setKind(x86_64::PCRel32GOTLoadREXRelaxable);
129       break;
130     case x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable:
131       E.setKind(x86_64::PCRel32GOTLoadRelaxable);
132       break;
133     case x86_64::RequestGOTAndTransformToDelta64:
134       E.setKind(x86_64::Delta64);
135       break;
136     case x86_64::RequestGOTAndTransformToDelta64FromGOT:
137       E.setKind(x86_64::Delta64FromGOT);
138       break;
139     case x86_64::RequestGOTAndTransformToDelta32:
140       E.setKind(x86_64::Delta32);
141       break;
142     default:
143       llvm_unreachable("Unexpected GOT edge kind");
144     }
145 
146     E.setTarget(GOTEntry);
147     // Leave the edge addend as-is.
148   }
149 
150   bool isExternalBranchEdge(Edge &E) {
151     return E.getKind() == x86_64::BranchPCRel32 && !E.getTarget().isDefined();
152   }
153 
154   Symbol &createPLTStub(Symbol &Target) {
155     auto &StubContentBlock =
156         G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0);
157     // Re-use GOT entries for stub targets.
158     auto &GOTEntrySymbol = getGOTEntry(Target);
159     StubContentBlock.addEdge(x86_64::Delta32, 2, GOTEntrySymbol, -4);
160     return G.addAnonymousSymbol(StubContentBlock, 0, 6, true, false);
161   }
162 
163   void fixPLTEdge(Edge &E, Symbol &Stub) {
164     assert(E.getKind() == x86_64::BranchPCRel32 && "Not a Branch32 edge?");
165 
166     // Set the edge kind to Branch32ToPtrJumpStubBypassable to enable it to be
167     // optimized when the target is in-range.
168     E.setKind(x86_64::BranchPCRel32ToPtrJumpStubBypassable);
169     E.setTarget(Stub);
170   }
171 
172 private:
173   Section &getGOTSection() const {
174     if (!GOTSection)
175       GOTSection = &G.createSection(ELFGOTSectionName, sys::Memory::MF_READ);
176     return *GOTSection;
177   }
178 
179   Section &getStubsSection() const {
180     if (!StubsSection) {
181       auto StubsProt = static_cast<sys::Memory::ProtectionFlags>(
182           sys::Memory::MF_READ | sys::Memory::MF_EXEC);
183       StubsSection = &G.createSection("$__STUBS", StubsProt);
184     }
185     return *StubsSection;
186   }
187 
188   ArrayRef<char> getGOTEntryBlockContent() {
189     return {reinterpret_cast<const char *>(NullGOTEntryContent),
190             sizeof(NullGOTEntryContent)};
191   }
192 
193   ArrayRef<char> getStubBlockContent() {
194     return {reinterpret_cast<const char *>(StubContent), sizeof(StubContent)};
195   }
196 
197   mutable Section *GOTSection = nullptr;
198   mutable Section *StubsSection = nullptr;
199 };
200 
201 } // namespace
202 
203 const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::NullGOTEntryContent[8] =
204     {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
205 const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::StubContent[6] = {
206     0xFF, 0x25, 0x00, 0x00, 0x00, 0x00};
207 
208 static const char *getELFX86_64RelocName(uint32_t Type) {
209   switch (Type) {
210 #define ELF_RELOC(Name, Number)                                                \
211   case Number:                                                                 \
212     return #Name;
213 #include "llvm/BinaryFormat/ELFRelocs/x86_64.def"
214 #undef ELF_RELOC
215   }
216   return "Unrecognized ELF/x86-64 relocation type";
217 }
218 
219 namespace llvm {
220 namespace jitlink {
221 
222 // This should become a template as the ELFFile is so a lot of this could become
223 // generic
224 class ELFLinkGraphBuilder_x86_64 : public ELFLinkGraphBuilder<object::ELF64LE> {
225 private:
226   using ELFT = object::ELF64LE;
227 
228   static Expected<ELF_x86_64_Edges::ELFX86RelocationKind>
229   getRelocationKind(const uint32_t Type) {
230     switch (Type) {
231     case ELF::R_X86_64_32S:
232       return ELF_x86_64_Edges::ELFX86RelocationKind::Pointer32Signed;
233     case ELF::R_X86_64_PC32:
234       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32;
235     case ELF::R_X86_64_PC64:
236     case ELF::R_X86_64_GOTPC64:
237       return ELF_x86_64_Edges::ELFX86RelocationKind::Delta64;
238     case ELF::R_X86_64_64:
239       return ELF_x86_64_Edges::ELFX86RelocationKind::Pointer64;
240     case ELF::R_X86_64_GOTPCREL:
241       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32GOTLoad;
242     case ELF::R_X86_64_GOTPCRELX:
243       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32GOTLoadRelaxable;
244     case ELF::R_X86_64_REX_GOTPCRELX:
245       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32REXGOTLoadRelaxable;
246     case ELF::R_X86_64_GOTPCREL64:
247       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel64GOT;
248     case ELF::R_X86_64_GOT64:
249       return ELF_x86_64_Edges::ELFX86RelocationKind::GOT64;
250     case ELF::R_X86_64_GOTOFF64:
251       return ELF_x86_64_Edges::ELFX86RelocationKind::GOTOFF64;
252     case ELF::R_X86_64_PLT32:
253       return ELF_x86_64_Edges::ELFX86RelocationKind::Branch32;
254     case ELF::R_X86_64_TLSGD:
255       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32TLV;
256     }
257     return make_error<JITLinkError>("Unsupported x86-64 relocation type " +
258                                     formatv("{0:d}: ", Type) +
259                                     getELFX86_64RelocName(Type));
260   }
261 
262   Error addRelocations() override {
263     LLVM_DEBUG(dbgs() << "Processing relocations:\n");
264 
265     using Base = ELFLinkGraphBuilder<ELFT>;
266     using Self = ELFLinkGraphBuilder_x86_64;
267     for (const auto &RelSect : Base::Sections) {
268       // Sanity check the section to read relocation entries from.
269       if (RelSect.sh_type == ELF::SHT_REL)
270         return make_error<StringError>(
271             "No SHT_REL in valid x64 ELF object files",
272             inconvertibleErrorCode());
273 
274       if (Error Err = Base::forEachRelocation(RelSect, this,
275                                               &Self::addSingleRelocation))
276         return Err;
277     }
278 
279     return Error::success();
280   }
281 
282   Error addSingleRelocation(const typename ELFT::Rela &Rel,
283                             const typename ELFT::Shdr &FixupSection,
284                             Section &GraphSection) {
285     using Base = ELFLinkGraphBuilder<ELFT>;
286 
287     uint32_t SymbolIndex = Rel.getSymbol(false);
288     auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec);
289     if (!ObjSymbol)
290       return ObjSymbol.takeError();
291 
292     Symbol *GraphSymbol = Base::getGraphSymbol(SymbolIndex);
293     if (!GraphSymbol)
294       return make_error<StringError>(
295           formatv("Could not find symbol at given index, did you add it to "
296                   "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}",
297                   SymbolIndex, (*ObjSymbol)->st_shndx,
298                   Base::GraphSymbols.size()),
299           inconvertibleErrorCode());
300 
301     // Sanity check the relocation kind.
302     auto ELFRelocKind = getRelocationKind(Rel.getType(false));
303     if (!ELFRelocKind)
304       return ELFRelocKind.takeError();
305 
306     int64_t Addend = Rel.r_addend;
307     Edge::Kind Kind = Edge::Invalid;
308     switch (*ELFRelocKind) {
309     case PCRel32:
310       Kind = x86_64::Delta32;
311       break;
312     case Delta64:
313       Kind = x86_64::Delta64;
314       break;
315     case Pointer32Signed:
316       Kind = x86_64::Pointer32Signed;
317       break;
318     case Pointer64:
319       Kind = x86_64::Pointer64;
320       break;
321     case PCRel32GOTLoad: {
322       Kind = x86_64::RequestGOTAndTransformToDelta32;
323       break;
324     }
325     case PCRel32REXGOTLoadRelaxable: {
326       Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable;
327       Addend = 0;
328       break;
329     }
330     case PCRel32TLV: {
331       Kind = x86_64::RequestTLSDescInGOTAndTransformToDelta32;
332       break;
333     }
334     case PCRel32GOTLoadRelaxable: {
335       Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable;
336       Addend = 0;
337       break;
338     }
339     case PCRel64GOT: {
340       Kind = x86_64::RequestGOTAndTransformToDelta64;
341       break;
342     }
343     case GOT64: {
344       Kind = x86_64::RequestGOTAndTransformToDelta64FromGOT;
345       break;
346     }
347     case GOTOFF64: {
348       Kind = x86_64::Delta64FromGOT;
349       break;
350     }
351     case Branch32: {
352       Kind = x86_64::BranchPCRel32;
353       Addend = 0;
354       break;
355     }
356     }
357 
358     Block *BlockToFix = *(GraphSection.blocks().begin());
359     JITTargetAddress FixupAddress = FixupSection.sh_addr + Rel.r_offset;
360     Edge::OffsetT Offset = FixupAddress - BlockToFix->getAddress();
361     Edge GE(Kind, Offset, *GraphSymbol, Addend);
362     LLVM_DEBUG({
363       dbgs() << "    ";
364       printEdge(dbgs(), *BlockToFix, GE, getELFX86RelocationKindName(Kind));
365       dbgs() << "\n";
366     });
367 
368     BlockToFix->addEdge(std::move(GE));
369     return Error::success();
370   }
371 
372 public:
373   ELFLinkGraphBuilder_x86_64(StringRef FileName,
374                              const object::ELFFile<object::ELF64LE> &Obj)
375       : ELFLinkGraphBuilder(Obj, Triple("x86_64-unknown-linux"), FileName,
376                             x86_64::getEdgeKindName) {}
377 };
378 
379 class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> {
380   friend class JITLinker<ELFJITLinker_x86_64>;
381 
382 public:
383   ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx,
384                       std::unique_ptr<LinkGraph> G,
385                       PassConfiguration PassConfig)
386       : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {
387     getPassConfig().PostAllocationPasses.push_back(
388         [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); });
389   }
390 
391 private:
392   Symbol *GOTSymbol = nullptr;
393 
394   Error getOrCreateGOTSymbol(LinkGraph &G) {
395     auto DefineExternalGOTSymbolIfPresent =
396         createDefineExternalSectionStartAndEndSymbolsPass(
397             [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc {
398               if (Sym.getName() == ELFGOTSymbolName)
399                 if (auto *GOTSection = G.findSectionByName(ELFGOTSectionName)) {
400                   GOTSymbol = &Sym;
401                   return {*GOTSection, true};
402                 }
403               return {};
404             });
405 
406     // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an
407     // external.
408     if (auto Err = DefineExternalGOTSymbolIfPresent(G))
409       return Err;
410 
411     // If we succeeded then we're done.
412     if (GOTSymbol)
413       return Error::success();
414 
415     // Otherwise look for a GOT section: If it already has a start symbol we'll
416     // record it, otherwise we'll create our own.
417     // If there's a GOT section but we didn't find an external GOT symbol...
418     if (auto *GOTSection = G.findSectionByName(ELFGOTSectionName)) {
419 
420       // Check for an existing defined symbol.
421       for (auto *Sym : GOTSection->symbols())
422         if (Sym->getName() == ELFGOTSymbolName) {
423           GOTSymbol = Sym;
424           return Error::success();
425         }
426 
427       // If there's no defined symbol then create one.
428       SectionRange SR(*GOTSection);
429       if (SR.empty())
430         GOTSymbol = &G.addAbsoluteSymbol(ELFGOTSymbolName, 0, 0,
431                                          Linkage::Strong, Scope::Local, true);
432       else
433         GOTSymbol =
434             &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0,
435                                 Linkage::Strong, Scope::Local, false, true);
436     }
437 
438     return Error::success();
439   }
440 
441   Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const {
442     return x86_64::applyFixup(G, B, E, GOTSymbol);
443   }
444 };
445 
446 Expected<std::unique_ptr<LinkGraph>>
447 createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) {
448   LLVM_DEBUG({
449     dbgs() << "Building jitlink graph for new input "
450            << ObjectBuffer.getBufferIdentifier() << "...\n";
451   });
452 
453   auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer);
454   if (!ELFObj)
455     return ELFObj.takeError();
456 
457   auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj);
458   return ELFLinkGraphBuilder_x86_64((*ELFObj)->getFileName(),
459                                     ELFObjFile.getELFFile())
460       .buildGraph();
461 }
462 
463 static SectionRangeSymbolDesc
464 identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) {
465   constexpr StringRef StartSymbolPrefix = "__start";
466   constexpr StringRef EndSymbolPrefix = "__end";
467 
468   auto SymName = Sym.getName();
469   if (SymName.startswith(StartSymbolPrefix)) {
470     if (auto *Sec =
471             G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size())))
472       return {*Sec, true};
473   } else if (SymName.startswith(EndSymbolPrefix)) {
474     if (auto *Sec =
475             G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size())))
476       return {*Sec, false};
477   }
478   return {};
479 }
480 
481 void link_ELF_x86_64(std::unique_ptr<LinkGraph> G,
482                      std::unique_ptr<JITLinkContext> Ctx) {
483   PassConfiguration Config;
484 
485   if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) {
486 
487     Config.PrePrunePasses.push_back(EHFrameSplitter(".eh_frame"));
488     Config.PrePrunePasses.push_back(
489         EHFrameEdgeFixer(".eh_frame", x86_64::PointerSize, x86_64::Delta64,
490                          x86_64::Delta32, x86_64::NegDelta32));
491     Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame"));
492 
493     // Construct a JITLinker and run the link function.
494     // Add a mark-live pass.
495     if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple()))
496       Config.PrePrunePasses.push_back(std::move(MarkLive));
497     else
498       Config.PrePrunePasses.push_back(markAllSymbolsLive);
499 
500     // Add an in-place GOT/Stubs pass.
501 
502     Config.PostPrunePasses.push_back(PerGraphTLSInfoBuilder_ELF_x86_64::asPass);
503     Config.PostPrunePasses.push_back(
504         PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::asPass);
505 
506     // Resolve any external section start / end symbols.
507     Config.PostAllocationPasses.push_back(
508         createDefineExternalSectionStartAndEndSymbolsPass(
509             identifyELFSectionStartAndEndSymbols));
510 
511     // Add GOT/Stubs optimizer pass.
512     Config.PreFixupPasses.push_back(x86_64::optimize_x86_64_GOTAndStubs);
513   }
514 
515   if (auto Err = Ctx->modifyPassConfig(*G, Config))
516     return Ctx->notifyFailed(std::move(Err));
517 
518   ELFJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config));
519 }
520 const char *getELFX86RelocationKindName(Edge::Kind R) {
521   switch (R) {
522   case Branch32:
523     return "Branch32";
524   case Pointer32Signed:
525     return "Pointer32Signed";
526   case Pointer64:
527     return "Pointer64";
528   case PCRel32:
529     return "PCRel32";
530   case PCRel32GOTLoad:
531     return "PCRel32GOTLoad";
532   case PCRel32GOTLoadRelaxable:
533     return "PCRel32GOTLoadRelaxable";
534   case PCRel32REXGOTLoadRelaxable:
535     return "PCRel32REXGOTLoad";
536   case PCRel64GOT:
537     return "PCRel64GOT";
538   case Delta64:
539     return "Delta64";
540   case GOT64:
541     return "GOT64";
542   case GOTOFF64:
543     return "GOTOFF64";
544   }
545   return getGenericEdgeKindName(static_cast<Edge::Kind>(R));
546 }
547 } // end namespace jitlink
548 } // end namespace llvm
549