1 //===---- ELF_x86_64.cpp -JIT linker implementation for ELF/x86-64 ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // ELF/x86-64 jit-link implementation.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
14 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
15 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
16 #include "llvm/Object/ELFObjectFile.h"
17 #include "llvm/Support/Endian.h"
18 
19 #include "DefineExternalSectionStartAndEndSymbols.h"
20 #include "EHFrameSupportImpl.h"
21 #include "ELFLinkGraphBuilder.h"
22 #include "JITLinkGeneric.h"
23 #include "PerGraphGOTAndPLTStubsBuilder.h"
24 
25 #define DEBUG_TYPE "jitlink"
26 
27 using namespace llvm;
28 using namespace llvm::jitlink;
29 using namespace llvm::jitlink::ELF_x86_64_Edges;
30 
31 namespace {
32 
33 constexpr StringRef ELFGOTSectionName = "$__GOT";
34 constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_";
35 
36 class PerGraphGOTAndPLTStubsBuilder_ELF_x86_64
37     : public PerGraphGOTAndPLTStubsBuilder<
38           PerGraphGOTAndPLTStubsBuilder_ELF_x86_64> {
39 public:
40   static const uint8_t NullGOTEntryContent[8];
41   static const uint8_t StubContent[6];
42 
43   using PerGraphGOTAndPLTStubsBuilder<
44       PerGraphGOTAndPLTStubsBuilder_ELF_x86_64>::PerGraphGOTAndPLTStubsBuilder;
45 
46   bool isGOTEdgeToFix(Edge &E) const {
47     if (E.getKind() == x86_64::Delta64FromGOT) {
48       // We need to make sure that the GOT section exists, but don't otherwise
49       // need to fix up this edge.
50       getGOTSection();
51       return false;
52     }
53     return E.getKind() == x86_64::RequestGOTAndTransformToDelta32 ||
54            E.getKind() == x86_64::RequestGOTAndTransformToDelta64 ||
55            E.getKind() ==
56                x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable ||
57            E.getKind() == x86_64::RequestGOTAndTransformToDelta64FromGOT ||
58            E.getKind() ==
59                x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable;
60   }
61 
62   Symbol &createGOTEntry(Symbol &Target) {
63     auto &GOTEntryBlock = G.createContentBlock(
64         getGOTSection(), getGOTEntryBlockContent(), 0, 8, 0);
65     GOTEntryBlock.addEdge(x86_64::Pointer64, 0, Target, 0);
66     return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false);
67   }
68 
69   void fixGOTEdge(Edge &E, Symbol &GOTEntry) {
70     // If this is a PCRel32GOT/PCRel64GOT then change it to an ordinary
71     // PCRel32/PCRel64. If it is a PCRel32GOTLoad then leave it as-is for now:
72     // We will use the kind to check for GOT optimization opportunities in the
73     // optimizeMachO_x86_64_GOTAndStubs pass below.
74     // If it's a GOT64 leave it as is.
75     switch (E.getKind()) {
76     case x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable:
77       E.setKind(x86_64::PCRel32GOTLoadREXRelaxable);
78       break;
79     case x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable:
80       E.setKind(x86_64::PCRel32GOTLoadRelaxable);
81       break;
82     case x86_64::RequestGOTAndTransformToDelta64:
83       E.setKind(x86_64::Delta64);
84       break;
85     case x86_64::RequestGOTAndTransformToDelta64FromGOT:
86       E.setKind(x86_64::Delta64FromGOT);
87       break;
88     case x86_64::RequestGOTAndTransformToDelta32:
89       E.setKind(x86_64::Delta32);
90       break;
91     default:
92       llvm_unreachable("Unexpected GOT edge kind");
93     }
94 
95     E.setTarget(GOTEntry);
96     // Leave the edge addend as-is.
97   }
98 
99   bool isExternalBranchEdge(Edge &E) {
100     return E.getKind() == x86_64::BranchPCRel32 && !E.getTarget().isDefined();
101   }
102 
103   Symbol &createPLTStub(Symbol &Target) {
104     auto &StubContentBlock =
105         G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0);
106     // Re-use GOT entries for stub targets.
107     auto &GOTEntrySymbol = getGOTEntry(Target);
108     StubContentBlock.addEdge(x86_64::Delta32, 2, GOTEntrySymbol, -4);
109     return G.addAnonymousSymbol(StubContentBlock, 0, 6, true, false);
110   }
111 
112   void fixPLTEdge(Edge &E, Symbol &Stub) {
113     assert(E.getKind() == x86_64::BranchPCRel32 && "Not a Branch32 edge?");
114 
115     // Set the edge kind to Branch32ToPtrJumpStubBypassable to enable it to be
116     // optimized when the target is in-range.
117     E.setKind(x86_64::BranchPCRel32ToPtrJumpStubBypassable);
118     E.setTarget(Stub);
119   }
120 
121 private:
122   Section &getGOTSection() const {
123     if (!GOTSection)
124       GOTSection = &G.createSection(ELFGOTSectionName, sys::Memory::MF_READ);
125     return *GOTSection;
126   }
127 
128   Section &getStubsSection() const {
129     if (!StubsSection) {
130       auto StubsProt = static_cast<sys::Memory::ProtectionFlags>(
131           sys::Memory::MF_READ | sys::Memory::MF_EXEC);
132       StubsSection = &G.createSection("$__STUBS", StubsProt);
133     }
134     return *StubsSection;
135   }
136 
137   ArrayRef<char> getGOTEntryBlockContent() {
138     return {reinterpret_cast<const char *>(NullGOTEntryContent),
139             sizeof(NullGOTEntryContent)};
140   }
141 
142   ArrayRef<char> getStubBlockContent() {
143     return {reinterpret_cast<const char *>(StubContent), sizeof(StubContent)};
144   }
145 
146   mutable Section *GOTSection = nullptr;
147   mutable Section *StubsSection = nullptr;
148 };
149 
150 } // namespace
151 
152 const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::NullGOTEntryContent[8] =
153     {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
154 const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::StubContent[6] = {
155     0xFF, 0x25, 0x00, 0x00, 0x00, 0x00};
156 
157 static const char *getELFX86_64RelocName(uint32_t Type) {
158   switch (Type) {
159 #define ELF_RELOC(Name, Number)                                                \
160   case Number:                                                                 \
161     return #Name;
162 #include "llvm/BinaryFormat/ELFRelocs/x86_64.def"
163 #undef ELF_RELOC
164   }
165   return "Unrecognized ELF/x86-64 relocation type";
166 }
167 
168 namespace llvm {
169 namespace jitlink {
170 
171 // This should become a template as the ELFFile is so a lot of this could become
172 // generic
173 class ELFLinkGraphBuilder_x86_64 : public ELFLinkGraphBuilder<object::ELF64LE> {
174 private:
175 
176   static Expected<ELF_x86_64_Edges::ELFX86RelocationKind>
177   getRelocationKind(const uint32_t Type) {
178     switch (Type) {
179     case ELF::R_X86_64_32S:
180       return ELF_x86_64_Edges::ELFX86RelocationKind::Pointer32Signed;
181     case ELF::R_X86_64_PC32:
182       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32;
183     case ELF::R_X86_64_PC64:
184     case ELF::R_X86_64_GOTPC64:
185       return ELF_x86_64_Edges::ELFX86RelocationKind::Delta64;
186     case ELF::R_X86_64_64:
187       return ELF_x86_64_Edges::ELFX86RelocationKind::Pointer64;
188     case ELF::R_X86_64_GOTPCREL:
189       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32GOTLoad;
190     case ELF::R_X86_64_GOTPCRELX:
191       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32GOTLoadRelaxable;
192     case ELF::R_X86_64_REX_GOTPCRELX:
193       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32REXGOTLoadRelaxable;
194     case ELF::R_X86_64_GOTPCREL64:
195       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel64GOT;
196     case ELF::R_X86_64_GOT64:
197       return ELF_x86_64_Edges::ELFX86RelocationKind::GOT64;
198     case ELF::R_X86_64_GOTOFF64:
199       return ELF_x86_64_Edges::ELFX86RelocationKind::GOTOFF64;
200     case ELF::R_X86_64_PLT32:
201       return ELF_x86_64_Edges::ELFX86RelocationKind::Branch32;
202     }
203     return make_error<JITLinkError>("Unsupported x86-64 relocation type " +
204                                     formatv("{0:d}: ", Type) +
205                                     getELFX86_64RelocName(Type));
206   }
207 
208   Error addRelocations() override {
209     LLVM_DEBUG(dbgs() << "Adding relocations\n");
210     // TODO a partern is forming of iterate some sections but only give me
211     // ones I am interested, i should abstract that concept some where
212     for (auto &SecRef : Sections) {
213       if (SecRef.sh_type != ELF::SHT_RELA && SecRef.sh_type != ELF::SHT_REL)
214         continue;
215       // TODO can the elf obj file do this for me?
216       if (SecRef.sh_type == ELF::SHT_REL)
217         return make_error<llvm::StringError>("Shouldn't have REL in x64",
218                                              llvm::inconvertibleErrorCode());
219 
220       auto RelSectName = Obj.getSectionName(SecRef);
221       if (!RelSectName)
222         return RelSectName.takeError();
223 
224       LLVM_DEBUG({
225         dbgs() << "Adding relocations from section " << *RelSectName << "\n";
226       });
227 
228       auto UpdateSection = Obj.getSection(SecRef.sh_info);
229       if (!UpdateSection)
230         return UpdateSection.takeError();
231 
232       auto UpdateSectionName = Obj.getSectionName(**UpdateSection);
233       if (!UpdateSectionName)
234         return UpdateSectionName.takeError();
235 
236       // Don't process relocations for debug sections.
237       if (isDwarfSection(*UpdateSectionName)) {
238         LLVM_DEBUG({
239           dbgs() << "  Target is dwarf section " << *UpdateSectionName
240                  << ". Skipping.\n";
241         });
242         continue;
243       } else
244         LLVM_DEBUG({
245           dbgs() << "  For target section " << *UpdateSectionName << "\n";
246         });
247 
248       auto JITSection = G->findSectionByName(*UpdateSectionName);
249       if (!JITSection)
250         return make_error<llvm::StringError>(
251             "Refencing a a section that wasn't added to graph" +
252                 *UpdateSectionName,
253             llvm::inconvertibleErrorCode());
254 
255       auto Relocations = Obj.relas(SecRef);
256       if (!Relocations)
257         return Relocations.takeError();
258 
259       for (const auto &Rela : *Relocations) {
260         auto Type = Rela.getType(false);
261 
262         LLVM_DEBUG({
263           dbgs() << "Relocation Type: " << Type << "\n"
264                  << "Name: " << Obj.getRelocationTypeName(Type) << "\n";
265         });
266         auto SymbolIndex = Rela.getSymbol(false);
267         auto Symbol = Obj.getRelocationSymbol(Rela, SymTabSec);
268         if (!Symbol)
269           return Symbol.takeError();
270 
271         auto BlockToFix = *(JITSection->blocks().begin());
272         auto *TargetSymbol = getGraphSymbol(SymbolIndex);
273 
274         if (!TargetSymbol) {
275           return make_error<llvm::StringError>(
276               "Could not find symbol at given index, did you add it to "
277               "JITSymbolTable? index: " +
278                   std::to_string(SymbolIndex) +
279                   ", shndx: " + std::to_string((*Symbol)->st_shndx) +
280                   " Size of table: " + std::to_string(GraphSymbols.size()),
281               llvm::inconvertibleErrorCode());
282         }
283         int64_t Addend = Rela.r_addend;
284         JITTargetAddress FixupAddress =
285             (*UpdateSection)->sh_addr + Rela.r_offset;
286 
287         LLVM_DEBUG({
288           dbgs() << "Processing relocation at "
289                  << format("0x%016" PRIx64, FixupAddress) << "\n";
290         });
291         auto ELFRelocKind = getRelocationKind(Type);
292         if (!ELFRelocKind)
293           return ELFRelocKind.takeError();
294 
295         Edge::Kind Kind = Edge::Invalid;
296         switch (*ELFRelocKind) {
297         case PCRel32:
298           Kind = x86_64::Delta32;
299           break;
300         case Delta64:
301           Kind = x86_64::Delta64;
302           break;
303         case Pointer32Signed:
304           Kind = x86_64::Pointer32Signed;
305           break;
306         case Pointer64:
307           Kind = x86_64::Pointer64;
308           break;
309         case PCRel32GOTLoad: {
310           Kind = x86_64::RequestGOTAndTransformToDelta32;
311           break;
312         }
313         case PCRel32REXGOTLoadRelaxable: {
314           Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable;
315           Addend = 0;
316           break;
317         }
318         case PCRel32GOTLoadRelaxable: {
319           Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable;
320           Addend = 0;
321           break;
322         }
323         case PCRel64GOT: {
324           Kind = x86_64::RequestGOTAndTransformToDelta64;
325           break;
326         }
327         case GOT64: {
328           Kind = x86_64::RequestGOTAndTransformToDelta64FromGOT;
329           break;
330         }
331         case GOTOFF64: {
332           Kind = x86_64::Delta64FromGOT;
333           break;
334         }
335         case Branch32: {
336           Kind = x86_64::BranchPCRel32;
337           Addend = 0;
338           break;
339         }
340         }
341 
342         LLVM_DEBUG({
343           Edge GE(Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol,
344                   Addend);
345           printEdge(dbgs(), *BlockToFix, GE, getELFX86RelocationKindName(Kind));
346           dbgs() << "\n";
347         });
348         BlockToFix->addEdge(Kind, FixupAddress - BlockToFix->getAddress(),
349                             *TargetSymbol, Addend);
350       }
351     }
352     return Error::success();
353   }
354 
355 public:
356   ELFLinkGraphBuilder_x86_64(StringRef FileName,
357                              const object::ELFFile<object::ELF64LE> &Obj)
358       : ELFLinkGraphBuilder(Obj, Triple("x86_64-unknown-linux"), FileName,
359                             x86_64::getEdgeKindName) {}
360 };
361 
362 class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> {
363   friend class JITLinker<ELFJITLinker_x86_64>;
364 
365 public:
366   ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx,
367                       std::unique_ptr<LinkGraph> G,
368                       PassConfiguration PassConfig)
369       : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {
370     getPassConfig().PostAllocationPasses.push_back(
371         [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); });
372   }
373 
374 private:
375   Symbol *GOTSymbol = nullptr;
376 
377   Error getOrCreateGOTSymbol(LinkGraph &G) {
378     auto DefineExternalGOTSymbolIfPresent =
379         createDefineExternalSectionStartAndEndSymbolsPass(
380             [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc {
381               if (Sym.getName() == ELFGOTSymbolName)
382                 if (auto *GOTSection = G.findSectionByName(ELFGOTSectionName)) {
383                   GOTSymbol = &Sym;
384                   return {*GOTSection, true};
385                 }
386               return {};
387             });
388 
389     // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an
390     // external.
391     if (auto Err = DefineExternalGOTSymbolIfPresent(G))
392       return Err;
393 
394     // If we succeeded then we're done.
395     if (GOTSymbol)
396       return Error::success();
397 
398     // Otherwise look for a GOT section: If it already has a start symbol we'll
399     // record it, otherwise we'll create our own.
400     // If there's a GOT section but we didn't find an external GOT symbol...
401     if (auto *GOTSection = G.findSectionByName(ELFGOTSectionName)) {
402 
403       // Check for an existing defined symbol.
404       for (auto *Sym : GOTSection->symbols())
405         if (Sym->getName() == ELFGOTSymbolName) {
406           GOTSymbol = Sym;
407           return Error::success();
408         }
409 
410       // If there's no defined symbol then create one.
411       SectionRange SR(*GOTSection);
412       if (SR.empty())
413         GOTSymbol = &G.addAbsoluteSymbol(ELFGOTSymbolName, 0, 0,
414                                          Linkage::Strong, Scope::Local, true);
415       else
416         GOTSymbol =
417             &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0,
418                                 Linkage::Strong, Scope::Local, false, true);
419     }
420 
421     return Error::success();
422   }
423 
424   Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const {
425     return x86_64::applyFixup(G, B, E, GOTSymbol);
426   }
427 };
428 
429 Expected<std::unique_ptr<LinkGraph>>
430 createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) {
431   LLVM_DEBUG({
432     dbgs() << "Building jitlink graph for new input "
433            << ObjectBuffer.getBufferIdentifier() << "...\n";
434   });
435 
436   auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer);
437   if (!ELFObj)
438     return ELFObj.takeError();
439 
440   auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj);
441   return ELFLinkGraphBuilder_x86_64((*ELFObj)->getFileName(),
442                                     ELFObjFile.getELFFile())
443       .buildGraph();
444 }
445 
446 static SectionRangeSymbolDesc
447 identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) {
448   constexpr StringRef StartSymbolPrefix = "__start";
449   constexpr StringRef EndSymbolPrefix = "__end";
450 
451   auto SymName = Sym.getName();
452   if (SymName.startswith(StartSymbolPrefix)) {
453     if (auto *Sec =
454             G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size())))
455       return {*Sec, true};
456   } else if (SymName.startswith(EndSymbolPrefix)) {
457     if (auto *Sec =
458             G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size())))
459       return {*Sec, false};
460   }
461   return {};
462 }
463 
464 void link_ELF_x86_64(std::unique_ptr<LinkGraph> G,
465                      std::unique_ptr<JITLinkContext> Ctx) {
466   PassConfiguration Config;
467 
468   if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) {
469 
470     Config.PrePrunePasses.push_back(EHFrameSplitter(".eh_frame"));
471     Config.PrePrunePasses.push_back(
472         EHFrameEdgeFixer(".eh_frame", x86_64::PointerSize, x86_64::Delta64,
473                          x86_64::Delta32, x86_64::NegDelta32));
474     Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame"));
475 
476     // Construct a JITLinker and run the link function.
477     // Add a mark-live pass.
478     if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple()))
479       Config.PrePrunePasses.push_back(std::move(MarkLive));
480     else
481       Config.PrePrunePasses.push_back(markAllSymbolsLive);
482 
483     // Add an in-place GOT/Stubs pass.
484     Config.PostPrunePasses.push_back(
485         PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::asPass);
486 
487     // Resolve any external section start / end symbols.
488     Config.PostAllocationPasses.push_back(
489         createDefineExternalSectionStartAndEndSymbolsPass(
490             identifyELFSectionStartAndEndSymbols));
491 
492     // Add GOT/Stubs optimizer pass.
493     Config.PreFixupPasses.push_back(x86_64::optimize_x86_64_GOTAndStubs);
494   }
495 
496   if (auto Err = Ctx->modifyPassConfig(*G, Config))
497     return Ctx->notifyFailed(std::move(Err));
498 
499   ELFJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config));
500 }
501 const char *getELFX86RelocationKindName(Edge::Kind R) {
502   switch (R) {
503   case Branch32:
504     return "Branch32";
505   case Pointer32Signed:
506     return "Pointer32Signed";
507   case Pointer64:
508     return "Pointer64";
509   case PCRel32:
510     return "PCRel32";
511   case PCRel32GOTLoad:
512     return "PCRel32GOTLoad";
513   case PCRel32GOTLoadRelaxable:
514     return "PCRel32GOTLoadRelaxable";
515   case PCRel32REXGOTLoadRelaxable:
516     return "PCRel32REXGOTLoad";
517   case PCRel64GOT:
518     return "PCRel64GOT";
519   case Delta64:
520     return "Delta64";
521   case GOT64:
522     return "GOT64";
523   case GOTOFF64:
524     return "GOTOFF64";
525   }
526   return getGenericEdgeKindName(static_cast<Edge::Kind>(R));
527 }
528 } // end namespace jitlink
529 } // end namespace llvm
530