1 //===---- ELF_x86_64.cpp -JIT linker implementation for ELF/x86-64 ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // ELF/x86-64 jit-link implementation.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
14 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
15 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
16 #include "llvm/Object/ELFObjectFile.h"
17 #include "llvm/Support/Endian.h"
18 
19 #include "DefineExternalSectionStartAndEndSymbols.h"
20 #include "EHFrameSupportImpl.h"
21 #include "ELFLinkGraphBuilder.h"
22 #include "JITLinkGeneric.h"
23 #include "PerGraphGOTAndPLTStubsBuilder.h"
24 
25 #define DEBUG_TYPE "jitlink"
26 
27 using namespace llvm;
28 using namespace llvm::jitlink;
29 using namespace llvm::jitlink::ELF_x86_64_Edges;
30 
31 namespace {
32 
33 constexpr StringRef ELFGOTSectionName = "$__GOT";
34 constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_";
35 
36 class PerGraphGOTAndPLTStubsBuilder_ELF_x86_64
37     : public PerGraphGOTAndPLTStubsBuilder<
38           PerGraphGOTAndPLTStubsBuilder_ELF_x86_64> {
39 public:
40   static const uint8_t NullGOTEntryContent[8];
41   static const uint8_t StubContent[6];
42 
43   using PerGraphGOTAndPLTStubsBuilder<
44       PerGraphGOTAndPLTStubsBuilder_ELF_x86_64>::PerGraphGOTAndPLTStubsBuilder;
45 
46   bool isGOTEdgeToFix(Edge &E) const {
47     if (E.getKind() == x86_64::Delta64FromGOT) {
48       // We need to make sure that the GOT section exists, but don't otherwise
49       // need to fix up this edge.
50       getGOTSection();
51       return false;
52     }
53     return E.getKind() == x86_64::RequestGOTAndTransformToDelta32 ||
54            E.getKind() == x86_64::RequestGOTAndTransformToDelta64 ||
55            E.getKind() ==
56                x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable ||
57            E.getKind() == x86_64::RequestGOTAndTransformToDelta64FromGOT ||
58            E.getKind() ==
59                x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable;
60   }
61 
62   Symbol &createGOTEntry(Symbol &Target) {
63     auto &GOTEntryBlock = G.createContentBlock(
64         getGOTSection(), getGOTEntryBlockContent(), 0, 8, 0);
65     GOTEntryBlock.addEdge(x86_64::Pointer64, 0, Target, 0);
66     return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false);
67   }
68 
69   void fixGOTEdge(Edge &E, Symbol &GOTEntry) {
70     // If this is a PCRel32GOT/PCRel64GOT then change it to an ordinary
71     // PCRel32/PCRel64. If it is a PCRel32GOTLoad then leave it as-is for now:
72     // We will use the kind to check for GOT optimization opportunities in the
73     // optimizeMachO_x86_64_GOTAndStubs pass below.
74     // If it's a GOT64 leave it as is.
75     switch (E.getKind()) {
76     case x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable:
77       E.setKind(x86_64::PCRel32GOTLoadREXRelaxable);
78       break;
79     case x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable:
80       E.setKind(x86_64::PCRel32GOTLoadRelaxable);
81       break;
82     case x86_64::RequestGOTAndTransformToDelta64:
83       E.setKind(x86_64::Delta64);
84       break;
85     case x86_64::RequestGOTAndTransformToDelta64FromGOT:
86       E.setKind(x86_64::Delta64FromGOT);
87       break;
88     case x86_64::RequestGOTAndTransformToDelta32:
89       E.setKind(x86_64::Delta32);
90       break;
91     default:
92       llvm_unreachable("Unexpected GOT edge kind");
93     }
94 
95     E.setTarget(GOTEntry);
96     // Leave the edge addend as-is.
97   }
98 
99   bool isExternalBranchEdge(Edge &E) {
100     return E.getKind() == x86_64::BranchPCRel32 && !E.getTarget().isDefined();
101   }
102 
103   Symbol &createPLTStub(Symbol &Target) {
104     auto &StubContentBlock =
105         G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0);
106     // Re-use GOT entries for stub targets.
107     auto &GOTEntrySymbol = getGOTEntry(Target);
108     StubContentBlock.addEdge(x86_64::Delta32, 2, GOTEntrySymbol, -4);
109     return G.addAnonymousSymbol(StubContentBlock, 0, 6, true, false);
110   }
111 
112   void fixPLTEdge(Edge &E, Symbol &Stub) {
113     assert(E.getKind() == x86_64::BranchPCRel32 && "Not a Branch32 edge?");
114 
115     // Set the edge kind to Branch32ToPtrJumpStubBypassable to enable it to be
116     // optimized when the target is in-range.
117     E.setKind(x86_64::BranchPCRel32ToPtrJumpStubBypassable);
118     E.setTarget(Stub);
119   }
120 
121 private:
122   Section &getGOTSection() const {
123     if (!GOTSection)
124       GOTSection = &G.createSection(ELFGOTSectionName, sys::Memory::MF_READ);
125     return *GOTSection;
126   }
127 
128   Section &getStubsSection() const {
129     if (!StubsSection) {
130       auto StubsProt = static_cast<sys::Memory::ProtectionFlags>(
131           sys::Memory::MF_READ | sys::Memory::MF_EXEC);
132       StubsSection = &G.createSection("$__STUBS", StubsProt);
133     }
134     return *StubsSection;
135   }
136 
137   ArrayRef<char> getGOTEntryBlockContent() {
138     return {reinterpret_cast<const char *>(NullGOTEntryContent),
139             sizeof(NullGOTEntryContent)};
140   }
141 
142   ArrayRef<char> getStubBlockContent() {
143     return {reinterpret_cast<const char *>(StubContent), sizeof(StubContent)};
144   }
145 
146   mutable Section *GOTSection = nullptr;
147   mutable Section *StubsSection = nullptr;
148 };
149 
150 } // namespace
151 
152 const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::NullGOTEntryContent[8] =
153     {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
154 const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::StubContent[6] = {
155     0xFF, 0x25, 0x00, 0x00, 0x00, 0x00};
156 
157 static const char *getELFX86_64RelocName(uint32_t Type) {
158   switch (Type) {
159 #define ELF_RELOC(Name, Number)                                                \
160   case Number:                                                                 \
161     return #Name;
162 #include "llvm/BinaryFormat/ELFRelocs/x86_64.def"
163 #undef ELF_RELOC
164   }
165   return "Unrecognized ELF/x86-64 relocation type";
166 }
167 
168 namespace llvm {
169 namespace jitlink {
170 
171 // This should become a template as the ELFFile is so a lot of this could become
172 // generic
173 class ELFLinkGraphBuilder_x86_64 : public ELFLinkGraphBuilder<object::ELF64LE> {
174 private:
175 
176   static Expected<ELF_x86_64_Edges::ELFX86RelocationKind>
177   getRelocationKind(const uint32_t Type) {
178     switch (Type) {
179     case ELF::R_X86_64_PC32:
180       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32;
181     case ELF::R_X86_64_PC64:
182     case ELF::R_X86_64_GOTPC64:
183       return ELF_x86_64_Edges::ELFX86RelocationKind::Delta64;
184     case ELF::R_X86_64_64:
185       return ELF_x86_64_Edges::ELFX86RelocationKind::Pointer64;
186     case ELF::R_X86_64_GOTPCREL:
187       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32GOTLoad;
188     case ELF::R_X86_64_GOTPCRELX:
189       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32GOTLoadRelaxable;
190     case ELF::R_X86_64_REX_GOTPCRELX:
191       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32REXGOTLoadRelaxable;
192     case ELF::R_X86_64_GOTPCREL64:
193       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel64GOT;
194     case ELF::R_X86_64_GOT64:
195       return ELF_x86_64_Edges::ELFX86RelocationKind::GOT64;
196     case ELF::R_X86_64_GOTOFF64:
197       return ELF_x86_64_Edges::ELFX86RelocationKind::GOTOFF64;
198     case ELF::R_X86_64_PLT32:
199       return ELF_x86_64_Edges::ELFX86RelocationKind::Branch32;
200     }
201     return make_error<JITLinkError>("Unsupported x86-64 relocation type " +
202                                     formatv("{0:d}: ", Type) +
203                                     getELFX86_64RelocName(Type));
204   }
205 
206   Error addRelocations() override {
207     LLVM_DEBUG(dbgs() << "Adding relocations\n");
208     // TODO a partern is forming of iterate some sections but only give me
209     // ones I am interested, i should abstract that concept some where
210     for (auto &SecRef : Sections) {
211       if (SecRef.sh_type != ELF::SHT_RELA && SecRef.sh_type != ELF::SHT_REL)
212         continue;
213       // TODO can the elf obj file do this for me?
214       if (SecRef.sh_type == ELF::SHT_REL)
215         return make_error<llvm::StringError>("Shouldn't have REL in x64",
216                                              llvm::inconvertibleErrorCode());
217 
218       auto RelSectName = Obj.getSectionName(SecRef);
219       if (!RelSectName)
220         return RelSectName.takeError();
221 
222       LLVM_DEBUG({
223         dbgs() << "Adding relocations from section " << *RelSectName << "\n";
224       });
225 
226       auto UpdateSection = Obj.getSection(SecRef.sh_info);
227       if (!UpdateSection)
228         return UpdateSection.takeError();
229 
230       auto UpdateSectionName = Obj.getSectionName(**UpdateSection);
231       if (!UpdateSectionName)
232         return UpdateSectionName.takeError();
233 
234       // Don't process relocations for debug sections.
235       if (isDwarfSection(*UpdateSectionName)) {
236         LLVM_DEBUG({
237           dbgs() << "  Target is dwarf section " << *UpdateSectionName
238                  << ". Skipping.\n";
239         });
240         continue;
241       } else
242         LLVM_DEBUG({
243           dbgs() << "  For target section " << *UpdateSectionName << "\n";
244         });
245 
246       auto JITSection = G->findSectionByName(*UpdateSectionName);
247       if (!JITSection)
248         return make_error<llvm::StringError>(
249             "Refencing a a section that wasn't added to graph" +
250                 *UpdateSectionName,
251             llvm::inconvertibleErrorCode());
252 
253       auto Relocations = Obj.relas(SecRef);
254       if (!Relocations)
255         return Relocations.takeError();
256 
257       for (const auto &Rela : *Relocations) {
258         auto Type = Rela.getType(false);
259 
260         LLVM_DEBUG({
261           dbgs() << "Relocation Type: " << Type << "\n"
262                  << "Name: " << Obj.getRelocationTypeName(Type) << "\n";
263         });
264         auto SymbolIndex = Rela.getSymbol(false);
265         auto Symbol = Obj.getRelocationSymbol(Rela, SymTabSec);
266         if (!Symbol)
267           return Symbol.takeError();
268 
269         auto BlockToFix = *(JITSection->blocks().begin());
270         auto *TargetSymbol = getGraphSymbol(SymbolIndex);
271 
272         if (!TargetSymbol) {
273           return make_error<llvm::StringError>(
274               "Could not find symbol at given index, did you add it to "
275               "JITSymbolTable? index: " +
276                   std::to_string(SymbolIndex) +
277                   ", shndx: " + std::to_string((*Symbol)->st_shndx) +
278                   " Size of table: " + std::to_string(GraphSymbols.size()),
279               llvm::inconvertibleErrorCode());
280         }
281         int64_t Addend = Rela.r_addend;
282         JITTargetAddress FixupAddress =
283             (*UpdateSection)->sh_addr + Rela.r_offset;
284 
285         LLVM_DEBUG({
286           dbgs() << "Processing relocation at "
287                  << format("0x%016" PRIx64, FixupAddress) << "\n";
288         });
289         auto ELFRelocKind = getRelocationKind(Type);
290         if (!ELFRelocKind)
291           return ELFRelocKind.takeError();
292 
293         Edge::Kind Kind = Edge::Invalid;
294         switch (*ELFRelocKind) {
295         case PCRel32:
296           Kind = x86_64::Delta32;
297           break;
298         case Delta64:
299           Kind = x86_64::Delta64;
300           break;
301         case Pointer64:
302           Kind = x86_64::Pointer64;
303           break;
304         case PCRel32GOTLoad: {
305           Kind = x86_64::RequestGOTAndTransformToDelta32;
306           break;
307         }
308         case PCRel32REXGOTLoadRelaxable: {
309           Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable;
310           Addend = 0;
311           break;
312         }
313         case PCRel32GOTLoadRelaxable: {
314           Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable;
315           Addend = 0;
316           break;
317         }
318         case PCRel64GOT: {
319           Kind = x86_64::RequestGOTAndTransformToDelta64;
320           break;
321         }
322         case GOT64: {
323           Kind = x86_64::RequestGOTAndTransformToDelta64FromGOT;
324           break;
325         }
326         case GOTOFF64: {
327           Kind = x86_64::Delta64FromGOT;
328           break;
329         }
330         case Branch32: {
331           Kind = x86_64::BranchPCRel32;
332           Addend = 0;
333           break;
334         }
335         }
336 
337         LLVM_DEBUG({
338           Edge GE(Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol,
339                   Addend);
340           printEdge(dbgs(), *BlockToFix, GE, getELFX86RelocationKindName(Kind));
341           dbgs() << "\n";
342         });
343         BlockToFix->addEdge(Kind, FixupAddress - BlockToFix->getAddress(),
344                             *TargetSymbol, Addend);
345       }
346     }
347     return Error::success();
348   }
349 
350 public:
351   ELFLinkGraphBuilder_x86_64(StringRef FileName,
352                              const object::ELFFile<object::ELF64LE> &Obj)
353       : ELFLinkGraphBuilder(Obj, Triple("x86_64-unknown-linux"), FileName,
354                             x86_64::getEdgeKindName) {}
355 };
356 
357 class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> {
358   friend class JITLinker<ELFJITLinker_x86_64>;
359 
360 public:
361   ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx,
362                       std::unique_ptr<LinkGraph> G,
363                       PassConfiguration PassConfig)
364       : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {
365     getPassConfig().PostAllocationPasses.push_back(
366         [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); });
367   }
368 
369 private:
370   Symbol *GOTSymbol = nullptr;
371 
372   Error getOrCreateGOTSymbol(LinkGraph &G) {
373     auto DefineExternalGOTSymbolIfPresent =
374         createDefineExternalSectionStartAndEndSymbolsPass(
375             [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc {
376               if (Sym.getName() == ELFGOTSymbolName)
377                 if (auto *GOTSection = G.findSectionByName(ELFGOTSectionName)) {
378                   GOTSymbol = &Sym;
379                   return {*GOTSection, true};
380                 }
381               return {};
382             });
383 
384     // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an
385     // external.
386     if (auto Err = DefineExternalGOTSymbolIfPresent(G))
387       return Err;
388 
389     // If we succeeded then we're done.
390     if (GOTSymbol)
391       return Error::success();
392 
393     // Otherwise look for a GOT section: If it already has a start symbol we'll
394     // record it, otherwise we'll create our own.
395     // If there's a GOT section but we didn't find an external GOT symbol...
396     if (auto *GOTSection = G.findSectionByName(ELFGOTSectionName)) {
397 
398       // Check for an existing defined symbol.
399       for (auto *Sym : GOTSection->symbols())
400         if (Sym->getName() == ELFGOTSymbolName) {
401           GOTSymbol = Sym;
402           return Error::success();
403         }
404 
405       // If there's no defined symbol then create one.
406       SectionRange SR(*GOTSection);
407       if (SR.empty())
408         GOTSymbol = &G.addAbsoluteSymbol(ELFGOTSymbolName, 0, 0,
409                                          Linkage::Strong, Scope::Local, true);
410       else
411         GOTSymbol =
412             &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0,
413                                 Linkage::Strong, Scope::Local, false, true);
414     }
415 
416     return Error::success();
417   }
418 
419   Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const {
420     return x86_64::applyFixup(G, B, E, GOTSymbol);
421   }
422 };
423 
424 Expected<std::unique_ptr<LinkGraph>>
425 createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) {
426   LLVM_DEBUG({
427     dbgs() << "Building jitlink graph for new input "
428            << ObjectBuffer.getBufferIdentifier() << "...\n";
429   });
430 
431   auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer);
432   if (!ELFObj)
433     return ELFObj.takeError();
434 
435   auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj);
436   return ELFLinkGraphBuilder_x86_64((*ELFObj)->getFileName(),
437                                     ELFObjFile.getELFFile())
438       .buildGraph();
439 }
440 
441 static SectionRangeSymbolDesc
442 identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) {
443   constexpr StringRef StartSymbolPrefix = "__start";
444   constexpr StringRef EndSymbolPrefix = "__end";
445 
446   auto SymName = Sym.getName();
447   if (SymName.startswith(StartSymbolPrefix)) {
448     if (auto *Sec =
449             G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size())))
450       return {*Sec, true};
451   } else if (SymName.startswith(EndSymbolPrefix)) {
452     if (auto *Sec =
453             G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size())))
454       return {*Sec, false};
455   }
456   return {};
457 }
458 
459 void link_ELF_x86_64(std::unique_ptr<LinkGraph> G,
460                      std::unique_ptr<JITLinkContext> Ctx) {
461   PassConfiguration Config;
462 
463   if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) {
464 
465     Config.PrePrunePasses.push_back(EHFrameSplitter(".eh_frame"));
466     Config.PrePrunePasses.push_back(
467         EHFrameEdgeFixer(".eh_frame", x86_64::PointerSize, x86_64::Delta64,
468                          x86_64::Delta32, x86_64::NegDelta32));
469     Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame"));
470 
471     // Construct a JITLinker and run the link function.
472     // Add a mark-live pass.
473     if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple()))
474       Config.PrePrunePasses.push_back(std::move(MarkLive));
475     else
476       Config.PrePrunePasses.push_back(markAllSymbolsLive);
477 
478     // Add an in-place GOT/Stubs pass.
479     Config.PostPrunePasses.push_back(
480         PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::asPass);
481 
482     // Resolve any external section start / end symbols.
483     Config.PostAllocationPasses.push_back(
484         createDefineExternalSectionStartAndEndSymbolsPass(
485             identifyELFSectionStartAndEndSymbols));
486 
487     // Add GOT/Stubs optimizer pass.
488     Config.PreFixupPasses.push_back(x86_64::optimize_x86_64_GOTAndStubs);
489   }
490 
491   if (auto Err = Ctx->modifyPassConfig(*G, Config))
492     return Ctx->notifyFailed(std::move(Err));
493 
494   ELFJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config));
495 }
496 const char *getELFX86RelocationKindName(Edge::Kind R) {
497   switch (R) {
498   case Branch32:
499     return "Branch32";
500   case Pointer64:
501     return "Pointer64";
502   case PCRel32:
503     return "PCRel32";
504   case PCRel32GOTLoad:
505     return "PCRel32GOTLoad";
506   case PCRel32GOTLoadRelaxable:
507     return "PCRel32GOTLoadRelaxable";
508   case PCRel32REXGOTLoadRelaxable:
509     return "PCRel32REXGOTLoad";
510   case PCRel64GOT:
511     return "PCRel64GOT";
512   case Delta64:
513     return "Delta64";
514   case GOT64:
515     return "GOT64";
516   case GOTOFF64:
517     return "GOTOFF64";
518   }
519   return getGenericEdgeKindName(static_cast<Edge::Kind>(R));
520 }
521 } // end namespace jitlink
522 } // end namespace llvm
523