1 //===---- ELF_x86_64.cpp -JIT linker implementation for ELF/x86-64 ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // ELF/x86-64 jit-link implementation.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
14 #include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h"
15 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
16 #include "llvm/ExecutionEngine/JITLink/TableManager.h"
17 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
18 #include "llvm/Object/ELFObjectFile.h"
19 #include "llvm/Support/Endian.h"
20 
21 #include "DefineExternalSectionStartAndEndSymbols.h"
22 #include "EHFrameSupportImpl.h"
23 #include "ELFLinkGraphBuilder.h"
24 #include "JITLinkGeneric.h"
25 
26 #define DEBUG_TYPE "jitlink"
27 
28 using namespace llvm;
29 using namespace llvm::jitlink;
30 using namespace llvm::jitlink::ELF_x86_64_Edges;
31 
32 namespace {
33 
34 constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_";
35 constexpr StringRef ELFTLSInfoSectionName = "$__TLSINFO";
36 
37 class TLSInfoTableManager_ELF_x86_64
38     : public TableManager<TLSInfoTableManager_ELF_x86_64> {
39 public:
40   static const uint8_t TLSInfoEntryContent[16];
41 
getSectionName()42   static StringRef getSectionName() { return ELFTLSInfoSectionName; }
43 
visitEdge(LinkGraph & G,Block * B,Edge & E)44   bool visitEdge(LinkGraph &G, Block *B, Edge &E) {
45     if (E.getKind() == x86_64::RequestTLSDescInGOTAndTransformToDelta32) {
46       LLVM_DEBUG({
47         dbgs() << "  Fixing " << G.getEdgeKindName(E.getKind()) << " edge at "
48                << formatv("{0:x}", B->getFixupAddress(E)) << " ("
49                << formatv("{0:x}", B->getAddress()) << " + "
50                << formatv("{0:x}", E.getOffset()) << ")\n";
51       });
52       E.setKind(x86_64::Delta32);
53       E.setTarget(getEntryForTarget(G, E.getTarget()));
54       return true;
55     }
56     return false;
57   }
58 
createEntry(LinkGraph & G,Symbol & Target)59   Symbol &createEntry(LinkGraph &G, Symbol &Target) {
60     // the TLS Info entry's key value will be written by the fixTLVSectionByName
61     // pass, so create mutable content.
62     auto &TLSInfoEntry = G.createMutableContentBlock(
63         getTLSInfoSection(G), G.allocateContent(getTLSInfoEntryContent()),
64         orc::ExecutorAddr(), 8, 0);
65     TLSInfoEntry.addEdge(x86_64::Pointer64, 8, Target, 0);
66     return G.addAnonymousSymbol(TLSInfoEntry, 0, 16, false, false);
67   }
68 
69 private:
getTLSInfoSection(LinkGraph & G)70   Section &getTLSInfoSection(LinkGraph &G) {
71     if (!TLSInfoTable)
72       TLSInfoTable = &G.createSection(ELFTLSInfoSectionName, MemProt::Read);
73     return *TLSInfoTable;
74   }
75 
getTLSInfoEntryContent() const76   ArrayRef<char> getTLSInfoEntryContent() const {
77     return {reinterpret_cast<const char *>(TLSInfoEntryContent),
78             sizeof(TLSInfoEntryContent)};
79   }
80 
81   Section *TLSInfoTable = nullptr;
82 };
83 
84 const uint8_t TLSInfoTableManager_ELF_x86_64::TLSInfoEntryContent[16] = {
85     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*pthread key */
86     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00  /*data address*/
87 };
88 
buildTables_ELF_x86_64(LinkGraph & G)89 Error buildTables_ELF_x86_64(LinkGraph &G) {
90   LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n");
91 
92   x86_64::GOTTableManager GOT;
93   x86_64::PLTTableManager PLT(GOT);
94   TLSInfoTableManager_ELF_x86_64 TLSInfo;
95   visitExistingEdges(G, GOT, PLT, TLSInfo);
96   return Error::success();
97 }
98 } // namespace
99 
100 namespace llvm {
101 namespace jitlink {
102 
103 // This should become a template as the ELFFile is so a lot of this could become
104 // generic
105 class ELFLinkGraphBuilder_x86_64 : public ELFLinkGraphBuilder<object::ELF64LE> {
106 private:
107   using ELFT = object::ELF64LE;
108 
109   static Expected<ELF_x86_64_Edges::ELFX86RelocationKind>
getRelocationKind(const uint32_t Type)110   getRelocationKind(const uint32_t Type) {
111     switch (Type) {
112     case ELF::R_X86_64_32S:
113       return ELF_x86_64_Edges::ELFX86RelocationKind::Pointer32Signed;
114     case ELF::R_X86_64_PC32:
115       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32;
116     case ELF::R_X86_64_PC64:
117     case ELF::R_X86_64_GOTPC64:
118       return ELF_x86_64_Edges::ELFX86RelocationKind::Delta64;
119     case ELF::R_X86_64_64:
120       return ELF_x86_64_Edges::ELFX86RelocationKind::Pointer64;
121     case ELF::R_X86_64_GOTPCREL:
122       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32GOTLoad;
123     case ELF::R_X86_64_GOTPCRELX:
124       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32GOTLoadRelaxable;
125     case ELF::R_X86_64_REX_GOTPCRELX:
126       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32REXGOTLoadRelaxable;
127     case ELF::R_X86_64_GOTPCREL64:
128       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel64GOT;
129     case ELF::R_X86_64_GOT64:
130       return ELF_x86_64_Edges::ELFX86RelocationKind::GOT64;
131     case ELF::R_X86_64_GOTOFF64:
132       return ELF_x86_64_Edges::ELFX86RelocationKind::GOTOFF64;
133     case ELF::R_X86_64_PLT32:
134       return ELF_x86_64_Edges::ELFX86RelocationKind::Branch32;
135     case ELF::R_X86_64_TLSGD:
136       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32TLV;
137     }
138     return make_error<JITLinkError>(
139         "Unsupported x86-64 relocation type " + formatv("{0:d}: ", Type) +
140         object::getELFRelocationTypeName(ELF::EM_X86_64, Type));
141   }
142 
addRelocations()143   Error addRelocations() override {
144     LLVM_DEBUG(dbgs() << "Processing relocations:\n");
145 
146     using Base = ELFLinkGraphBuilder<ELFT>;
147     using Self = ELFLinkGraphBuilder_x86_64;
148     for (const auto &RelSect : Base::Sections) {
149       // Validate the section to read relocation entries from.
150       if (RelSect.sh_type == ELF::SHT_REL)
151         return make_error<StringError>(
152             "No SHT_REL in valid x64 ELF object files",
153             inconvertibleErrorCode());
154 
155       if (Error Err = Base::forEachRelocation(RelSect, this,
156                                               &Self::addSingleRelocation))
157         return Err;
158     }
159 
160     return Error::success();
161   }
162 
addSingleRelocation(const typename ELFT::Rela & Rel,const typename ELFT::Shdr & FixupSection,Block & BlockToFix)163   Error addSingleRelocation(const typename ELFT::Rela &Rel,
164                             const typename ELFT::Shdr &FixupSection,
165                             Block &BlockToFix) {
166     using Base = ELFLinkGraphBuilder<ELFT>;
167 
168     uint32_t SymbolIndex = Rel.getSymbol(false);
169     auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec);
170     if (!ObjSymbol)
171       return ObjSymbol.takeError();
172 
173     Symbol *GraphSymbol = Base::getGraphSymbol(SymbolIndex);
174     if (!GraphSymbol)
175       return make_error<StringError>(
176           formatv("Could not find symbol at given index, did you add it to "
177                   "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}",
178                   SymbolIndex, (*ObjSymbol)->st_shndx,
179                   Base::GraphSymbols.size()),
180           inconvertibleErrorCode());
181 
182     // Validate the relocation kind.
183     auto ELFRelocKind = getRelocationKind(Rel.getType(false));
184     if (!ELFRelocKind)
185       return ELFRelocKind.takeError();
186 
187     int64_t Addend = Rel.r_addend;
188     Edge::Kind Kind = Edge::Invalid;
189     switch (*ELFRelocKind) {
190     case PCRel32:
191       Kind = x86_64::Delta32;
192       break;
193     case Delta64:
194       Kind = x86_64::Delta64;
195       break;
196     case Pointer32Signed:
197       Kind = x86_64::Pointer32Signed;
198       break;
199     case Pointer64:
200       Kind = x86_64::Pointer64;
201       break;
202     case PCRel32GOTLoad: {
203       Kind = x86_64::RequestGOTAndTransformToDelta32;
204       break;
205     }
206     case PCRel32REXGOTLoadRelaxable: {
207       Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable;
208       Addend = 0;
209       break;
210     }
211     case PCRel32TLV: {
212       Kind = x86_64::RequestTLSDescInGOTAndTransformToDelta32;
213       break;
214     }
215     case PCRel32GOTLoadRelaxable: {
216       Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable;
217       Addend = 0;
218       break;
219     }
220     case PCRel64GOT: {
221       Kind = x86_64::RequestGOTAndTransformToDelta64;
222       break;
223     }
224     case GOT64: {
225       Kind = x86_64::RequestGOTAndTransformToDelta64FromGOT;
226       break;
227     }
228     case GOTOFF64: {
229       Kind = x86_64::Delta64FromGOT;
230       break;
231     }
232     case Branch32: {
233       Kind = x86_64::BranchPCRel32;
234       // BranchPCRel32 implicitly handles the '-4' PC adjustment, so we have to
235       // adjust the addend by '+4' to compensate.
236       Addend += 4;
237       break;
238     }
239     }
240 
241     auto FixupAddress = orc::ExecutorAddr(FixupSection.sh_addr) + Rel.r_offset;
242     Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress();
243     Edge GE(Kind, Offset, *GraphSymbol, Addend);
244     LLVM_DEBUG({
245       dbgs() << "    ";
246       printEdge(dbgs(), BlockToFix, GE, x86_64::getEdgeKindName(Kind));
247       dbgs() << "\n";
248     });
249 
250     BlockToFix.addEdge(std::move(GE));
251     return Error::success();
252   }
253 
254 public:
ELFLinkGraphBuilder_x86_64(StringRef FileName,const object::ELFFile<object::ELF64LE> & Obj)255   ELFLinkGraphBuilder_x86_64(StringRef FileName,
256                              const object::ELFFile<object::ELF64LE> &Obj)
257       : ELFLinkGraphBuilder(Obj, Triple("x86_64-unknown-linux"), FileName,
258                             x86_64::getEdgeKindName) {}
259 };
260 
261 class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> {
262   friend class JITLinker<ELFJITLinker_x86_64>;
263 
264 public:
ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx,std::unique_ptr<LinkGraph> G,PassConfiguration PassConfig)265   ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx,
266                       std::unique_ptr<LinkGraph> G,
267                       PassConfiguration PassConfig)
268       : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {
269     getPassConfig().PostAllocationPasses.push_back(
270         [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); });
271   }
272 
273 private:
274   Symbol *GOTSymbol = nullptr;
275 
getOrCreateGOTSymbol(LinkGraph & G)276   Error getOrCreateGOTSymbol(LinkGraph &G) {
277     auto DefineExternalGOTSymbolIfPresent =
278         createDefineExternalSectionStartAndEndSymbolsPass(
279             [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc {
280               if (Sym.getName() == ELFGOTSymbolName)
281                 if (auto *GOTSection = G.findSectionByName(
282                         x86_64::GOTTableManager::getSectionName())) {
283                   GOTSymbol = &Sym;
284                   return {*GOTSection, true};
285                 }
286               return {};
287             });
288 
289     // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an
290     // external.
291     if (auto Err = DefineExternalGOTSymbolIfPresent(G))
292       return Err;
293 
294     // If we succeeded then we're done.
295     if (GOTSymbol)
296       return Error::success();
297 
298     // Otherwise look for a GOT section: If it already has a start symbol we'll
299     // record it, otherwise we'll create our own.
300     // If there's a GOT section but we didn't find an external GOT symbol...
301     if (auto *GOTSection =
302             G.findSectionByName(x86_64::GOTTableManager::getSectionName())) {
303 
304       // Check for an existing defined symbol.
305       for (auto *Sym : GOTSection->symbols())
306         if (Sym->getName() == ELFGOTSymbolName) {
307           GOTSymbol = Sym;
308           return Error::success();
309         }
310 
311       // If there's no defined symbol then create one.
312       SectionRange SR(*GOTSection);
313       if (SR.empty())
314         GOTSymbol =
315             &G.addAbsoluteSymbol(ELFGOTSymbolName, orc::ExecutorAddr(), 0,
316                                  Linkage::Strong, Scope::Local, true);
317       else
318         GOTSymbol =
319             &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0,
320                                 Linkage::Strong, Scope::Local, false, true);
321     }
322 
323     return Error::success();
324   }
325 
applyFixup(LinkGraph & G,Block & B,const Edge & E) const326   Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const {
327     return x86_64::applyFixup(G, B, E, GOTSymbol);
328   }
329 };
330 
331 Expected<std::unique_ptr<LinkGraph>>
createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer)332 createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) {
333   LLVM_DEBUG({
334     dbgs() << "Building jitlink graph for new input "
335            << ObjectBuffer.getBufferIdentifier() << "...\n";
336   });
337 
338   auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer);
339   if (!ELFObj)
340     return ELFObj.takeError();
341 
342   auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj);
343   return ELFLinkGraphBuilder_x86_64((*ELFObj)->getFileName(),
344                                     ELFObjFile.getELFFile())
345       .buildGraph();
346 }
347 
348 static SectionRangeSymbolDesc
identifyELFSectionStartAndEndSymbols(LinkGraph & G,Symbol & Sym)349 identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) {
350   constexpr StringRef StartSymbolPrefix = "__start";
351   constexpr StringRef EndSymbolPrefix = "__end";
352 
353   auto SymName = Sym.getName();
354   if (SymName.startswith(StartSymbolPrefix)) {
355     if (auto *Sec =
356             G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size())))
357       return {*Sec, true};
358   } else if (SymName.startswith(EndSymbolPrefix)) {
359     if (auto *Sec =
360             G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size())))
361       return {*Sec, false};
362   }
363   return {};
364 }
365 
link_ELF_x86_64(std::unique_ptr<LinkGraph> G,std::unique_ptr<JITLinkContext> Ctx)366 void link_ELF_x86_64(std::unique_ptr<LinkGraph> G,
367                      std::unique_ptr<JITLinkContext> Ctx) {
368   PassConfiguration Config;
369 
370   if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) {
371 
372     Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame"));
373     Config.PrePrunePasses.push_back(EHFrameEdgeFixer(
374         ".eh_frame", x86_64::PointerSize, x86_64::Pointer32, x86_64::Pointer64,
375         x86_64::Delta32, x86_64::Delta64, x86_64::NegDelta32));
376     Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame"));
377 
378     // Construct a JITLinker and run the link function.
379     // Add a mark-live pass.
380     if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple()))
381       Config.PrePrunePasses.push_back(std::move(MarkLive));
382     else
383       Config.PrePrunePasses.push_back(markAllSymbolsLive);
384 
385     // Add an in-place GOT/Stubs/TLSInfoEntry build pass.
386     Config.PostPrunePasses.push_back(buildTables_ELF_x86_64);
387 
388     // Resolve any external section start / end symbols.
389     Config.PostAllocationPasses.push_back(
390         createDefineExternalSectionStartAndEndSymbolsPass(
391             identifyELFSectionStartAndEndSymbols));
392 
393     // Add GOT/Stubs optimizer pass.
394     Config.PreFixupPasses.push_back(x86_64::optimizeGOTAndStubAccesses);
395   }
396 
397   if (auto Err = Ctx->modifyPassConfig(*G, Config))
398     return Ctx->notifyFailed(std::move(Err));
399 
400   ELFJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config));
401 }
getELFX86RelocationKindName(Edge::Kind R)402 const char *getELFX86RelocationKindName(Edge::Kind R) {
403   switch (R) {
404   case Branch32:
405     return "Branch32";
406   case Pointer32Signed:
407     return "Pointer32Signed";
408   case Pointer64:
409     return "Pointer64";
410   case PCRel32:
411     return "PCRel32";
412   case PCRel32GOTLoad:
413     return "PCRel32GOTLoad";
414   case PCRel32GOTLoadRelaxable:
415     return "PCRel32GOTLoadRelaxable";
416   case PCRel32REXGOTLoadRelaxable:
417     return "PCRel32REXGOTLoad";
418   case PCRel64GOT:
419     return "PCRel64GOT";
420   case Delta64:
421     return "Delta64";
422   case GOT64:
423     return "GOT64";
424   case GOTOFF64:
425     return "GOTOFF64";
426   }
427   return getGenericEdgeKindName(static_cast<Edge::Kind>(R));
428 }
429 } // end namespace jitlink
430 } // end namespace llvm
431