1 //===------ ELFNixPlatform.cpp - Utilities for executing MachO in Orc -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
10 
11 #include "llvm/BinaryFormat/ELF.h"
12 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
13 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
14 #include "llvm/ExecutionEngine/Orc/DebugUtils.h"
15 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
16 #include "llvm/Support/BinaryByteStream.h"
17 #include "llvm/Support/Debug.h"
18 
19 #define DEBUG_TYPE "orc"
20 
21 using namespace llvm;
22 using namespace llvm::orc;
23 using namespace llvm::orc::shared;
24 
25 namespace {
26 
27 class DSOHandleMaterializationUnit : public MaterializationUnit {
28 public:
29   DSOHandleMaterializationUnit(ELFNixPlatform &ENP,
30                                const SymbolStringPtr &DSOHandleSymbol)
31       : MaterializationUnit(
32             createDSOHandleSectionInterface(ENP, DSOHandleSymbol)),
33         ENP(ENP) {}
34 
35   StringRef getName() const override { return "DSOHandleMU"; }
36 
37   void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
38     unsigned PointerSize;
39     support::endianness Endianness;
40     jitlink::Edge::Kind EdgeKind;
41     const auto &TT =
42         ENP.getExecutionSession().getExecutorProcessControl().getTargetTriple();
43 
44     switch (TT.getArch()) {
45     case Triple::x86_64:
46       PointerSize = 8;
47       Endianness = support::endianness::little;
48       EdgeKind = jitlink::x86_64::Pointer64;
49       break;
50     default:
51       llvm_unreachable("Unrecognized architecture");
52     }
53 
54     // void *__dso_handle = &__dso_handle;
55     auto G = std::make_unique<jitlink::LinkGraph>(
56         "<DSOHandleMU>", TT, PointerSize, Endianness,
57         jitlink::getGenericEdgeKindName);
58     auto &DSOHandleSection =
59         G->createSection(".data.__dso_handle", jitlink::MemProt::Read);
60     auto &DSOHandleBlock = G->createContentBlock(
61         DSOHandleSection, getDSOHandleContent(PointerSize), orc::ExecutorAddr(),
62         8, 0);
63     auto &DSOHandleSymbol = G->addDefinedSymbol(
64         DSOHandleBlock, 0, *R->getInitializerSymbol(), DSOHandleBlock.getSize(),
65         jitlink::Linkage::Strong, jitlink::Scope::Default, false, true);
66     DSOHandleBlock.addEdge(EdgeKind, 0, DSOHandleSymbol, 0);
67 
68     ENP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
69   }
70 
71   void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
72 
73 private:
74   static MaterializationUnit::Interface
75   createDSOHandleSectionInterface(ELFNixPlatform &ENP,
76                                   const SymbolStringPtr &DSOHandleSymbol) {
77     SymbolFlagsMap SymbolFlags;
78     SymbolFlags[DSOHandleSymbol] = JITSymbolFlags::Exported;
79     return MaterializationUnit::Interface(std::move(SymbolFlags),
80                                           DSOHandleSymbol);
81   }
82 
83   ArrayRef<char> getDSOHandleContent(size_t PointerSize) {
84     static const char Content[8] = {0};
85     assert(PointerSize <= sizeof Content);
86     return {Content, PointerSize};
87   }
88 
89   ELFNixPlatform &ENP;
90 };
91 
92 StringRef EHFrameSectionName = ".eh_frame";
93 StringRef InitArrayFuncSectionName = ".init_array";
94 
95 StringRef ThreadBSSSectionName = ".tbss";
96 StringRef ThreadDataSectionName = ".tdata";
97 
98 StringRef InitSectionNames[] = {InitArrayFuncSectionName};
99 
100 } // end anonymous namespace
101 
102 namespace llvm {
103 namespace orc {
104 
105 Expected<std::unique_ptr<ELFNixPlatform>>
106 ELFNixPlatform::Create(ExecutionSession &ES,
107                        ObjectLinkingLayer &ObjLinkingLayer,
108                        JITDylib &PlatformJD, const char *OrcRuntimePath,
109                        Optional<SymbolAliasMap> RuntimeAliases) {
110 
111   auto &EPC = ES.getExecutorProcessControl();
112 
113   // If the target is not supported then bail out immediately.
114   if (!supportedTarget(EPC.getTargetTriple()))
115     return make_error<StringError>("Unsupported ELFNixPlatform triple: " +
116                                        EPC.getTargetTriple().str(),
117                                    inconvertibleErrorCode());
118 
119   // Create default aliases if the caller didn't supply any.
120   if (!RuntimeAliases) {
121     auto StandardRuntimeAliases = standardPlatformAliases(ES, PlatformJD);
122     if (!StandardRuntimeAliases)
123       return StandardRuntimeAliases.takeError();
124     RuntimeAliases = std::move(*StandardRuntimeAliases);
125   }
126 
127   // Define the aliases.
128   if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases))))
129     return std::move(Err);
130 
131   // Add JIT-dispatch function support symbols.
132   if (auto Err = PlatformJD.define(absoluteSymbols(
133           {{ES.intern("__orc_rt_jit_dispatch"),
134             {EPC.getJITDispatchInfo().JITDispatchFunction.getValue(),
135              JITSymbolFlags::Exported}},
136            {ES.intern("__orc_rt_jit_dispatch_ctx"),
137             {EPC.getJITDispatchInfo().JITDispatchContext.getValue(),
138              JITSymbolFlags::Exported}}})))
139     return std::move(Err);
140 
141   // Create a generator for the ORC runtime archive.
142   auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Load(
143       ObjLinkingLayer, OrcRuntimePath, EPC.getTargetTriple());
144   if (!OrcRuntimeArchiveGenerator)
145     return OrcRuntimeArchiveGenerator.takeError();
146 
147   // Create the instance.
148   Error Err = Error::success();
149   auto P = std::unique_ptr<ELFNixPlatform>(
150       new ELFNixPlatform(ES, ObjLinkingLayer, PlatformJD,
151                          std::move(*OrcRuntimeArchiveGenerator), Err));
152   if (Err)
153     return std::move(Err);
154   return std::move(P);
155 }
156 
157 Error ELFNixPlatform::setupJITDylib(JITDylib &JD) {
158   return JD.define(
159       std::make_unique<DSOHandleMaterializationUnit>(*this, DSOHandleSymbol));
160 }
161 
162 Error ELFNixPlatform::teardownJITDylib(JITDylib &JD) {
163   return Error::success();
164 }
165 
166 Error ELFNixPlatform::notifyAdding(ResourceTracker &RT,
167                                    const MaterializationUnit &MU) {
168   auto &JD = RT.getJITDylib();
169   const auto &InitSym = MU.getInitializerSymbol();
170   if (!InitSym)
171     return Error::success();
172 
173   RegisteredInitSymbols[&JD].add(InitSym,
174                                  SymbolLookupFlags::WeaklyReferencedSymbol);
175   LLVM_DEBUG({
176     dbgs() << "ELFNixPlatform: Registered init symbol " << *InitSym
177            << " for MU " << MU.getName() << "\n";
178   });
179   return Error::success();
180 }
181 
182 Error ELFNixPlatform::notifyRemoving(ResourceTracker &RT) {
183   llvm_unreachable("Not supported yet");
184 }
185 
186 static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
187                        ArrayRef<std::pair<const char *, const char *>> AL) {
188   for (auto &KV : AL) {
189     auto AliasName = ES.intern(KV.first);
190     assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
191     Aliases[std::move(AliasName)] = {ES.intern(KV.second),
192                                      JITSymbolFlags::Exported};
193   }
194 }
195 
196 Expected<SymbolAliasMap>
197 ELFNixPlatform::standardPlatformAliases(ExecutionSession &ES,
198                                         JITDylib &PlatformJD) {
199   SymbolAliasMap Aliases;
200   addAliases(ES, Aliases, requiredCXXAliases());
201   addAliases(ES, Aliases, standardRuntimeUtilityAliases());
202 
203   // Determine whether or not the libunwind extended-API function for
204   // dynamically registering an entire .eh_frame section is available.
205   // If it is not, we assume that libgcc_s is being used, and alias to
206   // its __register_frame with the same functionality.
207   auto RTRegisterFrame = ES.intern("__orc_rt_register_eh_frame_section");
208   auto LibUnwindRegisterFrame = ES.intern("__unw_add_dynamic_eh_frame_section");
209   auto RTDeregisterFrame = ES.intern("__orc_rt_deregister_eh_frame_section");
210   auto LibUnwindDeregisterFrame =
211       ES.intern("__unw_remove_dynamic_eh_frame_section");
212   auto SM = ES.lookup(makeJITDylibSearchOrder(&PlatformJD),
213                       SymbolLookupSet()
214                           .add(LibUnwindRegisterFrame,
215                                SymbolLookupFlags::WeaklyReferencedSymbol)
216                           .add(LibUnwindDeregisterFrame,
217                                SymbolLookupFlags::WeaklyReferencedSymbol));
218   if (!SM) { // Weak-ref means no "missing symbol" errors, so this must be
219              // something more serious that we should report.
220     return SM.takeError();
221   } else if (SM->size() == 2) {
222     LLVM_DEBUG({
223       dbgs() << "Using libunwind " << LibUnwindRegisterFrame
224              << " for unwind info registration\n";
225     });
226     Aliases[std::move(RTRegisterFrame)] = {LibUnwindRegisterFrame,
227                                            JITSymbolFlags::Exported};
228     Aliases[std::move(RTDeregisterFrame)] = {LibUnwindDeregisterFrame,
229                                              JITSymbolFlags::Exported};
230   } else {
231     // Since LLVM libunwind is not present, we assume that unwinding
232     // is provided by libgcc
233     LLVM_DEBUG({
234       dbgs() << "Using libgcc __register_frame"
235              << " for unwind info registration\n";
236     });
237     Aliases[std::move(RTRegisterFrame)] = {ES.intern("__register_frame"),
238                                            JITSymbolFlags::Exported};
239     Aliases[std::move(RTDeregisterFrame)] = {ES.intern("__deregister_frame"),
240                                              JITSymbolFlags::Exported};
241   }
242 
243   return Aliases;
244 }
245 
246 ArrayRef<std::pair<const char *, const char *>>
247 ELFNixPlatform::requiredCXXAliases() {
248   static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
249       {"__cxa_atexit", "__orc_rt_elfnix_cxa_atexit"},
250       {"atexit", "__orc_rt_elfnix_atexit"}};
251 
252   return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
253 }
254 
255 ArrayRef<std::pair<const char *, const char *>>
256 ELFNixPlatform::standardRuntimeUtilityAliases() {
257   static const std::pair<const char *, const char *>
258       StandardRuntimeUtilityAliases[] = {
259           {"__orc_rt_run_program", "__orc_rt_elfnix_run_program"},
260           {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
261 
262   return ArrayRef<std::pair<const char *, const char *>>(
263       StandardRuntimeUtilityAliases);
264 }
265 
266 bool ELFNixPlatform::isInitializerSection(StringRef SecName) {
267   for (auto &Name : InitSectionNames) {
268     if (Name.equals(SecName))
269       return true;
270   }
271   return false;
272 }
273 
274 bool ELFNixPlatform::supportedTarget(const Triple &TT) {
275   switch (TT.getArch()) {
276   case Triple::x86_64:
277     return true;
278   default:
279     return false;
280   }
281 }
282 
283 ELFNixPlatform::ELFNixPlatform(
284     ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
285     JITDylib &PlatformJD,
286     std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err)
287     : ES(ES), ObjLinkingLayer(ObjLinkingLayer),
288       DSOHandleSymbol(ES.intern("__dso_handle")) {
289   ErrorAsOutParameter _(&Err);
290 
291   ObjLinkingLayer.addPlugin(std::make_unique<ELFNixPlatformPlugin>(*this));
292 
293   PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));
294 
295   // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating
296   // the platform now), so set it up.
297   if (auto E2 = setupJITDylib(PlatformJD)) {
298     Err = std::move(E2);
299     return;
300   }
301 
302   RegisteredInitSymbols[&PlatformJD].add(
303       DSOHandleSymbol, SymbolLookupFlags::WeaklyReferencedSymbol);
304 
305   // Associate wrapper function tags with JIT-side function implementations.
306   if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
307     Err = std::move(E2);
308     return;
309   }
310 
311   // Lookup addresses of runtime functions callable by the platform,
312   // call the platform bootstrap function to initialize the platform-state
313   // object in the executor.
314   if (auto E2 = bootstrapELFNixRuntime(PlatformJD)) {
315     Err = std::move(E2);
316     return;
317   }
318 }
319 
320 Error ELFNixPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
321   ExecutionSession::JITDispatchHandlerAssociationMap WFs;
322 
323   using GetInitializersSPSSig =
324       SPSExpected<SPSELFNixJITDylibInitializerSequence>(SPSString);
325   WFs[ES.intern("__orc_rt_elfnix_get_initializers_tag")] =
326       ES.wrapAsyncWithSPS<GetInitializersSPSSig>(
327           this, &ELFNixPlatform::rt_getInitializers);
328 
329   using GetDeinitializersSPSSig =
330       SPSExpected<SPSELFJITDylibDeinitializerSequence>(SPSExecutorAddr);
331   WFs[ES.intern("__orc_rt_elfnix_get_deinitializers_tag")] =
332       ES.wrapAsyncWithSPS<GetDeinitializersSPSSig>(
333           this, &ELFNixPlatform::rt_getDeinitializers);
334 
335   using LookupSymbolSPSSig =
336       SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
337   WFs[ES.intern("__orc_rt_elfnix_symbol_lookup_tag")] =
338       ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this,
339                                               &ELFNixPlatform::rt_lookupSymbol);
340 
341   return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
342 }
343 
344 void ELFNixPlatform::getInitializersBuildSequencePhase(
345     SendInitializerSequenceFn SendResult, JITDylib &JD,
346     std::vector<JITDylibSP> DFSLinkOrder) {
347   ELFNixJITDylibInitializerSequence FullInitSeq;
348   {
349     std::lock_guard<std::mutex> Lock(PlatformMutex);
350     for (auto &InitJD : reverse(DFSLinkOrder)) {
351       LLVM_DEBUG({
352         dbgs() << "ELFNixPlatform: Appending inits for \"" << InitJD->getName()
353                << "\" to sequence\n";
354       });
355       auto ISItr = InitSeqs.find(InitJD.get());
356       if (ISItr != InitSeqs.end()) {
357         FullInitSeq.emplace_back(std::move(ISItr->second));
358         InitSeqs.erase(ISItr);
359       }
360     }
361   }
362 
363   SendResult(std::move(FullInitSeq));
364 }
365 
366 void ELFNixPlatform::getInitializersLookupPhase(
367     SendInitializerSequenceFn SendResult, JITDylib &JD) {
368 
369   auto DFSLinkOrder = JD.getDFSLinkOrder();
370   if (!DFSLinkOrder) {
371     SendResult(DFSLinkOrder.takeError());
372     return;
373   }
374 
375   DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
376   ES.runSessionLocked([&]() {
377     for (auto &InitJD : *DFSLinkOrder) {
378       auto RISItr = RegisteredInitSymbols.find(InitJD.get());
379       if (RISItr != RegisteredInitSymbols.end()) {
380         NewInitSymbols[InitJD.get()] = std::move(RISItr->second);
381         RegisteredInitSymbols.erase(RISItr);
382       }
383     }
384   });
385 
386   // If there are no further init symbols to look up then move on to the next
387   // phase.
388   if (NewInitSymbols.empty()) {
389     getInitializersBuildSequencePhase(std::move(SendResult), JD,
390                                       std::move(*DFSLinkOrder));
391     return;
392   }
393 
394   // Otherwise issue a lookup and re-run this phase when it completes.
395   lookupInitSymbolsAsync(
396       [this, SendResult = std::move(SendResult), &JD](Error Err) mutable {
397         if (Err)
398           SendResult(std::move(Err));
399         else
400           getInitializersLookupPhase(std::move(SendResult), JD);
401       },
402       ES, std::move(NewInitSymbols));
403 }
404 
405 void ELFNixPlatform::rt_getInitializers(SendInitializerSequenceFn SendResult,
406                                         StringRef JDName) {
407   LLVM_DEBUG({
408     dbgs() << "ELFNixPlatform::rt_getInitializers(\"" << JDName << "\")\n";
409   });
410 
411   JITDylib *JD = ES.getJITDylibByName(JDName);
412   if (!JD) {
413     LLVM_DEBUG({
414       dbgs() << "  No such JITDylib \"" << JDName << "\". Sending error.\n";
415     });
416     SendResult(make_error<StringError>("No JITDylib named " + JDName,
417                                        inconvertibleErrorCode()));
418     return;
419   }
420 
421   getInitializersLookupPhase(std::move(SendResult), *JD);
422 }
423 
424 void ELFNixPlatform::rt_getDeinitializers(
425     SendDeinitializerSequenceFn SendResult, ExecutorAddr Handle) {
426   LLVM_DEBUG({
427     dbgs() << "ELFNixPlatform::rt_getDeinitializers(\""
428            << formatv("{0:x}", Handle.getValue()) << "\")\n";
429   });
430 
431   JITDylib *JD = nullptr;
432 
433   {
434     std::lock_guard<std::mutex> Lock(PlatformMutex);
435     auto I = HandleAddrToJITDylib.find(Handle);
436     if (I != HandleAddrToJITDylib.end())
437       JD = I->second;
438   }
439 
440   if (!JD) {
441     LLVM_DEBUG({
442       dbgs() << "  No JITDylib for handle "
443              << formatv("{0:x}", Handle.getValue()) << "\n";
444     });
445     SendResult(make_error<StringError>("No JITDylib associated with handle " +
446                                            formatv("{0:x}", Handle.getValue()),
447                                        inconvertibleErrorCode()));
448     return;
449   }
450 
451   SendResult(ELFNixJITDylibDeinitializerSequence());
452 }
453 
454 void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
455                                      ExecutorAddr Handle,
456                                      StringRef SymbolName) {
457   LLVM_DEBUG({
458     dbgs() << "ELFNixPlatform::rt_lookupSymbol(\""
459            << formatv("{0:x}", Handle.getValue()) << "\")\n";
460   });
461 
462   JITDylib *JD = nullptr;
463 
464   {
465     std::lock_guard<std::mutex> Lock(PlatformMutex);
466     auto I = HandleAddrToJITDylib.find(Handle);
467     if (I != HandleAddrToJITDylib.end())
468       JD = I->second;
469   }
470 
471   if (!JD) {
472     LLVM_DEBUG({
473       dbgs() << "  No JITDylib for handle "
474              << formatv("{0:x}", Handle.getValue()) << "\n";
475     });
476     SendResult(make_error<StringError>("No JITDylib associated with handle " +
477                                            formatv("{0:x}", Handle.getValue()),
478                                        inconvertibleErrorCode()));
479     return;
480   }
481 
482   // Use functor class to work around XL build compiler issue on AIX.
483   class RtLookupNotifyComplete {
484   public:
485     RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
486         : SendResult(std::move(SendResult)) {}
487     void operator()(Expected<SymbolMap> Result) {
488       if (Result) {
489         assert(Result->size() == 1 && "Unexpected result map count");
490         SendResult(ExecutorAddr(Result->begin()->second.getAddress()));
491       } else {
492         SendResult(Result.takeError());
493       }
494     }
495 
496   private:
497     SendSymbolAddressFn SendResult;
498   };
499 
500   ES.lookup(
501       LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
502       SymbolLookupSet(ES.intern(SymbolName)), SymbolState::Ready,
503       RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister);
504 }
505 
506 Error ELFNixPlatform::bootstrapELFNixRuntime(JITDylib &PlatformJD) {
507 
508   std::pair<const char *, ExecutorAddr *> Symbols[] = {
509       {"__orc_rt_elfnix_platform_bootstrap", &orc_rt_elfnix_platform_bootstrap},
510       {"__orc_rt_elfnix_platform_shutdown", &orc_rt_elfnix_platform_shutdown},
511       {"__orc_rt_elfnix_register_object_sections",
512        &orc_rt_elfnix_register_object_sections},
513       {"__orc_rt_elfnix_create_pthread_key",
514        &orc_rt_elfnix_create_pthread_key}};
515 
516   SymbolLookupSet RuntimeSymbols;
517   std::vector<std::pair<SymbolStringPtr, ExecutorAddr *>> AddrsToRecord;
518   for (const auto &KV : Symbols) {
519     auto Name = ES.intern(KV.first);
520     RuntimeSymbols.add(Name);
521     AddrsToRecord.push_back({std::move(Name), KV.second});
522   }
523 
524   auto RuntimeSymbolAddrs = ES.lookup(
525       {{&PlatformJD, JITDylibLookupFlags::MatchAllSymbols}}, RuntimeSymbols);
526   if (!RuntimeSymbolAddrs)
527     return RuntimeSymbolAddrs.takeError();
528 
529   for (const auto &KV : AddrsToRecord) {
530     auto &Name = KV.first;
531     assert(RuntimeSymbolAddrs->count(Name) && "Missing runtime symbol?");
532     KV.second->setValue((*RuntimeSymbolAddrs)[Name].getAddress());
533   }
534 
535   auto PJDDSOHandle = ES.lookup(
536       {{&PlatformJD, JITDylibLookupFlags::MatchAllSymbols}}, DSOHandleSymbol);
537   if (!PJDDSOHandle)
538     return PJDDSOHandle.takeError();
539 
540   if (auto Err = ES.callSPSWrapper<void(uint64_t)>(
541           orc_rt_elfnix_platform_bootstrap, PJDDSOHandle->getAddress()))
542     return Err;
543 
544   // FIXME: Ordering is fuzzy here. We're probably best off saying
545   // "behavior is undefined if code that uses the runtime is added before
546   // the platform constructor returns", then move all this to the constructor.
547   RuntimeBootstrapped = true;
548   std::vector<ELFPerObjectSectionsToRegister> DeferredPOSRs;
549   {
550     std::lock_guard<std::mutex> Lock(PlatformMutex);
551     DeferredPOSRs = std::move(BootstrapPOSRs);
552   }
553 
554   for (auto &D : DeferredPOSRs)
555     if (auto Err = registerPerObjectSections(D))
556       return Err;
557 
558   return Error::success();
559 }
560 
561 Error ELFNixPlatform::registerInitInfo(
562     JITDylib &JD, ArrayRef<jitlink::Section *> InitSections) {
563 
564   std::unique_lock<std::mutex> Lock(PlatformMutex);
565 
566   ELFNixJITDylibInitializers *InitSeq = nullptr;
567   {
568     auto I = InitSeqs.find(&JD);
569     if (I == InitSeqs.end()) {
570       // If there's no init sequence entry yet then we need to look up the
571       // header symbol to force creation of one.
572       Lock.unlock();
573 
574       auto SearchOrder =
575           JD.withLinkOrderDo([](const JITDylibSearchOrder &SO) { return SO; });
576       if (auto Err = ES.lookup(SearchOrder, DSOHandleSymbol).takeError())
577         return Err;
578 
579       Lock.lock();
580       I = InitSeqs.find(&JD);
581       assert(I != InitSeqs.end() &&
582              "Entry missing after header symbol lookup?");
583     }
584     InitSeq = &I->second;
585   }
586 
587   for (auto *Sec : InitSections) {
588     // FIXME: Avoid copy here.
589     jitlink::SectionRange R(*Sec);
590     InitSeq->InitSections[Sec->getName()].push_back(
591         {ExecutorAddr(R.getStart()), ExecutorAddr(R.getEnd())});
592   }
593 
594   return Error::success();
595 }
596 
597 Error ELFNixPlatform::registerPerObjectSections(
598     const ELFPerObjectSectionsToRegister &POSR) {
599 
600   if (!orc_rt_elfnix_register_object_sections)
601     return make_error<StringError>("Attempting to register per-object "
602                                    "sections, but runtime support has not "
603                                    "been loaded yet",
604                                    inconvertibleErrorCode());
605 
606   Error ErrResult = Error::success();
607   if (auto Err = ES.callSPSWrapper<shared::SPSError(
608                      SPSELFPerObjectSectionsToRegister)>(
609           orc_rt_elfnix_register_object_sections, ErrResult, POSR))
610     return Err;
611   return ErrResult;
612 }
613 
614 Expected<uint64_t> ELFNixPlatform::createPThreadKey() {
615   if (!orc_rt_elfnix_create_pthread_key)
616     return make_error<StringError>(
617         "Attempting to create pthread key in target, but runtime support has "
618         "not been loaded yet",
619         inconvertibleErrorCode());
620 
621   Expected<uint64_t> Result(0);
622   if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>(
623           orc_rt_elfnix_create_pthread_key, Result))
624     return std::move(Err);
625   return Result;
626 }
627 
628 void ELFNixPlatform::ELFNixPlatformPlugin::modifyPassConfig(
629     MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
630     jitlink::PassConfiguration &Config) {
631 
632   // If the initializer symbol is the __dso_handle symbol then just add
633   // the DSO handle support passes.
634   if (MR.getInitializerSymbol() == MP.DSOHandleSymbol) {
635     addDSOHandleSupportPasses(MR, Config);
636     // The DSOHandle materialization unit doesn't require any other
637     // support, so we can bail out early.
638     return;
639   }
640 
641   // If the object contains initializers then add passes to record them.
642   if (MR.getInitializerSymbol())
643     addInitializerSupportPasses(MR, Config);
644 
645   // Add passes for eh-frame and TLV support.
646   addEHAndTLVSupportPasses(MR, Config);
647 }
648 
649 ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap
650 ELFNixPlatform::ELFNixPlatformPlugin::getSyntheticSymbolDependencies(
651     MaterializationResponsibility &MR) {
652   std::lock_guard<std::mutex> Lock(PluginMutex);
653   auto I = InitSymbolDeps.find(&MR);
654   if (I != InitSymbolDeps.end()) {
655     SyntheticSymbolDependenciesMap Result;
656     Result[MR.getInitializerSymbol()] = std::move(I->second);
657     InitSymbolDeps.erase(&MR);
658     return Result;
659   }
660   return SyntheticSymbolDependenciesMap();
661 }
662 
663 void ELFNixPlatform::ELFNixPlatformPlugin::addInitializerSupportPasses(
664     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
665 
666   /// Preserve init sections.
667   Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) -> Error {
668     if (auto Err = preserveInitSections(G, MR))
669       return Err;
670     return Error::success();
671   });
672 
673   Config.PostFixupPasses.push_back(
674       [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
675         return registerInitSections(G, JD);
676       });
677 }
678 
679 void ELFNixPlatform::ELFNixPlatformPlugin::addDSOHandleSupportPasses(
680     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
681 
682   Config.PostAllocationPasses.push_back([this, &JD = MR.getTargetJITDylib()](
683                                             jitlink::LinkGraph &G) -> Error {
684     auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {
685       return Sym->getName() == *MP.DSOHandleSymbol;
686     });
687     assert(I != G.defined_symbols().end() && "Missing DSO handle symbol");
688     {
689       std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
690       auto HandleAddr = (*I)->getAddress();
691       MP.HandleAddrToJITDylib[HandleAddr] = &JD;
692       assert(!MP.InitSeqs.count(&JD) && "InitSeq entry for JD already exists");
693       MP.InitSeqs.insert(std::make_pair(
694           &JD, ELFNixJITDylibInitializers(JD.getName(), HandleAddr)));
695     }
696     return Error::success();
697   });
698 }
699 
700 void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses(
701     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
702 
703   // Insert TLV lowering at the start of the PostPrunePasses, since we want
704   // it to run before GOT/PLT lowering.
705 
706   // TODO: Check that before the fixTLVSectionsAndEdges pass, the GOT/PLT build
707   // pass has done. Because the TLS descriptor need to be allocate in GOT.
708   Config.PostPrunePasses.push_back(
709       [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
710         return fixTLVSectionsAndEdges(G, JD);
711       });
712 
713   // Add a pass to register the final addresses of the eh-frame and TLV sections
714   // with the runtime.
715   Config.PostFixupPasses.push_back([this](jitlink::LinkGraph &G) -> Error {
716     ELFPerObjectSectionsToRegister POSR;
717 
718     if (auto *EHFrameSection = G.findSectionByName(EHFrameSectionName)) {
719       jitlink::SectionRange R(*EHFrameSection);
720       if (!R.empty())
721         POSR.EHFrameSection = {ExecutorAddr(R.getStart()),
722                                ExecutorAddr(R.getEnd())};
723     }
724 
725     // Get a pointer to the thread data section if there is one. It will be used
726     // below.
727     jitlink::Section *ThreadDataSection =
728         G.findSectionByName(ThreadDataSectionName);
729 
730     // Handle thread BSS section if there is one.
731     if (auto *ThreadBSSSection = G.findSectionByName(ThreadBSSSectionName)) {
732       // If there's already a thread data section in this graph then merge the
733       // thread BSS section content into it, otherwise just treat the thread
734       // BSS section as the thread data section.
735       if (ThreadDataSection)
736         G.mergeSections(*ThreadDataSection, *ThreadBSSSection);
737       else
738         ThreadDataSection = ThreadBSSSection;
739     }
740 
741     // Having merged thread BSS (if present) and thread data (if present),
742     // record the resulting section range.
743     if (ThreadDataSection) {
744       jitlink::SectionRange R(*ThreadDataSection);
745       if (!R.empty())
746         POSR.ThreadDataSection = {ExecutorAddr(R.getStart()),
747                                   ExecutorAddr(R.getEnd())};
748     }
749 
750     if (POSR.EHFrameSection.Start || POSR.ThreadDataSection.Start) {
751 
752       // If we're still bootstrapping the runtime then just record this
753       // frame for now.
754       if (!MP.RuntimeBootstrapped) {
755         std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
756         MP.BootstrapPOSRs.push_back(POSR);
757         return Error::success();
758       }
759 
760       // Otherwise register it immediately.
761       if (auto Err = MP.registerPerObjectSections(POSR))
762         return Err;
763     }
764 
765     return Error::success();
766   });
767 }
768 
769 Error ELFNixPlatform::ELFNixPlatformPlugin::preserveInitSections(
770     jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
771 
772   JITLinkSymbolSet InitSectionSymbols;
773   for (auto &InitSectionName : InitSectionNames) {
774     // Skip non-init sections.
775     auto *InitSection = G.findSectionByName(InitSectionName);
776     if (!InitSection)
777       continue;
778 
779     // Make a pass over live symbols in the section: those blocks are already
780     // preserved.
781     DenseSet<jitlink::Block *> AlreadyLiveBlocks;
782     for (auto &Sym : InitSection->symbols()) {
783       auto &B = Sym->getBlock();
784       if (Sym->isLive() && Sym->getOffset() == 0 &&
785           Sym->getSize() == B.getSize() && !AlreadyLiveBlocks.count(&B)) {
786         InitSectionSymbols.insert(Sym);
787         AlreadyLiveBlocks.insert(&B);
788       }
789     }
790 
791     // Add anonymous symbols to preserve any not-already-preserved blocks.
792     for (auto *B : InitSection->blocks())
793       if (!AlreadyLiveBlocks.count(B))
794         InitSectionSymbols.insert(
795             &G.addAnonymousSymbol(*B, 0, B->getSize(), false, true));
796   }
797 
798   if (!InitSectionSymbols.empty()) {
799     std::lock_guard<std::mutex> Lock(PluginMutex);
800     InitSymbolDeps[&MR] = std::move(InitSectionSymbols);
801   }
802 
803   return Error::success();
804 }
805 
806 Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections(
807     jitlink::LinkGraph &G, JITDylib &JD) {
808 
809   SmallVector<jitlink::Section *> InitSections;
810 
811   LLVM_DEBUG({ dbgs() << "ELFNixPlatform::registerInitSections\n"; });
812 
813   for (auto InitSectionName : InitSectionNames) {
814     if (auto *Sec = G.findSectionByName(InitSectionName)) {
815       InitSections.push_back(Sec);
816     }
817   }
818 
819   // Dump the scraped inits.
820   LLVM_DEBUG({
821     dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " init sections:\n";
822     for (auto *Sec : InitSections) {
823       jitlink::SectionRange R(*Sec);
824       dbgs() << "  " << Sec->getName() << ": "
825              << formatv("[ {0:x} -- {1:x} ]", R.getStart(), R.getEnd()) << "\n";
826     }
827   });
828 
829   return MP.registerInitInfo(JD, InitSections);
830 }
831 
832 Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges(
833     jitlink::LinkGraph &G, JITDylib &JD) {
834 
835   // TODO implement TLV support
836   for (auto *Sym : G.external_symbols())
837     if (Sym->getName() == "__tls_get_addr") {
838       Sym->setName("___orc_rt_elfnix_tls_get_addr");
839     }
840 
841   auto *TLSInfoEntrySection = G.findSectionByName("$__TLSINFO");
842 
843   if (TLSInfoEntrySection) {
844     Optional<uint64_t> Key;
845     {
846       std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
847       auto I = MP.JITDylibToPThreadKey.find(&JD);
848       if (I != MP.JITDylibToPThreadKey.end())
849         Key = I->second;
850     }
851     if (!Key) {
852       if (auto KeyOrErr = MP.createPThreadKey())
853         Key = *KeyOrErr;
854       else
855         return KeyOrErr.takeError();
856     }
857 
858     uint64_t PlatformKeyBits =
859         support::endian::byte_swap(*Key, G.getEndianness());
860 
861     for (auto *B : TLSInfoEntrySection->blocks()) {
862       // FIXME: The TLS descriptor byte length may different with different
863       // ISA
864       assert(B->getSize() == (G.getPointerSize() * 2) &&
865              "TLS descriptor must be 2 words length");
866       auto TLSInfoEntryContent = B->getMutableContent(G);
867       memcpy(TLSInfoEntryContent.data(), &PlatformKeyBits, G.getPointerSize());
868     }
869   }
870 
871   return Error::success();
872 }
873 
874 } // End namespace orc.
875 } // End namespace llvm.
876