1 //===- elfnix_platform.cpp ------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains code required to load the rest of the ELF-on-*IX runtime.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "elfnix_platform.h"
14 #include "common.h"
15 #include "error.h"
16 #include "wrapper_function_utils.h"
17 
18 #include <algorithm>
19 #include <map>
20 #include <mutex>
21 #include <sstream>
22 #include <unordered_map>
23 #include <vector>
24 
25 using namespace __orc_rt;
26 using namespace __orc_rt::elfnix;
27 
28 // Declare function tags for functions in the JIT process.
29 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_get_initializers_tag)
30 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_get_deinitializers_tag)
31 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_symbol_lookup_tag)
32 
33 // eh-frame registration functions, made available via aliases
34 // installed by the Platform
35 extern "C" void __orc_rt_register_eh_frame_section(const void *);
36 extern "C" void __orc_rt_deregister_eh_frame_section(const void *);
37 
38 namespace {
39 
40 Error validatePointerSectionExtent(const char *SectionName,
41                                    const ExecutorAddrRange &SE) {
42   if (SE.size().getValue() % sizeof(uintptr_t)) {
43     std::ostringstream ErrMsg;
44     ErrMsg << std::hex << "Size of " << SectionName << " 0x"
45            << SE.Start.getValue() << " -- 0x" << SE.End.getValue()
46            << " is not a pointer multiple";
47     return make_error<StringError>(ErrMsg.str());
48   }
49   return Error::success();
50 }
51 
52 Error runInitArray(const std::vector<ExecutorAddrRange> &InitArraySections,
53                    const ELFNixJITDylibInitializers &MOJDIs) {
54 
55   for (const auto &ModInits : InitArraySections) {
56     if (auto Err = validatePointerSectionExtent(".init_array", ModInits))
57       return Err;
58 
59     using InitFunc = void (*)();
60     for (auto *Init : ModInits.toSpan<InitFunc>())
61       (*Init)();
62   }
63 
64   return Error::success();
65 }
66 struct TLSInfoEntry {
67   unsigned long Key = 0;
68   unsigned long DataAddress = 0;
69 };
70 
71 class ELFNixPlatformRuntimeState {
72 private:
73   struct AtExitEntry {
74     void (*Func)(void *);
75     void *Arg;
76   };
77 
78   using AtExitsVector = std::vector<AtExitEntry>;
79 
80   struct PerJITDylibState {
81     void *Header = nullptr;
82     size_t RefCount = 0;
83     bool AllowReinitialization = false;
84     AtExitsVector AtExits;
85   };
86 
87 public:
88   static void initialize(void *DSOHandle);
89   static ELFNixPlatformRuntimeState &get();
90   static void destroy();
91 
92   ELFNixPlatformRuntimeState(void *DSOHandle)
93       : PlatformJDDSOHandle(DSOHandle) {}
94 
95   // Delete copy and move constructors.
96   ELFNixPlatformRuntimeState(const ELFNixPlatformRuntimeState &) = delete;
97   ELFNixPlatformRuntimeState &
98   operator=(const ELFNixPlatformRuntimeState &) = delete;
99   ELFNixPlatformRuntimeState(ELFNixPlatformRuntimeState &&) = delete;
100   ELFNixPlatformRuntimeState &operator=(ELFNixPlatformRuntimeState &&) = delete;
101 
102   Error registerObjectSections(ELFNixPerObjectSectionsToRegister POSR);
103   Error deregisterObjectSections(ELFNixPerObjectSectionsToRegister POSR);
104 
105   const char *dlerror();
106   void *dlopen(string_view Name, int Mode);
107   int dlclose(void *DSOHandle);
108   void *dlsym(void *DSOHandle, string_view Symbol);
109 
110   int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle);
111   void runAtExits(void *DSOHandle);
112 
113   /// Returns the base address of the section containing ThreadData.
114   Expected<std::pair<const char *, size_t>>
115   getThreadDataSectionFor(const char *ThreadData);
116 
117   void *getPlatformJDDSOHandle() { return PlatformJDDSOHandle; }
118 
119 private:
120   PerJITDylibState *getJITDylibStateByHeaderAddr(void *DSOHandle);
121   PerJITDylibState *getJITDylibStateByName(string_view Path);
122   PerJITDylibState &
123   getOrCreateJITDylibState(ELFNixJITDylibInitializers &MOJDIs);
124 
125   Error registerThreadDataSection(span<const char> ThreadDataSection);
126 
127   Expected<ExecutorAddr> lookupSymbolInJITDylib(void *DSOHandle,
128                                                 string_view Symbol);
129 
130   Expected<ELFNixJITDylibInitializerSequence>
131   getJITDylibInitializersByName(string_view Path);
132   Expected<void *> dlopenInitialize(string_view Path, int Mode);
133   Error initializeJITDylib(ELFNixJITDylibInitializers &MOJDIs);
134 
135   static ELFNixPlatformRuntimeState *MOPS;
136 
137   void *PlatformJDDSOHandle;
138 
139   // FIXME: Move to thread-state.
140   std::string DLFcnError;
141 
142   std::recursive_mutex JDStatesMutex;
143   std::unordered_map<void *, PerJITDylibState> JDStates;
144   std::unordered_map<std::string, void *> JDNameToHeader;
145 
146   std::mutex ThreadDataSectionsMutex;
147   std::map<const char *, size_t> ThreadDataSections;
148 };
149 
150 ELFNixPlatformRuntimeState *ELFNixPlatformRuntimeState::MOPS = nullptr;
151 
152 void ELFNixPlatformRuntimeState::initialize(void *DSOHandle) {
153   assert(!MOPS && "ELFNixPlatformRuntimeState should be null");
154   MOPS = new ELFNixPlatformRuntimeState(DSOHandle);
155 }
156 
157 ELFNixPlatformRuntimeState &ELFNixPlatformRuntimeState::get() {
158   assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
159   return *MOPS;
160 }
161 
162 void ELFNixPlatformRuntimeState::destroy() {
163   assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
164   delete MOPS;
165 }
166 
167 Error ELFNixPlatformRuntimeState::registerObjectSections(
168     ELFNixPerObjectSectionsToRegister POSR) {
169   if (POSR.EHFrameSection.Start)
170     __orc_rt_register_eh_frame_section(
171         POSR.EHFrameSection.Start.toPtr<const char *>());
172 
173   if (POSR.ThreadDataSection.Start) {
174     if (auto Err = registerThreadDataSection(
175             POSR.ThreadDataSection.toSpan<const char>()))
176       return Err;
177   }
178 
179   return Error::success();
180 }
181 
182 Error ELFNixPlatformRuntimeState::deregisterObjectSections(
183     ELFNixPerObjectSectionsToRegister POSR) {
184   if (POSR.EHFrameSection.Start)
185     __orc_rt_deregister_eh_frame_section(
186         POSR.EHFrameSection.Start.toPtr<const char *>());
187 
188   return Error::success();
189 }
190 
191 const char *ELFNixPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }
192 
193 void *ELFNixPlatformRuntimeState::dlopen(string_view Path, int Mode) {
194   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
195 
196   // Use fast path if all JITDylibs are already loaded and don't require
197   // re-running initializers.
198   if (auto *JDS = getJITDylibStateByName(Path)) {
199     if (!JDS->AllowReinitialization) {
200       ++JDS->RefCount;
201       return JDS->Header;
202     }
203   }
204 
205   auto H = dlopenInitialize(Path, Mode);
206   if (!H) {
207     DLFcnError = toString(H.takeError());
208     return nullptr;
209   }
210 
211   return *H;
212 }
213 
214 int ELFNixPlatformRuntimeState::dlclose(void *DSOHandle) {
215   runAtExits(DSOHandle);
216   return 0;
217 }
218 
219 void *ELFNixPlatformRuntimeState::dlsym(void *DSOHandle, string_view Symbol) {
220   auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol);
221   if (!Addr) {
222     DLFcnError = toString(Addr.takeError());
223     return 0;
224   }
225 
226   return Addr->toPtr<void *>();
227 }
228 
229 int ELFNixPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg,
230                                                void *DSOHandle) {
231   // FIXME: Handle out-of-memory errors, returning -1 if OOM.
232   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
233   auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
234   assert(JDS && "JITDylib state not initialized");
235   JDS->AtExits.push_back({F, Arg});
236   return 0;
237 }
238 
239 void ELFNixPlatformRuntimeState::runAtExits(void *DSOHandle) {
240   // FIXME: Should atexits be allowed to run concurrently with access to
241   // JDState?
242   AtExitsVector V;
243   {
244     std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
245     auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
246     assert(JDS && "JITDlybi state not initialized");
247     std::swap(V, JDS->AtExits);
248   }
249 
250   while (!V.empty()) {
251     auto &AE = V.back();
252     AE.Func(AE.Arg);
253     V.pop_back();
254   }
255 }
256 
257 Expected<std::pair<const char *, size_t>>
258 ELFNixPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) {
259   std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
260   auto I = ThreadDataSections.upper_bound(ThreadData);
261   // Check that we have a valid entry conovering this address.
262   if (I == ThreadDataSections.begin())
263     return make_error<StringError>("No thread local data section for key");
264   I = std::prev(I);
265   if (ThreadData >= I->first + I->second)
266     return make_error<StringError>("No thread local data section for key");
267   return *I;
268 }
269 
270 ELFNixPlatformRuntimeState::PerJITDylibState *
271 ELFNixPlatformRuntimeState::getJITDylibStateByHeaderAddr(void *DSOHandle) {
272   auto I = JDStates.find(DSOHandle);
273   if (I == JDStates.end())
274     return nullptr;
275   return &I->second;
276 }
277 
278 ELFNixPlatformRuntimeState::PerJITDylibState *
279 ELFNixPlatformRuntimeState::getJITDylibStateByName(string_view Name) {
280   // FIXME: Avoid creating string copy here.
281   auto I = JDNameToHeader.find(std::string(Name.data(), Name.size()));
282   if (I == JDNameToHeader.end())
283     return nullptr;
284   void *H = I->second;
285   auto J = JDStates.find(H);
286   assert(J != JDStates.end() &&
287          "JITDylib has name map entry but no header map entry");
288   return &J->second;
289 }
290 
291 ELFNixPlatformRuntimeState::PerJITDylibState &
292 ELFNixPlatformRuntimeState::getOrCreateJITDylibState(
293     ELFNixJITDylibInitializers &MOJDIs) {
294   void *Header = MOJDIs.DSOHandleAddress.toPtr<void *>();
295 
296   auto &JDS = JDStates[Header];
297 
298   // If this entry hasn't been created yet.
299   if (!JDS.Header) {
300     assert(!JDNameToHeader.count(MOJDIs.Name) &&
301            "JITDylib has header map entry but no name map entry");
302     JDNameToHeader[MOJDIs.Name] = Header;
303     JDS.Header = Header;
304   }
305 
306   return JDS;
307 }
308 
309 Error ELFNixPlatformRuntimeState::registerThreadDataSection(
310     span<const char> ThreadDataSection) {
311   std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
312   auto I = ThreadDataSections.upper_bound(ThreadDataSection.data());
313   if (I != ThreadDataSections.begin()) {
314     auto J = std::prev(I);
315     if (J->first + J->second > ThreadDataSection.data())
316       return make_error<StringError>("Overlapping .tdata sections");
317   }
318   ThreadDataSections.insert(
319       I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size()));
320   return Error::success();
321 }
322 
323 Expected<ExecutorAddr>
324 ELFNixPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle,
325                                                    string_view Sym) {
326   Expected<ExecutorAddr> Result((ExecutorAddr()));
327   if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddr>(
328           SPSExecutorAddr, SPSString)>::call(&__orc_rt_elfnix_symbol_lookup_tag,
329                                              Result,
330                                              ExecutorAddr::fromPtr(DSOHandle),
331                                              Sym))
332     return std::move(Err);
333   return Result;
334 }
335 
336 Expected<ELFNixJITDylibInitializerSequence>
337 ELFNixPlatformRuntimeState::getJITDylibInitializersByName(string_view Path) {
338   Expected<ELFNixJITDylibInitializerSequence> Result(
339       (ELFNixJITDylibInitializerSequence()));
340   std::string PathStr(Path.data(), Path.size());
341   if (auto Err =
342           WrapperFunction<SPSExpected<SPSELFNixJITDylibInitializerSequence>(
343               SPSString)>::call(&__orc_rt_elfnix_get_initializers_tag, Result,
344                                 Path))
345     return std::move(Err);
346   return Result;
347 }
348 
349 Expected<void *> ELFNixPlatformRuntimeState::dlopenInitialize(string_view Path,
350                                                               int Mode) {
351   // Either our JITDylib wasn't loaded, or it or one of its dependencies allows
352   // reinitialization. We need to call in to the JIT to see if there's any new
353   // work pending.
354   auto InitSeq = getJITDylibInitializersByName(Path);
355   if (!InitSeq)
356     return InitSeq.takeError();
357 
358   // Init sequences should be non-empty.
359   if (InitSeq->empty())
360     return make_error<StringError>(
361         "__orc_rt_elfnix_get_initializers returned an "
362         "empty init sequence");
363 
364   // Otherwise register and run initializers for each JITDylib.
365   for (auto &MOJDIs : *InitSeq)
366     if (auto Err = initializeJITDylib(MOJDIs))
367       return std::move(Err);
368 
369   // Return the header for the last item in the list.
370   auto *JDS = getJITDylibStateByHeaderAddr(
371       InitSeq->back().DSOHandleAddress.toPtr<void *>());
372   assert(JDS && "Missing state entry for JD");
373   return JDS->Header;
374 }
375 
376 long getPriority(const std::string &name) {
377   auto pos = name.find_last_not_of("0123456789");
378   if (pos == name.size() - 1)
379     return 65535;
380   else
381     return std::strtol(name.c_str() + pos + 1, nullptr, 10);
382 }
383 
384 Error ELFNixPlatformRuntimeState::initializeJITDylib(
385     ELFNixJITDylibInitializers &MOJDIs) {
386 
387   auto &JDS = getOrCreateJITDylibState(MOJDIs);
388   ++JDS.RefCount;
389 
390   using SectionList = std::vector<ExecutorAddrRange>;
391   std::sort(MOJDIs.InitSections.begin(), MOJDIs.InitSections.end(),
392             [](const std::pair<std::string, SectionList> &LHS,
393                const std::pair<std::string, SectionList> &RHS) -> bool {
394               return getPriority(LHS.first) < getPriority(RHS.first);
395             });
396   for (auto &Entry : MOJDIs.InitSections)
397     if (auto Err = runInitArray(Entry.second, MOJDIs))
398       return Err;
399 
400   return Error::success();
401 }
402 class ELFNixPlatformRuntimeTLVManager {
403 public:
404   void *getInstance(const char *ThreadData);
405 
406 private:
407   std::unordered_map<const char *, char *> Instances;
408   std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections;
409 };
410 
411 void *ELFNixPlatformRuntimeTLVManager::getInstance(const char *ThreadData) {
412   auto I = Instances.find(ThreadData);
413   if (I != Instances.end())
414     return I->second;
415   auto TDS =
416       ELFNixPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData);
417   if (!TDS) {
418     __orc_rt_log_error(toString(TDS.takeError()).c_str());
419     return nullptr;
420   }
421 
422   auto &Allocated = AllocatedSections[TDS->first];
423   if (!Allocated) {
424     Allocated = std::make_unique<char[]>(TDS->second);
425     memcpy(Allocated.get(), TDS->first, TDS->second);
426   }
427   size_t ThreadDataDelta = ThreadData - TDS->first;
428   assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds");
429 
430   char *Instance = Allocated.get() + ThreadDataDelta;
431   Instances[ThreadData] = Instance;
432   return Instance;
433 }
434 
435 void destroyELFNixTLVMgr(void *ELFNixTLVMgr) {
436   delete static_cast<ELFNixPlatformRuntimeTLVManager *>(ELFNixTLVMgr);
437 }
438 
439 } // end anonymous namespace
440 
441 //------------------------------------------------------------------------------
442 //                             JIT entry points
443 //------------------------------------------------------------------------------
444 
445 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
446 __orc_rt_elfnix_platform_bootstrap(char *ArgData, size_t ArgSize) {
447   return WrapperFunction<void(uint64_t)>::handle(
448              ArgData, ArgSize,
449              [](uint64_t &DSOHandle) {
450                ELFNixPlatformRuntimeState::initialize(
451                    reinterpret_cast<void *>(DSOHandle));
452              })
453       .release();
454 }
455 
456 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
457 __orc_rt_elfnix_platform_shutdown(char *ArgData, size_t ArgSize) {
458   ELFNixPlatformRuntimeState::destroy();
459   return WrapperFunctionResult().release();
460 }
461 
462 /// Wrapper function for registering metadata on a per-object basis.
463 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
464 __orc_rt_elfnix_register_object_sections(char *ArgData, size_t ArgSize) {
465   return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
466       handle(ArgData, ArgSize,
467              [](ELFNixPerObjectSectionsToRegister &POSR) {
468                return ELFNixPlatformRuntimeState::get().registerObjectSections(
469                    std::move(POSR));
470              })
471           .release();
472 }
473 
474 /// Wrapper for releasing per-object metadat.
475 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
476 __orc_rt_elfnix_deregister_object_sections(char *ArgData, size_t ArgSize) {
477   return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
478       handle(ArgData, ArgSize,
479              [](ELFNixPerObjectSectionsToRegister &POSR) {
480                return ELFNixPlatformRuntimeState::get()
481                    .deregisterObjectSections(std::move(POSR));
482              })
483           .release();
484 }
485 
486 //------------------------------------------------------------------------------
487 //                           TLV support
488 //------------------------------------------------------------------------------
489 
490 ORC_RT_INTERFACE void *__orc_rt_elfnix_tls_get_addr_impl(TLSInfoEntry *D) {
491   auto *TLVMgr = static_cast<ELFNixPlatformRuntimeTLVManager *>(
492       pthread_getspecific(D->Key));
493   if (!TLVMgr)
494     TLVMgr = new ELFNixPlatformRuntimeTLVManager();
495   if (pthread_setspecific(D->Key, TLVMgr)) {
496     __orc_rt_log_error("Call to pthread_setspecific failed");
497     return nullptr;
498   }
499 
500   return TLVMgr->getInstance(
501       reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress)));
502 }
503 
504 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
505 __orc_rt_elfnix_create_pthread_key(char *ArgData, size_t ArgSize) {
506   return WrapperFunction<SPSExpected<uint64_t>(void)>::handle(
507              ArgData, ArgSize,
508              []() -> Expected<uint64_t> {
509                pthread_key_t Key;
510                if (int Err = pthread_key_create(&Key, destroyELFNixTLVMgr)) {
511                  __orc_rt_log_error("Call to pthread_key_create failed");
512                  return make_error<StringError>(strerror(Err));
513                }
514                return static_cast<uint64_t>(Key);
515              })
516       .release();
517 }
518 
519 //------------------------------------------------------------------------------
520 //                           cxa_atexit support
521 //------------------------------------------------------------------------------
522 
523 int __orc_rt_elfnix_cxa_atexit(void (*func)(void *), void *arg,
524                                void *dso_handle) {
525   return ELFNixPlatformRuntimeState::get().registerAtExit(func, arg,
526                                                           dso_handle);
527 }
528 
529 int __orc_rt_elfnix_atexit(void (*func)(void *)) {
530   auto &PlatformRTState = ELFNixPlatformRuntimeState::get();
531   return ELFNixPlatformRuntimeState::get().registerAtExit(
532       func, NULL, PlatformRTState.getPlatformJDDSOHandle());
533 }
534 
535 void __orc_rt_elfnix_cxa_finalize(void *dso_handle) {
536   ELFNixPlatformRuntimeState::get().runAtExits(dso_handle);
537 }
538 
539 //------------------------------------------------------------------------------
540 //                        JIT'd dlfcn alternatives.
541 //------------------------------------------------------------------------------
542 
543 const char *__orc_rt_elfnix_jit_dlerror() {
544   return ELFNixPlatformRuntimeState::get().dlerror();
545 }
546 
547 void *__orc_rt_elfnix_jit_dlopen(const char *path, int mode) {
548   return ELFNixPlatformRuntimeState::get().dlopen(path, mode);
549 }
550 
551 int __orc_rt_elfnix_jit_dlclose(void *dso_handle) {
552   return ELFNixPlatformRuntimeState::get().dlclose(dso_handle);
553 }
554 
555 void *__orc_rt_elfnix_jit_dlsym(void *dso_handle, const char *symbol) {
556   return ELFNixPlatformRuntimeState::get().dlsym(dso_handle, symbol);
557 }
558 
559 //------------------------------------------------------------------------------
560 //                             ELFNix Run Program
561 //------------------------------------------------------------------------------
562 
563 ORC_RT_INTERFACE int64_t __orc_rt_elfnix_run_program(
564     const char *JITDylibName, const char *EntrySymbolName, int argc,
565     char *argv[]) {
566   using MainTy = int (*)(int, char *[]);
567 
568   void *H = __orc_rt_elfnix_jit_dlopen(JITDylibName,
569                                        __orc_rt::elfnix::ORC_RT_RTLD_LAZY);
570   if (!H) {
571     __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
572     return -1;
573   }
574 
575   auto *Main =
576       reinterpret_cast<MainTy>(__orc_rt_elfnix_jit_dlsym(H, EntrySymbolName));
577 
578   if (!Main) {
579     __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
580     return -1;
581   }
582 
583   int Result = Main(argc, argv);
584 
585   if (__orc_rt_elfnix_jit_dlclose(H) == -1)
586     __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
587 
588   return Result;
589 }
590