1 //===- elfnix_platform.cpp ------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains code required to load the rest of the ELF-on-*IX runtime.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "elfnix_platform.h"
14 #include "common.h"
15 #include "error.h"
16 #include "wrapper_function_utils.h"
17 
18 #include <map>
19 #include <mutex>
20 #include <sstream>
21 #include <unordered_map>
22 #include <vector>
23 
24 using namespace __orc_rt;
25 using namespace __orc_rt::elfnix;
26 
27 // Declare function tags for functions in the JIT process.
28 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_get_initializers_tag)
29 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_get_deinitializers_tag)
30 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_symbol_lookup_tag)
31 
32 // eh-frame registration functions.
33 // We expect these to be available for all processes.
34 extern "C" void __register_frame(const void *);
35 extern "C" void __deregister_frame(const void *);
36 
37 namespace {
38 
39 Error validatePointerSectionExtent(const char *SectionName,
40                                    const ExecutorAddressRange &SE) {
41   if (SE.size().getValue() % sizeof(uintptr_t)) {
42     std::ostringstream ErrMsg;
43     ErrMsg << std::hex << "Size of " << SectionName << " 0x"
44            << SE.StartAddress.getValue() << " -- 0x" << SE.EndAddress.getValue()
45            << " is not a pointer multiple";
46     return make_error<StringError>(ErrMsg.str());
47   }
48   return Error::success();
49 }
50 
51 Error runInitArray(const std::vector<ExecutorAddressRange> &InitArraySections,
52                    const ELFNixJITDylibInitializers &MOJDIs) {
53 
54   for (const auto &ModInits : InitArraySections) {
55     if (auto Err = validatePointerSectionExtent(".init_array", ModInits))
56       return Err;
57 
58     using InitFunc = void (*)();
59     for (auto *Init : ModInits.toSpan<InitFunc>())
60       (*Init)();
61   }
62 
63   return Error::success();
64 }
65 
66 class ELFNixPlatformRuntimeState {
67 private:
68   struct AtExitEntry {
69     void (*Func)(void *);
70     void *Arg;
71   };
72 
73   using AtExitsVector = std::vector<AtExitEntry>;
74 
75   struct PerJITDylibState {
76     void *Header = nullptr;
77     size_t RefCount = 0;
78     bool AllowReinitialization = false;
79     AtExitsVector AtExits;
80   };
81 
82 public:
83   static void initialize();
84   static ELFNixPlatformRuntimeState &get();
85   static void destroy();
86 
87   ELFNixPlatformRuntimeState() = default;
88 
89   // Delete copy and move constructors.
90   ELFNixPlatformRuntimeState(const ELFNixPlatformRuntimeState &) = delete;
91   ELFNixPlatformRuntimeState &
92   operator=(const ELFNixPlatformRuntimeState &) = delete;
93   ELFNixPlatformRuntimeState(ELFNixPlatformRuntimeState &&) = delete;
94   ELFNixPlatformRuntimeState &operator=(ELFNixPlatformRuntimeState &&) = delete;
95 
96   Error registerObjectSections(ELFNixPerObjectSectionsToRegister POSR);
97   Error deregisterObjectSections(ELFNixPerObjectSectionsToRegister POSR);
98 
99   const char *dlerror();
100   void *dlopen(string_view Name, int Mode);
101   int dlclose(void *DSOHandle);
102   void *dlsym(void *DSOHandle, string_view Symbol);
103 
104   int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle);
105   void runAtExits(void *DSOHandle);
106 
107 private:
108   PerJITDylibState *getJITDylibStateByHeaderAddr(void *DSOHandle);
109   PerJITDylibState *getJITDylibStateByName(string_view Path);
110   PerJITDylibState &
111   getOrCreateJITDylibState(ELFNixJITDylibInitializers &MOJDIs);
112 
113   Expected<ExecutorAddress> lookupSymbolInJITDylib(void *DSOHandle,
114                                                    string_view Symbol);
115 
116   Expected<ELFNixJITDylibInitializerSequence>
117   getJITDylibInitializersByName(string_view Path);
118   Expected<void *> dlopenInitialize(string_view Path, int Mode);
119   Error initializeJITDylib(ELFNixJITDylibInitializers &MOJDIs);
120 
121   static ELFNixPlatformRuntimeState *MOPS;
122 
123   using InitSectionHandler =
124       Error (*)(const std::vector<ExecutorAddressRange> &Sections,
125                 const ELFNixJITDylibInitializers &MOJDIs);
126   const std::vector<std::pair<const char *, InitSectionHandler>> InitSections =
127       {{".init_array", runInitArray}};
128 
129   // FIXME: Move to thread-state.
130   std::string DLFcnError;
131 
132   std::recursive_mutex JDStatesMutex;
133   std::unordered_map<void *, PerJITDylibState> JDStates;
134   std::unordered_map<std::string, void *> JDNameToHeader;
135 };
136 
137 ELFNixPlatformRuntimeState *ELFNixPlatformRuntimeState::MOPS = nullptr;
138 
139 void ELFNixPlatformRuntimeState::initialize() {
140   assert(!MOPS && "ELFNixPlatformRuntimeState should be null");
141   MOPS = new ELFNixPlatformRuntimeState();
142 }
143 
144 ELFNixPlatformRuntimeState &ELFNixPlatformRuntimeState::get() {
145   assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
146   return *MOPS;
147 }
148 
149 void ELFNixPlatformRuntimeState::destroy() {
150   assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
151   delete MOPS;
152 }
153 
154 Error ELFNixPlatformRuntimeState::registerObjectSections(
155     ELFNixPerObjectSectionsToRegister POSR) {
156   if (POSR.EHFrameSection.StartAddress)
157     __register_frame(POSR.EHFrameSection.StartAddress.toPtr<const char *>());
158 
159   // TODO: Register thread data sections.
160 
161   return Error::success();
162 }
163 
164 Error ELFNixPlatformRuntimeState::deregisterObjectSections(
165     ELFNixPerObjectSectionsToRegister POSR) {
166   if (POSR.EHFrameSection.StartAddress)
167     __deregister_frame(POSR.EHFrameSection.StartAddress.toPtr<const char *>());
168 
169   return Error::success();
170 }
171 
172 const char *ELFNixPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }
173 
174 void *ELFNixPlatformRuntimeState::dlopen(string_view Path, int Mode) {
175   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
176 
177   // Use fast path if all JITDylibs are already loaded and don't require
178   // re-running initializers.
179   if (auto *JDS = getJITDylibStateByName(Path)) {
180     if (!JDS->AllowReinitialization) {
181       ++JDS->RefCount;
182       return JDS->Header;
183     }
184   }
185 
186   auto H = dlopenInitialize(Path, Mode);
187   if (!H) {
188     DLFcnError = toString(H.takeError());
189     return nullptr;
190   }
191 
192   return *H;
193 }
194 
195 int ELFNixPlatformRuntimeState::dlclose(void *DSOHandle) {
196   runAtExits(DSOHandle);
197   return 0;
198 }
199 
200 void *ELFNixPlatformRuntimeState::dlsym(void *DSOHandle, string_view Symbol) {
201   auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol);
202   if (!Addr) {
203     DLFcnError = toString(Addr.takeError());
204     return 0;
205   }
206 
207   return Addr->toPtr<void *>();
208 }
209 
210 int ELFNixPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg,
211                                                void *DSOHandle) {
212   // FIXME: Handle out-of-memory errors, returning -1 if OOM.
213   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
214   auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
215   assert(JDS && "JITDylib state not initialized");
216   JDS->AtExits.push_back({F, Arg});
217   return 0;
218 }
219 
220 void ELFNixPlatformRuntimeState::runAtExits(void *DSOHandle) {
221   // FIXME: Should atexits be allowed to run concurrently with access to
222   // JDState?
223   AtExitsVector V;
224   {
225     std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
226     auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
227     assert(JDS && "JITDlybi state not initialized");
228     std::swap(V, JDS->AtExits);
229   }
230 
231   while (!V.empty()) {
232     auto &AE = V.back();
233     AE.Func(AE.Arg);
234     V.pop_back();
235   }
236 }
237 
238 ELFNixPlatformRuntimeState::PerJITDylibState *
239 ELFNixPlatformRuntimeState::getJITDylibStateByHeaderAddr(void *DSOHandle) {
240   auto I = JDStates.find(DSOHandle);
241   if (I == JDStates.end())
242     return nullptr;
243   return &I->second;
244 }
245 
246 ELFNixPlatformRuntimeState::PerJITDylibState *
247 ELFNixPlatformRuntimeState::getJITDylibStateByName(string_view Name) {
248   // FIXME: Avoid creating string copy here.
249   auto I = JDNameToHeader.find(std::string(Name.data(), Name.size()));
250   if (I == JDNameToHeader.end())
251     return nullptr;
252   void *H = I->second;
253   auto J = JDStates.find(H);
254   assert(J != JDStates.end() &&
255          "JITDylib has name map entry but no header map entry");
256   return &J->second;
257 }
258 
259 ELFNixPlatformRuntimeState::PerJITDylibState &
260 ELFNixPlatformRuntimeState::getOrCreateJITDylibState(
261     ELFNixJITDylibInitializers &MOJDIs) {
262   void *Header = MOJDIs.DSOHandleAddress.toPtr<void *>();
263 
264   auto &JDS = JDStates[Header];
265 
266   // If this entry hasn't been created yet.
267   if (!JDS.Header) {
268     assert(!JDNameToHeader.count(MOJDIs.Name) &&
269            "JITDylib has header map entry but no name map entry");
270     JDNameToHeader[MOJDIs.Name] = Header;
271     JDS.Header = Header;
272   }
273 
274   return JDS;
275 }
276 
277 Expected<ExecutorAddress>
278 ELFNixPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle,
279                                                    string_view Sym) {
280   Expected<ExecutorAddress> Result((ExecutorAddress()));
281   if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddress>(
282           SPSExecutorAddress,
283           SPSString)>::call(&__orc_rt_elfnix_symbol_lookup_tag, Result,
284                             ExecutorAddress::fromPtr(DSOHandle), Sym))
285     return std::move(Err);
286   return Result;
287 }
288 
289 Expected<ELFNixJITDylibInitializerSequence>
290 ELFNixPlatformRuntimeState::getJITDylibInitializersByName(string_view Path) {
291   Expected<ELFNixJITDylibInitializerSequence> Result(
292       (ELFNixJITDylibInitializerSequence()));
293   std::string PathStr(Path.data(), Path.size());
294   if (auto Err =
295           WrapperFunction<SPSExpected<SPSELFNixJITDylibInitializerSequence>(
296               SPSString)>::call(&__orc_rt_elfnix_get_initializers_tag, Result,
297                                 Path))
298     return std::move(Err);
299   return Result;
300 }
301 
302 Expected<void *> ELFNixPlatformRuntimeState::dlopenInitialize(string_view Path,
303                                                               int Mode) {
304   // Either our JITDylib wasn't loaded, or it or one of its dependencies allows
305   // reinitialization. We need to call in to the JIT to see if there's any new
306   // work pending.
307   auto InitSeq = getJITDylibInitializersByName(Path);
308   if (!InitSeq)
309     return InitSeq.takeError();
310 
311   // Init sequences should be non-empty.
312   if (InitSeq->empty())
313     return make_error<StringError>(
314         "__orc_rt_elfnix_get_initializers returned an "
315         "empty init sequence");
316 
317   // Otherwise register and run initializers for each JITDylib.
318   for (auto &MOJDIs : *InitSeq)
319     if (auto Err = initializeJITDylib(MOJDIs))
320       return std::move(Err);
321 
322   // Return the header for the last item in the list.
323   auto *JDS = getJITDylibStateByHeaderAddr(
324       InitSeq->back().DSOHandleAddress.toPtr<void *>());
325   assert(JDS && "Missing state entry for JD");
326   return JDS->Header;
327 }
328 
329 Error ELFNixPlatformRuntimeState::initializeJITDylib(
330     ELFNixJITDylibInitializers &MOJDIs) {
331 
332   auto &JDS = getOrCreateJITDylibState(MOJDIs);
333   ++JDS.RefCount;
334 
335   for (auto &KV : InitSections) {
336     const auto &Name = KV.first;
337     const auto &Handler = KV.second;
338     auto I = MOJDIs.InitSections.find(Name);
339     if (I != MOJDIs.InitSections.end()) {
340       if (auto Err = Handler(I->second, MOJDIs))
341         return Err;
342     }
343   }
344 
345   return Error::success();
346 }
347 
348 } // end anonymous namespace
349 
350 //------------------------------------------------------------------------------
351 //                             JIT entry points
352 //------------------------------------------------------------------------------
353 
354 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
355 __orc_rt_elfnix_platform_bootstrap(char *ArgData, size_t ArgSize) {
356   ELFNixPlatformRuntimeState::initialize();
357   return WrapperFunctionResult().release();
358 }
359 
360 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
361 __orc_rt_elfnix_platform_shutdown(char *ArgData, size_t ArgSize) {
362   ELFNixPlatformRuntimeState::destroy();
363   return WrapperFunctionResult().release();
364 }
365 
366 /// Wrapper function for registering metadata on a per-object basis.
367 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
368 __orc_rt_elfnix_register_object_sections(char *ArgData, size_t ArgSize) {
369   return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
370       handle(ArgData, ArgSize,
371              [](ELFNixPerObjectSectionsToRegister &POSR) {
372                return ELFNixPlatformRuntimeState::get().registerObjectSections(
373                    std::move(POSR));
374              })
375           .release();
376 }
377 
378 /// Wrapper for releasing per-object metadat.
379 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
380 __orc_rt_elfnix_deregister_object_sections(char *ArgData, size_t ArgSize) {
381   return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
382       handle(ArgData, ArgSize,
383              [](ELFNixPerObjectSectionsToRegister &POSR) {
384                return ELFNixPlatformRuntimeState::get()
385                    .deregisterObjectSections(std::move(POSR));
386              })
387           .release();
388 }
389 
390 //------------------------------------------------------------------------------
391 //                           cxa_atexit support
392 //------------------------------------------------------------------------------
393 
394 int __orc_rt_elfnix_cxa_atexit(void (*func)(void *), void *arg,
395                                void *dso_handle) {
396   return ELFNixPlatformRuntimeState::get().registerAtExit(func, arg,
397                                                           dso_handle);
398 }
399 
400 void __orc_rt_elfnix_cxa_finalize(void *dso_handle) {
401   ELFNixPlatformRuntimeState::get().runAtExits(dso_handle);
402 }
403 
404 //------------------------------------------------------------------------------
405 //                        JIT'd dlfcn alternatives.
406 //------------------------------------------------------------------------------
407 
408 const char *__orc_rt_elfnix_jit_dlerror() {
409   return ELFNixPlatformRuntimeState::get().dlerror();
410 }
411 
412 void *__orc_rt_elfnix_jit_dlopen(const char *path, int mode) {
413   return ELFNixPlatformRuntimeState::get().dlopen(path, mode);
414 }
415 
416 int __orc_rt_elfnix_jit_dlclose(void *dso_handle) {
417   return ELFNixPlatformRuntimeState::get().dlclose(dso_handle);
418 }
419 
420 void *__orc_rt_elfnix_jit_dlsym(void *dso_handle, const char *symbol) {
421   return ELFNixPlatformRuntimeState::get().dlsym(dso_handle, symbol);
422 }
423 
424 //------------------------------------------------------------------------------
425 //                             ELFNix Run Program
426 //------------------------------------------------------------------------------
427 
428 ORC_RT_INTERFACE int64_t __orc_rt_elfnix_run_program(
429     const char *JITDylibName, const char *EntrySymbolName, int argc,
430     char *argv[]) {
431   using MainTy = int (*)(int, char *[]);
432 
433   void *H = __orc_rt_elfnix_jit_dlopen(JITDylibName,
434                                        __orc_rt::elfnix::ORC_RT_RTLD_LAZY);
435   if (!H) {
436     __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
437     return -1;
438   }
439 
440   auto *Main =
441       reinterpret_cast<MainTy>(__orc_rt_elfnix_jit_dlsym(H, EntrySymbolName));
442 
443   if (!Main) {
444     __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
445     return -1;
446   }
447 
448   int Result = Main(argc, argv);
449 
450   if (__orc_rt_elfnix_jit_dlclose(H) == -1)
451     __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
452 
453   return Result;
454 }
455