1 //===- OrcCBindingsStack.h - Orc JIT stack for C bindings -----*- C++ -*---===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9
10 #ifndef LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H
11 #define LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H
12
13 #include "llvm-c/OrcBindings.h"
14 #include "llvm-c/TargetMachine.h"
15 #include "llvm/ADT/STLExtras.h"
16 #include "llvm/ADT/StringRef.h"
17 #include "llvm/ExecutionEngine/JITSymbol.h"
18 #include "llvm/ExecutionEngine/JITEventListener.h"
19 #include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h"
20 #include "llvm/ExecutionEngine/Orc/CompileUtils.h"
21 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
22 #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
23 #include "llvm/ExecutionEngine/Orc/LambdaResolver.h"
24 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
25 #include "llvm/ExecutionEngine/RuntimeDyld.h"
26 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
27 #include "llvm/IR/DataLayout.h"
28 #include "llvm/IR/Mangler.h"
29 #include "llvm/IR/Module.h"
30 #include "llvm/Support/CBindingWrapping.h"
31 #include "llvm/Support/Error.h"
32 #include "llvm/Support/raw_ostream.h"
33 #include "llvm/Target/TargetMachine.h"
34 #include <algorithm>
35 #include <cstdint>
36 #include <functional>
37 #include <map>
38 #include <memory>
39 #include <set>
40 #include <string>
41 #include <vector>
42
43 namespace llvm {
44
45 class OrcCBindingsStack;
46
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcCBindingsStack,LLVMOrcJITStackRef)47 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcCBindingsStack, LLVMOrcJITStackRef)
48 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
49
50 namespace detail {
51
52 // FIXME: Kill this off once the Layer concept becomes an interface.
53 class GenericLayer {
54 public:
55 virtual ~GenericLayer() = default;
56
57 virtual JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
58 bool ExportedSymbolsOnly) = 0;
59 virtual Error removeModule(orc::VModuleKey K) = 0;
60 };
61
62 template <typename LayerT> class GenericLayerImpl : public GenericLayer {
63 public:
64 GenericLayerImpl(LayerT &Layer) : Layer(Layer) {}
65
66 JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
67 bool ExportedSymbolsOnly) override {
68 return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly);
69 }
70
71 Error removeModule(orc::VModuleKey K) override {
72 return Layer.removeModule(K);
73 }
74
75 private:
76 LayerT &Layer;
77 };
78
79 template <>
80 class GenericLayerImpl<orc::LegacyRTDyldObjectLinkingLayer> : public GenericLayer {
81 private:
82 using LayerT = orc::LegacyRTDyldObjectLinkingLayer;
83 public:
84 GenericLayerImpl(LayerT &Layer) : Layer(Layer) {}
85
86 JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
87 bool ExportedSymbolsOnly) override {
88 return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly);
89 }
90
91 Error removeModule(orc::VModuleKey K) override {
92 return Layer.removeObject(K);
93 }
94
95 private:
96 LayerT &Layer;
97 };
98
99 template <typename LayerT>
100 std::unique_ptr<GenericLayerImpl<LayerT>> createGenericLayer(LayerT &Layer) {
101 return llvm::make_unique<GenericLayerImpl<LayerT>>(Layer);
102 }
103
104 } // end namespace detail
105
106 class OrcCBindingsStack {
107 public:
108
109 using CompileCallbackMgr = orc::JITCompileCallbackManager;
110 using ObjLayerT = orc::LegacyRTDyldObjectLinkingLayer;
111 using CompileLayerT = orc::LegacyIRCompileLayer<ObjLayerT, orc::SimpleCompiler>;
112 using CODLayerT =
113 orc::LegacyCompileOnDemandLayer<CompileLayerT, CompileCallbackMgr>;
114
115 using CallbackManagerBuilder =
116 std::function<std::unique_ptr<CompileCallbackMgr>()>;
117
118 using IndirectStubsManagerBuilder = CODLayerT::IndirectStubsManagerBuilderT;
119
120 private:
121
122 using OwningObject = object::OwningBinary<object::ObjectFile>;
123
124 class CBindingsResolver : public orc::SymbolResolver {
125 public:
CBindingsResolver(OrcCBindingsStack & Stack,LLVMOrcSymbolResolverFn ExternalResolver,void * ExternalResolverCtx)126 CBindingsResolver(OrcCBindingsStack &Stack,
127 LLVMOrcSymbolResolverFn ExternalResolver,
128 void *ExternalResolverCtx)
129 : Stack(Stack), ExternalResolver(std::move(ExternalResolver)),
130 ExternalResolverCtx(std::move(ExternalResolverCtx)) {}
131
132 orc::SymbolNameSet
getResponsibilitySet(const orc::SymbolNameSet & Symbols)133 getResponsibilitySet(const orc::SymbolNameSet &Symbols) override {
134 orc::SymbolNameSet Result;
135
136 for (auto &S : Symbols) {
137 if (auto Sym = findSymbol(*S)) {
138 if (!Sym.getFlags().isStrong())
139 Result.insert(S);
140 } else if (auto Err = Sym.takeError()) {
141 Stack.reportError(std::move(Err));
142 return orc::SymbolNameSet();
143 }
144 }
145
146 return Result;
147 }
148
149 orc::SymbolNameSet
lookup(std::shared_ptr<orc::AsynchronousSymbolQuery> Query,orc::SymbolNameSet Symbols)150 lookup(std::shared_ptr<orc::AsynchronousSymbolQuery> Query,
151 orc::SymbolNameSet Symbols) override {
152 orc::SymbolNameSet UnresolvedSymbols;
153
154 for (auto &S : Symbols) {
155 if (auto Sym = findSymbol(*S)) {
156 if (auto Addr = Sym.getAddress()) {
157 Query->resolve(S, JITEvaluatedSymbol(*Addr, Sym.getFlags()));
158 Query->notifySymbolReady();
159 } else {
160 Stack.ES.legacyFailQuery(*Query, Addr.takeError());
161 return orc::SymbolNameSet();
162 }
163 } else if (auto Err = Sym.takeError()) {
164 Stack.ES.legacyFailQuery(*Query, std::move(Err));
165 return orc::SymbolNameSet();
166 } else
167 UnresolvedSymbols.insert(S);
168 }
169
170 if (Query->isFullyResolved())
171 Query->handleFullyResolved();
172
173 if (Query->isFullyReady())
174 Query->handleFullyReady();
175
176 return UnresolvedSymbols;
177 }
178
179 private:
findSymbol(const std::string & Name)180 JITSymbol findSymbol(const std::string &Name) {
181 // Search order:
182 // 1. JIT'd symbols.
183 // 2. Runtime overrides.
184 // 3. External resolver (if present).
185
186 if (Stack.CODLayer) {
187 if (auto Sym = Stack.CODLayer->findSymbol(Name, true))
188 return Sym;
189 else if (auto Err = Sym.takeError())
190 return Sym.takeError();
191 } else {
192 if (auto Sym = Stack.CompileLayer.findSymbol(Name, true))
193 return Sym;
194 else if (auto Err = Sym.takeError())
195 return Sym.takeError();
196 }
197
198 if (auto Sym = Stack.CXXRuntimeOverrides.searchOverrides(Name))
199 return Sym;
200
201 if (ExternalResolver)
202 return JITSymbol(ExternalResolver(Name.c_str(), ExternalResolverCtx),
203 JITSymbolFlags::Exported);
204
205 return JITSymbol(nullptr);
206 }
207
208 OrcCBindingsStack &Stack;
209 LLVMOrcSymbolResolverFn ExternalResolver;
210 void *ExternalResolverCtx = nullptr;
211 };
212
213 public:
OrcCBindingsStack(TargetMachine & TM,IndirectStubsManagerBuilder IndirectStubsMgrBuilder)214 OrcCBindingsStack(TargetMachine &TM,
215 IndirectStubsManagerBuilder IndirectStubsMgrBuilder)
216 : CCMgr(createCompileCallbackManager(TM, ES)), DL(TM.createDataLayout()),
217 IndirectStubsMgr(IndirectStubsMgrBuilder()),
218 ObjectLayer(ES,
219 [this](orc::VModuleKey K) {
220 auto ResolverI = Resolvers.find(K);
221 assert(ResolverI != Resolvers.end() &&
222 "No resolver for module K");
223 auto Resolver = std::move(ResolverI->second);
224 Resolvers.erase(ResolverI);
225 return ObjLayerT::Resources{
226 std::make_shared<SectionMemoryManager>(), Resolver};
227 },
228 nullptr,
229 [this](orc::VModuleKey K, const object::ObjectFile &Obj,
230 const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) {
231 this->notifyFinalized(K, Obj, LoadedObjInfo);
232 },
233 [this](orc::VModuleKey K, const object::ObjectFile &Obj) {
234 this->notifyFreed(K, Obj);
235 }),
236 CompileLayer(ObjectLayer, orc::SimpleCompiler(TM)),
237 CODLayer(createCODLayer(ES, CompileLayer, CCMgr.get(),
238 std::move(IndirectStubsMgrBuilder), Resolvers)),
239 CXXRuntimeOverrides(
240 [this](const std::string &S) { return mangle(S); }) {}
241
shutdown()242 Error shutdown() {
243 // Run any destructors registered with __cxa_atexit.
244 CXXRuntimeOverrides.runDestructors();
245 // Run any IR destructors.
246 for (auto &DtorRunner : IRStaticDestructorRunners)
247 if (auto Err = DtorRunner.runViaLayer(*this))
248 return Err;
249 return Error::success();
250 }
251
mangle(StringRef Name)252 std::string mangle(StringRef Name) {
253 std::string MangledName;
254 {
255 raw_string_ostream MangledNameStream(MangledName);
256 Mangler::getNameWithPrefix(MangledNameStream, Name, DL);
257 }
258 return MangledName;
259 }
260
261 template <typename PtrTy>
fromTargetAddress(JITTargetAddress Addr)262 static PtrTy fromTargetAddress(JITTargetAddress Addr) {
263 return reinterpret_cast<PtrTy>(static_cast<uintptr_t>(Addr));
264 }
265
266 Expected<JITTargetAddress>
createLazyCompileCallback(LLVMOrcLazyCompileCallbackFn Callback,void * CallbackCtx)267 createLazyCompileCallback(LLVMOrcLazyCompileCallbackFn Callback,
268 void *CallbackCtx) {
269 auto WrappedCallback = [=]() -> JITTargetAddress {
270 return Callback(wrap(this), CallbackCtx);
271 };
272
273 return CCMgr->getCompileCallback(std::move(WrappedCallback));
274 }
275
createIndirectStub(StringRef StubName,JITTargetAddress Addr)276 Error createIndirectStub(StringRef StubName, JITTargetAddress Addr) {
277 return IndirectStubsMgr->createStub(StubName, Addr,
278 JITSymbolFlags::Exported);
279 }
280
setIndirectStubPointer(StringRef Name,JITTargetAddress Addr)281 Error setIndirectStubPointer(StringRef Name, JITTargetAddress Addr) {
282 return IndirectStubsMgr->updatePointer(Name, Addr);
283 }
284
285 template <typename LayerT>
286 Expected<orc::VModuleKey>
addIRModule(LayerT & Layer,std::unique_ptr<Module> M,std::unique_ptr<RuntimeDyld::MemoryManager> MemMgr,LLVMOrcSymbolResolverFn ExternalResolver,void * ExternalResolverCtx)287 addIRModule(LayerT &Layer, std::unique_ptr<Module> M,
288 std::unique_ptr<RuntimeDyld::MemoryManager> MemMgr,
289 LLVMOrcSymbolResolverFn ExternalResolver,
290 void *ExternalResolverCtx) {
291
292 // Attach a data-layout if one isn't already present.
293 if (M->getDataLayout().isDefault())
294 M->setDataLayout(DL);
295
296 // Record the static constructors and destructors. We have to do this before
297 // we hand over ownership of the module to the JIT.
298 std::vector<std::string> CtorNames, DtorNames;
299 for (auto Ctor : orc::getConstructors(*M))
300 CtorNames.push_back(mangle(Ctor.Func->getName()));
301 for (auto Dtor : orc::getDestructors(*M))
302 DtorNames.push_back(mangle(Dtor.Func->getName()));
303
304 // Add the module to the JIT.
305 auto K = ES.allocateVModule();
306 Resolvers[K] = std::make_shared<CBindingsResolver>(*this, ExternalResolver,
307 ExternalResolverCtx);
308 if (auto Err = Layer.addModule(K, std::move(M)))
309 return std::move(Err);
310
311 KeyLayers[K] = detail::createGenericLayer(Layer);
312
313 // Run the static constructors, and save the static destructor runner for
314 // execution when the JIT is torn down.
315 orc::LegacyCtorDtorRunner<OrcCBindingsStack> CtorRunner(std::move(CtorNames), K);
316 if (auto Err = CtorRunner.runViaLayer(*this))
317 return std::move(Err);
318
319 IRStaticDestructorRunners.emplace_back(std::move(DtorNames), K);
320
321 return K;
322 }
323
324 Expected<orc::VModuleKey>
addIRModuleEager(std::unique_ptr<Module> M,LLVMOrcSymbolResolverFn ExternalResolver,void * ExternalResolverCtx)325 addIRModuleEager(std::unique_ptr<Module> M,
326 LLVMOrcSymbolResolverFn ExternalResolver,
327 void *ExternalResolverCtx) {
328 return addIRModule(CompileLayer, std::move(M),
329 llvm::make_unique<SectionMemoryManager>(),
330 std::move(ExternalResolver), ExternalResolverCtx);
331 }
332
333 Expected<orc::VModuleKey>
addIRModuleLazy(std::unique_ptr<Module> M,LLVMOrcSymbolResolverFn ExternalResolver,void * ExternalResolverCtx)334 addIRModuleLazy(std::unique_ptr<Module> M,
335 LLVMOrcSymbolResolverFn ExternalResolver,
336 void *ExternalResolverCtx) {
337 if (!CODLayer)
338 return make_error<StringError>("Can not add lazy module: No compile "
339 "callback manager available",
340 inconvertibleErrorCode());
341
342 return addIRModule(*CODLayer, std::move(M),
343 llvm::make_unique<SectionMemoryManager>(),
344 std::move(ExternalResolver), ExternalResolverCtx);
345 }
346
removeModule(orc::VModuleKey K)347 Error removeModule(orc::VModuleKey K) {
348 // FIXME: Should error release the module key?
349 if (auto Err = KeyLayers[K]->removeModule(K))
350 return Err;
351 ES.releaseVModule(K);
352 KeyLayers.erase(K);
353 return Error::success();
354 }
355
addObject(std::unique_ptr<MemoryBuffer> ObjBuffer,LLVMOrcSymbolResolverFn ExternalResolver,void * ExternalResolverCtx)356 Expected<orc::VModuleKey> addObject(std::unique_ptr<MemoryBuffer> ObjBuffer,
357 LLVMOrcSymbolResolverFn ExternalResolver,
358 void *ExternalResolverCtx) {
359 if (auto Obj = object::ObjectFile::createObjectFile(
360 ObjBuffer->getMemBufferRef())) {
361
362 auto K = ES.allocateVModule();
363 Resolvers[K] = std::make_shared<CBindingsResolver>(
364 *this, ExternalResolver, ExternalResolverCtx);
365
366 if (auto Err = ObjectLayer.addObject(K, std::move(ObjBuffer)))
367 return std::move(Err);
368
369 KeyLayers[K] = detail::createGenericLayer(ObjectLayer);
370
371 return K;
372 } else
373 return Obj.takeError();
374 }
375
findSymbol(const std::string & Name,bool ExportedSymbolsOnly)376 JITSymbol findSymbol(const std::string &Name,
377 bool ExportedSymbolsOnly) {
378 if (auto Sym = IndirectStubsMgr->findStub(Name, ExportedSymbolsOnly))
379 return Sym;
380 if (CODLayer)
381 return CODLayer->findSymbol(mangle(Name), ExportedSymbolsOnly);
382 return CompileLayer.findSymbol(mangle(Name), ExportedSymbolsOnly);
383 }
384
findSymbolIn(orc::VModuleKey K,const std::string & Name,bool ExportedSymbolsOnly)385 JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
386 bool ExportedSymbolsOnly) {
387 assert(KeyLayers.count(K) && "looking up symbol in unknown module");
388 return KeyLayers[K]->findSymbolIn(K, mangle(Name), ExportedSymbolsOnly);
389 }
390
findSymbolAddress(const std::string & Name,bool ExportedSymbolsOnly)391 Expected<JITTargetAddress> findSymbolAddress(const std::string &Name,
392 bool ExportedSymbolsOnly) {
393 if (auto Sym = findSymbol(Name, ExportedSymbolsOnly)) {
394 // Successful lookup, non-null symbol:
395 if (auto AddrOrErr = Sym.getAddress())
396 return *AddrOrErr;
397 else
398 return AddrOrErr.takeError();
399 } else if (auto Err = Sym.takeError()) {
400 // Lookup failure - report error.
401 return std::move(Err);
402 }
403
404 // No symbol not found. Return 0.
405 return 0;
406 }
407
findSymbolAddressIn(orc::VModuleKey K,const std::string & Name,bool ExportedSymbolsOnly)408 Expected<JITTargetAddress> findSymbolAddressIn(orc::VModuleKey K,
409 const std::string &Name,
410 bool ExportedSymbolsOnly) {
411 if (auto Sym = findSymbolIn(K, Name, ExportedSymbolsOnly)) {
412 // Successful lookup, non-null symbol:
413 if (auto AddrOrErr = Sym.getAddress())
414 return *AddrOrErr;
415 else
416 return AddrOrErr.takeError();
417 } else if (auto Err = Sym.takeError()) {
418 // Lookup failure - report error.
419 return std::move(Err);
420 }
421
422 // Symbol not found. Return 0.
423 return 0;
424 }
425
getErrorMessage()426 const std::string &getErrorMessage() const { return ErrMsg; }
427
RegisterJITEventListener(JITEventListener * L)428 void RegisterJITEventListener(JITEventListener *L) {
429 if (!L)
430 return;
431 EventListeners.push_back(L);
432 }
433
UnregisterJITEventListener(JITEventListener * L)434 void UnregisterJITEventListener(JITEventListener *L) {
435 if (!L)
436 return;
437
438 auto I = find(reverse(EventListeners), L);
439 if (I != EventListeners.rend()) {
440 std::swap(*I, EventListeners.back());
441 EventListeners.pop_back();
442 }
443 }
444
445 private:
446 using ResolverMap =
447 std::map<orc::VModuleKey, std::shared_ptr<orc::SymbolResolver>>;
448
449 static std::unique_ptr<CompileCallbackMgr>
createCompileCallbackManager(TargetMachine & TM,orc::ExecutionSession & ES)450 createCompileCallbackManager(TargetMachine &TM, orc::ExecutionSession &ES) {
451 auto CCMgr = createLocalCompileCallbackManager(TM.getTargetTriple(), ES, 0);
452 if (!CCMgr) {
453 // FIXME: It would be good if we could report this somewhere, but we do
454 // have an instance yet.
455 logAllUnhandledErrors(CCMgr.takeError(), errs(), "ORC error: ");
456 return nullptr;
457 }
458 return std::move(*CCMgr);
459 }
460
461 static std::unique_ptr<CODLayerT>
createCODLayer(orc::ExecutionSession & ES,CompileLayerT & CompileLayer,CompileCallbackMgr * CCMgr,IndirectStubsManagerBuilder IndirectStubsMgrBuilder,ResolverMap & Resolvers)462 createCODLayer(orc::ExecutionSession &ES, CompileLayerT &CompileLayer,
463 CompileCallbackMgr *CCMgr,
464 IndirectStubsManagerBuilder IndirectStubsMgrBuilder,
465 ResolverMap &Resolvers) {
466 // If there is no compile callback manager available we can not create a
467 // compile on demand layer.
468 if (!CCMgr)
469 return nullptr;
470
471 return llvm::make_unique<CODLayerT>(
472 ES, CompileLayer,
473 [&Resolvers](orc::VModuleKey K) {
474 auto ResolverI = Resolvers.find(K);
475 assert(ResolverI != Resolvers.end() && "No resolver for module K");
476 return ResolverI->second;
477 },
478 [&Resolvers](orc::VModuleKey K,
479 std::shared_ptr<orc::SymbolResolver> Resolver) {
480 assert(!Resolvers.count(K) && "Resolver already present");
481 Resolvers[K] = std::move(Resolver);
482 },
483 [](Function &F) { return std::set<Function *>({&F}); }, *CCMgr,
484 std::move(IndirectStubsMgrBuilder), false);
485 }
486
reportError(Error Err)487 void reportError(Error Err) {
488 // FIXME: Report errors on the execution session.
489 logAllUnhandledErrors(std::move(Err), errs(), "ORC error: ");
490 };
491
notifyFinalized(orc::VModuleKey K,const object::ObjectFile & Obj,const RuntimeDyld::LoadedObjectInfo & LoadedObjInfo)492 void notifyFinalized(orc::VModuleKey K,
493 const object::ObjectFile &Obj,
494 const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) {
495 uint64_t Key = static_cast<uint64_t>(
496 reinterpret_cast<uintptr_t>(Obj.getData().data()));
497 for (auto &Listener : EventListeners)
498 Listener->notifyObjectLoaded(Key, Obj, LoadedObjInfo);
499 }
500
notifyFreed(orc::VModuleKey K,const object::ObjectFile & Obj)501 void notifyFreed(orc::VModuleKey K, const object::ObjectFile &Obj) {
502 uint64_t Key = static_cast<uint64_t>(
503 reinterpret_cast<uintptr_t>(Obj.getData().data()));
504 for (auto &Listener : EventListeners)
505 Listener->notifyFreeingObject(Key);
506 }
507
508 orc::ExecutionSession ES;
509 std::unique_ptr<CompileCallbackMgr> CCMgr;
510
511 std::vector<JITEventListener *> EventListeners;
512
513 DataLayout DL;
514 SectionMemoryManager CCMgrMemMgr;
515
516 std::unique_ptr<orc::IndirectStubsManager> IndirectStubsMgr;
517
518 ObjLayerT ObjectLayer;
519 CompileLayerT CompileLayer;
520 std::unique_ptr<CODLayerT> CODLayer;
521
522 std::map<orc::VModuleKey, std::unique_ptr<detail::GenericLayer>> KeyLayers;
523
524 orc::LegacyLocalCXXRuntimeOverrides CXXRuntimeOverrides;
525 std::vector<orc::LegacyCtorDtorRunner<OrcCBindingsStack>> IRStaticDestructorRunners;
526 std::string ErrMsg;
527
528 ResolverMap Resolvers;
529 };
530
531 } // end namespace llvm
532
533 #endif // LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H
534