1 //===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9
10 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
11 #include "llvm/ADT/STLExtras.h"
12 #include "llvm/ADT/Triple.h"
13 #include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
14 #include "llvm/IR/CallSite.h"
15 #include "llvm/IR/IRBuilder.h"
16 #include "llvm/Support/Format.h"
17 #include "llvm/Transforms/Utils/Cloning.h"
18 #include <sstream>
19
20 using namespace llvm;
21 using namespace llvm::orc;
22
23 namespace {
24
25 class CompileCallbackMaterializationUnit : public orc::MaterializationUnit {
26 public:
27 using CompileFunction = JITCompileCallbackManager::CompileFunction;
28
CompileCallbackMaterializationUnit(SymbolStringPtr Name,CompileFunction Compile,VModuleKey K)29 CompileCallbackMaterializationUnit(SymbolStringPtr Name,
30 CompileFunction Compile, VModuleKey K)
31 : MaterializationUnit(SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}),
32 std::move(K)),
33 Name(std::move(Name)), Compile(std::move(Compile)) {}
34
getName() const35 StringRef getName() const override { return "<Compile Callbacks>"; }
36
37 private:
materialize(MaterializationResponsibility R)38 void materialize(MaterializationResponsibility R) override {
39 SymbolMap Result;
40 Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported);
41 R.resolve(Result);
42 R.emit();
43 }
44
discard(const JITDylib & JD,const SymbolStringPtr & Name)45 void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
46 llvm_unreachable("Discard should never occur on a LMU?");
47 }
48
49 SymbolStringPtr Name;
50 CompileFunction Compile;
51 };
52
53 } // namespace
54
55 namespace llvm {
56 namespace orc {
57
anchor()58 void IndirectStubsManager::anchor() {}
anchor()59 void TrampolinePool::anchor() {}
60
61 Expected<JITTargetAddress>
getCompileCallback(CompileFunction Compile)62 JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) {
63 if (auto TrampolineAddr = TP->getTrampoline()) {
64 auto CallbackName =
65 ES.intern(std::string("cc") + std::to_string(++NextCallbackId));
66
67 std::lock_guard<std::mutex> Lock(CCMgrMutex);
68 AddrToSymbol[*TrampolineAddr] = CallbackName;
69 cantFail(CallbacksJD.define(
70 llvm::make_unique<CompileCallbackMaterializationUnit>(
71 std::move(CallbackName), std::move(Compile),
72 ES.allocateVModule())));
73 return *TrampolineAddr;
74 } else
75 return TrampolineAddr.takeError();
76 }
77
executeCompileCallback(JITTargetAddress TrampolineAddr)78 JITTargetAddress JITCompileCallbackManager::executeCompileCallback(
79 JITTargetAddress TrampolineAddr) {
80 SymbolStringPtr Name;
81
82 {
83 std::unique_lock<std::mutex> Lock(CCMgrMutex);
84 auto I = AddrToSymbol.find(TrampolineAddr);
85
86 // If this address is not associated with a compile callback then report an
87 // error to the execution session and return ErrorHandlerAddress to the
88 // callee.
89 if (I == AddrToSymbol.end()) {
90 Lock.unlock();
91 std::string ErrMsg;
92 {
93 raw_string_ostream ErrMsgStream(ErrMsg);
94 ErrMsgStream << "No compile callback for trampoline at "
95 << format("0x%016" PRIx64, TrampolineAddr);
96 }
97 ES.reportError(
98 make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()));
99 return ErrorHandlerAddress;
100 } else
101 Name = I->second;
102 }
103
104 if (auto Sym = ES.lookup(JITDylibSearchList({{&CallbacksJD, true}}), Name))
105 return Sym->getAddress();
106 else {
107 llvm::dbgs() << "Didn't find callback.\n";
108 // If anything goes wrong materializing Sym then report it to the session
109 // and return the ErrorHandlerAddress;
110 ES.reportError(Sym.takeError());
111 return ErrorHandlerAddress;
112 }
113 }
114
115 Expected<std::unique_ptr<JITCompileCallbackManager>>
createLocalCompileCallbackManager(const Triple & T,ExecutionSession & ES,JITTargetAddress ErrorHandlerAddress)116 createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES,
117 JITTargetAddress ErrorHandlerAddress) {
118 switch (T.getArch()) {
119 default:
120 return make_error<StringError>(
121 std::string("No callback manager available for ") + T.str(),
122 inconvertibleErrorCode());
123 case Triple::aarch64: {
124 typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT;
125 return CCMgrT::Create(ES, ErrorHandlerAddress);
126 }
127
128 case Triple::x86: {
129 typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT;
130 return CCMgrT::Create(ES, ErrorHandlerAddress);
131 }
132
133 case Triple::mips: {
134 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT;
135 return CCMgrT::Create(ES, ErrorHandlerAddress);
136 }
137 case Triple::mipsel: {
138 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT;
139 return CCMgrT::Create(ES, ErrorHandlerAddress);
140 }
141
142 case Triple::mips64:
143 case Triple::mips64el: {
144 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT;
145 return CCMgrT::Create(ES, ErrorHandlerAddress);
146 }
147
148 case Triple::x86_64: {
149 if ( T.getOS() == Triple::OSType::Win32 ) {
150 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT;
151 return CCMgrT::Create(ES, ErrorHandlerAddress);
152 } else {
153 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT;
154 return CCMgrT::Create(ES, ErrorHandlerAddress);
155 }
156 }
157
158 }
159 }
160
161 std::function<std::unique_ptr<IndirectStubsManager>()>
createLocalIndirectStubsManagerBuilder(const Triple & T)162 createLocalIndirectStubsManagerBuilder(const Triple &T) {
163 switch (T.getArch()) {
164 default:
165 return [](){
166 return llvm::make_unique<
167 orc::LocalIndirectStubsManager<orc::OrcGenericABI>>();
168 };
169
170 case Triple::aarch64:
171 return [](){
172 return llvm::make_unique<
173 orc::LocalIndirectStubsManager<orc::OrcAArch64>>();
174 };
175
176 case Triple::x86:
177 return [](){
178 return llvm::make_unique<
179 orc::LocalIndirectStubsManager<orc::OrcI386>>();
180 };
181
182 case Triple::mips:
183 return [](){
184 return llvm::make_unique<
185 orc::LocalIndirectStubsManager<orc::OrcMips32Be>>();
186 };
187
188 case Triple::mipsel:
189 return [](){
190 return llvm::make_unique<
191 orc::LocalIndirectStubsManager<orc::OrcMips32Le>>();
192 };
193
194 case Triple::mips64:
195 case Triple::mips64el:
196 return [](){
197 return llvm::make_unique<
198 orc::LocalIndirectStubsManager<orc::OrcMips64>>();
199 };
200
201 case Triple::x86_64:
202 if (T.getOS() == Triple::OSType::Win32) {
203 return [](){
204 return llvm::make_unique<
205 orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>();
206 };
207 } else {
208 return [](){
209 return llvm::make_unique<
210 orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>();
211 };
212 }
213
214 }
215 }
216
createIRTypedAddress(FunctionType & FT,JITTargetAddress Addr)217 Constant* createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr) {
218 Constant *AddrIntVal =
219 ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr);
220 Constant *AddrPtrVal =
221 ConstantExpr::getCast(Instruction::IntToPtr, AddrIntVal,
222 PointerType::get(&FT, 0));
223 return AddrPtrVal;
224 }
225
createImplPointer(PointerType & PT,Module & M,const Twine & Name,Constant * Initializer)226 GlobalVariable* createImplPointer(PointerType &PT, Module &M,
227 const Twine &Name, Constant *Initializer) {
228 auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage,
229 Initializer, Name, nullptr,
230 GlobalValue::NotThreadLocal, 0, true);
231 IP->setVisibility(GlobalValue::HiddenVisibility);
232 return IP;
233 }
234
makeStub(Function & F,Value & ImplPointer)235 void makeStub(Function &F, Value &ImplPointer) {
236 assert(F.isDeclaration() && "Can't turn a definition into a stub.");
237 assert(F.getParent() && "Function isn't in a module.");
238 Module &M = *F.getParent();
239 BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F);
240 IRBuilder<> Builder(EntryBlock);
241 LoadInst *ImplAddr = Builder.CreateLoad(&ImplPointer);
242 std::vector<Value*> CallArgs;
243 for (auto &A : F.args())
244 CallArgs.push_back(&A);
245 CallInst *Call = Builder.CreateCall(ImplAddr, CallArgs);
246 Call->setTailCall();
247 Call->setAttributes(F.getAttributes());
248 if (F.getReturnType()->isVoidTy())
249 Builder.CreateRetVoid();
250 else
251 Builder.CreateRet(Call);
252 }
253
operator ()(Module & M)254 std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) {
255 std::vector<GlobalValue *> PromotedGlobals;
256
257 for (auto &GV : M.global_values()) {
258 bool Promoted = true;
259
260 // Rename if necessary.
261 if (!GV.hasName())
262 GV.setName("__orc_anon." + Twine(NextId++));
263 else if (GV.getName().startswith("\01L"))
264 GV.setName("__" + GV.getName().substr(1) + "." + Twine(NextId++));
265 else if (GV.hasLocalLinkage())
266 GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++));
267 else
268 Promoted = false;
269
270 if (GV.hasLocalLinkage()) {
271 GV.setLinkage(GlobalValue::ExternalLinkage);
272 GV.setVisibility(GlobalValue::HiddenVisibility);
273 Promoted = true;
274 }
275 GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None);
276
277 if (Promoted)
278 PromotedGlobals.push_back(&GV);
279 }
280
281 return PromotedGlobals;
282 }
283
cloneFunctionDecl(Module & Dst,const Function & F,ValueToValueMapTy * VMap)284 Function* cloneFunctionDecl(Module &Dst, const Function &F,
285 ValueToValueMapTy *VMap) {
286 Function *NewF =
287 Function::Create(cast<FunctionType>(F.getValueType()),
288 F.getLinkage(), F.getName(), &Dst);
289 NewF->copyAttributesFrom(&F);
290
291 if (VMap) {
292 (*VMap)[&F] = NewF;
293 auto NewArgI = NewF->arg_begin();
294 for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE;
295 ++ArgI, ++NewArgI)
296 (*VMap)[&*ArgI] = &*NewArgI;
297 }
298
299 return NewF;
300 }
301
moveFunctionBody(Function & OrigF,ValueToValueMapTy & VMap,ValueMaterializer * Materializer,Function * NewF)302 void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap,
303 ValueMaterializer *Materializer,
304 Function *NewF) {
305 assert(!OrigF.isDeclaration() && "Nothing to move");
306 if (!NewF)
307 NewF = cast<Function>(VMap[&OrigF]);
308 else
309 assert(VMap[&OrigF] == NewF && "Incorrect function mapping in VMap.");
310 assert(NewF && "Function mapping missing from VMap.");
311 assert(NewF->getParent() != OrigF.getParent() &&
312 "moveFunctionBody should only be used to move bodies between "
313 "modules.");
314
315 SmallVector<ReturnInst *, 8> Returns; // Ignore returns cloned.
316 CloneFunctionInto(NewF, &OrigF, VMap, /*ModuleLevelChanges=*/true, Returns,
317 "", nullptr, nullptr, Materializer);
318 OrigF.deleteBody();
319 }
320
cloneGlobalVariableDecl(Module & Dst,const GlobalVariable & GV,ValueToValueMapTy * VMap)321 GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
322 ValueToValueMapTy *VMap) {
323 GlobalVariable *NewGV = new GlobalVariable(
324 Dst, GV.getValueType(), GV.isConstant(),
325 GV.getLinkage(), nullptr, GV.getName(), nullptr,
326 GV.getThreadLocalMode(), GV.getType()->getAddressSpace());
327 NewGV->copyAttributesFrom(&GV);
328 if (VMap)
329 (*VMap)[&GV] = NewGV;
330 return NewGV;
331 }
332
moveGlobalVariableInitializer(GlobalVariable & OrigGV,ValueToValueMapTy & VMap,ValueMaterializer * Materializer,GlobalVariable * NewGV)333 void moveGlobalVariableInitializer(GlobalVariable &OrigGV,
334 ValueToValueMapTy &VMap,
335 ValueMaterializer *Materializer,
336 GlobalVariable *NewGV) {
337 assert(OrigGV.hasInitializer() && "Nothing to move");
338 if (!NewGV)
339 NewGV = cast<GlobalVariable>(VMap[&OrigGV]);
340 else
341 assert(VMap[&OrigGV] == NewGV &&
342 "Incorrect global variable mapping in VMap.");
343 assert(NewGV->getParent() != OrigGV.getParent() &&
344 "moveGlobalVariableInitializer should only be used to move "
345 "initializers between modules");
346
347 NewGV->setInitializer(MapValue(OrigGV.getInitializer(), VMap, RF_None,
348 nullptr, Materializer));
349 }
350
cloneGlobalAliasDecl(Module & Dst,const GlobalAlias & OrigA,ValueToValueMapTy & VMap)351 GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,
352 ValueToValueMapTy &VMap) {
353 assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?");
354 auto *NewA = GlobalAlias::create(OrigA.getValueType(),
355 OrigA.getType()->getPointerAddressSpace(),
356 OrigA.getLinkage(), OrigA.getName(), &Dst);
357 NewA->copyAttributesFrom(&OrigA);
358 VMap[&OrigA] = NewA;
359 return NewA;
360 }
361
cloneModuleFlagsMetadata(Module & Dst,const Module & Src,ValueToValueMapTy & VMap)362 void cloneModuleFlagsMetadata(Module &Dst, const Module &Src,
363 ValueToValueMapTy &VMap) {
364 auto *MFs = Src.getModuleFlagsMetadata();
365 if (!MFs)
366 return;
367 for (auto *MF : MFs->operands())
368 Dst.addModuleFlag(MapMetadata(MF, VMap));
369 }
370
371 } // End namespace orc.
372 } // End namespace llvm.
373