1 //===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
10 #include "llvm/ADT/STLExtras.h"
11 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
12 #include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
13 #include "llvm/IR/IRBuilder.h"
14 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
15 #include "llvm/MC/MCInstrAnalysis.h"
16 #include "llvm/Support/Format.h"
17 #include "llvm/TargetParser/Triple.h"
18 #include "llvm/Transforms/Utils/Cloning.h"
19 #include <sstream>
20 
21 #define DEBUG_TYPE "orc"
22 
23 using namespace llvm;
24 using namespace llvm::orc;
25 
26 namespace {
27 
28 class CompileCallbackMaterializationUnit : public orc::MaterializationUnit {
29 public:
30   using CompileFunction = JITCompileCallbackManager::CompileFunction;
31 
CompileCallbackMaterializationUnit(SymbolStringPtr Name,CompileFunction Compile)32   CompileCallbackMaterializationUnit(SymbolStringPtr Name,
33                                      CompileFunction Compile)
34       : MaterializationUnit(Interface(
35             SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), nullptr)),
36         Name(std::move(Name)), Compile(std::move(Compile)) {}
37 
getName() const38   StringRef getName() const override { return "<Compile Callbacks>"; }
39 
40 private:
materialize(std::unique_ptr<MaterializationResponsibility> R)41   void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
42     SymbolMap Result;
43     Result[Name] = {Compile(), JITSymbolFlags::Exported};
44     // No dependencies, so these calls cannot fail.
45     cantFail(R->notifyResolved(Result));
46     cantFail(R->notifyEmitted());
47   }
48 
discard(const JITDylib & JD,const SymbolStringPtr & Name)49   void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
50     llvm_unreachable("Discard should never occur on a LMU?");
51   }
52 
53   SymbolStringPtr Name;
54   CompileFunction Compile;
55 };
56 
57 } // namespace
58 
59 namespace llvm {
60 namespace orc {
61 
62 TrampolinePool::~TrampolinePool() = default;
anchor()63 void IndirectStubsManager::anchor() {}
64 
65 Expected<ExecutorAddr>
getCompileCallback(CompileFunction Compile)66 JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) {
67   if (auto TrampolineAddr = TP->getTrampoline()) {
68     auto CallbackName =
69         ES.intern(std::string("cc") + std::to_string(++NextCallbackId));
70 
71     std::lock_guard<std::mutex> Lock(CCMgrMutex);
72     AddrToSymbol[*TrampolineAddr] = CallbackName;
73     cantFail(
74         CallbacksJD.define(std::make_unique<CompileCallbackMaterializationUnit>(
75             std::move(CallbackName), std::move(Compile))));
76     return *TrampolineAddr;
77   } else
78     return TrampolineAddr.takeError();
79 }
80 
81 ExecutorAddr
executeCompileCallback(ExecutorAddr TrampolineAddr)82 JITCompileCallbackManager::executeCompileCallback(ExecutorAddr TrampolineAddr) {
83   SymbolStringPtr Name;
84 
85   {
86     std::unique_lock<std::mutex> Lock(CCMgrMutex);
87     auto I = AddrToSymbol.find(TrampolineAddr);
88 
89     // If this address is not associated with a compile callback then report an
90     // error to the execution session and return ErrorHandlerAddress to the
91     // callee.
92     if (I == AddrToSymbol.end()) {
93       Lock.unlock();
94       ES.reportError(
95           make_error<StringError>("No compile callback for trampoline at " +
96                                       formatv("{0:x}", TrampolineAddr),
97                                   inconvertibleErrorCode()));
98       return ErrorHandlerAddress;
99     } else
100       Name = I->second;
101   }
102 
103   if (auto Sym =
104           ES.lookup(makeJITDylibSearchOrder(
105                         &CallbacksJD, JITDylibLookupFlags::MatchAllSymbols),
106                     Name))
107     return Sym->getAddress();
108   else {
109     llvm::dbgs() << "Didn't find callback.\n";
110     // If anything goes wrong materializing Sym then report it to the session
111     // and return the ErrorHandlerAddress;
112     ES.reportError(Sym.takeError());
113     return ErrorHandlerAddress;
114   }
115 }
116 
117 Expected<std::unique_ptr<JITCompileCallbackManager>>
createLocalCompileCallbackManager(const Triple & T,ExecutionSession & ES,ExecutorAddr ErrorHandlerAddress)118 createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES,
119                                   ExecutorAddr ErrorHandlerAddress) {
120   switch (T.getArch()) {
121   default:
122     return make_error<StringError>(
123         std::string("No callback manager available for ") + T.str(),
124         inconvertibleErrorCode());
125   case Triple::aarch64:
126   case Triple::aarch64_32: {
127     typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT;
128     return CCMgrT::Create(ES, ErrorHandlerAddress);
129     }
130 
131     case Triple::x86: {
132       typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT;
133       return CCMgrT::Create(ES, ErrorHandlerAddress);
134     }
135 
136     case Triple::loongarch64: {
137       typedef orc::LocalJITCompileCallbackManager<orc::OrcLoongArch64> CCMgrT;
138       return CCMgrT::Create(ES, ErrorHandlerAddress);
139     }
140 
141     case Triple::mips: {
142       typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT;
143       return CCMgrT::Create(ES, ErrorHandlerAddress);
144     }
145     case Triple::mipsel: {
146       typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT;
147       return CCMgrT::Create(ES, ErrorHandlerAddress);
148     }
149 
150     case Triple::mips64:
151     case Triple::mips64el: {
152       typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT;
153       return CCMgrT::Create(ES, ErrorHandlerAddress);
154     }
155 
156     case Triple::riscv64: {
157       typedef orc::LocalJITCompileCallbackManager<orc::OrcRiscv64> CCMgrT;
158       return CCMgrT::Create(ES, ErrorHandlerAddress);
159     }
160 
161     case Triple::x86_64: {
162       if (T.getOS() == Triple::OSType::Win32) {
163         typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT;
164         return CCMgrT::Create(ES, ErrorHandlerAddress);
165       } else {
166         typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT;
167         return CCMgrT::Create(ES, ErrorHandlerAddress);
168       }
169     }
170 
171   }
172 }
173 
174 std::function<std::unique_ptr<IndirectStubsManager>()>
createLocalIndirectStubsManagerBuilder(const Triple & T)175 createLocalIndirectStubsManagerBuilder(const Triple &T) {
176   switch (T.getArch()) {
177     default:
178       return [](){
179         return std::make_unique<
180                        orc::LocalIndirectStubsManager<orc::OrcGenericABI>>();
181       };
182 
183     case Triple::aarch64:
184     case Triple::aarch64_32:
185       return [](){
186         return std::make_unique<
187                        orc::LocalIndirectStubsManager<orc::OrcAArch64>>();
188       };
189 
190     case Triple::x86:
191       return [](){
192         return std::make_unique<
193                        orc::LocalIndirectStubsManager<orc::OrcI386>>();
194       };
195 
196     case Triple::loongarch64:
197       return []() {
198         return std::make_unique<
199             orc::LocalIndirectStubsManager<orc::OrcLoongArch64>>();
200       };
201 
202     case Triple::mips:
203       return [](){
204           return std::make_unique<
205                       orc::LocalIndirectStubsManager<orc::OrcMips32Be>>();
206       };
207 
208     case Triple::mipsel:
209       return [](){
210           return std::make_unique<
211                       orc::LocalIndirectStubsManager<orc::OrcMips32Le>>();
212       };
213 
214     case Triple::mips64:
215     case Triple::mips64el:
216       return [](){
217           return std::make_unique<
218                       orc::LocalIndirectStubsManager<orc::OrcMips64>>();
219       };
220 
221     case Triple::riscv64:
222       return []() {
223         return std::make_unique<
224             orc::LocalIndirectStubsManager<orc::OrcRiscv64>>();
225       };
226 
227     case Triple::x86_64:
228       if (T.getOS() == Triple::OSType::Win32) {
229         return [](){
230           return std::make_unique<
231                      orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>();
232         };
233       } else {
234         return [](){
235           return std::make_unique<
236                      orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>();
237         };
238       }
239 
240   }
241 }
242 
createIRTypedAddress(FunctionType & FT,ExecutorAddr Addr)243 Constant* createIRTypedAddress(FunctionType &FT, ExecutorAddr Addr) {
244   Constant *AddrIntVal =
245     ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr.getValue());
246   Constant *AddrPtrVal =
247     ConstantExpr::getIntToPtr(AddrIntVal, PointerType::get(&FT, 0));
248   return AddrPtrVal;
249 }
250 
createImplPointer(PointerType & PT,Module & M,const Twine & Name,Constant * Initializer)251 GlobalVariable* createImplPointer(PointerType &PT, Module &M,
252                                   const Twine &Name, Constant *Initializer) {
253   auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage,
254                                Initializer, Name, nullptr,
255                                GlobalValue::NotThreadLocal, 0, true);
256   IP->setVisibility(GlobalValue::HiddenVisibility);
257   return IP;
258 }
259 
makeStub(Function & F,Value & ImplPointer)260 void makeStub(Function &F, Value &ImplPointer) {
261   assert(F.isDeclaration() && "Can't turn a definition into a stub.");
262   assert(F.getParent() && "Function isn't in a module.");
263   Module &M = *F.getParent();
264   BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F);
265   IRBuilder<> Builder(EntryBlock);
266   LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer);
267   std::vector<Value*> CallArgs;
268   for (auto &A : F.args())
269     CallArgs.push_back(&A);
270   CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs);
271   Call->setTailCall();
272   Call->setAttributes(F.getAttributes());
273   if (F.getReturnType()->isVoidTy())
274     Builder.CreateRetVoid();
275   else
276     Builder.CreateRet(Call);
277 }
278 
operator ()(Module & M)279 std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) {
280   std::vector<GlobalValue *> PromotedGlobals;
281 
282   for (auto &GV : M.global_values()) {
283     bool Promoted = true;
284 
285     // Rename if necessary.
286     if (!GV.hasName())
287       GV.setName("__orc_anon." + Twine(NextId++));
288     else if (GV.getName().starts_with("\01L"))
289       GV.setName("__" + GV.getName().substr(1) + "." + Twine(NextId++));
290     else if (GV.hasLocalLinkage())
291       GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++));
292     else
293       Promoted = false;
294 
295     if (GV.hasLocalLinkage()) {
296       GV.setLinkage(GlobalValue::ExternalLinkage);
297       GV.setVisibility(GlobalValue::HiddenVisibility);
298       Promoted = true;
299     }
300     GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None);
301 
302     if (Promoted)
303       PromotedGlobals.push_back(&GV);
304   }
305 
306   return PromotedGlobals;
307 }
308 
cloneFunctionDecl(Module & Dst,const Function & F,ValueToValueMapTy * VMap)309 Function* cloneFunctionDecl(Module &Dst, const Function &F,
310                             ValueToValueMapTy *VMap) {
311   Function *NewF =
312     Function::Create(cast<FunctionType>(F.getValueType()),
313                      F.getLinkage(), F.getName(), &Dst);
314   NewF->copyAttributesFrom(&F);
315 
316   if (VMap) {
317     (*VMap)[&F] = NewF;
318     auto NewArgI = NewF->arg_begin();
319     for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE;
320          ++ArgI, ++NewArgI)
321       (*VMap)[&*ArgI] = &*NewArgI;
322   }
323 
324   return NewF;
325 }
326 
cloneGlobalVariableDecl(Module & Dst,const GlobalVariable & GV,ValueToValueMapTy * VMap)327 GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
328                                         ValueToValueMapTy *VMap) {
329   GlobalVariable *NewGV = new GlobalVariable(
330       Dst, GV.getValueType(), GV.isConstant(),
331       GV.getLinkage(), nullptr, GV.getName(), nullptr,
332       GV.getThreadLocalMode(), GV.getType()->getAddressSpace());
333   NewGV->copyAttributesFrom(&GV);
334   if (VMap)
335     (*VMap)[&GV] = NewGV;
336   return NewGV;
337 }
338 
cloneGlobalAliasDecl(Module & Dst,const GlobalAlias & OrigA,ValueToValueMapTy & VMap)339 GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,
340                                   ValueToValueMapTy &VMap) {
341   assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?");
342   auto *NewA = GlobalAlias::create(OrigA.getValueType(),
343                                    OrigA.getType()->getPointerAddressSpace(),
344                                    OrigA.getLinkage(), OrigA.getName(), &Dst);
345   NewA->copyAttributesFrom(&OrigA);
346   VMap[&OrigA] = NewA;
347   return NewA;
348 }
349 
addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol & Sym,jitlink::LinkGraph & G,MCDisassembler & Disassembler,MCInstrAnalysis & MIA)350 Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym,
351                                                    jitlink::LinkGraph &G,
352                                                    MCDisassembler &Disassembler,
353                                                    MCInstrAnalysis &MIA) {
354   // AArch64 appears to already come with the necessary relocations. Among other
355   // architectures, only x86_64 is currently implemented here.
356   if (G.getTargetTriple().getArch() != Triple::x86_64)
357     return Error::success();
358 
359   raw_null_ostream CommentStream;
360   auto &STI = Disassembler.getSubtargetInfo();
361 
362   // Determine the function bounds
363   auto &B = Sym.getBlock();
364   assert(!B.isZeroFill() && "expected content block");
365   auto SymAddress = Sym.getAddress();
366   auto SymStartInBlock =
367       (const uint8_t *)B.getContent().data() + Sym.getOffset();
368   auto SymSize = Sym.getSize() ? Sym.getSize() : B.getSize() - Sym.getOffset();
369   auto Content = ArrayRef(SymStartInBlock, SymSize);
370 
371   LLVM_DEBUG(dbgs() << "Adding self-relocations to " << Sym.getName() << "\n");
372 
373   SmallDenseSet<uintptr_t, 8> ExistingRelocations;
374   for (auto &E : B.edges()) {
375     if (E.isRelocation())
376       ExistingRelocations.insert(E.getOffset());
377   }
378 
379   size_t I = 0;
380   while (I < Content.size()) {
381     MCInst Instr;
382     uint64_t InstrSize = 0;
383     uint64_t InstrStart = SymAddress.getValue() + I;
384     auto DecodeStatus = Disassembler.getInstruction(
385         Instr, InstrSize, Content.drop_front(I), InstrStart, CommentStream);
386     if (DecodeStatus != MCDisassembler::Success) {
387       LLVM_DEBUG(dbgs() << "Aborting due to disassembly failure at address "
388                         << InstrStart);
389       return make_error<StringError>(
390           formatv("failed to disassemble at address {0:x16}", InstrStart),
391           inconvertibleErrorCode());
392     }
393     // Advance to the next instruction.
394     I += InstrSize;
395 
396     // Check for a PC-relative address equal to the symbol itself.
397     auto PCRelAddr =
398         MIA.evaluateMemoryOperandAddress(Instr, &STI, InstrStart, InstrSize);
399     if (!PCRelAddr || *PCRelAddr != SymAddress.getValue())
400       continue;
401 
402     auto RelocOffInInstr =
403         MIA.getMemoryOperandRelocationOffset(Instr, InstrSize);
404     if (!RelocOffInInstr || InstrSize - *RelocOffInInstr != 4) {
405       LLVM_DEBUG(dbgs() << "Skipping unknown self-relocation at "
406                         << InstrStart);
407       continue;
408     }
409 
410     auto RelocOffInBlock = orc::ExecutorAddr(InstrStart) + *RelocOffInInstr -
411                            SymAddress + Sym.getOffset();
412     if (ExistingRelocations.contains(RelocOffInBlock))
413       continue;
414 
415     LLVM_DEBUG(dbgs() << "Adding delta32 self-relocation at " << InstrStart);
416     B.addEdge(jitlink::x86_64::Delta32, RelocOffInBlock, Sym, /*Addend=*/-4);
417   }
418   return Error::success();
419 }
420 
421 } // End namespace orc.
422 } // End namespace llvm.
423