1 //===------ MachOPlatform.cpp - Utilities for executing MachO in Orc ------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
10 
11 #include "llvm/BinaryFormat/MachO.h"
12 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
13 #include "llvm/ExecutionEngine/Orc/DebugUtils.h"
14 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
15 #include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h"
16 #include "llvm/Support/BinaryByteStream.h"
17 #include "llvm/Support/Debug.h"
18 
19 #define DEBUG_TYPE "orc"
20 
21 using namespace llvm;
22 using namespace llvm::orc;
23 using namespace llvm::orc::shared;
24 
25 namespace llvm {
26 namespace orc {
27 namespace shared {
28 
29 using SPSMachOJITDylibDepInfo = SPSTuple<bool, SPSSequence<SPSExecutorAddr>>;
30 using SPSMachOJITDylibDepInfoMap =
31     SPSSequence<SPSTuple<SPSExecutorAddr, SPSMachOJITDylibDepInfo>>;
32 
33 template <>
34 class SPSSerializationTraits<SPSMachOJITDylibDepInfo,
35                              MachOPlatform::MachOJITDylibDepInfo> {
36 public:
37   static size_t size(const MachOPlatform::MachOJITDylibDepInfo &DDI) {
38     return SPSMachOJITDylibDepInfo::AsArgList::size(DDI.Sealed, DDI.DepHeaders);
39   }
40 
41   static bool serialize(SPSOutputBuffer &OB,
42                         const MachOPlatform::MachOJITDylibDepInfo &DDI) {
43     return SPSMachOJITDylibDepInfo::AsArgList::serialize(OB, DDI.Sealed,
44                                                          DDI.DepHeaders);
45   }
46 
47   static bool deserialize(SPSInputBuffer &IB,
48                           MachOPlatform::MachOJITDylibDepInfo &DDI) {
49     return SPSMachOJITDylibDepInfo::AsArgList::deserialize(IB, DDI.Sealed,
50                                                            DDI.DepHeaders);
51   }
52 };
53 
54 } // namespace shared
55 } // namespace orc
56 } // namespace llvm
57 
58 namespace {
59 
60 class MachOHeaderMaterializationUnit : public MaterializationUnit {
61 public:
62   MachOHeaderMaterializationUnit(MachOPlatform &MOP,
63                                  const SymbolStringPtr &HeaderStartSymbol)
64       : MaterializationUnit(createHeaderInterface(MOP, HeaderStartSymbol)),
65         MOP(MOP) {}
66 
67   StringRef getName() const override { return "MachOHeaderMU"; }
68 
69   void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
70     unsigned PointerSize;
71     support::endianness Endianness;
72     const auto &TT =
73         MOP.getExecutionSession().getExecutorProcessControl().getTargetTriple();
74 
75     switch (TT.getArch()) {
76     case Triple::aarch64:
77     case Triple::x86_64:
78       PointerSize = 8;
79       Endianness = support::endianness::little;
80       break;
81     default:
82       llvm_unreachable("Unrecognized architecture");
83     }
84 
85     auto G = std::make_unique<jitlink::LinkGraph>(
86         "<MachOHeaderMU>", TT, PointerSize, Endianness,
87         jitlink::getGenericEdgeKindName);
88     auto &HeaderSection = G->createSection("__header", jitlink::MemProt::Read);
89     auto &HeaderBlock = createHeaderBlock(*G, HeaderSection);
90 
91     // Init symbol is header-start symbol.
92     G->addDefinedSymbol(HeaderBlock, 0, *R->getInitializerSymbol(),
93                         HeaderBlock.getSize(), jitlink::Linkage::Strong,
94                         jitlink::Scope::Default, false, true);
95     for (auto &HS : AdditionalHeaderSymbols)
96       G->addDefinedSymbol(HeaderBlock, HS.Offset, HS.Name,
97                           HeaderBlock.getSize(), jitlink::Linkage::Strong,
98                           jitlink::Scope::Default, false, true);
99 
100     MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
101   }
102 
103   void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
104 
105 private:
106   struct HeaderSymbol {
107     const char *Name;
108     uint64_t Offset;
109   };
110 
111   static constexpr HeaderSymbol AdditionalHeaderSymbols[] = {
112       {"___mh_executable_header", 0}};
113 
114   static jitlink::Block &createHeaderBlock(jitlink::LinkGraph &G,
115                                            jitlink::Section &HeaderSection) {
116     MachO::mach_header_64 Hdr;
117     Hdr.magic = MachO::MH_MAGIC_64;
118     switch (G.getTargetTriple().getArch()) {
119     case Triple::aarch64:
120       Hdr.cputype = MachO::CPU_TYPE_ARM64;
121       Hdr.cpusubtype = MachO::CPU_SUBTYPE_ARM64_ALL;
122       break;
123     case Triple::x86_64:
124       Hdr.cputype = MachO::CPU_TYPE_X86_64;
125       Hdr.cpusubtype = MachO::CPU_SUBTYPE_X86_64_ALL;
126       break;
127     default:
128       llvm_unreachable("Unrecognized architecture");
129     }
130     Hdr.filetype = MachO::MH_DYLIB; // Custom file type?
131     Hdr.ncmds = 0;
132     Hdr.sizeofcmds = 0;
133     Hdr.flags = 0;
134     Hdr.reserved = 0;
135 
136     if (G.getEndianness() != support::endian::system_endianness())
137       MachO::swapStruct(Hdr);
138 
139     auto HeaderContent = G.allocateString(
140         StringRef(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr)));
141 
142     return G.createContentBlock(HeaderSection, HeaderContent, ExecutorAddr(), 8,
143                                 0);
144   }
145 
146   static MaterializationUnit::Interface
147   createHeaderInterface(MachOPlatform &MOP,
148                         const SymbolStringPtr &HeaderStartSymbol) {
149     SymbolFlagsMap HeaderSymbolFlags;
150 
151     HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported;
152     for (auto &HS : AdditionalHeaderSymbols)
153       HeaderSymbolFlags[MOP.getExecutionSession().intern(HS.Name)] =
154           JITSymbolFlags::Exported;
155 
156     return MaterializationUnit::Interface(std::move(HeaderSymbolFlags),
157                                           HeaderStartSymbol);
158   }
159 
160   MachOPlatform &MOP;
161 };
162 
163 constexpr MachOHeaderMaterializationUnit::HeaderSymbol
164     MachOHeaderMaterializationUnit::AdditionalHeaderSymbols[];
165 
166 StringRef EHFrameSectionName = "__TEXT,__eh_frame";
167 StringRef ModInitFuncSectionName = "__DATA,__mod_init_func";
168 StringRef ObjCClassListSectionName = "__DATA,__objc_classlist";
169 StringRef ObjCImageInfoSectionName = "__DATA,__objc_image_info";
170 StringRef ObjCSelRefsSectionName = "__DATA,__objc_selrefs";
171 StringRef Swift5ProtoSectionName = "__TEXT,__swift5_proto";
172 StringRef Swift5ProtosSectionName = "__TEXT,__swift5_protos";
173 StringRef Swift5TypesSectionName = "__TEXT,__swift5_types";
174 StringRef ThreadBSSSectionName = "__DATA,__thread_bss";
175 StringRef ThreadDataSectionName = "__DATA,__thread_data";
176 StringRef ThreadVarsSectionName = "__DATA,__thread_vars";
177 
178 StringRef InitSectionNames[] = {
179     ModInitFuncSectionName, ObjCSelRefsSectionName, ObjCClassListSectionName,
180     Swift5ProtosSectionName, Swift5ProtoSectionName, Swift5TypesSectionName};
181 
182 } // end anonymous namespace
183 
184 namespace llvm {
185 namespace orc {
186 
187 Expected<std::unique_ptr<MachOPlatform>>
188 MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
189                       JITDylib &PlatformJD, const char *OrcRuntimePath,
190                       Optional<SymbolAliasMap> RuntimeAliases) {
191 
192   auto &EPC = ES.getExecutorProcessControl();
193 
194   // If the target is not supported then bail out immediately.
195   if (!supportedTarget(EPC.getTargetTriple()))
196     return make_error<StringError>("Unsupported MachOPlatform triple: " +
197                                        EPC.getTargetTriple().str(),
198                                    inconvertibleErrorCode());
199 
200   // Create default aliases if the caller didn't supply any.
201   if (!RuntimeAliases)
202     RuntimeAliases = standardPlatformAliases(ES);
203 
204   // Define the aliases.
205   if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases))))
206     return std::move(Err);
207 
208   // Add JIT-dispatch function support symbols.
209   if (auto Err = PlatformJD.define(absoluteSymbols(
210           {{ES.intern("___orc_rt_jit_dispatch"),
211             {EPC.getJITDispatchInfo().JITDispatchFunction.getValue(),
212              JITSymbolFlags::Exported}},
213            {ES.intern("___orc_rt_jit_dispatch_ctx"),
214             {EPC.getJITDispatchInfo().JITDispatchContext.getValue(),
215              JITSymbolFlags::Exported}}})))
216     return std::move(Err);
217 
218   // Create a generator for the ORC runtime archive.
219   auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Load(
220       ObjLinkingLayer, OrcRuntimePath, EPC.getTargetTriple());
221   if (!OrcRuntimeArchiveGenerator)
222     return OrcRuntimeArchiveGenerator.takeError();
223 
224   // Create the instance.
225   Error Err = Error::success();
226   auto P = std::unique_ptr<MachOPlatform>(
227       new MachOPlatform(ES, ObjLinkingLayer, PlatformJD,
228                         std::move(*OrcRuntimeArchiveGenerator), Err));
229   if (Err)
230     return std::move(Err);
231   return std::move(P);
232 }
233 
234 Error MachOPlatform::setupJITDylib(JITDylib &JD) {
235   if (auto Err = JD.define(std::make_unique<MachOHeaderMaterializationUnit>(
236           *this, MachOHeaderStartSymbol)))
237     return Err;
238 
239   return ES.lookup({&JD}, MachOHeaderStartSymbol).takeError();
240 }
241 
242 Error MachOPlatform::teardownJITDylib(JITDylib &JD) {
243   std::lock_guard<std::mutex> Lock(PlatformMutex);
244   auto I = JITDylibToHeaderAddr.find(&JD);
245   if (I != JITDylibToHeaderAddr.end()) {
246     assert(HeaderAddrToJITDylib.count(I->second) &&
247            "HeaderAddrToJITDylib missing entry");
248     HeaderAddrToJITDylib.erase(I->second);
249     JITDylibToHeaderAddr.erase(I);
250   }
251   JITDylibToPThreadKey.erase(&JD);
252   return Error::success();
253 }
254 
255 Error MachOPlatform::notifyAdding(ResourceTracker &RT,
256                                   const MaterializationUnit &MU) {
257   auto &JD = RT.getJITDylib();
258   const auto &InitSym = MU.getInitializerSymbol();
259   if (!InitSym)
260     return Error::success();
261 
262   RegisteredInitSymbols[&JD].add(InitSym,
263                                  SymbolLookupFlags::WeaklyReferencedSymbol);
264   LLVM_DEBUG({
265     dbgs() << "MachOPlatform: Registered init symbol " << *InitSym << " for MU "
266            << MU.getName() << "\n";
267   });
268   return Error::success();
269 }
270 
271 Error MachOPlatform::notifyRemoving(ResourceTracker &RT) {
272   llvm_unreachable("Not supported yet");
273 }
274 
275 static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
276                        ArrayRef<std::pair<const char *, const char *>> AL) {
277   for (auto &KV : AL) {
278     auto AliasName = ES.intern(KV.first);
279     assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
280     Aliases[std::move(AliasName)] = {ES.intern(KV.second),
281                                      JITSymbolFlags::Exported};
282   }
283 }
284 
285 SymbolAliasMap MachOPlatform::standardPlatformAliases(ExecutionSession &ES) {
286   SymbolAliasMap Aliases;
287   addAliases(ES, Aliases, requiredCXXAliases());
288   addAliases(ES, Aliases, standardRuntimeUtilityAliases());
289   return Aliases;
290 }
291 
292 ArrayRef<std::pair<const char *, const char *>>
293 MachOPlatform::requiredCXXAliases() {
294   static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
295       {"___cxa_atexit", "___orc_rt_macho_cxa_atexit"}};
296 
297   return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
298 }
299 
300 ArrayRef<std::pair<const char *, const char *>>
301 MachOPlatform::standardRuntimeUtilityAliases() {
302   static const std::pair<const char *, const char *>
303       StandardRuntimeUtilityAliases[] = {
304           {"___orc_rt_run_program", "___orc_rt_macho_run_program"},
305           {"___orc_rt_log_error", "___orc_rt_log_error_to_stderr"}};
306 
307   return ArrayRef<std::pair<const char *, const char *>>(
308       StandardRuntimeUtilityAliases);
309 }
310 
311 bool MachOPlatform::isInitializerSection(StringRef SegName,
312                                          StringRef SectName) {
313   for (auto &Name : InitSectionNames) {
314     if (Name.startswith(SegName) && Name.substr(7) == SectName)
315       return true;
316   }
317   return false;
318 }
319 
320 bool MachOPlatform::supportedTarget(const Triple &TT) {
321   switch (TT.getArch()) {
322   case Triple::aarch64:
323   case Triple::x86_64:
324     return true;
325   default:
326     return false;
327   }
328 }
329 
330 MachOPlatform::MachOPlatform(
331     ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
332     JITDylib &PlatformJD,
333     std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err)
334     : ES(ES), ObjLinkingLayer(ObjLinkingLayer),
335       MachOHeaderStartSymbol(ES.intern("___dso_handle")) {
336   ErrorAsOutParameter _(&Err);
337 
338   ObjLinkingLayer.addPlugin(std::make_unique<MachOPlatformPlugin>(*this));
339 
340   PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));
341 
342   // Force linking of eh-frame registration functions.
343   if (auto Err2 = lookupAndRecordAddrs(
344           ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD),
345           {{ES.intern("___orc_rt_macho_register_ehframe_section"),
346             &orc_rt_macho_register_ehframe_section},
347            {ES.intern("___orc_rt_macho_deregister_ehframe_section"),
348             &orc_rt_macho_deregister_ehframe_section}})) {
349     Err = std::move(Err2);
350     return;
351   }
352 
353   State = BootstrapPhase2;
354 
355   // Associate wrapper function tags with JIT-side function implementations.
356   if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
357     Err = std::move(E2);
358     return;
359   }
360 
361   // Lookup addresses of runtime functions callable by the platform,
362   // call the platform bootstrap function to initialize the platform-state
363   // object in the executor.
364   if (auto E2 = bootstrapMachORuntime(PlatformJD)) {
365     Err = std::move(E2);
366     return;
367   }
368 
369   // PlatformJD hasn't been set up by the platform yet (since we're creating
370   // the platform now), so set it up.
371   if (auto E2 = setupJITDylib(PlatformJD)) {
372     Err = std::move(E2);
373     return;
374   }
375 
376   State = Initialized;
377 }
378 
379 Error MachOPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
380   ExecutionSession::JITDispatchHandlerAssociationMap WFs;
381 
382   using PushInitializersSPSSig =
383       SPSExpected<SPSMachOJITDylibDepInfoMap>(SPSExecutorAddr);
384   WFs[ES.intern("___orc_rt_macho_push_initializers_tag")] =
385       ES.wrapAsyncWithSPS<PushInitializersSPSSig>(
386           this, &MachOPlatform::rt_pushInitializers);
387 
388   using LookupSymbolSPSSig =
389       SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
390   WFs[ES.intern("___orc_rt_macho_symbol_lookup_tag")] =
391       ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this,
392                                               &MachOPlatform::rt_lookupSymbol);
393 
394   return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
395 }
396 
397 void MachOPlatform::pushInitializersLoop(
398     PushInitializersSendResultFn SendResult, JITDylibSP JD) {
399   DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
400   DenseMap<JITDylib *, SmallVector<JITDylib *>> JDDepMap;
401   SmallVector<JITDylib *, 16> Worklist({JD.get()});
402 
403   ES.runSessionLocked([&]() {
404     while (!Worklist.empty()) {
405       // FIXME: Check for defunct dylibs.
406 
407       auto DepJD = Worklist.back();
408       Worklist.pop_back();
409 
410       // If we've already visited this JITDylib on this iteration then continue.
411       if (JDDepMap.count(DepJD))
412         continue;
413 
414       // Add dep info.
415       auto &DM = JDDepMap[DepJD];
416       DepJD->withLinkOrderDo([&](const JITDylibSearchOrder &O) {
417         for (auto &KV : O) {
418           if (KV.first == DepJD)
419             continue;
420           DM.push_back(KV.first);
421           Worklist.push_back(KV.first);
422         }
423       });
424 
425       // Add any registered init symbols.
426       auto RISItr = RegisteredInitSymbols.find(DepJD);
427       if (RISItr != RegisteredInitSymbols.end()) {
428         NewInitSymbols[DepJD] = std::move(RISItr->second);
429         RegisteredInitSymbols.erase(RISItr);
430       }
431     }
432   });
433 
434   // If there are no further init symbols to look up then send the link order
435   // (as a list of header addresses) to the caller.
436   if (NewInitSymbols.empty()) {
437 
438     // To make the list intelligible to the runtime we need to convert all
439     // JITDylib pointers to their header addresses.
440     DenseMap<JITDylib *, ExecutorAddr> HeaderAddrs;
441     HeaderAddrs.reserve(JDDepMap.size());
442     {
443       std::lock_guard<std::mutex> Lock(PlatformMutex);
444       for (auto &KV : JDDepMap) {
445         auto I = JITDylibToHeaderAddr.find(KV.first);
446         if (I == JITDylibToHeaderAddr.end()) {
447           // The header address should have been materialized by the previous
448           // round, but we need to handle the pathalogical case where someone
449           // removes the symbol on another thread while we're running.
450           SendResult(
451               make_error<StringError>("JITDylib " + KV.first->getName() +
452                                           " has no registered header address",
453                                       inconvertibleErrorCode()));
454           return;
455         }
456         HeaderAddrs[KV.first] = I->second;
457       }
458     }
459 
460     // Build the dep info map to return.
461     MachOJITDylibDepInfoMap DIM;
462     DIM.reserve(JDDepMap.size());
463     for (auto &KV : JDDepMap) {
464       assert(HeaderAddrs.count(KV.first) && "Missing header addr");
465       auto H = HeaderAddrs[KV.first];
466       MachOJITDylibDepInfo DepInfo;
467       for (auto &Dep : KV.second) {
468         assert(HeaderAddrs.count(Dep) && "Missing header addr");
469         DepInfo.DepHeaders.push_back(HeaderAddrs[Dep]);
470       }
471       DIM.push_back(std::make_pair(H, std::move(DepInfo)));
472     }
473     SendResult(DIM);
474     return;
475   }
476 
477   // Otherwise issue a lookup and re-run this phase when it completes.
478   lookupInitSymbolsAsync(
479       [this, SendResult = std::move(SendResult), &JD](Error Err) mutable {
480         if (Err)
481           SendResult(std::move(Err));
482         else
483           pushInitializersLoop(std::move(SendResult), JD);
484       },
485       ES, std::move(NewInitSymbols));
486 }
487 
488 void MachOPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult,
489                                         ExecutorAddr JDHeaderAddr) {
490   JITDylibSP JD;
491   {
492     std::lock_guard<std::mutex> Lock(PlatformMutex);
493     auto I = HeaderAddrToJITDylib.find(JDHeaderAddr);
494     if (I != HeaderAddrToJITDylib.end())
495       JD = I->second;
496   }
497 
498   LLVM_DEBUG({
499     dbgs() << "MachOPlatform::rt_pushInitializers(" << JDHeaderAddr << ") ";
500     if (JD)
501       dbgs() << "pushing initializers for " << JD->getName() << "\n";
502     else
503       dbgs() << "No JITDylib for header address.\n";
504   });
505 
506   if (!JD) {
507     SendResult(
508         make_error<StringError>("No JITDylib with header addr " +
509                                     formatv("{0:x}", JDHeaderAddr.getValue()),
510                                 inconvertibleErrorCode()));
511     return;
512   }
513 
514   pushInitializersLoop(std::move(SendResult), JD);
515 }
516 
517 void MachOPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
518                                     ExecutorAddr Handle, StringRef SymbolName) {
519   LLVM_DEBUG({
520     dbgs() << "MachOPlatform::rt_lookupSymbol(\""
521            << formatv("{0:x}", Handle.getValue()) << "\")\n";
522   });
523 
524   JITDylib *JD = nullptr;
525 
526   {
527     std::lock_guard<std::mutex> Lock(PlatformMutex);
528     auto I = HeaderAddrToJITDylib.find(Handle);
529     if (I != HeaderAddrToJITDylib.end())
530       JD = I->second;
531   }
532 
533   if (!JD) {
534     LLVM_DEBUG({
535       dbgs() << "  No JITDylib for handle "
536              << formatv("{0:x}", Handle.getValue()) << "\n";
537     });
538     SendResult(make_error<StringError>("No JITDylib associated with handle " +
539                                            formatv("{0:x}", Handle.getValue()),
540                                        inconvertibleErrorCode()));
541     return;
542   }
543 
544   // Use functor class to work around XL build compiler issue on AIX.
545   class RtLookupNotifyComplete {
546   public:
547     RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
548         : SendResult(std::move(SendResult)) {}
549     void operator()(Expected<SymbolMap> Result) {
550       if (Result) {
551         assert(Result->size() == 1 && "Unexpected result map count");
552         SendResult(ExecutorAddr(Result->begin()->second.getAddress()));
553       } else {
554         SendResult(Result.takeError());
555       }
556     }
557 
558   private:
559     SendSymbolAddressFn SendResult;
560   };
561 
562   // FIXME: Proper mangling.
563   auto MangledName = ("_" + SymbolName).str();
564   ES.lookup(
565       LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
566       SymbolLookupSet(ES.intern(MangledName)), SymbolState::Ready,
567       RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister);
568 }
569 
570 Error MachOPlatform::bootstrapMachORuntime(JITDylib &PlatformJD) {
571   if (auto Err = lookupAndRecordAddrs(
572           ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD),
573           {{ES.intern("___orc_rt_macho_platform_bootstrap"),
574             &orc_rt_macho_platform_bootstrap},
575            {ES.intern("___orc_rt_macho_platform_shutdown"),
576             &orc_rt_macho_platform_shutdown},
577            {ES.intern("___orc_rt_macho_register_jitdylib"),
578             &orc_rt_macho_register_jitdylib},
579            {ES.intern("___orc_rt_macho_deregister_jitdylib"),
580             &orc_rt_macho_deregister_jitdylib},
581            {ES.intern("___orc_rt_macho_register_object_platform_sections"),
582             &orc_rt_macho_register_object_platform_sections},
583            {ES.intern("___orc_rt_macho_deregister_object_platform_sections"),
584             &orc_rt_macho_deregister_object_platform_sections},
585            {ES.intern("___orc_rt_macho_create_pthread_key"),
586             &orc_rt_macho_create_pthread_key}}))
587     return Err;
588 
589   return ES.callSPSWrapper<void()>(orc_rt_macho_platform_bootstrap);
590 }
591 
592 Expected<uint64_t> MachOPlatform::createPThreadKey() {
593   if (!orc_rt_macho_create_pthread_key)
594     return make_error<StringError>(
595         "Attempting to create pthread key in target, but runtime support has "
596         "not been loaded yet",
597         inconvertibleErrorCode());
598 
599   Expected<uint64_t> Result(0);
600   if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>(
601           orc_rt_macho_create_pthread_key, Result))
602     return std::move(Err);
603   return Result;
604 }
605 
606 void MachOPlatform::MachOPlatformPlugin::modifyPassConfig(
607     MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
608     jitlink::PassConfiguration &Config) {
609 
610   auto PS = MP.State.load();
611 
612   // --- Handle Initializers ---
613   if (auto InitSymbol = MR.getInitializerSymbol()) {
614 
615     // If the initializer symbol is the MachOHeader start symbol then just
616     // register it and then bail out -- the header materialization unit
617     // definitely doesn't need any other passes.
618     if (InitSymbol == MP.MachOHeaderStartSymbol) {
619       Config.PostAllocationPasses.push_back([this, &MR](jitlink::LinkGraph &G) {
620         return associateJITDylibHeaderSymbol(G, MR);
621       });
622       return;
623     }
624 
625     // If the object contains an init symbol other than the header start symbol
626     // then add passes to preserve, process and register the init
627     // sections/symbols.
628     Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) {
629       if (auto Err = preserveInitSections(G, MR))
630         return Err;
631       return processObjCImageInfo(G, MR);
632     });
633   }
634 
635   // --- Add passes for eh-frame and TLV support ---
636   if (PS == MachOPlatform::BootstrapPhase1) {
637     Config.PostFixupPasses.push_back(
638         [this](jitlink::LinkGraph &G) { return registerEHSectionsPhase1(G); });
639     return;
640   }
641 
642   // Insert TLV lowering at the start of the PostPrunePasses, since we want
643   // it to run before GOT/PLT lowering.
644   Config.PostPrunePasses.insert(
645       Config.PostPrunePasses.begin(),
646       [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
647         return fixTLVSectionsAndEdges(G, JD);
648       });
649 
650   // Add a pass to register the final addresses of any special sections in the
651   // object with the runtime.
652   Config.PostAllocationPasses.push_back(
653       [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
654         return registerObjectPlatformSections(G, JD);
655       });
656 }
657 
658 ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap
659 MachOPlatform::MachOPlatformPlugin::getSyntheticSymbolDependencies(
660     MaterializationResponsibility &MR) {
661   std::lock_guard<std::mutex> Lock(PluginMutex);
662   auto I = InitSymbolDeps.find(&MR);
663   if (I != InitSymbolDeps.end()) {
664     SyntheticSymbolDependenciesMap Result;
665     Result[MR.getInitializerSymbol()] = std::move(I->second);
666     InitSymbolDeps.erase(&MR);
667     return Result;
668   }
669   return SyntheticSymbolDependenciesMap();
670 }
671 
672 Error MachOPlatform::MachOPlatformPlugin::associateJITDylibHeaderSymbol(
673     jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
674   auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {
675     return Sym->getName() == *MP.MachOHeaderStartSymbol;
676   });
677   assert(I != G.defined_symbols().end() && "Missing MachO header start symbol");
678 
679   auto &JD = MR.getTargetJITDylib();
680   std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
681   auto HeaderAddr = (*I)->getAddress();
682   MP.JITDylibToHeaderAddr[&JD] = HeaderAddr;
683   MP.HeaderAddrToJITDylib[HeaderAddr] = &JD;
684   G.allocActions().push_back(
685       {cantFail(
686            WrapperFunctionCall::Create<SPSArgList<SPSString, SPSExecutorAddr>>(
687                MP.orc_rt_macho_register_jitdylib, JD.getName(), HeaderAddr)),
688        cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
689            MP.orc_rt_macho_deregister_jitdylib, HeaderAddr))});
690   return Error::success();
691 }
692 
693 Error MachOPlatform::MachOPlatformPlugin::preserveInitSections(
694     jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
695 
696   JITLinkSymbolSet InitSectionSymbols;
697   for (auto &InitSectionName : InitSectionNames) {
698     // Skip non-init sections.
699     auto *InitSection = G.findSectionByName(InitSectionName);
700     if (!InitSection)
701       continue;
702 
703     // Make a pass over live symbols in the section: those blocks are already
704     // preserved.
705     DenseSet<jitlink::Block *> AlreadyLiveBlocks;
706     for (auto &Sym : InitSection->symbols()) {
707       auto &B = Sym->getBlock();
708       if (Sym->isLive() && Sym->getOffset() == 0 &&
709           Sym->getSize() == B.getSize() && !AlreadyLiveBlocks.count(&B)) {
710         InitSectionSymbols.insert(Sym);
711         AlreadyLiveBlocks.insert(&B);
712       }
713     }
714 
715     // Add anonymous symbols to preserve any not-already-preserved blocks.
716     for (auto *B : InitSection->blocks())
717       if (!AlreadyLiveBlocks.count(B))
718         InitSectionSymbols.insert(
719             &G.addAnonymousSymbol(*B, 0, B->getSize(), false, true));
720   }
721 
722   if (!InitSectionSymbols.empty()) {
723     std::lock_guard<std::mutex> Lock(PluginMutex);
724     InitSymbolDeps[&MR] = std::move(InitSectionSymbols);
725   }
726 
727   return Error::success();
728 }
729 
730 Error MachOPlatform::MachOPlatformPlugin::processObjCImageInfo(
731     jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
732 
733   // If there's an ObjC imagine info then either
734   //   (1) It's the first __objc_imageinfo we've seen in this JITDylib. In
735   //       this case we name and record it.
736   // OR
737   //   (2) We already have a recorded __objc_imageinfo for this JITDylib,
738   //       in which case we just verify it.
739   auto *ObjCImageInfo = G.findSectionByName(ObjCImageInfoSectionName);
740   if (!ObjCImageInfo)
741     return Error::success();
742 
743   auto ObjCImageInfoBlocks = ObjCImageInfo->blocks();
744 
745   // Check that the section is not empty if present.
746   if (llvm::empty(ObjCImageInfoBlocks))
747     return make_error<StringError>("Empty " + ObjCImageInfoSectionName +
748                                        " section in " + G.getName(),
749                                    inconvertibleErrorCode());
750 
751   // Check that there's only one block in the section.
752   if (std::next(ObjCImageInfoBlocks.begin()) != ObjCImageInfoBlocks.end())
753     return make_error<StringError>("Multiple blocks in " +
754                                        ObjCImageInfoSectionName +
755                                        " section in " + G.getName(),
756                                    inconvertibleErrorCode());
757 
758   // Check that the __objc_imageinfo section is unreferenced.
759   // FIXME: We could optimize this check if Symbols had a ref-count.
760   for (auto &Sec : G.sections()) {
761     if (&Sec != ObjCImageInfo)
762       for (auto *B : Sec.blocks())
763         for (auto &E : B->edges())
764           if (E.getTarget().isDefined() &&
765               &E.getTarget().getBlock().getSection() == ObjCImageInfo)
766             return make_error<StringError>(ObjCImageInfoSectionName +
767                                                " is referenced within file " +
768                                                G.getName(),
769                                            inconvertibleErrorCode());
770   }
771 
772   auto &ObjCImageInfoBlock = **ObjCImageInfoBlocks.begin();
773   auto *ObjCImageInfoData = ObjCImageInfoBlock.getContent().data();
774   auto Version = support::endian::read32(ObjCImageInfoData, G.getEndianness());
775   auto Flags =
776       support::endian::read32(ObjCImageInfoData + 4, G.getEndianness());
777 
778   // Lock the mutex while we verify / update the ObjCImageInfos map.
779   std::lock_guard<std::mutex> Lock(PluginMutex);
780 
781   auto ObjCImageInfoItr = ObjCImageInfos.find(&MR.getTargetJITDylib());
782   if (ObjCImageInfoItr != ObjCImageInfos.end()) {
783     // We've already registered an __objc_imageinfo section. Verify the
784     // content of this new section matches, then delete it.
785     if (ObjCImageInfoItr->second.first != Version)
786       return make_error<StringError>(
787           "ObjC version in " + G.getName() +
788               " does not match first registered version",
789           inconvertibleErrorCode());
790     if (ObjCImageInfoItr->second.second != Flags)
791       return make_error<StringError>("ObjC flags in " + G.getName() +
792                                          " do not match first registered flags",
793                                      inconvertibleErrorCode());
794 
795     // __objc_imageinfo is valid. Delete the block.
796     for (auto *S : ObjCImageInfo->symbols())
797       G.removeDefinedSymbol(*S);
798     G.removeBlock(ObjCImageInfoBlock);
799   } else {
800     // We haven't registered an __objc_imageinfo section yet. Register and
801     // move on. The section should already be marked no-dead-strip.
802     ObjCImageInfos[&MR.getTargetJITDylib()] = std::make_pair(Version, Flags);
803   }
804 
805   return Error::success();
806 }
807 
808 Error MachOPlatform::MachOPlatformPlugin::fixTLVSectionsAndEdges(
809     jitlink::LinkGraph &G, JITDylib &JD) {
810 
811   // Rename external references to __tlv_bootstrap to ___orc_rt_tlv_get_addr.
812   for (auto *Sym : G.external_symbols())
813     if (Sym->getName() == "__tlv_bootstrap") {
814       Sym->setName("___orc_rt_macho_tlv_get_addr");
815       break;
816     }
817 
818   // Store key in __thread_vars struct fields.
819   if (auto *ThreadDataSec = G.findSectionByName(ThreadVarsSectionName)) {
820     Optional<uint64_t> Key;
821     {
822       std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
823       auto I = MP.JITDylibToPThreadKey.find(&JD);
824       if (I != MP.JITDylibToPThreadKey.end())
825         Key = I->second;
826     }
827 
828     if (!Key) {
829       if (auto KeyOrErr = MP.createPThreadKey())
830         Key = *KeyOrErr;
831       else
832         return KeyOrErr.takeError();
833     }
834 
835     uint64_t PlatformKeyBits =
836         support::endian::byte_swap(*Key, G.getEndianness());
837 
838     for (auto *B : ThreadDataSec->blocks()) {
839       if (B->getSize() != 3 * G.getPointerSize())
840         return make_error<StringError>("__thread_vars block at " +
841                                            formatv("{0:x}", B->getAddress()) +
842                                            " has unexpected size",
843                                        inconvertibleErrorCode());
844 
845       auto NewBlockContent = G.allocateBuffer(B->getSize());
846       llvm::copy(B->getContent(), NewBlockContent.data());
847       memcpy(NewBlockContent.data() + G.getPointerSize(), &PlatformKeyBits,
848              G.getPointerSize());
849       B->setContent(NewBlockContent);
850     }
851   }
852 
853   // Transform any TLV edges into GOT edges.
854   for (auto *B : G.blocks())
855     for (auto &E : B->edges())
856       if (E.getKind() ==
857           jitlink::x86_64::RequestTLVPAndTransformToPCRel32TLVPLoadREXRelaxable)
858         E.setKind(jitlink::x86_64::
859                       RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable);
860 
861   return Error::success();
862 }
863 
864 Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections(
865     jitlink::LinkGraph &G, JITDylib &JD) {
866 
867   // Add an action to register the eh-frame.
868   if (auto *EHFrameSection = G.findSectionByName(EHFrameSectionName)) {
869     jitlink::SectionRange R(*EHFrameSection);
870     if (!R.empty())
871       G.allocActions().push_back(
872           {cantFail(
873                WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>(
874                    MP.orc_rt_macho_register_ehframe_section, R.getRange())),
875            cantFail(
876                WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>(
877                    MP.orc_rt_macho_deregister_ehframe_section, R.getRange()))});
878   }
879 
880   // Get a pointer to the thread data section if there is one. It will be used
881   // below.
882   jitlink::Section *ThreadDataSection =
883       G.findSectionByName(ThreadDataSectionName);
884 
885   // Handle thread BSS section if there is one.
886   if (auto *ThreadBSSSection = G.findSectionByName(ThreadBSSSectionName)) {
887     // If there's already a thread data section in this graph then merge the
888     // thread BSS section content into it, otherwise just treat the thread
889     // BSS section as the thread data section.
890     if (ThreadDataSection)
891       G.mergeSections(*ThreadDataSection, *ThreadBSSSection);
892     else
893       ThreadDataSection = ThreadBSSSection;
894   }
895 
896   SmallVector<std::pair<StringRef, ExecutorAddrRange>, 8> MachOPlatformSecs;
897 
898   // Having merged thread BSS (if present) and thread data (if present),
899   // record the resulting section range.
900   if (ThreadDataSection) {
901     jitlink::SectionRange R(*ThreadDataSection);
902     if (!R.empty()) {
903       if (MP.State != MachOPlatform::Initialized)
904         return make_error<StringError>("__thread_data section encountered, but "
905                                        "MachOPlatform has not finished booting",
906                                        inconvertibleErrorCode());
907 
908       MachOPlatformSecs.push_back({ThreadDataSectionName, R.getRange()});
909     }
910   }
911 
912   // If any platform sections were found then add an allocation action to call
913   // the registration function.
914   StringRef PlatformSections[] = {
915       ModInitFuncSectionName,   ObjCClassListSectionName,
916       ObjCImageInfoSectionName, ObjCSelRefsSectionName,
917       Swift5ProtoSectionName,   Swift5ProtosSectionName,
918       Swift5TypesSectionName,
919   };
920 
921   for (auto &SecName : PlatformSections) {
922     auto *Sec = G.findSectionByName(SecName);
923     if (!Sec)
924       continue;
925     jitlink::SectionRange R(*Sec);
926     if (R.empty())
927       continue;
928 
929     MachOPlatformSecs.push_back({SecName, R.getRange()});
930   }
931 
932   if (!MachOPlatformSecs.empty()) {
933     Optional<ExecutorAddr> HeaderAddr;
934     {
935       std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
936       auto I = MP.JITDylibToHeaderAddr.find(&JD);
937       if (I != MP.JITDylibToHeaderAddr.end())
938         HeaderAddr = I->second;
939     }
940 
941     if (!HeaderAddr)
942       return make_error<StringError>("Missing header for " + JD.getName(),
943                                      inconvertibleErrorCode());
944 
945     // Dump the scraped inits.
946     LLVM_DEBUG({
947       dbgs() << "MachOPlatform: Scraped " << G.getName() << " init sections:\n";
948       for (auto &KV : MachOPlatformSecs)
949         dbgs() << "  " << KV.first << ": " << KV.second << "\n";
950     });
951 
952     using SPSRegisterObjectPlatformSectionsArgs =
953         SPSArgList<SPSExecutorAddr,
954                    SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>>;
955     G.allocActions().push_back(
956         {cantFail(
957              WrapperFunctionCall::Create<SPSRegisterObjectPlatformSectionsArgs>(
958                  MP.orc_rt_macho_register_object_platform_sections, *HeaderAddr,
959                  MachOPlatformSecs)),
960          cantFail(
961              WrapperFunctionCall::Create<SPSRegisterObjectPlatformSectionsArgs>(
962                  MP.orc_rt_macho_deregister_object_platform_sections,
963                  *HeaderAddr, MachOPlatformSecs))});
964   }
965 
966   return Error::success();
967 }
968 
969 Error MachOPlatform::MachOPlatformPlugin::registerEHSectionsPhase1(
970     jitlink::LinkGraph &G) {
971 
972   // If there's no eh-frame there's nothing to do.
973   auto *EHFrameSection = G.findSectionByName(EHFrameSectionName);
974   if (!EHFrameSection)
975     return Error::success();
976 
977   // If the eh-frame section is empty there's nothing to do.
978   jitlink::SectionRange R(*EHFrameSection);
979   if (R.empty())
980     return Error::success();
981 
982   // Since we're linking the object containing the registration code now the
983   // addresses won't be ready in the platform. We'll have to find them in this
984   // graph instead.
985   ExecutorAddr orc_rt_macho_register_ehframe_section;
986   ExecutorAddr orc_rt_macho_deregister_ehframe_section;
987   for (auto *Sym : G.defined_symbols()) {
988     if (!Sym->hasName())
989       continue;
990     if (Sym->getName() == "___orc_rt_macho_register_ehframe_section")
991       orc_rt_macho_register_ehframe_section = ExecutorAddr(Sym->getAddress());
992     else if (Sym->getName() == "___orc_rt_macho_deregister_ehframe_section")
993       orc_rt_macho_deregister_ehframe_section = ExecutorAddr(Sym->getAddress());
994 
995     if (orc_rt_macho_register_ehframe_section &&
996         orc_rt_macho_deregister_ehframe_section)
997       break;
998   }
999 
1000   // If we failed to find the required functions then bail out.
1001   if (!orc_rt_macho_register_ehframe_section ||
1002       !orc_rt_macho_deregister_ehframe_section)
1003     return make_error<StringError>("Could not find eh-frame registration "
1004                                    "functions during platform bootstrap",
1005                                    inconvertibleErrorCode());
1006 
1007   // Otherwise, add allocation actions to the graph to register eh-frames for
1008   // this object.
1009   G.allocActions().push_back(
1010       {cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>(
1011            orc_rt_macho_register_ehframe_section, R.getRange())),
1012        cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>(
1013            orc_rt_macho_deregister_ehframe_section, R.getRange()))});
1014 
1015   return Error::success();
1016 }
1017 
1018 } // End namespace orc.
1019 } // End namespace llvm.
1020