compiler-rt/lib/orc/macho_platform.cpp

   1 //===- macho_platform.cpp -------------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file contains code required to load the rest of the MachO runtime.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "macho_platform.h"
  14 #include "common.h"
  15 #include "debug.h"
  16 #include "error.h"
  17 #include "interval_map.h"
  18 #include "wrapper_function_utils.h"
  19
  20 #include <algorithm>
  21 #include <ios>
  22 #include <map>
  23 #include <mutex>
  24 #include <sstream>
  25 #include <string_view>
  26 #include <unordered_map>
  27 #include <unordered_set>
  28 #include <vector>
  29
  30 #define DEBUG_TYPE "macho_platform"
  31
  32 using namespace __orc_rt;
  33 using namespace __orc_rt::macho;
  34
  35 // Declare function tags for functions in the JIT process.
  36 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_push_initializers_tag)
  37 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_symbol_lookup_tag)
  38
  39 struct objc_image_info;
  40 struct mach_header;
  41
  42 // Objective-C registration functions.
  43 // These are weakly imported. If the Objective-C runtime has not been loaded
  44 // then code containing Objective-C sections will generate an error.
  45 extern "C" void
  46 _objc_map_images(unsigned count, const char *const paths[],
  47                  const mach_header *const mhdrs[]) ORC_RT_WEAK_IMPORT;
  48
  49 extern "C" void _objc_load_image(const char *path,
  50                                  const mach_header *mh) ORC_RT_WEAK_IMPORT;
  51
  52 // Libunwind prototypes.
  53 struct unw_dynamic_unwind_sections {
  54   uintptr_t dso_base;
  55   uintptr_t dwarf_section;
  56   size_t dwarf_section_length;
  57   uintptr_t compact_unwind_section;
  58   size_t compact_unwind_section_length;
  59 };
  60
  61 typedef int (*unw_find_dynamic_unwind_sections)(
  62     uintptr_t addr, struct unw_dynamic_unwind_sections *info);
  63
  64 extern "C" int __unw_add_find_dynamic_unwind_sections(
  65     unw_find_dynamic_unwind_sections find_dynamic_unwind_sections)
  66     ORC_RT_WEAK_IMPORT;
  67
  68 extern "C" int __unw_remove_find_dynamic_unwind_sections(
  69     unw_find_dynamic_unwind_sections find_dynamic_unwind_sections)
  70     ORC_RT_WEAK_IMPORT;
  71
  72 namespace {
  73
  74 struct MachOJITDylibDepInfo {
  75   bool Sealed = false;
  76   std::vector<ExecutorAddr> DepHeaders;
  77 };
  78
  79 using MachOJITDylibDepInfoMap =
  80     std::unordered_map<ExecutorAddr, MachOJITDylibDepInfo>;
  81
  82 } // anonymous namespace
  83
  84 namespace __orc_rt {
  85
  86 using SPSMachOObjectPlatformSectionsMap =
  87     SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>;
  88
  89 using SPSMachOJITDylibDepInfo = SPSTuple<bool, SPSSequence<SPSExecutorAddr>>;
  90
  91 using SPSMachOJITDylibDepInfoMap =
  92     SPSSequence<SPSTuple<SPSExecutorAddr, SPSMachOJITDylibDepInfo>>;
  93
  94 template <>
  95 class SPSSerializationTraits<SPSMachOJITDylibDepInfo, MachOJITDylibDepInfo> {
  96 public:
  97   static size_t size(const MachOJITDylibDepInfo &JDI) {
  98     return SPSMachOJITDylibDepInfo::AsArgList::size(JDI.Sealed, JDI.DepHeaders);
  99   }
 100
 101   static bool serialize(SPSOutputBuffer &OB, const MachOJITDylibDepInfo &JDI) {
 102     return SPSMachOJITDylibDepInfo::AsArgList::serialize(OB, JDI.Sealed,
 103                                                          JDI.DepHeaders);
 104   }
 105
 106   static bool deserialize(SPSInputBuffer &IB, MachOJITDylibDepInfo &JDI) {
 107     return SPSMachOJITDylibDepInfo::AsArgList::deserialize(IB, JDI.Sealed,
 108                                                            JDI.DepHeaders);
 109   }
 110 };
 111
 112 struct UnwindSectionInfo {
 113   std::vector<ExecutorAddrRange> CodeRanges;
 114   ExecutorAddrRange DwarfSection;
 115   ExecutorAddrRange CompactUnwindSection;
 116 };
 117
 118 using SPSUnwindSectionInfo =
 119     SPSTuple<SPSSequence<SPSExecutorAddrRange>, SPSExecutorAddrRange,
 120              SPSExecutorAddrRange>;
 121
 122 template <>
 123 class SPSSerializationTraits<SPSUnwindSectionInfo, UnwindSectionInfo> {
 124 public:
 125   static size_t size(const UnwindSectionInfo &USI) {
 126     return SPSUnwindSectionInfo::AsArgList::size(
 127         USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection);
 128   }
 129
 130   static bool serialize(SPSOutputBuffer &OB, const UnwindSectionInfo &USI) {
 131     return SPSUnwindSectionInfo::AsArgList::serialize(
 132         OB, USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection);
 133   }
 134
 135   static bool deserialize(SPSInputBuffer &IB, UnwindSectionInfo &USI) {
 136     return SPSUnwindSectionInfo::AsArgList::deserialize(
 137         IB, USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection);
 138   }
 139 };
 140
 141 } // namespace __orc_rt
 142
 143 namespace {
 144 struct TLVDescriptor {
 145   void *(*Thunk)(TLVDescriptor *) = nullptr;
 146   unsigned long Key = 0;
 147   unsigned long DataAddress = 0;
 148 };
 149
 150 class MachOPlatformRuntimeState {
 151 private:
 152   struct AtExitEntry {
 153     void (*Func)(void *);
 154     void *Arg;
 155   };
 156
 157   using AtExitsVector = std::vector<AtExitEntry>;
 158
 159   /// Used to manage sections of fixed-sized metadata records (e.g. pointer
 160   /// sections, selector refs, etc.)
 161   template <typename RecordElement> class RecordSectionsTracker {
 162   public:
 163     /// Add a section to the "new" list.
 164     void add(span<RecordElement> Sec) { New.push_back(std::move(Sec)); }
 165
 166     /// Returns true if there are new sections to process.
 167     bool hasNewSections() const { return !New.empty(); }
 168
 169     /// Returns the number of new sections to process.
 170     size_t numNewSections() const { return New.size(); }
 171
 172     /// Process all new sections.
 173     template <typename ProcessSectionFunc>
 174     std::enable_if_t<std::is_void_v<
 175         std::invoke_result_t<ProcessSectionFunc, span<RecordElement>>>>
 176     processNewSections(ProcessSectionFunc &&ProcessSection) {
 177       for (auto &Sec : New)
 178         ProcessSection(Sec);
 179       moveNewToProcessed();
 180     }
 181
 182     /// Proces all new sections with a fallible handler.
 183     ///
 184     /// Successfully handled sections will be moved to the Processed
 185     /// list.
 186     template <typename ProcessSectionFunc>
 187     std::enable_if_t<
 188         std::is_same_v<Error, std::invoke_result_t<ProcessSectionFunc,
 189                                                    span<RecordElement>>>,
 190         Error>
 191     processNewSections(ProcessSectionFunc &&ProcessSection) {
 192       for (size_t I = 0; I != New.size(); ++I) {
 193         if (auto Err = ProcessSection(New[I])) {
 194           for (size_t J = 0; J != I; ++J)
 195             Processed.push_back(New[J]);
 196           New.erase(New.begin(), New.begin() + I);
 197           return Err;
 198         }
 199       }
 200       moveNewToProcessed();
 201       return Error::success();
 202     }
 203
 204     /// Move all sections back to New for reprocessing.
 205     void reset() {
 206       moveNewToProcessed();
 207       New = std::move(Processed);
 208     }
 209
 210     /// Remove the section with the given range.
 211     bool removeIfPresent(ExecutorAddrRange R) {
 212       if (removeIfPresent(New, R))
 213         return true;
 214       return removeIfPresent(Processed, R);
 215     }
 216
 217   private:
 218     void moveNewToProcessed() {
 219       if (Processed.empty())
 220         Processed = std::move(New);
 221       else {
 222         Processed.reserve(Processed.size() + New.size());
 223         std::copy(New.begin(), New.end(), std::back_inserter(Processed));
 224         New.clear();
 225       }
 226     }
 227
 228     bool removeIfPresent(std::vector<span<RecordElement>> &V,
 229                          ExecutorAddrRange R) {
 230       auto RI = std::find_if(
 231           V.rbegin(), V.rend(),
 232           [RS = R.toSpan<RecordElement>()](const span<RecordElement> &E) {
 233             return E.data() == RS.data();
 234           });
 235       if (RI != V.rend()) {
 236         V.erase(std::next(RI).base());
 237         return true;
 238       }
 239       return false;
 240     }
 241
 242     std::vector<span<RecordElement>> Processed;
 243     std::vector<span<RecordElement>> New;
 244   };
 245
 246   struct UnwindSections {
 247     UnwindSections(const UnwindSectionInfo &USI)
 248         : DwarfSection(USI.DwarfSection.toSpan<char>()),
 249           CompactUnwindSection(USI.CompactUnwindSection.toSpan<char>()) {}
 250
 251     span<char> DwarfSection;
 252     span<char> CompactUnwindSection;
 253   };
 254
 255   using UnwindSectionsMap =
 256       IntervalMap<char *, UnwindSections, IntervalCoalescing::Disabled>;
 257
 258   struct JITDylibState {
 259     std::string Name;
 260     void *Header = nullptr;
 261     bool Sealed = false;
 262     size_t LinkedAgainstRefCount = 0;
 263     size_t DlRefCount = 0;
 264     std::vector<JITDylibState *> Deps;
 265     AtExitsVector AtExits;
 266     const objc_image_info *ObjCImageInfo = nullptr;
 267     std::unordered_map<void *, std::vector<char>> DataSectionContent;
 268     std::unordered_map<void *, size_t> ZeroInitRanges;
 269     UnwindSectionsMap UnwindSections;
 270     RecordSectionsTracker<void (*)()> ModInitsSections;
 271     RecordSectionsTracker<char> ObjCRuntimeRegistrationObjects;
 272
 273     bool referenced() const {
 274       return LinkedAgainstRefCount != 0 || DlRefCount != 0;
 275     }
 276   };
 277
 278 public:
 279   static Error create();
 280   static MachOPlatformRuntimeState &get();
 281   static Error destroy();
 282
 283   MachOPlatformRuntimeState() = default;
 284
 285   // Delete copy and move constructors.
 286   MachOPlatformRuntimeState(const MachOPlatformRuntimeState &) = delete;
 287   MachOPlatformRuntimeState &
 288   operator=(const MachOPlatformRuntimeState &) = delete;
 289   MachOPlatformRuntimeState(MachOPlatformRuntimeState &&) = delete;
 290   MachOPlatformRuntimeState &operator=(MachOPlatformRuntimeState &&) = delete;
 291
 292   Error initialize();
 293   Error shutdown();
 294
 295   Error registerJITDylib(std::string Name, void *Header);
 296   Error deregisterJITDylib(void *Header);
 297   Error registerThreadDataSection(span<const char> ThreadDataSection);
 298   Error deregisterThreadDataSection(span<const char> ThreadDataSection);
 299   Error registerObjectPlatformSections(
 300       ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindSections,
 301       std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs);
 302   Error deregisterObjectPlatformSections(
 303       ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindSections,
 304       std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs);
 305
 306   const char *dlerror();
 307   void *dlopen(std::string_view Name, int Mode);
 308   int dlclose(void *DSOHandle);
 309   void *dlsym(void *DSOHandle, std::string_view Symbol);
 310
 311   int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle);
 312   void runAtExits(std::unique_lock<std::mutex> &JDStatesLock,
 313                   JITDylibState &JDS);
 314   void runAtExits(void *DSOHandle);
 315
 316   /// Returns the base address of the section containing ThreadData.
 317   Expected<std::pair<const char *, size_t>>
 318   getThreadDataSectionFor(const char *ThreadData);
 319
 320 private:
 321   JITDylibState *getJITDylibStateByHeader(void *DSOHandle);
 322   JITDylibState *getJITDylibStateByName(std::string_view Path);
 323
 324   Expected<ExecutorAddr> lookupSymbolInJITDylib(void *DSOHandle,
 325                                                 std::string_view Symbol);
 326
 327   bool lookupUnwindSections(void *Addr, unw_dynamic_unwind_sections &Info);
 328
 329   static int findDynamicUnwindSections(uintptr_t addr,
 330                                        unw_dynamic_unwind_sections *info);
 331   static Error registerEHFrames(span<const char> EHFrameSection);
 332   static Error deregisterEHFrames(span<const char> EHFrameSection);
 333
 334   static Error registerObjCRegistrationObjects(JITDylibState &JDS);
 335   static Error runModInits(std::unique_lock<std::mutex> &JDStatesLock,
 336                            JITDylibState &JDS);
 337
 338   Expected<void *> dlopenImpl(std::string_view Path, int Mode);
 339   Error dlopenFull(std::unique_lock<std::mutex> &JDStatesLock,
 340                    JITDylibState &JDS);
 341   Error dlopenInitialize(std::unique_lock<std::mutex> &JDStatesLock,
 342                          JITDylibState &JDS, MachOJITDylibDepInfoMap &DepInfo);
 343
 344   Error dlcloseImpl(void *DSOHandle);
 345   Error dlcloseDeinitialize(std::unique_lock<std::mutex> &JDStatesLock,
 346                             JITDylibState &JDS);
 347
 348   static MachOPlatformRuntimeState *MOPS;
 349
 350   bool UseCallbackStyleUnwindInfo = false;
 351
 352   // FIXME: Move to thread-state.
 353   std::string DLFcnError;
 354
 355   // APIMutex guards against concurrent entry into key "dyld" API functions
 356   // (e.g. dlopen, dlclose).
 357   std::recursive_mutex DyldAPIMutex;
 358
 359   // JDStatesMutex guards the data structures that hold JITDylib state.
 360   std::mutex JDStatesMutex;
 361   std::unordered_map<void *, JITDylibState> JDStates;
 362   std::unordered_map<std::string_view, void *> JDNameToHeader;
 363
 364   // ThreadDataSectionsMutex guards thread local data section state.
 365   std::mutex ThreadDataSectionsMutex;
 366   std::map<const char *, size_t> ThreadDataSections;
 367 };
 368
 369 MachOPlatformRuntimeState *MachOPlatformRuntimeState::MOPS = nullptr;
 370
 371 Error MachOPlatformRuntimeState::create() {
 372   assert(!MOPS && "MachOPlatformRuntimeState should be null");
 373   MOPS = new MachOPlatformRuntimeState();
 374   return MOPS->initialize();
 375 }
 376
 377 MachOPlatformRuntimeState &MachOPlatformRuntimeState::get() {
 378   assert(MOPS && "MachOPlatformRuntimeState not initialized");
 379   return *MOPS;
 380 }
 381
 382 Error MachOPlatformRuntimeState::destroy() {
 383   assert(MOPS && "MachOPlatformRuntimeState not initialized");
 384   auto Err = MOPS->shutdown();
 385   delete MOPS;
 386   return Err;
 387 }
 388
 389 Error MachOPlatformRuntimeState::initialize() {
 390   UseCallbackStyleUnwindInfo = __unw_add_find_dynamic_unwind_sections &&
 391                                __unw_remove_find_dynamic_unwind_sections;
 392   if (UseCallbackStyleUnwindInfo) {
 393     ORC_RT_DEBUG({
 394       printdbg("__unw_add/remove_find_dynamic_unwind_sections available."
 395                " Using callback-based frame info lookup.\n");
 396     });
 397     if (__unw_add_find_dynamic_unwind_sections(&findDynamicUnwindSections))
 398       return make_error<StringError>(
 399           "Could not register findDynamicUnwindSections");
 400   } else {
 401     ORC_RT_DEBUG({
 402       printdbg("__unw_add/remove_find_dynamic_unwind_sections not available."
 403                " Using classic frame info registration.\n");
 404     });
 405   }
 406   return Error::success();
 407 }
 408
 409 Error MachOPlatformRuntimeState::shutdown() {
 410   if (UseCallbackStyleUnwindInfo) {
 411     if (__unw_remove_find_dynamic_unwind_sections(&findDynamicUnwindSections)) {
 412       ORC_RT_DEBUG(
 413           { printdbg("__unw_remove_find_dynamic_unwind_sections failed.\n"); });
 414     }
 415   }
 416   return Error::success();
 417 }
 418
 419 Error MachOPlatformRuntimeState::registerJITDylib(std::string Name,
 420                                                   void *Header) {
 421   ORC_RT_DEBUG({
 422     printdbg("Registering JITDylib %s: Header = %p\n", Name.c_str(), Header);
 423   });
 424   std::lock_guard<std::mutex> Lock(JDStatesMutex);
 425   if (JDStates.count(Header)) {
 426     std::ostringstream ErrStream;
 427     ErrStream << "Duplicate JITDylib registration for header " << Header
 428               << " (name = " << Name << ")";
 429     return make_error<StringError>(ErrStream.str());
 430   }
 431   if (JDNameToHeader.count(Name)) {
 432     std::ostringstream ErrStream;
 433     ErrStream << "Duplicate JITDylib registration for header " << Header
 434               << " (header = " << Header << ")";
 435     return make_error<StringError>(ErrStream.str());
 436   }
 437
 438   auto &JDS = JDStates[Header];
 439   JDS.Name = std::move(Name);
 440   JDS.Header = Header;
 441   JDNameToHeader[JDS.Name] = Header;
 442   return Error::success();
 443 }
 444
 445 Error MachOPlatformRuntimeState::deregisterJITDylib(void *Header) {
 446   std::lock_guard<std::mutex> Lock(JDStatesMutex);
 447   auto I = JDStates.find(Header);
 448   if (I == JDStates.end()) {
 449     std::ostringstream ErrStream;
 450     ErrStream << "Attempted to deregister unrecognized header " << Header;
 451     return make_error<StringError>(ErrStream.str());
 452   }
 453
 454   // Remove std::string construction once we can use C++20.
 455   auto J = JDNameToHeader.find(
 456       std::string(I->second.Name.data(), I->second.Name.size()));
 457   assert(J != JDNameToHeader.end() &&
 458          "Missing JDNameToHeader entry for JITDylib");
 459
 460   ORC_RT_DEBUG({
 461     printdbg("Deregistering JITDylib %s: Header = %p\n", I->second.Name.c_str(),
 462              Header);
 463   });
 464
 465   JDNameToHeader.erase(J);
 466   JDStates.erase(I);
 467   return Error::success();
 468 }
 469
 470 Error MachOPlatformRuntimeState::registerThreadDataSection(
 471     span<const char> ThreadDataSection) {
 472   std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
 473   auto I = ThreadDataSections.upper_bound(ThreadDataSection.data());
 474   if (I != ThreadDataSections.begin()) {
 475     auto J = std::prev(I);
 476     if (J->first + J->second > ThreadDataSection.data())
 477       return make_error<StringError>("Overlapping __thread_data sections");
 478   }
 479   ThreadDataSections.insert(
 480       I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size()));
 481   return Error::success();
 482 }
 483
 484 Error MachOPlatformRuntimeState::deregisterThreadDataSection(
 485     span<const char> ThreadDataSection) {
 486   std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
 487   auto I = ThreadDataSections.find(ThreadDataSection.data());
 488   if (I == ThreadDataSections.end())
 489     return make_error<StringError>("Attempt to deregister unknown thread data "
 490                                    "section");
 491   ThreadDataSections.erase(I);
 492   return Error::success();
 493 }
 494
 495 Error MachOPlatformRuntimeState::registerObjectPlatformSections(
 496     ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindInfo,
 497     std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs) {
 498
 499   // FIXME: Reject platform section registration after the JITDylib is
 500   // sealed?
 501
 502   ORC_RT_DEBUG({
 503     printdbg("MachOPlatform: Registering object sections for %p.\n",
 504              HeaderAddr.toPtr<void *>());
 505   });
 506
 507   std::lock_guard<std::mutex> Lock(JDStatesMutex);
 508   auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>());
 509   if (!JDS) {
 510     std::ostringstream ErrStream;
 511     ErrStream << "Could not register object platform sections for "
 512                  "unrecognized header "
 513               << HeaderAddr.toPtr<void *>();
 514     return make_error<StringError>(ErrStream.str());
 515   }
 516
 517   if (UnwindInfo && UseCallbackStyleUnwindInfo) {
 518     ORC_RT_DEBUG({
 519       printdbg("  Registering new-style unwind info for:\n"
 520                "    DWARF: %p -- %p\n"
 521                "    Compact-unwind: %p -- %p\n"
 522                "  for:\n",
 523                UnwindInfo->DwarfSection.Start.toPtr<void *>(),
 524                UnwindInfo->DwarfSection.End.toPtr<void *>(),
 525                UnwindInfo->CompactUnwindSection.Start.toPtr<void *>(),
 526                UnwindInfo->CompactUnwindSection.End.toPtr<void *>());
 527     });
 528     for (auto &CodeRange : UnwindInfo->CodeRanges) {
 529       JDS->UnwindSections.insert(CodeRange.Start.toPtr<char *>(),
 530                                  CodeRange.End.toPtr<char *>(), *UnwindInfo);
 531       ORC_RT_DEBUG({
 532         printdbg("    [ %p -- %p ]\n", CodeRange.Start.toPtr<void *>(),
 533                  CodeRange.End.toPtr<void *>());
 534       });
 535     }
 536   }
 537
 538   for (auto &KV : Secs) {
 539     // FIXME: Validate section ranges?
 540     if (KV.first == "__TEXT,__eh_frame") {
 541       if (!UseCallbackStyleUnwindInfo) {
 542         // Use classic libunwind registration.
 543         if (auto Err = registerEHFrames(KV.second.toSpan<const char>()))
 544           return Err;
 545       }
 546     } else if (KV.first == "__DATA,__data") {
 547       assert(!JDS->DataSectionContent.count(KV.second.Start.toPtr<char *>()) &&
 548              "Address already registered.");
 549       auto S = KV.second.toSpan<char>();
 550       JDS->DataSectionContent[KV.second.Start.toPtr<char *>()] =
 551           std::vector<char>(S.begin(), S.end());
 552     } else if (KV.first == "__DATA,__common") {
 553       JDS->ZeroInitRanges[KV.second.Start.toPtr<char *>()] = KV.second.size();
 554     } else if (KV.first == "__DATA,__thread_data") {
 555       if (auto Err = registerThreadDataSection(KV.second.toSpan<const char>()))
 556         return Err;
 557     } else if (KV.first == "__llvm_jitlink_ObjCRuntimeRegistrationObject")
 558       JDS->ObjCRuntimeRegistrationObjects.add(KV.second.toSpan<char>());
 559     else if (KV.first == "__DATA,__mod_init_func")
 560       JDS->ModInitsSections.add(KV.second.toSpan<void (*)()>());
 561     else {
 562       // Should this be a warning instead?
 563       return make_error<StringError>(
 564           "Encountered unexpected section " +
 565           std::string(KV.first.data(), KV.first.size()) +
 566           " while registering object platform sections");
 567     }
 568   }
 569
 570   return Error::success();
 571 }
 572
 573 Error MachOPlatformRuntimeState::deregisterObjectPlatformSections(
 574     ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindInfo,
 575     std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs) {
 576   // TODO: Make this more efficient? (maybe unnecessary if removal is rare?)
 577   // TODO: Add a JITDylib prepare-for-teardown operation that clears all
 578   //       registered sections, causing this function to take the fast-path.
 579   ORC_RT_DEBUG({
 580     printdbg("MachOPlatform: Deregistering object sections for %p.\n",
 581              HeaderAddr.toPtr<void *>());
 582   });
 583
 584   std::lock_guard<std::mutex> Lock(JDStatesMutex);
 585   auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>());
 586   if (!JDS) {
 587     std::ostringstream ErrStream;
 588     ErrStream << "Could not register object platform sections for unrecognized "
 589                  "header "
 590               << HeaderAddr.toPtr<void *>();
 591     return make_error<StringError>(ErrStream.str());
 592   }
 593
 594   // FIXME: Implement faster-path by returning immediately if JDS is being
 595   // torn down entirely?
 596
 597   // TODO: Make library permanent (i.e. not able to be dlclosed) if it contains
 598   // any Swift or ObjC. Once this happens we can clear (and no longer record)
 599   // data section content, as the library could never be re-initialized.
 600
 601   if (UnwindInfo && UseCallbackStyleUnwindInfo) {
 602     ORC_RT_DEBUG({
 603       printdbg("  Deregistering new-style unwind info for:\n"
 604                "    DWARF: %p -- %p\n"
 605                "    Compact-unwind: %p -- %p\n"
 606                "  for:\n",
 607                UnwindInfo->DwarfSection.Start.toPtr<void *>(),
 608                UnwindInfo->DwarfSection.End.toPtr<void *>(),
 609                UnwindInfo->CompactUnwindSection.Start.toPtr<void *>(),
 610                UnwindInfo->CompactUnwindSection.End.toPtr<void *>());
 611     });
 612     for (auto &CodeRange : UnwindInfo->CodeRanges) {
 613       JDS->UnwindSections.erase(CodeRange.Start.toPtr<char *>(),
 614                                 CodeRange.End.toPtr<char *>());
 615       ORC_RT_DEBUG({
 616         printdbg("    [ %p -- %p ]\n", CodeRange.Start.toPtr<void *>(),
 617                  CodeRange.End.toPtr<void *>());
 618       });
 619     }
 620   }
 621
 622   for (auto &KV : Secs) {
 623     // FIXME: Validate section ranges?
 624     if (KV.first == "__TEXT,__eh_frame") {
 625       if (!UseCallbackStyleUnwindInfo) {
 626         // Use classic libunwind registration.
 627         if (auto Err = deregisterEHFrames(KV.second.toSpan<const char>()))
 628           return Err;
 629       }
 630     } else if (KV.first == "__DATA,__data") {
 631       JDS->DataSectionContent.erase(KV.second.Start.toPtr<char *>());
 632     } else if (KV.first == "__DATA,__common") {
 633       JDS->ZeroInitRanges.erase(KV.second.Start.toPtr<char *>());
 634     } else if (KV.first == "__DATA,__thread_data") {
 635       if (auto Err =
 636               deregisterThreadDataSection(KV.second.toSpan<const char>()))
 637         return Err;
 638     } else if (KV.first == "__llvm_jitlink_ObjCRuntimeRegistrationObject")
 639       JDS->ObjCRuntimeRegistrationObjects.removeIfPresent(KV.second);
 640     else if (KV.first == "__DATA,__mod_init_func")
 641       JDS->ModInitsSections.removeIfPresent(KV.second);
 642     else {
 643       // Should this be a warning instead?
 644       return make_error<StringError>(
 645           "Encountered unexpected section " +
 646           std::string(KV.first.data(), KV.first.size()) +
 647           " while deregistering object platform sections");
 648     }
 649   }
 650   return Error::success();
 651 }
 652
 653 const char *MachOPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }
 654
 655 void *MachOPlatformRuntimeState::dlopen(std::string_view Path, int Mode) {
 656   ORC_RT_DEBUG({
 657     std::string S(Path.data(), Path.size());
 658     printdbg("MachOPlatform::dlopen(\"%s\")\n", S.c_str());
 659   });
 660   std::lock_guard<std::recursive_mutex> Lock(DyldAPIMutex);
 661   if (auto H = dlopenImpl(Path, Mode))
 662     return *H;
 663   else {
 664     // FIXME: Make dlerror thread safe.
 665     DLFcnError = toString(H.takeError());
 666     return nullptr;
 667   }
 668 }
 669
 670 int MachOPlatformRuntimeState::dlclose(void *DSOHandle) {
 671   ORC_RT_DEBUG({
 672     auto *JDS = getJITDylibStateByHeader(DSOHandle);
 673     std::string DylibName;
 674     if (JDS) {
 675       std::string S;
 676       printdbg("MachOPlatform::dlclose(%p) (%s)\n", DSOHandle, S.c_str());
 677     } else
 678       printdbg("MachOPlatform::dlclose(%p) (%s)\n", DSOHandle,
 679                "invalid handle");
 680   });
 681   std::lock_guard<std::recursive_mutex> Lock(DyldAPIMutex);
 682   if (auto Err = dlcloseImpl(DSOHandle)) {
 683     // FIXME: Make dlerror thread safe.
 684     DLFcnError = toString(std::move(Err));
 685     return -1;
 686   }
 687   return 0;
 688 }
 689
 690 void *MachOPlatformRuntimeState::dlsym(void *DSOHandle,
 691                                        std::string_view Symbol) {
 692   auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol);
 693   if (!Addr) {
 694     DLFcnError = toString(Addr.takeError());
 695     return 0;
 696   }
 697
 698   return Addr->toPtr<void *>();
 699 }
 700
 701 int MachOPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg,
 702                                               void *DSOHandle) {
 703   // FIXME: Handle out-of-memory errors, returning -1 if OOM.
 704   std::lock_guard<std::mutex> Lock(JDStatesMutex);
 705   auto *JDS = getJITDylibStateByHeader(DSOHandle);
 706   if (!JDS) {
 707     ORC_RT_DEBUG({
 708       printdbg("MachOPlatformRuntimeState::registerAtExit called with "
 709                "unrecognized dso handle %p\n",
 710                DSOHandle);
 711     });
 712     return -1;
 713   }
 714   JDS->AtExits.push_back({F, Arg});
 715   return 0;
 716 }
 717
 718 void MachOPlatformRuntimeState::runAtExits(
 719     std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
 720   auto AtExits = std::move(JDS.AtExits);
 721
 722   // Unlock while running atexits, as they may trigger operations that modify
 723   // JDStates.
 724   JDStatesLock.unlock();
 725   while (!AtExits.empty()) {
 726     auto &AE = AtExits.back();
 727     AE.Func(AE.Arg);
 728     AtExits.pop_back();
 729   }
 730   JDStatesLock.lock();
 731 }
 732
 733 void MachOPlatformRuntimeState::runAtExits(void *DSOHandle) {
 734   std::unique_lock<std::mutex> Lock(JDStatesMutex);
 735   auto *JDS = getJITDylibStateByHeader(DSOHandle);
 736   ORC_RT_DEBUG({
 737     printdbg("MachOPlatformRuntimeState::runAtExits called on unrecognized "
 738              "dso_handle %p\n",
 739              DSOHandle);
 740   });
 741   if (JDS)
 742     runAtExits(Lock, *JDS);
 743 }
 744
 745 Expected<std::pair<const char *, size_t>>
 746 MachOPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) {
 747   std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
 748   auto I = ThreadDataSections.upper_bound(ThreadData);
 749   // Check that we have a valid entry covering this address.
 750   if (I == ThreadDataSections.begin())
 751     return make_error<StringError>("No thread local data section for key");
 752   I = std::prev(I);
 753   if (ThreadData >= I->first + I->second)
 754     return make_error<StringError>("No thread local data section for key");
 755   return *I;
 756 }
 757
 758 MachOPlatformRuntimeState::JITDylibState *
 759 MachOPlatformRuntimeState::getJITDylibStateByHeader(void *DSOHandle) {
 760   auto I = JDStates.find(DSOHandle);
 761   if (I == JDStates.end()) {
 762     I = JDStates.insert(std::make_pair(DSOHandle, JITDylibState())).first;
 763     I->second.Header = DSOHandle;
 764   }
 765   return &I->second;
 766 }
 767
 768 MachOPlatformRuntimeState::JITDylibState *
 769 MachOPlatformRuntimeState::getJITDylibStateByName(std::string_view Name) {
 770   // FIXME: Avoid creating string once we have C++20.
 771   auto I = JDNameToHeader.find(std::string(Name.data(), Name.size()));
 772   if (I != JDNameToHeader.end())
 773     return getJITDylibStateByHeader(I->second);
 774   return nullptr;
 775 }
 776
 777 Expected<ExecutorAddr>
 778 MachOPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle,
 779                                                   std::string_view Sym) {
 780   Expected<ExecutorAddr> Result((ExecutorAddr()));
 781   if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddr>(
 782           SPSExecutorAddr, SPSString)>::call(&__orc_rt_macho_symbol_lookup_tag,
 783                                              Result,
 784                                              ExecutorAddr::fromPtr(DSOHandle),
 785                                              Sym))
 786     return std::move(Err);
 787   return Result;
 788 }
 789
 790 // eh-frame registration functions.
 791 // We expect these to be available for all processes.
 792 extern "C" void __register_frame(const void *);
 793 extern "C" void __deregister_frame(const void *);
 794
 795 template <typename HandleFDEFn>
 796 void walkEHFrameSection(span<const char> EHFrameSection,
 797                         HandleFDEFn HandleFDE) {
 798   const char *CurCFIRecord = EHFrameSection.data();
 799   uint64_t Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord);
 800
 801   while (CurCFIRecord != EHFrameSection.end() && Size != 0) {
 802     const char *OffsetField = CurCFIRecord + (Size == 0xffffffff ? 12 : 4);
 803     if (Size == 0xffffffff)
 804       Size = *reinterpret_cast<const uint64_t *>(CurCFIRecord + 4) + 12;
 805     else
 806       Size += 4;
 807     uint32_t Offset = *reinterpret_cast<const uint32_t *>(OffsetField);
 808
 809     if (Offset != 0)
 810       HandleFDE(CurCFIRecord);
 811
 812     CurCFIRecord += Size;
 813     Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord);
 814   }
 815 }
 816
 817 bool MachOPlatformRuntimeState::lookupUnwindSections(
 818     void *Addr, unw_dynamic_unwind_sections &Info) {
 819   ORC_RT_DEBUG(
 820       { printdbg("Tried to lookup unwind-info via new lookup call.\n"); });
 821   std::lock_guard<std::mutex> Lock(JDStatesMutex);
 822   for (auto &KV : JDStates) {
 823     auto &JD = KV.second;
 824     auto I = JD.UnwindSections.find(reinterpret_cast<char *>(Addr));
 825     if (I != JD.UnwindSections.end()) {
 826       Info.dso_base = reinterpret_cast<uintptr_t>(JD.Header);
 827       Info.dwarf_section =
 828           reinterpret_cast<uintptr_t>(I->second.DwarfSection.data());
 829       Info.dwarf_section_length = I->second.DwarfSection.size();
 830       Info.compact_unwind_section =
 831           reinterpret_cast<uintptr_t>(I->second.CompactUnwindSection.data());
 832       Info.compact_unwind_section_length =
 833           I->second.CompactUnwindSection.size();
 834       return true;
 835     }
 836   }
 837   return false;
 838 }
 839
 840 int MachOPlatformRuntimeState::findDynamicUnwindSections(
 841     uintptr_t addr, unw_dynamic_unwind_sections *info) {
 842   if (!info)
 843     return 0;
 844   return MachOPlatformRuntimeState::get().lookupUnwindSections((void *)addr,
 845                                                                *info);
 846 }
 847
 848 Error MachOPlatformRuntimeState::registerEHFrames(
 849     span<const char> EHFrameSection) {
 850   walkEHFrameSection(EHFrameSection, __register_frame);
 851   return Error::success();
 852 }
 853
 854 Error MachOPlatformRuntimeState::deregisterEHFrames(
 855     span<const char> EHFrameSection) {
 856   walkEHFrameSection(EHFrameSection, __deregister_frame);
 857   return Error::success();
 858 }
 859
 860 Error MachOPlatformRuntimeState::registerObjCRegistrationObjects(
 861     JITDylibState &JDS) {
 862   ORC_RT_DEBUG(printdbg("Registering Objective-C / Swift metadata.\n"));
 863
 864   std::vector<char *> RegObjBases;
 865   JDS.ObjCRuntimeRegistrationObjects.processNewSections(
 866       [&](span<char> RegObj) { RegObjBases.push_back(RegObj.data()); });
 867
 868   if (RegObjBases.empty())
 869     return Error::success();
 870
 871   if (!_objc_map_images || !_objc_load_image)
 872     return make_error<StringError>(
 873         "Could not register Objective-C / Swift metadata: _objc_map_images / "
 874         "_objc_load_image not found");
 875
 876   std::vector<char *> Paths;
 877   Paths.resize(RegObjBases.size());
 878   _objc_map_images(RegObjBases.size(), Paths.data(),
 879                    reinterpret_cast<mach_header **>(RegObjBases.data()));
 880
 881   for (void *RegObjBase : RegObjBases)
 882     _objc_load_image(nullptr, reinterpret_cast<mach_header *>(RegObjBase));
 883
 884   return Error::success();
 885 }
 886
 887 Error MachOPlatformRuntimeState::runModInits(
 888     std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
 889   std::vector<span<void (*)()>> InitSections;
 890   InitSections.reserve(JDS.ModInitsSections.numNewSections());
 891
 892   // Copy initializer sections: If the JITDylib is unsealed then the
 893   // initializers could reach back into the JIT and cause more initializers to
 894   // be added.
 895   // FIXME: Skip unlock and run in-place on sealed JITDylibs?
 896   JDS.ModInitsSections.processNewSections(
 897       [&](span<void (*)()> Inits) { InitSections.push_back(Inits); });
 898
 899   JDStatesLock.unlock();
 900   for (auto InitSec : InitSections)
 901     for (auto *Init : InitSec)
 902       Init();
 903   JDStatesLock.lock();
 904
 905   return Error::success();
 906 }
 907
 908 Expected<void *> MachOPlatformRuntimeState::dlopenImpl(std::string_view Path,
 909                                                        int Mode) {
 910   std::unique_lock<std::mutex> Lock(JDStatesMutex);
 911
 912   // Try to find JITDylib state by name.
 913   auto *JDS = getJITDylibStateByName(Path);
 914
 915   if (!JDS)
 916     return make_error<StringError>("No registered JTIDylib for path " +
 917                                    std::string(Path.data(), Path.size()));
 918
 919   // If this JITDylib is unsealed, or this is the first dlopen then run
 920   // full dlopen path (update deps, push and run initializers, update ref
 921   // counts on all JITDylibs in the dep tree).
 922   if (!JDS->referenced() || !JDS->Sealed) {
 923     if (auto Err = dlopenFull(Lock, *JDS))
 924       return std::move(Err);
 925   }
 926
 927   // Bump the ref-count on this dylib.
 928   ++JDS->DlRefCount;
 929
 930   // Return the header address.
 931   return JDS->Header;
 932 }
 933
 934 Error MachOPlatformRuntimeState::dlopenFull(
 935     std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
 936   // Call back to the JIT to push the initializers.
 937   Expected<MachOJITDylibDepInfoMap> DepInfo((MachOJITDylibDepInfoMap()));
 938   // Unlock so that we can accept the initializer update.
 939   JDStatesLock.unlock();
 940   if (auto Err = WrapperFunction<SPSExpected<SPSMachOJITDylibDepInfoMap>(
 941           SPSExecutorAddr)>::call(&__orc_rt_macho_push_initializers_tag,
 942                                   DepInfo, ExecutorAddr::fromPtr(JDS.Header)))
 943     return Err;
 944   JDStatesLock.lock();
 945
 946   if (!DepInfo)
 947     return DepInfo.takeError();
 948
 949   if (auto Err = dlopenInitialize(JDStatesLock, JDS, *DepInfo))
 950     return Err;
 951
 952   if (!DepInfo->empty()) {
 953     ORC_RT_DEBUG({
 954       printdbg("Unrecognized dep-info key headers in dlopen of %s\n",
 955                JDS.Name.c_str());
 956     });
 957     std::ostringstream ErrStream;
 958     ErrStream << "Encountered unrecognized dep-info key headers "
 959                  "while processing dlopen of "
 960               << JDS.Name;
 961     return make_error<StringError>(ErrStream.str());
 962   }
 963
 964   return Error::success();
 965 }
 966
 967 Error MachOPlatformRuntimeState::dlopenInitialize(
 968     std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS,
 969     MachOJITDylibDepInfoMap &DepInfo) {
 970   ORC_RT_DEBUG({
 971     printdbg("MachOPlatformRuntimeState::dlopenInitialize(\"%s\")\n",
 972              JDS.Name.c_str());
 973   });
 974
 975   // If the header is not present in the dep map then assume that we
 976   // already processed it earlier in the dlopenInitialize traversal and
 977   // return.
 978   // TODO: Keep a visited set instead so that we can error out on missing
 979   //       entries?
 980   auto I = DepInfo.find(ExecutorAddr::fromPtr(JDS.Header));
 981   if (I == DepInfo.end())
 982     return Error::success();
 983
 984   auto DI = std::move(I->second);
 985   DepInfo.erase(I);
 986
 987   // We don't need to re-initialize sealed JITDylibs that have already been
 988   // initialized. Just check that their dep-map entry is empty as expected.
 989   if (JDS.Sealed) {
 990     if (!DI.DepHeaders.empty()) {
 991       std::ostringstream ErrStream;
 992       ErrStream << "Sealed JITDylib " << JDS.Header
 993                 << " already has registered dependencies";
 994       return make_error<StringError>(ErrStream.str());
 995     }
 996     if (JDS.referenced())
 997       return Error::success();
 998   } else
 999     JDS.Sealed = DI.Sealed;
1000
1001   // This is an unsealed or newly sealed JITDylib. Run initializers.
1002   std::vector<JITDylibState *> OldDeps;
1003   std::swap(JDS.Deps, OldDeps);
1004   JDS.Deps.reserve(DI.DepHeaders.size());
1005   for (auto DepHeaderAddr : DI.DepHeaders) {
1006     auto *DepJDS = getJITDylibStateByHeader(DepHeaderAddr.toPtr<void *>());
1007     if (!DepJDS) {
1008       std::ostringstream ErrStream;
1009       ErrStream << "Encountered unrecognized dep header "
1010                 << DepHeaderAddr.toPtr<void *>() << " while initializing "
1011                 << JDS.Name;
1012       return make_error<StringError>(ErrStream.str());
1013     }
1014     ++DepJDS->LinkedAgainstRefCount;
1015     if (auto Err = dlopenInitialize(JDStatesLock, *DepJDS, DepInfo))
1016       return Err;
1017   }
1018
1019   // Initialize this JITDylib.
1020   if (auto Err = registerObjCRegistrationObjects(JDS))
1021     return Err;
1022   if (auto Err = runModInits(JDStatesLock, JDS))
1023     return Err;
1024
1025   // Decrement old deps.
1026   // FIXME: We should probably continue and just report deinitialize errors
1027   // here.
1028   for (auto *DepJDS : OldDeps) {
1029     --DepJDS->LinkedAgainstRefCount;
1030     if (!DepJDS->referenced())
1031       if (auto Err = dlcloseDeinitialize(JDStatesLock, *DepJDS))
1032         return Err;
1033   }
1034
1035   return Error::success();
1036 }
1037
1038 Error MachOPlatformRuntimeState::dlcloseImpl(void *DSOHandle) {
1039   std::unique_lock<std::mutex> Lock(JDStatesMutex);
1040
1041   // Try to find JITDylib state by header.
1042   auto *JDS = getJITDylibStateByHeader(DSOHandle);
1043
1044   if (!JDS) {
1045     std::ostringstream ErrStream;
1046     ErrStream << "No registered JITDylib for " << DSOHandle;
1047     return make_error<StringError>(ErrStream.str());
1048   }
1049
1050   // Bump the ref-count.
1051   --JDS->DlRefCount;
1052
1053   if (!JDS->referenced())
1054     return dlcloseDeinitialize(Lock, *JDS);
1055
1056   return Error::success();
1057 }
1058
1059 Error MachOPlatformRuntimeState::dlcloseDeinitialize(
1060     std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
1061
1062   ORC_RT_DEBUG({
1063     printdbg("MachOPlatformRuntimeState::dlcloseDeinitialize(\"%s\")\n",
1064              JDS.Name.c_str());
1065   });
1066
1067   runAtExits(JDStatesLock, JDS);
1068
1069   // Reset mod-inits
1070   JDS.ModInitsSections.reset();
1071
1072   // Reset data section contents.
1073   for (auto &KV : JDS.DataSectionContent)
1074     memcpy(KV.first, KV.second.data(), KV.second.size());
1075   for (auto &KV : JDS.ZeroInitRanges)
1076     memset(KV.first, 0, KV.second);
1077
1078   // Deinitialize any dependencies.
1079   for (auto *DepJDS : JDS.Deps) {
1080     --DepJDS->LinkedAgainstRefCount;
1081     if (!DepJDS->referenced())
1082       if (auto Err = dlcloseDeinitialize(JDStatesLock, *DepJDS))
1083         return Err;
1084   }
1085
1086   return Error::success();
1087 }
1088
1089 class MachOPlatformRuntimeTLVManager {
1090 public:
1091   void *getInstance(const char *ThreadData);
1092
1093 private:
1094   std::unordered_map<const char *, char *> Instances;
1095   std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections;
1096 };
1097
1098 void *MachOPlatformRuntimeTLVManager::getInstance(const char *ThreadData) {
1099   auto I = Instances.find(ThreadData);
1100   if (I != Instances.end())
1101     return I->second;
1102
1103   auto TDS =
1104       MachOPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData);
1105   if (!TDS) {
1106     __orc_rt_log_error(toString(TDS.takeError()).c_str());
1107     return nullptr;
1108   }
1109
1110   auto &Allocated = AllocatedSections[TDS->first];
1111   if (!Allocated) {
1112     Allocated = std::make_unique<char[]>(TDS->second);
1113     memcpy(Allocated.get(), TDS->first, TDS->second);
1114   }
1115
1116   size_t ThreadDataDelta = ThreadData - TDS->first;
1117   assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds");
1118
1119   char *Instance = Allocated.get() + ThreadDataDelta;
1120   Instances[ThreadData] = Instance;
1121   return Instance;
1122 }
1123
1124 void destroyMachOTLVMgr(void *MachOTLVMgr) {
1125   delete static_cast<MachOPlatformRuntimeTLVManager *>(MachOTLVMgr);
1126 }
1127
1128 Error runWrapperFunctionCalls(std::vector<WrapperFunctionCall> WFCs) {
1129   for (auto &WFC : WFCs)
1130     if (auto Err = WFC.runWithSPSRet<void>())
1131       return Err;
1132   return Error::success();
1133 }
1134
1135 } // end anonymous namespace
1136
1137 //------------------------------------------------------------------------------
1138 //                             JIT entry points
1139 //------------------------------------------------------------------------------
1140
1141 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
1142 __orc_rt_macho_platform_bootstrap(char *ArgData, size_t ArgSize) {
1143   return WrapperFunction<SPSError()>::handle(
1144              ArgData, ArgSize,
1145              []() { return MachOPlatformRuntimeState::create(); })
1146       .release();
1147 }
1148
1149 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
1150 __orc_rt_macho_platform_shutdown(char *ArgData, size_t ArgSize) {
1151   return WrapperFunction<SPSError()>::handle(
1152              ArgData, ArgSize,
1153              []() { return MachOPlatformRuntimeState::destroy(); })
1154       .release();
1155 }
1156
1157 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
1158 __orc_rt_macho_register_jitdylib(char *ArgData, size_t ArgSize) {
1159   return WrapperFunction<SPSError(SPSString, SPSExecutorAddr)>::handle(
1160              ArgData, ArgSize,
1161              [](std::string &Name, ExecutorAddr HeaderAddr) {
1162                return MachOPlatformRuntimeState::get().registerJITDylib(
1163                    std::move(Name), HeaderAddr.toPtr<void *>());
1164              })
1165       .release();
1166 }
1167
1168 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
1169 __orc_rt_macho_deregister_jitdylib(char *ArgData, size_t ArgSize) {
1170   return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
1171              ArgData, ArgSize,
1172              [](ExecutorAddr HeaderAddr) {
1173                return MachOPlatformRuntimeState::get().deregisterJITDylib(
1174                    HeaderAddr.toPtr<void *>());
1175              })
1176       .release();
1177 }
1178
1179 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
1180 __orc_rt_macho_register_object_platform_sections(char *ArgData,
1181                                                  size_t ArgSize) {
1182   return WrapperFunction<SPSError(SPSExecutorAddr,
1183                                   SPSOptional<SPSUnwindSectionInfo>,
1184                                   SPSMachOObjectPlatformSectionsMap)>::
1185       handle(ArgData, ArgSize,
1186              [](ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> USI,
1187                 std::vector<std::pair<std::string_view, ExecutorAddrRange>>
1188                     &Secs) {
1189                return MachOPlatformRuntimeState::get()
1190                    .registerObjectPlatformSections(HeaderAddr, std::move(USI),
1191                                                    std::move(Secs));
1192              })
1193           .release();
1194 }
1195
1196 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
1197 __orc_rt_macho_deregister_object_platform_sections(char *ArgData,
1198                                                    size_t ArgSize) {
1199   return WrapperFunction<SPSError(SPSExecutorAddr,
1200                                   SPSOptional<SPSUnwindSectionInfo>,
1201                                   SPSMachOObjectPlatformSectionsMap)>::
1202       handle(ArgData, ArgSize,
1203              [](ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> USI,
1204                 std::vector<std::pair<std::string_view, ExecutorAddrRange>>
1205                     &Secs) {
1206                return MachOPlatformRuntimeState::get()
1207                    .deregisterObjectPlatformSections(HeaderAddr, std::move(USI),
1208                                                      std::move(Secs));
1209              })
1210           .release();
1211 }
1212
1213 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
1214 __orc_rt_macho_run_wrapper_function_calls(char *ArgData, size_t ArgSize) {
1215   return WrapperFunction<SPSError(SPSSequence<SPSWrapperFunctionCall>)>::handle(
1216              ArgData, ArgSize, runWrapperFunctionCalls)
1217       .release();
1218 }
1219
1220 //------------------------------------------------------------------------------
1221 //                            TLV support
1222 //------------------------------------------------------------------------------
1223
1224 ORC_RT_INTERFACE void *__orc_rt_macho_tlv_get_addr_impl(TLVDescriptor *D) {
1225   auto *TLVMgr = static_cast<MachOPlatformRuntimeTLVManager *>(
1226       pthread_getspecific(D->Key));
1227   if (!TLVMgr) {
1228     TLVMgr = new MachOPlatformRuntimeTLVManager();
1229     if (pthread_setspecific(D->Key, TLVMgr)) {
1230       __orc_rt_log_error("Call to pthread_setspecific failed");
1231       return nullptr;
1232     }
1233   }
1234
1235   return TLVMgr->getInstance(
1236       reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress)));
1237 }
1238
1239 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
1240 __orc_rt_macho_create_pthread_key(char *ArgData, size_t ArgSize) {
1241   return WrapperFunction<SPSExpected<uint64_t>(void)>::handle(
1242              ArgData, ArgSize,
1243              []() -> Expected<uint64_t> {
1244                pthread_key_t Key;
1245                if (int Err = pthread_key_create(&Key, destroyMachOTLVMgr)) {
1246                  __orc_rt_log_error("Call to pthread_key_create failed");
1247                  return make_error<StringError>(strerror(Err));
1248                }
1249                return static_cast<uint64_t>(Key);
1250              })
1251       .release();
1252 }
1253
1254 //------------------------------------------------------------------------------
1255 //                           cxa_atexit support
1256 //------------------------------------------------------------------------------
1257
1258 int __orc_rt_macho_cxa_atexit(void (*func)(void *), void *arg,
1259                               void *dso_handle) {
1260   return MachOPlatformRuntimeState::get().registerAtExit(func, arg, dso_handle);
1261 }
1262
1263 void __orc_rt_macho_cxa_finalize(void *dso_handle) {
1264   MachOPlatformRuntimeState::get().runAtExits(dso_handle);
1265 }
1266
1267 //------------------------------------------------------------------------------
1268 //                        JIT'd dlfcn alternatives.
1269 //------------------------------------------------------------------------------
1270
1271 const char *__orc_rt_macho_jit_dlerror() {
1272   return MachOPlatformRuntimeState::get().dlerror();
1273 }
1274
1275 void *__orc_rt_macho_jit_dlopen(const char *path, int mode) {
1276   return MachOPlatformRuntimeState::get().dlopen(path, mode);
1277 }
1278
1279 int __orc_rt_macho_jit_dlclose(void *dso_handle) {
1280   return MachOPlatformRuntimeState::get().dlclose(dso_handle);
1281 }
1282
1283 void *__orc_rt_macho_jit_dlsym(void *dso_handle, const char *symbol) {
1284   return MachOPlatformRuntimeState::get().dlsym(dso_handle, symbol);
1285 }
1286
1287 //------------------------------------------------------------------------------
1288 //                             MachO Run Program
1289 //------------------------------------------------------------------------------
1290
1291 ORC_RT_INTERFACE int64_t __orc_rt_macho_run_program(const char *JITDylibName,
1292                                                     const char *EntrySymbolName,
1293                                                     int argc, char *argv[]) {
1294   using MainTy = int (*)(int, char *[]);
1295
1296   void *H = __orc_rt_macho_jit_dlopen(JITDylibName,
1297                                       __orc_rt::macho::ORC_RT_RTLD_LAZY);
1298   if (!H) {
1299     __orc_rt_log_error(__orc_rt_macho_jit_dlerror());
1300     return -1;
1301   }
1302
1303   auto *Main =
1304       reinterpret_cast<MainTy>(__orc_rt_macho_jit_dlsym(H, EntrySymbolName));
1305
1306   if (!Main) {
1307     __orc_rt_log_error(__orc_rt_macho_jit_dlerror());
1308     return -1;
1309   }
1310
1311   int Result = Main(argc, argv);
1312
1313   if (__orc_rt_macho_jit_dlclose(H) == -1)
1314     __orc_rt_log_error(__orc_rt_macho_jit_dlerror());
1315
1316   return Result;
1317 }