compiler-rt/lib/orc/macho_platform.cpp

   1 //===- macho_platform.cpp -------------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file contains code required to load the rest of the MachO runtime.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "macho_platform.h"
  14 #include "common.h"
  15 #include "debug.h"
  16 #include "error.h"
  17 #include "interval_map.h"
  18 #include "wrapper_function_utils.h"
  19
  20 #include <algorithm>
  21 #include <ios>
  22 #include <map>
  23 #include <mutex>
  24 #include <sstream>
  25 #include <string_view>
  26 #include <unordered_map>
  27 #include <unordered_set>
  28 #include <vector>
  29
  30 #define DEBUG_TYPE "macho_platform"
  31
  32 using namespace __orc_rt;
  33 using namespace __orc_rt::macho;
  34
  35 // Declare function tags for functions in the JIT process.
  36 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_push_initializers_tag)
  37 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_symbol_lookup_tag)
  38
  39 // Objective-C types.
  40 struct objc_class;
  41 struct objc_image_info;
  42 struct objc_object;
  43 struct objc_selector;
  44
  45 using Class = objc_class *;
  46 using id = objc_object *;
  47 using SEL = objc_selector *;
  48
  49 // Objective-C registration functions.
  50 // These are weakly imported. If the Objective-C runtime has not been loaded
  51 // then code containing Objective-C sections will generate an error.
  52 extern "C" id objc_msgSend(id, SEL, ...) ORC_RT_WEAK_IMPORT;
  53 extern "C" Class objc_readClassPair(Class,
  54                                     const objc_image_info *) ORC_RT_WEAK_IMPORT;
  55 extern "C" SEL sel_registerName(const char *) ORC_RT_WEAK_IMPORT;
  56
  57 // Swift types.
  58 class ProtocolRecord;
  59 class ProtocolConformanceRecord;
  60 class TypeMetadataRecord;
  61
  62 extern "C" void
  63 swift_registerProtocols(const ProtocolRecord *begin,
  64                         const ProtocolRecord *end) ORC_RT_WEAK_IMPORT;
  65
  66 extern "C" void swift_registerProtocolConformances(
  67     const ProtocolConformanceRecord *begin,
  68     const ProtocolConformanceRecord *end) ORC_RT_WEAK_IMPORT;
  69
  70 extern "C" void swift_registerTypeMetadataRecords(
  71     const TypeMetadataRecord *begin,
  72     const TypeMetadataRecord *end) ORC_RT_WEAK_IMPORT;
  73
  74 // Libunwind prototypes.
  75 struct unw_dynamic_unwind_sections {
  76   uintptr_t dso_base;
  77   uintptr_t dwarf_section;
  78   size_t dwarf_section_length;
  79   uintptr_t compact_unwind_section;
  80   size_t compact_unwind_section_length;
  81 };
  82
  83 typedef int (*unw_find_dynamic_unwind_sections)(
  84     uintptr_t addr, struct unw_dynamic_unwind_sections *info);
  85
  86 extern "C" int __unw_add_find_dynamic_unwind_sections(
  87     unw_find_dynamic_unwind_sections find_dynamic_unwind_sections)
  88     ORC_RT_WEAK_IMPORT;
  89
  90 extern "C" int __unw_remove_find_dynamic_unwind_sections(
  91     unw_find_dynamic_unwind_sections find_dynamic_unwind_sections)
  92     ORC_RT_WEAK_IMPORT;
  93
  94 namespace {
  95
  96 struct MachOJITDylibDepInfo {
  97   bool Sealed = false;
  98   std::vector<ExecutorAddr> DepHeaders;
  99 };
 100
 101 using MachOJITDylibDepInfoMap =
 102     std::unordered_map<ExecutorAddr, MachOJITDylibDepInfo>;
 103
 104 } // anonymous namespace
 105
 106 namespace __orc_rt {
 107
 108 using SPSMachOObjectPlatformSectionsMap =
 109     SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>;
 110
 111 using SPSMachOJITDylibDepInfo = SPSTuple<bool, SPSSequence<SPSExecutorAddr>>;
 112
 113 using SPSMachOJITDylibDepInfoMap =
 114     SPSSequence<SPSTuple<SPSExecutorAddr, SPSMachOJITDylibDepInfo>>;
 115
 116 template <>
 117 class SPSSerializationTraits<SPSMachOJITDylibDepInfo, MachOJITDylibDepInfo> {
 118 public:
 119   static size_t size(const MachOJITDylibDepInfo &JDI) {
 120     return SPSMachOJITDylibDepInfo::AsArgList::size(JDI.Sealed, JDI.DepHeaders);
 121   }
 122
 123   static bool serialize(SPSOutputBuffer &OB, const MachOJITDylibDepInfo &JDI) {
 124     return SPSMachOJITDylibDepInfo::AsArgList::serialize(OB, JDI.Sealed,
 125                                                          JDI.DepHeaders);
 126   }
 127
 128   static bool deserialize(SPSInputBuffer &IB, MachOJITDylibDepInfo &JDI) {
 129     return SPSMachOJITDylibDepInfo::AsArgList::deserialize(IB, JDI.Sealed,
 130                                                            JDI.DepHeaders);
 131   }
 132 };
 133
 134 struct UnwindSectionInfo {
 135   std::vector<ExecutorAddrRange> CodeRanges;
 136   ExecutorAddrRange DwarfSection;
 137   ExecutorAddrRange CompactUnwindSection;
 138 };
 139
 140 using SPSUnwindSectionInfo =
 141     SPSTuple<SPSSequence<SPSExecutorAddrRange>, SPSExecutorAddrRange,
 142              SPSExecutorAddrRange>;
 143
 144 template <>
 145 class SPSSerializationTraits<SPSUnwindSectionInfo, UnwindSectionInfo> {
 146 public:
 147   static size_t size(const UnwindSectionInfo &USI) {
 148     return SPSUnwindSectionInfo::AsArgList::size(
 149         USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection);
 150   }
 151
 152   static bool serialize(SPSOutputBuffer &OB, const UnwindSectionInfo &USI) {
 153     return SPSUnwindSectionInfo::AsArgList::serialize(
 154         OB, USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection);
 155   }
 156
 157   static bool deserialize(SPSInputBuffer &IB, UnwindSectionInfo &USI) {
 158     return SPSUnwindSectionInfo::AsArgList::deserialize(
 159         IB, USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection);
 160   }
 161 };
 162
 163 } // namespace __orc_rt
 164
 165 namespace {
 166 struct TLVDescriptor {
 167   void *(*Thunk)(TLVDescriptor *) = nullptr;
 168   unsigned long Key = 0;
 169   unsigned long DataAddress = 0;
 170 };
 171
 172 class MachOPlatformRuntimeState {
 173 private:
 174   struct AtExitEntry {
 175     void (*Func)(void *);
 176     void *Arg;
 177   };
 178
 179   using AtExitsVector = std::vector<AtExitEntry>;
 180
 181   /// Used to manage sections of fixed-sized metadata records (e.g. pointer
 182   /// sections, selector refs, etc.)
 183   template <typename RecordElement> class RecordSectionsTracker {
 184   public:
 185     /// Add a section to the "new" list.
 186     void add(span<RecordElement> Sec) { New.push_back(std::move(Sec)); }
 187
 188     /// Returns true if there are new sections to process.
 189     bool hasNewSections() const { return !New.empty(); }
 190
 191     /// Returns the number of new sections to process.
 192     size_t numNewSections() const { return New.size(); }
 193
 194     /// Process all new sections.
 195     template <typename ProcessSectionFunc>
 196     std::enable_if_t<std::is_void_v<
 197         std::invoke_result_t<ProcessSectionFunc, span<RecordElement>>>>
 198     processNewSections(ProcessSectionFunc &&ProcessSection) {
 199       for (auto &Sec : New)
 200         ProcessSection(Sec);
 201       moveNewToProcessed();
 202     }
 203
 204     /// Proces all new sections with a fallible handler.
 205     ///
 206     /// Successfully handled sections will be moved to the Processed
 207     /// list.
 208     template <typename ProcessSectionFunc>
 209     std::enable_if_t<
 210         std::is_same_v<Error, std::invoke_result_t<ProcessSectionFunc,
 211                                                    span<RecordElement>>>,
 212         Error>
 213     processNewSections(ProcessSectionFunc &&ProcessSection) {
 214       for (size_t I = 0; I != New.size(); ++I) {
 215         if (auto Err = ProcessSection(New[I])) {
 216           for (size_t J = 0; J != I; ++J)
 217             Processed.push_back(New[J]);
 218           New.erase(New.begin(), New.begin() + I);
 219           return Err;
 220         }
 221       }
 222       moveNewToProcessed();
 223       return Error::success();
 224     }
 225
 226     /// Move all sections back to New for reprocessing.
 227     void reset() {
 228       moveNewToProcessed();
 229       New = std::move(Processed);
 230     }
 231
 232     /// Remove the section with the given range.
 233     bool removeIfPresent(ExecutorAddrRange R) {
 234       if (removeIfPresent(New, R))
 235         return true;
 236       return removeIfPresent(Processed, R);
 237     }
 238
 239   private:
 240     void moveNewToProcessed() {
 241       if (Processed.empty())
 242         Processed = std::move(New);
 243       else {
 244         Processed.reserve(Processed.size() + New.size());
 245         std::copy(New.begin(), New.end(), std::back_inserter(Processed));
 246         New.clear();
 247       }
 248     }
 249
 250     bool removeIfPresent(std::vector<span<RecordElement>> &V,
 251                          ExecutorAddrRange R) {
 252       auto RI = std::find_if(
 253           V.rbegin(), V.rend(),
 254           [RS = R.toSpan<RecordElement>()](const span<RecordElement> &E) {
 255             return E.data() == RS.data();
 256           });
 257       if (RI != V.rend()) {
 258         V.erase(std::next(RI).base());
 259         return true;
 260       }
 261       return false;
 262     }
 263
 264     std::vector<span<RecordElement>> Processed;
 265     std::vector<span<RecordElement>> New;
 266   };
 267
 268   struct UnwindSections {
 269     UnwindSections(const UnwindSectionInfo &USI)
 270         : DwarfSection(USI.DwarfSection.toSpan<char>()),
 271           CompactUnwindSection(USI.CompactUnwindSection.toSpan<char>()) {}
 272
 273     span<char> DwarfSection;
 274     span<char> CompactUnwindSection;
 275   };
 276
 277   using UnwindSectionsMap =
 278       IntervalMap<char *, UnwindSections, IntervalCoalescing::Disabled>;
 279
 280   struct JITDylibState {
 281     std::string Name;
 282     void *Header = nullptr;
 283     bool Sealed = false;
 284     size_t LinkedAgainstRefCount = 0;
 285     size_t DlRefCount = 0;
 286     std::vector<JITDylibState *> Deps;
 287     AtExitsVector AtExits;
 288     const objc_image_info *ObjCImageInfo = nullptr;
 289     std::unordered_map<void *, std::vector<char>> DataSectionContent;
 290     std::unordered_map<void *, size_t> ZeroInitRanges;
 291     UnwindSectionsMap UnwindSections;
 292     RecordSectionsTracker<void (*)()> ModInitsSections;
 293     RecordSectionsTracker<void *> ObjCClassListSections;
 294     RecordSectionsTracker<void *> ObjCSelRefsSections;
 295     RecordSectionsTracker<char> Swift5ProtocolsSections;
 296     RecordSectionsTracker<char> Swift5ProtocolConformancesSections;
 297     RecordSectionsTracker<char> Swift5TypesSections;
 298
 299     bool referenced() const {
 300       return LinkedAgainstRefCount != 0 || DlRefCount != 0;
 301     }
 302   };
 303
 304 public:
 305   static Error create();
 306   static MachOPlatformRuntimeState &get();
 307   static Error destroy();
 308
 309   MachOPlatformRuntimeState() = default;
 310
 311   // Delete copy and move constructors.
 312   MachOPlatformRuntimeState(const MachOPlatformRuntimeState &) = delete;
 313   MachOPlatformRuntimeState &
 314   operator=(const MachOPlatformRuntimeState &) = delete;
 315   MachOPlatformRuntimeState(MachOPlatformRuntimeState &&) = delete;
 316   MachOPlatformRuntimeState &operator=(MachOPlatformRuntimeState &&) = delete;
 317
 318   Error initialize();
 319   Error shutdown();
 320
 321   Error registerJITDylib(std::string Name, void *Header);
 322   Error deregisterJITDylib(void *Header);
 323   Error registerThreadDataSection(span<const char> ThreadDataSection);
 324   Error deregisterThreadDataSection(span<const char> ThreadDataSection);
 325   Error registerObjectPlatformSections(
 326       ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindSections,
 327       std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs);
 328   Error deregisterObjectPlatformSections(
 329       ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindSections,
 330       std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs);
 331
 332   const char *dlerror();
 333   void *dlopen(std::string_view Name, int Mode);
 334   int dlclose(void *DSOHandle);
 335   void *dlsym(void *DSOHandle, std::string_view Symbol);
 336
 337   int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle);
 338   void runAtExits(std::unique_lock<std::mutex> &JDStatesLock,
 339                   JITDylibState &JDS);
 340   void runAtExits(void *DSOHandle);
 341
 342   /// Returns the base address of the section containing ThreadData.
 343   Expected<std::pair<const char *, size_t>>
 344   getThreadDataSectionFor(const char *ThreadData);
 345
 346 private:
 347   JITDylibState *getJITDylibStateByHeader(void *DSOHandle);
 348   JITDylibState *getJITDylibStateByName(std::string_view Path);
 349
 350   Expected<ExecutorAddr> lookupSymbolInJITDylib(void *DSOHandle,
 351                                                 std::string_view Symbol);
 352
 353   bool lookupUnwindSections(void *Addr, unw_dynamic_unwind_sections &Info);
 354
 355   static int findDynamicUnwindSections(uintptr_t addr,
 356                                        unw_dynamic_unwind_sections *info);
 357   static Error registerEHFrames(span<const char> EHFrameSection);
 358   static Error deregisterEHFrames(span<const char> EHFrameSection);
 359
 360   static Error registerObjCSelectors(JITDylibState &JDS);
 361   static Error registerObjCClasses(JITDylibState &JDS);
 362   static Error registerSwift5Protocols(JITDylibState &JDS);
 363   static Error registerSwift5ProtocolConformances(JITDylibState &JDS);
 364   static Error registerSwift5Types(JITDylibState &JDS);
 365   static Error runModInits(std::unique_lock<std::mutex> &JDStatesLock,
 366                            JITDylibState &JDS);
 367
 368   Expected<void *> dlopenImpl(std::string_view Path, int Mode);
 369   Error dlopenFull(std::unique_lock<std::mutex> &JDStatesLock,
 370                    JITDylibState &JDS);
 371   Error dlopenInitialize(std::unique_lock<std::mutex> &JDStatesLock,
 372                          JITDylibState &JDS, MachOJITDylibDepInfoMap &DepInfo);
 373
 374   Error dlcloseImpl(void *DSOHandle);
 375   Error dlcloseDeinitialize(std::unique_lock<std::mutex> &JDStatesLock,
 376                             JITDylibState &JDS);
 377
 378   static MachOPlatformRuntimeState *MOPS;
 379
 380   bool UseCallbackStyleUnwindInfo = false;
 381
 382   // FIXME: Move to thread-state.
 383   std::string DLFcnError;
 384
 385   // APIMutex guards against concurrent entry into key "dyld" API functions
 386   // (e.g. dlopen, dlclose).
 387   std::recursive_mutex DyldAPIMutex;
 388
 389   // JDStatesMutex guards the data structures that hold JITDylib state.
 390   std::mutex JDStatesMutex;
 391   std::unordered_map<void *, JITDylibState> JDStates;
 392   std::unordered_map<std::string_view, void *> JDNameToHeader;
 393
 394   // ThreadDataSectionsMutex guards thread local data section state.
 395   std::mutex ThreadDataSectionsMutex;
 396   std::map<const char *, size_t> ThreadDataSections;
 397 };
 398
 399 MachOPlatformRuntimeState *MachOPlatformRuntimeState::MOPS = nullptr;
 400
 401 Error MachOPlatformRuntimeState::create() {
 402   assert(!MOPS && "MachOPlatformRuntimeState should be null");
 403   MOPS = new MachOPlatformRuntimeState();
 404   return MOPS->initialize();
 405 }
 406
 407 MachOPlatformRuntimeState &MachOPlatformRuntimeState::get() {
 408   assert(MOPS && "MachOPlatformRuntimeState not initialized");
 409   return *MOPS;
 410 }
 411
 412 Error MachOPlatformRuntimeState::destroy() {
 413   assert(MOPS && "MachOPlatformRuntimeState not initialized");
 414   auto Err = MOPS->shutdown();
 415   delete MOPS;
 416   return Err;
 417 }
 418
 419 Error MachOPlatformRuntimeState::initialize() {
 420   UseCallbackStyleUnwindInfo = __unw_add_find_dynamic_unwind_sections &&
 421                                __unw_remove_find_dynamic_unwind_sections;
 422   if (UseCallbackStyleUnwindInfo) {
 423     ORC_RT_DEBUG({
 424       printdbg("__unw_add/remove_find_dynamic_unwind_sections available."
 425                " Using callback-based frame info lookup.\n");
 426     });
 427     if (__unw_add_find_dynamic_unwind_sections(&findDynamicUnwindSections))
 428       return make_error<StringError>(
 429           "Could not register findDynamicUnwindSections");
 430   } else {
 431     ORC_RT_DEBUG({
 432       printdbg("__unw_add/remove_find_dynamic_unwind_sections not available."
 433                " Using classic frame info registration.\n");
 434     });
 435   }
 436   return Error::success();
 437 }
 438
 439 Error MachOPlatformRuntimeState::shutdown() {
 440   if (__unw_add_find_dynamic_unwind_sections &&
 441       __unw_remove_find_dynamic_unwind_sections) {
 442     if (__unw_remove_find_dynamic_unwind_sections(&findDynamicUnwindSections)) {
 443       ORC_RT_DEBUG(
 444           { printdbg("__unw_remove_find_dynamic_unwind_sections failed.\n"); });
 445     }
 446   }
 447   return Error::success();
 448 }
 449
 450 Error MachOPlatformRuntimeState::registerJITDylib(std::string Name,
 451                                                   void *Header) {
 452   ORC_RT_DEBUG({
 453     printdbg("Registering JITDylib %s: Header = %p\n", Name.c_str(), Header);
 454   });
 455   std::lock_guard<std::mutex> Lock(JDStatesMutex);
 456   if (JDStates.count(Header)) {
 457     std::ostringstream ErrStream;
 458     ErrStream << "Duplicate JITDylib registration for header " << Header
 459               << " (name = " << Name << ")";
 460     return make_error<StringError>(ErrStream.str());
 461   }
 462   if (JDNameToHeader.count(Name)) {
 463     std::ostringstream ErrStream;
 464     ErrStream << "Duplicate JITDylib registration for header " << Header
 465               << " (header = " << Header << ")";
 466     return make_error<StringError>(ErrStream.str());
 467   }
 468
 469   auto &JDS = JDStates[Header];
 470   JDS.Name = std::move(Name);
 471   JDS.Header = Header;
 472   JDNameToHeader[JDS.Name] = Header;
 473   return Error::success();
 474 }
 475
 476 Error MachOPlatformRuntimeState::deregisterJITDylib(void *Header) {
 477   std::lock_guard<std::mutex> Lock(JDStatesMutex);
 478   auto I = JDStates.find(Header);
 479   if (I == JDStates.end()) {
 480     std::ostringstream ErrStream;
 481     ErrStream << "Attempted to deregister unrecognized header " << Header;
 482     return make_error<StringError>(ErrStream.str());
 483   }
 484
 485   // Remove std::string construction once we can use C++20.
 486   auto J = JDNameToHeader.find(
 487       std::string(I->second.Name.data(), I->second.Name.size()));
 488   assert(J != JDNameToHeader.end() &&
 489          "Missing JDNameToHeader entry for JITDylib");
 490
 491   ORC_RT_DEBUG({
 492     printdbg("Deregistering JITDylib %s: Header = %p\n", I->second.Name.c_str(),
 493              Header);
 494   });
 495
 496   JDNameToHeader.erase(J);
 497   JDStates.erase(I);
 498   return Error::success();
 499 }
 500
 501 Error MachOPlatformRuntimeState::registerThreadDataSection(
 502     span<const char> ThreadDataSection) {
 503   std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
 504   auto I = ThreadDataSections.upper_bound(ThreadDataSection.data());
 505   if (I != ThreadDataSections.begin()) {
 506     auto J = std::prev(I);
 507     if (J->first + J->second > ThreadDataSection.data())
 508       return make_error<StringError>("Overlapping __thread_data sections");
 509   }
 510   ThreadDataSections.insert(
 511       I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size()));
 512   return Error::success();
 513 }
 514
 515 Error MachOPlatformRuntimeState::deregisterThreadDataSection(
 516     span<const char> ThreadDataSection) {
 517   std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
 518   auto I = ThreadDataSections.find(ThreadDataSection.data());
 519   if (I == ThreadDataSections.end())
 520     return make_error<StringError>("Attempt to deregister unknown thread data "
 521                                    "section");
 522   ThreadDataSections.erase(I);
 523   return Error::success();
 524 }
 525
 526 Error MachOPlatformRuntimeState::registerObjectPlatformSections(
 527     ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindInfo,
 528     std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs) {
 529
 530   // FIXME: Reject platform section registration after the JITDylib is
 531   // sealed?
 532
 533   ORC_RT_DEBUG({
 534     printdbg("MachOPlatform: Registering object sections for %p.\n",
 535              HeaderAddr.toPtr<void *>());
 536   });
 537
 538   std::lock_guard<std::mutex> Lock(JDStatesMutex);
 539   auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>());
 540   if (!JDS) {
 541     std::ostringstream ErrStream;
 542     ErrStream << "Could not register object platform sections for "
 543                  "unrecognized header "
 544               << HeaderAddr.toPtr<void *>();
 545     return make_error<StringError>(ErrStream.str());
 546   }
 547
 548   if (UnwindInfo && UseCallbackStyleUnwindInfo) {
 549     ORC_RT_DEBUG({
 550       printdbg("  Registering new-style unwind info for:\n"
 551                "    DWARF: %p -- %p\n"
 552                "    Compact-unwind: %p -- %p\n"
 553                "  for:\n",
 554                UnwindInfo->DwarfSection.Start.toPtr<void *>(),
 555                UnwindInfo->DwarfSection.End.toPtr<void *>(),
 556                UnwindInfo->CompactUnwindSection.Start.toPtr<void *>(),
 557                UnwindInfo->CompactUnwindSection.End.toPtr<void *>());
 558     });
 559     for (auto &CodeRange : UnwindInfo->CodeRanges) {
 560       JDS->UnwindSections.insert(CodeRange.Start.toPtr<char *>(),
 561                                  CodeRange.End.toPtr<char *>(), *UnwindInfo);
 562       ORC_RT_DEBUG({
 563         printdbg("    [ %p -- %p ]\n", CodeRange.Start.toPtr<void *>(),
 564                  CodeRange.End.toPtr<void *>());
 565       });
 566     }
 567   }
 568
 569   for (auto &KV : Secs) {
 570     // FIXME: Validate section ranges?
 571     if (KV.first == "__TEXT,__eh_frame") {
 572       if (!UseCallbackStyleUnwindInfo) {
 573         // Use classic libunwind registration.
 574         if (auto Err = registerEHFrames(KV.second.toSpan<const char>()))
 575           return Err;
 576       }
 577     } else if (KV.first == "__DATA,__data") {
 578       assert(!JDS->DataSectionContent.count(KV.second.Start.toPtr<char *>()) &&
 579              "Address already registered.");
 580       auto S = KV.second.toSpan<char>();
 581       JDS->DataSectionContent[KV.second.Start.toPtr<char *>()] =
 582           std::vector<char>(S.begin(), S.end());
 583     } else if (KV.first == "__DATA,__common") {
 584       // fprintf(stderr, "Adding zero-init range %llx -- %llx\n",
 585       // KV.second.Start.getValue(), KV.second.size());
 586       JDS->ZeroInitRanges[KV.second.Start.toPtr<char *>()] = KV.second.size();
 587     } else if (KV.first == "__DATA,__thread_data") {
 588       if (auto Err = registerThreadDataSection(KV.second.toSpan<const char>()))
 589         return Err;
 590     } else if (KV.first == "__DATA,__objc_selrefs")
 591       JDS->ObjCSelRefsSections.add(KV.second.toSpan<void *>());
 592     else if (KV.first == "__DATA,__objc_classlist")
 593       JDS->ObjCClassListSections.add(KV.second.toSpan<void *>());
 594     else if (KV.first == "__TEXT,__swift5_protos")
 595       JDS->Swift5ProtocolsSections.add(KV.second.toSpan<char>());
 596     else if (KV.first == "__TEXT,__swift5_proto")
 597       JDS->Swift5ProtocolConformancesSections.add(KV.second.toSpan<char>());
 598     else if (KV.first == "__TEXT,__swift5_types")
 599       JDS->Swift5TypesSections.add(KV.second.toSpan<char>());
 600     else if (KV.first == "__DATA,__mod_init_func")
 601       JDS->ModInitsSections.add(KV.second.toSpan<void (*)()>());
 602     else {
 603       // Should this be a warning instead?
 604       return make_error<StringError>(
 605           "Encountered unexpected section " +
 606           std::string(KV.first.data(), KV.first.size()) +
 607           " while registering object platform sections");
 608     }
 609   }
 610
 611   return Error::success();
 612 }
 613
 614 Error MachOPlatformRuntimeState::deregisterObjectPlatformSections(
 615     ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindInfo,
 616     std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs) {
 617   // TODO: Make this more efficient? (maybe unnecessary if removal is rare?)
 618   // TODO: Add a JITDylib prepare-for-teardown operation that clears all
 619   //       registered sections, causing this function to take the fast-path.
 620   ORC_RT_DEBUG({
 621     printdbg("MachOPlatform: Registering object sections for %p.\n",
 622              HeaderAddr.toPtr<void *>());
 623   });
 624
 625   std::lock_guard<std::mutex> Lock(JDStatesMutex);
 626   auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>());
 627   if (!JDS) {
 628     std::ostringstream ErrStream;
 629     ErrStream << "Could not register object platform sections for unrecognized "
 630                  "header "
 631               << HeaderAddr.toPtr<void *>();
 632     return make_error<StringError>(ErrStream.str());
 633   }
 634
 635   // FIXME: Implement faster-path by returning immediately if JDS is being
 636   // torn down entirely?
 637
 638   // TODO: Make library permanent (i.e. not able to be dlclosed) if it contains
 639   // any Swift or ObjC. Once this happens we can clear (and no longer record)
 640   // data section content, as the library could never be re-initialized.
 641
 642   if (UnwindInfo && UseCallbackStyleUnwindInfo) {
 643     ORC_RT_DEBUG({
 644       printdbg("  Deregistering new-style unwind info for:\n"
 645                "    DWARF: %p -- %p\n"
 646                "    Compact-unwind: %p -- %p\n"
 647                "  for:\n",
 648                UnwindInfo->DwarfSection.Start.toPtr<void *>(),
 649                UnwindInfo->DwarfSection.End.toPtr<void *>(),
 650                UnwindInfo->CompactUnwindSection.Start.toPtr<void *>(),
 651                UnwindInfo->CompactUnwindSection.End.toPtr<void *>());
 652     });
 653     for (auto &CodeRange : UnwindInfo->CodeRanges) {
 654       JDS->UnwindSections.erase(CodeRange.Start.toPtr<char *>(),
 655                                 CodeRange.End.toPtr<char *>());
 656       ORC_RT_DEBUG({
 657         printdbg("    [ %p -- %p ]\n", CodeRange.Start.toPtr<void *>(),
 658                  CodeRange.End.toPtr<void *>());
 659       });
 660     }
 661   }
 662
 663   for (auto &KV : Secs) {
 664     // FIXME: Validate section ranges?
 665     if (KV.first == "__TEXT,__eh_frame") {
 666       if (!UseCallbackStyleUnwindInfo) {
 667         // Use classic libunwind registration.
 668         if (auto Err = deregisterEHFrames(KV.second.toSpan<const char>()))
 669           return Err;
 670       }
 671     } else if (KV.first == "__DATA,__data") {
 672       JDS->DataSectionContent.erase(KV.second.Start.toPtr<char *>());
 673     } else if (KV.first == "__DATA,__common") {
 674       JDS->ZeroInitRanges.erase(KV.second.Start.toPtr<char *>());
 675     } else if (KV.first == "__DATA,__thread_data") {
 676       if (auto Err =
 677               deregisterThreadDataSection(KV.second.toSpan<const char>()))
 678         return Err;
 679     } else if (KV.first == "__DATA,__objc_selrefs")
 680       JDS->ObjCSelRefsSections.removeIfPresent(KV.second);
 681     else if (KV.first == "__DATA,__objc_classlist")
 682       JDS->ObjCClassListSections.removeIfPresent(KV.second);
 683     else if (KV.first == "__TEXT,__swift5_protos")
 684       JDS->Swift5ProtocolsSections.removeIfPresent(KV.second);
 685     else if (KV.first == "__TEXT,__swift5_proto")
 686       JDS->Swift5ProtocolConformancesSections.removeIfPresent(KV.second);
 687     else if (KV.first == "__TEXT,__swift5_types")
 688       JDS->Swift5TypesSections.removeIfPresent(KV.second);
 689     else if (KV.first == "__DATA,__mod_init_func")
 690       JDS->ModInitsSections.removeIfPresent(KV.second);
 691     else {
 692       // Should this be a warning instead?
 693       return make_error<StringError>(
 694           "Encountered unexpected section " +
 695           std::string(KV.first.data(), KV.first.size()) +
 696           " while deregistering object platform sections");
 697     }
 698   }
 699   return Error::success();
 700 }
 701
 702 const char *MachOPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }
 703
 704 void *MachOPlatformRuntimeState::dlopen(std::string_view Path, int Mode) {
 705   ORC_RT_DEBUG({
 706     std::string S(Path.data(), Path.size());
 707     printdbg("MachOPlatform::dlopen(\"%s\")\n", S.c_str());
 708   });
 709   std::lock_guard<std::recursive_mutex> Lock(DyldAPIMutex);
 710   if (auto H = dlopenImpl(Path, Mode))
 711     return *H;
 712   else {
 713     // FIXME: Make dlerror thread safe.
 714     DLFcnError = toString(H.takeError());
 715     return nullptr;
 716   }
 717 }
 718
 719 int MachOPlatformRuntimeState::dlclose(void *DSOHandle) {
 720   ORC_RT_DEBUG({
 721     auto *JDS = getJITDylibStateByHeader(DSOHandle);
 722     std::string DylibName;
 723     if (JDS) {
 724       std::string S;
 725       printdbg("MachOPlatform::dlclose(%p) (%s)\n", DSOHandle, S.c_str());
 726     } else
 727       printdbg("MachOPlatform::dlclose(%p) (%s)\n", DSOHandle,
 728                "invalid handle");
 729   });
 730   std::lock_guard<std::recursive_mutex> Lock(DyldAPIMutex);
 731   if (auto Err = dlcloseImpl(DSOHandle)) {
 732     // FIXME: Make dlerror thread safe.
 733     DLFcnError = toString(std::move(Err));
 734     return -1;
 735   }
 736   return 0;
 737 }
 738
 739 void *MachOPlatformRuntimeState::dlsym(void *DSOHandle,
 740                                        std::string_view Symbol) {
 741   auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol);
 742   if (!Addr) {
 743     DLFcnError = toString(Addr.takeError());
 744     return 0;
 745   }
 746
 747   return Addr->toPtr<void *>();
 748 }
 749
 750 int MachOPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg,
 751                                               void *DSOHandle) {
 752   // FIXME: Handle out-of-memory errors, returning -1 if OOM.
 753   std::lock_guard<std::mutex> Lock(JDStatesMutex);
 754   auto *JDS = getJITDylibStateByHeader(DSOHandle);
 755   if (!JDS) {
 756     ORC_RT_DEBUG({
 757       printdbg("MachOPlatformRuntimeState::registerAtExit called with "
 758                "unrecognized dso handle %p\n",
 759                DSOHandle);
 760     });
 761     return -1;
 762   }
 763   JDS->AtExits.push_back({F, Arg});
 764   return 0;
 765 }
 766
 767 void MachOPlatformRuntimeState::runAtExits(
 768     std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
 769   auto AtExits = std::move(JDS.AtExits);
 770
 771   // Unlock while running atexits, as they may trigger operations that modify
 772   // JDStates.
 773   JDStatesLock.unlock();
 774   while (!AtExits.empty()) {
 775     auto &AE = AtExits.back();
 776     AE.Func(AE.Arg);
 777     AtExits.pop_back();
 778   }
 779   JDStatesLock.lock();
 780 }
 781
 782 void MachOPlatformRuntimeState::runAtExits(void *DSOHandle) {
 783   std::unique_lock<std::mutex> Lock(JDStatesMutex);
 784   auto *JDS = getJITDylibStateByHeader(DSOHandle);
 785   ORC_RT_DEBUG({
 786     printdbg("MachOPlatformRuntimeState::runAtExits called on unrecognized "
 787              "dso_handle %p\n",
 788              DSOHandle);
 789   });
 790   if (JDS)
 791     runAtExits(Lock, *JDS);
 792 }
 793
 794 Expected<std::pair<const char *, size_t>>
 795 MachOPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) {
 796   std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
 797   auto I = ThreadDataSections.upper_bound(ThreadData);
 798   // Check that we have a valid entry covering this address.
 799   if (I == ThreadDataSections.begin())
 800     return make_error<StringError>("No thread local data section for key");
 801   I = std::prev(I);
 802   if (ThreadData >= I->first + I->second)
 803     return make_error<StringError>("No thread local data section for key");
 804   return *I;
 805 }
 806
 807 MachOPlatformRuntimeState::JITDylibState *
 808 MachOPlatformRuntimeState::getJITDylibStateByHeader(void *DSOHandle) {
 809   auto I = JDStates.find(DSOHandle);
 810   if (I == JDStates.end()) {
 811     I = JDStates.insert(std::make_pair(DSOHandle, JITDylibState())).first;
 812     I->second.Header = DSOHandle;
 813   }
 814   return &I->second;
 815 }
 816
 817 MachOPlatformRuntimeState::JITDylibState *
 818 MachOPlatformRuntimeState::getJITDylibStateByName(std::string_view Name) {
 819   // FIXME: Avoid creating string once we have C++20.
 820   auto I = JDNameToHeader.find(std::string(Name.data(), Name.size()));
 821   if (I != JDNameToHeader.end())
 822     return getJITDylibStateByHeader(I->second);
 823   return nullptr;
 824 }
 825
 826 Expected<ExecutorAddr>
 827 MachOPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle,
 828                                                   std::string_view Sym) {
 829   Expected<ExecutorAddr> Result((ExecutorAddr()));
 830   if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddr>(
 831           SPSExecutorAddr, SPSString)>::call(&__orc_rt_macho_symbol_lookup_tag,
 832                                              Result,
 833                                              ExecutorAddr::fromPtr(DSOHandle),
 834                                              Sym))
 835     return std::move(Err);
 836   return Result;
 837 }
 838
 839 // eh-frame registration functions.
 840 // We expect these to be available for all processes.
 841 extern "C" void __register_frame(const void *);
 842 extern "C" void __deregister_frame(const void *);
 843
 844 template <typename HandleFDEFn>
 845 void walkEHFrameSection(span<const char> EHFrameSection,
 846                         HandleFDEFn HandleFDE) {
 847   const char *CurCFIRecord = EHFrameSection.data();
 848   uint64_t Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord);
 849
 850   while (CurCFIRecord != EHFrameSection.end() && Size != 0) {
 851     const char *OffsetField = CurCFIRecord + (Size == 0xffffffff ? 12 : 4);
 852     if (Size == 0xffffffff)
 853       Size = *reinterpret_cast<const uint64_t *>(CurCFIRecord + 4) + 12;
 854     else
 855       Size += 4;
 856     uint32_t Offset = *reinterpret_cast<const uint32_t *>(OffsetField);
 857
 858     if (Offset != 0)
 859       HandleFDE(CurCFIRecord);
 860
 861     CurCFIRecord += Size;
 862     Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord);
 863   }
 864 }
 865
 866 bool MachOPlatformRuntimeState::lookupUnwindSections(
 867     void *Addr, unw_dynamic_unwind_sections &Info) {
 868   ORC_RT_DEBUG(
 869       { printdbg("Tried to lookup unwind-info via new lookup call.\n"); });
 870   std::lock_guard<std::mutex> Lock(JDStatesMutex);
 871   for (auto &KV : JDStates) {
 872     auto &JD = KV.second;
 873     auto I = JD.UnwindSections.find(reinterpret_cast<char *>(Addr));
 874     if (I != JD.UnwindSections.end()) {
 875       Info.dso_base = reinterpret_cast<uintptr_t>(JD.Header);
 876       Info.dwarf_section =
 877           reinterpret_cast<uintptr_t>(I->second.DwarfSection.data());
 878       Info.dwarf_section_length = I->second.DwarfSection.size();
 879       Info.compact_unwind_section =
 880           reinterpret_cast<uintptr_t>(I->second.CompactUnwindSection.data());
 881       Info.compact_unwind_section_length =
 882           I->second.CompactUnwindSection.size();
 883       return true;
 884     }
 885   }
 886   return false;
 887 }
 888
 889 int MachOPlatformRuntimeState::findDynamicUnwindSections(
 890     uintptr_t addr, unw_dynamic_unwind_sections *info) {
 891   if (!info)
 892     return 0;
 893   return MachOPlatformRuntimeState::get().lookupUnwindSections((void *)addr,
 894                                                                *info);
 895 }
 896
 897 Error MachOPlatformRuntimeState::registerEHFrames(
 898     span<const char> EHFrameSection) {
 899   walkEHFrameSection(EHFrameSection, __register_frame);
 900   return Error::success();
 901 }
 902
 903 Error MachOPlatformRuntimeState::deregisterEHFrames(
 904     span<const char> EHFrameSection) {
 905   walkEHFrameSection(EHFrameSection, __deregister_frame);
 906   return Error::success();
 907 }
 908
 909 Error MachOPlatformRuntimeState::registerObjCSelectors(JITDylibState &JDS) {
 910   if (!JDS.ObjCSelRefsSections.hasNewSections())
 911     return Error::success();
 912
 913   if (ORC_RT_UNLIKELY(!sel_registerName))
 914     return make_error<StringError>("sel_registerName is not available");
 915
 916   JDS.ObjCSelRefsSections.processNewSections([](span<void *> SelRefs) {
 917     for (void *&SelEntry : SelRefs) {
 918       const char *SelName = reinterpret_cast<const char *>(SelEntry);
 919       auto Sel = sel_registerName(SelName);
 920       *reinterpret_cast<SEL *>(&SelEntry) = Sel;
 921     }
 922   });
 923
 924   return Error::success();
 925 }
 926
 927 Error MachOPlatformRuntimeState::registerObjCClasses(JITDylibState &JDS) {
 928   if (!JDS.ObjCClassListSections.hasNewSections())
 929     return Error::success();
 930
 931   if (ORC_RT_UNLIKELY(!objc_msgSend))
 932     return make_error<StringError>("objc_msgSend is not available");
 933   if (ORC_RT_UNLIKELY(!objc_readClassPair))
 934     return make_error<StringError>("objc_readClassPair is not available");
 935
 936   struct ObjCClassCompiled {
 937     void *Metaclass;
 938     void *Parent;
 939     void *Cache1;
 940     void *Cache2;
 941     void *Data;
 942   };
 943
 944   auto ClassSelector = sel_registerName("class");
 945
 946   return JDS.ObjCClassListSections.processNewSections(
 947       [&](span<void *> ClassPtrs) -> Error {
 948         for (void *ClassPtr : ClassPtrs) {
 949           auto *Cls = reinterpret_cast<Class>(ClassPtr);
 950           auto *ClassCompiled = reinterpret_cast<ObjCClassCompiled *>(ClassPtr);
 951           objc_msgSend(reinterpret_cast<id>(ClassCompiled->Parent),
 952                        ClassSelector);
 953           auto Registered = objc_readClassPair(Cls, JDS.ObjCImageInfo);
 954           // FIXME: Improve diagnostic by reporting the failed class's name.
 955           if (Registered != Cls)
 956             return make_error<StringError>(
 957                 "Unable to register Objective-C class");
 958         }
 959         return Error::success();
 960       });
 961 }
 962
 963 Error MachOPlatformRuntimeState::registerSwift5Protocols(JITDylibState &JDS) {
 964
 965   if (!JDS.Swift5ProtocolsSections.hasNewSections())
 966     return Error::success();
 967
 968   if (ORC_RT_UNLIKELY(!swift_registerProtocols))
 969     return make_error<StringError>("swift_registerProtocols is not available");
 970
 971   JDS.Swift5ProtocolsSections.processNewSections([](span<char> ProtoSec) {
 972     swift_registerProtocols(
 973         reinterpret_cast<const ProtocolRecord *>(ProtoSec.data()),
 974         reinterpret_cast<const ProtocolRecord *>(ProtoSec.data() +
 975                                                  ProtoSec.size()));
 976   });
 977
 978   return Error::success();
 979 }
 980
 981 Error MachOPlatformRuntimeState::registerSwift5ProtocolConformances(
 982     JITDylibState &JDS) {
 983
 984   if (!JDS.Swift5ProtocolConformancesSections.hasNewSections())
 985     return Error::success();
 986
 987   if (ORC_RT_UNLIKELY(!swift_registerProtocolConformances))
 988     return make_error<StringError>(
 989         "swift_registerProtocolConformances is not available");
 990
 991   JDS.Swift5ProtocolConformancesSections.processNewSections(
 992       [](span<char> ProtoConfSec) {
 993         swift_registerProtocolConformances(
 994             reinterpret_cast<const ProtocolConformanceRecord *>(
 995                 ProtoConfSec.data()),
 996             reinterpret_cast<const ProtocolConformanceRecord *>(
 997                 ProtoConfSec.data() + ProtoConfSec.size()));
 998       });
 999
1000   return Error::success();
1001 }
1002
1003 Error MachOPlatformRuntimeState::registerSwift5Types(JITDylibState &JDS) {
1004
1005   if (!JDS.Swift5TypesSections.hasNewSections())
1006     return Error::success();
1007
1008   if (ORC_RT_UNLIKELY(!swift_registerTypeMetadataRecords))
1009     return make_error<StringError>(
1010         "swift_registerTypeMetadataRecords is not available");
1011
1012   JDS.Swift5TypesSections.processNewSections([&](span<char> TypesSec) {
1013     swift_registerTypeMetadataRecords(
1014         reinterpret_cast<const TypeMetadataRecord *>(TypesSec.data()),
1015         reinterpret_cast<const TypeMetadataRecord *>(TypesSec.data() +
1016                                                      TypesSec.size()));
1017   });
1018
1019   return Error::success();
1020 }
1021
1022 Error MachOPlatformRuntimeState::runModInits(
1023     std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
1024   std::vector<span<void (*)()>> InitSections;
1025   InitSections.reserve(JDS.ModInitsSections.numNewSections());
1026
1027   // Copy initializer sections: If the JITDylib is unsealed then the
1028   // initializers could reach back into the JIT and cause more initializers to
1029   // be added.
1030   // FIXME: Skip unlock and run in-place on sealed JITDylibs?
1031   JDS.ModInitsSections.processNewSections(
1032       [&](span<void (*)()> Inits) { InitSections.push_back(Inits); });
1033
1034   JDStatesLock.unlock();
1035   for (auto InitSec : InitSections)
1036     for (auto *Init : InitSec)
1037       Init();
1038   JDStatesLock.lock();
1039
1040   return Error::success();
1041 }
1042
1043 Expected<void *> MachOPlatformRuntimeState::dlopenImpl(std::string_view Path,
1044                                                        int Mode) {
1045   std::unique_lock<std::mutex> Lock(JDStatesMutex);
1046
1047   // Try to find JITDylib state by name.
1048   auto *JDS = getJITDylibStateByName(Path);
1049
1050   if (!JDS)
1051     return make_error<StringError>("No registered JTIDylib for path " +
1052                                    std::string(Path.data(), Path.size()));
1053
1054   // If this JITDylib is unsealed, or this is the first dlopen then run
1055   // full dlopen path (update deps, push and run initializers, update ref
1056   // counts on all JITDylibs in the dep tree).
1057   if (!JDS->referenced() || !JDS->Sealed) {
1058     if (auto Err = dlopenFull(Lock, *JDS))
1059       return std::move(Err);
1060   }
1061
1062   // Bump the ref-count on this dylib.
1063   ++JDS->DlRefCount;
1064
1065   // Return the header address.
1066   return JDS->Header;
1067 }
1068
1069 Error MachOPlatformRuntimeState::dlopenFull(
1070     std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
1071   // Call back to the JIT to push the initializers.
1072   Expected<MachOJITDylibDepInfoMap> DepInfo((MachOJITDylibDepInfoMap()));
1073   // Unlock so that we can accept the initializer update.
1074   JDStatesLock.unlock();
1075   if (auto Err = WrapperFunction<SPSExpected<SPSMachOJITDylibDepInfoMap>(
1076           SPSExecutorAddr)>::call(&__orc_rt_macho_push_initializers_tag,
1077                                   DepInfo, ExecutorAddr::fromPtr(JDS.Header)))
1078     return Err;
1079   JDStatesLock.lock();
1080
1081   if (!DepInfo)
1082     return DepInfo.takeError();
1083
1084   if (auto Err = dlopenInitialize(JDStatesLock, JDS, *DepInfo))
1085     return Err;
1086
1087   if (!DepInfo->empty()) {
1088     ORC_RT_DEBUG({
1089       printdbg("Unrecognized dep-info key headers in dlopen of %s\n",
1090                JDS.Name.c_str());
1091     });
1092     std::ostringstream ErrStream;
1093     ErrStream << "Encountered unrecognized dep-info key headers "
1094                  "while processing dlopen of "
1095               << JDS.Name;
1096     return make_error<StringError>(ErrStream.str());
1097   }
1098
1099   return Error::success();
1100 }
1101
1102 Error MachOPlatformRuntimeState::dlopenInitialize(
1103     std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS,
1104     MachOJITDylibDepInfoMap &DepInfo) {
1105   ORC_RT_DEBUG({
1106     printdbg("MachOPlatformRuntimeState::dlopenInitialize(\"%s\")\n",
1107              JDS.Name.c_str());
1108   });
1109
1110   // If the header is not present in the dep map then assume that we
1111   // already processed it earlier in the dlopenInitialize traversal and
1112   // return.
1113   // TODO: Keep a visited set instead so that we can error out on missing
1114   //       entries?
1115   auto I = DepInfo.find(ExecutorAddr::fromPtr(JDS.Header));
1116   if (I == DepInfo.end())
1117     return Error::success();
1118
1119   auto DI = std::move(I->second);
1120   DepInfo.erase(I);
1121
1122   // We don't need to re-initialize sealed JITDylibs that have already been
1123   // initialized. Just check that their dep-map entry is empty as expected.
1124   if (JDS.Sealed) {
1125     if (!DI.DepHeaders.empty()) {
1126       std::ostringstream ErrStream;
1127       ErrStream << "Sealed JITDylib " << JDS.Header
1128                 << " already has registered dependencies";
1129       return make_error<StringError>(ErrStream.str());
1130     }
1131     if (JDS.referenced())
1132       return Error::success();
1133   } else
1134     JDS.Sealed = DI.Sealed;
1135
1136   // This is an unsealed or newly sealed JITDylib. Run initializers.
1137   std::vector<JITDylibState *> OldDeps;
1138   std::swap(JDS.Deps, OldDeps);
1139   JDS.Deps.reserve(DI.DepHeaders.size());
1140   for (auto DepHeaderAddr : DI.DepHeaders) {
1141     auto *DepJDS = getJITDylibStateByHeader(DepHeaderAddr.toPtr<void *>());
1142     if (!DepJDS) {
1143       std::ostringstream ErrStream;
1144       ErrStream << "Encountered unrecognized dep header "
1145                 << DepHeaderAddr.toPtr<void *>() << " while initializing "
1146                 << JDS.Name;
1147       return make_error<StringError>(ErrStream.str());
1148     }
1149     ++DepJDS->LinkedAgainstRefCount;
1150     if (auto Err = dlopenInitialize(JDStatesLock, *DepJDS, DepInfo))
1151       return Err;
1152   }
1153
1154   // Initialize this JITDylib.
1155   if (auto Err = registerObjCSelectors(JDS))
1156     return Err;
1157   if (auto Err = registerObjCClasses(JDS))
1158     return Err;
1159   if (auto Err = registerSwift5Protocols(JDS))
1160     return Err;
1161   if (auto Err = registerSwift5ProtocolConformances(JDS))
1162     return Err;
1163   if (auto Err = registerSwift5Types(JDS))
1164     return Err;
1165   if (auto Err = runModInits(JDStatesLock, JDS))
1166     return Err;
1167
1168   // Decrement old deps.
1169   // FIXME: We should probably continue and just report deinitialize errors
1170   // here.
1171   for (auto *DepJDS : OldDeps) {
1172     --DepJDS->LinkedAgainstRefCount;
1173     if (!DepJDS->referenced())
1174       if (auto Err = dlcloseDeinitialize(JDStatesLock, *DepJDS))
1175         return Err;
1176   }
1177
1178   return Error::success();
1179 }
1180
1181 Error MachOPlatformRuntimeState::dlcloseImpl(void *DSOHandle) {
1182   std::unique_lock<std::mutex> Lock(JDStatesMutex);
1183
1184   // Try to find JITDylib state by header.
1185   auto *JDS = getJITDylibStateByHeader(DSOHandle);
1186
1187   if (!JDS) {
1188     std::ostringstream ErrStream;
1189     ErrStream << "No registered JITDylib for " << DSOHandle;
1190     return make_error<StringError>(ErrStream.str());
1191   }
1192
1193   // Bump the ref-count.
1194   --JDS->DlRefCount;
1195
1196   if (!JDS->referenced())
1197     return dlcloseDeinitialize(Lock, *JDS);
1198
1199   return Error::success();
1200 }
1201
1202 Error MachOPlatformRuntimeState::dlcloseDeinitialize(
1203     std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
1204
1205   ORC_RT_DEBUG({
1206     printdbg("MachOPlatformRuntimeState::dlcloseDeinitialize(\"%s\")\n",
1207              JDS.Name.c_str());
1208   });
1209
1210   runAtExits(JDStatesLock, JDS);
1211
1212   // Reset mod-inits
1213   JDS.ModInitsSections.reset();
1214
1215   // Reset data section contents.
1216   for (auto &KV : JDS.DataSectionContent)
1217     memcpy(KV.first, KV.second.data(), KV.second.size());
1218   for (auto &KV : JDS.ZeroInitRanges)
1219     memset(KV.first, 0, KV.second);
1220
1221   // Deinitialize any dependencies.
1222   for (auto *DepJDS : JDS.Deps) {
1223     --DepJDS->LinkedAgainstRefCount;
1224     if (!DepJDS->referenced())
1225       if (auto Err = dlcloseDeinitialize(JDStatesLock, *DepJDS))
1226         return Err;
1227   }
1228
1229   return Error::success();
1230 }
1231
1232 class MachOPlatformRuntimeTLVManager {
1233 public:
1234   void *getInstance(const char *ThreadData);
1235
1236 private:
1237   std::unordered_map<const char *, char *> Instances;
1238   std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections;
1239 };
1240
1241 void *MachOPlatformRuntimeTLVManager::getInstance(const char *ThreadData) {
1242   auto I = Instances.find(ThreadData);
1243   if (I != Instances.end())
1244     return I->second;
1245
1246   auto TDS =
1247       MachOPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData);
1248   if (!TDS) {
1249     __orc_rt_log_error(toString(TDS.takeError()).c_str());
1250     return nullptr;
1251   }
1252
1253   auto &Allocated = AllocatedSections[TDS->first];
1254   if (!Allocated) {
1255     Allocated = std::make_unique<char[]>(TDS->second);
1256     memcpy(Allocated.get(), TDS->first, TDS->second);
1257   }
1258
1259   size_t ThreadDataDelta = ThreadData - TDS->first;
1260   assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds");
1261
1262   char *Instance = Allocated.get() + ThreadDataDelta;
1263   Instances[ThreadData] = Instance;
1264   return Instance;
1265 }
1266
1267 void destroyMachOTLVMgr(void *MachOTLVMgr) {
1268   delete static_cast<MachOPlatformRuntimeTLVManager *>(MachOTLVMgr);
1269 }
1270
1271 Error runWrapperFunctionCalls(std::vector<WrapperFunctionCall> WFCs) {
1272   for (auto &WFC : WFCs)
1273     if (auto Err = WFC.runWithSPSRet<void>())
1274       return Err;
1275   return Error::success();
1276 }
1277
1278 } // end anonymous namespace
1279
1280 //------------------------------------------------------------------------------
1281 //                             JIT entry points
1282 //------------------------------------------------------------------------------
1283
1284 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
1285 __orc_rt_macho_platform_bootstrap(char *ArgData, size_t ArgSize) {
1286   return WrapperFunction<SPSError()>::handle(
1287              ArgData, ArgSize,
1288              []() { return MachOPlatformRuntimeState::create(); })
1289       .release();
1290 }
1291
1292 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
1293 __orc_rt_macho_platform_shutdown(char *ArgData, size_t ArgSize) {
1294   return WrapperFunction<SPSError()>::handle(
1295              ArgData, ArgSize,
1296              []() { return MachOPlatformRuntimeState::destroy(); })
1297       .release();
1298 }
1299
1300 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
1301 __orc_rt_macho_register_jitdylib(char *ArgData, size_t ArgSize) {
1302   return WrapperFunction<SPSError(SPSString, SPSExecutorAddr)>::handle(
1303              ArgData, ArgSize,
1304              [](std::string &Name, ExecutorAddr HeaderAddr) {
1305                return MachOPlatformRuntimeState::get().registerJITDylib(
1306                    std::move(Name), HeaderAddr.toPtr<void *>());
1307              })
1308       .release();
1309 }
1310
1311 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
1312 __orc_rt_macho_deregister_jitdylib(char *ArgData, size_t ArgSize) {
1313   return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
1314              ArgData, ArgSize,
1315              [](ExecutorAddr HeaderAddr) {
1316                return MachOPlatformRuntimeState::get().deregisterJITDylib(
1317                    HeaderAddr.toPtr<void *>());
1318              })
1319       .release();
1320 }
1321
1322 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
1323 __orc_rt_macho_register_object_platform_sections(char *ArgData,
1324                                                  size_t ArgSize) {
1325   return WrapperFunction<SPSError(SPSExecutorAddr,
1326                                   SPSOptional<SPSUnwindSectionInfo>,
1327                                   SPSMachOObjectPlatformSectionsMap)>::
1328       handle(ArgData, ArgSize,
1329              [](ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> USI,
1330                 std::vector<std::pair<std::string_view, ExecutorAddrRange>>
1331                     &Secs) {
1332                return MachOPlatformRuntimeState::get()
1333                    .registerObjectPlatformSections(HeaderAddr, std::move(USI),
1334                                                    std::move(Secs));
1335              })
1336           .release();
1337 }
1338
1339 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
1340 __orc_rt_macho_deregister_object_platform_sections(char *ArgData,
1341                                                    size_t ArgSize) {
1342   return WrapperFunction<SPSError(SPSExecutorAddr,
1343                                   SPSOptional<SPSUnwindSectionInfo>,
1344                                   SPSMachOObjectPlatformSectionsMap)>::
1345       handle(ArgData, ArgSize,
1346              [](ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> USI,
1347                 std::vector<std::pair<std::string_view, ExecutorAddrRange>>
1348                     &Secs) {
1349                return MachOPlatformRuntimeState::get()
1350                    .deregisterObjectPlatformSections(HeaderAddr, std::move(USI),
1351                                                      std::move(Secs));
1352              })
1353           .release();
1354 }
1355
1356 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
1357 __orc_rt_macho_run_wrapper_function_calls(char *ArgData, size_t ArgSize) {
1358   return WrapperFunction<SPSError(SPSSequence<SPSWrapperFunctionCall>)>::handle(
1359              ArgData, ArgSize, runWrapperFunctionCalls)
1360       .release();
1361 }
1362
1363 //------------------------------------------------------------------------------
1364 //                            TLV support
1365 //------------------------------------------------------------------------------
1366
1367 ORC_RT_INTERFACE void *__orc_rt_macho_tlv_get_addr_impl(TLVDescriptor *D) {
1368   auto *TLVMgr = static_cast<MachOPlatformRuntimeTLVManager *>(
1369       pthread_getspecific(D->Key));
1370   if (!TLVMgr) {
1371     TLVMgr = new MachOPlatformRuntimeTLVManager();
1372     if (pthread_setspecific(D->Key, TLVMgr)) {
1373       __orc_rt_log_error("Call to pthread_setspecific failed");
1374       return nullptr;
1375     }
1376   }
1377
1378   return TLVMgr->getInstance(
1379       reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress)));
1380 }
1381
1382 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
1383 __orc_rt_macho_create_pthread_key(char *ArgData, size_t ArgSize) {
1384   return WrapperFunction<SPSExpected<uint64_t>(void)>::handle(
1385              ArgData, ArgSize,
1386              []() -> Expected<uint64_t> {
1387                pthread_key_t Key;
1388                if (int Err = pthread_key_create(&Key, destroyMachOTLVMgr)) {
1389                  __orc_rt_log_error("Call to pthread_key_create failed");
1390                  return make_error<StringError>(strerror(Err));
1391                }
1392                return static_cast<uint64_t>(Key);
1393              })
1394       .release();
1395 }
1396
1397 //------------------------------------------------------------------------------
1398 //                           cxa_atexit support
1399 //------------------------------------------------------------------------------
1400
1401 int __orc_rt_macho_cxa_atexit(void (*func)(void *), void *arg,
1402                               void *dso_handle) {
1403   return MachOPlatformRuntimeState::get().registerAtExit(func, arg, dso_handle);
1404 }
1405
1406 void __orc_rt_macho_cxa_finalize(void *dso_handle) {
1407   MachOPlatformRuntimeState::get().runAtExits(dso_handle);
1408 }
1409
1410 //------------------------------------------------------------------------------
1411 //                        JIT'd dlfcn alternatives.
1412 //------------------------------------------------------------------------------
1413
1414 const char *__orc_rt_macho_jit_dlerror() {
1415   return MachOPlatformRuntimeState::get().dlerror();
1416 }
1417
1418 void *__orc_rt_macho_jit_dlopen(const char *path, int mode) {
1419   return MachOPlatformRuntimeState::get().dlopen(path, mode);
1420 }
1421
1422 int __orc_rt_macho_jit_dlclose(void *dso_handle) {
1423   return MachOPlatformRuntimeState::get().dlclose(dso_handle);
1424 }
1425
1426 void *__orc_rt_macho_jit_dlsym(void *dso_handle, const char *symbol) {
1427   return MachOPlatformRuntimeState::get().dlsym(dso_handle, symbol);
1428 }
1429
1430 //------------------------------------------------------------------------------
1431 //                             MachO Run Program
1432 //------------------------------------------------------------------------------
1433
1434 ORC_RT_INTERFACE int64_t __orc_rt_macho_run_program(const char *JITDylibName,
1435                                                     const char *EntrySymbolName,
1436                                                     int argc, char *argv[]) {
1437   using MainTy = int (*)(int, char *[]);
1438
1439   void *H = __orc_rt_macho_jit_dlopen(JITDylibName,
1440                                       __orc_rt::macho::ORC_RT_RTLD_LAZY);
1441   if (!H) {
1442     __orc_rt_log_error(__orc_rt_macho_jit_dlerror());
1443     return -1;
1444   }
1445
1446   auto *Main =
1447       reinterpret_cast<MainTy>(__orc_rt_macho_jit_dlsym(H, EntrySymbolName));
1448
1449   if (!Main) {
1450     __orc_rt_log_error(__orc_rt_macho_jit_dlerror());
1451     return -1;
1452   }
1453
1454   int Result = Main(argc, argv);
1455
1456   if (__orc_rt_macho_jit_dlclose(H) == -1)
1457     __orc_rt_log_error(__orc_rt_macho_jit_dlerror());
1458
1459   return Result;
1460 }