llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp

   1 //===------ MachOPlatform.cpp - Utilities for executing MachO in Orc ------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
  10
  11 #include "llvm/BinaryFormat/MachO.h"
  12 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
  13 #include "llvm/ExecutionEngine/Orc/DebugUtils.h"
  14 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
  15 #include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h"
  16 #include "llvm/Support/BinaryByteStream.h"
  17 #include "llvm/Support/Debug.h"
  18
  19 #define DEBUG_TYPE "orc"
  20
  21 using namespace llvm;
  22 using namespace llvm::orc;
  23 using namespace llvm::orc::shared;
  24
  25 namespace {
  26
  27 class MachOHeaderMaterializationUnit : public MaterializationUnit {
  28 public:
  29   MachOHeaderMaterializationUnit(MachOPlatform &MOP,
  30                                  const SymbolStringPtr &HeaderStartSymbol)
  31       : MaterializationUnit(createHeaderSymbols(MOP, HeaderStartSymbol),
  32                             HeaderStartSymbol),
  33         MOP(MOP) {}
  34
  35   StringRef getName() const override { return "MachOHeaderMU"; }
  36
  37   void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
  38     unsigned PointerSize;
  39     support::endianness Endianness;
  40     const auto &TT =
  41         MOP.getExecutionSession().getExecutorProcessControl().getTargetTriple();
  42
  43     switch (TT.getArch()) {
  44     case Triple::aarch64:
  45     case Triple::x86_64:
  46       PointerSize = 8;
  47       Endianness = support::endianness::little;
  48       break;
  49     default:
  50       llvm_unreachable("Unrecognized architecture");
  51     }
  52
  53     auto G = std::make_unique<jitlink::LinkGraph>(
  54         "<MachOHeaderMU>", TT, PointerSize, Endianness,
  55         jitlink::getGenericEdgeKindName);
  56     auto &HeaderSection = G->createSection("__header", sys::Memory::MF_READ);
  57     auto &HeaderBlock = createHeaderBlock(*G, HeaderSection);
  58
  59     // Init symbol is header-start symbol.
  60     G->addDefinedSymbol(HeaderBlock, 0, *R->getInitializerSymbol(),
  61                         HeaderBlock.getSize(), jitlink::Linkage::Strong,
  62                         jitlink::Scope::Default, false, true);
  63     for (auto &HS : AdditionalHeaderSymbols)
  64       G->addDefinedSymbol(HeaderBlock, HS.Offset, HS.Name,
  65                           HeaderBlock.getSize(), jitlink::Linkage::Strong,
  66                           jitlink::Scope::Default, false, true);
  67
  68     MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
  69   }
  70
  71   void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
  72
  73 private:
  74   struct HeaderSymbol {
  75     const char *Name;
  76     uint64_t Offset;
  77   };
  78
  79   static constexpr HeaderSymbol AdditionalHeaderSymbols[] = {
  80       {"___mh_executable_header", 0}};
  81
  82   static jitlink::Block &createHeaderBlock(jitlink::LinkGraph &G,
  83                                            jitlink::Section &HeaderSection) {
  84     MachO::mach_header_64 Hdr;
  85     Hdr.magic = MachO::MH_MAGIC_64;
  86     switch (G.getTargetTriple().getArch()) {
  87     case Triple::aarch64:
  88       Hdr.cputype = MachO::CPU_TYPE_ARM64;
  89       Hdr.cpusubtype = MachO::CPU_SUBTYPE_ARM64_ALL;
  90       break;
  91     case Triple::x86_64:
  92       Hdr.cputype = MachO::CPU_TYPE_X86_64;
  93       Hdr.cpusubtype = MachO::CPU_SUBTYPE_X86_64_ALL;
  94       break;
  95     default:
  96       llvm_unreachable("Unrecognized architecture");
  97     }
  98     Hdr.filetype = MachO::MH_DYLIB; // Custom file type?
  99     Hdr.ncmds = 0;
 100     Hdr.sizeofcmds = 0;
 101     Hdr.flags = 0;
 102     Hdr.reserved = 0;
 103
 104     if (G.getEndianness() != support::endian::system_endianness())
 105       MachO::swapStruct(Hdr);
 106
 107     auto HeaderContent = G.allocateString(
 108         StringRef(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr)));
 109
 110     return G.createContentBlock(HeaderSection, HeaderContent, 0, 8, 0);
 111   }
 112
 113   static SymbolFlagsMap
 114   createHeaderSymbols(MachOPlatform &MOP,
 115                       const SymbolStringPtr &HeaderStartSymbol) {
 116     SymbolFlagsMap HeaderSymbolFlags;
 117
 118     HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported;
 119     for (auto &HS : AdditionalHeaderSymbols)
 120       HeaderSymbolFlags[MOP.getExecutionSession().intern(HS.Name)] =
 121           JITSymbolFlags::Exported;
 122
 123     return HeaderSymbolFlags;
 124   }
 125
 126   MachOPlatform &MOP;
 127 };
 128
 129 constexpr MachOHeaderMaterializationUnit::HeaderSymbol
 130     MachOHeaderMaterializationUnit::AdditionalHeaderSymbols[];
 131
 132 StringRef EHFrameSectionName = "__TEXT,__eh_frame";
 133 StringRef ModInitFuncSectionName = "__DATA,__mod_init_func";
 134 StringRef ObjCClassListSectionName = "__DATA,__objc_classlist";
 135 StringRef ObjCImageInfoSectionName = "__DATA,__objc_image_info";
 136 StringRef ObjCSelRefsSectionName = "__DATA,__objc_selrefs";
 137 StringRef Swift5ProtoSectionName = "__TEXT,__swift5_proto";
 138 StringRef Swift5ProtosSectionName = "__TEXT,__swift5_protos";
 139 StringRef ThreadBSSSectionName = "__DATA,__thread_bss";
 140 StringRef ThreadDataSectionName = "__DATA,__thread_data";
 141 StringRef ThreadVarsSectionName = "__DATA,__thread_vars";
 142
 143 StringRef InitSectionNames[] = {
 144     ModInitFuncSectionName, ObjCSelRefsSectionName, ObjCClassListSectionName,
 145     Swift5ProtosSectionName, Swift5ProtoSectionName};
 146
 147 } // end anonymous namespace
 148
 149 namespace llvm {
 150 namespace orc {
 151
 152 Expected<std::unique_ptr<MachOPlatform>>
 153 MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
 154                       JITDylib &PlatformJD, const char *OrcRuntimePath,
 155                       Optional<SymbolAliasMap> RuntimeAliases) {
 156
 157   auto &EPC = ES.getExecutorProcessControl();
 158
 159   // If the target is not supported then bail out immediately.
 160   if (!supportedTarget(EPC.getTargetTriple()))
 161     return make_error<StringError>("Unsupported MachOPlatform triple: " +
 162                                        EPC.getTargetTriple().str(),
 163                                    inconvertibleErrorCode());
 164
 165   // Create default aliases if the caller didn't supply any.
 166   if (!RuntimeAliases)
 167     RuntimeAliases = standardPlatformAliases(ES);
 168
 169   // Define the aliases.
 170   if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases))))
 171     return std::move(Err);
 172
 173   // Add JIT-dispatch function support symbols.
 174   if (auto Err = PlatformJD.define(absoluteSymbols(
 175           {{ES.intern("___orc_rt_jit_dispatch"),
 176             {EPC.getJITDispatchInfo().JITDispatchFunctionAddress.getValue(),
 177              JITSymbolFlags::Exported}},
 178            {ES.intern("___orc_rt_jit_dispatch_ctx"),
 179             {EPC.getJITDispatchInfo().JITDispatchContextAddress.getValue(),
 180              JITSymbolFlags::Exported}}})))
 181     return std::move(Err);
 182
 183   // Create a generator for the ORC runtime archive.
 184   auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Load(
 185       ObjLinkingLayer, OrcRuntimePath, EPC.getTargetTriple());
 186   if (!OrcRuntimeArchiveGenerator)
 187     return OrcRuntimeArchiveGenerator.takeError();
 188
 189   // Create the instance.
 190   Error Err = Error::success();
 191   auto P = std::unique_ptr<MachOPlatform>(
 192       new MachOPlatform(ES, ObjLinkingLayer, PlatformJD,
 193                         std::move(*OrcRuntimeArchiveGenerator), Err));
 194   if (Err)
 195     return std::move(Err);
 196   return std::move(P);
 197 }
 198
 199 Error MachOPlatform::setupJITDylib(JITDylib &JD) {
 200   return JD.define(std::make_unique<MachOHeaderMaterializationUnit>(
 201       *this, MachOHeaderStartSymbol));
 202 }
 203
 204 Error MachOPlatform::notifyAdding(ResourceTracker &RT,
 205                                   const MaterializationUnit &MU) {
 206   auto &JD = RT.getJITDylib();
 207   const auto &InitSym = MU.getInitializerSymbol();
 208   if (!InitSym)
 209     return Error::success();
 210
 211   RegisteredInitSymbols[&JD].add(InitSym,
 212                                  SymbolLookupFlags::WeaklyReferencedSymbol);
 213   LLVM_DEBUG({
 214     dbgs() << "MachOPlatform: Registered init symbol " << *InitSym << " for MU "
 215            << MU.getName() << "\n";
 216   });
 217   return Error::success();
 218 }
 219
 220 Error MachOPlatform::notifyRemoving(ResourceTracker &RT) {
 221   llvm_unreachable("Not supported yet");
 222 }
 223
 224 static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
 225                        ArrayRef<std::pair<const char *, const char *>> AL) {
 226   for (auto &KV : AL) {
 227     auto AliasName = ES.intern(KV.first);
 228     assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
 229     Aliases[std::move(AliasName)] = {ES.intern(KV.second),
 230                                      JITSymbolFlags::Exported};
 231   }
 232 }
 233
 234 SymbolAliasMap MachOPlatform::standardPlatformAliases(ExecutionSession &ES) {
 235   SymbolAliasMap Aliases;
 236   addAliases(ES, Aliases, requiredCXXAliases());
 237   addAliases(ES, Aliases, standardRuntimeUtilityAliases());
 238   return Aliases;
 239 }
 240
 241 ArrayRef<std::pair<const char *, const char *>>
 242 MachOPlatform::requiredCXXAliases() {
 243   static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
 244       {"___cxa_atexit", "___orc_rt_macho_cxa_atexit"}};
 245
 246   return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
 247 }
 248
 249 ArrayRef<std::pair<const char *, const char *>>
 250 MachOPlatform::standardRuntimeUtilityAliases() {
 251   static const std::pair<const char *, const char *>
 252       StandardRuntimeUtilityAliases[] = {
 253           {"___orc_rt_run_program", "___orc_rt_macho_run_program"},
 254           {"___orc_rt_log_error", "___orc_rt_log_error_to_stderr"}};
 255
 256   return ArrayRef<std::pair<const char *, const char *>>(
 257       StandardRuntimeUtilityAliases);
 258 }
 259
 260 bool MachOPlatform::isInitializerSection(StringRef SegName,
 261                                          StringRef SectName) {
 262   for (auto &Name : InitSectionNames) {
 263     if (Name.startswith(SegName) && Name.substr(7) == SectName)
 264       return true;
 265   }
 266   return false;
 267 }
 268
 269 bool MachOPlatform::supportedTarget(const Triple &TT) {
 270   switch (TT.getArch()) {
 271   case Triple::x86_64:
 272     return true;
 273   default:
 274     return false;
 275   }
 276 }
 277
 278 MachOPlatform::MachOPlatform(
 279     ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
 280     JITDylib &PlatformJD,
 281     std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err)
 282     : ES(ES), ObjLinkingLayer(ObjLinkingLayer),
 283       MachOHeaderStartSymbol(ES.intern("___dso_handle")) {
 284   ErrorAsOutParameter _(&Err);
 285
 286   ObjLinkingLayer.addPlugin(std::make_unique<MachOPlatformPlugin>(*this));
 287
 288   PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));
 289
 290   // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating
 291   // the platform now), so set it up.
 292   if (auto E2 = setupJITDylib(PlatformJD)) {
 293     Err = std::move(E2);
 294     return;
 295   }
 296
 297   RegisteredInitSymbols[&PlatformJD].add(
 298       MachOHeaderStartSymbol, SymbolLookupFlags::WeaklyReferencedSymbol);
 299
 300   // Associate wrapper function tags with JIT-side function implementations.
 301   if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
 302     Err = std::move(E2);
 303     return;
 304   }
 305
 306   // Lookup addresses of runtime functions callable by the platform,
 307   // call the platform bootstrap function to initialize the platform-state
 308   // object in the executor.
 309   if (auto E2 = bootstrapMachORuntime(PlatformJD)) {
 310     Err = std::move(E2);
 311     return;
 312   }
 313 }
 314
 315 Error MachOPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
 316   ExecutionSession::JITDispatchHandlerAssociationMap WFs;
 317
 318   using GetInitializersSPSSig =
 319       SPSExpected<SPSMachOJITDylibInitializerSequence>(SPSString);
 320   WFs[ES.intern("___orc_rt_macho_get_initializers_tag")] =
 321       ES.wrapAsyncWithSPS<GetInitializersSPSSig>(
 322           this, &MachOPlatform::rt_getInitializers);
 323
 324   using GetDeinitializersSPSSig =
 325       SPSExpected<SPSMachOJITDylibDeinitializerSequence>(SPSExecutorAddress);
 326   WFs[ES.intern("___orc_rt_macho_get_deinitializers_tag")] =
 327       ES.wrapAsyncWithSPS<GetDeinitializersSPSSig>(
 328           this, &MachOPlatform::rt_getDeinitializers);
 329
 330   using LookupSymbolSPSSig =
 331       SPSExpected<SPSExecutorAddress>(SPSExecutorAddress, SPSString);
 332   WFs[ES.intern("___orc_rt_macho_symbol_lookup_tag")] =
 333       ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this,
 334                                               &MachOPlatform::rt_lookupSymbol);
 335
 336   return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
 337 }
 338
 339 void MachOPlatform::getInitializersBuildSequencePhase(
 340     SendInitializerSequenceFn SendResult, JITDylib &JD,
 341     std::vector<JITDylibSP> DFSLinkOrder) {
 342   MachOJITDylibInitializerSequence FullInitSeq;
 343   {
 344     std::lock_guard<std::mutex> Lock(PlatformMutex);
 345     for (auto &InitJD : reverse(DFSLinkOrder)) {
 346       LLVM_DEBUG({
 347         dbgs() << "MachOPlatform: Appending inits for \"" << InitJD->getName()
 348                << "\" to sequence\n";
 349       });
 350       auto ISItr = InitSeqs.find(InitJD.get());
 351       if (ISItr != InitSeqs.end()) {
 352         FullInitSeq.emplace_back(std::move(ISItr->second));
 353         InitSeqs.erase(ISItr);
 354       }
 355     }
 356   }
 357
 358   SendResult(std::move(FullInitSeq));
 359 }
 360
 361 void MachOPlatform::getInitializersLookupPhase(
 362     SendInitializerSequenceFn SendResult, JITDylib &JD) {
 363
 364   auto DFSLinkOrder = JD.getDFSLinkOrder();
 365   DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
 366   ES.runSessionLocked([&]() {
 367     for (auto &InitJD : DFSLinkOrder) {
 368       auto RISItr = RegisteredInitSymbols.find(InitJD.get());
 369       if (RISItr != RegisteredInitSymbols.end()) {
 370         NewInitSymbols[InitJD.get()] = std::move(RISItr->second);
 371         RegisteredInitSymbols.erase(RISItr);
 372       }
 373     }
 374   });
 375
 376   // If there are no further init symbols to look up then move on to the next
 377   // phase.
 378   if (NewInitSymbols.empty()) {
 379     getInitializersBuildSequencePhase(std::move(SendResult), JD,
 380                                       std::move(DFSLinkOrder));
 381     return;
 382   }
 383
 384   // Otherwise issue a lookup and re-run this phase when it completes.
 385   lookupInitSymbolsAsync(
 386       [this, SendResult = std::move(SendResult), &JD](Error Err) mutable {
 387         if (Err)
 388           SendResult(std::move(Err));
 389         else
 390           getInitializersLookupPhase(std::move(SendResult), JD);
 391       },
 392       ES, std::move(NewInitSymbols));
 393 }
 394
 395 void MachOPlatform::rt_getInitializers(SendInitializerSequenceFn SendResult,
 396                                        StringRef JDName) {
 397   LLVM_DEBUG({
 398     dbgs() << "MachOPlatform::rt_getInitializers(\"" << JDName << "\")\n";
 399   });
 400
 401   JITDylib *JD = ES.getJITDylibByName(JDName);
 402   if (!JD) {
 403     LLVM_DEBUG({
 404       dbgs() << "  No such JITDylib \"" << JDName << "\". Sending error.\n";
 405     });
 406     SendResult(make_error<StringError>("No JITDylib named " + JDName,
 407                                        inconvertibleErrorCode()));
 408     return;
 409   }
 410
 411   getInitializersLookupPhase(std::move(SendResult), *JD);
 412 }
 413
 414 void MachOPlatform::rt_getDeinitializers(SendDeinitializerSequenceFn SendResult,
 415                                          ExecutorAddress Handle) {
 416   LLVM_DEBUG({
 417     dbgs() << "MachOPlatform::rt_getDeinitializers(\""
 418            << formatv("{0:x}", Handle.getValue()) << "\")\n";
 419   });
 420
 421   JITDylib *JD = nullptr;
 422
 423   {
 424     std::lock_guard<std::mutex> Lock(PlatformMutex);
 425     auto I = HeaderAddrToJITDylib.find(Handle.getValue());
 426     if (I != HeaderAddrToJITDylib.end())
 427       JD = I->second;
 428   }
 429
 430   if (!JD) {
 431     LLVM_DEBUG({
 432       dbgs() << "  No JITDylib for handle "
 433              << formatv("{0:x}", Handle.getValue()) << "\n";
 434     });
 435     SendResult(make_error<StringError>("No JITDylib associated with handle " +
 436                                            formatv("{0:x}", Handle.getValue()),
 437                                        inconvertibleErrorCode()));
 438     return;
 439   }
 440
 441   SendResult(MachOJITDylibDeinitializerSequence());
 442 }
 443
 444 void MachOPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
 445                                     ExecutorAddress Handle,
 446                                     StringRef SymbolName) {
 447   LLVM_DEBUG({
 448     dbgs() << "MachOPlatform::rt_lookupSymbol(\""
 449            << formatv("{0:x}", Handle.getValue()) << "\")\n";
 450   });
 451
 452   JITDylib *JD = nullptr;
 453
 454   {
 455     std::lock_guard<std::mutex> Lock(PlatformMutex);
 456     auto I = HeaderAddrToJITDylib.find(Handle.getValue());
 457     if (I != HeaderAddrToJITDylib.end())
 458       JD = I->second;
 459   }
 460
 461   if (!JD) {
 462     LLVM_DEBUG({
 463       dbgs() << "  No JITDylib for handle "
 464              << formatv("{0:x}", Handle.getValue()) << "\n";
 465     });
 466     SendResult(make_error<StringError>("No JITDylib associated with handle " +
 467                                            formatv("{0:x}", Handle.getValue()),
 468                                        inconvertibleErrorCode()));
 469     return;
 470   }
 471
 472   // Use functor class to work around XL build compiler issue on AIX.
 473   class RtLookupNotifyComplete {
 474   public:
 475     RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
 476         : SendResult(std::move(SendResult)) {}
 477     void operator()(Expected<SymbolMap> Result) {
 478       if (Result) {
 479         assert(Result->size() == 1 && "Unexpected result map count");
 480         SendResult(ExecutorAddress(Result->begin()->second.getAddress()));
 481       } else {
 482         SendResult(Result.takeError());
 483       }
 484     }
 485
 486   private:
 487     SendSymbolAddressFn SendResult;
 488   };
 489
 490   // FIXME: Proper mangling.
 491   auto MangledName = ("_" + SymbolName).str();
 492   ES.lookup(
 493       LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
 494       SymbolLookupSet(ES.intern(MangledName)), SymbolState::Ready,
 495       RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister);
 496 }
 497
 498 Error MachOPlatform::bootstrapMachORuntime(JITDylib &PlatformJD) {
 499
 500   if (auto Err = lookupAndRecordAddrs(
 501           ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD),
 502           {{ES.intern("___orc_rt_macho_platform_bootstrap"),
 503             &orc_rt_macho_platform_bootstrap},
 504            {ES.intern("___orc_rt_macho_platform_shutdown"),
 505             &orc_rt_macho_platform_shutdown},
 506            {ES.intern("___orc_rt_macho_register_object_sections"),
 507             &orc_rt_macho_register_object_sections},
 508            {ES.intern("___orc_rt_macho_create_pthread_key"),
 509             &orc_rt_macho_create_pthread_key}}))
 510     return Err;
 511
 512   if (auto Err =
 513           ES.callSPSWrapper<void()>(orc_rt_macho_platform_bootstrap.getValue()))
 514     return Err;
 515
 516   // FIXME: Ordering is fuzzy here. We're probably best off saying
 517   // "behavior is undefined if code that uses the runtime is added before
 518   // the platform constructor returns", then move all this to the constructor.
 519   RuntimeBootstrapped = true;
 520   std::vector<MachOPerObjectSectionsToRegister> DeferredPOSRs;
 521   {
 522     std::lock_guard<std::mutex> Lock(PlatformMutex);
 523     DeferredPOSRs = std::move(BootstrapPOSRs);
 524   }
 525
 526   for (auto &D : DeferredPOSRs)
 527     if (auto Err = registerPerObjectSections(D))
 528       return Err;
 529
 530   return Error::success();
 531 }
 532
 533 Error MachOPlatform::registerInitInfo(
 534     JITDylib &JD, ExecutorAddress ObjCImageInfoAddr,
 535     ArrayRef<jitlink::Section *> InitSections) {
 536
 537   std::unique_lock<std::mutex> Lock(PlatformMutex);
 538
 539   MachOJITDylibInitializers *InitSeq = nullptr;
 540   {
 541     auto I = InitSeqs.find(&JD);
 542     if (I == InitSeqs.end()) {
 543       // If there's no init sequence entry yet then we need to look up the
 544       // header symbol to force creation of one.
 545       Lock.unlock();
 546
 547       auto SearchOrder =
 548           JD.withLinkOrderDo([](const JITDylibSearchOrder &SO) { return SO; });
 549       if (auto Err = ES.lookup(SearchOrder, MachOHeaderStartSymbol).takeError())
 550         return Err;
 551
 552       Lock.lock();
 553       I = InitSeqs.find(&JD);
 554       assert(I != InitSeqs.end() &&
 555              "Entry missing after header symbol lookup?");
 556     }
 557     InitSeq = &I->second;
 558   }
 559
 560   InitSeq->ObjCImageInfoAddress = ObjCImageInfoAddr;
 561
 562   for (auto *Sec : InitSections) {
 563     // FIXME: Avoid copy here.
 564     jitlink::SectionRange R(*Sec);
 565     InitSeq->InitSections[Sec->getName()].push_back(
 566         {ExecutorAddress(R.getStart()), ExecutorAddress(R.getEnd())});
 567   }
 568
 569   return Error::success();
 570 }
 571
 572 Error MachOPlatform::registerPerObjectSections(
 573     const MachOPerObjectSectionsToRegister &POSR) {
 574
 575   if (!orc_rt_macho_register_object_sections)
 576     return make_error<StringError>("Attempting to register per-object "
 577                                    "sections, but runtime support has not "
 578                                    "been loaded yet",
 579                                    inconvertibleErrorCode());
 580
 581   Error ErrResult = Error::success();
 582   if (auto Err = ES.callSPSWrapper<shared::SPSError(
 583                      SPSMachOPerObjectSectionsToRegister)>(
 584           orc_rt_macho_register_object_sections.getValue(), ErrResult, POSR))
 585     return Err;
 586   return ErrResult;
 587 }
 588
 589 Expected<uint64_t> MachOPlatform::createPThreadKey() {
 590   if (!orc_rt_macho_create_pthread_key)
 591     return make_error<StringError>(
 592         "Attempting to create pthread key in target, but runtime support has "
 593         "not been loaded yet",
 594         inconvertibleErrorCode());
 595
 596   Expected<uint64_t> Result(0);
 597   if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>(
 598           orc_rt_macho_create_pthread_key.getValue(), Result))
 599     return std::move(Err);
 600   return Result;
 601 }
 602
 603 void MachOPlatform::MachOPlatformPlugin::modifyPassConfig(
 604     MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
 605     jitlink::PassConfiguration &Config) {
 606
 607   // If the initializer symbol is the MachOHeader start symbol then just add
 608   // the macho header support passes.
 609   if (MR.getInitializerSymbol() == MP.MachOHeaderStartSymbol) {
 610     addMachOHeaderSupportPasses(MR, Config);
 611     // The header materialization unit doesn't require any other support, so we
 612     // can bail out early.
 613     return;
 614   }
 615
 616   // If the object contains initializers then add passes to record them.
 617   if (MR.getInitializerSymbol())
 618     addInitializerSupportPasses(MR, Config);
 619
 620   // Add passes for eh-frame and TLV support.
 621   addEHAndTLVSupportPasses(MR, Config);
 622 }
 623
 624 ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap
 625 MachOPlatform::MachOPlatformPlugin::getSyntheticSymbolDependencies(
 626     MaterializationResponsibility &MR) {
 627   std::lock_guard<std::mutex> Lock(PluginMutex);
 628   auto I = InitSymbolDeps.find(&MR);
 629   if (I != InitSymbolDeps.end()) {
 630     SyntheticSymbolDependenciesMap Result;
 631     Result[MR.getInitializerSymbol()] = std::move(I->second);
 632     InitSymbolDeps.erase(&MR);
 633     return Result;
 634   }
 635   return SyntheticSymbolDependenciesMap();
 636 }
 637
 638 void MachOPlatform::MachOPlatformPlugin::addInitializerSupportPasses(
 639     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
 640
 641   /// Preserve init sections.
 642   Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) {
 643     if (auto Err = preserveInitSections(G, MR))
 644       return Err;
 645     return processObjCImageInfo(G, MR);
 646   });
 647
 648   Config.PostFixupPasses.push_back(
 649       [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
 650         return registerInitSections(G, JD);
 651       });
 652 }
 653
 654 void MachOPlatform::MachOPlatformPlugin::addMachOHeaderSupportPasses(
 655     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
 656
 657   Config.PostAllocationPasses.push_back([this, &JD = MR.getTargetJITDylib()](
 658                                             jitlink::LinkGraph &G) -> Error {
 659     auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {
 660       return Sym->getName() == *MP.MachOHeaderStartSymbol;
 661     });
 662     assert(I != G.defined_symbols().end() &&
 663            "Missing MachO header start symbol");
 664     {
 665       std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
 666       JITTargetAddress HeaderAddr = (*I)->getAddress();
 667       MP.HeaderAddrToJITDylib[HeaderAddr] = &JD;
 668       assert(!MP.InitSeqs.count(&JD) && "InitSeq entry for JD already exists");
 669       MP.InitSeqs.insert(
 670           std::make_pair(&JD, MachOJITDylibInitializers(
 671                                   JD.getName(), ExecutorAddress(HeaderAddr))));
 672     }
 673     return Error::success();
 674   });
 675 }
 676
 677 void MachOPlatform::MachOPlatformPlugin::addEHAndTLVSupportPasses(
 678     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
 679
 680   // Insert TLV lowering at the start of the PostPrunePasses, since we want
 681   // it to run before GOT/PLT lowering.
 682   Config.PostPrunePasses.insert(
 683       Config.PostPrunePasses.begin(),
 684       [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
 685         return fixTLVSectionsAndEdges(G, JD);
 686       });
 687
 688   // Add a pass to register the final addresses of the eh-frame and TLV sections
 689   // with the runtime.
 690   Config.PostFixupPasses.push_back([this](jitlink::LinkGraph &G) -> Error {
 691     MachOPerObjectSectionsToRegister POSR;
 692
 693     if (auto *EHFrameSection = G.findSectionByName(EHFrameSectionName)) {
 694       jitlink::SectionRange R(*EHFrameSection);
 695       if (!R.empty())
 696         POSR.EHFrameSection = {ExecutorAddress(R.getStart()),
 697                                ExecutorAddress(R.getEnd())};
 698     }
 699
 700     // Get a pointer to the thread data section if there is one. It will be used
 701     // below.
 702     jitlink::Section *ThreadDataSection =
 703         G.findSectionByName(ThreadDataSectionName);
 704
 705     // Handle thread BSS section if there is one.
 706     if (auto *ThreadBSSSection = G.findSectionByName(ThreadBSSSectionName)) {
 707       // If there's already a thread data section in this graph then merge the
 708       // thread BSS section content into it, otherwise just treat the thread
 709       // BSS section as the thread data section.
 710       if (ThreadDataSection)
 711         G.mergeSections(*ThreadDataSection, *ThreadBSSSection);
 712       else
 713         ThreadDataSection = ThreadBSSSection;
 714     }
 715
 716     // Having merged thread BSS (if present) and thread data (if present),
 717     // record the resulting section range.
 718     if (ThreadDataSection) {
 719       jitlink::SectionRange R(*ThreadDataSection);
 720       if (!R.empty())
 721         POSR.ThreadDataSection = {ExecutorAddress(R.getStart()),
 722                                   ExecutorAddress(R.getEnd())};
 723     }
 724
 725     if (POSR.EHFrameSection.StartAddress ||
 726         POSR.ThreadDataSection.StartAddress) {
 727
 728       // If we're still bootstrapping the runtime then just record this
 729       // frame for now.
 730       if (!MP.RuntimeBootstrapped) {
 731         std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
 732         MP.BootstrapPOSRs.push_back(POSR);
 733         return Error::success();
 734       }
 735
 736       // Otherwise register it immediately.
 737       if (auto Err = MP.registerPerObjectSections(POSR))
 738         return Err;
 739     }
 740
 741     return Error::success();
 742   });
 743 }
 744
 745 Error MachOPlatform::MachOPlatformPlugin::preserveInitSections(
 746     jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
 747
 748   JITLinkSymbolSet InitSectionSymbols;
 749   for (auto &InitSectionName : InitSectionNames) {
 750     // Skip non-init sections.
 751     auto *InitSection = G.findSectionByName(InitSectionName);
 752     if (!InitSection)
 753       continue;
 754
 755     // Make a pass over live symbols in the section: those blocks are already
 756     // preserved.
 757     DenseSet<jitlink::Block *> AlreadyLiveBlocks;
 758     for (auto &Sym : InitSection->symbols()) {
 759       auto &B = Sym->getBlock();
 760       if (Sym->isLive() && Sym->getOffset() == 0 &&
 761           Sym->getSize() == B.getSize() && !AlreadyLiveBlocks.count(&B)) {
 762         InitSectionSymbols.insert(Sym);
 763         AlreadyLiveBlocks.insert(&B);
 764       }
 765     }
 766
 767     // Add anonymous symbols to preserve any not-already-preserved blocks.
 768     for (auto *B : InitSection->blocks())
 769       if (!AlreadyLiveBlocks.count(B))
 770         InitSectionSymbols.insert(
 771             &G.addAnonymousSymbol(*B, 0, B->getSize(), false, true));
 772   }
 773
 774   if (!InitSectionSymbols.empty()) {
 775     std::lock_guard<std::mutex> Lock(PluginMutex);
 776     InitSymbolDeps[&MR] = std::move(InitSectionSymbols);
 777   }
 778
 779   return Error::success();
 780 }
 781
 782 Error MachOPlatform::MachOPlatformPlugin::processObjCImageInfo(
 783     jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
 784
 785   // If there's an ObjC imagine info then either
 786   //   (1) It's the first __objc_imageinfo we've seen in this JITDylib. In
 787   //       this case we name and record it.
 788   // OR
 789   //   (2) We already have a recorded __objc_imageinfo for this JITDylib,
 790   //       in which case we just verify it.
 791   auto *ObjCImageInfo = G.findSectionByName(ObjCImageInfoSectionName);
 792   if (!ObjCImageInfo)
 793     return Error::success();
 794
 795   auto ObjCImageInfoBlocks = ObjCImageInfo->blocks();
 796
 797   // Check that the section is not empty if present.
 798   if (llvm::empty(ObjCImageInfoBlocks))
 799     return make_error<StringError>("Empty " + ObjCImageInfoSectionName +
 800                                        " section in " + G.getName(),
 801                                    inconvertibleErrorCode());
 802
 803   // Check that there's only one block in the section.
 804   if (std::next(ObjCImageInfoBlocks.begin()) != ObjCImageInfoBlocks.end())
 805     return make_error<StringError>("Multiple blocks in " +
 806                                        ObjCImageInfoSectionName +
 807                                        " section in " + G.getName(),
 808                                    inconvertibleErrorCode());
 809
 810   // Check that the __objc_imageinfo section is unreferenced.
 811   // FIXME: We could optimize this check if Symbols had a ref-count.
 812   for (auto &Sec : G.sections()) {
 813     if (&Sec != ObjCImageInfo)
 814       for (auto *B : Sec.blocks())
 815         for (auto &E : B->edges())
 816           if (E.getTarget().isDefined() &&
 817               &E.getTarget().getBlock().getSection() == ObjCImageInfo)
 818             return make_error<StringError>(ObjCImageInfoSectionName +
 819                                                " is referenced within file " +
 820                                                G.getName(),
 821                                            inconvertibleErrorCode());
 822   }
 823
 824   auto &ObjCImageInfoBlock = **ObjCImageInfoBlocks.begin();
 825   auto *ObjCImageInfoData = ObjCImageInfoBlock.getContent().data();
 826   auto Version = support::endian::read32(ObjCImageInfoData, G.getEndianness());
 827   auto Flags =
 828       support::endian::read32(ObjCImageInfoData + 4, G.getEndianness());
 829
 830   // Lock the mutex while we verify / update the ObjCImageInfos map.
 831   std::lock_guard<std::mutex> Lock(PluginMutex);
 832
 833   auto ObjCImageInfoItr = ObjCImageInfos.find(&MR.getTargetJITDylib());
 834   if (ObjCImageInfoItr != ObjCImageInfos.end()) {
 835     // We've already registered an __objc_imageinfo section. Verify the
 836     // content of this new section matches, then delete it.
 837     if (ObjCImageInfoItr->second.first != Version)
 838       return make_error<StringError>(
 839           "ObjC version in " + G.getName() +
 840               " does not match first registered version",
 841           inconvertibleErrorCode());
 842     if (ObjCImageInfoItr->second.second != Flags)
 843       return make_error<StringError>("ObjC flags in " + G.getName() +
 844                                          " do not match first registered flags",
 845                                      inconvertibleErrorCode());
 846
 847     // __objc_imageinfo is valid. Delete the block.
 848     for (auto *S : ObjCImageInfo->symbols())
 849       G.removeDefinedSymbol(*S);
 850     G.removeBlock(ObjCImageInfoBlock);
 851   } else {
 852     // We haven't registered an __objc_imageinfo section yet. Register and
 853     // move on. The section should already be marked no-dead-strip.
 854     ObjCImageInfos[&MR.getTargetJITDylib()] = std::make_pair(Version, Flags);
 855   }
 856
 857   return Error::success();
 858 }
 859
 860 Error MachOPlatform::MachOPlatformPlugin::registerInitSections(
 861     jitlink::LinkGraph &G, JITDylib &JD) {
 862
 863   ExecutorAddress ObjCImageInfoAddr;
 864   SmallVector<jitlink::Section *> InitSections;
 865
 866   if (auto *ObjCImageInfoSec = G.findSectionByName(ObjCImageInfoSectionName)) {
 867     if (auto Addr = jitlink::SectionRange(*ObjCImageInfoSec).getStart())
 868       ObjCImageInfoAddr.setValue(Addr);
 869   }
 870
 871   for (auto InitSectionName : InitSectionNames)
 872     if (auto *Sec = G.findSectionByName(InitSectionName))
 873       InitSections.push_back(Sec);
 874
 875   // Dump the scraped inits.
 876   LLVM_DEBUG({
 877     dbgs() << "MachOPlatform: Scraped " << G.getName() << " init sections:\n";
 878     if (ObjCImageInfoAddr)
 879       dbgs() << "  " << ObjCImageInfoSectionName << ": "
 880              << formatv("{0:x}", ObjCImageInfoAddr.getValue()) << "\n";
 881     for (auto *Sec : InitSections) {
 882       jitlink::SectionRange R(*Sec);
 883       dbgs() << "  " << Sec->getName() << ": "
 884              << formatv("[ {0:x} -- {1:x} ]", R.getStart(), R.getEnd()) << "\n";
 885     }
 886   });
 887
 888   return MP.registerInitInfo(JD, ObjCImageInfoAddr, InitSections);
 889 }
 890
 891 Error MachOPlatform::MachOPlatformPlugin::fixTLVSectionsAndEdges(
 892     jitlink::LinkGraph &G, JITDylib &JD) {
 893
 894   // Rename external references to __tlv_bootstrap to ___orc_rt_tlv_get_addr.
 895   for (auto *Sym : G.external_symbols())
 896     if (Sym->getName() == "__tlv_bootstrap") {
 897       Sym->setName("___orc_rt_macho_tlv_get_addr");
 898       break;
 899     }
 900
 901   // Store key in __thread_vars struct fields.
 902   if (auto *ThreadDataSec = G.findSectionByName(ThreadVarsSectionName)) {
 903     Optional<uint64_t> Key;
 904     {
 905       std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
 906       auto I = MP.JITDylibToPThreadKey.find(&JD);
 907       if (I != MP.JITDylibToPThreadKey.end())
 908         Key = I->second;
 909     }
 910
 911     if (!Key) {
 912       if (auto KeyOrErr = MP.createPThreadKey())
 913         Key = *KeyOrErr;
 914       else
 915         return KeyOrErr.takeError();
 916     }
 917
 918     uint64_t PlatformKeyBits =
 919         support::endian::byte_swap(*Key, G.getEndianness());
 920
 921     for (auto *B : ThreadDataSec->blocks()) {
 922       if (B->getSize() != 3 * G.getPointerSize())
 923         return make_error<StringError>("__thread_vars block at " +
 924                                            formatv("{0:x}", B->getAddress()) +
 925                                            " has unexpected size",
 926                                        inconvertibleErrorCode());
 927
 928       auto NewBlockContent = G.allocateBuffer(B->getSize());
 929       llvm::copy(B->getContent(), NewBlockContent.data());
 930       memcpy(NewBlockContent.data() + G.getPointerSize(), &PlatformKeyBits,
 931              G.getPointerSize());
 932       B->setContent(NewBlockContent);
 933     }
 934   }
 935
 936   // Transform any TLV edges into GOT edges.
 937   for (auto *B : G.blocks())
 938     for (auto &E : B->edges())
 939       if (E.getKind() ==
 940           jitlink::x86_64::RequestTLVPAndTransformToPCRel32TLVPLoadREXRelaxable)
 941         E.setKind(jitlink::x86_64::
 942                       RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable);
 943
 944   return Error::success();
 945 }
 946
 947 } // End namespace orc.
 948 } // End namespace llvm.