llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp

   1 //===------- COFFPlatform.cpp - Utilities for executing COFF in Orc -------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ExecutionEngine/Orc/COFFPlatform.h"
  10 #include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h"
  11 #include "llvm/ExecutionEngine/Orc/DebugUtils.h"
  12 #include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h"
  13 #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
  14 #include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
  15
  16 #include "llvm/Object/COFF.h"
  17
  18 #include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
  19
  20 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
  21
  22 #define DEBUG_TYPE "orc"
  23
  24 using namespace llvm;
  25 using namespace llvm::orc;
  26 using namespace llvm::orc::shared;
  27
  28 namespace llvm {
  29 namespace orc {
  30 namespace shared {
  31
  32 using SPSCOFFJITDylibDepInfo = SPSSequence<SPSExecutorAddr>;
  33 using SPSCOFFJITDylibDepInfoMap =
  34     SPSSequence<SPSTuple<SPSExecutorAddr, SPSCOFFJITDylibDepInfo>>;
  35 using SPSCOFFObjectSectionsMap =
  36     SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>;
  37 using SPSCOFFRegisterObjectSectionsArgs =
  38     SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap, bool>;
  39 using SPSCOFFDeregisterObjectSectionsArgs =
  40     SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap>;
  41
  42 } // namespace shared
  43 } // namespace orc
  44 } // namespace llvm
  45 namespace {
  46
  47 class COFFHeaderMaterializationUnit : public MaterializationUnit {
  48 public:
  49   COFFHeaderMaterializationUnit(COFFPlatform &CP,
  50                                 const SymbolStringPtr &HeaderStartSymbol)
  51       : MaterializationUnit(createHeaderInterface(CP, HeaderStartSymbol)),
  52         CP(CP) {}
  53
  54   StringRef getName() const override { return "COFFHeaderMU"; }
  55
  56   void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
  57     unsigned PointerSize;
  58     llvm::endianness Endianness;
  59     const auto &TT = CP.getExecutionSession().getTargetTriple();
  60
  61     switch (TT.getArch()) {
  62     case Triple::x86_64:
  63       PointerSize = 8;
  64       Endianness = llvm::endianness::little;
  65       break;
  66     default:
  67       llvm_unreachable("Unrecognized architecture");
  68     }
  69
  70     auto G = std::make_unique<jitlink::LinkGraph>(
  71         "<COFFHeaderMU>", CP.getExecutionSession().getSymbolStringPool(), TT,
  72         PointerSize, Endianness, jitlink::getGenericEdgeKindName);
  73     auto &HeaderSection = G->createSection("__header", MemProt::Read);
  74     auto &HeaderBlock = createHeaderBlock(*G, HeaderSection);
  75
  76     // Init symbol is __ImageBase symbol.
  77     auto &ImageBaseSymbol = G->addDefinedSymbol(
  78         HeaderBlock, 0, *R->getInitializerSymbol(), HeaderBlock.getSize(),
  79         jitlink::Linkage::Strong, jitlink::Scope::Default, false, true);
  80
  81     addImageBaseRelocationEdge(HeaderBlock, ImageBaseSymbol);
  82
  83     CP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
  84   }
  85
  86   void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
  87
  88 private:
  89   struct HeaderSymbol {
  90     const char *Name;
  91     uint64_t Offset;
  92   };
  93
  94   struct NTHeader {
  95     support::ulittle32_t PEMagic;
  96     object::coff_file_header FileHeader;
  97     struct PEHeader {
  98       object::pe32plus_header Header;
  99       object::data_directory DataDirectory[COFF::NUM_DATA_DIRECTORIES + 1];
 100     } OptionalHeader;
 101   };
 102
 103   struct HeaderBlockContent {
 104     object::dos_header DOSHeader;
 105     COFFHeaderMaterializationUnit::NTHeader NTHeader;
 106   };
 107
 108   static jitlink::Block &createHeaderBlock(jitlink::LinkGraph &G,
 109                                            jitlink::Section &HeaderSection) {
 110     HeaderBlockContent Hdr = {};
 111
 112     // Set up magic
 113     Hdr.DOSHeader.Magic[0] = 'M';
 114     Hdr.DOSHeader.Magic[1] = 'Z';
 115     Hdr.DOSHeader.AddressOfNewExeHeader =
 116         offsetof(HeaderBlockContent, NTHeader);
 117     uint32_t PEMagic = *reinterpret_cast<const uint32_t *>(COFF::PEMagic);
 118     Hdr.NTHeader.PEMagic = PEMagic;
 119     Hdr.NTHeader.OptionalHeader.Header.Magic = COFF::PE32Header::PE32_PLUS;
 120
 121     switch (G.getTargetTriple().getArch()) {
 122     case Triple::x86_64:
 123       Hdr.NTHeader.FileHeader.Machine = COFF::IMAGE_FILE_MACHINE_AMD64;
 124       break;
 125     default:
 126       llvm_unreachable("Unrecognized architecture");
 127     }
 128
 129     auto HeaderContent = G.allocateContent(
 130         ArrayRef<char>(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr)));
 131
 132     return G.createContentBlock(HeaderSection, HeaderContent, ExecutorAddr(), 8,
 133                                 0);
 134   }
 135
 136   static void addImageBaseRelocationEdge(jitlink::Block &B,
 137                                          jitlink::Symbol &ImageBase) {
 138     auto ImageBaseOffset = offsetof(HeaderBlockContent, NTHeader) +
 139                            offsetof(NTHeader, OptionalHeader) +
 140                            offsetof(object::pe32plus_header, ImageBase);
 141     B.addEdge(jitlink::x86_64::Pointer64, ImageBaseOffset, ImageBase, 0);
 142   }
 143
 144   static MaterializationUnit::Interface
 145   createHeaderInterface(COFFPlatform &MOP,
 146                         const SymbolStringPtr &HeaderStartSymbol) {
 147     SymbolFlagsMap HeaderSymbolFlags;
 148
 149     HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported;
 150
 151     return MaterializationUnit::Interface(std::move(HeaderSymbolFlags),
 152                                           HeaderStartSymbol);
 153   }
 154
 155   COFFPlatform &CP;
 156 };
 157
 158 } // end anonymous namespace
 159
 160 namespace llvm {
 161 namespace orc {
 162
 163 Expected<std::unique_ptr<COFFPlatform>>
 164 COFFPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
 165                      std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,
 166                      LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
 167                      const char *VCRuntimePath,
 168                      std::optional<SymbolAliasMap> RuntimeAliases) {
 169
 170   auto &ES = ObjLinkingLayer.getExecutionSession();
 171
 172   // If the target is not supported then bail out immediately.
 173   if (!supportedTarget(ES.getTargetTriple()))
 174     return make_error<StringError>("Unsupported COFFPlatform triple: " +
 175                                        ES.getTargetTriple().str(),
 176                                    inconvertibleErrorCode());
 177
 178   auto &EPC = ES.getExecutorProcessControl();
 179
 180   auto GeneratorArchive =
 181       object::Archive::create(OrcRuntimeArchiveBuffer->getMemBufferRef());
 182   if (!GeneratorArchive)
 183     return GeneratorArchive.takeError();
 184
 185   auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Create(
 186       ObjLinkingLayer, nullptr, std::move(*GeneratorArchive));
 187   if (!OrcRuntimeArchiveGenerator)
 188     return OrcRuntimeArchiveGenerator.takeError();
 189
 190   // We need a second instance of the archive (for now) for the Platform. We
 191   // can `cantFail` this call, since if it were going to fail it would have
 192   // failed above.
 193   auto RuntimeArchive = cantFail(
 194       object::Archive::create(OrcRuntimeArchiveBuffer->getMemBufferRef()));
 195
 196   // Create default aliases if the caller didn't supply any.
 197   if (!RuntimeAliases)
 198     RuntimeAliases = standardPlatformAliases(ES);
 199
 200   // Define the aliases.
 201   if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases))))
 202     return std::move(Err);
 203
 204   auto &HostFuncJD = ES.createBareJITDylib("$<PlatformRuntimeHostFuncJD>");
 205
 206   // Add JIT-dispatch function support symbols.
 207   if (auto Err = HostFuncJD.define(
 208           absoluteSymbols({{ES.intern("__orc_rt_jit_dispatch"),
 209                             {EPC.getJITDispatchInfo().JITDispatchFunction,
 210                              JITSymbolFlags::Exported}},
 211                            {ES.intern("__orc_rt_jit_dispatch_ctx"),
 212                             {EPC.getJITDispatchInfo().JITDispatchContext,
 213                              JITSymbolFlags::Exported}}})))
 214     return std::move(Err);
 215
 216   PlatformJD.addToLinkOrder(HostFuncJD);
 217
 218   // Create the instance.
 219   Error Err = Error::success();
 220   auto P = std::unique_ptr<COFFPlatform>(new COFFPlatform(
 221       ObjLinkingLayer, PlatformJD, std::move(*OrcRuntimeArchiveGenerator),
 222       std::move(OrcRuntimeArchiveBuffer), std::move(RuntimeArchive),
 223       std::move(LoadDynLibrary), StaticVCRuntime, VCRuntimePath, Err));
 224   if (Err)
 225     return std::move(Err);
 226   return std::move(P);
 227 }
 228
 229 Expected<std::unique_ptr<COFFPlatform>>
 230 COFFPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
 231                      const char *OrcRuntimePath,
 232                      LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
 233                      const char *VCRuntimePath,
 234                      std::optional<SymbolAliasMap> RuntimeAliases) {
 235
 236   auto ArchiveBuffer = MemoryBuffer::getFile(OrcRuntimePath);
 237   if (!ArchiveBuffer)
 238     return createFileError(OrcRuntimePath, ArchiveBuffer.getError());
 239
 240   return Create(ObjLinkingLayer, PlatformJD, std::move(*ArchiveBuffer),
 241                 std::move(LoadDynLibrary), StaticVCRuntime, VCRuntimePath,
 242                 std::move(RuntimeAliases));
 243 }
 244
 245 Expected<MemoryBufferRef> COFFPlatform::getPerJDObjectFile() {
 246   auto PerJDObj = OrcRuntimeArchive->findSym("__orc_rt_coff_per_jd_marker");
 247   if (!PerJDObj)
 248     return PerJDObj.takeError();
 249
 250   if (!*PerJDObj)
 251     return make_error<StringError>("Could not find per jd object file",
 252                                    inconvertibleErrorCode());
 253
 254   auto Buffer = (*PerJDObj)->getAsBinary();
 255   if (!Buffer)
 256     return Buffer.takeError();
 257
 258   return (*Buffer)->getMemoryBufferRef();
 259 }
 260
 261 static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
 262                        ArrayRef<std::pair<const char *, const char *>> AL) {
 263   for (auto &KV : AL) {
 264     auto AliasName = ES.intern(KV.first);
 265     assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
 266     Aliases[std::move(AliasName)] = {ES.intern(KV.second),
 267                                      JITSymbolFlags::Exported};
 268   }
 269 }
 270
 271 Error COFFPlatform::setupJITDylib(JITDylib &JD) {
 272   if (auto Err = JD.define(std::make_unique<COFFHeaderMaterializationUnit>(
 273           *this, COFFHeaderStartSymbol)))
 274     return Err;
 275
 276   if (auto Err = ES.lookup({&JD}, COFFHeaderStartSymbol).takeError())
 277     return Err;
 278
 279   // Define the CXX aliases.
 280   SymbolAliasMap CXXAliases;
 281   addAliases(ES, CXXAliases, requiredCXXAliases());
 282   if (auto Err = JD.define(symbolAliases(std::move(CXXAliases))))
 283     return Err;
 284
 285   auto PerJDObj = getPerJDObjectFile();
 286   if (!PerJDObj)
 287     return PerJDObj.takeError();
 288
 289   auto I = getObjectFileInterface(ES, *PerJDObj);
 290   if (!I)
 291     return I.takeError();
 292
 293   if (auto Err = ObjLinkingLayer.add(
 294           JD, MemoryBuffer::getMemBuffer(*PerJDObj, false), std::move(*I)))
 295     return Err;
 296
 297   if (!Bootstrapping) {
 298     auto ImportedLibs = StaticVCRuntime
 299                             ? VCRuntimeBootstrap->loadStaticVCRuntime(JD)
 300                             : VCRuntimeBootstrap->loadDynamicVCRuntime(JD);
 301     if (!ImportedLibs)
 302       return ImportedLibs.takeError();
 303     for (auto &Lib : *ImportedLibs)
 304       if (auto Err = LoadDynLibrary(JD, Lib))
 305         return Err;
 306     if (StaticVCRuntime)
 307       if (auto Err = VCRuntimeBootstrap->initializeStaticVCRuntime(JD))
 308         return Err;
 309   }
 310
 311   JD.addGenerator(DLLImportDefinitionGenerator::Create(ES, ObjLinkingLayer));
 312   return Error::success();
 313 }
 314
 315 Error COFFPlatform::teardownJITDylib(JITDylib &JD) {
 316   std::lock_guard<std::mutex> Lock(PlatformMutex);
 317   auto I = JITDylibToHeaderAddr.find(&JD);
 318   if (I != JITDylibToHeaderAddr.end()) {
 319     assert(HeaderAddrToJITDylib.count(I->second) &&
 320            "HeaderAddrToJITDylib missing entry");
 321     HeaderAddrToJITDylib.erase(I->second);
 322     JITDylibToHeaderAddr.erase(I);
 323   }
 324   return Error::success();
 325 }
 326
 327 Error COFFPlatform::notifyAdding(ResourceTracker &RT,
 328                                  const MaterializationUnit &MU) {
 329   auto &JD = RT.getJITDylib();
 330   const auto &InitSym = MU.getInitializerSymbol();
 331   if (!InitSym)
 332     return Error::success();
 333
 334   RegisteredInitSymbols[&JD].add(InitSym,
 335                                  SymbolLookupFlags::WeaklyReferencedSymbol);
 336
 337   LLVM_DEBUG({
 338     dbgs() << "COFFPlatform: Registered init symbol " << *InitSym << " for MU "
 339            << MU.getName() << "\n";
 340   });
 341   return Error::success();
 342 }
 343
 344 Error COFFPlatform::notifyRemoving(ResourceTracker &RT) {
 345   llvm_unreachable("Not supported yet");
 346 }
 347
 348 SymbolAliasMap COFFPlatform::standardPlatformAliases(ExecutionSession &ES) {
 349   SymbolAliasMap Aliases;
 350   addAliases(ES, Aliases, standardRuntimeUtilityAliases());
 351   return Aliases;
 352 }
 353
 354 ArrayRef<std::pair<const char *, const char *>>
 355 COFFPlatform::requiredCXXAliases() {
 356   static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
 357       {"_CxxThrowException", "__orc_rt_coff_cxx_throw_exception"},
 358       {"_onexit", "__orc_rt_coff_onexit_per_jd"},
 359       {"atexit", "__orc_rt_coff_atexit_per_jd"}};
 360
 361   return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
 362 }
 363
 364 ArrayRef<std::pair<const char *, const char *>>
 365 COFFPlatform::standardRuntimeUtilityAliases() {
 366   static const std::pair<const char *, const char *>
 367       StandardRuntimeUtilityAliases[] = {
 368           {"__orc_rt_run_program", "__orc_rt_coff_run_program"},
 369           {"__orc_rt_jit_dlerror", "__orc_rt_coff_jit_dlerror"},
 370           {"__orc_rt_jit_dlopen", "__orc_rt_coff_jit_dlopen"},
 371           {"__orc_rt_jit_dlclose", "__orc_rt_coff_jit_dlclose"},
 372           {"__orc_rt_jit_dlsym", "__orc_rt_coff_jit_dlsym"},
 373           {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
 374
 375   return ArrayRef<std::pair<const char *, const char *>>(
 376       StandardRuntimeUtilityAliases);
 377 }
 378
 379 bool COFFPlatform::supportedTarget(const Triple &TT) {
 380   switch (TT.getArch()) {
 381   case Triple::x86_64:
 382     return true;
 383   default:
 384     return false;
 385   }
 386 }
 387
 388 COFFPlatform::COFFPlatform(
 389     ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
 390     std::unique_ptr<StaticLibraryDefinitionGenerator> OrcRuntimeGenerator,
 391     std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,
 392     std::unique_ptr<object::Archive> OrcRuntimeArchive,
 393     LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
 394     const char *VCRuntimePath, Error &Err)
 395     : ES(ObjLinkingLayer.getExecutionSession()),
 396       ObjLinkingLayer(ObjLinkingLayer),
 397       LoadDynLibrary(std::move(LoadDynLibrary)),
 398       OrcRuntimeArchiveBuffer(std::move(OrcRuntimeArchiveBuffer)),
 399       OrcRuntimeArchive(std::move(OrcRuntimeArchive)),
 400       StaticVCRuntime(StaticVCRuntime),
 401       COFFHeaderStartSymbol(ES.intern("__ImageBase")) {
 402   ErrorAsOutParameter _(Err);
 403
 404   Bootstrapping.store(true);
 405   ObjLinkingLayer.addPlugin(std::make_unique<COFFPlatformPlugin>(*this));
 406
 407   // Load vc runtime
 408   auto VCRT =
 409       COFFVCRuntimeBootstrapper::Create(ES, ObjLinkingLayer, VCRuntimePath);
 410   if (!VCRT) {
 411     Err = VCRT.takeError();
 412     return;
 413   }
 414   VCRuntimeBootstrap = std::move(*VCRT);
 415
 416   for (auto &Lib : OrcRuntimeGenerator->getImportedDynamicLibraries())
 417     DylibsToPreload.insert(Lib);
 418
 419   auto ImportedLibs =
 420       StaticVCRuntime ? VCRuntimeBootstrap->loadStaticVCRuntime(PlatformJD)
 421                       : VCRuntimeBootstrap->loadDynamicVCRuntime(PlatformJD);
 422   if (!ImportedLibs) {
 423     Err = ImportedLibs.takeError();
 424     return;
 425   }
 426
 427   for (auto &Lib : *ImportedLibs)
 428     DylibsToPreload.insert(Lib);
 429
 430   PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));
 431
 432   // PlatformJD hasn't been set up by the platform yet (since we're creating
 433   // the platform now), so set it up.
 434   if (auto E2 = setupJITDylib(PlatformJD)) {
 435     Err = std::move(E2);
 436     return;
 437   }
 438
 439   for (auto& Lib : DylibsToPreload)
 440     if (auto E2 = this->LoadDynLibrary(PlatformJD, Lib)) {
 441       Err = std::move(E2);
 442       return;
 443     }
 444
 445   if (StaticVCRuntime)
 446       if (auto E2 = VCRuntimeBootstrap->initializeStaticVCRuntime(PlatformJD)) {
 447           Err = std::move(E2);
 448           return;
 449       }
 450
 451   // Associate wrapper function tags with JIT-side function implementations.
 452   if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
 453       Err = std::move(E2);
 454       return;
 455   }
 456
 457   // Lookup addresses of runtime functions callable by the platform,
 458   // call the platform bootstrap function to initialize the platform-state
 459   // object in the executor.
 460   if (auto E2 = bootstrapCOFFRuntime(PlatformJD)) {
 461       Err = std::move(E2);
 462       return;
 463   }
 464
 465   Bootstrapping.store(false);
 466   JDBootstrapStates.clear();
 467 }
 468
 469 Expected<COFFPlatform::JITDylibDepMap>
 470 COFFPlatform::buildJDDepMap(JITDylib &JD) {
 471   return ES.runSessionLocked([&]() -> Expected<JITDylibDepMap> {
 472     JITDylibDepMap JDDepMap;
 473
 474     SmallVector<JITDylib *, 16> Worklist({&JD});
 475     while (!Worklist.empty()) {
 476       auto CurJD = Worklist.back();
 477       Worklist.pop_back();
 478
 479       auto &DM = JDDepMap[CurJD];
 480       CurJD->withLinkOrderDo([&](const JITDylibSearchOrder &O) {
 481         DM.reserve(O.size());
 482         for (auto &KV : O) {
 483           if (KV.first == CurJD)
 484             continue;
 485           {
 486             // Bare jitdylibs not known to the platform
 487             std::lock_guard<std::mutex> Lock(PlatformMutex);
 488             if (!JITDylibToHeaderAddr.count(KV.first)) {
 489               LLVM_DEBUG({
 490                 dbgs() << "JITDylib unregistered to COFFPlatform detected in "
 491                           "LinkOrder: "
 492                        << CurJD->getName() << "\n";
 493               });
 494               continue;
 495             }
 496           }
 497           DM.push_back(KV.first);
 498           // Push unvisited entry.
 499           if (!JDDepMap.count(KV.first)) {
 500             Worklist.push_back(KV.first);
 501             JDDepMap[KV.first] = {};
 502           }
 503         }
 504       });
 505     }
 506     return std::move(JDDepMap);
 507   });
 508 }
 509
 510 void COFFPlatform::pushInitializersLoop(PushInitializersSendResultFn SendResult,
 511                                         JITDylibSP JD,
 512                                         JITDylibDepMap &JDDepMap) {
 513   SmallVector<JITDylib *, 16> Worklist({JD.get()});
 514   DenseSet<JITDylib *> Visited({JD.get()});
 515   DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
 516   ES.runSessionLocked([&]() {
 517     while (!Worklist.empty()) {
 518       auto CurJD = Worklist.back();
 519       Worklist.pop_back();
 520
 521       auto RISItr = RegisteredInitSymbols.find(CurJD);
 522       if (RISItr != RegisteredInitSymbols.end()) {
 523         NewInitSymbols[CurJD] = std::move(RISItr->second);
 524         RegisteredInitSymbols.erase(RISItr);
 525       }
 526
 527       for (auto *DepJD : JDDepMap[CurJD])
 528         if (Visited.insert(DepJD).second)
 529           Worklist.push_back(DepJD);
 530     }
 531   });
 532
 533   // If there are no further init symbols to look up then send the link order
 534   // (as a list of header addresses) to the caller.
 535   if (NewInitSymbols.empty()) {
 536     // Build the dep info map to return.
 537     COFFJITDylibDepInfoMap DIM;
 538     DIM.reserve(JDDepMap.size());
 539     for (auto &KV : JDDepMap) {
 540       std::lock_guard<std::mutex> Lock(PlatformMutex);
 541       COFFJITDylibDepInfo DepInfo;
 542       DepInfo.reserve(KV.second.size());
 543       for (auto &Dep : KV.second) {
 544         DepInfo.push_back(JITDylibToHeaderAddr[Dep]);
 545       }
 546       auto H = JITDylibToHeaderAddr[KV.first];
 547       DIM.push_back(std::make_pair(H, std::move(DepInfo)));
 548     }
 549     SendResult(DIM);
 550     return;
 551   }
 552
 553   // Otherwise issue a lookup and re-run this phase when it completes.
 554   lookupInitSymbolsAsync(
 555       [this, SendResult = std::move(SendResult), &JD,
 556        JDDepMap = std::move(JDDepMap)](Error Err) mutable {
 557         if (Err)
 558           SendResult(std::move(Err));
 559         else
 560           pushInitializersLoop(std::move(SendResult), JD, JDDepMap);
 561       },
 562       ES, std::move(NewInitSymbols));
 563 }
 564
 565 void COFFPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult,
 566                                        ExecutorAddr JDHeaderAddr) {
 567   JITDylibSP JD;
 568   {
 569     std::lock_guard<std::mutex> Lock(PlatformMutex);
 570     auto I = HeaderAddrToJITDylib.find(JDHeaderAddr);
 571     if (I != HeaderAddrToJITDylib.end())
 572       JD = I->second;
 573   }
 574
 575   LLVM_DEBUG({
 576     dbgs() << "COFFPlatform::rt_pushInitializers(" << JDHeaderAddr << ") ";
 577     if (JD)
 578       dbgs() << "pushing initializers for " << JD->getName() << "\n";
 579     else
 580       dbgs() << "No JITDylib for header address.\n";
 581   });
 582
 583   if (!JD) {
 584     SendResult(make_error<StringError>("No JITDylib with header addr " +
 585                                            formatv("{0:x}", JDHeaderAddr),
 586                                        inconvertibleErrorCode()));
 587     return;
 588   }
 589
 590   auto JDDepMap = buildJDDepMap(*JD);
 591   if (!JDDepMap) {
 592     SendResult(JDDepMap.takeError());
 593     return;
 594   }
 595
 596   pushInitializersLoop(std::move(SendResult), JD, *JDDepMap);
 597 }
 598
 599 void COFFPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
 600                                    ExecutorAddr Handle, StringRef SymbolName) {
 601   LLVM_DEBUG(dbgs() << "COFFPlatform::rt_lookupSymbol(\"" << Handle << "\")\n");
 602
 603   JITDylib *JD = nullptr;
 604
 605   {
 606     std::lock_guard<std::mutex> Lock(PlatformMutex);
 607     auto I = HeaderAddrToJITDylib.find(Handle);
 608     if (I != HeaderAddrToJITDylib.end())
 609       JD = I->second;
 610   }
 611
 612   if (!JD) {
 613     LLVM_DEBUG(dbgs() << "  No JITDylib for handle " << Handle << "\n");
 614     SendResult(make_error<StringError>("No JITDylib associated with handle " +
 615                                            formatv("{0:x}", Handle),
 616                                        inconvertibleErrorCode()));
 617     return;
 618   }
 619
 620   // Use functor class to work around XL build compiler issue on AIX.
 621   class RtLookupNotifyComplete {
 622   public:
 623     RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
 624         : SendResult(std::move(SendResult)) {}
 625     void operator()(Expected<SymbolMap> Result) {
 626       if (Result) {
 627         assert(Result->size() == 1 && "Unexpected result map count");
 628         SendResult(Result->begin()->second.getAddress());
 629       } else {
 630         SendResult(Result.takeError());
 631       }
 632     }
 633
 634   private:
 635     SendSymbolAddressFn SendResult;
 636   };
 637
 638   ES.lookup(
 639       LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
 640       SymbolLookupSet(ES.intern(SymbolName)), SymbolState::Ready,
 641       RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister);
 642 }
 643
 644 Error COFFPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
 645   ExecutionSession::JITDispatchHandlerAssociationMap WFs;
 646
 647   using LookupSymbolSPSSig =
 648       SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
 649   WFs[ES.intern("__orc_rt_coff_symbol_lookup_tag")] =
 650       ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this,
 651                                               &COFFPlatform::rt_lookupSymbol);
 652   using PushInitializersSPSSig =
 653       SPSExpected<SPSCOFFJITDylibDepInfoMap>(SPSExecutorAddr);
 654   WFs[ES.intern("__orc_rt_coff_push_initializers_tag")] =
 655       ES.wrapAsyncWithSPS<PushInitializersSPSSig>(
 656           this, &COFFPlatform::rt_pushInitializers);
 657
 658   return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
 659 }
 660
 661 Error COFFPlatform::runBootstrapInitializers(JDBootstrapState &BState) {
 662   llvm::sort(BState.Initializers);
 663   if (auto Err =
 664           runBootstrapSubsectionInitializers(BState, ".CRT$XIA", ".CRT$XIZ"))
 665     return Err;
 666
 667   if (auto Err = runSymbolIfExists(*BState.JD, "__run_after_c_init"))
 668     return Err;
 669
 670   if (auto Err =
 671           runBootstrapSubsectionInitializers(BState, ".CRT$XCA", ".CRT$XCZ"))
 672     return Err;
 673   return Error::success();
 674 }
 675
 676 Error COFFPlatform::runBootstrapSubsectionInitializers(JDBootstrapState &BState,
 677                                                        StringRef Start,
 678                                                        StringRef End) {
 679   for (auto &Initializer : BState.Initializers)
 680     if (Initializer.first >= Start && Initializer.first <= End &&
 681         Initializer.second) {
 682       auto Res =
 683           ES.getExecutorProcessControl().runAsVoidFunction(Initializer.second);
 684       if (!Res)
 685         return Res.takeError();
 686     }
 687   return Error::success();
 688 }
 689
 690 Error COFFPlatform::bootstrapCOFFRuntime(JITDylib &PlatformJD) {
 691   // Lookup of runtime symbols causes the collection of initializers if
 692   // it's static linking setting.
 693   if (auto Err = lookupAndRecordAddrs(
 694           ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD),
 695           {
 696               {ES.intern("__orc_rt_coff_platform_bootstrap"),
 697                &orc_rt_coff_platform_bootstrap},
 698               {ES.intern("__orc_rt_coff_platform_shutdown"),
 699                &orc_rt_coff_platform_shutdown},
 700               {ES.intern("__orc_rt_coff_register_jitdylib"),
 701                &orc_rt_coff_register_jitdylib},
 702               {ES.intern("__orc_rt_coff_deregister_jitdylib"),
 703                &orc_rt_coff_deregister_jitdylib},
 704               {ES.intern("__orc_rt_coff_register_object_sections"),
 705                &orc_rt_coff_register_object_sections},
 706               {ES.intern("__orc_rt_coff_deregister_object_sections"),
 707                &orc_rt_coff_deregister_object_sections},
 708           }))
 709     return Err;
 710
 711   // Call bootstrap functions
 712   if (auto Err = ES.callSPSWrapper<void()>(orc_rt_coff_platform_bootstrap))
 713     return Err;
 714
 715   // Do the pending jitdylib registration actions that we couldn't do
 716   // because orc runtime was not linked fully.
 717   for (auto KV : JDBootstrapStates) {
 718     auto &JDBState = KV.second;
 719     if (auto Err = ES.callSPSWrapper<void(SPSString, SPSExecutorAddr)>(
 720             orc_rt_coff_register_jitdylib, JDBState.JDName,
 721             JDBState.HeaderAddr))
 722       return Err;
 723
 724     for (auto &ObjSectionMap : JDBState.ObjectSectionsMaps)
 725       if (auto Err = ES.callSPSWrapper<void(SPSExecutorAddr,
 726                                             SPSCOFFObjectSectionsMap, bool)>(
 727               orc_rt_coff_register_object_sections, JDBState.HeaderAddr,
 728               ObjSectionMap, false))
 729         return Err;
 730   }
 731
 732   // Run static initializers collected in bootstrap stage.
 733   for (auto KV : JDBootstrapStates) {
 734     auto &JDBState = KV.second;
 735     if (auto Err = runBootstrapInitializers(JDBState))
 736       return Err;
 737   }
 738
 739   return Error::success();
 740 }
 741
 742 Error COFFPlatform::runSymbolIfExists(JITDylib &PlatformJD,
 743                                       StringRef SymbolName) {
 744   ExecutorAddr jit_function;
 745   auto AfterCLookupErr = lookupAndRecordAddrs(
 746       ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD),
 747       {{ES.intern(SymbolName), &jit_function}});
 748   if (!AfterCLookupErr) {
 749     auto Res = ES.getExecutorProcessControl().runAsVoidFunction(jit_function);
 750     if (!Res)
 751       return Res.takeError();
 752     return Error::success();
 753   }
 754   if (!AfterCLookupErr.isA<SymbolsNotFound>())
 755     return AfterCLookupErr;
 756   consumeError(std::move(AfterCLookupErr));
 757   return Error::success();
 758 }
 759
 760 void COFFPlatform::COFFPlatformPlugin::modifyPassConfig(
 761     MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
 762     jitlink::PassConfiguration &Config) {
 763
 764   bool IsBootstrapping = CP.Bootstrapping.load();
 765
 766   if (auto InitSymbol = MR.getInitializerSymbol()) {
 767     if (InitSymbol == CP.COFFHeaderStartSymbol) {
 768       Config.PostAllocationPasses.push_back(
 769           [this, &MR, IsBootstrapping](jitlink::LinkGraph &G) {
 770             return associateJITDylibHeaderSymbol(G, MR, IsBootstrapping);
 771           });
 772       return;
 773     }
 774     Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) {
 775       return preserveInitializerSections(G, MR);
 776     });
 777   }
 778
 779   if (!IsBootstrapping)
 780     Config.PostFixupPasses.push_back(
 781         [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
 782           return registerObjectPlatformSections(G, JD);
 783         });
 784   else
 785     Config.PostFixupPasses.push_back(
 786         [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
 787           return registerObjectPlatformSectionsInBootstrap(G, JD);
 788         });
 789 }
 790
 791 Error COFFPlatform::COFFPlatformPlugin::associateJITDylibHeaderSymbol(
 792     jitlink::LinkGraph &G, MaterializationResponsibility &MR,
 793     bool IsBootstraping) {
 794   auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {
 795     return *Sym->getName() == *CP.COFFHeaderStartSymbol;
 796   });
 797   assert(I != G.defined_symbols().end() && "Missing COFF header start symbol");
 798
 799   auto &JD = MR.getTargetJITDylib();
 800   std::lock_guard<std::mutex> Lock(CP.PlatformMutex);
 801   auto HeaderAddr = (*I)->getAddress();
 802   CP.JITDylibToHeaderAddr[&JD] = HeaderAddr;
 803   CP.HeaderAddrToJITDylib[HeaderAddr] = &JD;
 804   if (!IsBootstraping) {
 805     G.allocActions().push_back(
 806         {cantFail(WrapperFunctionCall::Create<
 807                   SPSArgList<SPSString, SPSExecutorAddr>>(
 808              CP.orc_rt_coff_register_jitdylib, JD.getName(), HeaderAddr)),
 809          cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
 810              CP.orc_rt_coff_deregister_jitdylib, HeaderAddr))});
 811   } else {
 812     G.allocActions().push_back(
 813         {{},
 814          cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
 815              CP.orc_rt_coff_deregister_jitdylib, HeaderAddr))});
 816     JDBootstrapState BState;
 817     BState.JD = &JD;
 818     BState.JDName = JD.getName();
 819     BState.HeaderAddr = HeaderAddr;
 820     CP.JDBootstrapStates.emplace(&JD, BState);
 821   }
 822
 823   return Error::success();
 824 }
 825
 826 Error COFFPlatform::COFFPlatformPlugin::registerObjectPlatformSections(
 827     jitlink::LinkGraph &G, JITDylib &JD) {
 828   COFFObjectSectionsMap ObjSecs;
 829   auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];
 830   assert(HeaderAddr && "Must be registered jitdylib");
 831   for (auto &S : G.sections()) {
 832     jitlink::SectionRange Range(S);
 833     if (Range.getSize())
 834       ObjSecs.push_back(std::make_pair(S.getName().str(), Range.getRange()));
 835   }
 836
 837   G.allocActions().push_back(
 838       {cantFail(WrapperFunctionCall::Create<SPSCOFFRegisterObjectSectionsArgs>(
 839            CP.orc_rt_coff_register_object_sections, HeaderAddr, ObjSecs, true)),
 840        cantFail(
 841            WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(
 842                CP.orc_rt_coff_deregister_object_sections, HeaderAddr,
 843                ObjSecs))});
 844
 845   return Error::success();
 846 }
 847
 848 Error COFFPlatform::COFFPlatformPlugin::preserveInitializerSections(
 849     jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
 850
 851   if (const auto &InitSymName = MR.getInitializerSymbol()) {
 852
 853     jitlink::Symbol *InitSym = nullptr;
 854
 855     for (auto &InitSection : G.sections()) {
 856       // Skip non-init sections.
 857       if (!isCOFFInitializerSection(InitSection.getName()) ||
 858           InitSection.empty())
 859         continue;
 860
 861       // Create the init symbol if it has not been created already and attach it
 862       // to the first block.
 863       if (!InitSym) {
 864         auto &B = **InitSection.blocks().begin();
 865         InitSym = &G.addDefinedSymbol(
 866             B, 0, *InitSymName, B.getSize(), jitlink::Linkage::Strong,
 867             jitlink::Scope::SideEffectsOnly, false, true);
 868       }
 869
 870       // Add keep-alive edges to anonymous symbols in all other init blocks.
 871       for (auto *B : InitSection.blocks()) {
 872         if (B == &InitSym->getBlock())
 873           continue;
 874
 875         auto &S = G.addAnonymousSymbol(*B, 0, B->getSize(), false, true);
 876         InitSym->getBlock().addEdge(jitlink::Edge::KeepAlive, 0, S, 0);
 877       }
 878     }
 879   }
 880
 881   return Error::success();
 882 }
 883
 884 Error COFFPlatform::COFFPlatformPlugin::
 885     registerObjectPlatformSectionsInBootstrap(jitlink::LinkGraph &G,
 886                                               JITDylib &JD) {
 887   std::lock_guard<std::mutex> Lock(CP.PlatformMutex);
 888   auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];
 889   COFFObjectSectionsMap ObjSecs;
 890   for (auto &S : G.sections()) {
 891     jitlink::SectionRange Range(S);
 892     if (Range.getSize())
 893       ObjSecs.push_back(std::make_pair(S.getName().str(), Range.getRange()));
 894   }
 895
 896   G.allocActions().push_back(
 897       {{},
 898        cantFail(
 899            WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(
 900                CP.orc_rt_coff_deregister_object_sections, HeaderAddr,
 901                ObjSecs))});
 902
 903   auto &BState = CP.JDBootstrapStates[&JD];
 904   BState.ObjectSectionsMaps.push_back(std::move(ObjSecs));
 905
 906   // Collect static initializers
 907   for (auto &S : G.sections())
 908     if (isCOFFInitializerSection(S.getName()))
 909       for (auto *B : S.blocks()) {
 910         if (B->edges_empty())
 911           continue;
 912         for (auto &E : B->edges())
 913           BState.Initializers.push_back(std::make_pair(
 914               S.getName().str(), E.getTarget().getAddress() + E.getAddend()));
 915       }
 916
 917   return Error::success();
 918 }
 919
 920 } // End namespace orc.
 921 } // End namespace llvm.