llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp

   1 //===------- COFFPlatform.cpp - Utilities for executing COFF in Orc -------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ExecutionEngine/Orc/COFFPlatform.h"
  10 #include "llvm/ExecutionEngine/Orc/DebugUtils.h"
  11 #include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h"
  12 #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
  13 #include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
  14
  15 #include "llvm/Object/COFF.h"
  16
  17 #include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
  18
  19 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
  20
  21 #define DEBUG_TYPE "orc"
  22
  23 using namespace llvm;
  24 using namespace llvm::orc;
  25 using namespace llvm::orc::shared;
  26
  27 namespace llvm {
  28 namespace orc {
  29 namespace shared {
  30
  31 using SPSCOFFJITDylibDepInfo = SPSSequence<SPSExecutorAddr>;
  32 using SPSCOFFJITDylibDepInfoMap =
  33     SPSSequence<SPSTuple<SPSExecutorAddr, SPSCOFFJITDylibDepInfo>>;
  34 using SPSCOFFObjectSectionsMap =
  35     SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>;
  36 using SPSCOFFRegisterObjectSectionsArgs =
  37     SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap, bool>;
  38 using SPSCOFFDeregisterObjectSectionsArgs =
  39     SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap>;
  40
  41 } // namespace shared
  42 } // namespace orc
  43 } // namespace llvm
  44 namespace {
  45
  46 class COFFHeaderMaterializationUnit : public MaterializationUnit {
  47 public:
  48   COFFHeaderMaterializationUnit(COFFPlatform &CP,
  49                                 const SymbolStringPtr &HeaderStartSymbol)
  50       : MaterializationUnit(createHeaderInterface(CP, HeaderStartSymbol)),
  51         CP(CP) {}
  52
  53   StringRef getName() const override { return "COFFHeaderMU"; }
  54
  55   void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
  56     unsigned PointerSize;
  57     llvm::endianness Endianness;
  58     const auto &TT = CP.getExecutionSession().getTargetTriple();
  59
  60     switch (TT.getArch()) {
  61     case Triple::x86_64:
  62       PointerSize = 8;
  63       Endianness = llvm::endianness::little;
  64       break;
  65     default:
  66       llvm_unreachable("Unrecognized architecture");
  67     }
  68
  69     auto G = std::make_unique<jitlink::LinkGraph>(
  70         "<COFFHeaderMU>", TT, PointerSize, Endianness,
  71         jitlink::getGenericEdgeKindName);
  72     auto &HeaderSection = G->createSection("__header", MemProt::Read);
  73     auto &HeaderBlock = createHeaderBlock(*G, HeaderSection);
  74
  75     // Init symbol is __ImageBase symbol.
  76     auto &ImageBaseSymbol = G->addDefinedSymbol(
  77         HeaderBlock, 0, *R->getInitializerSymbol(), HeaderBlock.getSize(),
  78         jitlink::Linkage::Strong, jitlink::Scope::Default, false, true);
  79
  80     addImageBaseRelocationEdge(HeaderBlock, ImageBaseSymbol);
  81
  82     CP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
  83   }
  84
  85   void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
  86
  87 private:
  88   struct HeaderSymbol {
  89     const char *Name;
  90     uint64_t Offset;
  91   };
  92
  93   struct NTHeader {
  94     support::ulittle32_t PEMagic;
  95     object::coff_file_header FileHeader;
  96     struct PEHeader {
  97       object::pe32plus_header Header;
  98       object::data_directory DataDirectory[COFF::NUM_DATA_DIRECTORIES + 1];
  99     } OptionalHeader;
 100   };
 101
 102   struct HeaderBlockContent {
 103     object::dos_header DOSHeader;
 104     COFFHeaderMaterializationUnit::NTHeader NTHeader;
 105   };
 106
 107   static jitlink::Block &createHeaderBlock(jitlink::LinkGraph &G,
 108                                            jitlink::Section &HeaderSection) {
 109     HeaderBlockContent Hdr = {};
 110
 111     // Set up magic
 112     Hdr.DOSHeader.Magic[0] = 'M';
 113     Hdr.DOSHeader.Magic[1] = 'Z';
 114     Hdr.DOSHeader.AddressOfNewExeHeader =
 115         offsetof(HeaderBlockContent, NTHeader);
 116     uint32_t PEMagic = *reinterpret_cast<const uint32_t *>(COFF::PEMagic);
 117     Hdr.NTHeader.PEMagic = PEMagic;
 118     Hdr.NTHeader.OptionalHeader.Header.Magic = COFF::PE32Header::PE32_PLUS;
 119
 120     switch (G.getTargetTriple().getArch()) {
 121     case Triple::x86_64:
 122       Hdr.NTHeader.FileHeader.Machine = COFF::IMAGE_FILE_MACHINE_AMD64;
 123       break;
 124     default:
 125       llvm_unreachable("Unrecognized architecture");
 126     }
 127
 128     auto HeaderContent = G.allocateContent(
 129         ArrayRef<char>(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr)));
 130
 131     return G.createContentBlock(HeaderSection, HeaderContent, ExecutorAddr(), 8,
 132                                 0);
 133   }
 134
 135   static void addImageBaseRelocationEdge(jitlink::Block &B,
 136                                          jitlink::Symbol &ImageBase) {
 137     auto ImageBaseOffset = offsetof(HeaderBlockContent, NTHeader) +
 138                            offsetof(NTHeader, OptionalHeader) +
 139                            offsetof(object::pe32plus_header, ImageBase);
 140     B.addEdge(jitlink::x86_64::Pointer64, ImageBaseOffset, ImageBase, 0);
 141   }
 142
 143   static MaterializationUnit::Interface
 144   createHeaderInterface(COFFPlatform &MOP,
 145                         const SymbolStringPtr &HeaderStartSymbol) {
 146     SymbolFlagsMap HeaderSymbolFlags;
 147
 148     HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported;
 149
 150     return MaterializationUnit::Interface(std::move(HeaderSymbolFlags),
 151                                           HeaderStartSymbol);
 152   }
 153
 154   COFFPlatform &CP;
 155 };
 156
 157 } // end anonymous namespace
 158
 159 namespace llvm {
 160 namespace orc {
 161
 162 Expected<std::unique_ptr<COFFPlatform>> COFFPlatform::Create(
 163     ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
 164     JITDylib &PlatformJD, std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,
 165     LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
 166     const char *VCRuntimePath, std::optional<SymbolAliasMap> RuntimeAliases) {
 167
 168   // If the target is not supported then bail out immediately.
 169   if (!supportedTarget(ES.getTargetTriple()))
 170     return make_error<StringError>("Unsupported COFFPlatform triple: " +
 171                                        ES.getTargetTriple().str(),
 172                                    inconvertibleErrorCode());
 173
 174   auto &EPC = ES.getExecutorProcessControl();
 175
 176   auto GeneratorArchive =
 177       object::Archive::create(OrcRuntimeArchiveBuffer->getMemBufferRef());
 178   if (!GeneratorArchive)
 179     return GeneratorArchive.takeError();
 180
 181   auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Create(
 182       ObjLinkingLayer, nullptr, std::move(*GeneratorArchive));
 183   if (!OrcRuntimeArchiveGenerator)
 184     return OrcRuntimeArchiveGenerator.takeError();
 185
 186   // We need a second instance of the archive (for now) for the Platform. We
 187   // can `cantFail` this call, since if it were going to fail it would have
 188   // failed above.
 189   auto RuntimeArchive = cantFail(
 190       object::Archive::create(OrcRuntimeArchiveBuffer->getMemBufferRef()));
 191
 192   // Create default aliases if the caller didn't supply any.
 193   if (!RuntimeAliases)
 194     RuntimeAliases = standardPlatformAliases(ES);
 195
 196   // Define the aliases.
 197   if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases))))
 198     return std::move(Err);
 199
 200   auto &HostFuncJD = ES.createBareJITDylib("$<PlatformRuntimeHostFuncJD>");
 201
 202   // Add JIT-dispatch function support symbols.
 203   if (auto Err = HostFuncJD.define(
 204           absoluteSymbols({{ES.intern("__orc_rt_jit_dispatch"),
 205                             {EPC.getJITDispatchInfo().JITDispatchFunction,
 206                              JITSymbolFlags::Exported}},
 207                            {ES.intern("__orc_rt_jit_dispatch_ctx"),
 208                             {EPC.getJITDispatchInfo().JITDispatchContext,
 209                              JITSymbolFlags::Exported}}})))
 210     return std::move(Err);
 211
 212   PlatformJD.addToLinkOrder(HostFuncJD);
 213
 214   // Create the instance.
 215   Error Err = Error::success();
 216   auto P = std::unique_ptr<COFFPlatform>(new COFFPlatform(
 217       ES, ObjLinkingLayer, PlatformJD, std::move(*OrcRuntimeArchiveGenerator),
 218       std::move(OrcRuntimeArchiveBuffer), std::move(RuntimeArchive),
 219       std::move(LoadDynLibrary), StaticVCRuntime, VCRuntimePath, Err));
 220   if (Err)
 221     return std::move(Err);
 222   return std::move(P);
 223 }
 224
 225 Expected<std::unique_ptr<COFFPlatform>>
 226 COFFPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
 227                      JITDylib &PlatformJD, const char *OrcRuntimePath,
 228                      LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
 229                      const char *VCRuntimePath,
 230                      std::optional<SymbolAliasMap> RuntimeAliases) {
 231
 232   auto ArchiveBuffer = MemoryBuffer::getFile(OrcRuntimePath);
 233   if (!ArchiveBuffer)
 234     return createFileError(OrcRuntimePath, ArchiveBuffer.getError());
 235
 236   return Create(ES, ObjLinkingLayer, PlatformJD, std::move(*ArchiveBuffer),
 237                 std::move(LoadDynLibrary), StaticVCRuntime, VCRuntimePath,
 238                 std::move(RuntimeAliases));
 239 }
 240
 241 Expected<MemoryBufferRef> COFFPlatform::getPerJDObjectFile() {
 242   auto PerJDObj = OrcRuntimeArchive->findSym("__orc_rt_coff_per_jd_marker");
 243   if (!PerJDObj)
 244     return PerJDObj.takeError();
 245
 246   if (!*PerJDObj)
 247     return make_error<StringError>("Could not find per jd object file",
 248                                    inconvertibleErrorCode());
 249
 250   auto Buffer = (*PerJDObj)->getAsBinary();
 251   if (!Buffer)
 252     return Buffer.takeError();
 253
 254   return (*Buffer)->getMemoryBufferRef();
 255 }
 256
 257 static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
 258                        ArrayRef<std::pair<const char *, const char *>> AL) {
 259   for (auto &KV : AL) {
 260     auto AliasName = ES.intern(KV.first);
 261     assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
 262     Aliases[std::move(AliasName)] = {ES.intern(KV.second),
 263                                      JITSymbolFlags::Exported};
 264   }
 265 }
 266
 267 Error COFFPlatform::setupJITDylib(JITDylib &JD) {
 268   if (auto Err = JD.define(std::make_unique<COFFHeaderMaterializationUnit>(
 269           *this, COFFHeaderStartSymbol)))
 270     return Err;
 271
 272   if (auto Err = ES.lookup({&JD}, COFFHeaderStartSymbol).takeError())
 273     return Err;
 274
 275   // Define the CXX aliases.
 276   SymbolAliasMap CXXAliases;
 277   addAliases(ES, CXXAliases, requiredCXXAliases());
 278   if (auto Err = JD.define(symbolAliases(std::move(CXXAliases))))
 279     return Err;
 280
 281   auto PerJDObj = getPerJDObjectFile();
 282   if (!PerJDObj)
 283     return PerJDObj.takeError();
 284
 285   auto I = getObjectFileInterface(ES, *PerJDObj);
 286   if (!I)
 287     return I.takeError();
 288
 289   if (auto Err = ObjLinkingLayer.add(
 290           JD, MemoryBuffer::getMemBuffer(*PerJDObj, false), std::move(*I)))
 291     return Err;
 292
 293   if (!Bootstrapping) {
 294     auto ImportedLibs = StaticVCRuntime
 295                             ? VCRuntimeBootstrap->loadStaticVCRuntime(JD)
 296                             : VCRuntimeBootstrap->loadDynamicVCRuntime(JD);
 297     if (!ImportedLibs)
 298       return ImportedLibs.takeError();
 299     for (auto &Lib : *ImportedLibs)
 300       if (auto Err = LoadDynLibrary(JD, Lib))
 301         return Err;
 302     if (StaticVCRuntime)
 303       if (auto Err = VCRuntimeBootstrap->initializeStaticVCRuntime(JD))
 304         return Err;
 305   }
 306
 307   JD.addGenerator(DLLImportDefinitionGenerator::Create(ES, ObjLinkingLayer));
 308   return Error::success();
 309 }
 310
 311 Error COFFPlatform::teardownJITDylib(JITDylib &JD) {
 312   std::lock_guard<std::mutex> Lock(PlatformMutex);
 313   auto I = JITDylibToHeaderAddr.find(&JD);
 314   if (I != JITDylibToHeaderAddr.end()) {
 315     assert(HeaderAddrToJITDylib.count(I->second) &&
 316            "HeaderAddrToJITDylib missing entry");
 317     HeaderAddrToJITDylib.erase(I->second);
 318     JITDylibToHeaderAddr.erase(I);
 319   }
 320   return Error::success();
 321 }
 322
 323 Error COFFPlatform::notifyAdding(ResourceTracker &RT,
 324                                  const MaterializationUnit &MU) {
 325   auto &JD = RT.getJITDylib();
 326   const auto &InitSym = MU.getInitializerSymbol();
 327   if (!InitSym)
 328     return Error::success();
 329
 330   RegisteredInitSymbols[&JD].add(InitSym,
 331                                  SymbolLookupFlags::WeaklyReferencedSymbol);
 332
 333   LLVM_DEBUG({
 334     dbgs() << "COFFPlatform: Registered init symbol " << *InitSym << " for MU "
 335            << MU.getName() << "\n";
 336   });
 337   return Error::success();
 338 }
 339
 340 Error COFFPlatform::notifyRemoving(ResourceTracker &RT) {
 341   llvm_unreachable("Not supported yet");
 342 }
 343
 344 SymbolAliasMap COFFPlatform::standardPlatformAliases(ExecutionSession &ES) {
 345   SymbolAliasMap Aliases;
 346   addAliases(ES, Aliases, standardRuntimeUtilityAliases());
 347   return Aliases;
 348 }
 349
 350 ArrayRef<std::pair<const char *, const char *>>
 351 COFFPlatform::requiredCXXAliases() {
 352   static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
 353       {"_CxxThrowException", "__orc_rt_coff_cxx_throw_exception"},
 354       {"_onexit", "__orc_rt_coff_onexit_per_jd"},
 355       {"atexit", "__orc_rt_coff_atexit_per_jd"}};
 356
 357   return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
 358 }
 359
 360 ArrayRef<std::pair<const char *, const char *>>
 361 COFFPlatform::standardRuntimeUtilityAliases() {
 362   static const std::pair<const char *, const char *>
 363       StandardRuntimeUtilityAliases[] = {
 364           {"__orc_rt_run_program", "__orc_rt_coff_run_program"},
 365           {"__orc_rt_jit_dlerror", "__orc_rt_coff_jit_dlerror"},
 366           {"__orc_rt_jit_dlopen", "__orc_rt_coff_jit_dlopen"},
 367           {"__orc_rt_jit_dlclose", "__orc_rt_coff_jit_dlclose"},
 368           {"__orc_rt_jit_dlsym", "__orc_rt_coff_jit_dlsym"},
 369           {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
 370
 371   return ArrayRef<std::pair<const char *, const char *>>(
 372       StandardRuntimeUtilityAliases);
 373 }
 374
 375 bool COFFPlatform::supportedTarget(const Triple &TT) {
 376   switch (TT.getArch()) {
 377   case Triple::x86_64:
 378     return true;
 379   default:
 380     return false;
 381   }
 382 }
 383
 384 COFFPlatform::COFFPlatform(
 385     ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
 386     JITDylib &PlatformJD,
 387     std::unique_ptr<StaticLibraryDefinitionGenerator> OrcRuntimeGenerator,
 388     std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,
 389     std::unique_ptr<object::Archive> OrcRuntimeArchive,
 390     LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
 391     const char *VCRuntimePath, Error &Err)
 392     : ES(ES), ObjLinkingLayer(ObjLinkingLayer),
 393       LoadDynLibrary(std::move(LoadDynLibrary)),
 394       OrcRuntimeArchiveBuffer(std::move(OrcRuntimeArchiveBuffer)),
 395       OrcRuntimeArchive(std::move(OrcRuntimeArchive)),
 396       StaticVCRuntime(StaticVCRuntime),
 397       COFFHeaderStartSymbol(ES.intern("__ImageBase")) {
 398   ErrorAsOutParameter _(&Err);
 399
 400   Bootstrapping.store(true);
 401   ObjLinkingLayer.addPlugin(std::make_unique<COFFPlatformPlugin>(*this));
 402
 403   // Load vc runtime
 404   auto VCRT =
 405       COFFVCRuntimeBootstrapper::Create(ES, ObjLinkingLayer, VCRuntimePath);
 406   if (!VCRT) {
 407     Err = VCRT.takeError();
 408     return;
 409   }
 410   VCRuntimeBootstrap = std::move(*VCRT);
 411
 412   for (auto &Lib : OrcRuntimeGenerator->getImportedDynamicLibraries())
 413     DylibsToPreload.insert(Lib);
 414
 415   auto ImportedLibs =
 416       StaticVCRuntime ? VCRuntimeBootstrap->loadStaticVCRuntime(PlatformJD)
 417                       : VCRuntimeBootstrap->loadDynamicVCRuntime(PlatformJD);
 418   if (!ImportedLibs) {
 419     Err = ImportedLibs.takeError();
 420     return;
 421   }
 422
 423   for (auto &Lib : *ImportedLibs)
 424     DylibsToPreload.insert(Lib);
 425
 426   PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));
 427
 428   // PlatformJD hasn't been set up by the platform yet (since we're creating
 429   // the platform now), so set it up.
 430   if (auto E2 = setupJITDylib(PlatformJD)) {
 431     Err = std::move(E2);
 432     return;
 433   }
 434
 435   for (auto& Lib : DylibsToPreload)
 436     if (auto E2 = this->LoadDynLibrary(PlatformJD, Lib)) {
 437       Err = std::move(E2);
 438       return;
 439     }
 440
 441   if (StaticVCRuntime)
 442       if (auto E2 = VCRuntimeBootstrap->initializeStaticVCRuntime(PlatformJD)) {
 443           Err = std::move(E2);
 444           return;
 445       }
 446
 447   // Associate wrapper function tags with JIT-side function implementations.
 448   if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
 449       Err = std::move(E2);
 450       return;
 451   }
 452
 453   // Lookup addresses of runtime functions callable by the platform,
 454   // call the platform bootstrap function to initialize the platform-state
 455   // object in the executor.
 456   if (auto E2 = bootstrapCOFFRuntime(PlatformJD)) {
 457       Err = std::move(E2);
 458       return;
 459   }
 460
 461   Bootstrapping.store(false);
 462   JDBootstrapStates.clear();
 463 }
 464
 465 Expected<COFFPlatform::JITDylibDepMap>
 466 COFFPlatform::buildJDDepMap(JITDylib &JD) {
 467   return ES.runSessionLocked([&]() -> Expected<JITDylibDepMap> {
 468     JITDylibDepMap JDDepMap;
 469
 470     SmallVector<JITDylib *, 16> Worklist({&JD});
 471     while (!Worklist.empty()) {
 472       auto CurJD = Worklist.back();
 473       Worklist.pop_back();
 474
 475       auto &DM = JDDepMap[CurJD];
 476       CurJD->withLinkOrderDo([&](const JITDylibSearchOrder &O) {
 477         DM.reserve(O.size());
 478         for (auto &KV : O) {
 479           if (KV.first == CurJD)
 480             continue;
 481           {
 482             // Bare jitdylibs not known to the platform
 483             std::lock_guard<std::mutex> Lock(PlatformMutex);
 484             if (!JITDylibToHeaderAddr.count(KV.first)) {
 485               LLVM_DEBUG({
 486                 dbgs() << "JITDylib unregistered to COFFPlatform detected in "
 487                           "LinkOrder: "
 488                        << CurJD->getName() << "\n";
 489               });
 490               continue;
 491             }
 492           }
 493           DM.push_back(KV.first);
 494           // Push unvisited entry.
 495           if (!JDDepMap.count(KV.first)) {
 496             Worklist.push_back(KV.first);
 497             JDDepMap[KV.first] = {};
 498           }
 499         }
 500       });
 501     }
 502     return std::move(JDDepMap);
 503   });
 504 }
 505
 506 void COFFPlatform::pushInitializersLoop(PushInitializersSendResultFn SendResult,
 507                                         JITDylibSP JD,
 508                                         JITDylibDepMap &JDDepMap) {
 509   SmallVector<JITDylib *, 16> Worklist({JD.get()});
 510   DenseSet<JITDylib *> Visited({JD.get()});
 511   DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
 512   ES.runSessionLocked([&]() {
 513     while (!Worklist.empty()) {
 514       auto CurJD = Worklist.back();
 515       Worklist.pop_back();
 516
 517       auto RISItr = RegisteredInitSymbols.find(CurJD);
 518       if (RISItr != RegisteredInitSymbols.end()) {
 519         NewInitSymbols[CurJD] = std::move(RISItr->second);
 520         RegisteredInitSymbols.erase(RISItr);
 521       }
 522
 523       for (auto *DepJD : JDDepMap[CurJD])
 524         if (!Visited.count(DepJD)) {
 525           Worklist.push_back(DepJD);
 526           Visited.insert(DepJD);
 527         }
 528     }
 529   });
 530
 531   // If there are no further init symbols to look up then send the link order
 532   // (as a list of header addresses) to the caller.
 533   if (NewInitSymbols.empty()) {
 534     // Build the dep info map to return.
 535     COFFJITDylibDepInfoMap DIM;
 536     DIM.reserve(JDDepMap.size());
 537     for (auto &KV : JDDepMap) {
 538       std::lock_guard<std::mutex> Lock(PlatformMutex);
 539       COFFJITDylibDepInfo DepInfo;
 540       DepInfo.reserve(KV.second.size());
 541       for (auto &Dep : KV.second) {
 542         DepInfo.push_back(JITDylibToHeaderAddr[Dep]);
 543       }
 544       auto H = JITDylibToHeaderAddr[KV.first];
 545       DIM.push_back(std::make_pair(H, std::move(DepInfo)));
 546     }
 547     SendResult(DIM);
 548     return;
 549   }
 550
 551   // Otherwise issue a lookup and re-run this phase when it completes.
 552   lookupInitSymbolsAsync(
 553       [this, SendResult = std::move(SendResult), &JD,
 554        JDDepMap = std::move(JDDepMap)](Error Err) mutable {
 555         if (Err)
 556           SendResult(std::move(Err));
 557         else
 558           pushInitializersLoop(std::move(SendResult), JD, JDDepMap);
 559       },
 560       ES, std::move(NewInitSymbols));
 561 }
 562
 563 void COFFPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult,
 564                                        ExecutorAddr JDHeaderAddr) {
 565   JITDylibSP JD;
 566   {
 567     std::lock_guard<std::mutex> Lock(PlatformMutex);
 568     auto I = HeaderAddrToJITDylib.find(JDHeaderAddr);
 569     if (I != HeaderAddrToJITDylib.end())
 570       JD = I->second;
 571   }
 572
 573   LLVM_DEBUG({
 574     dbgs() << "COFFPlatform::rt_pushInitializers(" << JDHeaderAddr << ") ";
 575     if (JD)
 576       dbgs() << "pushing initializers for " << JD->getName() << "\n";
 577     else
 578       dbgs() << "No JITDylib for header address.\n";
 579   });
 580
 581   if (!JD) {
 582     SendResult(make_error<StringError>("No JITDylib with header addr " +
 583                                            formatv("{0:x}", JDHeaderAddr),
 584                                        inconvertibleErrorCode()));
 585     return;
 586   }
 587
 588   auto JDDepMap = buildJDDepMap(*JD);
 589   if (!JDDepMap) {
 590     SendResult(JDDepMap.takeError());
 591     return;
 592   }
 593
 594   pushInitializersLoop(std::move(SendResult), JD, *JDDepMap);
 595 }
 596
 597 void COFFPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
 598                                    ExecutorAddr Handle, StringRef SymbolName) {
 599   LLVM_DEBUG(dbgs() << "COFFPlatform::rt_lookupSymbol(\"" << Handle << "\")\n");
 600
 601   JITDylib *JD = nullptr;
 602
 603   {
 604     std::lock_guard<std::mutex> Lock(PlatformMutex);
 605     auto I = HeaderAddrToJITDylib.find(Handle);
 606     if (I != HeaderAddrToJITDylib.end())
 607       JD = I->second;
 608   }
 609
 610   if (!JD) {
 611     LLVM_DEBUG(dbgs() << "  No JITDylib for handle " << Handle << "\n");
 612     SendResult(make_error<StringError>("No JITDylib associated with handle " +
 613                                            formatv("{0:x}", Handle),
 614                                        inconvertibleErrorCode()));
 615     return;
 616   }
 617
 618   // Use functor class to work around XL build compiler issue on AIX.
 619   class RtLookupNotifyComplete {
 620   public:
 621     RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
 622         : SendResult(std::move(SendResult)) {}
 623     void operator()(Expected<SymbolMap> Result) {
 624       if (Result) {
 625         assert(Result->size() == 1 && "Unexpected result map count");
 626         SendResult(Result->begin()->second.getAddress());
 627       } else {
 628         SendResult(Result.takeError());
 629       }
 630     }
 631
 632   private:
 633     SendSymbolAddressFn SendResult;
 634   };
 635
 636   ES.lookup(
 637       LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
 638       SymbolLookupSet(ES.intern(SymbolName)), SymbolState::Ready,
 639       RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister);
 640 }
 641
 642 Error COFFPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
 643   ExecutionSession::JITDispatchHandlerAssociationMap WFs;
 644
 645   using LookupSymbolSPSSig =
 646       SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
 647   WFs[ES.intern("__orc_rt_coff_symbol_lookup_tag")] =
 648       ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this,
 649                                               &COFFPlatform::rt_lookupSymbol);
 650   using PushInitializersSPSSig =
 651       SPSExpected<SPSCOFFJITDylibDepInfoMap>(SPSExecutorAddr);
 652   WFs[ES.intern("__orc_rt_coff_push_initializers_tag")] =
 653       ES.wrapAsyncWithSPS<PushInitializersSPSSig>(
 654           this, &COFFPlatform::rt_pushInitializers);
 655
 656   return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
 657 }
 658
 659 Error COFFPlatform::runBootstrapInitializers(JDBootstrapState &BState) {
 660   llvm::sort(BState.Initializers);
 661   if (auto Err =
 662           runBootstrapSubsectionInitializers(BState, ".CRT$XIA", ".CRT$XIZ"))
 663     return Err;
 664
 665   if (auto Err = runSymbolIfExists(*BState.JD, "__run_after_c_init"))
 666     return Err;
 667
 668   if (auto Err =
 669           runBootstrapSubsectionInitializers(BState, ".CRT$XCA", ".CRT$XCZ"))
 670     return Err;
 671   return Error::success();
 672 }
 673
 674 Error COFFPlatform::runBootstrapSubsectionInitializers(JDBootstrapState &BState,
 675                                                        StringRef Start,
 676                                                        StringRef End) {
 677   for (auto &Initializer : BState.Initializers)
 678     if (Initializer.first >= Start && Initializer.first <= End &&
 679         Initializer.second) {
 680       auto Res =
 681           ES.getExecutorProcessControl().runAsVoidFunction(Initializer.second);
 682       if (!Res)
 683         return Res.takeError();
 684     }
 685   return Error::success();
 686 }
 687
 688 Error COFFPlatform::bootstrapCOFFRuntime(JITDylib &PlatformJD) {
 689   // Lookup of runtime symbols causes the collection of initializers if
 690   // it's static linking setting.
 691   if (auto Err = lookupAndRecordAddrs(
 692           ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD),
 693           {
 694               {ES.intern("__orc_rt_coff_platform_bootstrap"),
 695                &orc_rt_coff_platform_bootstrap},
 696               {ES.intern("__orc_rt_coff_platform_shutdown"),
 697                &orc_rt_coff_platform_shutdown},
 698               {ES.intern("__orc_rt_coff_register_jitdylib"),
 699                &orc_rt_coff_register_jitdylib},
 700               {ES.intern("__orc_rt_coff_deregister_jitdylib"),
 701                &orc_rt_coff_deregister_jitdylib},
 702               {ES.intern("__orc_rt_coff_register_object_sections"),
 703                &orc_rt_coff_register_object_sections},
 704               {ES.intern("__orc_rt_coff_deregister_object_sections"),
 705                &orc_rt_coff_deregister_object_sections},
 706           }))
 707     return Err;
 708
 709   // Call bootstrap functions
 710   if (auto Err = ES.callSPSWrapper<void()>(orc_rt_coff_platform_bootstrap))
 711     return Err;
 712
 713   // Do the pending jitdylib registration actions that we couldn't do
 714   // because orc runtime was not linked fully.
 715   for (auto KV : JDBootstrapStates) {
 716     auto &JDBState = KV.second;
 717     if (auto Err = ES.callSPSWrapper<void(SPSString, SPSExecutorAddr)>(
 718             orc_rt_coff_register_jitdylib, JDBState.JDName,
 719             JDBState.HeaderAddr))
 720       return Err;
 721
 722     for (auto &ObjSectionMap : JDBState.ObjectSectionsMaps)
 723       if (auto Err = ES.callSPSWrapper<void(SPSExecutorAddr,
 724                                             SPSCOFFObjectSectionsMap, bool)>(
 725               orc_rt_coff_register_object_sections, JDBState.HeaderAddr,
 726               ObjSectionMap, false))
 727         return Err;
 728   }
 729
 730   // Run static initializers collected in bootstrap stage.
 731   for (auto KV : JDBootstrapStates) {
 732     auto &JDBState = KV.second;
 733     if (auto Err = runBootstrapInitializers(JDBState))
 734       return Err;
 735   }
 736
 737   return Error::success();
 738 }
 739
 740 Error COFFPlatform::runSymbolIfExists(JITDylib &PlatformJD,
 741                                       StringRef SymbolName) {
 742   ExecutorAddr jit_function;
 743   auto AfterCLookupErr = lookupAndRecordAddrs(
 744       ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD),
 745       {{ES.intern(SymbolName), &jit_function}});
 746   if (!AfterCLookupErr) {
 747     auto Res = ES.getExecutorProcessControl().runAsVoidFunction(jit_function);
 748     if (!Res)
 749       return Res.takeError();
 750     return Error::success();
 751   }
 752   if (!AfterCLookupErr.isA<SymbolsNotFound>())
 753     return AfterCLookupErr;
 754   consumeError(std::move(AfterCLookupErr));
 755   return Error::success();
 756 }
 757
 758 void COFFPlatform::COFFPlatformPlugin::modifyPassConfig(
 759     MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
 760     jitlink::PassConfiguration &Config) {
 761
 762   bool IsBootstrapping = CP.Bootstrapping.load();
 763
 764   if (auto InitSymbol = MR.getInitializerSymbol()) {
 765     if (InitSymbol == CP.COFFHeaderStartSymbol) {
 766       Config.PostAllocationPasses.push_back(
 767           [this, &MR, IsBootstrapping](jitlink::LinkGraph &G) {
 768             return associateJITDylibHeaderSymbol(G, MR, IsBootstrapping);
 769           });
 770       return;
 771     }
 772     Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) {
 773       return preserveInitializerSections(G, MR);
 774     });
 775   }
 776
 777   if (!IsBootstrapping)
 778     Config.PostFixupPasses.push_back(
 779         [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
 780           return registerObjectPlatformSections(G, JD);
 781         });
 782   else
 783     Config.PostFixupPasses.push_back(
 784         [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
 785           return registerObjectPlatformSectionsInBootstrap(G, JD);
 786         });
 787 }
 788
 789 ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap
 790 COFFPlatform::COFFPlatformPlugin::getSyntheticSymbolDependencies(
 791     MaterializationResponsibility &MR) {
 792   std::lock_guard<std::mutex> Lock(PluginMutex);
 793   auto I = InitSymbolDeps.find(&MR);
 794   if (I != InitSymbolDeps.end()) {
 795     SyntheticSymbolDependenciesMap Result;
 796     Result[MR.getInitializerSymbol()] = std::move(I->second);
 797     InitSymbolDeps.erase(&MR);
 798     return Result;
 799   }
 800   return SyntheticSymbolDependenciesMap();
 801 }
 802
 803 Error COFFPlatform::COFFPlatformPlugin::associateJITDylibHeaderSymbol(
 804     jitlink::LinkGraph &G, MaterializationResponsibility &MR,
 805     bool IsBootstraping) {
 806   auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {
 807     return Sym->getName() == *CP.COFFHeaderStartSymbol;
 808   });
 809   assert(I != G.defined_symbols().end() && "Missing COFF header start symbol");
 810
 811   auto &JD = MR.getTargetJITDylib();
 812   std::lock_guard<std::mutex> Lock(CP.PlatformMutex);
 813   auto HeaderAddr = (*I)->getAddress();
 814   CP.JITDylibToHeaderAddr[&JD] = HeaderAddr;
 815   CP.HeaderAddrToJITDylib[HeaderAddr] = &JD;
 816   if (!IsBootstraping) {
 817     G.allocActions().push_back(
 818         {cantFail(WrapperFunctionCall::Create<
 819                   SPSArgList<SPSString, SPSExecutorAddr>>(
 820              CP.orc_rt_coff_register_jitdylib, JD.getName(), HeaderAddr)),
 821          cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
 822              CP.orc_rt_coff_deregister_jitdylib, HeaderAddr))});
 823   } else {
 824     G.allocActions().push_back(
 825         {{},
 826          cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
 827              CP.orc_rt_coff_deregister_jitdylib, HeaderAddr))});
 828     JDBootstrapState BState;
 829     BState.JD = &JD;
 830     BState.JDName = JD.getName();
 831     BState.HeaderAddr = HeaderAddr;
 832     CP.JDBootstrapStates.emplace(&JD, BState);
 833   }
 834
 835   return Error::success();
 836 }
 837
 838 Error COFFPlatform::COFFPlatformPlugin::registerObjectPlatformSections(
 839     jitlink::LinkGraph &G, JITDylib &JD) {
 840   COFFObjectSectionsMap ObjSecs;
 841   auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];
 842   assert(HeaderAddr && "Must be registered jitdylib");
 843   for (auto &S : G.sections()) {
 844     jitlink::SectionRange Range(S);
 845     if (Range.getSize())
 846       ObjSecs.push_back(std::make_pair(S.getName().str(), Range.getRange()));
 847   }
 848
 849   G.allocActions().push_back(
 850       {cantFail(WrapperFunctionCall::Create<SPSCOFFRegisterObjectSectionsArgs>(
 851            CP.orc_rt_coff_register_object_sections, HeaderAddr, ObjSecs, true)),
 852        cantFail(
 853            WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(
 854                CP.orc_rt_coff_deregister_object_sections, HeaderAddr,
 855                ObjSecs))});
 856
 857   return Error::success();
 858 }
 859
 860 Error COFFPlatform::COFFPlatformPlugin::preserveInitializerSections(
 861     jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
 862   JITLinkSymbolSet InitSectionSymbols;
 863   for (auto &Sec : G.sections())
 864     if (isCOFFInitializerSection(Sec.getName()))
 865       for (auto *B : Sec.blocks())
 866         if (!B->edges_empty())
 867           InitSectionSymbols.insert(
 868               &G.addAnonymousSymbol(*B, 0, 0, false, true));
 869
 870   std::lock_guard<std::mutex> Lock(PluginMutex);
 871   InitSymbolDeps[&MR] = InitSectionSymbols;
 872   return Error::success();
 873 }
 874
 875 Error COFFPlatform::COFFPlatformPlugin::
 876     registerObjectPlatformSectionsInBootstrap(jitlink::LinkGraph &G,
 877                                               JITDylib &JD) {
 878   std::lock_guard<std::mutex> Lock(CP.PlatformMutex);
 879   auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];
 880   COFFObjectSectionsMap ObjSecs;
 881   for (auto &S : G.sections()) {
 882     jitlink::SectionRange Range(S);
 883     if (Range.getSize())
 884       ObjSecs.push_back(std::make_pair(S.getName().str(), Range.getRange()));
 885   }
 886
 887   G.allocActions().push_back(
 888       {{},
 889        cantFail(
 890            WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(
 891                CP.orc_rt_coff_deregister_object_sections, HeaderAddr,
 892                ObjSecs))});
 893
 894   auto &BState = CP.JDBootstrapStates[&JD];
 895   BState.ObjectSectionsMaps.push_back(std::move(ObjSecs));
 896
 897   // Collect static initializers
 898   for (auto &S : G.sections())
 899     if (isCOFFInitializerSection(S.getName()))
 900       for (auto *B : S.blocks()) {
 901         if (B->edges_empty())
 902           continue;
 903         for (auto &E : B->edges())
 904           BState.Initializers.push_back(std::make_pair(
 905               S.getName().str(), E.getTarget().getAddress() + E.getAddend()));
 906       }
 907
 908   return Error::success();
 909 }
 910
 911 } // End namespace orc.
 912 } // End namespace llvm.