llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp

   1 //===------ ELFNixPlatform.cpp - Utilities for executing ELFNix in Orc
   2 //-----===//
   3 //
   4 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   5 // See https://llvm.org/LICENSE.txt for license information.
   6 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   7 //
   8 //===----------------------------------------------------------------------===//
   9
  10 #include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
  11
  12 #include "llvm/ExecutionEngine/JITLink/aarch64.h"
  13 #include "llvm/ExecutionEngine/JITLink/loongarch.h"
  14 #include "llvm/ExecutionEngine/JITLink/ppc64.h"
  15 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
  16 #include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h"
  17 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
  18 #include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
  19 #include "llvm/Support/Debug.h"
  20 #include <optional>
  21
  22 #define DEBUG_TYPE "orc"
  23
  24 using namespace llvm;
  25 using namespace llvm::orc;
  26 using namespace llvm::orc::shared;
  27
  28 namespace {
  29
  30 template <typename SPSSerializer, typename... ArgTs>
  31 shared::WrapperFunctionCall::ArgDataBufferType
  32 getArgDataBufferType(const ArgTs &...Args) {
  33   shared::WrapperFunctionCall::ArgDataBufferType ArgData;
  34   ArgData.resize(SPSSerializer::size(Args...));
  35   SPSOutputBuffer OB(ArgData.empty() ? nullptr : ArgData.data(),
  36                      ArgData.size());
  37   if (SPSSerializer::serialize(OB, Args...))
  38     return ArgData;
  39   return {};
  40 }
  41
  42 std::unique_ptr<jitlink::LinkGraph> createPlatformGraph(ELFNixPlatform &MOP,
  43                                                         std::string Name) {
  44   auto &ES = MOP.getExecutionSession();
  45   return std::make_unique<jitlink::LinkGraph>(
  46       std::move(Name), ES.getSymbolStringPool(), ES.getTargetTriple(),
  47       SubtargetFeatures(), jitlink::getGenericEdgeKindName);
  48 }
  49
  50 // Creates a Bootstrap-Complete LinkGraph to run deferred actions.
  51 class ELFNixPlatformCompleteBootstrapMaterializationUnit
  52     : public MaterializationUnit {
  53 public:
  54   ELFNixPlatformCompleteBootstrapMaterializationUnit(
  55       ELFNixPlatform &MOP, StringRef PlatformJDName,
  56       SymbolStringPtr CompleteBootstrapSymbol, DeferredRuntimeFnMap DeferredAAs,
  57       ExecutorAddr ELFNixHeaderAddr, ExecutorAddr PlatformBootstrap,
  58       ExecutorAddr PlatformShutdown, ExecutorAddr RegisterJITDylib,
  59       ExecutorAddr DeregisterJITDylib)
  60       : MaterializationUnit(
  61             {{{CompleteBootstrapSymbol, JITSymbolFlags::None}}, nullptr}),
  62         MOP(MOP), PlatformJDName(PlatformJDName),
  63         CompleteBootstrapSymbol(std::move(CompleteBootstrapSymbol)),
  64         DeferredAAsMap(std::move(DeferredAAs)),
  65         ELFNixHeaderAddr(ELFNixHeaderAddr),
  66         PlatformBootstrap(PlatformBootstrap),
  67         PlatformShutdown(PlatformShutdown), RegisterJITDylib(RegisterJITDylib),
  68         DeregisterJITDylib(DeregisterJITDylib) {}
  69
  70   StringRef getName() const override {
  71     return "ELFNixPlatformCompleteBootstrap";
  72   }
  73
  74   void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
  75     using namespace jitlink;
  76     auto G = createPlatformGraph(MOP, "<OrcRTCompleteBootstrap>");
  77     auto &PlaceholderSection =
  78         G->createSection("__orc_rt_cplt_bs", MemProt::Read);
  79     auto &PlaceholderBlock =
  80         G->createZeroFillBlock(PlaceholderSection, 1, ExecutorAddr(), 1, 0);
  81     G->addDefinedSymbol(PlaceholderBlock, 0, *CompleteBootstrapSymbol, 1,
  82                         Linkage::Strong, Scope::Hidden, false, true);
  83
  84     // 1. Bootstrap the platform support code.
  85     G->allocActions().push_back(
  86         {cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
  87              PlatformBootstrap, ELFNixHeaderAddr)),
  88          cantFail(
  89              WrapperFunctionCall::Create<SPSArgList<>>(PlatformShutdown))});
  90
  91     // 2. Register the platform JITDylib.
  92     G->allocActions().push_back(
  93         {cantFail(WrapperFunctionCall::Create<
  94                   SPSArgList<SPSString, SPSExecutorAddr>>(
  95              RegisterJITDylib, PlatformJDName, ELFNixHeaderAddr)),
  96          cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
  97              DeregisterJITDylib, ELFNixHeaderAddr))});
  98
  99     // 4. Add the deferred actions to the graph.
 100     for (auto &[Fn, CallDatas] : DeferredAAsMap) {
 101       for (auto &CallData : CallDatas) {
 102         G->allocActions().push_back(
 103             {WrapperFunctionCall(Fn.first->Addr, std::move(CallData.first)),
 104              WrapperFunctionCall(Fn.second->Addr, std::move(CallData.second))});
 105       }
 106     }
 107
 108     MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
 109   }
 110
 111   void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
 112
 113 private:
 114   ELFNixPlatform &MOP;
 115   StringRef PlatformJDName;
 116   SymbolStringPtr CompleteBootstrapSymbol;
 117   DeferredRuntimeFnMap DeferredAAsMap;
 118   ExecutorAddr ELFNixHeaderAddr;
 119   ExecutorAddr PlatformBootstrap;
 120   ExecutorAddr PlatformShutdown;
 121   ExecutorAddr RegisterJITDylib;
 122   ExecutorAddr DeregisterJITDylib;
 123 };
 124
 125 class DSOHandleMaterializationUnit : public MaterializationUnit {
 126 public:
 127   DSOHandleMaterializationUnit(ELFNixPlatform &ENP,
 128                                const SymbolStringPtr &DSOHandleSymbol)
 129       : MaterializationUnit(
 130             createDSOHandleSectionInterface(ENP, DSOHandleSymbol)),
 131         ENP(ENP) {}
 132
 133   StringRef getName() const override { return "DSOHandleMU"; }
 134
 135   void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
 136
 137     auto &ES = ENP.getExecutionSession();
 138
 139     jitlink::Edge::Kind EdgeKind;
 140
 141     switch (ES.getTargetTriple().getArch()) {
 142     case Triple::x86_64:
 143       EdgeKind = jitlink::x86_64::Pointer64;
 144       break;
 145     case Triple::aarch64:
 146       EdgeKind = jitlink::aarch64::Pointer64;
 147       break;
 148     case Triple::ppc64:
 149       EdgeKind = jitlink::ppc64::Pointer64;
 150       break;
 151     case Triple::ppc64le:
 152       EdgeKind = jitlink::ppc64::Pointer64;
 153       break;
 154     case Triple::loongarch64:
 155       EdgeKind = jitlink::loongarch::Pointer64;
 156       break;
 157     default:
 158       llvm_unreachable("Unrecognized architecture");
 159     }
 160
 161     // void *__dso_handle = &__dso_handle;
 162     auto G = std::make_unique<jitlink::LinkGraph>(
 163         "<DSOHandleMU>", ES.getSymbolStringPool(), ES.getTargetTriple(),
 164         SubtargetFeatures(), jitlink::getGenericEdgeKindName);
 165     auto &DSOHandleSection =
 166         G->createSection(".data.__dso_handle", MemProt::Read);
 167     auto &DSOHandleBlock = G->createContentBlock(
 168         DSOHandleSection, getDSOHandleContent(G->getPointerSize()),
 169         orc::ExecutorAddr(), 8, 0);
 170     auto &DSOHandleSymbol = G->addDefinedSymbol(
 171         DSOHandleBlock, 0, *R->getInitializerSymbol(), DSOHandleBlock.getSize(),
 172         jitlink::Linkage::Strong, jitlink::Scope::Default, false, true);
 173     DSOHandleBlock.addEdge(EdgeKind, 0, DSOHandleSymbol, 0);
 174
 175     ENP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
 176   }
 177
 178   void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
 179
 180 private:
 181   static MaterializationUnit::Interface
 182   createDSOHandleSectionInterface(ELFNixPlatform &ENP,
 183                                   const SymbolStringPtr &DSOHandleSymbol) {
 184     SymbolFlagsMap SymbolFlags;
 185     SymbolFlags[DSOHandleSymbol] = JITSymbolFlags::Exported;
 186     return MaterializationUnit::Interface(std::move(SymbolFlags),
 187                                           DSOHandleSymbol);
 188   }
 189
 190   ArrayRef<char> getDSOHandleContent(size_t PointerSize) {
 191     static const char Content[8] = {0};
 192     assert(PointerSize <= sizeof Content);
 193     return {Content, PointerSize};
 194   }
 195
 196   ELFNixPlatform &ENP;
 197 };
 198
 199 } // end anonymous namespace
 200
 201 namespace llvm {
 202 namespace orc {
 203
 204 Expected<std::unique_ptr<ELFNixPlatform>>
 205 ELFNixPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer,
 206                        JITDylib &PlatformJD,
 207                        std::unique_ptr<DefinitionGenerator> OrcRuntime,
 208                        std::optional<SymbolAliasMap> RuntimeAliases) {
 209
 210   auto &ES = ObjLinkingLayer.getExecutionSession();
 211
 212   // If the target is not supported then bail out immediately.
 213   if (!supportedTarget(ES.getTargetTriple()))
 214     return make_error<StringError>("Unsupported ELFNixPlatform triple: " +
 215                                        ES.getTargetTriple().str(),
 216                                    inconvertibleErrorCode());
 217
 218   auto &EPC = ES.getExecutorProcessControl();
 219
 220   // Create default aliases if the caller didn't supply any.
 221   if (!RuntimeAliases) {
 222     auto StandardRuntimeAliases = standardPlatformAliases(ES, PlatformJD);
 223     if (!StandardRuntimeAliases)
 224       return StandardRuntimeAliases.takeError();
 225     RuntimeAliases = std::move(*StandardRuntimeAliases);
 226   }
 227
 228   // Define the aliases.
 229   if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases))))
 230     return std::move(Err);
 231
 232   // Add JIT-dispatch function support symbols.
 233   if (auto Err = PlatformJD.define(
 234           absoluteSymbols({{ES.intern("__orc_rt_jit_dispatch"),
 235                             {EPC.getJITDispatchInfo().JITDispatchFunction,
 236                              JITSymbolFlags::Exported}},
 237                            {ES.intern("__orc_rt_jit_dispatch_ctx"),
 238                             {EPC.getJITDispatchInfo().JITDispatchContext,
 239                              JITSymbolFlags::Exported}}})))
 240     return std::move(Err);
 241
 242   // Create the instance.
 243   Error Err = Error::success();
 244   auto P = std::unique_ptr<ELFNixPlatform>(new ELFNixPlatform(
 245       ObjLinkingLayer, PlatformJD, std::move(OrcRuntime), Err));
 246   if (Err)
 247     return std::move(Err);
 248   return std::move(P);
 249 }
 250
 251 Expected<std::unique_ptr<ELFNixPlatform>>
 252 ELFNixPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer,
 253                        JITDylib &PlatformJD, const char *OrcRuntimePath,
 254                        std::optional<SymbolAliasMap> RuntimeAliases) {
 255
 256   // Create a generator for the ORC runtime archive.
 257   auto OrcRuntimeArchiveGenerator =
 258       StaticLibraryDefinitionGenerator::Load(ObjLinkingLayer, OrcRuntimePath);
 259   if (!OrcRuntimeArchiveGenerator)
 260     return OrcRuntimeArchiveGenerator.takeError();
 261
 262   return Create(ObjLinkingLayer, PlatformJD,
 263                 std::move(*OrcRuntimeArchiveGenerator),
 264                 std::move(RuntimeAliases));
 265 }
 266
 267 Error ELFNixPlatform::setupJITDylib(JITDylib &JD) {
 268   if (auto Err = JD.define(std::make_unique<DSOHandleMaterializationUnit>(
 269           *this, DSOHandleSymbol)))
 270     return Err;
 271
 272   return ES.lookup({&JD}, DSOHandleSymbol).takeError();
 273 }
 274
 275 Error ELFNixPlatform::teardownJITDylib(JITDylib &JD) {
 276   std::lock_guard<std::mutex> Lock(PlatformMutex);
 277   auto I = JITDylibToHandleAddr.find(&JD);
 278   if (I != JITDylibToHandleAddr.end()) {
 279     assert(HandleAddrToJITDylib.count(I->second) &&
 280            "HandleAddrToJITDylib missing entry");
 281     HandleAddrToJITDylib.erase(I->second);
 282     JITDylibToHandleAddr.erase(I);
 283   }
 284   return Error::success();
 285 }
 286
 287 Error ELFNixPlatform::notifyAdding(ResourceTracker &RT,
 288                                    const MaterializationUnit &MU) {
 289
 290   auto &JD = RT.getJITDylib();
 291   const auto &InitSym = MU.getInitializerSymbol();
 292   if (!InitSym)
 293     return Error::success();
 294
 295   RegisteredInitSymbols[&JD].add(InitSym,
 296                                  SymbolLookupFlags::WeaklyReferencedSymbol);
 297   LLVM_DEBUG({
 298     dbgs() << "ELFNixPlatform: Registered init symbol " << *InitSym
 299            << " for MU " << MU.getName() << "\n";
 300   });
 301   return Error::success();
 302 }
 303
 304 Error ELFNixPlatform::notifyRemoving(ResourceTracker &RT) {
 305   llvm_unreachable("Not supported yet");
 306 }
 307
 308 static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
 309                        ArrayRef<std::pair<const char *, const char *>> AL) {
 310   for (auto &KV : AL) {
 311     auto AliasName = ES.intern(KV.first);
 312     assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
 313     Aliases[std::move(AliasName)] = {ES.intern(KV.second),
 314                                      JITSymbolFlags::Exported};
 315   }
 316 }
 317
 318 Expected<SymbolAliasMap>
 319 ELFNixPlatform::standardPlatformAliases(ExecutionSession &ES,
 320                                         JITDylib &PlatformJD) {
 321   SymbolAliasMap Aliases;
 322   addAliases(ES, Aliases, requiredCXXAliases());
 323   addAliases(ES, Aliases, standardRuntimeUtilityAliases());
 324   addAliases(ES, Aliases, standardLazyCompilationAliases());
 325   return Aliases;
 326 }
 327
 328 ArrayRef<std::pair<const char *, const char *>>
 329 ELFNixPlatform::requiredCXXAliases() {
 330   static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
 331       {"__cxa_atexit", "__orc_rt_elfnix_cxa_atexit"},
 332       {"atexit", "__orc_rt_elfnix_atexit"}};
 333
 334   return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
 335 }
 336
 337 ArrayRef<std::pair<const char *, const char *>>
 338 ELFNixPlatform::standardRuntimeUtilityAliases() {
 339   static const std::pair<const char *, const char *>
 340       StandardRuntimeUtilityAliases[] = {
 341           {"__orc_rt_run_program", "__orc_rt_elfnix_run_program"},
 342           {"__orc_rt_jit_dlerror", "__orc_rt_elfnix_jit_dlerror"},
 343           {"__orc_rt_jit_dlopen", "__orc_rt_elfnix_jit_dlopen"},
 344           {"__orc_rt_jit_dlupdate", "__orc_rt_elfnix_jit_dlupdate"},
 345           {"__orc_rt_jit_dlclose", "__orc_rt_elfnix_jit_dlclose"},
 346           {"__orc_rt_jit_dlsym", "__orc_rt_elfnix_jit_dlsym"},
 347           {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
 348
 349   return ArrayRef<std::pair<const char *, const char *>>(
 350       StandardRuntimeUtilityAliases);
 351 }
 352
 353 ArrayRef<std::pair<const char *, const char *>>
 354 ELFNixPlatform::standardLazyCompilationAliases() {
 355   static const std::pair<const char *, const char *>
 356       StandardLazyCompilationAliases[] = {
 357           {"__orc_rt_reenter", "__orc_rt_sysv_reenter"}};
 358
 359   return ArrayRef<std::pair<const char *, const char *>>(
 360       StandardLazyCompilationAliases);
 361 }
 362
 363 bool ELFNixPlatform::supportedTarget(const Triple &TT) {
 364   switch (TT.getArch()) {
 365   case Triple::x86_64:
 366   case Triple::aarch64:
 367   // FIXME: jitlink for ppc64 hasn't been well tested, leave it unsupported
 368   // right now.
 369   case Triple::ppc64le:
 370   case Triple::loongarch64:
 371     return true;
 372   default:
 373     return false;
 374   }
 375 }
 376
 377 ELFNixPlatform::ELFNixPlatform(
 378     ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
 379     std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err)
 380     : ES(ObjLinkingLayer.getExecutionSession()), PlatformJD(PlatformJD),
 381       ObjLinkingLayer(ObjLinkingLayer),
 382       DSOHandleSymbol(ES.intern("__dso_handle")) {
 383   ErrorAsOutParameter _(Err);
 384   ObjLinkingLayer.addPlugin(std::make_unique<ELFNixPlatformPlugin>(*this));
 385
 386   PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));
 387
 388   BootstrapInfo BI;
 389   Bootstrap = &BI;
 390
 391   // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating
 392   // the platform now), so set it up.
 393   if (auto E2 = setupJITDylib(PlatformJD)) {
 394     Err = std::move(E2);
 395     return;
 396   }
 397
 398   // Step (2) Request runtime registration functions to trigger
 399   // materialization..
 400   if ((Err = ES.lookup(
 401                    makeJITDylibSearchOrder(&PlatformJD),
 402                    SymbolLookupSet(
 403                        {PlatformBootstrap.Name, PlatformShutdown.Name,
 404                         RegisterJITDylib.Name, DeregisterJITDylib.Name,
 405                         RegisterInitSections.Name, DeregisterInitSections.Name,
 406                         RegisterObjectSections.Name,
 407                         DeregisterObjectSections.Name, CreatePThreadKey.Name}))
 408                  .takeError()))
 409     return;
 410
 411   // Step (3) Wait for any incidental linker work to complete.
 412   {
 413     std::unique_lock<std::mutex> Lock(BI.Mutex);
 414     BI.CV.wait(Lock, [&]() { return BI.ActiveGraphs == 0; });
 415     Bootstrap = nullptr;
 416   }
 417
 418   // Step (4) Add complete-bootstrap materialization unit and request.
 419   auto BootstrapCompleteSymbol =
 420       ES.intern("__orc_rt_elfnix_complete_bootstrap");
 421   if ((Err = PlatformJD.define(
 422            std::make_unique<ELFNixPlatformCompleteBootstrapMaterializationUnit>(
 423                *this, PlatformJD.getName(), BootstrapCompleteSymbol,
 424                std::move(BI.DeferredRTFnMap), BI.ELFNixHeaderAddr,
 425                PlatformBootstrap.Addr, PlatformShutdown.Addr,
 426                RegisterJITDylib.Addr, DeregisterJITDylib.Addr))))
 427     return;
 428   if ((Err = ES.lookup(makeJITDylibSearchOrder(
 429                            &PlatformJD, JITDylibLookupFlags::MatchAllSymbols),
 430                        std::move(BootstrapCompleteSymbol))
 431                  .takeError()))
 432     return;
 433
 434   // Associate wrapper function tags with JIT-side function implementations.
 435   if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
 436     Err = std::move(E2);
 437     return;
 438   }
 439 }
 440
 441 Error ELFNixPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
 442   ExecutionSession::JITDispatchHandlerAssociationMap WFs;
 443
 444   using RecordInitializersSPSSig =
 445       SPSExpected<SPSELFNixJITDylibDepInfoMap>(SPSExecutorAddr);
 446   WFs[ES.intern("__orc_rt_elfnix_push_initializers_tag")] =
 447       ES.wrapAsyncWithSPS<RecordInitializersSPSSig>(
 448           this, &ELFNixPlatform::rt_recordInitializers);
 449
 450   using LookupSymbolSPSSig =
 451       SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
 452   WFs[ES.intern("__orc_rt_elfnix_symbol_lookup_tag")] =
 453       ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this,
 454                                               &ELFNixPlatform::rt_lookupSymbol);
 455
 456   return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
 457 }
 458
 459 void ELFNixPlatform::pushInitializersLoop(
 460     PushInitializersSendResultFn SendResult, JITDylibSP JD) {
 461   DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
 462   DenseMap<JITDylib *, SmallVector<JITDylib *>> JDDepMap;
 463   SmallVector<JITDylib *, 16> Worklist({JD.get()});
 464
 465   ES.runSessionLocked([&]() {
 466     while (!Worklist.empty()) {
 467       // FIXME: Check for defunct dylibs.
 468
 469       auto DepJD = Worklist.back();
 470       Worklist.pop_back();
 471
 472       // If we've already visited this JITDylib on this iteration then continue.
 473       if (JDDepMap.count(DepJD))
 474         continue;
 475
 476       // Add dep info.
 477       auto &DM = JDDepMap[DepJD];
 478       DepJD->withLinkOrderDo([&](const JITDylibSearchOrder &O) {
 479         for (auto &KV : O) {
 480           if (KV.first == DepJD)
 481             continue;
 482           DM.push_back(KV.first);
 483           Worklist.push_back(KV.first);
 484         }
 485       });
 486
 487       // Add any registered init symbols.
 488       auto RISItr = RegisteredInitSymbols.find(DepJD);
 489       if (RISItr != RegisteredInitSymbols.end()) {
 490         NewInitSymbols[DepJD] = std::move(RISItr->second);
 491         RegisteredInitSymbols.erase(RISItr);
 492       }
 493     }
 494   });
 495
 496   // If there are no further init symbols to look up then send the link order
 497   // (as a list of header addresses) to the caller.
 498   if (NewInitSymbols.empty()) {
 499
 500     // To make the list intelligible to the runtime we need to convert all
 501     // JITDylib pointers to their header addresses. Only include JITDylibs
 502     // that appear in the JITDylibToHandleAddr map (i.e. those that have been
 503     // through setupJITDylib) -- bare JITDylibs aren't managed by the platform.
 504     DenseMap<JITDylib *, ExecutorAddr> HeaderAddrs;
 505     HeaderAddrs.reserve(JDDepMap.size());
 506     {
 507       std::lock_guard<std::mutex> Lock(PlatformMutex);
 508       for (auto &KV : JDDepMap) {
 509         auto I = JITDylibToHandleAddr.find(KV.first);
 510         if (I != JITDylibToHandleAddr.end())
 511           HeaderAddrs[KV.first] = I->second;
 512       }
 513     }
 514
 515     // Build the dep info map to return.
 516     ELFNixJITDylibDepInfoMap DIM;
 517     DIM.reserve(JDDepMap.size());
 518     for (auto &KV : JDDepMap) {
 519       auto HI = HeaderAddrs.find(KV.first);
 520       // Skip unmanaged JITDylibs.
 521       if (HI == HeaderAddrs.end())
 522         continue;
 523       auto H = HI->second;
 524       ELFNixJITDylibDepInfo DepInfo;
 525       for (auto &Dep : KV.second) {
 526         auto HJ = HeaderAddrs.find(Dep);
 527         if (HJ != HeaderAddrs.end())
 528           DepInfo.push_back(HJ->second);
 529       }
 530       DIM.push_back(std::make_pair(H, std::move(DepInfo)));
 531     }
 532     SendResult(DIM);
 533     return;
 534   }
 535
 536   // Otherwise issue a lookup and re-run this phase when it completes.
 537   lookupInitSymbolsAsync(
 538       [this, SendResult = std::move(SendResult), JD](Error Err) mutable {
 539         if (Err)
 540           SendResult(std::move(Err));
 541         else
 542           pushInitializersLoop(std::move(SendResult), JD);
 543       },
 544       ES, std::move(NewInitSymbols));
 545 }
 546
 547 void ELFNixPlatform::rt_recordInitializers(
 548     PushInitializersSendResultFn SendResult, ExecutorAddr JDHeaderAddr) {
 549   JITDylibSP JD;
 550   {
 551     std::lock_guard<std::mutex> Lock(PlatformMutex);
 552     auto I = HandleAddrToJITDylib.find(JDHeaderAddr);
 553     if (I != HandleAddrToJITDylib.end())
 554       JD = I->second;
 555   }
 556
 557   LLVM_DEBUG({
 558     dbgs() << "ELFNixPlatform::rt_recordInitializers(" << JDHeaderAddr << ") ";
 559     if (JD)
 560       dbgs() << "pushing initializers for " << JD->getName() << "\n";
 561     else
 562       dbgs() << "No JITDylib for header address.\n";
 563   });
 564
 565   if (!JD) {
 566     SendResult(make_error<StringError>("No JITDylib with header addr " +
 567                                            formatv("{0:x}", JDHeaderAddr),
 568                                        inconvertibleErrorCode()));
 569     return;
 570   }
 571
 572   pushInitializersLoop(std::move(SendResult), JD);
 573 }
 574
 575 void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
 576                                      ExecutorAddr Handle,
 577                                      StringRef SymbolName) {
 578   LLVM_DEBUG({
 579     dbgs() << "ELFNixPlatform::rt_lookupSymbol(\"" << Handle << "\")\n";
 580   });
 581
 582   JITDylib *JD = nullptr;
 583
 584   {
 585     std::lock_guard<std::mutex> Lock(PlatformMutex);
 586     auto I = HandleAddrToJITDylib.find(Handle);
 587     if (I != HandleAddrToJITDylib.end())
 588       JD = I->second;
 589   }
 590
 591   if (!JD) {
 592     LLVM_DEBUG(dbgs() << "  No JITDylib for handle " << Handle << "\n");
 593     SendResult(make_error<StringError>("No JITDylib associated with handle " +
 594                                            formatv("{0:x}", Handle),
 595                                        inconvertibleErrorCode()));
 596     return;
 597   }
 598
 599   // Use functor class to work around XL build compiler issue on AIX.
 600   class RtLookupNotifyComplete {
 601   public:
 602     RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
 603         : SendResult(std::move(SendResult)) {}
 604     void operator()(Expected<SymbolMap> Result) {
 605       if (Result) {
 606         assert(Result->size() == 1 && "Unexpected result map count");
 607         SendResult(Result->begin()->second.getAddress());
 608       } else {
 609         SendResult(Result.takeError());
 610       }
 611     }
 612
 613   private:
 614     SendSymbolAddressFn SendResult;
 615   };
 616
 617   ES.lookup(
 618       LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
 619       SymbolLookupSet(ES.intern(SymbolName)), SymbolState::Ready,
 620       RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister);
 621 }
 622
 623 Error ELFNixPlatform::ELFNixPlatformPlugin::bootstrapPipelineStart(
 624     jitlink::LinkGraph &G) {
 625   // Increment the active graphs count in BootstrapInfo.
 626   std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);
 627   ++MP.Bootstrap.load()->ActiveGraphs;
 628   return Error::success();
 629 }
 630
 631 Error ELFNixPlatform::ELFNixPlatformPlugin::
 632     bootstrapPipelineRecordRuntimeFunctions(jitlink::LinkGraph &G) {
 633   // Record bootstrap function names.
 634   std::pair<StringRef, ExecutorAddr *> RuntimeSymbols[] = {
 635       {*MP.DSOHandleSymbol, &MP.Bootstrap.load()->ELFNixHeaderAddr},
 636       {*MP.PlatformBootstrap.Name, &MP.PlatformBootstrap.Addr},
 637       {*MP.PlatformShutdown.Name, &MP.PlatformShutdown.Addr},
 638       {*MP.RegisterJITDylib.Name, &MP.RegisterJITDylib.Addr},
 639       {*MP.DeregisterJITDylib.Name, &MP.DeregisterJITDylib.Addr},
 640       {*MP.RegisterObjectSections.Name, &MP.RegisterObjectSections.Addr},
 641       {*MP.DeregisterObjectSections.Name, &MP.DeregisterObjectSections.Addr},
 642       {*MP.RegisterInitSections.Name, &MP.RegisterInitSections.Addr},
 643       {*MP.DeregisterInitSections.Name, &MP.DeregisterInitSections.Addr},
 644       {*MP.CreatePThreadKey.Name, &MP.CreatePThreadKey.Addr}};
 645
 646   bool RegisterELFNixHeader = false;
 647
 648   for (auto *Sym : G.defined_symbols()) {
 649     for (auto &RTSym : RuntimeSymbols) {
 650       if (Sym->hasName() && *Sym->getName() == RTSym.first) {
 651         if (*RTSym.second)
 652           return make_error<StringError>(
 653               "Duplicate " + RTSym.first +
 654                   " detected during ELFNixPlatform bootstrap",
 655               inconvertibleErrorCode());
 656
 657         if (*Sym->getName() == *MP.DSOHandleSymbol)
 658           RegisterELFNixHeader = true;
 659
 660         *RTSym.second = Sym->getAddress();
 661       }
 662     }
 663   }
 664
 665   if (RegisterELFNixHeader) {
 666     // If this graph defines the elfnix header symbol then create the internal
 667     // mapping between it and PlatformJD.
 668     std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
 669     MP.JITDylibToHandleAddr[&MP.PlatformJD] =
 670         MP.Bootstrap.load()->ELFNixHeaderAddr;
 671     MP.HandleAddrToJITDylib[MP.Bootstrap.load()->ELFNixHeaderAddr] =
 672         &MP.PlatformJD;
 673   }
 674
 675   return Error::success();
 676 }
 677
 678 Error ELFNixPlatform::ELFNixPlatformPlugin::bootstrapPipelineEnd(
 679     jitlink::LinkGraph &G) {
 680   std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);
 681   assert(MP.Bootstrap && "DeferredAAs reset before bootstrap completed");
 682   --MP.Bootstrap.load()->ActiveGraphs;
 683   // Notify Bootstrap->CV while holding the mutex because the mutex is
 684   // also keeping Bootstrap->CV alive.
 685   if (MP.Bootstrap.load()->ActiveGraphs == 0)
 686     MP.Bootstrap.load()->CV.notify_all();
 687   return Error::success();
 688 }
 689
 690 Error ELFNixPlatform::registerPerObjectSections(
 691     jitlink::LinkGraph &G, const ELFPerObjectSectionsToRegister &POSR,
 692     bool IsBootstrapping) {
 693   using SPSRegisterPerObjSectionsArgs =
 694       SPSArgList<SPSELFPerObjectSectionsToRegister>;
 695
 696   if (LLVM_UNLIKELY(IsBootstrapping)) {
 697     Bootstrap.load()->addArgumentsToRTFnMap(
 698         &RegisterObjectSections, &DeregisterObjectSections,
 699         getArgDataBufferType<SPSRegisterPerObjSectionsArgs>(POSR),
 700         getArgDataBufferType<SPSRegisterPerObjSectionsArgs>(POSR));
 701     return Error::success();
 702   }
 703
 704   G.allocActions().push_back(
 705       {cantFail(WrapperFunctionCall::Create<SPSRegisterPerObjSectionsArgs>(
 706            RegisterObjectSections.Addr, POSR)),
 707        cantFail(WrapperFunctionCall::Create<SPSRegisterPerObjSectionsArgs>(
 708            DeregisterObjectSections.Addr, POSR))});
 709
 710   return Error::success();
 711 }
 712
 713 Expected<uint64_t> ELFNixPlatform::createPThreadKey() {
 714   if (!CreatePThreadKey.Addr)
 715     return make_error<StringError>(
 716         "Attempting to create pthread key in target, but runtime support has "
 717         "not been loaded yet",
 718         inconvertibleErrorCode());
 719
 720   Expected<uint64_t> Result(0);
 721   if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>(
 722           CreatePThreadKey.Addr, Result))
 723     return std::move(Err);
 724   return Result;
 725 }
 726
 727 void ELFNixPlatform::ELFNixPlatformPlugin::modifyPassConfig(
 728     MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
 729     jitlink::PassConfiguration &Config) {
 730   using namespace jitlink;
 731
 732   bool InBootstrapPhase =
 733       &MR.getTargetJITDylib() == &MP.PlatformJD && MP.Bootstrap;
 734
 735   // If we're in the bootstrap phase then increment the active graphs.
 736   if (InBootstrapPhase) {
 737     Config.PrePrunePasses.push_back(
 738         [this](LinkGraph &G) { return bootstrapPipelineStart(G); });
 739     Config.PostAllocationPasses.push_back([this](LinkGraph &G) {
 740       return bootstrapPipelineRecordRuntimeFunctions(G);
 741     });
 742   }
 743
 744   // If the initializer symbol is the __dso_handle symbol then just add
 745   // the DSO handle support passes.
 746   if (auto InitSymbol = MR.getInitializerSymbol()) {
 747     if (InitSymbol == MP.DSOHandleSymbol && !InBootstrapPhase) {
 748       addDSOHandleSupportPasses(MR, Config);
 749       // The DSOHandle materialization unit doesn't require any other
 750       // support, so we can bail out early.
 751       return;
 752     }
 753
 754     /// Preserve init sections.
 755     Config.PrePrunePasses.push_back(
 756         [this, &MR](jitlink::LinkGraph &G) -> Error {
 757           if (auto Err = preserveInitSections(G, MR))
 758             return Err;
 759           return Error::success();
 760         });
 761   }
 762
 763   // Add passes for eh-frame and TLV support.
 764   addEHAndTLVSupportPasses(MR, Config, InBootstrapPhase);
 765
 766   // If the object contains initializers then add passes to record them.
 767   Config.PostFixupPasses.push_back([this, &JD = MR.getTargetJITDylib(),
 768                                     InBootstrapPhase](jitlink::LinkGraph &G) {
 769     return registerInitSections(G, JD, InBootstrapPhase);
 770   });
 771
 772   // If we're in the bootstrap phase then steal allocation actions and then
 773   // decrement the active graphs.
 774   if (InBootstrapPhase)
 775     Config.PostFixupPasses.push_back(
 776         [this](LinkGraph &G) { return bootstrapPipelineEnd(G); });
 777 }
 778
 779 void ELFNixPlatform::ELFNixPlatformPlugin::addDSOHandleSupportPasses(
 780     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
 781
 782   Config.PostAllocationPasses.push_back([this, &JD = MR.getTargetJITDylib()](
 783                                             jitlink::LinkGraph &G) -> Error {
 784     auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {
 785       return Sym->getName() == MP.DSOHandleSymbol;
 786     });
 787     assert(I != G.defined_symbols().end() && "Missing DSO handle symbol");
 788     {
 789       std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
 790       auto HandleAddr = (*I)->getAddress();
 791       MP.HandleAddrToJITDylib[HandleAddr] = &JD;
 792       MP.JITDylibToHandleAddr[&JD] = HandleAddr;
 793
 794       G.allocActions().push_back(
 795           {cantFail(WrapperFunctionCall::Create<
 796                     SPSArgList<SPSString, SPSExecutorAddr>>(
 797                MP.RegisterJITDylib.Addr, JD.getName(), HandleAddr)),
 798            cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
 799                MP.DeregisterJITDylib.Addr, HandleAddr))});
 800     }
 801     return Error::success();
 802   });
 803 }
 804
 805 void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses(
 806     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config,
 807     bool IsBootstrapping) {
 808
 809   // Insert TLV lowering at the start of the PostPrunePasses, since we want
 810   // it to run before GOT/PLT lowering.
 811
 812   // TODO: Check that before the fixTLVSectionsAndEdges pass, the GOT/PLT build
 813   // pass has done. Because the TLS descriptor need to be allocate in GOT.
 814   Config.PostPrunePasses.push_back(
 815       [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
 816         return fixTLVSectionsAndEdges(G, JD);
 817       });
 818
 819   // Add a pass to register the final addresses of the eh-frame and TLV sections
 820   // with the runtime.
 821   Config.PostFixupPasses.push_back([this, IsBootstrapping](
 822                                        jitlink::LinkGraph &G) -> Error {
 823     ELFPerObjectSectionsToRegister POSR;
 824
 825     if (auto *EHFrameSection = G.findSectionByName(ELFEHFrameSectionName)) {
 826       jitlink::SectionRange R(*EHFrameSection);
 827       if (!R.empty())
 828         POSR.EHFrameSection = R.getRange();
 829     }
 830
 831     // Get a pointer to the thread data section if there is one. It will be used
 832     // below.
 833     jitlink::Section *ThreadDataSection =
 834         G.findSectionByName(ELFThreadDataSectionName);
 835
 836     // Handle thread BSS section if there is one.
 837     if (auto *ThreadBSSSection = G.findSectionByName(ELFThreadBSSSectionName)) {
 838       // If there's already a thread data section in this graph then merge the
 839       // thread BSS section content into it, otherwise just treat the thread
 840       // BSS section as the thread data section.
 841       if (ThreadDataSection)
 842         G.mergeSections(*ThreadDataSection, *ThreadBSSSection);
 843       else
 844         ThreadDataSection = ThreadBSSSection;
 845     }
 846
 847     // Having merged thread BSS (if present) and thread data (if present),
 848     // record the resulting section range.
 849     if (ThreadDataSection) {
 850       jitlink::SectionRange R(*ThreadDataSection);
 851       if (!R.empty())
 852         POSR.ThreadDataSection = R.getRange();
 853     }
 854
 855     if (POSR.EHFrameSection.Start || POSR.ThreadDataSection.Start) {
 856       if (auto Err = MP.registerPerObjectSections(G, POSR, IsBootstrapping))
 857         return Err;
 858     }
 859
 860     return Error::success();
 861   });
 862 }
 863
 864 Error ELFNixPlatform::ELFNixPlatformPlugin::preserveInitSections(
 865     jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
 866
 867   if (const auto &InitSymName = MR.getInitializerSymbol()) {
 868
 869     jitlink::Symbol *InitSym = nullptr;
 870
 871     for (auto &InitSection : G.sections()) {
 872       // Skip non-init sections.
 873       if (!isELFInitializerSection(InitSection.getName()) ||
 874           InitSection.empty())
 875         continue;
 876
 877       // Create the init symbol if it has not been created already and attach it
 878       // to the first block.
 879       if (!InitSym) {
 880         auto &B = **InitSection.blocks().begin();
 881         InitSym = &G.addDefinedSymbol(
 882             B, 0, *InitSymName, B.getSize(), jitlink::Linkage::Strong,
 883             jitlink::Scope::SideEffectsOnly, false, true);
 884       }
 885
 886       // Add keep-alive edges to anonymous symbols in all other init blocks.
 887       for (auto *B : InitSection.blocks()) {
 888         if (B == &InitSym->getBlock())
 889           continue;
 890
 891         auto &S = G.addAnonymousSymbol(*B, 0, B->getSize(), false, true);
 892         InitSym->getBlock().addEdge(jitlink::Edge::KeepAlive, 0, S, 0);
 893       }
 894     }
 895   }
 896
 897   return Error::success();
 898 }
 899
 900 Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections(
 901     jitlink::LinkGraph &G, JITDylib &JD, bool IsBootstrapping) {
 902   SmallVector<ExecutorAddrRange> ELFNixPlatformSecs;
 903   LLVM_DEBUG(dbgs() << "ELFNixPlatform::registerInitSections\n");
 904
 905   SmallVector<jitlink::Section *> OrderedInitSections;
 906   for (auto &Sec : G.sections())
 907     if (isELFInitializerSection(Sec.getName()))
 908       OrderedInitSections.push_back(&Sec);
 909
 910   // FIXME: This handles priority order within the current graph, but we'll need
 911   //        to include priority information in the initializer allocation
 912   //        actions in order to respect the ordering across multiple graphs.
 913   llvm::sort(OrderedInitSections, [](const jitlink::Section *LHS,
 914                                      const jitlink::Section *RHS) {
 915     if (LHS->getName().starts_with(".init_array")) {
 916       if (RHS->getName().starts_with(".init_array")) {
 917         StringRef LHSPrioStr(LHS->getName());
 918         StringRef RHSPrioStr(RHS->getName());
 919         uint64_t LHSPriority;
 920         bool LHSHasPriority = LHSPrioStr.consume_front(".init_array.") &&
 921                               !LHSPrioStr.getAsInteger(10, LHSPriority);
 922         uint64_t RHSPriority;
 923         bool RHSHasPriority = RHSPrioStr.consume_front(".init_array.") &&
 924                               !RHSPrioStr.getAsInteger(10, RHSPriority);
 925         if (LHSHasPriority)
 926           return RHSHasPriority ? LHSPriority < RHSPriority : true;
 927         else if (RHSHasPriority)
 928           return false;
 929         // If we get here we'll fall through to the
 930         // LHS->getName() < RHS->getName() test below.
 931       } else {
 932         // .init_array[.N] comes before any non-.init_array[.N] section.
 933         return true;
 934       }
 935     }
 936     return LHS->getName() < RHS->getName();
 937   });
 938
 939   for (auto &Sec : OrderedInitSections)
 940     ELFNixPlatformSecs.push_back(jitlink::SectionRange(*Sec).getRange());
 941
 942   // Dump the scraped inits.
 943   LLVM_DEBUG({
 944     dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " init sections:\n";
 945     for (auto &Sec : G.sections()) {
 946       jitlink::SectionRange R(Sec);
 947       dbgs() << "  " << Sec.getName() << ": " << R.getRange() << "\n";
 948     }
 949   });
 950
 951   ExecutorAddr HeaderAddr;
 952   {
 953     std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
 954     auto I = MP.JITDylibToHandleAddr.find(&JD);
 955     assert(I != MP.JITDylibToHandleAddr.end() && "No header registered for JD");
 956     assert(I->second && "Null header registered for JD");
 957     HeaderAddr = I->second;
 958   }
 959
 960   using SPSRegisterInitSectionsArgs =
 961       SPSArgList<SPSExecutorAddr, SPSSequence<SPSExecutorAddrRange>>;
 962
 963   if (LLVM_UNLIKELY(IsBootstrapping)) {
 964     MP.Bootstrap.load()->addArgumentsToRTFnMap(
 965         &MP.RegisterInitSections, &MP.DeregisterInitSections,
 966         getArgDataBufferType<SPSRegisterInitSectionsArgs>(HeaderAddr,
 967                                                           ELFNixPlatformSecs),
 968         getArgDataBufferType<SPSRegisterInitSectionsArgs>(HeaderAddr,
 969                                                           ELFNixPlatformSecs));
 970     return Error::success();
 971   }
 972
 973   G.allocActions().push_back(
 974       {cantFail(WrapperFunctionCall::Create<SPSRegisterInitSectionsArgs>(
 975            MP.RegisterInitSections.Addr, HeaderAddr, ELFNixPlatformSecs)),
 976        cantFail(WrapperFunctionCall::Create<SPSRegisterInitSectionsArgs>(
 977            MP.DeregisterInitSections.Addr, HeaderAddr, ELFNixPlatformSecs))});
 978
 979   return Error::success();
 980 }
 981
 982 Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges(
 983     jitlink::LinkGraph &G, JITDylib &JD) {
 984   auto TLSGetAddrSymbolName = G.intern("__tls_get_addr");
 985   auto TLSDescResolveSymbolName = G.intern("__tlsdesc_resolver");
 986   for (auto *Sym : G.external_symbols()) {
 987     if (Sym->getName() == TLSGetAddrSymbolName) {
 988       auto TLSGetAddr =
 989           MP.getExecutionSession().intern("___orc_rt_elfnix_tls_get_addr");
 990       Sym->setName(std::move(TLSGetAddr));
 991     } else if (Sym->getName() == TLSDescResolveSymbolName) {
 992       auto TLSGetAddr =
 993           MP.getExecutionSession().intern("___orc_rt_elfnix_tlsdesc_resolver");
 994       Sym->setName(std::move(TLSGetAddr));
 995     }
 996   }
 997
 998   auto *TLSInfoEntrySection = G.findSectionByName("$__TLSINFO");
 999
1000   if (TLSInfoEntrySection) {
1001     std::optional<uint64_t> Key;
1002     {
1003       std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
1004       auto I = MP.JITDylibToPThreadKey.find(&JD);
1005       if (I != MP.JITDylibToPThreadKey.end())
1006         Key = I->second;
1007     }
1008     if (!Key) {
1009       if (auto KeyOrErr = MP.createPThreadKey())
1010         Key = *KeyOrErr;
1011       else
1012         return KeyOrErr.takeError();
1013     }
1014
1015     uint64_t PlatformKeyBits =
1016         support::endian::byte_swap(*Key, G.getEndianness());
1017
1018     for (auto *B : TLSInfoEntrySection->blocks()) {
1019       // FIXME: The TLS descriptor byte length may different with different
1020       // ISA
1021       assert(B->getSize() == (G.getPointerSize() * 2) &&
1022              "TLS descriptor must be 2 words length");
1023       auto TLSInfoEntryContent = B->getMutableContent(G);
1024       memcpy(TLSInfoEntryContent.data(), &PlatformKeyBits, G.getPointerSize());
1025     }
1026   }
1027
1028   return Error::success();
1029 }
1030
1031 } // End namespace orc.
1032 } // End namespace llvm.