llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp

   1 //===------ ELFNixPlatform.cpp - Utilities for executing ELFNix in Orc
   2 //-----===//
   3 //
   4 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   5 // See https://llvm.org/LICENSE.txt for license information.
   6 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   7 //
   8 //===----------------------------------------------------------------------===//
   9
  10 #include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
  11
  12 #include "llvm/ExecutionEngine/JITLink/aarch64.h"
  13 #include "llvm/ExecutionEngine/JITLink/ppc64.h"
  14 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
  15 #include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h"
  16 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
  17 #include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
  18 #include "llvm/Support/Debug.h"
  19 #include <optional>
  20
  21 #define DEBUG_TYPE "orc"
  22
  23 using namespace llvm;
  24 using namespace llvm::orc;
  25 using namespace llvm::orc::shared;
  26
  27 namespace {
  28
  29 template <typename SPSSerializer, typename... ArgTs>
  30 shared::WrapperFunctionCall::ArgDataBufferType
  31 getArgDataBufferType(const ArgTs &...Args) {
  32   shared::WrapperFunctionCall::ArgDataBufferType ArgData;
  33   ArgData.resize(SPSSerializer::size(Args...));
  34   SPSOutputBuffer OB(ArgData.empty() ? nullptr : ArgData.data(),
  35                      ArgData.size());
  36   if (SPSSerializer::serialize(OB, Args...))
  37     return ArgData;
  38   return {};
  39 }
  40
  41 std::unique_ptr<jitlink::LinkGraph> createPlatformGraph(ELFNixPlatform &MOP,
  42                                                         std::string Name) {
  43   unsigned PointerSize;
  44   llvm::endianness Endianness;
  45   const auto &TT = MOP.getExecutionSession().getTargetTriple();
  46
  47   switch (TT.getArch()) {
  48   case Triple::x86_64:
  49     PointerSize = 8;
  50     Endianness = llvm::endianness::little;
  51     break;
  52   case Triple::aarch64:
  53     PointerSize = 8;
  54     Endianness = llvm::endianness::little;
  55     break;
  56   case Triple::ppc64:
  57     PointerSize = 8;
  58     Endianness = llvm::endianness::big;
  59     break;
  60   case Triple::ppc64le:
  61     PointerSize = 8;
  62     Endianness = llvm::endianness::little;
  63     break;
  64   default:
  65     llvm_unreachable("Unrecognized architecture");
  66   }
  67
  68   return std::make_unique<jitlink::LinkGraph>(std::move(Name), TT, PointerSize,
  69                                               Endianness,
  70                                               jitlink::getGenericEdgeKindName);
  71 }
  72
  73 // Creates a Bootstrap-Complete LinkGraph to run deferred actions.
  74 class ELFNixPlatformCompleteBootstrapMaterializationUnit
  75     : public MaterializationUnit {
  76 public:
  77   ELFNixPlatformCompleteBootstrapMaterializationUnit(
  78       ELFNixPlatform &MOP, StringRef PlatformJDName,
  79       SymbolStringPtr CompleteBootstrapSymbol, DeferredRuntimeFnMap DeferredAAs,
  80       ExecutorAddr ELFNixHeaderAddr, ExecutorAddr PlatformBootstrap,
  81       ExecutorAddr PlatformShutdown, ExecutorAddr RegisterJITDylib,
  82       ExecutorAddr DeregisterJITDylib)
  83       : MaterializationUnit(
  84             {{{CompleteBootstrapSymbol, JITSymbolFlags::None}}, nullptr}),
  85         MOP(MOP), PlatformJDName(PlatformJDName),
  86         CompleteBootstrapSymbol(std::move(CompleteBootstrapSymbol)),
  87         DeferredAAsMap(std::move(DeferredAAs)),
  88         ELFNixHeaderAddr(ELFNixHeaderAddr),
  89         PlatformBootstrap(PlatformBootstrap),
  90         PlatformShutdown(PlatformShutdown), RegisterJITDylib(RegisterJITDylib),
  91         DeregisterJITDylib(DeregisterJITDylib) {}
  92
  93   StringRef getName() const override {
  94     return "ELFNixPlatformCompleteBootstrap";
  95   }
  96
  97   void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
  98     using namespace jitlink;
  99     auto G = createPlatformGraph(MOP, "<OrcRTCompleteBootstrap>");
 100     auto &PlaceholderSection =
 101         G->createSection("__orc_rt_cplt_bs", MemProt::Read);
 102     auto &PlaceholderBlock =
 103         G->createZeroFillBlock(PlaceholderSection, 1, ExecutorAddr(), 1, 0);
 104     G->addDefinedSymbol(PlaceholderBlock, 0, *CompleteBootstrapSymbol, 1,
 105                         Linkage::Strong, Scope::Hidden, false, true);
 106
 107     // 1. Bootstrap the platform support code.
 108     G->allocActions().push_back(
 109         {cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
 110              PlatformBootstrap, ELFNixHeaderAddr)),
 111          cantFail(
 112              WrapperFunctionCall::Create<SPSArgList<>>(PlatformShutdown))});
 113
 114     // 2. Register the platform JITDylib.
 115     G->allocActions().push_back(
 116         {cantFail(WrapperFunctionCall::Create<
 117                   SPSArgList<SPSString, SPSExecutorAddr>>(
 118              RegisterJITDylib, PlatformJDName, ELFNixHeaderAddr)),
 119          cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
 120              DeregisterJITDylib, ELFNixHeaderAddr))});
 121
 122     // 4. Add the deferred actions to the graph.
 123     for (auto &[Fn, CallDatas] : DeferredAAsMap) {
 124       for (auto &CallData : CallDatas) {
 125         G->allocActions().push_back(
 126             {WrapperFunctionCall(Fn.first->Addr, std::move(CallData.first)),
 127              WrapperFunctionCall(Fn.second->Addr, std::move(CallData.second))});
 128       }
 129     }
 130
 131     MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
 132   }
 133
 134   void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
 135
 136 private:
 137   ELFNixPlatform &MOP;
 138   StringRef PlatformJDName;
 139   SymbolStringPtr CompleteBootstrapSymbol;
 140   DeferredRuntimeFnMap DeferredAAsMap;
 141   ExecutorAddr ELFNixHeaderAddr;
 142   ExecutorAddr PlatformBootstrap;
 143   ExecutorAddr PlatformShutdown;
 144   ExecutorAddr RegisterJITDylib;
 145   ExecutorAddr DeregisterJITDylib;
 146 };
 147
 148 class DSOHandleMaterializationUnit : public MaterializationUnit {
 149 public:
 150   DSOHandleMaterializationUnit(ELFNixPlatform &ENP,
 151                                const SymbolStringPtr &DSOHandleSymbol)
 152       : MaterializationUnit(
 153             createDSOHandleSectionInterface(ENP, DSOHandleSymbol)),
 154         ENP(ENP) {}
 155
 156   StringRef getName() const override { return "DSOHandleMU"; }
 157
 158   void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
 159     unsigned PointerSize;
 160     llvm::endianness Endianness;
 161     jitlink::Edge::Kind EdgeKind;
 162     const auto &TT = ENP.getExecutionSession().getTargetTriple();
 163
 164     switch (TT.getArch()) {
 165     case Triple::x86_64:
 166       PointerSize = 8;
 167       Endianness = llvm::endianness::little;
 168       EdgeKind = jitlink::x86_64::Pointer64;
 169       break;
 170     case Triple::aarch64:
 171       PointerSize = 8;
 172       Endianness = llvm::endianness::little;
 173       EdgeKind = jitlink::aarch64::Pointer64;
 174       break;
 175     case Triple::ppc64:
 176       PointerSize = 8;
 177       Endianness = llvm::endianness::big;
 178       EdgeKind = jitlink::ppc64::Pointer64;
 179       break;
 180     case Triple::ppc64le:
 181       PointerSize = 8;
 182       Endianness = llvm::endianness::little;
 183       EdgeKind = jitlink::ppc64::Pointer64;
 184       break;
 185     default:
 186       llvm_unreachable("Unrecognized architecture");
 187     }
 188
 189     // void *__dso_handle = &__dso_handle;
 190     auto G = std::make_unique<jitlink::LinkGraph>(
 191         "<DSOHandleMU>", TT, PointerSize, Endianness,
 192         jitlink::getGenericEdgeKindName);
 193     auto &DSOHandleSection =
 194         G->createSection(".data.__dso_handle", MemProt::Read);
 195     auto &DSOHandleBlock = G->createContentBlock(
 196         DSOHandleSection, getDSOHandleContent(PointerSize), orc::ExecutorAddr(),
 197         8, 0);
 198     auto &DSOHandleSymbol = G->addDefinedSymbol(
 199         DSOHandleBlock, 0, *R->getInitializerSymbol(), DSOHandleBlock.getSize(),
 200         jitlink::Linkage::Strong, jitlink::Scope::Default, false, true);
 201     DSOHandleBlock.addEdge(EdgeKind, 0, DSOHandleSymbol, 0);
 202
 203     ENP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
 204   }
 205
 206   void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
 207
 208 private:
 209   static MaterializationUnit::Interface
 210   createDSOHandleSectionInterface(ELFNixPlatform &ENP,
 211                                   const SymbolStringPtr &DSOHandleSymbol) {
 212     SymbolFlagsMap SymbolFlags;
 213     SymbolFlags[DSOHandleSymbol] = JITSymbolFlags::Exported;
 214     return MaterializationUnit::Interface(std::move(SymbolFlags),
 215                                           DSOHandleSymbol);
 216   }
 217
 218   ArrayRef<char> getDSOHandleContent(size_t PointerSize) {
 219     static const char Content[8] = {0};
 220     assert(PointerSize <= sizeof Content);
 221     return {Content, PointerSize};
 222   }
 223
 224   ELFNixPlatform &ENP;
 225 };
 226
 227 } // end anonymous namespace
 228
 229 namespace llvm {
 230 namespace orc {
 231
 232 Expected<std::unique_ptr<ELFNixPlatform>>
 233 ELFNixPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer,
 234                        JITDylib &PlatformJD,
 235                        std::unique_ptr<DefinitionGenerator> OrcRuntime,
 236                        std::optional<SymbolAliasMap> RuntimeAliases) {
 237
 238   auto &ES = ObjLinkingLayer.getExecutionSession();
 239
 240   // If the target is not supported then bail out immediately.
 241   if (!supportedTarget(ES.getTargetTriple()))
 242     return make_error<StringError>("Unsupported ELFNixPlatform triple: " +
 243                                        ES.getTargetTriple().str(),
 244                                    inconvertibleErrorCode());
 245
 246   auto &EPC = ES.getExecutorProcessControl();
 247
 248   // Create default aliases if the caller didn't supply any.
 249   if (!RuntimeAliases) {
 250     auto StandardRuntimeAliases = standardPlatformAliases(ES, PlatformJD);
 251     if (!StandardRuntimeAliases)
 252       return StandardRuntimeAliases.takeError();
 253     RuntimeAliases = std::move(*StandardRuntimeAliases);
 254   }
 255
 256   // Define the aliases.
 257   if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases))))
 258     return std::move(Err);
 259
 260   // Add JIT-dispatch function support symbols.
 261   if (auto Err = PlatformJD.define(
 262           absoluteSymbols({{ES.intern("__orc_rt_jit_dispatch"),
 263                             {EPC.getJITDispatchInfo().JITDispatchFunction,
 264                              JITSymbolFlags::Exported}},
 265                            {ES.intern("__orc_rt_jit_dispatch_ctx"),
 266                             {EPC.getJITDispatchInfo().JITDispatchContext,
 267                              JITSymbolFlags::Exported}}})))
 268     return std::move(Err);
 269
 270   // Create the instance.
 271   Error Err = Error::success();
 272   auto P = std::unique_ptr<ELFNixPlatform>(new ELFNixPlatform(
 273       ObjLinkingLayer, PlatformJD, std::move(OrcRuntime), Err));
 274   if (Err)
 275     return std::move(Err);
 276   return std::move(P);
 277 }
 278
 279 Expected<std::unique_ptr<ELFNixPlatform>>
 280 ELFNixPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer,
 281                        JITDylib &PlatformJD, const char *OrcRuntimePath,
 282                        std::optional<SymbolAliasMap> RuntimeAliases) {
 283
 284   // Create a generator for the ORC runtime archive.
 285   auto OrcRuntimeArchiveGenerator =
 286       StaticLibraryDefinitionGenerator::Load(ObjLinkingLayer, OrcRuntimePath);
 287   if (!OrcRuntimeArchiveGenerator)
 288     return OrcRuntimeArchiveGenerator.takeError();
 289
 290   return Create(ObjLinkingLayer, PlatformJD,
 291                 std::move(*OrcRuntimeArchiveGenerator),
 292                 std::move(RuntimeAliases));
 293 }
 294
 295 Error ELFNixPlatform::setupJITDylib(JITDylib &JD) {
 296   if (auto Err = JD.define(std::make_unique<DSOHandleMaterializationUnit>(
 297           *this, DSOHandleSymbol)))
 298     return Err;
 299
 300   return ES.lookup({&JD}, DSOHandleSymbol).takeError();
 301 }
 302
 303 Error ELFNixPlatform::teardownJITDylib(JITDylib &JD) {
 304   std::lock_guard<std::mutex> Lock(PlatformMutex);
 305   auto I = JITDylibToHandleAddr.find(&JD);
 306   if (I != JITDylibToHandleAddr.end()) {
 307     assert(HandleAddrToJITDylib.count(I->second) &&
 308            "HandleAddrToJITDylib missing entry");
 309     HandleAddrToJITDylib.erase(I->second);
 310     JITDylibToHandleAddr.erase(I);
 311   }
 312   return Error::success();
 313 }
 314
 315 Error ELFNixPlatform::notifyAdding(ResourceTracker &RT,
 316                                    const MaterializationUnit &MU) {
 317
 318   auto &JD = RT.getJITDylib();
 319   const auto &InitSym = MU.getInitializerSymbol();
 320   if (!InitSym)
 321     return Error::success();
 322
 323   RegisteredInitSymbols[&JD].add(InitSym,
 324                                  SymbolLookupFlags::WeaklyReferencedSymbol);
 325   LLVM_DEBUG({
 326     dbgs() << "ELFNixPlatform: Registered init symbol " << *InitSym
 327            << " for MU " << MU.getName() << "\n";
 328   });
 329   return Error::success();
 330 }
 331
 332 Error ELFNixPlatform::notifyRemoving(ResourceTracker &RT) {
 333   llvm_unreachable("Not supported yet");
 334 }
 335
 336 static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
 337                        ArrayRef<std::pair<const char *, const char *>> AL) {
 338   for (auto &KV : AL) {
 339     auto AliasName = ES.intern(KV.first);
 340     assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
 341     Aliases[std::move(AliasName)] = {ES.intern(KV.second),
 342                                      JITSymbolFlags::Exported};
 343   }
 344 }
 345
 346 Expected<SymbolAliasMap>
 347 ELFNixPlatform::standardPlatformAliases(ExecutionSession &ES,
 348                                         JITDylib &PlatformJD) {
 349   SymbolAliasMap Aliases;
 350   addAliases(ES, Aliases, requiredCXXAliases());
 351   addAliases(ES, Aliases, standardRuntimeUtilityAliases());
 352   return Aliases;
 353 }
 354
 355 ArrayRef<std::pair<const char *, const char *>>
 356 ELFNixPlatform::requiredCXXAliases() {
 357   static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
 358       {"__cxa_atexit", "__orc_rt_elfnix_cxa_atexit"},
 359       {"atexit", "__orc_rt_elfnix_atexit"}};
 360
 361   return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
 362 }
 363
 364 ArrayRef<std::pair<const char *, const char *>>
 365 ELFNixPlatform::standardRuntimeUtilityAliases() {
 366   static const std::pair<const char *, const char *>
 367       StandardRuntimeUtilityAliases[] = {
 368           {"__orc_rt_run_program", "__orc_rt_elfnix_run_program"},
 369           {"__orc_rt_jit_dlerror", "__orc_rt_elfnix_jit_dlerror"},
 370           {"__orc_rt_jit_dlopen", "__orc_rt_elfnix_jit_dlopen"},
 371           {"__orc_rt_jit_dlupdate", "__orc_rt_elfnix_jit_dlupdate"},
 372           {"__orc_rt_jit_dlclose", "__orc_rt_elfnix_jit_dlclose"},
 373           {"__orc_rt_jit_dlsym", "__orc_rt_elfnix_jit_dlsym"},
 374           {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
 375
 376   return ArrayRef<std::pair<const char *, const char *>>(
 377       StandardRuntimeUtilityAliases);
 378 }
 379
 380 bool ELFNixPlatform::supportedTarget(const Triple &TT) {
 381   switch (TT.getArch()) {
 382   case Triple::x86_64:
 383   case Triple::aarch64:
 384   // FIXME: jitlink for ppc64 hasn't been well tested, leave it unsupported
 385   // right now.
 386   case Triple::ppc64le:
 387     return true;
 388   default:
 389     return false;
 390   }
 391 }
 392
 393 ELFNixPlatform::ELFNixPlatform(
 394     ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
 395     std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err)
 396     : ES(ObjLinkingLayer.getExecutionSession()), PlatformJD(PlatformJD),
 397       ObjLinkingLayer(ObjLinkingLayer),
 398       DSOHandleSymbol(ES.intern("__dso_handle")) {
 399   ErrorAsOutParameter _(&Err);
 400   ObjLinkingLayer.addPlugin(std::make_unique<ELFNixPlatformPlugin>(*this));
 401
 402   PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));
 403
 404   BootstrapInfo BI;
 405   Bootstrap = &BI;
 406
 407   // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating
 408   // the platform now), so set it up.
 409   if (auto E2 = setupJITDylib(PlatformJD)) {
 410     Err = std::move(E2);
 411     return;
 412   }
 413
 414   // Step (2) Request runtime registration functions to trigger
 415   // materialization..
 416   if ((Err = ES.lookup(
 417                    makeJITDylibSearchOrder(&PlatformJD),
 418                    SymbolLookupSet(
 419                        {PlatformBootstrap.Name, PlatformShutdown.Name,
 420                         RegisterJITDylib.Name, DeregisterJITDylib.Name,
 421                         RegisterInitSections.Name, DeregisterInitSections.Name,
 422                         RegisterObjectSections.Name,
 423                         DeregisterObjectSections.Name, CreatePThreadKey.Name}))
 424                  .takeError()))
 425     return;
 426
 427   // Step (3) Wait for any incidental linker work to complete.
 428   {
 429     std::unique_lock<std::mutex> Lock(BI.Mutex);
 430     BI.CV.wait(Lock, [&]() { return BI.ActiveGraphs == 0; });
 431     Bootstrap = nullptr;
 432   }
 433
 434   // Step (4) Add complete-bootstrap materialization unit and request.
 435   auto BootstrapCompleteSymbol =
 436       ES.intern("__orc_rt_elfnix_complete_bootstrap");
 437   if ((Err = PlatformJD.define(
 438            std::make_unique<ELFNixPlatformCompleteBootstrapMaterializationUnit>(
 439                *this, PlatformJD.getName(), BootstrapCompleteSymbol,
 440                std::move(BI.DeferredRTFnMap), BI.ELFNixHeaderAddr,
 441                PlatformBootstrap.Addr, PlatformShutdown.Addr,
 442                RegisterJITDylib.Addr, DeregisterJITDylib.Addr))))
 443     return;
 444   if ((Err = ES.lookup(makeJITDylibSearchOrder(
 445                            &PlatformJD, JITDylibLookupFlags::MatchAllSymbols),
 446                        std::move(BootstrapCompleteSymbol))
 447                  .takeError()))
 448     return;
 449
 450   // Associate wrapper function tags with JIT-side function implementations.
 451   if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
 452     Err = std::move(E2);
 453     return;
 454   }
 455 }
 456
 457 Error ELFNixPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
 458   ExecutionSession::JITDispatchHandlerAssociationMap WFs;
 459
 460   using RecordInitializersSPSSig =
 461       SPSExpected<SPSELFNixJITDylibDepInfoMap>(SPSExecutorAddr);
 462   WFs[ES.intern("__orc_rt_elfnix_push_initializers_tag")] =
 463       ES.wrapAsyncWithSPS<RecordInitializersSPSSig>(
 464           this, &ELFNixPlatform::rt_recordInitializers);
 465
 466   using LookupSymbolSPSSig =
 467       SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
 468   WFs[ES.intern("__orc_rt_elfnix_symbol_lookup_tag")] =
 469       ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this,
 470                                               &ELFNixPlatform::rt_lookupSymbol);
 471
 472   return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
 473 }
 474
 475 void ELFNixPlatform::pushInitializersLoop(
 476     PushInitializersSendResultFn SendResult, JITDylibSP JD) {
 477   DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
 478   DenseMap<JITDylib *, SmallVector<JITDylib *>> JDDepMap;
 479   SmallVector<JITDylib *, 16> Worklist({JD.get()});
 480
 481   ES.runSessionLocked([&]() {
 482     while (!Worklist.empty()) {
 483       // FIXME: Check for defunct dylibs.
 484
 485       auto DepJD = Worklist.back();
 486       Worklist.pop_back();
 487
 488       // If we've already visited this JITDylib on this iteration then continue.
 489       if (JDDepMap.count(DepJD))
 490         continue;
 491
 492       // Add dep info.
 493       auto &DM = JDDepMap[DepJD];
 494       DepJD->withLinkOrderDo([&](const JITDylibSearchOrder &O) {
 495         for (auto &KV : O) {
 496           if (KV.first == DepJD)
 497             continue;
 498           DM.push_back(KV.first);
 499           Worklist.push_back(KV.first);
 500         }
 501       });
 502
 503       // Add any registered init symbols.
 504       auto RISItr = RegisteredInitSymbols.find(DepJD);
 505       if (RISItr != RegisteredInitSymbols.end()) {
 506         NewInitSymbols[DepJD] = std::move(RISItr->second);
 507         RegisteredInitSymbols.erase(RISItr);
 508       }
 509     }
 510   });
 511
 512   // If there are no further init symbols to look up then send the link order
 513   // (as a list of header addresses) to the caller.
 514   if (NewInitSymbols.empty()) {
 515
 516     // To make the list intelligible to the runtime we need to convert all
 517     // JITDylib pointers to their header addresses. Only include JITDylibs
 518     // that appear in the JITDylibToHandleAddr map (i.e. those that have been
 519     // through setupJITDylib) -- bare JITDylibs aren't managed by the platform.
 520     DenseMap<JITDylib *, ExecutorAddr> HeaderAddrs;
 521     HeaderAddrs.reserve(JDDepMap.size());
 522     {
 523       std::lock_guard<std::mutex> Lock(PlatformMutex);
 524       for (auto &KV : JDDepMap) {
 525         auto I = JITDylibToHandleAddr.find(KV.first);
 526         if (I != JITDylibToHandleAddr.end())
 527           HeaderAddrs[KV.first] = I->second;
 528       }
 529     }
 530
 531     // Build the dep info map to return.
 532     ELFNixJITDylibDepInfoMap DIM;
 533     DIM.reserve(JDDepMap.size());
 534     for (auto &KV : JDDepMap) {
 535       auto HI = HeaderAddrs.find(KV.first);
 536       // Skip unmanaged JITDylibs.
 537       if (HI == HeaderAddrs.end())
 538         continue;
 539       auto H = HI->second;
 540       ELFNixJITDylibDepInfo DepInfo;
 541       for (auto &Dep : KV.second) {
 542         auto HJ = HeaderAddrs.find(Dep);
 543         if (HJ != HeaderAddrs.end())
 544           DepInfo.push_back(HJ->second);
 545       }
 546       DIM.push_back(std::make_pair(H, std::move(DepInfo)));
 547     }
 548     SendResult(DIM);
 549     return;
 550   }
 551
 552   // Otherwise issue a lookup and re-run this phase when it completes.
 553   lookupInitSymbolsAsync(
 554       [this, SendResult = std::move(SendResult), JD](Error Err) mutable {
 555         if (Err)
 556           SendResult(std::move(Err));
 557         else
 558           pushInitializersLoop(std::move(SendResult), JD);
 559       },
 560       ES, std::move(NewInitSymbols));
 561 }
 562
 563 void ELFNixPlatform::rt_recordInitializers(
 564     PushInitializersSendResultFn SendResult, ExecutorAddr JDHeaderAddr) {
 565   JITDylibSP JD;
 566   {
 567     std::lock_guard<std::mutex> Lock(PlatformMutex);
 568     auto I = HandleAddrToJITDylib.find(JDHeaderAddr);
 569     if (I != HandleAddrToJITDylib.end())
 570       JD = I->second;
 571   }
 572
 573   LLVM_DEBUG({
 574     dbgs() << "ELFNixPlatform::rt_recordInitializers(" << JDHeaderAddr << ") ";
 575     if (JD)
 576       dbgs() << "pushing initializers for " << JD->getName() << "\n";
 577     else
 578       dbgs() << "No JITDylib for header address.\n";
 579   });
 580
 581   if (!JD) {
 582     SendResult(make_error<StringError>("No JITDylib with header addr " +
 583                                            formatv("{0:x}", JDHeaderAddr),
 584                                        inconvertibleErrorCode()));
 585     return;
 586   }
 587
 588   pushInitializersLoop(std::move(SendResult), JD);
 589 }
 590
 591 void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
 592                                      ExecutorAddr Handle,
 593                                      StringRef SymbolName) {
 594   LLVM_DEBUG({
 595     dbgs() << "ELFNixPlatform::rt_lookupSymbol(\"" << Handle << "\")\n";
 596   });
 597
 598   JITDylib *JD = nullptr;
 599
 600   {
 601     std::lock_guard<std::mutex> Lock(PlatformMutex);
 602     auto I = HandleAddrToJITDylib.find(Handle);
 603     if (I != HandleAddrToJITDylib.end())
 604       JD = I->second;
 605   }
 606
 607   if (!JD) {
 608     LLVM_DEBUG(dbgs() << "  No JITDylib for handle " << Handle << "\n");
 609     SendResult(make_error<StringError>("No JITDylib associated with handle " +
 610                                            formatv("{0:x}", Handle),
 611                                        inconvertibleErrorCode()));
 612     return;
 613   }
 614
 615   // Use functor class to work around XL build compiler issue on AIX.
 616   class RtLookupNotifyComplete {
 617   public:
 618     RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
 619         : SendResult(std::move(SendResult)) {}
 620     void operator()(Expected<SymbolMap> Result) {
 621       if (Result) {
 622         assert(Result->size() == 1 && "Unexpected result map count");
 623         SendResult(Result->begin()->second.getAddress());
 624       } else {
 625         SendResult(Result.takeError());
 626       }
 627     }
 628
 629   private:
 630     SendSymbolAddressFn SendResult;
 631   };
 632
 633   ES.lookup(
 634       LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
 635       SymbolLookupSet(ES.intern(SymbolName)), SymbolState::Ready,
 636       RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister);
 637 }
 638
 639 Error ELFNixPlatform::ELFNixPlatformPlugin::bootstrapPipelineStart(
 640     jitlink::LinkGraph &G) {
 641   // Increment the active graphs count in BootstrapInfo.
 642   std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);
 643   ++MP.Bootstrap.load()->ActiveGraphs;
 644   return Error::success();
 645 }
 646
 647 Error ELFNixPlatform::ELFNixPlatformPlugin::
 648     bootstrapPipelineRecordRuntimeFunctions(jitlink::LinkGraph &G) {
 649   // Record bootstrap function names.
 650   std::pair<StringRef, ExecutorAddr *> RuntimeSymbols[] = {
 651       {*MP.DSOHandleSymbol, &MP.Bootstrap.load()->ELFNixHeaderAddr},
 652       {*MP.PlatformBootstrap.Name, &MP.PlatformBootstrap.Addr},
 653       {*MP.PlatformShutdown.Name, &MP.PlatformShutdown.Addr},
 654       {*MP.RegisterJITDylib.Name, &MP.RegisterJITDylib.Addr},
 655       {*MP.DeregisterJITDylib.Name, &MP.DeregisterJITDylib.Addr},
 656       {*MP.RegisterObjectSections.Name, &MP.RegisterObjectSections.Addr},
 657       {*MP.DeregisterObjectSections.Name, &MP.DeregisterObjectSections.Addr},
 658       {*MP.RegisterInitSections.Name, &MP.RegisterInitSections.Addr},
 659       {*MP.DeregisterInitSections.Name, &MP.DeregisterInitSections.Addr},
 660       {*MP.CreatePThreadKey.Name, &MP.CreatePThreadKey.Addr}};
 661
 662   bool RegisterELFNixHeader = false;
 663
 664   for (auto *Sym : G.defined_symbols()) {
 665     for (auto &RTSym : RuntimeSymbols) {
 666       if (Sym->hasName() && Sym->getName() == RTSym.first) {
 667         if (*RTSym.second)
 668           return make_error<StringError>(
 669               "Duplicate " + RTSym.first +
 670                   " detected during ELFNixPlatform bootstrap",
 671               inconvertibleErrorCode());
 672
 673         if (Sym->getName() == *MP.DSOHandleSymbol)
 674           RegisterELFNixHeader = true;
 675
 676         *RTSym.second = Sym->getAddress();
 677       }
 678     }
 679   }
 680
 681   if (RegisterELFNixHeader) {
 682     // If this graph defines the elfnix header symbol then create the internal
 683     // mapping between it and PlatformJD.
 684     std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
 685     MP.JITDylibToHandleAddr[&MP.PlatformJD] =
 686         MP.Bootstrap.load()->ELFNixHeaderAddr;
 687     MP.HandleAddrToJITDylib[MP.Bootstrap.load()->ELFNixHeaderAddr] =
 688         &MP.PlatformJD;
 689   }
 690
 691   return Error::success();
 692 }
 693
 694 Error ELFNixPlatform::ELFNixPlatformPlugin::bootstrapPipelineEnd(
 695     jitlink::LinkGraph &G) {
 696   std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);
 697   assert(MP.Bootstrap && "DeferredAAs reset before bootstrap completed");
 698   --MP.Bootstrap.load()->ActiveGraphs;
 699   // Notify Bootstrap->CV while holding the mutex because the mutex is
 700   // also keeping Bootstrap->CV alive.
 701   if (MP.Bootstrap.load()->ActiveGraphs == 0)
 702     MP.Bootstrap.load()->CV.notify_all();
 703   return Error::success();
 704 }
 705
 706 Error ELFNixPlatform::registerPerObjectSections(
 707     jitlink::LinkGraph &G, const ELFPerObjectSectionsToRegister &POSR,
 708     bool IsBootstrapping) {
 709   using SPSRegisterPerObjSectionsArgs =
 710       SPSArgList<SPSELFPerObjectSectionsToRegister>;
 711
 712   if (LLVM_UNLIKELY(IsBootstrapping)) {
 713     Bootstrap.load()->addArgumentsToRTFnMap(
 714         &RegisterObjectSections, &DeregisterObjectSections,
 715         getArgDataBufferType<SPSRegisterPerObjSectionsArgs>(POSR),
 716         getArgDataBufferType<SPSRegisterPerObjSectionsArgs>(POSR));
 717     return Error::success();
 718   }
 719
 720   G.allocActions().push_back(
 721       {cantFail(WrapperFunctionCall::Create<SPSRegisterPerObjSectionsArgs>(
 722            RegisterObjectSections.Addr, POSR)),
 723        cantFail(WrapperFunctionCall::Create<SPSRegisterPerObjSectionsArgs>(
 724            DeregisterObjectSections.Addr, POSR))});
 725
 726   return Error::success();
 727 }
 728
 729 Expected<uint64_t> ELFNixPlatform::createPThreadKey() {
 730   if (!CreatePThreadKey.Addr)
 731     return make_error<StringError>(
 732         "Attempting to create pthread key in target, but runtime support has "
 733         "not been loaded yet",
 734         inconvertibleErrorCode());
 735
 736   Expected<uint64_t> Result(0);
 737   if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>(
 738           CreatePThreadKey.Addr, Result))
 739     return std::move(Err);
 740   return Result;
 741 }
 742
 743 void ELFNixPlatform::ELFNixPlatformPlugin::modifyPassConfig(
 744     MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
 745     jitlink::PassConfiguration &Config) {
 746   using namespace jitlink;
 747
 748   bool InBootstrapPhase =
 749       &MR.getTargetJITDylib() == &MP.PlatformJD && MP.Bootstrap;
 750
 751   // If we're in the bootstrap phase then increment the active graphs.
 752   if (InBootstrapPhase) {
 753     Config.PrePrunePasses.push_back(
 754         [this](LinkGraph &G) { return bootstrapPipelineStart(G); });
 755     Config.PostAllocationPasses.push_back([this](LinkGraph &G) {
 756       return bootstrapPipelineRecordRuntimeFunctions(G);
 757     });
 758   }
 759
 760   // If the initializer symbol is the __dso_handle symbol then just add
 761   // the DSO handle support passes.
 762   if (auto InitSymbol = MR.getInitializerSymbol()) {
 763     if (InitSymbol == MP.DSOHandleSymbol && !InBootstrapPhase) {
 764       addDSOHandleSupportPasses(MR, Config);
 765       // The DSOHandle materialization unit doesn't require any other
 766       // support, so we can bail out early.
 767       return;
 768     }
 769
 770     /// Preserve init sections.
 771     Config.PrePrunePasses.push_back(
 772         [this, &MR](jitlink::LinkGraph &G) -> Error {
 773           if (auto Err = preserveInitSections(G, MR))
 774             return Err;
 775           return Error::success();
 776         });
 777   }
 778
 779   // Add passes for eh-frame and TLV support.
 780   addEHAndTLVSupportPasses(MR, Config, InBootstrapPhase);
 781
 782   // If the object contains initializers then add passes to record them.
 783   Config.PostFixupPasses.push_back([this, &JD = MR.getTargetJITDylib(),
 784                                     InBootstrapPhase](jitlink::LinkGraph &G) {
 785     return registerInitSections(G, JD, InBootstrapPhase);
 786   });
 787
 788   // If we're in the bootstrap phase then steal allocation actions and then
 789   // decrement the active graphs.
 790   if (InBootstrapPhase)
 791     Config.PostFixupPasses.push_back(
 792         [this](LinkGraph &G) { return bootstrapPipelineEnd(G); });
 793 }
 794
 795 void ELFNixPlatform::ELFNixPlatformPlugin::addDSOHandleSupportPasses(
 796     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
 797
 798   Config.PostAllocationPasses.push_back([this, &JD = MR.getTargetJITDylib()](
 799                                             jitlink::LinkGraph &G) -> Error {
 800     auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {
 801       return Sym->getName() == *MP.DSOHandleSymbol;
 802     });
 803     assert(I != G.defined_symbols().end() && "Missing DSO handle symbol");
 804     {
 805       std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
 806       auto HandleAddr = (*I)->getAddress();
 807       MP.HandleAddrToJITDylib[HandleAddr] = &JD;
 808       MP.JITDylibToHandleAddr[&JD] = HandleAddr;
 809
 810       G.allocActions().push_back(
 811           {cantFail(WrapperFunctionCall::Create<
 812                     SPSArgList<SPSString, SPSExecutorAddr>>(
 813                MP.RegisterJITDylib.Addr, JD.getName(), HandleAddr)),
 814            cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
 815                MP.DeregisterJITDylib.Addr, HandleAddr))});
 816     }
 817     return Error::success();
 818   });
 819 }
 820
 821 void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses(
 822     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config,
 823     bool IsBootstrapping) {
 824
 825   // Insert TLV lowering at the start of the PostPrunePasses, since we want
 826   // it to run before GOT/PLT lowering.
 827
 828   // TODO: Check that before the fixTLVSectionsAndEdges pass, the GOT/PLT build
 829   // pass has done. Because the TLS descriptor need to be allocate in GOT.
 830   Config.PostPrunePasses.push_back(
 831       [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
 832         return fixTLVSectionsAndEdges(G, JD);
 833       });
 834
 835   // Add a pass to register the final addresses of the eh-frame and TLV sections
 836   // with the runtime.
 837   Config.PostFixupPasses.push_back([this, IsBootstrapping](
 838                                        jitlink::LinkGraph &G) -> Error {
 839     ELFPerObjectSectionsToRegister POSR;
 840
 841     if (auto *EHFrameSection = G.findSectionByName(ELFEHFrameSectionName)) {
 842       jitlink::SectionRange R(*EHFrameSection);
 843       if (!R.empty())
 844         POSR.EHFrameSection = R.getRange();
 845     }
 846
 847     // Get a pointer to the thread data section if there is one. It will be used
 848     // below.
 849     jitlink::Section *ThreadDataSection =
 850         G.findSectionByName(ELFThreadDataSectionName);
 851
 852     // Handle thread BSS section if there is one.
 853     if (auto *ThreadBSSSection = G.findSectionByName(ELFThreadBSSSectionName)) {
 854       // If there's already a thread data section in this graph then merge the
 855       // thread BSS section content into it, otherwise just treat the thread
 856       // BSS section as the thread data section.
 857       if (ThreadDataSection)
 858         G.mergeSections(*ThreadDataSection, *ThreadBSSSection);
 859       else
 860         ThreadDataSection = ThreadBSSSection;
 861     }
 862
 863     // Having merged thread BSS (if present) and thread data (if present),
 864     // record the resulting section range.
 865     if (ThreadDataSection) {
 866       jitlink::SectionRange R(*ThreadDataSection);
 867       if (!R.empty())
 868         POSR.ThreadDataSection = R.getRange();
 869     }
 870
 871     if (POSR.EHFrameSection.Start || POSR.ThreadDataSection.Start) {
 872       if (auto Err = MP.registerPerObjectSections(G, POSR, IsBootstrapping))
 873         return Err;
 874     }
 875
 876     return Error::success();
 877   });
 878 }
 879
 880 Error ELFNixPlatform::ELFNixPlatformPlugin::preserveInitSections(
 881     jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
 882
 883   if (const auto &InitSymName = MR.getInitializerSymbol()) {
 884
 885     jitlink::Symbol *InitSym = nullptr;
 886
 887     for (auto &InitSection : G.sections()) {
 888       // Skip non-init sections.
 889       if (!isELFInitializerSection(InitSection.getName()) ||
 890           InitSection.empty())
 891         continue;
 892
 893       // Create the init symbol if it has not been created already and attach it
 894       // to the first block.
 895       if (!InitSym) {
 896         auto &B = **InitSection.blocks().begin();
 897         InitSym = &G.addDefinedSymbol(B, 0, *InitSymName, B.getSize(),
 898                                       jitlink::Linkage::Strong,
 899                                       jitlink::Scope::Default, false, true);
 900       }
 901
 902       // Add keep-alive edges to anonymous symbols in all other init blocks.
 903       for (auto *B : InitSection.blocks()) {
 904         if (B == &InitSym->getBlock())
 905           continue;
 906
 907         auto &S = G.addAnonymousSymbol(*B, 0, B->getSize(), false, true);
 908         InitSym->getBlock().addEdge(jitlink::Edge::KeepAlive, 0, S, 0);
 909       }
 910     }
 911   }
 912
 913   return Error::success();
 914 }
 915
 916 Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections(
 917     jitlink::LinkGraph &G, JITDylib &JD, bool IsBootstrapping) {
 918   SmallVector<ExecutorAddrRange> ELFNixPlatformSecs;
 919   LLVM_DEBUG(dbgs() << "ELFNixPlatform::registerInitSections\n");
 920
 921   SmallVector<jitlink::Section *> OrderedInitSections;
 922   for (auto &Sec : G.sections())
 923     if (isELFInitializerSection(Sec.getName()))
 924       OrderedInitSections.push_back(&Sec);
 925
 926   // FIXME: This handles priority order within the current graph, but we'll need
 927   //        to include priority information in the initializer allocation
 928   //        actions in order to respect the ordering across multiple graphs.
 929   llvm::sort(OrderedInitSections, [](const jitlink::Section *LHS,
 930                                      const jitlink::Section *RHS) {
 931     if (LHS->getName().starts_with(".init_array")) {
 932       if (RHS->getName().starts_with(".init_array")) {
 933         StringRef LHSPrioStr(LHS->getName());
 934         StringRef RHSPrioStr(RHS->getName());
 935         uint64_t LHSPriority;
 936         bool LHSHasPriority = LHSPrioStr.consume_front(".init_array.") &&
 937                               !LHSPrioStr.getAsInteger(10, LHSPriority);
 938         uint64_t RHSPriority;
 939         bool RHSHasPriority = RHSPrioStr.consume_front(".init_array.") &&
 940                               !RHSPrioStr.getAsInteger(10, RHSPriority);
 941         if (LHSHasPriority)
 942           return RHSHasPriority ? LHSPriority < RHSPriority : true;
 943         else if (RHSHasPriority)
 944           return false;
 945         // If we get here we'll fall through to the
 946         // LHS->getName() < RHS->getName() test below.
 947       } else {
 948         // .init_array[.N] comes before any non-.init_array[.N] section.
 949         return true;
 950       }
 951     }
 952     return LHS->getName() < RHS->getName();
 953   });
 954
 955   for (auto &Sec : OrderedInitSections)
 956     ELFNixPlatformSecs.push_back(jitlink::SectionRange(*Sec).getRange());
 957
 958   // Dump the scraped inits.
 959   LLVM_DEBUG({
 960     dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " init sections:\n";
 961     for (auto &Sec : G.sections()) {
 962       jitlink::SectionRange R(Sec);
 963       dbgs() << "  " << Sec.getName() << ": " << R.getRange() << "\n";
 964     }
 965   });
 966
 967   ExecutorAddr HeaderAddr;
 968   {
 969     std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
 970     auto I = MP.JITDylibToHandleAddr.find(&JD);
 971     assert(I != MP.JITDylibToHandleAddr.end() && "No header registered for JD");
 972     assert(I->second && "Null header registered for JD");
 973     HeaderAddr = I->second;
 974   }
 975
 976   using SPSRegisterInitSectionsArgs =
 977       SPSArgList<SPSExecutorAddr, SPSSequence<SPSExecutorAddrRange>>;
 978
 979   if (LLVM_UNLIKELY(IsBootstrapping)) {
 980     MP.Bootstrap.load()->addArgumentsToRTFnMap(
 981         &MP.RegisterInitSections, &MP.DeregisterInitSections,
 982         getArgDataBufferType<SPSRegisterInitSectionsArgs>(HeaderAddr,
 983                                                           ELFNixPlatformSecs),
 984         getArgDataBufferType<SPSRegisterInitSectionsArgs>(HeaderAddr,
 985                                                           ELFNixPlatformSecs));
 986     return Error::success();
 987   }
 988
 989   G.allocActions().push_back(
 990       {cantFail(WrapperFunctionCall::Create<SPSRegisterInitSectionsArgs>(
 991            MP.RegisterInitSections.Addr, HeaderAddr, ELFNixPlatformSecs)),
 992        cantFail(WrapperFunctionCall::Create<SPSRegisterInitSectionsArgs>(
 993            MP.DeregisterInitSections.Addr, HeaderAddr, ELFNixPlatformSecs))});
 994
 995   return Error::success();
 996 }
 997
 998 Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges(
 999     jitlink::LinkGraph &G, JITDylib &JD) {
1000
1001   for (auto *Sym : G.external_symbols()) {
1002     if (Sym->getName() == "__tls_get_addr") {
1003       Sym->setName("___orc_rt_elfnix_tls_get_addr");
1004     } else if (Sym->getName() == "__tlsdesc_resolver") {
1005       Sym->setName("___orc_rt_elfnix_tlsdesc_resolver");
1006     }
1007   }
1008
1009   auto *TLSInfoEntrySection = G.findSectionByName("$__TLSINFO");
1010
1011   if (TLSInfoEntrySection) {
1012     std::optional<uint64_t> Key;
1013     {
1014       std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
1015       auto I = MP.JITDylibToPThreadKey.find(&JD);
1016       if (I != MP.JITDylibToPThreadKey.end())
1017         Key = I->second;
1018     }
1019     if (!Key) {
1020       if (auto KeyOrErr = MP.createPThreadKey())
1021         Key = *KeyOrErr;
1022       else
1023         return KeyOrErr.takeError();
1024     }
1025
1026     uint64_t PlatformKeyBits =
1027         support::endian::byte_swap(*Key, G.getEndianness());
1028
1029     for (auto *B : TLSInfoEntrySection->blocks()) {
1030       // FIXME: The TLS descriptor byte length may different with different
1031       // ISA
1032       assert(B->getSize() == (G.getPointerSize() * 2) &&
1033              "TLS descriptor must be 2 words length");
1034       auto TLSInfoEntryContent = B->getMutableContent(G);
1035       memcpy(TLSInfoEntryContent.data(), &PlatformKeyBits, G.getPointerSize());
1036     }
1037   }
1038
1039   return Error::success();
1040 }
1041
1042 } // End namespace orc.
1043 } // End namespace llvm.