llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp

   1 //===------ ELFNixPlatform.cpp - Utilities for executing MachO in Orc -----===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
  10
  11 #include "llvm/BinaryFormat/ELF.h"
  12 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
  13 #include "llvm/ExecutionEngine/JITLink/aarch64.h"
  14 #include "llvm/ExecutionEngine/JITLink/ppc64.h"
  15 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
  16 #include "llvm/ExecutionEngine/Orc/DebugUtils.h"
  17 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
  18 #include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
  19 #include "llvm/Support/BinaryByteStream.h"
  20 #include "llvm/Support/Debug.h"
  21 #include <optional>
  22
  23 #define DEBUG_TYPE "orc"
  24
  25 using namespace llvm;
  26 using namespace llvm::orc;
  27 using namespace llvm::orc::shared;
  28
  29 namespace {
  30
  31 class DSOHandleMaterializationUnit : public MaterializationUnit {
  32 public:
  33   DSOHandleMaterializationUnit(ELFNixPlatform &ENP,
  34                                const SymbolStringPtr &DSOHandleSymbol)
  35       : MaterializationUnit(
  36             createDSOHandleSectionInterface(ENP, DSOHandleSymbol)),
  37         ENP(ENP) {}
  38
  39   StringRef getName() const override { return "DSOHandleMU"; }
  40
  41   void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
  42     unsigned PointerSize;
  43     support::endianness Endianness;
  44     jitlink::Edge::Kind EdgeKind;
  45     const auto &TT = ENP.getExecutionSession().getTargetTriple();
  46
  47     switch (TT.getArch()) {
  48     case Triple::x86_64:
  49       PointerSize = 8;
  50       Endianness = support::endianness::little;
  51       EdgeKind = jitlink::x86_64::Pointer64;
  52       break;
  53     case Triple::aarch64:
  54       PointerSize = 8;
  55       Endianness = support::endianness::little;
  56       EdgeKind = jitlink::aarch64::Pointer64;
  57       break;
  58     case Triple::ppc64:
  59       PointerSize = 8;
  60       Endianness = support::endianness::big;
  61       EdgeKind = jitlink::ppc64::Pointer64;
  62       break;
  63     case Triple::ppc64le:
  64       PointerSize = 8;
  65       Endianness = support::endianness::little;
  66       EdgeKind = jitlink::ppc64::Pointer64;
  67       break;
  68     default:
  69       llvm_unreachable("Unrecognized architecture");
  70     }
  71
  72     // void *__dso_handle = &__dso_handle;
  73     auto G = std::make_unique<jitlink::LinkGraph>(
  74         "<DSOHandleMU>", TT, PointerSize, Endianness,
  75         jitlink::getGenericEdgeKindName);
  76     auto &DSOHandleSection =
  77         G->createSection(".data.__dso_handle", MemProt::Read);
  78     auto &DSOHandleBlock = G->createContentBlock(
  79         DSOHandleSection, getDSOHandleContent(PointerSize), orc::ExecutorAddr(),
  80         8, 0);
  81     auto &DSOHandleSymbol = G->addDefinedSymbol(
  82         DSOHandleBlock, 0, *R->getInitializerSymbol(), DSOHandleBlock.getSize(),
  83         jitlink::Linkage::Strong, jitlink::Scope::Default, false, true);
  84     DSOHandleBlock.addEdge(EdgeKind, 0, DSOHandleSymbol, 0);
  85
  86     ENP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
  87   }
  88
  89   void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
  90
  91 private:
  92   static MaterializationUnit::Interface
  93   createDSOHandleSectionInterface(ELFNixPlatform &ENP,
  94                                   const SymbolStringPtr &DSOHandleSymbol) {
  95     SymbolFlagsMap SymbolFlags;
  96     SymbolFlags[DSOHandleSymbol] = JITSymbolFlags::Exported;
  97     return MaterializationUnit::Interface(std::move(SymbolFlags),
  98                                           DSOHandleSymbol);
  99   }
 100
 101   ArrayRef<char> getDSOHandleContent(size_t PointerSize) {
 102     static const char Content[8] = {0};
 103     assert(PointerSize <= sizeof Content);
 104     return {Content, PointerSize};
 105   }
 106
 107   ELFNixPlatform &ENP;
 108 };
 109
 110 } // end anonymous namespace
 111
 112 namespace llvm {
 113 namespace orc {
 114
 115 Expected<std::unique_ptr<ELFNixPlatform>> ELFNixPlatform::Create(
 116     ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
 117     JITDylib &PlatformJD, std::unique_ptr<DefinitionGenerator> OrcRuntime,
 118     std::optional<SymbolAliasMap> RuntimeAliases) {
 119
 120   // If the target is not supported then bail out immediately.
 121   if (!supportedTarget(ES.getTargetTriple()))
 122     return make_error<StringError>("Unsupported ELFNixPlatform triple: " +
 123                                        ES.getTargetTriple().str(),
 124                                    inconvertibleErrorCode());
 125
 126   auto &EPC = ES.getExecutorProcessControl();
 127
 128   // Create default aliases if the caller didn't supply any.
 129   if (!RuntimeAliases) {
 130     auto StandardRuntimeAliases = standardPlatformAliases(ES, PlatformJD);
 131     if (!StandardRuntimeAliases)
 132       return StandardRuntimeAliases.takeError();
 133     RuntimeAliases = std::move(*StandardRuntimeAliases);
 134   }
 135
 136   // Define the aliases.
 137   if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases))))
 138     return std::move(Err);
 139
 140   // Add JIT-dispatch function support symbols.
 141   if (auto Err = PlatformJD.define(
 142           absoluteSymbols({{ES.intern("__orc_rt_jit_dispatch"),
 143                             {EPC.getJITDispatchInfo().JITDispatchFunction,
 144                              JITSymbolFlags::Exported}},
 145                            {ES.intern("__orc_rt_jit_dispatch_ctx"),
 146                             {EPC.getJITDispatchInfo().JITDispatchContext,
 147                              JITSymbolFlags::Exported}}})))
 148     return std::move(Err);
 149
 150   // Create the instance.
 151   Error Err = Error::success();
 152   auto P = std::unique_ptr<ELFNixPlatform>(new ELFNixPlatform(
 153       ES, ObjLinkingLayer, PlatformJD, std::move(OrcRuntime), Err));
 154   if (Err)
 155     return std::move(Err);
 156   return std::move(P);
 157 }
 158
 159 Expected<std::unique_ptr<ELFNixPlatform>>
 160 ELFNixPlatform::Create(ExecutionSession &ES,
 161                        ObjectLinkingLayer &ObjLinkingLayer,
 162                        JITDylib &PlatformJD, const char *OrcRuntimePath,
 163                        std::optional<SymbolAliasMap> RuntimeAliases) {
 164
 165   // Create a generator for the ORC runtime archive.
 166   auto OrcRuntimeArchiveGenerator =
 167       StaticLibraryDefinitionGenerator::Load(ObjLinkingLayer, OrcRuntimePath);
 168   if (!OrcRuntimeArchiveGenerator)
 169     return OrcRuntimeArchiveGenerator.takeError();
 170
 171   return Create(ES, ObjLinkingLayer, PlatformJD,
 172                 std::move(*OrcRuntimeArchiveGenerator),
 173                 std::move(RuntimeAliases));
 174 }
 175
 176 Error ELFNixPlatform::setupJITDylib(JITDylib &JD) {
 177   return JD.define(
 178       std::make_unique<DSOHandleMaterializationUnit>(*this, DSOHandleSymbol));
 179 }
 180
 181 Error ELFNixPlatform::teardownJITDylib(JITDylib &JD) {
 182   return Error::success();
 183 }
 184
 185 Error ELFNixPlatform::notifyAdding(ResourceTracker &RT,
 186                                    const MaterializationUnit &MU) {
 187   auto &JD = RT.getJITDylib();
 188   const auto &InitSym = MU.getInitializerSymbol();
 189   if (!InitSym)
 190     return Error::success();
 191
 192   RegisteredInitSymbols[&JD].add(InitSym,
 193                                  SymbolLookupFlags::WeaklyReferencedSymbol);
 194   LLVM_DEBUG({
 195     dbgs() << "ELFNixPlatform: Registered init symbol " << *InitSym
 196            << " for MU " << MU.getName() << "\n";
 197   });
 198   return Error::success();
 199 }
 200
 201 Error ELFNixPlatform::notifyRemoving(ResourceTracker &RT) {
 202   llvm_unreachable("Not supported yet");
 203 }
 204
 205 static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
 206                        ArrayRef<std::pair<const char *, const char *>> AL) {
 207   for (auto &KV : AL) {
 208     auto AliasName = ES.intern(KV.first);
 209     assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
 210     Aliases[std::move(AliasName)] = {ES.intern(KV.second),
 211                                      JITSymbolFlags::Exported};
 212   }
 213 }
 214
 215 Expected<SymbolAliasMap>
 216 ELFNixPlatform::standardPlatformAliases(ExecutionSession &ES,
 217                                         JITDylib &PlatformJD) {
 218   SymbolAliasMap Aliases;
 219   addAliases(ES, Aliases, requiredCXXAliases());
 220   addAliases(ES, Aliases, standardRuntimeUtilityAliases());
 221   return Aliases;
 222 }
 223
 224 ArrayRef<std::pair<const char *, const char *>>
 225 ELFNixPlatform::requiredCXXAliases() {
 226   static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
 227       {"__cxa_atexit", "__orc_rt_elfnix_cxa_atexit"},
 228       {"atexit", "__orc_rt_elfnix_atexit"}};
 229
 230   return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
 231 }
 232
 233 ArrayRef<std::pair<const char *, const char *>>
 234 ELFNixPlatform::standardRuntimeUtilityAliases() {
 235   static const std::pair<const char *, const char *>
 236       StandardRuntimeUtilityAliases[] = {
 237           {"__orc_rt_run_program", "__orc_rt_elfnix_run_program"},
 238           {"__orc_rt_jit_dlerror", "__orc_rt_elfnix_jit_dlerror"},
 239           {"__orc_rt_jit_dlopen", "__orc_rt_elfnix_jit_dlopen"},
 240           {"__orc_rt_jit_dlclose", "__orc_rt_elfnix_jit_dlclose"},
 241           {"__orc_rt_jit_dlsym", "__orc_rt_elfnix_jit_dlsym"},
 242           {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
 243
 244   return ArrayRef<std::pair<const char *, const char *>>(
 245       StandardRuntimeUtilityAliases);
 246 }
 247
 248 bool ELFNixPlatform::supportedTarget(const Triple &TT) {
 249   switch (TT.getArch()) {
 250   case Triple::x86_64:
 251   case Triple::aarch64:
 252   // FIXME: jitlink for ppc64 hasn't been well tested, leave it unsupported
 253   // right now.
 254   case Triple::ppc64le:
 255     return true;
 256   default:
 257     return false;
 258   }
 259 }
 260
 261 ELFNixPlatform::ELFNixPlatform(
 262     ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
 263     JITDylib &PlatformJD,
 264     std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err)
 265     : ES(ES), ObjLinkingLayer(ObjLinkingLayer),
 266       DSOHandleSymbol(ES.intern("__dso_handle")) {
 267   ErrorAsOutParameter _(&Err);
 268
 269   ObjLinkingLayer.addPlugin(std::make_unique<ELFNixPlatformPlugin>(*this));
 270
 271   PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));
 272
 273   // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating
 274   // the platform now), so set it up.
 275   if (auto E2 = setupJITDylib(PlatformJD)) {
 276     Err = std::move(E2);
 277     return;
 278   }
 279
 280   RegisteredInitSymbols[&PlatformJD].add(
 281       DSOHandleSymbol, SymbolLookupFlags::WeaklyReferencedSymbol);
 282
 283   // Associate wrapper function tags with JIT-side function implementations.
 284   if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
 285     Err = std::move(E2);
 286     return;
 287   }
 288
 289   // Lookup addresses of runtime functions callable by the platform,
 290   // call the platform bootstrap function to initialize the platform-state
 291   // object in the executor.
 292   if (auto E2 = bootstrapELFNixRuntime(PlatformJD)) {
 293     Err = std::move(E2);
 294     return;
 295   }
 296 }
 297
 298 Error ELFNixPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
 299   ExecutionSession::JITDispatchHandlerAssociationMap WFs;
 300
 301   using GetInitializersSPSSig =
 302       SPSExpected<SPSELFNixJITDylibInitializerSequence>(SPSString);
 303   WFs[ES.intern("__orc_rt_elfnix_get_initializers_tag")] =
 304       ES.wrapAsyncWithSPS<GetInitializersSPSSig>(
 305           this, &ELFNixPlatform::rt_getInitializers);
 306
 307   using GetDeinitializersSPSSig =
 308       SPSExpected<SPSELFJITDylibDeinitializerSequence>(SPSExecutorAddr);
 309   WFs[ES.intern("__orc_rt_elfnix_get_deinitializers_tag")] =
 310       ES.wrapAsyncWithSPS<GetDeinitializersSPSSig>(
 311           this, &ELFNixPlatform::rt_getDeinitializers);
 312
 313   using LookupSymbolSPSSig =
 314       SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
 315   WFs[ES.intern("__orc_rt_elfnix_symbol_lookup_tag")] =
 316       ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this,
 317                                               &ELFNixPlatform::rt_lookupSymbol);
 318
 319   return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
 320 }
 321
 322 void ELFNixPlatform::getInitializersBuildSequencePhase(
 323     SendInitializerSequenceFn SendResult, JITDylib &JD,
 324     std::vector<JITDylibSP> DFSLinkOrder) {
 325   ELFNixJITDylibInitializerSequence FullInitSeq;
 326   {
 327     std::lock_guard<std::mutex> Lock(PlatformMutex);
 328     for (auto &InitJD : reverse(DFSLinkOrder)) {
 329       LLVM_DEBUG({
 330         dbgs() << "ELFNixPlatform: Appending inits for \"" << InitJD->getName()
 331                << "\" to sequence\n";
 332       });
 333       auto ISItr = InitSeqs.find(InitJD.get());
 334       if (ISItr != InitSeqs.end()) {
 335         FullInitSeq.emplace_back(std::move(ISItr->second));
 336         InitSeqs.erase(ISItr);
 337       }
 338     }
 339   }
 340
 341   SendResult(std::move(FullInitSeq));
 342 }
 343
 344 void ELFNixPlatform::getInitializersLookupPhase(
 345     SendInitializerSequenceFn SendResult, JITDylib &JD) {
 346
 347   auto DFSLinkOrder = JD.getDFSLinkOrder();
 348   if (!DFSLinkOrder) {
 349     SendResult(DFSLinkOrder.takeError());
 350     return;
 351   }
 352
 353   DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
 354   ES.runSessionLocked([&]() {
 355     for (auto &InitJD : *DFSLinkOrder) {
 356       auto RISItr = RegisteredInitSymbols.find(InitJD.get());
 357       if (RISItr != RegisteredInitSymbols.end()) {
 358         NewInitSymbols[InitJD.get()] = std::move(RISItr->second);
 359         RegisteredInitSymbols.erase(RISItr);
 360       }
 361     }
 362   });
 363
 364   // If there are no further init symbols to look up then move on to the next
 365   // phase.
 366   if (NewInitSymbols.empty()) {
 367     getInitializersBuildSequencePhase(std::move(SendResult), JD,
 368                                       std::move(*DFSLinkOrder));
 369     return;
 370   }
 371
 372   // Otherwise issue a lookup and re-run this phase when it completes.
 373   lookupInitSymbolsAsync(
 374       [this, SendResult = std::move(SendResult), &JD](Error Err) mutable {
 375         if (Err)
 376           SendResult(std::move(Err));
 377         else
 378           getInitializersLookupPhase(std::move(SendResult), JD);
 379       },
 380       ES, std::move(NewInitSymbols));
 381 }
 382
 383 void ELFNixPlatform::rt_getInitializers(SendInitializerSequenceFn SendResult,
 384                                         StringRef JDName) {
 385   LLVM_DEBUG({
 386     dbgs() << "ELFNixPlatform::rt_getInitializers(\"" << JDName << "\")\n";
 387   });
 388
 389   JITDylib *JD = ES.getJITDylibByName(JDName);
 390   if (!JD) {
 391     LLVM_DEBUG({
 392       dbgs() << "  No such JITDylib \"" << JDName << "\". Sending error.\n";
 393     });
 394     SendResult(make_error<StringError>("No JITDylib named " + JDName,
 395                                        inconvertibleErrorCode()));
 396     return;
 397   }
 398
 399   getInitializersLookupPhase(std::move(SendResult), *JD);
 400 }
 401
 402 void ELFNixPlatform::rt_getDeinitializers(
 403     SendDeinitializerSequenceFn SendResult, ExecutorAddr Handle) {
 404   LLVM_DEBUG({
 405     dbgs() << "ELFNixPlatform::rt_getDeinitializers(\"" << Handle << "\")\n";
 406   });
 407
 408   JITDylib *JD = nullptr;
 409
 410   {
 411     std::lock_guard<std::mutex> Lock(PlatformMutex);
 412     auto I = HandleAddrToJITDylib.find(Handle);
 413     if (I != HandleAddrToJITDylib.end())
 414       JD = I->second;
 415   }
 416
 417   if (!JD) {
 418     LLVM_DEBUG(dbgs() << "  No JITDylib for handle " << Handle << "\n");
 419     SendResult(make_error<StringError>("No JITDylib associated with handle " +
 420                                            formatv("{0:x}", Handle),
 421                                        inconvertibleErrorCode()));
 422     return;
 423   }
 424
 425   SendResult(ELFNixJITDylibDeinitializerSequence());
 426 }
 427
 428 void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
 429                                      ExecutorAddr Handle,
 430                                      StringRef SymbolName) {
 431   LLVM_DEBUG({
 432     dbgs() << "ELFNixPlatform::rt_lookupSymbol(\"" << Handle << "\")\n";
 433   });
 434
 435   JITDylib *JD = nullptr;
 436
 437   {
 438     std::lock_guard<std::mutex> Lock(PlatformMutex);
 439     auto I = HandleAddrToJITDylib.find(Handle);
 440     if (I != HandleAddrToJITDylib.end())
 441       JD = I->second;
 442   }
 443
 444   if (!JD) {
 445     LLVM_DEBUG(dbgs() << "  No JITDylib for handle " << Handle << "\n");
 446     SendResult(make_error<StringError>("No JITDylib associated with handle " +
 447                                            formatv("{0:x}", Handle),
 448                                        inconvertibleErrorCode()));
 449     return;
 450   }
 451
 452   // Use functor class to work around XL build compiler issue on AIX.
 453   class RtLookupNotifyComplete {
 454   public:
 455     RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
 456         : SendResult(std::move(SendResult)) {}
 457     void operator()(Expected<SymbolMap> Result) {
 458       if (Result) {
 459         assert(Result->size() == 1 && "Unexpected result map count");
 460         SendResult(Result->begin()->second.getAddress());
 461       } else {
 462         SendResult(Result.takeError());
 463       }
 464     }
 465
 466   private:
 467     SendSymbolAddressFn SendResult;
 468   };
 469
 470   ES.lookup(
 471       LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
 472       SymbolLookupSet(ES.intern(SymbolName)), SymbolState::Ready,
 473       RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister);
 474 }
 475
 476 Error ELFNixPlatform::bootstrapELFNixRuntime(JITDylib &PlatformJD) {
 477
 478   std::pair<const char *, ExecutorAddr *> Symbols[] = {
 479       {"__orc_rt_elfnix_platform_bootstrap", &orc_rt_elfnix_platform_bootstrap},
 480       {"__orc_rt_elfnix_platform_shutdown", &orc_rt_elfnix_platform_shutdown},
 481       {"__orc_rt_elfnix_register_object_sections",
 482        &orc_rt_elfnix_register_object_sections},
 483       {"__orc_rt_elfnix_create_pthread_key",
 484        &orc_rt_elfnix_create_pthread_key}};
 485
 486   SymbolLookupSet RuntimeSymbols;
 487   std::vector<std::pair<SymbolStringPtr, ExecutorAddr *>> AddrsToRecord;
 488   for (const auto &KV : Symbols) {
 489     auto Name = ES.intern(KV.first);
 490     RuntimeSymbols.add(Name);
 491     AddrsToRecord.push_back({std::move(Name), KV.second});
 492   }
 493
 494   auto RuntimeSymbolAddrs = ES.lookup(
 495       {{&PlatformJD, JITDylibLookupFlags::MatchAllSymbols}}, RuntimeSymbols);
 496   if (!RuntimeSymbolAddrs)
 497     return RuntimeSymbolAddrs.takeError();
 498
 499   for (const auto &KV : AddrsToRecord) {
 500     auto &Name = KV.first;
 501     assert(RuntimeSymbolAddrs->count(Name) && "Missing runtime symbol?");
 502     *KV.second = (*RuntimeSymbolAddrs)[Name].getAddress();
 503   }
 504
 505   auto PJDDSOHandle = ES.lookup(
 506       {{&PlatformJD, JITDylibLookupFlags::MatchAllSymbols}}, DSOHandleSymbol);
 507   if (!PJDDSOHandle)
 508     return PJDDSOHandle.takeError();
 509
 510   if (auto Err = ES.callSPSWrapper<void(uint64_t)>(
 511           orc_rt_elfnix_platform_bootstrap,
 512           PJDDSOHandle->getAddress().getValue()))
 513     return Err;
 514
 515   // FIXME: Ordering is fuzzy here. We're probably best off saying
 516   // "behavior is undefined if code that uses the runtime is added before
 517   // the platform constructor returns", then move all this to the constructor.
 518   RuntimeBootstrapped = true;
 519   std::vector<ELFPerObjectSectionsToRegister> DeferredPOSRs;
 520   {
 521     std::lock_guard<std::mutex> Lock(PlatformMutex);
 522     DeferredPOSRs = std::move(BootstrapPOSRs);
 523   }
 524
 525   for (auto &D : DeferredPOSRs)
 526     if (auto Err = registerPerObjectSections(D))
 527       return Err;
 528
 529   return Error::success();
 530 }
 531
 532 Error ELFNixPlatform::registerInitInfo(
 533     JITDylib &JD, ArrayRef<jitlink::Section *> InitSections) {
 534
 535   std::unique_lock<std::mutex> Lock(PlatformMutex);
 536
 537   ELFNixJITDylibInitializers *InitSeq = nullptr;
 538   {
 539     auto I = InitSeqs.find(&JD);
 540     if (I == InitSeqs.end()) {
 541       // If there's no init sequence entry yet then we need to look up the
 542       // header symbol to force creation of one.
 543       Lock.unlock();
 544
 545       auto SearchOrder =
 546           JD.withLinkOrderDo([](const JITDylibSearchOrder &SO) { return SO; });
 547       if (auto Err = ES.lookup(SearchOrder, DSOHandleSymbol).takeError())
 548         return Err;
 549
 550       Lock.lock();
 551       I = InitSeqs.find(&JD);
 552       assert(I != InitSeqs.end() &&
 553              "Entry missing after header symbol lookup?");
 554     }
 555     InitSeq = &I->second;
 556   }
 557
 558   for (auto *Sec : InitSections) {
 559     // FIXME: Avoid copy here.
 560     jitlink::SectionRange R(*Sec);
 561     InitSeq->InitSections[Sec->getName()].push_back(R.getRange());
 562   }
 563
 564   return Error::success();
 565 }
 566
 567 Error ELFNixPlatform::registerPerObjectSections(
 568     const ELFPerObjectSectionsToRegister &POSR) {
 569
 570   if (!orc_rt_elfnix_register_object_sections)
 571     return make_error<StringError>("Attempting to register per-object "
 572                                    "sections, but runtime support has not "
 573                                    "been loaded yet",
 574                                    inconvertibleErrorCode());
 575
 576   Error ErrResult = Error::success();
 577   if (auto Err = ES.callSPSWrapper<shared::SPSError(
 578                      SPSELFPerObjectSectionsToRegister)>(
 579           orc_rt_elfnix_register_object_sections, ErrResult, POSR))
 580     return Err;
 581   return ErrResult;
 582 }
 583
 584 Expected<uint64_t> ELFNixPlatform::createPThreadKey() {
 585   if (!orc_rt_elfnix_create_pthread_key)
 586     return make_error<StringError>(
 587         "Attempting to create pthread key in target, but runtime support has "
 588         "not been loaded yet",
 589         inconvertibleErrorCode());
 590
 591   Expected<uint64_t> Result(0);
 592   if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>(
 593           orc_rt_elfnix_create_pthread_key, Result))
 594     return std::move(Err);
 595   return Result;
 596 }
 597
 598 void ELFNixPlatform::ELFNixPlatformPlugin::modifyPassConfig(
 599     MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
 600     jitlink::PassConfiguration &Config) {
 601
 602   // If the initializer symbol is the __dso_handle symbol then just add
 603   // the DSO handle support passes.
 604   if (MR.getInitializerSymbol() == MP.DSOHandleSymbol) {
 605     addDSOHandleSupportPasses(MR, Config);
 606     // The DSOHandle materialization unit doesn't require any other
 607     // support, so we can bail out early.
 608     return;
 609   }
 610
 611   // If the object contains initializers then add passes to record them.
 612   if (MR.getInitializerSymbol())
 613     addInitializerSupportPasses(MR, Config);
 614
 615   // Add passes for eh-frame and TLV support.
 616   addEHAndTLVSupportPasses(MR, Config);
 617 }
 618
 619 ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap
 620 ELFNixPlatform::ELFNixPlatformPlugin::getSyntheticSymbolDependencies(
 621     MaterializationResponsibility &MR) {
 622   std::lock_guard<std::mutex> Lock(PluginMutex);
 623   auto I = InitSymbolDeps.find(&MR);
 624   if (I != InitSymbolDeps.end()) {
 625     SyntheticSymbolDependenciesMap Result;
 626     Result[MR.getInitializerSymbol()] = std::move(I->second);
 627     InitSymbolDeps.erase(&MR);
 628     return Result;
 629   }
 630   return SyntheticSymbolDependenciesMap();
 631 }
 632
 633 void ELFNixPlatform::ELFNixPlatformPlugin::addInitializerSupportPasses(
 634     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
 635
 636   /// Preserve init sections.
 637   Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) -> Error {
 638     if (auto Err = preserveInitSections(G, MR))
 639       return Err;
 640     return Error::success();
 641   });
 642
 643   Config.PostFixupPasses.push_back(
 644       [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
 645         return registerInitSections(G, JD);
 646       });
 647 }
 648
 649 void ELFNixPlatform::ELFNixPlatformPlugin::addDSOHandleSupportPasses(
 650     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
 651
 652   Config.PostAllocationPasses.push_back([this, &JD = MR.getTargetJITDylib()](
 653                                             jitlink::LinkGraph &G) -> Error {
 654     auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {
 655       return Sym->getName() == *MP.DSOHandleSymbol;
 656     });
 657     assert(I != G.defined_symbols().end() && "Missing DSO handle symbol");
 658     {
 659       std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
 660       auto HandleAddr = (*I)->getAddress();
 661       MP.HandleAddrToJITDylib[HandleAddr] = &JD;
 662       assert(!MP.InitSeqs.count(&JD) && "InitSeq entry for JD already exists");
 663       MP.InitSeqs.insert(std::make_pair(
 664           &JD, ELFNixJITDylibInitializers(JD.getName(), HandleAddr)));
 665     }
 666     return Error::success();
 667   });
 668 }
 669
 670 void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses(
 671     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
 672
 673   // Insert TLV lowering at the start of the PostPrunePasses, since we want
 674   // it to run before GOT/PLT lowering.
 675
 676   // TODO: Check that before the fixTLVSectionsAndEdges pass, the GOT/PLT build
 677   // pass has done. Because the TLS descriptor need to be allocate in GOT.
 678   Config.PostPrunePasses.push_back(
 679       [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
 680         return fixTLVSectionsAndEdges(G, JD);
 681       });
 682
 683   // Add a pass to register the final addresses of the eh-frame and TLV sections
 684   // with the runtime.
 685   Config.PostFixupPasses.push_back([this](jitlink::LinkGraph &G) -> Error {
 686     ELFPerObjectSectionsToRegister POSR;
 687
 688     if (auto *EHFrameSection = G.findSectionByName(ELFEHFrameSectionName)) {
 689       jitlink::SectionRange R(*EHFrameSection);
 690       if (!R.empty())
 691         POSR.EHFrameSection = R.getRange();
 692     }
 693
 694     // Get a pointer to the thread data section if there is one. It will be used
 695     // below.
 696     jitlink::Section *ThreadDataSection =
 697         G.findSectionByName(ELFThreadDataSectionName);
 698
 699     // Handle thread BSS section if there is one.
 700     if (auto *ThreadBSSSection = G.findSectionByName(ELFThreadBSSSectionName)) {
 701       // If there's already a thread data section in this graph then merge the
 702       // thread BSS section content into it, otherwise just treat the thread
 703       // BSS section as the thread data section.
 704       if (ThreadDataSection)
 705         G.mergeSections(*ThreadDataSection, *ThreadBSSSection);
 706       else
 707         ThreadDataSection = ThreadBSSSection;
 708     }
 709
 710     // Having merged thread BSS (if present) and thread data (if present),
 711     // record the resulting section range.
 712     if (ThreadDataSection) {
 713       jitlink::SectionRange R(*ThreadDataSection);
 714       if (!R.empty())
 715         POSR.ThreadDataSection = R.getRange();
 716     }
 717
 718     if (POSR.EHFrameSection.Start || POSR.ThreadDataSection.Start) {
 719
 720       // If we're still bootstrapping the runtime then just record this
 721       // frame for now.
 722       if (!MP.RuntimeBootstrapped) {
 723         std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
 724         MP.BootstrapPOSRs.push_back(POSR);
 725         return Error::success();
 726       }
 727
 728       // Otherwise register it immediately.
 729       if (auto Err = MP.registerPerObjectSections(POSR))
 730         return Err;
 731     }
 732
 733     return Error::success();
 734   });
 735 }
 736
 737 Error ELFNixPlatform::ELFNixPlatformPlugin::preserveInitSections(
 738     jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
 739
 740   JITLinkSymbolSet InitSectionSymbols;
 741   for (auto &InitSection : G.sections()) {
 742     // Skip non-init sections.
 743     if (!isELFInitializerSection(InitSection.getName()))
 744       continue;
 745
 746     // Make a pass over live symbols in the section: those blocks are already
 747     // preserved.
 748     DenseSet<jitlink::Block *> AlreadyLiveBlocks;
 749     for (auto &Sym : InitSection.symbols()) {
 750       auto &B = Sym->getBlock();
 751       if (Sym->isLive() && Sym->getOffset() == 0 &&
 752           Sym->getSize() == B.getSize() && !AlreadyLiveBlocks.count(&B)) {
 753         InitSectionSymbols.insert(Sym);
 754         AlreadyLiveBlocks.insert(&B);
 755       }
 756     }
 757
 758     // Add anonymous symbols to preserve any not-already-preserved blocks.
 759     for (auto *B : InitSection.blocks())
 760       if (!AlreadyLiveBlocks.count(B))
 761         InitSectionSymbols.insert(
 762             &G.addAnonymousSymbol(*B, 0, B->getSize(), false, true));
 763   }
 764
 765   if (!InitSectionSymbols.empty()) {
 766     std::lock_guard<std::mutex> Lock(PluginMutex);
 767     InitSymbolDeps[&MR] = std::move(InitSectionSymbols);
 768   }
 769
 770   return Error::success();
 771 }
 772
 773 Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections(
 774     jitlink::LinkGraph &G, JITDylib &JD) {
 775
 776   SmallVector<jitlink::Section *> InitSections;
 777
 778   LLVM_DEBUG(dbgs() << "ELFNixPlatform::registerInitSections\n");
 779
 780   for (auto &Sec : G.sections()) {
 781     if (isELFInitializerSection(Sec.getName())) {
 782       InitSections.push_back(&Sec);
 783     }
 784   }
 785
 786   // Dump the scraped inits.
 787   LLVM_DEBUG({
 788     dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " init sections:\n";
 789     for (auto *Sec : InitSections) {
 790       jitlink::SectionRange R(*Sec);
 791       dbgs() << "  " << Sec->getName() << ": " << R.getRange() << "\n";
 792     }
 793   });
 794
 795   return MP.registerInitInfo(JD, InitSections);
 796 }
 797
 798 Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges(
 799     jitlink::LinkGraph &G, JITDylib &JD) {
 800
 801   for (auto *Sym : G.external_symbols()) {
 802     if (Sym->getName() == "__tls_get_addr") {
 803       Sym->setName("___orc_rt_elfnix_tls_get_addr");
 804     } else if (Sym->getName() == "__tlsdesc_resolver") {
 805       Sym->setName("___orc_rt_elfnix_tlsdesc_resolver");
 806     }
 807   }
 808
 809   auto *TLSInfoEntrySection = G.findSectionByName("$__TLSINFO");
 810
 811   if (TLSInfoEntrySection) {
 812     std::optional<uint64_t> Key;
 813     {
 814       std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
 815       auto I = MP.JITDylibToPThreadKey.find(&JD);
 816       if (I != MP.JITDylibToPThreadKey.end())
 817         Key = I->second;
 818     }
 819     if (!Key) {
 820       if (auto KeyOrErr = MP.createPThreadKey())
 821         Key = *KeyOrErr;
 822       else
 823         return KeyOrErr.takeError();
 824     }
 825
 826     uint64_t PlatformKeyBits =
 827         support::endian::byte_swap(*Key, G.getEndianness());
 828
 829     for (auto *B : TLSInfoEntrySection->blocks()) {
 830       // FIXME: The TLS descriptor byte length may different with different
 831       // ISA
 832       assert(B->getSize() == (G.getPointerSize() * 2) &&
 833              "TLS descriptor must be 2 words length");
 834       auto TLSInfoEntryContent = B->getMutableContent(G);
 835       memcpy(TLSInfoEntryContent.data(), &PlatformKeyBits, G.getPointerSize());
 836     }
 837   }
 838
 839   return Error::success();
 840 }
 841
 842 } // End namespace orc.
 843 } // End namespace llvm.