compiler-rt/lib/orc/elfnix_platform.cpp

   1 //===- elfnix_platform.cpp ------------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file contains code required to load the rest of the ELF-on-*IX runtime.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "elfnix_platform.h"
  14 #include "common.h"
  15 #include "compiler.h"
  16 #include "error.h"
  17 #include "jit_dispatch.h"
  18 #include "record_section_tracker.h"
  19 #include "wrapper_function_utils.h"
  20
  21 #include <algorithm>
  22 #include <map>
  23 #include <mutex>
  24 #include <sstream>
  25 #include <string_view>
  26 #include <unordered_map>
  27 #include <vector>
  28
  29 using namespace orc_rt;
  30 using namespace orc_rt::elfnix;
  31
  32 // Declare function tags for functions in the JIT process.
  33 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_reoptimize_tag)
  34 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_push_initializers_tag)
  35 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_symbol_lookup_tag)
  36
  37 // eh-frame registration functions, made available via aliases
  38 // installed by the Platform
  39 extern "C" void __register_frame(const void *);
  40 extern "C" void __deregister_frame(const void *);
  41
  42 extern "C" void
  43 __unw_add_dynamic_eh_frame_section(const void *) ORC_RT_WEAK_IMPORT;
  44 extern "C" void
  45 __unw_remove_dynamic_eh_frame_section(const void *) ORC_RT_WEAK_IMPORT;
  46
  47 namespace {
  48
  49 struct TLSInfoEntry {
  50   unsigned long Key = 0;
  51   unsigned long DataAddress = 0;
  52 };
  53
  54 struct TLSDescriptor {
  55   void (*Resolver)(void *);
  56   TLSInfoEntry *InfoEntry;
  57 };
  58
  59 class ELFNixPlatformRuntimeState {
  60 private:
  61   struct AtExitEntry {
  62     void (*Func)(void *);
  63     void *Arg;
  64   };
  65
  66   using AtExitsVector = std::vector<AtExitEntry>;
  67
  68   struct PerJITDylibState {
  69     std::string Name;
  70     void *Header = nullptr;
  71     size_t RefCount = 0;
  72     size_t LinkedAgainstRefCount = 0;
  73     bool AllowReinitialization = false;
  74     AtExitsVector AtExits;
  75     std::vector<PerJITDylibState *> Deps;
  76     RecordSectionsTracker<void (*)()> RecordedInits;
  77
  78     bool referenced() const {
  79       return LinkedAgainstRefCount != 0 || RefCount != 0;
  80     }
  81   };
  82
  83 public:
  84   static void initialize(void *DSOHandle);
  85   static ELFNixPlatformRuntimeState &get();
  86   static void destroy();
  87
  88   ELFNixPlatformRuntimeState(void *DSOHandle);
  89
  90   // Delete copy and move constructors.
  91   ELFNixPlatformRuntimeState(const ELFNixPlatformRuntimeState &) = delete;
  92   ELFNixPlatformRuntimeState &
  93   operator=(const ELFNixPlatformRuntimeState &) = delete;
  94   ELFNixPlatformRuntimeState(ELFNixPlatformRuntimeState &&) = delete;
  95   ELFNixPlatformRuntimeState &operator=(ELFNixPlatformRuntimeState &&) = delete;
  96
  97   Error registerObjectSections(ELFNixPerObjectSectionsToRegister POSR);
  98   Error registerJITDylib(std::string &Name, void *Handle);
  99   Error deregisterJITDylib(void *Handle);
 100   Error registerInits(ExecutorAddr HeaderAddr,
 101                       std::vector<ExecutorAddrRange> Inits);
 102   Error deregisterInits(ExecutorAddr HeaderAddr,
 103                         std::vector<ExecutorAddrRange> Inits);
 104   Error deregisterObjectSections(ELFNixPerObjectSectionsToRegister POSR);
 105
 106   const char *dlerror();
 107   void *dlopen(std::string_view Name, int Mode);
 108   int dlupdate(void *DSOHandle);
 109   int dlclose(void *DSOHandle);
 110   void *dlsym(void *DSOHandle, std::string_view Symbol);
 111
 112   int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle);
 113   void runAtExits(void *DSOHandle);
 114   void runAtExits(std::unique_lock<std::recursive_mutex> &JDStateLock,
 115                   PerJITDylibState &JDS);
 116
 117   /// Returns the base address of the section containing ThreadData.
 118   Expected<std::pair<const char *, size_t>>
 119   getThreadDataSectionFor(const char *ThreadData);
 120
 121   void *getPlatformJDDSOHandle() { return PlatformJDDSOHandle; }
 122
 123 private:
 124   PerJITDylibState *getJITDylibStateByHeaderAddr(void *DSOHandle);
 125   PerJITDylibState *getJITDylibStateByName(std::string_view Path);
 126
 127   Error registerThreadDataSection(span<const char> ThreadDataSection);
 128
 129   Expected<ExecutorAddr> lookupSymbolInJITDylib(void *DSOHandle,
 130                                                 std::string_view Symbol);
 131
 132   Error runInits(std::unique_lock<std::recursive_mutex> &JDStatesLock,
 133                  PerJITDylibState &JDS);
 134   Expected<void *> dlopenImpl(std::string_view Path, int Mode);
 135   Error dlopenFull(std::unique_lock<std::recursive_mutex> &JDStatesLock,
 136                    PerJITDylibState &JDS);
 137   Error dlopenInitialize(std::unique_lock<std::recursive_mutex> &JDStatesLock,
 138                          PerJITDylibState &JDS,
 139                          ELFNixJITDylibDepInfoMap &DepInfo);
 140   Error dlupdateImpl(void *DSOHandle);
 141   Error dlupdateFull(std::unique_lock<std::recursive_mutex> &JDStatesLock,
 142                      PerJITDylibState &JDS);
 143
 144   Error dlcloseImpl(void *DSOHandle);
 145   Error dlcloseInitialize(std::unique_lock<std::recursive_mutex> &JDStatesLock,
 146                           PerJITDylibState &JDS);
 147
 148   static ELFNixPlatformRuntimeState *MOPS;
 149
 150   void *PlatformJDDSOHandle;
 151
 152   // Frame registration functions:
 153   void (*registerEHFrameSection)(const void *) = nullptr;
 154   void (*deregisterEHFrameSection)(const void *) = nullptr;
 155
 156   // FIXME: Move to thread-state.
 157   std::string DLFcnError;
 158
 159   std::recursive_mutex JDStatesMutex;
 160   std::unordered_map<void *, PerJITDylibState> JDStates;
 161   std::unordered_map<std::string, void *> JDNameToHeader;
 162
 163   std::mutex ThreadDataSectionsMutex;
 164   std::map<const char *, size_t> ThreadDataSections;
 165 };
 166
 167 ELFNixPlatformRuntimeState *ELFNixPlatformRuntimeState::MOPS = nullptr;
 168
 169 void ELFNixPlatformRuntimeState::initialize(void *DSOHandle) {
 170   assert(!MOPS && "ELFNixPlatformRuntimeState should be null");
 171   MOPS = new ELFNixPlatformRuntimeState(DSOHandle);
 172 }
 173
 174 ELFNixPlatformRuntimeState &ELFNixPlatformRuntimeState::get() {
 175   assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
 176   return *MOPS;
 177 }
 178
 179 void ELFNixPlatformRuntimeState::destroy() {
 180   assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
 181   delete MOPS;
 182 }
 183
 184 ELFNixPlatformRuntimeState::ELFNixPlatformRuntimeState(void *DSOHandle)
 185     : PlatformJDDSOHandle(DSOHandle) {
 186   if (__unw_add_dynamic_eh_frame_section &&
 187       __unw_remove_dynamic_eh_frame_section) {
 188     registerEHFrameSection = __unw_add_dynamic_eh_frame_section;
 189     deregisterEHFrameSection = __unw_remove_dynamic_eh_frame_section;
 190   } else {
 191     registerEHFrameSection = __register_frame;
 192     deregisterEHFrameSection = __deregister_frame;
 193   }
 194 }
 195
 196 Error ELFNixPlatformRuntimeState::registerObjectSections(
 197     ELFNixPerObjectSectionsToRegister POSR) {
 198   if (POSR.EHFrameSection.Start)
 199     registerEHFrameSection(POSR.EHFrameSection.Start.toPtr<const char *>());
 200
 201   if (POSR.ThreadDataSection.Start) {
 202     if (auto Err = registerThreadDataSection(
 203             POSR.ThreadDataSection.toSpan<const char>()))
 204       return Err;
 205   }
 206
 207   return Error::success();
 208 }
 209
 210 Error ELFNixPlatformRuntimeState::deregisterObjectSections(
 211     ELFNixPerObjectSectionsToRegister POSR) {
 212   if (POSR.EHFrameSection.Start)
 213     deregisterEHFrameSection(POSR.EHFrameSection.Start.toPtr<const char *>());
 214
 215   return Error::success();
 216 }
 217
 218 Error ELFNixPlatformRuntimeState::registerJITDylib(std::string &Name,
 219                                                    void *Handle) {
 220   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
 221
 222   if (JDStates.count(Handle)) {
 223     std::ostringstream ErrStream;
 224     ErrStream << "Duplicate JITDylib registration for header " << Handle
 225               << " (name = " << Name << ")";
 226     return make_error<StringError>(ErrStream.str());
 227   }
 228
 229   if (JDNameToHeader.count(Name)) {
 230     std::ostringstream ErrStream;
 231     ErrStream << "Duplicate JITDylib registration for header " << Handle
 232               << " (header = " << Handle << ")";
 233     return make_error<StringError>(ErrStream.str());
 234   }
 235
 236   auto &JD = JDStates[Handle];
 237   JD.Header = Handle;
 238   JD.Name = std::move(Name);
 239   JDNameToHeader[JD.Name] = Handle;
 240   return Error::success();
 241 }
 242
 243 Error ELFNixPlatformRuntimeState::deregisterJITDylib(void *Handle) {
 244   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
 245
 246   auto I = JDStates.find(Handle);
 247   if (I == JDStates.end()) {
 248     std::ostringstream ErrStream;
 249     ErrStream << "Attempted to deregister unrecognized header " << Handle;
 250     return make_error<StringError>(ErrStream.str());
 251   }
 252
 253   auto J = JDNameToHeader.find(
 254       std::string(I->second.Name.data(), I->second.Name.size()));
 255   assert(J != JDNameToHeader.end() &&
 256          "Missing JDNameToHeader entry for JITDylib");
 257   JDNameToHeader.erase(J);
 258   JDStates.erase(I);
 259   return Error::success();
 260 }
 261
 262 Error ELFNixPlatformRuntimeState::registerInits(
 263     ExecutorAddr HeaderAddr, std::vector<ExecutorAddrRange> Inits) {
 264   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
 265   PerJITDylibState *JDS =
 266       getJITDylibStateByHeaderAddr(HeaderAddr.toPtr<void *>());
 267
 268   if (!JDS) {
 269     std::ostringstream ErrStream;
 270     ErrStream << "Could not register object platform sections for "
 271                  "unrecognized header "
 272               << HeaderAddr.toPtr<void *>();
 273     return make_error<StringError>(ErrStream.str());
 274   }
 275
 276   for (auto &I : Inits) {
 277     JDS->RecordedInits.add(I.toSpan<void (*)()>());
 278   }
 279
 280   return Error::success();
 281 }
 282
 283 Error ELFNixPlatformRuntimeState::deregisterInits(
 284     ExecutorAddr HeaderAddr, std::vector<ExecutorAddrRange> Inits) {
 285
 286   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
 287   PerJITDylibState *JDS =
 288       getJITDylibStateByHeaderAddr(HeaderAddr.toPtr<void *>());
 289
 290   if (!JDS) {
 291     std::ostringstream ErrStream;
 292     ErrStream << "Could not register object platform sections for unrecognized "
 293                  "header "
 294               << HeaderAddr.toPtr<void *>();
 295     return make_error<StringError>(ErrStream.str());
 296   }
 297
 298   for (auto &I : Inits) {
 299     JDS->RecordedInits.removeIfPresent(I);
 300   }
 301
 302   return Error::success();
 303 }
 304
 305 const char *ELFNixPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }
 306
 307 void *ELFNixPlatformRuntimeState::dlopen(std::string_view Path, int Mode) {
 308   if (auto H = dlopenImpl(Path, Mode))
 309     return *H;
 310   else {
 311     // FIXME: Make dlerror thread safe.
 312     DLFcnError = toString(H.takeError());
 313     return nullptr;
 314   }
 315 }
 316
 317 int ELFNixPlatformRuntimeState::dlupdate(void *DSOHandle) {
 318   if (auto Err = dlupdateImpl(DSOHandle)) {
 319     // FIXME: Make dlerror thread safe.
 320     DLFcnError = toString(std::move(Err));
 321     return -1;
 322   }
 323   return 0;
 324 }
 325
 326 int ELFNixPlatformRuntimeState::dlclose(void *DSOHandle) {
 327   if (auto Err = dlcloseImpl(DSOHandle)) {
 328     DLFcnError = toString(std::move(Err));
 329     return -1;
 330   }
 331   return 0;
 332 }
 333
 334 void *ELFNixPlatformRuntimeState::dlsym(void *DSOHandle,
 335                                         std::string_view Symbol) {
 336   auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol);
 337   if (!Addr) {
 338     DLFcnError = toString(Addr.takeError());
 339     return 0;
 340   }
 341
 342   return Addr->toPtr<void *>();
 343 }
 344
 345 int ELFNixPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg,
 346                                                void *DSOHandle) {
 347   // FIXME: Handle out-of-memory errors, returning -1 if OOM.
 348   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
 349   auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
 350   assert(JDS && "JITDylib state not initialized");
 351   JDS->AtExits.push_back({F, Arg});
 352   return 0;
 353 }
 354
 355 void ELFNixPlatformRuntimeState::runAtExits(void *DSOHandle) {
 356   std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex);
 357   PerJITDylibState *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
 358
 359   if (JDS)
 360     runAtExits(Lock, *JDS);
 361 }
 362
 363 void ELFNixPlatformRuntimeState::runAtExits(
 364     std::unique_lock<std::recursive_mutex> &JDStateLock,
 365     PerJITDylibState &JDS) {
 366   AtExitsVector V = std::move(JDS.AtExits);
 367
 368   while (!V.empty()) {
 369     auto &AE = V.back();
 370     AE.Func(AE.Arg);
 371     V.pop_back();
 372   }
 373 }
 374
 375 Expected<std::pair<const char *, size_t>>
 376 ELFNixPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) {
 377   std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
 378   auto I = ThreadDataSections.upper_bound(ThreadData);
 379   // Check that we have a valid entry conovering this address.
 380   if (I == ThreadDataSections.begin())
 381     return make_error<StringError>("No thread local data section for key");
 382   I = std::prev(I);
 383   if (ThreadData >= I->first + I->second)
 384     return make_error<StringError>("No thread local data section for key");
 385   return *I;
 386 }
 387
 388 ELFNixPlatformRuntimeState::PerJITDylibState *
 389 ELFNixPlatformRuntimeState::getJITDylibStateByHeaderAddr(void *DSOHandle) {
 390   auto I = JDStates.find(DSOHandle);
 391   if (I == JDStates.end())
 392     return nullptr;
 393
 394   return &I->second;
 395 }
 396
 397 ELFNixPlatformRuntimeState::PerJITDylibState *
 398 ELFNixPlatformRuntimeState::getJITDylibStateByName(std::string_view Name) {
 399   // FIXME: Avoid creating string copy here.
 400   auto I = JDNameToHeader.find(std::string(Name.data(), Name.size()));
 401   if (I == JDNameToHeader.end())
 402     return nullptr;
 403   void *H = I->second;
 404   auto J = JDStates.find(H);
 405   assert(J != JDStates.end() &&
 406          "JITDylib has name map entry but no header map entry");
 407   return &J->second;
 408 }
 409
 410 Error ELFNixPlatformRuntimeState::registerThreadDataSection(
 411     span<const char> ThreadDataSection) {
 412   std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
 413   auto I = ThreadDataSections.upper_bound(ThreadDataSection.data());
 414   if (I != ThreadDataSections.begin()) {
 415     auto J = std::prev(I);
 416     if (J->first + J->second > ThreadDataSection.data())
 417       return make_error<StringError>("Overlapping .tdata sections");
 418   }
 419   ThreadDataSections.insert(
 420       I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size()));
 421   return Error::success();
 422 }
 423
 424 Expected<ExecutorAddr>
 425 ELFNixPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle,
 426                                                    std::string_view Sym) {
 427   Expected<ExecutorAddr> Result((ExecutorAddr()));
 428   if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddr>(
 429           SPSExecutorAddr,
 430           SPSString)>::call(JITDispatch(&__orc_rt_elfnix_symbol_lookup_tag),
 431                             Result, ExecutorAddr::fromPtr(DSOHandle), Sym))
 432     return std::move(Err);
 433   return Result;
 434 }
 435
 436 Error ELFNixPlatformRuntimeState::runInits(
 437     std::unique_lock<std::recursive_mutex> &JDStatesLock,
 438     PerJITDylibState &JDS) {
 439   std::vector<span<void (*)()>> InitSections;
 440   InitSections.reserve(JDS.RecordedInits.numNewSections());
 441
 442   JDS.RecordedInits.processNewSections(
 443       [&](span<void (*)()> Inits) { InitSections.push_back(Inits); });
 444
 445   JDStatesLock.unlock();
 446   for (auto Sec : InitSections)
 447     for (auto *Init : Sec)
 448       Init();
 449
 450   JDStatesLock.lock();
 451
 452   return Error::success();
 453 }
 454
 455 Expected<void *> ELFNixPlatformRuntimeState::dlopenImpl(std::string_view Path,
 456                                                         int Mode) {
 457   std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex);
 458   PerJITDylibState *JDS = getJITDylibStateByName(Path);
 459
 460   if (!JDS)
 461     return make_error<StringError>("No registered JTIDylib for path " +
 462                                    std::string(Path.data(), Path.size()));
 463
 464   if (auto Err = dlopenFull(Lock, *JDS))
 465     return std::move(Err);
 466
 467   ++JDS->RefCount;
 468
 469   return JDS->Header;
 470 }
 471
 472 Error ELFNixPlatformRuntimeState::dlopenFull(
 473     std::unique_lock<std::recursive_mutex> &JDStateLock,
 474     PerJITDylibState &JDS) {
 475   Expected<ELFNixJITDylibDepInfoMap> DepInfo((ELFNixJITDylibDepInfoMap()));
 476   JDStateLock.unlock();
 477   if (auto Err = WrapperFunction<SPSExpected<SPSELFNixJITDylibDepInfoMap>(
 478           SPSExecutorAddr)>::
 479           call(JITDispatch(&__orc_rt_elfnix_push_initializers_tag), DepInfo,
 480                ExecutorAddr::fromPtr(JDS.Header)))
 481     return Err;
 482   JDStateLock.lock();
 483
 484   if (!DepInfo)
 485     return DepInfo.takeError();
 486
 487   if (auto Err = dlopenInitialize(JDStateLock, JDS, *DepInfo))
 488     return Err;
 489
 490   if (!DepInfo->empty()) {
 491     std::ostringstream ErrStream;
 492     ErrStream << "Encountered unrecognized dep-info key headers "
 493                  "while processing dlopen of "
 494               << JDS.Name;
 495     return make_error<StringError>(ErrStream.str());
 496   }
 497
 498   return Error::success();
 499 }
 500
 501 Error ELFNixPlatformRuntimeState::dlopenInitialize(
 502     std::unique_lock<std::recursive_mutex> &JDStatesLock, PerJITDylibState &JDS,
 503     ELFNixJITDylibDepInfoMap &DepInfo) {
 504
 505   auto I = DepInfo.find(ExecutorAddr::fromPtr(JDS.Header));
 506   if (I == DepInfo.end())
 507     return Error::success();
 508
 509   auto Deps = std::move(I->second);
 510   DepInfo.erase(I);
 511
 512   std::vector<PerJITDylibState *> OldDeps;
 513   std::swap(JDS.Deps, OldDeps);
 514   JDS.Deps.reserve(Deps.size());
 515   for (auto H : Deps) {
 516     PerJITDylibState *DepJDS = getJITDylibStateByHeaderAddr(H.toPtr<void *>());
 517     if (!DepJDS) {
 518       std::ostringstream ErrStream;
 519       ErrStream << "Encountered unrecognized dep header " << H.toPtr<void *>()
 520                 << " while initializing " << JDS.Name;
 521       return make_error<StringError>(ErrStream.str());
 522     }
 523     ++DepJDS->LinkedAgainstRefCount;
 524     if (auto Err = dlopenInitialize(JDStatesLock, *DepJDS, DepInfo))
 525       return Err;
 526   }
 527
 528   if (auto Err = runInits(JDStatesLock, JDS))
 529     return Err;
 530
 531   for (auto *DepJDS : OldDeps) {
 532     --DepJDS->LinkedAgainstRefCount;
 533     if (!DepJDS->referenced())
 534       if (auto Err = dlcloseInitialize(JDStatesLock, *DepJDS))
 535         return Err;
 536   }
 537   return Error::success();
 538 }
 539
 540 Error ELFNixPlatformRuntimeState::dlupdateImpl(void *DSOHandle) {
 541   std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex);
 542
 543   // Try to find JITDylib state by name.
 544   auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
 545
 546   if (!JDS) {
 547     std::ostringstream ErrStream;
 548     ErrStream << "No registered JITDylib for " << DSOHandle;
 549     return make_error<StringError>(ErrStream.str());
 550   }
 551
 552   if (!JDS->referenced())
 553     return make_error<StringError>("dlupdate failed, JITDylib must be open.");
 554
 555   if (auto Err = dlupdateFull(Lock, *JDS))
 556     return Err;
 557
 558   return Error::success();
 559 }
 560
 561 Error ELFNixPlatformRuntimeState::dlupdateFull(
 562     std::unique_lock<std::recursive_mutex> &JDStatesLock,
 563     PerJITDylibState &JDS) {
 564   // Call back to the JIT to push the initializers.
 565   Expected<ELFNixJITDylibDepInfoMap> DepInfo((ELFNixJITDylibDepInfoMap()));
 566   // Unlock so that we can accept the initializer update.
 567   JDStatesLock.unlock();
 568   if (auto Err = WrapperFunction<SPSExpected<SPSELFNixJITDylibDepInfoMap>(
 569           SPSExecutorAddr)>::
 570           call(JITDispatch(&__orc_rt_elfnix_push_initializers_tag), DepInfo,
 571                ExecutorAddr::fromPtr(JDS.Header)))
 572     return Err;
 573   JDStatesLock.lock();
 574
 575   if (!DepInfo)
 576     return DepInfo.takeError();
 577
 578   if (auto Err = runInits(JDStatesLock, JDS))
 579     return Err;
 580
 581   return Error::success();
 582 }
 583
 584 Error ELFNixPlatformRuntimeState::dlcloseImpl(void *DSOHandle) {
 585
 586   std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex);
 587   PerJITDylibState *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
 588
 589   if (!JDS) {
 590     std::ostringstream ErrStream;
 591     ErrStream << "No registered JITDylib for " << DSOHandle;
 592     return make_error<StringError>(ErrStream.str());
 593   }
 594
 595   --JDS->RefCount;
 596
 597   if (!JDS->referenced())
 598     return dlcloseInitialize(Lock, *JDS);
 599
 600   return Error::success();
 601 }
 602
 603 Error ELFNixPlatformRuntimeState::dlcloseInitialize(
 604     std::unique_lock<std::recursive_mutex> &JDStatesLock,
 605     PerJITDylibState &JDS) {
 606   runAtExits(JDStatesLock, JDS);
 607   JDS.RecordedInits.reset();
 608   for (auto *DepJDS : JDS.Deps)
 609     if (!JDS.referenced())
 610       if (auto Err = dlcloseInitialize(JDStatesLock, *DepJDS))
 611         return Err;
 612
 613   return Error::success();
 614 }
 615
 616 class ELFNixPlatformRuntimeTLVManager {
 617 public:
 618   void *getInstance(const char *ThreadData);
 619
 620 private:
 621   std::unordered_map<const char *, char *> Instances;
 622   std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections;
 623 };
 624
 625 void *ELFNixPlatformRuntimeTLVManager::getInstance(const char *ThreadData) {
 626   auto I = Instances.find(ThreadData);
 627   if (I != Instances.end())
 628     return I->second;
 629   auto TDS =
 630       ELFNixPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData);
 631   if (!TDS) {
 632     __orc_rt_log_error(toString(TDS.takeError()).c_str());
 633     return nullptr;
 634   }
 635
 636   auto &Allocated = AllocatedSections[TDS->first];
 637   if (!Allocated) {
 638     Allocated = std::make_unique<char[]>(TDS->second);
 639     memcpy(Allocated.get(), TDS->first, TDS->second);
 640   }
 641   size_t ThreadDataDelta = ThreadData - TDS->first;
 642   assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds");
 643
 644   char *Instance = Allocated.get() + ThreadDataDelta;
 645   Instances[ThreadData] = Instance;
 646   return Instance;
 647 }
 648
 649 void destroyELFNixTLVMgr(void *ELFNixTLVMgr) {
 650   delete static_cast<ELFNixPlatformRuntimeTLVManager *>(ELFNixTLVMgr);
 651 }
 652
 653 } // end anonymous namespace
 654
 655 //------------------------------------------------------------------------------
 656 //                             JIT entry points
 657 //------------------------------------------------------------------------------
 658
 659 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
 660 __orc_rt_elfnix_platform_bootstrap(char *ArgData, size_t ArgSize) {
 661   return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
 662              ArgData, ArgSize,
 663              [](ExecutorAddr DSOHandle) {
 664                ELFNixPlatformRuntimeState::initialize(
 665                    DSOHandle.toPtr<void *>());
 666                return Error::success();
 667              })
 668       .release();
 669 }
 670
 671 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
 672 __orc_rt_elfnix_platform_shutdown(char *ArgData, size_t ArgSize) {
 673   return WrapperFunction<SPSError()>::handle(
 674              ArgData, ArgSize,
 675              []() {
 676                ELFNixPlatformRuntimeState::destroy();
 677                return Error::success();
 678              })
 679       .release();
 680 }
 681
 682 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
 683 __orc_rt_elfnix_register_jitdylib(char *ArgData, size_t ArgSize) {
 684   return WrapperFunction<SPSError(SPSString, SPSExecutorAddr)>::handle(
 685              ArgData, ArgSize,
 686              [](std::string &JDName, ExecutorAddr HeaderAddr) {
 687                return ELFNixPlatformRuntimeState::get().registerJITDylib(
 688                    JDName, HeaderAddr.toPtr<void *>());
 689              })
 690       .release();
 691 }
 692
 693 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
 694 __orc_rt_elfnix_deregister_jitdylib(char *ArgData, size_t ArgSize) {
 695   return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
 696              ArgData, ArgSize,
 697              [](ExecutorAddr HeaderAddr) {
 698                return ELFNixPlatformRuntimeState::get().deregisterJITDylib(
 699                    HeaderAddr.toPtr<void *>());
 700              })
 701       .release();
 702 }
 703
 704 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
 705 __orc_rt_elfnix_register_init_sections(char *ArgData, size_t ArgSize) {
 706   return WrapperFunction<SPSError(SPSExecutorAddr,
 707                                   SPSSequence<SPSExecutorAddrRange>)>::
 708       handle(ArgData, ArgSize,
 709              [](ExecutorAddr HeaderAddr,
 710                 std::vector<ExecutorAddrRange> &Inits) {
 711                return ELFNixPlatformRuntimeState::get().registerInits(
 712                    HeaderAddr, std::move(Inits));
 713              })
 714           .release();
 715 }
 716
 717 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
 718 __orc_rt_elfnix_deregister_init_sections(char *ArgData, size_t ArgSize) {
 719   return WrapperFunction<SPSError(SPSExecutorAddr,
 720                                   SPSSequence<SPSExecutorAddrRange>)>::
 721       handle(ArgData, ArgSize,
 722              [](ExecutorAddr HeaderAddr,
 723                 std::vector<ExecutorAddrRange> &Inits) {
 724                return ELFNixPlatformRuntimeState::get().deregisterInits(
 725                    HeaderAddr, std::move(Inits));
 726              })
 727           .release();
 728 }
 729
 730 /// Wrapper function for registering metadata on a per-object basis.
 731 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
 732 __orc_rt_elfnix_register_object_sections(char *ArgData, size_t ArgSize) {
 733   return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
 734       handle(ArgData, ArgSize,
 735              [](ELFNixPerObjectSectionsToRegister &POSR) {
 736                return ELFNixPlatformRuntimeState::get().registerObjectSections(
 737                    std::move(POSR));
 738              })
 739           .release();
 740 }
 741
 742 /// Wrapper for releasing per-object metadat.
 743 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
 744 __orc_rt_elfnix_deregister_object_sections(char *ArgData, size_t ArgSize) {
 745   return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
 746       handle(ArgData, ArgSize,
 747              [](ELFNixPerObjectSectionsToRegister &POSR) {
 748                return ELFNixPlatformRuntimeState::get()
 749                    .deregisterObjectSections(std::move(POSR));
 750              })
 751           .release();
 752 }
 753
 754 //------------------------------------------------------------------------------
 755 //                           TLV support
 756 //------------------------------------------------------------------------------
 757
 758 ORC_RT_INTERFACE void *__orc_rt_elfnix_tls_get_addr_impl(TLSInfoEntry *D) {
 759   auto *TLVMgr = static_cast<ELFNixPlatformRuntimeTLVManager *>(
 760       pthread_getspecific(D->Key));
 761   if (!TLVMgr)
 762     TLVMgr = new ELFNixPlatformRuntimeTLVManager();
 763   if (pthread_setspecific(D->Key, TLVMgr)) {
 764     __orc_rt_log_error("Call to pthread_setspecific failed");
 765     return nullptr;
 766   }
 767
 768   return TLVMgr->getInstance(
 769       reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress)));
 770 }
 771
 772 ORC_RT_INTERFACE ptrdiff_t ___orc_rt_elfnix_tlsdesc_resolver_impl(
 773     TLSDescriptor *D, const char *ThreadPointer) {
 774   const char *TLVPtr = reinterpret_cast<const char *>(
 775       __orc_rt_elfnix_tls_get_addr_impl(D->InfoEntry));
 776   return TLVPtr - ThreadPointer;
 777 }
 778
 779 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
 780 __orc_rt_elfnix_create_pthread_key(char *ArgData, size_t ArgSize) {
 781   return WrapperFunction<SPSExpected<uint64_t>(void)>::handle(
 782              ArgData, ArgSize,
 783              []() -> Expected<uint64_t> {
 784                pthread_key_t Key;
 785                if (int Err = pthread_key_create(&Key, destroyELFNixTLVMgr)) {
 786                  __orc_rt_log_error("Call to pthread_key_create failed");
 787                  return make_error<StringError>(strerror(Err));
 788                }
 789                return static_cast<uint64_t>(Key);
 790              })
 791       .release();
 792 }
 793
 794 //------------------------------------------------------------------------------
 795 //                           cxa_atexit support
 796 //------------------------------------------------------------------------------
 797
 798 int __orc_rt_elfnix_cxa_atexit(void (*func)(void *), void *arg,
 799                                void *dso_handle) {
 800   return ELFNixPlatformRuntimeState::get().registerAtExit(func, arg,
 801                                                           dso_handle);
 802 }
 803
 804 int __orc_rt_elfnix_atexit(void (*func)(void *)) {
 805   auto &PlatformRTState = ELFNixPlatformRuntimeState::get();
 806   return ELFNixPlatformRuntimeState::get().registerAtExit(
 807       func, NULL, PlatformRTState.getPlatformJDDSOHandle());
 808 }
 809
 810 void __orc_rt_elfnix_cxa_finalize(void *dso_handle) {
 811   ELFNixPlatformRuntimeState::get().runAtExits(dso_handle);
 812 }
 813
 814 //------------------------------------------------------------------------------
 815 //                        JIT'd dlfcn alternatives.
 816 //------------------------------------------------------------------------------
 817
 818 const char *__orc_rt_elfnix_jit_dlerror() {
 819   return ELFNixPlatformRuntimeState::get().dlerror();
 820 }
 821
 822 void *__orc_rt_elfnix_jit_dlopen(const char *path, int mode) {
 823   return ELFNixPlatformRuntimeState::get().dlopen(path, mode);
 824 }
 825
 826 int __orc_rt_elfnix_jit_dlupdate(void *dso_handle) {
 827   return ELFNixPlatformRuntimeState::get().dlupdate(dso_handle);
 828 }
 829
 830 int __orc_rt_elfnix_jit_dlclose(void *dso_handle) {
 831   return ELFNixPlatformRuntimeState::get().dlclose(dso_handle);
 832 }
 833
 834 void *__orc_rt_elfnix_jit_dlsym(void *dso_handle, const char *symbol) {
 835   return ELFNixPlatformRuntimeState::get().dlsym(dso_handle, symbol);
 836 }
 837
 838 //------------------------------------------------------------------------------
 839 //                             ELFNix Run Program
 840 //------------------------------------------------------------------------------
 841
 842 ORC_RT_INTERFACE int64_t __orc_rt_elfnix_run_program(
 843     const char *JITDylibName, const char *EntrySymbolName, int argc,
 844     char *argv[]) {
 845   using MainTy = int (*)(int, char *[]);
 846
 847   void *H = __orc_rt_elfnix_jit_dlopen(JITDylibName,
 848                                        orc_rt::elfnix::ORC_RT_RTLD_LAZY);
 849   if (!H) {
 850     __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
 851     return -1;
 852   }
 853
 854   auto *Main =
 855       reinterpret_cast<MainTy>(__orc_rt_elfnix_jit_dlsym(H, EntrySymbolName));
 856
 857   if (!Main) {
 858     __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
 859     return -1;
 860   }
 861
 862   int Result = Main(argc, argv);
 863
 864   if (__orc_rt_elfnix_jit_dlclose(H) == -1)
 865     __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
 866
 867   return Result;
 868 }