llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp

   1 //===---------- LazyReexports.cpp - Utilities for lazy reexports ----------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ExecutionEngine/Orc/LazyReexports.h"
  10
  11 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
  12 #include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
  13 #include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h"
  14 #include "llvm/TargetParser/Triple.h"
  15
  16 #define DEBUG_TYPE "orc"
  17
  18 namespace llvm {
  19 namespace orc {
  20
  21 LazyCallThroughManager::LazyCallThroughManager(ExecutionSession &ES,
  22                                                ExecutorAddr ErrorHandlerAddr,
  23                                                TrampolinePool *TP)
  24     : ES(ES), ErrorHandlerAddr(ErrorHandlerAddr), TP(TP) {}
  25
  26 Expected<ExecutorAddr> LazyCallThroughManager::getCallThroughTrampoline(
  27     JITDylib &SourceJD, SymbolStringPtr SymbolName,
  28     NotifyResolvedFunction NotifyResolved) {
  29   assert(TP && "TrampolinePool not set");
  30
  31   std::lock_guard<std::mutex> Lock(LCTMMutex);
  32   auto Trampoline = TP->getTrampoline();
  33
  34   if (!Trampoline)
  35     return Trampoline.takeError();
  36
  37   Reexports[*Trampoline] = ReexportsEntry{&SourceJD, std::move(SymbolName)};
  38   Notifiers[*Trampoline] = std::move(NotifyResolved);
  39   return *Trampoline;
  40 }
  41
  42 ExecutorAddr LazyCallThroughManager::reportCallThroughError(Error Err) {
  43   ES.reportError(std::move(Err));
  44   return ErrorHandlerAddr;
  45 }
  46
  47 Expected<LazyCallThroughManager::ReexportsEntry>
  48 LazyCallThroughManager::findReexport(ExecutorAddr TrampolineAddr) {
  49   std::lock_guard<std::mutex> Lock(LCTMMutex);
  50   auto I = Reexports.find(TrampolineAddr);
  51   if (I == Reexports.end())
  52     return createStringError(inconvertibleErrorCode(),
  53                              "Missing reexport for trampoline address %p" +
  54                                  formatv("{0:x}", TrampolineAddr));
  55   return I->second;
  56 }
  57
  58 Error LazyCallThroughManager::notifyResolved(ExecutorAddr TrampolineAddr,
  59                                              ExecutorAddr ResolvedAddr) {
  60   NotifyResolvedFunction NotifyResolved;
  61   {
  62     std::lock_guard<std::mutex> Lock(LCTMMutex);
  63     auto I = Notifiers.find(TrampolineAddr);
  64     if (I != Notifiers.end()) {
  65       NotifyResolved = std::move(I->second);
  66       Notifiers.erase(I);
  67     }
  68   }
  69
  70   return NotifyResolved ? NotifyResolved(ResolvedAddr) : Error::success();
  71 }
  72
  73 void LazyCallThroughManager::resolveTrampolineLandingAddress(
  74     ExecutorAddr TrampolineAddr,
  75     NotifyLandingResolvedFunction NotifyLandingResolved) {
  76
  77   auto Entry = findReexport(TrampolineAddr);
  78   if (!Entry)
  79     return NotifyLandingResolved(reportCallThroughError(Entry.takeError()));
  80
  81   // Declaring SLS and the callback outside of the call to ES.lookup is a
  82   // workaround to fix build failures on AIX and on z/OS platforms.
  83   SymbolLookupSet SLS({Entry->SymbolName});
  84   auto Callback = [this, TrampolineAddr, SymbolName = Entry->SymbolName,
  85                    NotifyLandingResolved = std::move(NotifyLandingResolved)](
  86                       Expected<SymbolMap> Result) mutable {
  87     if (Result) {
  88       assert(Result->size() == 1 && "Unexpected result size");
  89       assert(Result->count(SymbolName) && "Unexpected result value");
  90       ExecutorAddr LandingAddr = (*Result)[SymbolName].getAddress();
  91
  92       if (auto Err = notifyResolved(TrampolineAddr, LandingAddr))
  93         NotifyLandingResolved(reportCallThroughError(std::move(Err)));
  94       else
  95         NotifyLandingResolved(LandingAddr);
  96     } else {
  97       NotifyLandingResolved(reportCallThroughError(Result.takeError()));
  98     }
  99   };
 100
 101   ES.lookup(LookupKind::Static,
 102             makeJITDylibSearchOrder(Entry->SourceJD,
 103                                     JITDylibLookupFlags::MatchAllSymbols),
 104             std::move(SLS), SymbolState::Ready, std::move(Callback),
 105             NoDependenciesToRegister);
 106 }
 107
 108 Expected<std::unique_ptr<LazyCallThroughManager>>
 109 createLocalLazyCallThroughManager(const Triple &T, ExecutionSession &ES,
 110                                   ExecutorAddr ErrorHandlerAddr) {
 111   switch (T.getArch()) {
 112   default:
 113     return make_error<StringError>(
 114         std::string("No callback manager available for ") + T.str(),
 115         inconvertibleErrorCode());
 116
 117   case Triple::aarch64:
 118   case Triple::aarch64_32:
 119     return LocalLazyCallThroughManager::Create<OrcAArch64>(ES,
 120                                                            ErrorHandlerAddr);
 121
 122   case Triple::x86:
 123     return LocalLazyCallThroughManager::Create<OrcI386>(ES, ErrorHandlerAddr);
 124
 125   case Triple::loongarch64:
 126     return LocalLazyCallThroughManager::Create<OrcLoongArch64>(
 127         ES, ErrorHandlerAddr);
 128
 129   case Triple::mips:
 130     return LocalLazyCallThroughManager::Create<OrcMips32Be>(ES,
 131                                                             ErrorHandlerAddr);
 132
 133   case Triple::mipsel:
 134     return LocalLazyCallThroughManager::Create<OrcMips32Le>(ES,
 135                                                             ErrorHandlerAddr);
 136
 137   case Triple::mips64:
 138   case Triple::mips64el:
 139     return LocalLazyCallThroughManager::Create<OrcMips64>(ES, ErrorHandlerAddr);
 140
 141   case Triple::riscv64:
 142     return LocalLazyCallThroughManager::Create<OrcRiscv64>(ES,
 143                                                            ErrorHandlerAddr);
 144
 145   case Triple::x86_64:
 146     if (T.getOS() == Triple::OSType::Win32)
 147       return LocalLazyCallThroughManager::Create<OrcX86_64_Win32>(
 148           ES, ErrorHandlerAddr);
 149     else
 150       return LocalLazyCallThroughManager::Create<OrcX86_64_SysV>(
 151           ES, ErrorHandlerAddr);
 152   }
 153 }
 154
 155 LazyReexportsMaterializationUnit::LazyReexportsMaterializationUnit(
 156     LazyCallThroughManager &LCTManager, RedirectableSymbolManager &RSManager,
 157     JITDylib &SourceJD, SymbolAliasMap CallableAliases, ImplSymbolMap *SrcJDLoc)
 158     : MaterializationUnit(extractFlags(CallableAliases)),
 159       LCTManager(LCTManager), RSManager(RSManager), SourceJD(SourceJD),
 160       CallableAliases(std::move(CallableAliases)), AliaseeTable(SrcJDLoc) {}
 161
 162 StringRef LazyReexportsMaterializationUnit::getName() const {
 163   return "<Lazy Reexports>";
 164 }
 165
 166 void LazyReexportsMaterializationUnit::materialize(
 167     std::unique_ptr<MaterializationResponsibility> R) {
 168   auto RequestedSymbols = R->getRequestedSymbols();
 169
 170   SymbolAliasMap RequestedAliases;
 171   for (auto &RequestedSymbol : RequestedSymbols) {
 172     auto I = CallableAliases.find(RequestedSymbol);
 173     assert(I != CallableAliases.end() && "Symbol not found in alias map?");
 174     RequestedAliases[I->first] = std::move(I->second);
 175     CallableAliases.erase(I);
 176   }
 177
 178   if (!CallableAliases.empty())
 179     if (auto Err = R->replace(lazyReexports(LCTManager, RSManager, SourceJD,
 180                                             std::move(CallableAliases),
 181                                             AliaseeTable))) {
 182       R->getExecutionSession().reportError(std::move(Err));
 183       R->failMaterialization();
 184       return;
 185     }
 186
 187   SymbolMap Inits;
 188   for (auto &Alias : RequestedAliases) {
 189     auto CallThroughTrampoline = LCTManager.getCallThroughTrampoline(
 190         SourceJD, Alias.second.Aliasee,
 191         [&TargetJD = R->getTargetJITDylib(), &RSManager = this->RSManager,
 192          StubSym = Alias.first](ExecutorAddr ResolvedAddr) -> Error {
 193           return RSManager.redirect(TargetJD, StubSym,
 194                                     ExecutorSymbolDef(ResolvedAddr, {}));
 195         });
 196
 197     if (!CallThroughTrampoline) {
 198       R->getExecutionSession().reportError(CallThroughTrampoline.takeError());
 199       R->failMaterialization();
 200       return;
 201     }
 202
 203     Inits[Alias.first] = {*CallThroughTrampoline, Alias.second.AliasFlags};
 204   }
 205
 206   if (AliaseeTable != nullptr && !RequestedAliases.empty())
 207     AliaseeTable->trackImpls(RequestedAliases, &SourceJD);
 208
 209   if (auto Err = R->replace(std::make_unique<RedirectableMaterializationUnit>(
 210           RSManager, std::move(Inits)))) {
 211     R->getExecutionSession().reportError(std::move(Err));
 212     return R->failMaterialization();
 213   }
 214 }
 215
 216 void LazyReexportsMaterializationUnit::discard(const JITDylib &JD,
 217                                                const SymbolStringPtr &Name) {
 218   assert(CallableAliases.count(Name) &&
 219          "Symbol not covered by this MaterializationUnit");
 220   CallableAliases.erase(Name);
 221 }
 222
 223 MaterializationUnit::Interface
 224 LazyReexportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) {
 225   SymbolFlagsMap SymbolFlags;
 226   for (auto &KV : Aliases) {
 227     assert(KV.second.AliasFlags.isCallable() &&
 228            "Lazy re-exports must be callable symbols");
 229     SymbolFlags[KV.first] = KV.second.AliasFlags;
 230   }
 231   return MaterializationUnit::Interface(std::move(SymbolFlags), nullptr);
 232 }
 233
 234 class LazyReexportsManager::MU : public MaterializationUnit {
 235 public:
 236   MU(LazyReexportsManager &LRMgr, SymbolAliasMap Reexports)
 237       : MaterializationUnit(getInterface(Reexports)), LRMgr(LRMgr),
 238         Reexports(std::move(Reexports)) {}
 239
 240 private:
 241   Interface getInterface(const SymbolAliasMap &Reexports) {
 242     SymbolFlagsMap SF;
 243     for (auto &[Alias, AI] : Reexports)
 244       SF[Alias] = AI.AliasFlags;
 245     return {std::move(SF), nullptr};
 246   }
 247
 248   StringRef getName() const override { return "LazyReexportsManager::MU"; }
 249
 250   void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
 251     LRMgr.emitReentryTrampolines(std::move(R), std::move(Reexports));
 252   }
 253
 254   void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
 255     Reexports.erase(Name);
 256   }
 257
 258   LazyReexportsManager &LRMgr;
 259   SymbolAliasMap Reexports;
 260 };
 261
 262 class LazyReexportsManager::Plugin : public ObjectLinkingLayer::Plugin {
 263 public:
 264   void modifyPassConfig(MaterializationResponsibility &MR,
 265                         jitlink::LinkGraph &G,
 266                         jitlink::PassConfiguration &Config) override {}
 267
 268   Error notifyFailed(MaterializationResponsibility &MR) override {
 269     return Error::success();
 270   }
 271
 272   Error notifyRemovingResources(JITDylib &JD, ResourceKey K) override {
 273     return Error::success();
 274   }
 275
 276   void notifyTransferringResources(JITDylib &JD, ResourceKey DstKey,
 277                                    ResourceKey SrcKey) override {}
 278
 279 private:
 280   std::mutex M;
 281 };
 282
 283 Expected<std::unique_ptr<LazyReexportsManager>>
 284 LazyReexportsManager::Create(EmitTrampolinesFn EmitTrampolines,
 285                              RedirectableSymbolManager &RSMgr,
 286                              JITDylib &PlatformJD) {
 287   Error Err = Error::success();
 288   std::unique_ptr<LazyReexportsManager> LRM(new LazyReexportsManager(
 289       std::move(EmitTrampolines), RSMgr, PlatformJD, Err));
 290   if (Err)
 291     return std::move(Err);
 292   return std::move(LRM);
 293 }
 294
 295 Error LazyReexportsManager::handleRemoveResources(JITDylib &JD, ResourceKey K) {
 296   JD.getExecutionSession().runSessionLocked([&]() {
 297     auto I = KeyToReentryAddrs.find(K);
 298     if (I != KeyToReentryAddrs.end()) {
 299       auto &ReentryAddrs = I->second;
 300       for (auto &ReentryAddr : ReentryAddrs) {
 301         assert(CallThroughs.count(ReentryAddr) && "CallTrhough missing");
 302         CallThroughs.erase(ReentryAddr);
 303       }
 304       KeyToReentryAddrs.erase(I);
 305     }
 306   });
 307   return Error::success();
 308 }
 309
 310 void LazyReexportsManager::handleTransferResources(JITDylib &JD,
 311                                                    ResourceKey DstK,
 312                                                    ResourceKey SrcK) {
 313   auto I = KeyToReentryAddrs.find(SrcK);
 314   if (I != KeyToReentryAddrs.end()) {
 315     auto J = KeyToReentryAddrs.find(DstK);
 316     if (J == KeyToReentryAddrs.end()) {
 317       auto Tmp = std::move(I->second);
 318       KeyToReentryAddrs.erase(I);
 319       KeyToReentryAddrs[DstK] = std::move(Tmp);
 320     } else {
 321       auto &SrcAddrs = I->second;
 322       auto &DstAddrs = J->second;
 323       DstAddrs.insert(DstAddrs.end(), SrcAddrs.begin(), SrcAddrs.end());
 324       KeyToReentryAddrs.erase(I);
 325     }
 326   }
 327 }
 328
 329 LazyReexportsManager::LazyReexportsManager(EmitTrampolinesFn EmitTrampolines,
 330                                            RedirectableSymbolManager &RSMgr,
 331                                            JITDylib &PlatformJD, Error &Err)
 332     : ES(PlatformJD.getExecutionSession()),
 333       EmitTrampolines(std::move(EmitTrampolines)), RSMgr(RSMgr) {
 334
 335   using namespace shared;
 336
 337   ErrorAsOutParameter _(&Err);
 338
 339   ExecutionSession::JITDispatchHandlerAssociationMap WFs;
 340
 341   WFs[ES.intern("__orc_rt_resolve_tag")] =
 342       ES.wrapAsyncWithSPS<SPSExpected<SPSExecutorSymbolDef>(SPSExecutorAddr)>(
 343           this, &LazyReexportsManager::resolve);
 344
 345   Err = ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
 346 }
 347
 348 std::unique_ptr<MaterializationUnit>
 349 LazyReexportsManager::createLazyReexports(SymbolAliasMap Reexports) {
 350   return std::make_unique<MU>(*this, std::move(Reexports));
 351 }
 352
 353 void LazyReexportsManager::emitReentryTrampolines(
 354     std::unique_ptr<MaterializationResponsibility> MR,
 355     SymbolAliasMap Reexports) {
 356   size_t NumTrampolines = Reexports.size();
 357   auto RT = MR->getResourceTracker();
 358   EmitTrampolines(
 359       std::move(RT), NumTrampolines,
 360       [this, MR = std::move(MR), Reexports = std::move(Reexports)](
 361           Expected<std::vector<ExecutorSymbolDef>> ReentryPoints) mutable {
 362         emitRedirectableSymbols(std::move(MR), std::move(Reexports),
 363                                 std::move(ReentryPoints));
 364       });
 365 }
 366
 367 void LazyReexportsManager::emitRedirectableSymbols(
 368     std::unique_ptr<MaterializationResponsibility> MR, SymbolAliasMap Reexports,
 369     Expected<std::vector<ExecutorSymbolDef>> ReentryPoints) {
 370
 371   if (!ReentryPoints) {
 372     MR->getExecutionSession().reportError(ReentryPoints.takeError());
 373     MR->failMaterialization();
 374     return;
 375   }
 376
 377   assert(Reexports.size() == ReentryPoints->size() &&
 378          "Number of reentry points doesn't match number of reexports");
 379
 380   // Bind entry points to names.
 381   SymbolMap Redirs;
 382   size_t I = 0;
 383   for (auto &[Name, AI] : Reexports)
 384     Redirs[Name] = (*ReentryPoints)[I++];
 385
 386   I = 0;
 387   if (auto Err = MR->withResourceKeyDo([&](ResourceKey K) {
 388         for (auto &[Name, AI] : Reexports) {
 389           const auto &ReentryPoint = (*ReentryPoints)[I++];
 390           CallThroughs[ReentryPoint.getAddress()] = {Name, AI.Aliasee,
 391                                                      &MR->getTargetJITDylib()};
 392           KeyToReentryAddrs[K].push_back(ReentryPoint.getAddress());
 393         }
 394       })) {
 395     MR->getExecutionSession().reportError(std::move(Err));
 396     MR->failMaterialization();
 397     return;
 398   }
 399
 400   RSMgr.emitRedirectableSymbols(std::move(MR), std::move(Redirs));
 401 }
 402
 403 void LazyReexportsManager::resolve(ResolveSendResultFn SendResult,
 404                                    ExecutorAddr ReentryStubAddr) {
 405
 406   CallThroughInfo LandingInfo;
 407
 408   ES.runSessionLocked([&]() {
 409     auto I = CallThroughs.find(ReentryStubAddr);
 410     if (I == CallThroughs.end())
 411       return SendResult(make_error<StringError>(
 412           "Reentry address " + formatv("{0:x}", ReentryStubAddr) +
 413               " not registered",
 414           inconvertibleErrorCode()));
 415     LandingInfo = I->second;
 416   });
 417
 418   SymbolInstance LandingSym(LandingInfo.JD, std::move(LandingInfo.BodyName));
 419   LandingSym.lookupAsync([this, JD = std::move(LandingInfo.JD),
 420                           ReentryName = std::move(LandingInfo.Name),
 421                           SendResult = std::move(SendResult)](
 422                              Expected<ExecutorSymbolDef> Result) mutable {
 423     if (Result) {
 424       // FIXME: Make RedirectionManager operations async, then use the async
 425       //        APIs here.
 426       if (auto Err = RSMgr.redirect(*JD, ReentryName, *Result))
 427         SendResult(std::move(Err));
 428       else
 429         SendResult(std::move(Result));
 430     } else
 431       SendResult(std::move(Result));
 432   });
 433 }
 434
 435 } // End namespace orc.
 436 } // End namespace llvm.