lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp

   1 //===------- ObjectLinkingLayer.cpp - JITLink backed ORC ObjectLayer ------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
  10
  11 #include "llvm/ADT/Optional.h"
  12 #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h"
  13
  14 #include <vector>
  15
  16 #define DEBUG_TYPE "orc"
  17
  18 using namespace llvm;
  19 using namespace llvm::jitlink;
  20 using namespace llvm::orc;
  21
  22 namespace llvm {
  23 namespace orc {
  24
  25 class ObjectLinkingLayerJITLinkContext final : public JITLinkContext {
  26 public:
  27   ObjectLinkingLayerJITLinkContext(ObjectLinkingLayer &Layer,
  28                                    MaterializationResponsibility MR,
  29                                    std::unique_ptr<MemoryBuffer> ObjBuffer)
  30       : Layer(Layer), MR(std::move(MR)), ObjBuffer(std::move(ObjBuffer)) {}
  31
  32   JITLinkMemoryManager &getMemoryManager() override { return Layer.MemMgr; }
  33
  34   MemoryBufferRef getObjectBuffer() const override {
  35     return ObjBuffer->getMemBufferRef();
  36   }
  37
  38   void notifyFailed(Error Err) override {
  39     Layer.getExecutionSession().reportError(std::move(Err));
  40     MR.failMaterialization();
  41   }
  42
  43   void lookup(const DenseSet<StringRef> &Symbols,
  44               JITLinkAsyncLookupContinuation LookupContinuation) override {
  45
  46     JITDylibSearchList SearchOrder;
  47     MR.getTargetJITDylib().withSearchOrderDo(
  48         [&](const JITDylibSearchList &JDs) { SearchOrder = JDs; });
  49
  50     auto &ES = Layer.getExecutionSession();
  51
  52     SymbolNameSet InternedSymbols;
  53     for (auto &S : Symbols)
  54       InternedSymbols.insert(ES.intern(S));
  55
  56     // OnResolve -- De-intern the symbols and pass the result to the linker.
  57     // FIXME: Capture LookupContinuation by move once we have c++14.
  58     auto SharedLookupContinuation =
  59         std::make_shared<JITLinkAsyncLookupContinuation>(
  60             std::move(LookupContinuation));
  61     auto OnResolve = [SharedLookupContinuation](Expected<SymbolMap> Result) {
  62       if (!Result)
  63         (*SharedLookupContinuation)(Result.takeError());
  64       else {
  65         AsyncLookupResult LR;
  66         for (auto &KV : *Result)
  67           LR[*KV.first] = KV.second;
  68         (*SharedLookupContinuation)(std::move(LR));
  69       }
  70     };
  71
  72     ES.lookup(SearchOrder, std::move(InternedSymbols), SymbolState::Resolved,
  73               std::move(OnResolve), [this](const SymbolDependenceMap &Deps) {
  74                 registerDependencies(Deps);
  75               });
  76   }
  77
  78   void notifyResolved(AtomGraph &G) override {
  79     auto &ES = Layer.getExecutionSession();
  80
  81     SymbolFlagsMap ExtraSymbolsToClaim;
  82     bool AutoClaim = Layer.AutoClaimObjectSymbols;
  83
  84     SymbolMap InternedResult;
  85     for (auto *DA : G.defined_atoms())
  86       if (DA->hasName() && DA->isGlobal()) {
  87         auto InternedName = ES.intern(DA->getName());
  88         JITSymbolFlags Flags;
  89
  90         if (DA->isExported())
  91           Flags |= JITSymbolFlags::Exported;
  92         if (DA->isWeak())
  93           Flags |= JITSymbolFlags::Weak;
  94         if (DA->isCallable())
  95           Flags |= JITSymbolFlags::Callable;
  96         if (DA->isCommon())
  97           Flags |= JITSymbolFlags::Common;
  98
  99         InternedResult[InternedName] =
 100             JITEvaluatedSymbol(DA->getAddress(), Flags);
 101         if (AutoClaim && !MR.getSymbols().count(InternedName)) {
 102           assert(!ExtraSymbolsToClaim.count(InternedName) &&
 103                  "Duplicate symbol to claim?");
 104           ExtraSymbolsToClaim[InternedName] = Flags;
 105         }
 106       }
 107
 108     for (auto *A : G.absolute_atoms())
 109       if (A->hasName()) {
 110         auto InternedName = ES.intern(A->getName());
 111         JITSymbolFlags Flags;
 112         Flags |= JITSymbolFlags::Absolute;
 113         if (A->isWeak())
 114           Flags |= JITSymbolFlags::Weak;
 115         if (A->isCallable())
 116           Flags |= JITSymbolFlags::Callable;
 117         InternedResult[InternedName] =
 118             JITEvaluatedSymbol(A->getAddress(), Flags);
 119         if (AutoClaim && !MR.getSymbols().count(InternedName)) {
 120           assert(!ExtraSymbolsToClaim.count(InternedName) &&
 121                  "Duplicate symbol to claim?");
 122           ExtraSymbolsToClaim[InternedName] = Flags;
 123         }
 124       }
 125
 126     if (!ExtraSymbolsToClaim.empty())
 127       if (auto Err = MR.defineMaterializing(ExtraSymbolsToClaim))
 128         return notifyFailed(std::move(Err));
 129
 130     if (auto Err = MR.notifyResolved(InternedResult)) {
 131       Layer.getExecutionSession().reportError(std::move(Err));
 132       MR.failMaterialization();
 133       return;
 134     }
 135
 136     Layer.notifyLoaded(MR);
 137   }
 138
 139   void notifyFinalized(
 140       std::unique_ptr<JITLinkMemoryManager::Allocation> A) override {
 141
 142     if (auto Err = Layer.notifyEmitted(MR, std::move(A))) {
 143       Layer.getExecutionSession().reportError(std::move(Err));
 144       MR.failMaterialization();
 145       return;
 146     }
 147     if (auto Err = MR.notifyEmitted()) {
 148       Layer.getExecutionSession().reportError(std::move(Err));
 149       MR.failMaterialization();
 150     }
 151   }
 152
 153   AtomGraphPassFunction getMarkLivePass(const Triple &TT) const override {
 154     return [this](AtomGraph &G) { return markResponsibilitySymbolsLive(G); };
 155   }
 156
 157   Error modifyPassConfig(const Triple &TT, PassConfiguration &Config) override {
 158     // Add passes to mark duplicate defs as should-discard, and to walk the
 159     // atom graph to build the symbol dependence graph.
 160     Config.PrePrunePasses.push_back(
 161         [this](AtomGraph &G) { return markSymbolsToDiscard(G); });
 162     Config.PostPrunePasses.push_back(
 163         [this](AtomGraph &G) { return computeNamedSymbolDependencies(G); });
 164
 165     Layer.modifyPassConfig(MR, TT, Config);
 166
 167     return Error::success();
 168   }
 169
 170 private:
 171   using AnonAtomNamedDependenciesMap =
 172       DenseMap<const DefinedAtom *, SymbolNameSet>;
 173
 174   Error markSymbolsToDiscard(AtomGraph &G) {
 175     auto &ES = Layer.getExecutionSession();
 176     for (auto *DA : G.defined_atoms())
 177       if (DA->isWeak() && DA->hasName()) {
 178         auto S = ES.intern(DA->getName());
 179         auto I = MR.getSymbols().find(S);
 180         if (I == MR.getSymbols().end())
 181           DA->setShouldDiscard(true);
 182       }
 183
 184     for (auto *A : G.absolute_atoms())
 185       if (A->isWeak() && A->hasName()) {
 186         auto S = ES.intern(A->getName());
 187         auto I = MR.getSymbols().find(S);
 188         if (I == MR.getSymbols().end())
 189           A->setShouldDiscard(true);
 190       }
 191
 192     return Error::success();
 193   }
 194
 195   Error markResponsibilitySymbolsLive(AtomGraph &G) const {
 196     auto &ES = Layer.getExecutionSession();
 197     for (auto *DA : G.defined_atoms())
 198       if (DA->hasName() &&
 199           MR.getSymbols().count(ES.intern(DA->getName())))
 200         DA->setLive(true);
 201     return Error::success();
 202   }
 203
 204   Error computeNamedSymbolDependencies(AtomGraph &G) {
 205     auto &ES = MR.getTargetJITDylib().getExecutionSession();
 206     auto AnonDeps = computeAnonDeps(G);
 207
 208     for (auto *DA : G.defined_atoms()) {
 209
 210       // Skip anonymous and non-global atoms: we do not need dependencies for
 211       // these.
 212       if (!DA->hasName() || !DA->isGlobal())
 213         continue;
 214
 215       auto DAName = ES.intern(DA->getName());
 216       SymbolNameSet &DADeps = NamedSymbolDeps[DAName];
 217
 218       for (auto &E : DA->edges()) {
 219         auto &TA = E.getTarget();
 220
 221         if (TA.hasName())
 222           DADeps.insert(ES.intern(TA.getName()));
 223         else {
 224           assert(TA.isDefined() && "Anonymous atoms must be defined");
 225           auto &DTA = static_cast<DefinedAtom &>(TA);
 226           auto I = AnonDeps.find(&DTA);
 227           if (I != AnonDeps.end())
 228             for (auto &S : I->second)
 229               DADeps.insert(S);
 230         }
 231       }
 232     }
 233
 234     return Error::success();
 235   }
 236
 237   AnonAtomNamedDependenciesMap computeAnonDeps(AtomGraph &G) {
 238
 239     auto &ES = MR.getTargetJITDylib().getExecutionSession();
 240     AnonAtomNamedDependenciesMap DepMap;
 241
 242     // For all anonymous atoms:
 243     // (1) Add their named dependencies.
 244     // (2) Add them to the worklist for further iteration if they have any
 245     //     depend on any other anonymous atoms.
 246     struct WorklistEntry {
 247       WorklistEntry(DefinedAtom *DA, DenseSet<DefinedAtom *> DAAnonDeps)
 248           : DA(DA), DAAnonDeps(std::move(DAAnonDeps)) {}
 249
 250       DefinedAtom *DA = nullptr;
 251       DenseSet<DefinedAtom *> DAAnonDeps;
 252     };
 253     std::vector<WorklistEntry> Worklist;
 254     for (auto *DA : G.defined_atoms())
 255       if (!DA->hasName()) {
 256         auto &DANamedDeps = DepMap[DA];
 257         DenseSet<DefinedAtom *> DAAnonDeps;
 258
 259         for (auto &E : DA->edges()) {
 260           auto &TA = E.getTarget();
 261           if (TA.hasName())
 262             DANamedDeps.insert(ES.intern(TA.getName()));
 263           else {
 264             assert(TA.isDefined() && "Anonymous atoms must be defined");
 265             DAAnonDeps.insert(static_cast<DefinedAtom *>(&TA));
 266           }
 267         }
 268
 269         if (!DAAnonDeps.empty())
 270           Worklist.push_back(WorklistEntry(DA, std::move(DAAnonDeps)));
 271       }
 272
 273     // Loop over all anonymous atoms with anonymous dependencies, propagating
 274     // their respective *named* dependencies. Iterate until we hit a stable
 275     // state.
 276     bool Changed;
 277     do {
 278       Changed = false;
 279       for (auto &WLEntry : Worklist) {
 280         auto *DA = WLEntry.DA;
 281         auto &DANamedDeps = DepMap[DA];
 282         auto &DAAnonDeps = WLEntry.DAAnonDeps;
 283
 284         for (auto *TA : DAAnonDeps) {
 285           auto I = DepMap.find(TA);
 286           if (I != DepMap.end())
 287             for (const auto &S : I->second)
 288               Changed |= DANamedDeps.insert(S).second;
 289         }
 290       }
 291     } while (Changed);
 292
 293     return DepMap;
 294   }
 295
 296   void registerDependencies(const SymbolDependenceMap &QueryDeps) {
 297     for (auto &NamedDepsEntry : NamedSymbolDeps) {
 298       auto &Name = NamedDepsEntry.first;
 299       auto &NameDeps = NamedDepsEntry.second;
 300       SymbolDependenceMap SymbolDeps;
 301
 302       for (const auto &QueryDepsEntry : QueryDeps) {
 303         JITDylib &SourceJD = *QueryDepsEntry.first;
 304         const SymbolNameSet &Symbols = QueryDepsEntry.second;
 305         auto &DepsForJD = SymbolDeps[&SourceJD];
 306
 307         for (const auto &S : Symbols)
 308           if (NameDeps.count(S))
 309             DepsForJD.insert(S);
 310
 311         if (DepsForJD.empty())
 312           SymbolDeps.erase(&SourceJD);
 313       }
 314
 315       MR.addDependencies(Name, SymbolDeps);
 316     }
 317   }
 318
 319   ObjectLinkingLayer &Layer;
 320   MaterializationResponsibility MR;
 321   std::unique_ptr<MemoryBuffer> ObjBuffer;
 322   DenseMap<SymbolStringPtr, SymbolNameSet> NamedSymbolDeps;
 323 };
 324
 325 ObjectLinkingLayer::Plugin::~Plugin() {}
 326
 327 ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES,
 328                                        JITLinkMemoryManager &MemMgr)
 329     : ObjectLayer(ES), MemMgr(MemMgr) {}
 330
 331 ObjectLinkingLayer::~ObjectLinkingLayer() {
 332   if (auto Err = removeAllModules())
 333     getExecutionSession().reportError(std::move(Err));
 334 }
 335
 336 void ObjectLinkingLayer::emit(MaterializationResponsibility R,
 337                               std::unique_ptr<MemoryBuffer> O) {
 338   assert(O && "Object must not be null");
 339   jitLink(std::make_unique<ObjectLinkingLayerJITLinkContext>(
 340       *this, std::move(R), std::move(O)));
 341 }
 342
 343 void ObjectLinkingLayer::modifyPassConfig(MaterializationResponsibility &MR,
 344                                           const Triple &TT,
 345                                           PassConfiguration &PassConfig) {
 346   for (auto &P : Plugins)
 347     P->modifyPassConfig(MR, TT, PassConfig);
 348 }
 349
 350 void ObjectLinkingLayer::notifyLoaded(MaterializationResponsibility &MR) {
 351   for (auto &P : Plugins)
 352     P->notifyLoaded(MR);
 353 }
 354
 355 Error ObjectLinkingLayer::notifyEmitted(MaterializationResponsibility &MR,
 356                                         AllocPtr Alloc) {
 357   Error Err = Error::success();
 358   for (auto &P : Plugins)
 359     Err = joinErrors(std::move(Err), P->notifyEmitted(MR));
 360
 361   if (Err)
 362     return Err;
 363
 364   {
 365     std::lock_guard<std::mutex> Lock(LayerMutex);
 366     UntrackedAllocs.push_back(std::move(Alloc));
 367   }
 368
 369   return Error::success();
 370 }
 371
 372 Error ObjectLinkingLayer::removeModule(VModuleKey K) {
 373   Error Err = Error::success();
 374
 375   for (auto &P : Plugins)
 376     Err = joinErrors(std::move(Err), P->notifyRemovingModule(K));
 377
 378   AllocPtr Alloc;
 379
 380   {
 381     std::lock_guard<std::mutex> Lock(LayerMutex);
 382     auto AllocItr = TrackedAllocs.find(K);
 383     Alloc = std::move(AllocItr->second);
 384     TrackedAllocs.erase(AllocItr);
 385   }
 386
 387   assert(Alloc && "No allocation for key K");
 388
 389   return joinErrors(std::move(Err), Alloc->deallocate());
 390 }
 391
 392 Error ObjectLinkingLayer::removeAllModules() {
 393
 394   Error Err = Error::success();
 395
 396   for (auto &P : Plugins)
 397     Err = joinErrors(std::move(Err), P->notifyRemovingAllModules());
 398
 399   std::vector<AllocPtr> Allocs;
 400   {
 401     std::lock_guard<std::mutex> Lock(LayerMutex);
 402     Allocs = std::move(UntrackedAllocs);
 403
 404     for (auto &KV : TrackedAllocs)
 405       Allocs.push_back(std::move(KV.second));
 406
 407     TrackedAllocs.clear();
 408   }
 409
 410   while (!Allocs.empty()) {
 411     Err = joinErrors(std::move(Err), Allocs.back()->deallocate());
 412     Allocs.pop_back();
 413   }
 414
 415   return Err;
 416 }
 417
 418 EHFrameRegistrationPlugin::EHFrameRegistrationPlugin(
 419     jitlink::EHFrameRegistrar &Registrar)
 420     : Registrar(Registrar) {}
 421
 422 void EHFrameRegistrationPlugin::modifyPassConfig(
 423     MaterializationResponsibility &MR, const Triple &TT,
 424     PassConfiguration &PassConfig) {
 425   assert(!InProcessLinks.count(&MR) && "Link for MR already being tracked?");
 426
 427   PassConfig.PostFixupPasses.push_back(
 428       createEHFrameRecorderPass(TT, [this, &MR](JITTargetAddress Addr) {
 429         if (Addr)
 430           InProcessLinks[&MR] = Addr;
 431       }));
 432 }
 433
 434 Error EHFrameRegistrationPlugin::notifyEmitted(
 435     MaterializationResponsibility &MR) {
 436
 437   auto EHFrameAddrItr = InProcessLinks.find(&MR);
 438   if (EHFrameAddrItr == InProcessLinks.end())
 439     return Error::success();
 440
 441   auto EHFrameAddr = EHFrameAddrItr->second;
 442   assert(EHFrameAddr && "eh-frame addr to register can not be null");
 443
 444   InProcessLinks.erase(EHFrameAddrItr);
 445   if (auto Key = MR.getVModuleKey())
 446     TrackedEHFrameAddrs[Key] = EHFrameAddr;
 447   else
 448     UntrackedEHFrameAddrs.push_back(EHFrameAddr);
 449
 450   return Registrar.registerEHFrames(EHFrameAddr);
 451 }
 452
 453 Error EHFrameRegistrationPlugin::notifyRemovingModule(VModuleKey K) {
 454   auto EHFrameAddrItr = TrackedEHFrameAddrs.find(K);
 455   if (EHFrameAddrItr == TrackedEHFrameAddrs.end())
 456     return Error::success();
 457
 458   auto EHFrameAddr = EHFrameAddrItr->second;
 459   assert(EHFrameAddr && "Tracked eh-frame addr must not be null");
 460
 461   TrackedEHFrameAddrs.erase(EHFrameAddrItr);
 462
 463   return Registrar.deregisterEHFrames(EHFrameAddr);
 464 }
 465
 466 Error EHFrameRegistrationPlugin::notifyRemovingAllModules() {
 467
 468   std::vector<JITTargetAddress> EHFrameAddrs = std::move(UntrackedEHFrameAddrs);
 469   EHFrameAddrs.reserve(EHFrameAddrs.size() + TrackedEHFrameAddrs.size());
 470
 471   for (auto &KV : TrackedEHFrameAddrs)
 472     EHFrameAddrs.push_back(KV.second);
 473
 474   TrackedEHFrameAddrs.clear();
 475
 476   Error Err = Error::success();
 477
 478   while (!EHFrameAddrs.empty()) {
 479     auto EHFrameAddr = EHFrameAddrs.back();
 480     assert(EHFrameAddr && "Untracked eh-frame addr must not be null");
 481     EHFrameAddrs.pop_back();
 482     Err = joinErrors(std::move(Err), Registrar.deregisterEHFrames(EHFrameAddr));
 483   }
 484
 485   return Err;
 486 }
 487
 488 } // End namespace orc.
 489 } // End namespace llvm.