lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp

   1 //===------- ObjectLinkingLayer.cpp - JITLink backed ORC ObjectLayer ------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
  10
  11 #include "llvm/ADT/Optional.h"
  12 #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h"
  13
  14 #include <vector>
  15
  16 #define DEBUG_TYPE "orc"
  17
  18 using namespace llvm;
  19 using namespace llvm::jitlink;
  20 using namespace llvm::orc;
  21
  22 namespace llvm {
  23 namespace orc {
  24
  25 class ObjectLinkingLayerJITLinkContext final : public JITLinkContext {
  26 public:
  27   ObjectLinkingLayerJITLinkContext(ObjectLinkingLayer &Layer,
  28                                    MaterializationResponsibility MR,
  29                                    std::unique_ptr<MemoryBuffer> ObjBuffer)
  30       : Layer(Layer), MR(std::move(MR)), ObjBuffer(std::move(ObjBuffer)) {}
  31
  32   ~ObjectLinkingLayerJITLinkContext() {
  33     // If there is an object buffer return function then use it to
  34     // return ownership of the buffer.
  35     if (Layer.ReturnObjectBuffer)
  36       Layer.ReturnObjectBuffer(std::move(ObjBuffer));
  37   }
  38
  39   JITLinkMemoryManager &getMemoryManager() override { return Layer.MemMgr; }
  40
  41   MemoryBufferRef getObjectBuffer() const override {
  42     return ObjBuffer->getMemBufferRef();
  43   }
  44
  45   void notifyFailed(Error Err) override {
  46     Layer.getExecutionSession().reportError(std::move(Err));
  47     MR.failMaterialization();
  48   }
  49
  50   void lookup(const DenseSet<StringRef> &Symbols,
  51               std::unique_ptr<JITLinkAsyncLookupContinuation> LC) override {
  52
  53     JITDylibSearchList SearchOrder;
  54     MR.getTargetJITDylib().withSearchOrderDo(
  55         [&](const JITDylibSearchList &JDs) { SearchOrder = JDs; });
  56
  57     auto &ES = Layer.getExecutionSession();
  58
  59     SymbolNameSet InternedSymbols;
  60     for (auto &S : Symbols)
  61       InternedSymbols.insert(ES.intern(S));
  62
  63     // OnResolve -- De-intern the symbols and pass the result to the linker.
  64     auto OnResolve = [this, LookupContinuation = std::move(LC)](
  65                          Expected<SymbolMap> Result) mutable {
  66       auto Main = Layer.getExecutionSession().intern("_main");
  67       if (!Result)
  68         LookupContinuation->run(Result.takeError());
  69       else {
  70         AsyncLookupResult LR;
  71         for (auto &KV : *Result)
  72           LR[*KV.first] = KV.second;
  73         LookupContinuation->run(std::move(LR));
  74       }
  75     };
  76
  77     ES.lookup(SearchOrder, std::move(InternedSymbols), SymbolState::Resolved,
  78               std::move(OnResolve), [this](const SymbolDependenceMap &Deps) {
  79                 registerDependencies(Deps);
  80               });
  81   }
  82
  83   void notifyResolved(LinkGraph &G) override {
  84     auto &ES = Layer.getExecutionSession();
  85
  86     SymbolFlagsMap ExtraSymbolsToClaim;
  87     bool AutoClaim = Layer.AutoClaimObjectSymbols;
  88
  89     SymbolMap InternedResult;
  90     for (auto *Sym : G.defined_symbols())
  91       if (Sym->hasName() && Sym->getScope() != Scope::Local) {
  92         auto InternedName = ES.intern(Sym->getName());
  93         JITSymbolFlags Flags;
  94
  95         if (Sym->isCallable())
  96           Flags |= JITSymbolFlags::Callable;
  97         if (Sym->getScope() == Scope::Default)
  98           Flags |= JITSymbolFlags::Exported;
  99
 100         InternedResult[InternedName] =
 101             JITEvaluatedSymbol(Sym->getAddress(), Flags);
 102         if (AutoClaim && !MR.getSymbols().count(InternedName)) {
 103           assert(!ExtraSymbolsToClaim.count(InternedName) &&
 104                  "Duplicate symbol to claim?");
 105           ExtraSymbolsToClaim[InternedName] = Flags;
 106         }
 107       }
 108
 109     for (auto *Sym : G.absolute_symbols())
 110       if (Sym->hasName()) {
 111         auto InternedName = ES.intern(Sym->getName());
 112         JITSymbolFlags Flags;
 113         Flags |= JITSymbolFlags::Absolute;
 114         if (Sym->isCallable())
 115           Flags |= JITSymbolFlags::Callable;
 116         if (Sym->getLinkage() == Linkage::Weak)
 117           Flags |= JITSymbolFlags::Weak;
 118         InternedResult[InternedName] =
 119             JITEvaluatedSymbol(Sym->getAddress(), Flags);
 120         if (AutoClaim && !MR.getSymbols().count(InternedName)) {
 121           assert(!ExtraSymbolsToClaim.count(InternedName) &&
 122                  "Duplicate symbol to claim?");
 123           ExtraSymbolsToClaim[InternedName] = Flags;
 124         }
 125       }
 126
 127     if (!ExtraSymbolsToClaim.empty())
 128       if (auto Err = MR.defineMaterializing(ExtraSymbolsToClaim))
 129         return notifyFailed(std::move(Err));
 130     if (auto Err = MR.notifyResolved(InternedResult)) {
 131       Layer.getExecutionSession().reportError(std::move(Err));
 132       MR.failMaterialization();
 133       return;
 134     }
 135     Layer.notifyLoaded(MR);
 136   }
 137
 138   void notifyFinalized(
 139       std::unique_ptr<JITLinkMemoryManager::Allocation> A) override {
 140     if (auto Err = Layer.notifyEmitted(MR, std::move(A))) {
 141       Layer.getExecutionSession().reportError(std::move(Err));
 142       MR.failMaterialization();
 143       return;
 144     }
 145     if (auto Err = MR.notifyEmitted()) {
 146       Layer.getExecutionSession().reportError(std::move(Err));
 147       MR.failMaterialization();
 148     }
 149   }
 150
 151   LinkGraphPassFunction getMarkLivePass(const Triple &TT) const override {
 152     return [this](LinkGraph &G) { return markResponsibilitySymbolsLive(G); };
 153   }
 154
 155   Error modifyPassConfig(const Triple &TT, PassConfiguration &Config) override {
 156     // Add passes to mark duplicate defs as should-discard, and to walk the
 157     // link graph to build the symbol dependence graph.
 158     Config.PrePrunePasses.push_back(
 159         [this](LinkGraph &G) { return externalizeWeakAndCommonSymbols(G); });
 160     Config.PostPrunePasses.push_back(
 161         [this](LinkGraph &G) { return computeNamedSymbolDependencies(G); });
 162
 163     Layer.modifyPassConfig(MR, TT, Config);
 164
 165     return Error::success();
 166   }
 167
 168 private:
 169   using AnonToNamedDependenciesMap = DenseMap<const Symbol *, SymbolNameSet>;
 170
 171   Error externalizeWeakAndCommonSymbols(LinkGraph &G) {
 172     auto &ES = Layer.getExecutionSession();
 173     for (auto *Sym : G.defined_symbols())
 174       if (Sym->hasName() && Sym->getLinkage() == Linkage::Weak) {
 175         if (!MR.getSymbols().count(ES.intern(Sym->getName())))
 176           G.makeExternal(*Sym);
 177       }
 178
 179     for (auto *Sym : G.absolute_symbols())
 180       if (Sym->hasName() && Sym->getLinkage() == Linkage::Weak) {
 181         if (!MR.getSymbols().count(ES.intern(Sym->getName())))
 182           G.makeExternal(*Sym);
 183       }
 184
 185     return Error::success();
 186   }
 187
 188   Error markResponsibilitySymbolsLive(LinkGraph &G) const {
 189     auto &ES = Layer.getExecutionSession();
 190     for (auto *Sym : G.defined_symbols())
 191       if (Sym->hasName() && MR.getSymbols().count(ES.intern(Sym->getName())))
 192         Sym->setLive(true);
 193     return Error::success();
 194   }
 195
 196   Error computeNamedSymbolDependencies(LinkGraph &G) {
 197     auto &ES = MR.getTargetJITDylib().getExecutionSession();
 198     auto AnonDeps = computeAnonDeps(G);
 199
 200     for (auto *Sym : G.defined_symbols()) {
 201
 202       // Skip anonymous and non-global atoms: we do not need dependencies for
 203       // these.
 204       if (Sym->getScope() == Scope::Local)
 205         continue;
 206
 207       auto SymName = ES.intern(Sym->getName());
 208       SymbolNameSet &SymDeps = NamedSymbolDeps[SymName];
 209
 210       for (auto &E : Sym->getBlock().edges()) {
 211         auto &TargetSym = E.getTarget();
 212
 213         if (TargetSym.getScope() != Scope::Local)
 214           SymDeps.insert(ES.intern(TargetSym.getName()));
 215         else {
 216           assert(TargetSym.isDefined() &&
 217                  "Anonymous/local symbols must be defined");
 218           auto I = AnonDeps.find(&TargetSym);
 219           if (I != AnonDeps.end())
 220             for (auto &S : I->second)
 221               SymDeps.insert(S);
 222         }
 223       }
 224     }
 225
 226     return Error::success();
 227   }
 228
 229   AnonToNamedDependenciesMap computeAnonDeps(LinkGraph &G) {
 230
 231     auto &ES = MR.getTargetJITDylib().getExecutionSession();
 232     AnonToNamedDependenciesMap DepMap;
 233
 234     // For all anonymous symbols:
 235     // (1) Add their named dependencies.
 236     // (2) Add them to the worklist for further iteration if they have any
 237     //     depend on any other anonymous symbols.
 238     struct WorklistEntry {
 239       WorklistEntry(Symbol *Sym, DenseSet<Symbol *> SymAnonDeps)
 240           : Sym(Sym), SymAnonDeps(std::move(SymAnonDeps)) {}
 241
 242       Symbol *Sym = nullptr;
 243       DenseSet<Symbol *> SymAnonDeps;
 244     };
 245     std::vector<WorklistEntry> Worklist;
 246     for (auto *Sym : G.defined_symbols())
 247       if (!Sym->hasName()) {
 248         auto &SymNamedDeps = DepMap[Sym];
 249         DenseSet<Symbol *> SymAnonDeps;
 250
 251         for (auto &E : Sym->getBlock().edges()) {
 252           auto &TargetSym = E.getTarget();
 253           if (TargetSym.hasName())
 254             SymNamedDeps.insert(ES.intern(TargetSym.getName()));
 255           else {
 256             assert(TargetSym.isDefined() &&
 257                    "Anonymous symbols must be defined");
 258             SymAnonDeps.insert(&TargetSym);
 259           }
 260         }
 261
 262         if (!SymAnonDeps.empty())
 263           Worklist.push_back(WorklistEntry(Sym, std::move(SymAnonDeps)));
 264       }
 265
 266     // Loop over all anonymous symbols with anonymous dependencies, propagating
 267     // their respective *named* dependencies. Iterate until we hit a stable
 268     // state.
 269     bool Changed;
 270     do {
 271       Changed = false;
 272       for (auto &WLEntry : Worklist) {
 273         auto *Sym = WLEntry.Sym;
 274         auto &SymNamedDeps = DepMap[Sym];
 275         auto &SymAnonDeps = WLEntry.SymAnonDeps;
 276
 277         for (auto *TargetSym : SymAnonDeps) {
 278           auto I = DepMap.find(TargetSym);
 279           if (I != DepMap.end())
 280             for (const auto &S : I->second)
 281               Changed |= SymNamedDeps.insert(S).second;
 282         }
 283       }
 284     } while (Changed);
 285
 286     return DepMap;
 287   }
 288
 289   void registerDependencies(const SymbolDependenceMap &QueryDeps) {
 290     for (auto &NamedDepsEntry : NamedSymbolDeps) {
 291       auto &Name = NamedDepsEntry.first;
 292       auto &NameDeps = NamedDepsEntry.second;
 293       SymbolDependenceMap SymbolDeps;
 294
 295       for (const auto &QueryDepsEntry : QueryDeps) {
 296         JITDylib &SourceJD = *QueryDepsEntry.first;
 297         const SymbolNameSet &Symbols = QueryDepsEntry.second;
 298         auto &DepsForJD = SymbolDeps[&SourceJD];
 299
 300         for (const auto &S : Symbols)
 301           if (NameDeps.count(S))
 302             DepsForJD.insert(S);
 303
 304         if (DepsForJD.empty())
 305           SymbolDeps.erase(&SourceJD);
 306       }
 307
 308       MR.addDependencies(Name, SymbolDeps);
 309     }
 310   }
 311
 312   ObjectLinkingLayer &Layer;
 313   MaterializationResponsibility MR;
 314   std::unique_ptr<MemoryBuffer> ObjBuffer;
 315   DenseMap<SymbolStringPtr, SymbolNameSet> NamedSymbolDeps;
 316 };
 317
 318 ObjectLinkingLayer::Plugin::~Plugin() {}
 319
 320 ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES,
 321                                        JITLinkMemoryManager &MemMgr)
 322     : ObjectLayer(ES), MemMgr(MemMgr) {}
 323
 324 ObjectLinkingLayer::~ObjectLinkingLayer() {
 325   if (auto Err = removeAllModules())
 326     getExecutionSession().reportError(std::move(Err));
 327 }
 328
 329 void ObjectLinkingLayer::emit(MaterializationResponsibility R,
 330                               std::unique_ptr<MemoryBuffer> O) {
 331   assert(O && "Object must not be null");
 332   jitLink(std::make_unique<ObjectLinkingLayerJITLinkContext>(
 333       *this, std::move(R), std::move(O)));
 334 }
 335
 336 void ObjectLinkingLayer::modifyPassConfig(MaterializationResponsibility &MR,
 337                                           const Triple &TT,
 338                                           PassConfiguration &PassConfig) {
 339   for (auto &P : Plugins)
 340     P->modifyPassConfig(MR, TT, PassConfig);
 341 }
 342
 343 void ObjectLinkingLayer::notifyLoaded(MaterializationResponsibility &MR) {
 344   for (auto &P : Plugins)
 345     P->notifyLoaded(MR);
 346 }
 347
 348 Error ObjectLinkingLayer::notifyEmitted(MaterializationResponsibility &MR,
 349                                         AllocPtr Alloc) {
 350   Error Err = Error::success();
 351   for (auto &P : Plugins)
 352     Err = joinErrors(std::move(Err), P->notifyEmitted(MR));
 353
 354   if (Err)
 355     return Err;
 356
 357   {
 358     std::lock_guard<std::mutex> Lock(LayerMutex);
 359     UntrackedAllocs.push_back(std::move(Alloc));
 360   }
 361
 362   return Error::success();
 363 }
 364
 365 Error ObjectLinkingLayer::removeModule(VModuleKey K) {
 366   Error Err = Error::success();
 367
 368   for (auto &P : Plugins)
 369     Err = joinErrors(std::move(Err), P->notifyRemovingModule(K));
 370
 371   AllocPtr Alloc;
 372
 373   {
 374     std::lock_guard<std::mutex> Lock(LayerMutex);
 375     auto AllocItr = TrackedAllocs.find(K);
 376     Alloc = std::move(AllocItr->second);
 377     TrackedAllocs.erase(AllocItr);
 378   }
 379
 380   assert(Alloc && "No allocation for key K");
 381
 382   return joinErrors(std::move(Err), Alloc->deallocate());
 383 }
 384
 385 Error ObjectLinkingLayer::removeAllModules() {
 386
 387   Error Err = Error::success();
 388
 389   for (auto &P : Plugins)
 390     Err = joinErrors(std::move(Err), P->notifyRemovingAllModules());
 391
 392   std::vector<AllocPtr> Allocs;
 393   {
 394     std::lock_guard<std::mutex> Lock(LayerMutex);
 395     Allocs = std::move(UntrackedAllocs);
 396
 397     for (auto &KV : TrackedAllocs)
 398       Allocs.push_back(std::move(KV.second));
 399
 400     TrackedAllocs.clear();
 401   }
 402
 403   while (!Allocs.empty()) {
 404     Err = joinErrors(std::move(Err), Allocs.back()->deallocate());
 405     Allocs.pop_back();
 406   }
 407
 408   return Err;
 409 }
 410
 411 EHFrameRegistrationPlugin::EHFrameRegistrationPlugin(
 412     EHFrameRegistrar &Registrar)
 413     : Registrar(Registrar) {}
 414
 415 void EHFrameRegistrationPlugin::modifyPassConfig(
 416     MaterializationResponsibility &MR, const Triple &TT,
 417     PassConfiguration &PassConfig) {
 418   assert(!InProcessLinks.count(&MR) && "Link for MR already being tracked?");
 419
 420   PassConfig.PostFixupPasses.push_back(
 421       createEHFrameRecorderPass(TT, [this, &MR](JITTargetAddress Addr,
 422                                                 size_t Size) {
 423         if (Addr)
 424           InProcessLinks[&MR] = { Addr, Size };
 425       }));
 426 }
 427
 428 Error EHFrameRegistrationPlugin::notifyEmitted(
 429     MaterializationResponsibility &MR) {
 430
 431   auto EHFrameRangeItr = InProcessLinks.find(&MR);
 432   if (EHFrameRangeItr == InProcessLinks.end())
 433     return Error::success();
 434
 435   auto EHFrameRange = EHFrameRangeItr->second;
 436   assert(EHFrameRange.Addr &&
 437          "eh-frame addr to register can not be null");
 438
 439   InProcessLinks.erase(EHFrameRangeItr);
 440   if (auto Key = MR.getVModuleKey())
 441     TrackedEHFrameRanges[Key] = EHFrameRange;
 442   else
 443     UntrackedEHFrameRanges.push_back(EHFrameRange);
 444
 445   return Registrar.registerEHFrames(EHFrameRange.Addr, EHFrameRange.Size);
 446 }
 447
 448 Error EHFrameRegistrationPlugin::notifyRemovingModule(VModuleKey K) {
 449   auto EHFrameRangeItr = TrackedEHFrameRanges.find(K);
 450   if (EHFrameRangeItr == TrackedEHFrameRanges.end())
 451     return Error::success();
 452
 453   auto EHFrameRange = EHFrameRangeItr->second;
 454   assert(EHFrameRange.Addr && "Tracked eh-frame range must not be null");
 455
 456   TrackedEHFrameRanges.erase(EHFrameRangeItr);
 457
 458   return Registrar.deregisterEHFrames(EHFrameRange.Addr, EHFrameRange.Size);
 459 }
 460
 461 Error EHFrameRegistrationPlugin::notifyRemovingAllModules() {
 462
 463   std::vector<EHFrameRange> EHFrameRanges =
 464     std::move(UntrackedEHFrameRanges);
 465   EHFrameRanges.reserve(EHFrameRanges.size() + TrackedEHFrameRanges.size());
 466
 467   for (auto &KV : TrackedEHFrameRanges)
 468     EHFrameRanges.push_back(KV.second);
 469
 470   TrackedEHFrameRanges.clear();
 471
 472   Error Err = Error::success();
 473
 474   while (!EHFrameRanges.empty()) {
 475     auto EHFrameRange = EHFrameRanges.back();
 476     assert(EHFrameRange.Addr && "Untracked eh-frame range must not be null");
 477     EHFrameRanges.pop_back();
 478     Err = joinErrors(std::move(Err),
 479                      Registrar.deregisterEHFrames(EHFrameRange.Addr,
 480                                                   EHFrameRange.Size));
 481   }
 482
 483   return Err;
 484 }
 485
 486 } // End namespace orc.
 487 } // End namespace llvm.