lib/ExecutionEngine/Orc/OrcCBindingsStack.h

   1 //===- OrcCBindingsStack.h - Orc JIT stack for C bindings -----*- C++ -*---===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #ifndef LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H
  10 #define LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H
  11
  12 #include "llvm-c/OrcBindings.h"
  13 #include "llvm-c/TargetMachine.h"
  14 #include "llvm/ADT/STLExtras.h"
  15 #include "llvm/ADT/StringRef.h"
  16 #include "llvm/ExecutionEngine/JITSymbol.h"
  17 #include "llvm/ExecutionEngine/JITEventListener.h"
  18 #include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h"
  19 #include "llvm/ExecutionEngine/Orc/CompileUtils.h"
  20 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
  21 #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
  22 #include "llvm/ExecutionEngine/Orc/LambdaResolver.h"
  23 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
  24 #include "llvm/ExecutionEngine/RuntimeDyld.h"
  25 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
  26 #include "llvm/IR/DataLayout.h"
  27 #include "llvm/IR/Mangler.h"
  28 #include "llvm/IR/Module.h"
  29 #include "llvm/Support/CBindingWrapping.h"
  30 #include "llvm/Support/Error.h"
  31 #include "llvm/Support/raw_ostream.h"
  32 #include "llvm/Target/TargetMachine.h"
  33 #include <algorithm>
  34 #include <cstdint>
  35 #include <functional>
  36 #include <map>
  37 #include <memory>
  38 #include <set>
  39 #include <string>
  40 #include <vector>
  41
  42 namespace llvm {
  43
  44 class OrcCBindingsStack;
  45
  46 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcCBindingsStack, LLVMOrcJITStackRef)
  47 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
  48
  49 namespace detail {
  50
  51 // FIXME: Kill this off once the Layer concept becomes an interface.
  52 class GenericLayer {
  53 public:
  54   virtual ~GenericLayer() = default;
  55
  56   virtual JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
  57                                  bool ExportedSymbolsOnly) = 0;
  58   virtual Error removeModule(orc::VModuleKey K) = 0;
  59   };
  60
  61   template <typename LayerT> class GenericLayerImpl : public GenericLayer {
  62   public:
  63     GenericLayerImpl(LayerT &Layer) : Layer(Layer) {}
  64
  65     JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
  66                            bool ExportedSymbolsOnly) override {
  67       return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly);
  68     }
  69
  70     Error removeModule(orc::VModuleKey K) override {
  71       return Layer.removeModule(K);
  72     }
  73
  74   private:
  75     LayerT &Layer;
  76   };
  77
  78   template <>
  79   class GenericLayerImpl<orc::LegacyRTDyldObjectLinkingLayer> : public GenericLayer {
  80   private:
  81     using LayerT = orc::LegacyRTDyldObjectLinkingLayer;
  82   public:
  83     GenericLayerImpl(LayerT &Layer) : Layer(Layer) {}
  84
  85     JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
  86                            bool ExportedSymbolsOnly) override {
  87       return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly);
  88     }
  89
  90     Error removeModule(orc::VModuleKey K) override {
  91       return Layer.removeObject(K);
  92     }
  93
  94   private:
  95     LayerT &Layer;
  96   };
  97
  98   template <typename LayerT>
  99   std::unique_ptr<GenericLayerImpl<LayerT>> createGenericLayer(LayerT &Layer) {
 100     return std::make_unique<GenericLayerImpl<LayerT>>(Layer);
 101   }
 102
 103 } // end namespace detail
 104
 105 class OrcCBindingsStack {
 106 public:
 107
 108   using CompileCallbackMgr = orc::JITCompileCallbackManager;
 109   using ObjLayerT = orc::LegacyRTDyldObjectLinkingLayer;
 110   using CompileLayerT = orc::LegacyIRCompileLayer<ObjLayerT, orc::SimpleCompiler>;
 111   using CODLayerT =
 112         orc::LegacyCompileOnDemandLayer<CompileLayerT, CompileCallbackMgr>;
 113
 114   using CallbackManagerBuilder =
 115       std::function<std::unique_ptr<CompileCallbackMgr>()>;
 116
 117   using IndirectStubsManagerBuilder = CODLayerT::IndirectStubsManagerBuilderT;
 118
 119 private:
 120
 121   using OwningObject = object::OwningBinary<object::ObjectFile>;
 122
 123   class CBindingsResolver : public orc::SymbolResolver {
 124   public:
 125     CBindingsResolver(OrcCBindingsStack &Stack,
 126                       LLVMOrcSymbolResolverFn ExternalResolver,
 127                       void *ExternalResolverCtx)
 128         : Stack(Stack), ExternalResolver(std::move(ExternalResolver)),
 129           ExternalResolverCtx(std::move(ExternalResolverCtx)) {}
 130
 131     orc::SymbolNameSet
 132     getResponsibilitySet(const orc::SymbolNameSet &Symbols) override {
 133       orc::SymbolNameSet Result;
 134
 135       for (auto &S : Symbols) {
 136         if (auto Sym = findSymbol(*S)) {
 137           if (!Sym.getFlags().isStrong())
 138             Result.insert(S);
 139         } else if (auto Err = Sym.takeError()) {
 140           Stack.reportError(std::move(Err));
 141           return orc::SymbolNameSet();
 142         }
 143       }
 144
 145       return Result;
 146     }
 147
 148     orc::SymbolNameSet
 149     lookup(std::shared_ptr<orc::AsynchronousSymbolQuery> Query,
 150            orc::SymbolNameSet Symbols) override {
 151       orc::SymbolNameSet UnresolvedSymbols;
 152
 153       for (auto &S : Symbols) {
 154         if (auto Sym = findSymbol(*S)) {
 155           if (auto Addr = Sym.getAddress()) {
 156             Query->notifySymbolMetRequiredState(
 157                 S, JITEvaluatedSymbol(*Addr, Sym.getFlags()));
 158           } else {
 159             Stack.ES.legacyFailQuery(*Query, Addr.takeError());
 160             return orc::SymbolNameSet();
 161           }
 162         } else if (auto Err = Sym.takeError()) {
 163           Stack.ES.legacyFailQuery(*Query, std::move(Err));
 164           return orc::SymbolNameSet();
 165         } else
 166           UnresolvedSymbols.insert(S);
 167       }
 168
 169       if (Query->isComplete())
 170         Query->handleComplete();
 171
 172       return UnresolvedSymbols;
 173     }
 174
 175   private:
 176     JITSymbol findSymbol(const std::string &Name) {
 177       // Search order:
 178       // 1. JIT'd symbols.
 179       // 2. Runtime overrides.
 180       // 3. External resolver (if present).
 181
 182       if (Stack.CODLayer) {
 183         if (auto Sym = Stack.CODLayer->findSymbol(Name, true))
 184           return Sym;
 185         else if (auto Err = Sym.takeError())
 186           return Sym.takeError();
 187       } else {
 188         if (auto Sym = Stack.CompileLayer.findSymbol(Name, true))
 189           return Sym;
 190         else if (auto Err = Sym.takeError())
 191           return Sym.takeError();
 192       }
 193
 194       if (auto Sym = Stack.CXXRuntimeOverrides.searchOverrides(Name))
 195         return Sym;
 196
 197       if (ExternalResolver)
 198         return JITSymbol(ExternalResolver(Name.c_str(), ExternalResolverCtx),
 199                          JITSymbolFlags::Exported);
 200
 201       return JITSymbol(nullptr);
 202     }
 203
 204     OrcCBindingsStack &Stack;
 205     LLVMOrcSymbolResolverFn ExternalResolver;
 206     void *ExternalResolverCtx = nullptr;
 207   };
 208
 209 public:
 210   OrcCBindingsStack(TargetMachine &TM,
 211                     IndirectStubsManagerBuilder IndirectStubsMgrBuilder)
 212       : CCMgr(createCompileCallbackManager(TM, ES)), DL(TM.createDataLayout()),
 213         IndirectStubsMgr(IndirectStubsMgrBuilder()),
 214         ObjectLayer(
 215             AcknowledgeORCv1Deprecation, ES,
 216             [this](orc::VModuleKey K) {
 217               auto ResolverI = Resolvers.find(K);
 218               assert(ResolverI != Resolvers.end() &&
 219                      "No resolver for module K");
 220               auto Resolver = std::move(ResolverI->second);
 221               Resolvers.erase(ResolverI);
 222               return ObjLayerT::Resources{
 223                   std::make_shared<SectionMemoryManager>(), Resolver};
 224             },
 225             nullptr,
 226             [this](orc::VModuleKey K, const object::ObjectFile &Obj,
 227                    const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) {
 228               this->notifyFinalized(K, Obj, LoadedObjInfo);
 229             },
 230             [this](orc::VModuleKey K, const object::ObjectFile &Obj) {
 231               this->notifyFreed(K, Obj);
 232             }),
 233         CompileLayer(AcknowledgeORCv1Deprecation, ObjectLayer,
 234                      orc::SimpleCompiler(TM)),
 235         CODLayer(createCODLayer(ES, CompileLayer, CCMgr.get(),
 236                                 std::move(IndirectStubsMgrBuilder), Resolvers)),
 237         CXXRuntimeOverrides(
 238             AcknowledgeORCv1Deprecation,
 239             [this](const std::string &S) { return mangle(S); }) {}
 240
 241   Error shutdown() {
 242     // Run any destructors registered with __cxa_atexit.
 243     CXXRuntimeOverrides.runDestructors();
 244     // Run any IR destructors.
 245     for (auto &DtorRunner : IRStaticDestructorRunners)
 246       if (auto Err = DtorRunner.runViaLayer(*this))
 247         return Err;
 248     return Error::success();
 249   }
 250
 251   std::string mangle(StringRef Name) {
 252     std::string MangledName;
 253     {
 254       raw_string_ostream MangledNameStream(MangledName);
 255       Mangler::getNameWithPrefix(MangledNameStream, Name, DL);
 256     }
 257     return MangledName;
 258   }
 259
 260   template <typename PtrTy>
 261   static PtrTy fromTargetAddress(JITTargetAddress Addr) {
 262     return reinterpret_cast<PtrTy>(static_cast<uintptr_t>(Addr));
 263   }
 264
 265   Expected<JITTargetAddress>
 266   createLazyCompileCallback(LLVMOrcLazyCompileCallbackFn Callback,
 267                             void *CallbackCtx) {
 268     auto WrappedCallback = [=]() -> JITTargetAddress {
 269       return Callback(wrap(this), CallbackCtx);
 270     };
 271
 272     return CCMgr->getCompileCallback(std::move(WrappedCallback));
 273   }
 274
 275   Error createIndirectStub(StringRef StubName, JITTargetAddress Addr) {
 276     return IndirectStubsMgr->createStub(StubName, Addr,
 277                                         JITSymbolFlags::Exported);
 278   }
 279
 280   Error setIndirectStubPointer(StringRef Name, JITTargetAddress Addr) {
 281     return IndirectStubsMgr->updatePointer(Name, Addr);
 282   }
 283
 284   template <typename LayerT>
 285   Expected<orc::VModuleKey>
 286   addIRModule(LayerT &Layer, std::unique_ptr<Module> M,
 287               std::unique_ptr<RuntimeDyld::MemoryManager> MemMgr,
 288               LLVMOrcSymbolResolverFn ExternalResolver,
 289               void *ExternalResolverCtx) {
 290
 291     // Attach a data-layout if one isn't already present.
 292     if (M->getDataLayout().isDefault())
 293       M->setDataLayout(DL);
 294
 295     // Record the static constructors and destructors. We have to do this before
 296     // we hand over ownership of the module to the JIT.
 297     std::vector<std::string> CtorNames, DtorNames;
 298     for (auto Ctor : orc::getConstructors(*M))
 299       CtorNames.push_back(mangle(Ctor.Func->getName()));
 300     for (auto Dtor : orc::getDestructors(*M))
 301       DtorNames.push_back(mangle(Dtor.Func->getName()));
 302
 303     // Add the module to the JIT.
 304     auto K = ES.allocateVModule();
 305     Resolvers[K] = std::make_shared<CBindingsResolver>(*this, ExternalResolver,
 306                                                        ExternalResolverCtx);
 307     if (auto Err = Layer.addModule(K, std::move(M)))
 308       return std::move(Err);
 309
 310     KeyLayers[K] = detail::createGenericLayer(Layer);
 311
 312     // Run the static constructors, and save the static destructor runner for
 313     // execution when the JIT is torn down.
 314     orc::LegacyCtorDtorRunner<OrcCBindingsStack> CtorRunner(
 315         AcknowledgeORCv1Deprecation, std::move(CtorNames), K);
 316     if (auto Err = CtorRunner.runViaLayer(*this))
 317       return std::move(Err);
 318
 319     IRStaticDestructorRunners.emplace_back(AcknowledgeORCv1Deprecation,
 320                                            std::move(DtorNames), K);
 321
 322     return K;
 323   }
 324
 325   Expected<orc::VModuleKey>
 326   addIRModuleEager(std::unique_ptr<Module> M,
 327                    LLVMOrcSymbolResolverFn ExternalResolver,
 328                    void *ExternalResolverCtx) {
 329     return addIRModule(CompileLayer, std::move(M),
 330                        std::make_unique<SectionMemoryManager>(),
 331                        std::move(ExternalResolver), ExternalResolverCtx);
 332   }
 333
 334   Expected<orc::VModuleKey>
 335   addIRModuleLazy(std::unique_ptr<Module> M,
 336                   LLVMOrcSymbolResolverFn ExternalResolver,
 337                   void *ExternalResolverCtx) {
 338     if (!CODLayer)
 339       return make_error<StringError>("Can not add lazy module: No compile "
 340                                      "callback manager available",
 341                                      inconvertibleErrorCode());
 342
 343     return addIRModule(*CODLayer, std::move(M),
 344                        std::make_unique<SectionMemoryManager>(),
 345                        std::move(ExternalResolver), ExternalResolverCtx);
 346   }
 347
 348   Error removeModule(orc::VModuleKey K) {
 349     // FIXME: Should error release the module key?
 350     if (auto Err = KeyLayers[K]->removeModule(K))
 351       return Err;
 352     ES.releaseVModule(K);
 353     KeyLayers.erase(K);
 354     return Error::success();
 355   }
 356
 357   Expected<orc::VModuleKey> addObject(std::unique_ptr<MemoryBuffer> ObjBuffer,
 358                                       LLVMOrcSymbolResolverFn ExternalResolver,
 359                                       void *ExternalResolverCtx) {
 360     if (auto Obj = object::ObjectFile::createObjectFile(
 361             ObjBuffer->getMemBufferRef())) {
 362
 363       auto K = ES.allocateVModule();
 364       Resolvers[K] = std::make_shared<CBindingsResolver>(
 365           *this, ExternalResolver, ExternalResolverCtx);
 366
 367       if (auto Err = ObjectLayer.addObject(K, std::move(ObjBuffer)))
 368         return std::move(Err);
 369
 370       KeyLayers[K] = detail::createGenericLayer(ObjectLayer);
 371
 372       return K;
 373     } else
 374       return Obj.takeError();
 375   }
 376
 377   JITSymbol findSymbol(const std::string &Name,
 378                                  bool ExportedSymbolsOnly) {
 379     if (auto Sym = IndirectStubsMgr->findStub(Name, ExportedSymbolsOnly))
 380       return Sym;
 381     if (CODLayer)
 382       return CODLayer->findSymbol(mangle(Name), ExportedSymbolsOnly);
 383     return CompileLayer.findSymbol(mangle(Name), ExportedSymbolsOnly);
 384   }
 385
 386   JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
 387                          bool ExportedSymbolsOnly) {
 388     assert(KeyLayers.count(K) && "looking up symbol in unknown module");
 389     return KeyLayers[K]->findSymbolIn(K, mangle(Name), ExportedSymbolsOnly);
 390   }
 391
 392   Expected<JITTargetAddress> findSymbolAddress(const std::string &Name,
 393                                                bool ExportedSymbolsOnly) {
 394     if (auto Sym = findSymbol(Name, ExportedSymbolsOnly)) {
 395       // Successful lookup, non-null symbol:
 396       if (auto AddrOrErr = Sym.getAddress())
 397         return *AddrOrErr;
 398       else
 399         return AddrOrErr.takeError();
 400     } else if (auto Err = Sym.takeError()) {
 401       // Lookup failure - report error.
 402       return std::move(Err);
 403     }
 404
 405     // No symbol not found. Return 0.
 406     return 0;
 407   }
 408
 409   Expected<JITTargetAddress> findSymbolAddressIn(orc::VModuleKey K,
 410                                                  const std::string &Name,
 411                                                  bool ExportedSymbolsOnly) {
 412     if (auto Sym = findSymbolIn(K, Name, ExportedSymbolsOnly)) {
 413       // Successful lookup, non-null symbol:
 414       if (auto AddrOrErr = Sym.getAddress())
 415         return *AddrOrErr;
 416       else
 417         return AddrOrErr.takeError();
 418     } else if (auto Err = Sym.takeError()) {
 419       // Lookup failure - report error.
 420       return std::move(Err);
 421     }
 422
 423     // Symbol not found. Return 0.
 424     return 0;
 425   }
 426
 427   const std::string &getErrorMessage() const { return ErrMsg; }
 428
 429   void RegisterJITEventListener(JITEventListener *L) {
 430     if (!L)
 431       return;
 432     EventListeners.push_back(L);
 433   }
 434
 435   void UnregisterJITEventListener(JITEventListener *L) {
 436     if (!L)
 437       return;
 438
 439     auto I = find(reverse(EventListeners), L);
 440     if (I != EventListeners.rend()) {
 441       std::swap(*I, EventListeners.back());
 442       EventListeners.pop_back();
 443     }
 444   }
 445
 446 private:
 447   using ResolverMap =
 448       std::map<orc::VModuleKey, std::shared_ptr<orc::SymbolResolver>>;
 449
 450   static std::unique_ptr<CompileCallbackMgr>
 451   createCompileCallbackManager(TargetMachine &TM, orc::ExecutionSession &ES) {
 452     auto CCMgr = createLocalCompileCallbackManager(TM.getTargetTriple(), ES, 0);
 453     if (!CCMgr) {
 454       // FIXME: It would be good if we could report this somewhere, but we do
 455       //        have an instance yet.
 456       logAllUnhandledErrors(CCMgr.takeError(), errs(), "ORC error: ");
 457       return nullptr;
 458     }
 459     return std::move(*CCMgr);
 460   }
 461
 462   static std::unique_ptr<CODLayerT>
 463   createCODLayer(orc::ExecutionSession &ES, CompileLayerT &CompileLayer,
 464                  CompileCallbackMgr *CCMgr,
 465                  IndirectStubsManagerBuilder IndirectStubsMgrBuilder,
 466                  ResolverMap &Resolvers) {
 467     // If there is no compile callback manager available we can not create a
 468     // compile on demand layer.
 469     if (!CCMgr)
 470       return nullptr;
 471
 472     return std::make_unique<CODLayerT>(
 473         AcknowledgeORCv1Deprecation, ES, CompileLayer,
 474         [&Resolvers](orc::VModuleKey K) {
 475           auto ResolverI = Resolvers.find(K);
 476           assert(ResolverI != Resolvers.end() && "No resolver for module K");
 477           return ResolverI->second;
 478         },
 479         [&Resolvers](orc::VModuleKey K,
 480                      std::shared_ptr<orc::SymbolResolver> Resolver) {
 481           assert(!Resolvers.count(K) && "Resolver already present");
 482           Resolvers[K] = std::move(Resolver);
 483         },
 484         [](Function &F) { return std::set<Function *>({&F}); }, *CCMgr,
 485         std::move(IndirectStubsMgrBuilder), false);
 486   }
 487
 488   void reportError(Error Err) {
 489     // FIXME: Report errors on the execution session.
 490     logAllUnhandledErrors(std::move(Err), errs(), "ORC error: ");
 491   };
 492
 493   void notifyFinalized(orc::VModuleKey K,
 494                        const object::ObjectFile &Obj,
 495                        const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) {
 496     uint64_t Key = static_cast<uint64_t>(
 497         reinterpret_cast<uintptr_t>(Obj.getData().data()));
 498     for (auto &Listener : EventListeners)
 499       Listener->notifyObjectLoaded(Key, Obj, LoadedObjInfo);
 500   }
 501
 502   void notifyFreed(orc::VModuleKey K, const object::ObjectFile &Obj) {
 503     uint64_t Key = static_cast<uint64_t>(
 504         reinterpret_cast<uintptr_t>(Obj.getData().data()));
 505     for (auto &Listener : EventListeners)
 506       Listener->notifyFreeingObject(Key);
 507   }
 508
 509   orc::ExecutionSession ES;
 510   std::unique_ptr<CompileCallbackMgr> CCMgr;
 511
 512   std::vector<JITEventListener *> EventListeners;
 513
 514   DataLayout DL;
 515   SectionMemoryManager CCMgrMemMgr;
 516
 517   std::unique_ptr<orc::IndirectStubsManager> IndirectStubsMgr;
 518
 519   ObjLayerT ObjectLayer;
 520   CompileLayerT CompileLayer;
 521   std::unique_ptr<CODLayerT> CODLayer;
 522
 523   std::map<orc::VModuleKey, std::unique_ptr<detail::GenericLayer>> KeyLayers;
 524
 525   orc::LegacyLocalCXXRuntimeOverrides CXXRuntimeOverrides;
 526   std::vector<orc::LegacyCtorDtorRunner<OrcCBindingsStack>> IRStaticDestructorRunners;
 527   std::string ErrMsg;
 528
 529   ResolverMap Resolvers;
 530 };
 531
 532 } // end namespace llvm
 533
 534 #endif // LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H