llvm/lib/ExecutionEngine/Orc/LLJIT.cpp

   1 //===--------- LLJIT.cpp - An ORC-based JIT for compiling LLVM IR ---------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ExecutionEngine/Orc/LLJIT.h"
  10
  11 #include "llvm/Analysis/TargetLibraryInfo.h"
  12 #include "llvm/Config/llvm-config.h" // for LLVM_ENABLE_THREADS
  13 #include "llvm/ExecutionEngine/Orc/COFFPlatform.h"
  14 #include "llvm/ExecutionEngine/Orc/EHFrameRegistrationPlugin.h"
  15 #include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
  16 #include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
  17 #include "llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h"
  18 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
  19 #include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
  20 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
  21 #include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h"
  22 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
  23 #include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h"
  24 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
  25 #include "llvm/IR/GlobalVariable.h"
  26 #include "llvm/IR/IRBuilder.h"
  27 #include "llvm/IR/Mangler.h"
  28 #include "llvm/IR/Module.h"
  29 #include "llvm/Support/DynamicLibrary.h"
  30
  31 #define DEBUG_TYPE "orc"
  32
  33 using namespace llvm;
  34 using namespace llvm::orc;
  35
  36 namespace {
  37
  38 /// Adds helper function decls and wrapper functions that call the helper with
  39 /// some additional prefix arguments.
  40 ///
  41 /// E.g. For wrapper "foo" with type i8(i8, i64), helper "bar", and prefix
  42 /// args i32 4 and i16 12345, this function will add:
  43 ///
  44 /// declare i8 @bar(i32, i16, i8, i64)
  45 ///
  46 /// define i8 @foo(i8, i64) {
  47 /// entry:
  48 ///   %2 = call i8 @bar(i32 4, i16 12345, i8 %0, i64 %1)
  49 ///   ret i8 %2
  50 /// }
  51 ///
  52 Function *addHelperAndWrapper(Module &M, StringRef WrapperName,
  53                               FunctionType *WrapperFnType,
  54                               GlobalValue::VisibilityTypes WrapperVisibility,
  55                               StringRef HelperName,
  56                               ArrayRef<Value *> HelperPrefixArgs) {
  57   std::vector<Type *> HelperArgTypes;
  58   for (auto *Arg : HelperPrefixArgs)
  59     HelperArgTypes.push_back(Arg->getType());
  60   for (auto *T : WrapperFnType->params())
  61     HelperArgTypes.push_back(T);
  62   auto *HelperFnType =
  63       FunctionType::get(WrapperFnType->getReturnType(), HelperArgTypes, false);
  64   auto *HelperFn = Function::Create(HelperFnType, GlobalValue::ExternalLinkage,
  65                                     HelperName, M);
  66
  67   auto *WrapperFn = Function::Create(
  68       WrapperFnType, GlobalValue::ExternalLinkage, WrapperName, M);
  69   WrapperFn->setVisibility(WrapperVisibility);
  70
  71   auto *EntryBlock = BasicBlock::Create(M.getContext(), "entry", WrapperFn);
  72   IRBuilder<> IB(EntryBlock);
  73
  74   std::vector<Value *> HelperArgs;
  75   for (auto *Arg : HelperPrefixArgs)
  76     HelperArgs.push_back(Arg);
  77   for (auto &Arg : WrapperFn->args())
  78     HelperArgs.push_back(&Arg);
  79   auto *HelperResult = IB.CreateCall(HelperFn, HelperArgs);
  80   if (HelperFn->getReturnType()->isVoidTy())
  81     IB.CreateRetVoid();
  82   else
  83     IB.CreateRet(HelperResult);
  84
  85   return WrapperFn;
  86 }
  87
  88 class GenericLLVMIRPlatformSupport;
  89
  90 /// orc::Platform component of Generic LLVM IR Platform support.
  91 /// Just forwards calls to the GenericLLVMIRPlatformSupport class below.
  92 class GenericLLVMIRPlatform : public Platform {
  93 public:
  94   GenericLLVMIRPlatform(GenericLLVMIRPlatformSupport &S) : S(S) {}
  95   Error setupJITDylib(JITDylib &JD) override;
  96   Error teardownJITDylib(JITDylib &JD) override;
  97   Error notifyAdding(ResourceTracker &RT,
  98                      const MaterializationUnit &MU) override;
  99   Error notifyRemoving(ResourceTracker &RT) override {
 100     // Noop -- Nothing to do (yet).
 101     return Error::success();
 102   }
 103
 104 private:
 105   GenericLLVMIRPlatformSupport &S;
 106 };
 107
 108 /// This transform parses llvm.global_ctors to produce a single initialization
 109 /// function for the module, records the function, then deletes
 110 /// llvm.global_ctors.
 111 class GlobalCtorDtorScraper {
 112 public:
 113   GlobalCtorDtorScraper(GenericLLVMIRPlatformSupport &PS,
 114                         StringRef InitFunctionPrefix,
 115                         StringRef DeInitFunctionPrefix)
 116       : PS(PS), InitFunctionPrefix(InitFunctionPrefix),
 117         DeInitFunctionPrefix(DeInitFunctionPrefix) {}
 118   Expected<ThreadSafeModule> operator()(ThreadSafeModule TSM,
 119                                         MaterializationResponsibility &R);
 120
 121 private:
 122   GenericLLVMIRPlatformSupport &PS;
 123   StringRef InitFunctionPrefix;
 124   StringRef DeInitFunctionPrefix;
 125 };
 126
 127 /// Generic IR Platform Support
 128 ///
 129 /// Scrapes llvm.global_ctors and llvm.global_dtors and replaces them with
 130 /// specially named 'init' and 'deinit'. Injects definitions / interposes for
 131 /// some runtime API, including __cxa_atexit, dlopen, and dlclose.
 132 class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport {
 133 public:
 134   GenericLLVMIRPlatformSupport(LLJIT &J, JITDylib &PlatformJD)
 135       : J(J), InitFunctionPrefix(J.mangle("__orc_init_func.")),
 136         DeInitFunctionPrefix(J.mangle("__orc_deinit_func.")) {
 137
 138     getExecutionSession().setPlatform(
 139         std::make_unique<GenericLLVMIRPlatform>(*this));
 140
 141     setInitTransform(J, GlobalCtorDtorScraper(*this, InitFunctionPrefix,
 142                                               DeInitFunctionPrefix));
 143
 144     SymbolMap StdInterposes;
 145
 146     StdInterposes[J.mangleAndIntern("__lljit.platform_support_instance")] = {
 147         ExecutorAddr::fromPtr(this), JITSymbolFlags::Exported};
 148     StdInterposes[J.mangleAndIntern("__lljit.cxa_atexit_helper")] = {
 149         ExecutorAddr::fromPtr(registerCxaAtExitHelper), JITSymbolFlags()};
 150
 151     cantFail(PlatformJD.define(absoluteSymbols(std::move(StdInterposes))));
 152     cantFail(setupJITDylib(PlatformJD));
 153     cantFail(J.addIRModule(PlatformJD, createPlatformRuntimeModule()));
 154   }
 155
 156   ExecutionSession &getExecutionSession() { return J.getExecutionSession(); }
 157
 158   /// Adds a module that defines the __dso_handle global.
 159   Error setupJITDylib(JITDylib &JD) {
 160
 161     // Add per-jitdylib standard interposes.
 162     SymbolMap PerJDInterposes;
 163     PerJDInterposes[J.mangleAndIntern("__lljit.run_atexits_helper")] = {
 164         ExecutorAddr::fromPtr(runAtExitsHelper), JITSymbolFlags()};
 165     PerJDInterposes[J.mangleAndIntern("__lljit.atexit_helper")] = {
 166         ExecutorAddr::fromPtr(registerAtExitHelper), JITSymbolFlags()};
 167     cantFail(JD.define(absoluteSymbols(std::move(PerJDInterposes))));
 168
 169     auto Ctx = std::make_unique<LLVMContext>();
 170     auto M = std::make_unique<Module>("__standard_lib", *Ctx);
 171     M->setDataLayout(J.getDataLayout());
 172
 173     auto *Int64Ty = Type::getInt64Ty(*Ctx);
 174     auto *DSOHandle = new GlobalVariable(
 175         *M, Int64Ty, true, GlobalValue::ExternalLinkage,
 176         ConstantInt::get(Int64Ty, reinterpret_cast<uintptr_t>(&JD)),
 177         "__dso_handle");
 178     DSOHandle->setVisibility(GlobalValue::DefaultVisibility);
 179     DSOHandle->setInitializer(
 180         ConstantInt::get(Int64Ty, ExecutorAddr::fromPtr(&JD).getValue()));
 181
 182     auto *GenericIRPlatformSupportTy =
 183         StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport");
 184
 185     auto *PlatformInstanceDecl = new GlobalVariable(
 186         *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
 187         nullptr, "__lljit.platform_support_instance");
 188
 189     auto *VoidTy = Type::getVoidTy(*Ctx);
 190     addHelperAndWrapper(
 191         *M, "__lljit_run_atexits", FunctionType::get(VoidTy, {}, false),
 192         GlobalValue::HiddenVisibility, "__lljit.run_atexits_helper",
 193         {PlatformInstanceDecl, DSOHandle});
 194
 195     auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT);
 196     auto *AtExitCallbackTy = FunctionType::get(VoidTy, {}, false);
 197     auto *AtExitCallbackPtrTy = PointerType::getUnqual(AtExitCallbackTy);
 198     auto *AtExit = addHelperAndWrapper(
 199         *M, "atexit", FunctionType::get(IntTy, {AtExitCallbackPtrTy}, false),
 200         GlobalValue::HiddenVisibility, "__lljit.atexit_helper",
 201         {PlatformInstanceDecl, DSOHandle});
 202     Attribute::AttrKind AtExitExtAttr =
 203         TargetLibraryInfo::getExtAttrForI32Return(J.getTargetTriple());
 204     if (AtExitExtAttr != Attribute::None)
 205       AtExit->addRetAttr(AtExitExtAttr);
 206
 207     return J.addIRModule(JD, ThreadSafeModule(std::move(M), std::move(Ctx)));
 208   }
 209
 210   Error notifyAdding(ResourceTracker &RT, const MaterializationUnit &MU) {
 211     auto &JD = RT.getJITDylib();
 212     if (auto &InitSym = MU.getInitializerSymbol())
 213       InitSymbols[&JD].add(InitSym, SymbolLookupFlags::WeaklyReferencedSymbol);
 214     else {
 215       // If there's no identified init symbol attached, but there is a symbol
 216       // with the GenericIRPlatform::InitFunctionPrefix, then treat that as
 217       // an init function. Add the symbol to both the InitSymbols map (which
 218       // will trigger a lookup to materialize the module) and the InitFunctions
 219       // map (which holds the names of the symbols to execute).
 220       for (auto &KV : MU.getSymbols())
 221         if ((*KV.first).starts_with(InitFunctionPrefix)) {
 222           InitSymbols[&JD].add(KV.first,
 223                                SymbolLookupFlags::WeaklyReferencedSymbol);
 224           InitFunctions[&JD].add(KV.first);
 225         } else if ((*KV.first).starts_with(DeInitFunctionPrefix)) {
 226           DeInitFunctions[&JD].add(KV.first);
 227         }
 228     }
 229     return Error::success();
 230   }
 231
 232   Error initialize(JITDylib &JD) override {
 233     LLVM_DEBUG({
 234       dbgs() << "GenericLLVMIRPlatformSupport getting initializers to run\n";
 235     });
 236     if (auto Initializers = getInitializers(JD)) {
 237       LLVM_DEBUG(
 238           { dbgs() << "GenericLLVMIRPlatformSupport running initializers\n"; });
 239       for (auto InitFnAddr : *Initializers) {
 240         LLVM_DEBUG({
 241           dbgs() << "  Running init " << formatv("{0:x16}", InitFnAddr)
 242                  << "...\n";
 243         });
 244         auto *InitFn = InitFnAddr.toPtr<void (*)()>();
 245         InitFn();
 246       }
 247     } else
 248       return Initializers.takeError();
 249     return Error::success();
 250   }
 251
 252   Error deinitialize(JITDylib &JD) override {
 253     LLVM_DEBUG({
 254       dbgs() << "GenericLLVMIRPlatformSupport getting deinitializers to run\n";
 255     });
 256     if (auto Deinitializers = getDeinitializers(JD)) {
 257       LLVM_DEBUG({
 258         dbgs() << "GenericLLVMIRPlatformSupport running deinitializers\n";
 259       });
 260       for (auto DeinitFnAddr : *Deinitializers) {
 261         LLVM_DEBUG({
 262           dbgs() << "  Running deinit " << formatv("{0:x16}", DeinitFnAddr)
 263                  << "...\n";
 264         });
 265         auto *DeinitFn = DeinitFnAddr.toPtr<void (*)()>();
 266         DeinitFn();
 267       }
 268     } else
 269       return Deinitializers.takeError();
 270
 271     return Error::success();
 272   }
 273
 274   void registerInitFunc(JITDylib &JD, SymbolStringPtr InitName) {
 275     getExecutionSession().runSessionLocked([&]() {
 276         InitFunctions[&JD].add(InitName);
 277       });
 278   }
 279
 280   void registerDeInitFunc(JITDylib &JD, SymbolStringPtr DeInitName) {
 281     getExecutionSession().runSessionLocked(
 282         [&]() { DeInitFunctions[&JD].add(DeInitName); });
 283   }
 284
 285 private:
 286   Expected<std::vector<ExecutorAddr>> getInitializers(JITDylib &JD) {
 287     if (auto Err = issueInitLookups(JD))
 288       return std::move(Err);
 289
 290     DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
 291     std::vector<JITDylibSP> DFSLinkOrder;
 292
 293     if (auto Err = getExecutionSession().runSessionLocked([&]() -> Error {
 294           if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
 295             DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
 296           else
 297             return DFSLinkOrderOrErr.takeError();
 298
 299           for (auto &NextJD : DFSLinkOrder) {
 300             auto IFItr = InitFunctions.find(NextJD.get());
 301             if (IFItr != InitFunctions.end()) {
 302               LookupSymbols[NextJD.get()] = std::move(IFItr->second);
 303               InitFunctions.erase(IFItr);
 304             }
 305           }
 306           return Error::success();
 307         }))
 308       return std::move(Err);
 309
 310     LLVM_DEBUG({
 311       dbgs() << "JITDylib init order is [ ";
 312       for (auto &JD : llvm::reverse(DFSLinkOrder))
 313         dbgs() << "\"" << JD->getName() << "\" ";
 314       dbgs() << "]\n";
 315       dbgs() << "Looking up init functions:\n";
 316       for (auto &KV : LookupSymbols)
 317         dbgs() << "  \"" << KV.first->getName() << "\": " << KV.second << "\n";
 318     });
 319
 320     auto &ES = getExecutionSession();
 321     auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols);
 322
 323     if (!LookupResult)
 324       return LookupResult.takeError();
 325
 326     std::vector<ExecutorAddr> Initializers;
 327     while (!DFSLinkOrder.empty()) {
 328       auto &NextJD = *DFSLinkOrder.back();
 329       DFSLinkOrder.pop_back();
 330       auto InitsItr = LookupResult->find(&NextJD);
 331       if (InitsItr == LookupResult->end())
 332         continue;
 333       for (auto &KV : InitsItr->second)
 334         Initializers.push_back(KV.second.getAddress());
 335     }
 336
 337     return Initializers;
 338   }
 339
 340   Expected<std::vector<ExecutorAddr>> getDeinitializers(JITDylib &JD) {
 341     auto &ES = getExecutionSession();
 342
 343     auto LLJITRunAtExits = J.mangleAndIntern("__lljit_run_atexits");
 344
 345     DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
 346     std::vector<JITDylibSP> DFSLinkOrder;
 347
 348     if (auto Err = ES.runSessionLocked([&]() -> Error {
 349           if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
 350             DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
 351           else
 352             return DFSLinkOrderOrErr.takeError();
 353
 354           for (auto &NextJD : DFSLinkOrder) {
 355             auto &JDLookupSymbols = LookupSymbols[NextJD.get()];
 356             auto DIFItr = DeInitFunctions.find(NextJD.get());
 357             if (DIFItr != DeInitFunctions.end()) {
 358               LookupSymbols[NextJD.get()] = std::move(DIFItr->second);
 359               DeInitFunctions.erase(DIFItr);
 360             }
 361             JDLookupSymbols.add(LLJITRunAtExits,
 362                                 SymbolLookupFlags::WeaklyReferencedSymbol);
 363           }
 364           return Error::success();
 365         }))
 366       return std::move(Err);
 367
 368     LLVM_DEBUG({
 369       dbgs() << "JITDylib deinit order is [ ";
 370       for (auto &JD : DFSLinkOrder)
 371         dbgs() << "\"" << JD->getName() << "\" ";
 372       dbgs() << "]\n";
 373       dbgs() << "Looking up deinit functions:\n";
 374       for (auto &KV : LookupSymbols)
 375         dbgs() << "  \"" << KV.first->getName() << "\": " << KV.second << "\n";
 376     });
 377
 378     auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols);
 379
 380     if (!LookupResult)
 381       return LookupResult.takeError();
 382
 383     std::vector<ExecutorAddr> DeInitializers;
 384     for (auto &NextJD : DFSLinkOrder) {
 385       auto DeInitsItr = LookupResult->find(NextJD.get());
 386       assert(DeInitsItr != LookupResult->end() &&
 387              "Every JD should have at least __lljit_run_atexits");
 388
 389       auto RunAtExitsItr = DeInitsItr->second.find(LLJITRunAtExits);
 390       if (RunAtExitsItr != DeInitsItr->second.end())
 391         DeInitializers.push_back(RunAtExitsItr->second.getAddress());
 392
 393       for (auto &KV : DeInitsItr->second)
 394         if (KV.first != LLJITRunAtExits)
 395           DeInitializers.push_back(KV.second.getAddress());
 396     }
 397
 398     return DeInitializers;
 399   }
 400
 401   /// Issue lookups for all init symbols required to initialize JD (and any
 402   /// JITDylibs that it depends on).
 403   Error issueInitLookups(JITDylib &JD) {
 404     DenseMap<JITDylib *, SymbolLookupSet> RequiredInitSymbols;
 405     std::vector<JITDylibSP> DFSLinkOrder;
 406
 407     if (auto Err = getExecutionSession().runSessionLocked([&]() -> Error {
 408           if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
 409             DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
 410           else
 411             return DFSLinkOrderOrErr.takeError();
 412
 413           for (auto &NextJD : DFSLinkOrder) {
 414             auto ISItr = InitSymbols.find(NextJD.get());
 415             if (ISItr != InitSymbols.end()) {
 416               RequiredInitSymbols[NextJD.get()] = std::move(ISItr->second);
 417               InitSymbols.erase(ISItr);
 418             }
 419           }
 420           return Error::success();
 421         }))
 422       return Err;
 423
 424     return Platform::lookupInitSymbols(getExecutionSession(),
 425                                        RequiredInitSymbols)
 426         .takeError();
 427   }
 428
 429   static void registerCxaAtExitHelper(void *Self, void (*F)(void *), void *Ctx,
 430                                       void *DSOHandle) {
 431     LLVM_DEBUG({
 432       dbgs() << "Registering cxa atexit function " << (void *)F << " for JD "
 433              << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
 434     });
 435     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit(
 436         F, Ctx, DSOHandle);
 437   }
 438
 439   static void registerAtExitHelper(void *Self, void *DSOHandle, void (*F)()) {
 440     LLVM_DEBUG({
 441       dbgs() << "Registering atexit function " << (void *)F << " for JD "
 442              << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
 443     });
 444     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit(
 445         reinterpret_cast<void (*)(void *)>(F), nullptr, DSOHandle);
 446   }
 447
 448   static void runAtExitsHelper(void *Self, void *DSOHandle) {
 449     LLVM_DEBUG({
 450       dbgs() << "Running atexit functions for JD "
 451              << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
 452     });
 453     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.runAtExits(
 454         DSOHandle);
 455   }
 456
 457   // Constructs an LLVM IR module containing platform runtime globals,
 458   // functions, and interposes.
 459   ThreadSafeModule createPlatformRuntimeModule() {
 460     auto Ctx = std::make_unique<LLVMContext>();
 461     auto M = std::make_unique<Module>("__standard_lib", *Ctx);
 462     M->setDataLayout(J.getDataLayout());
 463
 464     auto *GenericIRPlatformSupportTy =
 465         StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport");
 466
 467     auto *PlatformInstanceDecl = new GlobalVariable(
 468         *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
 469         nullptr, "__lljit.platform_support_instance");
 470
 471     auto *Int8Ty = Type::getInt8Ty(*Ctx);
 472     auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT);
 473     auto *VoidTy = Type::getVoidTy(*Ctx);
 474     auto *BytePtrTy = PointerType::getUnqual(Int8Ty);
 475     auto *CxaAtExitCallbackTy = FunctionType::get(VoidTy, {BytePtrTy}, false);
 476     auto *CxaAtExitCallbackPtrTy = PointerType::getUnqual(CxaAtExitCallbackTy);
 477
 478     auto *CxaAtExit = addHelperAndWrapper(
 479         *M, "__cxa_atexit",
 480         FunctionType::get(IntTy, {CxaAtExitCallbackPtrTy, BytePtrTy, BytePtrTy},
 481                           false),
 482         GlobalValue::DefaultVisibility, "__lljit.cxa_atexit_helper",
 483         {PlatformInstanceDecl});
 484     Attribute::AttrKind CxaAtExitExtAttr =
 485         TargetLibraryInfo::getExtAttrForI32Return(J.getTargetTriple());
 486     if (CxaAtExitExtAttr != Attribute::None)
 487       CxaAtExit->addRetAttr(CxaAtExitExtAttr);
 488
 489     return ThreadSafeModule(std::move(M), std::move(Ctx));
 490   }
 491
 492   LLJIT &J;
 493   std::string InitFunctionPrefix;
 494   std::string DeInitFunctionPrefix;
 495   DenseMap<JITDylib *, SymbolLookupSet> InitSymbols;
 496   DenseMap<JITDylib *, SymbolLookupSet> InitFunctions;
 497   DenseMap<JITDylib *, SymbolLookupSet> DeInitFunctions;
 498   ItaniumCXAAtExitSupport AtExitMgr;
 499 };
 500
 501 Error GenericLLVMIRPlatform::setupJITDylib(JITDylib &JD) {
 502   return S.setupJITDylib(JD);
 503 }
 504
 505 Error GenericLLVMIRPlatform::teardownJITDylib(JITDylib &JD) {
 506   return Error::success();
 507 }
 508
 509 Error GenericLLVMIRPlatform::notifyAdding(ResourceTracker &RT,
 510                                           const MaterializationUnit &MU) {
 511   return S.notifyAdding(RT, MU);
 512 }
 513
 514 Expected<ThreadSafeModule>
 515 GlobalCtorDtorScraper::operator()(ThreadSafeModule TSM,
 516                                   MaterializationResponsibility &R) {
 517   auto Err = TSM.withModuleDo([&](Module &M) -> Error {
 518     auto &Ctx = M.getContext();
 519     auto *GlobalCtors = M.getNamedGlobal("llvm.global_ctors");
 520     auto *GlobalDtors = M.getNamedGlobal("llvm.global_dtors");
 521
 522     auto RegisterCOrDtors = [&](GlobalVariable *GlobalCOrDtors,
 523                                 bool isCtor) -> Error {
 524       // If there's no llvm.global_c/dtor or it's just a decl then skip.
 525       if (!GlobalCOrDtors || GlobalCOrDtors->isDeclaration())
 526         return Error::success();
 527       std::string InitOrDeInitFunctionName;
 528       if (isCtor)
 529         raw_string_ostream(InitOrDeInitFunctionName)
 530             << InitFunctionPrefix << M.getModuleIdentifier();
 531       else
 532         raw_string_ostream(InitOrDeInitFunctionName)
 533             << DeInitFunctionPrefix << M.getModuleIdentifier();
 534
 535       MangleAndInterner Mangle(PS.getExecutionSession(), M.getDataLayout());
 536       auto InternedInitOrDeInitName = Mangle(InitOrDeInitFunctionName);
 537       if (auto Err = R.defineMaterializing(
 538               {{InternedInitOrDeInitName, JITSymbolFlags::Callable}}))
 539         return Err;
 540
 541       auto *InitOrDeInitFunc = Function::Create(
 542           FunctionType::get(Type::getVoidTy(Ctx), {}, false),
 543           GlobalValue::ExternalLinkage, InitOrDeInitFunctionName, &M);
 544       InitOrDeInitFunc->setVisibility(GlobalValue::HiddenVisibility);
 545       std::vector<std::pair<Function *, unsigned>> InitsOrDeInits;
 546       auto COrDtors = isCtor ? getConstructors(M) : getDestructors(M);
 547
 548       for (auto E : COrDtors)
 549         InitsOrDeInits.push_back(std::make_pair(E.Func, E.Priority));
 550       llvm::stable_sort(InitsOrDeInits, llvm::less_second());
 551
 552       auto *InitOrDeInitFuncEntryBlock =
 553           BasicBlock::Create(Ctx, "entry", InitOrDeInitFunc);
 554       IRBuilder<> IB(InitOrDeInitFuncEntryBlock);
 555       for (auto &KV : InitsOrDeInits)
 556         IB.CreateCall(KV.first);
 557       IB.CreateRetVoid();
 558
 559       if (isCtor)
 560         PS.registerInitFunc(R.getTargetJITDylib(), InternedInitOrDeInitName);
 561       else
 562         PS.registerDeInitFunc(R.getTargetJITDylib(), InternedInitOrDeInitName);
 563
 564       GlobalCOrDtors->eraseFromParent();
 565       return Error::success();
 566     };
 567
 568     if (auto Err = RegisterCOrDtors(GlobalCtors, true))
 569       return Err;
 570     if (auto Err = RegisterCOrDtors(GlobalDtors, false))
 571       return Err;
 572
 573     return Error::success();
 574   });
 575
 576   if (Err)
 577     return std::move(Err);
 578
 579   return std::move(TSM);
 580 }
 581
 582 /// Inactive Platform Support
 583 ///
 584 /// Explicitly disables platform support. JITDylibs are not scanned for special
 585 /// init/deinit symbols. No runtime API interposes are injected.
 586 class InactivePlatformSupport : public LLJIT::PlatformSupport {
 587 public:
 588   InactivePlatformSupport() = default;
 589
 590   Error initialize(JITDylib &JD) override {
 591     LLVM_DEBUG(dbgs() << "InactivePlatformSupport: no initializers running for "
 592                       << JD.getName() << "\n");
 593     return Error::success();
 594   }
 595
 596   Error deinitialize(JITDylib &JD) override {
 597     LLVM_DEBUG(
 598         dbgs() << "InactivePlatformSupport: no deinitializers running for "
 599                << JD.getName() << "\n");
 600     return Error::success();
 601   }
 602 };
 603
 604 } // end anonymous namespace
 605
 606 namespace llvm {
 607 namespace orc {
 608
 609 Error ORCPlatformSupport::initialize(orc::JITDylib &JD) {
 610   using llvm::orc::shared::SPSExecutorAddr;
 611   using llvm::orc::shared::SPSString;
 612   using SPSDLOpenSig = SPSExecutorAddr(SPSString, int32_t);
 613   using SPSDLUpdateSig = int32_t(SPSExecutorAddr);
 614   enum dlopen_mode : int32_t {
 615     ORC_RT_RTLD_LAZY = 0x1,
 616     ORC_RT_RTLD_NOW = 0x2,
 617     ORC_RT_RTLD_LOCAL = 0x4,
 618     ORC_RT_RTLD_GLOBAL = 0x8
 619   };
 620
 621   auto &ES = J.getExecutionSession();
 622   auto MainSearchOrder = J.getMainJITDylib().withLinkOrderDo(
 623       [](const JITDylibSearchOrder &SO) { return SO; });
 624   StringRef WrapperToCall = "__orc_rt_jit_dlopen_wrapper";
 625   bool dlupdate = false;
 626   const Triple &TT = ES.getTargetTriple();
 627   if (TT.isOSBinFormatMachO() || TT.isOSBinFormatELF()) {
 628     if (InitializedDylib.contains(&JD)) {
 629       WrapperToCall = "__orc_rt_jit_dlupdate_wrapper";
 630       dlupdate = true;
 631     } else
 632       InitializedDylib.insert(&JD);
 633   }
 634
 635   if (auto WrapperAddr =
 636           ES.lookup(MainSearchOrder, J.mangleAndIntern(WrapperToCall))) {
 637     if (dlupdate) {
 638       int32_t result;
 639       auto E = ES.callSPSWrapper<SPSDLUpdateSig>(WrapperAddr->getAddress(),
 640                                                  result, DSOHandles[&JD]);
 641       if (result)
 642         return make_error<StringError>("dlupdate failed",
 643                                        inconvertibleErrorCode());
 644       return E;
 645     }
 646     return ES.callSPSWrapper<SPSDLOpenSig>(WrapperAddr->getAddress(),
 647                                            DSOHandles[&JD], JD.getName(),
 648                                            int32_t(ORC_RT_RTLD_LAZY));
 649   } else
 650     return WrapperAddr.takeError();
 651 }
 652
 653 Error ORCPlatformSupport::deinitialize(orc::JITDylib &JD) {
 654   using llvm::orc::shared::SPSExecutorAddr;
 655   using SPSDLCloseSig = int32_t(SPSExecutorAddr);
 656
 657   auto &ES = J.getExecutionSession();
 658   auto MainSearchOrder = J.getMainJITDylib().withLinkOrderDo(
 659       [](const JITDylibSearchOrder &SO) { return SO; });
 660
 661   if (auto WrapperAddr = ES.lookup(
 662           MainSearchOrder, J.mangleAndIntern("__orc_rt_jit_dlclose_wrapper"))) {
 663     int32_t result;
 664     auto E = J.getExecutionSession().callSPSWrapper<SPSDLCloseSig>(
 665         WrapperAddr->getAddress(), result, DSOHandles[&JD]);
 666     if (E)
 667       return E;
 668     else if (result)
 669       return make_error<StringError>("dlclose failed",
 670                                      inconvertibleErrorCode());
 671     DSOHandles.erase(&JD);
 672     InitializedDylib.erase(&JD);
 673   } else
 674     return WrapperAddr.takeError();
 675   return Error::success();
 676 }
 677
 678 void LLJIT::PlatformSupport::setInitTransform(
 679     LLJIT &J, IRTransformLayer::TransformFunction T) {
 680   J.InitHelperTransformLayer->setTransform(std::move(T));
 681 }
 682
 683 LLJIT::PlatformSupport::~PlatformSupport() = default;
 684
 685 Error LLJITBuilderState::prepareForConstruction() {
 686
 687   LLVM_DEBUG(dbgs() << "Preparing to create LLJIT instance...\n");
 688
 689   if (!JTMB) {
 690     LLVM_DEBUG({
 691       dbgs() << "  No explicitly set JITTargetMachineBuilder. "
 692                 "Detecting host...\n";
 693     });
 694     if (auto JTMBOrErr = JITTargetMachineBuilder::detectHost())
 695       JTMB = std::move(*JTMBOrErr);
 696     else
 697       return JTMBOrErr.takeError();
 698   }
 699
 700   if ((ES || EPC) && NumCompileThreads)
 701     return make_error<StringError>(
 702         "NumCompileThreads cannot be used with a custom ExecutionSession or "
 703         "ExecutorProcessControl",
 704         inconvertibleErrorCode());
 705
 706 #if !LLVM_ENABLE_THREADS
 707   if (NumCompileThreads)
 708     return make_error<StringError>(
 709         "LLJIT num-compile-threads is " + Twine(NumCompileThreads) +
 710             " but LLVM was compiled with LLVM_ENABLE_THREADS=Off",
 711         inconvertibleErrorCode());
 712 #endif // !LLVM_ENABLE_THREADS
 713
 714   // Only used in debug builds.
 715   [[maybe_unused]] bool ConcurrentCompilationSettingDefaulted =
 716       !SupportConcurrentCompilation;
 717
 718   if (!SupportConcurrentCompilation) {
 719 #if LLVM_ENABLE_THREADS
 720     SupportConcurrentCompilation = NumCompileThreads || ES || EPC;
 721 #else
 722     SupportConcurrentCompilation = false;
 723 #endif // LLVM_ENABLE_THREADS
 724   } else {
 725 #if !LLVM_ENABLE_THREADS
 726     if (*SupportConcurrentCompilation)
 727       return make_error<StringError>(
 728           "LLJIT concurrent compilation support requested, but LLVM was built "
 729           "with LLVM_ENABLE_THREADS=Off",
 730           inconvertibleErrorCode());
 731 #endif // !LLVM_ENABLE_THREADS
 732   }
 733
 734   LLVM_DEBUG({
 735     dbgs() << "  JITTargetMachineBuilder is "
 736            << JITTargetMachineBuilderPrinter(*JTMB, "  ")
 737            << "  Pre-constructed ExecutionSession: " << (ES ? "Yes" : "No")
 738            << "\n"
 739            << "  DataLayout: ";
 740     if (DL)
 741       dbgs() << DL->getStringRepresentation() << "\n";
 742     else
 743       dbgs() << "None (will be created by JITTargetMachineBuilder)\n";
 744
 745     dbgs() << "  Custom object-linking-layer creator: "
 746            << (CreateObjectLinkingLayer ? "Yes" : "No") << "\n"
 747            << "  Custom compile-function creator: "
 748            << (CreateCompileFunction ? "Yes" : "No") << "\n"
 749            << "  Custom platform-setup function: "
 750            << (SetUpPlatform ? "Yes" : "No") << "\n"
 751            << "  Support concurrent compilation: "
 752            << (*SupportConcurrentCompilation ? "Yes" : "No");
 753     if (ConcurrentCompilationSettingDefaulted)
 754       dbgs() << " (defaulted based on ES / EPC / NumCompileThreads)\n";
 755     else
 756       dbgs() << "\n";
 757     dbgs() << "  Number of compile threads: " << NumCompileThreads << "\n";
 758   });
 759
 760   // Create DL if not specified.
 761   if (!DL) {
 762     if (auto DLOrErr = JTMB->getDefaultDataLayoutForTarget())
 763       DL = std::move(*DLOrErr);
 764     else
 765       return DLOrErr.takeError();
 766   }
 767
 768   // If neither ES nor EPC has been set then create an EPC instance.
 769   if (!ES && !EPC) {
 770     LLVM_DEBUG({
 771       dbgs() << "ExecutorProcessControl not specified, "
 772                 "Creating SelfExecutorProcessControl instance\n";
 773     });
 774
 775     std::unique_ptr<TaskDispatcher> D = nullptr;
 776 #if LLVM_ENABLE_THREADS
 777     if (*SupportConcurrentCompilation) {
 778       std::optional<size_t> NumThreads = std ::nullopt;
 779       if (NumCompileThreads)
 780         NumThreads = NumCompileThreads;
 781       D = std::make_unique<DynamicThreadPoolTaskDispatcher>(NumThreads);
 782     } else
 783       D = std::make_unique<InPlaceTaskDispatcher>();
 784 #endif // LLVM_ENABLE_THREADS
 785     if (auto EPCOrErr =
 786             SelfExecutorProcessControl::Create(nullptr, std::move(D), nullptr))
 787       EPC = std::move(*EPCOrErr);
 788     else
 789       return EPCOrErr.takeError();
 790   } else if (EPC) {
 791     LLVM_DEBUG({
 792       dbgs() << "Using explicitly specified ExecutorProcessControl instance "
 793              << EPC.get() << "\n";
 794     });
 795   } else {
 796     LLVM_DEBUG({
 797       dbgs() << "Using explicitly specified ExecutionSession instance "
 798              << ES.get() << "\n";
 799     });
 800   }
 801
 802   // If the client didn't configure any linker options then auto-configure the
 803   // JIT linker.
 804   if (!CreateObjectLinkingLayer) {
 805     auto &TT = JTMB->getTargetTriple();
 806     bool UseJITLink = false;
 807     switch (TT.getArch()) {
 808     case Triple::riscv64:
 809     case Triple::loongarch64:
 810       UseJITLink = true;
 811       break;
 812     case Triple::aarch64:
 813       UseJITLink = !TT.isOSBinFormatCOFF();
 814       break;
 815     case Triple::arm:
 816     case Triple::armeb:
 817     case Triple::thumb:
 818     case Triple::thumbeb:
 819       UseJITLink = TT.isOSBinFormatELF();
 820       break;
 821     case Triple::x86_64:
 822       UseJITLink = !TT.isOSBinFormatCOFF();
 823       break;
 824     case Triple::ppc64:
 825       UseJITLink = TT.isPPC64ELFv2ABI();
 826       break;
 827     case Triple::ppc64le:
 828       UseJITLink = TT.isOSBinFormatELF();
 829       break;
 830     default:
 831       break;
 832     }
 833     if (UseJITLink) {
 834       if (!JTMB->getCodeModel())
 835         JTMB->setCodeModel(CodeModel::Small);
 836       JTMB->setRelocationModel(Reloc::PIC_);
 837       CreateObjectLinkingLayer =
 838           [](ExecutionSession &ES,
 839              const Triple &) -> Expected<std::unique_ptr<ObjectLayer>> {
 840         auto ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>(ES);
 841         if (auto EHFrameRegistrar = EPCEHFrameRegistrar::Create(ES))
 842           ObjLinkingLayer->addPlugin(
 843               std::make_unique<EHFrameRegistrationPlugin>(
 844                   ES, std::move(*EHFrameRegistrar)));
 845         else
 846           return EHFrameRegistrar.takeError();
 847         return std::move(ObjLinkingLayer);
 848       };
 849     }
 850   }
 851
 852   // If we need a process JITDylib but no setup function has been given then
 853   // create a default one.
 854   if (!SetupProcessSymbolsJITDylib && LinkProcessSymbolsByDefault) {
 855     LLVM_DEBUG(dbgs() << "Creating default Process JD setup function\n");
 856     SetupProcessSymbolsJITDylib = [](LLJIT &J) -> Expected<JITDylibSP> {
 857       auto &JD =
 858           J.getExecutionSession().createBareJITDylib("<Process Symbols>");
 859       auto G = EPCDynamicLibrarySearchGenerator::GetForTargetProcess(
 860           J.getExecutionSession());
 861       if (!G)
 862         return G.takeError();
 863       JD.addGenerator(std::move(*G));
 864       return &JD;
 865     };
 866   }
 867
 868   return Error::success();
 869 }
 870
 871 LLJIT::~LLJIT() {
 872   if (auto Err = ES->endSession())
 873     ES->reportError(std::move(Err));
 874 }
 875
 876 JITDylibSP LLJIT::getProcessSymbolsJITDylib() { return ProcessSymbols; }
 877
 878 JITDylibSP LLJIT::getPlatformJITDylib() { return Platform; }
 879
 880 Expected<JITDylib &> LLJIT::createJITDylib(std::string Name) {
 881   auto JD = ES->createJITDylib(std::move(Name));
 882   if (!JD)
 883     return JD.takeError();
 884
 885   JD->addToLinkOrder(DefaultLinks);
 886   return JD;
 887 }
 888
 889 Expected<JITDylib &> LLJIT::loadPlatformDynamicLibrary(const char *Path) {
 890   auto G = EPCDynamicLibrarySearchGenerator::Load(*ES, Path);
 891   if (!G)
 892     return G.takeError();
 893
 894   if (auto *ExistingJD = ES->getJITDylibByName(Path))
 895     return *ExistingJD;
 896
 897   auto &JD = ES->createBareJITDylib(Path);
 898   JD.addGenerator(std::move(*G));
 899   return JD;
 900 }
 901
 902 Error LLJIT::linkStaticLibraryInto(JITDylib &JD,
 903                                    std::unique_ptr<MemoryBuffer> LibBuffer) {
 904   auto G = StaticLibraryDefinitionGenerator::Create(*ObjLinkingLayer,
 905                                                     std::move(LibBuffer));
 906   if (!G)
 907     return G.takeError();
 908
 909   JD.addGenerator(std::move(*G));
 910
 911   return Error::success();
 912 }
 913
 914 Error LLJIT::linkStaticLibraryInto(JITDylib &JD, const char *Path) {
 915   auto G = StaticLibraryDefinitionGenerator::Load(*ObjLinkingLayer, Path);
 916   if (!G)
 917     return G.takeError();
 918
 919   JD.addGenerator(std::move(*G));
 920
 921   return Error::success();
 922 }
 923
 924 Error LLJIT::addIRModule(ResourceTrackerSP RT, ThreadSafeModule TSM) {
 925   assert(TSM && "Can not add null module");
 926
 927   if (auto Err =
 928           TSM.withModuleDo([&](Module &M) { return applyDataLayout(M); }))
 929     return Err;
 930
 931   return InitHelperTransformLayer->add(std::move(RT), std::move(TSM));
 932 }
 933
 934 Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) {
 935   return addIRModule(JD.getDefaultResourceTracker(), std::move(TSM));
 936 }
 937
 938 Error LLJIT::addObjectFile(ResourceTrackerSP RT,
 939                            std::unique_ptr<MemoryBuffer> Obj) {
 940   assert(Obj && "Can not add null object");
 941
 942   return ObjTransformLayer->add(std::move(RT), std::move(Obj));
 943 }
 944
 945 Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) {
 946   return addObjectFile(JD.getDefaultResourceTracker(), std::move(Obj));
 947 }
 948
 949 Expected<ExecutorAddr> LLJIT::lookupLinkerMangled(JITDylib &JD,
 950                                                   SymbolStringPtr Name) {
 951   if (auto Sym = ES->lookup(
 952         makeJITDylibSearchOrder(&JD, JITDylibLookupFlags::MatchAllSymbols),
 953         Name))
 954     return Sym->getAddress();
 955   else
 956     return Sym.takeError();
 957 }
 958
 959 Expected<std::unique_ptr<ObjectLayer>>
 960 LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) {
 961
 962   // If the config state provided an ObjectLinkingLayer factory then use it.
 963   if (S.CreateObjectLinkingLayer)
 964     return S.CreateObjectLinkingLayer(ES, S.JTMB->getTargetTriple());
 965
 966   // Otherwise default to creating an RTDyldObjectLinkingLayer that constructs
 967   // a new SectionMemoryManager for each object.
 968   auto GetMemMgr = []() { return std::make_unique<SectionMemoryManager>(); };
 969   auto Layer =
 970       std::make_unique<RTDyldObjectLinkingLayer>(ES, std::move(GetMemMgr));
 971
 972   if (S.JTMB->getTargetTriple().isOSBinFormatCOFF()) {
 973     Layer->setOverrideObjectFlagsWithResponsibilityFlags(true);
 974     Layer->setAutoClaimResponsibilityForObjectSymbols(true);
 975   }
 976
 977   if (S.JTMB->getTargetTriple().isOSBinFormatELF() &&
 978       (S.JTMB->getTargetTriple().getArch() == Triple::ArchType::ppc64 ||
 979        S.JTMB->getTargetTriple().getArch() == Triple::ArchType::ppc64le))
 980     Layer->setAutoClaimResponsibilityForObjectSymbols(true);
 981
 982   // FIXME: Explicit conversion to std::unique_ptr<ObjectLayer> added to silence
 983   //        errors from some GCC / libstdc++ bots. Remove this conversion (i.e.
 984   //        just return ObjLinkingLayer) once those bots are upgraded.
 985   return std::unique_ptr<ObjectLayer>(std::move(Layer));
 986 }
 987
 988 Expected<std::unique_ptr<IRCompileLayer::IRCompiler>>
 989 LLJIT::createCompileFunction(LLJITBuilderState &S,
 990                              JITTargetMachineBuilder JTMB) {
 991
 992   /// If there is a custom compile function creator set then use it.
 993   if (S.CreateCompileFunction)
 994     return S.CreateCompileFunction(std::move(JTMB));
 995
 996   // If using a custom EPC then use a ConcurrentIRCompiler by default.
 997   if (*S.SupportConcurrentCompilation)
 998     return std::make_unique<ConcurrentIRCompiler>(std::move(JTMB));
 999
1000   auto TM = JTMB.createTargetMachine();
1001   if (!TM)
1002     return TM.takeError();
1003
1004   return std::make_unique<TMOwningSimpleCompiler>(std::move(*TM));
1005 }
1006
1007 LLJIT::LLJIT(LLJITBuilderState &S, Error &Err)
1008     : DL(std::move(*S.DL)), TT(S.JTMB->getTargetTriple()) {
1009
1010   ErrorAsOutParameter _(Err);
1011
1012   assert(!(S.EPC && S.ES) && "EPC and ES should not both be set");
1013
1014   if (S.EPC) {
1015     ES = std::make_unique<ExecutionSession>(std::move(S.EPC));
1016   } else if (S.ES)
1017     ES = std::move(S.ES);
1018   else {
1019     if (auto EPC = SelfExecutorProcessControl::Create()) {
1020       ES = std::make_unique<ExecutionSession>(std::move(*EPC));
1021     } else {
1022       Err = EPC.takeError();
1023       return;
1024     }
1025   }
1026
1027   auto ObjLayer = createObjectLinkingLayer(S, *ES);
1028   if (!ObjLayer) {
1029     Err = ObjLayer.takeError();
1030     return;
1031   }
1032   ObjLinkingLayer = std::move(*ObjLayer);
1033   ObjTransformLayer =
1034       std::make_unique<ObjectTransformLayer>(*ES, *ObjLinkingLayer);
1035
1036   {
1037     auto CompileFunction = createCompileFunction(S, std::move(*S.JTMB));
1038     if (!CompileFunction) {
1039       Err = CompileFunction.takeError();
1040       return;
1041     }
1042     CompileLayer = std::make_unique<IRCompileLayer>(
1043         *ES, *ObjTransformLayer, std::move(*CompileFunction));
1044     TransformLayer = std::make_unique<IRTransformLayer>(*ES, *CompileLayer);
1045     InitHelperTransformLayer =
1046         std::make_unique<IRTransformLayer>(*ES, *TransformLayer);
1047   }
1048
1049   if (*S.SupportConcurrentCompilation)
1050     InitHelperTransformLayer->setCloneToNewContextOnEmit(true);
1051
1052   if (S.SetupProcessSymbolsJITDylib) {
1053     if (auto ProcSymsJD = S.SetupProcessSymbolsJITDylib(*this)) {
1054       ProcessSymbols = ProcSymsJD->get();
1055     } else {
1056       Err = ProcSymsJD.takeError();
1057       return;
1058     }
1059   }
1060
1061   if (S.PrePlatformSetup) {
1062     if (auto Err2 = S.PrePlatformSetup(*this)) {
1063       Err = std::move(Err2);
1064       return;
1065     }
1066   }
1067
1068   if (!S.SetUpPlatform)
1069     S.SetUpPlatform = setUpGenericLLVMIRPlatform;
1070
1071   if (auto PlatformJDOrErr = S.SetUpPlatform(*this)) {
1072     Platform = PlatformJDOrErr->get();
1073     if (Platform)
1074       DefaultLinks.push_back(
1075           {Platform, JITDylibLookupFlags::MatchExportedSymbolsOnly});
1076   } else {
1077     Err = PlatformJDOrErr.takeError();
1078     return;
1079   }
1080
1081   if (S.LinkProcessSymbolsByDefault)
1082     DefaultLinks.push_back(
1083         {ProcessSymbols, JITDylibLookupFlags::MatchExportedSymbolsOnly});
1084
1085   if (auto MainOrErr = createJITDylib("main"))
1086     Main = &*MainOrErr;
1087   else {
1088     Err = MainOrErr.takeError();
1089     return;
1090   }
1091 }
1092
1093 std::string LLJIT::mangle(StringRef UnmangledName) const {
1094   std::string MangledName;
1095   {
1096     raw_string_ostream MangledNameStream(MangledName);
1097     Mangler::getNameWithPrefix(MangledNameStream, UnmangledName, DL);
1098   }
1099   return MangledName;
1100 }
1101
1102 Error LLJIT::applyDataLayout(Module &M) {
1103   if (M.getDataLayout().isDefault())
1104     M.setDataLayout(DL);
1105
1106   if (M.getDataLayout() != DL)
1107     return make_error<StringError>(
1108         "Added modules have incompatible data layouts: " +
1109             M.getDataLayout().getStringRepresentation() + " (module) vs " +
1110             DL.getStringRepresentation() + " (jit)",
1111         inconvertibleErrorCode());
1112
1113   return Error::success();
1114 }
1115
1116 Error setUpOrcPlatformManually(LLJIT &J) {
1117   LLVM_DEBUG({ dbgs() << "Setting up orc platform support for LLJIT\n"; });
1118   J.setPlatformSupport(std::make_unique<ORCPlatformSupport>(J));
1119   return Error::success();
1120 }
1121
1122 class LoadAndLinkDynLibrary {
1123 public:
1124   LoadAndLinkDynLibrary(LLJIT &J) : J(J) {}
1125   Error operator()(JITDylib &JD, StringRef DLLName) {
1126     if (!DLLName.ends_with_insensitive(".dll"))
1127       return make_error<StringError>("DLLName not ending with .dll",
1128                                      inconvertibleErrorCode());
1129     auto DLLNameStr = DLLName.str(); // Guarantees null-termination.
1130     auto DLLJD = J.loadPlatformDynamicLibrary(DLLNameStr.c_str());
1131     if (!DLLJD)
1132       return DLLJD.takeError();
1133     JD.addToLinkOrder(*DLLJD);
1134     return Error::success();
1135   }
1136
1137 private:
1138   LLJIT &J;
1139 };
1140
1141 Expected<JITDylibSP> ExecutorNativePlatform::operator()(LLJIT &J) {
1142   auto ProcessSymbolsJD = J.getProcessSymbolsJITDylib();
1143   if (!ProcessSymbolsJD)
1144     return make_error<StringError>(
1145         "Native platforms require a process symbols JITDylib",
1146         inconvertibleErrorCode());
1147
1148   const Triple &TT = J.getTargetTriple();
1149   ObjectLinkingLayer *ObjLinkingLayer =
1150       dyn_cast<ObjectLinkingLayer>(&J.getObjLinkingLayer());
1151
1152   if (!ObjLinkingLayer)
1153     return make_error<StringError>(
1154         "ExecutorNativePlatform requires ObjectLinkingLayer",
1155         inconvertibleErrorCode());
1156
1157   std::unique_ptr<MemoryBuffer> RuntimeArchiveBuffer;
1158   if (OrcRuntime.index() == 0) {
1159     auto A = errorOrToExpected(MemoryBuffer::getFile(std::get<0>(OrcRuntime)));
1160     if (!A)
1161       return A.takeError();
1162     RuntimeArchiveBuffer = std::move(*A);
1163   } else
1164     RuntimeArchiveBuffer = std::move(std::get<1>(OrcRuntime));
1165
1166   auto &ES = J.getExecutionSession();
1167   auto &PlatformJD = ES.createBareJITDylib("<Platform>");
1168   PlatformJD.addToLinkOrder(*ProcessSymbolsJD);
1169
1170   J.setPlatformSupport(std::make_unique<ORCPlatformSupport>(J));
1171
1172   switch (TT.getObjectFormat()) {
1173   case Triple::COFF: {
1174     const char *VCRuntimePath = nullptr;
1175     bool StaticVCRuntime = false;
1176     if (VCRuntime) {
1177       VCRuntimePath = VCRuntime->first.c_str();
1178       StaticVCRuntime = VCRuntime->second;
1179     }
1180     if (auto P = COFFPlatform::Create(
1181             *ObjLinkingLayer, PlatformJD, std::move(RuntimeArchiveBuffer),
1182             LoadAndLinkDynLibrary(J), StaticVCRuntime, VCRuntimePath))
1183       J.getExecutionSession().setPlatform(std::move(*P));
1184     else
1185       return P.takeError();
1186     break;
1187   }
1188   case Triple::ELF: {
1189     auto G = StaticLibraryDefinitionGenerator::Create(
1190         *ObjLinkingLayer, std::move(RuntimeArchiveBuffer));
1191     if (!G)
1192       return G.takeError();
1193
1194     if (auto P =
1195             ELFNixPlatform::Create(*ObjLinkingLayer, PlatformJD, std::move(*G)))
1196       J.getExecutionSession().setPlatform(std::move(*P));
1197     else
1198       return P.takeError();
1199     break;
1200   }
1201   case Triple::MachO: {
1202     auto G = StaticLibraryDefinitionGenerator::Create(
1203         *ObjLinkingLayer, std::move(RuntimeArchiveBuffer));
1204     if (!G)
1205       return G.takeError();
1206
1207     if (auto P =
1208             MachOPlatform::Create(*ObjLinkingLayer, PlatformJD, std::move(*G)))
1209       ES.setPlatform(std::move(*P));
1210     else
1211       return P.takeError();
1212     break;
1213   }
1214   default:
1215     return make_error<StringError>("Unsupported object format in triple " +
1216                                        TT.str(),
1217                                    inconvertibleErrorCode());
1218   }
1219
1220   return &PlatformJD;
1221 }
1222
1223 Expected<JITDylibSP> setUpGenericLLVMIRPlatform(LLJIT &J) {
1224   LLVM_DEBUG(
1225       { dbgs() << "Setting up GenericLLVMIRPlatform support for LLJIT\n"; });
1226   auto ProcessSymbolsJD = J.getProcessSymbolsJITDylib();
1227   if (!ProcessSymbolsJD)
1228     return make_error<StringError>(
1229         "Native platforms require a process symbols JITDylib",
1230         inconvertibleErrorCode());
1231
1232   auto &PlatformJD = J.getExecutionSession().createBareJITDylib("<Platform>");
1233   PlatformJD.addToLinkOrder(*ProcessSymbolsJD);
1234
1235   J.setPlatformSupport(
1236       std::make_unique<GenericLLVMIRPlatformSupport>(J, PlatformJD));
1237
1238   return &PlatformJD;
1239 }
1240
1241 Expected<JITDylibSP> setUpInactivePlatform(LLJIT &J) {
1242   LLVM_DEBUG(
1243       { dbgs() << "Explicitly deactivated platform support for LLJIT\n"; });
1244   J.setPlatformSupport(std::make_unique<InactivePlatformSupport>());
1245   return nullptr;
1246 }
1247
1248 Error LLLazyJITBuilderState::prepareForConstruction() {
1249   if (auto Err = LLJITBuilderState::prepareForConstruction())
1250     return Err;
1251   TT = JTMB->getTargetTriple();
1252   return Error::success();
1253 }
1254
1255 Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) {
1256   assert(TSM && "Can not add null module");
1257
1258   if (auto Err = TSM.withModuleDo(
1259           [&](Module &M) -> Error { return applyDataLayout(M); }))
1260     return Err;
1261
1262   return CODLayer->add(JD, std::move(TSM));
1263 }
1264
1265 LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) {
1266
1267   // If LLJIT construction failed then bail out.
1268   if (Err)
1269     return;
1270
1271   ErrorAsOutParameter _(&Err);
1272
1273   /// Take/Create the lazy-compile callthrough manager.
1274   if (S.LCTMgr)
1275     LCTMgr = std::move(S.LCTMgr);
1276   else {
1277     if (auto LCTMgrOrErr = createLocalLazyCallThroughManager(
1278             S.TT, *ES, S.LazyCompileFailureAddr))
1279       LCTMgr = std::move(*LCTMgrOrErr);
1280     else {
1281       Err = LCTMgrOrErr.takeError();
1282       return;
1283     }
1284   }
1285
1286   // Take/Create the indirect stubs manager builder.
1287   auto ISMBuilder = std::move(S.ISMBuilder);
1288
1289   // If none was provided, try to build one.
1290   if (!ISMBuilder)
1291     ISMBuilder = createLocalIndirectStubsManagerBuilder(S.TT);
1292
1293   // No luck. Bail out.
1294   if (!ISMBuilder) {
1295     Err = make_error<StringError>("Could not construct "
1296                                   "IndirectStubsManagerBuilder for target " +
1297                                       S.TT.str(),
1298                                   inconvertibleErrorCode());
1299     return;
1300   }
1301
1302   // Create the IP Layer.
1303   IPLayer = std::make_unique<IRPartitionLayer>(*ES, *InitHelperTransformLayer);
1304
1305   // Create the COD layer.
1306   CODLayer = std::make_unique<CompileOnDemandLayer>(*ES, *IPLayer, *LCTMgr,
1307                                                     std::move(ISMBuilder));
1308
1309   if (*S.SupportConcurrentCompilation)
1310     CODLayer->setCloneToNewContextOnEmit(true);
1311 }
1312
1313 // In-process LLJIT uses eh-frame section wrappers via EPC, so we need to force
1314 // them to be linked in.
1315 LLVM_ATTRIBUTE_USED void linkComponents() {
1316   errs() << (void *)&llvm_orc_registerEHFrameSectionWrapper
1317          << (void *)&llvm_orc_deregisterEHFrameSectionWrapper;
1318 }
1319
1320 } // End namespace orc.
1321 } // End namespace llvm.