llvm/lib/ExecutionEngine/Orc/LLJIT.cpp

   1 //===--------- LLJIT.cpp - An ORC-based JIT for compiling LLVM IR ---------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ExecutionEngine/Orc/LLJIT.h"
  10 #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h"
  11 #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h"
  12 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
  13 #include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
  14 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
  15 #include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h"
  16 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
  17 #include "llvm/ExecutionEngine/Orc/Shared/OrcError.h"
  18 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
  19 #include "llvm/IR/GlobalVariable.h"
  20 #include "llvm/IR/IRBuilder.h"
  21 #include "llvm/IR/Mangler.h"
  22 #include "llvm/IR/Module.h"
  23 #include "llvm/Support/DynamicLibrary.h"
  24
  25 #include <map>
  26
  27 #define DEBUG_TYPE "orc"
  28
  29 using namespace llvm;
  30 using namespace llvm::orc;
  31
  32 namespace {
  33
  34 /// Adds helper function decls and wrapper functions that call the helper with
  35 /// some additional prefix arguments.
  36 ///
  37 /// E.g. For wrapper "foo" with type i8(i8, i64), helper "bar", and prefix
  38 /// args i32 4 and i16 12345, this function will add:
  39 ///
  40 /// declare i8 @bar(i32, i16, i8, i64)
  41 ///
  42 /// define i8 @foo(i8, i64) {
  43 /// entry:
  44 ///   %2 = call i8 @bar(i32 4, i16 12345, i8 %0, i64 %1)
  45 ///   ret i8 %2
  46 /// }
  47 ///
  48 Function *addHelperAndWrapper(Module &M, StringRef WrapperName,
  49                               FunctionType *WrapperFnType,
  50                               GlobalValue::VisibilityTypes WrapperVisibility,
  51                               StringRef HelperName,
  52                               ArrayRef<Value *> HelperPrefixArgs) {
  53   std::vector<Type *> HelperArgTypes;
  54   for (auto *Arg : HelperPrefixArgs)
  55     HelperArgTypes.push_back(Arg->getType());
  56   for (auto *T : WrapperFnType->params())
  57     HelperArgTypes.push_back(T);
  58   auto *HelperFnType =
  59       FunctionType::get(WrapperFnType->getReturnType(), HelperArgTypes, false);
  60   auto *HelperFn = Function::Create(HelperFnType, GlobalValue::ExternalLinkage,
  61                                     HelperName, M);
  62
  63   auto *WrapperFn = Function::Create(
  64       WrapperFnType, GlobalValue::ExternalLinkage, WrapperName, M);
  65   WrapperFn->setVisibility(WrapperVisibility);
  66
  67   auto *EntryBlock = BasicBlock::Create(M.getContext(), "entry", WrapperFn);
  68   IRBuilder<> IB(EntryBlock);
  69
  70   std::vector<Value *> HelperArgs;
  71   for (auto *Arg : HelperPrefixArgs)
  72     HelperArgs.push_back(Arg);
  73   for (auto &Arg : WrapperFn->args())
  74     HelperArgs.push_back(&Arg);
  75   auto *HelperResult = IB.CreateCall(HelperFn, HelperArgs);
  76   if (HelperFn->getReturnType()->isVoidTy())
  77     IB.CreateRetVoid();
  78   else
  79     IB.CreateRet(HelperResult);
  80
  81   return WrapperFn;
  82 }
  83
  84 class GenericLLVMIRPlatformSupport;
  85
  86 /// orc::Platform component of Generic LLVM IR Platform support.
  87 /// Just forwards calls to the GenericLLVMIRPlatformSupport class below.
  88 class GenericLLVMIRPlatform : public Platform {
  89 public:
  90   GenericLLVMIRPlatform(GenericLLVMIRPlatformSupport &S) : S(S) {}
  91   Error setupJITDylib(JITDylib &JD) override;
  92   Error notifyAdding(ResourceTracker &RT,
  93                      const MaterializationUnit &MU) override;
  94   Error notifyRemoving(ResourceTracker &RT) override {
  95     // Noop -- Nothing to do (yet).
  96     return Error::success();
  97   }
  98
  99 private:
 100   GenericLLVMIRPlatformSupport &S;
 101 };
 102
 103 /// This transform parses llvm.global_ctors to produce a single initialization
 104 /// function for the module, records the function, then deletes
 105 /// llvm.global_ctors.
 106 class GlobalCtorDtorScraper {
 107 public:
 108
 109   GlobalCtorDtorScraper(GenericLLVMIRPlatformSupport &PS,
 110                         StringRef InitFunctionPrefix)
 111     : PS(PS), InitFunctionPrefix(InitFunctionPrefix) {}
 112   Expected<ThreadSafeModule> operator()(ThreadSafeModule TSM,
 113                                         MaterializationResponsibility &R);
 114
 115 private:
 116   GenericLLVMIRPlatformSupport &PS;
 117   StringRef InitFunctionPrefix;
 118 };
 119
 120 /// Generic IR Platform Support
 121 ///
 122 /// Scrapes llvm.global_ctors and llvm.global_dtors and replaces them with
 123 /// specially named 'init' and 'deinit'. Injects definitions / interposes for
 124 /// some runtime API, including __cxa_atexit, dlopen, and dlclose.
 125 class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport {
 126 public:
 127   GenericLLVMIRPlatformSupport(LLJIT &J)
 128       : J(J), InitFunctionPrefix(J.mangle("__orc_init_func.")) {
 129
 130     getExecutionSession().setPlatform(
 131         std::make_unique<GenericLLVMIRPlatform>(*this));
 132
 133     setInitTransform(J, GlobalCtorDtorScraper(*this, InitFunctionPrefix));
 134
 135     SymbolMap StdInterposes;
 136
 137     StdInterposes[J.mangleAndIntern("__lljit.platform_support_instance")] =
 138         JITEvaluatedSymbol(pointerToJITTargetAddress(this),
 139                            JITSymbolFlags::Exported);
 140     StdInterposes[J.mangleAndIntern("__lljit.cxa_atexit_helper")] =
 141         JITEvaluatedSymbol(pointerToJITTargetAddress(registerAtExitHelper),
 142                            JITSymbolFlags());
 143
 144     cantFail(
 145         J.getMainJITDylib().define(absoluteSymbols(std::move(StdInterposes))));
 146     cantFail(setupJITDylib(J.getMainJITDylib()));
 147     cantFail(J.addIRModule(J.getMainJITDylib(), createPlatformRuntimeModule()));
 148   }
 149
 150   ExecutionSession &getExecutionSession() { return J.getExecutionSession(); }
 151
 152   /// Adds a module that defines the __dso_handle global.
 153   Error setupJITDylib(JITDylib &JD) {
 154
 155     // Add per-jitdylib standard interposes.
 156     SymbolMap PerJDInterposes;
 157     PerJDInterposes[J.mangleAndIntern("__lljit.run_atexits_helper")] =
 158         JITEvaluatedSymbol(pointerToJITTargetAddress(runAtExitsHelper),
 159                            JITSymbolFlags());
 160     cantFail(JD.define(absoluteSymbols(std::move(PerJDInterposes))));
 161
 162     auto Ctx = std::make_unique<LLVMContext>();
 163     auto M = std::make_unique<Module>("__standard_lib", *Ctx);
 164     M->setDataLayout(J.getDataLayout());
 165
 166     auto *Int64Ty = Type::getInt64Ty(*Ctx);
 167     auto *DSOHandle = new GlobalVariable(
 168         *M, Int64Ty, true, GlobalValue::ExternalLinkage,
 169         ConstantInt::get(Int64Ty, reinterpret_cast<uintptr_t>(&JD)),
 170         "__dso_handle");
 171     DSOHandle->setVisibility(GlobalValue::DefaultVisibility);
 172     DSOHandle->setInitializer(
 173         ConstantInt::get(Int64Ty, pointerToJITTargetAddress(&JD)));
 174
 175     auto *GenericIRPlatformSupportTy =
 176         StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport");
 177
 178     auto *PlatformInstanceDecl = new GlobalVariable(
 179         *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
 180         nullptr, "__lljit.platform_support_instance");
 181
 182     auto *VoidTy = Type::getVoidTy(*Ctx);
 183     addHelperAndWrapper(
 184         *M, "__lljit_run_atexits", FunctionType::get(VoidTy, {}, false),
 185         GlobalValue::HiddenVisibility, "__lljit.run_atexits_helper",
 186         {PlatformInstanceDecl, DSOHandle});
 187
 188     return J.addIRModule(JD, ThreadSafeModule(std::move(M), std::move(Ctx)));
 189   }
 190
 191   Error notifyAdding(ResourceTracker &RT, const MaterializationUnit &MU) {
 192     auto &JD = RT.getJITDylib();
 193     if (auto &InitSym = MU.getInitializerSymbol())
 194       InitSymbols[&JD].add(InitSym, SymbolLookupFlags::WeaklyReferencedSymbol);
 195     else {
 196       // If there's no identified init symbol attached, but there is a symbol
 197       // with the GenericIRPlatform::InitFunctionPrefix, then treat that as
 198       // an init function. Add the symbol to both the InitSymbols map (which
 199       // will trigger a lookup to materialize the module) and the InitFunctions
 200       // map (which holds the names of the symbols to execute).
 201       for (auto &KV : MU.getSymbols())
 202         if ((*KV.first).startswith(InitFunctionPrefix)) {
 203           InitSymbols[&JD].add(KV.first,
 204                                SymbolLookupFlags::WeaklyReferencedSymbol);
 205           InitFunctions[&JD].add(KV.first);
 206         }
 207     }
 208     return Error::success();
 209   }
 210
 211   Error initialize(JITDylib &JD) override {
 212     LLVM_DEBUG({
 213       dbgs() << "GenericLLVMIRPlatformSupport getting initializers to run\n";
 214     });
 215     if (auto Initializers = getInitializers(JD)) {
 216       LLVM_DEBUG(
 217           { dbgs() << "GenericLLVMIRPlatformSupport running initializers\n"; });
 218       for (auto InitFnAddr : *Initializers) {
 219         LLVM_DEBUG({
 220           dbgs() << "  Running init " << formatv("{0:x16}", InitFnAddr)
 221                  << "...\n";
 222         });
 223         auto *InitFn = jitTargetAddressToFunction<void (*)()>(InitFnAddr);
 224         InitFn();
 225       }
 226     } else
 227       return Initializers.takeError();
 228     return Error::success();
 229   }
 230
 231   Error deinitialize(JITDylib &JD) override {
 232     LLVM_DEBUG({
 233       dbgs() << "GenericLLVMIRPlatformSupport getting deinitializers to run\n";
 234     });
 235     if (auto Deinitializers = getDeinitializers(JD)) {
 236       LLVM_DEBUG({
 237         dbgs() << "GenericLLVMIRPlatformSupport running deinitializers\n";
 238       });
 239       for (auto DeinitFnAddr : *Deinitializers) {
 240         LLVM_DEBUG({
 241           dbgs() << "  Running deinit " << formatv("{0:x16}", DeinitFnAddr)
 242                  << "...\n";
 243         });
 244         auto *DeinitFn = jitTargetAddressToFunction<void (*)()>(DeinitFnAddr);
 245         DeinitFn();
 246       }
 247     } else
 248       return Deinitializers.takeError();
 249
 250     return Error::success();
 251   }
 252
 253   void registerInitFunc(JITDylib &JD, SymbolStringPtr InitName) {
 254     getExecutionSession().runSessionLocked([&]() {
 255         InitFunctions[&JD].add(InitName);
 256       });
 257   }
 258
 259 private:
 260
 261   Expected<std::vector<JITTargetAddress>> getInitializers(JITDylib &JD) {
 262     if (auto Err = issueInitLookups(JD))
 263       return std::move(Err);
 264
 265     DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
 266     std::vector<JITDylibSP> DFSLinkOrder;
 267
 268     getExecutionSession().runSessionLocked([&]() {
 269       DFSLinkOrder = JD.getDFSLinkOrder();
 270
 271       for (auto &NextJD : DFSLinkOrder) {
 272         auto IFItr = InitFunctions.find(NextJD.get());
 273         if (IFItr != InitFunctions.end()) {
 274           LookupSymbols[NextJD.get()] = std::move(IFItr->second);
 275           InitFunctions.erase(IFItr);
 276         }
 277       }
 278     });
 279
 280     LLVM_DEBUG({
 281       dbgs() << "JITDylib init order is [ ";
 282       for (auto &JD : llvm::reverse(DFSLinkOrder))
 283         dbgs() << "\"" << JD->getName() << "\" ";
 284       dbgs() << "]\n";
 285       dbgs() << "Looking up init functions:\n";
 286       for (auto &KV : LookupSymbols)
 287         dbgs() << "  \"" << KV.first->getName() << "\": " << KV.second << "\n";
 288     });
 289
 290     auto &ES = getExecutionSession();
 291     auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols);
 292
 293     if (!LookupResult)
 294       return LookupResult.takeError();
 295
 296     std::vector<JITTargetAddress> Initializers;
 297     while (!DFSLinkOrder.empty()) {
 298       auto &NextJD = *DFSLinkOrder.back();
 299       DFSLinkOrder.pop_back();
 300       auto InitsItr = LookupResult->find(&NextJD);
 301       if (InitsItr == LookupResult->end())
 302         continue;
 303       for (auto &KV : InitsItr->second)
 304         Initializers.push_back(KV.second.getAddress());
 305     }
 306
 307     return Initializers;
 308   }
 309
 310   Expected<std::vector<JITTargetAddress>> getDeinitializers(JITDylib &JD) {
 311     auto &ES = getExecutionSession();
 312
 313     auto LLJITRunAtExits = J.mangleAndIntern("__lljit_run_atexits");
 314
 315     DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
 316     std::vector<JITDylibSP> DFSLinkOrder;
 317
 318     ES.runSessionLocked([&]() {
 319       DFSLinkOrder = JD.getDFSLinkOrder();
 320
 321       for (auto &NextJD : DFSLinkOrder) {
 322         auto &JDLookupSymbols = LookupSymbols[NextJD.get()];
 323         auto DIFItr = DeInitFunctions.find(NextJD.get());
 324         if (DIFItr != DeInitFunctions.end()) {
 325           LookupSymbols[NextJD.get()] = std::move(DIFItr->second);
 326           DeInitFunctions.erase(DIFItr);
 327         }
 328         JDLookupSymbols.add(LLJITRunAtExits,
 329                             SymbolLookupFlags::WeaklyReferencedSymbol);
 330       }
 331     });
 332
 333     LLVM_DEBUG({
 334       dbgs() << "JITDylib deinit order is [ ";
 335       for (auto &JD : DFSLinkOrder)
 336         dbgs() << "\"" << JD->getName() << "\" ";
 337       dbgs() << "]\n";
 338       dbgs() << "Looking up deinit functions:\n";
 339       for (auto &KV : LookupSymbols)
 340         dbgs() << "  \"" << KV.first->getName() << "\": " << KV.second << "\n";
 341     });
 342
 343     auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols);
 344
 345     if (!LookupResult)
 346       return LookupResult.takeError();
 347
 348     std::vector<JITTargetAddress> DeInitializers;
 349     for (auto &NextJD : DFSLinkOrder) {
 350       auto DeInitsItr = LookupResult->find(NextJD.get());
 351       assert(DeInitsItr != LookupResult->end() &&
 352              "Every JD should have at least __lljit_run_atexits");
 353
 354       auto RunAtExitsItr = DeInitsItr->second.find(LLJITRunAtExits);
 355       if (RunAtExitsItr != DeInitsItr->second.end())
 356         DeInitializers.push_back(RunAtExitsItr->second.getAddress());
 357
 358       for (auto &KV : DeInitsItr->second)
 359         if (KV.first != LLJITRunAtExits)
 360           DeInitializers.push_back(KV.second.getAddress());
 361     }
 362
 363     return DeInitializers;
 364   }
 365
 366   /// Issue lookups for all init symbols required to initialize JD (and any
 367   /// JITDylibs that it depends on).
 368   Error issueInitLookups(JITDylib &JD) {
 369     DenseMap<JITDylib *, SymbolLookupSet> RequiredInitSymbols;
 370     std::vector<JITDylibSP> DFSLinkOrder;
 371
 372     getExecutionSession().runSessionLocked([&]() {
 373       DFSLinkOrder = JD.getDFSLinkOrder();
 374
 375       for (auto &NextJD : DFSLinkOrder) {
 376         auto ISItr = InitSymbols.find(NextJD.get());
 377         if (ISItr != InitSymbols.end()) {
 378           RequiredInitSymbols[NextJD.get()] = std::move(ISItr->second);
 379           InitSymbols.erase(ISItr);
 380         }
 381       }
 382     });
 383
 384     return Platform::lookupInitSymbols(getExecutionSession(),
 385                                        RequiredInitSymbols)
 386         .takeError();
 387   }
 388
 389   static void registerAtExitHelper(void *Self, void (*F)(void *), void *Ctx,
 390                                    void *DSOHandle) {
 391     LLVM_DEBUG({
 392       dbgs() << "Registering atexit function " << (void *)F << " for JD "
 393              << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
 394     });
 395     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit(
 396         F, Ctx, DSOHandle);
 397   }
 398
 399   static void runAtExitsHelper(void *Self, void *DSOHandle) {
 400     LLVM_DEBUG({
 401       dbgs() << "Running atexit functions for JD "
 402              << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
 403     });
 404     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.runAtExits(
 405         DSOHandle);
 406   }
 407
 408   // Constructs an LLVM IR module containing platform runtime globals,
 409   // functions, and interposes.
 410   ThreadSafeModule createPlatformRuntimeModule() {
 411     auto Ctx = std::make_unique<LLVMContext>();
 412     auto M = std::make_unique<Module>("__standard_lib", *Ctx);
 413     M->setDataLayout(J.getDataLayout());
 414
 415     auto *GenericIRPlatformSupportTy =
 416         StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport");
 417
 418     auto *PlatformInstanceDecl = new GlobalVariable(
 419         *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
 420         nullptr, "__lljit.platform_support_instance");
 421
 422     auto *Int8Ty = Type::getInt8Ty(*Ctx);
 423     auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT);
 424     auto *VoidTy = Type::getVoidTy(*Ctx);
 425     auto *BytePtrTy = PointerType::getUnqual(Int8Ty);
 426     auto *AtExitCallbackTy = FunctionType::get(VoidTy, {BytePtrTy}, false);
 427     auto *AtExitCallbackPtrTy = PointerType::getUnqual(AtExitCallbackTy);
 428
 429     addHelperAndWrapper(
 430         *M, "__cxa_atexit",
 431         FunctionType::get(IntTy, {AtExitCallbackPtrTy, BytePtrTy, BytePtrTy},
 432                           false),
 433         GlobalValue::DefaultVisibility, "__lljit.cxa_atexit_helper",
 434         {PlatformInstanceDecl});
 435
 436     return ThreadSafeModule(std::move(M), std::move(Ctx));
 437   }
 438
 439   LLJIT &J;
 440   std::string InitFunctionPrefix;
 441   DenseMap<JITDylib *, SymbolLookupSet> InitSymbols;
 442   DenseMap<JITDylib *, SymbolLookupSet> InitFunctions;
 443   DenseMap<JITDylib *, SymbolLookupSet> DeInitFunctions;
 444   ItaniumCXAAtExitSupport AtExitMgr;
 445 };
 446
 447 Error GenericLLVMIRPlatform::setupJITDylib(JITDylib &JD) {
 448   return S.setupJITDylib(JD);
 449 }
 450
 451 Error GenericLLVMIRPlatform::notifyAdding(ResourceTracker &RT,
 452                                           const MaterializationUnit &MU) {
 453   return S.notifyAdding(RT, MU);
 454 }
 455
 456 Expected<ThreadSafeModule>
 457 GlobalCtorDtorScraper::operator()(ThreadSafeModule TSM,
 458                                   MaterializationResponsibility &R) {
 459   auto Err = TSM.withModuleDo([&](Module &M) -> Error {
 460     auto &Ctx = M.getContext();
 461     auto *GlobalCtors = M.getNamedGlobal("llvm.global_ctors");
 462
 463     // If there's no llvm.global_ctors or it's just a decl then skip.
 464     if (!GlobalCtors || GlobalCtors->isDeclaration())
 465       return Error::success();
 466
 467     std::string InitFunctionName;
 468     raw_string_ostream(InitFunctionName)
 469         << InitFunctionPrefix << M.getModuleIdentifier();
 470
 471     MangleAndInterner Mangle(PS.getExecutionSession(), M.getDataLayout());
 472     auto InternedName = Mangle(InitFunctionName);
 473     if (auto Err =
 474             R.defineMaterializing({{InternedName, JITSymbolFlags::Callable}}))
 475       return Err;
 476
 477     auto *InitFunc =
 478         Function::Create(FunctionType::get(Type::getVoidTy(Ctx), {}, false),
 479                          GlobalValue::ExternalLinkage, InitFunctionName, &M);
 480     InitFunc->setVisibility(GlobalValue::HiddenVisibility);
 481     std::vector<std::pair<Function *, unsigned>> Inits;
 482     for (auto E : getConstructors(M))
 483       Inits.push_back(std::make_pair(E.Func, E.Priority));
 484     llvm::sort(Inits, [](const std::pair<Function *, unsigned> &LHS,
 485                          const std::pair<Function *, unsigned> &RHS) {
 486       return LHS.first < RHS.first;
 487     });
 488     auto *EntryBlock = BasicBlock::Create(Ctx, "entry", InitFunc);
 489     IRBuilder<> IB(EntryBlock);
 490     for (auto &KV : Inits)
 491       IB.CreateCall(KV.first);
 492     IB.CreateRetVoid();
 493
 494     PS.registerInitFunc(R.getTargetJITDylib(), InternedName);
 495     GlobalCtors->eraseFromParent();
 496     return Error::success();
 497   });
 498
 499   if (Err)
 500     return std::move(Err);
 501
 502   return std::move(TSM);
 503 }
 504
 505 /// Inactive Platform Support
 506 ///
 507 /// Explicitly disables platform support. JITDylibs are not scanned for special
 508 /// init/deinit symbols. No runtime API interposes are injected.
 509 class InactivePlatformSupport : public LLJIT::PlatformSupport {
 510 public:
 511   InactivePlatformSupport() = default;
 512
 513   Error initialize(JITDylib &JD) override {
 514     LLVM_DEBUG(dbgs() << "InactivePlatformSupport: no initializers running for "
 515                       << JD.getName() << "\n");
 516     return Error::success();
 517   }
 518
 519   Error deinitialize(JITDylib &JD) override {
 520     LLVM_DEBUG(
 521         dbgs() << "InactivePlatformSupport: no deinitializers running for "
 522                << JD.getName() << "\n");
 523     return Error::success();
 524   }
 525 };
 526
 527 } // end anonymous namespace
 528
 529 namespace llvm {
 530 namespace orc {
 531
 532 void LLJIT::PlatformSupport::setInitTransform(
 533     LLJIT &J, IRTransformLayer::TransformFunction T) {
 534   J.InitHelperTransformLayer->setTransform(std::move(T));
 535 }
 536
 537 LLJIT::PlatformSupport::~PlatformSupport() {}
 538
 539 Error LLJITBuilderState::prepareForConstruction() {
 540
 541   LLVM_DEBUG(dbgs() << "Preparing to create LLJIT instance...\n");
 542
 543   if (!JTMB) {
 544     LLVM_DEBUG({
 545       dbgs() << "  No explicitly set JITTargetMachineBuilder. "
 546                 "Detecting host...\n";
 547     });
 548     if (auto JTMBOrErr = JITTargetMachineBuilder::detectHost())
 549       JTMB = std::move(*JTMBOrErr);
 550     else
 551       return JTMBOrErr.takeError();
 552   }
 553
 554   LLVM_DEBUG({
 555     dbgs() << "  JITTargetMachineBuilder is "
 556            << JITTargetMachineBuilderPrinter(*JTMB, "  ")
 557            << "  Pre-constructed ExecutionSession: " << (ES ? "Yes" : "No")
 558            << "\n"
 559            << "  DataLayout: ";
 560     if (DL)
 561       dbgs() << DL->getStringRepresentation() << "\n";
 562     else
 563       dbgs() << "None (will be created by JITTargetMachineBuilder)\n";
 564
 565     dbgs() << "  Custom object-linking-layer creator: "
 566            << (CreateObjectLinkingLayer ? "Yes" : "No") << "\n"
 567            << "  Custom compile-function creator: "
 568            << (CreateCompileFunction ? "Yes" : "No") << "\n"
 569            << "  Custom platform-setup function: "
 570            << (SetUpPlatform ? "Yes" : "No") << "\n"
 571            << "  Number of compile threads: " << NumCompileThreads;
 572     if (!NumCompileThreads)
 573       dbgs() << " (code will be compiled on the execution thread)\n";
 574     else
 575       dbgs() << "\n";
 576   });
 577
 578   // If neither ES nor EPC has been set then create an EPC instance.
 579   if (!ES && !EPC) {
 580     LLVM_DEBUG({
 581       dbgs() << "ExecutorProcessControl not specified, "
 582                 "Creating SelfExecutorProcessControl instance\n";
 583     });
 584     if (auto EPCOrErr = SelfExecutorProcessControl::Create())
 585       EPC = std::move(*EPCOrErr);
 586     else
 587       return EPCOrErr.takeError();
 588   } else
 589     LLVM_DEBUG({
 590       dbgs() << "Using explicitly specified ExecutorProcessControl instance "
 591              << EPC.get() << "\n";
 592     });
 593
 594   // If the client didn't configure any linker options then auto-configure the
 595   // JIT linker.
 596   if (!CreateObjectLinkingLayer) {
 597     auto &TT = JTMB->getTargetTriple();
 598     if (TT.isOSBinFormatMachO() &&
 599         (TT.getArch() == Triple::aarch64 || TT.getArch() == Triple::x86_64)) {
 600
 601       JTMB->setRelocationModel(Reloc::PIC_);
 602       JTMB->setCodeModel(CodeModel::Small);
 603       CreateObjectLinkingLayer =
 604           [](ExecutionSession &ES,
 605              const Triple &) -> Expected<std::unique_ptr<ObjectLayer>> {
 606         auto ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>(ES);
 607         ObjLinkingLayer->addPlugin(std::make_unique<EHFrameRegistrationPlugin>(
 608             ES, std::make_unique<jitlink::InProcessEHFrameRegistrar>()));
 609         return std::move(ObjLinkingLayer);
 610       };
 611     }
 612   }
 613
 614   return Error::success();
 615 }
 616
 617 LLJIT::~LLJIT() {
 618   if (CompileThreads)
 619     CompileThreads->wait();
 620   if (auto Err = ES->endSession())
 621     ES->reportError(std::move(Err));
 622 }
 623
 624 Error LLJIT::addIRModule(ResourceTrackerSP RT, ThreadSafeModule TSM) {
 625   assert(TSM && "Can not add null module");
 626
 627   if (auto Err =
 628           TSM.withModuleDo([&](Module &M) { return applyDataLayout(M); }))
 629     return Err;
 630
 631   return InitHelperTransformLayer->add(std::move(RT), std::move(TSM));
 632 }
 633
 634 Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) {
 635   return addIRModule(JD.getDefaultResourceTracker(), std::move(TSM));
 636 }
 637
 638 Error LLJIT::addObjectFile(ResourceTrackerSP RT,
 639                            std::unique_ptr<MemoryBuffer> Obj) {
 640   assert(Obj && "Can not add null object");
 641
 642   return ObjTransformLayer->add(std::move(RT), std::move(Obj));
 643 }
 644
 645 Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) {
 646   return addObjectFile(JD.getDefaultResourceTracker(), std::move(Obj));
 647 }
 648
 649 Expected<JITEvaluatedSymbol> LLJIT::lookupLinkerMangled(JITDylib &JD,
 650                                                         SymbolStringPtr Name) {
 651   return ES->lookup(
 652       makeJITDylibSearchOrder(&JD, JITDylibLookupFlags::MatchAllSymbols), Name);
 653 }
 654
 655 Expected<std::unique_ptr<ObjectLayer>>
 656 LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) {
 657
 658   // If the config state provided an ObjectLinkingLayer factory then use it.
 659   if (S.CreateObjectLinkingLayer)
 660     return S.CreateObjectLinkingLayer(ES, S.JTMB->getTargetTriple());
 661
 662   // Otherwise default to creating an RTDyldObjectLinkingLayer that constructs
 663   // a new SectionMemoryManager for each object.
 664   auto GetMemMgr = []() { return std::make_unique<SectionMemoryManager>(); };
 665   auto Layer =
 666       std::make_unique<RTDyldObjectLinkingLayer>(ES, std::move(GetMemMgr));
 667
 668   if (S.JTMB->getTargetTriple().isOSBinFormatCOFF()) {
 669     Layer->setOverrideObjectFlagsWithResponsibilityFlags(true);
 670     Layer->setAutoClaimResponsibilityForObjectSymbols(true);
 671   }
 672
 673   // FIXME: Explicit conversion to std::unique_ptr<ObjectLayer> added to silence
 674   //        errors from some GCC / libstdc++ bots. Remove this conversion (i.e.
 675   //        just return ObjLinkingLayer) once those bots are upgraded.
 676   return std::unique_ptr<ObjectLayer>(std::move(Layer));
 677 }
 678
 679 Expected<std::unique_ptr<IRCompileLayer::IRCompiler>>
 680 LLJIT::createCompileFunction(LLJITBuilderState &S,
 681                              JITTargetMachineBuilder JTMB) {
 682
 683   /// If there is a custom compile function creator set then use it.
 684   if (S.CreateCompileFunction)
 685     return S.CreateCompileFunction(std::move(JTMB));
 686
 687   // Otherwise default to creating a SimpleCompiler, or ConcurrentIRCompiler,
 688   // depending on the number of threads requested.
 689   if (S.NumCompileThreads > 0)
 690     return std::make_unique<ConcurrentIRCompiler>(std::move(JTMB));
 691
 692   auto TM = JTMB.createTargetMachine();
 693   if (!TM)
 694     return TM.takeError();
 695
 696   return std::make_unique<TMOwningSimpleCompiler>(std::move(*TM));
 697 }
 698
 699 LLJIT::LLJIT(LLJITBuilderState &S, Error &Err)
 700     : DL(""), TT(S.JTMB->getTargetTriple()) {
 701
 702   ErrorAsOutParameter _(&Err);
 703
 704   assert(!(S.EPC && S.ES) && "EPC and ES should not both be set");
 705
 706   if (S.EPC) {
 707     ES = std::make_unique<ExecutionSession>(std::move(S.EPC));
 708   } else if (S.ES)
 709     ES = std::move(S.ES);
 710   else {
 711     if (auto EPC = SelfExecutorProcessControl::Create()) {
 712       ES = std::make_unique<ExecutionSession>(std::move(*EPC));
 713     } else {
 714       Err = EPC.takeError();
 715       return;
 716     }
 717   }
 718
 719   if (auto MainOrErr = this->ES->createJITDylib("main"))
 720     Main = &*MainOrErr;
 721   else {
 722     Err = MainOrErr.takeError();
 723     return;
 724   }
 725
 726   if (S.DL)
 727     DL = std::move(*S.DL);
 728   else if (auto DLOrErr = S.JTMB->getDefaultDataLayoutForTarget())
 729     DL = std::move(*DLOrErr);
 730   else {
 731     Err = DLOrErr.takeError();
 732     return;
 733   }
 734
 735   auto ObjLayer = createObjectLinkingLayer(S, *ES);
 736   if (!ObjLayer) {
 737     Err = ObjLayer.takeError();
 738     return;
 739   }
 740   ObjLinkingLayer = std::move(*ObjLayer);
 741   ObjTransformLayer =
 742       std::make_unique<ObjectTransformLayer>(*ES, *ObjLinkingLayer);
 743
 744   {
 745     auto CompileFunction = createCompileFunction(S, std::move(*S.JTMB));
 746     if (!CompileFunction) {
 747       Err = CompileFunction.takeError();
 748       return;
 749     }
 750     CompileLayer = std::make_unique<IRCompileLayer>(
 751         *ES, *ObjTransformLayer, std::move(*CompileFunction));
 752     TransformLayer = std::make_unique<IRTransformLayer>(*ES, *CompileLayer);
 753     InitHelperTransformLayer =
 754         std::make_unique<IRTransformLayer>(*ES, *TransformLayer);
 755   }
 756
 757   if (S.NumCompileThreads > 0) {
 758     InitHelperTransformLayer->setCloneToNewContextOnEmit(true);
 759     CompileThreads =
 760         std::make_unique<ThreadPool>(hardware_concurrency(S.NumCompileThreads));
 761     ES->setDispatchTask([this](std::unique_ptr<Task> T) {
 762       // FIXME: We should be able to use move-capture here, but ThreadPool's
 763       // AsyncTaskTys are std::functions rather than unique_functions
 764       // (because MSVC's std::packaged_tasks don't support move-only types).
 765       // Fix this when all the above gets sorted out.
 766       CompileThreads->async([UnownedT = T.release()]() mutable {
 767         std::unique_ptr<Task> T(UnownedT);
 768         T->run();
 769       });
 770     });
 771   }
 772
 773   if (S.SetUpPlatform)
 774     Err = S.SetUpPlatform(*this);
 775   else
 776     setUpGenericLLVMIRPlatform(*this);
 777 }
 778
 779 std::string LLJIT::mangle(StringRef UnmangledName) const {
 780   std::string MangledName;
 781   {
 782     raw_string_ostream MangledNameStream(MangledName);
 783     Mangler::getNameWithPrefix(MangledNameStream, UnmangledName, DL);
 784   }
 785   return MangledName;
 786 }
 787
 788 Error LLJIT::applyDataLayout(Module &M) {
 789   if (M.getDataLayout().isDefault())
 790     M.setDataLayout(DL);
 791
 792   if (M.getDataLayout() != DL)
 793     return make_error<StringError>(
 794         "Added modules have incompatible data layouts: " +
 795             M.getDataLayout().getStringRepresentation() + " (module) vs " +
 796             DL.getStringRepresentation() + " (jit)",
 797         inconvertibleErrorCode());
 798
 799   return Error::success();
 800 }
 801
 802 void setUpGenericLLVMIRPlatform(LLJIT &J) {
 803   LLVM_DEBUG(
 804       { dbgs() << "Setting up GenericLLVMIRPlatform support for LLJIT\n"; });
 805   J.setPlatformSupport(std::make_unique<GenericLLVMIRPlatformSupport>(J));
 806 }
 807
 808 Error setUpInactivePlatform(LLJIT &J) {
 809   LLVM_DEBUG(
 810       { dbgs() << "Explicitly deactivated platform support for LLJIT\n"; });
 811   J.setPlatformSupport(std::make_unique<InactivePlatformSupport>());
 812   return Error::success();
 813 }
 814
 815 Error LLLazyJITBuilderState::prepareForConstruction() {
 816   if (auto Err = LLJITBuilderState::prepareForConstruction())
 817     return Err;
 818   TT = JTMB->getTargetTriple();
 819   return Error::success();
 820 }
 821
 822 Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) {
 823   assert(TSM && "Can not add null module");
 824
 825   if (auto Err = TSM.withModuleDo(
 826           [&](Module &M) -> Error { return applyDataLayout(M); }))
 827     return Err;
 828
 829   return CODLayer->add(JD, std::move(TSM));
 830 }
 831
 832 LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) {
 833
 834   // If LLJIT construction failed then bail out.
 835   if (Err)
 836     return;
 837
 838   ErrorAsOutParameter _(&Err);
 839
 840   /// Take/Create the lazy-compile callthrough manager.
 841   if (S.LCTMgr)
 842     LCTMgr = std::move(S.LCTMgr);
 843   else {
 844     if (auto LCTMgrOrErr = createLocalLazyCallThroughManager(
 845             S.TT, *ES, S.LazyCompileFailureAddr))
 846       LCTMgr = std::move(*LCTMgrOrErr);
 847     else {
 848       Err = LCTMgrOrErr.takeError();
 849       return;
 850     }
 851   }
 852
 853   // Take/Create the indirect stubs manager builder.
 854   auto ISMBuilder = std::move(S.ISMBuilder);
 855
 856   // If none was provided, try to build one.
 857   if (!ISMBuilder)
 858     ISMBuilder = createLocalIndirectStubsManagerBuilder(S.TT);
 859
 860   // No luck. Bail out.
 861   if (!ISMBuilder) {
 862     Err = make_error<StringError>("Could not construct "
 863                                   "IndirectStubsManagerBuilder for target " +
 864                                       S.TT.str(),
 865                                   inconvertibleErrorCode());
 866     return;
 867   }
 868
 869   // Create the COD layer.
 870   CODLayer = std::make_unique<CompileOnDemandLayer>(
 871       *ES, *InitHelperTransformLayer, *LCTMgr, std::move(ISMBuilder));
 872
 873   if (S.NumCompileThreads > 0)
 874     CODLayer->setCloneToNewContextOnEmit(true);
 875 }
 876
 877 } // End namespace orc.
 878 } // End namespace llvm.