llvm/lib/ExecutionEngine/Orc/LLJIT.cpp

   1 //===--------- LLJIT.cpp - An ORC-based JIT for compiling LLVM IR ---------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ExecutionEngine/Orc/LLJIT.h"
  10 #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h"
  11 #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h"
  12 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
  13 #include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
  14 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
  15 #include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h"
  16 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
  17 #include "llvm/ExecutionEngine/Orc/Shared/OrcError.h"
  18 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
  19 #include "llvm/IR/GlobalVariable.h"
  20 #include "llvm/IR/IRBuilder.h"
  21 #include "llvm/IR/Mangler.h"
  22 #include "llvm/IR/Module.h"
  23 #include "llvm/Support/DynamicLibrary.h"
  24
  25 #include <map>
  26
  27 #define DEBUG_TYPE "orc"
  28
  29 using namespace llvm;
  30 using namespace llvm::orc;
  31
  32 namespace {
  33
  34 /// Adds helper function decls and wrapper functions that call the helper with
  35 /// some additional prefix arguments.
  36 ///
  37 /// E.g. For wrapper "foo" with type i8(i8, i64), helper "bar", and prefix
  38 /// args i32 4 and i16 12345, this function will add:
  39 ///
  40 /// declare i8 @bar(i32, i16, i8, i64)
  41 ///
  42 /// define i8 @foo(i8, i64) {
  43 /// entry:
  44 ///   %2 = call i8 @bar(i32 4, i16 12345, i8 %0, i64 %1)
  45 ///   ret i8 %2
  46 /// }
  47 ///
  48 Function *addHelperAndWrapper(Module &M, StringRef WrapperName,
  49                               FunctionType *WrapperFnType,
  50                               GlobalValue::VisibilityTypes WrapperVisibility,
  51                               StringRef HelperName,
  52                               ArrayRef<Value *> HelperPrefixArgs) {
  53   std::vector<Type *> HelperArgTypes;
  54   for (auto *Arg : HelperPrefixArgs)
  55     HelperArgTypes.push_back(Arg->getType());
  56   for (auto *T : WrapperFnType->params())
  57     HelperArgTypes.push_back(T);
  58   auto *HelperFnType =
  59       FunctionType::get(WrapperFnType->getReturnType(), HelperArgTypes, false);
  60   auto *HelperFn = Function::Create(HelperFnType, GlobalValue::ExternalLinkage,
  61                                     HelperName, M);
  62
  63   auto *WrapperFn = Function::Create(
  64       WrapperFnType, GlobalValue::ExternalLinkage, WrapperName, M);
  65   WrapperFn->setVisibility(WrapperVisibility);
  66
  67   auto *EntryBlock = BasicBlock::Create(M.getContext(), "entry", WrapperFn);
  68   IRBuilder<> IB(EntryBlock);
  69
  70   std::vector<Value *> HelperArgs;
  71   for (auto *Arg : HelperPrefixArgs)
  72     HelperArgs.push_back(Arg);
  73   for (auto &Arg : WrapperFn->args())
  74     HelperArgs.push_back(&Arg);
  75   auto *HelperResult = IB.CreateCall(HelperFn, HelperArgs);
  76   if (HelperFn->getReturnType()->isVoidTy())
  77     IB.CreateRetVoid();
  78   else
  79     IB.CreateRet(HelperResult);
  80
  81   return WrapperFn;
  82 }
  83
  84 class GenericLLVMIRPlatformSupport;
  85
  86 /// orc::Platform component of Generic LLVM IR Platform support.
  87 /// Just forwards calls to the GenericLLVMIRPlatformSupport class below.
  88 class GenericLLVMIRPlatform : public Platform {
  89 public:
  90   GenericLLVMIRPlatform(GenericLLVMIRPlatformSupport &S) : S(S) {}
  91   Error setupJITDylib(JITDylib &JD) override;
  92   Error teardownJITDylib(JITDylib &JD) override;
  93   Error notifyAdding(ResourceTracker &RT,
  94                      const MaterializationUnit &MU) override;
  95   Error notifyRemoving(ResourceTracker &RT) override {
  96     // Noop -- Nothing to do (yet).
  97     return Error::success();
  98   }
  99
 100 private:
 101   GenericLLVMIRPlatformSupport &S;
 102 };
 103
 104 /// This transform parses llvm.global_ctors to produce a single initialization
 105 /// function for the module, records the function, then deletes
 106 /// llvm.global_ctors.
 107 class GlobalCtorDtorScraper {
 108 public:
 109   GlobalCtorDtorScraper(GenericLLVMIRPlatformSupport &PS,
 110                         StringRef InitFunctionPrefix,
 111                         StringRef DeInitFunctionPrefix)
 112       : PS(PS), InitFunctionPrefix(InitFunctionPrefix),
 113         DeInitFunctionPrefix(DeInitFunctionPrefix) {}
 114   Expected<ThreadSafeModule> operator()(ThreadSafeModule TSM,
 115                                         MaterializationResponsibility &R);
 116
 117 private:
 118   GenericLLVMIRPlatformSupport &PS;
 119   StringRef InitFunctionPrefix;
 120   StringRef DeInitFunctionPrefix;
 121 };
 122
 123 /// Generic IR Platform Support
 124 ///
 125 /// Scrapes llvm.global_ctors and llvm.global_dtors and replaces them with
 126 /// specially named 'init' and 'deinit'. Injects definitions / interposes for
 127 /// some runtime API, including __cxa_atexit, dlopen, and dlclose.
 128 class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport {
 129 public:
 130   GenericLLVMIRPlatformSupport(LLJIT &J)
 131       : J(J), InitFunctionPrefix(J.mangle("__orc_init_func.")),
 132         DeInitFunctionPrefix(J.mangle("__orc_deinit_func.")) {
 133
 134     getExecutionSession().setPlatform(
 135         std::make_unique<GenericLLVMIRPlatform>(*this));
 136
 137     setInitTransform(J, GlobalCtorDtorScraper(*this, InitFunctionPrefix,
 138                                               DeInitFunctionPrefix));
 139
 140     SymbolMap StdInterposes;
 141
 142     StdInterposes[J.mangleAndIntern("__lljit.platform_support_instance")] =
 143         JITEvaluatedSymbol(pointerToJITTargetAddress(this),
 144                            JITSymbolFlags::Exported);
 145     StdInterposes[J.mangleAndIntern("__lljit.cxa_atexit_helper")] =
 146         JITEvaluatedSymbol(pointerToJITTargetAddress(registerAtExitHelper),
 147                            JITSymbolFlags());
 148
 149     cantFail(
 150         J.getMainJITDylib().define(absoluteSymbols(std::move(StdInterposes))));
 151     cantFail(setupJITDylib(J.getMainJITDylib()));
 152     cantFail(J.addIRModule(J.getMainJITDylib(), createPlatformRuntimeModule()));
 153   }
 154
 155   ExecutionSession &getExecutionSession() { return J.getExecutionSession(); }
 156
 157   /// Adds a module that defines the __dso_handle global.
 158   Error setupJITDylib(JITDylib &JD) {
 159
 160     // Add per-jitdylib standard interposes.
 161     SymbolMap PerJDInterposes;
 162     PerJDInterposes[J.mangleAndIntern("__lljit.run_atexits_helper")] =
 163         JITEvaluatedSymbol(pointerToJITTargetAddress(runAtExitsHelper),
 164                            JITSymbolFlags());
 165     cantFail(JD.define(absoluteSymbols(std::move(PerJDInterposes))));
 166
 167     auto Ctx = std::make_unique<LLVMContext>();
 168     auto M = std::make_unique<Module>("__standard_lib", *Ctx);
 169     M->setDataLayout(J.getDataLayout());
 170
 171     auto *Int64Ty = Type::getInt64Ty(*Ctx);
 172     auto *DSOHandle = new GlobalVariable(
 173         *M, Int64Ty, true, GlobalValue::ExternalLinkage,
 174         ConstantInt::get(Int64Ty, reinterpret_cast<uintptr_t>(&JD)),
 175         "__dso_handle");
 176     DSOHandle->setVisibility(GlobalValue::DefaultVisibility);
 177     DSOHandle->setInitializer(
 178         ConstantInt::get(Int64Ty, pointerToJITTargetAddress(&JD)));
 179
 180     auto *GenericIRPlatformSupportTy =
 181         StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport");
 182
 183     auto *PlatformInstanceDecl = new GlobalVariable(
 184         *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
 185         nullptr, "__lljit.platform_support_instance");
 186
 187     auto *VoidTy = Type::getVoidTy(*Ctx);
 188     addHelperAndWrapper(
 189         *M, "__lljit_run_atexits", FunctionType::get(VoidTy, {}, false),
 190         GlobalValue::HiddenVisibility, "__lljit.run_atexits_helper",
 191         {PlatformInstanceDecl, DSOHandle});
 192
 193     return J.addIRModule(JD, ThreadSafeModule(std::move(M), std::move(Ctx)));
 194   }
 195
 196   Error notifyAdding(ResourceTracker &RT, const MaterializationUnit &MU) {
 197     auto &JD = RT.getJITDylib();
 198     if (auto &InitSym = MU.getInitializerSymbol())
 199       InitSymbols[&JD].add(InitSym, SymbolLookupFlags::WeaklyReferencedSymbol);
 200     else {
 201       // If there's no identified init symbol attached, but there is a symbol
 202       // with the GenericIRPlatform::InitFunctionPrefix, then treat that as
 203       // an init function. Add the symbol to both the InitSymbols map (which
 204       // will trigger a lookup to materialize the module) and the InitFunctions
 205       // map (which holds the names of the symbols to execute).
 206       for (auto &KV : MU.getSymbols())
 207         if ((*KV.first).startswith(InitFunctionPrefix)) {
 208           InitSymbols[&JD].add(KV.first,
 209                                SymbolLookupFlags::WeaklyReferencedSymbol);
 210           InitFunctions[&JD].add(KV.first);
 211         } else if ((*KV.first).startswith(DeInitFunctionPrefix)) {
 212           DeInitFunctions[&JD].add(KV.first);
 213         }
 214     }
 215     return Error::success();
 216   }
 217
 218   Error initialize(JITDylib &JD) override {
 219     LLVM_DEBUG({
 220       dbgs() << "GenericLLVMIRPlatformSupport getting initializers to run\n";
 221     });
 222     if (auto Initializers = getInitializers(JD)) {
 223       LLVM_DEBUG(
 224           { dbgs() << "GenericLLVMIRPlatformSupport running initializers\n"; });
 225       for (auto InitFnAddr : *Initializers) {
 226         LLVM_DEBUG({
 227           dbgs() << "  Running init " << formatv("{0:x16}", InitFnAddr)
 228                  << "...\n";
 229         });
 230         auto *InitFn = jitTargetAddressToFunction<void (*)()>(InitFnAddr);
 231         InitFn();
 232       }
 233     } else
 234       return Initializers.takeError();
 235     return Error::success();
 236   }
 237
 238   Error deinitialize(JITDylib &JD) override {
 239     LLVM_DEBUG({
 240       dbgs() << "GenericLLVMIRPlatformSupport getting deinitializers to run\n";
 241     });
 242     if (auto Deinitializers = getDeinitializers(JD)) {
 243       LLVM_DEBUG({
 244         dbgs() << "GenericLLVMIRPlatformSupport running deinitializers\n";
 245       });
 246       for (auto DeinitFnAddr : *Deinitializers) {
 247         LLVM_DEBUG({
 248           dbgs() << "  Running deinit " << formatv("{0:x16}", DeinitFnAddr)
 249                  << "...\n";
 250         });
 251         auto *DeinitFn = jitTargetAddressToFunction<void (*)()>(DeinitFnAddr);
 252         DeinitFn();
 253       }
 254     } else
 255       return Deinitializers.takeError();
 256
 257     return Error::success();
 258   }
 259
 260   void registerInitFunc(JITDylib &JD, SymbolStringPtr InitName) {
 261     getExecutionSession().runSessionLocked([&]() {
 262         InitFunctions[&JD].add(InitName);
 263       });
 264   }
 265
 266   void registerDeInitFunc(JITDylib &JD, SymbolStringPtr DeInitName) {
 267     getExecutionSession().runSessionLocked(
 268         [&]() { DeInitFunctions[&JD].add(DeInitName); });
 269   }
 270
 271 private:
 272
 273   Expected<std::vector<JITTargetAddress>> getInitializers(JITDylib &JD) {
 274     if (auto Err = issueInitLookups(JD))
 275       return std::move(Err);
 276
 277     DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
 278     std::vector<JITDylibSP> DFSLinkOrder;
 279
 280     getExecutionSession().runSessionLocked([&]() {
 281       DFSLinkOrder = JD.getDFSLinkOrder();
 282
 283       for (auto &NextJD : DFSLinkOrder) {
 284         auto IFItr = InitFunctions.find(NextJD.get());
 285         if (IFItr != InitFunctions.end()) {
 286           LookupSymbols[NextJD.get()] = std::move(IFItr->second);
 287           InitFunctions.erase(IFItr);
 288         }
 289       }
 290     });
 291
 292     LLVM_DEBUG({
 293       dbgs() << "JITDylib init order is [ ";
 294       for (auto &JD : llvm::reverse(DFSLinkOrder))
 295         dbgs() << "\"" << JD->getName() << "\" ";
 296       dbgs() << "]\n";
 297       dbgs() << "Looking up init functions:\n";
 298       for (auto &KV : LookupSymbols)
 299         dbgs() << "  \"" << KV.first->getName() << "\": " << KV.second << "\n";
 300     });
 301
 302     auto &ES = getExecutionSession();
 303     auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols);
 304
 305     if (!LookupResult)
 306       return LookupResult.takeError();
 307
 308     std::vector<JITTargetAddress> Initializers;
 309     while (!DFSLinkOrder.empty()) {
 310       auto &NextJD = *DFSLinkOrder.back();
 311       DFSLinkOrder.pop_back();
 312       auto InitsItr = LookupResult->find(&NextJD);
 313       if (InitsItr == LookupResult->end())
 314         continue;
 315       for (auto &KV : InitsItr->second)
 316         Initializers.push_back(KV.second.getAddress());
 317     }
 318
 319     return Initializers;
 320   }
 321
 322   Expected<std::vector<JITTargetAddress>> getDeinitializers(JITDylib &JD) {
 323     auto &ES = getExecutionSession();
 324
 325     auto LLJITRunAtExits = J.mangleAndIntern("__lljit_run_atexits");
 326
 327     DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
 328     std::vector<JITDylibSP> DFSLinkOrder;
 329
 330     ES.runSessionLocked([&]() {
 331       DFSLinkOrder = JD.getDFSLinkOrder();
 332
 333       for (auto &NextJD : DFSLinkOrder) {
 334         auto &JDLookupSymbols = LookupSymbols[NextJD.get()];
 335         auto DIFItr = DeInitFunctions.find(NextJD.get());
 336         if (DIFItr != DeInitFunctions.end()) {
 337           LookupSymbols[NextJD.get()] = std::move(DIFItr->second);
 338           DeInitFunctions.erase(DIFItr);
 339         }
 340         JDLookupSymbols.add(LLJITRunAtExits,
 341                             SymbolLookupFlags::WeaklyReferencedSymbol);
 342       }
 343     });
 344
 345     LLVM_DEBUG({
 346       dbgs() << "JITDylib deinit order is [ ";
 347       for (auto &JD : DFSLinkOrder)
 348         dbgs() << "\"" << JD->getName() << "\" ";
 349       dbgs() << "]\n";
 350       dbgs() << "Looking up deinit functions:\n";
 351       for (auto &KV : LookupSymbols)
 352         dbgs() << "  \"" << KV.first->getName() << "\": " << KV.second << "\n";
 353     });
 354
 355     auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols);
 356
 357     if (!LookupResult)
 358       return LookupResult.takeError();
 359
 360     std::vector<JITTargetAddress> DeInitializers;
 361     for (auto &NextJD : DFSLinkOrder) {
 362       auto DeInitsItr = LookupResult->find(NextJD.get());
 363       assert(DeInitsItr != LookupResult->end() &&
 364              "Every JD should have at least __lljit_run_atexits");
 365
 366       auto RunAtExitsItr = DeInitsItr->second.find(LLJITRunAtExits);
 367       if (RunAtExitsItr != DeInitsItr->second.end())
 368         DeInitializers.push_back(RunAtExitsItr->second.getAddress());
 369
 370       for (auto &KV : DeInitsItr->second)
 371         if (KV.first != LLJITRunAtExits)
 372           DeInitializers.push_back(KV.second.getAddress());
 373     }
 374
 375     return DeInitializers;
 376   }
 377
 378   /// Issue lookups for all init symbols required to initialize JD (and any
 379   /// JITDylibs that it depends on).
 380   Error issueInitLookups(JITDylib &JD) {
 381     DenseMap<JITDylib *, SymbolLookupSet> RequiredInitSymbols;
 382     std::vector<JITDylibSP> DFSLinkOrder;
 383
 384     getExecutionSession().runSessionLocked([&]() {
 385       DFSLinkOrder = JD.getDFSLinkOrder();
 386
 387       for (auto &NextJD : DFSLinkOrder) {
 388         auto ISItr = InitSymbols.find(NextJD.get());
 389         if (ISItr != InitSymbols.end()) {
 390           RequiredInitSymbols[NextJD.get()] = std::move(ISItr->second);
 391           InitSymbols.erase(ISItr);
 392         }
 393       }
 394     });
 395
 396     return Platform::lookupInitSymbols(getExecutionSession(),
 397                                        RequiredInitSymbols)
 398         .takeError();
 399   }
 400
 401   static void registerAtExitHelper(void *Self, void (*F)(void *), void *Ctx,
 402                                    void *DSOHandle) {
 403     LLVM_DEBUG({
 404       dbgs() << "Registering atexit function " << (void *)F << " for JD "
 405              << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
 406     });
 407     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit(
 408         F, Ctx, DSOHandle);
 409   }
 410
 411   static void runAtExitsHelper(void *Self, void *DSOHandle) {
 412     LLVM_DEBUG({
 413       dbgs() << "Running atexit functions for JD "
 414              << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
 415     });
 416     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.runAtExits(
 417         DSOHandle);
 418   }
 419
 420   // Constructs an LLVM IR module containing platform runtime globals,
 421   // functions, and interposes.
 422   ThreadSafeModule createPlatformRuntimeModule() {
 423     auto Ctx = std::make_unique<LLVMContext>();
 424     auto M = std::make_unique<Module>("__standard_lib", *Ctx);
 425     M->setDataLayout(J.getDataLayout());
 426
 427     auto *GenericIRPlatformSupportTy =
 428         StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport");
 429
 430     auto *PlatformInstanceDecl = new GlobalVariable(
 431         *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
 432         nullptr, "__lljit.platform_support_instance");
 433
 434     auto *Int8Ty = Type::getInt8Ty(*Ctx);
 435     auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT);
 436     auto *VoidTy = Type::getVoidTy(*Ctx);
 437     auto *BytePtrTy = PointerType::getUnqual(Int8Ty);
 438     auto *AtExitCallbackTy = FunctionType::get(VoidTy, {BytePtrTy}, false);
 439     auto *AtExitCallbackPtrTy = PointerType::getUnqual(AtExitCallbackTy);
 440
 441     addHelperAndWrapper(
 442         *M, "__cxa_atexit",
 443         FunctionType::get(IntTy, {AtExitCallbackPtrTy, BytePtrTy, BytePtrTy},
 444                           false),
 445         GlobalValue::DefaultVisibility, "__lljit.cxa_atexit_helper",
 446         {PlatformInstanceDecl});
 447
 448     return ThreadSafeModule(std::move(M), std::move(Ctx));
 449   }
 450
 451   LLJIT &J;
 452   std::string InitFunctionPrefix;
 453   std::string DeInitFunctionPrefix;
 454   DenseMap<JITDylib *, SymbolLookupSet> InitSymbols;
 455   DenseMap<JITDylib *, SymbolLookupSet> InitFunctions;
 456   DenseMap<JITDylib *, SymbolLookupSet> DeInitFunctions;
 457   ItaniumCXAAtExitSupport AtExitMgr;
 458 };
 459
 460 Error GenericLLVMIRPlatform::setupJITDylib(JITDylib &JD) {
 461   return S.setupJITDylib(JD);
 462 }
 463
 464 Error GenericLLVMIRPlatform::teardownJITDylib(JITDylib &JD) {
 465   return Error::success();
 466 }
 467
 468 Error GenericLLVMIRPlatform::notifyAdding(ResourceTracker &RT,
 469                                           const MaterializationUnit &MU) {
 470   return S.notifyAdding(RT, MU);
 471 }
 472
 473 Expected<ThreadSafeModule>
 474 GlobalCtorDtorScraper::operator()(ThreadSafeModule TSM,
 475                                   MaterializationResponsibility &R) {
 476   auto Err = TSM.withModuleDo([&](Module &M) -> Error {
 477     auto &Ctx = M.getContext();
 478     auto *GlobalCtors = M.getNamedGlobal("llvm.global_ctors");
 479     auto *GlobalDtors = M.getNamedGlobal("llvm.global_dtors");
 480
 481     auto RegisterCOrDtors = [&](GlobalVariable *GlobalCOrDtors,
 482                                 bool isCtor) -> Error {
 483       // If there's no llvm.global_c/dtor or it's just a decl then skip.
 484       if (!GlobalCOrDtors || GlobalCOrDtors->isDeclaration())
 485         return Error::success();
 486       std::string InitOrDeInitFunctionName;
 487       if (isCtor)
 488         raw_string_ostream(InitOrDeInitFunctionName)
 489             << InitFunctionPrefix << M.getModuleIdentifier();
 490       else
 491         raw_string_ostream(InitOrDeInitFunctionName)
 492             << DeInitFunctionPrefix << M.getModuleIdentifier();
 493
 494       MangleAndInterner Mangle(PS.getExecutionSession(), M.getDataLayout());
 495       auto InternedInitOrDeInitName = Mangle(InitOrDeInitFunctionName);
 496       if (auto Err = R.defineMaterializing(
 497               {{InternedInitOrDeInitName, JITSymbolFlags::Callable}}))
 498         return Err;
 499
 500       auto *InitOrDeInitFunc = Function::Create(
 501           FunctionType::get(Type::getVoidTy(Ctx), {}, false),
 502           GlobalValue::ExternalLinkage, InitOrDeInitFunctionName, &M);
 503       InitOrDeInitFunc->setVisibility(GlobalValue::HiddenVisibility);
 504       std::vector<std::pair<Function *, unsigned>> InitsOrDeInits;
 505       auto COrDtors = isCtor ? getConstructors(M) : getDestructors(M);
 506
 507       for (auto E : COrDtors)
 508         InitsOrDeInits.push_back(std::make_pair(E.Func, E.Priority));
 509       llvm::sort(InitsOrDeInits,
 510                  [](const std::pair<Function *, unsigned> &LHS,
 511                     const std::pair<Function *, unsigned> &RHS) {
 512                    return LHS.first < RHS.first;
 513                  });
 514
 515       auto *InitOrDeInitFuncEntryBlock =
 516           BasicBlock::Create(Ctx, "entry", InitOrDeInitFunc);
 517       IRBuilder<> IB(InitOrDeInitFuncEntryBlock);
 518       for (auto &KV : InitsOrDeInits)
 519         IB.CreateCall(KV.first);
 520       IB.CreateRetVoid();
 521
 522       if (isCtor)
 523         PS.registerInitFunc(R.getTargetJITDylib(), InternedInitOrDeInitName);
 524       else
 525         PS.registerDeInitFunc(R.getTargetJITDylib(), InternedInitOrDeInitName);
 526
 527       GlobalCOrDtors->eraseFromParent();
 528       return Error::success();
 529     };
 530
 531     if (auto Err = RegisterCOrDtors(GlobalCtors, true))
 532       return Err;
 533     if (auto Err = RegisterCOrDtors(GlobalDtors, false))
 534       return Err;
 535
 536     return Error::success();
 537   });
 538
 539   if (Err)
 540     return std::move(Err);
 541
 542   return std::move(TSM);
 543 }
 544
 545 /// Inactive Platform Support
 546 ///
 547 /// Explicitly disables platform support. JITDylibs are not scanned for special
 548 /// init/deinit symbols. No runtime API interposes are injected.
 549 class InactivePlatformSupport : public LLJIT::PlatformSupport {
 550 public:
 551   InactivePlatformSupport() = default;
 552
 553   Error initialize(JITDylib &JD) override {
 554     LLVM_DEBUG(dbgs() << "InactivePlatformSupport: no initializers running for "
 555                       << JD.getName() << "\n");
 556     return Error::success();
 557   }
 558
 559   Error deinitialize(JITDylib &JD) override {
 560     LLVM_DEBUG(
 561         dbgs() << "InactivePlatformSupport: no deinitializers running for "
 562                << JD.getName() << "\n");
 563     return Error::success();
 564   }
 565 };
 566
 567 } // end anonymous namespace
 568
 569 namespace llvm {
 570 namespace orc {
 571
 572 void LLJIT::PlatformSupport::setInitTransform(
 573     LLJIT &J, IRTransformLayer::TransformFunction T) {
 574   J.InitHelperTransformLayer->setTransform(std::move(T));
 575 }
 576
 577 LLJIT::PlatformSupport::~PlatformSupport() {}
 578
 579 Error LLJITBuilderState::prepareForConstruction() {
 580
 581   LLVM_DEBUG(dbgs() << "Preparing to create LLJIT instance...\n");
 582
 583   if (!JTMB) {
 584     LLVM_DEBUG({
 585       dbgs() << "  No explicitly set JITTargetMachineBuilder. "
 586                 "Detecting host...\n";
 587     });
 588     if (auto JTMBOrErr = JITTargetMachineBuilder::detectHost())
 589       JTMB = std::move(*JTMBOrErr);
 590     else
 591       return JTMBOrErr.takeError();
 592   }
 593
 594   LLVM_DEBUG({
 595     dbgs() << "  JITTargetMachineBuilder is "
 596            << JITTargetMachineBuilderPrinter(*JTMB, "  ")
 597            << "  Pre-constructed ExecutionSession: " << (ES ? "Yes" : "No")
 598            << "\n"
 599            << "  DataLayout: ";
 600     if (DL)
 601       dbgs() << DL->getStringRepresentation() << "\n";
 602     else
 603       dbgs() << "None (will be created by JITTargetMachineBuilder)\n";
 604
 605     dbgs() << "  Custom object-linking-layer creator: "
 606            << (CreateObjectLinkingLayer ? "Yes" : "No") << "\n"
 607            << "  Custom compile-function creator: "
 608            << (CreateCompileFunction ? "Yes" : "No") << "\n"
 609            << "  Custom platform-setup function: "
 610            << (SetUpPlatform ? "Yes" : "No") << "\n"
 611            << "  Number of compile threads: " << NumCompileThreads;
 612     if (!NumCompileThreads)
 613       dbgs() << " (code will be compiled on the execution thread)\n";
 614     else
 615       dbgs() << "\n";
 616   });
 617
 618   // If neither ES nor EPC has been set then create an EPC instance.
 619   if (!ES && !EPC) {
 620     LLVM_DEBUG({
 621       dbgs() << "ExecutorProcessControl not specified, "
 622                 "Creating SelfExecutorProcessControl instance\n";
 623     });
 624     if (auto EPCOrErr = SelfExecutorProcessControl::Create())
 625       EPC = std::move(*EPCOrErr);
 626     else
 627       return EPCOrErr.takeError();
 628   } else
 629     LLVM_DEBUG({
 630       dbgs() << "Using explicitly specified ExecutorProcessControl instance "
 631              << EPC.get() << "\n";
 632     });
 633
 634   // If the client didn't configure any linker options then auto-configure the
 635   // JIT linker.
 636   if (!CreateObjectLinkingLayer) {
 637     auto &TT = JTMB->getTargetTriple();
 638     if (TT.isOSBinFormatMachO() &&
 639         (TT.getArch() == Triple::aarch64 || TT.getArch() == Triple::x86_64)) {
 640
 641       JTMB->setRelocationModel(Reloc::PIC_);
 642       JTMB->setCodeModel(CodeModel::Small);
 643       CreateObjectLinkingLayer =
 644           [](ExecutionSession &ES,
 645              const Triple &) -> Expected<std::unique_ptr<ObjectLayer>> {
 646         auto ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>(ES);
 647         ObjLinkingLayer->addPlugin(std::make_unique<EHFrameRegistrationPlugin>(
 648             ES, std::make_unique<jitlink::InProcessEHFrameRegistrar>()));
 649         return std::move(ObjLinkingLayer);
 650       };
 651     }
 652   }
 653
 654   return Error::success();
 655 }
 656
 657 LLJIT::~LLJIT() {
 658   if (CompileThreads)
 659     CompileThreads->wait();
 660   if (auto Err = ES->endSession())
 661     ES->reportError(std::move(Err));
 662 }
 663
 664 Error LLJIT::addIRModule(ResourceTrackerSP RT, ThreadSafeModule TSM) {
 665   assert(TSM && "Can not add null module");
 666
 667   if (auto Err =
 668           TSM.withModuleDo([&](Module &M) { return applyDataLayout(M); }))
 669     return Err;
 670
 671   return InitHelperTransformLayer->add(std::move(RT), std::move(TSM));
 672 }
 673
 674 Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) {
 675   return addIRModule(JD.getDefaultResourceTracker(), std::move(TSM));
 676 }
 677
 678 Error LLJIT::addObjectFile(ResourceTrackerSP RT,
 679                            std::unique_ptr<MemoryBuffer> Obj) {
 680   assert(Obj && "Can not add null object");
 681
 682   return ObjTransformLayer->add(std::move(RT), std::move(Obj));
 683 }
 684
 685 Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) {
 686   return addObjectFile(JD.getDefaultResourceTracker(), std::move(Obj));
 687 }
 688
 689 Expected<JITEvaluatedSymbol> LLJIT::lookupLinkerMangled(JITDylib &JD,
 690                                                         SymbolStringPtr Name) {
 691   return ES->lookup(
 692       makeJITDylibSearchOrder(&JD, JITDylibLookupFlags::MatchAllSymbols), Name);
 693 }
 694
 695 Expected<std::unique_ptr<ObjectLayer>>
 696 LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) {
 697
 698   // If the config state provided an ObjectLinkingLayer factory then use it.
 699   if (S.CreateObjectLinkingLayer)
 700     return S.CreateObjectLinkingLayer(ES, S.JTMB->getTargetTriple());
 701
 702   // Otherwise default to creating an RTDyldObjectLinkingLayer that constructs
 703   // a new SectionMemoryManager for each object.
 704   auto GetMemMgr = []() { return std::make_unique<SectionMemoryManager>(); };
 705   auto Layer =
 706       std::make_unique<RTDyldObjectLinkingLayer>(ES, std::move(GetMemMgr));
 707
 708   if (S.JTMB->getTargetTriple().isOSBinFormatCOFF()) {
 709     Layer->setOverrideObjectFlagsWithResponsibilityFlags(true);
 710     Layer->setAutoClaimResponsibilityForObjectSymbols(true);
 711   }
 712
 713   // FIXME: Explicit conversion to std::unique_ptr<ObjectLayer> added to silence
 714   //        errors from some GCC / libstdc++ bots. Remove this conversion (i.e.
 715   //        just return ObjLinkingLayer) once those bots are upgraded.
 716   return std::unique_ptr<ObjectLayer>(std::move(Layer));
 717 }
 718
 719 Expected<std::unique_ptr<IRCompileLayer::IRCompiler>>
 720 LLJIT::createCompileFunction(LLJITBuilderState &S,
 721                              JITTargetMachineBuilder JTMB) {
 722
 723   /// If there is a custom compile function creator set then use it.
 724   if (S.CreateCompileFunction)
 725     return S.CreateCompileFunction(std::move(JTMB));
 726
 727   // Otherwise default to creating a SimpleCompiler, or ConcurrentIRCompiler,
 728   // depending on the number of threads requested.
 729   if (S.NumCompileThreads > 0)
 730     return std::make_unique<ConcurrentIRCompiler>(std::move(JTMB));
 731
 732   auto TM = JTMB.createTargetMachine();
 733   if (!TM)
 734     return TM.takeError();
 735
 736   return std::make_unique<TMOwningSimpleCompiler>(std::move(*TM));
 737 }
 738
 739 LLJIT::LLJIT(LLJITBuilderState &S, Error &Err)
 740     : DL(""), TT(S.JTMB->getTargetTriple()) {
 741
 742   ErrorAsOutParameter _(&Err);
 743
 744   assert(!(S.EPC && S.ES) && "EPC and ES should not both be set");
 745
 746   if (S.EPC) {
 747     ES = std::make_unique<ExecutionSession>(std::move(S.EPC));
 748   } else if (S.ES)
 749     ES = std::move(S.ES);
 750   else {
 751     if (auto EPC = SelfExecutorProcessControl::Create()) {
 752       ES = std::make_unique<ExecutionSession>(std::move(*EPC));
 753     } else {
 754       Err = EPC.takeError();
 755       return;
 756     }
 757   }
 758
 759   if (auto MainOrErr = this->ES->createJITDylib("main"))
 760     Main = &*MainOrErr;
 761   else {
 762     Err = MainOrErr.takeError();
 763     return;
 764   }
 765
 766   if (S.DL)
 767     DL = std::move(*S.DL);
 768   else if (auto DLOrErr = S.JTMB->getDefaultDataLayoutForTarget())
 769     DL = std::move(*DLOrErr);
 770   else {
 771     Err = DLOrErr.takeError();
 772     return;
 773   }
 774
 775   auto ObjLayer = createObjectLinkingLayer(S, *ES);
 776   if (!ObjLayer) {
 777     Err = ObjLayer.takeError();
 778     return;
 779   }
 780   ObjLinkingLayer = std::move(*ObjLayer);
 781   ObjTransformLayer =
 782       std::make_unique<ObjectTransformLayer>(*ES, *ObjLinkingLayer);
 783
 784   {
 785     auto CompileFunction = createCompileFunction(S, std::move(*S.JTMB));
 786     if (!CompileFunction) {
 787       Err = CompileFunction.takeError();
 788       return;
 789     }
 790     CompileLayer = std::make_unique<IRCompileLayer>(
 791         *ES, *ObjTransformLayer, std::move(*CompileFunction));
 792     TransformLayer = std::make_unique<IRTransformLayer>(*ES, *CompileLayer);
 793     InitHelperTransformLayer =
 794         std::make_unique<IRTransformLayer>(*ES, *TransformLayer);
 795   }
 796
 797   if (S.NumCompileThreads > 0) {
 798     InitHelperTransformLayer->setCloneToNewContextOnEmit(true);
 799     CompileThreads =
 800         std::make_unique<ThreadPool>(hardware_concurrency(S.NumCompileThreads));
 801     ES->setDispatchTask([this](std::unique_ptr<Task> T) {
 802       // FIXME: We should be able to use move-capture here, but ThreadPool's
 803       // AsyncTaskTys are std::functions rather than unique_functions
 804       // (because MSVC's std::packaged_tasks don't support move-only types).
 805       // Fix this when all the above gets sorted out.
 806       CompileThreads->async([UnownedT = T.release()]() mutable {
 807         std::unique_ptr<Task> T(UnownedT);
 808         T->run();
 809       });
 810     });
 811   }
 812
 813   if (S.SetUpPlatform)
 814     Err = S.SetUpPlatform(*this);
 815   else
 816     setUpGenericLLVMIRPlatform(*this);
 817 }
 818
 819 std::string LLJIT::mangle(StringRef UnmangledName) const {
 820   std::string MangledName;
 821   {
 822     raw_string_ostream MangledNameStream(MangledName);
 823     Mangler::getNameWithPrefix(MangledNameStream, UnmangledName, DL);
 824   }
 825   return MangledName;
 826 }
 827
 828 Error LLJIT::applyDataLayout(Module &M) {
 829   if (M.getDataLayout().isDefault())
 830     M.setDataLayout(DL);
 831
 832   if (M.getDataLayout() != DL)
 833     return make_error<StringError>(
 834         "Added modules have incompatible data layouts: " +
 835             M.getDataLayout().getStringRepresentation() + " (module) vs " +
 836             DL.getStringRepresentation() + " (jit)",
 837         inconvertibleErrorCode());
 838
 839   return Error::success();
 840 }
 841
 842 void setUpGenericLLVMIRPlatform(LLJIT &J) {
 843   LLVM_DEBUG(
 844       { dbgs() << "Setting up GenericLLVMIRPlatform support for LLJIT\n"; });
 845   J.setPlatformSupport(std::make_unique<GenericLLVMIRPlatformSupport>(J));
 846 }
 847
 848 Error setUpInactivePlatform(LLJIT &J) {
 849   LLVM_DEBUG(
 850       { dbgs() << "Explicitly deactivated platform support for LLJIT\n"; });
 851   J.setPlatformSupport(std::make_unique<InactivePlatformSupport>());
 852   return Error::success();
 853 }
 854
 855 Error LLLazyJITBuilderState::prepareForConstruction() {
 856   if (auto Err = LLJITBuilderState::prepareForConstruction())
 857     return Err;
 858   TT = JTMB->getTargetTriple();
 859   return Error::success();
 860 }
 861
 862 Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) {
 863   assert(TSM && "Can not add null module");
 864
 865   if (auto Err = TSM.withModuleDo(
 866           [&](Module &M) -> Error { return applyDataLayout(M); }))
 867     return Err;
 868
 869   return CODLayer->add(JD, std::move(TSM));
 870 }
 871
 872 LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) {
 873
 874   // If LLJIT construction failed then bail out.
 875   if (Err)
 876     return;
 877
 878   ErrorAsOutParameter _(&Err);
 879
 880   /// Take/Create the lazy-compile callthrough manager.
 881   if (S.LCTMgr)
 882     LCTMgr = std::move(S.LCTMgr);
 883   else {
 884     if (auto LCTMgrOrErr = createLocalLazyCallThroughManager(
 885             S.TT, *ES, S.LazyCompileFailureAddr))
 886       LCTMgr = std::move(*LCTMgrOrErr);
 887     else {
 888       Err = LCTMgrOrErr.takeError();
 889       return;
 890     }
 891   }
 892
 893   // Take/Create the indirect stubs manager builder.
 894   auto ISMBuilder = std::move(S.ISMBuilder);
 895
 896   // If none was provided, try to build one.
 897   if (!ISMBuilder)
 898     ISMBuilder = createLocalIndirectStubsManagerBuilder(S.TT);
 899
 900   // No luck. Bail out.
 901   if (!ISMBuilder) {
 902     Err = make_error<StringError>("Could not construct "
 903                                   "IndirectStubsManagerBuilder for target " +
 904                                       S.TT.str(),
 905                                   inconvertibleErrorCode());
 906     return;
 907   }
 908
 909   // Create the COD layer.
 910   CODLayer = std::make_unique<CompileOnDemandLayer>(
 911       *ES, *InitHelperTransformLayer, *LCTMgr, std::move(ISMBuilder));
 912
 913   if (S.NumCompileThreads > 0)
 914     CODLayer->setCloneToNewContextOnEmit(true);
 915 }
 916
 917 } // End namespace orc.
 918 } // End namespace llvm.