llvm/lib/ExecutionEngine/Orc/LLJIT.cpp

   1 //===--------- LLJIT.cpp - An ORC-based JIT for compiling LLVM IR ---------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ExecutionEngine/Orc/LLJIT.h"
  10
  11 #include "llvm/Analysis/TargetLibraryInfo.h"
  12 #include "llvm/Config/llvm-config.h" // for LLVM_ENABLE_THREADS
  13 #include "llvm/ExecutionEngine/Orc/COFFPlatform.h"
  14 #include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
  15 #include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
  16 #include "llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h"
  17 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
  18 #include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
  19 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
  20 #include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h"
  21 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
  22 #include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h"
  23 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
  24 #include "llvm/IR/GlobalVariable.h"
  25 #include "llvm/IR/IRBuilder.h"
  26 #include "llvm/IR/Mangler.h"
  27 #include "llvm/IR/Module.h"
  28 #include "llvm/Support/DynamicLibrary.h"
  29
  30 #define DEBUG_TYPE "orc"
  31
  32 using namespace llvm;
  33 using namespace llvm::orc;
  34
  35 namespace {
  36
  37 /// Adds helper function decls and wrapper functions that call the helper with
  38 /// some additional prefix arguments.
  39 ///
  40 /// E.g. For wrapper "foo" with type i8(i8, i64), helper "bar", and prefix
  41 /// args i32 4 and i16 12345, this function will add:
  42 ///
  43 /// declare i8 @bar(i32, i16, i8, i64)
  44 ///
  45 /// define i8 @foo(i8, i64) {
  46 /// entry:
  47 ///   %2 = call i8 @bar(i32 4, i16 12345, i8 %0, i64 %1)
  48 ///   ret i8 %2
  49 /// }
  50 ///
  51 Function *addHelperAndWrapper(Module &M, StringRef WrapperName,
  52                               FunctionType *WrapperFnType,
  53                               GlobalValue::VisibilityTypes WrapperVisibility,
  54                               StringRef HelperName,
  55                               ArrayRef<Value *> HelperPrefixArgs) {
  56   std::vector<Type *> HelperArgTypes;
  57   for (auto *Arg : HelperPrefixArgs)
  58     HelperArgTypes.push_back(Arg->getType());
  59   for (auto *T : WrapperFnType->params())
  60     HelperArgTypes.push_back(T);
  61   auto *HelperFnType =
  62       FunctionType::get(WrapperFnType->getReturnType(), HelperArgTypes, false);
  63   auto *HelperFn = Function::Create(HelperFnType, GlobalValue::ExternalLinkage,
  64                                     HelperName, M);
  65
  66   auto *WrapperFn = Function::Create(
  67       WrapperFnType, GlobalValue::ExternalLinkage, WrapperName, M);
  68   WrapperFn->setVisibility(WrapperVisibility);
  69
  70   auto *EntryBlock = BasicBlock::Create(M.getContext(), "entry", WrapperFn);
  71   IRBuilder<> IB(EntryBlock);
  72
  73   std::vector<Value *> HelperArgs;
  74   for (auto *Arg : HelperPrefixArgs)
  75     HelperArgs.push_back(Arg);
  76   for (auto &Arg : WrapperFn->args())
  77     HelperArgs.push_back(&Arg);
  78   auto *HelperResult = IB.CreateCall(HelperFn, HelperArgs);
  79   if (HelperFn->getReturnType()->isVoidTy())
  80     IB.CreateRetVoid();
  81   else
  82     IB.CreateRet(HelperResult);
  83
  84   return WrapperFn;
  85 }
  86
  87 class GenericLLVMIRPlatformSupport;
  88
  89 /// orc::Platform component of Generic LLVM IR Platform support.
  90 /// Just forwards calls to the GenericLLVMIRPlatformSupport class below.
  91 class GenericLLVMIRPlatform : public Platform {
  92 public:
  93   GenericLLVMIRPlatform(GenericLLVMIRPlatformSupport &S) : S(S) {}
  94   Error setupJITDylib(JITDylib &JD) override;
  95   Error teardownJITDylib(JITDylib &JD) override;
  96   Error notifyAdding(ResourceTracker &RT,
  97                      const MaterializationUnit &MU) override;
  98   Error notifyRemoving(ResourceTracker &RT) override {
  99     // Noop -- Nothing to do (yet).
 100     return Error::success();
 101   }
 102
 103 private:
 104   GenericLLVMIRPlatformSupport &S;
 105 };
 106
 107 /// This transform parses llvm.global_ctors to produce a single initialization
 108 /// function for the module, records the function, then deletes
 109 /// llvm.global_ctors.
 110 class GlobalCtorDtorScraper {
 111 public:
 112   GlobalCtorDtorScraper(GenericLLVMIRPlatformSupport &PS,
 113                         StringRef InitFunctionPrefix,
 114                         StringRef DeInitFunctionPrefix)
 115       : PS(PS), InitFunctionPrefix(InitFunctionPrefix),
 116         DeInitFunctionPrefix(DeInitFunctionPrefix) {}
 117   Expected<ThreadSafeModule> operator()(ThreadSafeModule TSM,
 118                                         MaterializationResponsibility &R);
 119
 120 private:
 121   GenericLLVMIRPlatformSupport &PS;
 122   StringRef InitFunctionPrefix;
 123   StringRef DeInitFunctionPrefix;
 124 };
 125
 126 /// Generic IR Platform Support
 127 ///
 128 /// Scrapes llvm.global_ctors and llvm.global_dtors and replaces them with
 129 /// specially named 'init' and 'deinit'. Injects definitions / interposes for
 130 /// some runtime API, including __cxa_atexit, dlopen, and dlclose.
 131 class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport {
 132 public:
 133   GenericLLVMIRPlatformSupport(LLJIT &J, JITDylib &PlatformJD)
 134       : J(J), InitFunctionPrefix(J.mangle("__orc_init_func.")),
 135         DeInitFunctionPrefix(J.mangle("__orc_deinit_func.")) {
 136
 137     getExecutionSession().setPlatform(
 138         std::make_unique<GenericLLVMIRPlatform>(*this));
 139
 140     setInitTransform(J, GlobalCtorDtorScraper(*this, InitFunctionPrefix,
 141                                               DeInitFunctionPrefix));
 142
 143     SymbolMap StdInterposes;
 144
 145     StdInterposes[J.mangleAndIntern("__lljit.platform_support_instance")] = {
 146         ExecutorAddr::fromPtr(this), JITSymbolFlags::Exported};
 147     StdInterposes[J.mangleAndIntern("__lljit.cxa_atexit_helper")] = {
 148         ExecutorAddr::fromPtr(registerCxaAtExitHelper), JITSymbolFlags()};
 149
 150     cantFail(PlatformJD.define(absoluteSymbols(std::move(StdInterposes))));
 151     cantFail(setupJITDylib(PlatformJD));
 152     cantFail(J.addIRModule(PlatformJD, createPlatformRuntimeModule()));
 153   }
 154
 155   ExecutionSession &getExecutionSession() { return J.getExecutionSession(); }
 156
 157   /// Adds a module that defines the __dso_handle global.
 158   Error setupJITDylib(JITDylib &JD) {
 159
 160     // Add per-jitdylib standard interposes.
 161     SymbolMap PerJDInterposes;
 162     PerJDInterposes[J.mangleAndIntern("__lljit.run_atexits_helper")] = {
 163         ExecutorAddr::fromPtr(runAtExitsHelper), JITSymbolFlags()};
 164     PerJDInterposes[J.mangleAndIntern("__lljit.atexit_helper")] = {
 165         ExecutorAddr::fromPtr(registerAtExitHelper), JITSymbolFlags()};
 166     cantFail(JD.define(absoluteSymbols(std::move(PerJDInterposes))));
 167
 168     auto Ctx = std::make_unique<LLVMContext>();
 169     auto M = std::make_unique<Module>("__standard_lib", *Ctx);
 170     M->setDataLayout(J.getDataLayout());
 171
 172     auto *Int64Ty = Type::getInt64Ty(*Ctx);
 173     auto *DSOHandle = new GlobalVariable(
 174         *M, Int64Ty, true, GlobalValue::ExternalLinkage,
 175         ConstantInt::get(Int64Ty, reinterpret_cast<uintptr_t>(&JD)),
 176         "__dso_handle");
 177     DSOHandle->setVisibility(GlobalValue::DefaultVisibility);
 178     DSOHandle->setInitializer(
 179         ConstantInt::get(Int64Ty, ExecutorAddr::fromPtr(&JD).getValue()));
 180
 181     auto *GenericIRPlatformSupportTy =
 182         StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport");
 183
 184     auto *PlatformInstanceDecl = new GlobalVariable(
 185         *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
 186         nullptr, "__lljit.platform_support_instance");
 187
 188     auto *VoidTy = Type::getVoidTy(*Ctx);
 189     addHelperAndWrapper(
 190         *M, "__lljit_run_atexits", FunctionType::get(VoidTy, {}, false),
 191         GlobalValue::HiddenVisibility, "__lljit.run_atexits_helper",
 192         {PlatformInstanceDecl, DSOHandle});
 193
 194     auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT);
 195     auto *AtExitCallbackTy = FunctionType::get(VoidTy, {}, false);
 196     auto *AtExitCallbackPtrTy = PointerType::getUnqual(AtExitCallbackTy);
 197     auto *AtExit = addHelperAndWrapper(
 198         *M, "atexit", FunctionType::get(IntTy, {AtExitCallbackPtrTy}, false),
 199         GlobalValue::HiddenVisibility, "__lljit.atexit_helper",
 200         {PlatformInstanceDecl, DSOHandle});
 201     Attribute::AttrKind AtExitExtAttr =
 202         TargetLibraryInfo::getExtAttrForI32Return(J.getTargetTriple());
 203     if (AtExitExtAttr != Attribute::None)
 204       AtExit->addRetAttr(AtExitExtAttr);
 205
 206     return J.addIRModule(JD, ThreadSafeModule(std::move(M), std::move(Ctx)));
 207   }
 208
 209   Error notifyAdding(ResourceTracker &RT, const MaterializationUnit &MU) {
 210     auto &JD = RT.getJITDylib();
 211     if (auto &InitSym = MU.getInitializerSymbol())
 212       InitSymbols[&JD].add(InitSym, SymbolLookupFlags::WeaklyReferencedSymbol);
 213     else {
 214       // If there's no identified init symbol attached, but there is a symbol
 215       // with the GenericIRPlatform::InitFunctionPrefix, then treat that as
 216       // an init function. Add the symbol to both the InitSymbols map (which
 217       // will trigger a lookup to materialize the module) and the InitFunctions
 218       // map (which holds the names of the symbols to execute).
 219       for (auto &KV : MU.getSymbols())
 220         if ((*KV.first).starts_with(InitFunctionPrefix)) {
 221           InitSymbols[&JD].add(KV.first,
 222                                SymbolLookupFlags::WeaklyReferencedSymbol);
 223           InitFunctions[&JD].add(KV.first);
 224         } else if ((*KV.first).starts_with(DeInitFunctionPrefix)) {
 225           DeInitFunctions[&JD].add(KV.first);
 226         }
 227     }
 228     return Error::success();
 229   }
 230
 231   Error initialize(JITDylib &JD) override {
 232     LLVM_DEBUG({
 233       dbgs() << "GenericLLVMIRPlatformSupport getting initializers to run\n";
 234     });
 235     if (auto Initializers = getInitializers(JD)) {
 236       LLVM_DEBUG(
 237           { dbgs() << "GenericLLVMIRPlatformSupport running initializers\n"; });
 238       for (auto InitFnAddr : *Initializers) {
 239         LLVM_DEBUG({
 240           dbgs() << "  Running init " << formatv("{0:x16}", InitFnAddr)
 241                  << "...\n";
 242         });
 243         auto *InitFn = InitFnAddr.toPtr<void (*)()>();
 244         InitFn();
 245       }
 246     } else
 247       return Initializers.takeError();
 248     return Error::success();
 249   }
 250
 251   Error deinitialize(JITDylib &JD) override {
 252     LLVM_DEBUG({
 253       dbgs() << "GenericLLVMIRPlatformSupport getting deinitializers to run\n";
 254     });
 255     if (auto Deinitializers = getDeinitializers(JD)) {
 256       LLVM_DEBUG({
 257         dbgs() << "GenericLLVMIRPlatformSupport running deinitializers\n";
 258       });
 259       for (auto DeinitFnAddr : *Deinitializers) {
 260         LLVM_DEBUG({
 261           dbgs() << "  Running deinit " << formatv("{0:x16}", DeinitFnAddr)
 262                  << "...\n";
 263         });
 264         auto *DeinitFn = DeinitFnAddr.toPtr<void (*)()>();
 265         DeinitFn();
 266       }
 267     } else
 268       return Deinitializers.takeError();
 269
 270     return Error::success();
 271   }
 272
 273   void registerInitFunc(JITDylib &JD, SymbolStringPtr InitName) {
 274     getExecutionSession().runSessionLocked([&]() {
 275         InitFunctions[&JD].add(InitName);
 276       });
 277   }
 278
 279   void registerDeInitFunc(JITDylib &JD, SymbolStringPtr DeInitName) {
 280     getExecutionSession().runSessionLocked(
 281         [&]() { DeInitFunctions[&JD].add(DeInitName); });
 282   }
 283
 284 private:
 285   Expected<std::vector<ExecutorAddr>> getInitializers(JITDylib &JD) {
 286     if (auto Err = issueInitLookups(JD))
 287       return std::move(Err);
 288
 289     DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
 290     std::vector<JITDylibSP> DFSLinkOrder;
 291
 292     if (auto Err = getExecutionSession().runSessionLocked([&]() -> Error {
 293           if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
 294             DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
 295           else
 296             return DFSLinkOrderOrErr.takeError();
 297
 298           for (auto &NextJD : DFSLinkOrder) {
 299             auto IFItr = InitFunctions.find(NextJD.get());
 300             if (IFItr != InitFunctions.end()) {
 301               LookupSymbols[NextJD.get()] = std::move(IFItr->second);
 302               InitFunctions.erase(IFItr);
 303             }
 304           }
 305           return Error::success();
 306         }))
 307       return std::move(Err);
 308
 309     LLVM_DEBUG({
 310       dbgs() << "JITDylib init order is [ ";
 311       for (auto &JD : llvm::reverse(DFSLinkOrder))
 312         dbgs() << "\"" << JD->getName() << "\" ";
 313       dbgs() << "]\n";
 314       dbgs() << "Looking up init functions:\n";
 315       for (auto &KV : LookupSymbols)
 316         dbgs() << "  \"" << KV.first->getName() << "\": " << KV.second << "\n";
 317     });
 318
 319     auto &ES = getExecutionSession();
 320     auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols);
 321
 322     if (!LookupResult)
 323       return LookupResult.takeError();
 324
 325     std::vector<ExecutorAddr> Initializers;
 326     while (!DFSLinkOrder.empty()) {
 327       auto &NextJD = *DFSLinkOrder.back();
 328       DFSLinkOrder.pop_back();
 329       auto InitsItr = LookupResult->find(&NextJD);
 330       if (InitsItr == LookupResult->end())
 331         continue;
 332       for (auto &KV : InitsItr->second)
 333         Initializers.push_back(KV.second.getAddress());
 334     }
 335
 336     return Initializers;
 337   }
 338
 339   Expected<std::vector<ExecutorAddr>> getDeinitializers(JITDylib &JD) {
 340     auto &ES = getExecutionSession();
 341
 342     auto LLJITRunAtExits = J.mangleAndIntern("__lljit_run_atexits");
 343
 344     DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
 345     std::vector<JITDylibSP> DFSLinkOrder;
 346
 347     if (auto Err = ES.runSessionLocked([&]() -> Error {
 348           if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
 349             DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
 350           else
 351             return DFSLinkOrderOrErr.takeError();
 352
 353           for (auto &NextJD : DFSLinkOrder) {
 354             auto &JDLookupSymbols = LookupSymbols[NextJD.get()];
 355             auto DIFItr = DeInitFunctions.find(NextJD.get());
 356             if (DIFItr != DeInitFunctions.end()) {
 357               LookupSymbols[NextJD.get()] = std::move(DIFItr->second);
 358               DeInitFunctions.erase(DIFItr);
 359             }
 360             JDLookupSymbols.add(LLJITRunAtExits,
 361                                 SymbolLookupFlags::WeaklyReferencedSymbol);
 362           }
 363           return Error::success();
 364         }))
 365       return std::move(Err);
 366
 367     LLVM_DEBUG({
 368       dbgs() << "JITDylib deinit order is [ ";
 369       for (auto &JD : DFSLinkOrder)
 370         dbgs() << "\"" << JD->getName() << "\" ";
 371       dbgs() << "]\n";
 372       dbgs() << "Looking up deinit functions:\n";
 373       for (auto &KV : LookupSymbols)
 374         dbgs() << "  \"" << KV.first->getName() << "\": " << KV.second << "\n";
 375     });
 376
 377     auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols);
 378
 379     if (!LookupResult)
 380       return LookupResult.takeError();
 381
 382     std::vector<ExecutorAddr> DeInitializers;
 383     for (auto &NextJD : DFSLinkOrder) {
 384       auto DeInitsItr = LookupResult->find(NextJD.get());
 385       assert(DeInitsItr != LookupResult->end() &&
 386              "Every JD should have at least __lljit_run_atexits");
 387
 388       auto RunAtExitsItr = DeInitsItr->second.find(LLJITRunAtExits);
 389       if (RunAtExitsItr != DeInitsItr->second.end())
 390         DeInitializers.push_back(RunAtExitsItr->second.getAddress());
 391
 392       for (auto &KV : DeInitsItr->second)
 393         if (KV.first != LLJITRunAtExits)
 394           DeInitializers.push_back(KV.second.getAddress());
 395     }
 396
 397     return DeInitializers;
 398   }
 399
 400   /// Issue lookups for all init symbols required to initialize JD (and any
 401   /// JITDylibs that it depends on).
 402   Error issueInitLookups(JITDylib &JD) {
 403     DenseMap<JITDylib *, SymbolLookupSet> RequiredInitSymbols;
 404     std::vector<JITDylibSP> DFSLinkOrder;
 405
 406     if (auto Err = getExecutionSession().runSessionLocked([&]() -> Error {
 407           if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
 408             DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
 409           else
 410             return DFSLinkOrderOrErr.takeError();
 411
 412           for (auto &NextJD : DFSLinkOrder) {
 413             auto ISItr = InitSymbols.find(NextJD.get());
 414             if (ISItr != InitSymbols.end()) {
 415               RequiredInitSymbols[NextJD.get()] = std::move(ISItr->second);
 416               InitSymbols.erase(ISItr);
 417             }
 418           }
 419           return Error::success();
 420         }))
 421       return Err;
 422
 423     return Platform::lookupInitSymbols(getExecutionSession(),
 424                                        RequiredInitSymbols)
 425         .takeError();
 426   }
 427
 428   static void registerCxaAtExitHelper(void *Self, void (*F)(void *), void *Ctx,
 429                                       void *DSOHandle) {
 430     LLVM_DEBUG({
 431       dbgs() << "Registering cxa atexit function " << (void *)F << " for JD "
 432              << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
 433     });
 434     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit(
 435         F, Ctx, DSOHandle);
 436   }
 437
 438   static void registerAtExitHelper(void *Self, void *DSOHandle, void (*F)()) {
 439     LLVM_DEBUG({
 440       dbgs() << "Registering atexit function " << (void *)F << " for JD "
 441              << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
 442     });
 443     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit(
 444         reinterpret_cast<void (*)(void *)>(F), nullptr, DSOHandle);
 445   }
 446
 447   static void runAtExitsHelper(void *Self, void *DSOHandle) {
 448     LLVM_DEBUG({
 449       dbgs() << "Running atexit functions for JD "
 450              << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
 451     });
 452     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.runAtExits(
 453         DSOHandle);
 454   }
 455
 456   // Constructs an LLVM IR module containing platform runtime globals,
 457   // functions, and interposes.
 458   ThreadSafeModule createPlatformRuntimeModule() {
 459     auto Ctx = std::make_unique<LLVMContext>();
 460     auto M = std::make_unique<Module>("__standard_lib", *Ctx);
 461     M->setDataLayout(J.getDataLayout());
 462
 463     auto *GenericIRPlatformSupportTy =
 464         StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport");
 465
 466     auto *PlatformInstanceDecl = new GlobalVariable(
 467         *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
 468         nullptr, "__lljit.platform_support_instance");
 469
 470     auto *Int8Ty = Type::getInt8Ty(*Ctx);
 471     auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT);
 472     auto *VoidTy = Type::getVoidTy(*Ctx);
 473     auto *BytePtrTy = PointerType::getUnqual(Int8Ty);
 474     auto *CxaAtExitCallbackTy = FunctionType::get(VoidTy, {BytePtrTy}, false);
 475     auto *CxaAtExitCallbackPtrTy = PointerType::getUnqual(CxaAtExitCallbackTy);
 476
 477     auto *CxaAtExit = addHelperAndWrapper(
 478         *M, "__cxa_atexit",
 479         FunctionType::get(IntTy, {CxaAtExitCallbackPtrTy, BytePtrTy, BytePtrTy},
 480                           false),
 481         GlobalValue::DefaultVisibility, "__lljit.cxa_atexit_helper",
 482         {PlatformInstanceDecl});
 483     Attribute::AttrKind CxaAtExitExtAttr =
 484         TargetLibraryInfo::getExtAttrForI32Return(J.getTargetTriple());
 485     if (CxaAtExitExtAttr != Attribute::None)
 486       CxaAtExit->addRetAttr(CxaAtExitExtAttr);
 487
 488     return ThreadSafeModule(std::move(M), std::move(Ctx));
 489   }
 490
 491   LLJIT &J;
 492   std::string InitFunctionPrefix;
 493   std::string DeInitFunctionPrefix;
 494   DenseMap<JITDylib *, SymbolLookupSet> InitSymbols;
 495   DenseMap<JITDylib *, SymbolLookupSet> InitFunctions;
 496   DenseMap<JITDylib *, SymbolLookupSet> DeInitFunctions;
 497   ItaniumCXAAtExitSupport AtExitMgr;
 498 };
 499
 500 Error GenericLLVMIRPlatform::setupJITDylib(JITDylib &JD) {
 501   return S.setupJITDylib(JD);
 502 }
 503
 504 Error GenericLLVMIRPlatform::teardownJITDylib(JITDylib &JD) {
 505   return Error::success();
 506 }
 507
 508 Error GenericLLVMIRPlatform::notifyAdding(ResourceTracker &RT,
 509                                           const MaterializationUnit &MU) {
 510   return S.notifyAdding(RT, MU);
 511 }
 512
 513 Expected<ThreadSafeModule>
 514 GlobalCtorDtorScraper::operator()(ThreadSafeModule TSM,
 515                                   MaterializationResponsibility &R) {
 516   auto Err = TSM.withModuleDo([&](Module &M) -> Error {
 517     auto &Ctx = M.getContext();
 518     auto *GlobalCtors = M.getNamedGlobal("llvm.global_ctors");
 519     auto *GlobalDtors = M.getNamedGlobal("llvm.global_dtors");
 520
 521     auto RegisterCOrDtors = [&](GlobalVariable *GlobalCOrDtors,
 522                                 bool isCtor) -> Error {
 523       // If there's no llvm.global_c/dtor or it's just a decl then skip.
 524       if (!GlobalCOrDtors || GlobalCOrDtors->isDeclaration())
 525         return Error::success();
 526       std::string InitOrDeInitFunctionName;
 527       if (isCtor)
 528         raw_string_ostream(InitOrDeInitFunctionName)
 529             << InitFunctionPrefix << M.getModuleIdentifier();
 530       else
 531         raw_string_ostream(InitOrDeInitFunctionName)
 532             << DeInitFunctionPrefix << M.getModuleIdentifier();
 533
 534       MangleAndInterner Mangle(PS.getExecutionSession(), M.getDataLayout());
 535       auto InternedInitOrDeInitName = Mangle(InitOrDeInitFunctionName);
 536       if (auto Err = R.defineMaterializing(
 537               {{InternedInitOrDeInitName, JITSymbolFlags::Callable}}))
 538         return Err;
 539
 540       auto *InitOrDeInitFunc = Function::Create(
 541           FunctionType::get(Type::getVoidTy(Ctx), {}, false),
 542           GlobalValue::ExternalLinkage, InitOrDeInitFunctionName, &M);
 543       InitOrDeInitFunc->setVisibility(GlobalValue::HiddenVisibility);
 544       std::vector<std::pair<Function *, unsigned>> InitsOrDeInits;
 545       auto COrDtors = isCtor ? getConstructors(M) : getDestructors(M);
 546
 547       for (auto E : COrDtors)
 548         InitsOrDeInits.push_back(std::make_pair(E.Func, E.Priority));
 549       llvm::stable_sort(InitsOrDeInits, llvm::less_second());
 550
 551       auto *InitOrDeInitFuncEntryBlock =
 552           BasicBlock::Create(Ctx, "entry", InitOrDeInitFunc);
 553       IRBuilder<> IB(InitOrDeInitFuncEntryBlock);
 554       for (auto &KV : InitsOrDeInits)
 555         IB.CreateCall(KV.first);
 556       IB.CreateRetVoid();
 557
 558       if (isCtor)
 559         PS.registerInitFunc(R.getTargetJITDylib(), InternedInitOrDeInitName);
 560       else
 561         PS.registerDeInitFunc(R.getTargetJITDylib(), InternedInitOrDeInitName);
 562
 563       GlobalCOrDtors->eraseFromParent();
 564       return Error::success();
 565     };
 566
 567     if (auto Err = RegisterCOrDtors(GlobalCtors, true))
 568       return Err;
 569     if (auto Err = RegisterCOrDtors(GlobalDtors, false))
 570       return Err;
 571
 572     return Error::success();
 573   });
 574
 575   if (Err)
 576     return std::move(Err);
 577
 578   return std::move(TSM);
 579 }
 580
 581 /// Inactive Platform Support
 582 ///
 583 /// Explicitly disables platform support. JITDylibs are not scanned for special
 584 /// init/deinit symbols. No runtime API interposes are injected.
 585 class InactivePlatformSupport : public LLJIT::PlatformSupport {
 586 public:
 587   InactivePlatformSupport() = default;
 588
 589   Error initialize(JITDylib &JD) override {
 590     LLVM_DEBUG(dbgs() << "InactivePlatformSupport: no initializers running for "
 591                       << JD.getName() << "\n");
 592     return Error::success();
 593   }
 594
 595   Error deinitialize(JITDylib &JD) override {
 596     LLVM_DEBUG(
 597         dbgs() << "InactivePlatformSupport: no deinitializers running for "
 598                << JD.getName() << "\n");
 599     return Error::success();
 600   }
 601 };
 602
 603 } // end anonymous namespace
 604
 605 namespace llvm {
 606 namespace orc {
 607
 608 Error ORCPlatformSupport::initialize(orc::JITDylib &JD) {
 609   using llvm::orc::shared::SPSExecutorAddr;
 610   using llvm::orc::shared::SPSString;
 611   using SPSDLOpenSig = SPSExecutorAddr(SPSString, int32_t);
 612   using SPSDLUpdateSig = int32_t(SPSExecutorAddr);
 613   enum dlopen_mode : int32_t {
 614     ORC_RT_RTLD_LAZY = 0x1,
 615     ORC_RT_RTLD_NOW = 0x2,
 616     ORC_RT_RTLD_LOCAL = 0x4,
 617     ORC_RT_RTLD_GLOBAL = 0x8
 618   };
 619
 620   auto &ES = J.getExecutionSession();
 621   auto MainSearchOrder = J.getMainJITDylib().withLinkOrderDo(
 622       [](const JITDylibSearchOrder &SO) { return SO; });
 623   StringRef WrapperToCall = "__orc_rt_jit_dlopen_wrapper";
 624   bool dlupdate = false;
 625   const Triple &TT = ES.getTargetTriple();
 626   if (TT.isOSBinFormatMachO() || TT.isOSBinFormatELF()) {
 627     if (InitializedDylib.contains(&JD)) {
 628       WrapperToCall = "__orc_rt_jit_dlupdate_wrapper";
 629       dlupdate = true;
 630     } else
 631       InitializedDylib.insert(&JD);
 632   }
 633
 634   if (auto WrapperAddr =
 635           ES.lookup(MainSearchOrder, J.mangleAndIntern(WrapperToCall))) {
 636     if (dlupdate) {
 637       int32_t result;
 638       auto E = ES.callSPSWrapper<SPSDLUpdateSig>(WrapperAddr->getAddress(),
 639                                                  result, DSOHandles[&JD]);
 640       if (result)
 641         return make_error<StringError>("dlupdate failed",
 642                                        inconvertibleErrorCode());
 643       return E;
 644     }
 645     return ES.callSPSWrapper<SPSDLOpenSig>(WrapperAddr->getAddress(),
 646                                            DSOHandles[&JD], JD.getName(),
 647                                            int32_t(ORC_RT_RTLD_LAZY));
 648   } else
 649     return WrapperAddr.takeError();
 650 }
 651
 652 Error ORCPlatformSupport::deinitialize(orc::JITDylib &JD) {
 653   using llvm::orc::shared::SPSExecutorAddr;
 654   using SPSDLCloseSig = int32_t(SPSExecutorAddr);
 655
 656   auto &ES = J.getExecutionSession();
 657   auto MainSearchOrder = J.getMainJITDylib().withLinkOrderDo(
 658       [](const JITDylibSearchOrder &SO) { return SO; });
 659
 660   if (auto WrapperAddr = ES.lookup(
 661           MainSearchOrder, J.mangleAndIntern("__orc_rt_jit_dlclose_wrapper"))) {
 662     int32_t result;
 663     auto E = J.getExecutionSession().callSPSWrapper<SPSDLCloseSig>(
 664         WrapperAddr->getAddress(), result, DSOHandles[&JD]);
 665     if (E)
 666       return E;
 667     else if (result)
 668       return make_error<StringError>("dlclose failed",
 669                                      inconvertibleErrorCode());
 670     DSOHandles.erase(&JD);
 671     InitializedDylib.erase(&JD);
 672   } else
 673     return WrapperAddr.takeError();
 674   return Error::success();
 675 }
 676
 677 void LLJIT::PlatformSupport::setInitTransform(
 678     LLJIT &J, IRTransformLayer::TransformFunction T) {
 679   J.InitHelperTransformLayer->setTransform(std::move(T));
 680 }
 681
 682 LLJIT::PlatformSupport::~PlatformSupport() = default;
 683
 684 Error LLJITBuilderState::prepareForConstruction() {
 685
 686   LLVM_DEBUG(dbgs() << "Preparing to create LLJIT instance...\n");
 687
 688   if (!JTMB) {
 689     LLVM_DEBUG({
 690       dbgs() << "  No explicitly set JITTargetMachineBuilder. "
 691                 "Detecting host...\n";
 692     });
 693     if (auto JTMBOrErr = JITTargetMachineBuilder::detectHost())
 694       JTMB = std::move(*JTMBOrErr);
 695     else
 696       return JTMBOrErr.takeError();
 697   }
 698
 699   if ((ES || EPC) && NumCompileThreads)
 700     return make_error<StringError>(
 701         "NumCompileThreads cannot be used with a custom ExecutionSession or "
 702         "ExecutorProcessControl",
 703         inconvertibleErrorCode());
 704
 705 #if !LLVM_ENABLE_THREADS
 706   if (NumCompileThreads)
 707     return make_error<StringError>(
 708         "LLJIT num-compile-threads is " + Twine(NumCompileThreads) +
 709             " but LLVM was compiled with LLVM_ENABLE_THREADS=Off",
 710         inconvertibleErrorCode());
 711 #endif // !LLVM_ENABLE_THREADS
 712
 713   // Only used in debug builds.
 714   [[maybe_unused]] bool ConcurrentCompilationSettingDefaulted =
 715       !SupportConcurrentCompilation;
 716
 717   if (!SupportConcurrentCompilation) {
 718 #if LLVM_ENABLE_THREADS
 719     SupportConcurrentCompilation = NumCompileThreads || ES || EPC;
 720 #else
 721     SupportConcurrentCompilation = false;
 722 #endif // LLVM_ENABLE_THREADS
 723   } else {
 724 #if !LLVM_ENABLE_THREADS
 725     if (*SupportConcurrentCompilation)
 726       return make_error<StringError>(
 727           "LLJIT concurrent compilation support requested, but LLVM was built "
 728           "with LLVM_ENABLE_THREADS=Off",
 729           inconvertibleErrorCode());
 730 #endif // !LLVM_ENABLE_THREADS
 731   }
 732
 733   LLVM_DEBUG({
 734     dbgs() << "  JITTargetMachineBuilder is "
 735            << JITTargetMachineBuilderPrinter(*JTMB, "  ")
 736            << "  Pre-constructed ExecutionSession: " << (ES ? "Yes" : "No")
 737            << "\n"
 738            << "  DataLayout: ";
 739     if (DL)
 740       dbgs() << DL->getStringRepresentation() << "\n";
 741     else
 742       dbgs() << "None (will be created by JITTargetMachineBuilder)\n";
 743
 744     dbgs() << "  Custom object-linking-layer creator: "
 745            << (CreateObjectLinkingLayer ? "Yes" : "No") << "\n"
 746            << "  Custom compile-function creator: "
 747            << (CreateCompileFunction ? "Yes" : "No") << "\n"
 748            << "  Custom platform-setup function: "
 749            << (SetUpPlatform ? "Yes" : "No") << "\n"
 750            << "  Support concurrent compilation: "
 751            << (*SupportConcurrentCompilation ? "Yes" : "No");
 752     if (ConcurrentCompilationSettingDefaulted)
 753       dbgs() << " (defaulted based on ES / EPC / NumCompileThreads)\n";
 754     else
 755       dbgs() << "\n";
 756     dbgs() << "  Number of compile threads: " << NumCompileThreads << "\n";
 757   });
 758
 759   // Create DL if not specified.
 760   if (!DL) {
 761     if (auto DLOrErr = JTMB->getDefaultDataLayoutForTarget())
 762       DL = std::move(*DLOrErr);
 763     else
 764       return DLOrErr.takeError();
 765   }
 766
 767   // If neither ES nor EPC has been set then create an EPC instance.
 768   if (!ES && !EPC) {
 769     LLVM_DEBUG({
 770       dbgs() << "ExecutorProcessControl not specified, "
 771                 "Creating SelfExecutorProcessControl instance\n";
 772     });
 773
 774     std::unique_ptr<TaskDispatcher> D = nullptr;
 775 #if LLVM_ENABLE_THREADS
 776     if (*SupportConcurrentCompilation) {
 777       std::optional<size_t> NumThreads = std ::nullopt;
 778       if (NumCompileThreads)
 779         NumThreads = NumCompileThreads;
 780       D = std::make_unique<DynamicThreadPoolTaskDispatcher>(NumThreads);
 781     } else
 782       D = std::make_unique<InPlaceTaskDispatcher>();
 783 #endif // LLVM_ENABLE_THREADS
 784     if (auto EPCOrErr =
 785             SelfExecutorProcessControl::Create(nullptr, std::move(D), nullptr))
 786       EPC = std::move(*EPCOrErr);
 787     else
 788       return EPCOrErr.takeError();
 789   } else if (EPC) {
 790     LLVM_DEBUG({
 791       dbgs() << "Using explicitly specified ExecutorProcessControl instance "
 792              << EPC.get() << "\n";
 793     });
 794   } else {
 795     LLVM_DEBUG({
 796       dbgs() << "Using explicitly specified ExecutionSession instance "
 797              << ES.get() << "\n";
 798     });
 799   }
 800
 801   // If the client didn't configure any linker options then auto-configure the
 802   // JIT linker.
 803   if (!CreateObjectLinkingLayer) {
 804     auto &TT = JTMB->getTargetTriple();
 805     bool UseJITLink = false;
 806     switch (TT.getArch()) {
 807     case Triple::riscv64:
 808     case Triple::loongarch64:
 809       UseJITLink = true;
 810       break;
 811     case Triple::aarch64:
 812       UseJITLink = !TT.isOSBinFormatCOFF();
 813       break;
 814     case Triple::arm:
 815     case Triple::armeb:
 816     case Triple::thumb:
 817     case Triple::thumbeb:
 818       UseJITLink = TT.isOSBinFormatELF();
 819       break;
 820     case Triple::x86_64:
 821       UseJITLink = !TT.isOSBinFormatCOFF();
 822       break;
 823     case Triple::ppc64:
 824       UseJITLink = TT.isPPC64ELFv2ABI();
 825       break;
 826     case Triple::ppc64le:
 827       UseJITLink = TT.isOSBinFormatELF();
 828       break;
 829     default:
 830       break;
 831     }
 832     if (UseJITLink) {
 833       if (!JTMB->getCodeModel())
 834         JTMB->setCodeModel(CodeModel::Small);
 835       JTMB->setRelocationModel(Reloc::PIC_);
 836       CreateObjectLinkingLayer =
 837           [](ExecutionSession &ES,
 838              const Triple &) -> Expected<std::unique_ptr<ObjectLayer>> {
 839         auto ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>(ES);
 840         if (auto EHFrameRegistrar = EPCEHFrameRegistrar::Create(ES))
 841           ObjLinkingLayer->addPlugin(
 842               std::make_unique<EHFrameRegistrationPlugin>(
 843                   ES, std::move(*EHFrameRegistrar)));
 844         else
 845           return EHFrameRegistrar.takeError();
 846         return std::move(ObjLinkingLayer);
 847       };
 848     }
 849   }
 850
 851   // If we need a process JITDylib but no setup function has been given then
 852   // create a default one.
 853   if (!SetupProcessSymbolsJITDylib && LinkProcessSymbolsByDefault) {
 854     LLVM_DEBUG(dbgs() << "Creating default Process JD setup function\n");
 855     SetupProcessSymbolsJITDylib = [](LLJIT &J) -> Expected<JITDylibSP> {
 856       auto &JD =
 857           J.getExecutionSession().createBareJITDylib("<Process Symbols>");
 858       auto G = EPCDynamicLibrarySearchGenerator::GetForTargetProcess(
 859           J.getExecutionSession());
 860       if (!G)
 861         return G.takeError();
 862       JD.addGenerator(std::move(*G));
 863       return &JD;
 864     };
 865   }
 866
 867   return Error::success();
 868 }
 869
 870 LLJIT::~LLJIT() {
 871   if (auto Err = ES->endSession())
 872     ES->reportError(std::move(Err));
 873 }
 874
 875 JITDylibSP LLJIT::getProcessSymbolsJITDylib() { return ProcessSymbols; }
 876
 877 JITDylibSP LLJIT::getPlatformJITDylib() { return Platform; }
 878
 879 Expected<JITDylib &> LLJIT::createJITDylib(std::string Name) {
 880   auto JD = ES->createJITDylib(std::move(Name));
 881   if (!JD)
 882     return JD.takeError();
 883
 884   JD->addToLinkOrder(DefaultLinks);
 885   return JD;
 886 }
 887
 888 Expected<JITDylib &> LLJIT::loadPlatformDynamicLibrary(const char *Path) {
 889   auto G = EPCDynamicLibrarySearchGenerator::Load(*ES, Path);
 890   if (!G)
 891     return G.takeError();
 892
 893   if (auto *ExistingJD = ES->getJITDylibByName(Path))
 894     return *ExistingJD;
 895
 896   auto &JD = ES->createBareJITDylib(Path);
 897   JD.addGenerator(std::move(*G));
 898   return JD;
 899 }
 900
 901 Error LLJIT::linkStaticLibraryInto(JITDylib &JD,
 902                                    std::unique_ptr<MemoryBuffer> LibBuffer) {
 903   auto G = StaticLibraryDefinitionGenerator::Create(*ObjLinkingLayer,
 904                                                     std::move(LibBuffer));
 905   if (!G)
 906     return G.takeError();
 907
 908   JD.addGenerator(std::move(*G));
 909
 910   return Error::success();
 911 }
 912
 913 Error LLJIT::linkStaticLibraryInto(JITDylib &JD, const char *Path) {
 914   auto G = StaticLibraryDefinitionGenerator::Load(*ObjLinkingLayer, Path);
 915   if (!G)
 916     return G.takeError();
 917
 918   JD.addGenerator(std::move(*G));
 919
 920   return Error::success();
 921 }
 922
 923 Error LLJIT::addIRModule(ResourceTrackerSP RT, ThreadSafeModule TSM) {
 924   assert(TSM && "Can not add null module");
 925
 926   if (auto Err =
 927           TSM.withModuleDo([&](Module &M) { return applyDataLayout(M); }))
 928     return Err;
 929
 930   return InitHelperTransformLayer->add(std::move(RT), std::move(TSM));
 931 }
 932
 933 Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) {
 934   return addIRModule(JD.getDefaultResourceTracker(), std::move(TSM));
 935 }
 936
 937 Error LLJIT::addObjectFile(ResourceTrackerSP RT,
 938                            std::unique_ptr<MemoryBuffer> Obj) {
 939   assert(Obj && "Can not add null object");
 940
 941   return ObjTransformLayer->add(std::move(RT), std::move(Obj));
 942 }
 943
 944 Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) {
 945   return addObjectFile(JD.getDefaultResourceTracker(), std::move(Obj));
 946 }
 947
 948 Expected<ExecutorAddr> LLJIT::lookupLinkerMangled(JITDylib &JD,
 949                                                   SymbolStringPtr Name) {
 950   if (auto Sym = ES->lookup(
 951         makeJITDylibSearchOrder(&JD, JITDylibLookupFlags::MatchAllSymbols),
 952         Name))
 953     return Sym->getAddress();
 954   else
 955     return Sym.takeError();
 956 }
 957
 958 Expected<std::unique_ptr<ObjectLayer>>
 959 LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) {
 960
 961   // If the config state provided an ObjectLinkingLayer factory then use it.
 962   if (S.CreateObjectLinkingLayer)
 963     return S.CreateObjectLinkingLayer(ES, S.JTMB->getTargetTriple());
 964
 965   // Otherwise default to creating an RTDyldObjectLinkingLayer that constructs
 966   // a new SectionMemoryManager for each object.
 967   auto GetMemMgr = []() { return std::make_unique<SectionMemoryManager>(); };
 968   auto Layer =
 969       std::make_unique<RTDyldObjectLinkingLayer>(ES, std::move(GetMemMgr));
 970
 971   if (S.JTMB->getTargetTriple().isOSBinFormatCOFF()) {
 972     Layer->setOverrideObjectFlagsWithResponsibilityFlags(true);
 973     Layer->setAutoClaimResponsibilityForObjectSymbols(true);
 974   }
 975
 976   if (S.JTMB->getTargetTriple().isOSBinFormatELF() &&
 977       (S.JTMB->getTargetTriple().getArch() == Triple::ArchType::ppc64 ||
 978        S.JTMB->getTargetTriple().getArch() == Triple::ArchType::ppc64le))
 979     Layer->setAutoClaimResponsibilityForObjectSymbols(true);
 980
 981   // FIXME: Explicit conversion to std::unique_ptr<ObjectLayer> added to silence
 982   //        errors from some GCC / libstdc++ bots. Remove this conversion (i.e.
 983   //        just return ObjLinkingLayer) once those bots are upgraded.
 984   return std::unique_ptr<ObjectLayer>(std::move(Layer));
 985 }
 986
 987 Expected<std::unique_ptr<IRCompileLayer::IRCompiler>>
 988 LLJIT::createCompileFunction(LLJITBuilderState &S,
 989                              JITTargetMachineBuilder JTMB) {
 990
 991   /// If there is a custom compile function creator set then use it.
 992   if (S.CreateCompileFunction)
 993     return S.CreateCompileFunction(std::move(JTMB));
 994
 995   // If using a custom EPC then use a ConcurrentIRCompiler by default.
 996   if (*S.SupportConcurrentCompilation)
 997     return std::make_unique<ConcurrentIRCompiler>(std::move(JTMB));
 998
 999   auto TM = JTMB.createTargetMachine();
1000   if (!TM)
1001     return TM.takeError();
1002
1003   return std::make_unique<TMOwningSimpleCompiler>(std::move(*TM));
1004 }
1005
1006 LLJIT::LLJIT(LLJITBuilderState &S, Error &Err)
1007     : DL(std::move(*S.DL)), TT(S.JTMB->getTargetTriple()) {
1008
1009   ErrorAsOutParameter _(&Err);
1010
1011   assert(!(S.EPC && S.ES) && "EPC and ES should not both be set");
1012
1013   if (S.EPC) {
1014     ES = std::make_unique<ExecutionSession>(std::move(S.EPC));
1015   } else if (S.ES)
1016     ES = std::move(S.ES);
1017   else {
1018     if (auto EPC = SelfExecutorProcessControl::Create()) {
1019       ES = std::make_unique<ExecutionSession>(std::move(*EPC));
1020     } else {
1021       Err = EPC.takeError();
1022       return;
1023     }
1024   }
1025
1026   auto ObjLayer = createObjectLinkingLayer(S, *ES);
1027   if (!ObjLayer) {
1028     Err = ObjLayer.takeError();
1029     return;
1030   }
1031   ObjLinkingLayer = std::move(*ObjLayer);
1032   ObjTransformLayer =
1033       std::make_unique<ObjectTransformLayer>(*ES, *ObjLinkingLayer);
1034
1035   {
1036     auto CompileFunction = createCompileFunction(S, std::move(*S.JTMB));
1037     if (!CompileFunction) {
1038       Err = CompileFunction.takeError();
1039       return;
1040     }
1041     CompileLayer = std::make_unique<IRCompileLayer>(
1042         *ES, *ObjTransformLayer, std::move(*CompileFunction));
1043     TransformLayer = std::make_unique<IRTransformLayer>(*ES, *CompileLayer);
1044     InitHelperTransformLayer =
1045         std::make_unique<IRTransformLayer>(*ES, *TransformLayer);
1046   }
1047
1048   if (*S.SupportConcurrentCompilation)
1049     InitHelperTransformLayer->setCloneToNewContextOnEmit(true);
1050
1051   if (S.SetupProcessSymbolsJITDylib) {
1052     if (auto ProcSymsJD = S.SetupProcessSymbolsJITDylib(*this)) {
1053       ProcessSymbols = ProcSymsJD->get();
1054     } else {
1055       Err = ProcSymsJD.takeError();
1056       return;
1057     }
1058   }
1059
1060   if (S.PrePlatformSetup) {
1061     if (auto Err2 = S.PrePlatformSetup(*this)) {
1062       Err = std::move(Err2);
1063       return;
1064     }
1065   }
1066
1067   if (!S.SetUpPlatform)
1068     S.SetUpPlatform = setUpGenericLLVMIRPlatform;
1069
1070   if (auto PlatformJDOrErr = S.SetUpPlatform(*this)) {
1071     Platform = PlatformJDOrErr->get();
1072     if (Platform)
1073       DefaultLinks.push_back(
1074           {Platform, JITDylibLookupFlags::MatchExportedSymbolsOnly});
1075   } else {
1076     Err = PlatformJDOrErr.takeError();
1077     return;
1078   }
1079
1080   if (S.LinkProcessSymbolsByDefault)
1081     DefaultLinks.push_back(
1082         {ProcessSymbols, JITDylibLookupFlags::MatchExportedSymbolsOnly});
1083
1084   if (auto MainOrErr = createJITDylib("main"))
1085     Main = &*MainOrErr;
1086   else {
1087     Err = MainOrErr.takeError();
1088     return;
1089   }
1090 }
1091
1092 std::string LLJIT::mangle(StringRef UnmangledName) const {
1093   std::string MangledName;
1094   {
1095     raw_string_ostream MangledNameStream(MangledName);
1096     Mangler::getNameWithPrefix(MangledNameStream, UnmangledName, DL);
1097   }
1098   return MangledName;
1099 }
1100
1101 Error LLJIT::applyDataLayout(Module &M) {
1102   if (M.getDataLayout().isDefault())
1103     M.setDataLayout(DL);
1104
1105   if (M.getDataLayout() != DL)
1106     return make_error<StringError>(
1107         "Added modules have incompatible data layouts: " +
1108             M.getDataLayout().getStringRepresentation() + " (module) vs " +
1109             DL.getStringRepresentation() + " (jit)",
1110         inconvertibleErrorCode());
1111
1112   return Error::success();
1113 }
1114
1115 Error setUpOrcPlatformManually(LLJIT &J) {
1116   LLVM_DEBUG({ dbgs() << "Setting up orc platform support for LLJIT\n"; });
1117   J.setPlatformSupport(std::make_unique<ORCPlatformSupport>(J));
1118   return Error::success();
1119 }
1120
1121 class LoadAndLinkDynLibrary {
1122 public:
1123   LoadAndLinkDynLibrary(LLJIT &J) : J(J) {}
1124   Error operator()(JITDylib &JD, StringRef DLLName) {
1125     if (!DLLName.ends_with_insensitive(".dll"))
1126       return make_error<StringError>("DLLName not ending with .dll",
1127                                      inconvertibleErrorCode());
1128     auto DLLNameStr = DLLName.str(); // Guarantees null-termination.
1129     auto DLLJD = J.loadPlatformDynamicLibrary(DLLNameStr.c_str());
1130     if (!DLLJD)
1131       return DLLJD.takeError();
1132     JD.addToLinkOrder(*DLLJD);
1133     return Error::success();
1134   }
1135
1136 private:
1137   LLJIT &J;
1138 };
1139
1140 Expected<JITDylibSP> ExecutorNativePlatform::operator()(LLJIT &J) {
1141   auto ProcessSymbolsJD = J.getProcessSymbolsJITDylib();
1142   if (!ProcessSymbolsJD)
1143     return make_error<StringError>(
1144         "Native platforms require a process symbols JITDylib",
1145         inconvertibleErrorCode());
1146
1147   const Triple &TT = J.getTargetTriple();
1148   ObjectLinkingLayer *ObjLinkingLayer =
1149       dyn_cast<ObjectLinkingLayer>(&J.getObjLinkingLayer());
1150
1151   if (!ObjLinkingLayer)
1152     return make_error<StringError>(
1153         "ExecutorNativePlatform requires ObjectLinkingLayer",
1154         inconvertibleErrorCode());
1155
1156   std::unique_ptr<MemoryBuffer> RuntimeArchiveBuffer;
1157   if (OrcRuntime.index() == 0) {
1158     auto A = errorOrToExpected(MemoryBuffer::getFile(std::get<0>(OrcRuntime)));
1159     if (!A)
1160       return A.takeError();
1161     RuntimeArchiveBuffer = std::move(*A);
1162   } else
1163     RuntimeArchiveBuffer = std::move(std::get<1>(OrcRuntime));
1164
1165   auto &ES = J.getExecutionSession();
1166   auto &PlatformJD = ES.createBareJITDylib("<Platform>");
1167   PlatformJD.addToLinkOrder(*ProcessSymbolsJD);
1168
1169   J.setPlatformSupport(std::make_unique<ORCPlatformSupport>(J));
1170
1171   switch (TT.getObjectFormat()) {
1172   case Triple::COFF: {
1173     const char *VCRuntimePath = nullptr;
1174     bool StaticVCRuntime = false;
1175     if (VCRuntime) {
1176       VCRuntimePath = VCRuntime->first.c_str();
1177       StaticVCRuntime = VCRuntime->second;
1178     }
1179     if (auto P = COFFPlatform::Create(
1180             *ObjLinkingLayer, PlatformJD, std::move(RuntimeArchiveBuffer),
1181             LoadAndLinkDynLibrary(J), StaticVCRuntime, VCRuntimePath))
1182       J.getExecutionSession().setPlatform(std::move(*P));
1183     else
1184       return P.takeError();
1185     break;
1186   }
1187   case Triple::ELF: {
1188     auto G = StaticLibraryDefinitionGenerator::Create(
1189         *ObjLinkingLayer, std::move(RuntimeArchiveBuffer));
1190     if (!G)
1191       return G.takeError();
1192
1193     if (auto P =
1194             ELFNixPlatform::Create(*ObjLinkingLayer, PlatformJD, std::move(*G)))
1195       J.getExecutionSession().setPlatform(std::move(*P));
1196     else
1197       return P.takeError();
1198     break;
1199   }
1200   case Triple::MachO: {
1201     auto G = StaticLibraryDefinitionGenerator::Create(
1202         *ObjLinkingLayer, std::move(RuntimeArchiveBuffer));
1203     if (!G)
1204       return G.takeError();
1205
1206     if (auto P =
1207             MachOPlatform::Create(*ObjLinkingLayer, PlatformJD, std::move(*G)))
1208       ES.setPlatform(std::move(*P));
1209     else
1210       return P.takeError();
1211     break;
1212   }
1213   default:
1214     return make_error<StringError>("Unsupported object format in triple " +
1215                                        TT.str(),
1216                                    inconvertibleErrorCode());
1217   }
1218
1219   return &PlatformJD;
1220 }
1221
1222 Expected<JITDylibSP> setUpGenericLLVMIRPlatform(LLJIT &J) {
1223   LLVM_DEBUG(
1224       { dbgs() << "Setting up GenericLLVMIRPlatform support for LLJIT\n"; });
1225   auto ProcessSymbolsJD = J.getProcessSymbolsJITDylib();
1226   if (!ProcessSymbolsJD)
1227     return make_error<StringError>(
1228         "Native platforms require a process symbols JITDylib",
1229         inconvertibleErrorCode());
1230
1231   auto &PlatformJD = J.getExecutionSession().createBareJITDylib("<Platform>");
1232   PlatformJD.addToLinkOrder(*ProcessSymbolsJD);
1233
1234   J.setPlatformSupport(
1235       std::make_unique<GenericLLVMIRPlatformSupport>(J, PlatformJD));
1236
1237   return &PlatformJD;
1238 }
1239
1240 Expected<JITDylibSP> setUpInactivePlatform(LLJIT &J) {
1241   LLVM_DEBUG(
1242       { dbgs() << "Explicitly deactivated platform support for LLJIT\n"; });
1243   J.setPlatformSupport(std::make_unique<InactivePlatformSupport>());
1244   return nullptr;
1245 }
1246
1247 Error LLLazyJITBuilderState::prepareForConstruction() {
1248   if (auto Err = LLJITBuilderState::prepareForConstruction())
1249     return Err;
1250   TT = JTMB->getTargetTriple();
1251   return Error::success();
1252 }
1253
1254 Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) {
1255   assert(TSM && "Can not add null module");
1256
1257   if (auto Err = TSM.withModuleDo(
1258           [&](Module &M) -> Error { return applyDataLayout(M); }))
1259     return Err;
1260
1261   return CODLayer->add(JD, std::move(TSM));
1262 }
1263
1264 LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) {
1265
1266   // If LLJIT construction failed then bail out.
1267   if (Err)
1268     return;
1269
1270   ErrorAsOutParameter _(&Err);
1271
1272   /// Take/Create the lazy-compile callthrough manager.
1273   if (S.LCTMgr)
1274     LCTMgr = std::move(S.LCTMgr);
1275   else {
1276     if (auto LCTMgrOrErr = createLocalLazyCallThroughManager(
1277             S.TT, *ES, S.LazyCompileFailureAddr))
1278       LCTMgr = std::move(*LCTMgrOrErr);
1279     else {
1280       Err = LCTMgrOrErr.takeError();
1281       return;
1282     }
1283   }
1284
1285   // Take/Create the indirect stubs manager builder.
1286   auto ISMBuilder = std::move(S.ISMBuilder);
1287
1288   // If none was provided, try to build one.
1289   if (!ISMBuilder)
1290     ISMBuilder = createLocalIndirectStubsManagerBuilder(S.TT);
1291
1292   // No luck. Bail out.
1293   if (!ISMBuilder) {
1294     Err = make_error<StringError>("Could not construct "
1295                                   "IndirectStubsManagerBuilder for target " +
1296                                       S.TT.str(),
1297                                   inconvertibleErrorCode());
1298     return;
1299   }
1300
1301   // Create the IP Layer.
1302   IPLayer = std::make_unique<IRPartitionLayer>(*ES, *InitHelperTransformLayer);
1303
1304   // Create the COD layer.
1305   CODLayer = std::make_unique<CompileOnDemandLayer>(*ES, *IPLayer, *LCTMgr,
1306                                                     std::move(ISMBuilder));
1307
1308   if (*S.SupportConcurrentCompilation)
1309     CODLayer->setCloneToNewContextOnEmit(true);
1310 }
1311
1312 // In-process LLJIT uses eh-frame section wrappers via EPC, so we need to force
1313 // them to be linked in.
1314 LLVM_ATTRIBUTE_USED void linkComponents() {
1315   errs() << (void *)&llvm_orc_registerEHFrameSectionWrapper
1316          << (void *)&llvm_orc_deregisterEHFrameSectionWrapper;
1317 }
1318
1319 } // End namespace orc.
1320 } // End namespace llvm.