llvm/lib/ExecutionEngine/Orc/LLJIT.cpp

   1 //===--------- LLJIT.cpp - An ORC-based JIT for compiling LLVM IR ---------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ExecutionEngine/Orc/LLJIT.h"
  10 #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h"
  11 #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h"
  12 #include "llvm/ExecutionEngine/Orc/COFFPlatform.h"
  13 #include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h"
  14 #include "llvm/ExecutionEngine/Orc/DebuggerSupportPlugin.h"
  15 #include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
  16 #include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
  17 #include "llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h"
  18 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
  19 #include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
  20 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
  21 #include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h"
  22 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
  23 #include "llvm/ExecutionEngine/Orc/Shared/OrcError.h"
  24 #include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h"
  25 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
  26 #include "llvm/IR/GlobalVariable.h"
  27 #include "llvm/IR/IRBuilder.h"
  28 #include "llvm/IR/Mangler.h"
  29 #include "llvm/IR/Module.h"
  30 #include "llvm/Support/DynamicLibrary.h"
  31
  32 #include <map>
  33
  34 #define DEBUG_TYPE "orc"
  35
  36 using namespace llvm;
  37 using namespace llvm::orc;
  38
  39 namespace {
  40
  41 /// Adds helper function decls and wrapper functions that call the helper with
  42 /// some additional prefix arguments.
  43 ///
  44 /// E.g. For wrapper "foo" with type i8(i8, i64), helper "bar", and prefix
  45 /// args i32 4 and i16 12345, this function will add:
  46 ///
  47 /// declare i8 @bar(i32, i16, i8, i64)
  48 ///
  49 /// define i8 @foo(i8, i64) {
  50 /// entry:
  51 ///   %2 = call i8 @bar(i32 4, i16 12345, i8 %0, i64 %1)
  52 ///   ret i8 %2
  53 /// }
  54 ///
  55 Function *addHelperAndWrapper(Module &M, StringRef WrapperName,
  56                               FunctionType *WrapperFnType,
  57                               GlobalValue::VisibilityTypes WrapperVisibility,
  58                               StringRef HelperName,
  59                               ArrayRef<Value *> HelperPrefixArgs) {
  60   std::vector<Type *> HelperArgTypes;
  61   for (auto *Arg : HelperPrefixArgs)
  62     HelperArgTypes.push_back(Arg->getType());
  63   for (auto *T : WrapperFnType->params())
  64     HelperArgTypes.push_back(T);
  65   auto *HelperFnType =
  66       FunctionType::get(WrapperFnType->getReturnType(), HelperArgTypes, false);
  67   auto *HelperFn = Function::Create(HelperFnType, GlobalValue::ExternalLinkage,
  68                                     HelperName, M);
  69
  70   auto *WrapperFn = Function::Create(
  71       WrapperFnType, GlobalValue::ExternalLinkage, WrapperName, M);
  72   WrapperFn->setVisibility(WrapperVisibility);
  73
  74   auto *EntryBlock = BasicBlock::Create(M.getContext(), "entry", WrapperFn);
  75   IRBuilder<> IB(EntryBlock);
  76
  77   std::vector<Value *> HelperArgs;
  78   for (auto *Arg : HelperPrefixArgs)
  79     HelperArgs.push_back(Arg);
  80   for (auto &Arg : WrapperFn->args())
  81     HelperArgs.push_back(&Arg);
  82   auto *HelperResult = IB.CreateCall(HelperFn, HelperArgs);
  83   if (HelperFn->getReturnType()->isVoidTy())
  84     IB.CreateRetVoid();
  85   else
  86     IB.CreateRet(HelperResult);
  87
  88   return WrapperFn;
  89 }
  90
  91 class ORCPlatformSupport : public LLJIT::PlatformSupport {
  92 public:
  93   ORCPlatformSupport(orc::LLJIT &J) : J(J) {}
  94
  95   Error initialize(orc::JITDylib &JD) override {
  96     using llvm::orc::shared::SPSExecutorAddr;
  97     using llvm::orc::shared::SPSString;
  98     using SPSDLOpenSig = SPSExecutorAddr(SPSString, int32_t);
  99     enum dlopen_mode : int32_t {
 100       ORC_RT_RTLD_LAZY = 0x1,
 101       ORC_RT_RTLD_NOW = 0x2,
 102       ORC_RT_RTLD_LOCAL = 0x4,
 103       ORC_RT_RTLD_GLOBAL = 0x8
 104     };
 105
 106     auto &ES = J.getExecutionSession();
 107     auto MainSearchOrder = J.getMainJITDylib().withLinkOrderDo(
 108         [](const JITDylibSearchOrder &SO) { return SO; });
 109
 110     if (auto WrapperAddr =
 111             ES.lookup(MainSearchOrder,
 112                       J.mangleAndIntern("__orc_rt_jit_dlopen_wrapper"))) {
 113       return ES.callSPSWrapper<SPSDLOpenSig>(WrapperAddr->getAddress(),
 114                                              DSOHandles[&JD], JD.getName(),
 115                                              int32_t(ORC_RT_RTLD_LAZY));
 116     } else
 117       return WrapperAddr.takeError();
 118   }
 119
 120   Error deinitialize(orc::JITDylib &JD) override {
 121     using llvm::orc::shared::SPSExecutorAddr;
 122     using SPSDLCloseSig = int32_t(SPSExecutorAddr);
 123
 124     auto &ES = J.getExecutionSession();
 125     auto MainSearchOrder = J.getMainJITDylib().withLinkOrderDo(
 126         [](const JITDylibSearchOrder &SO) { return SO; });
 127
 128     if (auto WrapperAddr =
 129             ES.lookup(MainSearchOrder,
 130                       J.mangleAndIntern("__orc_rt_jit_dlclose_wrapper"))) {
 131       int32_t result;
 132       auto E = J.getExecutionSession().callSPSWrapper<SPSDLCloseSig>(
 133           WrapperAddr->getAddress(), result, DSOHandles[&JD]);
 134       if (E)
 135         return E;
 136       else if (result)
 137         return make_error<StringError>("dlclose failed",
 138                                        inconvertibleErrorCode());
 139       DSOHandles.erase(&JD);
 140     } else
 141       return WrapperAddr.takeError();
 142     return Error::success();
 143   }
 144
 145 private:
 146   orc::LLJIT &J;
 147   DenseMap<orc::JITDylib *, orc::ExecutorAddr> DSOHandles;
 148 };
 149
 150 class GenericLLVMIRPlatformSupport;
 151
 152 /// orc::Platform component of Generic LLVM IR Platform support.
 153 /// Just forwards calls to the GenericLLVMIRPlatformSupport class below.
 154 class GenericLLVMIRPlatform : public Platform {
 155 public:
 156   GenericLLVMIRPlatform(GenericLLVMIRPlatformSupport &S) : S(S) {}
 157   Error setupJITDylib(JITDylib &JD) override;
 158   Error teardownJITDylib(JITDylib &JD) override;
 159   Error notifyAdding(ResourceTracker &RT,
 160                      const MaterializationUnit &MU) override;
 161   Error notifyRemoving(ResourceTracker &RT) override {
 162     // Noop -- Nothing to do (yet).
 163     return Error::success();
 164   }
 165
 166 private:
 167   GenericLLVMIRPlatformSupport &S;
 168 };
 169
 170 /// This transform parses llvm.global_ctors to produce a single initialization
 171 /// function for the module, records the function, then deletes
 172 /// llvm.global_ctors.
 173 class GlobalCtorDtorScraper {
 174 public:
 175   GlobalCtorDtorScraper(GenericLLVMIRPlatformSupport &PS,
 176                         StringRef InitFunctionPrefix,
 177                         StringRef DeInitFunctionPrefix)
 178       : PS(PS), InitFunctionPrefix(InitFunctionPrefix),
 179         DeInitFunctionPrefix(DeInitFunctionPrefix) {}
 180   Expected<ThreadSafeModule> operator()(ThreadSafeModule TSM,
 181                                         MaterializationResponsibility &R);
 182
 183 private:
 184   GenericLLVMIRPlatformSupport &PS;
 185   StringRef InitFunctionPrefix;
 186   StringRef DeInitFunctionPrefix;
 187 };
 188
 189 /// Generic IR Platform Support
 190 ///
 191 /// Scrapes llvm.global_ctors and llvm.global_dtors and replaces them with
 192 /// specially named 'init' and 'deinit'. Injects definitions / interposes for
 193 /// some runtime API, including __cxa_atexit, dlopen, and dlclose.
 194 class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport {
 195 public:
 196   GenericLLVMIRPlatformSupport(LLJIT &J, JITDylib &PlatformJD)
 197       : J(J), InitFunctionPrefix(J.mangle("__orc_init_func.")),
 198         DeInitFunctionPrefix(J.mangle("__orc_deinit_func.")) {
 199
 200     getExecutionSession().setPlatform(
 201         std::make_unique<GenericLLVMIRPlatform>(*this));
 202
 203     setInitTransform(J, GlobalCtorDtorScraper(*this, InitFunctionPrefix,
 204                                               DeInitFunctionPrefix));
 205
 206     SymbolMap StdInterposes;
 207
 208     StdInterposes[J.mangleAndIntern("__lljit.platform_support_instance")] = {
 209         ExecutorAddr::fromPtr(this), JITSymbolFlags::Exported};
 210     StdInterposes[J.mangleAndIntern("__lljit.cxa_atexit_helper")] = {
 211         ExecutorAddr::fromPtr(registerCxaAtExitHelper), JITSymbolFlags()};
 212
 213     cantFail(PlatformJD.define(absoluteSymbols(std::move(StdInterposes))));
 214     cantFail(setupJITDylib(PlatformJD));
 215     cantFail(J.addIRModule(PlatformJD, createPlatformRuntimeModule()));
 216   }
 217
 218   ExecutionSession &getExecutionSession() { return J.getExecutionSession(); }
 219
 220   /// Adds a module that defines the __dso_handle global.
 221   Error setupJITDylib(JITDylib &JD) {
 222
 223     // Add per-jitdylib standard interposes.
 224     SymbolMap PerJDInterposes;
 225     PerJDInterposes[J.mangleAndIntern("__lljit.run_atexits_helper")] = {
 226         ExecutorAddr::fromPtr(runAtExitsHelper), JITSymbolFlags()};
 227     PerJDInterposes[J.mangleAndIntern("__lljit.atexit_helper")] = {
 228         ExecutorAddr::fromPtr(registerAtExitHelper), JITSymbolFlags()};
 229     cantFail(JD.define(absoluteSymbols(std::move(PerJDInterposes))));
 230
 231     auto Ctx = std::make_unique<LLVMContext>();
 232     auto M = std::make_unique<Module>("__standard_lib", *Ctx);
 233     M->setDataLayout(J.getDataLayout());
 234
 235     auto *Int64Ty = Type::getInt64Ty(*Ctx);
 236     auto *DSOHandle = new GlobalVariable(
 237         *M, Int64Ty, true, GlobalValue::ExternalLinkage,
 238         ConstantInt::get(Int64Ty, reinterpret_cast<uintptr_t>(&JD)),
 239         "__dso_handle");
 240     DSOHandle->setVisibility(GlobalValue::DefaultVisibility);
 241     DSOHandle->setInitializer(
 242         ConstantInt::get(Int64Ty, ExecutorAddr::fromPtr(&JD).getValue()));
 243
 244     auto *GenericIRPlatformSupportTy =
 245         StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport");
 246
 247     auto *PlatformInstanceDecl = new GlobalVariable(
 248         *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
 249         nullptr, "__lljit.platform_support_instance");
 250
 251     auto *VoidTy = Type::getVoidTy(*Ctx);
 252     addHelperAndWrapper(
 253         *M, "__lljit_run_atexits", FunctionType::get(VoidTy, {}, false),
 254         GlobalValue::HiddenVisibility, "__lljit.run_atexits_helper",
 255         {PlatformInstanceDecl, DSOHandle});
 256
 257     auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT);
 258     auto *AtExitCallbackTy = FunctionType::get(VoidTy, {}, false);
 259     auto *AtExitCallbackPtrTy = PointerType::getUnqual(AtExitCallbackTy);
 260     addHelperAndWrapper(*M, "atexit",
 261                         FunctionType::get(IntTy, {AtExitCallbackPtrTy}, false),
 262                         GlobalValue::HiddenVisibility, "__lljit.atexit_helper",
 263                         {PlatformInstanceDecl, DSOHandle});
 264
 265     return J.addIRModule(JD, ThreadSafeModule(std::move(M), std::move(Ctx)));
 266   }
 267
 268   Error notifyAdding(ResourceTracker &RT, const MaterializationUnit &MU) {
 269     auto &JD = RT.getJITDylib();
 270     if (auto &InitSym = MU.getInitializerSymbol())
 271       InitSymbols[&JD].add(InitSym, SymbolLookupFlags::WeaklyReferencedSymbol);
 272     else {
 273       // If there's no identified init symbol attached, but there is a symbol
 274       // with the GenericIRPlatform::InitFunctionPrefix, then treat that as
 275       // an init function. Add the symbol to both the InitSymbols map (which
 276       // will trigger a lookup to materialize the module) and the InitFunctions
 277       // map (which holds the names of the symbols to execute).
 278       for (auto &KV : MU.getSymbols())
 279         if ((*KV.first).startswith(InitFunctionPrefix)) {
 280           InitSymbols[&JD].add(KV.first,
 281                                SymbolLookupFlags::WeaklyReferencedSymbol);
 282           InitFunctions[&JD].add(KV.first);
 283         } else if ((*KV.first).startswith(DeInitFunctionPrefix)) {
 284           DeInitFunctions[&JD].add(KV.first);
 285         }
 286     }
 287     return Error::success();
 288   }
 289
 290   Error initialize(JITDylib &JD) override {
 291     LLVM_DEBUG({
 292       dbgs() << "GenericLLVMIRPlatformSupport getting initializers to run\n";
 293     });
 294     if (auto Initializers = getInitializers(JD)) {
 295       LLVM_DEBUG(
 296           { dbgs() << "GenericLLVMIRPlatformSupport running initializers\n"; });
 297       for (auto InitFnAddr : *Initializers) {
 298         LLVM_DEBUG({
 299           dbgs() << "  Running init " << formatv("{0:x16}", InitFnAddr)
 300                  << "...\n";
 301         });
 302         auto *InitFn = InitFnAddr.toPtr<void (*)()>();
 303         InitFn();
 304       }
 305     } else
 306       return Initializers.takeError();
 307     return Error::success();
 308   }
 309
 310   Error deinitialize(JITDylib &JD) override {
 311     LLVM_DEBUG({
 312       dbgs() << "GenericLLVMIRPlatformSupport getting deinitializers to run\n";
 313     });
 314     if (auto Deinitializers = getDeinitializers(JD)) {
 315       LLVM_DEBUG({
 316         dbgs() << "GenericLLVMIRPlatformSupport running deinitializers\n";
 317       });
 318       for (auto DeinitFnAddr : *Deinitializers) {
 319         LLVM_DEBUG({
 320           dbgs() << "  Running deinit " << formatv("{0:x16}", DeinitFnAddr)
 321                  << "...\n";
 322         });
 323         auto *DeinitFn = DeinitFnAddr.toPtr<void (*)()>();
 324         DeinitFn();
 325       }
 326     } else
 327       return Deinitializers.takeError();
 328
 329     return Error::success();
 330   }
 331
 332   void registerInitFunc(JITDylib &JD, SymbolStringPtr InitName) {
 333     getExecutionSession().runSessionLocked([&]() {
 334         InitFunctions[&JD].add(InitName);
 335       });
 336   }
 337
 338   void registerDeInitFunc(JITDylib &JD, SymbolStringPtr DeInitName) {
 339     getExecutionSession().runSessionLocked(
 340         [&]() { DeInitFunctions[&JD].add(DeInitName); });
 341   }
 342
 343 private:
 344   Expected<std::vector<ExecutorAddr>> getInitializers(JITDylib &JD) {
 345     if (auto Err = issueInitLookups(JD))
 346       return std::move(Err);
 347
 348     DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
 349     std::vector<JITDylibSP> DFSLinkOrder;
 350
 351     if (auto Err = getExecutionSession().runSessionLocked([&]() -> Error {
 352           if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
 353             DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
 354           else
 355             return DFSLinkOrderOrErr.takeError();
 356
 357           for (auto &NextJD : DFSLinkOrder) {
 358             auto IFItr = InitFunctions.find(NextJD.get());
 359             if (IFItr != InitFunctions.end()) {
 360               LookupSymbols[NextJD.get()] = std::move(IFItr->second);
 361               InitFunctions.erase(IFItr);
 362             }
 363           }
 364           return Error::success();
 365         }))
 366       return std::move(Err);
 367
 368     LLVM_DEBUG({
 369       dbgs() << "JITDylib init order is [ ";
 370       for (auto &JD : llvm::reverse(DFSLinkOrder))
 371         dbgs() << "\"" << JD->getName() << "\" ";
 372       dbgs() << "]\n";
 373       dbgs() << "Looking up init functions:\n";
 374       for (auto &KV : LookupSymbols)
 375         dbgs() << "  \"" << KV.first->getName() << "\": " << KV.second << "\n";
 376     });
 377
 378     auto &ES = getExecutionSession();
 379     auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols);
 380
 381     if (!LookupResult)
 382       return LookupResult.takeError();
 383
 384     std::vector<ExecutorAddr> Initializers;
 385     while (!DFSLinkOrder.empty()) {
 386       auto &NextJD = *DFSLinkOrder.back();
 387       DFSLinkOrder.pop_back();
 388       auto InitsItr = LookupResult->find(&NextJD);
 389       if (InitsItr == LookupResult->end())
 390         continue;
 391       for (auto &KV : InitsItr->second)
 392         Initializers.push_back(KV.second.getAddress());
 393     }
 394
 395     return Initializers;
 396   }
 397
 398   Expected<std::vector<ExecutorAddr>> getDeinitializers(JITDylib &JD) {
 399     auto &ES = getExecutionSession();
 400
 401     auto LLJITRunAtExits = J.mangleAndIntern("__lljit_run_atexits");
 402
 403     DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
 404     std::vector<JITDylibSP> DFSLinkOrder;
 405
 406     if (auto Err = ES.runSessionLocked([&]() -> Error {
 407           if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
 408             DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
 409           else
 410             return DFSLinkOrderOrErr.takeError();
 411
 412           for (auto &NextJD : DFSLinkOrder) {
 413             auto &JDLookupSymbols = LookupSymbols[NextJD.get()];
 414             auto DIFItr = DeInitFunctions.find(NextJD.get());
 415             if (DIFItr != DeInitFunctions.end()) {
 416               LookupSymbols[NextJD.get()] = std::move(DIFItr->second);
 417               DeInitFunctions.erase(DIFItr);
 418             }
 419             JDLookupSymbols.add(LLJITRunAtExits,
 420                                 SymbolLookupFlags::WeaklyReferencedSymbol);
 421           }
 422           return Error::success();
 423         }))
 424       return std::move(Err);
 425
 426     LLVM_DEBUG({
 427       dbgs() << "JITDylib deinit order is [ ";
 428       for (auto &JD : DFSLinkOrder)
 429         dbgs() << "\"" << JD->getName() << "\" ";
 430       dbgs() << "]\n";
 431       dbgs() << "Looking up deinit functions:\n";
 432       for (auto &KV : LookupSymbols)
 433         dbgs() << "  \"" << KV.first->getName() << "\": " << KV.second << "\n";
 434     });
 435
 436     auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols);
 437
 438     if (!LookupResult)
 439       return LookupResult.takeError();
 440
 441     std::vector<ExecutorAddr> DeInitializers;
 442     for (auto &NextJD : DFSLinkOrder) {
 443       auto DeInitsItr = LookupResult->find(NextJD.get());
 444       assert(DeInitsItr != LookupResult->end() &&
 445              "Every JD should have at least __lljit_run_atexits");
 446
 447       auto RunAtExitsItr = DeInitsItr->second.find(LLJITRunAtExits);
 448       if (RunAtExitsItr != DeInitsItr->second.end())
 449         DeInitializers.push_back(RunAtExitsItr->second.getAddress());
 450
 451       for (auto &KV : DeInitsItr->second)
 452         if (KV.first != LLJITRunAtExits)
 453           DeInitializers.push_back(KV.second.getAddress());
 454     }
 455
 456     return DeInitializers;
 457   }
 458
 459   /// Issue lookups for all init symbols required to initialize JD (and any
 460   /// JITDylibs that it depends on).
 461   Error issueInitLookups(JITDylib &JD) {
 462     DenseMap<JITDylib *, SymbolLookupSet> RequiredInitSymbols;
 463     std::vector<JITDylibSP> DFSLinkOrder;
 464
 465     if (auto Err = getExecutionSession().runSessionLocked([&]() -> Error {
 466           if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
 467             DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
 468           else
 469             return DFSLinkOrderOrErr.takeError();
 470
 471           for (auto &NextJD : DFSLinkOrder) {
 472             auto ISItr = InitSymbols.find(NextJD.get());
 473             if (ISItr != InitSymbols.end()) {
 474               RequiredInitSymbols[NextJD.get()] = std::move(ISItr->second);
 475               InitSymbols.erase(ISItr);
 476             }
 477           }
 478           return Error::success();
 479         }))
 480       return Err;
 481
 482     return Platform::lookupInitSymbols(getExecutionSession(),
 483                                        RequiredInitSymbols)
 484         .takeError();
 485   }
 486
 487   static void registerCxaAtExitHelper(void *Self, void (*F)(void *), void *Ctx,
 488                                       void *DSOHandle) {
 489     LLVM_DEBUG({
 490       dbgs() << "Registering cxa atexit function " << (void *)F << " for JD "
 491              << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
 492     });
 493     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit(
 494         F, Ctx, DSOHandle);
 495   }
 496
 497   static void registerAtExitHelper(void *Self, void *DSOHandle, void (*F)()) {
 498     LLVM_DEBUG({
 499       dbgs() << "Registering atexit function " << (void *)F << " for JD "
 500              << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
 501     });
 502     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit(
 503         reinterpret_cast<void (*)(void *)>(F), nullptr, DSOHandle);
 504   }
 505
 506   static void runAtExitsHelper(void *Self, void *DSOHandle) {
 507     LLVM_DEBUG({
 508       dbgs() << "Running atexit functions for JD "
 509              << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
 510     });
 511     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.runAtExits(
 512         DSOHandle);
 513   }
 514
 515   // Constructs an LLVM IR module containing platform runtime globals,
 516   // functions, and interposes.
 517   ThreadSafeModule createPlatformRuntimeModule() {
 518     auto Ctx = std::make_unique<LLVMContext>();
 519     auto M = std::make_unique<Module>("__standard_lib", *Ctx);
 520     M->setDataLayout(J.getDataLayout());
 521
 522     auto *GenericIRPlatformSupportTy =
 523         StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport");
 524
 525     auto *PlatformInstanceDecl = new GlobalVariable(
 526         *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
 527         nullptr, "__lljit.platform_support_instance");
 528
 529     auto *Int8Ty = Type::getInt8Ty(*Ctx);
 530     auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT);
 531     auto *VoidTy = Type::getVoidTy(*Ctx);
 532     auto *BytePtrTy = PointerType::getUnqual(Int8Ty);
 533     auto *CxaAtExitCallbackTy = FunctionType::get(VoidTy, {BytePtrTy}, false);
 534     auto *CxaAtExitCallbackPtrTy = PointerType::getUnqual(CxaAtExitCallbackTy);
 535
 536     addHelperAndWrapper(
 537         *M, "__cxa_atexit",
 538         FunctionType::get(IntTy, {CxaAtExitCallbackPtrTy, BytePtrTy, BytePtrTy},
 539                           false),
 540         GlobalValue::DefaultVisibility, "__lljit.cxa_atexit_helper",
 541         {PlatformInstanceDecl});
 542
 543     return ThreadSafeModule(std::move(M), std::move(Ctx));
 544   }
 545
 546   LLJIT &J;
 547   std::string InitFunctionPrefix;
 548   std::string DeInitFunctionPrefix;
 549   DenseMap<JITDylib *, SymbolLookupSet> InitSymbols;
 550   DenseMap<JITDylib *, SymbolLookupSet> InitFunctions;
 551   DenseMap<JITDylib *, SymbolLookupSet> DeInitFunctions;
 552   ItaniumCXAAtExitSupport AtExitMgr;
 553 };
 554
 555 Error GenericLLVMIRPlatform::setupJITDylib(JITDylib &JD) {
 556   return S.setupJITDylib(JD);
 557 }
 558
 559 Error GenericLLVMIRPlatform::teardownJITDylib(JITDylib &JD) {
 560   return Error::success();
 561 }
 562
 563 Error GenericLLVMIRPlatform::notifyAdding(ResourceTracker &RT,
 564                                           const MaterializationUnit &MU) {
 565   return S.notifyAdding(RT, MU);
 566 }
 567
 568 Expected<ThreadSafeModule>
 569 GlobalCtorDtorScraper::operator()(ThreadSafeModule TSM,
 570                                   MaterializationResponsibility &R) {
 571   auto Err = TSM.withModuleDo([&](Module &M) -> Error {
 572     auto &Ctx = M.getContext();
 573     auto *GlobalCtors = M.getNamedGlobal("llvm.global_ctors");
 574     auto *GlobalDtors = M.getNamedGlobal("llvm.global_dtors");
 575
 576     auto RegisterCOrDtors = [&](GlobalVariable *GlobalCOrDtors,
 577                                 bool isCtor) -> Error {
 578       // If there's no llvm.global_c/dtor or it's just a decl then skip.
 579       if (!GlobalCOrDtors || GlobalCOrDtors->isDeclaration())
 580         return Error::success();
 581       std::string InitOrDeInitFunctionName;
 582       if (isCtor)
 583         raw_string_ostream(InitOrDeInitFunctionName)
 584             << InitFunctionPrefix << M.getModuleIdentifier();
 585       else
 586         raw_string_ostream(InitOrDeInitFunctionName)
 587             << DeInitFunctionPrefix << M.getModuleIdentifier();
 588
 589       MangleAndInterner Mangle(PS.getExecutionSession(), M.getDataLayout());
 590       auto InternedInitOrDeInitName = Mangle(InitOrDeInitFunctionName);
 591       if (auto Err = R.defineMaterializing(
 592               {{InternedInitOrDeInitName, JITSymbolFlags::Callable}}))
 593         return Err;
 594
 595       auto *InitOrDeInitFunc = Function::Create(
 596           FunctionType::get(Type::getVoidTy(Ctx), {}, false),
 597           GlobalValue::ExternalLinkage, InitOrDeInitFunctionName, &M);
 598       InitOrDeInitFunc->setVisibility(GlobalValue::HiddenVisibility);
 599       std::vector<std::pair<Function *, unsigned>> InitsOrDeInits;
 600       auto COrDtors = isCtor ? getConstructors(M) : getDestructors(M);
 601
 602       for (auto E : COrDtors)
 603         InitsOrDeInits.push_back(std::make_pair(E.Func, E.Priority));
 604       llvm::sort(InitsOrDeInits, llvm::less_second());
 605
 606       auto *InitOrDeInitFuncEntryBlock =
 607           BasicBlock::Create(Ctx, "entry", InitOrDeInitFunc);
 608       IRBuilder<> IB(InitOrDeInitFuncEntryBlock);
 609       for (auto &KV : InitsOrDeInits)
 610         IB.CreateCall(KV.first);
 611       IB.CreateRetVoid();
 612
 613       if (isCtor)
 614         PS.registerInitFunc(R.getTargetJITDylib(), InternedInitOrDeInitName);
 615       else
 616         PS.registerDeInitFunc(R.getTargetJITDylib(), InternedInitOrDeInitName);
 617
 618       GlobalCOrDtors->eraseFromParent();
 619       return Error::success();
 620     };
 621
 622     if (auto Err = RegisterCOrDtors(GlobalCtors, true))
 623       return Err;
 624     if (auto Err = RegisterCOrDtors(GlobalDtors, false))
 625       return Err;
 626
 627     return Error::success();
 628   });
 629
 630   if (Err)
 631     return std::move(Err);
 632
 633   return std::move(TSM);
 634 }
 635
 636 /// Inactive Platform Support
 637 ///
 638 /// Explicitly disables platform support. JITDylibs are not scanned for special
 639 /// init/deinit symbols. No runtime API interposes are injected.
 640 class InactivePlatformSupport : public LLJIT::PlatformSupport {
 641 public:
 642   InactivePlatformSupport() = default;
 643
 644   Error initialize(JITDylib &JD) override {
 645     LLVM_DEBUG(dbgs() << "InactivePlatformSupport: no initializers running for "
 646                       << JD.getName() << "\n");
 647     return Error::success();
 648   }
 649
 650   Error deinitialize(JITDylib &JD) override {
 651     LLVM_DEBUG(
 652         dbgs() << "InactivePlatformSupport: no deinitializers running for "
 653                << JD.getName() << "\n");
 654     return Error::success();
 655   }
 656 };
 657
 658 } // end anonymous namespace
 659
 660 namespace llvm {
 661 namespace orc {
 662
 663 void LLJIT::PlatformSupport::setInitTransform(
 664     LLJIT &J, IRTransformLayer::TransformFunction T) {
 665   J.InitHelperTransformLayer->setTransform(std::move(T));
 666 }
 667
 668 LLJIT::PlatformSupport::~PlatformSupport() = default;
 669
 670 Error LLJITBuilderState::prepareForConstruction() {
 671
 672   LLVM_DEBUG(dbgs() << "Preparing to create LLJIT instance...\n");
 673
 674   if (!JTMB) {
 675     LLVM_DEBUG({
 676       dbgs() << "  No explicitly set JITTargetMachineBuilder. "
 677                 "Detecting host...\n";
 678     });
 679     if (auto JTMBOrErr = JITTargetMachineBuilder::detectHost())
 680       JTMB = std::move(*JTMBOrErr);
 681     else
 682       return JTMBOrErr.takeError();
 683   }
 684
 685   LLVM_DEBUG({
 686     dbgs() << "  JITTargetMachineBuilder is "
 687            << JITTargetMachineBuilderPrinter(*JTMB, "  ")
 688            << "  Pre-constructed ExecutionSession: " << (ES ? "Yes" : "No")
 689            << "\n"
 690            << "  DataLayout: ";
 691     if (DL)
 692       dbgs() << DL->getStringRepresentation() << "\n";
 693     else
 694       dbgs() << "None (will be created by JITTargetMachineBuilder)\n";
 695
 696     dbgs() << "  Custom object-linking-layer creator: "
 697            << (CreateObjectLinkingLayer ? "Yes" : "No") << "\n"
 698            << "  Custom compile-function creator: "
 699            << (CreateCompileFunction ? "Yes" : "No") << "\n"
 700            << "  Custom platform-setup function: "
 701            << (SetUpPlatform ? "Yes" : "No") << "\n"
 702            << "  Number of compile threads: " << NumCompileThreads;
 703     if (!NumCompileThreads)
 704       dbgs() << " (code will be compiled on the execution thread)\n";
 705     else
 706       dbgs() << "\n";
 707   });
 708
 709   // Create DL if not specified.
 710   if (!DL) {
 711     if (auto DLOrErr = JTMB->getDefaultDataLayoutForTarget())
 712       DL = std::move(*DLOrErr);
 713     else
 714       return DLOrErr.takeError();
 715   }
 716
 717   // If neither ES nor EPC has been set then create an EPC instance.
 718   if (!ES && !EPC) {
 719     LLVM_DEBUG({
 720       dbgs() << "ExecutorProcessControl not specified, "
 721                 "Creating SelfExecutorProcessControl instance\n";
 722     });
 723     if (auto EPCOrErr = SelfExecutorProcessControl::Create())
 724       EPC = std::move(*EPCOrErr);
 725     else
 726       return EPCOrErr.takeError();
 727   } else if (EPC) {
 728     LLVM_DEBUG({
 729       dbgs() << "Using explicitly specified ExecutorProcessControl instance "
 730              << EPC.get() << "\n";
 731     });
 732   } else {
 733     LLVM_DEBUG({
 734       dbgs() << "Using explicitly specified ExecutionSession instance "
 735              << ES.get() << "\n";
 736     });
 737   }
 738
 739   // If the client didn't configure any linker options then auto-configure the
 740   // JIT linker.
 741   if (!CreateObjectLinkingLayer) {
 742     auto &TT = JTMB->getTargetTriple();
 743     bool UseJITLink = false;
 744     switch (TT.getArch()) {
 745     case Triple::riscv64:
 746     case Triple::loongarch64:
 747       UseJITLink = true;
 748       break;
 749     case Triple::aarch64:
 750       UseJITLink = !TT.isOSBinFormatCOFF();
 751       break;
 752     case Triple::x86_64:
 753       UseJITLink = !TT.isOSBinFormatCOFF();
 754       break;
 755     case Triple::ppc64:
 756       UseJITLink = TT.isPPC64ELFv2ABI();
 757       break;
 758     case Triple::ppc64le:
 759       UseJITLink = TT.isOSBinFormatELF();
 760       break;
 761     default:
 762       break;
 763     }
 764     if (UseJITLink) {
 765       JTMB->setRelocationModel(Reloc::PIC_);
 766       JTMB->setCodeModel(CodeModel::Small);
 767       CreateObjectLinkingLayer =
 768           [](ExecutionSession &ES,
 769              const Triple &) -> Expected<std::unique_ptr<ObjectLayer>> {
 770         auto ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>(ES);
 771         if (auto EHFrameRegistrar = EPCEHFrameRegistrar::Create(ES))
 772           ObjLinkingLayer->addPlugin(
 773               std::make_unique<EHFrameRegistrationPlugin>(
 774                   ES, std::move(*EHFrameRegistrar)));
 775         else
 776           return EHFrameRegistrar.takeError();
 777         return std::move(ObjLinkingLayer);
 778       };
 779     }
 780   }
 781
 782   // If we need a process JITDylib but no setup function has been given then
 783   // create a default one.
 784   if (!SetupProcessSymbolsJITDylib &&
 785       (LinkProcessSymbolsByDefault || EnableDebuggerSupport)) {
 786
 787     LLVM_DEBUG({
 788       dbgs() << "Creating default Process JD setup function (neeeded for";
 789       if (LinkProcessSymbolsByDefault)
 790         dbgs() << " <link-process-syms-by-default>";
 791       if (EnableDebuggerSupport)
 792         dbgs() << " <debugger-support>";
 793       dbgs() << ")\n";
 794     });
 795
 796     SetupProcessSymbolsJITDylib = [this](LLJIT &J) -> Expected<JITDylibSP> {
 797       auto &JD =
 798           J.getExecutionSession().createBareJITDylib("<Process Symbols>");
 799       auto G = orc::DynamicLibrarySearchGenerator::GetForCurrentProcess(
 800           DL->getGlobalPrefix());
 801       if (!G)
 802         return G.takeError();
 803       JD.addGenerator(std::move(*G));
 804       return &JD;
 805     };
 806   }
 807
 808   return Error::success();
 809 }
 810
 811 LLJIT::~LLJIT() {
 812   if (CompileThreads)
 813     CompileThreads->wait();
 814   if (auto Err = ES->endSession())
 815     ES->reportError(std::move(Err));
 816 }
 817
 818 JITDylibSP LLJIT::getProcessSymbolsJITDylib() { return ProcessSymbols; }
 819
 820 JITDylibSP LLJIT::getPlatformJITDylib() { return Platform; }
 821
 822 Expected<JITDylib &> LLJIT::createJITDylib(std::string Name) {
 823   auto JD = ES->createJITDylib(std::move(Name));
 824   if (!JD)
 825     return JD.takeError();
 826
 827   JD->addToLinkOrder(DefaultLinks);
 828   return JD;
 829 }
 830
 831 Expected<JITDylib &> LLJIT::loadPlatformDynamicLibrary(const char *Path) {
 832   auto G = EPCDynamicLibrarySearchGenerator::Load(*ES, Path);
 833   if (!G)
 834     return G.takeError();
 835
 836   if (auto *ExistingJD = ES->getJITDylibByName(Path))
 837     return *ExistingJD;
 838
 839   auto &JD = ES->createBareJITDylib(Path);
 840   JD.addGenerator(std::move(*G));
 841   return JD;
 842 }
 843
 844 Error LLJIT::linkStaticLibraryInto(JITDylib &JD,
 845                                    std::unique_ptr<MemoryBuffer> LibBuffer) {
 846   auto G = StaticLibraryDefinitionGenerator::Create(*ObjLinkingLayer,
 847                                                     std::move(LibBuffer));
 848   if (!G)
 849     return G.takeError();
 850
 851   JD.addGenerator(std::move(*G));
 852
 853   return Error::success();
 854 }
 855
 856 Error LLJIT::linkStaticLibraryInto(JITDylib &JD, const char *Path) {
 857   auto G = StaticLibraryDefinitionGenerator::Load(*ObjLinkingLayer, Path);
 858   if (!G)
 859     return G.takeError();
 860
 861   JD.addGenerator(std::move(*G));
 862
 863   return Error::success();
 864 }
 865
 866 Error LLJIT::addIRModule(ResourceTrackerSP RT, ThreadSafeModule TSM) {
 867   assert(TSM && "Can not add null module");
 868
 869   if (auto Err =
 870           TSM.withModuleDo([&](Module &M) { return applyDataLayout(M); }))
 871     return Err;
 872
 873   return InitHelperTransformLayer->add(std::move(RT), std::move(TSM));
 874 }
 875
 876 Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) {
 877   return addIRModule(JD.getDefaultResourceTracker(), std::move(TSM));
 878 }
 879
 880 Error LLJIT::addObjectFile(ResourceTrackerSP RT,
 881                            std::unique_ptr<MemoryBuffer> Obj) {
 882   assert(Obj && "Can not add null object");
 883
 884   return ObjTransformLayer->add(std::move(RT), std::move(Obj));
 885 }
 886
 887 Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) {
 888   return addObjectFile(JD.getDefaultResourceTracker(), std::move(Obj));
 889 }
 890
 891 Expected<ExecutorAddr> LLJIT::lookupLinkerMangled(JITDylib &JD,
 892                                                   SymbolStringPtr Name) {
 893   if (auto Sym = ES->lookup(
 894         makeJITDylibSearchOrder(&JD, JITDylibLookupFlags::MatchAllSymbols),
 895         Name))
 896     return Sym->getAddress();
 897   else
 898     return Sym.takeError();
 899 }
 900
 901 Expected<std::unique_ptr<ObjectLayer>>
 902 LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) {
 903
 904   // If the config state provided an ObjectLinkingLayer factory then use it.
 905   if (S.CreateObjectLinkingLayer)
 906     return S.CreateObjectLinkingLayer(ES, S.JTMB->getTargetTriple());
 907
 908   // Otherwise default to creating an RTDyldObjectLinkingLayer that constructs
 909   // a new SectionMemoryManager for each object.
 910   auto GetMemMgr = []() { return std::make_unique<SectionMemoryManager>(); };
 911   auto Layer =
 912       std::make_unique<RTDyldObjectLinkingLayer>(ES, std::move(GetMemMgr));
 913
 914   if (S.JTMB->getTargetTriple().isOSBinFormatCOFF()) {
 915     Layer->setOverrideObjectFlagsWithResponsibilityFlags(true);
 916     Layer->setAutoClaimResponsibilityForObjectSymbols(true);
 917   }
 918
 919   if (S.JTMB->getTargetTriple().isOSBinFormatELF() &&
 920       (S.JTMB->getTargetTriple().getArch() == Triple::ArchType::ppc64 ||
 921        S.JTMB->getTargetTriple().getArch() == Triple::ArchType::ppc64le))
 922     Layer->setAutoClaimResponsibilityForObjectSymbols(true);
 923
 924   // FIXME: Explicit conversion to std::unique_ptr<ObjectLayer> added to silence
 925   //        errors from some GCC / libstdc++ bots. Remove this conversion (i.e.
 926   //        just return ObjLinkingLayer) once those bots are upgraded.
 927   return std::unique_ptr<ObjectLayer>(std::move(Layer));
 928 }
 929
 930 Expected<std::unique_ptr<IRCompileLayer::IRCompiler>>
 931 LLJIT::createCompileFunction(LLJITBuilderState &S,
 932                              JITTargetMachineBuilder JTMB) {
 933
 934   /// If there is a custom compile function creator set then use it.
 935   if (S.CreateCompileFunction)
 936     return S.CreateCompileFunction(std::move(JTMB));
 937
 938   // Otherwise default to creating a SimpleCompiler, or ConcurrentIRCompiler,
 939   // depending on the number of threads requested.
 940   if (S.NumCompileThreads > 0)
 941     return std::make_unique<ConcurrentIRCompiler>(std::move(JTMB));
 942
 943   auto TM = JTMB.createTargetMachine();
 944   if (!TM)
 945     return TM.takeError();
 946
 947   return std::make_unique<TMOwningSimpleCompiler>(std::move(*TM));
 948 }
 949
 950 LLJIT::LLJIT(LLJITBuilderState &S, Error &Err)
 951     : DL(std::move(*S.DL)), TT(S.JTMB->getTargetTriple()) {
 952
 953   ErrorAsOutParameter _(&Err);
 954
 955   assert(!(S.EPC && S.ES) && "EPC and ES should not both be set");
 956
 957   if (S.EPC) {
 958     ES = std::make_unique<ExecutionSession>(std::move(S.EPC));
 959   } else if (S.ES)
 960     ES = std::move(S.ES);
 961   else {
 962     if (auto EPC = SelfExecutorProcessControl::Create()) {
 963       ES = std::make_unique<ExecutionSession>(std::move(*EPC));
 964     } else {
 965       Err = EPC.takeError();
 966       return;
 967     }
 968   }
 969
 970   auto ObjLayer = createObjectLinkingLayer(S, *ES);
 971   if (!ObjLayer) {
 972     Err = ObjLayer.takeError();
 973     return;
 974   }
 975   ObjLinkingLayer = std::move(*ObjLayer);
 976   ObjTransformLayer =
 977       std::make_unique<ObjectTransformLayer>(*ES, *ObjLinkingLayer);
 978
 979   {
 980     auto CompileFunction = createCompileFunction(S, std::move(*S.JTMB));
 981     if (!CompileFunction) {
 982       Err = CompileFunction.takeError();
 983       return;
 984     }
 985     CompileLayer = std::make_unique<IRCompileLayer>(
 986         *ES, *ObjTransformLayer, std::move(*CompileFunction));
 987     TransformLayer = std::make_unique<IRTransformLayer>(*ES, *CompileLayer);
 988     InitHelperTransformLayer =
 989         std::make_unique<IRTransformLayer>(*ES, *TransformLayer);
 990   }
 991
 992   if (S.NumCompileThreads > 0) {
 993     InitHelperTransformLayer->setCloneToNewContextOnEmit(true);
 994     CompileThreads =
 995         std::make_unique<ThreadPool>(hardware_concurrency(S.NumCompileThreads));
 996     ES->setDispatchTask([this](std::unique_ptr<Task> T) {
 997       // FIXME: We should be able to use move-capture here, but ThreadPool's
 998       // AsyncTaskTys are std::functions rather than unique_functions
 999       // (because MSVC's std::packaged_tasks don't support move-only types).
1000       // Fix this when all the above gets sorted out.
1001       CompileThreads->async([UnownedT = T.release()]() mutable {
1002         std::unique_ptr<Task> T(UnownedT);
1003         T->run();
1004       });
1005     });
1006   }
1007
1008   if (S.SetupProcessSymbolsJITDylib) {
1009     if (auto ProcSymsJD = S.SetupProcessSymbolsJITDylib(*this)) {
1010       ProcessSymbols = ProcSymsJD->get();
1011     } else {
1012       Err = ProcSymsJD.takeError();
1013       return;
1014     }
1015   }
1016
1017   if (S.EnableDebuggerSupport) {
1018     if (auto *OLL = dyn_cast<ObjectLinkingLayer>(ObjLinkingLayer.get())) {
1019       switch (TT.getObjectFormat()) {
1020       case Triple::ELF: {
1021         auto Registrar = createJITLoaderGDBRegistrar(*ES);
1022         if (!Registrar) {
1023           Err = Registrar.takeError();
1024           return;
1025         }
1026         OLL->addPlugin(std::make_unique<DebugObjectManagerPlugin>(
1027             *ES, std::move(*Registrar), true, true));
1028         break;
1029       }
1030       case Triple::MachO: {
1031         assert(ProcessSymbols && "ProcessSymbols JD should be available when "
1032                                  "EnableDebuggerSupport is set");
1033         auto DS =
1034             GDBJITDebugInfoRegistrationPlugin::Create(*ES, *ProcessSymbols, TT);
1035         if (!DS) {
1036           Err = DS.takeError();
1037           return;
1038         }
1039         OLL->addPlugin(std::move(*DS));
1040         break;
1041       }
1042       default:
1043         LLVM_DEBUG({
1044           dbgs() << "Cannot enable LLJIT debugger support: "
1045                  << Triple::getObjectFormatTypeName(TT.getObjectFormat())
1046                  << " not supported.\n";
1047         });
1048       }
1049     } else {
1050       LLVM_DEBUG({
1051         dbgs() << "Cannot enable LLJIT debugger support: "
1052                   " debugger support is only available when using JITLink.\n";
1053       });
1054     }
1055   }
1056
1057   if (S.PrePlatformSetup) {
1058     if (auto Err2 = S.PrePlatformSetup(*this)) {
1059       Err = std::move(Err2);
1060       return;
1061     }
1062   }
1063
1064   if (!S.SetUpPlatform)
1065     S.SetUpPlatform = setUpGenericLLVMIRPlatform;
1066
1067   if (auto PlatformJDOrErr = S.SetUpPlatform(*this)) {
1068     Platform = PlatformJDOrErr->get();
1069     if (Platform)
1070       DefaultLinks.push_back(
1071           {Platform, JITDylibLookupFlags::MatchExportedSymbolsOnly});
1072   } else {
1073     Err = PlatformJDOrErr.takeError();
1074     return;
1075   }
1076
1077   if (S.LinkProcessSymbolsByDefault)
1078     DefaultLinks.push_back(
1079         {ProcessSymbols, JITDylibLookupFlags::MatchExportedSymbolsOnly});
1080
1081   if (auto MainOrErr = createJITDylib("main"))
1082     Main = &*MainOrErr;
1083   else {
1084     Err = MainOrErr.takeError();
1085     return;
1086   }
1087 }
1088
1089 std::string LLJIT::mangle(StringRef UnmangledName) const {
1090   std::string MangledName;
1091   {
1092     raw_string_ostream MangledNameStream(MangledName);
1093     Mangler::getNameWithPrefix(MangledNameStream, UnmangledName, DL);
1094   }
1095   return MangledName;
1096 }
1097
1098 Error LLJIT::applyDataLayout(Module &M) {
1099   if (M.getDataLayout().isDefault())
1100     M.setDataLayout(DL);
1101
1102   if (M.getDataLayout() != DL)
1103     return make_error<StringError>(
1104         "Added modules have incompatible data layouts: " +
1105             M.getDataLayout().getStringRepresentation() + " (module) vs " +
1106             DL.getStringRepresentation() + " (jit)",
1107         inconvertibleErrorCode());
1108
1109   return Error::success();
1110 }
1111
1112 Error setUpOrcPlatformManually(LLJIT &J) {
1113   LLVM_DEBUG({ dbgs() << "Setting up orc platform support for LLJIT\n"; });
1114   J.setPlatformSupport(std::make_unique<ORCPlatformSupport>(J));
1115   return Error::success();
1116 }
1117
1118 class LoadAndLinkDynLibrary {
1119 public:
1120   LoadAndLinkDynLibrary(LLJIT &J) : J(J) {}
1121   Error operator()(JITDylib &JD, StringRef DLLName) {
1122     if (!DLLName.ends_with_insensitive(".dll"))
1123       return make_error<StringError>("DLLName not ending with .dll",
1124                                      inconvertibleErrorCode());
1125     auto DLLNameStr = DLLName.str(); // Guarantees null-termination.
1126     auto DLLJD = J.loadPlatformDynamicLibrary(DLLNameStr.c_str());
1127     if (!DLLJD)
1128       return DLLJD.takeError();
1129     JD.addToLinkOrder(*DLLJD);
1130     return Error::success();
1131   }
1132
1133 private:
1134   LLJIT &J;
1135 };
1136
1137 Expected<JITDylibSP> ExecutorNativePlatform::operator()(LLJIT &J) {
1138   auto ProcessSymbolsJD = J.getProcessSymbolsJITDylib();
1139   if (!ProcessSymbolsJD)
1140     return make_error<StringError>(
1141         "Native platforms require a process symbols JITDylib",
1142         inconvertibleErrorCode());
1143
1144   const Triple &TT = J.getTargetTriple();
1145   ObjectLinkingLayer *ObjLinkingLayer =
1146       dyn_cast<ObjectLinkingLayer>(&J.getObjLinkingLayer());
1147
1148   if (!ObjLinkingLayer)
1149     return make_error<StringError>(
1150         "ExecutorNativePlatform requires ObjectLinkingLayer",
1151         inconvertibleErrorCode());
1152
1153   std::unique_ptr<MemoryBuffer> RuntimeArchiveBuffer;
1154   if (OrcRuntime.index() == 0) {
1155     auto A = errorOrToExpected(MemoryBuffer::getFile(std::get<0>(OrcRuntime)));
1156     if (!A)
1157       return A.takeError();
1158     RuntimeArchiveBuffer = std::move(*A);
1159   } else
1160     RuntimeArchiveBuffer = std::move(std::get<1>(OrcRuntime));
1161
1162   auto &ES = J.getExecutionSession();
1163   auto &PlatformJD = ES.createBareJITDylib("<Platform>");
1164   PlatformJD.addToLinkOrder(*ProcessSymbolsJD);
1165
1166   J.setPlatformSupport(std::make_unique<ORCPlatformSupport>(J));
1167
1168   switch (TT.getObjectFormat()) {
1169   case Triple::COFF: {
1170     const char *VCRuntimePath = nullptr;
1171     bool StaticVCRuntime = false;
1172     if (VCRuntime) {
1173       VCRuntimePath = VCRuntime->first.c_str();
1174       StaticVCRuntime = VCRuntime->second;
1175     }
1176     if (auto P = COFFPlatform::Create(
1177             ES, *ObjLinkingLayer, PlatformJD, std::move(RuntimeArchiveBuffer),
1178             LoadAndLinkDynLibrary(J), StaticVCRuntime, VCRuntimePath))
1179       J.getExecutionSession().setPlatform(std::move(*P));
1180     else
1181       return P.takeError();
1182     break;
1183   }
1184   case Triple::ELF: {
1185     auto G = StaticLibraryDefinitionGenerator::Create(
1186         *ObjLinkingLayer, std::move(RuntimeArchiveBuffer));
1187     if (!G)
1188       return G.takeError();
1189
1190     if (auto P = ELFNixPlatform::Create(ES, *ObjLinkingLayer, PlatformJD,
1191                                         std::move(*G)))
1192       J.getExecutionSession().setPlatform(std::move(*P));
1193     else
1194       return P.takeError();
1195     break;
1196   }
1197   case Triple::MachO: {
1198     auto G = StaticLibraryDefinitionGenerator::Create(
1199         *ObjLinkingLayer, std::move(RuntimeArchiveBuffer));
1200     if (!G)
1201       return G.takeError();
1202
1203     if (auto P = MachOPlatform::Create(ES, *ObjLinkingLayer, PlatformJD,
1204                                        std::move(*G)))
1205       ES.setPlatform(std::move(*P));
1206     else
1207       return P.takeError();
1208     break;
1209   }
1210   default:
1211     return make_error<StringError>("Unsupported object format in triple " +
1212                                        TT.str(),
1213                                    inconvertibleErrorCode());
1214   }
1215
1216   return &PlatformJD;
1217 }
1218
1219 Expected<JITDylibSP> setUpGenericLLVMIRPlatform(LLJIT &J) {
1220   LLVM_DEBUG(
1221       { dbgs() << "Setting up GenericLLVMIRPlatform support for LLJIT\n"; });
1222   auto ProcessSymbolsJD = J.getProcessSymbolsJITDylib();
1223   if (!ProcessSymbolsJD)
1224     return make_error<StringError>(
1225         "Native platforms require a process symbols JITDylib",
1226         inconvertibleErrorCode());
1227
1228   auto &PlatformJD = J.getExecutionSession().createBareJITDylib("<Platform>");
1229   PlatformJD.addToLinkOrder(*ProcessSymbolsJD);
1230
1231   J.setPlatformSupport(
1232       std::make_unique<GenericLLVMIRPlatformSupport>(J, PlatformJD));
1233
1234   return &PlatformJD;
1235 }
1236
1237 Expected<JITDylibSP> setUpInactivePlatform(LLJIT &J) {
1238   LLVM_DEBUG(
1239       { dbgs() << "Explicitly deactivated platform support for LLJIT\n"; });
1240   J.setPlatformSupport(std::make_unique<InactivePlatformSupport>());
1241   return nullptr;
1242 }
1243
1244 Error LLLazyJITBuilderState::prepareForConstruction() {
1245   if (auto Err = LLJITBuilderState::prepareForConstruction())
1246     return Err;
1247   TT = JTMB->getTargetTriple();
1248   return Error::success();
1249 }
1250
1251 Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) {
1252   assert(TSM && "Can not add null module");
1253
1254   if (auto Err = TSM.withModuleDo(
1255           [&](Module &M) -> Error { return applyDataLayout(M); }))
1256     return Err;
1257
1258   return CODLayer->add(JD, std::move(TSM));
1259 }
1260
1261 LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) {
1262
1263   // If LLJIT construction failed then bail out.
1264   if (Err)
1265     return;
1266
1267   ErrorAsOutParameter _(&Err);
1268
1269   /// Take/Create the lazy-compile callthrough manager.
1270   if (S.LCTMgr)
1271     LCTMgr = std::move(S.LCTMgr);
1272   else {
1273     if (auto LCTMgrOrErr = createLocalLazyCallThroughManager(
1274             S.TT, *ES, S.LazyCompileFailureAddr))
1275       LCTMgr = std::move(*LCTMgrOrErr);
1276     else {
1277       Err = LCTMgrOrErr.takeError();
1278       return;
1279     }
1280   }
1281
1282   // Take/Create the indirect stubs manager builder.
1283   auto ISMBuilder = std::move(S.ISMBuilder);
1284
1285   // If none was provided, try to build one.
1286   if (!ISMBuilder)
1287     ISMBuilder = createLocalIndirectStubsManagerBuilder(S.TT);
1288
1289   // No luck. Bail out.
1290   if (!ISMBuilder) {
1291     Err = make_error<StringError>("Could not construct "
1292                                   "IndirectStubsManagerBuilder for target " +
1293                                       S.TT.str(),
1294                                   inconvertibleErrorCode());
1295     return;
1296   }
1297
1298   // Create the COD layer.
1299   CODLayer = std::make_unique<CompileOnDemandLayer>(
1300       *ES, *InitHelperTransformLayer, *LCTMgr, std::move(ISMBuilder));
1301
1302   if (S.NumCompileThreads > 0)
1303     CODLayer->setCloneToNewContextOnEmit(true);
1304 }
1305
1306 // In-process LLJIT uses eh-frame section wrappers via EPC, so we need to force
1307 // them to be linked in.
1308 LLVM_ATTRIBUTE_USED void linkComponents() {
1309   errs() << (void *)&llvm_orc_registerEHFrameSectionWrapper
1310          << (void *)&llvm_orc_deregisterEHFrameSectionWrapper;
1311 }
1312
1313 } // End namespace orc.
1314 } // End namespace llvm.