llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp

   1 //===------- EPCIndirectionUtils.cpp -- EPC based indirection APIs --------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h"
  10
  11 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
  12 #include "llvm/Support/MathExtras.h"
  13
  14 #include <future>
  15
  16 using namespace llvm;
  17 using namespace llvm::orc;
  18
  19 namespace llvm {
  20 namespace orc {
  21
  22 class EPCIndirectionUtilsAccess {
  23 public:
  24   using IndirectStubInfo = EPCIndirectionUtils::IndirectStubInfo;
  25   using IndirectStubInfoVector = EPCIndirectionUtils::IndirectStubInfoVector;
  26
  27   static Expected<IndirectStubInfoVector>
  28   getIndirectStubs(EPCIndirectionUtils &EPCIU, unsigned NumStubs) {
  29     return EPCIU.getIndirectStubs(NumStubs);
  30   };
  31 };
  32
  33 } // end namespace orc
  34 } // end namespace llvm
  35
  36 namespace {
  37
  38 class EPCTrampolinePool : public TrampolinePool {
  39 public:
  40   EPCTrampolinePool(EPCIndirectionUtils &EPCIU);
  41   Error deallocatePool();
  42
  43 protected:
  44   Error grow() override;
  45
  46   using FinalizedAlloc = jitlink::JITLinkMemoryManager::FinalizedAlloc;
  47
  48   EPCIndirectionUtils &EPCIU;
  49   unsigned TrampolineSize = 0;
  50   unsigned TrampolinesPerPage = 0;
  51   std::vector<FinalizedAlloc> TrampolineBlocks;
  52 };
  53
  54 class EPCIndirectStubsManager : public IndirectStubsManager,
  55                                 private EPCIndirectionUtilsAccess {
  56 public:
  57   EPCIndirectStubsManager(EPCIndirectionUtils &EPCIU) : EPCIU(EPCIU) {}
  58
  59   Error deallocateStubs();
  60
  61   Error createStub(StringRef StubName, ExecutorAddr StubAddr,
  62                    JITSymbolFlags StubFlags) override;
  63
  64   Error createStubs(const StubInitsMap &StubInits) override;
  65
  66   ExecutorSymbolDef findStub(StringRef Name, bool ExportedStubsOnly) override;
  67
  68   ExecutorSymbolDef findPointer(StringRef Name) override;
  69
  70   Error updatePointer(StringRef Name, ExecutorAddr NewAddr) override;
  71
  72 private:
  73   using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>;
  74
  75   std::mutex ISMMutex;
  76   EPCIndirectionUtils &EPCIU;
  77   StringMap<StubInfo> StubInfos;
  78 };
  79
  80 EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU)
  81     : EPCIU(EPCIU) {
  82   auto &EPC = EPCIU.getExecutorProcessControl();
  83   auto &ABI = EPCIU.getABISupport();
  84
  85   TrampolineSize = ABI.getTrampolineSize();
  86   TrampolinesPerPage =
  87       (EPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize;
  88 }
  89
  90 Error EPCTrampolinePool::deallocatePool() {
  91   std::promise<MSVCPError> DeallocResultP;
  92   auto DeallocResultF = DeallocResultP.get_future();
  93
  94   EPCIU.getExecutorProcessControl().getMemMgr().deallocate(
  95       std::move(TrampolineBlocks),
  96       [&](Error Err) { DeallocResultP.set_value(std::move(Err)); });
  97
  98   return DeallocResultF.get();
  99 }
 100
 101 Error EPCTrampolinePool::grow() {
 102   using namespace jitlink;
 103
 104   assert(AvailableTrampolines.empty() &&
 105          "Grow called with trampolines still available");
 106
 107   auto ResolverAddress = EPCIU.getResolverBlockAddress();
 108   assert(ResolverAddress && "Resolver address can not be null");
 109
 110   auto &EPC = EPCIU.getExecutorProcessControl();
 111   auto PageSize = EPC.getPageSize();
 112   auto Alloc = SimpleSegmentAlloc::Create(
 113       EPC.getMemMgr(), nullptr,
 114       {{MemProt::Read | MemProt::Exec, {PageSize, Align(PageSize)}}});
 115   if (!Alloc)
 116     return Alloc.takeError();
 117
 118   unsigned NumTrampolines = TrampolinesPerPage;
 119
 120   auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec);
 121   EPCIU.getABISupport().writeTrampolines(
 122       SegInfo.WorkingMem.data(), SegInfo.Addr, ResolverAddress, NumTrampolines);
 123   for (unsigned I = 0; I < NumTrampolines; ++I)
 124     AvailableTrampolines.push_back(SegInfo.Addr + (I * TrampolineSize));
 125
 126   auto FA = Alloc->finalize();
 127   if (!FA)
 128     return FA.takeError();
 129
 130   TrampolineBlocks.push_back(std::move(*FA));
 131
 132   return Error::success();
 133 }
 134
 135 Error EPCIndirectStubsManager::createStub(StringRef StubName,
 136                                           ExecutorAddr StubAddr,
 137                                           JITSymbolFlags StubFlags) {
 138   StubInitsMap SIM;
 139   SIM[StubName] = std::make_pair(StubAddr, StubFlags);
 140   return createStubs(SIM);
 141 }
 142
 143 Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) {
 144   auto AvailableStubInfos = getIndirectStubs(EPCIU, StubInits.size());
 145   if (!AvailableStubInfos)
 146     return AvailableStubInfos.takeError();
 147
 148   {
 149     std::lock_guard<std::mutex> Lock(ISMMutex);
 150     unsigned ASIdx = 0;
 151     for (auto &SI : StubInits) {
 152       auto &A = (*AvailableStubInfos)[ASIdx++];
 153       StubInfos[SI.first()] = std::make_pair(A, SI.second.second);
 154     }
 155   }
 156
 157   auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess();
 158   switch (EPCIU.getABISupport().getPointerSize()) {
 159   case 4: {
 160     unsigned ASIdx = 0;
 161     std::vector<tpctypes::UInt32Write> PtrUpdates;
 162     for (auto &SI : StubInits)
 163       PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress,
 164                             static_cast<uint32_t>(SI.second.first.getValue())});
 165     return MemAccess.writeUInt32s(PtrUpdates);
 166   }
 167   case 8: {
 168     unsigned ASIdx = 0;
 169     std::vector<tpctypes::UInt64Write> PtrUpdates;
 170     for (auto &SI : StubInits)
 171       PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress,
 172                             static_cast<uint64_t>(SI.second.first.getValue())});
 173     return MemAccess.writeUInt64s(PtrUpdates);
 174   }
 175   default:
 176     return make_error<StringError>("Unsupported pointer size",
 177                                    inconvertibleErrorCode());
 178   }
 179 }
 180
 181 ExecutorSymbolDef EPCIndirectStubsManager::findStub(StringRef Name,
 182                                                     bool ExportedStubsOnly) {
 183   std::lock_guard<std::mutex> Lock(ISMMutex);
 184   auto I = StubInfos.find(Name);
 185   if (I == StubInfos.end())
 186     return ExecutorSymbolDef();
 187   return {I->second.first.StubAddress, I->second.second};
 188 }
 189
 190 ExecutorSymbolDef EPCIndirectStubsManager::findPointer(StringRef Name) {
 191   std::lock_guard<std::mutex> Lock(ISMMutex);
 192   auto I = StubInfos.find(Name);
 193   if (I == StubInfos.end())
 194     return ExecutorSymbolDef();
 195   return {I->second.first.PointerAddress, I->second.second};
 196 }
 197
 198 Error EPCIndirectStubsManager::updatePointer(StringRef Name,
 199                                              ExecutorAddr NewAddr) {
 200
 201   ExecutorAddr PtrAddr;
 202   {
 203     std::lock_guard<std::mutex> Lock(ISMMutex);
 204     auto I = StubInfos.find(Name);
 205     if (I == StubInfos.end())
 206       return make_error<StringError>("Unknown stub name",
 207                                      inconvertibleErrorCode());
 208     PtrAddr = I->second.first.PointerAddress;
 209   }
 210
 211   auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess();
 212   switch (EPCIU.getABISupport().getPointerSize()) {
 213   case 4: {
 214     tpctypes::UInt32Write PUpdate(PtrAddr, NewAddr.getValue());
 215     return MemAccess.writeUInt32s(PUpdate);
 216   }
 217   case 8: {
 218     tpctypes::UInt64Write PUpdate(PtrAddr, NewAddr.getValue());
 219     return MemAccess.writeUInt64s(PUpdate);
 220   }
 221   default:
 222     return make_error<StringError>("Unsupported pointer size",
 223                                    inconvertibleErrorCode());
 224   }
 225 }
 226
 227 } // end anonymous namespace.
 228
 229 namespace llvm {
 230 namespace orc {
 231
 232 EPCIndirectionUtils::ABISupport::~ABISupport() = default;
 233
 234 Expected<std::unique_ptr<EPCIndirectionUtils>>
 235 EPCIndirectionUtils::Create(ExecutorProcessControl &EPC) {
 236   const auto &TT = EPC.getTargetTriple();
 237   switch (TT.getArch()) {
 238   default:
 239     return make_error<StringError>(
 240         std::string("No EPCIndirectionUtils available for ") + TT.str(),
 241         inconvertibleErrorCode());
 242   case Triple::aarch64:
 243   case Triple::aarch64_32:
 244     return CreateWithABI<OrcAArch64>(EPC);
 245
 246   case Triple::x86:
 247     return CreateWithABI<OrcI386>(EPC);
 248
 249   case Triple::loongarch64:
 250     return CreateWithABI<OrcLoongArch64>(EPC);
 251
 252   case Triple::mips:
 253     return CreateWithABI<OrcMips32Be>(EPC);
 254
 255   case Triple::mipsel:
 256     return CreateWithABI<OrcMips32Le>(EPC);
 257
 258   case Triple::mips64:
 259   case Triple::mips64el:
 260     return CreateWithABI<OrcMips64>(EPC);
 261
 262   case Triple::riscv64:
 263     return CreateWithABI<OrcRiscv64>(EPC);
 264
 265   case Triple::x86_64:
 266     if (TT.getOS() == Triple::OSType::Win32)
 267       return CreateWithABI<OrcX86_64_Win32>(EPC);
 268     else
 269       return CreateWithABI<OrcX86_64_SysV>(EPC);
 270   }
 271 }
 272
 273 Error EPCIndirectionUtils::cleanup() {
 274
 275   auto &MemMgr = EPC.getMemMgr();
 276   auto Err = MemMgr.deallocate(std::move(IndirectStubAllocs));
 277
 278   if (TP)
 279     Err = joinErrors(std::move(Err),
 280                      static_cast<EPCTrampolinePool &>(*TP).deallocatePool());
 281
 282   if (ResolverBlock)
 283     Err =
 284         joinErrors(std::move(Err), MemMgr.deallocate(std::move(ResolverBlock)));
 285
 286   return Err;
 287 }
 288
 289 Expected<ExecutorAddr>
 290 EPCIndirectionUtils::writeResolverBlock(ExecutorAddr ReentryFnAddr,
 291                                         ExecutorAddr ReentryCtxAddr) {
 292   using namespace jitlink;
 293
 294   assert(ABI && "ABI can not be null");
 295   auto ResolverSize = ABI->getResolverCodeSize();
 296
 297   auto Alloc =
 298       SimpleSegmentAlloc::Create(EPC.getMemMgr(), nullptr,
 299                                  {{MemProt::Read | MemProt::Exec,
 300                                    {ResolverSize, Align(EPC.getPageSize())}}});
 301
 302   if (!Alloc)
 303     return Alloc.takeError();
 304
 305   auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec);
 306   ResolverBlockAddr = SegInfo.Addr;
 307   ABI->writeResolverCode(SegInfo.WorkingMem.data(), ResolverBlockAddr,
 308                          ReentryFnAddr, ReentryCtxAddr);
 309
 310   auto FA = Alloc->finalize();
 311   if (!FA)
 312     return FA.takeError();
 313
 314   ResolverBlock = std::move(*FA);
 315   return ResolverBlockAddr;
 316 }
 317
 318 std::unique_ptr<IndirectStubsManager>
 319 EPCIndirectionUtils::createIndirectStubsManager() {
 320   return std::make_unique<EPCIndirectStubsManager>(*this);
 321 }
 322
 323 TrampolinePool &EPCIndirectionUtils::getTrampolinePool() {
 324   if (!TP)
 325     TP = std::make_unique<EPCTrampolinePool>(*this);
 326   return *TP;
 327 }
 328
 329 LazyCallThroughManager &EPCIndirectionUtils::createLazyCallThroughManager(
 330     ExecutionSession &ES, ExecutorAddr ErrorHandlerAddr) {
 331   assert(!LCTM &&
 332          "createLazyCallThroughManager can not have been called before");
 333   LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr,
 334                                                   &getTrampolinePool());
 335   return *LCTM;
 336 }
 337
 338 EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC,
 339                                          std::unique_ptr<ABISupport> ABI)
 340     : EPC(EPC), ABI(std::move(ABI)) {
 341   assert(this->ABI && "ABI can not be null");
 342
 343   assert(EPC.getPageSize() > getABISupport().getStubSize() &&
 344          "Stubs larger than one page are not supported");
 345 }
 346
 347 Expected<EPCIndirectionUtils::IndirectStubInfoVector>
 348 EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) {
 349   using namespace jitlink;
 350
 351   std::lock_guard<std::mutex> Lock(EPCUIMutex);
 352
 353   // If there aren't enough stubs available then allocate some more.
 354   if (NumStubs > AvailableIndirectStubs.size()) {
 355     auto NumStubsToAllocate = NumStubs;
 356     auto PageSize = EPC.getPageSize();
 357     auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize);
 358     NumStubsToAllocate = StubBytes / ABI->getStubSize();
 359     auto PtrBytes =
 360         alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize);
 361
 362     auto StubProt = MemProt::Read | MemProt::Exec;
 363     auto PtrProt = MemProt::Read | MemProt::Write;
 364
 365     auto Alloc = SimpleSegmentAlloc::Create(
 366         EPC.getMemMgr(), nullptr,
 367         {{StubProt, {static_cast<size_t>(StubBytes), Align(PageSize)}},
 368          {PtrProt, {static_cast<size_t>(PtrBytes), Align(PageSize)}}});
 369
 370     if (!Alloc)
 371       return Alloc.takeError();
 372
 373     auto StubSeg = Alloc->getSegInfo(StubProt);
 374     auto PtrSeg = Alloc->getSegInfo(PtrProt);
 375
 376     ABI->writeIndirectStubsBlock(StubSeg.WorkingMem.data(), StubSeg.Addr,
 377                                  PtrSeg.Addr, NumStubsToAllocate);
 378
 379     auto FA = Alloc->finalize();
 380     if (!FA)
 381       return FA.takeError();
 382
 383     IndirectStubAllocs.push_back(std::move(*FA));
 384
 385     auto StubExecutorAddr = StubSeg.Addr;
 386     auto PtrExecutorAddr = PtrSeg.Addr;
 387     for (unsigned I = 0; I != NumStubsToAllocate; ++I) {
 388       AvailableIndirectStubs.push_back(
 389           IndirectStubInfo(StubExecutorAddr, PtrExecutorAddr));
 390       StubExecutorAddr += ABI->getStubSize();
 391       PtrExecutorAddr += ABI->getPointerSize();
 392     }
 393   }
 394
 395   assert(NumStubs <= AvailableIndirectStubs.size() &&
 396          "Sufficient stubs should have been allocated above");
 397
 398   IndirectStubInfoVector Result;
 399   while (NumStubs--) {
 400     Result.push_back(AvailableIndirectStubs.back());
 401     AvailableIndirectStubs.pop_back();
 402   }
 403
 404   return std::move(Result);
 405 }
 406
 407 static JITTargetAddress reentry(JITTargetAddress LCTMAddr,
 408                                 JITTargetAddress TrampolineAddr) {
 409   auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr);
 410   std::promise<ExecutorAddr> LandingAddrP;
 411   auto LandingAddrF = LandingAddrP.get_future();
 412   LCTM.resolveTrampolineLandingAddress(
 413       ExecutorAddr(TrampolineAddr),
 414       [&](ExecutorAddr Addr) { LandingAddrP.set_value(Addr); });
 415   return LandingAddrF.get().getValue();
 416 }
 417
 418 Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU) {
 419   auto &LCTM = EPCIU.getLazyCallThroughManager();
 420   return EPCIU
 421       .writeResolverBlock(ExecutorAddr::fromPtr(&reentry),
 422                           ExecutorAddr::fromPtr(&LCTM))
 423       .takeError();
 424 }
 425
 426 } // end namespace orc
 427 } // end namespace llvm