llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp

   1 //===------- EPCIndirectionUtils.cpp -- EPC based indirection APIs --------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h"
  10
  11 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
  12 #include "llvm/Support/MathExtras.h"
  13
  14 #include <future>
  15
  16 using namespace llvm;
  17 using namespace llvm::orc;
  18
  19 namespace llvm {
  20 namespace orc {
  21
  22 class EPCIndirectionUtilsAccess {
  23 public:
  24   using IndirectStubInfo = EPCIndirectionUtils::IndirectStubInfo;
  25   using IndirectStubInfoVector = EPCIndirectionUtils::IndirectStubInfoVector;
  26
  27   static Expected<IndirectStubInfoVector>
  28   getIndirectStubs(EPCIndirectionUtils &EPCIU, unsigned NumStubs) {
  29     return EPCIU.getIndirectStubs(NumStubs);
  30   };
  31 };
  32
  33 } // end namespace orc
  34 } // end namespace llvm
  35
  36 namespace {
  37
  38 class EPCTrampolinePool : public TrampolinePool {
  39 public:
  40   EPCTrampolinePool(EPCIndirectionUtils &EPCIU);
  41   Error deallocatePool();
  42
  43 protected:
  44   Error grow() override;
  45
  46   using Allocation = jitlink::JITLinkMemoryManager::Allocation;
  47
  48   EPCIndirectionUtils &EPCIU;
  49   unsigned TrampolineSize = 0;
  50   unsigned TrampolinesPerPage = 0;
  51   std::vector<std::unique_ptr<Allocation>> TrampolineBlocks;
  52 };
  53
  54 class EPCIndirectStubsManager : public IndirectStubsManager,
  55                                 private EPCIndirectionUtilsAccess {
  56 public:
  57   EPCIndirectStubsManager(EPCIndirectionUtils &EPCIU) : EPCIU(EPCIU) {}
  58
  59   Error deallocateStubs();
  60
  61   Error createStub(StringRef StubName, JITTargetAddress StubAddr,
  62                    JITSymbolFlags StubFlags) override;
  63
  64   Error createStubs(const StubInitsMap &StubInits) override;
  65
  66   JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override;
  67
  68   JITEvaluatedSymbol findPointer(StringRef Name) override;
  69
  70   Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override;
  71
  72 private:
  73   using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>;
  74
  75   std::mutex ISMMutex;
  76   EPCIndirectionUtils &EPCIU;
  77   StringMap<StubInfo> StubInfos;
  78 };
  79
  80 EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU)
  81     : EPCIU(EPCIU) {
  82   auto &EPC = EPCIU.getExecutorProcessControl();
  83   auto &ABI = EPCIU.getABISupport();
  84
  85   TrampolineSize = ABI.getTrampolineSize();
  86   TrampolinesPerPage =
  87       (EPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize;
  88 }
  89
  90 Error EPCTrampolinePool::deallocatePool() {
  91   Error Err = Error::success();
  92   for (auto &Alloc : TrampolineBlocks)
  93     Err = joinErrors(std::move(Err), Alloc->deallocate());
  94   return Err;
  95 }
  96
  97 Error EPCTrampolinePool::grow() {
  98   assert(AvailableTrampolines.empty() &&
  99          "Grow called with trampolines still available");
 100
 101   auto ResolverAddress = EPCIU.getResolverBlockAddress();
 102   assert(ResolverAddress && "Resolver address can not be null");
 103
 104   auto &EPC = EPCIU.getExecutorProcessControl();
 105   constexpr auto TrampolinePagePermissions =
 106       static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
 107                                                 sys::Memory::MF_EXEC);
 108   auto PageSize = EPC.getPageSize();
 109   jitlink::JITLinkMemoryManager::SegmentsRequestMap Request;
 110   Request[TrampolinePagePermissions] = {PageSize, static_cast<size_t>(PageSize),
 111                                         0};
 112   auto Alloc = EPC.getMemMgr().allocate(nullptr, Request);
 113
 114   if (!Alloc)
 115     return Alloc.takeError();
 116
 117   unsigned NumTrampolines = TrampolinesPerPage;
 118
 119   auto WorkingMemory = (*Alloc)->getWorkingMemory(TrampolinePagePermissions);
 120   auto TargetAddress = (*Alloc)->getTargetMemory(TrampolinePagePermissions);
 121
 122   EPCIU.getABISupport().writeTrampolines(WorkingMemory.data(), TargetAddress,
 123                                          ResolverAddress, NumTrampolines);
 124
 125   auto TargetAddr = (*Alloc)->getTargetMemory(TrampolinePagePermissions);
 126   for (unsigned I = 0; I < NumTrampolines; ++I)
 127     AvailableTrampolines.push_back(TargetAddr + (I * TrampolineSize));
 128
 129   if (auto Err = (*Alloc)->finalize())
 130     return Err;
 131
 132   TrampolineBlocks.push_back(std::move(*Alloc));
 133
 134   return Error::success();
 135 }
 136
 137 Error EPCIndirectStubsManager::createStub(StringRef StubName,
 138                                           JITTargetAddress StubAddr,
 139                                           JITSymbolFlags StubFlags) {
 140   StubInitsMap SIM;
 141   SIM[StubName] = std::make_pair(StubAddr, StubFlags);
 142   return createStubs(SIM);
 143 }
 144
 145 Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) {
 146   auto AvailableStubInfos = getIndirectStubs(EPCIU, StubInits.size());
 147   if (!AvailableStubInfos)
 148     return AvailableStubInfos.takeError();
 149
 150   {
 151     std::lock_guard<std::mutex> Lock(ISMMutex);
 152     unsigned ASIdx = 0;
 153     for (auto &SI : StubInits) {
 154       auto &A = (*AvailableStubInfos)[ASIdx++];
 155       StubInfos[SI.first()] = std::make_pair(A, SI.second.second);
 156     }
 157   }
 158
 159   auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess();
 160   switch (EPCIU.getABISupport().getPointerSize()) {
 161   case 4: {
 162     unsigned ASIdx = 0;
 163     std::vector<tpctypes::UInt32Write> PtrUpdates;
 164     for (auto &SI : StubInits)
 165       PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress,
 166                             static_cast<uint32_t>(SI.second.first)});
 167     return MemAccess.writeUInt32s(PtrUpdates);
 168   }
 169   case 8: {
 170     unsigned ASIdx = 0;
 171     std::vector<tpctypes::UInt64Write> PtrUpdates;
 172     for (auto &SI : StubInits)
 173       PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress,
 174                             static_cast<uint64_t>(SI.second.first)});
 175     return MemAccess.writeUInt64s(PtrUpdates);
 176   }
 177   default:
 178     return make_error<StringError>("Unsupported pointer size",
 179                                    inconvertibleErrorCode());
 180   }
 181 }
 182
 183 JITEvaluatedSymbol EPCIndirectStubsManager::findStub(StringRef Name,
 184                                                      bool ExportedStubsOnly) {
 185   std::lock_guard<std::mutex> Lock(ISMMutex);
 186   auto I = StubInfos.find(Name);
 187   if (I == StubInfos.end())
 188     return nullptr;
 189   return {I->second.first.StubAddress, I->second.second};
 190 }
 191
 192 JITEvaluatedSymbol EPCIndirectStubsManager::findPointer(StringRef Name) {
 193   std::lock_guard<std::mutex> Lock(ISMMutex);
 194   auto I = StubInfos.find(Name);
 195   if (I == StubInfos.end())
 196     return nullptr;
 197   return {I->second.first.PointerAddress, I->second.second};
 198 }
 199
 200 Error EPCIndirectStubsManager::updatePointer(StringRef Name,
 201                                              JITTargetAddress NewAddr) {
 202
 203   JITTargetAddress PtrAddr = 0;
 204   {
 205     std::lock_guard<std::mutex> Lock(ISMMutex);
 206     auto I = StubInfos.find(Name);
 207     if (I == StubInfos.end())
 208       return make_error<StringError>("Unknown stub name",
 209                                      inconvertibleErrorCode());
 210     PtrAddr = I->second.first.PointerAddress;
 211   }
 212
 213   auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess();
 214   switch (EPCIU.getABISupport().getPointerSize()) {
 215   case 4: {
 216     tpctypes::UInt32Write PUpdate(PtrAddr, NewAddr);
 217     return MemAccess.writeUInt32s(PUpdate);
 218   }
 219   case 8: {
 220     tpctypes::UInt64Write PUpdate(PtrAddr, NewAddr);
 221     return MemAccess.writeUInt64s(PUpdate);
 222   }
 223   default:
 224     return make_error<StringError>("Unsupported pointer size",
 225                                    inconvertibleErrorCode());
 226   }
 227 }
 228
 229 } // end anonymous namespace.
 230
 231 namespace llvm {
 232 namespace orc {
 233
 234 EPCIndirectionUtils::ABISupport::~ABISupport() {}
 235
 236 Expected<std::unique_ptr<EPCIndirectionUtils>>
 237 EPCIndirectionUtils::Create(ExecutorProcessControl &EPC) {
 238   const auto &TT = EPC.getTargetTriple();
 239   switch (TT.getArch()) {
 240   default:
 241     return make_error<StringError>(
 242         std::string("No EPCIndirectionUtils available for ") + TT.str(),
 243         inconvertibleErrorCode());
 244   case Triple::aarch64:
 245   case Triple::aarch64_32:
 246     return CreateWithABI<OrcAArch64>(EPC);
 247
 248   case Triple::x86:
 249     return CreateWithABI<OrcI386>(EPC);
 250
 251   case Triple::mips:
 252     return CreateWithABI<OrcMips32Be>(EPC);
 253
 254   case Triple::mipsel:
 255     return CreateWithABI<OrcMips32Le>(EPC);
 256
 257   case Triple::mips64:
 258   case Triple::mips64el:
 259     return CreateWithABI<OrcMips64>(EPC);
 260
 261   case Triple::x86_64:
 262     if (TT.getOS() == Triple::OSType::Win32)
 263       return CreateWithABI<OrcX86_64_Win32>(EPC);
 264     else
 265       return CreateWithABI<OrcX86_64_SysV>(EPC);
 266   }
 267 }
 268
 269 Error EPCIndirectionUtils::cleanup() {
 270   Error Err = Error::success();
 271
 272   for (auto &A : IndirectStubAllocs)
 273     Err = joinErrors(std::move(Err), A->deallocate());
 274
 275   if (TP)
 276     Err = joinErrors(std::move(Err),
 277                      static_cast<EPCTrampolinePool &>(*TP).deallocatePool());
 278
 279   if (ResolverBlock)
 280     Err = joinErrors(std::move(Err), ResolverBlock->deallocate());
 281
 282   return Err;
 283 }
 284
 285 Expected<JITTargetAddress>
 286 EPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr,
 287                                         JITTargetAddress ReentryCtxAddr) {
 288   assert(ABI && "ABI can not be null");
 289   constexpr auto ResolverBlockPermissions =
 290       static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
 291                                                 sys::Memory::MF_EXEC);
 292   auto ResolverSize = ABI->getResolverCodeSize();
 293
 294   jitlink::JITLinkMemoryManager::SegmentsRequestMap Request;
 295   Request[ResolverBlockPermissions] = {EPC.getPageSize(),
 296                                        static_cast<size_t>(ResolverSize), 0};
 297   auto Alloc = EPC.getMemMgr().allocate(nullptr, Request);
 298   if (!Alloc)
 299     return Alloc.takeError();
 300
 301   auto WorkingMemory = (*Alloc)->getWorkingMemory(ResolverBlockPermissions);
 302   ResolverBlockAddr = (*Alloc)->getTargetMemory(ResolverBlockPermissions);
 303   ABI->writeResolverCode(WorkingMemory.data(), ResolverBlockAddr, ReentryFnAddr,
 304                          ReentryCtxAddr);
 305
 306   if (auto Err = (*Alloc)->finalize())
 307     return std::move(Err);
 308
 309   ResolverBlock = std::move(*Alloc);
 310   return ResolverBlockAddr;
 311 }
 312
 313 std::unique_ptr<IndirectStubsManager>
 314 EPCIndirectionUtils::createIndirectStubsManager() {
 315   return std::make_unique<EPCIndirectStubsManager>(*this);
 316 }
 317
 318 TrampolinePool &EPCIndirectionUtils::getTrampolinePool() {
 319   if (!TP)
 320     TP = std::make_unique<EPCTrampolinePool>(*this);
 321   return *TP;
 322 }
 323
 324 LazyCallThroughManager &EPCIndirectionUtils::createLazyCallThroughManager(
 325     ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr) {
 326   assert(!LCTM &&
 327          "createLazyCallThroughManager can not have been called before");
 328   LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr,
 329                                                   &getTrampolinePool());
 330   return *LCTM;
 331 }
 332
 333 EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC,
 334                                          std::unique_ptr<ABISupport> ABI)
 335     : EPC(EPC), ABI(std::move(ABI)) {
 336   assert(this->ABI && "ABI can not be null");
 337
 338   assert(EPC.getPageSize() > getABISupport().getStubSize() &&
 339          "Stubs larger than one page are not supported");
 340 }
 341
 342 Expected<EPCIndirectionUtils::IndirectStubInfoVector>
 343 EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) {
 344
 345   std::lock_guard<std::mutex> Lock(EPCUIMutex);
 346
 347   // If there aren't enough stubs available then allocate some more.
 348   if (NumStubs > AvailableIndirectStubs.size()) {
 349     auto NumStubsToAllocate = NumStubs;
 350     auto PageSize = EPC.getPageSize();
 351     auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize);
 352     NumStubsToAllocate = StubBytes / ABI->getStubSize();
 353     auto PointerBytes =
 354         alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize);
 355
 356     constexpr auto StubPagePermissions =
 357         static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
 358                                                   sys::Memory::MF_EXEC);
 359     constexpr auto PointerPagePermissions =
 360         static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
 361                                                   sys::Memory::MF_WRITE);
 362
 363     jitlink::JITLinkMemoryManager::SegmentsRequestMap Request;
 364     Request[StubPagePermissions] = {PageSize, static_cast<size_t>(StubBytes),
 365                                     0};
 366     Request[PointerPagePermissions] = {PageSize, 0, PointerBytes};
 367     auto Alloc = EPC.getMemMgr().allocate(nullptr, Request);
 368     if (!Alloc)
 369       return Alloc.takeError();
 370
 371     auto StubTargetAddr = (*Alloc)->getTargetMemory(StubPagePermissions);
 372     auto PointerTargetAddr = (*Alloc)->getTargetMemory(PointerPagePermissions);
 373
 374     ABI->writeIndirectStubsBlock(
 375         (*Alloc)->getWorkingMemory(StubPagePermissions).data(), StubTargetAddr,
 376         PointerTargetAddr, NumStubsToAllocate);
 377
 378     if (auto Err = (*Alloc)->finalize())
 379       return std::move(Err);
 380
 381     for (unsigned I = 0; I != NumStubsToAllocate; ++I) {
 382       AvailableIndirectStubs.push_back(
 383           IndirectStubInfo(StubTargetAddr, PointerTargetAddr));
 384       StubTargetAddr += ABI->getStubSize();
 385       PointerTargetAddr += ABI->getPointerSize();
 386     }
 387
 388     IndirectStubAllocs.push_back(std::move(*Alloc));
 389   }
 390
 391   assert(NumStubs <= AvailableIndirectStubs.size() &&
 392          "Sufficient stubs should have been allocated above");
 393
 394   IndirectStubInfoVector Result;
 395   while (NumStubs--) {
 396     Result.push_back(AvailableIndirectStubs.back());
 397     AvailableIndirectStubs.pop_back();
 398   }
 399
 400   return std::move(Result);
 401 }
 402
 403 static JITTargetAddress reentry(JITTargetAddress LCTMAddr,
 404                                 JITTargetAddress TrampolineAddr) {
 405   auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr);
 406   std::promise<JITTargetAddress> LandingAddrP;
 407   auto LandingAddrF = LandingAddrP.get_future();
 408   LCTM.resolveTrampolineLandingAddress(
 409       TrampolineAddr,
 410       [&](JITTargetAddress Addr) { LandingAddrP.set_value(Addr); });
 411   return LandingAddrF.get();
 412 }
 413
 414 Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU) {
 415   auto &LCTM = EPCIU.getLazyCallThroughManager();
 416   return EPCIU
 417       .writeResolverBlock(pointerToJITTargetAddress(&reentry),
 418                           pointerToJITTargetAddress(&LCTM))
 419       .takeError();
 420 }
 421
 422 } // end namespace orc
 423 } // end namespace llvm