llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp

   1 //===------- EPCIndirectionUtils.cpp -- EPC based indirection APIs --------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h"
  10
  11 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
  12 #include "llvm/Support/MathExtras.h"
  13
  14 #include <future>
  15
  16 using namespace llvm;
  17 using namespace llvm::orc;
  18
  19 namespace llvm {
  20 namespace orc {
  21
  22 class EPCIndirectionUtilsAccess {
  23 public:
  24   using IndirectStubInfo = EPCIndirectionUtils::IndirectStubInfo;
  25   using IndirectStubInfoVector = EPCIndirectionUtils::IndirectStubInfoVector;
  26
  27   static Expected<IndirectStubInfoVector>
  28   getIndirectStubs(EPCIndirectionUtils &EPCIU, unsigned NumStubs) {
  29     return EPCIU.getIndirectStubs(NumStubs);
  30   };
  31 };
  32
  33 } // end namespace orc
  34 } // end namespace llvm
  35
  36 namespace {
  37
  38 class EPCTrampolinePool : public TrampolinePool {
  39 public:
  40   EPCTrampolinePool(EPCIndirectionUtils &EPCIU);
  41   Error deallocatePool();
  42
  43 protected:
  44   Error grow() override;
  45
  46   using FinalizedAlloc = jitlink::JITLinkMemoryManager::FinalizedAlloc;
  47
  48   EPCIndirectionUtils &EPCIU;
  49   unsigned TrampolineSize = 0;
  50   unsigned TrampolinesPerPage = 0;
  51   std::vector<FinalizedAlloc> TrampolineBlocks;
  52 };
  53
  54 class EPCIndirectStubsManager : public IndirectStubsManager,
  55                                 private EPCIndirectionUtilsAccess {
  56 public:
  57   EPCIndirectStubsManager(EPCIndirectionUtils &EPCIU) : EPCIU(EPCIU) {}
  58
  59   Error deallocateStubs();
  60
  61   Error createStub(StringRef StubName, JITTargetAddress StubAddr,
  62                    JITSymbolFlags StubFlags) override;
  63
  64   Error createStubs(const StubInitsMap &StubInits) override;
  65
  66   JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override;
  67
  68   JITEvaluatedSymbol findPointer(StringRef Name) override;
  69
  70   Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override;
  71
  72 private:
  73   using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>;
  74
  75   std::mutex ISMMutex;
  76   EPCIndirectionUtils &EPCIU;
  77   StringMap<StubInfo> StubInfos;
  78 };
  79
  80 EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU)
  81     : EPCIU(EPCIU) {
  82   auto &EPC = EPCIU.getExecutorProcessControl();
  83   auto &ABI = EPCIU.getABISupport();
  84
  85   TrampolineSize = ABI.getTrampolineSize();
  86   TrampolinesPerPage =
  87       (EPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize;
  88 }
  89
  90 Error EPCTrampolinePool::deallocatePool() {
  91   Error Err = Error::success();
  92   std::promise<MSVCPError> DeallocResultP;
  93   auto DeallocResultF = DeallocResultP.get_future();
  94
  95   EPCIU.getExecutorProcessControl().getMemMgr().deallocate(
  96       std::move(TrampolineBlocks),
  97       [&](Error Err) { DeallocResultP.set_value(std::move(Err)); });
  98
  99   return DeallocResultF.get();
 100 }
 101
 102 Error EPCTrampolinePool::grow() {
 103   using namespace jitlink;
 104
 105   assert(AvailableTrampolines.empty() &&
 106          "Grow called with trampolines still available");
 107
 108   auto ResolverAddress = EPCIU.getResolverBlockAddress();
 109   assert(ResolverAddress && "Resolver address can not be null");
 110
 111   auto &EPC = EPCIU.getExecutorProcessControl();
 112   auto PageSize = EPC.getPageSize();
 113   auto Alloc = SimpleSegmentAlloc::Create(
 114       EPC.getMemMgr(), nullptr,
 115       {{MemProt::Read | MemProt::Exec, {PageSize, Align(PageSize)}}});
 116   if (!Alloc)
 117     return Alloc.takeError();
 118
 119   unsigned NumTrampolines = TrampolinesPerPage;
 120
 121   auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec);
 122   EPCIU.getABISupport().writeTrampolines(SegInfo.WorkingMem.data(),
 123                                          SegInfo.Addr.getValue(),
 124                                          ResolverAddress, NumTrampolines);
 125   for (unsigned I = 0; I < NumTrampolines; ++I)
 126     AvailableTrampolines.push_back(SegInfo.Addr.getValue() +
 127                                    (I * TrampolineSize));
 128
 129   auto FA = Alloc->finalize();
 130   if (!FA)
 131     return FA.takeError();
 132
 133   TrampolineBlocks.push_back(std::move(*FA));
 134
 135   return Error::success();
 136 }
 137
 138 Error EPCIndirectStubsManager::createStub(StringRef StubName,
 139                                           JITTargetAddress StubAddr,
 140                                           JITSymbolFlags StubFlags) {
 141   StubInitsMap SIM;
 142   SIM[StubName] = std::make_pair(StubAddr, StubFlags);
 143   return createStubs(SIM);
 144 }
 145
 146 Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) {
 147   auto AvailableStubInfos = getIndirectStubs(EPCIU, StubInits.size());
 148   if (!AvailableStubInfos)
 149     return AvailableStubInfos.takeError();
 150
 151   {
 152     std::lock_guard<std::mutex> Lock(ISMMutex);
 153     unsigned ASIdx = 0;
 154     for (auto &SI : StubInits) {
 155       auto &A = (*AvailableStubInfos)[ASIdx++];
 156       StubInfos[SI.first()] = std::make_pair(A, SI.second.second);
 157     }
 158   }
 159
 160   auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess();
 161   switch (EPCIU.getABISupport().getPointerSize()) {
 162   case 4: {
 163     unsigned ASIdx = 0;
 164     std::vector<tpctypes::UInt32Write> PtrUpdates;
 165     for (auto &SI : StubInits)
 166       PtrUpdates.push_back(
 167           {ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress),
 168            static_cast<uint32_t>(SI.second.first)});
 169     return MemAccess.writeUInt32s(PtrUpdates);
 170   }
 171   case 8: {
 172     unsigned ASIdx = 0;
 173     std::vector<tpctypes::UInt64Write> PtrUpdates;
 174     for (auto &SI : StubInits)
 175       PtrUpdates.push_back(
 176           {ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress),
 177            static_cast<uint64_t>(SI.second.first)});
 178     return MemAccess.writeUInt64s(PtrUpdates);
 179   }
 180   default:
 181     return make_error<StringError>("Unsupported pointer size",
 182                                    inconvertibleErrorCode());
 183   }
 184 }
 185
 186 JITEvaluatedSymbol EPCIndirectStubsManager::findStub(StringRef Name,
 187                                                      bool ExportedStubsOnly) {
 188   std::lock_guard<std::mutex> Lock(ISMMutex);
 189   auto I = StubInfos.find(Name);
 190   if (I == StubInfos.end())
 191     return nullptr;
 192   return {I->second.first.StubAddress, I->second.second};
 193 }
 194
 195 JITEvaluatedSymbol EPCIndirectStubsManager::findPointer(StringRef Name) {
 196   std::lock_guard<std::mutex> Lock(ISMMutex);
 197   auto I = StubInfos.find(Name);
 198   if (I == StubInfos.end())
 199     return nullptr;
 200   return {I->second.first.PointerAddress, I->second.second};
 201 }
 202
 203 Error EPCIndirectStubsManager::updatePointer(StringRef Name,
 204                                              JITTargetAddress NewAddr) {
 205
 206   JITTargetAddress PtrAddr = 0;
 207   {
 208     std::lock_guard<std::mutex> Lock(ISMMutex);
 209     auto I = StubInfos.find(Name);
 210     if (I == StubInfos.end())
 211       return make_error<StringError>("Unknown stub name",
 212                                      inconvertibleErrorCode());
 213     PtrAddr = I->second.first.PointerAddress;
 214   }
 215
 216   auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess();
 217   switch (EPCIU.getABISupport().getPointerSize()) {
 218   case 4: {
 219     tpctypes::UInt32Write PUpdate(ExecutorAddr(PtrAddr), NewAddr);
 220     return MemAccess.writeUInt32s(PUpdate);
 221   }
 222   case 8: {
 223     tpctypes::UInt64Write PUpdate(ExecutorAddr(PtrAddr), NewAddr);
 224     return MemAccess.writeUInt64s(PUpdate);
 225   }
 226   default:
 227     return make_error<StringError>("Unsupported pointer size",
 228                                    inconvertibleErrorCode());
 229   }
 230 }
 231
 232 } // end anonymous namespace.
 233
 234 namespace llvm {
 235 namespace orc {
 236
 237 EPCIndirectionUtils::ABISupport::~ABISupport() {}
 238
 239 Expected<std::unique_ptr<EPCIndirectionUtils>>
 240 EPCIndirectionUtils::Create(ExecutorProcessControl &EPC) {
 241   const auto &TT = EPC.getTargetTriple();
 242   switch (TT.getArch()) {
 243   default:
 244     return make_error<StringError>(
 245         std::string("No EPCIndirectionUtils available for ") + TT.str(),
 246         inconvertibleErrorCode());
 247   case Triple::aarch64:
 248   case Triple::aarch64_32:
 249     return CreateWithABI<OrcAArch64>(EPC);
 250
 251   case Triple::x86:
 252     return CreateWithABI<OrcI386>(EPC);
 253
 254   case Triple::mips:
 255     return CreateWithABI<OrcMips32Be>(EPC);
 256
 257   case Triple::mipsel:
 258     return CreateWithABI<OrcMips32Le>(EPC);
 259
 260   case Triple::mips64:
 261   case Triple::mips64el:
 262     return CreateWithABI<OrcMips64>(EPC);
 263
 264   case Triple::x86_64:
 265     if (TT.getOS() == Triple::OSType::Win32)
 266       return CreateWithABI<OrcX86_64_Win32>(EPC);
 267     else
 268       return CreateWithABI<OrcX86_64_SysV>(EPC);
 269   }
 270 }
 271
 272 Error EPCIndirectionUtils::cleanup() {
 273
 274   auto &MemMgr = EPC.getMemMgr();
 275   auto Err = MemMgr.deallocate(std::move(IndirectStubAllocs));
 276
 277   if (TP)
 278     Err = joinErrors(std::move(Err),
 279                      static_cast<EPCTrampolinePool &>(*TP).deallocatePool());
 280
 281   if (ResolverBlock)
 282     Err =
 283         joinErrors(std::move(Err), MemMgr.deallocate(std::move(ResolverBlock)));
 284
 285   return Err;
 286 }
 287
 288 Expected<JITTargetAddress>
 289 EPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr,
 290                                         JITTargetAddress ReentryCtxAddr) {
 291   using namespace jitlink;
 292
 293   assert(ABI && "ABI can not be null");
 294   auto ResolverSize = ABI->getResolverCodeSize();
 295
 296   auto Alloc =
 297       SimpleSegmentAlloc::Create(EPC.getMemMgr(), nullptr,
 298                                  {{MemProt::Read | MemProt::Exec,
 299                                    {ResolverSize, Align(EPC.getPageSize())}}});
 300
 301   if (!Alloc)
 302     return Alloc.takeError();
 303
 304   auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec);
 305   ABI->writeResolverCode(SegInfo.WorkingMem.data(), SegInfo.Addr.getValue(),
 306                          ReentryFnAddr, ReentryCtxAddr);
 307
 308   auto FA = Alloc->finalize();
 309   if (!FA)
 310     return FA.takeError();
 311
 312   ResolverBlock = std::move(*FA);
 313   return SegInfo.Addr.getValue();
 314 }
 315
 316 std::unique_ptr<IndirectStubsManager>
 317 EPCIndirectionUtils::createIndirectStubsManager() {
 318   return std::make_unique<EPCIndirectStubsManager>(*this);
 319 }
 320
 321 TrampolinePool &EPCIndirectionUtils::getTrampolinePool() {
 322   if (!TP)
 323     TP = std::make_unique<EPCTrampolinePool>(*this);
 324   return *TP;
 325 }
 326
 327 LazyCallThroughManager &EPCIndirectionUtils::createLazyCallThroughManager(
 328     ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr) {
 329   assert(!LCTM &&
 330          "createLazyCallThroughManager can not have been called before");
 331   LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr,
 332                                                   &getTrampolinePool());
 333   return *LCTM;
 334 }
 335
 336 EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC,
 337                                          std::unique_ptr<ABISupport> ABI)
 338     : EPC(EPC), ABI(std::move(ABI)) {
 339   assert(this->ABI && "ABI can not be null");
 340
 341   assert(EPC.getPageSize() > getABISupport().getStubSize() &&
 342          "Stubs larger than one page are not supported");
 343 }
 344
 345 Expected<EPCIndirectionUtils::IndirectStubInfoVector>
 346 EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) {
 347   using namespace jitlink;
 348
 349   std::lock_guard<std::mutex> Lock(EPCUIMutex);
 350
 351   // If there aren't enough stubs available then allocate some more.
 352   if (NumStubs > AvailableIndirectStubs.size()) {
 353     auto NumStubsToAllocate = NumStubs;
 354     auto PageSize = EPC.getPageSize();
 355     auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize);
 356     NumStubsToAllocate = StubBytes / ABI->getStubSize();
 357     auto PtrBytes =
 358         alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize);
 359
 360     auto StubProt = MemProt::Read | MemProt::Exec;
 361     auto PtrProt = MemProt::Read | MemProt::Write;
 362
 363     auto Alloc = SimpleSegmentAlloc::Create(
 364         EPC.getMemMgr(), nullptr,
 365         {{StubProt, {static_cast<size_t>(StubBytes), Align(PageSize)}},
 366          {PtrProt, {static_cast<size_t>(PtrBytes), Align(PageSize)}}});
 367
 368     if (!Alloc)
 369       return Alloc.takeError();
 370
 371     auto StubSeg = Alloc->getSegInfo(StubProt);
 372     auto PtrSeg = Alloc->getSegInfo(PtrProt);
 373
 374     ABI->writeIndirectStubsBlock(StubSeg.WorkingMem.data(),
 375                                  StubSeg.Addr.getValue(),
 376                                  PtrSeg.Addr.getValue(), NumStubsToAllocate);
 377
 378     auto FA = Alloc->finalize();
 379     if (!FA)
 380       return FA.takeError();
 381
 382     IndirectStubAllocs.push_back(std::move(*FA));
 383
 384     auto StubExecutorAddr = StubSeg.Addr;
 385     auto PtrExecutorAddr = PtrSeg.Addr;
 386     for (unsigned I = 0; I != NumStubsToAllocate; ++I) {
 387       AvailableIndirectStubs.push_back(IndirectStubInfo(
 388           StubExecutorAddr.getValue(), PtrExecutorAddr.getValue()));
 389       StubExecutorAddr += ABI->getStubSize();
 390       PtrExecutorAddr += ABI->getPointerSize();
 391     }
 392   }
 393
 394   assert(NumStubs <= AvailableIndirectStubs.size() &&
 395          "Sufficient stubs should have been allocated above");
 396
 397   IndirectStubInfoVector Result;
 398   while (NumStubs--) {
 399     Result.push_back(AvailableIndirectStubs.back());
 400     AvailableIndirectStubs.pop_back();
 401   }
 402
 403   return std::move(Result);
 404 }
 405
 406 static JITTargetAddress reentry(JITTargetAddress LCTMAddr,
 407                                 JITTargetAddress TrampolineAddr) {
 408   auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr);
 409   std::promise<JITTargetAddress> LandingAddrP;
 410   auto LandingAddrF = LandingAddrP.get_future();
 411   LCTM.resolveTrampolineLandingAddress(
 412       TrampolineAddr,
 413       [&](JITTargetAddress Addr) { LandingAddrP.set_value(Addr); });
 414   return LandingAddrF.get();
 415 }
 416
 417 Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU) {
 418   auto &LCTM = EPCIU.getLazyCallThroughManager();
 419   return EPCIU
 420       .writeResolverBlock(pointerToJITTargetAddress(&reentry),
 421                           pointerToJITTargetAddress(&LCTM))
 422       .takeError();
 423 }
 424
 425 } // end namespace orc
 426 } // end namespace llvm