llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp

   1 //===- SimpleExecuorMemoryManagare.cpp - Simple executor-side memory mgmt -===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h"
  10
  11 #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
  12 #include "llvm/Support/FormatVariadic.h"
  13
  14 #define DEBUG_TYPE "orc"
  15
  16 namespace llvm {
  17 namespace orc {
  18 namespace rt_bootstrap {
  19
  20 SimpleExecutorMemoryManager::~SimpleExecutorMemoryManager() {
  21   assert(Allocations.empty() && "shutdown not called?");
  22 }
  23
  24 Expected<ExecutorAddr> SimpleExecutorMemoryManager::allocate(uint64_t Size) {
  25   std::error_code EC;
  26   auto MB = sys::Memory::allocateMappedMemory(
  27       Size, nullptr, sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC);
  28   if (EC)
  29     return errorCodeToError(EC);
  30   std::lock_guard<std::mutex> Lock(M);
  31   assert(!Allocations.count(MB.base()) && "Duplicate allocation addr");
  32   Allocations[MB.base()].Size = Size;
  33   return ExecutorAddr::fromPtr(MB.base());
  34 }
  35
  36 Error SimpleExecutorMemoryManager::finalize(tpctypes::FinalizeRequest &FR) {
  37   ExecutorAddr Base(~0ULL);
  38   std::vector<shared::WrapperFunctionCall> DeallocationActions;
  39   size_t SuccessfulFinalizationActions = 0;
  40
  41   if (FR.Segments.empty()) {
  42     // NOTE: Finalizing nothing is currently a no-op. Should it be an error?
  43     if (FR.Actions.empty())
  44       return Error::success();
  45     else
  46       return make_error<StringError>("Finalization actions attached to empty "
  47                                      "finalization request",
  48                                      inconvertibleErrorCode());
  49   }
  50
  51   for (auto &Seg : FR.Segments)
  52     Base = std::min(Base, Seg.Addr);
  53
  54   for (auto &ActPair : FR.Actions)
  55     if (ActPair.Dealloc)
  56       DeallocationActions.push_back(ActPair.Dealloc);
  57
  58   // Get the Allocation for this finalization.
  59   size_t AllocSize = 0;
  60   {
  61     std::lock_guard<std::mutex> Lock(M);
  62     auto I = Allocations.find(Base.toPtr<void *>());
  63     if (I == Allocations.end())
  64       return make_error<StringError>("Attempt to finalize unrecognized "
  65                                      "allocation " +
  66                                          formatv("{0:x}", Base.getValue()),
  67                                      inconvertibleErrorCode());
  68     AllocSize = I->second.Size;
  69     I->second.DeallocationActions = std::move(DeallocationActions);
  70   }
  71   ExecutorAddr AllocEnd = Base + ExecutorAddrDiff(AllocSize);
  72
  73   // Bail-out function: this will run deallocation actions corresponding to any
  74   // completed finalization actions, then deallocate memory.
  75   auto BailOut = [&](Error Err) {
  76     std::pair<void *, Allocation> AllocToDestroy;
  77
  78     // Get allocation to destroy.
  79     {
  80       std::lock_guard<std::mutex> Lock(M);
  81       auto I = Allocations.find(Base.toPtr<void *>());
  82
  83       // Check for missing allocation (effective a double free).
  84       if (I == Allocations.end())
  85         return joinErrors(
  86             std::move(Err),
  87             make_error<StringError>("No allocation entry found "
  88                                     "for " +
  89                                         formatv("{0:x}", Base.getValue()),
  90                                     inconvertibleErrorCode()));
  91       AllocToDestroy = std::move(*I);
  92       Allocations.erase(I);
  93     }
  94
  95     // Run deallocation actions for all completed finalization actions.
  96     while (SuccessfulFinalizationActions)
  97       Err =
  98           joinErrors(std::move(Err), FR.Actions[--SuccessfulFinalizationActions]
  99                                          .Dealloc.runWithSPSRetErrorMerged());
 100
 101     // Deallocate memory.
 102     sys::MemoryBlock MB(AllocToDestroy.first, AllocToDestroy.second.Size);
 103     if (auto EC = sys::Memory::releaseMappedMemory(MB))
 104       Err = joinErrors(std::move(Err), errorCodeToError(EC));
 105
 106     return Err;
 107   };
 108
 109   // Copy content and apply permissions.
 110   for (auto &Seg : FR.Segments) {
 111
 112     // Check segment ranges.
 113     if (LLVM_UNLIKELY(Seg.Size < Seg.Content.size()))
 114       return BailOut(make_error<StringError>(
 115           formatv("Segment {0:x} content size ({1:x} bytes) "
 116                   "exceeds segment size ({2:x} bytes)",
 117                   Seg.Addr.getValue(), Seg.Content.size(), Seg.Size),
 118           inconvertibleErrorCode()));
 119     ExecutorAddr SegEnd = Seg.Addr + ExecutorAddrDiff(Seg.Size);
 120     if (LLVM_UNLIKELY(Seg.Addr < Base || SegEnd > AllocEnd))
 121       return BailOut(make_error<StringError>(
 122           formatv("Segment {0:x} -- {1:x} crosses boundary of "
 123                   "allocation {2:x} -- {3:x}",
 124                   Seg.Addr.getValue(), SegEnd.getValue(), Base.getValue(),
 125                   AllocEnd.getValue()),
 126           inconvertibleErrorCode()));
 127
 128     char *Mem = Seg.Addr.toPtr<char *>();
 129     if (!Seg.Content.empty())
 130       memcpy(Mem, Seg.Content.data(), Seg.Content.size());
 131     memset(Mem + Seg.Content.size(), 0, Seg.Size - Seg.Content.size());
 132     assert(Seg.Size <= std::numeric_limits<size_t>::max());
 133     if (auto EC = sys::Memory::protectMappedMemory(
 134             {Mem, static_cast<size_t>(Seg.Size)},
 135             toSysMemoryProtectionFlags(Seg.RAG.Prot)))
 136       return BailOut(errorCodeToError(EC));
 137     if ((Seg.RAG.Prot & MemProt::Exec) == MemProt::Exec)
 138       sys::Memory::InvalidateInstructionCache(Mem, Seg.Size);
 139   }
 140
 141   // Run finalization actions.
 142   for (auto &ActPair : FR.Actions) {
 143     if (auto Err = ActPair.Finalize.runWithSPSRetErrorMerged())
 144       return BailOut(std::move(Err));
 145     ++SuccessfulFinalizationActions;
 146   }
 147
 148   return Error::success();
 149 }
 150
 151 Error SimpleExecutorMemoryManager::deallocate(
 152     const std::vector<ExecutorAddr> &Bases) {
 153   std::vector<std::pair<void *, Allocation>> AllocPairs;
 154   AllocPairs.reserve(Bases.size());
 155
 156   // Get allocation to destroy.
 157   Error Err = Error::success();
 158   {
 159     std::lock_guard<std::mutex> Lock(M);
 160     for (auto &Base : Bases) {
 161       auto I = Allocations.find(Base.toPtr<void *>());
 162
 163       // Check for missing allocation (effective a double free).
 164       if (I != Allocations.end()) {
 165         AllocPairs.push_back(std::move(*I));
 166         Allocations.erase(I);
 167       } else
 168         Err = joinErrors(
 169             std::move(Err),
 170             make_error<StringError>("No allocation entry found "
 171                                     "for " +
 172                                         formatv("{0:x}", Base.getValue()),
 173                                     inconvertibleErrorCode()));
 174     }
 175   }
 176
 177   while (!AllocPairs.empty()) {
 178     auto &P = AllocPairs.back();
 179     Err = joinErrors(std::move(Err), deallocateImpl(P.first, P.second));
 180     AllocPairs.pop_back();
 181   }
 182
 183   return Err;
 184 }
 185
 186 Error SimpleExecutorMemoryManager::shutdown() {
 187
 188   AllocationsMap AM;
 189   {
 190     std::lock_guard<std::mutex> Lock(M);
 191     AM = std::move(Allocations);
 192   }
 193
 194   Error Err = Error::success();
 195   for (auto &KV : AM)
 196     Err = joinErrors(std::move(Err), deallocateImpl(KV.first, KV.second));
 197   return Err;
 198 }
 199
 200 void SimpleExecutorMemoryManager::addBootstrapSymbols(
 201     StringMap<ExecutorAddr> &M) {
 202   M[rt::SimpleExecutorMemoryManagerInstanceName] = ExecutorAddr::fromPtr(this);
 203   M[rt::SimpleExecutorMemoryManagerReserveWrapperName] =
 204       ExecutorAddr::fromPtr(&reserveWrapper);
 205   M[rt::SimpleExecutorMemoryManagerFinalizeWrapperName] =
 206       ExecutorAddr::fromPtr(&finalizeWrapper);
 207   M[rt::SimpleExecutorMemoryManagerDeallocateWrapperName] =
 208       ExecutorAddr::fromPtr(&deallocateWrapper);
 209 }
 210
 211 Error SimpleExecutorMemoryManager::deallocateImpl(void *Base, Allocation &A) {
 212   Error Err = Error::success();
 213
 214   while (!A.DeallocationActions.empty()) {
 215     Err = joinErrors(std::move(Err),
 216                      A.DeallocationActions.back().runWithSPSRetErrorMerged());
 217     A.DeallocationActions.pop_back();
 218   }
 219
 220   sys::MemoryBlock MB(Base, A.Size);
 221   if (auto EC = sys::Memory::releaseMappedMemory(MB))
 222     Err = joinErrors(std::move(Err), errorCodeToError(EC));
 223
 224   return Err;
 225 }
 226
 227 llvm::orc::shared::CWrapperFunctionResult
 228 SimpleExecutorMemoryManager::reserveWrapper(const char *ArgData,
 229                                             size_t ArgSize) {
 230   return shared::WrapperFunction<
 231              rt::SPSSimpleExecutorMemoryManagerReserveSignature>::
 232       handle(ArgData, ArgSize,
 233              shared::makeMethodWrapperHandler(
 234                  &SimpleExecutorMemoryManager::allocate))
 235           .release();
 236 }
 237
 238 llvm::orc::shared::CWrapperFunctionResult
 239 SimpleExecutorMemoryManager::finalizeWrapper(const char *ArgData,
 240                                              size_t ArgSize) {
 241   return shared::WrapperFunction<
 242              rt::SPSSimpleExecutorMemoryManagerFinalizeSignature>::
 243       handle(ArgData, ArgSize,
 244              shared::makeMethodWrapperHandler(
 245                  &SimpleExecutorMemoryManager::finalize))
 246           .release();
 247 }
 248
 249 llvm::orc::shared::CWrapperFunctionResult
 250 SimpleExecutorMemoryManager::deallocateWrapper(const char *ArgData,
 251                                                size_t ArgSize) {
 252   return shared::WrapperFunction<
 253              rt::SPSSimpleExecutorMemoryManagerDeallocateSignature>::
 254       handle(ArgData, ArgSize,
 255              shared::makeMethodWrapperHandler(
 256                  &SimpleExecutorMemoryManager::deallocate))
 257           .release();
 258 }
 259
 260 } // namespace rt_bootstrap
 261 } // end namespace orc
 262 } // end namespace llvm