llvm/lib/ExecutionEngine/SectionMemoryManager.cpp

   1 //===- SectionMemoryManager.cpp - Memory manager for MCJIT/RtDyld *- C++ -*-==//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the section-based memory manager used by the MCJIT
  10 // execution engine and RuntimeDyld
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
  15 #include "llvm/Config/config.h"
  16 #include "llvm/Support/Process.h"
  17
  18 namespace llvm {
  19
  20 uint8_t *SectionMemoryManager::allocateDataSection(uintptr_t Size,
  21                                                    unsigned Alignment,
  22                                                    unsigned SectionID,
  23                                                    StringRef SectionName,
  24                                                    bool IsReadOnly) {
  25   if (IsReadOnly)
  26     return allocateSection(SectionMemoryManager::AllocationPurpose::ROData,
  27                            Size, Alignment);
  28   return allocateSection(SectionMemoryManager::AllocationPurpose::RWData, Size,
  29                          Alignment);
  30 }
  31
  32 uint8_t *SectionMemoryManager::allocateCodeSection(uintptr_t Size,
  33                                                    unsigned Alignment,
  34                                                    unsigned SectionID,
  35                                                    StringRef SectionName) {
  36   return allocateSection(SectionMemoryManager::AllocationPurpose::Code, Size,
  37                          Alignment);
  38 }
  39
  40 uint8_t *SectionMemoryManager::allocateSection(
  41     SectionMemoryManager::AllocationPurpose Purpose, uintptr_t Size,
  42     unsigned Alignment) {
  43   if (!Alignment)
  44     Alignment = 16;
  45
  46   assert(!(Alignment & (Alignment - 1)) && "Alignment must be a power of two.");
  47
  48   uintptr_t RequiredSize = Alignment * ((Size + Alignment - 1) / Alignment + 1);
  49   uintptr_t Addr = 0;
  50
  51   MemoryGroup &MemGroup = [&]() -> MemoryGroup & {
  52     switch (Purpose) {
  53     case AllocationPurpose::Code:
  54       return CodeMem;
  55     case AllocationPurpose::ROData:
  56       return RODataMem;
  57     case AllocationPurpose::RWData:
  58       return RWDataMem;
  59     }
  60     llvm_unreachable("Unknown SectionMemoryManager::AllocationPurpose");
  61   }();
  62
  63   // Look in the list of free memory regions and use a block there if one
  64   // is available.
  65   for (FreeMemBlock &FreeMB : MemGroup.FreeMem) {
  66     if (FreeMB.Free.allocatedSize() >= RequiredSize) {
  67       Addr = (uintptr_t)FreeMB.Free.base();
  68       uintptr_t EndOfBlock = Addr + FreeMB.Free.allocatedSize();
  69       // Align the address.
  70       Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);
  71
  72       if (FreeMB.PendingPrefixIndex == (unsigned)-1) {
  73         // The part of the block we're giving out to the user is now pending
  74         MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size));
  75
  76         // Remember this pending block, such that future allocations can just
  77         // modify it rather than creating a new one
  78         FreeMB.PendingPrefixIndex = MemGroup.PendingMem.size() - 1;
  79       } else {
  80         sys::MemoryBlock &PendingMB =
  81             MemGroup.PendingMem[FreeMB.PendingPrefixIndex];
  82         PendingMB = sys::MemoryBlock(PendingMB.base(),
  83                                      Addr + Size - (uintptr_t)PendingMB.base());
  84       }
  85
  86       // Remember how much free space is now left in this block
  87       FreeMB.Free =
  88           sys::MemoryBlock((void *)(Addr + Size), EndOfBlock - Addr - Size);
  89       return (uint8_t *)Addr;
  90     }
  91   }
  92
  93   // No pre-allocated free block was large enough. Allocate a new memory region.
  94   // Note that all sections get allocated as read-write.  The permissions will
  95   // be updated later based on memory group.
  96   //
  97   // FIXME: It would be useful to define a default allocation size (or add
  98   // it as a constructor parameter) to minimize the number of allocations.
  99   //
 100   // FIXME: Initialize the Near member for each memory group to avoid
 101   // interleaving.
 102   std::error_code ec;
 103   sys::MemoryBlock MB = MMapper->allocateMappedMemory(
 104       Purpose, RequiredSize, &MemGroup.Near,
 105       sys::Memory::MF_READ | sys::Memory::MF_WRITE, ec);
 106   if (ec) {
 107     // FIXME: Add error propagation to the interface.
 108     return nullptr;
 109   }
 110
 111   // Save this address as the basis for our next request
 112   MemGroup.Near = MB;
 113
 114   // Copy the address to all the other groups, if they have not
 115   // been initialized.
 116   if (CodeMem.Near.base() == nullptr)
 117     CodeMem.Near = MB;
 118   if (RODataMem.Near.base() == nullptr)
 119     RODataMem.Near = MB;
 120   if (RWDataMem.Near.base() == nullptr)
 121     RWDataMem.Near = MB;
 122
 123   // Remember that we allocated this memory
 124   MemGroup.AllocatedMem.push_back(MB);
 125   Addr = (uintptr_t)MB.base();
 126   uintptr_t EndOfBlock = Addr + MB.allocatedSize();
 127
 128   // Align the address.
 129   Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);
 130
 131   // The part of the block we're giving out to the user is now pending
 132   MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size));
 133
 134   // The allocateMappedMemory may allocate much more memory than we need. In
 135   // this case, we store the unused memory as a free memory block.
 136   unsigned FreeSize = EndOfBlock - Addr - Size;
 137   if (FreeSize > 16) {
 138     FreeMemBlock FreeMB;
 139     FreeMB.Free = sys::MemoryBlock((void *)(Addr + Size), FreeSize);
 140     FreeMB.PendingPrefixIndex = (unsigned)-1;
 141     MemGroup.FreeMem.push_back(FreeMB);
 142   }
 143
 144   // Return aligned address
 145   return (uint8_t *)Addr;
 146 }
 147
 148 bool SectionMemoryManager::finalizeMemory(std::string *ErrMsg) {
 149   // FIXME: Should in-progress permissions be reverted if an error occurs?
 150   std::error_code ec;
 151
 152   // Make code memory executable.
 153   ec = applyMemoryGroupPermissions(CodeMem,
 154                                    sys::Memory::MF_READ | sys::Memory::MF_EXEC);
 155   if (ec) {
 156     if (ErrMsg) {
 157       *ErrMsg = ec.message();
 158     }
 159     return true;
 160   }
 161
 162   // Make read-only data memory read-only.
 163   ec = applyMemoryGroupPermissions(RODataMem, sys::Memory::MF_READ);
 164   if (ec) {
 165     if (ErrMsg) {
 166       *ErrMsg = ec.message();
 167     }
 168     return true;
 169   }
 170
 171   // Read-write data memory already has the correct permissions
 172
 173   // Some platforms with separate data cache and instruction cache require
 174   // explicit cache flush, otherwise JIT code manipulations (like resolved
 175   // relocations) will get to the data cache but not to the instruction cache.
 176   invalidateInstructionCache();
 177
 178   return false;
 179 }
 180
 181 static sys::MemoryBlock trimBlockToPageSize(sys::MemoryBlock M) {
 182   static const size_t PageSize = sys::Process::getPageSizeEstimate();
 183
 184   size_t StartOverlap =
 185       (PageSize - ((uintptr_t)M.base() % PageSize)) % PageSize;
 186
 187   size_t TrimmedSize = M.allocatedSize();
 188   TrimmedSize -= StartOverlap;
 189   TrimmedSize -= TrimmedSize % PageSize;
 190
 191   sys::MemoryBlock Trimmed((void *)((uintptr_t)M.base() + StartOverlap),
 192                            TrimmedSize);
 193
 194   assert(((uintptr_t)Trimmed.base() % PageSize) == 0);
 195   assert((Trimmed.allocatedSize() % PageSize) == 0);
 196   assert(M.base() <= Trimmed.base() &&
 197          Trimmed.allocatedSize() <= M.allocatedSize());
 198
 199   return Trimmed;
 200 }
 201
 202 std::error_code
 203 SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup,
 204                                                   unsigned Permissions) {
 205   for (sys::MemoryBlock &MB : MemGroup.PendingMem)
 206     if (std::error_code EC = MMapper->protectMappedMemory(MB, Permissions))
 207       return EC;
 208
 209   MemGroup.PendingMem.clear();
 210
 211   // Now go through free blocks and trim any of them that don't span the entire
 212   // page because one of the pending blocks may have overlapped it.
 213   for (FreeMemBlock &FreeMB : MemGroup.FreeMem) {
 214     FreeMB.Free = trimBlockToPageSize(FreeMB.Free);
 215     // We cleared the PendingMem list, so all these pointers are now invalid
 216     FreeMB.PendingPrefixIndex = (unsigned)-1;
 217   }
 218
 219   // Remove all blocks which are now empty
 220   erase_if(MemGroup.FreeMem, [](FreeMemBlock &FreeMB) {
 221     return FreeMB.Free.allocatedSize() == 0;
 222   });
 223
 224   return std::error_code();
 225 }
 226
 227 void SectionMemoryManager::invalidateInstructionCache() {
 228   for (sys::MemoryBlock &Block : CodeMem.PendingMem)
 229     sys::Memory::InvalidateInstructionCache(Block.base(),
 230                                             Block.allocatedSize());
 231 }
 232
 233 SectionMemoryManager::~SectionMemoryManager() {
 234   for (MemoryGroup *Group : {&CodeMem, &RWDataMem, &RODataMem}) {
 235     for (sys::MemoryBlock &Block : Group->AllocatedMem)
 236       MMapper->releaseMappedMemory(Block);
 237   }
 238 }
 239
 240 SectionMemoryManager::MemoryMapper::~MemoryMapper() = default;
 241
 242 void SectionMemoryManager::anchor() {}
 243
 244 namespace {
 245 // Trivial implementation of SectionMemoryManager::MemoryMapper that just calls
 246 // into sys::Memory.
 247 class DefaultMMapper final : public SectionMemoryManager::MemoryMapper {
 248 public:
 249   sys::MemoryBlock
 250   allocateMappedMemory(SectionMemoryManager::AllocationPurpose Purpose,
 251                        size_t NumBytes, const sys::MemoryBlock *const NearBlock,
 252                        unsigned Flags, std::error_code &EC) override {
 253     return sys::Memory::allocateMappedMemory(NumBytes, NearBlock, Flags, EC);
 254   }
 255
 256   std::error_code protectMappedMemory(const sys::MemoryBlock &Block,
 257                                       unsigned Flags) override {
 258     return sys::Memory::protectMappedMemory(Block, Flags);
 259   }
 260
 261   std::error_code releaseMappedMemory(sys::MemoryBlock &M) override {
 262     return sys::Memory::releaseMappedMemory(M);
 263   }
 264 };
 265 } // namespace
 266
 267 SectionMemoryManager::SectionMemoryManager(MemoryMapper *UnownedMM)
 268     : MMapper(UnownedMM), OwnedMMapper(nullptr) {
 269   if (!MMapper) {
 270     OwnedMMapper = std::make_unique<DefaultMMapper>();
 271     MMapper = OwnedMMapper.get();
 272   }
 273 }
 274
 275 } // namespace llvm