1 //===- SectionMemoryManager.cpp - Memory manager for MCJIT/RtDyld *- C++ -*-==//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file implements the section-based memory manager used by the MCJIT
10 // execution engine and RuntimeDyld
12 //===----------------------------------------------------------------------===//
14 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
15 #include "llvm/Config/config.h"
16 #include "llvm/Support/MathExtras.h"
17 #include "llvm/Support/Process.h"
21 uint8_t *SectionMemoryManager::allocateDataSection(uintptr_t Size
,
24 StringRef SectionName
,
27 return allocateSection(SectionMemoryManager::AllocationPurpose::ROData
,
29 return allocateSection(SectionMemoryManager::AllocationPurpose::RWData
, Size
,
33 uint8_t *SectionMemoryManager::allocateCodeSection(uintptr_t Size
,
36 StringRef SectionName
) {
37 return allocateSection(SectionMemoryManager::AllocationPurpose::Code
, Size
,
41 uint8_t *SectionMemoryManager::allocateSection(
42 SectionMemoryManager::AllocationPurpose Purpose
, uintptr_t Size
,
47 assert(!(Alignment
& (Alignment
- 1)) && "Alignment must be a power of two.");
49 uintptr_t RequiredSize
= Alignment
* ((Size
+ Alignment
- 1) / Alignment
+ 1);
52 MemoryGroup
&MemGroup
= [&]() -> MemoryGroup
& {
54 case AllocationPurpose::Code
:
56 case AllocationPurpose::ROData
:
58 case AllocationPurpose::RWData
:
61 llvm_unreachable("Unknown SectionMemoryManager::AllocationPurpose");
64 // Look in the list of free memory regions and use a block there if one
66 for (FreeMemBlock
&FreeMB
: MemGroup
.FreeMem
) {
67 if (FreeMB
.Free
.allocatedSize() >= RequiredSize
) {
68 Addr
= (uintptr_t)FreeMB
.Free
.base();
69 uintptr_t EndOfBlock
= Addr
+ FreeMB
.Free
.allocatedSize();
71 Addr
= (Addr
+ Alignment
- 1) & ~(uintptr_t)(Alignment
- 1);
73 if (FreeMB
.PendingPrefixIndex
== (unsigned)-1) {
74 // The part of the block we're giving out to the user is now pending
75 MemGroup
.PendingMem
.push_back(sys::MemoryBlock((void *)Addr
, Size
));
77 // Remember this pending block, such that future allocations can just
78 // modify it rather than creating a new one
79 FreeMB
.PendingPrefixIndex
= MemGroup
.PendingMem
.size() - 1;
81 sys::MemoryBlock
&PendingMB
=
82 MemGroup
.PendingMem
[FreeMB
.PendingPrefixIndex
];
83 PendingMB
= sys::MemoryBlock(PendingMB
.base(),
84 Addr
+ Size
- (uintptr_t)PendingMB
.base());
87 // Remember how much free space is now left in this block
89 sys::MemoryBlock((void *)(Addr
+ Size
), EndOfBlock
- Addr
- Size
);
90 return (uint8_t *)Addr
;
94 // No pre-allocated free block was large enough. Allocate a new memory region.
95 // Note that all sections get allocated as read-write. The permissions will
96 // be updated later based on memory group.
98 // FIXME: It would be useful to define a default allocation size (or add
99 // it as a constructor parameter) to minimize the number of allocations.
101 // FIXME: Initialize the Near member for each memory group to avoid
104 sys::MemoryBlock MB
= MMapper
.allocateMappedMemory(
105 Purpose
, RequiredSize
, &MemGroup
.Near
,
106 sys::Memory::MF_READ
| sys::Memory::MF_WRITE
, ec
);
108 // FIXME: Add error propagation to the interface.
112 // Save this address as the basis for our next request
115 // Remember that we allocated this memory
116 MemGroup
.AllocatedMem
.push_back(MB
);
117 Addr
= (uintptr_t)MB
.base();
118 uintptr_t EndOfBlock
= Addr
+ MB
.allocatedSize();
120 // Align the address.
121 Addr
= (Addr
+ Alignment
- 1) & ~(uintptr_t)(Alignment
- 1);
123 // The part of the block we're giving out to the user is now pending
124 MemGroup
.PendingMem
.push_back(sys::MemoryBlock((void *)Addr
, Size
));
126 // The allocateMappedMemory may allocate much more memory than we need. In
127 // this case, we store the unused memory as a free memory block.
128 unsigned FreeSize
= EndOfBlock
- Addr
- Size
;
131 FreeMB
.Free
= sys::MemoryBlock((void *)(Addr
+ Size
), FreeSize
);
132 FreeMB
.PendingPrefixIndex
= (unsigned)-1;
133 MemGroup
.FreeMem
.push_back(FreeMB
);
136 // Return aligned address
137 return (uint8_t *)Addr
;
140 bool SectionMemoryManager::finalizeMemory(std::string
*ErrMsg
) {
141 // FIXME: Should in-progress permissions be reverted if an error occurs?
144 // Make code memory executable.
145 ec
= applyMemoryGroupPermissions(CodeMem
,
146 sys::Memory::MF_READ
| sys::Memory::MF_EXEC
);
149 *ErrMsg
= ec
.message();
154 // Make read-only data memory read-only.
155 ec
= applyMemoryGroupPermissions(RODataMem
,
156 sys::Memory::MF_READ
| sys::Memory::MF_EXEC
);
159 *ErrMsg
= ec
.message();
164 // Read-write data memory already has the correct permissions
166 // Some platforms with separate data cache and instruction cache require
167 // explicit cache flush, otherwise JIT code manipulations (like resolved
168 // relocations) will get to the data cache but not to the instruction cache.
169 invalidateInstructionCache();
174 static sys::MemoryBlock
trimBlockToPageSize(sys::MemoryBlock M
) {
175 static const size_t PageSize
= sys::Process::getPageSizeEstimate();
177 size_t StartOverlap
=
178 (PageSize
- ((uintptr_t)M
.base() % PageSize
)) % PageSize
;
180 size_t TrimmedSize
= M
.allocatedSize();
181 TrimmedSize
-= StartOverlap
;
182 TrimmedSize
-= TrimmedSize
% PageSize
;
184 sys::MemoryBlock
Trimmed((void *)((uintptr_t)M
.base() + StartOverlap
),
187 assert(((uintptr_t)Trimmed
.base() % PageSize
) == 0);
188 assert((Trimmed
.allocatedSize() % PageSize
) == 0);
189 assert(M
.base() <= Trimmed
.base() &&
190 Trimmed
.allocatedSize() <= M
.allocatedSize());
196 SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup
&MemGroup
,
197 unsigned Permissions
) {
198 for (sys::MemoryBlock
&MB
: MemGroup
.PendingMem
)
199 if (std::error_code EC
= MMapper
.protectMappedMemory(MB
, Permissions
))
202 MemGroup
.PendingMem
.clear();
204 // Now go through free blocks and trim any of them that don't span the entire
205 // page because one of the pending blocks may have overlapped it.
206 for (FreeMemBlock
&FreeMB
: MemGroup
.FreeMem
) {
207 FreeMB
.Free
= trimBlockToPageSize(FreeMB
.Free
);
208 // We cleared the PendingMem list, so all these pointers are now invalid
209 FreeMB
.PendingPrefixIndex
= (unsigned)-1;
212 // Remove all blocks which are now empty
213 MemGroup
.FreeMem
.erase(remove_if(MemGroup
.FreeMem
,
214 [](FreeMemBlock
&FreeMB
) {
215 return FreeMB
.Free
.allocatedSize() == 0;
217 MemGroup
.FreeMem
.end());
219 return std::error_code();
222 void SectionMemoryManager::invalidateInstructionCache() {
223 for (sys::MemoryBlock
&Block
: CodeMem
.PendingMem
)
224 sys::Memory::InvalidateInstructionCache(Block
.base(),
225 Block
.allocatedSize());
228 SectionMemoryManager::~SectionMemoryManager() {
229 for (MemoryGroup
*Group
: {&CodeMem
, &RWDataMem
, &RODataMem
}) {
230 for (sys::MemoryBlock
&Block
: Group
->AllocatedMem
)
231 MMapper
.releaseMappedMemory(Block
);
235 SectionMemoryManager::MemoryMapper::~MemoryMapper() {}
237 void SectionMemoryManager::anchor() {}
240 // Trivial implementation of SectionMemoryManager::MemoryMapper that just calls
242 class DefaultMMapper final
: public SectionMemoryManager::MemoryMapper
{
245 allocateMappedMemory(SectionMemoryManager::AllocationPurpose Purpose
,
246 size_t NumBytes
, const sys::MemoryBlock
*const NearBlock
,
247 unsigned Flags
, std::error_code
&EC
) override
{
248 return sys::Memory::allocateMappedMemory(NumBytes
, NearBlock
, Flags
, EC
);
251 std::error_code
protectMappedMemory(const sys::MemoryBlock
&Block
,
252 unsigned Flags
) override
{
253 return sys::Memory::protectMappedMemory(Block
, Flags
);
256 std::error_code
releaseMappedMemory(sys::MemoryBlock
&M
) override
{
257 return sys::Memory::releaseMappedMemory(M
);
261 DefaultMMapper DefaultMMapperInstance
;
264 SectionMemoryManager::SectionMemoryManager(MemoryMapper
*MM
)
265 : MMapper(MM
? *MM
: DefaultMMapperInstance
) {}