compiler-rt/lib/xray/xray_allocator.h

   1 //===-- xray_allocator.h ---------------------------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file is a part of XRay, a dynamic runtime instrumentation system.
  10 //
  11 // Defines the allocator interface for an arena allocator, used primarily for
  12 // the profiling runtime.
  13 //
  14 //===----------------------------------------------------------------------===//
  15 #ifndef XRAY_ALLOCATOR_H
  16 #define XRAY_ALLOCATOR_H
  17
  18 #include "sanitizer_common/sanitizer_common.h"
  19 #include "sanitizer_common/sanitizer_internal_defs.h"
  20 #include "sanitizer_common/sanitizer_mutex.h"
  21 #if SANITIZER_FUCHSIA
  22 #include <zircon/process.h>
  23 #include <zircon/status.h>
  24 #include <zircon/syscalls.h>
  25 #else
  26 #include "sanitizer_common/sanitizer_posix.h"
  27 #endif
  28 #include "xray_defs.h"
  29 #include "xray_utils.h"
  30 #include <cstddef>
  31 #include <cstdint>
  32 #include <sys/mman.h>
  33
  34 namespace __xray {
  35
  36 // We implement our own memory allocation routine which will bypass the
  37 // internal allocator. This allows us to manage the memory directly, using
  38 // mmap'ed memory to back the allocators.
  39 template <class T> T *allocate() XRAY_NEVER_INSTRUMENT {
  40   uptr RoundedSize = RoundUpTo(sizeof(T), GetPageSizeCached());
  41 #if SANITIZER_FUCHSIA
  42   zx_handle_t Vmo;
  43   zx_status_t Status = _zx_vmo_create(RoundedSize, 0, &Vmo);
  44   if (Status != ZX_OK) {
  45     if (Verbosity())
  46       Report("XRay Profiling: Failed to create VMO of size %zu: %s\n",
  47              sizeof(T), _zx_status_get_string(Status));
  48     return nullptr;
  49   }
  50   uintptr_t B;
  51   Status =
  52       _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0,
  53                    Vmo, 0, sizeof(T), &B);
  54   _zx_handle_close(Vmo);
  55   if (Status != ZX_OK) {
  56     if (Verbosity())
  57       Report("XRay Profiling: Failed to map VMAR of size %zu: %s\n", sizeof(T),
  58              _zx_status_get_string(Status));
  59     return nullptr;
  60   }
  61   return reinterpret_cast<T *>(B);
  62 #else
  63   uptr B = internal_mmap(NULL, RoundedSize, PROT_READ | PROT_WRITE,
  64                          MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
  65   int ErrNo = 0;
  66   if (UNLIKELY(internal_iserror(B, &ErrNo))) {
  67     if (Verbosity())
  68       Report("XRay Profiling: Failed to allocate memory of size %zu; Error = "
  69              "%zu\n",
  70              RoundedSize, B);
  71     return nullptr;
  72   }
  73 #endif
  74   return reinterpret_cast<T *>(B);
  75 }
  76
  77 template <class T> void deallocate(T *B) XRAY_NEVER_INSTRUMENT {
  78   if (B == nullptr)
  79     return;
  80   uptr RoundedSize = RoundUpTo(sizeof(T), GetPageSizeCached());
  81 #if SANITIZER_FUCHSIA
  82   _zx_vmar_unmap(_zx_vmar_root_self(), reinterpret_cast<uintptr_t>(B),
  83                  RoundedSize);
  84 #else
  85   internal_munmap(B, RoundedSize);
  86 #endif
  87 }
  88
  89 template <class T = unsigned char>
  90 T *allocateBuffer(size_t S) XRAY_NEVER_INSTRUMENT {
  91   uptr RoundedSize = RoundUpTo(S * sizeof(T), GetPageSizeCached());
  92 #if SANITIZER_FUCHSIA
  93   zx_handle_t Vmo;
  94   zx_status_t Status = _zx_vmo_create(RoundedSize, 0, &Vmo);
  95   if (Status != ZX_OK) {
  96     if (Verbosity())
  97       Report("XRay Profiling: Failed to create VMO of size %zu: %s\n", S,
  98              _zx_status_get_string(Status));
  99     return nullptr;
 100   }
 101   uintptr_t B;
 102   Status = _zx_vmar_map(_zx_vmar_root_self(),
 103                         ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0, Vmo, 0, S, &B);
 104   _zx_handle_close(Vmo);
 105   if (Status != ZX_OK) {
 106     if (Verbosity())
 107       Report("XRay Profiling: Failed to map VMAR of size %zu: %s\n", S,
 108              _zx_status_get_string(Status));
 109     return nullptr;
 110   }
 111 #else
 112   uptr B = internal_mmap(NULL, RoundedSize, PROT_READ | PROT_WRITE,
 113                          MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
 114   int ErrNo = 0;
 115   if (UNLIKELY(internal_iserror(B, &ErrNo))) {
 116     if (Verbosity())
 117       Report("XRay Profiling: Failed to allocate memory of size %zu; Error = "
 118              "%zu\n",
 119              RoundedSize, B);
 120     return nullptr;
 121   }
 122 #endif
 123   return reinterpret_cast<T *>(B);
 124 }
 125
 126 template <class T> void deallocateBuffer(T *B, size_t S) XRAY_NEVER_INSTRUMENT {
 127   if (B == nullptr)
 128     return;
 129   uptr RoundedSize = RoundUpTo(S * sizeof(T), GetPageSizeCached());
 130 #if SANITIZER_FUCHSIA
 131   _zx_vmar_unmap(_zx_vmar_root_self(), reinterpret_cast<uintptr_t>(B),
 132                  RoundedSize);
 133 #else
 134   internal_munmap(B, RoundedSize);
 135 #endif
 136 }
 137
 138 template <class T, class... U>
 139 T *initArray(size_t N, U &&... Us) XRAY_NEVER_INSTRUMENT {
 140   auto A = allocateBuffer<T>(N);
 141   if (A != nullptr)
 142     while (N > 0)
 143       new (A + (--N)) T(std::forward<U>(Us)...);
 144   return A;
 145 }
 146
 147 /// The Allocator type hands out fixed-sized chunks of memory that are
 148 /// cache-line aligned and sized. This is useful for placement of
 149 /// performance-sensitive data in memory that's frequently accessed. The
 150 /// allocator also self-limits the peak memory usage to a dynamically defined
 151 /// maximum.
 152 ///
 153 /// N is the lower-bound size of the block of memory to return from the
 154 /// allocation function. N is used to compute the size of a block, which is
 155 /// cache-line-size multiples worth of memory. We compute the size of a block by
 156 /// determining how many cache lines worth of memory is required to subsume N.
 157 ///
 158 /// The Allocator instance will manage its own memory acquired through mmap.
 159 /// This severely constrains the platforms on which this can be used to POSIX
 160 /// systems where mmap semantics are well-defined.
 161 ///
 162 /// FIXME: Isolate the lower-level memory management to a different abstraction
 163 /// that can be platform-specific.
 164 template <size_t N> struct Allocator {
 165   // The Allocator returns memory as Block instances.
 166   struct Block {
 167     /// Compute the minimum cache-line size multiple that is >= N.
 168     static constexpr auto Size = nearest_boundary(N, kCacheLineSize);
 169     void *Data;
 170   };
 171
 172 private:
 173   size_t MaxMemory{0};
 174   unsigned char *BackingStore = nullptr;
 175   unsigned char *AlignedNextBlock = nullptr;
 176   size_t AllocatedBlocks = 0;
 177   bool Owned;
 178   SpinMutex Mutex{};
 179
 180   void *Alloc() XRAY_NEVER_INSTRUMENT {
 181     SpinMutexLock Lock(&Mutex);
 182     if (UNLIKELY(BackingStore == nullptr)) {
 183       BackingStore = allocateBuffer(MaxMemory);
 184       if (BackingStore == nullptr) {
 185         if (Verbosity())
 186           Report("XRay Profiling: Failed to allocate memory for allocator\n");
 187         return nullptr;
 188       }
 189
 190       AlignedNextBlock = BackingStore;
 191
 192       // Ensure that NextBlock is aligned appropriately.
 193       auto BackingStoreNum = reinterpret_cast<uintptr_t>(BackingStore);
 194       auto AlignedNextBlockNum = nearest_boundary(
 195           reinterpret_cast<uintptr_t>(AlignedNextBlock), kCacheLineSize);
 196       if (diff(AlignedNextBlockNum, BackingStoreNum) > ptrdiff_t(MaxMemory)) {
 197         deallocateBuffer(BackingStore, MaxMemory);
 198         AlignedNextBlock = BackingStore = nullptr;
 199         if (Verbosity())
 200           Report("XRay Profiling: Cannot obtain enough memory from "
 201                  "preallocated region\n");
 202         return nullptr;
 203       }
 204
 205       AlignedNextBlock = reinterpret_cast<unsigned char *>(AlignedNextBlockNum);
 206
 207       // Assert that AlignedNextBlock is cache-line aligned.
 208       DCHECK_EQ(reinterpret_cast<uintptr_t>(AlignedNextBlock) % kCacheLineSize,
 209                 0);
 210     }
 211
 212     if (((AllocatedBlocks + 1) * Block::Size) > MaxMemory)
 213       return nullptr;
 214
 215     // Align the pointer we'd like to return to an appropriate alignment, then
 216     // advance the pointer from where to start allocations.
 217     void *Result = AlignedNextBlock;
 218     AlignedNextBlock =
 219         reinterpret_cast<unsigned char *>(AlignedNextBlock) + Block::Size;
 220     ++AllocatedBlocks;
 221     return Result;
 222   }
 223
 224 public:
 225   explicit Allocator(size_t M) XRAY_NEVER_INSTRUMENT
 226       : MaxMemory(RoundUpTo(M, kCacheLineSize)),
 227         BackingStore(nullptr),
 228         AlignedNextBlock(nullptr),
 229         AllocatedBlocks(0),
 230         Owned(true),
 231         Mutex() {}
 232
 233   explicit Allocator(void *P, size_t M) XRAY_NEVER_INSTRUMENT
 234       : MaxMemory(M),
 235         BackingStore(reinterpret_cast<unsigned char *>(P)),
 236         AlignedNextBlock(reinterpret_cast<unsigned char *>(P)),
 237         AllocatedBlocks(0),
 238         Owned(false),
 239         Mutex() {}
 240
 241   Allocator(const Allocator &) = delete;
 242   Allocator &operator=(const Allocator &) = delete;
 243
 244   Allocator(Allocator &&O) XRAY_NEVER_INSTRUMENT {
 245     SpinMutexLock L0(&Mutex);
 246     SpinMutexLock L1(&O.Mutex);
 247     MaxMemory = O.MaxMemory;
 248     O.MaxMemory = 0;
 249     BackingStore = O.BackingStore;
 250     O.BackingStore = nullptr;
 251     AlignedNextBlock = O.AlignedNextBlock;
 252     O.AlignedNextBlock = nullptr;
 253     AllocatedBlocks = O.AllocatedBlocks;
 254     O.AllocatedBlocks = 0;
 255     Owned = O.Owned;
 256     O.Owned = false;
 257   }
 258
 259   Allocator &operator=(Allocator &&O) XRAY_NEVER_INSTRUMENT {
 260     SpinMutexLock L0(&Mutex);
 261     SpinMutexLock L1(&O.Mutex);
 262     MaxMemory = O.MaxMemory;
 263     O.MaxMemory = 0;
 264     if (BackingStore != nullptr)
 265       deallocateBuffer(BackingStore, MaxMemory);
 266     BackingStore = O.BackingStore;
 267     O.BackingStore = nullptr;
 268     AlignedNextBlock = O.AlignedNextBlock;
 269     O.AlignedNextBlock = nullptr;
 270     AllocatedBlocks = O.AllocatedBlocks;
 271     O.AllocatedBlocks = 0;
 272     Owned = O.Owned;
 273     O.Owned = false;
 274     return *this;
 275   }
 276
 277   Block Allocate() XRAY_NEVER_INSTRUMENT { return {Alloc()}; }
 278
 279   ~Allocator() NOEXCEPT XRAY_NEVER_INSTRUMENT {
 280     if (Owned && BackingStore != nullptr) {
 281       deallocateBuffer(BackingStore, MaxMemory);
 282     }
 283   }
 284 };
 285
 286 } // namespace __xray
 287
 288 #endif // XRAY_ALLOCATOR_H