compiler-rt/lib/dfsan/dfsan_allocator.cpp

   1 //===-- dfsan_allocator.cpp -------------------------- --------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file is a part of DataflowSanitizer.
  10 //
  11 // DataflowSanitizer allocator.
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "dfsan_allocator.h"
  15
  16 #include "dfsan.h"
  17 #include "dfsan_flags.h"
  18 #include "dfsan_thread.h"
  19 #include "sanitizer_common/sanitizer_allocator.h"
  20 #include "sanitizer_common/sanitizer_allocator_checks.h"
  21 #include "sanitizer_common/sanitizer_allocator_interface.h"
  22 #include "sanitizer_common/sanitizer_allocator_report.h"
  23 #include "sanitizer_common/sanitizer_errno.h"
  24
  25 using namespace __dfsan;
  26
  27 namespace {
  28
  29 struct Metadata {
  30   uptr requested_size;
  31 };
  32
  33 struct DFsanMapUnmapCallback {
  34   void OnMap(uptr p, uptr size) const { dfsan_set_label(0, (void *)p, size); }
  35   void OnMapSecondary(uptr p, uptr size, uptr user_begin,
  36                       uptr user_size) const {
  37     OnMap(p, size);
  38   }
  39   void OnUnmap(uptr p, uptr size) const { dfsan_set_label(0, (void *)p, size); }
  40 };
  41
  42 // Note: to ensure that the allocator is compatible with the application memory
  43 // layout (especially with high-entropy ASLR), kSpaceBeg and kSpaceSize must be
  44 // duplicated as MappingDesc::ALLOCATOR in dfsan_platform.h.
  45 #if defined(__aarch64__)
  46 const uptr kAllocatorSpace = 0xE00000000000ULL;
  47 #else
  48 const uptr kAllocatorSpace = 0x700000000000ULL;
  49 #endif
  50 const uptr kMaxAllowedMallocSize = 1ULL << 40;
  51
  52 struct AP64 {  // Allocator64 parameters. Deliberately using a short name.
  53   static const uptr kSpaceBeg = kAllocatorSpace;
  54   static const uptr kSpaceSize = 0x40000000000;  // 4T.
  55   static const uptr kMetadataSize = sizeof(Metadata);
  56   typedef DefaultSizeClassMap SizeClassMap;
  57   typedef DFsanMapUnmapCallback MapUnmapCallback;
  58   static const uptr kFlags = 0;
  59   using AddressSpaceView = LocalAddressSpaceView;
  60 };
  61
  62 typedef SizeClassAllocator64<AP64> PrimaryAllocator;
  63
  64 typedef CombinedAllocator<PrimaryAllocator> Allocator;
  65 typedef Allocator::AllocatorCache AllocatorCache;
  66
  67 static Allocator allocator;
  68 static AllocatorCache fallback_allocator_cache;
  69 static StaticSpinMutex fallback_mutex;
  70
  71 static uptr max_malloc_size;
  72 }  // namespace
  73
  74 void __dfsan::dfsan_allocator_init() {
  75   SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
  76   allocator.Init(common_flags()->allocator_release_to_os_interval_ms);
  77   if (common_flags()->max_allocation_size_mb)
  78     max_malloc_size = Min(common_flags()->max_allocation_size_mb << 20,
  79                           kMaxAllowedMallocSize);
  80   else
  81     max_malloc_size = kMaxAllowedMallocSize;
  82 }
  83
  84 static AllocatorCache *GetAllocatorCache(DFsanThreadLocalMallocStorage *ms) {
  85   CHECK(ms);
  86   CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache));
  87   return reinterpret_cast<AllocatorCache *>(ms->allocator_cache);
  88 }
  89
  90 void DFsanThreadLocalMallocStorage::CommitBack() {
  91   allocator.SwallowCache(GetAllocatorCache(this));
  92 }
  93
  94 static void *DFsanAllocate(uptr size, uptr alignment, bool zeroise) {
  95   if (size > max_malloc_size) {
  96     if (AllocatorMayReturnNull()) {
  97       Report("WARNING: DataflowSanitizer failed to allocate 0x%zx bytes\n",
  98              size);
  99       return nullptr;
 100     }
 101     UNINITIALIZED BufferedStackTrace stack;
 102     ReportAllocationSizeTooBig(size, max_malloc_size, &stack);
 103   }
 104   if (UNLIKELY(IsRssLimitExceeded())) {
 105     if (AllocatorMayReturnNull())
 106       return nullptr;
 107     UNINITIALIZED BufferedStackTrace stack;
 108     ReportRssLimitExceeded(&stack);
 109   }
 110   DFsanThread *t = GetCurrentThread();
 111   void *allocated;
 112   if (t) {
 113     AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
 114     allocated = allocator.Allocate(cache, size, alignment);
 115   } else {
 116     SpinMutexLock l(&fallback_mutex);
 117     AllocatorCache *cache = &fallback_allocator_cache;
 118     allocated = allocator.Allocate(cache, size, alignment);
 119   }
 120   if (UNLIKELY(!allocated)) {
 121     SetAllocatorOutOfMemory();
 122     if (AllocatorMayReturnNull())
 123       return nullptr;
 124     UNINITIALIZED BufferedStackTrace stack;
 125     ReportOutOfMemory(size, &stack);
 126   }
 127   Metadata *meta =
 128       reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated));
 129   meta->requested_size = size;
 130   if (zeroise) {
 131     internal_memset(allocated, 0, size);
 132     dfsan_set_label(0, allocated, size);
 133   } else if (flags().zero_in_malloc) {
 134     dfsan_set_label(0, allocated, size);
 135   }
 136   return allocated;
 137 }
 138
 139 void __dfsan::dfsan_deallocate(void *p) {
 140   CHECK(p);
 141   Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p));
 142   uptr size = meta->requested_size;
 143   meta->requested_size = 0;
 144   if (flags().zero_in_free)
 145     dfsan_set_label(0, p, size);
 146   DFsanThread *t = GetCurrentThread();
 147   if (t) {
 148     AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
 149     allocator.Deallocate(cache, p);
 150   } else {
 151     SpinMutexLock l(&fallback_mutex);
 152     AllocatorCache *cache = &fallback_allocator_cache;
 153     allocator.Deallocate(cache, p);
 154   }
 155 }
 156
 157 static void *DFsanReallocate(void *old_p, uptr new_size, uptr alignment) {
 158   Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(old_p));
 159   uptr old_size = meta->requested_size;
 160   uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(old_p);
 161   if (new_size <= actually_allocated_size) {
 162     // We are not reallocating here.
 163     meta->requested_size = new_size;
 164     if (new_size > old_size && flags().zero_in_malloc)
 165       dfsan_set_label(0, (char *)old_p + old_size, new_size - old_size);
 166     return old_p;
 167   }
 168   uptr memcpy_size = Min(new_size, old_size);
 169   void *new_p = DFsanAllocate(new_size, alignment, false /*zeroise*/);
 170   if (new_p) {
 171     dfsan_copy_memory(new_p, old_p, memcpy_size);
 172     dfsan_deallocate(old_p);
 173   }
 174   return new_p;
 175 }
 176
 177 static void *DFsanCalloc(uptr nmemb, uptr size) {
 178   if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
 179     if (AllocatorMayReturnNull())
 180       return nullptr;
 181     UNINITIALIZED BufferedStackTrace stack;
 182     ReportCallocOverflow(nmemb, size, &stack);
 183   }
 184   return DFsanAllocate(nmemb * size, sizeof(u64), true /*zeroise*/);
 185 }
 186
 187 static const void *AllocationBegin(const void *p) {
 188   if (!p)
 189     return nullptr;
 190   void *beg = allocator.GetBlockBegin(p);
 191   if (!beg)
 192     return nullptr;
 193   Metadata *b = (Metadata *)allocator.GetMetaData(beg);
 194   if (!b)
 195     return nullptr;
 196   if (b->requested_size == 0)
 197     return nullptr;
 198   return (const void *)beg;
 199 }
 200
 201 static uptr AllocationSize(const void *p) {
 202   if (!p)
 203     return 0;
 204   const void *beg = allocator.GetBlockBegin(p);
 205   if (beg != p)
 206     return 0;
 207   Metadata *b = (Metadata *)allocator.GetMetaData(p);
 208   return b->requested_size;
 209 }
 210
 211 static uptr AllocationSizeFast(const void *p) {
 212   return reinterpret_cast<Metadata *>(allocator.GetMetaData(p))->requested_size;
 213 }
 214
 215 void *__dfsan::dfsan_malloc(uptr size) {
 216   return SetErrnoOnNull(DFsanAllocate(size, sizeof(u64), false /*zeroise*/));
 217 }
 218
 219 void *__dfsan::dfsan_calloc(uptr nmemb, uptr size) {
 220   return SetErrnoOnNull(DFsanCalloc(nmemb, size));
 221 }
 222
 223 void *__dfsan::dfsan_realloc(void *ptr, uptr size) {
 224   if (!ptr)
 225     return SetErrnoOnNull(DFsanAllocate(size, sizeof(u64), false /*zeroise*/));
 226   if (size == 0) {
 227     dfsan_deallocate(ptr);
 228     return nullptr;
 229   }
 230   return SetErrnoOnNull(DFsanReallocate(ptr, size, sizeof(u64)));
 231 }
 232
 233 void *__dfsan::dfsan_reallocarray(void *ptr, uptr nmemb, uptr size) {
 234   if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
 235     errno = errno_ENOMEM;
 236     if (AllocatorMayReturnNull())
 237       return nullptr;
 238     UNINITIALIZED BufferedStackTrace stack;
 239     ReportReallocArrayOverflow(nmemb, size, &stack);
 240   }
 241   return dfsan_realloc(ptr, nmemb * size);
 242 }
 243
 244 void *__dfsan::dfsan_valloc(uptr size) {
 245   return SetErrnoOnNull(
 246       DFsanAllocate(size, GetPageSizeCached(), false /*zeroise*/));
 247 }
 248
 249 void *__dfsan::dfsan_pvalloc(uptr size) {
 250   uptr PageSize = GetPageSizeCached();
 251   if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
 252     errno = errno_ENOMEM;
 253     if (AllocatorMayReturnNull())
 254       return nullptr;
 255     UNINITIALIZED BufferedStackTrace stack;
 256     ReportPvallocOverflow(size, &stack);
 257   }
 258   // pvalloc(0) should allocate one page.
 259   size = size ? RoundUpTo(size, PageSize) : PageSize;
 260   return SetErrnoOnNull(DFsanAllocate(size, PageSize, false /*zeroise*/));
 261 }
 262
 263 void *__dfsan::dfsan_aligned_alloc(uptr alignment, uptr size) {
 264   if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
 265     errno = errno_EINVAL;
 266     if (AllocatorMayReturnNull())
 267       return nullptr;
 268     UNINITIALIZED BufferedStackTrace stack;
 269     ReportInvalidAlignedAllocAlignment(size, alignment, &stack);
 270   }
 271   return SetErrnoOnNull(DFsanAllocate(size, alignment, false /*zeroise*/));
 272 }
 273
 274 void *__dfsan::dfsan_memalign(uptr alignment, uptr size) {
 275   if (UNLIKELY(!IsPowerOfTwo(alignment))) {
 276     errno = errno_EINVAL;
 277     if (AllocatorMayReturnNull())
 278       return nullptr;
 279     UNINITIALIZED BufferedStackTrace stack;
 280     ReportInvalidAllocationAlignment(alignment, &stack);
 281   }
 282   return SetErrnoOnNull(DFsanAllocate(size, alignment, false /*zeroise*/));
 283 }
 284
 285 int __dfsan::dfsan_posix_memalign(void **memptr, uptr alignment, uptr size) {
 286   if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
 287     if (AllocatorMayReturnNull())
 288       return errno_EINVAL;
 289     UNINITIALIZED BufferedStackTrace stack;
 290     ReportInvalidPosixMemalignAlignment(alignment, &stack);
 291   }
 292   void *ptr = DFsanAllocate(size, alignment, false /*zeroise*/);
 293   if (UNLIKELY(!ptr))
 294     // OOM error is already taken care of by DFsanAllocate.
 295     return errno_ENOMEM;
 296   CHECK(IsAligned((uptr)ptr, alignment));
 297   *memptr = ptr;
 298   return 0;
 299 }
 300
 301 extern "C" {
 302 uptr __sanitizer_get_current_allocated_bytes() {
 303   uptr stats[AllocatorStatCount];
 304   allocator.GetStats(stats);
 305   return stats[AllocatorStatAllocated];
 306 }
 307
 308 uptr __sanitizer_get_heap_size() {
 309   uptr stats[AllocatorStatCount];
 310   allocator.GetStats(stats);
 311   return stats[AllocatorStatMapped];
 312 }
 313
 314 uptr __sanitizer_get_free_bytes() { return 1; }
 315
 316 uptr __sanitizer_get_unmapped_bytes() { return 1; }
 317
 318 uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
 319
 320 int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; }
 321
 322 const void *__sanitizer_get_allocated_begin(const void *p) {
 323   return AllocationBegin(p);
 324 }
 325
 326 uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); }
 327
 328 uptr __sanitizer_get_allocated_size_fast(const void *p) {
 329   DCHECK_EQ(p, __sanitizer_get_allocated_begin(p));
 330   uptr ret = AllocationSizeFast(p);
 331   DCHECK_EQ(ret, __sanitizer_get_allocated_size(p));
 332   return ret;
 333 }
 334 }