base/allocator/allocator_shim.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/allocator/allocator_shim.h"
   6
   7 #include <config.h>
   8 #include "base/allocator/allocator_extension_thunks.h"
   9 #include "base/profiler/alternate_timer.h"
  10 #include "base/sysinfo.h"
  11 #include "jemalloc.h"
  12
  13 // When defined, different heap allocators can be used via an environment
  14 // variable set before running the program.  This may reduce the amount
  15 // of inlining that we get with malloc/free/etc.  Disabling makes it
  16 // so that only tcmalloc can be used.
  17 #define ENABLE_DYNAMIC_ALLOCATOR_SWITCHING
  18
  19 // TODO(mbelshe): Ensure that all calls to tcmalloc have the proper call depth
  20 // from the "user code" so that debugging tools (HeapChecker) can work.
  21
  22 // __THROW is defined in glibc systems.  It means, counter-intuitively,
  23 // "This function will never throw an exception."  It's an optional
  24 // optimization tool, but we may need to use it to match glibc prototypes.
  25 #ifndef __THROW    // I guess we're not on a glibc system
  26 # define __THROW   // __THROW is just an optimization, so ok to make it ""
  27 #endif
  28
  29 // new_mode behaves similarly to MSVC's _set_new_mode.
  30 // If flag is 0 (default), calls to malloc will behave normally.
  31 // If flag is 1, calls to malloc will behave like calls to new,
  32 // and the std_new_handler will be invoked on failure.
  33 // Can be set by calling _set_new_mode().
  34 static int new_mode = 0;
  35
  36 typedef enum {
  37   TCMALLOC,    // TCMalloc is the default allocator.
  38   JEMALLOC,    // JEMalloc.
  39   WINHEAP,  // Windows Heap (standard Windows allocator).
  40   WINLFH,      // Windows LFH Heap.
  41 } Allocator;
  42
  43 // This is the default allocator. This value can be changed at startup by
  44 // specifying environment variables shown below it.
  45 // See SetupSubprocessAllocator() to specify a default secondary (subprocess)
  46 // allocator.
  47 // TODO(jar): Switch to using TCMALLOC for the renderer as well.
  48 #if (defined(ADDRESS_SANITIZER) && defined(OS_WIN))
  49 // The Windows implementation of Asan requires the use of "WINHEAP".
  50 static Allocator allocator = WINHEAP;
  51 #else
  52 static Allocator allocator = TCMALLOC;
  53 #endif
  54 // The names of the environment variables that can optionally control the
  55 // selection of the allocator.  The primary may be used to control overall
  56 // allocator selection, and the secondary can be used to specify an allocator
  57 // to use in sub-processes.
  58 static const char primary_name[] = "CHROME_ALLOCATOR";
  59 static const char secondary_name[] = "CHROME_ALLOCATOR_2";
  60
  61 // We include tcmalloc and the win_allocator to get as much inlining as
  62 // possible.
  63 #include "debugallocation_shim.cc"
  64 #include "win_allocator.cc"
  65
  66 // Forward declarations from jemalloc.
  67 extern "C" {
  68 void* je_malloc(size_t s);
  69 void* je_realloc(void* p, size_t s);
  70 void je_free(void* s);
  71 size_t je_msize(void* p);
  72 bool je_malloc_init_hard();
  73 void* je_memalign(size_t a, size_t s);
  74 }
  75
  76 // Call the new handler, if one has been set.
  77 // Returns true on successfully calling the handler, false otherwise.
  78 inline bool call_new_handler(bool nothrow) {
  79   // Get the current new handler.  NB: this function is not
  80   // thread-safe.  We make a feeble stab at making it so here, but
  81   // this lock only protects against tcmalloc interfering with
  82   // itself, not with other libraries calling set_new_handler.
  83   std::new_handler nh;
  84   {
  85     SpinLockHolder h(&set_new_handler_lock);
  86     nh = std::set_new_handler(0);
  87     (void) std::set_new_handler(nh);
  88   }
  89 #if (defined(__GNUC__) && !defined(__EXCEPTIONS)) || \
  90     (defined(_HAS_EXCEPTIONS) && !_HAS_EXCEPTIONS)
  91   if (!nh)
  92     return false;
  93   // Since exceptions are disabled, we don't really know if new_handler
  94   // failed.  Assume it will abort if it fails.
  95   (*nh)();
  96   return false;  // break out of the retry loop.
  97 #else
  98   // If no new_handler is established, the allocation failed.
  99   if (!nh) {
 100     if (nothrow)
 101       return false;
 102     throw std::bad_alloc();
 103   }
 104   // Otherwise, try the new_handler.  If it returns, retry the
 105   // allocation.  If it throws std::bad_alloc, fail the allocation.
 106   // if it throws something else, don't interfere.
 107   try {
 108     (*nh)();
 109   } catch (const std::bad_alloc&) {
 110     if (!nothrow)
 111       throw;
 112     return true;
 113   }
 114 #endif  // (defined(__GNUC__) && !defined(__EXCEPTIONS)) || (defined(_HAS_EXCEPTIONS) && !_HAS_EXCEPTIONS)
 115   return false;
 116 }
 117
 118 extern "C" {
 119 void* malloc(size_t size) __THROW {
 120   void* ptr;
 121   for (;;) {
 122 #ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING
 123     switch (allocator) {
 124       case JEMALLOC:
 125         ptr = je_malloc(size);
 126         break;
 127       case WINHEAP:
 128       case WINLFH:
 129         ptr = win_heap_malloc(size);
 130         break;
 131       case TCMALLOC:
 132       default:
 133         ptr = do_malloc(size);
 134         break;
 135     }
 136 #else
 137     // TCMalloc case.
 138     ptr = do_malloc(size);
 139 #endif
 140     if (ptr)
 141       return ptr;
 142
 143     if (!new_mode || !call_new_handler(true))
 144       break;
 145   }
 146   return ptr;
 147 }
 148
 149 void free(void* p) __THROW {
 150 #ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING
 151   switch (allocator) {
 152     case JEMALLOC:
 153       je_free(p);
 154       return;
 155     case WINHEAP:
 156     case WINLFH:
 157       win_heap_free(p);
 158       return;
 159   }
 160 #endif
 161   // TCMalloc case.
 162   do_free(p);
 163 }
 164
 165 void* realloc(void* ptr, size_t size) __THROW {
 166   // Webkit is brittle for allocators that return NULL for malloc(0).  The
 167   // realloc(0, 0) code path does not guarantee a non-NULL return, so be sure
 168   // to call malloc for this case.
 169   if (!ptr)
 170     return malloc(size);
 171
 172   void* new_ptr;
 173   for (;;) {
 174 #ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING
 175     switch (allocator) {
 176       case JEMALLOC:
 177         new_ptr = je_realloc(ptr, size);
 178         break;
 179       case WINHEAP:
 180       case WINLFH:
 181         new_ptr = win_heap_realloc(ptr, size);
 182         break;
 183       case TCMALLOC:
 184       default:
 185         new_ptr = do_realloc(ptr, size);
 186         break;
 187     }
 188 #else
 189     // TCMalloc case.
 190     new_ptr = do_realloc(ptr, size);
 191 #endif
 192
 193     // Subtle warning:  NULL return does not alwas indicate out-of-memory.  If
 194     // the requested new size is zero, realloc should free the ptr and return
 195     // NULL.
 196     if (new_ptr || !size)
 197       return new_ptr;
 198     if (!new_mode || !call_new_handler(true))
 199       break;
 200   }
 201   return new_ptr;
 202 }
 203
 204 // TODO(mbelshe): Implement this for other allocators.
 205 void malloc_stats(void) __THROW {
 206 #ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING
 207   switch (allocator) {
 208     case JEMALLOC:
 209       // No stats.
 210       return;
 211     case WINHEAP:
 212     case WINLFH:
 213       // No stats.
 214       return;
 215   }
 216 #endif
 217   tc_malloc_stats();
 218 }
 219
 220 #ifdef WIN32
 221
 222 extern "C" size_t _msize(void* p) {
 223 #ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING
 224   switch (allocator) {
 225     case JEMALLOC:
 226       return je_msize(p);
 227     case WINHEAP:
 228     case WINLFH:
 229       return win_heap_msize(p);
 230   }
 231 #endif
 232   return MallocExtension::instance()->GetAllocatedSize(p);
 233 }
 234
 235 // This is included to resolve references from libcmt.
 236 extern "C" intptr_t _get_heap_handle() {
 237   return 0;
 238 }
 239
 240 static bool get_allocator_waste_size_thunk(size_t* size) {
 241 #ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING
 242   switch (allocator) {
 243     case JEMALLOC:
 244     case WINHEAP:
 245     case WINLFH:
 246       // TODO(alexeif): Implement for allocators other than tcmalloc.
 247       return false;
 248   }
 249 #endif
 250   size_t heap_size, allocated_bytes, unmapped_bytes;
 251   MallocExtension* ext = MallocExtension::instance();
 252   if (ext->GetNumericProperty("generic.heap_size", &heap_size) &&
 253       ext->GetNumericProperty("generic.current_allocated_bytes",
 254                               &allocated_bytes) &&
 255       ext->GetNumericProperty("tcmalloc.pageheap_unmapped_bytes",
 256                               &unmapped_bytes)) {
 257     *size = heap_size - allocated_bytes - unmapped_bytes;
 258     return true;
 259   }
 260   return false;
 261 }
 262
 263 static void get_stats_thunk(char* buffer, int buffer_length) {
 264   MallocExtension::instance()->GetStats(buffer, buffer_length);
 265 }
 266
 267 static void release_free_memory_thunk() {
 268   MallocExtension::instance()->ReleaseFreeMemory();
 269 }
 270
 271 // The CRT heap initialization stub.
 272 extern "C" int _heap_init() {
 273 #ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING
 274 // Don't use the environment variable if ADDRESS_SANITIZER is defined on
 275 // Windows, as the implementation requires Winheap to be the allocator.
 276 #if !(defined(ADDRESS_SANITIZER) && defined(OS_WIN))
 277   const char* environment_value = GetenvBeforeMain(primary_name);
 278   if (environment_value) {
 279     if (!stricmp(environment_value, "jemalloc"))
 280       allocator = JEMALLOC;
 281     else if (!stricmp(environment_value, "winheap"))
 282       allocator = WINHEAP;
 283     else if (!stricmp(environment_value, "winlfh"))
 284       allocator = WINLFH;
 285     else if (!stricmp(environment_value, "tcmalloc"))
 286       allocator = TCMALLOC;
 287   }
 288 #endif
 289
 290   switch (allocator) {
 291     case JEMALLOC:
 292       return je_malloc_init_hard() ? 0 : 1;
 293     case WINHEAP:
 294       return win_heap_init(false) ? 1 : 0;
 295     case WINLFH:
 296       return win_heap_init(true) ? 1 : 0;
 297     case TCMALLOC:
 298     default:
 299       // fall through
 300       break;
 301   }
 302 #endif
 303   // Initializing tcmalloc.
 304   // We intentionally leak this object.  It lasts for the process
 305   // lifetime.  Trying to teardown at _heap_term() is so late that
 306   // you can't do anything useful anyway.
 307   new TCMallocGuard();
 308
 309   // Provide optional hook for monitoring allocation quantities on a per-thread
 310   // basis.  Only set the hook if the environment indicates this needs to be
 311   // enabled.
 312   const char* profiling =
 313       GetenvBeforeMain(tracked_objects::kAlternateProfilerTime);
 314   if (profiling && *profiling == '1') {
 315     tracked_objects::SetAlternateTimeSource(
 316         tcmalloc::ThreadCache::GetBytesAllocatedOnCurrentThread,
 317         tracked_objects::TIME_SOURCE_TYPE_TCMALLOC);
 318   }
 319
 320   base::allocator::thunks::SetGetAllocatorWasteSizeFunction(
 321       get_allocator_waste_size_thunk);
 322   base::allocator::thunks::SetGetStatsFunction(get_stats_thunk);
 323   base::allocator::thunks::SetReleaseFreeMemoryFunction(
 324       release_free_memory_thunk);
 325
 326   return 1;
 327 }
 328
 329 // The CRT heap cleanup stub.
 330 extern "C" void _heap_term() {}
 331
 332 // We set this to 1 because part of the CRT uses a check of _crtheap != 0
 333 // to test whether the CRT has been initialized.  Once we've ripped out
 334 // the allocators from libcmt, we need to provide this definition so that
 335 // the rest of the CRT is still usable.
 336 extern "C" void* _crtheap = reinterpret_cast<void*>(1);
 337
 338 // Provide support for aligned memory through Windows only _aligned_malloc().
 339 void* _aligned_malloc(size_t size, size_t alignment) {
 340   // _aligned_malloc guarantees parameter validation, so do so here.  These
 341   // checks are somewhat stricter than _aligned_malloc() since we're effectively
 342   // using memalign() under the hood.
 343   DCHECK_GT(size, 0U);
 344   DCHECK_EQ(alignment & (alignment - 1), 0U);
 345   DCHECK_EQ(alignment % sizeof(void*), 0U);
 346
 347   void* ptr;
 348   for (;;) {
 349 #ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING
 350     switch (allocator) {
 351       case JEMALLOC:
 352         ptr = je_memalign(alignment, size);
 353         break;
 354       case WINHEAP:
 355       case WINLFH:
 356         ptr = win_heap_memalign(alignment, size);
 357         break;
 358       case TCMALLOC:
 359       default:
 360         ptr = tc_memalign(alignment, size);
 361         break;
 362     }
 363 #else
 364     // TCMalloc case.
 365     ptr = tc_memalign(alignment, size);
 366 #endif
 367     if (ptr) {
 368       // Sanity check alignment.
 369       DCHECK_EQ(reinterpret_cast<uintptr_t>(ptr) & (alignment - 1), 0U);
 370       return ptr;
 371     }
 372
 373     if (!new_mode || !call_new_handler(true))
 374       break;
 375   }
 376   return ptr;
 377 }
 378
 379 void _aligned_free(void* p) {
 380   // Both JEMalloc and TCMalloc return pointers from memalign() that are safe to
 381   // use with free().  Pointers allocated with win_heap_memalign() MUST be freed
 382   // via win_heap_memalign_free() since the aligned pointer is not the real one.
 383 #ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING
 384   switch (allocator) {
 385     case JEMALLOC:
 386       je_free(p);
 387       return;
 388     case WINHEAP:
 389     case WINLFH:
 390       win_heap_memalign_free(p);
 391       return;
 392   }
 393 #endif
 394   // TCMalloc case.
 395   do_free(p);
 396 }
 397
 398 #endif  // WIN32
 399
 400 #include "generic_allocators.cc"
 401
 402 }  // extern C
 403
 404 namespace base {
 405 namespace allocator {
 406
 407 void SetupSubprocessAllocator() {
 408 #ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING
 409   size_t primary_length = 0;
 410   getenv_s(&primary_length, NULL, 0, primary_name);
 411
 412   size_t secondary_length = 0;
 413   char buffer[20];
 414   getenv_s(&secondary_length, buffer, sizeof(buffer), secondary_name);
 415   DCHECK_GT(sizeof(buffer), secondary_length);
 416   buffer[sizeof(buffer) - 1] = '\0';
 417
 418   if (secondary_length || !primary_length) {
 419     // Don't use the environment variable if ADDRESS_SANITIZER is defined on
 420     // Windows, as the implementation require Winheap to be the allocator.
 421 #if !(defined(ADDRESS_SANITIZER) && defined(OS_WIN))
 422     const char* secondary_value = secondary_length ? buffer : "TCMALLOC";
 423     // Force renderer (or other subprocesses) to use secondary_value.
 424 #else
 425     const char* secondary_value = "WINHEAP";
 426 #endif
 427     int ret_val = _putenv_s(primary_name, secondary_value);
 428     DCHECK_EQ(0, ret_val);
 429   }
 430 #endif  // ENABLE_DYNAMIC_ALLOCATOR_SWITCHING
 431 }
 432
 433 void* TCMallocDoMallocForTest(size_t size) {
 434   return do_malloc(size);
 435 }
 436
 437 void TCMallocDoFreeForTest(void* ptr) {
 438   do_free(ptr);
 439 }
 440
 441 size_t ExcludeSpaceForMarkForTest(size_t size) {
 442   return ExcludeSpaceForMark(size);
 443 }
 444
 445 }  // namespace allocator.
 446 }  // namespace base.