llvm/lib/Support/rpmalloc/malloc.c

   1 //===------------------------ malloc.c ------------------*- C -*-=============//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This library provides a cross-platform lock free thread caching malloc
  10 // implementation in C11.
  11 //
  12 //
  13 // This file provides overrides for the standard library malloc entry points for
  14 // C and new/delete operators for C++ It also provides automatic
  15 // initialization/finalization of process and threads
  16 //
  17 //===----------------------------------------------------------------------===//
  18
  19 #if defined(__TINYC__)
  20 #include <sys/types.h>
  21 #endif
  22
  23 #ifndef ARCH_64BIT
  24 #if defined(__LLP64__) || defined(__LP64__) || defined(_WIN64)
  25 #define ARCH_64BIT 1
  26 _Static_assert(sizeof(size_t) == 8, "Data type size mismatch");
  27 _Static_assert(sizeof(void *) == 8, "Data type size mismatch");
  28 #else
  29 #define ARCH_64BIT 0
  30 _Static_assert(sizeof(size_t) == 4, "Data type size mismatch");
  31 _Static_assert(sizeof(void *) == 4, "Data type size mismatch");
  32 #endif
  33 #endif
  34
  35 #if (defined(__GNUC__) || defined(__clang__))
  36 #pragma GCC visibility push(default)
  37 #endif
  38
  39 #define USE_IMPLEMENT 1
  40 #define USE_INTERPOSE 0
  41 #define USE_ALIAS 0
  42
  43 #if defined(__APPLE__)
  44 #undef USE_INTERPOSE
  45 #define USE_INTERPOSE 1
  46
  47 typedef struct interpose_t {
  48   void *new_func;
  49   void *orig_func;
  50 } interpose_t;
  51
  52 #define MAC_INTERPOSE_PAIR(newf, oldf) {(void *)newf, (void *)oldf}
  53 #define MAC_INTERPOSE_SINGLE(newf, oldf)                                       \
  54   __attribute__((used)) static const interpose_t macinterpose##newf##oldf      \
  55       __attribute__((section("__DATA, __interpose"))) =                        \
  56           MAC_INTERPOSE_PAIR(newf, oldf)
  57
  58 #endif
  59
  60 #if !defined(_WIN32) && !defined(__APPLE__)
  61 #undef USE_IMPLEMENT
  62 #undef USE_ALIAS
  63 #define USE_IMPLEMENT 0
  64 #define USE_ALIAS 1
  65 #endif
  66
  67 #ifdef _MSC_VER
  68 #pragma warning(disable : 4100)
  69 #undef malloc
  70 #undef free
  71 #undef calloc
  72 #define RPMALLOC_RESTRICT __declspec(restrict)
  73 #else
  74 #define RPMALLOC_RESTRICT
  75 #endif
  76
  77 #if ENABLE_OVERRIDE
  78
  79 typedef struct rp_nothrow_t {
  80   int __dummy;
  81 } rp_nothrow_t;
  82
  83 #if USE_IMPLEMENT
  84
  85 extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL malloc(size_t size) {
  86   return rpmalloc(size);
  87 }
  88 extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL calloc(size_t count,
  89                                                             size_t size) {
  90   return rpcalloc(count, size);
  91 }
  92 extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL realloc(void *ptr,
  93                                                              size_t size) {
  94   return rprealloc(ptr, size);
  95 }
  96 extern inline void *RPMALLOC_CDECL reallocf(void *ptr, size_t size) {
  97   return rprealloc(ptr, size);
  98 }
  99 extern inline void *RPMALLOC_CDECL aligned_alloc(size_t alignment,
 100                                                  size_t size) {
 101   return rpaligned_alloc(alignment, size);
 102 }
 103 extern inline void *RPMALLOC_CDECL memalign(size_t alignment, size_t size) {
 104   return rpmemalign(alignment, size);
 105 }
 106 extern inline int RPMALLOC_CDECL posix_memalign(void **memptr, size_t alignment,
 107                                                 size_t size) {
 108   return rpposix_memalign(memptr, alignment, size);
 109 }
 110 extern inline void RPMALLOC_CDECL free(void *ptr) { rpfree(ptr); }
 111 extern inline void RPMALLOC_CDECL cfree(void *ptr) { rpfree(ptr); }
 112 extern inline size_t RPMALLOC_CDECL malloc_usable_size(void *ptr) {
 113   return rpmalloc_usable_size(ptr);
 114 }
 115 extern inline size_t RPMALLOC_CDECL malloc_size(void *ptr) {
 116   return rpmalloc_usable_size(ptr);
 117 }
 118
 119 #ifdef _WIN32
 120 extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL _malloc_base(size_t size) {
 121   return rpmalloc(size);
 122 }
 123 extern inline void RPMALLOC_CDECL _free_base(void *ptr) { rpfree(ptr); }
 124 extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL _calloc_base(size_t count,
 125                                                                   size_t size) {
 126   return rpcalloc(count, size);
 127 }
 128 extern inline size_t RPMALLOC_CDECL _msize(void *ptr) {
 129   return rpmalloc_usable_size(ptr);
 130 }
 131 extern inline size_t RPMALLOC_CDECL _msize_base(void *ptr) {
 132   return rpmalloc_usable_size(ptr);
 133 }
 134 extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL
 135 _realloc_base(void *ptr, size_t size) {
 136   return rprealloc(ptr, size);
 137 }
 138 #endif
 139
 140 #ifdef _WIN32
 141 // For Windows, #include <rpnew.h> in one source file to get the C++ operator
 142 // overrides implemented in your module
 143 #else
 144 // Overload the C++ operators using the mangled names
 145 // (https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling) operators
 146 // delete and delete[]
 147 #define RPDEFVIS __attribute__((visibility("default")))
 148 extern void _ZdlPv(void *p);
 149 void RPDEFVIS _ZdlPv(void *p) { rpfree(p); }
 150 extern void _ZdaPv(void *p);
 151 void RPDEFVIS _ZdaPv(void *p) { rpfree(p); }
 152 #if ARCH_64BIT
 153 // 64-bit operators new and new[], normal and aligned
 154 extern void *_Znwm(uint64_t size);
 155 void *RPDEFVIS _Znwm(uint64_t size) { return rpmalloc(size); }
 156 extern void *_Znam(uint64_t size);
 157 void *RPDEFVIS _Znam(uint64_t size) { return rpmalloc(size); }
 158 extern void *_Znwmm(uint64_t size, uint64_t align);
 159 void *RPDEFVIS _Znwmm(uint64_t size, uint64_t align) {
 160   return rpaligned_alloc(align, size);
 161 }
 162 extern void *_Znamm(uint64_t size, uint64_t align);
 163 void *RPDEFVIS _Znamm(uint64_t size, uint64_t align) {
 164   return rpaligned_alloc(align, size);
 165 }
 166 extern void *_ZnwmSt11align_val_t(uint64_t size, uint64_t align);
 167 void *RPDEFVIS _ZnwmSt11align_val_t(uint64_t size, uint64_t align) {
 168   return rpaligned_alloc(align, size);
 169 }
 170 extern void *_ZnamSt11align_val_t(uint64_t size, uint64_t align);
 171 void *RPDEFVIS _ZnamSt11align_val_t(uint64_t size, uint64_t align) {
 172   return rpaligned_alloc(align, size);
 173 }
 174 extern void *_ZnwmRKSt9nothrow_t(uint64_t size, rp_nothrow_t t);
 175 void *RPDEFVIS _ZnwmRKSt9nothrow_t(uint64_t size, rp_nothrow_t t) {
 176   (void)sizeof(t);
 177   return rpmalloc(size);
 178 }
 179 extern void *_ZnamRKSt9nothrow_t(uint64_t size, rp_nothrow_t t);
 180 void *RPDEFVIS _ZnamRKSt9nothrow_t(uint64_t size, rp_nothrow_t t) {
 181   (void)sizeof(t);
 182   return rpmalloc(size);
 183 }
 184 extern void *_ZnwmSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align,
 185                                                 rp_nothrow_t t);
 186 void *RPDEFVIS _ZnwmSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align,
 187                                                   rp_nothrow_t t) {
 188   (void)sizeof(t);
 189   return rpaligned_alloc(align, size);
 190 }
 191 extern void *_ZnamSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align,
 192                                                 rp_nothrow_t t);
 193 void *RPDEFVIS _ZnamSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align,
 194                                                   rp_nothrow_t t) {
 195   (void)sizeof(t);
 196   return rpaligned_alloc(align, size);
 197 }
 198 // 64-bit operators sized delete and delete[], normal and aligned
 199 extern void _ZdlPvm(void *p, uint64_t size);
 200 void RPDEFVIS _ZdlPvm(void *p, uint64_t size) {
 201   rpfree(p);
 202   (void)sizeof(size);
 203 }
 204 extern void _ZdaPvm(void *p, uint64_t size);
 205 void RPDEFVIS _ZdaPvm(void *p, uint64_t size) {
 206   rpfree(p);
 207   (void)sizeof(size);
 208 }
 209 extern void _ZdlPvSt11align_val_t(void *p, uint64_t align);
 210 void RPDEFVIS _ZdlPvSt11align_val_t(void *p, uint64_t align) {
 211   rpfree(p);
 212   (void)sizeof(align);
 213 }
 214 extern void _ZdaPvSt11align_val_t(void *p, uint64_t align);
 215 void RPDEFVIS _ZdaPvSt11align_val_t(void *p, uint64_t align) {
 216   rpfree(p);
 217   (void)sizeof(align);
 218 }
 219 extern void _ZdlPvmSt11align_val_t(void *p, uint64_t size, uint64_t align);
 220 void RPDEFVIS _ZdlPvmSt11align_val_t(void *p, uint64_t size, uint64_t align) {
 221   rpfree(p);
 222   (void)sizeof(size);
 223   (void)sizeof(align);
 224 }
 225 extern void _ZdaPvmSt11align_val_t(void *p, uint64_t size, uint64_t align);
 226 void RPDEFVIS _ZdaPvmSt11align_val_t(void *p, uint64_t size, uint64_t align) {
 227   rpfree(p);
 228   (void)sizeof(size);
 229   (void)sizeof(align);
 230 }
 231 #else
 232 // 32-bit operators new and new[], normal and aligned
 233 extern void *_Znwj(uint32_t size);
 234 void *RPDEFVIS _Znwj(uint32_t size) { return rpmalloc(size); }
 235 extern void *_Znaj(uint32_t size);
 236 void *RPDEFVIS _Znaj(uint32_t size) { return rpmalloc(size); }
 237 extern void *_Znwjj(uint32_t size, uint32_t align);
 238 void *RPDEFVIS _Znwjj(uint32_t size, uint32_t align) {
 239   return rpaligned_alloc(align, size);
 240 }
 241 extern void *_Znajj(uint32_t size, uint32_t align);
 242 void *RPDEFVIS _Znajj(uint32_t size, uint32_t align) {
 243   return rpaligned_alloc(align, size);
 244 }
 245 extern void *_ZnwjSt11align_val_t(size_t size, size_t align);
 246 void *RPDEFVIS _ZnwjSt11align_val_t(size_t size, size_t align) {
 247   return rpaligned_alloc(align, size);
 248 }
 249 extern void *_ZnajSt11align_val_t(size_t size, size_t align);
 250 void *RPDEFVIS _ZnajSt11align_val_t(size_t size, size_t align) {
 251   return rpaligned_alloc(align, size);
 252 }
 253 extern void *_ZnwjRKSt9nothrow_t(size_t size, rp_nothrow_t t);
 254 void *RPDEFVIS _ZnwjRKSt9nothrow_t(size_t size, rp_nothrow_t t) {
 255   (void)sizeof(t);
 256   return rpmalloc(size);
 257 }
 258 extern void *_ZnajRKSt9nothrow_t(size_t size, rp_nothrow_t t);
 259 void *RPDEFVIS _ZnajRKSt9nothrow_t(size_t size, rp_nothrow_t t) {
 260   (void)sizeof(t);
 261   return rpmalloc(size);
 262 }
 263 extern void *_ZnwjSt11align_val_tRKSt9nothrow_t(size_t size, size_t align,
 264                                                 rp_nothrow_t t);
 265 void *RPDEFVIS _ZnwjSt11align_val_tRKSt9nothrow_t(size_t size, size_t align,
 266                                                   rp_nothrow_t t) {
 267   (void)sizeof(t);
 268   return rpaligned_alloc(align, size);
 269 }
 270 extern void *_ZnajSt11align_val_tRKSt9nothrow_t(size_t size, size_t align,
 271                                                 rp_nothrow_t t);
 272 void *RPDEFVIS _ZnajSt11align_val_tRKSt9nothrow_t(size_t size, size_t align,
 273                                                   rp_nothrow_t t) {
 274   (void)sizeof(t);
 275   return rpaligned_alloc(align, size);
 276 }
 277 // 32-bit operators sized delete and delete[], normal and aligned
 278 extern void _ZdlPvj(void *p, uint64_t size);
 279 void RPDEFVIS _ZdlPvj(void *p, uint64_t size) {
 280   rpfree(p);
 281   (void)sizeof(size);
 282 }
 283 extern void _ZdaPvj(void *p, uint64_t size);
 284 void RPDEFVIS _ZdaPvj(void *p, uint64_t size) {
 285   rpfree(p);
 286   (void)sizeof(size);
 287 }
 288 extern void _ZdlPvSt11align_val_t(void *p, uint32_t align);
 289 void RPDEFVIS _ZdlPvSt11align_val_t(void *p, uint64_t a) {
 290   rpfree(p);
 291   (void)sizeof(align);
 292 }
 293 extern void _ZdaPvSt11align_val_t(void *p, uint32_t align);
 294 void RPDEFVIS _ZdaPvSt11align_val_t(void *p, uint64_t a) {
 295   rpfree(p);
 296   (void)sizeof(align);
 297 }
 298 extern void _ZdlPvjSt11align_val_t(void *p, uint32_t size, uint32_t align);
 299 void RPDEFVIS _ZdlPvjSt11align_val_t(void *p, uint64_t size, uint64_t align) {
 300   rpfree(p);
 301   (void)sizeof(size);
 302   (void)sizeof(a);
 303 }
 304 extern void _ZdaPvjSt11align_val_t(void *p, uint32_t size, uint32_t align);
 305 void RPDEFVIS _ZdaPvjSt11align_val_t(void *p, uint64_t size, uint64_t align) {
 306   rpfree(p);
 307   (void)sizeof(size);
 308   (void)sizeof(a);
 309 }
 310 #endif
 311 #endif
 312 #endif
 313
 314 #if USE_INTERPOSE || USE_ALIAS
 315
 316 static void *rpmalloc_nothrow(size_t size, rp_nothrow_t t) {
 317   (void)sizeof(t);
 318   return rpmalloc(size);
 319 }
 320 static void *rpaligned_alloc_reverse(size_t size, size_t align) {
 321   return rpaligned_alloc(align, size);
 322 }
 323 static void *rpaligned_alloc_reverse_nothrow(size_t size, size_t align,
 324                                              rp_nothrow_t t) {
 325   (void)sizeof(t);
 326   return rpaligned_alloc(align, size);
 327 }
 328 static void rpfree_size(void *p, size_t size) {
 329   (void)sizeof(size);
 330   rpfree(p);
 331 }
 332 static void rpfree_aligned(void *p, size_t align) {
 333   (void)sizeof(align);
 334   rpfree(p);
 335 }
 336 static void rpfree_size_aligned(void *p, size_t size, size_t align) {
 337   (void)sizeof(size);
 338   (void)sizeof(align);
 339   rpfree(p);
 340 }
 341
 342 #endif
 343
 344 #if USE_INTERPOSE
 345
 346 __attribute__((used)) static const interpose_t macinterpose_malloc[]
 347     __attribute__((section("__DATA, __interpose"))) = {
 348         // new and new[]
 349         MAC_INTERPOSE_PAIR(rpmalloc, _Znwm),
 350         MAC_INTERPOSE_PAIR(rpmalloc, _Znam),
 351         MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _Znwmm),
 352         MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _Znamm),
 353         MAC_INTERPOSE_PAIR(rpmalloc_nothrow, _ZnwmRKSt9nothrow_t),
 354         MAC_INTERPOSE_PAIR(rpmalloc_nothrow, _ZnamRKSt9nothrow_t),
 355         MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _ZnwmSt11align_val_t),
 356         MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _ZnamSt11align_val_t),
 357         MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse_nothrow,
 358                            _ZnwmSt11align_val_tRKSt9nothrow_t),
 359         MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse_nothrow,
 360                            _ZnamSt11align_val_tRKSt9nothrow_t),
 361         // delete and delete[]
 362         MAC_INTERPOSE_PAIR(rpfree, _ZdlPv), MAC_INTERPOSE_PAIR(rpfree, _ZdaPv),
 363         MAC_INTERPOSE_PAIR(rpfree_size, _ZdlPvm),
 364         MAC_INTERPOSE_PAIR(rpfree_size, _ZdaPvm),
 365         MAC_INTERPOSE_PAIR(rpfree_aligned, _ZdlPvSt11align_val_t),
 366         MAC_INTERPOSE_PAIR(rpfree_aligned, _ZdaPvSt11align_val_t),
 367         MAC_INTERPOSE_PAIR(rpfree_size_aligned, _ZdlPvmSt11align_val_t),
 368         MAC_INTERPOSE_PAIR(rpfree_size_aligned, _ZdaPvmSt11align_val_t),
 369         // libc entry points
 370         MAC_INTERPOSE_PAIR(rpmalloc, malloc),
 371         MAC_INTERPOSE_PAIR(rpmalloc, calloc),
 372         MAC_INTERPOSE_PAIR(rprealloc, realloc),
 373         MAC_INTERPOSE_PAIR(rprealloc, reallocf),
 374 #if defined(__MAC_10_15) && __MAC_OS_X_VERSION_MIN_REQUIRED >= __MAC_10_15
 375         MAC_INTERPOSE_PAIR(rpaligned_alloc, aligned_alloc),
 376 #endif
 377         MAC_INTERPOSE_PAIR(rpmemalign, memalign),
 378         MAC_INTERPOSE_PAIR(rpposix_memalign, posix_memalign),
 379         MAC_INTERPOSE_PAIR(rpfree, free), MAC_INTERPOSE_PAIR(rpfree, cfree),
 380         MAC_INTERPOSE_PAIR(rpmalloc_usable_size, malloc_usable_size),
 381         MAC_INTERPOSE_PAIR(rpmalloc_usable_size, malloc_size)};
 382
 383 #endif
 384
 385 #if USE_ALIAS
 386
 387 #define RPALIAS(fn) __attribute__((alias(#fn), used, visibility("default")));
 388
 389 // Alias the C++ operators using the mangled names
 390 // (https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling)
 391
 392 // operators delete and delete[]
 393 void _ZdlPv(void *p) RPALIAS(rpfree) void _ZdaPv(void *p) RPALIAS(rpfree)
 394
 395 #if ARCH_64BIT
 396     // 64-bit operators new and new[], normal and aligned
 397     void *_Znwm(uint64_t size) RPMALLOC_ATTRIB_MALLOC
 398     RPMALLOC_ATTRIB_ALLOC_SIZE(1)
 399         RPALIAS(rpmalloc) void *_Znam(uint64_t size) RPMALLOC_ATTRIB_MALLOC
 400     RPMALLOC_ATTRIB_ALLOC_SIZE(1) RPALIAS(rpmalloc) void *_Znwmm(uint64_t size,
 401                                                                  uint64_t align)
 402         RPALIAS(rpaligned_alloc_reverse) void *_Znamm(uint64_t size,
 403                                                       uint64_t align)
 404             RPALIAS(rpaligned_alloc_reverse) void *_ZnwmSt11align_val_t(
 405                 size_t size, size_t align)
 406                 RPALIAS(rpaligned_alloc_reverse) void *_ZnamSt11align_val_t(
 407                     size_t size, size_t align)
 408                     RPALIAS(rpaligned_alloc_reverse) void *_ZnwmRKSt9nothrow_t(
 409                         size_t size, rp_nothrow_t t)
 410                         RPALIAS(rpmalloc_nothrow) void *_ZnamRKSt9nothrow_t(
 411                             size_t size,
 412                             rp_nothrow_t t) RPALIAS(rpmalloc_nothrow) void
 413                             *_ZnwmSt11align_val_tRKSt9nothrow_t(size_t size,
 414                                                                 size_t align,
 415                                                                 rp_nothrow_t t)
 416                                 RPALIAS(rpaligned_alloc_reverse_nothrow) void
 417                                     *_ZnamSt11align_val_tRKSt9nothrow_t(
 418                                         size_t size, size_t align,
 419                                         rp_nothrow_t t)
 420                                         RPALIAS(rpaligned_alloc_reverse_nothrow)
 421     // 64-bit operators delete and delete[], sized and aligned
 422     void _ZdlPvm(void *p, size_t n) RPALIAS(rpfree_size) void _ZdaPvm(void *p,
 423                                                                       size_t n)
 424         RPALIAS(rpfree_size) void _ZdlPvSt11align_val_t(void *p, size_t a)
 425             RPALIAS(rpfree_aligned) void _ZdaPvSt11align_val_t(void *p,
 426                                                                size_t a)
 427                 RPALIAS(rpfree_aligned) void _ZdlPvmSt11align_val_t(void *p,
 428                                                                     size_t n,
 429                                                                     size_t a)
 430                     RPALIAS(rpfree_size_aligned) void _ZdaPvmSt11align_val_t(
 431                         void *p, size_t n, size_t a)
 432                         RPALIAS(rpfree_size_aligned)
 433 #else
 434     // 32-bit operators new and new[], normal and aligned
 435     void *_Znwj(uint32_t size) RPMALLOC_ATTRIB_MALLOC
 436     RPMALLOC_ATTRIB_ALLOC_SIZE(1)
 437         RPALIAS(rpmalloc) void *_Znaj(uint32_t size) RPMALLOC_ATTRIB_MALLOC
 438     RPMALLOC_ATTRIB_ALLOC_SIZE(1) RPALIAS(rpmalloc) void *_Znwjj(uint32_t size,
 439                                                                  uint32_t align)
 440         RPALIAS(rpaligned_alloc_reverse) void *_Znajj(uint32_t size,
 441                                                       uint32_t align)
 442             RPALIAS(rpaligned_alloc_reverse) void *_ZnwjSt11align_val_t(
 443                 size_t size, size_t align)
 444                 RPALIAS(rpaligned_alloc_reverse) void *_ZnajSt11align_val_t(
 445                     size_t size, size_t align)
 446                     RPALIAS(rpaligned_alloc_reverse) void *_ZnwjRKSt9nothrow_t(
 447                         size_t size, rp_nothrow_t t)
 448                         RPALIAS(rpmalloc_nothrow) void *_ZnajRKSt9nothrow_t(
 449                             size_t size,
 450                             rp_nothrow_t t) RPALIAS(rpmalloc_nothrow) void
 451                             *_ZnwjSt11align_val_tRKSt9nothrow_t(size_t size,
 452                                                                 size_t align,
 453                                                                 rp_nothrow_t t)
 454                                 RPALIAS(rpaligned_alloc_reverse_nothrow) void
 455                                     *_ZnajSt11align_val_tRKSt9nothrow_t(
 456                                         size_t size, size_t align,
 457                                         rp_nothrow_t t)
 458                                         RPALIAS(rpaligned_alloc_reverse_nothrow)
 459     // 32-bit operators delete and delete[], sized and aligned
 460     void _ZdlPvj(void *p, size_t n) RPALIAS(rpfree_size) void _ZdaPvj(void *p,
 461                                                                       size_t n)
 462         RPALIAS(rpfree_size) void _ZdlPvSt11align_val_t(void *p, size_t a)
 463             RPALIAS(rpfree_aligned) void _ZdaPvSt11align_val_t(void *p,
 464                                                                size_t a)
 465                 RPALIAS(rpfree_aligned) void _ZdlPvjSt11align_val_t(void *p,
 466                                                                     size_t n,
 467                                                                     size_t a)
 468                     RPALIAS(rpfree_size_aligned) void _ZdaPvjSt11align_val_t(
 469                         void *p, size_t n, size_t a)
 470                         RPALIAS(rpfree_size_aligned)
 471 #endif
 472
 473                             void *malloc(size_t size) RPMALLOC_ATTRIB_MALLOC
 474     RPMALLOC_ATTRIB_ALLOC_SIZE(1)
 475         RPALIAS(rpmalloc) void *calloc(size_t count, size_t size)
 476             RPALIAS(rpcalloc) void *realloc(void *ptr, size_t size)
 477                 RPALIAS(rprealloc) void *reallocf(void *ptr, size_t size)
 478                     RPMALLOC_ATTRIB_MALLOC
 479     RPMALLOC_ATTRIB_ALLOC_SIZE(2)
 480         RPALIAS(rprealloc) void *aligned_alloc(size_t alignment, size_t size)
 481             RPALIAS(rpaligned_alloc) void *memalign(
 482                 size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC
 483     RPMALLOC_ATTRIB_ALLOC_SIZE(2)
 484         RPALIAS(rpmemalign) int posix_memalign(void **memptr, size_t alignment,
 485                                                size_t size)
 486             RPALIAS(rpposix_memalign) void free(void *ptr)
 487                 RPALIAS(rpfree) void cfree(void *ptr) RPALIAS(rpfree)
 488 #if defined(__ANDROID__) || defined(__FreeBSD__)
 489                     size_t
 490     malloc_usable_size(const void *ptr) RPALIAS(rpmalloc_usable_size)
 491 #else
 492                     size_t
 493     malloc_usable_size(void *ptr) RPALIAS(rpmalloc_usable_size)
 494 #endif
 495         size_t malloc_size(void *ptr) RPALIAS(rpmalloc_usable_size)
 496
 497 #endif
 498
 499             static inline size_t _rpmalloc_page_size(void) {
 500   return _memory_page_size;
 501 }
 502
 503 extern void *RPMALLOC_CDECL reallocarray(void *ptr, size_t count, size_t size);
 504
 505 extern void *RPMALLOC_CDECL reallocarray(void *ptr, size_t count, size_t size) {
 506   size_t total;
 507 #if ENABLE_VALIDATE_ARGS
 508 #ifdef _MSC_VER
 509   int err = SizeTMult(count, size, &total);
 510   if ((err != S_OK) || (total >= MAX_ALLOC_SIZE)) {
 511     errno = EINVAL;
 512     return 0;
 513   }
 514 #else
 515   int err = __builtin_umull_overflow(count, size, &total);
 516   if (err || (total >= MAX_ALLOC_SIZE)) {
 517     errno = EINVAL;
 518     return 0;
 519   }
 520 #endif
 521 #else
 522   total = count * size;
 523 #endif
 524   return realloc(ptr, total);
 525 }
 526
 527 extern inline void *RPMALLOC_CDECL valloc(size_t size) {
 528   get_thread_heap();
 529   return rpaligned_alloc(_rpmalloc_page_size(), size);
 530 }
 531
 532 extern inline void *RPMALLOC_CDECL pvalloc(size_t size) {
 533   get_thread_heap();
 534   const size_t page_size = _rpmalloc_page_size();
 535   const size_t aligned_size = ((size + page_size - 1) / page_size) * page_size;
 536 #if ENABLE_VALIDATE_ARGS
 537   if (aligned_size < size) {
 538     errno = EINVAL;
 539     return 0;
 540   }
 541 #endif
 542   return rpaligned_alloc(_rpmalloc_page_size(), aligned_size);
 543 }
 544
 545 #endif // ENABLE_OVERRIDE
 546
 547 #if ENABLE_PRELOAD
 548
 549 #ifdef _WIN32
 550
 551 #if defined(BUILD_DYNAMIC_LINK) && BUILD_DYNAMIC_LINK
 552
 553 extern __declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE instance,
 554                                                  DWORD reason, LPVOID reserved);
 555
 556 extern __declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE instance,
 557                                                  DWORD reason,
 558                                                  LPVOID reserved) {
 559   (void)sizeof(reserved);
 560   (void)sizeof(instance);
 561   if (reason == DLL_PROCESS_ATTACH)
 562     rpmalloc_initialize();
 563   else if (reason == DLL_PROCESS_DETACH)
 564     rpmalloc_finalize();
 565   else if (reason == DLL_THREAD_ATTACH)
 566     rpmalloc_thread_initialize();
 567   else if (reason == DLL_THREAD_DETACH)
 568     rpmalloc_thread_finalize(1);
 569   return TRUE;
 570 }
 571
 572 // end BUILD_DYNAMIC_LINK
 573 #else
 574
 575 extern void _global_rpmalloc_init(void) {
 576   rpmalloc_set_main_thread();
 577   rpmalloc_initialize();
 578 }
 579
 580 #if defined(__clang__) || defined(__GNUC__)
 581
 582 static void __attribute__((constructor)) initializer(void) {
 583   _global_rpmalloc_init();
 584 }
 585
 586 #elif defined(_MSC_VER)
 587
 588 static int _global_rpmalloc_xib(void) {
 589   _global_rpmalloc_init();
 590   return 0;
 591 }
 592
 593 #pragma section(".CRT$XIB", read)
 594 __declspec(allocate(".CRT$XIB")) void (*_rpmalloc_module_init)(void) =
 595     _global_rpmalloc_xib;
 596 #if defined(_M_IX86) || defined(__i386__)
 597 #pragma comment(linker, "/include:"                                            \
 598                         "__rpmalloc_module_init")
 599 #else
 600 #pragma comment(linker, "/include:"                                            \
 601                         "_rpmalloc_module_init")
 602 #endif
 603
 604 #endif
 605
 606 // end !BUILD_DYNAMIC_LINK
 607 #endif
 608
 609 #else
 610
 611 #include <pthread.h>
 612 #include <stdint.h>
 613 #include <stdlib.h>
 614 #include <unistd.h>
 615
 616 extern void rpmalloc_set_main_thread(void);
 617
 618 static pthread_key_t destructor_key;
 619
 620 static void thread_destructor(void *);
 621
 622 static void __attribute__((constructor)) initializer(void) {
 623   rpmalloc_set_main_thread();
 624   rpmalloc_initialize();
 625   pthread_key_create(&destructor_key, thread_destructor);
 626 }
 627
 628 static void __attribute__((destructor)) finalizer(void) { rpmalloc_finalize(); }
 629
 630 typedef struct {
 631   void *(*real_start)(void *);
 632   void *real_arg;
 633 } thread_starter_arg;
 634
 635 static void *thread_starter(void *argptr) {
 636   thread_starter_arg *arg = argptr;
 637   void *(*real_start)(void *) = arg->real_start;
 638   void *real_arg = arg->real_arg;
 639   rpmalloc_thread_initialize();
 640   rpfree(argptr);
 641   pthread_setspecific(destructor_key, (void *)1);
 642   return (*real_start)(real_arg);
 643 }
 644
 645 static void thread_destructor(void *value) {
 646   (void)sizeof(value);
 647   rpmalloc_thread_finalize(1);
 648 }
 649
 650 #ifdef __APPLE__
 651
 652 static int pthread_create_proxy(pthread_t *thread, const pthread_attr_t *attr,
 653                                 void *(*start_routine)(void *), void *arg) {
 654   rpmalloc_initialize();
 655   thread_starter_arg *starter_arg = rpmalloc(sizeof(thread_starter_arg));
 656   starter_arg->real_start = start_routine;
 657   starter_arg->real_arg = arg;
 658   return pthread_create(thread, attr, thread_starter, starter_arg);
 659 }
 660
 661 MAC_INTERPOSE_SINGLE(pthread_create_proxy, pthread_create);
 662
 663 #else
 664
 665 #include <dlfcn.h>
 666
 667 int pthread_create(pthread_t *thread, const pthread_attr_t *attr,
 668                    void *(*start_routine)(void *), void *arg) {
 669 #if defined(__linux__) || defined(__FreeBSD__) || defined(__OpenBSD__) ||      \
 670     defined(__NetBSD__) || defined(__DragonFly__) || defined(__APPLE__) ||     \
 671     defined(__HAIKU__)
 672   char fname[] = "pthread_create";
 673 #else
 674   char fname[] = "_pthread_create";
 675 #endif
 676   void *real_pthread_create = dlsym(RTLD_NEXT, fname);
 677   rpmalloc_thread_initialize();
 678   thread_starter_arg *starter_arg = rpmalloc(sizeof(thread_starter_arg));
 679   starter_arg->real_start = start_routine;
 680   starter_arg->real_arg = arg;
 681   return (*(int (*)(pthread_t *, const pthread_attr_t *, void *(*)(void *),
 682                     void *))real_pthread_create)(thread, attr, thread_starter,
 683                                                  starter_arg);
 684 }
 685
 686 #endif
 687
 688 #endif
 689
 690 #endif
 691
 692 #if ENABLE_OVERRIDE
 693
 694 #if defined(__GLIBC__) && defined(__linux__)
 695
 696 void *__libc_malloc(size_t size) RPMALLOC_ATTRIB_MALLOC
 697     RPMALLOC_ATTRIB_ALLOC_SIZE(1)
 698         RPALIAS(rpmalloc) void *__libc_calloc(size_t count, size_t size)
 699             RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE2(1, 2)
 700                 RPALIAS(rpcalloc) void *__libc_realloc(void *p, size_t size)
 701                     RPMALLOC_ATTRIB_MALLOC
 702     RPMALLOC_ATTRIB_ALLOC_SIZE(2) RPALIAS(rprealloc) void __libc_free(void *p)
 703         RPALIAS(rpfree) void __libc_cfree(void *p)
 704             RPALIAS(rpfree) void *__libc_memalign(size_t align, size_t size)
 705                 RPMALLOC_ATTRIB_MALLOC
 706     RPMALLOC_ATTRIB_ALLOC_SIZE(2)
 707         RPALIAS(rpmemalign) int __posix_memalign(void **p, size_t align,
 708                                                  size_t size)
 709             RPALIAS(rpposix_memalign)
 710
 711                 extern void *__libc_valloc(size_t size);
 712 extern void *__libc_pvalloc(size_t size);
 713
 714 void *__libc_valloc(size_t size) { return valloc(size); }
 715
 716 void *__libc_pvalloc(size_t size) { return pvalloc(size); }
 717
 718 #endif
 719
 720 #endif
 721
 722 #if (defined(__GNUC__) || defined(__clang__))
 723 #pragma GCC visibility pop
 724 #endif