lib/libc/stdlib/malloc.c

   1 /*      $NetBSD: malloc.c,v 1.52 2008/02/03 22:56:53 christos Exp $     */
   2
   3 /*
   4  * ----------------------------------------------------------------------------
   5  * "THE BEER-WARE LICENSE" (Revision 42):
   6  * <phk@FreeBSD.ORG> wrote this file.  As long as you retain this notice you
   7  * can do whatever you want with this stuff. If we meet some day, and you think
   8  * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
   9  * ----------------------------------------------------------------------------
  10  *
  11  * From FreeBSD: malloc.c,v 1.91 2006/01/12 07:28:20 jasone
  12  *
  13  */
  14
  15 #ifdef __minix
  16 #define mmap minix_mmap
  17 #define munmap minix_munmap
  18 #ifdef _LIBSYS
  19 #include <minix/sysutil.h>
  20 #define MALLOC_NO_SYSCALLS
  21 #define wrtwarning(w) printf("libminc malloc warning: %s\n", w)
  22 #define wrterror(w) panic("libminc malloc error: %s\n", w)
  23 #endif
  24 #endif
  25
  26 /*
  27  * Defining MALLOC_EXTRA_SANITY will enable extra checks which are related
  28  * to internal conditions and consistency in malloc.c. This has a
  29  * noticeable runtime performance hit, and generally will not do you
  30  * any good unless you fiddle with the internals of malloc or want
  31  * to catch random pointer corruption as early as possible.
  32  */
  33 #ifndef MALLOC_EXTRA_SANITY
  34 #undef MALLOC_EXTRA_SANITY
  35 #endif
  36
  37 /*
  38  * What to use for Junk.  This is the byte value we use to fill with
  39  * when the 'J' option is enabled.
  40  */
  41 #define SOME_JUNK       0xd0            /* as in "Duh" :-) */
  42
  43 /*
  44  * The basic parameters you can tweak.
  45  *
  46  * malloc_minsize       minimum size of an allocation in bytes.
  47  *                      If this is too small it's too much work
  48  *                      to manage them.  This is also the smallest
  49  *                      unit of alignment used for the storage
  50  *                      returned by malloc/realloc.
  51  *
  52  */
  53
  54 #include "namespace.h"
  55 #if defined(__FreeBSD__)
  56 #   if defined(__i386__)
  57 #       define malloc_minsize           16U
  58 #   endif
  59 #   if defined(__ia64__)
  60 #       define malloc_pageshift         13U
  61 #       define malloc_minsize           16U
  62 #   endif
  63 #   if defined(__alpha__)
  64 #       define malloc_pageshift         13U
  65 #       define malloc_minsize           16U
  66 #   endif
  67 #   if defined(__sparc64__)
  68 #       define malloc_pageshift         13U
  69 #       define malloc_minsize           16U
  70 #   endif
  71 #   if defined(__amd64__)
  72 #       define malloc_pageshift         12U
  73 #       define malloc_minsize           16U
  74 #   endif
  75 #   if defined(__arm__)
  76 #       define malloc_pageshift         12U
  77 #       define malloc_minsize           16U
  78 #   endif
  79 #ifndef __minix
  80 #   define HAS_UTRACE
  81 #   define UTRACE_LABEL
  82 #endif /* __minix */
  83
  84 #include <sys/cdefs.h>
  85 void utrace(struct ut *, int);
  86
  87     /*
  88      * Make malloc/free/realloc thread-safe in libc for use with
  89      * kernel threads.
  90      */
  91 #   include "libc_private.h"
  92 #   include "spinlock.h"
  93     static spinlock_t thread_lock       = _SPINLOCK_INITIALIZER;
  94 #   define _MALLOC_LOCK()               if (__isthreaded) _SPINLOCK(&thread_lock);
  95 #   define _MALLOC_UNLOCK()             if (__isthreaded) _SPINUNLOCK(&thread_lock);
  96 #endif /* __FreeBSD__ */
  97
  98 #include <assert.h>
  99
 100 #include <sys/types.h>
 101 #if defined(__NetBSD__)
 102 #   define malloc_minsize               16U
 103 #   define HAS_UTRACE
 104 #   define UTRACE_LABEL "malloc",
 105 #include <sys/cdefs.h>
 106 #include "extern.h"
 107 #if defined(LIBC_SCCS) && !defined(lint)
 108 __RCSID("$NetBSD: malloc.c,v 1.52 2008/02/03 22:56:53 christos Exp $");
 109 #endif /* LIBC_SCCS and not lint */
 110 int utrace(const char *, void *, size_t);
 111
 112 #include <reentrant.h>
 113 extern int __isthreaded;
 114 static mutex_t thread_lock = MUTEX_INITIALIZER;
 115 #define _MALLOC_LOCK()  if (__isthreaded) mutex_lock(&thread_lock);
 116 #define _MALLOC_UNLOCK()        if (__isthreaded) mutex_unlock(&thread_lock);
 117 #endif /* __NetBSD__ */
 118
 119 #if defined(__sparc__) && defined(sun)
 120 #   define malloc_minsize               16U
 121 #   define MAP_ANON                     (0)
 122     static int fdzero;
 123 #   define MMAP_FD      fdzero
 124 #   define INIT_MMAP() \
 125         { if ((fdzero = open(_PATH_DEVZERO, O_RDWR, 0000)) == -1) \
 126             wrterror("open of /dev/zero"); }
 127 #endif /* __sparc__ */
 128
 129 /* Insert your combination here... */
 130 #if defined(__FOOCPU__) && defined(__BAROS__)
 131 #   define malloc_minsize               16U
 132 #endif /* __FOOCPU__ && __BAROS__ */
 133
 134 #ifndef ZEROSIZEPTR
 135 #define ZEROSIZEPTR     ((void *)(uintptr_t)(1UL << (malloc_pageshift - 1)))
 136 #endif
 137
 138 /*
 139  * No user serviceable parts behind this point.
 140  */
 141 #include <sys/types.h>
 142 #include <sys/mman.h>
 143 #include <errno.h>
 144 #include <fcntl.h>
 145 #include <paths.h>
 146 #include <stddef.h>
 147 #include <stdio.h>
 148 #include <stdlib.h>
 149 #include <string.h>
 150 #include <unistd.h>
 151
 152 /*
 153  * This structure describes a page worth of chunks.
 154  */
 155
 156 struct pginfo {
 157     struct pginfo       *next;  /* next on the free list */
 158     void                *page;  /* Pointer to the page */
 159     u_short             size;   /* size of this page's chunks */
 160     u_short             shift;  /* How far to shift for this size chunks */
 161     u_short             free;   /* How many free chunks */
 162     u_short             total;  /* How many chunk */
 163     u_int               bits[1]; /* Which chunks are free */
 164 };
 165
 166 /*
 167  * This structure describes a number of free pages.
 168  */
 169
 170 struct pgfree {
 171     struct pgfree       *next;  /* next run of free pages */
 172     struct pgfree       *prev;  /* prev run of free pages */
 173     void                *page;  /* pointer to free pages */
 174     void                *end;   /* pointer to end of free pages */
 175     size_t              size;   /* number of bytes free */
 176 };
 177
 178 /*
 179  * How many bits per u_int in the bitmap.
 180  * Change only if not 8 bits/byte
 181  */
 182 #define MALLOC_BITS     ((int)(8*sizeof(u_int)))
 183
 184 /*
 185  * Magic values to put in the page_directory
 186  */
 187 #define MALLOC_NOT_MINE ((struct pginfo*) 0)
 188 #define MALLOC_FREE     ((struct pginfo*) 1)
 189 #define MALLOC_FIRST    ((struct pginfo*) 2)
 190 #define MALLOC_FOLLOW   ((struct pginfo*) 3)
 191 #define MALLOC_MAGIC    ((struct pginfo*) 4)
 192
 193 /*
 194  * Page size related parameters, computed at run-time.
 195  */
 196 static size_t malloc_pagesize;
 197 static size_t malloc_pageshift;
 198 static size_t malloc_pagemask;
 199
 200 #ifndef malloc_minsize
 201 #define malloc_minsize                  16U
 202 #endif
 203
 204 #ifndef malloc_maxsize
 205 #define malloc_maxsize                  ((malloc_pagesize)>>1)
 206 #endif
 207
 208 #define pageround(foo) (((foo) + (malloc_pagemask))&(~(malloc_pagemask)))
 209 #define ptr2idx(foo) \
 210     (((size_t)(uintptr_t)(foo) >> malloc_pageshift)-malloc_origo)
 211
 212 #ifndef _MALLOC_LOCK
 213 #define _MALLOC_LOCK()
 214 #endif
 215
 216 #ifndef _MALLOC_UNLOCK
 217 #define _MALLOC_UNLOCK()
 218 #endif
 219
 220 #ifndef MMAP_FD
 221 #define MMAP_FD (-1)
 222 #endif
 223
 224 #ifndef INIT_MMAP
 225 #define INIT_MMAP()
 226 #endif
 227
 228 #ifndef __minix
 229 #ifndef MADV_FREE
 230 #define MADV_FREE MADV_DONTNEED
 231 #endif
 232 #endif /* !__minix */
 233
 234 /* Number of free pages we cache */
 235 static size_t malloc_cache = 16;
 236
 237 /* The offset from pagenumber to index into the page directory */
 238 static size_t malloc_origo;
 239
 240 /* The last index in the page directory we care about */
 241 static size_t last_idx;
 242
 243 /* Pointer to page directory. Allocated "as if with" malloc */
 244 static struct   pginfo **page_dir;
 245
 246 /* How many slots in the page directory */
 247 static size_t   malloc_ninfo;
 248
 249 /* Free pages line up here */
 250 static struct pgfree free_list;
 251
 252 /* Abort(), user doesn't handle problems.  */
 253 static int malloc_abort;
 254
 255 /* Are we trying to die ?  */
 256 static int suicide;
 257
 258 /* always realloc ?  */
 259 static int malloc_realloc;
 260
 261 /* pass the kernel a hint on free pages ?  */
 262 #if defined(MADV_FREE)
 263 static int malloc_hint = 0;
 264 #endif
 265
 266 /* xmalloc behaviour ?  */
 267 static int malloc_xmalloc;
 268
 269 /* sysv behaviour for malloc(0) ?  */
 270 static int malloc_sysv;
 271
 272 /* zero fill ?  */
 273 static int malloc_zero;
 274
 275 /* junk fill ?  */
 276 static int malloc_junk;
 277
 278 #ifdef HAS_UTRACE
 279
 280 /* utrace ?  */
 281 static int malloc_utrace;
 282
 283 struct ut { void *p; size_t s; void *r; };
 284
 285 #define UTRACE(a, b, c) \
 286         if (malloc_utrace) {                    \
 287                 struct ut u;                    \
 288                 u.p=a; u.s = b; u.r=c;          \
 289                 utrace(UTRACE_LABEL (void *) &u, sizeof u);     \
 290         }
 291 #else /* !HAS_UTRACE */
 292 #define UTRACE(a,b,c)
 293 #endif /* HAS_UTRACE */
 294
 295 /* my last break. */
 296 static void *malloc_brk;
 297
 298 /* one location cache for free-list holders */
 299 static struct pgfree *px;
 300
 301 /* compile-time options */
 302 const char *_malloc_options;
 303
 304 /* Name of the current public function */
 305 static const char *malloc_func;
 306
 307 /* Macro for mmap */
 308 #define MMAP(size) \
 309         mmap(NULL, (size), PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, \
 310             MMAP_FD, (off_t)0);
 311
 312 /*
 313  * Necessary function declarations
 314  */
 315 static int extend_pgdir(size_t idx);
 316 static void *imalloc(size_t size);
 317 static void ifree(void *ptr);
 318 static void *irealloc(void *ptr, size_t size);
 319
 320 #ifndef MALLOC_NO_SYSCALLS
 321 static void
 322 wrtmessage(const char *p1, const char *p2, const char *p3, const char *p4)
 323 {
 324
 325     write(STDERR_FILENO, p1, strlen(p1));
 326     write(STDERR_FILENO, p2, strlen(p2));
 327     write(STDERR_FILENO, p3, strlen(p3));
 328     write(STDERR_FILENO, p4, strlen(p4));
 329 }
 330
 331 void (*_malloc_message)(const char *p1, const char *p2, const char *p3,
 332             const char *p4) = wrtmessage;
 333 static void
 334 wrterror(const char *p)
 335 {
 336
 337     suicide = 1;
 338     _malloc_message(getprogname(), malloc_func, " error: ", p);
 339     abort();
 340 }
 341
 342 static void
 343 wrtwarning(const char *p)
 344 {
 345
 346     /*
 347      * Sensitive processes, somewhat arbitrarily defined here as setuid,
 348      * setgid, root and wheel cannot afford to have malloc mistakes.
 349      */
 350     if (malloc_abort || issetugid() || getuid() == 0 || getgid() == 0)
 351         wrterror(p);
 352 }
 353 #endif
 354
 355 /*
 356  * Allocate a number of pages from the OS
 357  */
 358 static void *
 359 map_pages(size_t pages)
 360 {
 361     caddr_t result, rresult, tail;
 362     intptr_t bytes = pages << malloc_pageshift;
 363
 364     if (bytes < 0 || (size_t)bytes < pages) {
 365         errno = ENOMEM;
 366         return NULL;
 367     }
 368
 369     if ((result = sbrk(bytes)) == (void *)-1)
 370         return NULL;
 371
 372     /*
 373      * Round to a page, in case sbrk(2) did not do this for us
 374      */
 375     rresult = (caddr_t)pageround((size_t)(uintptr_t)result);
 376     if (result < rresult) {
 377         /* make sure we have enough space to fit bytes */
 378         if (sbrk((intptr_t)(rresult - result)) == (void *) -1) {
 379             /* we failed, put everything back */
 380             if (brk(result)) {
 381                 wrterror("brk(2) failed [internal error]\n");
 382             }
 383         }
 384     }
 385     tail = rresult + (size_t)bytes;
 386
 387     last_idx = ptr2idx(tail) - 1;
 388     malloc_brk = tail;
 389
 390     if ((last_idx+1) >= malloc_ninfo && !extend_pgdir(last_idx)) {
 391         malloc_brk = result;
 392         last_idx = ptr2idx(malloc_brk) - 1;
 393         /* Put back break point since we failed. */
 394         if (brk(malloc_brk))
 395             wrterror("brk(2) failed [internal error]\n");
 396         return 0;
 397     }
 398
 399     return rresult;
 400 }
 401
 402 /*
 403  * Extend page directory
 404  */
 405 static int
 406 extend_pgdir(size_t idx)
 407 {
 408     struct  pginfo **new, **old;
 409     size_t newlen, oldlen;
 410
 411     /* check for overflow */
 412     if ((((~(1UL << ((sizeof(size_t) * NBBY) - 1)) / sizeof(*page_dir)) + 1)
 413         + (malloc_pagesize / sizeof *page_dir)) < idx) {
 414         errno = ENOMEM;
 415         return 0;
 416     }
 417
 418     /* Make it this many pages */
 419     newlen = pageround(idx * sizeof *page_dir) + malloc_pagesize;
 420
 421     /* remember the old mapping size */
 422     oldlen = malloc_ninfo * sizeof *page_dir;
 423
 424     /*
 425      * NOTE: we allocate new pages and copy the directory rather than tempt
 426      * fate by trying to "grow" the region.. There is nothing to prevent
 427      * us from accidentally re-mapping space that's been allocated by our caller
 428      * via dlopen() or other mmap().
 429      *
 430      * The copy problem is not too bad, as there is 4K of page index per
 431      * 4MB of malloc arena.
 432      *
 433      * We can totally avoid the copy if we open a file descriptor to associate
 434      * the anon mappings with.  Then, when we remap the pages at the new
 435      * address, the old pages will be "magically" remapped..  But this means
 436      * keeping open a "secret" file descriptor.....
 437      */
 438
 439     /* Get new pages */
 440     new = MMAP(newlen);
 441     if (new == MAP_FAILED)
 442         return 0;
 443
 444     /* Copy the old stuff */
 445     memcpy(new, page_dir, oldlen);
 446
 447     /* register the new size */
 448     malloc_ninfo = newlen / sizeof *page_dir;
 449
 450     /* swap the pointers */
 451     old = page_dir;
 452     page_dir = new;
 453
 454     /* Now free the old stuff */
 455     munmap(old, oldlen);
 456     return 1;
 457 }
 458
 459 /*
 460  * Initialize the world
 461  */
 462 static void
 463 malloc_init(void)
 464 {
 465     int save_errno = errno;
 466 #ifndef MALLOC_NO_SYSCALLS
 467     const char *p;
 468     char b[64];
 469     size_t i;
 470     ssize_t j;
 471
 472     /*
 473      * Compute page-size related variables.
 474      */
 475     malloc_pagesize = (size_t)sysconf(_SC_PAGESIZE);
 476 #else
 477     malloc_pagesize = PAGE_SIZE;
 478 #endif
 479     malloc_pagemask = malloc_pagesize - 1;
 480     for (malloc_pageshift = 0;
 481          (1UL << malloc_pageshift) != malloc_pagesize;
 482          malloc_pageshift++)
 483         /* nothing */ ;
 484
 485     INIT_MMAP();
 486
 487 #ifdef MALLOC_EXTRA_SANITY
 488     malloc_junk = 1;
 489 #endif /* MALLOC_EXTRA_SANITY */
 490
 491 #ifndef MALLOC_NO_SYSCALLS
 492     for (i = 0; i < 3; i++) {
 493         if (i == 0) {
 494             j = readlink("/etc/malloc.conf", b, sizeof b - 1);
 495             if (j <= 0)
 496                 continue;
 497             b[j] = '\0';
 498             p = b;
 499         } else if (i == 1 && issetugid() == 0) {
 500             p = getenv("MALLOC_OPTIONS");
 501         } else if (i == 1) {
 502             continue;
 503         } else {
 504             p = _malloc_options;
 505         }
 506         for (; p != NULL && *p != '\0'; p++) {
 507             switch (*p) {
 508                 case '>': malloc_cache   <<= 1; break;
 509                 case '<': malloc_cache   >>= 1; break;
 510                 case 'a': malloc_abort   = 0; break;
 511                 case 'A': malloc_abort   = 1; break;
 512 #ifndef __minix
 513                 case 'h': malloc_hint    = 0; break;
 514                 case 'H': malloc_hint    = 1; break;
 515 #endif /* !__minix */
 516                 case 'r': malloc_realloc = 0; break;
 517                 case 'R': malloc_realloc = 1; break;
 518                 case 'j': malloc_junk    = 0; break;
 519                 case 'J': malloc_junk    = 1; break;
 520 #ifdef HAS_UTRACE
 521                 case 'u': malloc_utrace  = 0; break;
 522                 case 'U': malloc_utrace  = 1; break;
 523 #endif
 524                 case 'v': malloc_sysv    = 0; break;
 525                 case 'V': malloc_sysv    = 1; break;
 526                 case 'x': malloc_xmalloc = 0; break;
 527                 case 'X': malloc_xmalloc = 1; break;
 528                 case 'z': malloc_zero    = 0; break;
 529                 case 'Z': malloc_zero    = 1; break;
 530                 default:
 531                     _malloc_message(getprogname(), malloc_func,
 532                          " warning: ", "unknown char in MALLOC_OPTIONS\n");
 533                     break;
 534             }
 535         }
 536     }
 537 #endif
 538
 539     UTRACE(0, 0, 0);
 540
 541     /*
 542      * We want junk in the entire allocation, and zero only in the part
 543      * the user asked for.
 544      */
 545     if (malloc_zero)
 546         malloc_junk = 1;
 547
 548     /* Allocate one page for the page directory */
 549     page_dir = MMAP(malloc_pagesize);
 550
 551     if (page_dir == MAP_FAILED)
 552         wrterror("mmap(2) failed, check limits.\n");
 553
 554     /*
 555      * We need a maximum of malloc_pageshift buckets, steal these from the
 556      * front of the page_directory;
 557      */
 558     malloc_origo = pageround((size_t)(uintptr_t)sbrk((intptr_t)0))
 559         >> malloc_pageshift;
 560     malloc_origo -= malloc_pageshift;
 561
 562     malloc_ninfo = malloc_pagesize / sizeof *page_dir;
 563
 564     /* Recalculate the cache size in bytes, and make sure it's nonzero */
 565
 566     if (!malloc_cache)
 567         malloc_cache++;
 568
 569     malloc_cache <<= malloc_pageshift;
 570
 571     /*
 572      * This is a nice hack from Kaleb Keithly (kaleb@x.org).
 573      * We can sbrk(2) further back when we keep this on a low address.
 574      */
 575     px = imalloc(sizeof *px);
 576
 577     errno = save_errno;
 578 }
 579
 580 /*
 581  * Allocate a number of complete pages
 582  */
 583 static void *
 584 malloc_pages(size_t size)
 585 {
 586     void *p, *delay_free = NULL;
 587     size_t i;
 588     struct pgfree *pf;
 589     size_t idx;
 590
 591     idx = pageround(size);
 592     if (idx < size) {
 593         errno = ENOMEM;
 594         return NULL;
 595     } else
 596         size = idx;
 597
 598     p = NULL;
 599
 600     /* Look for free pages before asking for more */
 601     for(pf = free_list.next; pf; pf = pf->next) {
 602
 603 #ifdef MALLOC_EXTRA_SANITY
 604         if (pf->size & malloc_pagemask)
 605             wrterror("(ES): junk length entry on free_list.\n");
 606         if (!pf->size)
 607             wrterror("(ES): zero length entry on free_list.\n");
 608         if (pf->page == pf->end)
 609             wrterror("(ES): zero entry on free_list.\n");
 610         if (pf->page > pf->end)
 611             wrterror("(ES): sick entry on free_list.\n");
 612         if ((void*)pf->page >= (void*)sbrk(0))
 613             wrterror("(ES): entry on free_list past brk.\n");
 614         if (page_dir[ptr2idx(pf->page)] != MALLOC_FREE)
 615             wrterror("(ES): non-free first page on free-list.\n");
 616         if (page_dir[ptr2idx(pf->end)-1] != MALLOC_FREE)
 617             wrterror("(ES): non-free last page on free-list.\n");
 618 #endif /* MALLOC_EXTRA_SANITY */
 619
 620         if (pf->size < size)
 621             continue;
 622
 623         if (pf->size == size) {
 624             p = pf->page;
 625             if (pf->next != NULL)
 626                     pf->next->prev = pf->prev;
 627             pf->prev->next = pf->next;
 628             delay_free = pf;
 629             break;
 630         }
 631
 632         p = pf->page;
 633         pf->page = (char *)pf->page + size;
 634         pf->size -= size;
 635         break;
 636     }
 637
 638 #ifdef MALLOC_EXTRA_SANITY
 639     if (p != NULL && page_dir[ptr2idx(p)] != MALLOC_FREE)
 640         wrterror("(ES): allocated non-free page on free-list.\n");
 641 #endif /* MALLOC_EXTRA_SANITY */
 642
 643     size >>= malloc_pageshift;
 644
 645     /* Map new pages */
 646     if (p == NULL)
 647         p = map_pages(size);
 648
 649     if (p != NULL) {
 650
 651         idx = ptr2idx(p);
 652         page_dir[idx] = MALLOC_FIRST;
 653         for (i=1;i<size;i++)
 654             page_dir[idx+i] = MALLOC_FOLLOW;
 655
 656         if (malloc_junk)
 657             memset(p, SOME_JUNK, size << malloc_pageshift);
 658     }
 659
 660     if (delay_free) {
 661         if (px == NULL)
 662             px = delay_free;
 663         else
 664             ifree(delay_free);
 665     }
 666
 667     return p;
 668 }
 669
 670 /*
 671  * Allocate a page of fragments
 672  */
 673
 674 static inline int
 675 malloc_make_chunks(int bits)
 676 {
 677     struct  pginfo *bp;
 678     void *pp;
 679     int i, k;
 680     long l;
 681
 682     /* Allocate a new bucket */
 683     pp = malloc_pages(malloc_pagesize);
 684     if (pp == NULL)
 685         return 0;
 686
 687     /* Find length of admin structure */
 688     l = (long)offsetof(struct pginfo, bits[0]);
 689     l += (long)sizeof bp->bits[0] *
 690         (((malloc_pagesize >> bits)+MALLOC_BITS-1) / MALLOC_BITS);
 691
 692     /* Don't waste more than two chunks on this */
 693     if ((1<<(bits)) <= l+l) {
 694         bp = (struct  pginfo *)pp;
 695     } else {
 696         bp = imalloc((size_t)l);
 697         if (bp == NULL) {
 698             ifree(pp);
 699             return 0;
 700         }
 701     }
 702
 703     bp->size = (1<<bits);
 704     bp->shift = bits;
 705     bp->total = bp->free = (u_short)(malloc_pagesize >> bits);
 706     bp->page = pp;
 707
 708     /* set all valid bits in the bitmap */
 709     k = bp->total;
 710     i = 0;
 711
 712     /* Do a bunch at a time */
 713     for(;k-i >= MALLOC_BITS; i += MALLOC_BITS)
 714         bp->bits[i / MALLOC_BITS] = ~0U;
 715
 716     for(; i < k; i++)
 717         bp->bits[i/MALLOC_BITS] |= 1<<(i%MALLOC_BITS);
 718
 719     if (bp == bp->page) {
 720         /* Mark the ones we stole for ourselves */
 721         for(i = 0; l > 0; i++) {
 722             bp->bits[i / MALLOC_BITS] &= ~(1 << (i % MALLOC_BITS));
 723             bp->free--;
 724             bp->total--;
 725             l -= (long)(1 << bits);
 726         }
 727     }
 728
 729     /* MALLOC_LOCK */
 730
 731     page_dir[ptr2idx(pp)] = bp;
 732
 733     bp->next = page_dir[bits];
 734     page_dir[bits] = bp;
 735
 736     /* MALLOC_UNLOCK */
 737
 738     return 1;
 739 }
 740
 741 /*
 742  * Allocate a fragment
 743  */
 744 static void *
 745 malloc_bytes(size_t size)
 746 {
 747     size_t i;
 748     int j;
 749     u_int u;
 750     struct  pginfo *bp;
 751     size_t k;
 752     u_int *lp;
 753
 754     /* Don't bother with anything less than this */
 755     if (size < malloc_minsize)
 756         size = malloc_minsize;
 757
 758
 759     /* Find the right bucket */
 760     j = 1;
 761     i = size-1;
 762     while (i >>= 1)
 763         j++;
 764
 765     /* If it's empty, make a page more of that size chunks */
 766     if (page_dir[j] == NULL && !malloc_make_chunks(j))
 767         return NULL;
 768
 769     bp = page_dir[j];
 770
 771     /* Find first word of bitmap which isn't empty */
 772     for (lp = bp->bits; !*lp; lp++)
 773         ;
 774
 775     /* Find that bit, and tweak it */
 776     u = 1;
 777     k = 0;
 778     while (!(*lp & u)) {
 779         u += u;
 780         k++;
 781     }
 782     *lp ^= u;
 783
 784     /* If there are no more free, remove from free-list */
 785     if (!--bp->free) {
 786         page_dir[j] = bp->next;
 787         bp->next = NULL;
 788     }
 789
 790     /* Adjust to the real offset of that chunk */
 791     k += (lp-bp->bits)*MALLOC_BITS;
 792     k <<= bp->shift;
 793
 794     if (malloc_junk)
 795         memset((u_char*)bp->page + k, SOME_JUNK, (size_t)bp->size);
 796
 797     return (u_char *)bp->page + k;
 798 }
 799
 800 /*
 801  * Allocate a piece of memory
 802  */
 803 static void *
 804 imalloc(size_t size)
 805 {
 806     void *result;
 807
 808     if (suicide)
 809         abort();
 810
 811     if ((size + malloc_pagesize) < size)        /* Check for overflow */
 812         result = NULL;
 813     else if ((size + malloc_pagesize) >= (uintptr_t)page_dir)
 814         result = NULL;
 815     else if (size <= malloc_maxsize)
 816         result = malloc_bytes(size);
 817     else
 818         result = malloc_pages(size);
 819
 820     if (malloc_abort && result == NULL)
 821         wrterror("allocation failed.\n");
 822
 823     if (malloc_zero && result != NULL)
 824         memset(result, 0, size);
 825
 826     return result;
 827 }
 828
 829 /*
 830  * Change the size of an allocation.
 831  */
 832 static void *
 833 irealloc(void *ptr, size_t size)
 834 {
 835     void *p;
 836     size_t osize, idx;
 837     struct pginfo **mp;
 838     size_t i;
 839
 840     if (suicide)
 841         abort();
 842
 843     idx = ptr2idx(ptr);
 844
 845     if (idx < malloc_pageshift) {
 846         wrtwarning("junk pointer, too low to make sense.\n");
 847         return 0;
 848     }
 849
 850     if (idx > last_idx) {
 851         wrtwarning("junk pointer, too high to make sense.\n");
 852         return 0;
 853     }
 854
 855     mp = &page_dir[idx];
 856
 857     if (*mp == MALLOC_FIRST) {                  /* Page allocation */
 858
 859         /* Check the pointer */
 860         if ((size_t)(uintptr_t)ptr & malloc_pagemask) {
 861             wrtwarning("modified (page-) pointer.\n");
 862             return NULL;
 863         }
 864
 865         /* Find the size in bytes */
 866         for (osize = malloc_pagesize; *++mp == MALLOC_FOLLOW;)
 867             osize += malloc_pagesize;
 868
 869         if (!malloc_realloc &&                  /* unless we have to, */
 870           size <= osize &&                      /* .. or are too small, */
 871           size > (osize - malloc_pagesize)) {   /* .. or can free a page, */
 872             if (malloc_junk)
 873                 memset((u_char *)ptr + size, SOME_JUNK, osize-size);
 874             return ptr;                         /* don't do anything. */
 875         }
 876
 877     } else if (*mp >= MALLOC_MAGIC) {           /* Chunk allocation */
 878
 879         /* Check the pointer for sane values */
 880         if (((size_t)(uintptr_t)ptr & ((*mp)->size-1))) {
 881             wrtwarning("modified (chunk-) pointer.\n");
 882             return NULL;
 883         }
 884
 885         /* Find the chunk index in the page */
 886         i = ((size_t)(uintptr_t)ptr & malloc_pagemask) >> (*mp)->shift;
 887
 888         /* Verify that it isn't a free chunk already */
 889         if ((*mp)->bits[i/MALLOC_BITS] & (1UL << (i % MALLOC_BITS))) {
 890             wrtwarning("chunk is already free.\n");
 891             return NULL;
 892         }
 893
 894         osize = (*mp)->size;
 895
 896         if (!malloc_realloc &&          /* Unless we have to, */
 897           size <= osize &&              /* ..or are too small, */
 898           (size > osize / 2 ||          /* ..or could use a smaller size, */
 899           osize == malloc_minsize)) {   /* ..(if there is one) */
 900             if (malloc_junk)
 901                 memset((u_char *)ptr + size, SOME_JUNK, osize-size);
 902             return ptr;                 /* ..Don't do anything */
 903         }
 904
 905     } else {
 906         wrtwarning("pointer to wrong page.\n");
 907         return NULL;
 908     }
 909
 910     p = imalloc(size);
 911
 912     if (p != NULL) {
 913         /* copy the lesser of the two sizes, and free the old one */
 914         if (!size || !osize)
 915             ;
 916         else if (osize < size)
 917             memcpy(p, ptr, osize);
 918         else
 919             memcpy(p, ptr, size);
 920         ifree(ptr);
 921     }
 922     return p;
 923 }
 924
 925 /*
 926  * Free a sequence of pages
 927  */
 928
 929 static inline void
 930 free_pages(void *ptr, size_t idx, struct pginfo *info)
 931 {
 932     size_t i;
 933     struct pgfree *pf, *pt=NULL;
 934     size_t l;
 935     void *tail;
 936
 937     if (info == MALLOC_FREE) {
 938         wrtwarning("page is already free.\n");
 939         return;
 940     }
 941
 942     if (info != MALLOC_FIRST) {
 943         wrtwarning("pointer to wrong page.\n");
 944         return;
 945     }
 946
 947     if ((size_t)(uintptr_t)ptr & malloc_pagemask) {
 948         wrtwarning("modified (page-) pointer.\n");
 949         return;
 950     }
 951
 952     /* Count how many pages and mark them free at the same time */
 953     page_dir[idx] = MALLOC_FREE;
 954     for (i = 1; page_dir[idx+i] == MALLOC_FOLLOW; i++)
 955         page_dir[idx + i] = MALLOC_FREE;
 956
 957     l = i << malloc_pageshift;
 958
 959     if (malloc_junk)
 960         memset(ptr, SOME_JUNK, l);
 961
 962 #ifndef __minix
 963     if (malloc_hint)
 964         madvise(ptr, l, MADV_FREE);
 965 #endif /* !__minix */
 966
 967     tail = (char *)ptr+l;
 968
 969     /* add to free-list */
 970     if (px == NULL)
 971         px = imalloc(sizeof *px);       /* This cannot fail... */
 972     px->page = ptr;
 973     px->end =  tail;
 974     px->size = l;
 975     if (free_list.next == NULL) {
 976
 977         /* Nothing on free list, put this at head */
 978         px->next = free_list.next;
 979         px->prev = &free_list;
 980         free_list.next = px;
 981         pf = px;
 982         px = NULL;
 983
 984     } else {
 985
 986         /* Find the right spot, leave pf pointing to the modified entry. */
 987         tail = (char *)ptr+l;
 988
 989         for(pf = free_list.next; pf->end < ptr && pf->next != NULL;
 990             pf = pf->next)
 991             ; /* Race ahead here */
 992
 993         if (pf->page > tail) {
 994             /* Insert before entry */
 995             px->next = pf;
 996             px->prev = pf->prev;
 997             pf->prev = px;
 998             px->prev->next = px;
 999             pf = px;
1000             px = NULL;
1001         } else if (pf->end == ptr ) {
1002             /* Append to the previous entry */
1003             pf->end = (char *)pf->end + l;
1004             pf->size += l;
1005             if (pf->next != NULL && pf->end == pf->next->page ) {
1006                 /* And collapse the next too. */
1007                 pt = pf->next;
1008                 pf->end = pt->end;
1009                 pf->size += pt->size;
1010                 pf->next = pt->next;
1011                 if (pf->next != NULL)
1012                     pf->next->prev = pf;
1013             }
1014         } else if (pf->page == tail) {
1015             /* Prepend to entry */
1016             pf->size += l;
1017             pf->page = ptr;
1018         } else if (pf->next == NULL) {
1019             /* Append at tail of chain */
1020             px->next = NULL;
1021             px->prev = pf;
1022             pf->next = px;
1023             pf = px;
1024             px = NULL;
1025         } else {
1026             wrterror("freelist is destroyed.\n");
1027         }
1028     }
1029
1030     /* Return something to OS ? */
1031     if (pf->next == NULL &&                     /* If we're the last one, */
1032       pf->size > malloc_cache &&                /* ..and the cache is full, */
1033       pf->end == malloc_brk &&                  /* ..and none behind us, */
1034       malloc_brk == sbrk((intptr_t)0)) {        /* ..and it's OK to do... */
1035         int r;
1036         /*
1037          * Keep the cache intact.  Notice that the '>' above guarantees that
1038          * the pf will always have at least one page afterwards.
1039          */
1040         pf->end = (char *)pf->page + malloc_cache;
1041         pf->size = malloc_cache;
1042
1043         r = brk(pf->end);
1044         assert(r >= 0);
1045         malloc_brk = pf->end;
1046
1047         idx = ptr2idx(pf->end);
1048
1049         for(i=idx;i <= last_idx;)
1050             page_dir[i++] = MALLOC_NOT_MINE;
1051
1052         last_idx = idx - 1;
1053
1054         /* XXX: We could realloc/shrink the pagedir here I guess. */
1055     }
1056     if (pt != NULL)
1057         ifree(pt);
1058 }
1059
1060 /*
1061  * Free a chunk, and possibly the page it's on, if the page becomes empty.
1062  */
1063
1064 static inline void
1065 free_bytes(void *ptr, size_t idx, struct pginfo *info)
1066 {
1067     size_t i;
1068     struct pginfo **mp;
1069     void *vp;
1070
1071     /* Find the chunk number on the page */
1072     i = ((size_t)(uintptr_t)ptr & malloc_pagemask) >> info->shift;
1073
1074     if (((size_t)(uintptr_t)ptr & (info->size-1))) {
1075         wrtwarning("modified (chunk-) pointer.\n");
1076         return;
1077     }
1078
1079     if (info->bits[i/MALLOC_BITS] & (1UL << (i % MALLOC_BITS))) {
1080         wrtwarning("chunk is already free.\n");
1081         return;
1082     }
1083
1084     if (malloc_junk)
1085         memset(ptr, SOME_JUNK, (size_t)info->size);
1086
1087     info->bits[i/MALLOC_BITS] |= (u_int)(1UL << (i % MALLOC_BITS));
1088     info->free++;
1089
1090     mp = page_dir + info->shift;
1091
1092     if (info->free == 1) {
1093
1094         /* Page became non-full */
1095
1096         mp = page_dir + info->shift;
1097         /* Insert in address order */
1098         while (*mp && (*mp)->next && (*mp)->next->page < info->page)
1099             mp = &(*mp)->next;
1100         info->next = *mp;
1101         *mp = info;
1102         return;
1103     }
1104
1105     if (info->free != info->total)
1106         return;
1107
1108     /* Find & remove this page in the queue */
1109     while (*mp != info) {
1110         mp = &((*mp)->next);
1111 #ifdef MALLOC_EXTRA_SANITY
1112         if (!*mp)
1113                 wrterror("(ES): Not on queue.\n");
1114 #endif /* MALLOC_EXTRA_SANITY */
1115     }
1116     *mp = info->next;
1117
1118     /* Free the page & the info structure if need be */
1119     page_dir[idx] = MALLOC_FIRST;
1120     vp = info->page;            /* Order is important ! */
1121     if(vp != (void*)info)
1122         ifree(info);
1123     ifree(vp);
1124 }
1125
1126 static void
1127 ifree(void *ptr)
1128 {
1129     struct pginfo *info;
1130     size_t idx;
1131
1132     /* This is legal */
1133     if (ptr == NULL)
1134         return;
1135
1136     /* If we're already sinking, don't make matters any worse. */
1137     if (suicide)
1138         return;
1139
1140     idx = ptr2idx(ptr);
1141
1142     if (idx < malloc_pageshift) {
1143         wrtwarning("junk pointer, too low to make sense.\n");
1144         return;
1145     }
1146
1147     if (idx > last_idx) {
1148         wrtwarning("junk pointer, too high to make sense.\n");
1149         return;
1150     }
1151
1152     info = page_dir[idx];
1153
1154     if (info < MALLOC_MAGIC)
1155         free_pages(ptr, idx, info);
1156     else
1157         free_bytes(ptr, idx, info);
1158     return;
1159 }
1160
1161 static int malloc_active; /* Recusion flag for public interface. */
1162 static unsigned malloc_started; /* Set when initialization has been done */
1163
1164 static void *
1165 pubrealloc(void *ptr, size_t size, const char *func)
1166 {
1167     void *r;
1168     int err = 0;
1169
1170     /*
1171      * If a thread is inside our code with a functional lock held, and then
1172      * catches a signal which calls us again, we would get a deadlock if the
1173      * lock is not of a recursive type.
1174      */
1175     _MALLOC_LOCK();
1176     malloc_func = func;
1177     if (malloc_active > 0) {
1178         if (malloc_active == 1) {
1179             wrtwarning("recursive call\n");
1180             malloc_active = 2;
1181         }
1182         _MALLOC_UNLOCK();
1183         errno = EINVAL;
1184         return (NULL);
1185     }
1186     malloc_active = 1;
1187
1188     if (!malloc_started) {
1189         if (ptr != NULL) {
1190             wrtwarning("malloc() has never been called\n");
1191             malloc_active = 0;
1192             _MALLOC_UNLOCK();
1193             errno = EINVAL;
1194             return (NULL);
1195         }
1196         malloc_init();
1197         malloc_started = 1;
1198     }
1199
1200     if (ptr == ZEROSIZEPTR)
1201         ptr = NULL;
1202     if (malloc_sysv && !size) {
1203         if (ptr != NULL)
1204             ifree(ptr);
1205         r = NULL;
1206     } else if (!size) {
1207         if (ptr != NULL)
1208             ifree(ptr);
1209         r = ZEROSIZEPTR;
1210     } else if (ptr == NULL) {
1211         r = imalloc(size);
1212         err = (r == NULL);
1213     } else {
1214         r = irealloc(ptr, size);
1215         err = (r == NULL);
1216     }
1217     UTRACE(ptr, size, r);
1218     malloc_active = 0;
1219     _MALLOC_UNLOCK();
1220     if (malloc_xmalloc && err)
1221         wrterror("out of memory\n");
1222     if (err)
1223         errno = ENOMEM;
1224     return (r);
1225 }
1226
1227 /*
1228  * These are the public exported interface routines.
1229  */
1230
1231 void *
1232 malloc(size_t size)
1233 {
1234
1235     return pubrealloc(NULL, size, " in malloc():");
1236 }
1237
1238 int
1239 posix_memalign(void **memptr, size_t alignment, size_t size)
1240 {
1241     int err;
1242     void *result;
1243
1244     if (!malloc_started) {
1245             malloc_init();
1246             malloc_started = 1;
1247     }
1248     /* Make sure that alignment is a large enough power of 2. */
1249     if (((alignment - 1) & alignment) != 0 || alignment < sizeof(void *) ||
1250         alignment > malloc_pagesize)
1251             return EINVAL;
1252
1253     /*
1254      * (size | alignment) is enough to assure the requested alignment, since
1255      * the allocator always allocates power-of-two blocks.
1256      */
1257     err = errno; /* Protect errno against changes in pubrealloc(). */
1258     result = pubrealloc(NULL, (size | alignment), " in posix_memalign()");
1259     errno = err;
1260
1261     if (result == NULL)
1262         return ENOMEM;
1263
1264     *memptr = result;
1265     return 0;
1266 }
1267
1268 void *
1269 calloc(size_t num, size_t size)
1270 {
1271     void *ret;
1272
1273     if (size != 0 && (num * size) / size != num) {
1274         /* size_t overflow. */
1275         errno = ENOMEM;
1276         return (NULL);
1277     }
1278
1279     ret = pubrealloc(NULL, num * size, " in calloc():");
1280
1281     if (ret != NULL)
1282         memset(ret, 0, num * size);
1283
1284     return ret;
1285 }
1286
1287 void
1288 free(void *ptr)
1289 {
1290
1291     pubrealloc(ptr, 0, " in free():");
1292 }
1293
1294 void *
1295 realloc(void *ptr, size_t size)
1296 {
1297
1298     return pubrealloc(ptr, size, " in realloc():");
1299 }
1300
1301 /*
1302  * Begin library-private functions, used by threading libraries for protection
1303  * of malloc during fork().  These functions are only called if the program is
1304  * running in threaded mode, so there is no need to check whether the program
1305  * is threaded here.
1306  */
1307
1308 void
1309 _malloc_prefork(void)
1310 {
1311
1312         _MALLOC_LOCK();
1313 }
1314
1315 void
1316 _malloc_postfork(void)
1317 {
1318
1319         _MALLOC_UNLOCK();
1320 }