sal/rtl/source/alloc.c

   1 /*************************************************************************
   2  *
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * Copyright 2008 by Sun Microsystems, Inc.
   6  *
   7  * OpenOffice.org - a multi-platform office productivity suite
   8  *
   9  * $RCSfile: alloc.c,v $
  10  * $Revision: 1.20 $
  11  *
  12  * This file is part of OpenOffice.org.
  13  *
  14  * OpenOffice.org is free software: you can redistribute it and/or modify
  15  * it under the terms of the GNU Lesser General Public License version 3
  16  * only, as published by the Free Software Foundation.
  17  *
  18  * OpenOffice.org is distributed in the hope that it will be useful,
  19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21  * GNU Lesser General Public License version 3 for more details
  22  * (a copy is included in the LICENSE file that accompanied this code).
  23  *
  24  * You should have received a copy of the GNU Lesser General Public License
  25  * version 3 along with OpenOffice.org.  If not, see
  26  * <http://www.openoffice.org/license.html>
  27  * for a copy of the LGPLv3 License.
  28  *
  29  ************************************************************************/
  30
  31 #ifdef PROFILE
  32 #undef OSL_DEBUG_LEVEL
  33 #define OSL_DEBUG_LEVEL 0
  34 #endif /* PROFILE */
  35
  36 #include <sal/types.h>
  37 #include <osl/diagnose.h>
  38 #include <rtl/alloc.h>
  39
  40 #ifndef INCLUDED_STDDEF_H
  41 #include <stddef.h>
  42 #define INCLUDED_STDDEF_H
  43 #endif
  44
  45 #ifndef INCLUDED_STDLIB_H
  46 #include <stdlib.h>
  47 #define INCLUDED_STDLIB_H
  48 #endif
  49
  50 #ifndef INCLUDED_STRING_H
  51 #include <string.h>
  52 #define INCLUDED_STRING_H
  53 #endif
  54
  55 #ifndef FORCE_SYSALLOC
  56
  57 /*===========================================================================
  58  *
  59  * rtl_memory (UNX) internals.
  60  *
  61  *=========================================================================*/
  62 #ifdef SAL_UNX
  63
  64 #include <unistd.h>
  65 #include <pthread.h>
  66 #include <sys/mman.h>
  67 #include <fcntl.h>
  68
  69 typedef pthread_mutex_t mutex_type;
  70
  71 #define RTL_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
  72 #define RTL_MUTEX_ACQUIRE(a)  pthread_mutex_lock((a))
  73 #define RTL_MUTEX_RELEASE(a)  pthread_mutex_unlock((a))
  74
  75 #if   defined(FREEBSD) || defined(NETBSD) || defined(MACOSX)
  76 static sal_Size __rtl_memory_vmpagesize (void)
  77 {
  78     /* xBSD */
  79     return (sal_Size)(getpagesize());
  80 }
  81 #elif defined(IRIX) || defined(LINUX) || defined(SOLARIS)
  82 static sal_Size __rtl_memory_vmpagesize (void)
  83 {
  84     /* POSIX */
  85     return (sal_Size)(sysconf(_SC_PAGESIZE));
  86 }
  87 #else
  88 static sal_Size __rtl_memory_vmpagesize (void)
  89 {
  90     /* other */
  91     return (sal_Size)(0x2000);
  92 }
  93 #endif /* FREEBSD || NETBSD || MACOSX || IRIX || LINUX || SOLARIS */
  94
  95 #ifndef PROT_HEAP
  96 #define PROT_HEAP (PROT_READ | PROT_WRITE | PROT_EXEC)
  97 #endif
  98
  99 /* #95880# building on Solaris 8 provides MAP_ANON, but it
 100    is not available on Solaris 7 */
 101 #if defined (SOLARIS)
 102 #ifdef MAP_ANON
 103 #undef MAP_ANON
 104 #endif
 105 #endif
 106
 107 #ifndef MAP_ANON
 108 static void* __rtl_memory_vmalloc (sal_Size n)
 109 {
 110     /* SYSV */
 111     int fd = open("/dev/zero", O_RDWR);
 112     if (!(fd < 0))
 113     {
 114         void * p = mmap(NULL, n, PROT_HEAP, MAP_PRIVATE, fd, 0);
 115         close(fd);
 116         return ((p == MAP_FAILED) ? NULL : p);
 117     }
 118     return (NULL);
 119 }
 120 #else  /* MAP_ANON */
 121 static void* __rtl_memory_vmalloc (sal_Size n)
 122 {
 123     /* xBSD */
 124     void * p = mmap(NULL, n, PROT_HEAP, MAP_PRIVATE | MAP_ANON, -1, 0);
 125     return ((p == MAP_FAILED) ? NULL : p);
 126 }
 127 #endif /* MAP_ANON */
 128
 129 #define RTL_MEMORY_ALLOC(n) __rtl_memory_vmalloc((sal_Size)(n))
 130 #define RTL_MEMORY_FREE(p, n) munmap((void*)(p), (sal_Size)(n))
 131
 132 #endif /* SAL_UNX */
 133
 134 /*===========================================================================
 135  *
 136  * rtl_memory (W32) internals.
 137  *
 138  *=========================================================================*/
 139 #ifdef SAL_W32
 140
 141 #define WIN32_LEAN_AND_MEAN
 142 #ifdef _MSC_VER
 143 #pragma warning(push,1) /* disable warnings within system headers */
 144 #endif
 145 #include <windows.h>
 146 #include <wchar.h>
 147
 148 typedef CRITICAL_SECTION mutex_type;
 149
 150 /* Static initializer (struct declared in WINNT.H). */
 151 #define RTL_MUTEX_INITIALIZER { NULL, -1, 0, NULL, NULL, 0 }
 152
 153 /*
 154  * __rtl_mutex_init (dynamic initialization).
 155  *
 156  * Static initialization (with DebugInfo == NULL)
 157  * leads to Access Violation upon first contention.
 158  */
 159 static void __rtl_mutex_init (LPCRITICAL_SECTION lpCriticalSection)
 160 {
 161     static LONG g_spinlock = 0;
 162
 163     while (InterlockedExchange (&g_spinlock, 1) == 1)
 164     {
 165         /* Already locked, spin */
 166         Sleep (0);
 167     }
 168     if (!(lpCriticalSection->DebugInfo))
 169     {
 170         /* Dynamic initialization */
 171         InitializeCriticalSection (lpCriticalSection);
 172     }
 173     InterlockedExchange (&g_spinlock, 0);
 174 }
 175
 176 #define RTL_MUTEX_INIT(a)  __rtl_mutex_init((LPCRITICAL_SECTION)(a))
 177 #define RTL_MUTEX_ACQUIRE(a)  EnterCriticalSection((a))
 178 #define RTL_MUTEX_RELEASE(a)  LeaveCriticalSection((a))
 179
 180 static sal_Size __rtl_memory_vmpagesize (void)
 181 {
 182     SYSTEM_INFO info;
 183     GetSystemInfo (&info);
 184     return ((sal_Size)(info.dwPageSize));
 185 }
 186
 187 #define RTL_MEMORY_ALLOC(n) \
 188 (void*)(VirtualAlloc (NULL, (SIZE_T)(n), MEM_COMMIT, PAGE_READWRITE))
 189
 190 #define RTL_MEMORY_FREE(p, n) \
 191 (void)(VirtualFree ((LPVOID)(p), (SIZE_T)(0), MEM_RELEASE))
 192
 193 #endif /* SAL_W32 */
 194
 195 /*===========================================================================
 196  *
 197  * rtl_memory (OS2) internals.
 198  *
 199  *=========================================================================*/
 200 #ifdef SAL_OS2
 201
 202 #define INCL_DOS
 203 #include <os2.h>
 204
 205 typedef HMTX mutex_type;
 206
 207 /* Static initializer */
 208 #define RTL_MUTEX_INITIALIZER -1
 209
 210 /*
 211  * __rtl_mutex_init (dynamic initialization).
 212  *
 213  * Static initialization (with DebugInfo == NULL)
 214  * leads to Access Violation upon first contention.
 215  */
 216 static void __rtl_mutex_init (mutex_type* mutex)
 217 {
 218     APIRET              rc = 0;
 219
 220     rc = DosCreateMutexSem(NULL,mutex,0,0);
 221
 222 }
 223
 224 static int __rtl_mutex_destroy (mutex_type* mutex)
 225 {
 226     APIRET              rc = 0;
 227
 228
 229     do {
 230         rc = DosCloseMutexSem(*mutex);
 231         if (rc == 301) DosReleaseMutexSem(*mutex);
 232     } while (rc == 301);
 233
 234     *mutex = 0;
 235
 236     /* Return the completion status: */
 237     return (0);
 238 }
 239
 240
 241 static int __rtl_mutex_acquire(mutex_type* mutex)
 242 {
 243     int         ret = 0;
 244     int         status = 0;
 245     APIRET              rc = 0;
 246
 247     // initialize static semaphores created with PTHREAD_MUTEX_INITIALIZER state.
 248     if (*mutex == -1)
 249         __rtl_mutex_init( mutex);
 250
 251     rc = DosRequestMutexSem(*mutex,SEM_INDEFINITE_WAIT);
 252     if (rc)
 253         return(1);
 254
 255     /* Return the completion status: */
 256     return (0);
 257 }
 258
 259 static int __rtl_mutex_release(mutex_type* mutex)
 260 {
 261     int         ret = 0;
 262     APIRET              rc = 0;
 263     int         status;
 264
 265
 266     // initialize static semaphores created with PTHREAD_MUTEX_INITIALIZER state.
 267     if (*mutex == -1)
 268        __rtl_mutex_init( mutex);
 269
 270     rc = DosReleaseMutexSem(*mutex);
 271
 272     /* Return the completion status: */
 273     return (0);
 274 }
 275
 276 #define RTL_MUTEX_INIT(a)  __rtl_mutex_init((mutex_type*)(a))
 277 #define RTL_MUTEX_ACQUIRE(a)  __rtl_mutex_acquire((mutex_type*)(a))
 278 #define RTL_MUTEX_RELEASE(a)  __rtl_mutex_release((mutex_type*)(a))
 279
 280 static sal_Size __rtl_memory_vmpagesize (void)
 281 {
 282     return (sal_Size)(getpagesize());
 283 }
 284
 285 #define RTL_MEMORY_ALLOC(n) (void*)(malloc(n))
 286
 287 #define RTL_MEMORY_FREE(p, n) (void)(free(p))
 288
 289 #endif /* SAL_OS2 */
 290
 291 /*===========================================================================
 292  *
 293  * Determine allocation mode (debug/release) by examining environment
 294  * variable "OOO_FORCE_SYSALLOC".
 295  *
 296  *=========================================================================*/
 297
 298 #include <stdlib.h>   /* getenv */
 299 #include <stdio.h>    /* stderr */
 300
 301 typedef
 302    enum { AMode_CUSTOM, AMode_SYSTEM, AMode_UNSET }
 303    AllocMode;
 304
 305 static AllocMode alloc_mode = AMode_UNSET;
 306
 307 static void determine_alloc_mode ( void )
 308 {
 309    /* This shouldn't happen, but still ... */
 310    if (alloc_mode != AMode_UNSET)
 311       return;
 312
 313    if (getenv("OOO_FORCE_SYSALLOC") != NULL) {
 314       alloc_mode = AMode_SYSTEM;
 315       fprintf(stderr, "OOo: Using system memory allocator.\n");
 316       fprintf(stderr, "OOo: This is for debugging only.  To disable,\n");
 317       fprintf(stderr, "OOo: unset the environment variable"
 318                       " OOO_FORCE_SYSALLOC.\n");
 319    } else {
 320       alloc_mode = AMode_CUSTOM;
 321    }
 322 }
 323
 324 /*===========================================================================
 325  *
 326  * rtl_memory (global) internals.
 327  *
 328  *=========================================================================*/
 329 #define __L__ 32
 330 #define __P__ 24
 331 #define __N__ ((__L__) + (__P__))
 332 #define __M__ 0x10000
 333
 334 static const sal_Size __T__ = (__M__) * 2 / 3;
 335
 336 typedef struct __rtl_memory_desc_st memory_type;
 337 struct __rtl_memory_desc_st
 338 {
 339     sal_Size       m_length;
 340     sal_Size       m_offset;
 341     memory_type *m_flink;
 342     memory_type *m_blink;
 343 };
 344
 345 static const int __C__ = 2 * sizeof(sal_Size);
 346 static const int __Q__ = 2 * sizeof(memory_type*);
 347
 348 typedef struct __rtl_memory_stat_st memory_stat;
 349 struct __rtl_memory_stat_st
 350 {
 351     sal_uInt64 m_dequeue;
 352     sal_uInt64 m_enqueue;
 353     sal_Int32  m_delta_q;
 354
 355     sal_uInt64 m_deqsize;
 356     sal_uInt64 m_enqsize;
 357     sal_Int32  m_delta_n;
 358 };
 359
 360 #define RTL_MEMORY_ALIGN(n, m) (((n) + ((m) - 1)) & ~((m) - 1))
 361 #define RTL_MEMORY_SIZEOF(a) RTL_MEMORY_ALIGN(sizeof(a), sizeof(memory_type))
 362
 363 struct __rtl_memory_global_st
 364 {
 365     sal_Size  m_magic;
 366     sal_Size  m_align;
 367
 368     union {
 369         mutex_type m_lock;
 370         char       m_data[RTL_MEMORY_SIZEOF(mutex_type)];
 371     } m_mutex;
 372
 373     memory_type    m_alloc_head;
 374     memory_type    m_spare_head;
 375     memory_type    m_queue_head[__N__];
 376
 377 #if OSL_DEBUG_LEVEL > 0
 378     memory_stat    m_queue_stat[__N__];
 379 #endif /* OSL_DEBUG_LEVEL */
 380 };
 381
 382 static struct __rtl_memory_global_st g_memory =
 383 {
 384     0, 0, { RTL_MUTEX_INITIALIZER },
 385     { 0, 0, NULL, NULL }, { 0, 0, NULL, NULL }, { { 0, 0, NULL, NULL } },
 386 #if OSL_DEBUG_LEVEL > 0
 387     { { 0, 0, 0, 0, 0, 0 } }
 388 #endif /* OSL_DEBUG_LEVEL */
 389 };
 390
 391 void SAL_CALL ___rtl_memory_init (void);
 392 void SAL_CALL ___rtl_memory_fini (void);
 393
 394 #define RTL_MEMORY_ENTER() \
 395 { \
 396     if (!(g_memory.m_align)) ___rtl_memory_init(); \
 397     RTL_MUTEX_ACQUIRE(&(g_memory.m_mutex.m_lock)); \
 398 }
 399
 400 #define RTL_MEMORY_LEAVE() \
 401 { \
 402     RTL_MUTEX_RELEASE(&(g_memory.m_mutex.m_lock)); \
 403 }
 404
 405 /*===========================================================================
 406  *
 407  * rtl_memory (queue) internals.
 408  *
 409  *=========================================================================*/
 410 #if defined(PROFILE) || (OSL_DEBUG_LEVEL > 0)
 411 static sal_Size queue (sal_Size n)
 412 {
 413     /* k = n div __C__ */
 414     register sal_Size k = n / __C__, m = __L__;
 415
 416     OSL_PRECOND((__L__ == 32),
 417                 "__rtl_memory_queue(): internal logic error");
 418     if (k > m)
 419     {
 420         /* k = k div __L__ = k div 32 */
 421         k >>= 5;
 422         while ((k >>= 1) > 0) m++;
 423         k = m;
 424     }
 425
 426     OSL_POSTCOND((0 < k) && (k < __N__),
 427                  "__rtl_memory_queue(): "
 428                  "internal error: index out of bounds");
 429     return (k);
 430 }
 431 #else  /* PRODUCT */
 432 #define queue(k, n) \
 433 { \
 434     (k) = ((n) / __C__); \
 435     if ((k) > __L__) \
 436     { \
 437         register sal_Size m = __L__; \
 438         (k) >>= 5; \
 439         while (((k) >>= 1) > 0) m++; \
 440         (k) = m; \
 441     } \
 442 }
 443 #endif /* OSL_DEBUG_LEVEL || PRODUCT */
 444
 445 #define queue_start(entry) \
 446 { \
 447     (entry)->m_flink = (entry); \
 448     (entry)->m_blink = (entry); \
 449 }
 450
 451 #define queue_remove(entry) \
 452 { \
 453     (entry)->m_blink->m_flink = (entry)->m_flink; \
 454     (entry)->m_flink->m_blink = (entry)->m_blink; \
 455     queue_start(entry); \
 456 }
 457
 458 #define queue_insert_head(head, entry) \
 459 { \
 460     (entry)->m_blink = (head); \
 461     (entry)->m_flink = (head)->m_flink; \
 462     (head)->m_flink = (entry); \
 463     (entry)->m_flink->m_blink = (entry); \
 464 }
 465
 466 #define queue_insert_tail(head, entry) \
 467 { \
 468     (entry)->m_flink = (head); \
 469     (entry)->m_blink = (head)->m_blink; \
 470     (head)->m_blink = (entry); \
 471     (entry)->m_blink->m_flink = (entry); \
 472 }
 473
 474 /*===========================================================================
 475  *
 476  * rtl_memory (debug) internals.
 477  *
 478  *=========================================================================*/
 479 #if OSL_DEBUG_LEVEL > 0
 480
 481 #define __dbg_memory_succ(entry, length) \
 482 (memory_type*)((char*)((entry)) + ((length) & ~0x1))
 483
 484 #define __dbg_memory_pred(entry, offset) \
 485 (memory_type*)((char*)((entry)) - ((offset) & ~0x1))
 486
 487 #define __dbg_memory_ensure(entry) (!((sal_Size)(entry) & 0x7))
 488
 489 /*
 490  * __dbg_memory_dequeue.
 491  */
 492 static void __dbg_memory_dequeue (sal_Size n)
 493 {
 494     register sal_Size k = queue(n);
 495
 496     g_memory.m_queue_stat[k].m_dequeue += 1;
 497     g_memory.m_queue_stat[k].m_delta_q += 1;
 498
 499     g_memory.m_queue_stat[k].m_deqsize += n;
 500     g_memory.m_queue_stat[k].m_delta_n += n;
 501 }
 502
 503 /*
 504  * __dbg_memory_enqueue.
 505  */
 506 static void __dbg_memory_enqueue (sal_Size n)
 507 {
 508     register sal_Size k = queue(n);
 509
 510     g_memory.m_queue_stat[k].m_enqueue += 1;
 511     g_memory.m_queue_stat[k].m_delta_q -= 1;
 512
 513     g_memory.m_queue_stat[k].m_enqsize += n;
 514     g_memory.m_queue_stat[k].m_delta_n -= n;
 515 }
 516
 517 /*
 518  * __dbg_memory_insert.
 519  */
 520 static void __dbg_memory_insert (memory_type **ppMemory)
 521 {
 522     register memory_type * succ;
 523     succ = __dbg_memory_succ (*ppMemory, sizeof(memory_type));
 524
 525     succ->m_length = (*ppMemory)->m_length - sizeof(memory_type);
 526     succ->m_offset = (*ppMemory)->m_offset;
 527
 528     queue_insert_tail (&(g_memory.m_alloc_head), (*ppMemory));
 529     (*ppMemory) = succ;
 530 }
 531
 532 /*
 533  * __dbg_memory_remove.
 534  */
 535 static void __dbg_memory_remove (memory_type **ppMemory)
 536 {
 537     (*ppMemory) = __dbg_memory_pred (*ppMemory, sizeof(memory_type));
 538     queue_remove (*ppMemory);
 539 }
 540
 541 /*
 542  * __dbg_memory_verify_chain.
 543  */
 544 static int __dbg_memory_verify_chain (memory_type * x)
 545 {
 546     if (!__dbg_memory_ensure(x))
 547     {
 548         OSL_ENSURE(0, "__rtl_memory_verify(): invalid pointer alignment.");
 549         return (0);
 550     }
 551     if (!__dbg_memory_ensure(x->m_length & ~0x1))
 552     {
 553         OSL_ENSURE(0, "__rtl_memory_verify(): dynamic memory corruption");
 554         return (0);
 555     }
 556     if (!__dbg_memory_ensure(x->m_offset & ~0x1))
 557     {
 558         OSL_ENSURE(0, "__rtl_memory_verify(): dynamic memory corruption");
 559         return (0);
 560     }
 561     if (!(x->m_length & ~0x1))
 562     {
 563         OSL_ENSURE(0, "__rtl_memory_verify(): dynamic memory corruption");
 564         return (0);
 565     }
 566     return (1);
 567 }
 568
 569 /*
 570  * __dbg_memory_verify_queue.
 571  */
 572 static int __dbg_memory_verify_queue (memory_type * x)
 573 {
 574     if (!__dbg_memory_ensure(x))
 575     {
 576         OSL_ENSURE(0, "__rtl_memory_verify(): invalid pointer alignment.");
 577         return (0);
 578     }
 579     if (!__dbg_memory_ensure(x->m_flink))
 580     {
 581         OSL_ENSURE(0, "__rtl_memory_verify(): free memory corruption");
 582         return (0);
 583     }
 584     if (!__dbg_memory_ensure(x->m_blink))
 585     {
 586         OSL_ENSURE(0, "__rtl_memory_verify(): free memory corruption");
 587         return (0);
 588     }
 589     if ((x == x->m_flink) || (x == x->m_blink))
 590     {
 591         OSL_ENSURE(0, "__rtl_memory_verify(): internal logic error");
 592         return (0);
 593     }
 594     return (1);
 595 }
 596
 597 /*
 598  * __dbg_memory_verify_alloc.
 599  */
 600 static int __dbg_memory_verify_alloc (memory_type * x)
 601 {
 602     register memory_type *head, *entry;
 603     head = entry = &(g_memory.m_alloc_head);
 604
 605     if (!__dbg_memory_ensure(x))
 606     {
 607         OSL_ENSURE(0, "__rtl_memory_verify(): invalid pointer alignment.");
 608         return (0);
 609     }
 610     while (!((entry = entry->m_flink) == head))
 611     {
 612         if ((entry < x) && (x < __dbg_memory_succ(entry, entry->m_length)))
 613         {
 614             head = entry = __dbg_memory_succ(entry, sizeof(memory_type));
 615             while (!((x == entry) || (entry->m_offset & 0x1)))
 616             {
 617                 /* no match, not last */
 618                 if (!__dbg_memory_verify_chain (entry))
 619                     return (0);
 620                 entry = __dbg_memory_succ(entry, entry->m_length);
 621             }
 622
 623             /* match, or last */
 624             if (!__dbg_memory_verify_chain (entry))
 625                 return (0);
 626             break;
 627         }
 628     }
 629     if (!(x == entry))
 630     {
 631         OSL_ENSURE(0, "__rtl_memory_verify(): memory not allocated.");
 632         return (0);
 633     }
 634     return (1);
 635 }
 636
 637 /*
 638  * __dbg_memory_verify.
 639  */
 640 static int __dbg_memory_verify (memory_type * x, int debug)
 641 {
 642     /* dispatch upon 'debug' level */
 643     if (debug)
 644     {
 645         /* verify allocation */
 646         if (!__dbg_memory_verify_alloc (x))
 647             return (0);
 648     }
 649     else
 650     {
 651         /* verify 'chain' fields */
 652         if (!__dbg_memory_verify_chain (x))
 653             return (0);
 654     }
 655
 656     /* verify 'used' bit */
 657     if (!(x->m_length & 0x1))
 658     {
 659         OSL_ENSURE(0, "__rtl_memory_verify(): memory not used.");
 660         return (0);
 661     }
 662     return (1);
 663 }
 664
 665 #if OSL_DEBUG_LEVEL > 1
 666 /*
 667  * __dbg_memory_usage_update.
 668  */
 669 static sal_Size __dbg_memory_usage_update (memory_stat * stat, sal_Size length)
 670 {
 671     register sal_Size n = (length & ~0x1), k = queue(n);
 672
 673     stat[k].m_dequeue += 1;
 674     stat[k].m_deqsize += n;
 675
 676     if (!(length & 0x1))
 677     {
 678         /* not used */
 679         stat[k].m_enqueue += 1;
 680         stat[k].m_enqsize += n;
 681         return (n);
 682     }
 683     else
 684     {
 685         /* used */
 686         stat[k].m_delta_q += 1;
 687         stat[k].m_delta_n += n;
 688         return (0);
 689     }
 690 }
 691
 692 /*
 693  * __dbg_memory_usage.
 694  */
 695 static void __dbg_memory_usage (memory_stat * total)
 696 {
 697     register memory_type *head, *entry, *memory;
 698     memory_stat           stat[__N__];
 699
 700     memset (stat, 0, __N__ * sizeof(memory_stat));
 701
 702     head = entry = &(g_memory.m_alloc_head);
 703     while (!((entry = entry->m_flink) == head))
 704     {
 705         register sal_Size k = 0, n = entry->m_length - sizeof(memory_type);
 706
 707         memory = __dbg_memory_succ(entry, sizeof(memory_type));
 708         while (!(memory->m_offset & 0x1))
 709         {
 710             /* not last */
 711             k += __dbg_memory_usage_update (stat, memory->m_length);
 712             memory = __dbg_memory_succ(memory, memory->m_length);
 713         }
 714
 715         k += __dbg_memory_usage_update (stat, memory->m_length);
 716         OSL_TRACE("%x %10d %10d", (sal_Size)(entry), n, k);
 717     }
 718
 719     if (total)
 720     {
 721         sal_Size i;
 722
 723         memset (total, 0, sizeof(memory_stat));
 724         for (i = 0; i < __N__; i++)
 725         {
 726             total->m_dequeue += stat[i].m_dequeue;
 727             total->m_enqueue += stat[i].m_enqueue;
 728             total->m_delta_q += stat[i].m_delta_q;
 729
 730             total->m_deqsize += stat[i].m_deqsize;
 731             total->m_enqsize += stat[i].m_enqsize;
 732             total->m_delta_n += stat[i].m_delta_n;
 733         }
 734     }
 735 }
 736 #endif /* OSL_DEBUG_LEVEL */
 737
 738 #endif /* OSL_DEBUG_LEVEL */
 739 #if OSL_DEBUG_LEVEL > 0
 740
 741 #define DBG_MEMORY_DEQUEUE(n) __dbg_memory_dequeue((sal_Size)(n) & ~0x1)
 742 #define DBG_MEMORY_ENQUEUE(n) __dbg_memory_enqueue((sal_Size)(n) & ~0x1)
 743
 744 #define DBG_MEMORY_DEQFILL(entry, offset, length) \
 745     memset(((char*)(entry) + (offset)), 0x77777777, (length))
 746 #define DBG_MEMORY_ENQFILL(entry, offset, length) \
 747     memset(((char*)(entry) + (offset)), 0x33333333, (length))
 748
 749 #define DBG_MEMORY_INSERT(entry) __dbg_memory_insert((entry))
 750 #define DBG_MEMORY_REMOVE(entry) __dbg_memory_remove((entry))
 751
 752 #if OSL_DEBUG_LEVEL > 1
 753 #define DBG_MEMORY_VERIFY(entry) __dbg_memory_verify((entry), 1)
 754 #else  /* OSL_DEBUG_LEVEL > 0 */
 755 #define DBG_MEMORY_VERIFY(entry) __dbg_memory_verify((entry), 0)
 756 #endif /* OSL_DEBUG_LEVEL */
 757
 758 #define DBG_MEMORY_VERIFY_CHAIN(entry) __dbg_memory_verify_chain((entry))
 759 #define DBG_MEMORY_VERIFY_QUEUE(entry) __dbg_memory_verify_queue((entry))
 760
 761 #else  /* PRODUCT */
 762
 763 #define DBG_MEMORY_DEQUEUE(n)
 764 #define DBG_MEMORY_ENQUEUE(n)
 765
 766 #define DBG_MEMORY_DEQFILL(entry, offset, length)
 767 #define DBG_MEMORY_ENQFILL(entry, offset, length)
 768
 769 #define DBG_MEMORY_INSERT(entry)
 770 #define DBG_MEMORY_REMOVE(entry)
 771
 772 #define DBG_MEMORY_VERIFY(entry)
 773 #define DBG_MEMORY_VERIFY_CHAIN(entry)
 774 #define DBG_MEMORY_VERIFY_QUEUE(entry)
 775
 776 #endif /* OSL_DEBUG_LEVEL || PRODUCT */
 777
 778 /*===========================================================================
 779  *
 780  * rtl_memory (manager) internals.
 781  *
 782  *=========================================================================*/
 783 #define queue_cast(entry, offset) \
 784 ((memory_type*)((char*)(entry) + (ptrdiff_t)(offset)))
 785
 786 #define __rtl_memory_used(entry) ((entry)->m_length & 0x1)
 787 #define __rtl_memory_last(entry) ((entry)->m_offset & 0x1)
 788 #define __rtl_memory_offset(entry) \
 789     ((ptrdiff_t)((entry)->m_offset & ~0x1))
 790
 791 /*
 792  * ___rtl_memory_init.
 793  */
 794 void SAL_CALL ___rtl_memory_init (void)
 795 {
 796 #if defined(RTL_MUTEX_INIT)
 797     RTL_MUTEX_INIT (&(g_memory.m_mutex.m_lock));
 798 #endif /* RTL_MUTEX_INIT */
 799
 800     RTL_MUTEX_ACQUIRE(&(g_memory.m_mutex.m_lock));
 801     if (!(g_memory.m_align))
 802     {
 803         sal_Size pagesize;
 804         int    i;
 805
 806         queue_start (&(g_memory.m_alloc_head));
 807         queue_start (&(g_memory.m_spare_head));
 808
 809         for (i = 0; i < __N__; i++)
 810             queue_start (&(g_memory.m_queue_head[i]));
 811         for (i = 1; i <= __L__; i++)
 812             g_memory.m_queue_head[i].m_length = i * __C__;
 813         for (i = __L__ + 1; i < __N__; i++)
 814             g_memory.m_queue_head[i].m_length =
 815                 2 * g_memory.m_queue_head[i - 1].m_length;
 816
 817         pagesize = __rtl_memory_vmpagesize();
 818         g_memory.m_align = RTL_MEMORY_ALIGN(__M__, pagesize);
 819     }
 820     RTL_MUTEX_RELEASE(&(g_memory.m_mutex.m_lock));
 821 }
 822
 823 /*
 824  * ___rtl_memory_fini.
 825  */
 826 void SAL_CALL ___rtl_memory_fini (void)
 827 {
 828 #if OSL_DEBUG_LEVEL > 1
 829
 830     memory_stat total;
 831
 832     __dbg_memory_usage (&total);
 833     if (total.m_delta_n > 0)
 834     {
 835         OSL_TRACE("___rtl_memory_fini(): "
 836                   "Leak: %10d (Alloc: %10d, Free: %10d)",
 837                   total.m_delta_n,
 838                   (sal_uInt32)(total.m_deqsize & 0xffffffff),
 839                   (sal_uInt32)(total.m_enqsize & 0xffffffff));
 840     }
 841
 842 #endif /* OSL_DEBUG_LEVEL */
 843 }
 844
 845 /*
 846  * __rtl_memory_merge.
 847  */
 848 #if defined(PROFILE) || (OSL_DEBUG_LEVEL > 0)
 849 static void __rtl_memory_merge (memory_type * prev, memory_type * next)
 850 {
 851     /* adjust length */
 852     prev->m_length += next->m_length;
 853     if (!__rtl_memory_last(next))
 854     {
 855         /* not last, adjust offset */
 856         register memory_type * succ = queue_cast(prev, prev->m_length);
 857         DBG_MEMORY_VERIFY_CHAIN (succ);
 858         succ->m_offset = prev->m_length | __rtl_memory_last(succ);
 859     }
 860
 861     /* propagate 'last' bit */
 862     prev->m_offset |= __rtl_memory_last(next);
 863 }
 864 #else  /* PRODUCT */
 865 #define __rtl_memory_merge(prev, next) \
 866 { \
 867     (prev)->m_length += (next)->m_length; \
 868     if (!__rtl_memory_last((next))) \
 869     { \
 870         register memory_type * succ = queue_cast((prev), (prev)->m_length); \
 871         succ->m_offset = (prev)->m_length | __rtl_memory_last(succ); \
 872     } \
 873     (prev)->m_offset |= __rtl_memory_last((next)); \
 874 }
 875 #endif /* OSL_DEBUG_LEVEL || PRODUCT */
 876
 877 /*
 878  * __rtl_memory_split.
 879  */
 880 #if defined(PROFILE) || (OSL_DEBUG_LEVEL > 0)
 881 static void __rtl_memory_split (memory_type * prev, memory_type * next)
 882 {
 883     /* adjust length */
 884     prev->m_length -= next->m_length;
 885     if (!__rtl_memory_last(prev))
 886     {
 887         /* not last, adjust offset */
 888         register memory_type * succ = queue_cast(next, next->m_length);
 889         DBG_MEMORY_VERIFY_CHAIN (succ);
 890         succ->m_offset = next->m_length | __rtl_memory_last(succ);
 891     }
 892
 893     /* propagate 'last' bit */
 894     next->m_offset |= __rtl_memory_last(prev);
 895     prev->m_offset &= ~0x1;
 896 }
 897 #else  /* PRODUCT */
 898 #define __rtl_memory_split(prev, next) \
 899 { \
 900     (prev)->m_length -= (next)->m_length; \
 901     if (!__rtl_memory_last((prev))) \
 902     { \
 903         register memory_type * succ = queue_cast((next), (next)->m_length); \
 904         succ->m_offset = (next)->m_length | __rtl_memory_last(succ); \
 905     } \
 906 \
 907     (next)->m_offset |= __rtl_memory_last((prev)); \
 908     (prev)->m_offset &= ~0x1; \
 909 }
 910 #endif /* OSL_DEBUG_LEVEL || PRODUCT */
 911
 912 /*
 913  * __rtl_memory_insert.
 914  */
 915 #if defined(PROFILE) || (OSL_DEBUG_LEVEL > 0)
 916 static void __rtl_memory_insert (memory_type * memory, sal_Size n)
 917 {
 918     /* obtain queue head */
 919     register memory_type *head;
 920
 921     head = &(g_memory.m_queue_head[queue(n)]);
 922     DBG_MEMORY_VERIFY_CHAIN (head);
 923
 924     /* insert at queue tail (first-in first-out) */
 925     queue_insert_tail (head, memory);
 926 }
 927 #else  /* PRODUCT */
 928 #define __rtl_memory_insert(memory, n) \
 929 { \
 930     register sal_Size h; \
 931 \
 932     queue(h, (n)); \
 933     queue_insert_tail (&(g_memory.m_queue_head[h]), (memory)); \
 934 }
 935 #endif /* OSL_DEBUG_LEVEL || PRODUCT */
 936
 937 /*
 938  * __rtl_memory_resize.
 939  */
 940 #if defined(PROFILE) || (OSL_DEBUG_LEVEL > 0)
 941 static void __rtl_memory_resize (memory_type * memory, sal_Size n)
 942 {
 943     register sal_Size k = (memory->m_length - n);
 944
 945     OSL_ENSURE(!(memory->m_length & 0x1),
 946                "__rtl_memory_resize(): "
 947                "internal logic error.");
 948
 949     if ((k >= sizeof(memory_type)) && (n <= __T__))
 950     {
 951         /* split */
 952         register memory_type * remain = queue_cast(memory, n);
 953
 954         remain->m_length = k; remain->m_offset = n;
 955         __rtl_memory_split (memory, remain);
 956
 957         /* check postcond */
 958         if (!__rtl_memory_last(remain))
 959         {
 960             /* not last, verify used next entry */
 961             register memory_type *next;
 962
 963             next = queue_cast(remain, remain->m_length);
 964             DBG_MEMORY_VERIFY_CHAIN (next);
 965
 966             OSL_POSTCOND(__rtl_memory_used(next),
 967                          "__rtl_memory_resize(): "
 968                          "internal logic error.");
 969         }
 970
 971         /* enqueue */
 972         __rtl_memory_insert (remain, k);
 973         DBG_MEMORY_VERIFY_QUEUE (remain);
 974     }
 975
 976     DBG_MEMORY_DEQUEUE(memory->m_length);
 977 }
 978 #else  /* PRODUCT */
 979 #define __rtl_memory_resize(memory, n) \
 980 { \
 981     register sal_Size kn = ((memory)->m_length - (n)); \
 982     if ((kn >= sizeof(memory_type)) && (n <= __T__)) \
 983     { \
 984         register memory_type * remain = queue_cast((memory), (n)); \
 985 \
 986         remain->m_length = kn; remain->m_offset = (n); \
 987         __rtl_memory_split ((memory), remain); \
 988 \
 989         __rtl_memory_insert (remain, kn); \
 990     } \
 991 }
 992 #endif /* OSL_DEBUG_LEVEL || PRODUCT */
 993
 994 /*
 995  * __rtl_memory_dequeue.
 996  */
 997 #if defined(PROFILE) || (OSL_DEBUG_LEVEL > 0)
 998 static void __rtl_memory_dequeue (memory_type **ppMemory, sal_Size n)
 999 {
1000     register memory_type *head, *entry;
1001     register sal_Size       k, m = n;
1002
1003     OSL_PRECOND(!*ppMemory, "__rtl_memory_dequeue(): internal logic error.");
1004     for (k = queue(m); k < __N__; k++)
1005     {
1006         /* first fit (equals best fit w/ ascending insert) */
1007         head = &(g_memory.m_queue_head[k]);
1008         for (entry = head->m_flink; entry != head; entry = entry->m_flink)
1009         {
1010             /* queue not empty */
1011             DBG_MEMORY_VERIFY_CHAIN (entry);
1012             if (entry->m_length >= m)
1013             {
1014                 /* remove entry */
1015                 DBG_MEMORY_VERIFY_QUEUE (entry);
1016                 queue_remove (entry);
1017
1018                 /* assign result */
1019                 *ppMemory = entry;
1020                 goto dequeue_leave;
1021             }
1022         }
1023     }
1024
1025     head = &(g_memory.m_spare_head);
1026     for (entry = head->m_flink; entry != head; entry = entry->m_flink)
1027     {
1028         /* queue not empty */
1029         DBG_MEMORY_VERIFY_CHAIN (entry);
1030         if (entry->m_length >= m)
1031         {
1032             /* remove entry */
1033             DBG_MEMORY_VERIFY_QUEUE (entry);
1034             queue_remove (entry);
1035
1036             /* assign result */
1037             *ppMemory = entry;
1038             goto dequeue_leave;
1039         }
1040     }
1041
1042 #if OSL_DEBUG_LEVEL > 0
1043     /* adjust for DBG_MEMORY_INSERT() overhead */
1044     m += sizeof(memory_type);
1045 #endif /* OSL_DEBUG_LEVEL */
1046
1047     k = RTL_MEMORY_ALIGN((m > __M__) ? m : __M__, g_memory.m_align);
1048     if (!((entry = RTL_MEMORY_ALLOC(k)) == 0))
1049     {
1050         entry->m_length = k;
1051         entry->m_offset = 0x1; /* set 'last' bit */
1052
1053         *ppMemory = entry;
1054         DBG_MEMORY_INSERT(ppMemory);
1055     }
1056
1057 dequeue_leave:
1058     OSL_POSTCOND(*ppMemory, "__rtl_memory_dequeue(): out of memory.");
1059     if ((entry = *ppMemory) != 0)
1060     {
1061         /* adjust length */
1062         __rtl_memory_resize (entry, n);
1063
1064         /* fill w/ 'uninitialized' pattern */
1065         DBG_MEMORY_DEQFILL (entry, __C__, entry->m_length - __C__);
1066     }
1067 #if OSL_DEBUG_LEVEL > 1
1068     if (!entry)
1069     {
1070         memory_stat total;
1071         __dbg_memory_usage (&total);
1072     }
1073 #endif /* OSL_DEBUG_LEVEL */
1074 }
1075 #else  /* PRODUCT */
1076 #define __rtl_memory_dequeue(ppMemory, n, label) \
1077 { \
1078     register memory_type *head, *entry; \
1079     register sal_Size       h, m = (n); \
1080 \
1081     queue (h, m); \
1082     for (; h < __N__; h++) \
1083     { \
1084         head = &(g_memory.m_queue_head[h]); \
1085         for (entry = head->m_flink; entry != head; entry = entry->m_flink) \
1086         { \
1087             if (entry->m_length >= m) \
1088             { \
1089                 queue_remove (entry); \
1090                 goto label; \
1091             } \
1092         } \
1093     } \
1094 \
1095     head = &(g_memory.m_spare_head); \
1096     for (entry = head->m_flink; entry != head; entry = entry->m_flink) \
1097     { \
1098         if (entry->m_length >= m) \
1099         { \
1100             queue_remove (entry); \
1101             goto label; \
1102         } \
1103     } \
1104 \
1105     h = RTL_MEMORY_ALIGN((m > __M__) ? m : __M__, g_memory.m_align); \
1106     if (!((entry = RTL_MEMORY_ALLOC(h)) == 0)) \
1107     { \
1108         entry->m_length = h; \
1109         entry->m_offset = 0x1; \
1110     } \
1111 \
1112 label: \
1113     if (entry) \
1114     { \
1115         __rtl_memory_resize (entry, (n)); \
1116         *(ppMemory) = entry; \
1117     } \
1118 }
1119 #endif /* OSL_DEBUG_LEVEL || PRODUCT */
1120
1121 #if defined(PROFILE) || (OSL_DEBUG_LEVEL > 0)
1122 #define RTL_MEMORY_DEQUEUE(m, n, l) __rtl_memory_dequeue((m), (n))
1123 #else  /* PRODUCT */
1124 #define RTL_MEMORY_DEQUEUE(m, n, l) __rtl_memory_dequeue(m, n, l)
1125 #endif /* OSL_DEBUG_LEVEL || PRODUCT */
1126
1127 /*
1128  * __rtl_memory_enqueue.
1129  */
1130 #if defined(PROFILE) || (OSL_DEBUG_LEVEL > 0)
1131 static void __rtl_memory_enqueue (memory_type **ppMemory)
1132 {
1133     register memory_type *head = *ppMemory;
1134
1135     OSL_ENSURE(!__rtl_memory_used(head),
1136                "__rtl_memory_enqueue(): "
1137                "internal logic error.");
1138     DBG_MEMORY_ENQUEUE (head->m_length);
1139
1140     /* fill w/ 'deinitialized' pattern */
1141     DBG_MEMORY_ENQFILL (head, __C__, head->m_length - __C__);
1142
1143     /* try merge w/ next entry */
1144     if (!__rtl_memory_last(head))
1145     {
1146         /* not last, check next in chain */
1147         register memory_type * next;
1148
1149         next = queue_cast(head, head->m_length);
1150         DBG_MEMORY_VERIFY_CHAIN (next);
1151
1152         if (!__rtl_memory_used(next))
1153         {
1154             /* next not used */
1155             DBG_MEMORY_VERIFY_QUEUE (next);
1156             queue_remove (next);
1157
1158             /* merge w/ next */
1159             __rtl_memory_merge (head, next);
1160             DBG_MEMORY_ENQFILL (next, 0, sizeof(memory_type));
1161         }
1162     }
1163
1164     /* try merge w/ prev entry */
1165     if (__rtl_memory_offset(head) > 0)
1166     {
1167         /* not first, check prev in chain */
1168         register memory_type * prev;
1169
1170         prev = queue_cast(head, -(__rtl_memory_offset(head)));
1171         DBG_MEMORY_VERIFY_CHAIN (prev);
1172
1173         if (!__rtl_memory_used(prev))
1174         {
1175             /* prev not used */
1176             DBG_MEMORY_VERIFY_QUEUE (prev);
1177             queue_remove (prev);
1178
1179             /* merge w/ prev */
1180             __rtl_memory_merge (prev, head);
1181             DBG_MEMORY_ENQFILL (head, 0, sizeof(memory_type));
1182             head = prev;
1183         }
1184     }
1185
1186     if (!(head->m_offset == 0x1))
1187     {
1188         /* page still used, enqueue */
1189         __rtl_memory_insert (head, head->m_length);
1190         head = 0;
1191     }
1192     else if (head->m_length <= g_memory.m_align)
1193     {
1194         /* small page unused, check spare page */
1195         register memory_type * spare;
1196
1197         spare = &(g_memory.m_spare_head);
1198         if (spare->m_flink == spare)
1199         {
1200             /* keep as spare page */
1201             queue_insert_tail (spare, head);
1202             head = 0;
1203         }
1204     }
1205     if ((*ppMemory = head) != 0)
1206     {
1207         /* page unused, remove */
1208         DBG_MEMORY_REMOVE(ppMemory);
1209     }
1210 }
1211 #else  /* PRODUCT */
1212 #define __rtl_memory_enqueue(ppMemory) \
1213 { \
1214     register memory_type *head = *(ppMemory); \
1215 \
1216     if (!__rtl_memory_last(head)) \
1217     { \
1218         register memory_type * next; \
1219         next = queue_cast(head, head->m_length); \
1220         if (!__rtl_memory_used(next)) \
1221         { \
1222             queue_remove (next); \
1223             __rtl_memory_merge (head, next); \
1224         } \
1225     } \
1226 \
1227     if (__rtl_memory_offset(head) > 0) \
1228     { \
1229         register memory_type * prev; \
1230         prev = queue_cast(head, -(__rtl_memory_offset(head))); \
1231         if (!__rtl_memory_used(prev)) \
1232         { \
1233             queue_remove (prev); \
1234             __rtl_memory_merge (prev, head); \
1235             head = prev; \
1236         } \
1237     } \
1238 \
1239     if (!(head->m_offset == 0x1)) \
1240     { \
1241         register memory_type * used = head; \
1242         __rtl_memory_insert (used, used->m_length); \
1243         head = 0; \
1244     } \
1245     else if (head->m_length <= g_memory.m_align) \
1246     { \
1247         register memory_type * spare; \
1248         spare = &(g_memory.m_spare_head); \
1249         if (spare->m_flink == spare) \
1250         { \
1251             queue_insert_tail (spare, head); \
1252             head = 0; \
1253         } \
1254     } \
1255 \
1256     *(ppMemory) = head; \
1257 }
1258 #endif /* OSL_DEBUG_LEVEL || PRODUCT */
1259
1260 #define RTL_MEMORY_ENQUEUE(m) __rtl_memory_enqueue((m))
1261
1262 #endif /* FORCE_SYSALLOC */
1263
1264 /*===========================================================================
1265  *
1266  * rtl_memory (manager) implementation.
1267  *
1268  *=========================================================================*/
1269 /*
1270  * rtl_reallocateMemory.
1271  */
1272 static
1273 void* SAL_CALL rtl_reallocateMemory_CUSTOM (void * p, sal_Size n) SAL_THROW_EXTERN_C()
1274 {
1275     memory_type * memory;
1276     if (!(!p || !n))
1277     {
1278         /* reallocate */
1279         register sal_Size datlen;
1280
1281         memory = queue_cast(p, -(__C__)); p = 0;
1282         n = RTL_MEMORY_ALIGN(n, __Q__) + __C__;
1283
1284         RTL_MEMORY_ENTER();
1285         DBG_MEMORY_VERIFY(memory);
1286
1287         /* clear 'used' bit */
1288         DBG_MEMORY_ENQUEUE (memory->m_length);
1289         memory->m_length &= ~0x1;
1290
1291         /* amount of data to be moved or copied */
1292         datlen = ((memory->m_length < n) ? memory->m_length : n);
1293
1294         /* try merge w/ next entry */
1295         if (!__rtl_memory_last(memory))
1296         {
1297             /* not last, check next in chain */
1298             register memory_type * next;
1299
1300             next = queue_cast(memory, memory->m_length);
1301             DBG_MEMORY_VERIFY_CHAIN(next);
1302
1303             if (!__rtl_memory_used(next))
1304             {
1305                 /* next not used */
1306                 DBG_MEMORY_VERIFY_QUEUE(next);
1307                 queue_remove (next);
1308
1309                 /* merge w/ next */
1310                 __rtl_memory_merge (memory, next);
1311             }
1312         }
1313
1314         /* try merge w/ prev entry */
1315         if (__rtl_memory_offset(memory) > 0)
1316         {
1317             /* not first, check prev in chain */
1318             register memory_type * prev;
1319
1320             prev = queue_cast(memory, -(__rtl_memory_offset(memory)));
1321             DBG_MEMORY_VERIFY_CHAIN (prev);
1322
1323             if (!__rtl_memory_used(prev))
1324             {
1325                 /* prev not used, try merge, move */
1326                 if ((memory->m_length + prev->m_length) >= n)
1327                 {
1328                     /* prev does fit */
1329                     DBG_MEMORY_VERIFY_QUEUE (prev);
1330                     queue_remove (prev);
1331
1332                     /* merge w/ prev */
1333                     __rtl_memory_merge (prev, memory);
1334
1335                     /* move to prev */
1336                     memmove (
1337                         queue_cast(prev, __C__),
1338                         queue_cast(memory, __C__),
1339                         datlen - __C__);
1340                     memory = prev;
1341                 }
1342             }
1343         }
1344
1345         if (memory->m_length >= n)
1346         {
1347             /* adjust, set 'used' bit */
1348             __rtl_memory_resize (memory, n);
1349             memory->m_length |= 0x1;
1350
1351             /* assign result */
1352             p = queue_cast(memory, __C__);
1353         }
1354         else
1355         {
1356             /* allocate */
1357             memory_type * result = 0;
1358
1359             /* restore 'used' bit */
1360             DBG_MEMORY_DEQUEUE (memory->m_length);
1361             memory->m_length |= 0x80000000;
1362
1363             RTL_MEMORY_DEQUEUE (&result, n, realloc_label_1);
1364             if (result)
1365             {
1366                 /* set 'used' bit */
1367                 result->m_length |= 0x1;
1368
1369                 /* copy */
1370                 memcpy (
1371                     queue_cast(result, __C__),
1372                     queue_cast(memory, __C__),
1373                     datlen - __C__);
1374
1375                 /* clear 'used' bit, enqueue */
1376                 memory->m_length &= 0x7fffffff;
1377                 RTL_MEMORY_ENQUEUE (&memory);
1378                 if (memory)
1379                 {
1380                     /* free memory page */
1381                     RTL_MEMORY_FREE(memory, memory->m_length);
1382                 }
1383
1384                 /* assign result */
1385                 p = queue_cast(result, __C__);
1386             }
1387         }
1388         RTL_MEMORY_LEAVE();
1389     }
1390     else if (!p)
1391     {
1392         /* allocate */
1393         memory = 0;
1394         n = RTL_MEMORY_ALIGN(n, __Q__) + __C__;
1395
1396         RTL_MEMORY_ENTER();
1397         RTL_MEMORY_DEQUEUE (&memory, n, realloc_label_2);
1398         if (memory)
1399         {
1400             /* set 'used' bit */
1401             memory->m_length |= 0x1;
1402
1403             /* assign result */
1404             p = queue_cast(memory, __C__);
1405         }
1406         RTL_MEMORY_LEAVE();
1407     }
1408     else if (!n)
1409     {
1410         /* free */
1411         memory = queue_cast(p, -(__C__)); p = 0;
1412
1413         RTL_MEMORY_ENTER();
1414         DBG_MEMORY_VERIFY(memory);
1415
1416         /* clear 'used' bit, enqueue */
1417         memory->m_length &= ~0x1;
1418
1419         RTL_MEMORY_ENQUEUE (&memory);
1420         if (memory)
1421         {
1422             /* free memory page */
1423             RTL_MEMORY_FREE(memory, memory->m_length);
1424         }
1425         RTL_MEMORY_LEAVE();
1426     }
1427     return (p);
1428 }
1429
1430 static
1431 void* SAL_CALL rtl_reallocateMemory_SYSTEM (void * p, sal_Size n) SAL_THROW_EXTERN_C()
1432 {
1433     return realloc(p, (sal_Size)(n));
1434 }
1435
1436 void* SAL_CALL rtl_reallocateMemory (void * p, sal_Size n) SAL_THROW_EXTERN_C()
1437 {
1438    while (1) {
1439       if (alloc_mode == AMode_CUSTOM) {
1440          return rtl_reallocateMemory_CUSTOM(p,n);
1441       }
1442       if (alloc_mode == AMode_SYSTEM) {
1443          return rtl_reallocateMemory_SYSTEM(p,n);
1444       }
1445       determine_alloc_mode();
1446    }
1447 }
1448
1449
1450
1451 /*
1452  * rtl_allocateMemory.
1453  */
1454 static
1455 void* SAL_CALL rtl_allocateMemory_CUSTOM (sal_Size n) SAL_THROW_EXTERN_C()
1456 {
1457     void * p = 0;
1458     if (n > 0)
1459     {
1460         memory_type * memory = 0;
1461         n = RTL_MEMORY_ALIGN(n, __Q__) + __C__;
1462
1463         RTL_MEMORY_ENTER();
1464         RTL_MEMORY_DEQUEUE (&memory, n, alloc_label);
1465         if (memory)
1466         {
1467             /* set 'used' bit */
1468             memory->m_length |= 0x1;
1469
1470             /* assign result */
1471             p = queue_cast(memory, __C__);
1472         }
1473         RTL_MEMORY_LEAVE();
1474     }
1475     return (p);
1476 }
1477
1478 static
1479 void* SAL_CALL rtl_allocateMemory_SYSTEM (sal_Size n) SAL_THROW_EXTERN_C()
1480 {
1481     return malloc((sal_Size)(n));
1482 }
1483
1484 void* SAL_CALL rtl_allocateMemory (sal_Size n) SAL_THROW_EXTERN_C()
1485 {
1486    while (1) {
1487       if (alloc_mode == AMode_CUSTOM) {
1488          return rtl_allocateMemory_CUSTOM(n);
1489       }
1490       if (alloc_mode == AMode_SYSTEM) {
1491          return rtl_allocateMemory_SYSTEM(n);
1492       }
1493       determine_alloc_mode();
1494    }
1495 }
1496
1497
1498
1499 /*
1500  * rtl_freeMemory.
1501  */
1502 static
1503 void SAL_CALL rtl_freeMemory_CUSTOM (void * p) SAL_THROW_EXTERN_C()
1504 {
1505     if (p)
1506     {
1507         memory_type * memory = queue_cast(p, -(__C__));
1508
1509         RTL_MEMORY_ENTER();
1510         DBG_MEMORY_VERIFY(memory);
1511
1512         /* clear 'used' bit, enqueue */
1513         memory->m_length &= ~0x1;
1514
1515         RTL_MEMORY_ENQUEUE (&memory);
1516         if (memory)
1517         {
1518             /* free memory page */
1519             RTL_MEMORY_FREE(memory, memory->m_length);
1520         }
1521         RTL_MEMORY_LEAVE();
1522     }
1523 }
1524
1525 static
1526 void SAL_CALL rtl_freeMemory_SYSTEM (void * p) SAL_THROW_EXTERN_C()
1527 {
1528     free(p);
1529 }
1530
1531 void SAL_CALL rtl_freeMemory (void * p) SAL_THROW_EXTERN_C()
1532 {
1533    while (1) {
1534       if (alloc_mode == AMode_CUSTOM) {
1535          rtl_freeMemory_CUSTOM(p);
1536          return;
1537       }
1538       if (alloc_mode == AMode_SYSTEM) {
1539          rtl_freeMemory_SYSTEM(p);
1540          return;
1541       }
1542       determine_alloc_mode();
1543    }
1544 }
1545
1546
1547 /*
1548  * rtl_allocateZeroMemory.
1549  */
1550 static
1551 void* SAL_CALL rtl_allocateZeroMemory_CUSTOM (sal_Size n) SAL_THROW_EXTERN_C()
1552 {
1553     void * p = 0;
1554     if (n > 0)
1555     {
1556         memory_type * memory = 0;
1557         n = RTL_MEMORY_ALIGN(n, __Q__) + __C__;
1558
1559         RTL_MEMORY_ENTER();
1560         RTL_MEMORY_DEQUEUE (&memory, n, alloc_label); /* NYI: demand zero */
1561         if (memory)
1562         {
1563             /* zero, set 'used' bit */
1564             memset ((char*)memory + __C__, 0, memory->m_length - __C__);
1565             memory->m_length |= 0x1;
1566
1567             /* assign result */
1568             p = queue_cast(memory, __C__);
1569         }
1570         RTL_MEMORY_LEAVE();
1571     }
1572     return (p);
1573 }
1574
1575 static
1576 void* SAL_CALL rtl_allocateZeroMemory_SYSTEM (sal_Size n) SAL_THROW_EXTERN_C()
1577 {
1578     return calloc((sal_Size)(n), 1);
1579 }
1580
1581 void* SAL_CALL rtl_allocateZeroMemory (sal_Size n) SAL_THROW_EXTERN_C()
1582 {
1583    while (1) {
1584       if (alloc_mode == AMode_CUSTOM) {
1585          return rtl_allocateZeroMemory_CUSTOM(n);
1586       }
1587       if (alloc_mode == AMode_SYSTEM) {
1588          return rtl_allocateZeroMemory_SYSTEM(n);
1589       }
1590       determine_alloc_mode();
1591    }
1592 }
1593
1594
1595 /*
1596  * rtl_freeZeroMemory.
1597  */
1598 static
1599 void SAL_CALL rtl_freeZeroMemory_CUSTOM (void * p, sal_Size n) SAL_THROW_EXTERN_C()
1600 {
1601     (void) n; /* unused */
1602     if (p)
1603     {
1604         memory_type * memory = queue_cast(p, -(__C__));
1605
1606         RTL_MEMORY_ENTER();
1607         DBG_MEMORY_VERIFY(memory);
1608
1609         /* clear 'used' bit, zero, enqueue */
1610         memory->m_length &= ~0x1;
1611         memset ((char*)memory + __C__, 0, memory->m_length - __C__);
1612
1613         RTL_MEMORY_ENQUEUE (&memory); /* NYI: demand zero */
1614         if (memory)
1615         {
1616             /* free memory page */
1617             RTL_MEMORY_FREE(memory, memory->m_length);
1618         }
1619         RTL_MEMORY_LEAVE();
1620     }
1621 }
1622
1623 static
1624 void SAL_CALL rtl_freeZeroMemory_SYSTEM (void * p, sal_Size n) SAL_THROW_EXTERN_C()
1625 {
1626     if (p)
1627     {
1628         memset(p, 0, n);
1629         free(p);
1630     }
1631 }
1632
1633 void SAL_CALL rtl_freeZeroMemory (void * p, sal_Size n) SAL_THROW_EXTERN_C()
1634 {
1635    while (1) {
1636       if (alloc_mode == AMode_CUSTOM) {
1637          rtl_freeZeroMemory_CUSTOM(p,n);
1638          return;
1639       }
1640       if (alloc_mode == AMode_SYSTEM) {
1641          rtl_freeZeroMemory_SYSTEM(p,n);
1642          return;
1643       }
1644       determine_alloc_mode();
1645    }
1646 }
1647
1648 /*===========================================================================
1649  *
1650  * The End.
1651  *
1652  *=========================================================================*/