usr/src/lib/libumem/common/umem_impl.h

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License, Version 1.0 only
   6  * (the "License").  You may not use this file except in compliance
   7  * with the License.
   8  *
   9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  10  * or http://www.opensolaris.org/os/licensing.
  11  * See the License for the specific language governing permissions
  12  * and limitations under the License.
  13  *
  14  * When distributing Covered Code, include this CDDL HEADER in each
  15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  16  * If applicable, add the following below this CDDL HEADER, with the
  17  * fields enclosed by brackets "[]" replaced with your own identifying
  18  * information: Portions Copyright [yyyy] [name of copyright owner]
  19  *
  20  * CDDL HEADER END
  21  */
  22 /*
  23  * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26
  27 /*
  28  * Copyright (c) 2012 Joyent, Inc.  All rights reserved.
  29  * Copyright (c) 2015 by Delphix. All rights reserved.
  30  */
  31
  32 #ifndef _UMEM_IMPL_H
  33 #define _UMEM_IMPL_H
  34
  35 #include <umem.h>
  36
  37 #include <sys/sysmacros.h>
  38 #include <sys/time.h>
  39 #include <sys/vmem.h>
  40 #include <thread.h>
  41
  42 #ifdef  __cplusplus
  43 extern "C" {
  44 #endif
  45
  46 /*
  47  * umem memory allocator: implementation-private data structures
  48  */
  49
  50 /*
  51  * Internal flags for umem_cache_create
  52  */
  53 #define UMC_QCACHE      0x00100000
  54 #define UMC_INTERNAL    0x80000000
  55
  56 /*
  57  * Cache flags
  58  */
  59 #define UMF_AUDIT       0x00000001      /* transaction auditing */
  60 #define UMF_DEADBEEF    0x00000002      /* deadbeef checking */
  61 #define UMF_REDZONE     0x00000004      /* redzone checking */
  62 #define UMF_CONTENTS    0x00000008      /* freed-buffer content logging */
  63 #define UMF_CHECKSIGNAL 0x00000010      /* abort when in signal context */
  64 #define UMF_NOMAGAZINE  0x00000020      /* disable per-cpu magazines */
  65 #define UMF_FIREWALL    0x00000040      /* put all bufs before unmapped pages */
  66 #define UMF_LITE        0x00000100      /* lightweight debugging */
  67
  68 #define UMF_HASH        0x00000200      /* cache has hash table */
  69 #define UMF_RANDOMIZE   0x00000400      /* randomize other umem_flags */
  70 #define UMF_PTC         0x00000800      /* cache has per-thread caching */
  71
  72 #define UMF_CHECKNULL   0x00001000      /* heap exhaustion checking */
  73
  74 #define UMF_BUFTAG      (UMF_DEADBEEF | UMF_REDZONE)
  75 #define UMF_TOUCH       (UMF_BUFTAG | UMF_LITE | UMF_CONTENTS)
  76 #define UMF_RANDOM      (UMF_TOUCH | UMF_AUDIT | UMF_NOMAGAZINE)
  77 #define UMF_DEBUG       (UMF_RANDOM | UMF_FIREWALL)
  78
  79 #define UMEM_STACK_DEPTH        umem_stack_depth
  80
  81 #define UMEM_FREE_PATTERN               0xdeadbeefdeadbeefULL
  82 #define UMEM_UNINITIALIZED_PATTERN      0xbaddcafebaddcafeULL
  83 #define UMEM_REDZONE_PATTERN            0xfeedfacefeedfaceULL
  84 #define UMEM_REDZONE_BYTE               0xbb
  85
  86 #define UMEM_FATAL_FLAGS        (UMEM_NOFAIL)
  87 #define UMEM_SLEEP_FLAGS        (0)
  88
  89 /*
  90  * Redzone size encodings for umem_alloc() / umem_free().  We encode the
  91  * allocation size, rather than storing it directly, so that umem_free()
  92  * can distinguish frees of the wrong size from redzone violations.
  93  */
  94 #define UMEM_SIZE_ENCODE(x)     (251 * (x) + 1)
  95 #define UMEM_SIZE_DECODE(x)     ((x) / 251)
  96 #define UMEM_SIZE_VALID(x)      ((x) % 251 == 1)
  97
  98 /*
  99  * The bufctl (buffer control) structure keeps some minimal information
 100  * about each buffer: its address, its slab, and its current linkage,
 101  * which is either on the slab's freelist (if the buffer is free), or
 102  * on the cache's buf-to-bufctl hash table (if the buffer is allocated).
 103  * In the case of non-hashed, or "raw", caches (the common case), only
 104  * the freelist linkage is necessary: the buffer address is at a fixed
 105  * offset from the bufctl address, and the slab is at the end of the page.
 106  *
 107  * NOTE: bc_next must be the first field; raw buffers have linkage only.
 108  */
 109 typedef struct umem_bufctl {
 110         struct umem_bufctl      *bc_next;       /* next bufctl struct */
 111         void                    *bc_addr;       /* address of buffer */
 112         struct umem_slab        *bc_slab;       /* controlling slab */
 113 } umem_bufctl_t;
 114
 115 /*
 116  * The UMF_AUDIT version of the bufctl structure.  The beginning of this
 117  * structure must be identical to the normal bufctl structure so that
 118  * pointers are interchangeable.
 119  */
 120
 121 #define UMEM_BUFCTL_AUDIT_SIZE_DEPTH(frames) \
 122         ((size_t)(&((umem_bufctl_audit_t *)0)->bc_stack[frames]))
 123
 124 /*
 125  * umem_bufctl_audits must be allocated from a UMC_NOHASH cache, so we
 126  * require that 2 of them, plus 2 buftags, plus a umem_slab_t, all fit on
 127  * a single page.
 128  *
 129  * For ILP32, this is about 1000 frames.
 130  * For LP64, this is about 490 frames.
 131  */
 132
 133 #define UMEM_BUFCTL_AUDIT_ALIGN 32
 134
 135 #define UMEM_BUFCTL_AUDIT_MAX_SIZE                                      \
 136         (P2ALIGN((PAGESIZE - sizeof (umem_slab_t))/2 -                  \
 137             sizeof (umem_buftag_t), UMEM_BUFCTL_AUDIT_ALIGN))
 138
 139 #define UMEM_MAX_STACK_DEPTH                                            \
 140         ((UMEM_BUFCTL_AUDIT_MAX_SIZE -                                  \
 141             UMEM_BUFCTL_AUDIT_SIZE_DEPTH(0)) / sizeof (uintptr_t))
 142
 143 typedef struct umem_bufctl_audit {
 144         struct umem_bufctl      *bc_next;       /* next bufctl struct */
 145         void                    *bc_addr;       /* address of buffer */
 146         struct umem_slab        *bc_slab;       /* controlling slab */
 147         umem_cache_t            *bc_cache;      /* controlling cache */
 148         hrtime_t                bc_timestamp;   /* transaction time */
 149         thread_t                bc_thread;      /* thread doing transaction */
 150         struct umem_bufctl      *bc_lastlog;    /* last log entry */
 151         void                    *bc_contents;   /* contents at last free */
 152         int                     bc_depth;       /* stack depth */
 153         uintptr_t               bc_stack[1];    /* pc stack */
 154 } umem_bufctl_audit_t;
 155
 156 #define UMEM_LOCAL_BUFCTL_AUDIT(bcpp)                                   \
 157                 *(bcpp) = (umem_bufctl_audit_t *)                       \
 158                     alloca(UMEM_BUFCTL_AUDIT_SIZE)
 159
 160 #define UMEM_BUFCTL_AUDIT_SIZE                                          \
 161         UMEM_BUFCTL_AUDIT_SIZE_DEPTH(UMEM_STACK_DEPTH)
 162
 163 /*
 164  * A umem_buftag structure is appended to each buffer whenever any of the
 165  * UMF_BUFTAG flags (UMF_DEADBEEF, UMF_REDZONE, UMF_VERIFY) are set.
 166  */
 167 typedef struct umem_buftag {
 168         uint64_t                bt_redzone;     /* 64-bit redzone pattern */
 169         umem_bufctl_t           *bt_bufctl;     /* bufctl */
 170         intptr_t                bt_bxstat;      /* bufctl ^ (alloc/free) */
 171 } umem_buftag_t;
 172
 173 #define UMEM_BUFTAG(cp, buf)            \
 174         ((umem_buftag_t *)((char *)(buf) + (cp)->cache_buftag))
 175
 176 #define UMEM_BUFCTL(cp, buf)            \
 177         ((umem_bufctl_t *)((char *)(buf) + (cp)->cache_bufctl))
 178
 179 #define UMEM_BUF(cp, bcp)               \
 180         ((void *)((char *)(bcp) - (cp)->cache_bufctl))
 181
 182 #define UMEM_SLAB(cp, buf)              \
 183         ((umem_slab_t *)P2END((uintptr_t)(buf), (cp)->cache_slabsize) - 1)
 184
 185 #define UMEM_CPU_CACHE(cp, cpu)         \
 186         (umem_cpu_cache_t *)((char *)cp + cpu->cpu_cache_offset)
 187
 188 #define UMEM_MAGAZINE_VALID(cp, mp)     \
 189         (((umem_slab_t *)P2END((uintptr_t)(mp), PAGESIZE) - 1)->slab_cache == \
 190             (cp)->cache_magtype->mt_cache)
 191
 192 #define UMEM_SLAB_MEMBER(sp, buf)       \
 193         ((size_t)(buf) - (size_t)(sp)->slab_base < \
 194             (sp)->slab_cache->cache_slabsize)
 195
 196 #define UMEM_BUFTAG_ALLOC       0xa110c8edUL
 197 #define UMEM_BUFTAG_FREE        0xf4eef4eeUL
 198
 199 typedef struct umem_slab {
 200         struct umem_cache       *slab_cache;    /* controlling cache */
 201         void                    *slab_base;     /* base of allocated memory */
 202         struct umem_slab        *slab_next;     /* next slab on freelist */
 203         struct umem_slab        *slab_prev;     /* prev slab on freelist */
 204         struct umem_bufctl      *slab_head;     /* first free buffer */
 205         long                    slab_refcnt;    /* outstanding allocations */
 206         long                    slab_chunks;    /* chunks (bufs) in this slab */
 207 } umem_slab_t;
 208
 209 #define UMEM_HASH_INITIAL       64
 210
 211 #define UMEM_HASH(cp, buf)      \
 212         ((cp)->cache_hash_table +       \
 213         (((uintptr_t)(buf) >> (cp)->cache_hash_shift) & (cp)->cache_hash_mask))
 214
 215 typedef struct umem_magazine {
 216         void    *mag_next;
 217         void    *mag_round[1];          /* one or more rounds */
 218 } umem_magazine_t;
 219
 220 /*
 221  * The magazine types for fast per-cpu allocation
 222  */
 223 typedef struct umem_magtype {
 224         int             mt_magsize;     /* magazine size (number of rounds) */
 225         int             mt_align;       /* magazine alignment */
 226         size_t          mt_minbuf;      /* all smaller buffers qualify */
 227         size_t          mt_maxbuf;      /* no larger buffers qualify */
 228         umem_cache_t    *mt_cache;      /* magazine cache */
 229 } umem_magtype_t;
 230
 231 #define UMEM_CPU_CACHE_SIZE     64      /* must be power of 2 */
 232 #define UMEM_CPU_PAD            (UMEM_CPU_CACHE_SIZE - sizeof (mutex_t) - \
 233         2 * sizeof (uint_t) - 2 * sizeof (void *) - 4 * sizeof (int))
 234 #define UMEM_CACHE_SIZE(ncpus)  \
 235         ((size_t)(&((umem_cache_t *)0)->cache_cpu[ncpus]))
 236
 237 typedef struct umem_cpu_cache {
 238         mutex_t         cc_lock;        /* protects this cpu's local cache */
 239         uint_t          cc_alloc;       /* allocations from this cpu */
 240         uint_t          cc_free;        /* frees to this cpu */
 241         umem_magazine_t *cc_loaded;     /* the currently loaded magazine */
 242         umem_magazine_t *cc_ploaded;    /* the previously loaded magazine */
 243         int             cc_rounds;      /* number of objects in loaded mag */
 244         int             cc_prounds;     /* number of objects in previous mag */
 245         int             cc_magsize;     /* number of rounds in a full mag */
 246         int             cc_flags;       /* CPU-local copy of cache_flags */
 247 #ifndef _LP64
 248         char            cc_pad[UMEM_CPU_PAD]; /* for nice alignment (32-bit) */
 249 #endif
 250 } umem_cpu_cache_t;
 251
 252 /*
 253  * The magazine lists used in the depot.
 254  */
 255 typedef struct umem_maglist {
 256         umem_magazine_t *ml_list;       /* magazine list */
 257         long            ml_total;       /* number of magazines */
 258         long            ml_min;         /* min since last update */
 259         long            ml_reaplimit;   /* max reapable magazines */
 260         uint64_t        ml_alloc;       /* allocations from this list */
 261 } umem_maglist_t;
 262
 263 #define UMEM_CACHE_NAMELEN      31
 264
 265 struct umem_cache {
 266         /*
 267          * Statistics
 268          */
 269         uint64_t        cache_slab_create;      /* slab creates */
 270         uint64_t        cache_slab_destroy;     /* slab destroys */
 271         uint64_t        cache_slab_alloc;       /* slab layer allocations */
 272         uint64_t        cache_slab_free;        /* slab layer frees */
 273         uint64_t        cache_alloc_fail;       /* total failed allocations */
 274         uint64_t        cache_buftotal;         /* total buffers */
 275         uint64_t        cache_bufmax;           /* max buffers ever */
 276         uint64_t        cache_rescale;          /* # of hash table rescales */
 277         uint64_t        cache_lookup_depth;     /* hash lookup depth */
 278         uint64_t        cache_depot_contention; /* mutex contention count */
 279         uint64_t        cache_depot_contention_prev; /* previous snapshot */
 280
 281         /*
 282          * Cache properties
 283          */
 284         char            cache_name[UMEM_CACHE_NAMELEN + 1];
 285         size_t          cache_bufsize;          /* object size */
 286         size_t          cache_align;            /* object alignment */
 287         umem_constructor_t *cache_constructor;
 288         umem_destructor_t *cache_destructor;
 289         umem_reclaim_t  *cache_reclaim;
 290         void            *cache_private;         /* opaque arg to callbacks */
 291         vmem_t          *cache_arena;           /* vmem source for slabs */
 292         int             cache_cflags;           /* cache creation flags */
 293         int             cache_flags;            /* various cache state info */
 294         int             cache_uflags;           /* UMU_* flags */
 295         uint32_t        cache_mtbf;             /* induced alloc failure rate */
 296         umem_cache_t    *cache_next;            /* forward cache linkage */
 297         umem_cache_t    *cache_prev;            /* backward cache linkage */
 298         umem_cache_t    *cache_unext;           /* next in update list */
 299         umem_cache_t    *cache_uprev;           /* prev in update list */
 300         uint32_t        cache_cpu_mask;         /* mask for cpu offset */
 301
 302         /*
 303          * Slab layer
 304          */
 305         mutex_t         cache_lock;             /* protects slab layer */
 306         size_t          cache_chunksize;        /* buf + alignment [+ debug] */
 307         size_t          cache_slabsize;         /* size of a slab */
 308         size_t          cache_bufctl;           /* buf-to-bufctl distance */
 309         size_t          cache_buftag;           /* buf-to-buftag distance */
 310         size_t          cache_verify;           /* bytes to verify */
 311         size_t          cache_contents;         /* bytes of saved content */
 312         size_t          cache_color;            /* next slab color */
 313         size_t          cache_mincolor;         /* maximum slab color */
 314         size_t          cache_maxcolor;         /* maximum slab color */
 315         size_t          cache_hash_shift;       /* get to interesting bits */
 316         size_t          cache_hash_mask;        /* hash table mask */
 317         umem_slab_t     *cache_freelist;        /* slab free list */
 318         umem_slab_t     cache_nullslab;         /* end of freelist marker */
 319         umem_cache_t    *cache_bufctl_cache;    /* source of bufctls */
 320         umem_bufctl_t   **cache_hash_table;     /* hash table base */
 321         /*
 322          * Depot layer
 323          */
 324         mutex_t         cache_depot_lock;       /* protects depot */
 325         umem_magtype_t  *cache_magtype;         /* magazine type */
 326         umem_maglist_t  cache_full;             /* full magazines */
 327         umem_maglist_t  cache_empty;            /* empty magazines */
 328
 329         /*
 330          * Per-CPU layer
 331          */
 332         umem_cpu_cache_t cache_cpu[1];          /* cache_cpu_mask + 1 entries */
 333 };
 334
 335 typedef struct umem_cpu_log_header {
 336         mutex_t         clh_lock;
 337         char            *clh_current;
 338         size_t          clh_avail;
 339         int             clh_chunk;
 340         int             clh_hits;
 341         char            clh_pad[64 - sizeof (mutex_t) - sizeof (char *) -
 342                                 sizeof (size_t) - 2 * sizeof (int)];
 343 } umem_cpu_log_header_t;
 344
 345 typedef struct umem_log_header {
 346         mutex_t         lh_lock;
 347         char            *lh_base;
 348         int             *lh_free;
 349         size_t          lh_chunksize;
 350         int             lh_nchunks;
 351         int             lh_head;
 352         int             lh_tail;
 353         int             lh_hits;
 354         umem_cpu_log_header_t lh_cpu[1];        /* actually umem_max_ncpus */
 355 } umem_log_header_t;
 356
 357 typedef struct umem_cpu {
 358         uint32_t cpu_cache_offset;
 359         uint32_t cpu_number;
 360 } umem_cpu_t;
 361
 362 #define UMEM_MAXBUF     131072
 363
 364 #define UMEM_ALIGN              8       /* min guaranteed alignment */
 365 #define UMEM_ALIGN_SHIFT        3       /* log2(UMEM_ALIGN) */
 366 #define UMEM_VOID_FRACTION      8       /* never waste more than 1/8 of slab */
 367
 368 /*
 369  * For 64 bits, buffers >= 16 bytes must be 16-byte aligned
 370  */
 371 #ifdef _LP64
 372 #define UMEM_SECOND_ALIGN 16
 373 #else
 374 #define UMEM_SECOND_ALIGN UMEM_ALIGN
 375 #endif
 376
 377 #define MALLOC_MAGIC                    0x3a10c000 /* 8-byte tag */
 378 #define MEMALIGN_MAGIC                  0x3e3a1000
 379
 380 #ifdef _LP64
 381 #define MALLOC_SECOND_MAGIC             0x16ba7000 /* 8-byte tag, 16-aligned */
 382 #define MALLOC_OVERSIZE_MAGIC           0x06e47000 /* 16-byte tag, _LP64 */
 383 #endif
 384
 385 #define UMEM_MALLOC_ENCODE(type, sz)    (uint32_t)((type) - (sz))
 386 #define UMEM_MALLOC_DECODE(stat, sz)    (uint32_t)((stat) + (sz))
 387 #define UMEM_FREE_PATTERN_32            (uint32_t)(UMEM_FREE_PATTERN)
 388
 389 #define UMU_MAGAZINE_RESIZE     0x00000001
 390 #define UMU_HASH_RESCALE        0x00000002
 391 #define UMU_REAP                0x00000004
 392 #define UMU_NOTIFY              0x08000000
 393 #define UMU_ACTIVE              0x80000000
 394
 395 #define UMEM_READY_INIT_FAILED          -1
 396 #define UMEM_READY_STARTUP              1
 397 #define UMEM_READY_INITING              2
 398 #define UMEM_READY                      3
 399
 400 #ifdef UMEM_STANDALONE
 401 extern void umem_startup(caddr_t, size_t, size_t, caddr_t, caddr_t);
 402 extern int umem_add(caddr_t, size_t);
 403 #endif
 404
 405 /*
 406  * Private interface with libc for tcumem.
 407  */
 408 extern uintptr_t _tmem_get_base(void);
 409 extern int _tmem_get_nentries(void);
 410 extern void _tmem_set_cleanup(void(*)(void *, int));
 411
 412 #ifdef  __cplusplus
 413 }
 414 #endif
 415
 416 #endif  /* _UMEM_IMPL_H */