include/linux/slqb_def.h

   1 #ifndef _LINUX_SLQB_DEF_H
   2 #define _LINUX_SLQB_DEF_H
   3
   4 /*
   5  * SLQB : A slab allocator with object queues.
   6  *
   7  * (C) 2008 Nick Piggin <npiggin@suse.de>
   8  */
   9 #include <linux/types.h>
  10 #include <linux/gfp.h>
  11 #include <linux/workqueue.h>
  12 #include <linux/kobject.h>
  13 #include <linux/rcu_types.h>
  14 #include <linux/mm_types.h>
  15 #include <linux/kernel.h>
  16 #include <linux/kobject.h>
  17
  18 #define SLAB_NUMA               0x00000001UL    /* shortcut */
  19
  20 enum stat_item {
  21         ALLOC,                  /* Allocation count */
  22         ALLOC_SLAB_FILL,        /* Fill freelist from page list */
  23         ALLOC_SLAB_NEW,         /* New slab acquired from page allocator */
  24         FREE,                   /* Free count */
  25         FREE_REMOTE,            /* NUMA: freeing to remote list */
  26         FLUSH_FREE_LIST,        /* Freelist flushed */
  27         FLUSH_FREE_LIST_OBJECTS, /* Objects flushed from freelist */
  28         FLUSH_FREE_LIST_REMOTE, /* Objects flushed from freelist to remote */
  29         FLUSH_SLAB_PARTIAL,     /* Freeing moves slab to partial list */
  30         FLUSH_SLAB_FREE,        /* Slab freed to the page allocator */
  31         FLUSH_RFREE_LIST,       /* Rfree list flushed */
  32         FLUSH_RFREE_LIST_OBJECTS, /* Rfree objects flushed */
  33         CLAIM_REMOTE_LIST,      /* Remote freed list claimed */
  34         CLAIM_REMOTE_LIST_OBJECTS, /* Remote freed objects claimed */
  35         NR_SLQB_STAT_ITEMS
  36 };
  37
  38 /*
  39  * Singly-linked list with head, tail, and nr
  40  */
  41 struct kmlist {
  42         unsigned long   nr;
  43         void            **head;
  44         void            **tail;
  45 };
  46
  47 /*
  48  * Every kmem_cache_list has a kmem_cache_remote_free structure, by which
  49  * objects can be returned to the kmem_cache_list from remote CPUs.
  50  */
  51 struct kmem_cache_remote_free {
  52         spinlock_t      lock;
  53         struct kmlist   list;
  54 } ____cacheline_aligned;
  55
  56 /*
  57  * A kmem_cache_list manages all the slabs and objects allocated from a given
  58  * source. Per-cpu kmem_cache_lists allow node-local allocations. Per-node
  59  * kmem_cache_lists allow off-node allocations (but require locking).
  60  */
  61 struct kmem_cache_list {
  62                                 /* Fastpath LIFO freelist of objects */
  63         struct kmlist           freelist;
  64 #ifdef CONFIG_SMP
  65                                 /* remote_free has reached a watermark */
  66         int                     remote_free_check;
  67 #endif
  68                                 /* kmem_cache corresponding to this list */
  69         struct kmem_cache       *cache;
  70
  71                                 /* Number of partial slabs (pages) */
  72         unsigned long           nr_partial;
  73
  74                                 /* Slabs which have some free objects */
  75         struct list_head        partial;
  76
  77                                 /* Total number of slabs allocated */
  78         unsigned long           nr_slabs;
  79
  80                                 /* Protects nr_partial, nr_slabs, and partial */
  81         spinlock_t              page_lock;
  82
  83 #ifdef CONFIG_SMP
  84         /*
  85          * In the case of per-cpu lists, remote_free is for objects freed by
  86          * non-owner CPU back to its home list. For per-node lists, remote_free
  87          * is always used to free objects.
  88          */
  89         struct kmem_cache_remote_free remote_free;
  90 #endif
  91
  92 #ifdef CONFIG_SLQB_STATS
  93         unsigned long           stats[NR_SLQB_STAT_ITEMS];
  94 #endif
  95 } ____cacheline_aligned;
  96
  97 /*
  98  * Primary per-cpu, per-kmem_cache structure.
  99  */
 100 struct kmem_cache_cpu {
 101         struct kmem_cache_list  list;           /* List for node-local slabs */
 102         unsigned int            colour_next;    /* Next colour offset to use */
 103
 104 #ifdef CONFIG_SMP
 105         /*
 106          * rlist is a list of objects that don't fit on list.freelist (ie.
 107          * wrong node). The objects all correspond to a given kmem_cache_list,
 108          * remote_cache_list. To free objects to another list, we must first
 109          * flush the existing objects, then switch remote_cache_list.
 110          *
 111          * An NR_CPUS or MAX_NUMNODES array would be nice here, but then we
 112          * get to O(NR_CPUS^2) memory consumption situation.
 113          */
 114         struct kmlist           rlist;
 115         struct kmem_cache_list  *remote_cache_list;
 116 #endif
 117 } ____cacheline_aligned_in_smp;
 118
 119 /*
 120  * Per-node, per-kmem_cache structure. Used for node-specific allocations.
 121  */
 122 struct kmem_cache_node {
 123         struct kmem_cache_list  list;
 124         spinlock_t              list_lock;      /* protects access to list */
 125 } ____cacheline_aligned;
 126
 127 /*
 128  * Management object for a slab cache.
 129  */
 130 struct kmem_cache {
 131         unsigned long   flags;
 132         int             hiwater;        /* LIFO list high watermark */
 133         int             freebatch;      /* LIFO freelist batch flush size */
 134 #ifdef CONFIG_SMP
 135         struct kmem_cache_cpu   **cpu_slab; /* dynamic per-cpu structures */
 136 #else
 137         struct kmem_cache_cpu   cpu_slab;
 138 #endif
 139         int             objsize;        /* Size of object without meta data */
 140         int             offset;         /* Free pointer offset. */
 141         int             objects;        /* Number of objects in slab */
 142
 143 #ifdef CONFIG_NUMA
 144         struct kmem_cache_node  **node_slab; /* dynamic per-node structures */
 145 #endif
 146
 147         int             size;           /* Size of object including meta data */
 148         int             order;          /* Allocation order */
 149         gfp_t           allocflags;     /* gfp flags to use on allocation */
 150         unsigned int    colour_range;   /* range of colour counter */
 151         unsigned int    colour_off;     /* offset per colour */
 152         void            (*ctor)(void *);
 153
 154         const char      *name;          /* Name (only for display!) */
 155         struct list_head list;          /* List of slab caches */
 156
 157         int             align;          /* Alignment */
 158         int             inuse;          /* Offset to metadata */
 159
 160 #ifdef CONFIG_SLQB_SYSFS
 161         struct kobject  kobj;           /* For sysfs */
 162 #endif
 163 } ____cacheline_aligned;
 164
 165 /*
 166  * Kmalloc subsystem.
 167  */
 168 #if defined(ARCH_KMALLOC_MINALIGN) && ARCH_KMALLOC_MINALIGN > 8
 169 #define KMALLOC_MIN_SIZE ARCH_KMALLOC_MINALIGN
 170 #else
 171 #define KMALLOC_MIN_SIZE 8
 172 #endif
 173
 174 #define KMALLOC_SHIFT_LOW ilog2(KMALLOC_MIN_SIZE)
 175 #define KMALLOC_SHIFT_SLQB_HIGH (PAGE_SHIFT +                   \
 176                                  ((9 <= (MAX_ORDER - 1)) ? 9 : (MAX_ORDER - 1)))
 177
 178 extern struct kmem_cache kmalloc_caches[KMALLOC_SHIFT_SLQB_HIGH + 1];
 179 extern struct kmem_cache kmalloc_caches_dma[KMALLOC_SHIFT_SLQB_HIGH + 1];
 180
 181 /*
 182  * Constant size allocations use this path to find index into kmalloc caches
 183  * arrays. get_slab() function is used for non-constant sizes.
 184  */
 185 static __always_inline int kmalloc_index(size_t size)
 186 {
 187         extern int ____kmalloc_too_large(void);
 188
 189         if (unlikely(size <= KMALLOC_MIN_SIZE))
 190                 return KMALLOC_SHIFT_LOW;
 191
 192 #if L1_CACHE_BYTES < 64
 193         if (size > 64 && size <= 96)
 194                 return 1;
 195 #endif
 196 #if L1_CACHE_BYTES < 128
 197         if (size > 128 && size <= 192)
 198                 return 2;
 199 #endif
 200         if (size <=       8) return 3;
 201         if (size <=      16) return 4;
 202         if (size <=      32) return 5;
 203         if (size <=      64) return 6;
 204         if (size <=     128) return 7;
 205         if (size <=     256) return 8;
 206         if (size <=     512) return 9;
 207         if (size <=       1024) return 10;
 208         if (size <=   2 * 1024) return 11;
 209         if (size <=   4 * 1024) return 12;
 210         if (size <=   8 * 1024) return 13;
 211         if (size <=  16 * 1024) return 14;
 212         if (size <=  32 * 1024) return 15;
 213         if (size <=  64 * 1024) return 16;
 214         if (size <= 128 * 1024) return 17;
 215         if (size <= 256 * 1024) return 18;
 216         if (size <= 512 * 1024) return 19;
 217         if (size <= 1024 * 1024) return 20;
 218         if (size <=  2 * 1024 * 1024) return 21;
 219         if (size <=  4 * 1024 * 1024) return 22;
 220         if (size <=  8 * 1024 * 1024) return 23;
 221         if (size <=  16 * 1024 * 1024) return 24;
 222         if (size <=  32 * 1024 * 1024) return 25;
 223         return ____kmalloc_too_large();
 224 }
 225
 226 #ifdef CONFIG_ZONE_DMA
 227 #define SLQB_DMA __GFP_DMA
 228 #else
 229 /* Disable "DMA slabs" */
 230 #define SLQB_DMA (__force gfp_t)0
 231 #endif
 232
 233 /*
 234  * Find the kmalloc slab cache for a given combination of allocation flags and
 235  * size. Should really only be used for constant 'size' arguments, due to
 236  * bloat.
 237  */
 238 static __always_inline struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags)
 239 {
 240         int index;
 241
 242         if (unlikely(size > 1UL << KMALLOC_SHIFT_SLQB_HIGH))
 243                 return NULL;
 244         if (unlikely(!size))
 245                 return ZERO_SIZE_PTR;
 246
 247         index = kmalloc_index(size);
 248         if (likely(!(flags & SLQB_DMA)))
 249                 return &kmalloc_caches[index];
 250         else
 251                 return &kmalloc_caches_dma[index];
 252 }
 253
 254 void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
 255 void *__kmalloc(size_t size, gfp_t flags);
 256
 257 #ifndef ARCH_KMALLOC_MINALIGN
 258 #define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long)
 259 #endif
 260
 261 #ifndef ARCH_SLAB_MINALIGN
 262 #define ARCH_SLAB_MINALIGN __alignof__(unsigned long long)
 263 #endif
 264
 265 #define KMALLOC_HEADER (ARCH_KMALLOC_MINALIGN < sizeof(void *) ?        \
 266                                 sizeof(void *) : ARCH_KMALLOC_MINALIGN)
 267
 268 static __always_inline void *kmalloc(size_t size, gfp_t flags)
 269 {
 270         if (__builtin_constant_p(size)) {
 271                 struct kmem_cache *s;
 272
 273                 s = kmalloc_slab(size, flags);
 274                 if (unlikely(ZERO_OR_NULL_PTR(s)))
 275                         return s;
 276
 277                 return kmem_cache_alloc(s, flags);
 278         }
 279         return __kmalloc(size, flags);
 280 }
 281
 282 #ifdef CONFIG_NUMA
 283 void *__kmalloc_node(size_t size, gfp_t flags, int node);
 284 void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
 285
 286 static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
 287 {
 288         if (__builtin_constant_p(size)) {
 289                 struct kmem_cache *s;
 290
 291                 s = kmalloc_slab(size, flags);
 292                 if (unlikely(ZERO_OR_NULL_PTR(s)))
 293                         return s;
 294
 295                 return kmem_cache_alloc_node(s, flags, node);
 296         }
 297         return __kmalloc_node(size, flags, node);
 298 }
 299 #endif
 300
 301 #endif /* _LINUX_SLQB_DEF_H */