On Tue, Nov 06, 2007 at 02:33:53AM -0800, akpm@linux-foundation.org wrote:

[mmotm.git] / mm / slqb.c

/*
 * SLQB: A slab allocator that focuses on per-CPU scaling, and good performance
 * with order-0 allocations. Fastpaths emphasis is placed on local allocaiton
 * and freeing, but with a secondary goal of good remote freeing (freeing on
 * another CPU from that which allocated).
 *
 * Using ideas and code from mm/slab.c, mm/slob.c, and mm/slub.c.
 */

#include <linux/mm.h>
#include <linux/swap.h> /* struct reclaim_state */
#include <linux/module.h>
#include <linux/interrupt.h>
#include <linux/slab.h>
#include <linux/seq_file.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/mempolicy.h>
#include <linux/ctype.h>
#include <linux/kallsyms.h>
#include <linux/memory.h>
#include <linux/fault-inject.h>

/*
 * TODO
 * - fix up releasing of offlined data structures. Not a big deal because
 *   they don't get cumulatively leaked with successive online/offline cycles
 * - allow OOM conditions to flush back per-CPU pages to common lists to be
 *   reused by other CPUs.
 * - investiage performance with memoryless nodes. Perhaps CPUs can be given
 *   a default closest home node via which it can use fastpath functions.
 *   Perhaps it is not a big problem.
 */

/*
 * slqb_page overloads struct page, and is used to manage some slob allocation
 * aspects, however to avoid the horrible mess in include/linux/mm_types.h,
 * we'll just define our own struct slqb_page type variant here.
 */
struct slqb_page {
        union {
                struct {
                        unsigned long   flags;          /* mandatory */
                        atomic_t        _count;         /* mandatory */
                        unsigned int    inuse;          /* Nr of objects */
                        struct kmem_cache_list *list;   /* Pointer to list */
                        void             **freelist;    /* LIFO freelist */
                        union {
                                struct list_head lru;   /* misc. list */
                                struct rcu_head rcu_head; /* for rcu freeing */
                        };
                };
                struct page page;
        };
};
static inline void struct_slqb_page_wrong_size(void)
{ BUILD_BUG_ON(sizeof(struct slqb_page) != sizeof(struct page)); }

#define PG_SLQB_BIT (1 << PG_slab)

/*
 * slqb_min_order: minimum allocation order for slabs
 */
static int slqb_min_order;

/*
 * slqb_min_objects: minimum number of objects per slab. Increasing this
 * will increase the allocation order for slabs with larger objects
 */
static int slqb_min_objects = 1;

#ifdef CONFIG_NUMA
static inline int slab_numa(struct kmem_cache *s)
{
        return s->flags & SLAB_NUMA;
}
#else
static inline int slab_numa(struct kmem_cache *s)
{
        return 0;
}
#endif

static inline int slab_hiwater(struct kmem_cache *s)
{
        return s->hiwater;
}

static inline int slab_freebatch(struct kmem_cache *s)
{
        return s->freebatch;
}

/*
 * Lock order:
 * kmem_cache_node->list_lock
 *   kmem_cache_remote_free->lock
 *
 * Data structures:
 * SLQB is primarily per-cpu. For each kmem_cache, each CPU has:
 *
 * - A LIFO list of node-local objects. Allocation and freeing of node local
 *   objects goes first to this list.
 *
 * - 2 Lists of slab pages, free and partial pages. If an allocation misses
 *   the object list, it tries from the partial list, then the free list.
 *   After freeing an object to the object list, if it is over a watermark,
 *   some objects are freed back to pages. If an allocation misses these lists,
 *   a new slab page is allocated from the page allocator. If the free list
 *   reaches a watermark, some of its pages are returned to the page allocator.
 *
 * - A remote free queue, where objects freed that did not come from the local
 *   node are queued to. When this reaches a watermark, the objects are
 *   flushed.
 *
 * - A remotely freed queue, where objects allocated from this CPU are flushed
 *   to from other CPUs' remote free queues. kmem_cache_remote_free->lock is
 *   used to protect access to this queue.
 *
 *   When the remotely freed queue reaches a watermark, a flag is set to tell
 *   the owner CPU to check it. The owner CPU will then check the queue on the
 *   next allocation that misses the object list. It will move all objects from
 *   this list onto the object list and then allocate one.
 *
 *   This system of remote queueing is intended to reduce lock and remote
 *   cacheline acquisitions, and give a cooling off period for remotely freed
 *   objects before they are re-allocated.
 *
 * node specific allocations from somewhere other than the local node are
 * handled by a per-node list which is the same as the above per-CPU data
 * structures except for the following differences:
 *
 * - kmem_cache_node->list_lock is used to protect access for multiple CPUs to
 *   allocate from a given node.
 *
 * - There is no remote free queue. Nodes don't free objects, CPUs do.
 */

static inline void slqb_stat_inc(struct kmem_cache_list *list,
                                enum stat_item si)
{
#ifdef CONFIG_SLQB_STATS
        list->stats[si]++;
#endif
}

static inline void slqb_stat_add(struct kmem_cache_list *list,
                                enum stat_item si, unsigned long nr)
{
#ifdef CONFIG_SLQB_STATS
        list->stats[si] += nr;
#endif
}

static inline int slqb_page_to_nid(struct slqb_page *page)
{
        return page_to_nid(&page->page);
}

static inline void *slqb_page_address(struct slqb_page *page)
{
        return page_address(&page->page);
}

static inline struct zone *slqb_page_zone(struct slqb_page *page)
{
        return page_zone(&page->page);
}

static inline int virt_to_nid(const void *addr)
{
        return page_to_nid(virt_to_page(addr));
}

static inline struct slqb_page *virt_to_head_slqb_page(const void *addr)
{
        struct page *p;

        p = virt_to_head_page(addr);
        return (struct slqb_page *)p;
}

static inline void __free_slqb_pages(struct slqb_page *page, unsigned int order,
                                        int pages)
{
        struct page *p = &page->page;

        reset_page_mapcount(p);
        p->mapping = NULL;
        VM_BUG_ON(!(p->flags & PG_SLQB_BIT));
        p->flags &= ~PG_SLQB_BIT;

        if (current->reclaim_state)
                current->reclaim_state->reclaimed_slab += pages;
        __free_pages(p, order);
}

#ifdef CONFIG_SLQB_DEBUG
static inline int slab_debug(struct kmem_cache *s)
{
        return s->flags &
                        (SLAB_DEBUG_FREE |
                         SLAB_RED_ZONE |
                         SLAB_POISON |
                         SLAB_STORE_USER |
                         SLAB_TRACE);
}
static inline int slab_poison(struct kmem_cache *s)
{
        return s->flags & SLAB_POISON;
}
#else
static inline int slab_debug(struct kmem_cache *s)
{
        return 0;
}
static inline int slab_poison(struct kmem_cache *s)
{
        return 0;
}
#endif

#define DEBUG_DEFAULT_FLAGS (SLAB_DEBUG_FREE | SLAB_RED_ZONE | \
                                SLAB_POISON | SLAB_STORE_USER)

/* Internal SLQB flags */
#define __OBJECT_POISON         0x80000000 /* Poison object */

/* Not all arches define cache_line_size */
#ifndef cache_line_size
#define cache_line_size()       L1_CACHE_BYTES
#endif

#ifdef CONFIG_SMP
static struct notifier_block slab_notifier;
#endif

/*
 * slqb_lock protects slab_caches list and serialises hotplug operations.
 * hotplug operations take lock for write, other operations can hold off
 * hotplug by taking it for read (or write).
 */
static DECLARE_RWSEM(slqb_lock);

/*
 * A list of all slab caches on the system
 */
static LIST_HEAD(slab_caches);

/*
 * Tracking user of a slab.
 */
struct track {
        unsigned long addr;     /* Called from address */
        int cpu;                /* Was running on cpu */
        int pid;                /* Pid context */
        unsigned long when;     /* When did the operation occur */
};

enum track_item { TRACK_ALLOC, TRACK_FREE };

static struct kmem_cache kmem_cache_cache;

#ifdef CONFIG_SLQB_SYSFS
static int sysfs_slab_add(struct kmem_cache *s);
static void sysfs_slab_remove(struct kmem_cache *s);
#else
static inline int sysfs_slab_add(struct kmem_cache *s)
{
        return 0;
}
static inline void sysfs_slab_remove(struct kmem_cache *s)
{
        kmem_cache_free(&kmem_cache_cache, s);
}
#endif

/********************************************************************
 *                      Core slab cache functions
 *******************************************************************/

static int __slab_is_available __read_mostly;
int slab_is_available(void)
{
        return __slab_is_available;
}

static inline struct kmem_cache_cpu *get_cpu_slab(struct kmem_cache *s, int cpu)
{
#ifdef CONFIG_SMP
        VM_BUG_ON(!s->cpu_slab[cpu]);
        return s->cpu_slab[cpu];
#else
        return &s->cpu_slab;
#endif
}

static inline int check_valid_pointer(struct kmem_cache *s,
                                struct slqb_page *page, const void *object)
{
        void *base;

        base = slqb_page_address(page);
        if (object < base || object >= base + s->objects * s->size ||
                (object - base) % s->size) {
                return 0;
        }

        return 1;
}

static inline void *get_freepointer(struct kmem_cache *s, void *object)
{
        return *(void **)(object + s->offset);
}

static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
{
        *(void **)(object + s->offset) = fp;
}

/* Loop over all objects in a slab */
#define for_each_object(__p, __s, __addr) \
        for (__p = (__addr); __p < (__addr) + (__s)->objects * (__s)->size;\
                        __p += (__s)->size)

/* Scan freelist */
#define for_each_free_object(__p, __s, __free) \
        for (__p = (__free); (__p) != NULL; __p = get_freepointer((__s),\
                __p))

#ifdef CONFIG_SLQB_DEBUG
/*
 * Debug settings:
 */
#ifdef CONFIG_SLQB_DEBUG_ON
static int slqb_debug __read_mostly = DEBUG_DEFAULT_FLAGS;
#else
static int slqb_debug __read_mostly;
#endif

static char *slqb_debug_slabs;

/*
 * Object debugging
 */
static void print_section(char *text, u8 *addr, unsigned int length)
{
        int i, offset;
        int newline = 1;
        char ascii[17];

        ascii[16] = 0;

        for (i = 0; i < length; i++) {
                if (newline) {
                        printk(KERN_ERR "%8s 0x%p: ", text, addr + i);
                        newline = 0;
                }
                printk(KERN_CONT " %02x", addr[i]);
                offset = i % 16;
                ascii[offset] = isgraph(addr[i]) ? addr[i] : '.';
                if (offset == 15) {
                        printk(KERN_CONT " %s\n", ascii);
                        newline = 1;
                }
        }
        if (!newline) {
                i %= 16;
                while (i < 16) {
                        printk(KERN_CONT "   ");
                        ascii[i] = ' ';
                        i++;
                }
                printk(KERN_CONT " %s\n", ascii);
        }
}

static struct track *get_track(struct kmem_cache *s, void *object,
        enum track_item alloc)
{
        struct track *p;

        if (s->offset)
                p = object + s->offset + sizeof(void *);
        else
                p = object + s->inuse;

        return p + alloc;
}

static void set_track(struct kmem_cache *s, void *object,
                                enum track_item alloc, unsigned long addr)
{
        struct track *p;

        if (s->offset)
                p = object + s->offset + sizeof(void *);
        else
                p = object + s->inuse;

        p += alloc;
        if (addr) {
                p->addr = addr;
                p->cpu = raw_smp_processor_id();
                p->pid = current ? current->pid : -1;
                p->when = jiffies;
        } else
                memset(p, 0, sizeof(struct track));
}

static void init_tracking(struct kmem_cache *s, void *object)
{
        if (!(s->flags & SLAB_STORE_USER))
                return;

        set_track(s, object, TRACK_FREE, 0UL);
        set_track(s, object, TRACK_ALLOC, 0UL);
}

static void print_track(const char *s, struct track *t)
{
        if (!t->addr)
                return;

        printk(KERN_ERR "INFO: %s in ", s);
        __print_symbol("%s", (unsigned long)t->addr);
        printk(" age=%lu cpu=%u pid=%d\n", jiffies - t->when, t->cpu, t->pid);
}

static void print_tracking(struct kmem_cache *s, void *object)
{
        if (!(s->flags & SLAB_STORE_USER))
                return;

        print_track("Allocated", get_track(s, object, TRACK_ALLOC));
        print_track("Freed", get_track(s, object, TRACK_FREE));
}

static void print_page_info(struct slqb_page *page)
{
        printk(KERN_ERR "INFO: Slab 0x%p used=%u fp=0x%p flags=0x%04lx\n",
                page, page->inuse, page->freelist, page->flags);

}

#define MAX_ERR_STR 100
static void slab_bug(struct kmem_cache *s, char *fmt, ...)
{
        va_list args;
        char buf[MAX_ERR_STR];

        va_start(args, fmt);
        vsnprintf(buf, sizeof(buf), fmt, args);
        va_end(args);
        printk(KERN_ERR "========================================"
                        "=====================================\n");
        printk(KERN_ERR "BUG %s: %s\n", s->name, buf);
        printk(KERN_ERR "----------------------------------------"
                        "-------------------------------------\n\n");
}

static void slab_fix(struct kmem_cache *s, char *fmt, ...)
{
        va_list args;
        char buf[100];

        va_start(args, fmt);
        vsnprintf(buf, sizeof(buf), fmt, args);
        va_end(args);
        printk(KERN_ERR "FIX %s: %s\n", s->name, buf);
}

static void print_trailer(struct kmem_cache *s, struct slqb_page *page, u8 *p)
{
        unsigned int off;       /* Offset of last byte */
        u8 *addr = slqb_page_address(page);

        print_tracking(s, p);

        print_page_info(page);

        printk(KERN_ERR "INFO: Object 0x%p @offset=%tu fp=0x%p\n\n",
                        p, p - addr, get_freepointer(s, p));

        if (p > addr + 16)
                print_section("Bytes b4", p - 16, 16);

        print_section("Object", p, min(s->objsize, 128));

        if (s->flags & SLAB_RED_ZONE)
                print_section("Redzone", p + s->objsize, s->inuse - s->objsize);

        if (s->offset)
                off = s->offset + sizeof(void *);
        else
                off = s->inuse;

        if (s->flags & SLAB_STORE_USER)
                off += 2 * sizeof(struct track);

        if (off != s->size) {
                /* Beginning of the filler is the free pointer */
                print_section("Padding", p + off, s->size - off);
        }

        dump_stack();
}

static void object_err(struct kmem_cache *s, struct slqb_page *page,
                        u8 *object, char *reason)
{
        slab_bug(s, reason);
        print_trailer(s, page, object);
}

static void slab_err(struct kmem_cache *s, struct slqb_page *page,
                        char *fmt, ...)
{
        slab_bug(s, fmt);
        print_page_info(page);
        dump_stack();
}

static void init_object(struct kmem_cache *s, void *object, int active)
{
        u8 *p = object;

        if (s->flags & __OBJECT_POISON) {
                memset(p, POISON_FREE, s->objsize - 1);
                p[s->objsize - 1] = POISON_END;
        }

        if (s->flags & SLAB_RED_ZONE) {
                memset(p + s->objsize,
                        active ? SLUB_RED_ACTIVE : SLUB_RED_INACTIVE,
                        s->inuse - s->objsize);
        }
}

static u8 *check_bytes(u8 *start, unsigned int value, unsigned int bytes)
{
        while (bytes) {
                if (*start != (u8)value)
                        return start;
                start++;
                bytes--;
        }
        return NULL;
}

static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
                                void *from, void *to)
{
        slab_fix(s, "Restoring 0x%p-0x%p=0x%x\n", from, to - 1, data);
        memset(from, data, to - from);
}

static int check_bytes_and_report(struct kmem_cache *s, struct slqb_page *page,
                        u8 *object, char *what,
                        u8 *start, unsigned int value, unsigned int bytes)
{
        u8 *fault;
        u8 *end;

        fault = check_bytes(start, value, bytes);
        if (!fault)
                return 1;

        end = start + bytes;
        while (end > fault && end[-1] == value)
                end--;

        slab_bug(s, "%s overwritten", what);
        printk(KERN_ERR "INFO: 0x%p-0x%p. First byte 0x%x instead of 0x%x\n",
                                        fault, end - 1, fault[0], value);
        print_trailer(s, page, object);

        restore_bytes(s, what, value, fault, end);
        return 0;
}

/*
 * Object layout:
 *
 * object address
 *      Bytes of the object to be managed.
 *      If the freepointer may overlay the object then the free
 *      pointer is the first word of the object.
 *
 *      Poisoning uses 0x6b (POISON_FREE) and the last byte is
 *      0xa5 (POISON_END)
 *
 * object + s->objsize
 *      Padding to reach word boundary. This is also used for Redzoning.
 *      Padding is extended by another word if Redzoning is enabled and
 *      objsize == inuse.
 *
 *      We fill with 0xbb (RED_INACTIVE) for inactive objects and with
 *      0xcc (RED_ACTIVE) for objects in use.
 *
 * object + s->inuse
 *      Meta data starts here.
 *
 *      A. Free pointer (if we cannot overwrite object on free)
 *      B. Tracking data for SLAB_STORE_USER
 *      C. Padding to reach required alignment boundary or at mininum
 *              one word if debuggin is on to be able to detect writes
 *              before the word boundary.
 *
 *      Padding is done using 0x5a (POISON_INUSE)
 *
 * object + s->size
 *      Nothing is used beyond s->size.
 */

static int check_pad_bytes(struct kmem_cache *s, struct slqb_page *page, u8 *p)
{
        unsigned long off = s->inuse;   /* The end of info */

        if (s->offset) {
                /* Freepointer is placed after the object. */
                off += sizeof(void *);
        }

        if (s->flags & SLAB_STORE_USER) {
                /* We also have user information there */
                off += 2 * sizeof(struct track);
        }

        if (s->size == off)
                return 1;

        return check_bytes_and_report(s, page, p, "Object padding",
                                p + off, POISON_INUSE, s->size - off);
}

static int slab_pad_check(struct kmem_cache *s, struct slqb_page *page)
{
        u8 *start;
        u8 *fault;
        u8 *end;
        int length;
        int remainder;

        if (!(s->flags & SLAB_POISON))
                return 1;

        start = slqb_page_address(page);
        end = start + (PAGE_SIZE << s->order);
        length = s->objects * s->size;
        remainder = end - (start + length);
        if (!remainder)
                return 1;

        fault = check_bytes(start + length, POISON_INUSE, remainder);
        if (!fault)
                return 1;

        while (end > fault && end[-1] == POISON_INUSE)
                end--;

        slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1);
        print_section("Padding", start, length);

        restore_bytes(s, "slab padding", POISON_INUSE, start, end);
        return 0;
}

static int check_object(struct kmem_cache *s, struct slqb_page *page,
                                        void *object, int active)
{
        u8 *p = object;
        u8 *endobject = object + s->objsize;

        if (s->flags & SLAB_RED_ZONE) {
                unsigned int red =
                        active ? SLUB_RED_ACTIVE : SLUB_RED_INACTIVE;

                if (!check_bytes_and_report(s, page, object, "Redzone",
                        endobject, red, s->inuse - s->objsize))
                        return 0;
        } else {
                if ((s->flags & SLAB_POISON) && s->objsize < s->inuse) {
                        check_bytes_and_report(s, page, p, "Alignment padding",
                                endobject, POISON_INUSE, s->inuse - s->objsize);
                }
        }

        if (s->flags & SLAB_POISON) {
                if (!active && (s->flags & __OBJECT_POISON)) {
                        if (!check_bytes_and_report(s, page, p, "Poison", p,
                                        POISON_FREE, s->objsize - 1))
                                return 0;

                        if (!check_bytes_and_report(s, page, p, "Poison",
                                        p + s->objsize - 1, POISON_END, 1))
                                return 0;
                }

                /*
                 * check_pad_bytes cleans up on its own.
                 */
                check_pad_bytes(s, page, p);
        }

        return 1;
}

static int check_slab(struct kmem_cache *s, struct slqb_page *page)
{
        if (!(page->flags & PG_SLQB_BIT)) {
                slab_err(s, page, "Not a valid slab page");
                return 0;
        }
        if (page->inuse == 0) {
                slab_err(s, page, "inuse before free / after alloc", s->name);
                return 0;
        }
        if (page->inuse > s->objects) {
                slab_err(s, page, "inuse %u > max %u",
                        s->name, page->inuse, s->objects);
                return 0;
        }
        /* Slab_pad_check fixes things up after itself */
        slab_pad_check(s, page);
        return 1;
}

static void trace(struct kmem_cache *s, struct slqb_page *page,
                        void *object, int alloc)
{
        if (s->flags & SLAB_TRACE) {
                printk(KERN_INFO "TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
                        s->name,
                        alloc ? "alloc" : "free",
                        object, page->inuse,
                        page->freelist);

                if (!alloc)
                        print_section("Object", (void *)object, s->objsize);

                dump_stack();
        }
}

static void setup_object_debug(struct kmem_cache *s, struct slqb_page *page,
                                void *object)
{
        if (!slab_debug(s))
                return;

        if (!(s->flags & (SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON)))
                return;

        init_object(s, object, 0);
        init_tracking(s, object);
}

static int alloc_debug_processing(struct kmem_cache *s,
                                        void *object, unsigned long addr)
{
        struct slqb_page *page;
        page = virt_to_head_slqb_page(object);

        if (!check_slab(s, page))
                goto bad;

        if (!check_valid_pointer(s, page, object)) {
                object_err(s, page, object, "Freelist Pointer check fails");
                goto bad;
        }

        if (object && !check_object(s, page, object, 0))
                goto bad;

        /* Success perform special debug activities for allocs */
        if (s->flags & SLAB_STORE_USER)
                set_track(s, object, TRACK_ALLOC, addr);
        trace(s, page, object, 1);
        init_object(s, object, 1);
        return 1;

bad:
        return 0;
}

static int free_debug_processing(struct kmem_cache *s,
                                        void *object, unsigned long addr)
{
        struct slqb_page *page;
        page = virt_to_head_slqb_page(object);

        if (!check_slab(s, page))
                goto fail;

        if (!check_valid_pointer(s, page, object)) {
                slab_err(s, page, "Invalid object pointer 0x%p", object);
                goto fail;
        }

        if (!check_object(s, page, object, 1))
                return 0;

        /* Special debug activities for freeing objects */
        if (s->flags & SLAB_STORE_USER)
                set_track(s, object, TRACK_FREE, addr);
        trace(s, page, object, 0);
        init_object(s, object, 0);
        return 1;

fail:
        slab_fix(s, "Object at 0x%p not freed", object);
        return 0;
}

static int __init setup_slqb_debug(char *str)
{
        slqb_debug = DEBUG_DEFAULT_FLAGS;
        if (*str++ != '=' || !*str) {
                /*
                 * No options specified. Switch on full debugging.
                 */
                goto out;
        }

        if (*str == ',') {
                /*
                 * No options but restriction on slabs. This means full
                 * debugging for slabs matching a pattern.
                 */
                goto check_slabs;
        }

        slqb_debug = 0;
        if (*str == '-') {
                /*
                 * Switch off all debugging measures.
                 */
                goto out;
        }

        /*
         * Determine which debug features should be switched on
         */
        for (; *str && *str != ','; str++) {
                switch (tolower(*str)) {
                case 'f':
                        slqb_debug |= SLAB_DEBUG_FREE;
                        break;
                case 'z':
                        slqb_debug |= SLAB_RED_ZONE;
                        break;
                case 'p':
                        slqb_debug |= SLAB_POISON;
                        break;
                case 'u':
                        slqb_debug |= SLAB_STORE_USER;
                        break;
                case 't':
                        slqb_debug |= SLAB_TRACE;
                        break;
                default:
                        printk(KERN_ERR "slqb_debug option '%c' "
                                "unknown. skipped\n", *str);
                }
        }

check_slabs:
        if (*str == ',')
                slqb_debug_slabs = str + 1;
out:
        return 1;
}
__setup("slqb_debug", setup_slqb_debug);

static int __init setup_slqb_min_order(char *str)
{
        get_option(&str, &slqb_min_order);
        slqb_min_order = min(slqb_min_order, MAX_ORDER - 1);

        return 1;
}
__setup("slqb_min_order=", setup_slqb_min_order);

static int __init setup_slqb_min_objects(char *str)
{
        get_option(&str, &slqb_min_objects);

        return 1;
}

__setup("slqb_min_objects=", setup_slqb_min_objects);

static unsigned long kmem_cache_flags(unsigned long objsize,
                                unsigned long flags, const char *name,
                                void (*ctor)(void *))
{
        /*
         * Enable debugging if selected on the kernel commandline.
         */
        if (slqb_debug && (!slqb_debug_slabs ||
            strncmp(slqb_debug_slabs, name,
                strlen(slqb_debug_slabs)) == 0))
                        flags |= slqb_debug;

        if (num_possible_nodes() > 1)
                flags |= SLAB_NUMA;

        return flags;
}
#else
static inline void setup_object_debug(struct kmem_cache *s,
                        struct slqb_page *page, void *object)
{
}

static inline int alloc_debug_processing(struct kmem_cache *s,
                        void *object, unsigned long addr)
{
        return 0;
}

static inline int free_debug_processing(struct kmem_cache *s,
                        void *object, unsigned long addr)
{
        return 0;
}

static inline int slab_pad_check(struct kmem_cache *s, struct slqb_page *page)
{
        return 1;
}

static inline int check_object(struct kmem_cache *s, struct slqb_page *page,
                        void *object, int active)
{
        return 1;
}

static inline void add_full(struct kmem_cache_node *n, struct slqb_page *page)
{
}

static inline unsigned long kmem_cache_flags(unsigned long objsize,
        unsigned long flags, const char *name, void (*ctor)(void *))
{
        if (num_possible_nodes() > 1)
                flags |= SLAB_NUMA;
        return flags;
}

static const int slqb_debug;
#endif

/*
 * allocate a new slab (return its corresponding struct slqb_page)
 */
static struct slqb_page *allocate_slab(struct kmem_cache *s,
                                        gfp_t flags, int node)
{
        struct slqb_page *page;
        int pages = 1 << s->order;

        flags |= s->allocflags;

        page = (struct slqb_page *)alloc_pages_node(node, flags, s->order);
        if (!page)
                return NULL;

        mod_zone_page_state(slqb_page_zone(page),
                (s->flags & SLAB_RECLAIM_ACCOUNT) ?
                NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
                pages);

        return page;
}

/*
 * Called once for each object on a new slab page
 */
static void setup_object(struct kmem_cache *s,
                                struct slqb_page *page, void *object)
{
        setup_object_debug(s, page, object);
        if (unlikely(s->ctor))
                s->ctor(object);
}

/*
 * Allocate a new slab, set up its object list.
 */
static struct slqb_page *new_slab_page(struct kmem_cache *s,
                                gfp_t flags, int node, unsigned int colour)
{
        struct slqb_page *page;
        void *start;
        void *last;
        void *p;

        BUG_ON(flags & GFP_SLAB_BUG_MASK);

        page = allocate_slab(s,
                flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
        if (!page)
                goto out;

        page->flags |= PG_SLQB_BIT;

        start = page_address(&page->page);

        if (unlikely(slab_poison(s)))
                memset(start, POISON_INUSE, PAGE_SIZE << s->order);

        start += colour;

        last = start;
        for_each_object(p, s, start) {
                setup_object(s, page, p);
                set_freepointer(s, last, p);
                last = p;
        }
        set_freepointer(s, last, NULL);

        page->freelist = start;
        page->inuse = 0;
out:
        return page;
}

/*
 * Free a slab page back to the page allocator
 */
static void __free_slab(struct kmem_cache *s, struct slqb_page *page)
{
        int pages = 1 << s->order;

        if (unlikely(slab_debug(s))) {
                void *p;

                slab_pad_check(s, page);
                for_each_free_object(p, s, page->freelist)
                        check_object(s, page, p, 0);
        }

        mod_zone_page_state(slqb_page_zone(page),
                (s->flags & SLAB_RECLAIM_ACCOUNT) ?
                NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
                -pages);

        __free_slqb_pages(page, s->order, pages);
}

static void rcu_free_slab(struct rcu_head *h)
{
        struct slqb_page *page;

        page = container_of(h, struct slqb_page, rcu_head);
        __free_slab(page->list->cache, page);
}

static void free_slab(struct kmem_cache *s, struct slqb_page *page)
{
        VM_BUG_ON(page->inuse);
        if (unlikely(s->flags & SLAB_DESTROY_BY_RCU))
                call_rcu(&page->rcu_head, rcu_free_slab);
        else
                __free_slab(s, page);
}

/*
 * Return an object to its slab.
 *
 * Caller must be the owner CPU in the case of per-CPU list, or hold the node's
 * list_lock in the case of per-node list.
 */
static int free_object_to_page(struct kmem_cache *s,
                        struct kmem_cache_list *l, struct slqb_page *page,
                        void *object)
{
        VM_BUG_ON(page->list != l);

        set_freepointer(s, object, page->freelist);
        page->freelist = object;
        page->inuse--;

        if (!page->inuse) {
                if (likely(s->objects > 1)) {
                        l->nr_partial--;
                        list_del(&page->lru);
                }
                l->nr_slabs--;
                free_slab(s, page);
                slqb_stat_inc(l, FLUSH_SLAB_FREE);
                return 1;

        } else if (page->inuse + 1 == s->objects) {
                l->nr_partial++;
                list_add(&page->lru, &l->partial);
                slqb_stat_inc(l, FLUSH_SLAB_PARTIAL);
                return 0;
        }
        return 0;
}

#ifdef CONFIG_SMP
static void slab_free_to_remote(struct kmem_cache *s, struct slqb_page *page,
                                void *object, struct kmem_cache_cpu *c);
#endif

/*
 * Flush the LIFO list of objects on a list. They are sent back to their pages
 * in case the pages also belong to the list, or to our CPU's remote-free list
 * in the case they do not.
 *
 * Doesn't flush the entire list. flush_free_list_all does.
 *
 * Caller must be the owner CPU in the case of per-CPU list, or hold the node's
 * list_lock in the case of per-node list.
 */
static void flush_free_list(struct kmem_cache *s, struct kmem_cache_list *l)
{
        void **head;
        int nr;
        int locked = 0;

        nr = l->freelist.nr;
        if (unlikely(!nr))
                return;

        nr = min(slab_freebatch(s), nr);

        slqb_stat_inc(l, FLUSH_FREE_LIST);
        slqb_stat_add(l, FLUSH_FREE_LIST_OBJECTS, nr);

        l->freelist.nr -= nr;
        head = l->freelist.head;

        do {
                struct slqb_page *page;
                void **object;

                object = head;
                VM_BUG_ON(!object);
                head = get_freepointer(s, object);
                page = virt_to_head_slqb_page(object);

#ifdef CONFIG_SMP
                if (page->list != l) {
                        struct kmem_cache_cpu *c;

                        if (locked) {
                                spin_unlock(&l->page_lock);
                                locked = 0;
                        }

                        c = get_cpu_slab(s, smp_processor_id());

                        slab_free_to_remote(s, page, object, c);
                        slqb_stat_inc(l, FLUSH_FREE_LIST_REMOTE);
                } else
#endif
                {
                        if (!locked) {
                                spin_lock(&l->page_lock);
                                locked = 1;
                        }
                        free_object_to_page(s, l, page, object);
                }

                nr--;
        } while (nr);

        if (locked)
                spin_unlock(&l->page_lock);

        l->freelist.head = head;
        if (!l->freelist.nr)
                l->freelist.tail = NULL;
}

static void flush_free_list_all(struct kmem_cache *s, struct kmem_cache_list *l)
{
        while (l->freelist.nr)
                flush_free_list(s, l);
}

#ifdef CONFIG_SMP
/*
 * If enough objects have been remotely freed back to this list,
 * remote_free_check will be set. In which case, we'll eventually come here
 * to take those objects off our remote_free list and onto our LIFO freelist.
 *
 * Caller must be the owner CPU in the case of per-CPU list, or hold the node's
 * list_lock in the case of per-node list.
 */
static void claim_remote_free_list(struct kmem_cache *s,
                                        struct kmem_cache_list *l)
{
        void **head, **tail;
        int nr;

        if (!l->remote_free.list.nr)
                return;

        spin_lock(&l->remote_free.lock);

        l->remote_free_check = 0;
        head = l->remote_free.list.head;
        l->remote_free.list.head = NULL;
        tail = l->remote_free.list.tail;
        l->remote_free.list.tail = NULL;
        nr = l->remote_free.list.nr;
        l->remote_free.list.nr = 0;

        spin_unlock(&l->remote_free.lock);

        VM_BUG_ON(!nr);

        if (!l->freelist.nr) {
                /* Get head hot for likely subsequent allocation or flush */
                prefetchw(head);
                l->freelist.head = head;
        } else
                set_freepointer(s, l->freelist.tail, head);
        l->freelist.tail = tail;

        l->freelist.nr += nr;

        slqb_stat_inc(l, CLAIM_REMOTE_LIST);
        slqb_stat_add(l, CLAIM_REMOTE_LIST_OBJECTS, nr);
}
#else
static inline void claim_remote_free_list(struct kmem_cache *s,
                                        struct kmem_cache_list *l)
{
}
#endif

/*
 * Allocation fastpath. Get an object from the list's LIFO freelist, or
 * return NULL if it is empty.
 *
 * Caller must be the owner CPU in the case of per-CPU list, or hold the node's
 * list_lock in the case of per-node list.
 */
static __always_inline void *__cache_list_get_object(struct kmem_cache *s,
                                                struct kmem_cache_list *l)
{
        void *object;

        object = l->freelist.head;
        if (likely(object)) {
                void *next = get_freepointer(s, object);

                VM_BUG_ON(!l->freelist.nr);
                l->freelist.nr--;
                l->freelist.head = next;

                return object;
        }
        VM_BUG_ON(l->freelist.nr);

#ifdef CONFIG_SMP
        if (unlikely(l->remote_free_check)) {
                claim_remote_free_list(s, l);

                if (l->freelist.nr > slab_hiwater(s))
                        flush_free_list(s, l);

                /* repetition here helps gcc :( */
                object = l->freelist.head;
                if (likely(object)) {
                        void *next = get_freepointer(s, object);

                        VM_BUG_ON(!l->freelist.nr);
                        l->freelist.nr--;
                        l->freelist.head = next;

                        return object;
                }
                VM_BUG_ON(l->freelist.nr);
        }
#endif

        return NULL;
}

/*
 * Slow(er) path. Get a page from this list's existing pages. Will be a
 * new empty page in the case that __slab_alloc_page has just been called
 * (empty pages otherwise never get queued up on the lists), or a partial page
 * already on the list.
 *
 * Caller must be the owner CPU in the case of per-CPU list, or hold the node's
 * list_lock in the case of per-node list.
 */
static noinline void *__cache_list_get_page(struct kmem_cache *s,
                                struct kmem_cache_list *l)
{
        struct slqb_page *page;
        void *object;

        if (unlikely(!l->nr_partial))
                return NULL;

        page = list_first_entry(&l->partial, struct slqb_page, lru);
        VM_BUG_ON(page->inuse == s->objects);
        if (page->inuse + 1 == s->objects) {
                l->nr_partial--;
                list_del(&page->lru);
        }

        VM_BUG_ON(!page->freelist);

        page->inuse++;

        object = page->freelist;
        page->freelist = get_freepointer(s, object);
        if (page->freelist)
                prefetchw(page->freelist);
        VM_BUG_ON((page->inuse == s->objects) != (page->freelist == NULL));
        slqb_stat_inc(l, ALLOC_SLAB_FILL);

        return object;
}

static void *cache_list_get_page(struct kmem_cache *s,
                                struct kmem_cache_list *l)
{
        void *object;

        if (unlikely(!l->nr_partial))
                return NULL;

        spin_lock(&l->page_lock);
        object = __cache_list_get_page(s, l);
        spin_unlock(&l->page_lock);

        return object;
}

/*
 * Allocation slowpath. Allocate a new slab page from the page allocator, and
 * put it on the list's partial list. Must be followed by an allocation so
 * that we don't have dangling empty pages on the partial list.
 *
 * Returns 0 on allocation failure.
 *
 * Must be called with interrupts disabled.
 */
static noinline void *__slab_alloc_page(struct kmem_cache *s,
                                gfp_t gfpflags, int node)
{
        struct slqb_page *page;
        struct kmem_cache_list *l;
        struct kmem_cache_cpu *c;
        unsigned int colour;
        void *object;

        c = get_cpu_slab(s, smp_processor_id());
        colour = c->colour_next;
        c->colour_next += s->colour_off;
        if (c->colour_next >= s->colour_range)
                c->colour_next = 0;

        /* Caller handles __GFP_ZERO */
        gfpflags &= ~__GFP_ZERO;

        if (gfpflags & __GFP_WAIT)
                local_irq_enable();
        page = new_slab_page(s, gfpflags, node, colour);
        if (gfpflags & __GFP_WAIT)
                local_irq_disable();
        if (unlikely(!page))
                return page;

        if (!NUMA_BUILD || likely(slqb_page_to_nid(page) == numa_node_id())) {
                struct kmem_cache_cpu *c;
                int cpu = smp_processor_id();

                c = get_cpu_slab(s, cpu);
                l = &c->list;
                page->list = l;

                spin_lock(&l->page_lock);
                l->nr_slabs++;
                l->nr_partial++;
                list_add(&page->lru, &l->partial);
                slqb_stat_inc(l, ALLOC);
                slqb_stat_inc(l, ALLOC_SLAB_NEW);
                object = __cache_list_get_page(s, l);
                spin_unlock(&l->page_lock);
        } else {
#ifdef CONFIG_NUMA
                struct kmem_cache_node *n;

                n = s->node_slab[slqb_page_to_nid(page)];
                l = &n->list;
                page->list = l;

                spin_lock(&n->list_lock);
                spin_lock(&l->page_lock);
                l->nr_slabs++;
                l->nr_partial++;
                list_add(&page->lru, &l->partial);
                slqb_stat_inc(l, ALLOC);
                slqb_stat_inc(l, ALLOC_SLAB_NEW);
                object = __cache_list_get_page(s, l);
                spin_unlock(&l->page_lock);
                spin_unlock(&n->list_lock);
#endif
        }
        VM_BUG_ON(!object);
        return object;
}

#ifdef CONFIG_NUMA
static noinline int alternate_nid(struct kmem_cache *s,
                                gfp_t gfpflags, int node)
{
        if (in_interrupt() || (gfpflags & __GFP_THISNODE))
                return node;
        if (cpuset_do_slab_mem_spread() && (s->flags & SLAB_MEM_SPREAD))
                return cpuset_mem_spread_node();
        else if (current->mempolicy)
                return slab_node(current->mempolicy);
        return node;
}

/*
 * Allocate an object from a remote node. Return NULL if none could be found
 * (in which case, caller should allocate a new slab)
 *
 * Must be called with interrupts disabled.
 */
static void *__remote_slab_alloc_node(struct kmem_cache *s,
                                gfp_t gfpflags, int node)
{
        struct kmem_cache_node *n;
        struct kmem_cache_list *l;
        void *object;

        n = s->node_slab[node];
        if (unlikely(!n)) /* node has no memory */
                return NULL;
        l = &n->list;

        spin_lock(&n->list_lock);

        object = __cache_list_get_object(s, l);
        if (unlikely(!object)) {
                object = cache_list_get_page(s, l);
                if (unlikely(!object)) {
                        spin_unlock(&n->list_lock);
                        return __slab_alloc_page(s, gfpflags, node);
                }
        }
        if (likely(object))
                slqb_stat_inc(l, ALLOC);
        spin_unlock(&n->list_lock);
        return object;
}

static noinline void *__remote_slab_alloc(struct kmem_cache *s,
                                gfp_t gfpflags, int node)
{
        void *object;
        struct zonelist *zonelist;
        struct zoneref *z;
        struct zone *zone;
        enum zone_type high_zoneidx = gfp_zone(gfpflags);

        object = __remote_slab_alloc_node(s, gfpflags, node);
        if (likely(object || (gfpflags & __GFP_THISNODE)))
                return object;

        zonelist = node_zonelist(slab_node(current->mempolicy), gfpflags);
        for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
                if (!cpuset_zone_allowed_hardwall(zone, gfpflags))
                        continue;

                node = zone_to_nid(zone);
                object = __remote_slab_alloc_node(s, gfpflags, node);
                if (likely(object))
                        return object;
        }
        return NULL;
}
#endif

/*
 * Main allocation path. Return an object, or NULL on allocation failure.
 *
 * Must be called with interrupts disabled.
 */
static __always_inline void *__slab_alloc(struct kmem_cache *s,
                                gfp_t gfpflags, int node)
{
        void *object;
        struct kmem_cache_cpu *c;
        struct kmem_cache_list *l;

#ifdef CONFIG_NUMA
        if (unlikely(node != -1) && unlikely(node != numa_node_id())) {
try_remote:
                return __remote_slab_alloc(s, gfpflags, node);
        }
#endif

        c = get_cpu_slab(s, smp_processor_id());
        VM_BUG_ON(!c);
        l = &c->list;
        object = __cache_list_get_object(s, l);
        if (unlikely(!object)) {
#ifdef CONFIG_NUMA
                int thisnode = numa_node_id();

                /*
                 * If the local node is memoryless, try remote alloc before
                 * trying the page allocator. Otherwise, what happens is
                 * objects are always freed to remote lists but the allocation
                 * side always allocates a new page with only one object
                 * used in each page
                 */
                if (unlikely(!node_state(thisnode, N_HIGH_MEMORY)))
                        object = __remote_slab_alloc(s, gfpflags, thisnode);
#endif

                if (!object) {
                        object = cache_list_get_page(s, l);
                        if (unlikely(!object)) {
                                object = __slab_alloc_page(s, gfpflags, node);
#ifdef CONFIG_NUMA
                                if (unlikely(!object)) {
                                        node = numa_node_id();
                                        goto try_remote;
                                }
#endif
                                return object;
                        }
                }
        }
        if (likely(object))
                slqb_stat_inc(l, ALLOC);
        return object;
}

/*
 * Perform some interrupts-on processing around the main allocation path
 * (debug checking and memset()ing).
 */
static __always_inline void *slab_alloc(struct kmem_cache *s,
                                gfp_t gfpflags, int node, unsigned long addr)
{
        void *object;
        unsigned long flags;

        gfpflags &= gfp_allowed_mask;

        lockdep_trace_alloc(gfpflags);
        might_sleep_if(gfpflags & __GFP_WAIT);

        if (should_failslab(s->objsize, gfpflags))
                return NULL;

again:
        local_irq_save(flags);
        object = __slab_alloc(s, gfpflags, node);
        local_irq_restore(flags);

        if (unlikely(slab_debug(s)) && likely(object)) {
                if (unlikely(!alloc_debug_processing(s, object, addr)))
                        goto again;
        }

        if (unlikely(gfpflags & __GFP_ZERO) && likely(object))
                memset(object, 0, s->objsize);

        return object;
}

static __always_inline void *__kmem_cache_alloc(struct kmem_cache *s,
                                gfp_t gfpflags, unsigned long caller)
{
        int node = -1;

#ifdef CONFIG_NUMA
        if (unlikely(current->flags & (PF_SPREAD_SLAB | PF_MEMPOLICY)))
                node = alternate_nid(s, gfpflags, node);
#endif
        return slab_alloc(s, gfpflags, node, caller);
}

void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
{
        return __kmem_cache_alloc(s, gfpflags, _RET_IP_);
}
EXPORT_SYMBOL(kmem_cache_alloc);

#ifdef CONFIG_NUMA
void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
{
        return slab_alloc(s, gfpflags, node, _RET_IP_);
}
EXPORT_SYMBOL(kmem_cache_alloc_node);
#endif

#ifdef CONFIG_SMP
/*
 * Flush this CPU's remote free list of objects back to the list from where
 * they originate. They end up on that list's remotely freed list, and
 * eventually we set it's remote_free_check if there are enough objects on it.
 *
 * This seems convoluted, but it keeps is from stomping on the target CPU's
 * fastpath cachelines.
 *
 * Must be called with interrupts disabled.
 */
static void flush_remote_free_cache(struct kmem_cache *s,
                                struct kmem_cache_cpu *c)
{
        struct kmlist *src;
        struct kmem_cache_list *dst;
        unsigned int nr;
        int set;

        src = &c->rlist;
        nr = src->nr;
        if (unlikely(!nr))
                return;

#ifdef CONFIG_SLQB_STATS
        {
                struct kmem_cache_list *l = &c->list;

                slqb_stat_inc(l, FLUSH_RFREE_LIST);
                slqb_stat_add(l, FLUSH_RFREE_LIST_OBJECTS, nr);
        }
#endif

        dst = c->remote_cache_list;

        /*
         * Less common case, dst is filling up so free synchronously.
         * No point in having remote CPU free thse as it will just
         * free them back to the page list anyway.
         */
        if (unlikely(dst->remote_free.list.nr > (slab_hiwater(s) >> 1))) {
                void **head;

                head = src->head;
                spin_lock(&dst->page_lock);
                do {
                        struct slqb_page *page;
                        void **object;

                        object = head;
                        VM_BUG_ON(!object);
                        head = get_freepointer(s, object);
                        page = virt_to_head_slqb_page(object);

                        free_object_to_page(s, dst, page, object);
                        nr--;
                } while (nr);
                spin_unlock(&dst->page_lock);

                src->head = NULL;
                src->tail = NULL;
                src->nr = 0;

                return;
        }

        spin_lock(&dst->remote_free.lock);

        if (!dst->remote_free.list.head)
                dst->remote_free.list.head = src->head;
        else
                set_freepointer(s, dst->remote_free.list.tail, src->head);
        dst->remote_free.list.tail = src->tail;

        src->head = NULL;
        src->tail = NULL;
        src->nr = 0;

        if (dst->remote_free.list.nr < slab_freebatch(s))
                set = 1;
        else
                set = 0;

        dst->remote_free.list.nr += nr;

        if (unlikely(dst->remote_free.list.nr >= slab_freebatch(s) && set))
                dst->remote_free_check = 1;

        spin_unlock(&dst->remote_free.lock);
}

/*
 * Free an object to this CPU's remote free list.
 *
 * Must be called with interrupts disabled.
 */
static noinline void slab_free_to_remote(struct kmem_cache *s,
                                struct slqb_page *page, void *object,
                                struct kmem_cache_cpu *c)
{
        struct kmlist *r;

        /*
         * Our remote free list corresponds to a different list. Must
         * flush it and switch.
         */
        if (page->list != c->remote_cache_list) {
                flush_remote_free_cache(s, c);
                c->remote_cache_list = page->list;
        }

        r = &c->rlist;
        if (!r->head)
                r->head = object;
        else
                set_freepointer(s, r->tail, object);
        set_freepointer(s, object, NULL);
        r->tail = object;
        r->nr++;

        if (unlikely(r->nr >= slab_freebatch(s)))
                flush_remote_free_cache(s, c);
}
#endif

/*
 * Main freeing path. Return an object, or NULL on allocation failure.
 *
 * Must be called with interrupts disabled.
 */
static __always_inline void __slab_free(struct kmem_cache *s,
                                struct slqb_page *page, void *object)
{
        struct kmem_cache_cpu *c;
        struct kmem_cache_list *l;
        int thiscpu = smp_processor_id();

        c = get_cpu_slab(s, thiscpu);
        l = &c->list;

        slqb_stat_inc(l, FREE);

        if (!NUMA_BUILD || !slab_numa(s) ||
                        likely(slqb_page_to_nid(page) == numa_node_id())) {
                /*
                 * Freeing fastpath. Collects all local-node objects, not
                 * just those allocated from our per-CPU list. This allows
                 * fast transfer of objects from one CPU to another within
                 * a given node.
                 */
                set_freepointer(s, object, l->freelist.head);
                l->freelist.head = object;
                if (!l->freelist.nr)
                        l->freelist.tail = object;
                l->freelist.nr++;

                if (unlikely(l->freelist.nr > slab_hiwater(s)))
                        flush_free_list(s, l);

        } else {
#ifdef CONFIG_SMP
                /*
                 * Freeing an object that was allocated on a remote node.
                 */
                slab_free_to_remote(s, page, object, c);
                slqb_stat_inc(l, FREE_REMOTE);
#endif
        }
}

/*
 * Perform some interrupts-on processing around the main freeing path
 * (debug checking).
 */
static __always_inline void slab_free(struct kmem_cache *s,
                                struct slqb_page *page, void *object)
{
        unsigned long flags;

        prefetchw(object);

        debug_check_no_locks_freed(object, s->objsize);
        if (likely(object) && unlikely(slab_debug(s))) {
                if (unlikely(!free_debug_processing(s, object, _RET_IP_)))
                        return;
        }

        local_irq_save(flags);
        __slab_free(s, page, object);
        local_irq_restore(flags);
}

void kmem_cache_free(struct kmem_cache *s, void *object)
{
        struct slqb_page *page = NULL;

        if (slab_numa(s))
                page = virt_to_head_slqb_page(object);
        slab_free(s, page, object);
}
EXPORT_SYMBOL(kmem_cache_free);

/*
 * Calculate the order of allocation given an slab object size.
 *
 * Order 0 allocations are preferred since order 0 does not cause fragmentation
 * in the page allocator, and they have fastpaths in the page allocator. But
 * also minimise external fragmentation with large objects.
 */
static int slab_order(int size, int max_order, int frac)
{
        int order;

        if (fls(size - 1) <= PAGE_SHIFT)
                order = 0;
        else
                order = fls(size - 1) - PAGE_SHIFT;
        if (order < slqb_min_order)
                order = slqb_min_order;

        while (order <= max_order) {
                unsigned long slab_size = PAGE_SIZE << order;
                unsigned long objects;
                unsigned long waste;

                objects = slab_size / size;
                if (!objects)
                        goto next;

                if (order < MAX_ORDER && objects < slqb_min_objects) {
                        /*
                         * if we don't have enough objects for min_objects,
                         * then try the next size up. Unless we have reached
                         * our maximum possible page size.
                         */
                        goto next;
                }

                waste = slab_size - (objects * size);

                if (waste * frac <= slab_size)
                        break;

next:
                order++;
        }

        return order;
}

static int calculate_order(int size)
{
        int order;

        /*
         * Attempt to find best configuration for a slab. This
         * works by first attempting to generate a layout with
         * the best configuration and backing off gradually.
         */
        order = slab_order(size, 1, 4);
        if (order <= 1)
                return order;

        /*
         * This size cannot fit in order-1. Allow bigger orders, but
         * forget about trying to save space.
         */
        order = slab_order(size, MAX_ORDER - 1, 0);
        if (order < MAX_ORDER)
                return order;

        return -ENOSYS;
}

/*
 * Figure out what the alignment of the objects will be.
 */
static unsigned long calculate_alignment(unsigned long flags,
                                unsigned long align, unsigned long size)
{
        /*
         * If the user wants hardware cache aligned objects then follow that
         * suggestion if the object is sufficiently large.
         *
         * The hardware cache alignment cannot override the specified
         * alignment though. If that is greater then use it.
         */
        if (flags & SLAB_HWCACHE_ALIGN) {
                unsigned long ralign = cache_line_size();

                while (size <= ralign / 2)
                        ralign /= 2;
                align = max(align, ralign);
        }

        if (align < ARCH_SLAB_MINALIGN)
                align = ARCH_SLAB_MINALIGN;

        return ALIGN(align, sizeof(void *));
}

static void init_kmem_cache_list(struct kmem_cache *s,
                                struct kmem_cache_list *l)
{
        l->cache                = s;
        l->freelist.nr          = 0;
        l->freelist.head        = NULL;
        l->freelist.tail        = NULL;
        l->nr_partial           = 0;
        l->nr_slabs             = 0;
        INIT_LIST_HEAD(&l->partial);
        spin_lock_init(&l->page_lock);

#ifdef CONFIG_SMP
        l->remote_free_check    = 0;
        spin_lock_init(&l->remote_free.lock);
        l->remote_free.list.nr  = 0;
        l->remote_free.list.head = NULL;
        l->remote_free.list.tail = NULL;
#endif

#ifdef CONFIG_SLQB_STATS
        memset(l->stats, 0, sizeof(l->stats));
#endif
}

static void init_kmem_cache_cpu(struct kmem_cache *s,
                                struct kmem_cache_cpu *c)
{
        init_kmem_cache_list(s, &c->list);

        c->colour_next          = 0;
#ifdef CONFIG_SMP
        c->rlist.nr             = 0;
        c->rlist.head           = NULL;
        c->rlist.tail           = NULL;
        c->remote_cache_list    = NULL;
#endif
}

#ifdef CONFIG_NUMA
static void init_kmem_cache_node(struct kmem_cache *s,
                                struct kmem_cache_node *n)
{
        spin_lock_init(&n->list_lock);
        init_kmem_cache_list(s, &n->list);
}
#endif

/* Initial slabs. */
#ifdef CONFIG_SMP
static DEFINE_PER_CPU(struct kmem_cache_cpu, kmem_cache_cpus);
#endif
#ifdef CONFIG_NUMA
/* XXX: really need a DEFINE_PER_NODE for per-node data because a static
 *      array is wasteful */
static struct kmem_cache_node kmem_cache_nodes[MAX_NUMNODES];
#endif

#ifdef CONFIG_SMP
static struct kmem_cache kmem_cpu_cache;
static DEFINE_PER_CPU(struct kmem_cache_cpu, kmem_cpu_cpus);
#ifdef CONFIG_NUMA
static struct kmem_cache_node kmem_cpu_nodes[MAX_NUMNODES]; /* XXX per-nid */
#endif
#endif

#ifdef CONFIG_NUMA
static struct kmem_cache kmem_node_cache;
#ifdef CONFIG_SMP
static DEFINE_PER_CPU(struct kmem_cache_cpu, kmem_node_cpus);
#endif
static struct kmem_cache_node kmem_node_nodes[MAX_NUMNODES]; /*XXX per-nid */
#endif

#ifdef CONFIG_SMP
static struct kmem_cache_cpu *alloc_kmem_cache_cpu(struct kmem_cache *s,
                                int cpu)
{
        struct kmem_cache_cpu *c;
        int node;

        node = cpu_to_node(cpu);

        c = kmem_cache_alloc_node(&kmem_cpu_cache, GFP_KERNEL, node);
        if (!c)
                return NULL;

        init_kmem_cache_cpu(s, c);
        return c;
}

static void free_kmem_cache_cpus(struct kmem_cache *s)
{
        int cpu;

        for_each_online_cpu(cpu) {
                struct kmem_cache_cpu *c;

                c = s->cpu_slab[cpu];
                if (c) {
                        kmem_cache_free(&kmem_cpu_cache, c);
                        s->cpu_slab[cpu] = NULL;
                }
        }
}

static int alloc_kmem_cache_cpus(struct kmem_cache *s)
{
        int cpu;

        for_each_online_cpu(cpu) {
                struct kmem_cache_cpu *c;

                c = s->cpu_slab[cpu];
                if (c)
                        continue;

                c = alloc_kmem_cache_cpu(s, cpu);
                if (!c) {
                        free_kmem_cache_cpus(s);
                        return 0;
                }
                s->cpu_slab[cpu] = c;
        }
        return 1;
}

#else
static inline void free_kmem_cache_cpus(struct kmem_cache *s)
{
}

static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
{
        init_kmem_cache_cpu(s, &s->cpu_slab);
        return 1;
}
#endif

#ifdef CONFIG_NUMA
static void free_kmem_cache_nodes(struct kmem_cache *s)
{
        int node;

        for_each_node_state(node, N_NORMAL_MEMORY) {
                struct kmem_cache_node *n;

                n = s->node_slab[node];
                if (n) {
                        kmem_cache_free(&kmem_node_cache, n);
                        s->node_slab[node] = NULL;
                }
        }
}

static int alloc_kmem_cache_nodes(struct kmem_cache *s)
{
        int node;

        for_each_node_state(node, N_NORMAL_MEMORY) {
                struct kmem_cache_node *n;

                n = kmem_cache_alloc_node(&kmem_node_cache, GFP_KERNEL, node);
                if (!n) {
                        free_kmem_cache_nodes(s);
                        return 0;
                }
                init_kmem_cache_node(s, n);
                s->node_slab[node] = n;
        }
        return 1;
}
#else
static void free_kmem_cache_nodes(struct kmem_cache *s)
{
}

static int alloc_kmem_cache_nodes(struct kmem_cache *s)
{
        return 1;
}
#endif

/*
 * calculate_sizes() determines the order and the distribution of data within
 * a slab object.
 */
static int calculate_sizes(struct kmem_cache *s)
{
        unsigned long flags = s->flags;
        unsigned long size = s->objsize;
        unsigned long align = s->align;

        /*
         * Determine if we can poison the object itself. If the user of
         * the slab may touch the object after free or before allocation
         * then we should never poison the object itself.
         */
        if (slab_poison(s) && !(flags & SLAB_DESTROY_BY_RCU) && !s->ctor)
                s->flags |= __OBJECT_POISON;
        else
                s->flags &= ~__OBJECT_POISON;

        /*
         * Round up object size to the next word boundary. We can only
         * place the free pointer at word boundaries and this determines
         * the possible location of the free pointer.
         */
        size = ALIGN(size, sizeof(void *));

#ifdef CONFIG_SLQB_DEBUG
        /*
         * If we are Redzoning then check if there is some space between the
         * end of the object and the free pointer. If not then add an
         * additional word to have some bytes to store Redzone information.
         */
        if ((flags & SLAB_RED_ZONE) && size == s->objsize)
                size += sizeof(void *);
#endif

        /*
         * With that we have determined the number of bytes in actual use
         * by the object. This is the potential offset to the free pointer.
         */
        s->inuse = size;

        if (((flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON)) || s->ctor)) {
                /*
                 * Relocate free pointer after the object if it is not
                 * permitted to overwrite the first word of the object on
                 * kmem_cache_free.
                 *
                 * This is the case if we do RCU, have a constructor or
                 * destructor or are poisoning the objects.
                 */
                s->offset = size;
                size += sizeof(void *);
        }

#ifdef CONFIG_SLQB_DEBUG
        if (flags & SLAB_STORE_USER) {
                /*
                 * Need to store information about allocs and frees after
                 * the object.
                 */
                size += 2 * sizeof(struct track);
        }

        if (flags & SLAB_RED_ZONE) {
                /*
                 * Add some empty padding so that we can catch
                 * overwrites from earlier objects rather than let
                 * tracking information or the free pointer be
                 * corrupted if an user writes before the start
                 * of the object.
                 */
                size += sizeof(void *);
        }
#endif

        /*
         * Determine the alignment based on various parameters that the
         * user specified and the dynamic determination of cache line size
         * on bootup.
         */
        align = calculate_alignment(flags, align, s->objsize);

        /*
         * SLQB stores one object immediately after another beginning from
         * offset 0. In order to align the objects we have to simply size
         * each object to conform to the alignment.
         */
        size = ALIGN(size, align);
        s->size = size;
        s->order = calculate_order(size);

        if (s->order < 0)
                return 0;

        s->allocflags = 0;
        if (s->order)
                s->allocflags |= __GFP_COMP;

        if (s->flags & SLAB_CACHE_DMA)
                s->allocflags |= SLQB_DMA;

        if (s->flags & SLAB_RECLAIM_ACCOUNT)
                s->allocflags |= __GFP_RECLAIMABLE;

        /*
         * Determine the number of objects per slab
         */
        s->objects = (PAGE_SIZE << s->order) / size;

        s->freebatch = max(4UL*PAGE_SIZE / size,
                                min(256UL, 64*PAGE_SIZE / size));
        if (!s->freebatch)
                s->freebatch = 1;
        s->hiwater = s->freebatch << 2;

        return !!s->objects;

}

#ifdef CONFIG_SMP
/*
 * Per-cpu allocator can't be used because it always uses slab allocator,
 * and it can't do per-node allocations.
 */
static void *kmem_cache_dyn_array_alloc(int ids)
{
        size_t size = sizeof(void *) * ids;

        BUG_ON(!size);

        if (unlikely(!slab_is_available())) {
                static void *nextmem;
                static size_t nextleft;
                void *ret;

                /*
                 * Special case for setting up initial caches. These will
                 * never get freed by definition so we can do it rather
                 * simply.
                 */
                if (size > nextleft) {
                        nextmem = alloc_pages_exact(size, GFP_KERNEL);
                        if (!nextmem)
                                return NULL;
                        nextleft = roundup(size, PAGE_SIZE);
                }

                ret = nextmem;
                nextleft -= size;
                nextmem += size;
                memset(ret, 0, size);
                return ret;
        } else {
                return kzalloc(size, GFP_KERNEL);
        }
}

static void kmem_cache_dyn_array_free(void *array)
{
        if (unlikely(!slab_is_available()))
                return; /* error case without crashing here (will panic soon) */
        kfree(array);
}
#endif

/*
 * Except in early boot, this should be called with slqb_lock held for write
 * to lock out hotplug, and protect list modifications.
 */
static int kmem_cache_open(struct kmem_cache *s,
                        const char *name, size_t size, size_t align,
                        unsigned long flags, void (*ctor)(void *), int alloc)
{
        unsigned int left_over;

        memset(s, 0, sizeof(struct kmem_cache));
        s->name = name;
        s->ctor = ctor;
        s->objsize = size;
        s->align = align;
        s->flags = kmem_cache_flags(size, flags, name, ctor);

        if (!calculate_sizes(s))
                goto error;

        if (!slab_debug(s)) {
                left_over = (PAGE_SIZE << s->order) - (s->objects * s->size);
                s->colour_off = max(cache_line_size(), s->align);
                s->colour_range = left_over;
        } else {
                s->colour_off = 0;
                s->colour_range = 0;
        }

#ifdef CONFIG_SMP
        s->cpu_slab = kmem_cache_dyn_array_alloc(nr_cpu_ids);
        if (!s->cpu_slab)
                goto error;
# ifdef CONFIG_NUMA
        s->node_slab = kmem_cache_dyn_array_alloc(nr_node_ids);
        if (!s->node_slab)
                goto error_cpu_array;
# endif
#endif

        if (likely(alloc)) {
                if (!alloc_kmem_cache_nodes(s))
                        goto error_node_array;

                if (!alloc_kmem_cache_cpus(s))
                        goto error_nodes;
        }

        sysfs_slab_add(s);
        list_add(&s->list, &slab_caches);

        return 1;

error_nodes:
        free_kmem_cache_nodes(s);
error_node_array:
#if defined(CONFIG_NUMA) && defined(CONFIG_SMP)
        kmem_cache_dyn_array_free(s->node_slab);
error_cpu_array:
#endif
#ifdef CONFIG_SMP
        kmem_cache_dyn_array_free(s->cpu_slab);
#endif
error:
        if (flags & SLAB_PANIC)
                panic("%s: failed to create slab `%s'\n", __func__, name);
        return 0;
}

/**
 * kmem_ptr_validate - check if an untrusted pointer might be a slab entry.
 * @s: the cache we're checking against
 * @ptr: pointer to validate
 *
 * This verifies that the untrusted pointer looks sane;
 * it is _not_ a guarantee that the pointer is actually
 * part of the slab cache in question, but it at least
 * validates that the pointer can be dereferenced and
 * looks half-way sane.
 *
 * Currently only used for dentry validation.
 */
int kmem_ptr_validate(struct kmem_cache *s, const void *ptr)
{
        unsigned long addr = (unsigned long)ptr;
        struct slqb_page *page;

        if (unlikely(addr < PAGE_OFFSET))
                goto out;
        if (unlikely(addr > (unsigned long)high_memory - s->size))
                goto out;
        if (unlikely(!IS_ALIGNED(addr, s->align)))
                goto out;
        if (unlikely(!kern_addr_valid(addr)))
                goto out;
        if (unlikely(!kern_addr_valid(addr + s->size - 1)))
                goto out;
        if (unlikely(!pfn_valid(addr >> PAGE_SHIFT)))
                goto out;
        page = virt_to_head_slqb_page(ptr);
        if (unlikely(!(page->flags & PG_SLQB_BIT)))
                goto out;
        if (unlikely(page->list->cache != s)) /* XXX: ouch, racy */
                goto out;
        return 1;
out:
        return 0;
}
EXPORT_SYMBOL(kmem_ptr_validate);

/*
 * Determine the size of a slab object
 */
unsigned int kmem_cache_size(struct kmem_cache *s)
{
        return s->objsize;
}
EXPORT_SYMBOL(kmem_cache_size);

const char *kmem_cache_name(struct kmem_cache *s)
{
        return s->name;
}
EXPORT_SYMBOL(kmem_cache_name);

/*
 * Release all resources used by a slab cache. No more concurrency on the
 * slab, so we can touch remote kmem_cache_cpu structures.
 */
void kmem_cache_destroy(struct kmem_cache *s)
{
#ifdef CONFIG_NUMA
        int node;
#endif
        int cpu;

        down_write(&slqb_lock);
        list_del(&s->list);

        local_irq_disable();
#ifdef CONFIG_SMP
        for_each_online_cpu(cpu) {
                struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
                struct kmem_cache_list *l = &c->list;

                flush_free_list_all(s, l);
                flush_remote_free_cache(s, c);
        }
#endif

        for_each_online_cpu(cpu) {
                struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
                struct kmem_cache_list *l = &c->list;

                claim_remote_free_list(s, l);
                flush_free_list_all(s, l);

                WARN_ON(l->freelist.nr);
                WARN_ON(l->nr_slabs);
                WARN_ON(l->nr_partial);
        }

        free_kmem_cache_cpus(s);

#ifdef CONFIG_NUMA
        for_each_node_state(node, N_NORMAL_MEMORY) {
                struct kmem_cache_node *n;
                struct kmem_cache_list *l;

                n = s->node_slab[node];
                if (!n)
                        continue;
                l = &n->list;

                claim_remote_free_list(s, l);
                flush_free_list_all(s, l);

                WARN_ON(l->freelist.nr);
                WARN_ON(l->nr_slabs);
                WARN_ON(l->nr_partial);
        }

        free_kmem_cache_nodes(s);
#endif
        local_irq_enable();

        sysfs_slab_remove(s);
        up_write(&slqb_lock);
}
EXPORT_SYMBOL(kmem_cache_destroy);

/********************************************************************
 *              Kmalloc subsystem
 *******************************************************************/

struct kmem_cache kmalloc_caches[KMALLOC_SHIFT_SLQB_HIGH + 1] __cacheline_aligned;
EXPORT_SYMBOL(kmalloc_caches);

#ifdef CONFIG_ZONE_DMA
struct kmem_cache kmalloc_caches_dma[KMALLOC_SHIFT_SLQB_HIGH + 1] __cacheline_aligned;
EXPORT_SYMBOL(kmalloc_caches_dma);
#endif

#ifndef ARCH_KMALLOC_FLAGS
#define ARCH_KMALLOC_FLAGS SLAB_HWCACHE_ALIGN
#endif

static struct kmem_cache *open_kmalloc_cache(struct kmem_cache *s,
                                const char *name, int size, gfp_t gfp_flags)
{
        unsigned int flags = ARCH_KMALLOC_FLAGS | SLAB_PANIC;

        if (gfp_flags & SLQB_DMA)
                flags |= SLAB_CACHE_DMA;

        kmem_cache_open(s, name, size, ARCH_KMALLOC_MINALIGN, flags, NULL, 1);

        return s;
}

/*
 * Conversion table for small slabs sizes / 8 to the index in the
 * kmalloc array. This is necessary for slabs < 192 since we have non power
 * of two cache sizes there. The size of larger slabs can be determined using
 * fls.
 */
static s8 size_index[24] __cacheline_aligned = {
        3,      /* 8 */
        4,      /* 16 */
        5,      /* 24 */
        5,      /* 32 */
        6,      /* 40 */
        6,      /* 48 */
        6,      /* 56 */
        6,      /* 64 */
#if L1_CACHE_BYTES < 64
        1,      /* 72 */
        1,      /* 80 */
        1,      /* 88 */
        1,      /* 96 */
#else
        7,
        7,
        7,
        7,
#endif
        7,      /* 104 */
        7,      /* 112 */
        7,      /* 120 */
        7,      /* 128 */
#if L1_CACHE_BYTES < 128
        2,      /* 136 */
        2,      /* 144 */
        2,      /* 152 */
        2,      /* 160 */
        2,      /* 168 */
        2,      /* 176 */
        2,      /* 184 */
        2       /* 192 */
#else
        -1,
        -1,
        -1,
        -1,
        -1,
        -1,
        -1,
        -1
#endif
};

static struct kmem_cache *get_slab(size_t size, gfp_t flags)
{
        int index;

        if (unlikely(size <= KMALLOC_MIN_SIZE)) {
                if (unlikely(!size))
                        return ZERO_SIZE_PTR;

                index = KMALLOC_SHIFT_LOW;
                goto got_index;
        }

#if L1_CACHE_BYTES >= 128
        if (size <= 128) {
#else
        if (size <= 192) {
#endif
                index = size_index[(size - 1) / 8];
        } else {
                if (unlikely(size > 1UL << KMALLOC_SHIFT_SLQB_HIGH))
                        return NULL;

                index = fls(size - 1);
        }

got_index:
        if (unlikely((flags & SLQB_DMA)))
                return &kmalloc_caches_dma[index];
        else
                return &kmalloc_caches[index];
}

void *__kmalloc(size_t size, gfp_t flags)
{
        struct kmem_cache *s;

        s = get_slab(size, flags);
        if (unlikely(ZERO_OR_NULL_PTR(s)))
                return s;

        return __kmem_cache_alloc(s, flags, _RET_IP_);
}
EXPORT_SYMBOL(__kmalloc);

#ifdef CONFIG_NUMA
void *__kmalloc_node(size_t size, gfp_t flags, int node)
{
        struct kmem_cache *s;

        s = get_slab(size, flags);
        if (unlikely(ZERO_OR_NULL_PTR(s)))
                return s;

        return kmem_cache_alloc_node(s, flags, node);
}
EXPORT_SYMBOL(__kmalloc_node);
#endif

size_t ksize(const void *object)
{
        struct slqb_page *page;
        struct kmem_cache *s;

        BUG_ON(!object);
        if (unlikely(object == ZERO_SIZE_PTR))
                return 0;

        page = virt_to_head_slqb_page(object);
        BUG_ON(!(page->flags & PG_SLQB_BIT));

        s = page->list->cache;

        /*
         * Debugging requires use of the padding between object
         * and whatever may come after it.
         */
        if (s->flags & (SLAB_RED_ZONE | SLAB_POISON))
                return s->objsize;

        /*
         * If we have the need to store the freelist pointer
         * back there or track user information then we can
         * only use the space before that information.
         */
        if (s->flags & (SLAB_DESTROY_BY_RCU | SLAB_STORE_USER))
                return s->inuse;

        /*
         * Else we can use all the padding etc for the allocation
         */
        return s->size;
}
EXPORT_SYMBOL(ksize);

void kfree(const void *object)
{
        struct kmem_cache *s;
        struct slqb_page *page;

        if (unlikely(ZERO_OR_NULL_PTR(object)))
                return;

        page = virt_to_head_slqb_page(object);
        s = page->list->cache;

        slab_free(s, page, (void *)object);
}
EXPORT_SYMBOL(kfree);

static void kmem_cache_trim_percpu(void *arg)
{
        int cpu = smp_processor_id();
        struct kmem_cache *s = arg;
        struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
        struct kmem_cache_list *l = &c->list;

        claim_remote_free_list(s, l);
        flush_free_list(s, l);
#ifdef CONFIG_SMP
        flush_remote_free_cache(s, c);
#endif
}

int kmem_cache_shrink(struct kmem_cache *s)
{
#ifdef CONFIG_NUMA
        int node;
#endif

        on_each_cpu(kmem_cache_trim_percpu, s, 1);

#ifdef CONFIG_NUMA
        for_each_node_state(node, N_NORMAL_MEMORY) {
                struct kmem_cache_node *n;
                struct kmem_cache_list *l;

                n = s->node_slab[node];
                if (!n)
                        continue;
                l = &n->list;

                spin_lock_irq(&n->list_lock);
                claim_remote_free_list(s, l);
                flush_free_list(s, l);
                spin_unlock_irq(&n->list_lock);
        }
#endif

        return 0;
}
EXPORT_SYMBOL(kmem_cache_shrink);

#if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG)
static void kmem_cache_reap_percpu(void *arg)
{
        int cpu = smp_processor_id();
        struct kmem_cache *s;
        long phase = (long)arg;

        list_for_each_entry(s, &slab_caches, list) {
                struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
                struct kmem_cache_list *l = &c->list;

                if (phase == 0) {
                        flush_free_list_all(s, l);
                        flush_remote_free_cache(s, c);
                }

                if (phase == 1) {
                        claim_remote_free_list(s, l);
                        flush_free_list_all(s, l);
                }
        }
}

static void kmem_cache_reap(void)
{
        struct kmem_cache *s;
        int node;

        down_read(&slqb_lock);
        on_each_cpu(kmem_cache_reap_percpu, (void *)0, 1);
        on_each_cpu(kmem_cache_reap_percpu, (void *)1, 1);

        list_for_each_entry(s, &slab_caches, list) {
                for_each_node_state(node, N_NORMAL_MEMORY) {
                        struct kmem_cache_node *n;
                        struct kmem_cache_list *l;

                        n = s->node_slab[node];
                        if (!n)
                                continue;
                        l = &n->list;

                        spin_lock_irq(&n->list_lock);
                        claim_remote_free_list(s, l);
                        flush_free_list_all(s, l);
                        spin_unlock_irq(&n->list_lock);
                }
        }
        up_read(&slqb_lock);
}
#endif

static void cache_trim_worker(struct work_struct *w)
{
        struct delayed_work *work =
                container_of(w, struct delayed_work, work);
        struct kmem_cache *s;

        if (!down_read_trylock(&slqb_lock))
                goto out;

        list_for_each_entry(s, &slab_caches, list) {
#ifdef CONFIG_NUMA
                int node = numa_node_id();
                struct kmem_cache_node *n = s->node_slab[node];

                if (n) {
                        struct kmem_cache_list *l = &n->list;

                        spin_lock_irq(&n->list_lock);
                        claim_remote_free_list(s, l);
                        flush_free_list(s, l);
                        spin_unlock_irq(&n->list_lock);
                }
#endif

                local_irq_disable();
                kmem_cache_trim_percpu(s);
                local_irq_enable();
        }

        up_read(&slqb_lock);
out:
        schedule_delayed_work(work, round_jiffies_relative(3*HZ));
}

static DEFINE_PER_CPU(struct delayed_work, cache_trim_work);

static void __cpuinit start_cpu_timer(int cpu)
{
        struct delayed_work *cache_trim_work = &per_cpu(cache_trim_work, cpu);

        /*
         * When this gets called from do_initcalls via cpucache_init(),
         * init_workqueues() has already run, so keventd will be setup
         * at that time.
         */
        if (keventd_up() && cache_trim_work->work.func == NULL) {
                INIT_DELAYED_WORK(cache_trim_work, cache_trim_worker);
                schedule_delayed_work_on(cpu, cache_trim_work,
                                        __round_jiffies_relative(HZ, cpu));
        }
}

static int __init cpucache_init(void)
{
        int cpu;

        for_each_online_cpu(cpu)
                start_cpu_timer(cpu);

        return 0;
}
device_initcall(cpucache_init);

#if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG)
static void slab_mem_going_offline_callback(void *arg)
{
        kmem_cache_reap();
}

static void slab_mem_offline_callback(void *arg)
{
        /* XXX: should release structures, see CPU offline comment */
}

static int slab_mem_going_online_callback(void *arg)
{
        struct kmem_cache *s;
        struct kmem_cache_node *n;
        struct memory_notify *marg = arg;
        int nid = marg->status_change_nid;
        int ret = 0;

        /*
         * If the node's memory is already available, then kmem_cache_node is
         * already created. Nothing to do.
         */
        if (nid < 0)
                return 0;

        /*
         * We are bringing a node online. No memory is availabe yet. We must
         * allocate a kmem_cache_node structure in order to bring the node
         * online.
         */
        down_write(&slqb_lock);
        list_for_each_entry(s, &slab_caches, list) {
                /*
                 * XXX: kmem_cache_alloc_node will fallback to other nodes
                 *      since memory is not yet available from the node that
                 *      is brought up.
                 */
                if (s->node_slab[nid]) /* could be lefover from last online */
                        continue;
                n = kmem_cache_alloc(&kmem_node_cache, GFP_KERNEL);
                if (!n) {
                        ret = -ENOMEM;
                        goto out;
                }
                init_kmem_cache_node(s, n);
                s->node_slab[nid] = n;
        }
out:
        up_write(&slqb_lock);
        return ret;
}

static int slab_memory_callback(struct notifier_block *self,
                                unsigned long action, void *arg)
{
        int ret = 0;

        switch (action) {
        case MEM_GOING_ONLINE:
                ret = slab_mem_going_online_callback(arg);
                break;
        case MEM_GOING_OFFLINE:
                slab_mem_going_offline_callback(arg);
                break;
        case MEM_OFFLINE:
        case MEM_CANCEL_ONLINE:
                slab_mem_offline_callback(arg);
                break;
        case MEM_ONLINE:
        case MEM_CANCEL_OFFLINE:
                break;
        }

        if (ret)
                ret = notifier_from_errno(ret);
        else
                ret = NOTIFY_OK;
        return ret;
}

#endif /* CONFIG_MEMORY_HOTPLUG */

/********************************************************************
 *                      Basic setup of slabs
 *******************************************************************/

void __init kmem_cache_init(void)
{
        int i;
        unsigned int flags = SLAB_HWCACHE_ALIGN|SLAB_PANIC;

        /*
         * All the ifdefs are rather ugly here, but it's just the setup code,
         * so it doesn't have to be too readable :)
         */

        /*
         * No need to take slqb_lock here: there should be no concurrency
         * anyway, and spin_unlock_irq in rwsem code could enable interrupts
         * too early.
         */
        kmem_cache_open(&kmem_cache_cache, "kmem_cache",
                        sizeof(struct kmem_cache), 0, flags, NULL, 0);
#ifdef CONFIG_SMP
        kmem_cache_open(&kmem_cpu_cache, "kmem_cache_cpu",
                        sizeof(struct kmem_cache_cpu), 0, flags, NULL, 0);
#endif
#ifdef CONFIG_NUMA
        kmem_cache_open(&kmem_node_cache, "kmem_cache_node",
                        sizeof(struct kmem_cache_node), 0, flags, NULL, 0);
#endif

#ifdef CONFIG_SMP
        for_each_possible_cpu(i) {
                struct kmem_cache_cpu *c;

                c = &per_cpu(kmem_cache_cpus, i);
                init_kmem_cache_cpu(&kmem_cache_cache, c);
                kmem_cache_cache.cpu_slab[i] = c;

                c = &per_cpu(kmem_cpu_cpus, i);
                init_kmem_cache_cpu(&kmem_cpu_cache, c);
                kmem_cpu_cache.cpu_slab[i] = c;

#ifdef CONFIG_NUMA
                c = &per_cpu(kmem_node_cpus, i);
                init_kmem_cache_cpu(&kmem_node_cache, c);
                kmem_node_cache.cpu_slab[i] = c;
#endif
        }
#else
        init_kmem_cache_cpu(&kmem_cache_cache, &kmem_cache_cache.cpu_slab);
#endif

#ifdef CONFIG_NUMA
        for_each_node_state(i, N_NORMAL_MEMORY) {
                struct kmem_cache_node *n;

                n = &kmem_cache_nodes[i];
                init_kmem_cache_node(&kmem_cache_cache, n);
                kmem_cache_cache.node_slab[i] = n;
#ifdef CONFIG_SMP
                n = &kmem_cpu_nodes[i];
                init_kmem_cache_node(&kmem_cpu_cache, n);
                kmem_cpu_cache.node_slab[i] = n;
#endif
                n = &kmem_node_nodes[i];
                init_kmem_cache_node(&kmem_node_cache, n);
                kmem_node_cache.node_slab[i] = n;
        }
#endif

        /* Caches that are not of the two-to-the-power-of size */
        if (L1_CACHE_BYTES < 64 && KMALLOC_MIN_SIZE <= 64) {
                open_kmalloc_cache(&kmalloc_caches[1],
                                "kmalloc-96", 96, GFP_KERNEL);
#ifdef CONFIG_ZONE_DMA
                open_kmalloc_cache(&kmalloc_caches_dma[1],
                                "kmalloc_dma-96", 96, GFP_KERNEL|SLQB_DMA);
#endif
        }
        if (L1_CACHE_BYTES < 128 && KMALLOC_MIN_SIZE <= 128) {
                open_kmalloc_cache(&kmalloc_caches[2],
                                "kmalloc-192", 192, GFP_KERNEL);
#ifdef CONFIG_ZONE_DMA
                open_kmalloc_cache(&kmalloc_caches_dma[2],
                                "kmalloc_dma-192", 192, GFP_KERNEL|SLQB_DMA);
#endif
        }

        for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_SLQB_HIGH; i++) {
                open_kmalloc_cache(&kmalloc_caches[i],
                                "kmalloc", 1 << i, GFP_KERNEL);
#ifdef CONFIG_ZONE_DMA
                open_kmalloc_cache(&kmalloc_caches_dma[i],
                                "kmalloc_dma", 1 << i, GFP_KERNEL|SLQB_DMA);
#endif
        }

        /*
         * Patch up the size_index table if we have strange large alignment
         * requirements for the kmalloc array. This is only the case for
         * mips it seems. The standard arches will not generate any code here.
         *
         * Largest permitted alignment is 256 bytes due to the way we
         * handle the index determination for the smaller caches.
         *
         * Make sure that nothing crazy happens if someone starts tinkering
         * around with ARCH_KMALLOC_MINALIGN
         */
        BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 ||
                (KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1)));

        for (i = 8; i < KMALLOC_MIN_SIZE; i += 8)
                size_index[(i - 1) / 8] = KMALLOC_SHIFT_LOW;

        /* Provide the correct kmalloc names now that the caches are up */
        for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_SLQB_HIGH; i++) {
                kmalloc_caches[i].name =
                        kasprintf(GFP_KERNEL, "kmalloc-%d", 1 << i);
#ifdef CONFIG_ZONE_DMA
                kmalloc_caches_dma[i].name =
                        kasprintf(GFP_KERNEL, "kmalloc_dma-%d", 1 << i);
#endif
        }

#ifdef CONFIG_SMP
        register_cpu_notifier(&slab_notifier);
#endif
#ifdef CONFIG_NUMA
        hotplug_memory_notifier(slab_memory_callback, 1);
#endif
        /*
         * smp_init() has not yet been called, so no worries about memory
         * ordering with __slab_is_available.
         */
        __slab_is_available = 1;
}

void __init kmem_cache_init_late(void)
{
}

/*
 * Some basic slab creation sanity checks
 */
static int kmem_cache_create_ok(const char *name, size_t size,
                size_t align, unsigned long flags)
{
        struct kmem_cache *tmp;

        /*
         * Sanity checks... these are all serious usage bugs.
         */
        if (!name || in_interrupt() || (size < sizeof(void *))) {
                printk(KERN_ERR "kmem_cache_create(): early error in slab %s\n",
                                name);
                dump_stack();

                return 0;
        }

        list_for_each_entry(tmp, &slab_caches, list) {
                char x;
                int res;

                /*
                 * This happens when the module gets unloaded and doesn't
                 * destroy its slab cache and no-one else reuses the vmalloc
                 * area of the module.  Print a warning.
                 */
                res = probe_kernel_address(tmp->name, x);
                if (res) {
                        printk(KERN_ERR
                               "SLAB: cache with size %d has lost its name\n",
                               tmp->size);
                        continue;
                }

                if (!strcmp(tmp->name, name)) {
                        printk(KERN_ERR
                               "SLAB: duplicate cache %s\n", name);
                        dump_stack();

                        return 0;
                }
        }

        WARN_ON(strchr(name, ' '));     /* It confuses parsers */
        if (flags & SLAB_DESTROY_BY_RCU)
                WARN_ON(flags & SLAB_POISON);

        return 1;
}

struct kmem_cache *kmem_cache_create(const char *name, size_t size,
                size_t align, unsigned long flags, void (*ctor)(void *))
{
        struct kmem_cache *s;

        down_write(&slqb_lock);
        if (!kmem_cache_create_ok(name, size, align, flags))
                goto err;

        s = kmem_cache_alloc(&kmem_cache_cache, GFP_KERNEL);
        if (!s)
                goto err;

        if (kmem_cache_open(s, name, size, align, flags, ctor, 1)) {
                up_write(&slqb_lock);
                return s;
        }

        kmem_cache_free(&kmem_cache_cache, s);

err:
        up_write(&slqb_lock);
        if (flags & SLAB_PANIC)
                panic("%s: failed to create slab `%s'\n", __func__, name);

        return NULL;
}
EXPORT_SYMBOL(kmem_cache_create);

#ifdef CONFIG_SMP
/*
 * Use the cpu notifier to insure that the cpu slabs are flushed when
 * necessary.
 */
static int __cpuinit slab_cpuup_callback(struct notifier_block *nfb,
                                unsigned long action, void *hcpu)
{
        long cpu = (long)hcpu;
        struct kmem_cache *s;

        switch (action) {
        case CPU_UP_PREPARE:
        case CPU_UP_PREPARE_FROZEN:
                down_write(&slqb_lock);
                list_for_each_entry(s, &slab_caches, list) {
                        if (s->cpu_slab[cpu]) /* could be lefover last online */
                                continue;
                        s->cpu_slab[cpu] = alloc_kmem_cache_cpu(s, cpu);
                        if (!s->cpu_slab[cpu]) {
                                up_read(&slqb_lock);
                                return NOTIFY_BAD;
                        }
                }
                up_write(&slqb_lock);
                break;

        case CPU_ONLINE:
        case CPU_ONLINE_FROZEN:
        case CPU_DOWN_FAILED:
        case CPU_DOWN_FAILED_FROZEN:
                start_cpu_timer(cpu);
                break;

        case CPU_DOWN_PREPARE:
        case CPU_DOWN_PREPARE_FROZEN:
                cancel_rearming_delayed_work(&per_cpu(cache_trim_work, cpu));
                per_cpu(cache_trim_work, cpu).work.func = NULL;
                break;

        case CPU_UP_CANCELED:
        case CPU_UP_CANCELED_FROZEN:
        case CPU_DEAD:
        case CPU_DEAD_FROZEN:
                /*
                 * XXX: Freeing here doesn't work because objects can still be
                 * on this CPU's list. periodic timer needs to check if a CPU
                 * is offline and then try to cleanup from there. Same for node
                 * offline.
                 */
        default:
                break;
        }
        return NOTIFY_OK;
}

static struct notifier_block __cpuinitdata slab_notifier = {
        .notifier_call = slab_cpuup_callback
};

#endif

#ifdef CONFIG_SLQB_DEBUG
void *__kmalloc_track_caller(size_t size, gfp_t flags, unsigned long caller)
{
        struct kmem_cache *s;
        int node = -1;

        s = get_slab(size, flags);
        if (unlikely(ZERO_OR_NULL_PTR(s)))
                return s;

#ifdef CONFIG_NUMA
        if (unlikely(current->flags & (PF_SPREAD_SLAB | PF_MEMPOLICY)))
                node = alternate_nid(s, flags, node);
#endif
        return slab_alloc(s, flags, node, caller);
}

void *__kmalloc_node_track_caller(size_t size, gfp_t flags, int node,
                                unsigned long caller)
{
        struct kmem_cache *s;

        s = get_slab(size, flags);
        if (unlikely(ZERO_OR_NULL_PTR(s)))
                return s;

        return slab_alloc(s, flags, node, caller);
}
#endif

#if defined(CONFIG_SLQB_SYSFS) || defined(CONFIG_SLABINFO)
struct stats_gather {
        struct kmem_cache *s;
        spinlock_t lock;
        unsigned long nr_slabs;
        unsigned long nr_partial;
        unsigned long nr_inuse;
        unsigned long nr_objects;

#ifdef CONFIG_SLQB_STATS
        unsigned long stats[NR_SLQB_STAT_ITEMS];
#endif
};

static void __gather_stats(void *arg)
{
        unsigned long nr_slabs;
        unsigned long nr_partial;
        unsigned long nr_inuse;
        struct stats_gather *gather = arg;
        int cpu = smp_processor_id();
        struct kmem_cache *s = gather->s;
        struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
        struct kmem_cache_list *l = &c->list;
        struct slqb_page *page;
#ifdef CONFIG_SLQB_STATS
        int i;
#endif

        spin_lock(&l->page_lock);
        nr_slabs = l->nr_slabs;
        nr_partial = l->nr_partial;
        nr_inuse = (nr_slabs - nr_partial) * s->objects;

        list_for_each_entry(page, &l->partial, lru) {
                nr_inuse += page->inuse;
        }
        spin_unlock(&l->page_lock);

        spin_lock(&gather->lock);
        gather->nr_slabs += nr_slabs;
        gather->nr_partial += nr_partial;
        gather->nr_inuse += nr_inuse;
#ifdef CONFIG_SLQB_STATS
        for (i = 0; i < NR_SLQB_STAT_ITEMS; i++)
                gather->stats[i] += l->stats[i];
#endif
        spin_unlock(&gather->lock);
}

/* must be called with slqb_lock held */
static void gather_stats_locked(struct kmem_cache *s,
                                struct stats_gather *stats)
{
#ifdef CONFIG_NUMA
        int node;
#endif

        memset(stats, 0, sizeof(struct stats_gather));
        stats->s = s;
        spin_lock_init(&stats->lock);

        on_each_cpu(__gather_stats, stats, 1);

#ifdef CONFIG_NUMA
        for_each_online_node(node) {
                struct kmem_cache_node *n = s->node_slab[node];
                struct kmem_cache_list *l = &n->list;
                struct slqb_page *page;
                unsigned long flags;
#ifdef CONFIG_SLQB_STATS
                int i;
#endif

                spin_lock_irqsave(&n->list_lock, flags);
#ifdef CONFIG_SLQB_STATS
                for (i = 0; i < NR_SLQB_STAT_ITEMS; i++)
                        stats->stats[i] += l->stats[i];
#endif
                stats->nr_slabs += l->nr_slabs;
                stats->nr_partial += l->nr_partial;
                stats->nr_inuse += (l->nr_slabs - l->nr_partial) * s->objects;

                list_for_each_entry(page, &l->partial, lru) {
                        stats->nr_inuse += page->inuse;
                }
                spin_unlock_irqrestore(&n->list_lock, flags);
        }
#endif

        stats->nr_objects = stats->nr_slabs * s->objects;
}

#ifdef CONFIG_SLQB_SYSFS
static void gather_stats(struct kmem_cache *s, struct stats_gather *stats)
{
        down_read(&slqb_lock); /* hold off hotplug */
        gather_stats_locked(s, stats);
        up_read(&slqb_lock);
}
#endif
#endif

/*
 * The /proc/slabinfo ABI
 */
#ifdef CONFIG_SLABINFO
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
ssize_t slabinfo_write(struct file *file, const char __user * buffer,
                       size_t count, loff_t *ppos)
{
        return -EINVAL;
}

static void print_slabinfo_header(struct seq_file *m)
{
        seq_puts(m, "slabinfo - version: 2.1\n");
        seq_puts(m, "# name         <active_objs> <num_objs> <objsize> "
                 "<objperslab> <pagesperslab>");
        seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
        seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
        seq_putc(m, '\n');
}

static void *s_start(struct seq_file *m, loff_t *pos)
{
        loff_t n = *pos;

        down_read(&slqb_lock);
        if (!n)
                print_slabinfo_header(m);

        return seq_list_start(&slab_caches, *pos);
}

static void *s_next(struct seq_file *m, void *p, loff_t *pos)
{
        return seq_list_next(p, &slab_caches, pos);
}

static void s_stop(struct seq_file *m, void *p)
{
        up_read(&slqb_lock);
}

static int s_show(struct seq_file *m, void *p)
{
        struct stats_gather stats;
        struct kmem_cache *s;

        s = list_entry(p, struct kmem_cache, list);

        gather_stats_locked(s, &stats);

        seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", s->name, stats.nr_inuse,
                        stats.nr_objects, s->size, s->objects, (1 << s->order));
        seq_printf(m, " : tunables %4u %4u %4u", slab_hiwater(s),
                        slab_freebatch(s), 0);
        seq_printf(m, " : slabdata %6lu %6lu %6lu", stats.nr_slabs,
                        stats.nr_slabs, 0UL);
        seq_putc(m, '\n');
        return 0;
}

static const struct seq_operations slabinfo_op = {
        .start = s_start,
        .next = s_next,
        .stop = s_stop,
        .show = s_show,
};

static int slabinfo_open(struct inode *inode, struct file *file)
{
        return seq_open(file, &slabinfo_op);
}

static const struct file_operations proc_slabinfo_operations = {
        .open           = slabinfo_open,
        .read           = seq_read,
        .llseek         = seq_lseek,
        .release        = seq_release,
};

static int __init slab_proc_init(void)
{
        proc_create("slabinfo", S_IWUSR|S_IRUGO, NULL,
                        &proc_slabinfo_operations);
        return 0;
}
module_init(slab_proc_init);
#endif /* CONFIG_SLABINFO */

#ifdef CONFIG_SLQB_SYSFS
/*
 * sysfs API
 */
#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)
#define to_slab(n) container_of(n, struct kmem_cache, kobj);

struct slab_attribute {
        struct attribute attr;
        ssize_t (*show)(struct kmem_cache *s, char *buf);
        ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
};

#define SLAB_ATTR_RO(_name) \
        static struct slab_attribute _name##_attr = __ATTR_RO(_name)

#define SLAB_ATTR(_name) \
        static struct slab_attribute _name##_attr =  \
        __ATTR(_name, 0644, _name##_show, _name##_store)

static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
{
        return sprintf(buf, "%d\n", s->size);
}
SLAB_ATTR_RO(slab_size);

static ssize_t align_show(struct kmem_cache *s, char *buf)
{
        return sprintf(buf, "%d\n", s->align);
}
SLAB_ATTR_RO(align);

static ssize_t object_size_show(struct kmem_cache *s, char *buf)
{
        return sprintf(buf, "%d\n", s->objsize);
}
SLAB_ATTR_RO(object_size);

static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
{
        return sprintf(buf, "%d\n", s->objects);
}
SLAB_ATTR_RO(objs_per_slab);

static ssize_t order_show(struct kmem_cache *s, char *buf)
{
        return sprintf(buf, "%d\n", s->order);
}
SLAB_ATTR_RO(order);

static ssize_t ctor_show(struct kmem_cache *s, char *buf)
{
        if (s->ctor) {
                int n = sprint_symbol(buf, (unsigned long)s->ctor);

                return n + sprintf(buf + n, "\n");
        }
        return 0;
}
SLAB_ATTR_RO(ctor);

static ssize_t slabs_show(struct kmem_cache *s, char *buf)
{
        struct stats_gather stats;

        gather_stats(s, &stats);

        return sprintf(buf, "%lu\n", stats.nr_slabs);
}
SLAB_ATTR_RO(slabs);

static ssize_t objects_show(struct kmem_cache *s, char *buf)
{
        struct stats_gather stats;

        gather_stats(s, &stats);

        return sprintf(buf, "%lu\n", stats.nr_inuse);
}
SLAB_ATTR_RO(objects);

static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
{
        struct stats_gather stats;

        gather_stats(s, &stats);

        return sprintf(buf, "%lu\n", stats.nr_objects);
}
SLAB_ATTR_RO(total_objects);

static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
{
        return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
}
SLAB_ATTR_RO(reclaim_account);

static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
{
        return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
}
SLAB_ATTR_RO(hwcache_align);

#ifdef CONFIG_ZONE_DMA
static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)
{
        return sprintf(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
}
SLAB_ATTR_RO(cache_dma);
#endif

static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
{
        return sprintf(buf, "%d\n", !!(s->flags & SLAB_DESTROY_BY_RCU));
}
SLAB_ATTR_RO(destroy_by_rcu);

static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
{
        return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
}
SLAB_ATTR_RO(red_zone);

static ssize_t poison_show(struct kmem_cache *s, char *buf)
{
        return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON));
}
SLAB_ATTR_RO(poison);

static ssize_t store_user_show(struct kmem_cache *s, char *buf)
{
        return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
}
SLAB_ATTR_RO(store_user);

static ssize_t hiwater_store(struct kmem_cache *s,
                                const char *buf, size_t length)
{
        long hiwater;
        int err;

        err = strict_strtol(buf, 10, &hiwater);
        if (err)
                return err;

        if (hiwater < 0)
                return -EINVAL;

        s->hiwater = hiwater;

        return length;
}

static ssize_t hiwater_show(struct kmem_cache *s, char *buf)
{
        return sprintf(buf, "%d\n", slab_hiwater(s));
}
SLAB_ATTR(hiwater);

static ssize_t freebatch_store(struct kmem_cache *s,
                                const char *buf, size_t length)
{
        long freebatch;
        int err;

        err = strict_strtol(buf, 10, &freebatch);
        if (err)
                return err;

        if (freebatch <= 0 || freebatch - 1 > s->hiwater)
                return -EINVAL;

        s->freebatch = freebatch;

        return length;
}

static ssize_t freebatch_show(struct kmem_cache *s, char *buf)
{
        return sprintf(buf, "%d\n", slab_freebatch(s));
}
SLAB_ATTR(freebatch);

#ifdef CONFIG_SLQB_STATS
static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)
{
        struct stats_gather stats;
        int len;
#ifdef CONFIG_SMP
        int cpu;
#endif

        gather_stats(s, &stats);

        len = sprintf(buf, "%lu", stats.stats[si]);

#ifdef CONFIG_SMP
        for_each_online_cpu(cpu) {
                struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
                struct kmem_cache_list *l = &c->list;

                if (len < PAGE_SIZE - 20)
                        len += sprintf(buf+len, " C%d=%lu", cpu, l->stats[si]);
        }
#endif
        return len + sprintf(buf + len, "\n");
}

#define STAT_ATTR(si, text)                                     \
static ssize_t text##_show(struct kmem_cache *s, char *buf)     \
{                                                               \
        return show_stat(s, buf, si);                           \
}                                                               \
SLAB_ATTR_RO(text);                                             \

STAT_ATTR(ALLOC, alloc);
STAT_ATTR(ALLOC_SLAB_FILL, alloc_slab_fill);
STAT_ATTR(ALLOC_SLAB_NEW, alloc_slab_new);
STAT_ATTR(FREE, free);
STAT_ATTR(FREE_REMOTE, free_remote);
STAT_ATTR(FLUSH_FREE_LIST, flush_free_list);
STAT_ATTR(FLUSH_FREE_LIST_OBJECTS, flush_free_list_objects);
STAT_ATTR(FLUSH_FREE_LIST_REMOTE, flush_free_list_remote);
STAT_ATTR(FLUSH_SLAB_PARTIAL, flush_slab_partial);
STAT_ATTR(FLUSH_SLAB_FREE, flush_slab_free);
STAT_ATTR(FLUSH_RFREE_LIST, flush_rfree_list);
STAT_ATTR(FLUSH_RFREE_LIST_OBJECTS, flush_rfree_list_objects);
STAT_ATTR(CLAIM_REMOTE_LIST, claim_remote_list);
STAT_ATTR(CLAIM_REMOTE_LIST_OBJECTS, claim_remote_list_objects);
#endif

static struct attribute *slab_attrs[] = {
        &slab_size_attr.attr,
        &object_size_attr.attr,
        &objs_per_slab_attr.attr,
        &order_attr.attr,
        &objects_attr.attr,
        &total_objects_attr.attr,
        &slabs_attr.attr,
        &ctor_attr.attr,
        &align_attr.attr,
        &hwcache_align_attr.attr,
        &reclaim_account_attr.attr,
        &destroy_by_rcu_attr.attr,
        &red_zone_attr.attr,
        &poison_attr.attr,
        &store_user_attr.attr,
        &hiwater_attr.attr,
        &freebatch_attr.attr,
#ifdef CONFIG_ZONE_DMA
        &cache_dma_attr.attr,
#endif
#ifdef CONFIG_SLQB_STATS
        &alloc_attr.attr,
        &alloc_slab_fill_attr.attr,
        &alloc_slab_new_attr.attr,
        &free_attr.attr,
        &free_remote_attr.attr,
        &flush_free_list_attr.attr,
        &flush_free_list_objects_attr.attr,
        &flush_free_list_remote_attr.attr,
        &flush_slab_partial_attr.attr,
        &flush_slab_free_attr.attr,
        &flush_rfree_list_attr.attr,
        &flush_rfree_list_objects_attr.attr,
        &claim_remote_list_attr.attr,
        &claim_remote_list_objects_attr.attr,
#endif
        NULL
};

static struct attribute_group slab_attr_group = {
        .attrs = slab_attrs,
};

static ssize_t slab_attr_show(struct kobject *kobj,
                                struct attribute *attr, char *buf)
{
        struct slab_attribute *attribute;
        struct kmem_cache *s;
        int err;

        attribute = to_slab_attr(attr);
        s = to_slab(kobj);

        if (!attribute->show)
                return -EIO;

        err = attribute->show(s, buf);

        return err;
}

static ssize_t slab_attr_store(struct kobject *kobj,
                        struct attribute *attr, const char *buf, size_t len)
{
        struct slab_attribute *attribute;
        struct kmem_cache *s;
        int err;

        attribute = to_slab_attr(attr);
        s = to_slab(kobj);

        if (!attribute->store)
                return -EIO;

        err = attribute->store(s, buf, len);

        return err;
}

static void kmem_cache_release(struct kobject *kobj)
{
        struct kmem_cache *s = to_slab(kobj);

        kmem_cache_free(&kmem_cache_cache, s);
}

static struct sysfs_ops slab_sysfs_ops = {
        .show = slab_attr_show,
        .store = slab_attr_store,
};

static struct kobj_type slab_ktype = {
        .sysfs_ops = &slab_sysfs_ops,
        .release = kmem_cache_release
};

static int uevent_filter(struct kset *kset, struct kobject *kobj)
{
        struct kobj_type *ktype = get_ktype(kobj);

        if (ktype == &slab_ktype)
                return 1;
        return 0;
}

static struct kset_uevent_ops slab_uevent_ops = {
        .filter = uevent_filter,
};

static struct kset *slab_kset;

static int sysfs_available __read_mostly;

static int sysfs_slab_add(struct kmem_cache *s)
{
        int err;

        if (!sysfs_available)
                return 0;

        s->kobj.kset = slab_kset;
        err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, s->name);
        if (err) {
                kobject_put(&s->kobj);
                return err;
        }

        err = sysfs_create_group(&s->kobj, &slab_attr_group);
        if (err)
                return err;

        kobject_uevent(&s->kobj, KOBJ_ADD);

        return 0;
}

static void sysfs_slab_remove(struct kmem_cache *s)
{
        kobject_uevent(&s->kobj, KOBJ_REMOVE);
        kobject_del(&s->kobj);
        kobject_put(&s->kobj);
}

static int __init slab_sysfs_init(void)
{
        struct kmem_cache *s;
        int err;

        slab_kset = kset_create_and_add("slab", &slab_uevent_ops, kernel_kobj);
        if (!slab_kset) {
                printk(KERN_ERR "Cannot register slab subsystem.\n");
                return -ENOSYS;
        }

        down_write(&slqb_lock);

        sysfs_available = 1;

        list_for_each_entry(s, &slab_caches, list) {
                err = sysfs_slab_add(s);
                if (err)
                        printk(KERN_ERR "SLQB: Unable to add boot slab %s"
                                                " to sysfs\n", s->name);
        }

        up_write(&slqb_lock);

        return 0;
}
device_initcall(slab_sysfs_init);

#endif