include/linux/quicklist.h

   1 #ifndef LINUX_QUICKLIST_H
   2 #define LINUX_QUICKLIST_H
   3 /*
   4  * Fast allocations and disposal of pages. Pages must be in the condition
   5  * as needed after allocation when they are freed. Per cpu lists of pages
   6  * are kept that only contain node local pages.
   7  *
   8  * (C) 2007, SGI. Christoph Lameter <clameter@sgi.com>
   9  */
  10 #include <linux/kernel.h>
  11 #include <linux/gfp.h>
  12 #include <linux/percpu.h>
  13
  14 #ifdef CONFIG_QUICKLIST
  15
  16 struct quicklist {
  17         void *page;
  18         int nr_pages;
  19 };
  20
  21 DECLARE_PER_CPU(struct quicklist, quicklist)[CONFIG_NR_QUICK];
  22
  23 /*
  24  * The two key functions quicklist_alloc and quicklist_free are inline so
  25  * that they may be custom compiled for the platform.
  26  * Specifying a NULL ctor can remove constructor support. Specifying
  27  * a constant quicklist allows the determination of the exact address
  28  * in the per cpu area.
  29  *
  30  * The fast patch in quicklist_alloc touched only a per cpu cacheline and
  31  * the first cacheline of the page itself. There is minmal overhead involved.
  32  */
  33 static inline void *quicklist_alloc(int nr, gfp_t flags, void (*ctor)(void *))
  34 {
  35         struct quicklist *q;
  36         void **p = NULL;
  37
  38         q =&get_cpu_var(quicklist)[nr];
  39         p = q->page;
  40         if (likely(p)) {
  41                 q->page = p[0];
  42                 p[0] = NULL;
  43                 q->nr_pages--;
  44         }
  45         put_cpu_var(quicklist);
  46         if (likely(p))
  47                 return p;
  48
  49         p = (void *)__get_free_page(flags | __GFP_ZERO);
  50         if (ctor && p)
  51                 ctor(p);
  52         return p;
  53 }
  54
  55 static inline void __quicklist_free(int nr, void (*dtor)(void *), void *p,
  56         struct page *page)
  57 {
  58         struct quicklist *q;
  59         int nid = page_to_nid(page);
  60
  61         if (unlikely(nid != numa_node_id())) {
  62                 if (dtor)
  63                         dtor(p);
  64                 __free_page(page);
  65                 return;
  66         }
  67
  68         q = &get_cpu_var(quicklist)[nr];
  69         *(void **)p = q->page;
  70         q->page = p;
  71         q->nr_pages++;
  72         put_cpu_var(quicklist);
  73 }
  74
  75 static inline void quicklist_free(int nr, void (*dtor)(void *), void *pp)
  76 {
  77         __quicklist_free(nr, dtor, pp, virt_to_page(pp));
  78 }
  79
  80 static inline void quicklist_free_page(int nr, void (*dtor)(void *),
  81                                                         struct page *page)
  82 {
  83         __quicklist_free(nr, dtor, page_address(page), page);
  84 }
  85
  86 void quicklist_trim(int nr, void (*dtor)(void *),
  87         unsigned long min_pages, unsigned long max_free);
  88
  89 unsigned long quicklist_total_size(void);
  90
  91 #endif
  92
  93 #endif /* LINUX_QUICKLIST_H */
  94