mm/swap_cgroup.c

   1 #include <linux/swap_cgroup.h>
   2 #include <linux/vmalloc.h>
   3 #include <linux/mm.h>
   4
   5 #include <linux/swapops.h> /* depends on mm.h include */
   6
   7 static DEFINE_MUTEX(swap_cgroup_mutex);
   8 struct swap_cgroup_ctrl {
   9         struct page **map;
  10         unsigned long length;
  11         spinlock_t      lock;
  12 };
  13
  14 static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];
  15
  16 struct swap_cgroup {
  17         unsigned short          id;
  18 };
  19 #define SC_PER_PAGE     (PAGE_SIZE/sizeof(struct swap_cgroup))
  20
  21 /*
  22  * SwapCgroup implements "lookup" and "exchange" operations.
  23  * In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge
  24  * against SwapCache. At swap_free(), this is accessed directly from swap.
  25  *
  26  * This means,
  27  *  - we have no race in "exchange" when we're accessed via SwapCache because
  28  *    SwapCache(and its swp_entry) is under lock.
  29  *  - When called via swap_free(), there is no user of this entry and no race.
  30  * Then, we don't need lock around "exchange".
  31  *
  32  * TODO: we can push these buffers out to HIGHMEM.
  33  */
  34
  35 /*
  36  * allocate buffer for swap_cgroup.
  37  */
  38 static int swap_cgroup_prepare(int type)
  39 {
  40         struct page *page;
  41         struct swap_cgroup_ctrl *ctrl;
  42         unsigned long idx, max;
  43
  44         ctrl = &swap_cgroup_ctrl[type];
  45
  46         for (idx = 0; idx < ctrl->length; idx++) {
  47                 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  48                 if (!page)
  49                         goto not_enough_page;
  50                 ctrl->map[idx] = page;
  51
  52                 if (!(idx % SWAP_CLUSTER_MAX))
  53                         cond_resched();
  54         }
  55         return 0;
  56 not_enough_page:
  57         max = idx;
  58         for (idx = 0; idx < max; idx++)
  59                 __free_page(ctrl->map[idx]);
  60
  61         return -ENOMEM;
  62 }
  63
  64 static struct swap_cgroup *__lookup_swap_cgroup(struct swap_cgroup_ctrl *ctrl,
  65                                                 pgoff_t offset)
  66 {
  67         struct page *mappage;
  68         struct swap_cgroup *sc;
  69
  70         mappage = ctrl->map[offset / SC_PER_PAGE];
  71         sc = page_address(mappage);
  72         return sc + offset % SC_PER_PAGE;
  73 }
  74
  75 static struct swap_cgroup *lookup_swap_cgroup(swp_entry_t ent,
  76                                         struct swap_cgroup_ctrl **ctrlp)
  77 {
  78         pgoff_t offset = swp_offset(ent);
  79         struct swap_cgroup_ctrl *ctrl;
  80
  81         ctrl = &swap_cgroup_ctrl[swp_type(ent)];
  82         if (ctrlp)
  83                 *ctrlp = ctrl;
  84         return __lookup_swap_cgroup(ctrl, offset);
  85 }
  86
  87 /**
  88  * swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry.
  89  * @ent: swap entry to be cmpxchged
  90  * @old: old id
  91  * @new: new id
  92  *
  93  * Returns old id at success, 0 at failure.
  94  * (There is no mem_cgroup using 0 as its id)
  95  */
  96 unsigned short swap_cgroup_cmpxchg(swp_entry_t ent,
  97                                         unsigned short old, unsigned short new)
  98 {
  99         struct swap_cgroup_ctrl *ctrl;
 100         struct swap_cgroup *sc;
 101         unsigned long flags;
 102         unsigned short retval;
 103
 104         sc = lookup_swap_cgroup(ent, &ctrl);
 105
 106         spin_lock_irqsave(&ctrl->lock, flags);
 107         retval = sc->id;
 108         if (retval == old)
 109                 sc->id = new;
 110         else
 111                 retval = 0;
 112         spin_unlock_irqrestore(&ctrl->lock, flags);
 113         return retval;
 114 }
 115
 116 /**
 117  * swap_cgroup_record - record mem_cgroup for a set of swap entries
 118  * @ent: the first swap entry to be recorded into
 119  * @id: mem_cgroup to be recorded
 120  * @nr_ents: number of swap entries to be recorded
 121  *
 122  * Returns old value at success, 0 at failure.
 123  * (Of course, old value can be 0.)
 124  */
 125 unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id,
 126                                   unsigned int nr_ents)
 127 {
 128         struct swap_cgroup_ctrl *ctrl;
 129         struct swap_cgroup *sc;
 130         unsigned short old;
 131         unsigned long flags;
 132         pgoff_t offset = swp_offset(ent);
 133         pgoff_t end = offset + nr_ents;
 134
 135         sc = lookup_swap_cgroup(ent, &ctrl);
 136
 137         spin_lock_irqsave(&ctrl->lock, flags);
 138         old = sc->id;
 139         for (;;) {
 140                 VM_BUG_ON(sc->id != old);
 141                 sc->id = id;
 142                 offset++;
 143                 if (offset == end)
 144                         break;
 145                 if (offset % SC_PER_PAGE)
 146                         sc++;
 147                 else
 148                         sc = __lookup_swap_cgroup(ctrl, offset);
 149         }
 150         spin_unlock_irqrestore(&ctrl->lock, flags);
 151
 152         return old;
 153 }
 154
 155 /**
 156  * lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry
 157  * @ent: swap entry to be looked up.
 158  *
 159  * Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
 160  */
 161 unsigned short lookup_swap_cgroup_id(swp_entry_t ent)
 162 {
 163         return lookup_swap_cgroup(ent, NULL)->id;
 164 }
 165
 166 int swap_cgroup_swapon(int type, unsigned long max_pages)
 167 {
 168         void *array;
 169         unsigned long array_size;
 170         unsigned long length;
 171         struct swap_cgroup_ctrl *ctrl;
 172
 173         if (!do_swap_account)
 174                 return 0;
 175
 176         length = DIV_ROUND_UP(max_pages, SC_PER_PAGE);
 177         array_size = length * sizeof(void *);
 178
 179         array = vzalloc(array_size);
 180         if (!array)
 181                 goto nomem;
 182
 183         ctrl = &swap_cgroup_ctrl[type];
 184         mutex_lock(&swap_cgroup_mutex);
 185         ctrl->length = length;
 186         ctrl->map = array;
 187         spin_lock_init(&ctrl->lock);
 188         if (swap_cgroup_prepare(type)) {
 189                 /* memory shortage */
 190                 ctrl->map = NULL;
 191                 ctrl->length = 0;
 192                 mutex_unlock(&swap_cgroup_mutex);
 193                 vfree(array);
 194                 goto nomem;
 195         }
 196         mutex_unlock(&swap_cgroup_mutex);
 197
 198         return 0;
 199 nomem:
 200         pr_info("couldn't allocate enough memory for swap_cgroup\n");
 201         pr_info("swap_cgroup can be disabled by swapaccount=0 boot option\n");
 202         return -ENOMEM;
 203 }
 204
 205 void swap_cgroup_swapoff(int type)
 206 {
 207         struct page **map;
 208         unsigned long i, length;
 209         struct swap_cgroup_ctrl *ctrl;
 210
 211         if (!do_swap_account)
 212                 return;
 213
 214         mutex_lock(&swap_cgroup_mutex);
 215         ctrl = &swap_cgroup_ctrl[type];
 216         map = ctrl->map;
 217         length = ctrl->length;
 218         ctrl->map = NULL;
 219         ctrl->length = 0;
 220         mutex_unlock(&swap_cgroup_mutex);
 221
 222         if (map) {
 223                 for (i = 0; i < length; i++) {
 224                         struct page *page = map[i];
 225                         if (page)
 226                                 __free_page(page);
 227                         if (!(i % SWAP_CLUSTER_MAX))
 228                                 cond_resched();
 229                 }
 230                 vfree(map);
 231         }
 232 }