mm/balloon_compaction.c

   1 /*
   2  * mm/balloon_compaction.c
   3  *
   4  * Common interface for making balloon pages movable by compaction.
   5  *
   6  * Copyright (C) 2012, Red Hat, Inc.  Rafael Aquini <aquini@redhat.com>
   7  */
   8 #include <linux/mm.h>
   9 #include <linux/slab.h>
  10 #include <linux/export.h>
  11 #include <linux/balloon_compaction.h>
  12
  13 /*
  14  * balloon_devinfo_alloc - allocates a balloon device information descriptor.
  15  * @balloon_dev_descriptor: pointer to reference the balloon device which
  16  *                          this struct balloon_dev_info will be servicing.
  17  *
  18  * Driver must call it to properly allocate and initialize an instance of
  19  * struct balloon_dev_info which will be used to reference a balloon device
  20  * as well as to keep track of the balloon device page list.
  21  */
  22 struct balloon_dev_info *balloon_devinfo_alloc(void *balloon_dev_descriptor)
  23 {
  24         struct balloon_dev_info *b_dev_info;
  25         b_dev_info = kmalloc(sizeof(*b_dev_info), GFP_KERNEL);
  26         if (!b_dev_info)
  27                 return ERR_PTR(-ENOMEM);
  28
  29         b_dev_info->balloon_device = balloon_dev_descriptor;
  30         b_dev_info->mapping = NULL;
  31         b_dev_info->isolated_pages = 0;
  32         spin_lock_init(&b_dev_info->pages_lock);
  33         INIT_LIST_HEAD(&b_dev_info->pages);
  34
  35         return b_dev_info;
  36 }
  37 EXPORT_SYMBOL_GPL(balloon_devinfo_alloc);
  38
  39 /*
  40  * balloon_page_enqueue - allocates a new page and inserts it into the balloon
  41  *                        page list.
  42  * @b_dev_info: balloon device decriptor where we will insert a new page to
  43  *
  44  * Driver must call it to properly allocate a new enlisted balloon page
  45  * before definetively removing it from the guest system.
  46  * This function returns the page address for the recently enqueued page or
  47  * NULL in the case we fail to allocate a new page this turn.
  48  */
  49 struct page *balloon_page_enqueue(struct balloon_dev_info *b_dev_info)
  50 {
  51         unsigned long flags;
  52         struct page *page = alloc_page(balloon_mapping_gfp_mask() |
  53                                         __GFP_NOMEMALLOC | __GFP_NORETRY);
  54         if (!page)
  55                 return NULL;
  56
  57         /*
  58          * Block others from accessing the 'page' when we get around to
  59          * establishing additional references. We should be the only one
  60          * holding a reference to the 'page' at this point.
  61          */
  62         BUG_ON(!trylock_page(page));
  63         spin_lock_irqsave(&b_dev_info->pages_lock, flags);
  64         balloon_page_insert(page, b_dev_info->mapping, &b_dev_info->pages);
  65         spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
  66         unlock_page(page);
  67         return page;
  68 }
  69 EXPORT_SYMBOL_GPL(balloon_page_enqueue);
  70
  71 /*
  72  * balloon_page_dequeue - removes a page from balloon's page list and returns
  73  *                        the its address to allow the driver release the page.
  74  * @b_dev_info: balloon device decriptor where we will grab a page from.
  75  *
  76  * Driver must call it to properly de-allocate a previous enlisted balloon page
  77  * before definetively releasing it back to the guest system.
  78  * This function returns the page address for the recently dequeued page or
  79  * NULL in the case we find balloon's page list temporarily empty due to
  80  * compaction isolated pages.
  81  */
  82 struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info)
  83 {
  84         struct page *page, *tmp;
  85         unsigned long flags;
  86         bool dequeued_page;
  87
  88         dequeued_page = false;
  89         list_for_each_entry_safe(page, tmp, &b_dev_info->pages, lru) {
  90                 /*
  91                  * Block others from accessing the 'page' while we get around
  92                  * establishing additional references and preparing the 'page'
  93                  * to be released by the balloon driver.
  94                  */
  95                 if (trylock_page(page)) {
  96                         spin_lock_irqsave(&b_dev_info->pages_lock, flags);
  97                         /*
  98                          * Raise the page refcount here to prevent any wrong
  99                          * attempt to isolate this page, in case of coliding
 100                          * with balloon_page_isolate() just after we release
 101                          * the page lock.
 102                          *
 103                          * balloon_page_free() will take care of dropping
 104                          * this extra refcount later.
 105                          */
 106                         get_page(page);
 107                         balloon_page_delete(page);
 108                         spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
 109                         unlock_page(page);
 110                         dequeued_page = true;
 111                         break;
 112                 }
 113         }
 114
 115         if (!dequeued_page) {
 116                 /*
 117                  * If we are unable to dequeue a balloon page because the page
 118                  * list is empty and there is no isolated pages, then something
 119                  * went out of track and some balloon pages are lost.
 120                  * BUG() here, otherwise the balloon driver may get stuck into
 121                  * an infinite loop while attempting to release all its pages.
 122                  */
 123                 spin_lock_irqsave(&b_dev_info->pages_lock, flags);
 124                 if (unlikely(list_empty(&b_dev_info->pages) &&
 125                              !b_dev_info->isolated_pages))
 126                         BUG();
 127                 spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
 128                 page = NULL;
 129         }
 130         return page;
 131 }
 132 EXPORT_SYMBOL_GPL(balloon_page_dequeue);
 133
 134 #ifdef CONFIG_BALLOON_COMPACTION
 135 /*
 136  * balloon_mapping_alloc - allocates a special ->mapping for ballooned pages.
 137  * @b_dev_info: holds the balloon device information descriptor.
 138  * @a_ops: balloon_mapping address_space_operations descriptor.
 139  *
 140  * Driver must call it to properly allocate and initialize an instance of
 141  * struct address_space which will be used as the special page->mapping for
 142  * balloon device enlisted page instances.
 143  */
 144 struct address_space *balloon_mapping_alloc(struct balloon_dev_info *b_dev_info,
 145                                 const struct address_space_operations *a_ops)
 146 {
 147         struct address_space *mapping;
 148
 149         mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
 150         if (!mapping)
 151                 return ERR_PTR(-ENOMEM);
 152
 153         /*
 154          * Give a clean 'zeroed' status to all elements of this special
 155          * balloon page->mapping struct address_space instance.
 156          */
 157         address_space_init_once(mapping);
 158
 159         /*
 160          * Set mapping->flags appropriately, to allow balloon pages
 161          * ->mapping identification.
 162          */
 163         mapping_set_balloon(mapping);
 164         mapping_set_gfp_mask(mapping, balloon_mapping_gfp_mask());
 165
 166         /* balloon's page->mapping->a_ops callback descriptor */
 167         mapping->a_ops = a_ops;
 168
 169         /*
 170          * Establish a pointer reference back to the balloon device descriptor
 171          * this particular page->mapping will be servicing.
 172          * This is used by compaction / migration procedures to identify and
 173          * access the balloon device pageset while isolating / migrating pages.
 174          *
 175          * As some balloon drivers can register multiple balloon devices
 176          * for a single guest, this also helps compaction / migration to
 177          * properly deal with multiple balloon pagesets, when required.
 178          */
 179         mapping->private_data = b_dev_info;
 180         b_dev_info->mapping = mapping;
 181
 182         return mapping;
 183 }
 184 EXPORT_SYMBOL_GPL(balloon_mapping_alloc);
 185
 186 static inline void __isolate_balloon_page(struct page *page)
 187 {
 188         struct balloon_dev_info *b_dev_info = page->mapping->private_data;
 189         unsigned long flags;
 190         spin_lock_irqsave(&b_dev_info->pages_lock, flags);
 191         list_del(&page->lru);
 192         b_dev_info->isolated_pages++;
 193         spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
 194 }
 195
 196 static inline void __putback_balloon_page(struct page *page)
 197 {
 198         struct balloon_dev_info *b_dev_info = page->mapping->private_data;
 199         unsigned long flags;
 200         spin_lock_irqsave(&b_dev_info->pages_lock, flags);
 201         list_add(&page->lru, &b_dev_info->pages);
 202         b_dev_info->isolated_pages--;
 203         spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
 204 }
 205
 206 static inline int __migrate_balloon_page(struct address_space *mapping,
 207                 struct page *newpage, struct page *page, enum migrate_mode mode)
 208 {
 209         return page->mapping->a_ops->migratepage(mapping, newpage, page, mode);
 210 }
 211
 212 /* __isolate_lru_page() counterpart for a ballooned page */
 213 bool balloon_page_isolate(struct page *page)
 214 {
 215         /*
 216          * Avoid burning cycles with pages that are yet under __free_pages(),
 217          * or just got freed under us.
 218          *
 219          * In case we 'win' a race for a balloon page being freed under us and
 220          * raise its refcount preventing __free_pages() from doing its job
 221          * the put_page() at the end of this block will take care of
 222          * release this page, thus avoiding a nasty leakage.
 223          */
 224         if (likely(get_page_unless_zero(page))) {
 225                 /*
 226                  * As balloon pages are not isolated from LRU lists, concurrent
 227                  * compaction threads can race against page migration functions
 228                  * as well as race against the balloon driver releasing a page.
 229                  *
 230                  * In order to avoid having an already isolated balloon page
 231                  * being (wrongly) re-isolated while it is under migration,
 232                  * or to avoid attempting to isolate pages being released by
 233                  * the balloon driver, lets be sure we have the page lock
 234                  * before proceeding with the balloon page isolation steps.
 235                  */
 236                 if (likely(trylock_page(page))) {
 237                         /*
 238                          * A ballooned page, by default, has just one refcount.
 239                          * Prevent concurrent compaction threads from isolating
 240                          * an already isolated balloon page by refcount check.
 241                          */
 242                         if (__is_movable_balloon_page(page) &&
 243                             page_count(page) == 2) {
 244                                 __isolate_balloon_page(page);
 245                                 unlock_page(page);
 246                                 return true;
 247                         }
 248                         unlock_page(page);
 249                 }
 250                 put_page(page);
 251         }
 252         return false;
 253 }
 254
 255 /* putback_lru_page() counterpart for a ballooned page */
 256 void balloon_page_putback(struct page *page)
 257 {
 258         /*
 259          * 'lock_page()' stabilizes the page and prevents races against
 260          * concurrent isolation threads attempting to re-isolate it.
 261          */
 262         lock_page(page);
 263
 264         if (__is_movable_balloon_page(page)) {
 265                 __putback_balloon_page(page);
 266                 /* drop the extra ref count taken for page isolation */
 267                 put_page(page);
 268         } else {
 269                 WARN_ON(1);
 270                 dump_page(page);
 271         }
 272         unlock_page(page);
 273 }
 274
 275 /* move_to_new_page() counterpart for a ballooned page */
 276 int balloon_page_migrate(struct page *newpage,
 277                          struct page *page, enum migrate_mode mode)
 278 {
 279         struct address_space *mapping;
 280         int rc = -EAGAIN;
 281
 282         /*
 283          * Block others from accessing the 'newpage' when we get around to
 284          * establishing additional references. We should be the only one
 285          * holding a reference to the 'newpage' at this point.
 286          */
 287         BUG_ON(!trylock_page(newpage));
 288
 289         if (WARN_ON(!__is_movable_balloon_page(page))) {
 290                 dump_page(page);
 291                 unlock_page(newpage);
 292                 return rc;
 293         }
 294
 295         mapping = page->mapping;
 296         if (mapping)
 297                 rc = __migrate_balloon_page(mapping, newpage, page, mode);
 298
 299         unlock_page(newpage);
 300         return rc;
 301 }
 302 #endif /* CONFIG_BALLOON_COMPACTION */