i386/linux/linux-2.3.21/mm/swap_state.c

   1 /*
   2  *  linux/mm/swap_state.c
   3  *
   4  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
   5  *  Swap reorganised 29.12.95, Stephen Tweedie
   6  *
   7  *  Rewritten to use page cache, (C) 1998 Stephen Tweedie
   8  */
   9
  10 #include <linux/mm.h>
  11 #include <linux/kernel_stat.h>
  12 #include <linux/swap.h>
  13 #include <linux/swapctl.h>
  14 #include <linux/init.h>
  15 #include <linux/pagemap.h>
  16
  17 #include <asm/pgtable.h>
  18
  19 /*
  20  * Keep a reserved false inode which we will use to mark pages in the
  21  * page cache are acting as swap cache instead of file cache.
  22  *
  23  * We only need a unique pointer to satisfy the page cache, but we'll
  24  * reserve an entire zeroed inode structure for the purpose just to
  25  * ensure that any mistaken dereferences of this structure cause a
  26  * kernel oops.
  27  */
  28
  29 static struct inode_operations swapper_inode_operations = {
  30         NULL,                           /* default file operations */
  31         NULL,                           /* create */
  32         NULL,                           /* lookup */
  33         NULL,                           /* link */
  34         NULL,                           /* unlink */
  35         NULL,                           /* symlink */
  36         NULL,                           /* mkdir */
  37         NULL,                           /* rmdir */
  38         NULL,                           /* mknod */
  39         NULL,                           /* rename */
  40         NULL,                           /* readlink */
  41         NULL,                           /* follow_link */
  42         NULL,                           /* get_block */
  43         NULL,                           /* readpage */
  44         NULL,                           /* writepage */
  45         block_flushpage,                /* flushpage */
  46         NULL,                           /* truncate */
  47         NULL,                           /* permission */
  48         NULL,                           /* smap */
  49         NULL                            /* revalidate */
  50 };
  51
  52 struct inode swapper_inode = { i_op: &swapper_inode_operations };
  53
  54 #ifdef SWAP_CACHE_INFO
  55 unsigned long swap_cache_add_total = 0;
  56 unsigned long swap_cache_del_total = 0;
  57 unsigned long swap_cache_find_total = 0;
  58 unsigned long swap_cache_find_success = 0;
  59
  60 void show_swap_cache_info(void)
  61 {
  62         printk("Swap cache: add %ld, delete %ld, find %ld/%ld\n",
  63                 swap_cache_add_total,
  64                 swap_cache_del_total,
  65                 swap_cache_find_success, swap_cache_find_total);
  66 }
  67 #endif
  68
  69 void add_to_swap_cache(struct page *page, unsigned long entry)
  70 {
  71 #ifdef SWAP_CACHE_INFO
  72         swap_cache_add_total++;
  73 #endif
  74 #ifdef DEBUG_SWAP
  75         printk("DebugVM: add_to_swap_cache(%08lx count %d, entry %08lx)\n",
  76                    page_address(page), page_count(page), entry);
  77 #endif
  78         if (PageTestandSetSwapCache(page)) {
  79                 printk(KERN_ERR "swap_cache: replacing non-empty entry %08lx "
  80                            "on page %08lx\n",
  81                            page->offset, page_address(page));
  82         }
  83         if (page->inode) {
  84                 printk(KERN_ERR "swap_cache: replacing page-cached entry "
  85                            "on page %08lx\n", page_address(page));
  86         }
  87         add_to_page_cache(page, &swapper_inode, entry);
  88 }
  89
  90 /*
  91  * Verify that a swap entry is valid and increment its swap map count.
  92  *
  93  * Note: if swap_map[] reaches SWAP_MAP_MAX the entries are treated as
  94  * "permanent", but will be reclaimed by the next swapoff.
  95  */
  96 int swap_duplicate(unsigned long entry)
  97 {
  98         struct swap_info_struct * p;
  99         unsigned long offset, type;
 100         int result = 0;
 101
 102         if (!entry)
 103                 goto out;
 104         type = SWP_TYPE(entry);
 105         if (type & SHM_SWP_TYPE)
 106                 goto out;
 107         if (type >= nr_swapfiles)
 108                 goto bad_file;
 109         p = type + swap_info;
 110         offset = SWP_OFFSET(entry);
 111         if (offset >= p->max)
 112                 goto bad_offset;
 113         if (!p->swap_map[offset])
 114                 goto bad_unused;
 115         /*
 116          * Entry is valid, so increment the map count.
 117          */
 118         if (p->swap_map[offset] < SWAP_MAP_MAX)
 119                 p->swap_map[offset]++;
 120         else {
 121                 static int overflow = 0;
 122                 if (overflow++ < 5)
 123                         printk(KERN_WARNING
 124                                 "swap_duplicate: entry %08lx map count=%d\n",
 125                                 entry, p->swap_map[offset]);
 126                 p->swap_map[offset] = SWAP_MAP_MAX;
 127         }
 128         result = 1;
 129 #ifdef DEBUG_SWAP
 130         printk("DebugVM: swap_duplicate(entry %08lx, count now %d)\n",
 131                    entry, p->swap_map[offset]);
 132 #endif
 133 out:
 134         return result;
 135
 136 bad_file:
 137         printk(KERN_ERR
 138                 "swap_duplicate: entry %08lx, nonexistent swap file\n", entry);
 139         goto out;
 140 bad_offset:
 141         printk(KERN_ERR
 142                 "swap_duplicate: entry %08lx, offset exceeds max\n", entry);
 143         goto out;
 144 bad_unused:
 145         printk(KERN_ERR
 146                 "swap_duplicate at %8p: entry %08lx, unused page\n",
 147                    __builtin_return_address(0), entry);
 148         goto out;
 149 }
 150
 151 int swap_count(unsigned long entry)
 152 {
 153         struct swap_info_struct * p;
 154         unsigned long offset, type;
 155         int retval = 0;
 156
 157         if (!entry)
 158                 goto bad_entry;
 159         type = SWP_TYPE(entry);
 160         if (type & SHM_SWP_TYPE)
 161                 goto out;
 162         if (type >= nr_swapfiles)
 163                 goto bad_file;
 164         p = type + swap_info;
 165         offset = SWP_OFFSET(entry);
 166         if (offset >= p->max)
 167                 goto bad_offset;
 168         if (!p->swap_map[offset])
 169                 goto bad_unused;
 170         retval = p->swap_map[offset];
 171 #ifdef DEBUG_SWAP
 172         printk("DebugVM: swap_count(entry %08lx, count %d)\n",
 173                    entry, retval);
 174 #endif
 175 out:
 176         return retval;
 177
 178 bad_entry:
 179         printk(KERN_ERR "swap_count: null entry!\n");
 180         goto out;
 181 bad_file:
 182         printk(KERN_ERR
 183                    "swap_count: entry %08lx, nonexistent swap file!\n", entry);
 184         goto out;
 185 bad_offset:
 186         printk(KERN_ERR
 187                    "swap_count: entry %08lx, offset exceeds max!\n", entry);
 188         goto out;
 189 bad_unused:
 190         printk(KERN_ERR
 191                    "swap_count at %8p: entry %08lx, unused page!\n",
 192                    __builtin_return_address(0), entry);
 193         goto out;
 194 }
 195
 196 static inline void remove_from_swap_cache(struct page *page)
 197 {
 198         struct inode *inode = page->inode;
 199
 200         if (!inode) {
 201                 printk ("VM: Removing swap cache page with zero inode hash "
 202                         "on page %08lx\n", page_address(page));
 203                 return;
 204         }
 205         if (inode != &swapper_inode) {
 206                 printk ("VM: Removing swap cache page with wrong inode hash "
 207                         "on page %08lx\n", page_address(page));
 208         }
 209         if (!PageSwapCache(page))
 210                 PAGE_BUG(page);
 211
 212 #ifdef DEBUG_SWAP
 213         printk("DebugVM: remove_from_swap_cache(%08lx count %d)\n",
 214                    page_address(page), page_count(page));
 215 #endif
 216         PageClearSwapCache(page);
 217         remove_inode_page(page);
 218 }
 219
 220 /*
 221  * This must be called only on pages that have
 222  * been verified to be in the swap cache.
 223  */
 224 void __delete_from_swap_cache(struct page *page)
 225 {
 226         long entry = page->offset;
 227
 228 #ifdef SWAP_CACHE_INFO
 229         swap_cache_del_total++;
 230 #endif
 231 #ifdef DEBUG_SWAP
 232         printk("DebugVM: delete_from_swap_cache(%08lx count %d, "
 233                    "entry %08lx)\n",
 234                    page_address(page), page_count(page), entry);
 235 #endif
 236         remove_from_swap_cache (page);
 237         swap_free (entry);
 238 }
 239
 240 static void delete_from_swap_cache_nolock(struct page *page)
 241 {
 242         if (!swapper_inode.i_op->flushpage ||
 243             swapper_inode.i_op->flushpage(&swapper_inode, page, 0))
 244                 lru_cache_del(page);
 245
 246         __delete_from_swap_cache(page);
 247 }
 248
 249 /*
 250  * This must be called only on pages that have
 251  * been verified to be in the swap cache.
 252  */
 253 void delete_from_swap_cache(struct page *page)
 254 {
 255         lock_page(page);
 256
 257         delete_from_swap_cache_nolock(page);
 258
 259         UnlockPage(page);
 260         page_cache_release(page);
 261 }
 262
 263 /*
 264  * Perform a free_page(), also freeing any swap cache associated with
 265  * this page if it is the last user of the page.
 266  */
 267
 268 void free_page_and_swap_cache(unsigned long addr)
 269 {
 270         struct page *page = mem_map + MAP_NR(addr);
 271
 272         /*
 273          * If we are the only user, then free up the swap cache.
 274          */
 275         lock_page(page);
 276         if (PageSwapCache(page) && !is_page_shared(page)) {
 277                 delete_from_swap_cache_nolock(page);
 278                 page_cache_release(page);
 279         }
 280         UnlockPage(page);
 281
 282         clear_bit(PG_swap_entry, &page->flags);
 283
 284         __free_page(page);
 285 }
 286
 287
 288 /*
 289  * Lookup a swap entry in the swap cache. A found page will be returned
 290  * unlocked and with its refcount incremented - we rely on the kernel
 291  * lock getting page table operations atomic even if we drop the page
 292  * lock before returning.
 293  */
 294
 295 struct page * lookup_swap_cache(unsigned long entry)
 296 {
 297         struct page *found;
 298
 299 #ifdef SWAP_CACHE_INFO
 300         swap_cache_find_total++;
 301 #endif
 302         while (1) {
 303                 found = find_lock_page(&swapper_inode, entry);
 304                 if (!found)
 305                         return 0;
 306                 if (found->inode != &swapper_inode || !PageSwapCache(found))
 307                         goto out_bad;
 308 #ifdef SWAP_CACHE_INFO
 309                 swap_cache_find_success++;
 310 #endif
 311                 UnlockPage(found);
 312                 return found;
 313         }
 314
 315 out_bad:
 316         printk (KERN_ERR "VM: Found a non-swapper swap page!\n");
 317         UnlockPage(found);
 318         __free_page(found);
 319         return 0;
 320 }
 321
 322 /*
 323  * Locate a page of swap in physical memory, reserving swap cache space
 324  * and reading the disk if it is not already cached.  If wait==0, we are
 325  * only doing readahead, so don't worry if the page is already locked.
 326  *
 327  * A failure return means that either the page allocation failed or that
 328  * the swap entry is no longer in use.
 329  */
 330
 331 struct page * read_swap_cache_async(unsigned long entry, int wait)
 332 {
 333         struct page *found_page = 0, *new_page;
 334         unsigned long new_page_addr;
 335
 336 #ifdef DEBUG_SWAP
 337         printk("DebugVM: read_swap_cache_async entry %08lx%s\n",
 338                    entry, wait ? ", wait" : "");
 339 #endif
 340         /*
 341          * Make sure the swap entry is still in use.
 342          */
 343         if (!swap_duplicate(entry))     /* Account for the swap cache */
 344                 goto out;
 345         /*
 346          * Look for the page in the swap cache.
 347          */
 348         found_page = lookup_swap_cache(entry);
 349         if (found_page)
 350                 goto out_free_swap;
 351
 352         new_page_addr = __get_free_page(GFP_USER);
 353         if (!new_page_addr)
 354                 goto out_free_swap;     /* Out of memory */
 355         new_page = mem_map + MAP_NR(new_page_addr);
 356
 357         /*
 358          * Check the swap cache again, in case we stalled above.
 359          */
 360         found_page = lookup_swap_cache(entry);
 361         if (found_page)
 362                 goto out_free_page;
 363         /*
 364          * Add it to the swap cache and read its contents.
 365          */
 366         add_to_swap_cache(new_page, entry);
 367         rw_swap_page(READ, new_page, wait);
 368 #ifdef DEBUG_SWAP
 369         printk("DebugVM: read_swap_cache_async created "
 370                    "entry %08lx at %p\n",
 371                         entry, (char *) page_address(new_page));
 372 #endif
 373         return new_page;
 374
 375 out_free_page:
 376         __free_page(new_page);
 377 out_free_swap:
 378         swap_free(entry);
 379 out:
 380         return found_page;
 381 }