mm/highmem.c

   1 /*
   2  * High memory handling common code and variables.
   3  *
   4  * (C) 1999 Andrea Arcangeli, SuSE GmbH, andrea@suse.de
   5  *          Gerhard Wichert, Siemens AG, Gerhard.Wichert@pdb.siemens.de
   6  *
   7  *
   8  * Redesigned the x86 32-bit VM architecture to deal with
   9  * 64-bit physical space. With current x86 CPUs this
  10  * means up to 64 Gigabytes physical RAM.
  11  *
  12  * Rewrote high memory support to move the page cache into
  13  * high memory. Implemented permanent (schedulable) kmaps
  14  * based on Linus' idea.
  15  *
  16  * Copyright (C) 1999 Ingo Molnar <mingo@redhat.com>
  17  */
  18
  19 #include <linux/mm.h>
  20 #include <linux/export.h>
  21 #include <linux/swap.h>
  22 #include <linux/bio.h>
  23 #include <linux/pagemap.h>
  24 #include <linux/mempool.h>
  25 #include <linux/blkdev.h>
  26 #include <linux/init.h>
  27 #include <linux/hash.h>
  28 #include <linux/highmem.h>
  29 #include <linux/kgdb.h>
  30 #include <asm/tlbflush.h>
  31
  32 #ifndef CONFIG_PREEMPT_RT_FULL
  33 #if defined(CONFIG_HIGHMEM) || defined(CONFIG_X86_32)
  34 DEFINE_PER_CPU(int, __kmap_atomic_idx);
  35 #endif
  36 #endif
  37
  38 /*
  39  * Virtual_count is not a pure "count".
  40  *  0 means that it is not mapped, and has not been mapped
  41  *    since a TLB flush - it is usable.
  42  *  1 means that there are no users, but it has been mapped
  43  *    since the last TLB flush - so we can't use it.
  44  *  n means that there are (n-1) current users of it.
  45  */
  46 #ifdef CONFIG_HIGHMEM
  47
  48 unsigned long totalhigh_pages __read_mostly;
  49 EXPORT_SYMBOL(totalhigh_pages);
  50
  51 #ifndef CONFIG_PREEMPT_RT_FULL
  52 EXPORT_PER_CPU_SYMBOL(__kmap_atomic_idx);
  53 #endif
  54
  55 unsigned int nr_free_highpages (void)
  56 {
  57         pg_data_t *pgdat;
  58         unsigned int pages = 0;
  59
  60         for_each_online_pgdat(pgdat) {
  61                 pages += zone_page_state(&pgdat->node_zones[ZONE_HIGHMEM],
  62                         NR_FREE_PAGES);
  63                 if (zone_movable_is_highmem())
  64                         pages += zone_page_state(
  65                                         &pgdat->node_zones[ZONE_MOVABLE],
  66                                         NR_FREE_PAGES);
  67         }
  68
  69         return pages;
  70 }
  71
  72 static int pkmap_count[LAST_PKMAP];
  73 static unsigned int last_pkmap_nr;
  74 static  __cacheline_aligned_in_smp DEFINE_SPINLOCK(kmap_lock);
  75
  76 pte_t * pkmap_page_table;
  77
  78 static DECLARE_WAIT_QUEUE_HEAD(pkmap_map_wait);
  79
  80 /*
  81  * Most architectures have no use for kmap_high_get(), so let's abstract
  82  * the disabling of IRQ out of the locking in that case to save on a
  83  * potential useless overhead.
  84  */
  85 #ifdef ARCH_NEEDS_KMAP_HIGH_GET
  86 #define lock_kmap()             spin_lock_irq(&kmap_lock)
  87 #define unlock_kmap()           spin_unlock_irq(&kmap_lock)
  88 #define lock_kmap_any(flags)    spin_lock_irqsave(&kmap_lock, flags)
  89 #define unlock_kmap_any(flags)  spin_unlock_irqrestore(&kmap_lock, flags)
  90 #else
  91 #define lock_kmap()             spin_lock(&kmap_lock)
  92 #define unlock_kmap()           spin_unlock(&kmap_lock)
  93 #define lock_kmap_any(flags)    \
  94                 do { spin_lock(&kmap_lock); (void)(flags); } while (0)
  95 #define unlock_kmap_any(flags)  \
  96                 do { spin_unlock(&kmap_lock); (void)(flags); } while (0)
  97 #endif
  98
  99 struct page *kmap_to_page(void *vaddr)
 100 {
 101         unsigned long addr = (unsigned long)vaddr;
 102
 103         if (addr >= PKMAP_ADDR(0) && addr < PKMAP_ADDR(LAST_PKMAP)) {
 104                 int i = PKMAP_NR(addr);
 105                 return pte_page(pkmap_page_table[i]);
 106         }
 107
 108         return virt_to_page(addr);
 109 }
 110 EXPORT_SYMBOL(kmap_to_page);
 111
 112 static void flush_all_zero_pkmaps(void)
 113 {
 114         int i;
 115         int need_flush = 0;
 116
 117         flush_cache_kmaps();
 118
 119         for (i = 0; i < LAST_PKMAP; i++) {
 120                 struct page *page;
 121
 122                 /*
 123                  * zero means we don't have anything to do,
 124                  * >1 means that it is still in use. Only
 125                  * a count of 1 means that it is free but
 126                  * needs to be unmapped
 127                  */
 128                 if (pkmap_count[i] != 1)
 129                         continue;
 130                 pkmap_count[i] = 0;
 131
 132                 /* sanity check */
 133                 BUG_ON(pte_none(pkmap_page_table[i]));
 134
 135                 /*
 136                  * Don't need an atomic fetch-and-clear op here;
 137                  * no-one has the page mapped, and cannot get at
 138                  * its virtual address (and hence PTE) without first
 139                  * getting the kmap_lock (which is held here).
 140                  * So no dangers, even with speculative execution.
 141                  */
 142                 page = pte_page(pkmap_page_table[i]);
 143                 pte_clear(&init_mm, PKMAP_ADDR(i), &pkmap_page_table[i]);
 144
 145                 set_page_address(page, NULL);
 146                 need_flush = 1;
 147         }
 148         if (need_flush)
 149                 flush_tlb_kernel_range(PKMAP_ADDR(0), PKMAP_ADDR(LAST_PKMAP));
 150 }
 151
 152 /**
 153  * kmap_flush_unused - flush all unused kmap mappings in order to remove stray mappings
 154  */
 155 void kmap_flush_unused(void)
 156 {
 157         lock_kmap();
 158         flush_all_zero_pkmaps();
 159         unlock_kmap();
 160 }
 161
 162 static inline unsigned long map_new_virtual(struct page *page)
 163 {
 164         unsigned long vaddr;
 165         int count;
 166
 167 start:
 168         count = LAST_PKMAP;
 169         /* Find an empty entry */
 170         for (;;) {
 171                 last_pkmap_nr = (last_pkmap_nr + 1) & LAST_PKMAP_MASK;
 172                 if (!last_pkmap_nr) {
 173                         flush_all_zero_pkmaps();
 174                         count = LAST_PKMAP;
 175                 }
 176                 if (!pkmap_count[last_pkmap_nr])
 177                         break;  /* Found a usable entry */
 178                 if (--count)
 179                         continue;
 180
 181                 /*
 182                  * Sleep for somebody else to unmap their entries
 183                  */
 184                 {
 185                         DECLARE_WAITQUEUE(wait, current);
 186
 187                         __set_current_state(TASK_UNINTERRUPTIBLE);
 188                         add_wait_queue(&pkmap_map_wait, &wait);
 189                         unlock_kmap();
 190                         schedule();
 191                         remove_wait_queue(&pkmap_map_wait, &wait);
 192                         lock_kmap();
 193
 194                         /* Somebody else might have mapped it while we slept */
 195                         if (page_address(page))
 196                                 return (unsigned long)page_address(page);
 197
 198                         /* Re-start */
 199                         goto start;
 200                 }
 201         }
 202         vaddr = PKMAP_ADDR(last_pkmap_nr);
 203         set_pte_at(&init_mm, vaddr,
 204                    &(pkmap_page_table[last_pkmap_nr]), mk_pte(page, kmap_prot));
 205
 206         pkmap_count[last_pkmap_nr] = 1;
 207         set_page_address(page, (void *)vaddr);
 208
 209         return vaddr;
 210 }
 211
 212 /**
 213  * kmap_high - map a highmem page into memory
 214  * @page: &struct page to map
 215  *
 216  * Returns the page's virtual memory address.
 217  *
 218  * We cannot call this from interrupts, as it may block.
 219  */
 220 void *kmap_high(struct page *page)
 221 {
 222         unsigned long vaddr;
 223
 224         /*
 225          * For highmem pages, we can't trust "virtual" until
 226          * after we have the lock.
 227          */
 228         lock_kmap();
 229         vaddr = (unsigned long)page_address(page);
 230         if (!vaddr)
 231                 vaddr = map_new_virtual(page);
 232         pkmap_count[PKMAP_NR(vaddr)]++;
 233         BUG_ON(pkmap_count[PKMAP_NR(vaddr)] < 2);
 234         unlock_kmap();
 235         return (void*) vaddr;
 236 }
 237
 238 EXPORT_SYMBOL(kmap_high);
 239
 240 #ifdef ARCH_NEEDS_KMAP_HIGH_GET
 241 /**
 242  * kmap_high_get - pin a highmem page into memory
 243  * @page: &struct page to pin
 244  *
 245  * Returns the page's current virtual memory address, or NULL if no mapping
 246  * exists.  If and only if a non null address is returned then a
 247  * matching call to kunmap_high() is necessary.
 248  *
 249  * This can be called from any context.
 250  */
 251 void *kmap_high_get(struct page *page)
 252 {
 253         unsigned long vaddr, flags;
 254
 255         lock_kmap_any(flags);
 256         vaddr = (unsigned long)page_address(page);
 257         if (vaddr) {
 258                 BUG_ON(pkmap_count[PKMAP_NR(vaddr)] < 1);
 259                 pkmap_count[PKMAP_NR(vaddr)]++;
 260         }
 261         unlock_kmap_any(flags);
 262         return (void*) vaddr;
 263 }
 264 #endif
 265
 266 /**
 267  * kunmap_high - unmap a highmem page into memory
 268  * @page: &struct page to unmap
 269  *
 270  * If ARCH_NEEDS_KMAP_HIGH_GET is not defined then this may be called
 271  * only from user context.
 272  */
 273 void kunmap_high(struct page *page)
 274 {
 275         unsigned long vaddr;
 276         unsigned long nr;
 277         unsigned long flags;
 278         int need_wakeup;
 279
 280         lock_kmap_any(flags);
 281         vaddr = (unsigned long)page_address(page);
 282         BUG_ON(!vaddr);
 283         nr = PKMAP_NR(vaddr);
 284
 285         /*
 286          * A count must never go down to zero
 287          * without a TLB flush!
 288          */
 289         need_wakeup = 0;
 290         switch (--pkmap_count[nr]) {
 291         case 0:
 292                 BUG();
 293         case 1:
 294                 /*
 295                  * Avoid an unnecessary wake_up() function call.
 296                  * The common case is pkmap_count[] == 1, but
 297                  * no waiters.
 298                  * The tasks queued in the wait-queue are guarded
 299                  * by both the lock in the wait-queue-head and by
 300                  * the kmap_lock.  As the kmap_lock is held here,
 301                  * no need for the wait-queue-head's lock.  Simply
 302                  * test if the queue is empty.
 303                  */
 304                 need_wakeup = waitqueue_active(&pkmap_map_wait);
 305         }
 306         unlock_kmap_any(flags);
 307
 308         /* do wake-up, if needed, race-free outside of the spin lock */
 309         if (need_wakeup)
 310                 wake_up(&pkmap_map_wait);
 311 }
 312
 313 EXPORT_SYMBOL(kunmap_high);
 314 #endif
 315
 316 #if defined(HASHED_PAGE_VIRTUAL)
 317
 318 #define PA_HASH_ORDER   7
 319
 320 /*
 321  * Describes one page->virtual association
 322  */
 323 struct page_address_map {
 324         struct page *page;
 325         void *virtual;
 326         struct list_head list;
 327 };
 328
 329 static struct page_address_map page_address_maps[LAST_PKMAP];
 330
 331 /*
 332  * Hash table bucket
 333  */
 334 static struct page_address_slot {
 335         struct list_head lh;                    /* List of page_address_maps */
 336         spinlock_t lock;                        /* Protect this bucket's list */
 337 } ____cacheline_aligned_in_smp page_address_htable[1<<PA_HASH_ORDER];
 338
 339 static struct page_address_slot *page_slot(const struct page *page)
 340 {
 341         return &page_address_htable[hash_ptr(page, PA_HASH_ORDER)];
 342 }
 343
 344 /**
 345  * page_address - get the mapped virtual address of a page
 346  * @page: &struct page to get the virtual address of
 347  *
 348  * Returns the page's virtual address.
 349  */
 350 void *page_address(const struct page *page)
 351 {
 352         unsigned long flags;
 353         void *ret;
 354         struct page_address_slot *pas;
 355
 356         if (!PageHighMem(page))
 357                 return lowmem_page_address(page);
 358
 359         pas = page_slot(page);
 360         ret = NULL;
 361         spin_lock_irqsave(&pas->lock, flags);
 362         if (!list_empty(&pas->lh)) {
 363                 struct page_address_map *pam;
 364
 365                 list_for_each_entry(pam, &pas->lh, list) {
 366                         if (pam->page == page) {
 367                                 ret = pam->virtual;
 368                                 goto done;
 369                         }
 370                 }
 371         }
 372 done:
 373         spin_unlock_irqrestore(&pas->lock, flags);
 374         return ret;
 375 }
 376
 377 EXPORT_SYMBOL(page_address);
 378
 379 /**
 380  * set_page_address - set a page's virtual address
 381  * @page: &struct page to set
 382  * @virtual: virtual address to use
 383  */
 384 void set_page_address(struct page *page, void *virtual)
 385 {
 386         unsigned long flags;
 387         struct page_address_slot *pas;
 388         struct page_address_map *pam;
 389
 390         BUG_ON(!PageHighMem(page));
 391
 392         pas = page_slot(page);
 393         if (virtual) {          /* Add */
 394                 pam = &page_address_maps[PKMAP_NR((unsigned long)virtual)];
 395                 pam->page = page;
 396                 pam->virtual = virtual;
 397
 398                 spin_lock_irqsave(&pas->lock, flags);
 399                 list_add_tail(&pam->list, &pas->lh);
 400                 spin_unlock_irqrestore(&pas->lock, flags);
 401         } else {                /* Remove */
 402                 spin_lock_irqsave(&pas->lock, flags);
 403                 list_for_each_entry(pam, &pas->lh, list) {
 404                         if (pam->page == page) {
 405                                 list_del(&pam->list);
 406                                 spin_unlock_irqrestore(&pas->lock, flags);
 407                                 goto done;
 408                         }
 409                 }
 410                 spin_unlock_irqrestore(&pas->lock, flags);
 411         }
 412 done:
 413         return;
 414 }
 415
 416 void __init page_address_init(void)
 417 {
 418         int i;
 419
 420         for (i = 0; i < ARRAY_SIZE(page_address_htable); i++) {
 421                 INIT_LIST_HEAD(&page_address_htable[i].lh);
 422                 spin_lock_init(&page_address_htable[i].lock);
 423         }
 424 }
 425
 426 #endif  /* defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL) */