arch/tile/mm/highmem.c

   1 /*
   2  * Copyright 2010 Tilera Corporation. All Rights Reserved.
   3  *
   4  *   This program is free software; you can redistribute it and/or
   5  *   modify it under the terms of the GNU General Public License
   6  *   as published by the Free Software Foundation, version 2.
   7  *
   8  *   This program is distributed in the hope that it will be useful, but
   9  *   WITHOUT ANY WARRANTY; without even the implied warranty of
  10  *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
  11  *   NON INFRINGEMENT.  See the GNU General Public License for
  12  *   more details.
  13  */
  14
  15 #include <linux/highmem.h>
  16 #include <linux/module.h>
  17 #include <linux/pagemap.h>
  18 #include <asm/homecache.h>
  19
  20 #define kmap_get_pte(vaddr) \
  21         pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr), (vaddr)),\
  22                 (vaddr)), (vaddr))
  23
  24
  25 void *kmap(struct page *page)
  26 {
  27         void *kva;
  28         unsigned long flags;
  29         pte_t *ptep;
  30
  31         might_sleep();
  32         if (!PageHighMem(page))
  33                 return page_address(page);
  34         kva = kmap_high(page);
  35
  36         /*
  37          * Rewrite the PTE under the lock.  This ensures that the page
  38          * is not currently migrating.
  39          */
  40         ptep = kmap_get_pte((unsigned long)kva);
  41         flags = homecache_kpte_lock();
  42         set_pte_at(&init_mm, kva, ptep, mk_pte(page, page_to_kpgprot(page)));
  43         homecache_kpte_unlock(flags);
  44
  45         return kva;
  46 }
  47 EXPORT_SYMBOL(kmap);
  48
  49 void kunmap(struct page *page)
  50 {
  51         if (in_interrupt())
  52                 BUG();
  53         if (!PageHighMem(page))
  54                 return;
  55         kunmap_high(page);
  56 }
  57 EXPORT_SYMBOL(kunmap);
  58
  59 /*
  60  * Describe a single atomic mapping of a page on a given cpu at a
  61  * given address, and allow it to be linked into a list.
  62  */
  63 struct atomic_mapped_page {
  64         struct list_head list;
  65         struct page *page;
  66         int cpu;
  67         unsigned long va;
  68 };
  69
  70 static spinlock_t amp_lock = __SPIN_LOCK_UNLOCKED(&amp_lock);
  71 static struct list_head amp_list = LIST_HEAD_INIT(amp_list);
  72
  73 /*
  74  * Combining this structure with a per-cpu declaration lets us give
  75  * each cpu an atomic_mapped_page structure per type.
  76  */
  77 struct kmap_amps {
  78         struct atomic_mapped_page per_type[KM_TYPE_NR];
  79 };
  80 static DEFINE_PER_CPU(struct kmap_amps, amps);
  81
  82 /*
  83  * Add a page and va, on this cpu, to the list of kmap_atomic pages,
  84  * and write the new pte to memory.  Writing the new PTE under the
  85  * lock guarantees that it is either on the list before migration starts
  86  * (if we won the race), or set_pte() sets the migrating bit in the PTE
  87  * (if we lost the race).  And doing it under the lock guarantees
  88  * that when kmap_atomic_fix_one_pte() comes along, it finds a valid
  89  * PTE in memory, iff the mapping is still on the amp_list.
  90  *
  91  * Finally, doing it under the lock lets us safely examine the page
  92  * to see if it is immutable or not, for the generic kmap_atomic() case.
  93  * If we examine it earlier we are exposed to a race where it looks
  94  * writable earlier, but becomes immutable before we write the PTE.
  95  */
  96 static void kmap_atomic_register(struct page *page, int type,
  97                                  unsigned long va, pte_t *ptep, pte_t pteval)
  98 {
  99         unsigned long flags;
 100         struct atomic_mapped_page *amp;
 101
 102         flags = homecache_kpte_lock();
 103         spin_lock(&amp_lock);
 104
 105         /* With interrupts disabled, now fill in the per-cpu info. */
 106         amp = this_cpu_ptr(&amps.per_type[type]);
 107         amp->page = page;
 108         amp->cpu = smp_processor_id();
 109         amp->va = va;
 110
 111         /* For generic kmap_atomic(), choose the PTE writability now. */
 112         if (!pte_read(pteval))
 113                 pteval = mk_pte(page, page_to_kpgprot(page));
 114
 115         list_add(&amp->list, &amp_list);
 116         set_pte(ptep, pteval);
 117
 118         spin_unlock(&amp_lock);
 119         homecache_kpte_unlock(flags);
 120 }
 121
 122 /*
 123  * Remove a page and va, on this cpu, from the list of kmap_atomic pages.
 124  * Linear-time search, but we count on the lists being short.
 125  * We don't need to adjust the PTE under the lock (as opposed to the
 126  * kmap_atomic_register() case), since we're just unconditionally
 127  * zeroing the PTE after it's off the list.
 128  */
 129 static void kmap_atomic_unregister(struct page *page, unsigned long va)
 130 {
 131         unsigned long flags;
 132         struct atomic_mapped_page *amp;
 133         int cpu = smp_processor_id();
 134         spin_lock_irqsave(&amp_lock, flags);
 135         list_for_each_entry(amp, &amp_list, list) {
 136                 if (amp->page == page && amp->cpu == cpu && amp->va == va)
 137                         break;
 138         }
 139         BUG_ON(&amp->list == &amp_list);
 140         list_del(&amp->list);
 141         spin_unlock_irqrestore(&amp_lock, flags);
 142 }
 143
 144 /* Helper routine for kmap_atomic_fix_kpte(), below. */
 145 static void kmap_atomic_fix_one_kpte(struct atomic_mapped_page *amp,
 146                                      int finished)
 147 {
 148         pte_t *ptep = kmap_get_pte(amp->va);
 149         if (!finished) {
 150                 set_pte(ptep, pte_mkmigrate(*ptep));
 151                 flush_remote(0, 0, NULL, amp->va, PAGE_SIZE, PAGE_SIZE,
 152                              cpumask_of(amp->cpu), NULL, 0);
 153         } else {
 154                 /*
 155                  * Rewrite a default kernel PTE for this page.
 156                  * We rely on the fact that set_pte() writes the
 157                  * present+migrating bits last.
 158                  */
 159                 pte_t pte = mk_pte(amp->page, page_to_kpgprot(amp->page));
 160                 set_pte(ptep, pte);
 161         }
 162 }
 163
 164 /*
 165  * This routine is a helper function for homecache_fix_kpte(); see
 166  * its comments for more information on the "finished" argument here.
 167  *
 168  * Note that we hold the lock while doing the remote flushes, which
 169  * will stall any unrelated cpus trying to do kmap_atomic operations.
 170  * We could just update the PTEs under the lock, and save away copies
 171  * of the structs (or just the va+cpu), then flush them after we
 172  * release the lock, but it seems easier just to do it all under the lock.
 173  */
 174 void kmap_atomic_fix_kpte(struct page *page, int finished)
 175 {
 176         struct atomic_mapped_page *amp;
 177         unsigned long flags;
 178         spin_lock_irqsave(&amp_lock, flags);
 179         list_for_each_entry(amp, &amp_list, list) {
 180                 if (amp->page == page)
 181                         kmap_atomic_fix_one_kpte(amp, finished);
 182         }
 183         spin_unlock_irqrestore(&amp_lock, flags);
 184 }
 185
 186 /*
 187  * kmap_atomic/kunmap_atomic is significantly faster than kmap/kunmap
 188  * because the kmap code must perform a global TLB invalidation when
 189  * the kmap pool wraps.
 190  *
 191  * Note that they may be slower than on x86 (etc.) because unlike on
 192  * those platforms, we do have to take a global lock to map and unmap
 193  * pages on Tile (see above).
 194  *
 195  * When holding an atomic kmap is is not legal to sleep, so atomic
 196  * kmaps are appropriate for short, tight code paths only.
 197  */
 198 void *kmap_atomic_prot(struct page *page, pgprot_t prot)
 199 {
 200         unsigned long vaddr;
 201         int idx, type;
 202         pte_t *pte;
 203
 204         preempt_disable();
 205         pagefault_disable();
 206
 207         /* Avoid icache flushes by disallowing atomic executable mappings. */
 208         BUG_ON(pte_exec(prot));
 209
 210         if (!PageHighMem(page))
 211                 return page_address(page);
 212
 213         type = kmap_atomic_idx_push();
 214         idx = type + KM_TYPE_NR*smp_processor_id();
 215         vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
 216         pte = kmap_get_pte(vaddr);
 217         BUG_ON(!pte_none(*pte));
 218
 219         /* Register that this page is mapped atomically on this cpu. */
 220         kmap_atomic_register(page, type, vaddr, pte, mk_pte(page, prot));
 221
 222         return (void *)vaddr;
 223 }
 224 EXPORT_SYMBOL(kmap_atomic_prot);
 225
 226 void *kmap_atomic(struct page *page)
 227 {
 228         /* PAGE_NONE is a magic value that tells us to check immutability. */
 229         return kmap_atomic_prot(page, PAGE_NONE);
 230 }
 231 EXPORT_SYMBOL(kmap_atomic);
 232
 233 void __kunmap_atomic(void *kvaddr)
 234 {
 235         unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK;
 236
 237         if (vaddr >= __fix_to_virt(FIX_KMAP_END) &&
 238             vaddr <= __fix_to_virt(FIX_KMAP_BEGIN)) {
 239                 pte_t *pte = kmap_get_pte(vaddr);
 240                 pte_t pteval = *pte;
 241                 int idx, type;
 242
 243                 type = kmap_atomic_idx();
 244                 idx = type + KM_TYPE_NR*smp_processor_id();
 245
 246                 /*
 247                  * Force other mappings to Oops if they try to access this pte
 248                  * without first remapping it.  Keeping stale mappings around
 249                  * is a bad idea.
 250                  */
 251                 BUG_ON(!pte_present(pteval) && !pte_migrating(pteval));
 252                 kmap_atomic_unregister(pte_page(pteval), vaddr);
 253                 kpte_clear_flush(pte, vaddr);
 254                 kmap_atomic_idx_pop();
 255         } else {
 256                 /* Must be a lowmem page */
 257                 BUG_ON(vaddr < PAGE_OFFSET);
 258                 BUG_ON(vaddr >= (unsigned long)high_memory);
 259         }
 260
 261         pagefault_enable();
 262         preempt_enable();
 263 }
 264 EXPORT_SYMBOL(__kunmap_atomic);
 265
 266 /*
 267  * This API is supposed to allow us to map memory without a "struct page".
 268  * Currently we don't support this, though this may change in the future.
 269  */
 270 void *kmap_atomic_pfn(unsigned long pfn)
 271 {
 272         return kmap_atomic(pfn_to_page(pfn));
 273 }
 274 void *kmap_atomic_prot_pfn(unsigned long pfn, pgprot_t prot)
 275 {
 276         return kmap_atomic_prot(pfn_to_page(pfn), prot);
 277 }