x86/efi: Enforce CONFIG_RELOCATABLE for EFI boot stub
[linux/fpc-iii.git] / arch / tile / mm / pgtable.c
blob4fd9ec0b58edcb57d0b3346d41eea54c6f66c35f
1 /*
2 * Copyright 2010 Tilera Corporation. All Rights Reserved.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation, version 2.
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
11 * NON INFRINGEMENT. See the GNU General Public License for
12 * more details.
15 #include <linux/sched.h>
16 #include <linux/kernel.h>
17 #include <linux/errno.h>
18 #include <linux/mm.h>
19 #include <linux/swap.h>
20 #include <linux/highmem.h>
21 #include <linux/slab.h>
22 #include <linux/pagemap.h>
23 #include <linux/spinlock.h>
24 #include <linux/cpumask.h>
25 #include <linux/module.h>
26 #include <linux/io.h>
27 #include <linux/vmalloc.h>
28 #include <linux/smp.h>
30 #include <asm/pgtable.h>
31 #include <asm/pgalloc.h>
32 #include <asm/fixmap.h>
33 #include <asm/tlb.h>
34 #include <asm/tlbflush.h>
35 #include <asm/homecache.h>
37 #define K(x) ((x) << (PAGE_SHIFT-10))
40 * The normal show_free_areas() is too verbose on Tile, with dozens
41 * of processors and often four NUMA zones each with high and lowmem.
43 void show_mem(unsigned int filter)
45 struct zone *zone;
47 pr_err("Active:%lu inactive:%lu dirty:%lu writeback:%lu unstable:%lu"
48 " free:%lu\n slab:%lu mapped:%lu pagetables:%lu bounce:%lu"
49 " pagecache:%lu swap:%lu\n",
50 (global_page_state(NR_ACTIVE_ANON) +
51 global_page_state(NR_ACTIVE_FILE)),
52 (global_page_state(NR_INACTIVE_ANON) +
53 global_page_state(NR_INACTIVE_FILE)),
54 global_page_state(NR_FILE_DIRTY),
55 global_page_state(NR_WRITEBACK),
56 global_page_state(NR_UNSTABLE_NFS),
57 global_page_state(NR_FREE_PAGES),
58 (global_page_state(NR_SLAB_RECLAIMABLE) +
59 global_page_state(NR_SLAB_UNRECLAIMABLE)),
60 global_page_state(NR_FILE_MAPPED),
61 global_page_state(NR_PAGETABLE),
62 global_page_state(NR_BOUNCE),
63 global_page_state(NR_FILE_PAGES),
64 get_nr_swap_pages());
66 for_each_zone(zone) {
67 unsigned long flags, order, total = 0, largest_order = -1;
69 if (!populated_zone(zone))
70 continue;
72 spin_lock_irqsave(&zone->lock, flags);
73 for (order = 0; order < MAX_ORDER; order++) {
74 int nr = zone->free_area[order].nr_free;
75 total += nr << order;
76 if (nr)
77 largest_order = order;
79 spin_unlock_irqrestore(&zone->lock, flags);
80 pr_err("Node %d %7s: %lukB (largest %luKb)\n",
81 zone_to_nid(zone), zone->name,
82 K(total), largest_order ? K(1UL) << largest_order : 0);
86 /**
87 * shatter_huge_page() - ensure a given address is mapped by a small page.
89 * This function converts a huge PTE mapping kernel LOWMEM into a bunch
90 * of small PTEs with the same caching. No cache flush required, but we
91 * must do a global TLB flush.
93 * Any caller that wishes to modify a kernel mapping that might
94 * have been made with a huge page should call this function,
95 * since doing so properly avoids race conditions with installing the
96 * newly-shattered page and then flushing all the TLB entries.
98 * @addr: Address at which to shatter any existing huge page.
100 void shatter_huge_page(unsigned long addr)
102 pgd_t *pgd;
103 pud_t *pud;
104 pmd_t *pmd;
105 unsigned long flags = 0; /* happy compiler */
106 #ifdef __PAGETABLE_PMD_FOLDED
107 struct list_head *pos;
108 #endif
110 /* Get a pointer to the pmd entry that we need to change. */
111 addr &= HPAGE_MASK;
112 BUG_ON(pgd_addr_invalid(addr));
113 BUG_ON(addr < PAGE_OFFSET); /* only for kernel LOWMEM */
114 pgd = swapper_pg_dir + pgd_index(addr);
115 pud = pud_offset(pgd, addr);
116 BUG_ON(!pud_present(*pud));
117 pmd = pmd_offset(pud, addr);
118 BUG_ON(!pmd_present(*pmd));
119 if (!pmd_huge_page(*pmd))
120 return;
122 spin_lock_irqsave(&init_mm.page_table_lock, flags);
123 if (!pmd_huge_page(*pmd)) {
124 /* Lost the race to convert the huge page. */
125 spin_unlock_irqrestore(&init_mm.page_table_lock, flags);
126 return;
129 /* Shatter the huge page into the preallocated L2 page table. */
130 pmd_populate_kernel(&init_mm, pmd, get_prealloc_pte(pmd_pfn(*pmd)));
132 #ifdef __PAGETABLE_PMD_FOLDED
133 /* Walk every pgd on the system and update the pmd there. */
134 spin_lock(&pgd_lock);
135 list_for_each(pos, &pgd_list) {
136 pmd_t *copy_pmd;
137 pgd = list_to_pgd(pos) + pgd_index(addr);
138 pud = pud_offset(pgd, addr);
139 copy_pmd = pmd_offset(pud, addr);
140 __set_pmd(copy_pmd, *pmd);
142 spin_unlock(&pgd_lock);
143 #endif
145 /* Tell every cpu to notice the change. */
146 flush_remote(0, 0, NULL, addr, HPAGE_SIZE, HPAGE_SIZE,
147 cpu_possible_mask, NULL, 0);
149 /* Hold the lock until the TLB flush is finished to avoid races. */
150 spin_unlock_irqrestore(&init_mm.page_table_lock, flags);
154 * List of all pgd's needed so it can invalidate entries in both cached
155 * and uncached pgd's. This is essentially codepath-based locking
156 * against pageattr.c; it is the unique case in which a valid change
157 * of kernel pagetables can't be lazily synchronized by vmalloc faults.
158 * vmalloc faults work because attached pagetables are never freed.
160 * The lock is always taken with interrupts disabled, unlike on x86
161 * and other platforms, because we need to take the lock in
162 * shatter_huge_page(), which may be called from an interrupt context.
163 * We are not at risk from the tlbflush IPI deadlock that was seen on
164 * x86, since we use the flush_remote() API to have the hypervisor do
165 * the TLB flushes regardless of irq disabling.
167 DEFINE_SPINLOCK(pgd_lock);
168 LIST_HEAD(pgd_list);
170 static inline void pgd_list_add(pgd_t *pgd)
172 list_add(pgd_to_list(pgd), &pgd_list);
175 static inline void pgd_list_del(pgd_t *pgd)
177 list_del(pgd_to_list(pgd));
180 #define KERNEL_PGD_INDEX_START pgd_index(PAGE_OFFSET)
181 #define KERNEL_PGD_PTRS (PTRS_PER_PGD - KERNEL_PGD_INDEX_START)
183 static void pgd_ctor(pgd_t *pgd)
185 unsigned long flags;
187 memset(pgd, 0, KERNEL_PGD_INDEX_START*sizeof(pgd_t));
188 spin_lock_irqsave(&pgd_lock, flags);
190 #ifndef __tilegx__
192 * Check that the user interrupt vector has no L2.
193 * It never should for the swapper, and new page tables
194 * should always start with an empty user interrupt vector.
196 BUG_ON(((u64 *)swapper_pg_dir)[pgd_index(MEM_USER_INTRPT)] != 0);
197 #endif
199 memcpy(pgd + KERNEL_PGD_INDEX_START,
200 swapper_pg_dir + KERNEL_PGD_INDEX_START,
201 KERNEL_PGD_PTRS * sizeof(pgd_t));
203 pgd_list_add(pgd);
204 spin_unlock_irqrestore(&pgd_lock, flags);
207 static void pgd_dtor(pgd_t *pgd)
209 unsigned long flags; /* can be called from interrupt context */
211 spin_lock_irqsave(&pgd_lock, flags);
212 pgd_list_del(pgd);
213 spin_unlock_irqrestore(&pgd_lock, flags);
216 pgd_t *pgd_alloc(struct mm_struct *mm)
218 pgd_t *pgd = kmem_cache_alloc(pgd_cache, GFP_KERNEL);
219 if (pgd)
220 pgd_ctor(pgd);
221 return pgd;
224 void pgd_free(struct mm_struct *mm, pgd_t *pgd)
226 pgd_dtor(pgd);
227 kmem_cache_free(pgd_cache, pgd);
231 #define L2_USER_PGTABLE_PAGES (1 << L2_USER_PGTABLE_ORDER)
233 struct page *pgtable_alloc_one(struct mm_struct *mm, unsigned long address,
234 int order)
236 gfp_t flags = GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO;
237 struct page *p;
238 int i;
240 p = alloc_pages(flags, L2_USER_PGTABLE_ORDER);
241 if (p == NULL)
242 return NULL;
245 * Make every page have a page_count() of one, not just the first.
246 * We don't use __GFP_COMP since it doesn't look like it works
247 * correctly with tlb_remove_page().
249 for (i = 1; i < order; ++i) {
250 init_page_count(p+i);
251 inc_zone_page_state(p+i, NR_PAGETABLE);
254 pgtable_page_ctor(p);
255 return p;
259 * Free page immediately (used in __pte_alloc if we raced with another
260 * process). We have to correct whatever pte_alloc_one() did before
261 * returning the pages to the allocator.
263 void pgtable_free(struct mm_struct *mm, struct page *p, int order)
265 int i;
267 pgtable_page_dtor(p);
268 __free_page(p);
270 for (i = 1; i < order; ++i) {
271 __free_page(p+i);
272 dec_zone_page_state(p+i, NR_PAGETABLE);
276 void __pgtable_free_tlb(struct mmu_gather *tlb, struct page *pte,
277 unsigned long address, int order)
279 int i;
281 pgtable_page_dtor(pte);
282 tlb_remove_page(tlb, pte);
284 for (i = 1; i < order; ++i) {
285 tlb_remove_page(tlb, pte + i);
286 dec_zone_page_state(pte + i, NR_PAGETABLE);
290 #ifndef __tilegx__
293 * FIXME: needs to be atomic vs hypervisor writes. For now we make the
294 * window of vulnerability a bit smaller by doing an unlocked 8-bit update.
296 int ptep_test_and_clear_young(struct vm_area_struct *vma,
297 unsigned long addr, pte_t *ptep)
299 #if HV_PTE_INDEX_ACCESSED < 8 || HV_PTE_INDEX_ACCESSED >= 16
300 # error Code assumes HV_PTE "accessed" bit in second byte
301 #endif
302 u8 *tmp = (u8 *)ptep;
303 u8 second_byte = tmp[1];
304 if (!(second_byte & (1 << (HV_PTE_INDEX_ACCESSED - 8))))
305 return 0;
306 tmp[1] = second_byte & ~(1 << (HV_PTE_INDEX_ACCESSED - 8));
307 return 1;
311 * This implementation is atomic vs hypervisor writes, since the hypervisor
312 * always writes the low word (where "accessed" and "dirty" are) and this
313 * routine only writes the high word.
315 void ptep_set_wrprotect(struct mm_struct *mm,
316 unsigned long addr, pte_t *ptep)
318 #if HV_PTE_INDEX_WRITABLE < 32
319 # error Code assumes HV_PTE "writable" bit in high word
320 #endif
321 u32 *tmp = (u32 *)ptep;
322 tmp[1] = tmp[1] & ~(1 << (HV_PTE_INDEX_WRITABLE - 32));
325 #endif
328 * Return a pointer to the PTE that corresponds to the given
329 * address in the given page table. A NULL page table just uses
330 * the standard kernel page table; the preferred API in this case
331 * is virt_to_kpte().
333 * The returned pointer can point to a huge page in other levels
334 * of the page table than the bottom, if the huge page is present
335 * in the page table. For bottom-level PTEs, the returned pointer
336 * can point to a PTE that is either present or not.
338 pte_t *virt_to_pte(struct mm_struct* mm, unsigned long addr)
340 pgd_t *pgd;
341 pud_t *pud;
342 pmd_t *pmd;
344 if (pgd_addr_invalid(addr))
345 return NULL;
347 pgd = mm ? pgd_offset(mm, addr) : swapper_pg_dir + pgd_index(addr);
348 pud = pud_offset(pgd, addr);
349 if (!pud_present(*pud))
350 return NULL;
351 if (pud_huge_page(*pud))
352 return (pte_t *)pud;
353 pmd = pmd_offset(pud, addr);
354 if (!pmd_present(*pmd))
355 return NULL;
356 if (pmd_huge_page(*pmd))
357 return (pte_t *)pmd;
358 return pte_offset_kernel(pmd, addr);
360 EXPORT_SYMBOL(virt_to_pte);
362 pte_t *virt_to_kpte(unsigned long kaddr)
364 BUG_ON(kaddr < PAGE_OFFSET);
365 return virt_to_pte(NULL, kaddr);
367 EXPORT_SYMBOL(virt_to_kpte);
369 pgprot_t set_remote_cache_cpu(pgprot_t prot, int cpu)
371 unsigned int width = smp_width;
372 int x = cpu % width;
373 int y = cpu / width;
374 BUG_ON(y >= smp_height);
375 BUG_ON(hv_pte_get_mode(prot) != HV_PTE_MODE_CACHE_TILE_L3);
376 BUG_ON(cpu < 0 || cpu >= NR_CPUS);
377 BUG_ON(!cpu_is_valid_lotar(cpu));
378 return hv_pte_set_lotar(prot, HV_XY_TO_LOTAR(x, y));
381 int get_remote_cache_cpu(pgprot_t prot)
383 HV_LOTAR lotar = hv_pte_get_lotar(prot);
384 int x = HV_LOTAR_X(lotar);
385 int y = HV_LOTAR_Y(lotar);
386 BUG_ON(hv_pte_get_mode(prot) != HV_PTE_MODE_CACHE_TILE_L3);
387 return x + y * smp_width;
391 * Convert a kernel VA to a PA and homing information.
393 int va_to_cpa_and_pte(void *va, unsigned long long *cpa, pte_t *pte)
395 struct page *page = virt_to_page(va);
396 pte_t null_pte = { 0 };
398 *cpa = __pa(va);
400 /* Note that this is not writing a page table, just returning a pte. */
401 *pte = pte_set_home(null_pte, page_home(page));
403 return 0; /* return non-zero if not hfh? */
405 EXPORT_SYMBOL(va_to_cpa_and_pte);
407 void __set_pte(pte_t *ptep, pte_t pte)
409 #ifdef __tilegx__
410 *ptep = pte;
411 #else
412 # if HV_PTE_INDEX_PRESENT >= 32 || HV_PTE_INDEX_MIGRATING >= 32
413 # error Must write the present and migrating bits last
414 # endif
415 if (pte_present(pte)) {
416 ((u32 *)ptep)[1] = (u32)(pte_val(pte) >> 32);
417 barrier();
418 ((u32 *)ptep)[0] = (u32)(pte_val(pte));
419 } else {
420 ((u32 *)ptep)[0] = (u32)(pte_val(pte));
421 barrier();
422 ((u32 *)ptep)[1] = (u32)(pte_val(pte) >> 32);
424 #endif /* __tilegx__ */
427 void set_pte(pte_t *ptep, pte_t pte)
429 if (pte_present(pte) &&
430 (!CHIP_HAS_MMIO() || hv_pte_get_mode(pte) != HV_PTE_MODE_MMIO)) {
431 /* The PTE actually references physical memory. */
432 unsigned long pfn = pte_pfn(pte);
433 if (pfn_valid(pfn)) {
434 /* Update the home of the PTE from the struct page. */
435 pte = pte_set_home(pte, page_home(pfn_to_page(pfn)));
436 } else if (hv_pte_get_mode(pte) == 0) {
437 /* remap_pfn_range(), etc, must supply PTE mode. */
438 panic("set_pte(): out-of-range PFN and mode 0\n");
442 __set_pte(ptep, pte);
445 /* Can this mm load a PTE with cached_priority set? */
446 static inline int mm_is_priority_cached(struct mm_struct *mm)
448 return mm->context.priority_cached != 0;
452 * Add a priority mapping to an mm_context and
453 * notify the hypervisor if this is the first one.
455 void start_mm_caching(struct mm_struct *mm)
457 if (!mm_is_priority_cached(mm)) {
458 mm->context.priority_cached = -1UL;
459 hv_set_caching(-1UL);
464 * Validate and return the priority_cached flag. We know if it's zero
465 * that we don't need to scan, since we immediately set it non-zero
466 * when we first consider a MAP_CACHE_PRIORITY mapping.
468 * We only _try_ to acquire the mmap_sem semaphore; if we can't acquire it,
469 * since we're in an interrupt context (servicing switch_mm) we don't
470 * worry about it and don't unset the "priority_cached" field.
471 * Presumably we'll come back later and have more luck and clear
472 * the value then; for now we'll just keep the cache marked for priority.
474 static unsigned long update_priority_cached(struct mm_struct *mm)
476 if (mm->context.priority_cached && down_write_trylock(&mm->mmap_sem)) {
477 struct vm_area_struct *vm;
478 for (vm = mm->mmap; vm; vm = vm->vm_next) {
479 if (hv_pte_get_cached_priority(vm->vm_page_prot))
480 break;
482 if (vm == NULL)
483 mm->context.priority_cached = 0;
484 up_write(&mm->mmap_sem);
486 return mm->context.priority_cached;
489 /* Set caching correctly for an mm that we are switching to. */
490 void check_mm_caching(struct mm_struct *prev, struct mm_struct *next)
492 if (!mm_is_priority_cached(next)) {
494 * If the new mm doesn't use priority caching, just see if we
495 * need the hv_set_caching(), or can assume it's already zero.
497 if (mm_is_priority_cached(prev))
498 hv_set_caching(0);
499 } else {
500 hv_set_caching(update_priority_cached(next));
504 #if CHIP_HAS_MMIO()
506 /* Map an arbitrary MMIO address, homed according to pgprot, into VA space. */
507 void __iomem *ioremap_prot(resource_size_t phys_addr, unsigned long size,
508 pgprot_t home)
510 void *addr;
511 struct vm_struct *area;
512 unsigned long offset, last_addr;
513 pgprot_t pgprot;
515 /* Don't allow wraparound or zero size */
516 last_addr = phys_addr + size - 1;
517 if (!size || last_addr < phys_addr)
518 return NULL;
520 /* Create a read/write, MMIO VA mapping homed at the requested shim. */
521 pgprot = PAGE_KERNEL;
522 pgprot = hv_pte_set_mode(pgprot, HV_PTE_MODE_MMIO);
523 pgprot = hv_pte_set_lotar(pgprot, hv_pte_get_lotar(home));
526 * Mappings have to be page-aligned
528 offset = phys_addr & ~PAGE_MASK;
529 phys_addr &= PAGE_MASK;
530 size = PAGE_ALIGN(last_addr+1) - phys_addr;
533 * Ok, go for it..
535 area = get_vm_area(size, VM_IOREMAP /* | other flags? */);
536 if (!area)
537 return NULL;
538 area->phys_addr = phys_addr;
539 addr = area->addr;
540 if (ioremap_page_range((unsigned long)addr, (unsigned long)addr + size,
541 phys_addr, pgprot)) {
542 free_vm_area(area);
543 return NULL;
545 return (__force void __iomem *) (offset + (char *)addr);
547 EXPORT_SYMBOL(ioremap_prot);
549 /* Unmap an MMIO VA mapping. */
550 void iounmap(volatile void __iomem *addr_in)
552 volatile void __iomem *addr = (volatile void __iomem *)
553 (PAGE_MASK & (unsigned long __force)addr_in);
554 #if 1
555 vunmap((void * __force)addr);
556 #else
557 /* x86 uses this complicated flow instead of vunmap(). Is
558 * there any particular reason we should do the same? */
559 struct vm_struct *p, *o;
561 /* Use the vm area unlocked, assuming the caller
562 ensures there isn't another iounmap for the same address
563 in parallel. Reuse of the virtual address is prevented by
564 leaving it in the global lists until we're done with it.
565 cpa takes care of the direct mappings. */
566 p = find_vm_area((void *)addr);
568 if (!p) {
569 pr_err("iounmap: bad address %p\n", addr);
570 dump_stack();
571 return;
574 /* Finally remove it */
575 o = remove_vm_area((void *)addr);
576 BUG_ON(p != o || o == NULL);
577 kfree(p);
578 #endif
580 EXPORT_SYMBOL(iounmap);
582 #endif /* CHIP_HAS_MMIO() */