Full support for Ginger Console
[linux-ginger.git] / arch / powerpc / mm / tlb_nohash.c
blob2fbc680c2c7147f8ca2800d383c00a4500cb0b69
1 /*
2 * This file contains the routines for TLB flushing.
3 * On machines where the MMU does not use a hash table to store virtual to
4 * physical translations (ie, SW loaded TLBs or Book3E compilant processors,
5 * this does -not- include 603 however which shares the implementation with
6 * hash based processors)
8 * -- BenH
10 * Copyright 2008,2009 Ben Herrenschmidt <benh@kernel.crashing.org>
11 * IBM Corp.
13 * Derived from arch/ppc/mm/init.c:
14 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
16 * Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
17 * and Cort Dougan (PReP) (cort@cs.nmt.edu)
18 * Copyright (C) 1996 Paul Mackerras
20 * Derived from "arch/i386/mm/init.c"
21 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
23 * This program is free software; you can redistribute it and/or
24 * modify it under the terms of the GNU General Public License
25 * as published by the Free Software Foundation; either version
26 * 2 of the License, or (at your option) any later version.
30 #include <linux/kernel.h>
31 #include <linux/mm.h>
32 #include <linux/init.h>
33 #include <linux/highmem.h>
34 #include <linux/pagemap.h>
35 #include <linux/preempt.h>
36 #include <linux/spinlock.h>
37 #include <linux/lmb.h>
39 #include <asm/tlbflush.h>
40 #include <asm/tlb.h>
41 #include <asm/code-patching.h>
43 #include "mmu_decl.h"
45 #ifdef CONFIG_PPC_BOOK3E
46 struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
47 [MMU_PAGE_4K] = {
48 .shift = 12,
49 .enc = BOOK3E_PAGESZ_4K,
51 [MMU_PAGE_16K] = {
52 .shift = 14,
53 .enc = BOOK3E_PAGESZ_16K,
55 [MMU_PAGE_64K] = {
56 .shift = 16,
57 .enc = BOOK3E_PAGESZ_64K,
59 [MMU_PAGE_1M] = {
60 .shift = 20,
61 .enc = BOOK3E_PAGESZ_1M,
63 [MMU_PAGE_16M] = {
64 .shift = 24,
65 .enc = BOOK3E_PAGESZ_16M,
67 [MMU_PAGE_256M] = {
68 .shift = 28,
69 .enc = BOOK3E_PAGESZ_256M,
71 [MMU_PAGE_1G] = {
72 .shift = 30,
73 .enc = BOOK3E_PAGESZ_1GB,
76 static inline int mmu_get_tsize(int psize)
78 return mmu_psize_defs[psize].enc;
80 #else
81 static inline int mmu_get_tsize(int psize)
83 /* This isn't used on !Book3E for now */
84 return 0;
86 #endif
88 /* The variables below are currently only used on 64-bit Book3E
89 * though this will probably be made common with other nohash
90 * implementations at some point
92 #ifdef CONFIG_PPC64
94 int mmu_linear_psize; /* Page size used for the linear mapping */
95 int mmu_pte_psize; /* Page size used for PTE pages */
96 int mmu_vmemmap_psize; /* Page size used for the virtual mem map */
97 int book3e_htw_enabled; /* Is HW tablewalk enabled ? */
98 unsigned long linear_map_top; /* Top of linear mapping */
100 #endif /* CONFIG_PPC64 */
103 * Base TLB flushing operations:
105 * - flush_tlb_mm(mm) flushes the specified mm context TLB's
106 * - flush_tlb_page(vma, vmaddr) flushes one page
107 * - flush_tlb_range(vma, start, end) flushes a range of pages
108 * - flush_tlb_kernel_range(start, end) flushes kernel pages
110 * - local_* variants of page and mm only apply to the current
111 * processor
115 * These are the base non-SMP variants of page and mm flushing
117 void local_flush_tlb_mm(struct mm_struct *mm)
119 unsigned int pid;
121 preempt_disable();
122 pid = mm->context.id;
123 if (pid != MMU_NO_CONTEXT)
124 _tlbil_pid(pid);
125 preempt_enable();
127 EXPORT_SYMBOL(local_flush_tlb_mm);
129 void __local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
130 int tsize, int ind)
132 unsigned int pid;
134 preempt_disable();
135 pid = mm ? mm->context.id : 0;
136 if (pid != MMU_NO_CONTEXT)
137 _tlbil_va(vmaddr, pid, tsize, ind);
138 preempt_enable();
141 void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
143 __local_flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
144 mmu_get_tsize(mmu_virtual_psize), 0);
146 EXPORT_SYMBOL(local_flush_tlb_page);
149 * And here are the SMP non-local implementations
151 #ifdef CONFIG_SMP
153 static DEFINE_SPINLOCK(tlbivax_lock);
155 static int mm_is_core_local(struct mm_struct *mm)
157 return cpumask_subset(mm_cpumask(mm),
158 topology_thread_cpumask(smp_processor_id()));
161 struct tlb_flush_param {
162 unsigned long addr;
163 unsigned int pid;
164 unsigned int tsize;
165 unsigned int ind;
168 static void do_flush_tlb_mm_ipi(void *param)
170 struct tlb_flush_param *p = param;
172 _tlbil_pid(p ? p->pid : 0);
175 static void do_flush_tlb_page_ipi(void *param)
177 struct tlb_flush_param *p = param;
179 _tlbil_va(p->addr, p->pid, p->tsize, p->ind);
183 /* Note on invalidations and PID:
185 * We snapshot the PID with preempt disabled. At this point, it can still
186 * change either because:
187 * - our context is being stolen (PID -> NO_CONTEXT) on another CPU
188 * - we are invaliating some target that isn't currently running here
189 * and is concurrently acquiring a new PID on another CPU
190 * - some other CPU is re-acquiring a lost PID for this mm
191 * etc...
193 * However, this shouldn't be a problem as we only guarantee
194 * invalidation of TLB entries present prior to this call, so we
195 * don't care about the PID changing, and invalidating a stale PID
196 * is generally harmless.
199 void flush_tlb_mm(struct mm_struct *mm)
201 unsigned int pid;
203 preempt_disable();
204 pid = mm->context.id;
205 if (unlikely(pid == MMU_NO_CONTEXT))
206 goto no_context;
207 if (!mm_is_core_local(mm)) {
208 struct tlb_flush_param p = { .pid = pid };
209 /* Ignores smp_processor_id() even if set. */
210 smp_call_function_many(mm_cpumask(mm),
211 do_flush_tlb_mm_ipi, &p, 1);
213 _tlbil_pid(pid);
214 no_context:
215 preempt_enable();
217 EXPORT_SYMBOL(flush_tlb_mm);
219 void __flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
220 int tsize, int ind)
222 struct cpumask *cpu_mask;
223 unsigned int pid;
225 preempt_disable();
226 pid = mm ? mm->context.id : 0;
227 if (unlikely(pid == MMU_NO_CONTEXT))
228 goto bail;
229 cpu_mask = mm_cpumask(mm);
230 if (!mm_is_core_local(mm)) {
231 /* If broadcast tlbivax is supported, use it */
232 if (mmu_has_feature(MMU_FTR_USE_TLBIVAX_BCAST)) {
233 int lock = mmu_has_feature(MMU_FTR_LOCK_BCAST_INVAL);
234 if (lock)
235 spin_lock(&tlbivax_lock);
236 _tlbivax_bcast(vmaddr, pid, tsize, ind);
237 if (lock)
238 spin_unlock(&tlbivax_lock);
239 goto bail;
240 } else {
241 struct tlb_flush_param p = {
242 .pid = pid,
243 .addr = vmaddr,
244 .tsize = tsize,
245 .ind = ind,
247 /* Ignores smp_processor_id() even if set in cpu_mask */
248 smp_call_function_many(cpu_mask,
249 do_flush_tlb_page_ipi, &p, 1);
252 _tlbil_va(vmaddr, pid, tsize, ind);
253 bail:
254 preempt_enable();
257 void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
259 __flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
260 mmu_get_tsize(mmu_virtual_psize), 0);
262 EXPORT_SYMBOL(flush_tlb_page);
264 #endif /* CONFIG_SMP */
267 * Flush kernel TLB entries in the given range
269 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
271 #ifdef CONFIG_SMP
272 preempt_disable();
273 smp_call_function(do_flush_tlb_mm_ipi, NULL, 1);
274 _tlbil_pid(0);
275 preempt_enable();
276 #else
277 _tlbil_pid(0);
278 #endif
280 EXPORT_SYMBOL(flush_tlb_kernel_range);
283 * Currently, for range flushing, we just do a full mm flush. This should
284 * be optimized based on a threshold on the size of the range, since
285 * some implementation can stack multiple tlbivax before a tlbsync but
286 * for now, we keep it that way
288 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
289 unsigned long end)
292 flush_tlb_mm(vma->vm_mm);
294 EXPORT_SYMBOL(flush_tlb_range);
296 void tlb_flush(struct mmu_gather *tlb)
298 flush_tlb_mm(tlb->mm);
300 /* Push out batch of freed page tables */
301 pte_free_finish();
305 * Below are functions specific to the 64-bit variant of Book3E though that
306 * may change in the future
309 #ifdef CONFIG_PPC64
312 * Handling of virtual linear page tables or indirect TLB entries
313 * flushing when PTE pages are freed
315 void tlb_flush_pgtable(struct mmu_gather *tlb, unsigned long address)
317 int tsize = mmu_psize_defs[mmu_pte_psize].enc;
319 if (book3e_htw_enabled) {
320 unsigned long start = address & PMD_MASK;
321 unsigned long end = address + PMD_SIZE;
322 unsigned long size = 1UL << mmu_psize_defs[mmu_pte_psize].shift;
324 /* This isn't the most optimal, ideally we would factor out the
325 * while preempt & CPU mask mucking around, or even the IPI but
326 * it will do for now
328 while (start < end) {
329 __flush_tlb_page(tlb->mm, start, tsize, 1);
330 start += size;
332 } else {
333 unsigned long rmask = 0xf000000000000000ul;
334 unsigned long rid = (address & rmask) | 0x1000000000000000ul;
335 unsigned long vpte = address & ~rmask;
337 #ifdef CONFIG_PPC_64K_PAGES
338 vpte = (vpte >> (PAGE_SHIFT - 4)) & ~0xfffful;
339 #else
340 vpte = (vpte >> (PAGE_SHIFT - 3)) & ~0xffful;
341 #endif
342 vpte |= rid;
343 __flush_tlb_page(tlb->mm, vpte, tsize, 0);
348 * Early initialization of the MMU TLB code
350 static void __early_init_mmu(int boot_cpu)
352 extern unsigned int interrupt_base_book3e;
353 extern unsigned int exc_data_tlb_miss_htw_book3e;
354 extern unsigned int exc_instruction_tlb_miss_htw_book3e;
356 unsigned int *ibase = &interrupt_base_book3e;
357 unsigned int mas4;
359 /* XXX This will have to be decided at runtime, but right
360 * now our boot and TLB miss code hard wires it. Ideally
361 * we should find out a suitable page size and patch the
362 * TLB miss code (either that or use the PACA to store
363 * the value we want)
365 mmu_linear_psize = MMU_PAGE_1G;
367 /* XXX This should be decided at runtime based on supported
368 * page sizes in the TLB, but for now let's assume 16M is
369 * always there and a good fit (which it probably is)
371 mmu_vmemmap_psize = MMU_PAGE_16M;
373 /* Check if HW tablewalk is present, and if yes, enable it by:
375 * - patching the TLB miss handlers to branch to the
376 * one dedicates to it
378 * - setting the global book3e_htw_enabled
380 * - Set MAS4:INDD and default page size
383 /* XXX This code only checks for TLB 0 capabilities and doesn't
384 * check what page size combos are supported by the HW. It
385 * also doesn't handle the case where a separate array holds
386 * the IND entries from the array loaded by the PT.
388 if (boot_cpu) {
389 unsigned int tlb0cfg = mfspr(SPRN_TLB0CFG);
391 /* Check if HW loader is supported */
392 if ((tlb0cfg & TLBnCFG_IND) &&
393 (tlb0cfg & TLBnCFG_PT)) {
394 patch_branch(ibase + (0x1c0 / 4),
395 (unsigned long)&exc_data_tlb_miss_htw_book3e, 0);
396 patch_branch(ibase + (0x1e0 / 4),
397 (unsigned long)&exc_instruction_tlb_miss_htw_book3e, 0);
398 book3e_htw_enabled = 1;
400 pr_info("MMU: Book3E Page Tables %s\n",
401 book3e_htw_enabled ? "Enabled" : "Disabled");
404 /* Set MAS4 based on page table setting */
406 mas4 = 0x4 << MAS4_WIMGED_SHIFT;
407 if (book3e_htw_enabled) {
408 mas4 |= mas4 | MAS4_INDD;
409 #ifdef CONFIG_PPC_64K_PAGES
410 mas4 |= BOOK3E_PAGESZ_256M << MAS4_TSIZED_SHIFT;
411 mmu_pte_psize = MMU_PAGE_256M;
412 #else
413 mas4 |= BOOK3E_PAGESZ_1M << MAS4_TSIZED_SHIFT;
414 mmu_pte_psize = MMU_PAGE_1M;
415 #endif
416 } else {
417 #ifdef CONFIG_PPC_64K_PAGES
418 mas4 |= BOOK3E_PAGESZ_64K << MAS4_TSIZED_SHIFT;
419 #else
420 mas4 |= BOOK3E_PAGESZ_4K << MAS4_TSIZED_SHIFT;
421 #endif
422 mmu_pte_psize = mmu_virtual_psize;
424 mtspr(SPRN_MAS4, mas4);
426 /* Set the global containing the top of the linear mapping
427 * for use by the TLB miss code
429 linear_map_top = lmb_end_of_DRAM();
431 /* A sync won't hurt us after mucking around with
432 * the MMU configuration
434 mb();
437 void __init early_init_mmu(void)
439 __early_init_mmu(1);
442 void __cpuinit early_init_mmu_secondary(void)
444 __early_init_mmu(0);
447 #endif /* CONFIG_PPC64 */