1 /* arch/sparc64/mm/tsb.c
3 * Copyright (C) 2006, 2008 David S. Miller <davem@davemloft.net>
6 #include <linux/kernel.h>
7 #include <linux/preempt.h>
8 #include <linux/slab.h>
10 #include <asm/tlbflush.h>
12 #include <asm/mmu_context.h>
13 #include <asm/pgtable.h>
15 #include <asm/oplib.h>
17 extern struct tsb swapper_tsb
[KERNEL_TSB_NENTRIES
];
19 static inline unsigned long tsb_hash(unsigned long vaddr
, unsigned long hash_shift
, unsigned long nentries
)
22 return vaddr
& (nentries
- 1);
25 static inline int tag_compare(unsigned long tag
, unsigned long vaddr
)
27 return (tag
== (vaddr
>> 22));
30 /* TSB flushes need only occur on the processor initiating the address
31 * space modification, not on each cpu the address space has run on.
32 * Only the TLB flush needs that treatment.
35 void flush_tsb_kernel_range(unsigned long start
, unsigned long end
)
39 for (v
= start
; v
< end
; v
+= PAGE_SIZE
) {
40 unsigned long hash
= tsb_hash(v
, PAGE_SHIFT
,
42 struct tsb
*ent
= &swapper_tsb
[hash
];
44 if (tag_compare(ent
->tag
, v
))
45 ent
->tag
= (1UL << TSB_TAG_INVALID_BIT
);
49 static void __flush_tsb_one(struct tlb_batch
*tb
, unsigned long hash_shift
,
50 unsigned long tsb
, unsigned long nentries
)
54 for (i
= 0; i
< tb
->tlb_nr
; i
++) {
55 unsigned long v
= tb
->vaddrs
[i
];
56 unsigned long tag
, ent
, hash
;
60 hash
= tsb_hash(v
, hash_shift
, nentries
);
61 ent
= tsb
+ (hash
* sizeof(struct tsb
));
68 void flush_tsb_user(struct tlb_batch
*tb
)
70 struct mm_struct
*mm
= tb
->mm
;
71 unsigned long nentries
, base
, flags
;
73 spin_lock_irqsave(&mm
->context
.lock
, flags
);
75 base
= (unsigned long) mm
->context
.tsb_block
[MM_TSB_BASE
].tsb
;
76 nentries
= mm
->context
.tsb_block
[MM_TSB_BASE
].tsb_nentries
;
77 if (tlb_type
== cheetah_plus
|| tlb_type
== hypervisor
)
79 __flush_tsb_one(tb
, PAGE_SHIFT
, base
, nentries
);
81 #ifdef CONFIG_HUGETLB_PAGE
82 if (mm
->context
.tsb_block
[MM_TSB_HUGE
].tsb
) {
83 base
= (unsigned long) mm
->context
.tsb_block
[MM_TSB_HUGE
].tsb
;
84 nentries
= mm
->context
.tsb_block
[MM_TSB_HUGE
].tsb_nentries
;
85 if (tlb_type
== cheetah_plus
|| tlb_type
== hypervisor
)
87 __flush_tsb_one(tb
, HPAGE_SHIFT
, base
, nentries
);
90 spin_unlock_irqrestore(&mm
->context
.lock
, flags
);
93 #if defined(CONFIG_SPARC64_PAGE_SIZE_8KB)
94 #define HV_PGSZ_IDX_BASE HV_PGSZ_IDX_8K
95 #define HV_PGSZ_MASK_BASE HV_PGSZ_MASK_8K
96 #elif defined(CONFIG_SPARC64_PAGE_SIZE_64KB)
97 #define HV_PGSZ_IDX_BASE HV_PGSZ_IDX_64K
98 #define HV_PGSZ_MASK_BASE HV_PGSZ_MASK_64K
100 #error Broken base page size setting...
103 #ifdef CONFIG_HUGETLB_PAGE
104 #if defined(CONFIG_HUGETLB_PAGE_SIZE_64K)
105 #define HV_PGSZ_IDX_HUGE HV_PGSZ_IDX_64K
106 #define HV_PGSZ_MASK_HUGE HV_PGSZ_MASK_64K
107 #elif defined(CONFIG_HUGETLB_PAGE_SIZE_512K)
108 #define HV_PGSZ_IDX_HUGE HV_PGSZ_IDX_512K
109 #define HV_PGSZ_MASK_HUGE HV_PGSZ_MASK_512K
110 #elif defined(CONFIG_HUGETLB_PAGE_SIZE_4MB)
111 #define HV_PGSZ_IDX_HUGE HV_PGSZ_IDX_4MB
112 #define HV_PGSZ_MASK_HUGE HV_PGSZ_MASK_4MB
114 #error Broken huge page size setting...
118 static void setup_tsb_params(struct mm_struct
*mm
, unsigned long tsb_idx
, unsigned long tsb_bytes
)
120 unsigned long tsb_reg
, base
, tsb_paddr
;
121 unsigned long page_sz
, tte
;
123 mm
->context
.tsb_block
[tsb_idx
].tsb_nentries
=
124 tsb_bytes
/ sizeof(struct tsb
);
127 tte
= pgprot_val(PAGE_KERNEL_LOCKED
);
128 tsb_paddr
= __pa(mm
->context
.tsb_block
[tsb_idx
].tsb
);
129 BUG_ON(tsb_paddr
& (tsb_bytes
- 1UL));
131 /* Use the smallest page size that can map the whole TSB
137 #ifdef DCACHE_ALIASING_POSSIBLE
138 base
+= (tsb_paddr
& 8192);
160 page_sz
= 512 * 1024;
165 page_sz
= 512 * 1024;
170 page_sz
= 512 * 1024;
175 page_sz
= 4 * 1024 * 1024;
179 printk(KERN_ERR
"TSB[%s:%d]: Impossible TSB size %lu, killing process.\n",
180 current
->comm
, current
->pid
, tsb_bytes
);
183 tte
|= pte_sz_bits(page_sz
);
185 if (tlb_type
== cheetah_plus
|| tlb_type
== hypervisor
) {
186 /* Physical mapping, no locked TLB entry for TSB. */
187 tsb_reg
|= tsb_paddr
;
189 mm
->context
.tsb_block
[tsb_idx
].tsb_reg_val
= tsb_reg
;
190 mm
->context
.tsb_block
[tsb_idx
].tsb_map_vaddr
= 0;
191 mm
->context
.tsb_block
[tsb_idx
].tsb_map_pte
= 0;
194 tsb_reg
|= (tsb_paddr
& (page_sz
- 1UL));
195 tte
|= (tsb_paddr
& ~(page_sz
- 1UL));
197 mm
->context
.tsb_block
[tsb_idx
].tsb_reg_val
= tsb_reg
;
198 mm
->context
.tsb_block
[tsb_idx
].tsb_map_vaddr
= base
;
199 mm
->context
.tsb_block
[tsb_idx
].tsb_map_pte
= tte
;
202 /* Setup the Hypervisor TSB descriptor. */
203 if (tlb_type
== hypervisor
) {
204 struct hv_tsb_descr
*hp
= &mm
->context
.tsb_descr
[tsb_idx
];
208 hp
->pgsz_idx
= HV_PGSZ_IDX_BASE
;
210 #ifdef CONFIG_HUGETLB_PAGE
212 hp
->pgsz_idx
= HV_PGSZ_IDX_HUGE
;
219 hp
->num_ttes
= tsb_bytes
/ 16;
223 hp
->pgsz_mask
= HV_PGSZ_MASK_BASE
;
225 #ifdef CONFIG_HUGETLB_PAGE
227 hp
->pgsz_mask
= HV_PGSZ_MASK_HUGE
;
233 hp
->tsb_base
= tsb_paddr
;
238 struct kmem_cache
*pgtable_cache __read_mostly
;
240 static struct kmem_cache
*tsb_caches
[8] __read_mostly
;
242 static const char *tsb_cache_names
[8] = {
253 void __init
pgtable_cache_init(void)
257 pgtable_cache
= kmem_cache_create("pgtable_cache",
258 PAGE_SIZE
, PAGE_SIZE
,
261 if (!pgtable_cache
) {
262 prom_printf("pgtable_cache_init(): Could not create!\n");
266 for (i
= 0; i
< 8; i
++) {
267 unsigned long size
= 8192 << i
;
268 const char *name
= tsb_cache_names
[i
];
270 tsb_caches
[i
] = kmem_cache_create(name
,
273 if (!tsb_caches
[i
]) {
274 prom_printf("Could not create %s cache\n", name
);
280 int sysctl_tsb_ratio
= -2;
282 static unsigned long tsb_size_to_rss_limit(unsigned long new_size
)
284 unsigned long num_ents
= (new_size
/ sizeof(struct tsb
));
286 if (sysctl_tsb_ratio
< 0)
287 return num_ents
- (num_ents
>> -sysctl_tsb_ratio
);
289 return num_ents
+ (num_ents
>> sysctl_tsb_ratio
);
292 /* When the RSS of an address space exceeds tsb_rss_limit for a TSB,
293 * do_sparc64_fault() invokes this routine to try and grow it.
295 * When we reach the maximum TSB size supported, we stick ~0UL into
296 * tsb_rss_limit for that TSB so the grow checks in do_sparc64_fault()
297 * will not trigger any longer.
299 * The TSB can be anywhere from 8K to 1MB in size, in increasing powers
300 * of two. The TSB must be aligned to it's size, so f.e. a 512K TSB
301 * must be 512K aligned. It also must be physically contiguous, so we
302 * cannot use vmalloc().
304 * The idea here is to grow the TSB when the RSS of the process approaches
305 * the number of entries that the current TSB can hold at once. Currently,
306 * we trigger when the RSS hits 3/4 of the TSB capacity.
308 void tsb_grow(struct mm_struct
*mm
, unsigned long tsb_index
, unsigned long rss
)
310 unsigned long max_tsb_size
= 1 * 1024 * 1024;
311 unsigned long new_size
, old_size
, flags
;
312 struct tsb
*old_tsb
, *new_tsb
;
313 unsigned long new_cache_index
, old_cache_index
;
314 unsigned long new_rss_limit
;
317 if (max_tsb_size
> (PAGE_SIZE
<< MAX_ORDER
))
318 max_tsb_size
= (PAGE_SIZE
<< MAX_ORDER
);
321 for (new_size
= 8192; new_size
< max_tsb_size
; new_size
<<= 1UL) {
322 new_rss_limit
= tsb_size_to_rss_limit(new_size
);
323 if (new_rss_limit
> rss
)
328 if (new_size
== max_tsb_size
)
329 new_rss_limit
= ~0UL;
332 gfp_flags
= GFP_KERNEL
;
333 if (new_size
> (PAGE_SIZE
* 2))
334 gfp_flags
= __GFP_NOWARN
| __GFP_NORETRY
;
336 new_tsb
= kmem_cache_alloc_node(tsb_caches
[new_cache_index
],
337 gfp_flags
, numa_node_id());
338 if (unlikely(!new_tsb
)) {
339 /* Not being able to fork due to a high-order TSB
340 * allocation failure is very bad behavior. Just back
341 * down to a 0-order allocation and force no TSB
342 * growing for this address space.
344 if (mm
->context
.tsb_block
[tsb_index
].tsb
== NULL
&&
345 new_cache_index
> 0) {
348 new_rss_limit
= ~0UL;
349 goto retry_tsb_alloc
;
352 /* If we failed on a TSB grow, we are under serious
353 * memory pressure so don't try to grow any more.
355 if (mm
->context
.tsb_block
[tsb_index
].tsb
!= NULL
)
356 mm
->context
.tsb_block
[tsb_index
].tsb_rss_limit
= ~0UL;
360 /* Mark all tags as invalid. */
361 tsb_init(new_tsb
, new_size
);
363 /* Ok, we are about to commit the changes. If we are
364 * growing an existing TSB the locking is very tricky,
367 * We have to hold mm->context.lock while committing to the
368 * new TSB, this synchronizes us with processors in
369 * flush_tsb_user() and switch_mm() for this address space.
371 * But even with that lock held, processors run asynchronously
372 * accessing the old TSB via TLB miss handling. This is OK
373 * because those actions are just propagating state from the
374 * Linux page tables into the TSB, page table mappings are not
375 * being changed. If a real fault occurs, the processor will
376 * synchronize with us when it hits flush_tsb_user(), this is
377 * also true for the case where vmscan is modifying the page
378 * tables. The only thing we need to be careful with is to
379 * skip any locked TSB entries during copy_tsb().
381 * When we finish committing to the new TSB, we have to drop
382 * the lock and ask all other cpus running this address space
383 * to run tsb_context_switch() to see the new TSB table.
385 spin_lock_irqsave(&mm
->context
.lock
, flags
);
387 old_tsb
= mm
->context
.tsb_block
[tsb_index
].tsb
;
389 (mm
->context
.tsb_block
[tsb_index
].tsb_reg_val
& 0x7UL
);
390 old_size
= (mm
->context
.tsb_block
[tsb_index
].tsb_nentries
*
394 /* Handle multiple threads trying to grow the TSB at the same time.
395 * One will get in here first, and bump the size and the RSS limit.
396 * The others will get in here next and hit this check.
398 if (unlikely(old_tsb
&&
399 (rss
< mm
->context
.tsb_block
[tsb_index
].tsb_rss_limit
))) {
400 spin_unlock_irqrestore(&mm
->context
.lock
, flags
);
402 kmem_cache_free(tsb_caches
[new_cache_index
], new_tsb
);
406 mm
->context
.tsb_block
[tsb_index
].tsb_rss_limit
= new_rss_limit
;
409 extern void copy_tsb(unsigned long old_tsb_base
,
410 unsigned long old_tsb_size
,
411 unsigned long new_tsb_base
,
412 unsigned long new_tsb_size
);
413 unsigned long old_tsb_base
= (unsigned long) old_tsb
;
414 unsigned long new_tsb_base
= (unsigned long) new_tsb
;
416 if (tlb_type
== cheetah_plus
|| tlb_type
== hypervisor
) {
417 old_tsb_base
= __pa(old_tsb_base
);
418 new_tsb_base
= __pa(new_tsb_base
);
420 copy_tsb(old_tsb_base
, old_size
, new_tsb_base
, new_size
);
423 mm
->context
.tsb_block
[tsb_index
].tsb
= new_tsb
;
424 setup_tsb_params(mm
, tsb_index
, new_size
);
426 spin_unlock_irqrestore(&mm
->context
.lock
, flags
);
428 /* If old_tsb is NULL, we're being invoked for the first time
429 * from init_new_context().
432 /* Reload it on the local cpu. */
433 tsb_context_switch(mm
);
435 /* Now force other processors to do the same. */
440 /* Now it is safe to free the old tsb. */
441 kmem_cache_free(tsb_caches
[old_cache_index
], old_tsb
);
445 int init_new_context(struct task_struct
*tsk
, struct mm_struct
*mm
)
447 #ifdef CONFIG_HUGETLB_PAGE
448 unsigned long huge_pte_count
;
452 spin_lock_init(&mm
->context
.lock
);
454 mm
->context
.sparc64_ctx_val
= 0UL;
456 #ifdef CONFIG_HUGETLB_PAGE
457 /* We reset it to zero because the fork() page copying
458 * will re-increment the counters as the parent PTEs are
459 * copied into the child address space.
461 huge_pte_count
= mm
->context
.huge_pte_count
;
462 mm
->context
.huge_pte_count
= 0;
465 /* copy_mm() copies over the parent's mm_struct before calling
466 * us, so we need to zero out the TSB pointer or else tsb_grow()
467 * will be confused and think there is an older TSB to free up.
469 for (i
= 0; i
< MM_NUM_TSBS
; i
++)
470 mm
->context
.tsb_block
[i
].tsb
= NULL
;
472 /* If this is fork, inherit the parent's TSB size. We would
473 * grow it to that size on the first page fault anyways.
475 tsb_grow(mm
, MM_TSB_BASE
, get_mm_rss(mm
));
477 #ifdef CONFIG_HUGETLB_PAGE
478 if (unlikely(huge_pte_count
))
479 tsb_grow(mm
, MM_TSB_HUGE
, huge_pte_count
);
482 if (unlikely(!mm
->context
.tsb_block
[MM_TSB_BASE
].tsb
))
488 static void tsb_destroy_one(struct tsb_config
*tp
)
490 unsigned long cache_index
;
494 cache_index
= tp
->tsb_reg_val
& 0x7UL
;
495 kmem_cache_free(tsb_caches
[cache_index
], tp
->tsb
);
497 tp
->tsb_reg_val
= 0UL;
500 void destroy_context(struct mm_struct
*mm
)
502 unsigned long flags
, i
;
504 for (i
= 0; i
< MM_NUM_TSBS
; i
++)
505 tsb_destroy_one(&mm
->context
.tsb_block
[i
]);
507 spin_lock_irqsave(&ctx_alloc_lock
, flags
);
509 if (CTX_VALID(mm
->context
)) {
510 unsigned long nr
= CTX_NRBITS(mm
->context
);
511 mmu_context_bmap
[nr
>>6] &= ~(1UL << (nr
& 63));
514 spin_unlock_irqrestore(&ctx_alloc_lock
, flags
);