?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 2.6.17.7
[linux/fpc-iii.git] / arch / powerpc / mm / hash_native_64.c
blob994856e55b7c49f79f008936c249b2ac897ce9f5
1 /*
2 * native hashtable management.
3 *
4 * SMP scalability work:
5 * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version
10 * 2 of the License, or (at your option) any later version.
11 */
12
13 #undef DEBUG_LOW
14
15 #include <linux/spinlock.h>
16 #include <linux/bitops.h>
17 #include <linux/threads.h>
18 #include <linux/smp.h>
19
20 #include <asm/abs_addr.h>
21 #include <asm/machdep.h>
22 #include <asm/mmu.h>
23 #include <asm/mmu_context.h>
24 #include <asm/pgtable.h>
25 #include <asm/tlbflush.h>
26 #include <asm/tlb.h>
27 #include <asm/cputable.h>
28 #include <asm/udbg.h>
29
30 #ifdef DEBUG_LOW
31 #define DBG_LOW(fmt...) udbg_printf(fmt)
32 #else
33 #define DBG_LOW(fmt...)
34 #endif
35
36 #define HPTE_LOCK_BIT 3
37
38 static DEFINE_SPINLOCK(native_tlbie_lock);
39
40 static inline void __tlbie(unsigned long va, unsigned int psize)
41 {
42 unsigned int penc;
43
44 /* clear top 16 bits, non SLS segment */
45 va &= ~(0xffffULL << 48);
46
47 switch (psize) {
48 case MMU_PAGE_4K:
49 va &= ~0xffful;
50 asm volatile("tlbie %0,0" : : "r" (va) : "memory");
51 break;
52 default:
53 penc = mmu_psize_defs[psize].penc;
54 va &= ~((1ul << mmu_psize_defs[psize].shift) - 1);
55 va |= penc << 12;
56 asm volatile("tlbie %0,1" : : "r" (va) : "memory");
57 break;
58 }
59 }
60
61 static inline void __tlbiel(unsigned long va, unsigned int psize)
62 {
63 unsigned int penc;
64
65 /* clear top 16 bits, non SLS segment */
66 va &= ~(0xffffULL << 48);
67
68 switch (psize) {
69 case MMU_PAGE_4K:
70 va &= ~0xffful;
71 asm volatile(".long 0x7c000224 | (%0 << 11) | (0 << 21)"
72 : : "r"(va) : "memory");
73 break;
74 default:
75 penc = mmu_psize_defs[psize].penc;
76 va &= ~((1ul << mmu_psize_defs[psize].shift) - 1);
77 va |= penc << 12;
78 asm volatile(".long 0x7c000224 | (%0 << 11) | (1 << 21)"
79 : : "r"(va) : "memory");
80 break;
81 }
82
83 }
84
85 static inline void tlbie(unsigned long va, int psize, int local)
86 {
87 unsigned int use_local = local && cpu_has_feature(CPU_FTR_TLBIEL);
88 int lock_tlbie = !cpu_has_feature(CPU_FTR_LOCKLESS_TLBIE);
89
90 if (use_local)
91 use_local = mmu_psize_defs[psize].tlbiel;
92 if (lock_tlbie && !use_local)
93 spin_lock(&native_tlbie_lock);
94 asm volatile("ptesync": : :"memory");
95 if (use_local) {
96 __tlbiel(va, psize);
97 asm volatile("ptesync": : :"memory");
98 } else {
99 __tlbie(va, psize);
100 asm volatile("eieio; tlbsync; ptesync": : :"memory");
101 }
102 if (lock_tlbie && !use_local)
103 spin_unlock(&native_tlbie_lock);
104 }
105
106 static inline void native_lock_hpte(hpte_t *hptep)
107 {
108 unsigned long *word = &hptep->v;
109
110 while (1) {
111 if (!test_and_set_bit(HPTE_LOCK_BIT, word))
112 break;
113 while(test_bit(HPTE_LOCK_BIT, word))
114 cpu_relax();
115 }
116 }
117
118 static inline void native_unlock_hpte(hpte_t *hptep)
119 {
120 unsigned long *word = &hptep->v;
121
122 asm volatile("lwsync":::"memory");
123 clear_bit(HPTE_LOCK_BIT, word);
124 }
125
126 long native_hpte_insert(unsigned long hpte_group, unsigned long va,
127 unsigned long pa, unsigned long rflags,
128 unsigned long vflags, int psize)
129 {
130 hpte_t *hptep = htab_address + hpte_group;
131 unsigned long hpte_v, hpte_r;
132 int i;
133
134 if (!(vflags & HPTE_V_BOLTED)) {
135 DBG_LOW(" insert(group=%lx, va=%016lx, pa=%016lx,"
136 " rflags=%lx, vflags=%lx, psize=%d)\n",
137 hpte_group, va, pa, rflags, vflags, psize);
138 }
139
140 for (i = 0; i < HPTES_PER_GROUP; i++) {
141 if (! (hptep->v & HPTE_V_VALID)) {
142 /* retry with lock held */
143 native_lock_hpte(hptep);
144 if (! (hptep->v & HPTE_V_VALID))
145 break;
146 native_unlock_hpte(hptep);
147 }
148
149 hptep++;
150 }
151
152 if (i == HPTES_PER_GROUP)
153 return -1;
154
155 hpte_v = hpte_encode_v(va, psize) | vflags | HPTE_V_VALID;
156 hpte_r = hpte_encode_r(pa, psize) | rflags;
157
158 if (!(vflags & HPTE_V_BOLTED)) {
159 DBG_LOW(" i=%x hpte_v=%016lx, hpte_r=%016lx\n",
160 i, hpte_v, hpte_r);
161 }
162
163 hptep->r = hpte_r;
164 /* Guarantee the second dword is visible before the valid bit */
165 __asm__ __volatile__ ("eieio" : : : "memory");
166 /*
167 * Now set the first dword including the valid bit
168 * NOTE: this also unlocks the hpte
169 */
170 hptep->v = hpte_v;
171
172 __asm__ __volatile__ ("ptesync" : : : "memory");
173
174 return i | (!!(vflags & HPTE_V_SECONDARY) << 3);
175 }
176
177 static long native_hpte_remove(unsigned long hpte_group)
178 {
179 hpte_t *hptep;
180 int i;
181 int slot_offset;
182 unsigned long hpte_v;
183
184 DBG_LOW(" remove(group=%lx)\n", hpte_group);
185
186 /* pick a random entry to start at */
187 slot_offset = mftb() & 0x7;
188
189 for (i = 0; i < HPTES_PER_GROUP; i++) {
190 hptep = htab_address + hpte_group + slot_offset;
191 hpte_v = hptep->v;
192
193 if ((hpte_v & HPTE_V_VALID) && !(hpte_v & HPTE_V_BOLTED)) {
194 /* retry with lock held */
195 native_lock_hpte(hptep);
196 hpte_v = hptep->v;
197 if ((hpte_v & HPTE_V_VALID)
198 && !(hpte_v & HPTE_V_BOLTED))
199 break;
200 native_unlock_hpte(hptep);
201 }
202
203 slot_offset++;
204 slot_offset &= 0x7;
205 }
206
207 if (i == HPTES_PER_GROUP)
208 return -1;
209
210 /* Invalidate the hpte. NOTE: this also unlocks it */
211 hptep->v = 0;
212
213 return i;
214 }
215
216 static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
217 unsigned long va, int psize, int local)
218 {
219 hpte_t *hptep = htab_address + slot;
220 unsigned long hpte_v, want_v;
221 int ret = 0;
222
223 want_v = hpte_encode_v(va, psize);
224
225 DBG_LOW(" update(va=%016lx, avpnv=%016lx, hash=%016lx, newpp=%x)",
226 va, want_v & HPTE_V_AVPN, slot, newpp);
227
228 native_lock_hpte(hptep);
229
230 hpte_v = hptep->v;
231
232 /* Even if we miss, we need to invalidate the TLB */
233 if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) {
234 DBG_LOW(" -> miss\n");
235 native_unlock_hpte(hptep);
236 ret = -1;
237 } else {
238 DBG_LOW(" -> hit\n");
239 /* Update the HPTE */
240 hptep->r = (hptep->r & ~(HPTE_R_PP | HPTE_R_N)) |
241 (newpp & (HPTE_R_PP | HPTE_R_N));
242 native_unlock_hpte(hptep);
243 }
244
245 /* Ensure it is out of the tlb too. */
246 tlbie(va, psize, local);
247
248 return ret;
249 }
250
251 static long native_hpte_find(unsigned long va, int psize)
252 {
253 hpte_t *hptep;
254 unsigned long hash;
255 unsigned long i, j;
256 long slot;
257 unsigned long want_v, hpte_v;
258
259 hash = hpt_hash(va, mmu_psize_defs[psize].shift);
260 want_v = hpte_encode_v(va, psize);
261
262 for (j = 0; j < 2; j++) {
263 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
264 for (i = 0; i < HPTES_PER_GROUP; i++) {
265 hptep = htab_address + slot;
266 hpte_v = hptep->v;
267
268 if (HPTE_V_COMPARE(hpte_v, want_v)
269 && (hpte_v & HPTE_V_VALID)
270 && ( !!(hpte_v & HPTE_V_SECONDARY) == j)) {
271 /* HPTE matches */
272 if (j)
273 slot = -slot;
274 return slot;
275 }
276 ++slot;
277 }
278 hash = ~hash;
279 }
280
281 return -1;
282 }
283
284 /*
285 * Update the page protection bits. Intended to be used to create
286 * guard pages for kernel data structures on pages which are bolted
287 * in the HPT. Assumes pages being operated on will not be stolen.
288 *
289 * No need to lock here because we should be the only user.
290 */
291 static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
292 int psize)
293 {
294 unsigned long vsid, va;
295 long slot;
296 hpte_t *hptep;
297
298 vsid = get_kernel_vsid(ea);
299 va = (vsid << 28) | (ea & 0x0fffffff);
300
301 slot = native_hpte_find(va, psize);
302 if (slot == -1)
303 panic("could not find page to bolt\n");
304 hptep = htab_address + slot;
305
306 /* Update the HPTE */
307 hptep->r = (hptep->r & ~(HPTE_R_PP | HPTE_R_N)) |
308 (newpp & (HPTE_R_PP | HPTE_R_N));
309
310 /* Ensure it is out of the tlb too. */
311 tlbie(va, psize, 0);
312 }
313
314 static void native_hpte_invalidate(unsigned long slot, unsigned long va,
315 int psize, int local)
316 {
317 hpte_t *hptep = htab_address + slot;
318 unsigned long hpte_v;
319 unsigned long want_v;
320 unsigned long flags;
321
322 local_irq_save(flags);
323
324 DBG_LOW(" invalidate(va=%016lx, hash: %x)\n", va, slot);
325
326 want_v = hpte_encode_v(va, psize);
327 native_lock_hpte(hptep);
328 hpte_v = hptep->v;
329
330 /* Even if we miss, we need to invalidate the TLB */
331 if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID))
332 native_unlock_hpte(hptep);
333 else
334 /* Invalidate the hpte. NOTE: this also unlocks it */
335 hptep->v = 0;
336
337 /* Invalidate the TLB */
338 tlbie(va, psize, local);
339
340 local_irq_restore(flags);
341 }
342
343 /*
344 * XXX This need fixing based on page size. It's only used by
345 * native_hpte_clear() for now which needs fixing too so they
346 * make a good pair...
347 */
348 static unsigned long slot2va(unsigned long hpte_v, unsigned long slot)
349 {
350 unsigned long avpn = HPTE_V_AVPN_VAL(hpte_v);
351 unsigned long va;
352
353 va = avpn << 23;
354
355 if (! (hpte_v & HPTE_V_LARGE)) {
356 unsigned long vpi, pteg;
357
358 pteg = slot / HPTES_PER_GROUP;
359 if (hpte_v & HPTE_V_SECONDARY)
360 pteg = ~pteg;
361
362 vpi = ((va >> 28) ^ pteg) & htab_hash_mask;
363
364 va |= vpi << PAGE_SHIFT;
365 }
366
367 return va;
368 }
369
370 /*
371 * clear all mappings on kexec. All cpus are in real mode (or they will
372 * be when they isi), and we are the only one left. We rely on our kernel
373 * mapping being 0xC0's and the hardware ignoring those two real bits.
374 *
375 * TODO: add batching support when enabled. remember, no dynamic memory here,
376 * athough there is the control page available...
377 *
378 * XXX FIXME: 4k only for now !
379 */
380 static void native_hpte_clear(void)
381 {
382 unsigned long slot, slots, flags;
383 hpte_t *hptep = htab_address;
384 unsigned long hpte_v;
385 unsigned long pteg_count;
386
387 pteg_count = htab_hash_mask + 1;
388
389 local_irq_save(flags);
390
391 /* we take the tlbie lock and hold it. Some hardware will
392 * deadlock if we try to tlbie from two processors at once.
393 */
394 spin_lock(&native_tlbie_lock);
395
396 slots = pteg_count * HPTES_PER_GROUP;
397
398 for (slot = 0; slot < slots; slot++, hptep++) {
399 /*
400 * we could lock the pte here, but we are the only cpu
401 * running, right? and for crash dump, we probably
402 * don't want to wait for a maybe bad cpu.
403 */
404 hpte_v = hptep->v;
405
406 /*
407 * Call __tlbie() here rather than tlbie() since we
408 * already hold the native_tlbie_lock.
409 */
410 if (hpte_v & HPTE_V_VALID) {
411 hptep->v = 0;
412 __tlbie(slot2va(hpte_v, slot), MMU_PAGE_4K);
413 }
414 }
415
416 asm volatile("eieio; tlbsync; ptesync":::"memory");
417 spin_unlock(&native_tlbie_lock);
418 local_irq_restore(flags);
419 }
420
421 /*
422 * Batched hash table flush, we batch the tlbie's to avoid taking/releasing
423 * the lock all the time
424 */
425 static void native_flush_hash_range(unsigned long number, int local)
426 {
427 unsigned long va, hash, index, hidx, shift, slot;
428 hpte_t *hptep;
429 unsigned long hpte_v;
430 unsigned long want_v;
431 unsigned long flags;
432 real_pte_t pte;
433 struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);
434 unsigned long psize = batch->psize;
435 int i;
436
437 local_irq_save(flags);
438
439 for (i = 0; i < number; i++) {
440 va = batch->vaddr[i];
441 pte = batch->pte[i];
442
443 pte_iterate_hashed_subpages(pte, psize, va, index, shift) {
444 hash = hpt_hash(va, shift);
445 hidx = __rpte_to_hidx(pte, index);
446 if (hidx & _PTEIDX_SECONDARY)
447 hash = ~hash;
448 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
449 slot += hidx & _PTEIDX_GROUP_IX;
450 hptep = htab_address + slot;
451 want_v = hpte_encode_v(va, psize);
452 native_lock_hpte(hptep);
453 hpte_v = hptep->v;
454 if (!HPTE_V_COMPARE(hpte_v, want_v) ||
455 !(hpte_v & HPTE_V_VALID))
456 native_unlock_hpte(hptep);
457 else
458 hptep->v = 0;
459 } pte_iterate_hashed_end();
460 }
461
462 if (cpu_has_feature(CPU_FTR_TLBIEL) &&
463 mmu_psize_defs[psize].tlbiel && local) {
464 asm volatile("ptesync":::"memory");
465 for (i = 0; i < number; i++) {
466 va = batch->vaddr[i];
467 pte = batch->pte[i];
468
469 pte_iterate_hashed_subpages(pte, psize, va, index,
470 shift) {
471 __tlbiel(va, psize);
472 } pte_iterate_hashed_end();
473 }
474 asm volatile("ptesync":::"memory");
475 } else {
476 int lock_tlbie = !cpu_has_feature(CPU_FTR_LOCKLESS_TLBIE);
477
478 if (lock_tlbie)
479 spin_lock(&native_tlbie_lock);
480
481 asm volatile("ptesync":::"memory");
482 for (i = 0; i < number; i++) {
483 va = batch->vaddr[i];
484 pte = batch->pte[i];
485
486 pte_iterate_hashed_subpages(pte, psize, va, index,
487 shift) {
488 __tlbie(va, psize);
489 } pte_iterate_hashed_end();
490 }
491 asm volatile("eieio; tlbsync; ptesync":::"memory");
492
493 if (lock_tlbie)
494 spin_unlock(&native_tlbie_lock);
495 }
496
497 local_irq_restore(flags);
498 }
499
500 #ifdef CONFIG_PPC_PSERIES
501 /* Disable TLB batching on nighthawk */
502 static inline int tlb_batching_enabled(void)
503 {
504 struct device_node *root = of_find_node_by_path("/");
505 int enabled = 1;
506
507 if (root) {
508 const char *model = get_property(root, "model", NULL);
509 if (model && !strcmp(model, "IBM,9076-N81"))
510 enabled = 0;
511 of_node_put(root);
512 }
513
514 return enabled;
515 }
516 #else
517 static inline int tlb_batching_enabled(void)
518 {
519 return 1;
520 }
521 #endif
522
523 void hpte_init_native(void)
524 {
525 ppc_md.hpte_invalidate = native_hpte_invalidate;
526 ppc_md.hpte_updatepp = native_hpte_updatepp;
527 ppc_md.hpte_updateboltedpp = native_hpte_updateboltedpp;
528 ppc_md.hpte_insert = native_hpte_insert;
529 ppc_md.hpte_remove = native_hpte_remove;
530 ppc_md.hpte_clear_all = native_hpte_clear;
531 if (tlb_batching_enabled())
532 ppc_md.flush_hash_range = native_flush_hash_range;
533 htab_finish_init();
534 }
Linux with added FPC-III support