| blob | bd68df5fa78ac083bc0479946b1085c44934f5b7 |
1 /*
2 * $Id: hashtable.S,v 1.6 1999/10/08 01:56:15 paulus Exp $
3 *
4 * PowerPC version
5 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
6 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
7 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
8 * Adapted for Power Macintosh by Paul Mackerras.
9 * Low-level exception handlers and MMU support
10 * rewritten by Paul Mackerras.
11 * Copyright (C) 1996 Paul Mackerras.
12 *
13 * This file contains low-level assembler routines for managing
14 * the PowerPC MMU hash table. (PPC 8xx processors don't use a
15 * hash table, so this file is not used on them.)
16 *
17 * This program is free software; you can redistribute it and/or
18 * modify it under the terms of the GNU General Public License
19 * as published by the Free Software Foundation; either version
20 * 2 of the License, or (at your option) any later version.
21 *
22 */
24 #include <asm/reg.h>
25 #include <asm/page.h>
26 #include <asm/pgtable.h>
27 #include <asm/cputable.h>
28 #include <asm/ppc_asm.h>
29 #include <asm/thread_info.h>
30 #include <asm/asm-offsets.h>
32 #ifdef CONFIG_SMP
33 .comm mmu_hash_lock,4
34 #endif /* CONFIG_SMP */
36 /*
37 * Sync CPUs with hash_page taking & releasing the hash
38 * table lock
39 */
40 #ifdef CONFIG_SMP
41 .text
42 _GLOBAL(hash_page_sync)
43 lis r8,mmu_hash_lock@h
44 ori r8,r8,mmu_hash_lock@l
45 lis r0,0x0fff
46 b 10f
47 11: lwz r6,0(r8)
48 cmpwi 0,r6,0
49 bne 11b
50 10: lwarx r6,0,r8
51 cmpwi 0,r6,0
52 bne- 11b
53 stwcx. r0,0,r8
54 bne- 10b
55 isync
56 eieio
57 li r0,0
58 stw r0,0(r8)
59 blr
60 #endif
62 /*
63 * Load a PTE into the hash table, if possible.
64 * The address is in r4, and r3 contains an access flag:
65 * _PAGE_RW (0x400) if a write.
66 * r9 contains the SRR1 value, from which we use the MSR_PR bit.
67 * SPRG3 contains the physical address of the current task's thread.
68 *
69 * Returns to the caller if the access is illegal or there is no
70 * mapping for the address. Otherwise it places an appropriate PTE
71 * in the hash table and returns from the exception.
72 * Uses r0, r3 - r8, ctr, lr.
73 */
74 .text
75 _GLOBAL(hash_page)
76 tophys(r7,0) /* gets -KERNELBASE into r7 */
77 #ifdef CONFIG_SMP
78 addis r8,r7,mmu_hash_lock@h
79 ori r8,r8,mmu_hash_lock@l
80 lis r0,0x0fff
81 b 10f
82 11: lwz r6,0(r8)
83 cmpwi 0,r6,0
84 bne 11b
85 10: lwarx r6,0,r8
86 cmpwi 0,r6,0
87 bne- 11b
88 stwcx. r0,0,r8
89 bne- 10b
90 isync
91 #endif
92 /* Get PTE (linux-style) and check access */
93 lis r0,KERNELBASE@h /* check if kernel address */
94 cmplw 0,r4,r0
95 mfspr r8,SPRN_SPRG3 /* current task's THREAD (phys) */
96 ori r3,r3,_PAGE_USER|_PAGE_PRESENT /* test low addresses as user */
97 lwz r5,PGDIR(r8) /* virt page-table root */
98 blt+ 112f /* assume user more likely */
99 lis r5,swapper_pg_dir@ha /* if kernel address, use */
100 addi r5,r5,swapper_pg_dir@l /* kernel page table */
101 rlwimi r3,r9,32-12,29,29 /* MSR_PR -> _PAGE_USER */
102 112: add r5,r5,r7 /* convert to phys addr */
103 rlwimi r5,r4,12,20,29 /* insert top 10 bits of address */
104 lwz r8,0(r5) /* get pmd entry */
105 rlwinm. r8,r8,0,0,19 /* extract address of pte page */
106 #ifdef CONFIG_SMP
107 beq- hash_page_out /* return if no mapping */
108 #else
109 /* XXX it seems like the 601 will give a machine fault on the
110 rfi if its alignment is wrong (bottom 4 bits of address are
111 8 or 0xc) and we have had a not-taken conditional branch
112 to the address following the rfi. */
113 beqlr-
114 #endif
115 rlwimi r8,r4,22,20,29 /* insert next 10 bits of address */
116 rlwinm r0,r3,32-3,24,24 /* _PAGE_RW access -> _PAGE_DIRTY */
117 ori r0,r0,_PAGE_ACCESSED|_PAGE_HASHPTE
119 /*
120 * Update the linux PTE atomically. We do the lwarx up-front
121 * because almost always, there won't be a permission violation
122 * and there won't already be an HPTE, and thus we will have
123 * to update the PTE to set _PAGE_HASHPTE. -- paulus.
124 */
125 retry:
126 lwarx r6,0,r8 /* get linux-style pte */
127 andc. r5,r3,r6 /* check access & ~permission */
128 #ifdef CONFIG_SMP
129 bne- hash_page_out /* return if access not permitted */
130 #else
131 bnelr-
132 #endif
133 or r5,r0,r6 /* set accessed/dirty bits */
134 stwcx. r5,0,r8 /* attempt to update PTE */
135 bne- retry /* retry if someone got there first */
137 mfsrin r3,r4 /* get segment reg for segment */
138 mfctr r0
139 stw r0,_CTR(r11)
140 bl create_hpte /* add the hash table entry */
142 #ifdef CONFIG_SMP
143 eieio
144 addis r8,r7,mmu_hash_lock@ha
145 li r0,0
146 stw r0,mmu_hash_lock@l(r8)
147 #endif
149 /* Return from the exception */
150 lwz r5,_CTR(r11)
151 mtctr r5
152 lwz r0,GPR0(r11)
153 lwz r7,GPR7(r11)
154 lwz r8,GPR8(r11)
155 b fast_exception_return
157 #ifdef CONFIG_SMP
158 hash_page_out:
159 eieio
160 addis r8,r7,mmu_hash_lock@ha
161 li r0,0
162 stw r0,mmu_hash_lock@l(r8)
163 blr
164 #endif /* CONFIG_SMP */
166 /*
167 * Add an entry for a particular page to the hash table.
168 *
169 * add_hash_page(unsigned context, unsigned long va, unsigned long pmdval)
170 *
171 * We assume any necessary modifications to the pte (e.g. setting
172 * the accessed bit) have already been done and that there is actually
173 * a hash table in use (i.e. we're not on a 603).
174 */
175 _GLOBAL(add_hash_page)
176 mflr r0
177 stw r0,4(r1)
179 /* Convert context and va to VSID */
180 mulli r3,r3,897*16 /* multiply context by context skew */
181 rlwinm r0,r4,4,28,31 /* get ESID (top 4 bits of va) */
182 mulli r0,r0,0x111 /* multiply by ESID skew */
183 add r3,r3,r0 /* note create_hpte trims to 24 bits */
185 #ifdef CONFIG_SMP
186 rlwinm r8,r1,0,0,18 /* use cpu number to make tag */
187 lwz r8,TI_CPU(r8) /* to go in mmu_hash_lock */
188 oris r8,r8,12
189 #endif /* CONFIG_SMP */
191 /*
192 * We disable interrupts here, even on UP, because we don't
193 * want to race with hash_page, and because we want the
194 * _PAGE_HASHPTE bit to be a reliable indication of whether
195 * the HPTE exists (or at least whether one did once).
196 * We also turn off the MMU for data accesses so that we
197 * we can't take a hash table miss (assuming the code is
198 * covered by a BAT). -- paulus
199 */
200 mfmsr r10
201 SYNC
202 rlwinm r0,r10,0,17,15 /* clear bit 16 (MSR_EE) */
203 rlwinm r0,r0,0,28,26 /* clear MSR_DR */
204 mtmsr r0
205 SYNC_601
206 isync
208 tophys(r7,0)
210 #ifdef CONFIG_SMP
211 addis r9,r7,mmu_hash_lock@ha
212 addi r9,r9,mmu_hash_lock@l
213 10: lwarx r0,0,r9 /* take the mmu_hash_lock */
214 cmpi 0,r0,0
215 bne- 11f
216 stwcx. r8,0,r9
217 beq+ 12f
218 11: lwz r0,0(r9)
219 cmpi 0,r0,0
220 beq 10b
221 b 11b
222 12: isync
223 #endif
225 /*
226 * Fetch the linux pte and test and set _PAGE_HASHPTE atomically.
227 * If _PAGE_HASHPTE was already set, we don't replace the existing
228 * HPTE, so we just unlock and return.
229 */
230 mr r8,r5
231 rlwimi r8,r4,22,20,29
232 1: lwarx r6,0,r8
233 andi. r0,r6,_PAGE_HASHPTE
234 bne 9f /* if HASHPTE already set, done */
235 ori r5,r6,_PAGE_HASHPTE
236 stwcx. r5,0,r8
237 bne- 1b
239 bl create_hpte
241 9:
242 #ifdef CONFIG_SMP
243 eieio
244 li r0,0
245 stw r0,0(r9) /* clear mmu_hash_lock */
246 #endif
248 /* reenable interrupts and DR */
249 mtmsr r10
250 SYNC_601
251 isync
253 lwz r0,4(r1)
254 mtlr r0
255 blr
257 /*
258 * This routine adds a hardware PTE to the hash table.
259 * It is designed to be called with the MMU either on or off.
260 * r3 contains the VSID, r4 contains the virtual address,
261 * r5 contains the linux PTE, r6 contains the old value of the
262 * linux PTE (before setting _PAGE_HASHPTE) and r7 contains the
263 * offset to be added to addresses (0 if the MMU is on,
264 * -KERNELBASE if it is off).
265 * On SMP, the caller should have the mmu_hash_lock held.
266 * We assume that the caller has (or will) set the _PAGE_HASHPTE
267 * bit in the linux PTE in memory. The value passed in r6 should
268 * be the old linux PTE value; if it doesn't have _PAGE_HASHPTE set
269 * this routine will skip the search for an existing HPTE.
270 * This procedure modifies r0, r3 - r6, r8, cr0.
271 * -- paulus.
272 *
273 * For speed, 4 of the instructions get patched once the size and
274 * physical address of the hash table are known. These definitions
275 * of Hash_base and Hash_bits below are just an example.
276 */
277 Hash_base = 0xc0180000
278 Hash_bits = 12 /* e.g. 256kB hash table */
279 Hash_msk = (((1 << Hash_bits) - 1) * 64)
281 /* defines for the PTE format for 32-bit PPCs */
282 #define PTE_SIZE 8
283 #define PTEG_SIZE 64
284 #define LG_PTEG_SIZE 6
285 #define LDPTEu lwzu
286 #define STPTE stw
287 #define CMPPTE cmpw
288 #define PTE_H 0x40
289 #define PTE_V 0x80000000
290 #define TST_V(r) rlwinm. r,r,0,0,0
291 #define SET_V(r) oris r,r,PTE_V@h
292 #define CLR_V(r,t) rlwinm r,r,0,1,31
294 #define HASH_LEFT 31-(LG_PTEG_SIZE+Hash_bits-1)
295 #define HASH_RIGHT 31-LG_PTEG_SIZE
297 _GLOBAL(create_hpte)
298 /* Convert linux-style PTE (r5) to low word of PPC-style PTE (r8) */
299 rlwinm r8,r5,32-10,31,31 /* _PAGE_RW -> PP lsb */
300 rlwinm r0,r5,32-7,31,31 /* _PAGE_DIRTY -> PP lsb */
301 and r8,r8,r0 /* writable if _RW & _DIRTY */
302 rlwimi r5,r5,32-1,30,30 /* _PAGE_USER -> PP msb */
303 rlwimi r5,r5,32-2,31,31 /* _PAGE_USER -> PP lsb */
304 ori r8,r8,0xe14 /* clear out reserved bits and M */
305 andc r8,r5,r8 /* PP = user? (rw&dirty? 2: 3): 0 */
306 BEGIN_FTR_SECTION
307 ori r8,r8,_PAGE_COHERENT /* set M (coherence required) */
308 END_FTR_SECTION_IFSET(CPU_FTR_NEED_COHERENT)
310 /* Construct the high word of the PPC-style PTE (r5) */
311 rlwinm r5,r3,7,1,24 /* put VSID in 0x7fffff80 bits */
312 rlwimi r5,r4,10,26,31 /* put in API (abbrev page index) */
313 SET_V(r5) /* set V (valid) bit */
315 /* Get the address of the primary PTE group in the hash table (r3) */
316 _GLOBAL(hash_page_patch_A)
317 addis r0,r7,Hash_base@h /* base address of hash table */
318 rlwimi r0,r3,LG_PTEG_SIZE,HASH_LEFT,HASH_RIGHT /* VSID -> hash */
319 rlwinm r3,r4,20+LG_PTEG_SIZE,HASH_LEFT,HASH_RIGHT /* PI -> hash */
320 xor r3,r3,r0 /* make primary hash */
321 li r0,8 /* PTEs/group */
323 /*
324 * Test the _PAGE_HASHPTE bit in the old linux PTE, and skip the search
325 * if it is clear, meaning that the HPTE isn't there already...
326 */
327 andi. r6,r6,_PAGE_HASHPTE
328 beq+ 10f /* no PTE: go look for an empty slot */
329 tlbie r4
331 addis r4,r7,htab_hash_searches@ha
332 lwz r6,htab_hash_searches@l(r4)
333 addi r6,r6,1 /* count how many searches we do */
334 stw r6,htab_hash_searches@l(r4)
336 /* Search the primary PTEG for a PTE whose 1st (d)word matches r5 */
337 mtctr r0
338 addi r4,r3,-PTE_SIZE
339 1: LDPTEu r6,PTE_SIZE(r4) /* get next PTE */
340 CMPPTE 0,r6,r5
341 bdnzf 2,1b /* loop while ctr != 0 && !cr0.eq */
342 beq+ found_slot
344 /* Search the secondary PTEG for a matching PTE */
345 ori r5,r5,PTE_H /* set H (secondary hash) bit */
346 _GLOBAL(hash_page_patch_B)
347 xoris r4,r3,Hash_msk>>16 /* compute secondary hash */
348 xori r4,r4,(-PTEG_SIZE & 0xffff)
349 addi r4,r4,-PTE_SIZE
350 mtctr r0
351 2: LDPTEu r6,PTE_SIZE(r4)
352 CMPPTE 0,r6,r5
353 bdnzf 2,2b
354 beq+ found_slot
355 xori r5,r5,PTE_H /* clear H bit again */
357 /* Search the primary PTEG for an empty slot */
358 10: mtctr r0
359 addi r4,r3,-PTE_SIZE /* search primary PTEG */
360 1: LDPTEu r6,PTE_SIZE(r4) /* get next PTE */
361 TST_V(r6) /* test valid bit */
362 bdnzf 2,1b /* loop while ctr != 0 && !cr0.eq */
363 beq+ found_empty
365 /* update counter of times that the primary PTEG is full */
366 addis r4,r7,primary_pteg_full@ha
367 lwz r6,primary_pteg_full@l(r4)
368 addi r6,r6,1
369 stw r6,primary_pteg_full@l(r4)
371 /* Search the secondary PTEG for an empty slot */
372 ori r5,r5,PTE_H /* set H (secondary hash) bit */
373 _GLOBAL(hash_page_patch_C)
374 xoris r4,r3,Hash_msk>>16 /* compute secondary hash */
375 xori r4,r4,(-PTEG_SIZE & 0xffff)
376 addi r4,r4,-PTE_SIZE
377 mtctr r0
378 2: LDPTEu r6,PTE_SIZE(r4)
379 TST_V(r6)
380 bdnzf 2,2b
381 beq+ found_empty
382 xori r5,r5,PTE_H /* clear H bit again */
384 /*
385 * Choose an arbitrary slot in the primary PTEG to overwrite.
386 * Since both the primary and secondary PTEGs are full, and we
387 * have no information that the PTEs in the primary PTEG are
388 * more important or useful than those in the secondary PTEG,
389 * and we know there is a definite (although small) speed
390 * advantage to putting the PTE in the primary PTEG, we always
391 * put the PTE in the primary PTEG.
392 */
393 addis r4,r7,next_slot@ha
394 lwz r6,next_slot@l(r4)
395 addi r6,r6,PTE_SIZE
396 andi. r6,r6,7*PTE_SIZE
397 stw r6,next_slot@l(r4)
398 add r4,r3,r6
400 #ifndef CONFIG_SMP
401 /* Store PTE in PTEG */
402 found_empty:
403 STPTE r5,0(r4)
404 found_slot:
405 STPTE r8,PTE_SIZE/2(r4)
407 #else /* CONFIG_SMP */
408 /*
409 * Between the tlbie above and updating the hash table entry below,
410 * another CPU could read the hash table entry and put it in its TLB.
411 * There are 3 cases:
412 * 1. using an empty slot
413 * 2. updating an earlier entry to change permissions (i.e. enable write)
414 * 3. taking over the PTE for an unrelated address
415 *
416 * In each case it doesn't really matter if the other CPUs have the old
417 * PTE in their TLB. So we don't need to bother with another tlbie here,
418 * which is convenient as we've overwritten the register that had the
419 * address. :-) The tlbie above is mainly to make sure that this CPU comes
420 * and gets the new PTE from the hash table.
421 *
422 * We do however have to make sure that the PTE is never in an invalid
423 * state with the V bit set.
424 */
425 found_empty:
426 found_slot:
427 CLR_V(r5,r0) /* clear V (valid) bit in PTE */
428 STPTE r5,0(r4)
429 sync
430 TLBSYNC
431 STPTE r8,PTE_SIZE/2(r4) /* put in correct RPN, WIMG, PP bits */
432 sync
433 SET_V(r5)
434 STPTE r5,0(r4) /* finally set V bit in PTE */
435 #endif /* CONFIG_SMP */
437 sync /* make sure pte updates get to memory */
438 blr
440 .comm next_slot,4
441 .comm primary_pteg_full,4
442 .comm htab_hash_searches,4
444 /*
445 * Flush the entry for a particular page from the hash table.
446 *
447 * flush_hash_pages(unsigned context, unsigned long va, unsigned long pmdval,
448 * int count)
449 *
450 * We assume that there is a hash table in use (Hash != 0).
451 */
452 _GLOBAL(flush_hash_pages)
453 tophys(r7,0)
455 /*
456 * We disable interrupts here, even on UP, because we want
457 * the _PAGE_HASHPTE bit to be a reliable indication of
458 * whether the HPTE exists (or at least whether one did once).
459 * We also turn off the MMU for data accesses so that we
460 * we can't take a hash table miss (assuming the code is
461 * covered by a BAT). -- paulus
462 */
463 mfmsr r10
464 SYNC
465 rlwinm r0,r10,0,17,15 /* clear bit 16 (MSR_EE) */
466 rlwinm r0,r0,0,28,26 /* clear MSR_DR */
467 mtmsr r0
468 SYNC_601
469 isync
471 /* First find a PTE in the range that has _PAGE_HASHPTE set */
472 rlwimi r5,r4,22,20,29
473 1: lwz r0,0(r5)
474 cmpwi cr1,r6,1
475 andi. r0,r0,_PAGE_HASHPTE
476 bne 2f
477 ble cr1,19f
478 addi r4,r4,0x1000
479 addi r5,r5,4
480 addi r6,r6,-1
481 b 1b
483 /* Convert context and va to VSID */
484 2: mulli r3,r3,897*16 /* multiply context by context skew */
485 rlwinm r0,r4,4,28,31 /* get ESID (top 4 bits of va) */
486 mulli r0,r0,0x111 /* multiply by ESID skew */
487 add r3,r3,r0 /* note code below trims to 24 bits */
489 /* Construct the high word of the PPC-style PTE (r11) */
490 rlwinm r11,r3,7,1,24 /* put VSID in 0x7fffff80 bits */
491 rlwimi r11,r4,10,26,31 /* put in API (abbrev page index) */
492 SET_V(r11) /* set V (valid) bit */
494 #ifdef CONFIG_SMP
495 addis r9,r7,mmu_hash_lock@ha
496 addi r9,r9,mmu_hash_lock@l
497 rlwinm r8,r1,0,0,18
498 add r8,r8,r7
499 lwz r8,TI_CPU(r8)
500 oris r8,r8,9
501 10: lwarx r0,0,r9
502 cmpi 0,r0,0
503 bne- 11f
504 stwcx. r8,0,r9
505 beq+ 12f
506 11: lwz r0,0(r9)
507 cmpi 0,r0,0
508 beq 10b
509 b 11b
510 12: isync
511 #endif
513 /*
514 * Check the _PAGE_HASHPTE bit in the linux PTE. If it is
515 * already clear, we're done (for this pte). If not,
516 * clear it (atomically) and proceed. -- paulus.
517 */
518 33: lwarx r8,0,r5 /* fetch the pte */
519 andi. r0,r8,_PAGE_HASHPTE
520 beq 8f /* done if HASHPTE is already clear */
521 rlwinm r8,r8,0,31,29 /* clear HASHPTE bit */
522 stwcx. r8,0,r5 /* update the pte */
523 bne- 33b
525 /* Get the address of the primary PTE group in the hash table (r3) */
526 _GLOBAL(flush_hash_patch_A)
527 addis r8,r7,Hash_base@h /* base address of hash table */
528 rlwimi r8,r3,LG_PTEG_SIZE,HASH_LEFT,HASH_RIGHT /* VSID -> hash */
529 rlwinm r0,r4,20+LG_PTEG_SIZE,HASH_LEFT,HASH_RIGHT /* PI -> hash */
530 xor r8,r0,r8 /* make primary hash */
532 /* Search the primary PTEG for a PTE whose 1st (d)word matches r5 */
533 li r0,8 /* PTEs/group */
534 mtctr r0
535 addi r12,r8,-PTE_SIZE
536 1: LDPTEu r0,PTE_SIZE(r12) /* get next PTE */
537 CMPPTE 0,r0,r11
538 bdnzf 2,1b /* loop while ctr != 0 && !cr0.eq */
539 beq+ 3f
541 /* Search the secondary PTEG for a matching PTE */
542 ori r11,r11,PTE_H /* set H (secondary hash) bit */
543 li r0,8 /* PTEs/group */
544 _GLOBAL(flush_hash_patch_B)
545 xoris r12,r8,Hash_msk>>16 /* compute secondary hash */
546 xori r12,r12,(-PTEG_SIZE & 0xffff)
547 addi r12,r12,-PTE_SIZE
548 mtctr r0
549 2: LDPTEu r0,PTE_SIZE(r12)
550 CMPPTE 0,r0,r11
551 bdnzf 2,2b
552 xori r11,r11,PTE_H /* clear H again */
553 bne- 4f /* should rarely fail to find it */
555 3: li r0,0
556 STPTE r0,0(r12) /* invalidate entry */
557 4: sync
558 tlbie r4 /* in hw tlb too */
559 sync
561 8: ble cr1,9f /* if all ptes checked */
562 81: addi r6,r6,-1
563 addi r5,r5,4 /* advance to next pte */
564 addi r4,r4,0x1000
565 lwz r0,0(r5) /* check next pte */
566 cmpwi cr1,r6,1
567 andi. r0,r0,_PAGE_HASHPTE
568 bne 33b
569 bgt cr1,81b
571 9:
572 #ifdef CONFIG_SMP
573 TLBSYNC
574 li r0,0
575 stw r0,0(r9) /* clear mmu_hash_lock */
576 #endif
578 19: mtmsr r10
579 SYNC_601
580 isync
581 blr