| blob | a242b5d7cbe477c109a78152a344b663444738a1 |
1 /*
2 * ppc64 MMU hashtable management routines
3 *
4 * (c) Copyright IBM Corp. 2003, 2005
5 *
6 * Maintained by: Benjamin Herrenschmidt
7 * <benh@kernel.crashing.org>
8 *
9 * This file is covered by the GNU Public Licence v2 as
10 * described in the kernel's COPYING file.
11 */
13 #include <asm/reg.h>
14 #include <asm/pgtable.h>
15 #include <asm/mmu.h>
16 #include <asm/page.h>
17 #include <asm/types.h>
18 #include <asm/ppc_asm.h>
19 #include <asm/asm-offsets.h>
20 #include <asm/cputable.h>
22 .text
24 /*
25 * Stackframe:
26 *
27 * +-> Back chain (SP + 256)
28 * | General register save area (SP + 112)
29 * | Parameter save area (SP + 48)
30 * | TOC save area (SP + 40)
31 * | link editor doubleword (SP + 32)
32 * | compiler doubleword (SP + 24)
33 * | LR save area (SP + 16)
34 * | CR save area (SP + 8)
35 * SP ---> +-- Back chain (SP + 0)
36 */
37 #define STACKFRAMESIZE 256
39 /* Save parameters offsets */
40 #define STK_PARM(i) (STACKFRAMESIZE + 48 + ((i)-3)*8)
42 /* Save non-volatile offsets */
43 #define STK_REG(i) (112 + ((i)-14)*8)
46 #ifndef CONFIG_PPC_64K_PAGES
48 /*****************************************************************************
49 * *
50 * 4K SW & 4K HW pages implementation *
51 * *
52 *****************************************************************************/
55 /*
56 * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
57 * pte_t *ptep, unsigned long trap, int local, int ssize)
58 *
59 * Adds a 4K page to the hash table in a segment of 4K pages only
60 */
62 _GLOBAL(__hash_page_4K)
63 mflr r0
64 std r0,16(r1)
65 stdu r1,-STACKFRAMESIZE(r1)
66 /* Save all params that we need after a function call */
67 std r6,STK_PARM(r6)(r1)
68 std r8,STK_PARM(r8)(r1)
69 std r9,STK_PARM(r9)(r1)
71 /* Save non-volatile registers.
72 * r31 will hold "old PTE"
73 * r30 is "new PTE"
74 * r29 is "va"
75 * r28 is a hash value
76 * r27 is hashtab mask (maybe dynamic patched instead ?)
77 */
78 std r27,STK_REG(r27)(r1)
79 std r28,STK_REG(r28)(r1)
80 std r29,STK_REG(r29)(r1)
81 std r30,STK_REG(r30)(r1)
82 std r31,STK_REG(r31)(r1)
84 /* Step 1:
85 *
86 * Check permissions, atomically mark the linux PTE busy
87 * and hashed.
88 */
89 1:
90 ldarx r31,0,r6
91 /* Check access rights (access & ~(pte_val(*ptep))) */
92 andc. r0,r4,r31
93 bne- htab_wrong_access
94 /* Check if PTE is busy */
95 andi. r0,r31,_PAGE_BUSY
96 /* If so, just bail out and refault if needed. Someone else
97 * is changing this PTE anyway and might hash it.
98 */
99 bne- htab_bail_ok
101 /* Prepare new PTE value (turn access RW into DIRTY, then
102 * add BUSY,HASHPTE and ACCESSED)
103 */
104 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
105 or r30,r30,r31
106 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
107 /* Write the linux PTE atomically (setting busy) */
108 stdcx. r30,0,r6
109 bne- 1b
110 isync
112 /* Step 2:
113 *
114 * Insert/Update the HPTE in the hash table. At this point,
115 * r4 (access) is re-useable, we use it for the new HPTE flags
116 */
118 BEGIN_FTR_SECTION
119 cmpdi r9,0 /* check segment size */
120 bne 3f
121 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
122 /* Calc va and put it in r29 */
123 rldicr r29,r5,28,63-28
124 rldicl r3,r3,0,36
125 or r29,r3,r29
127 /* Calculate hash value for primary slot and store it in r28 */
128 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
129 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */
130 xor r28,r5,r0
131 b 4f
133 3: /* Calc VA and hash in r29 and r28 for 1T segment */
134 sldi r29,r5,40 /* vsid << 40 */
135 clrldi r3,r3,24 /* ea & 0xffffffffff */
136 rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */
137 clrldi r5,r5,40 /* vsid & 0xffffff */
138 rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */
139 xor r28,r28,r5
140 or r29,r3,r29 /* VA */
141 xor r28,r28,r0 /* hash */
143 /* Convert linux PTE bits into HW equivalents */
144 4: andi. r3,r30,0x1fe /* Get basic set of flags */
145 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
146 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
147 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
148 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
149 andc r0,r30,r0 /* r0 = pte & ~r0 */
150 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
151 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */
153 /* We eventually do the icache sync here (maybe inline that
154 * code rather than call a C function...)
155 */
156 BEGIN_FTR_SECTION
157 mr r4,r30
158 mr r5,r7
159 bl .hash_page_do_lazy_icache
160 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
162 /* At this point, r3 contains new PP bits, save them in
163 * place of "access" in the param area (sic)
164 */
165 std r3,STK_PARM(r4)(r1)
167 /* Get htab_hash_mask */
168 ld r4,htab_hash_mask@got(2)
169 ld r27,0(r4) /* htab_hash_mask -> r27 */
171 /* Check if we may already be in the hashtable, in this case, we
172 * go to out-of-line code to try to modify the HPTE
173 */
174 andi. r0,r31,_PAGE_HASHPTE
175 bne htab_modify_pte
177 htab_insert_pte:
178 /* Clear hpte bits in new pte (we also clear BUSY btw) and
179 * add _PAGE_HASHPTE
180 */
181 lis r0,_PAGE_HPTEFLAGS@h
182 ori r0,r0,_PAGE_HPTEFLAGS@l
183 andc r30,r30,r0
184 ori r30,r30,_PAGE_HASHPTE
186 /* physical address r5 */
187 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
188 sldi r5,r5,PAGE_SHIFT
190 /* Calculate primary group hash */
191 and r0,r28,r27
192 rldicr r3,r0,3,63-3 /* r3 = (hash & mask) << 3 */
194 /* Call ppc_md.hpte_insert */
195 ld r6,STK_PARM(r4)(r1) /* Retrieve new pp bits */
196 mr r4,r29 /* Retrieve va */
197 li r7,0 /* !bolted, !secondary */
198 li r8,MMU_PAGE_4K /* page size */
199 ld r9,STK_PARM(r9)(r1) /* segment size */
200 _GLOBAL(htab_call_hpte_insert1)
201 bl . /* Patched by htab_finish_init() */
202 cmpdi 0,r3,0
203 bge htab_pte_insert_ok /* Insertion successful */
204 cmpdi 0,r3,-2 /* Critical failure */
205 beq- htab_pte_insert_failure
207 /* Now try secondary slot */
209 /* physical address r5 */
210 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
211 sldi r5,r5,PAGE_SHIFT
213 /* Calculate secondary group hash */
214 andc r0,r27,r28
215 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
217 /* Call ppc_md.hpte_insert */
218 ld r6,STK_PARM(r4)(r1) /* Retrieve new pp bits */
219 mr r4,r29 /* Retrieve va */
220 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
221 li r8,MMU_PAGE_4K /* page size */
222 ld r9,STK_PARM(r9)(r1) /* segment size */
223 _GLOBAL(htab_call_hpte_insert2)
224 bl . /* Patched by htab_finish_init() */
225 cmpdi 0,r3,0
226 bge+ htab_pte_insert_ok /* Insertion successful */
227 cmpdi 0,r3,-2 /* Critical failure */
228 beq- htab_pte_insert_failure
230 /* Both are full, we need to evict something */
231 mftb r0
232 /* Pick a random group based on TB */
233 andi. r0,r0,1
234 mr r5,r28
235 bne 2f
236 not r5,r5
237 2: and r0,r5,r27
238 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
239 /* Call ppc_md.hpte_remove */
240 _GLOBAL(htab_call_hpte_remove)
241 bl . /* Patched by htab_finish_init() */
243 /* Try all again */
244 b htab_insert_pte
246 htab_bail_ok:
247 li r3,0
248 b htab_bail
250 htab_pte_insert_ok:
251 /* Insert slot number & secondary bit in PTE */
252 rldimi r30,r3,12,63-15
254 /* Write out the PTE with a normal write
255 * (maybe add eieio may be good still ?)
256 */
257 htab_write_out_pte:
258 ld r6,STK_PARM(r6)(r1)
259 std r30,0(r6)
260 li r3, 0
261 htab_bail:
262 ld r27,STK_REG(r27)(r1)
263 ld r28,STK_REG(r28)(r1)
264 ld r29,STK_REG(r29)(r1)
265 ld r30,STK_REG(r30)(r1)
266 ld r31,STK_REG(r31)(r1)
267 addi r1,r1,STACKFRAMESIZE
268 ld r0,16(r1)
269 mtlr r0
270 blr
272 htab_modify_pte:
273 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
274 mr r4,r3
275 rlwinm r3,r31,32-12,29,31
277 /* Secondary group ? if yes, get a inverted hash value */
278 mr r5,r28
279 andi. r0,r31,_PAGE_SECONDARY
280 beq 1f
281 not r5,r5
282 1:
283 /* Calculate proper slot value for ppc_md.hpte_updatepp */
284 and r0,r5,r27
285 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
286 add r3,r0,r3 /* add slot idx */
288 /* Call ppc_md.hpte_updatepp */
289 mr r5,r29 /* va */
290 li r6,MMU_PAGE_4K /* page size */
291 ld r7,STK_PARM(r9)(r1) /* segment size */
292 ld r8,STK_PARM(r8)(r1) /* get "local" param */
293 _GLOBAL(htab_call_hpte_updatepp)
294 bl . /* Patched by htab_finish_init() */
296 /* if we failed because typically the HPTE wasn't really here
297 * we try an insertion.
298 */
299 cmpdi 0,r3,-1
300 beq- htab_insert_pte
302 /* Clear the BUSY bit and Write out the PTE */
303 li r0,_PAGE_BUSY
304 andc r30,r30,r0
305 b htab_write_out_pte
307 htab_wrong_access:
308 /* Bail out clearing reservation */
309 stdcx. r31,0,r6
310 li r3,1
311 b htab_bail
313 htab_pte_insert_failure:
314 /* Bail out restoring old PTE */
315 ld r6,STK_PARM(r6)(r1)
316 std r31,0(r6)
317 li r3,-1
318 b htab_bail
321 #else /* CONFIG_PPC_64K_PAGES */
324 /*****************************************************************************
325 * *
326 * 64K SW & 4K or 64K HW in a 4K segment pages implementation *
327 * *
328 *****************************************************************************/
330 /* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
331 * pte_t *ptep, unsigned long trap, int local, int ssize,
332 * int subpg_prot)
333 */
335 /*
336 * For now, we do NOT implement Admixed pages
337 */
338 _GLOBAL(__hash_page_4K)
339 mflr r0
340 std r0,16(r1)
341 stdu r1,-STACKFRAMESIZE(r1)
342 /* Save all params that we need after a function call */
343 std r6,STK_PARM(r6)(r1)
344 std r8,STK_PARM(r8)(r1)
345 std r9,STK_PARM(r9)(r1)
347 /* Save non-volatile registers.
348 * r31 will hold "old PTE"
349 * r30 is "new PTE"
350 * r29 is "va"
351 * r28 is a hash value
352 * r27 is hashtab mask (maybe dynamic patched instead ?)
353 * r26 is the hidx mask
354 * r25 is the index in combo page
355 */
356 std r25,STK_REG(r25)(r1)
357 std r26,STK_REG(r26)(r1)
358 std r27,STK_REG(r27)(r1)
359 std r28,STK_REG(r28)(r1)
360 std r29,STK_REG(r29)(r1)
361 std r30,STK_REG(r30)(r1)
362 std r31,STK_REG(r31)(r1)
364 /* Step 1:
365 *
366 * Check permissions, atomically mark the linux PTE busy
367 * and hashed.
368 */
369 1:
370 ldarx r31,0,r6
371 /* Check access rights (access & ~(pte_val(*ptep))) */
372 andc. r0,r4,r31
373 bne- htab_wrong_access
374 /* Check if PTE is busy */
375 andi. r0,r31,_PAGE_BUSY
376 /* If so, just bail out and refault if needed. Someone else
377 * is changing this PTE anyway and might hash it.
378 */
379 bne- htab_bail_ok
380 /* Prepare new PTE value (turn access RW into DIRTY, then
381 * add BUSY and ACCESSED)
382 */
383 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
384 or r30,r30,r31
385 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
386 oris r30,r30,_PAGE_COMBO@h
387 /* Write the linux PTE atomically (setting busy) */
388 stdcx. r30,0,r6
389 bne- 1b
390 isync
392 /* Step 2:
393 *
394 * Insert/Update the HPTE in the hash table. At this point,
395 * r4 (access) is re-useable, we use it for the new HPTE flags
396 */
398 /* Load the hidx index */
399 rldicl r25,r3,64-12,60
401 BEGIN_FTR_SECTION
402 cmpdi r9,0 /* check segment size */
403 bne 3f
404 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
405 /* Calc va and put it in r29 */
406 rldicr r29,r5,28,63-28 /* r29 = (vsid << 28) */
407 rldicl r3,r3,0,36 /* r3 = (ea & 0x0fffffff) */
408 or r29,r3,r29 /* r29 = va */
410 /* Calculate hash value for primary slot and store it in r28 */
411 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
412 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */
413 xor r28,r5,r0
414 b 4f
416 3: /* Calc VA and hash in r29 and r28 for 1T segment */
417 sldi r29,r5,40 /* vsid << 40 */
418 clrldi r3,r3,24 /* ea & 0xffffffffff */
419 rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */
420 clrldi r5,r5,40 /* vsid & 0xffffff */
421 rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */
422 xor r28,r28,r5
423 or r29,r3,r29 /* VA */
424 xor r28,r28,r0 /* hash */
426 /* Convert linux PTE bits into HW equivalents */
427 4:
428 #ifdef CONFIG_PPC_SUBPAGE_PROT
429 andc r10,r30,r10
430 andi. r3,r10,0x1fe /* Get basic set of flags */
431 rlwinm r0,r10,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
432 #else
433 andi. r3,r30,0x1fe /* Get basic set of flags */
434 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
435 #endif
436 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
437 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
438 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
439 andc r0,r3,r0 /* r0 = pte & ~r0 */
440 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
441 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */
443 /* We eventually do the icache sync here (maybe inline that
444 * code rather than call a C function...)
445 */
446 BEGIN_FTR_SECTION
447 mr r4,r30
448 mr r5,r7
449 bl .hash_page_do_lazy_icache
450 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
452 /* At this point, r3 contains new PP bits, save them in
453 * place of "access" in the param area (sic)
454 */
455 std r3,STK_PARM(r4)(r1)
457 /* Get htab_hash_mask */
458 ld r4,htab_hash_mask@got(2)
459 ld r27,0(r4) /* htab_hash_mask -> r27 */
461 /* Check if we may already be in the hashtable, in this case, we
462 * go to out-of-line code to try to modify the HPTE. We look for
463 * the bit at (1 >> (index + 32))
464 */
465 rldicl. r0,r31,64-12,48
466 li r26,0 /* Default hidx */
467 beq htab_insert_pte
469 /*
470 * Check if the pte was already inserted into the hash table
471 * as a 64k HW page, and invalidate the 64k HPTE if so.
472 */
473 andis. r0,r31,_PAGE_COMBO@h
474 beq htab_inval_old_hpte
476 ld r6,STK_PARM(r6)(r1)
477 ori r26,r6,0x8000 /* Load the hidx mask */
478 ld r26,0(r26)
479 addi r5,r25,36 /* Check actual HPTE_SUB bit, this */
480 rldcr. r0,r31,r5,0 /* must match pgtable.h definition */
481 bne htab_modify_pte
483 htab_insert_pte:
484 /* real page number in r5, PTE RPN value + index */
485 andis. r0,r31,_PAGE_4K_PFN@h
486 srdi r5,r31,PTE_RPN_SHIFT
487 bne- htab_special_pfn
488 sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
489 add r5,r5,r25
490 htab_special_pfn:
491 sldi r5,r5,HW_PAGE_SHIFT
493 /* Calculate primary group hash */
494 and r0,r28,r27
495 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
497 /* Call ppc_md.hpte_insert */
498 ld r6,STK_PARM(r4)(r1) /* Retrieve new pp bits */
499 mr r4,r29 /* Retrieve va */
500 li r7,0 /* !bolted, !secondary */
501 li r8,MMU_PAGE_4K /* page size */
502 ld r9,STK_PARM(r9)(r1) /* segment size */
503 _GLOBAL(htab_call_hpte_insert1)
504 bl . /* patched by htab_finish_init() */
505 cmpdi 0,r3,0
506 bge htab_pte_insert_ok /* Insertion successful */
507 cmpdi 0,r3,-2 /* Critical failure */
508 beq- htab_pte_insert_failure
510 /* Now try secondary slot */
512 /* real page number in r5, PTE RPN value + index */
513 andis. r0,r31,_PAGE_4K_PFN@h
514 srdi r5,r31,PTE_RPN_SHIFT
515 bne- 3f
516 sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
517 add r5,r5,r25
518 3: sldi r5,r5,HW_PAGE_SHIFT
520 /* Calculate secondary group hash */
521 andc r0,r27,r28
522 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
524 /* Call ppc_md.hpte_insert */
525 ld r6,STK_PARM(r4)(r1) /* Retrieve new pp bits */
526 mr r4,r29 /* Retrieve va */
527 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
528 li r8,MMU_PAGE_4K /* page size */
529 ld r9,STK_PARM(r9)(r1) /* segment size */
530 _GLOBAL(htab_call_hpte_insert2)
531 bl . /* patched by htab_finish_init() */
532 cmpdi 0,r3,0
533 bge+ htab_pte_insert_ok /* Insertion successful */
534 cmpdi 0,r3,-2 /* Critical failure */
535 beq- htab_pte_insert_failure
537 /* Both are full, we need to evict something */
538 mftb r0
539 /* Pick a random group based on TB */
540 andi. r0,r0,1
541 mr r5,r28
542 bne 2f
543 not r5,r5
544 2: and r0,r5,r27
545 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
546 /* Call ppc_md.hpte_remove */
547 _GLOBAL(htab_call_hpte_remove)
548 bl . /* patched by htab_finish_init() */
550 /* Try all again */
551 b htab_insert_pte
553 /*
554 * Call out to C code to invalidate an 64k HW HPTE that is
555 * useless now that the segment has been switched to 4k pages.
556 */
557 htab_inval_old_hpte:
558 mr r3,r29 /* virtual addr */
559 mr r4,r31 /* PTE.pte */
560 li r5,0 /* PTE.hidx */
561 li r6,MMU_PAGE_64K /* psize */
562 ld r7,STK_PARM(r9)(r1) /* ssize */
563 ld r8,STK_PARM(r8)(r1) /* local */
564 bl .flush_hash_page
565 /* Clear out _PAGE_HPTE_SUB bits in the new linux PTE */
566 lis r0,_PAGE_HPTE_SUB@h
567 ori r0,r0,_PAGE_HPTE_SUB@l
568 andc r30,r30,r0
569 b htab_insert_pte
571 htab_bail_ok:
572 li r3,0
573 b htab_bail
575 htab_pte_insert_ok:
576 /* Insert slot number & secondary bit in PTE second half,
577 * clear _PAGE_BUSY and set approriate HPTE slot bit
578 */
579 ld r6,STK_PARM(r6)(r1)
580 li r0,_PAGE_BUSY
581 andc r30,r30,r0
582 /* HPTE SUB bit */
583 li r0,1
584 subfic r5,r25,27 /* Must match bit position in */
585 sld r0,r0,r5 /* pgtable.h */
586 or r30,r30,r0
587 /* hindx */
588 sldi r5,r25,2
589 sld r3,r3,r5
590 li r4,0xf
591 sld r4,r4,r5
592 andc r26,r26,r4
593 or r26,r26,r3
594 ori r5,r6,0x8000
595 std r26,0(r5)
596 lwsync
597 std r30,0(r6)
598 li r3, 0
599 htab_bail:
600 ld r25,STK_REG(r25)(r1)
601 ld r26,STK_REG(r26)(r1)
602 ld r27,STK_REG(r27)(r1)
603 ld r28,STK_REG(r28)(r1)
604 ld r29,STK_REG(r29)(r1)
605 ld r30,STK_REG(r30)(r1)
606 ld r31,STK_REG(r31)(r1)
607 addi r1,r1,STACKFRAMESIZE
608 ld r0,16(r1)
609 mtlr r0
610 blr
612 htab_modify_pte:
613 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
614 mr r4,r3
615 sldi r5,r25,2
616 srd r3,r26,r5
618 /* Secondary group ? if yes, get a inverted hash value */
619 mr r5,r28
620 andi. r0,r3,0x8 /* page secondary ? */
621 beq 1f
622 not r5,r5
623 1: andi. r3,r3,0x7 /* extract idx alone */
625 /* Calculate proper slot value for ppc_md.hpte_updatepp */
626 and r0,r5,r27
627 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
628 add r3,r0,r3 /* add slot idx */
630 /* Call ppc_md.hpte_updatepp */
631 mr r5,r29 /* va */
632 li r6,MMU_PAGE_4K /* page size */
633 ld r7,STK_PARM(r9)(r1) /* segment size */
634 ld r8,STK_PARM(r8)(r1) /* get "local" param */
635 _GLOBAL(htab_call_hpte_updatepp)
636 bl . /* patched by htab_finish_init() */
638 /* if we failed because typically the HPTE wasn't really here
639 * we try an insertion.
640 */
641 cmpdi 0,r3,-1
642 beq- htab_insert_pte
644 /* Clear the BUSY bit and Write out the PTE */
645 li r0,_PAGE_BUSY
646 andc r30,r30,r0
647 ld r6,STK_PARM(r6)(r1)
648 std r30,0(r6)
649 li r3,0
650 b htab_bail
652 htab_wrong_access:
653 /* Bail out clearing reservation */
654 stdcx. r31,0,r6
655 li r3,1
656 b htab_bail
658 htab_pte_insert_failure:
659 /* Bail out restoring old PTE */
660 ld r6,STK_PARM(r6)(r1)
661 std r31,0(r6)
662 li r3,-1
663 b htab_bail
665 #endif /* CONFIG_PPC_64K_PAGES */
667 #ifdef CONFIG_PPC_HAS_HASH_64K
669 /*****************************************************************************
670 * *
671 * 64K SW & 64K HW in a 64K segment pages implementation *
672 * *
673 *****************************************************************************/
675 _GLOBAL(__hash_page_64K)
676 mflr r0
677 std r0,16(r1)
678 stdu r1,-STACKFRAMESIZE(r1)
679 /* Save all params that we need after a function call */
680 std r6,STK_PARM(r6)(r1)
681 std r8,STK_PARM(r8)(r1)
682 std r9,STK_PARM(r9)(r1)
684 /* Save non-volatile registers.
685 * r31 will hold "old PTE"
686 * r30 is "new PTE"
687 * r29 is "va"
688 * r28 is a hash value
689 * r27 is hashtab mask (maybe dynamic patched instead ?)
690 */
691 std r27,STK_REG(r27)(r1)
692 std r28,STK_REG(r28)(r1)
693 std r29,STK_REG(r29)(r1)
694 std r30,STK_REG(r30)(r1)
695 std r31,STK_REG(r31)(r1)
697 /* Step 1:
698 *
699 * Check permissions, atomically mark the linux PTE busy
700 * and hashed.
701 */
702 1:
703 ldarx r31,0,r6
704 /* Check access rights (access & ~(pte_val(*ptep))) */
705 andc. r0,r4,r31
706 bne- ht64_wrong_access
707 /* Check if PTE is busy */
708 andi. r0,r31,_PAGE_BUSY
709 /* If so, just bail out and refault if needed. Someone else
710 * is changing this PTE anyway and might hash it.
711 */
712 bne- ht64_bail_ok
713 BEGIN_FTR_SECTION
714 /* Check if PTE has the cache-inhibit bit set */
715 andi. r0,r31,_PAGE_NO_CACHE
716 /* If so, bail out and refault as a 4k page */
717 bne- ht64_bail_ok
718 END_MMU_FTR_SECTION_IFCLR(MMU_FTR_CI_LARGE_PAGE)
719 /* Prepare new PTE value (turn access RW into DIRTY, then
720 * add BUSY and ACCESSED)
721 */
722 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
723 or r30,r30,r31
724 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
725 /* Write the linux PTE atomically (setting busy) */
726 stdcx. r30,0,r6
727 bne- 1b
728 isync
730 /* Step 2:
731 *
732 * Insert/Update the HPTE in the hash table. At this point,
733 * r4 (access) is re-useable, we use it for the new HPTE flags
734 */
736 BEGIN_FTR_SECTION
737 cmpdi r9,0 /* check segment size */
738 bne 3f
739 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
740 /* Calc va and put it in r29 */
741 rldicr r29,r5,28,63-28
742 rldicl r3,r3,0,36
743 or r29,r3,r29
745 /* Calculate hash value for primary slot and store it in r28 */
746 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
747 rldicl r0,r3,64-16,52 /* (ea >> 16) & 0xfff */
748 xor r28,r5,r0
749 b 4f
751 3: /* Calc VA and hash in r29 and r28 for 1T segment */
752 sldi r29,r5,40 /* vsid << 40 */
753 clrldi r3,r3,24 /* ea & 0xffffffffff */
754 rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */
755 clrldi r5,r5,40 /* vsid & 0xffffff */
756 rldicl r0,r3,64-16,40 /* (ea >> 16) & 0xffffff */
757 xor r28,r28,r5
758 or r29,r3,r29 /* VA */
759 xor r28,r28,r0 /* hash */
761 /* Convert linux PTE bits into HW equivalents */
762 4: andi. r3,r30,0x1fe /* Get basic set of flags */
763 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
764 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
765 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
766 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
767 andc r0,r30,r0 /* r0 = pte & ~r0 */
768 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
769 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */
771 /* We eventually do the icache sync here (maybe inline that
772 * code rather than call a C function...)
773 */
774 BEGIN_FTR_SECTION
775 mr r4,r30
776 mr r5,r7
777 bl .hash_page_do_lazy_icache
778 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
780 /* At this point, r3 contains new PP bits, save them in
781 * place of "access" in the param area (sic)
782 */
783 std r3,STK_PARM(r4)(r1)
785 /* Get htab_hash_mask */
786 ld r4,htab_hash_mask@got(2)
787 ld r27,0(r4) /* htab_hash_mask -> r27 */
789 /* Check if we may already be in the hashtable, in this case, we
790 * go to out-of-line code to try to modify the HPTE
791 */
792 rldicl. r0,r31,64-12,48
793 bne ht64_modify_pte
795 ht64_insert_pte:
796 /* Clear hpte bits in new pte (we also clear BUSY btw) and
797 * add _PAGE_HPTE_SUB0
798 */
799 lis r0,_PAGE_HPTEFLAGS@h
800 ori r0,r0,_PAGE_HPTEFLAGS@l
801 andc r30,r30,r0
802 #ifdef CONFIG_PPC_64K_PAGES
803 oris r30,r30,_PAGE_HPTE_SUB0@h
804 #else
805 ori r30,r30,_PAGE_HASHPTE
806 #endif
807 /* Phyical address in r5 */
808 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
809 sldi r5,r5,PAGE_SHIFT
811 /* Calculate primary group hash */
812 and r0,r28,r27
813 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
815 /* Call ppc_md.hpte_insert */
816 ld r6,STK_PARM(r4)(r1) /* Retrieve new pp bits */
817 mr r4,r29 /* Retrieve va */
818 li r7,0 /* !bolted, !secondary */
819 li r8,MMU_PAGE_64K
820 ld r9,STK_PARM(r9)(r1) /* segment size */
821 _GLOBAL(ht64_call_hpte_insert1)
822 bl . /* patched by htab_finish_init() */
823 cmpdi 0,r3,0
824 bge ht64_pte_insert_ok /* Insertion successful */
825 cmpdi 0,r3,-2 /* Critical failure */
826 beq- ht64_pte_insert_failure
828 /* Now try secondary slot */
830 /* Phyical address in r5 */
831 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
832 sldi r5,r5,PAGE_SHIFT
834 /* Calculate secondary group hash */
835 andc r0,r27,r28
836 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
838 /* Call ppc_md.hpte_insert */
839 ld r6,STK_PARM(r4)(r1) /* Retrieve new pp bits */
840 mr r4,r29 /* Retrieve va */
841 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
842 li r8,MMU_PAGE_64K
843 ld r9,STK_PARM(r9)(r1) /* segment size */
844 _GLOBAL(ht64_call_hpte_insert2)
845 bl . /* patched by htab_finish_init() */
846 cmpdi 0,r3,0
847 bge+ ht64_pte_insert_ok /* Insertion successful */
848 cmpdi 0,r3,-2 /* Critical failure */
849 beq- ht64_pte_insert_failure
851 /* Both are full, we need to evict something */
852 mftb r0
853 /* Pick a random group based on TB */
854 andi. r0,r0,1
855 mr r5,r28
856 bne 2f
857 not r5,r5
858 2: and r0,r5,r27
859 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
860 /* Call ppc_md.hpte_remove */
861 _GLOBAL(ht64_call_hpte_remove)
862 bl . /* patched by htab_finish_init() */
864 /* Try all again */
865 b ht64_insert_pte
867 ht64_bail_ok:
868 li r3,0
869 b ht64_bail
871 ht64_pte_insert_ok:
872 /* Insert slot number & secondary bit in PTE */
873 rldimi r30,r3,12,63-15
875 /* Write out the PTE with a normal write
876 * (maybe add eieio may be good still ?)
877 */
878 ht64_write_out_pte:
879 ld r6,STK_PARM(r6)(r1)
880 std r30,0(r6)
881 li r3, 0
882 ht64_bail:
883 ld r27,STK_REG(r27)(r1)
884 ld r28,STK_REG(r28)(r1)
885 ld r29,STK_REG(r29)(r1)
886 ld r30,STK_REG(r30)(r1)
887 ld r31,STK_REG(r31)(r1)
888 addi r1,r1,STACKFRAMESIZE
889 ld r0,16(r1)
890 mtlr r0
891 blr
893 ht64_modify_pte:
894 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
895 mr r4,r3
896 rlwinm r3,r31,32-12,29,31
898 /* Secondary group ? if yes, get a inverted hash value */
899 mr r5,r28
900 andi. r0,r31,_PAGE_F_SECOND
901 beq 1f
902 not r5,r5
903 1:
904 /* Calculate proper slot value for ppc_md.hpte_updatepp */
905 and r0,r5,r27
906 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
907 add r3,r0,r3 /* add slot idx */
909 /* Call ppc_md.hpte_updatepp */
910 mr r5,r29 /* va */
911 li r6,MMU_PAGE_64K
912 ld r7,STK_PARM(r9)(r1) /* segment size */
913 ld r8,STK_PARM(r8)(r1) /* get "local" param */
914 _GLOBAL(ht64_call_hpte_updatepp)
915 bl . /* patched by htab_finish_init() */
917 /* if we failed because typically the HPTE wasn't really here
918 * we try an insertion.
919 */
920 cmpdi 0,r3,-1
921 beq- ht64_insert_pte
923 /* Clear the BUSY bit and Write out the PTE */
924 li r0,_PAGE_BUSY
925 andc r30,r30,r0
926 b ht64_write_out_pte
928 ht64_wrong_access:
929 /* Bail out clearing reservation */
930 stdcx. r31,0,r6
931 li r3,1
932 b ht64_bail
934 ht64_pte_insert_failure:
935 /* Bail out restoring old PTE */
936 ld r6,STK_PARM(r6)(r1)
937 std r31,0(r6)
938 li r3,-1
939 b ht64_bail
942 #endif /* CONFIG_PPC_HAS_HASH_64K */
945 /*****************************************************************************
946 * *
947 * Huge pages implementation is in hugetlbpage.c *
948 * *
949 *****************************************************************************/