| blob | e0d02c4a2615f8787abc1b13737f7b78e67bc5a5 |
1 /*
2 * ppc64 MMU hashtable management routines
3 *
4 * (c) Copyright IBM Corp. 2003, 2005
5 *
6 * Maintained by: Benjamin Herrenschmidt
7 * <benh@kernel.crashing.org>
8 *
9 * This file is covered by the GNU Public Licence v2 as
10 * described in the kernel's COPYING file.
11 */
13 #include <linux/config.h>
14 #include <asm/reg.h>
15 #include <asm/pgtable.h>
16 #include <asm/mmu.h>
17 #include <asm/page.h>
18 #include <asm/types.h>
19 #include <asm/ppc_asm.h>
20 #include <asm/asm-offsets.h>
21 #include <asm/cputable.h>
23 .text
25 /*
26 * Stackframe:
27 *
28 * +-> Back chain (SP + 256)
29 * | General register save area (SP + 112)
30 * | Parameter save area (SP + 48)
31 * | TOC save area (SP + 40)
32 * | link editor doubleword (SP + 32)
33 * | compiler doubleword (SP + 24)
34 * | LR save area (SP + 16)
35 * | CR save area (SP + 8)
36 * SP ---> +-- Back chain (SP + 0)
37 */
38 #define STACKFRAMESIZE 256
40 /* Save parameters offsets */
41 #define STK_PARM(i) (STACKFRAMESIZE + 48 + ((i)-3)*8)
43 /* Save non-volatile offsets */
44 #define STK_REG(i) (112 + ((i)-14)*8)
47 #ifndef CONFIG_PPC_64K_PAGES
49 /*****************************************************************************
50 * *
51 * 4K SW & 4K HW pages implementation *
52 * *
53 *****************************************************************************/
56 /*
57 * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
58 * pte_t *ptep, unsigned long trap, int local)
59 *
60 * Adds a 4K page to the hash table in a segment of 4K pages only
61 */
63 _GLOBAL(__hash_page_4K)
64 mflr r0
65 std r0,16(r1)
66 stdu r1,-STACKFRAMESIZE(r1)
67 /* Save all params that we need after a function call */
68 std r6,STK_PARM(r6)(r1)
69 std r8,STK_PARM(r8)(r1)
71 /* Add _PAGE_PRESENT to access */
72 ori r4,r4,_PAGE_PRESENT
74 /* Save non-volatile registers.
75 * r31 will hold "old PTE"
76 * r30 is "new PTE"
77 * r29 is "va"
78 * r28 is a hash value
79 * r27 is hashtab mask (maybe dynamic patched instead ?)
80 */
81 std r27,STK_REG(r27)(r1)
82 std r28,STK_REG(r28)(r1)
83 std r29,STK_REG(r29)(r1)
84 std r30,STK_REG(r30)(r1)
85 std r31,STK_REG(r31)(r1)
87 /* Step 1:
88 *
89 * Check permissions, atomically mark the linux PTE busy
90 * and hashed.
91 */
92 1:
93 ldarx r31,0,r6
94 /* Check access rights (access & ~(pte_val(*ptep))) */
95 andc. r0,r4,r31
96 bne- htab_wrong_access
97 /* Check if PTE is busy */
98 andi. r0,r31,_PAGE_BUSY
99 /* If so, just bail out and refault if needed. Someone else
100 * is changing this PTE anyway and might hash it.
101 */
102 bne- htab_bail_ok
104 /* Prepare new PTE value (turn access RW into DIRTY, then
105 * add BUSY,HASHPTE and ACCESSED)
106 */
107 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
108 or r30,r30,r31
109 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
110 /* Write the linux PTE atomically (setting busy) */
111 stdcx. r30,0,r6
112 bne- 1b
113 isync
115 /* Step 2:
116 *
117 * Insert/Update the HPTE in the hash table. At this point,
118 * r4 (access) is re-useable, we use it for the new HPTE flags
119 */
121 /* Calc va and put it in r29 */
122 rldicr r29,r5,28,63-28
123 rldicl r3,r3,0,36
124 or r29,r3,r29
126 /* Calculate hash value for primary slot and store it in r28 */
127 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
128 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */
129 xor r28,r5,r0
131 /* Convert linux PTE bits into HW equivalents */
132 andi. r3,r30,0x1fe /* Get basic set of flags */
133 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
134 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
135 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
136 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
137 andc r0,r30,r0 /* r0 = pte & ~r0 */
138 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
140 /* We eventually do the icache sync here (maybe inline that
141 * code rather than call a C function...)
142 */
143 BEGIN_FTR_SECTION
144 mr r4,r30
145 mr r5,r7
146 bl .hash_page_do_lazy_icache
147 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
149 /* At this point, r3 contains new PP bits, save them in
150 * place of "access" in the param area (sic)
151 */
152 std r3,STK_PARM(r4)(r1)
154 /* Get htab_hash_mask */
155 ld r4,htab_hash_mask@got(2)
156 ld r27,0(r4) /* htab_hash_mask -> r27 */
158 /* Check if we may already be in the hashtable, in this case, we
159 * go to out-of-line code to try to modify the HPTE
160 */
161 andi. r0,r31,_PAGE_HASHPTE
162 bne htab_modify_pte
164 htab_insert_pte:
165 /* Clear hpte bits in new pte (we also clear BUSY btw) and
166 * add _PAGE_HASHPTE
167 */
168 lis r0,_PAGE_HPTEFLAGS@h
169 ori r0,r0,_PAGE_HPTEFLAGS@l
170 andc r30,r30,r0
171 ori r30,r30,_PAGE_HASHPTE
173 /* physical address r5 */
174 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
175 sldi r5,r5,PAGE_SHIFT
177 /* Calculate primary group hash */
178 and r0,r28,r27
179 rldicr r3,r0,3,63-3 /* r3 = (hash & mask) << 3 */
181 /* Call ppc_md.hpte_insert */
182 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
183 mr r4,r29 /* Retreive va */
184 li r7,0 /* !bolted, !secondary */
185 li r8,MMU_PAGE_4K /* page size */
186 _GLOBAL(htab_call_hpte_insert1)
187 bl . /* Patched by htab_finish_init() */
188 cmpdi 0,r3,0
189 bge htab_pte_insert_ok /* Insertion successful */
190 cmpdi 0,r3,-2 /* Critical failure */
191 beq- htab_pte_insert_failure
193 /* Now try secondary slot */
195 /* physical address r5 */
196 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
197 sldi r5,r5,PAGE_SHIFT
199 /* Calculate secondary group hash */
200 andc r0,r27,r28
201 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
203 /* Call ppc_md.hpte_insert */
204 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
205 mr r4,r29 /* Retreive va */
206 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
207 li r8,MMU_PAGE_4K /* page size */
208 _GLOBAL(htab_call_hpte_insert2)
209 bl . /* Patched by htab_finish_init() */
210 cmpdi 0,r3,0
211 bge+ htab_pte_insert_ok /* Insertion successful */
212 cmpdi 0,r3,-2 /* Critical failure */
213 beq- htab_pte_insert_failure
215 /* Both are full, we need to evict something */
216 mftb r0
217 /* Pick a random group based on TB */
218 andi. r0,r0,1
219 mr r5,r28
220 bne 2f
221 not r5,r5
222 2: and r0,r5,r27
223 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
224 /* Call ppc_md.hpte_remove */
225 _GLOBAL(htab_call_hpte_remove)
226 bl . /* Patched by htab_finish_init() */
228 /* Try all again */
229 b htab_insert_pte
231 htab_bail_ok:
232 li r3,0
233 b htab_bail
235 htab_pte_insert_ok:
236 /* Insert slot number & secondary bit in PTE */
237 rldimi r30,r3,12,63-15
239 /* Write out the PTE with a normal write
240 * (maybe add eieio may be good still ?)
241 */
242 htab_write_out_pte:
243 ld r6,STK_PARM(r6)(r1)
244 std r30,0(r6)
245 li r3, 0
246 htab_bail:
247 ld r27,STK_REG(r27)(r1)
248 ld r28,STK_REG(r28)(r1)
249 ld r29,STK_REG(r29)(r1)
250 ld r30,STK_REG(r30)(r1)
251 ld r31,STK_REG(r31)(r1)
252 addi r1,r1,STACKFRAMESIZE
253 ld r0,16(r1)
254 mtlr r0
255 blr
257 htab_modify_pte:
258 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
259 mr r4,r3
260 rlwinm r3,r31,32-12,29,31
262 /* Secondary group ? if yes, get a inverted hash value */
263 mr r5,r28
264 andi. r0,r31,_PAGE_SECONDARY
265 beq 1f
266 not r5,r5
267 1:
268 /* Calculate proper slot value for ppc_md.hpte_updatepp */
269 and r0,r5,r27
270 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
271 add r3,r0,r3 /* add slot idx */
273 /* Call ppc_md.hpte_updatepp */
274 mr r5,r29 /* va */
275 li r6,MMU_PAGE_4K /* page size */
276 ld r7,STK_PARM(r8)(r1) /* get "local" param */
277 _GLOBAL(htab_call_hpte_updatepp)
278 bl . /* Patched by htab_finish_init() */
280 /* if we failed because typically the HPTE wasn't really here
281 * we try an insertion.
282 */
283 cmpdi 0,r3,-1
284 beq- htab_insert_pte
286 /* Clear the BUSY bit and Write out the PTE */
287 li r0,_PAGE_BUSY
288 andc r30,r30,r0
289 b htab_write_out_pte
291 htab_wrong_access:
292 /* Bail out clearing reservation */
293 stdcx. r31,0,r6
294 li r3,1
295 b htab_bail
297 htab_pte_insert_failure:
298 /* Bail out restoring old PTE */
299 ld r6,STK_PARM(r6)(r1)
300 std r31,0(r6)
301 li r3,-1
302 b htab_bail
305 #else /* CONFIG_PPC_64K_PAGES */
308 /*****************************************************************************
309 * *
310 * 64K SW & 4K or 64K HW in a 4K segment pages implementation *
311 * *
312 *****************************************************************************/
314 /* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
315 * pte_t *ptep, unsigned long trap, int local)
316 */
318 /*
319 * For now, we do NOT implement Admixed pages
320 */
321 _GLOBAL(__hash_page_4K)
322 mflr r0
323 std r0,16(r1)
324 stdu r1,-STACKFRAMESIZE(r1)
325 /* Save all params that we need after a function call */
326 std r6,STK_PARM(r6)(r1)
327 std r8,STK_PARM(r8)(r1)
329 /* Add _PAGE_PRESENT to access */
330 ori r4,r4,_PAGE_PRESENT
332 /* Save non-volatile registers.
333 * r31 will hold "old PTE"
334 * r30 is "new PTE"
335 * r29 is "va"
336 * r28 is a hash value
337 * r27 is hashtab mask (maybe dynamic patched instead ?)
338 * r26 is the hidx mask
339 * r25 is the index in combo page
340 */
341 std r25,STK_REG(r25)(r1)
342 std r26,STK_REG(r26)(r1)
343 std r27,STK_REG(r27)(r1)
344 std r28,STK_REG(r28)(r1)
345 std r29,STK_REG(r29)(r1)
346 std r30,STK_REG(r30)(r1)
347 std r31,STK_REG(r31)(r1)
349 /* Step 1:
350 *
351 * Check permissions, atomically mark the linux PTE busy
352 * and hashed.
353 */
354 1:
355 ldarx r31,0,r6
356 /* Check access rights (access & ~(pte_val(*ptep))) */
357 andc. r0,r4,r31
358 bne- htab_wrong_access
359 /* Check if PTE is busy */
360 andi. r0,r31,_PAGE_BUSY
361 /* If so, just bail out and refault if needed. Someone else
362 * is changing this PTE anyway and might hash it.
363 */
364 bne- htab_bail_ok
365 /* Prepare new PTE value (turn access RW into DIRTY, then
366 * add BUSY and ACCESSED)
367 */
368 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
369 or r30,r30,r31
370 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
371 /* Write the linux PTE atomically (setting busy) */
372 stdcx. r30,0,r6
373 bne- 1b
374 isync
376 /* Step 2:
377 *
378 * Insert/Update the HPTE in the hash table. At this point,
379 * r4 (access) is re-useable, we use it for the new HPTE flags
380 */
382 /* Load the hidx index */
383 rldicl r25,r3,64-12,60
385 /* Calc va and put it in r29 */
386 rldicr r29,r5,28,63-28 /* r29 = (vsid << 28) */
387 rldicl r3,r3,0,36 /* r3 = (ea & 0x0fffffff) */
388 or r29,r3,r29 /* r29 = va
390 /* Calculate hash value for primary slot and store it in r28 */
391 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
392 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */
393 xor r28,r5,r0
395 /* Convert linux PTE bits into HW equivalents */
396 andi. r3,r30,0x1fe /* Get basic set of flags */
397 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
398 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
399 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
400 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
401 andc r0,r30,r0 /* r0 = pte & ~r0 */
402 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
404 /* We eventually do the icache sync here (maybe inline that
405 * code rather than call a C function...)
406 */
407 BEGIN_FTR_SECTION
408 mr r4,r30
409 mr r5,r7
410 bl .hash_page_do_lazy_icache
411 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
413 /* At this point, r3 contains new PP bits, save them in
414 * place of "access" in the param area (sic)
415 */
416 std r3,STK_PARM(r4)(r1)
418 /* Get htab_hash_mask */
419 ld r4,htab_hash_mask@got(2)
420 ld r27,0(r4) /* htab_hash_mask -> r27 */
422 /* Check if we may already be in the hashtable, in this case, we
423 * go to out-of-line code to try to modify the HPTE. We look for
424 * the bit at (1 >> (index + 32))
425 */
426 andi. r0,r31,_PAGE_HASHPTE
427 li r26,0 /* Default hidx */
428 beq htab_insert_pte
429 ld r6,STK_PARM(r6)(r1)
430 ori r26,r6,0x8000 /* Load the hidx mask */
431 ld r26,0(r26)
432 addi r5,r25,36 /* Check actual HPTE_SUB bit, this */
433 rldcr. r0,r31,r5,0 /* must match pgtable.h definition */
434 bne htab_modify_pte
436 htab_insert_pte:
437 /* real page number in r5, PTE RPN value + index */
438 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
439 sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
440 add r5,r5,r25
441 sldi r5,r5,HW_PAGE_SHIFT
443 /* Calculate primary group hash */
444 and r0,r28,r27
445 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
447 /* Call ppc_md.hpte_insert */
448 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
449 mr r4,r29 /* Retreive va */
450 li r7,0 /* !bolted, !secondary */
451 li r8,MMU_PAGE_4K /* page size */
452 _GLOBAL(htab_call_hpte_insert1)
453 bl . /* patched by htab_finish_init() */
454 cmpdi 0,r3,0
455 bge htab_pte_insert_ok /* Insertion successful */
456 cmpdi 0,r3,-2 /* Critical failure */
457 beq- htab_pte_insert_failure
459 /* Now try secondary slot */
461 /* real page number in r5, PTE RPN value + index */
462 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
463 sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
464 add r5,r5,r25
465 sldi r5,r5,HW_PAGE_SHIFT
467 /* Calculate secondary group hash */
468 andc r0,r27,r28
469 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
471 /* Call ppc_md.hpte_insert */
472 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
473 mr r4,r29 /* Retreive va */
474 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
475 li r8,MMU_PAGE_4K /* page size */
476 _GLOBAL(htab_call_hpte_insert2)
477 bl . /* patched by htab_finish_init() */
478 cmpdi 0,r3,0
479 bge+ htab_pte_insert_ok /* Insertion successful */
480 cmpdi 0,r3,-2 /* Critical failure */
481 beq- htab_pte_insert_failure
483 /* Both are full, we need to evict something */
484 mftb r0
485 /* Pick a random group based on TB */
486 andi. r0,r0,1
487 mr r5,r28
488 bne 2f
489 not r5,r5
490 2: and r0,r5,r27
491 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
492 /* Call ppc_md.hpte_remove */
493 _GLOBAL(htab_call_hpte_remove)
494 bl . /* patched by htab_finish_init() */
496 /* Try all again */
497 b htab_insert_pte
499 htab_bail_ok:
500 li r3,0
501 b htab_bail
503 htab_pte_insert_ok:
504 /* Insert slot number & secondary bit in PTE second half,
505 * clear _PAGE_BUSY and set approriate HPTE slot bit
506 */
507 ld r6,STK_PARM(r6)(r1)
508 li r0,_PAGE_BUSY
509 andc r30,r30,r0
510 /* HPTE SUB bit */
511 li r0,1
512 subfic r5,r25,27 /* Must match bit position in */
513 sld r0,r0,r5 /* pgtable.h */
514 or r30,r30,r0
515 /* hindx */
516 sldi r5,r25,2
517 sld r3,r3,r5
518 li r4,0xf
519 sld r4,r4,r5
520 andc r26,r26,r4
521 or r26,r26,r3
522 ori r5,r6,0x8000
523 std r26,0(r5)
524 lwsync
525 std r30,0(r6)
526 li r3, 0
527 htab_bail:
528 ld r25,STK_REG(r25)(r1)
529 ld r26,STK_REG(r26)(r1)
530 ld r27,STK_REG(r27)(r1)
531 ld r28,STK_REG(r28)(r1)
532 ld r29,STK_REG(r29)(r1)
533 ld r30,STK_REG(r30)(r1)
534 ld r31,STK_REG(r31)(r1)
535 addi r1,r1,STACKFRAMESIZE
536 ld r0,16(r1)
537 mtlr r0
538 blr
540 htab_modify_pte:
541 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
542 mr r4,r3
543 sldi r5,r25,2
544 srd r3,r26,r5
546 /* Secondary group ? if yes, get a inverted hash value */
547 mr r5,r28
548 andi. r0,r3,0x8 /* page secondary ? */
549 beq 1f
550 not r5,r5
551 1: andi. r3,r3,0x7 /* extract idx alone */
553 /* Calculate proper slot value for ppc_md.hpte_updatepp */
554 and r0,r5,r27
555 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
556 add r3,r0,r3 /* add slot idx */
558 /* Call ppc_md.hpte_updatepp */
559 mr r5,r29 /* va */
560 li r6,MMU_PAGE_4K /* page size */
561 ld r7,STK_PARM(r8)(r1) /* get "local" param */
562 _GLOBAL(htab_call_hpte_updatepp)
563 bl . /* patched by htab_finish_init() */
565 /* if we failed because typically the HPTE wasn't really here
566 * we try an insertion.
567 */
568 cmpdi 0,r3,-1
569 beq- htab_insert_pte
571 /* Clear the BUSY bit and Write out the PTE */
572 li r0,_PAGE_BUSY
573 andc r30,r30,r0
574 ld r6,STK_PARM(r6)(r1)
575 std r30,0(r6)
576 li r3,0
577 b htab_bail
579 htab_wrong_access:
580 /* Bail out clearing reservation */
581 stdcx. r31,0,r6
582 li r3,1
583 b htab_bail
585 htab_pte_insert_failure:
586 /* Bail out restoring old PTE */
587 ld r6,STK_PARM(r6)(r1)
588 std r31,0(r6)
589 li r3,-1
590 b htab_bail
593 /*****************************************************************************
594 * *
595 * 64K SW & 64K HW in a 64K segment pages implementation *
596 * *
597 *****************************************************************************/
599 _GLOBAL(__hash_page_64K)
600 mflr r0
601 std r0,16(r1)
602 stdu r1,-STACKFRAMESIZE(r1)
603 /* Save all params that we need after a function call */
604 std r6,STK_PARM(r6)(r1)
605 std r8,STK_PARM(r8)(r1)
607 /* Add _PAGE_PRESENT to access */
608 ori r4,r4,_PAGE_PRESENT
610 /* Save non-volatile registers.
611 * r31 will hold "old PTE"
612 * r30 is "new PTE"
613 * r29 is "va"
614 * r28 is a hash value
615 * r27 is hashtab mask (maybe dynamic patched instead ?)
616 */
617 std r27,STK_REG(r27)(r1)
618 std r28,STK_REG(r28)(r1)
619 std r29,STK_REG(r29)(r1)
620 std r30,STK_REG(r30)(r1)
621 std r31,STK_REG(r31)(r1)
623 /* Step 1:
624 *
625 * Check permissions, atomically mark the linux PTE busy
626 * and hashed.
627 */
628 1:
629 ldarx r31,0,r6
630 /* Check access rights (access & ~(pte_val(*ptep))) */
631 andc. r0,r4,r31
632 bne- ht64_wrong_access
633 /* Check if PTE is busy */
634 andi. r0,r31,_PAGE_BUSY
635 /* If so, just bail out and refault if needed. Someone else
636 * is changing this PTE anyway and might hash it.
637 */
638 bne- ht64_bail_ok
639 /* Prepare new PTE value (turn access RW into DIRTY, then
640 * add BUSY,HASHPTE and ACCESSED)
641 */
642 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
643 or r30,r30,r31
644 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
645 /* Write the linux PTE atomically (setting busy) */
646 stdcx. r30,0,r6
647 bne- 1b
648 isync
650 /* Step 2:
651 *
652 * Insert/Update the HPTE in the hash table. At this point,
653 * r4 (access) is re-useable, we use it for the new HPTE flags
654 */
656 /* Calc va and put it in r29 */
657 rldicr r29,r5,28,63-28
658 rldicl r3,r3,0,36
659 or r29,r3,r29
661 /* Calculate hash value for primary slot and store it in r28 */
662 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
663 rldicl r0,r3,64-16,52 /* (ea >> 16) & 0xfff */
664 xor r28,r5,r0
666 /* Convert linux PTE bits into HW equivalents */
667 andi. r3,r30,0x1fe /* Get basic set of flags */
668 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
669 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
670 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
671 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
672 andc r0,r30,r0 /* r0 = pte & ~r0 */
673 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
675 /* We eventually do the icache sync here (maybe inline that
676 * code rather than call a C function...)
677 */
678 BEGIN_FTR_SECTION
679 mr r4,r30
680 mr r5,r7
681 bl .hash_page_do_lazy_icache
682 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
684 /* At this point, r3 contains new PP bits, save them in
685 * place of "access" in the param area (sic)
686 */
687 std r3,STK_PARM(r4)(r1)
689 /* Get htab_hash_mask */
690 ld r4,htab_hash_mask@got(2)
691 ld r27,0(r4) /* htab_hash_mask -> r27 */
693 /* Check if we may already be in the hashtable, in this case, we
694 * go to out-of-line code to try to modify the HPTE
695 */
696 andi. r0,r31,_PAGE_HASHPTE
697 bne ht64_modify_pte
699 ht64_insert_pte:
700 /* Clear hpte bits in new pte (we also clear BUSY btw) and
701 * add _PAGE_HASHPTE
702 */
703 lis r0,_PAGE_HPTEFLAGS@h
704 ori r0,r0,_PAGE_HPTEFLAGS@l
705 andc r30,r30,r0
706 ori r30,r30,_PAGE_HASHPTE
708 /* Phyical address in r5 */
709 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
710 sldi r5,r5,PAGE_SHIFT
712 /* Calculate primary group hash */
713 and r0,r28,r27
714 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
716 /* Call ppc_md.hpte_insert */
717 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
718 mr r4,r29 /* Retreive va */
719 li r7,0 /* !bolted, !secondary */
720 li r8,MMU_PAGE_64K
721 _GLOBAL(ht64_call_hpte_insert1)
722 bl . /* patched by htab_finish_init() */
723 cmpdi 0,r3,0
724 bge ht64_pte_insert_ok /* Insertion successful */
725 cmpdi 0,r3,-2 /* Critical failure */
726 beq- ht64_pte_insert_failure
728 /* Now try secondary slot */
730 /* Phyical address in r5 */
731 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
732 sldi r5,r5,PAGE_SHIFT
734 /* Calculate secondary group hash */
735 andc r0,r27,r28
736 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
738 /* Call ppc_md.hpte_insert */
739 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
740 mr r4,r29 /* Retreive va */
741 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
742 li r8,MMU_PAGE_64K
743 _GLOBAL(ht64_call_hpte_insert2)
744 bl . /* patched by htab_finish_init() */
745 cmpdi 0,r3,0
746 bge+ ht64_pte_insert_ok /* Insertion successful */
747 cmpdi 0,r3,-2 /* Critical failure */
748 beq- ht64_pte_insert_failure
750 /* Both are full, we need to evict something */
751 mftb r0
752 /* Pick a random group based on TB */
753 andi. r0,r0,1
754 mr r5,r28
755 bne 2f
756 not r5,r5
757 2: and r0,r5,r27
758 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
759 /* Call ppc_md.hpte_remove */
760 _GLOBAL(ht64_call_hpte_remove)
761 bl . /* patched by htab_finish_init() */
763 /* Try all again */
764 b ht64_insert_pte
766 ht64_bail_ok:
767 li r3,0
768 b ht64_bail
770 ht64_pte_insert_ok:
771 /* Insert slot number & secondary bit in PTE */
772 rldimi r30,r3,12,63-15
774 /* Write out the PTE with a normal write
775 * (maybe add eieio may be good still ?)
776 */
777 ht64_write_out_pte:
778 ld r6,STK_PARM(r6)(r1)
779 std r30,0(r6)
780 li r3, 0
781 ht64_bail:
782 ld r27,STK_REG(r27)(r1)
783 ld r28,STK_REG(r28)(r1)
784 ld r29,STK_REG(r29)(r1)
785 ld r30,STK_REG(r30)(r1)
786 ld r31,STK_REG(r31)(r1)
787 addi r1,r1,STACKFRAMESIZE
788 ld r0,16(r1)
789 mtlr r0
790 blr
792 ht64_modify_pte:
793 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
794 mr r4,r3
795 rlwinm r3,r31,32-12,29,31
797 /* Secondary group ? if yes, get a inverted hash value */
798 mr r5,r28
799 andi. r0,r31,_PAGE_F_SECOND
800 beq 1f
801 not r5,r5
802 1:
803 /* Calculate proper slot value for ppc_md.hpte_updatepp */
804 and r0,r5,r27
805 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
806 add r3,r0,r3 /* add slot idx */
808 /* Call ppc_md.hpte_updatepp */
809 mr r5,r29 /* va */
810 li r6,MMU_PAGE_64K
811 ld r7,STK_PARM(r8)(r1) /* get "local" param */
812 _GLOBAL(ht64_call_hpte_updatepp)
813 bl . /* patched by htab_finish_init() */
815 /* if we failed because typically the HPTE wasn't really here
816 * we try an insertion.
817 */
818 cmpdi 0,r3,-1
819 beq- ht64_insert_pte
821 /* Clear the BUSY bit and Write out the PTE */
822 li r0,_PAGE_BUSY
823 andc r30,r30,r0
824 b ht64_write_out_pte
826 ht64_wrong_access:
827 /* Bail out clearing reservation */
828 stdcx. r31,0,r6
829 li r3,1
830 b ht64_bail
832 ht64_pte_insert_failure:
833 /* Bail out restoring old PTE */
834 ld r6,STK_PARM(r6)(r1)
835 std r31,0(r6)
836 li r3,-1
837 b ht64_bail
840 #endif /* CONFIG_PPC_64K_PAGES */
843 /*****************************************************************************
844 * *
845 * Huge pages implementation is in hugetlbpage.c *
846 * *
847 *****************************************************************************/