| blob | 602aeb06d298b1eccc72089b0251443dff397163 |
1 /*
2 * ppc64 MMU hashtable management routines
3 *
4 * (c) Copyright IBM Corp. 2003, 2005
5 *
6 * Maintained by: Benjamin Herrenschmidt
7 * <benh@kernel.crashing.org>
8 *
9 * This file is covered by the GNU Public Licence v2 as
10 * described in the kernel's COPYING file.
11 */
13 #include <asm/reg.h>
14 #include <asm/pgtable.h>
15 #include <asm/mmu.h>
16 #include <asm/page.h>
17 #include <asm/types.h>
18 #include <asm/ppc_asm.h>
19 #include <asm/asm-offsets.h>
20 #include <asm/cputable.h>
22 .text
24 /*
25 * Stackframe:
26 *
27 * +-> Back chain (SP + 256)
28 * | General register save area (SP + 112)
29 * | Parameter save area (SP + 48)
30 * | TOC save area (SP + 40)
31 * | link editor doubleword (SP + 32)
32 * | compiler doubleword (SP + 24)
33 * | LR save area (SP + 16)
34 * | CR save area (SP + 8)
35 * SP ---> +-- Back chain (SP + 0)
36 */
38 #ifndef CONFIG_PPC_64K_PAGES
40 /*****************************************************************************
41 * *
42 * 4K SW & 4K HW pages implementation *
43 * *
44 *****************************************************************************/
47 /*
48 * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
49 * pte_t *ptep, unsigned long trap, int local, int ssize)
50 *
51 * Adds a 4K page to the hash table in a segment of 4K pages only
52 */
54 _GLOBAL(__hash_page_4K)
55 mflr r0
56 std r0,16(r1)
57 stdu r1,-STACKFRAMESIZE(r1)
58 /* Save all params that we need after a function call */
59 std r6,STK_PARAM(R6)(r1)
60 std r8,STK_PARAM(R8)(r1)
61 std r9,STK_PARAM(R9)(r1)
63 /* Save non-volatile registers.
64 * r31 will hold "old PTE"
65 * r30 is "new PTE"
66 * r29 is "va"
67 * r28 is a hash value
68 * r27 is hashtab mask (maybe dynamic patched instead ?)
69 */
70 std r27,STK_REG(R27)(r1)
71 std r28,STK_REG(R28)(r1)
72 std r29,STK_REG(R29)(r1)
73 std r30,STK_REG(R30)(r1)
74 std r31,STK_REG(R31)(r1)
76 /* Step 1:
77 *
78 * Check permissions, atomically mark the linux PTE busy
79 * and hashed.
80 */
81 1:
82 ldarx r31,0,r6
83 /* Check access rights (access & ~(pte_val(*ptep))) */
84 andc. r0,r4,r31
85 bne- htab_wrong_access
86 /* Check if PTE is busy */
87 andi. r0,r31,_PAGE_BUSY
88 /* If so, just bail out and refault if needed. Someone else
89 * is changing this PTE anyway and might hash it.
90 */
91 bne- htab_bail_ok
93 /* Prepare new PTE value (turn access RW into DIRTY, then
94 * add BUSY,HASHPTE and ACCESSED)
95 */
96 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
97 or r30,r30,r31
98 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
99 /* Write the linux PTE atomically (setting busy) */
100 stdcx. r30,0,r6
101 bne- 1b
102 isync
104 /* Step 2:
105 *
106 * Insert/Update the HPTE in the hash table. At this point,
107 * r4 (access) is re-useable, we use it for the new HPTE flags
108 */
110 BEGIN_FTR_SECTION
111 cmpdi r9,0 /* check segment size */
112 bne 3f
113 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
114 /* Calc va and put it in r29 */
115 rldicr r29,r5,28,63-28
116 rldicl r3,r3,0,36
117 or r29,r3,r29
119 /* Calculate hash value for primary slot and store it in r28 */
120 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
121 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */
122 xor r28,r5,r0
123 b 4f
125 3: /* Calc VA and hash in r29 and r28 for 1T segment */
126 sldi r29,r5,40 /* vsid << 40 */
127 clrldi r3,r3,24 /* ea & 0xffffffffff */
128 rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */
129 clrldi r5,r5,40 /* vsid & 0xffffff */
130 rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */
131 xor r28,r28,r5
132 or r29,r3,r29 /* VA */
133 xor r28,r28,r0 /* hash */
135 /* Convert linux PTE bits into HW equivalents */
136 4: andi. r3,r30,0x1fe /* Get basic set of flags */
137 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
138 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
139 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
140 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
141 andc r0,r30,r0 /* r0 = pte & ~r0 */
142 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
143 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */
145 /* We eventually do the icache sync here (maybe inline that
146 * code rather than call a C function...)
147 */
148 BEGIN_FTR_SECTION
149 mr r4,r30
150 mr r5,r7
151 bl .hash_page_do_lazy_icache
152 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
154 /* At this point, r3 contains new PP bits, save them in
155 * place of "access" in the param area (sic)
156 */
157 std r3,STK_PARAM(R4)(r1)
159 /* Get htab_hash_mask */
160 ld r4,htab_hash_mask@got(2)
161 ld r27,0(r4) /* htab_hash_mask -> r27 */
163 /* Check if we may already be in the hashtable, in this case, we
164 * go to out-of-line code to try to modify the HPTE
165 */
166 andi. r0,r31,_PAGE_HASHPTE
167 bne htab_modify_pte
169 htab_insert_pte:
170 /* Clear hpte bits in new pte (we also clear BUSY btw) and
171 * add _PAGE_HASHPTE
172 */
173 lis r0,_PAGE_HPTEFLAGS@h
174 ori r0,r0,_PAGE_HPTEFLAGS@l
175 andc r30,r30,r0
176 ori r30,r30,_PAGE_HASHPTE
178 /* physical address r5 */
179 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
180 sldi r5,r5,PAGE_SHIFT
182 /* Calculate primary group hash */
183 and r0,r28,r27
184 rldicr r3,r0,3,63-3 /* r3 = (hash & mask) << 3 */
186 /* Call ppc_md.hpte_insert */
187 ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */
188 mr r4,r29 /* Retrieve va */
189 li r7,0 /* !bolted, !secondary */
190 li r8,MMU_PAGE_4K /* page size */
191 ld r9,STK_PARAM(R9)(r1) /* segment size */
192 _GLOBAL(htab_call_hpte_insert1)
193 bl . /* Patched by htab_finish_init() */
194 cmpdi 0,r3,0
195 bge htab_pte_insert_ok /* Insertion successful */
196 cmpdi 0,r3,-2 /* Critical failure */
197 beq- htab_pte_insert_failure
199 /* Now try secondary slot */
201 /* physical address r5 */
202 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
203 sldi r5,r5,PAGE_SHIFT
205 /* Calculate secondary group hash */
206 andc r0,r27,r28
207 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
209 /* Call ppc_md.hpte_insert */
210 ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */
211 mr r4,r29 /* Retrieve va */
212 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
213 li r8,MMU_PAGE_4K /* page size */
214 ld r9,STK_PARAM(R9)(r1) /* segment size */
215 _GLOBAL(htab_call_hpte_insert2)
216 bl . /* Patched by htab_finish_init() */
217 cmpdi 0,r3,0
218 bge+ htab_pte_insert_ok /* Insertion successful */
219 cmpdi 0,r3,-2 /* Critical failure */
220 beq- htab_pte_insert_failure
222 /* Both are full, we need to evict something */
223 mftb r0
224 /* Pick a random group based on TB */
225 andi. r0,r0,1
226 mr r5,r28
227 bne 2f
228 not r5,r5
229 2: and r0,r5,r27
230 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
231 /* Call ppc_md.hpte_remove */
232 _GLOBAL(htab_call_hpte_remove)
233 bl . /* Patched by htab_finish_init() */
235 /* Try all again */
236 b htab_insert_pte
238 htab_bail_ok:
239 li r3,0
240 b htab_bail
242 htab_pte_insert_ok:
243 /* Insert slot number & secondary bit in PTE */
244 rldimi r30,r3,12,63-15
246 /* Write out the PTE with a normal write
247 * (maybe add eieio may be good still ?)
248 */
249 htab_write_out_pte:
250 ld r6,STK_PARAM(R6)(r1)
251 std r30,0(r6)
252 li r3, 0
253 htab_bail:
254 ld r27,STK_REG(R27)(r1)
255 ld r28,STK_REG(R28)(r1)
256 ld r29,STK_REG(R29)(r1)
257 ld r30,STK_REG(R30)(r1)
258 ld r31,STK_REG(R31)(r1)
259 addi r1,r1,STACKFRAMESIZE
260 ld r0,16(r1)
261 mtlr r0
262 blr
264 htab_modify_pte:
265 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
266 mr r4,r3
267 rlwinm r3,r31,32-12,29,31
269 /* Secondary group ? if yes, get a inverted hash value */
270 mr r5,r28
271 andi. r0,r31,_PAGE_SECONDARY
272 beq 1f
273 not r5,r5
274 1:
275 /* Calculate proper slot value for ppc_md.hpte_updatepp */
276 and r0,r5,r27
277 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
278 add r3,r0,r3 /* add slot idx */
280 /* Call ppc_md.hpte_updatepp */
281 mr r5,r29 /* va */
282 li r6,MMU_PAGE_4K /* page size */
283 ld r7,STK_PARAM(R9)(r1) /* segment size */
284 ld r8,STK_PARAM(R8)(r1) /* get "local" param */
285 _GLOBAL(htab_call_hpte_updatepp)
286 bl . /* Patched by htab_finish_init() */
288 /* if we failed because typically the HPTE wasn't really here
289 * we try an insertion.
290 */
291 cmpdi 0,r3,-1
292 beq- htab_insert_pte
294 /* Clear the BUSY bit and Write out the PTE */
295 li r0,_PAGE_BUSY
296 andc r30,r30,r0
297 b htab_write_out_pte
299 htab_wrong_access:
300 /* Bail out clearing reservation */
301 stdcx. r31,0,r6
302 li r3,1
303 b htab_bail
305 htab_pte_insert_failure:
306 /* Bail out restoring old PTE */
307 ld r6,STK_PARAM(R6)(r1)
308 std r31,0(r6)
309 li r3,-1
310 b htab_bail
313 #else /* CONFIG_PPC_64K_PAGES */
316 /*****************************************************************************
317 * *
318 * 64K SW & 4K or 64K HW in a 4K segment pages implementation *
319 * *
320 *****************************************************************************/
322 /* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
323 * pte_t *ptep, unsigned long trap, int local, int ssize,
324 * int subpg_prot)
325 */
327 /*
328 * For now, we do NOT implement Admixed pages
329 */
330 _GLOBAL(__hash_page_4K)
331 mflr r0
332 std r0,16(r1)
333 stdu r1,-STACKFRAMESIZE(r1)
334 /* Save all params that we need after a function call */
335 std r6,STK_PARAM(R6)(r1)
336 std r8,STK_PARAM(R8)(r1)
337 std r9,STK_PARAM(R9)(r1)
339 /* Save non-volatile registers.
340 * r31 will hold "old PTE"
341 * r30 is "new PTE"
342 * r29 is "va"
343 * r28 is a hash value
344 * r27 is hashtab mask (maybe dynamic patched instead ?)
345 * r26 is the hidx mask
346 * r25 is the index in combo page
347 */
348 std r25,STK_REG(R25)(r1)
349 std r26,STK_REG(R26)(r1)
350 std r27,STK_REG(R27)(r1)
351 std r28,STK_REG(R28)(r1)
352 std r29,STK_REG(R29)(r1)
353 std r30,STK_REG(R30)(r1)
354 std r31,STK_REG(R31)(r1)
356 /* Step 1:
357 *
358 * Check permissions, atomically mark the linux PTE busy
359 * and hashed.
360 */
361 1:
362 ldarx r31,0,r6
363 /* Check access rights (access & ~(pte_val(*ptep))) */
364 andc. r0,r4,r31
365 bne- htab_wrong_access
366 /* Check if PTE is busy */
367 andi. r0,r31,_PAGE_BUSY
368 /* If so, just bail out and refault if needed. Someone else
369 * is changing this PTE anyway and might hash it.
370 */
371 bne- htab_bail_ok
372 /* Prepare new PTE value (turn access RW into DIRTY, then
373 * add BUSY and ACCESSED)
374 */
375 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
376 or r30,r30,r31
377 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
378 oris r30,r30,_PAGE_COMBO@h
379 /* Write the linux PTE atomically (setting busy) */
380 stdcx. r30,0,r6
381 bne- 1b
382 isync
384 /* Step 2:
385 *
386 * Insert/Update the HPTE in the hash table. At this point,
387 * r4 (access) is re-useable, we use it for the new HPTE flags
388 */
390 /* Load the hidx index */
391 rldicl r25,r3,64-12,60
393 BEGIN_FTR_SECTION
394 cmpdi r9,0 /* check segment size */
395 bne 3f
396 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
397 /* Calc va and put it in r29 */
398 rldicr r29,r5,28,63-28 /* r29 = (vsid << 28) */
399 rldicl r3,r3,0,36 /* r3 = (ea & 0x0fffffff) */
400 or r29,r3,r29 /* r29 = va */
402 /* Calculate hash value for primary slot and store it in r28 */
403 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
404 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */
405 xor r28,r5,r0
406 b 4f
408 3: /* Calc VA and hash in r29 and r28 for 1T segment */
409 sldi r29,r5,40 /* vsid << 40 */
410 clrldi r3,r3,24 /* ea & 0xffffffffff */
411 rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */
412 clrldi r5,r5,40 /* vsid & 0xffffff */
413 rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */
414 xor r28,r28,r5
415 or r29,r3,r29 /* VA */
416 xor r28,r28,r0 /* hash */
418 /* Convert linux PTE bits into HW equivalents */
419 4:
420 #ifdef CONFIG_PPC_SUBPAGE_PROT
421 andc r10,r30,r10
422 andi. r3,r10,0x1fe /* Get basic set of flags */
423 rlwinm r0,r10,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
424 #else
425 andi. r3,r30,0x1fe /* Get basic set of flags */
426 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
427 #endif
428 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
429 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
430 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
431 andc r0,r3,r0 /* r0 = pte & ~r0 */
432 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
433 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */
435 /* We eventually do the icache sync here (maybe inline that
436 * code rather than call a C function...)
437 */
438 BEGIN_FTR_SECTION
439 mr r4,r30
440 mr r5,r7
441 bl .hash_page_do_lazy_icache
442 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
444 /* At this point, r3 contains new PP bits, save them in
445 * place of "access" in the param area (sic)
446 */
447 std r3,STK_PARAM(R4)(r1)
449 /* Get htab_hash_mask */
450 ld r4,htab_hash_mask@got(2)
451 ld r27,0(r4) /* htab_hash_mask -> r27 */
453 /* Check if we may already be in the hashtable, in this case, we
454 * go to out-of-line code to try to modify the HPTE. We look for
455 * the bit at (1 >> (index + 32))
456 */
457 rldicl. r0,r31,64-12,48
458 li r26,0 /* Default hidx */
459 beq htab_insert_pte
461 /*
462 * Check if the pte was already inserted into the hash table
463 * as a 64k HW page, and invalidate the 64k HPTE if so.
464 */
465 andis. r0,r31,_PAGE_COMBO@h
466 beq htab_inval_old_hpte
468 ld r6,STK_PARAM(R6)(r1)
469 ori r26,r6,0x8000 /* Load the hidx mask */
470 ld r26,0(r26)
471 addi r5,r25,36 /* Check actual HPTE_SUB bit, this */
472 rldcr. r0,r31,r5,0 /* must match pgtable.h definition */
473 bne htab_modify_pte
475 htab_insert_pte:
476 /* real page number in r5, PTE RPN value + index */
477 andis. r0,r31,_PAGE_4K_PFN@h
478 srdi r5,r31,PTE_RPN_SHIFT
479 bne- htab_special_pfn
480 sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
481 add r5,r5,r25
482 htab_special_pfn:
483 sldi r5,r5,HW_PAGE_SHIFT
485 /* Calculate primary group hash */
486 and r0,r28,r27
487 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
489 /* Call ppc_md.hpte_insert */
490 ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */
491 mr r4,r29 /* Retrieve va */
492 li r7,0 /* !bolted, !secondary */
493 li r8,MMU_PAGE_4K /* page size */
494 ld r9,STK_PARAM(R9)(r1) /* segment size */
495 _GLOBAL(htab_call_hpte_insert1)
496 bl . /* patched by htab_finish_init() */
497 cmpdi 0,r3,0
498 bge htab_pte_insert_ok /* Insertion successful */
499 cmpdi 0,r3,-2 /* Critical failure */
500 beq- htab_pte_insert_failure
502 /* Now try secondary slot */
504 /* real page number in r5, PTE RPN value + index */
505 andis. r0,r31,_PAGE_4K_PFN@h
506 srdi r5,r31,PTE_RPN_SHIFT
507 bne- 3f
508 sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
509 add r5,r5,r25
510 3: sldi r5,r5,HW_PAGE_SHIFT
512 /* Calculate secondary group hash */
513 andc r0,r27,r28
514 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
516 /* Call ppc_md.hpte_insert */
517 ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */
518 mr r4,r29 /* Retrieve va */
519 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
520 li r8,MMU_PAGE_4K /* page size */
521 ld r9,STK_PARAM(R9)(r1) /* segment size */
522 _GLOBAL(htab_call_hpte_insert2)
523 bl . /* patched by htab_finish_init() */
524 cmpdi 0,r3,0
525 bge+ htab_pte_insert_ok /* Insertion successful */
526 cmpdi 0,r3,-2 /* Critical failure */
527 beq- htab_pte_insert_failure
529 /* Both are full, we need to evict something */
530 mftb r0
531 /* Pick a random group based on TB */
532 andi. r0,r0,1
533 mr r5,r28
534 bne 2f
535 not r5,r5
536 2: and r0,r5,r27
537 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
538 /* Call ppc_md.hpte_remove */
539 _GLOBAL(htab_call_hpte_remove)
540 bl . /* patched by htab_finish_init() */
542 /* Try all again */
543 b htab_insert_pte
545 /*
546 * Call out to C code to invalidate an 64k HW HPTE that is
547 * useless now that the segment has been switched to 4k pages.
548 */
549 htab_inval_old_hpte:
550 mr r3,r29 /* virtual addr */
551 mr r4,r31 /* PTE.pte */
552 li r5,0 /* PTE.hidx */
553 li r6,MMU_PAGE_64K /* psize */
554 ld r7,STK_PARAM(R9)(r1) /* ssize */
555 ld r8,STK_PARAM(R8)(r1) /* local */
556 bl .flush_hash_page
557 /* Clear out _PAGE_HPTE_SUB bits in the new linux PTE */
558 lis r0,_PAGE_HPTE_SUB@h
559 ori r0,r0,_PAGE_HPTE_SUB@l
560 andc r30,r30,r0
561 b htab_insert_pte
563 htab_bail_ok:
564 li r3,0
565 b htab_bail
567 htab_pte_insert_ok:
568 /* Insert slot number & secondary bit in PTE second half,
569 * clear _PAGE_BUSY and set approriate HPTE slot bit
570 */
571 ld r6,STK_PARAM(R6)(r1)
572 li r0,_PAGE_BUSY
573 andc r30,r30,r0
574 /* HPTE SUB bit */
575 li r0,1
576 subfic r5,r25,27 /* Must match bit position in */
577 sld r0,r0,r5 /* pgtable.h */
578 or r30,r30,r0
579 /* hindx */
580 sldi r5,r25,2
581 sld r3,r3,r5
582 li r4,0xf
583 sld r4,r4,r5
584 andc r26,r26,r4
585 or r26,r26,r3
586 ori r5,r6,0x8000
587 std r26,0(r5)
588 lwsync
589 std r30,0(r6)
590 li r3, 0
591 htab_bail:
592 ld r25,STK_REG(R25)(r1)
593 ld r26,STK_REG(R26)(r1)
594 ld r27,STK_REG(R27)(r1)
595 ld r28,STK_REG(R28)(r1)
596 ld r29,STK_REG(R29)(r1)
597 ld r30,STK_REG(R30)(r1)
598 ld r31,STK_REG(R31)(r1)
599 addi r1,r1,STACKFRAMESIZE
600 ld r0,16(r1)
601 mtlr r0
602 blr
604 htab_modify_pte:
605 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
606 mr r4,r3
607 sldi r5,r25,2
608 srd r3,r26,r5
610 /* Secondary group ? if yes, get a inverted hash value */
611 mr r5,r28
612 andi. r0,r3,0x8 /* page secondary ? */
613 beq 1f
614 not r5,r5
615 1: andi. r3,r3,0x7 /* extract idx alone */
617 /* Calculate proper slot value for ppc_md.hpte_updatepp */
618 and r0,r5,r27
619 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
620 add r3,r0,r3 /* add slot idx */
622 /* Call ppc_md.hpte_updatepp */
623 mr r5,r29 /* va */
624 li r6,MMU_PAGE_4K /* page size */
625 ld r7,STK_PARAM(R9)(r1) /* segment size */
626 ld r8,STK_PARAM(R8)(r1) /* get "local" param */
627 _GLOBAL(htab_call_hpte_updatepp)
628 bl . /* patched by htab_finish_init() */
630 /* if we failed because typically the HPTE wasn't really here
631 * we try an insertion.
632 */
633 cmpdi 0,r3,-1
634 beq- htab_insert_pte
636 /* Clear the BUSY bit and Write out the PTE */
637 li r0,_PAGE_BUSY
638 andc r30,r30,r0
639 ld r6,STK_PARAM(R6)(r1)
640 std r30,0(r6)
641 li r3,0
642 b htab_bail
644 htab_wrong_access:
645 /* Bail out clearing reservation */
646 stdcx. r31,0,r6
647 li r3,1
648 b htab_bail
650 htab_pte_insert_failure:
651 /* Bail out restoring old PTE */
652 ld r6,STK_PARAM(R6)(r1)
653 std r31,0(r6)
654 li r3,-1
655 b htab_bail
657 #endif /* CONFIG_PPC_64K_PAGES */
659 #ifdef CONFIG_PPC_HAS_HASH_64K
661 /*****************************************************************************
662 * *
663 * 64K SW & 64K HW in a 64K segment pages implementation *
664 * *
665 *****************************************************************************/
667 _GLOBAL(__hash_page_64K)
668 mflr r0
669 std r0,16(r1)
670 stdu r1,-STACKFRAMESIZE(r1)
671 /* Save all params that we need after a function call */
672 std r6,STK_PARAM(R6)(r1)
673 std r8,STK_PARAM(R8)(r1)
674 std r9,STK_PARAM(R9)(r1)
676 /* Save non-volatile registers.
677 * r31 will hold "old PTE"
678 * r30 is "new PTE"
679 * r29 is "va"
680 * r28 is a hash value
681 * r27 is hashtab mask (maybe dynamic patched instead ?)
682 */
683 std r27,STK_REG(R27)(r1)
684 std r28,STK_REG(R28)(r1)
685 std r29,STK_REG(R29)(r1)
686 std r30,STK_REG(R30)(r1)
687 std r31,STK_REG(R31)(r1)
689 /* Step 1:
690 *
691 * Check permissions, atomically mark the linux PTE busy
692 * and hashed.
693 */
694 1:
695 ldarx r31,0,r6
696 /* Check access rights (access & ~(pte_val(*ptep))) */
697 andc. r0,r4,r31
698 bne- ht64_wrong_access
699 /* Check if PTE is busy */
700 andi. r0,r31,_PAGE_BUSY
701 /* If so, just bail out and refault if needed. Someone else
702 * is changing this PTE anyway and might hash it.
703 */
704 bne- ht64_bail_ok
705 BEGIN_FTR_SECTION
706 /* Check if PTE has the cache-inhibit bit set */
707 andi. r0,r31,_PAGE_NO_CACHE
708 /* If so, bail out and refault as a 4k page */
709 bne- ht64_bail_ok
710 END_MMU_FTR_SECTION_IFCLR(MMU_FTR_CI_LARGE_PAGE)
711 /* Prepare new PTE value (turn access RW into DIRTY, then
712 * add BUSY and ACCESSED)
713 */
714 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
715 or r30,r30,r31
716 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
717 /* Write the linux PTE atomically (setting busy) */
718 stdcx. r30,0,r6
719 bne- 1b
720 isync
722 /* Step 2:
723 *
724 * Insert/Update the HPTE in the hash table. At this point,
725 * r4 (access) is re-useable, we use it for the new HPTE flags
726 */
728 BEGIN_FTR_SECTION
729 cmpdi r9,0 /* check segment size */
730 bne 3f
731 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
732 /* Calc va and put it in r29 */
733 rldicr r29,r5,28,63-28
734 rldicl r3,r3,0,36
735 or r29,r3,r29
737 /* Calculate hash value for primary slot and store it in r28 */
738 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
739 rldicl r0,r3,64-16,52 /* (ea >> 16) & 0xfff */
740 xor r28,r5,r0
741 b 4f
743 3: /* Calc VA and hash in r29 and r28 for 1T segment */
744 sldi r29,r5,40 /* vsid << 40 */
745 clrldi r3,r3,24 /* ea & 0xffffffffff */
746 rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */
747 clrldi r5,r5,40 /* vsid & 0xffffff */
748 rldicl r0,r3,64-16,40 /* (ea >> 16) & 0xffffff */
749 xor r28,r28,r5
750 or r29,r3,r29 /* VA */
751 xor r28,r28,r0 /* hash */
753 /* Convert linux PTE bits into HW equivalents */
754 4: andi. r3,r30,0x1fe /* Get basic set of flags */
755 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
756 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
757 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
758 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
759 andc r0,r30,r0 /* r0 = pte & ~r0 */
760 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
761 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */
763 /* We eventually do the icache sync here (maybe inline that
764 * code rather than call a C function...)
765 */
766 BEGIN_FTR_SECTION
767 mr r4,r30
768 mr r5,r7
769 bl .hash_page_do_lazy_icache
770 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
772 /* At this point, r3 contains new PP bits, save them in
773 * place of "access" in the param area (sic)
774 */
775 std r3,STK_PARAM(R4)(r1)
777 /* Get htab_hash_mask */
778 ld r4,htab_hash_mask@got(2)
779 ld r27,0(r4) /* htab_hash_mask -> r27 */
781 /* Check if we may already be in the hashtable, in this case, we
782 * go to out-of-line code to try to modify the HPTE
783 */
784 rldicl. r0,r31,64-12,48
785 bne ht64_modify_pte
787 ht64_insert_pte:
788 /* Clear hpte bits in new pte (we also clear BUSY btw) and
789 * add _PAGE_HPTE_SUB0
790 */
791 lis r0,_PAGE_HPTEFLAGS@h
792 ori r0,r0,_PAGE_HPTEFLAGS@l
793 andc r30,r30,r0
794 #ifdef CONFIG_PPC_64K_PAGES
795 oris r30,r30,_PAGE_HPTE_SUB0@h
796 #else
797 ori r30,r30,_PAGE_HASHPTE
798 #endif
799 /* Phyical address in r5 */
800 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
801 sldi r5,r5,PAGE_SHIFT
803 /* Calculate primary group hash */
804 and r0,r28,r27
805 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
807 /* Call ppc_md.hpte_insert */
808 ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */
809 mr r4,r29 /* Retrieve va */
810 li r7,0 /* !bolted, !secondary */
811 li r8,MMU_PAGE_64K
812 ld r9,STK_PARAM(R9)(r1) /* segment size */
813 _GLOBAL(ht64_call_hpte_insert1)
814 bl . /* patched by htab_finish_init() */
815 cmpdi 0,r3,0
816 bge ht64_pte_insert_ok /* Insertion successful */
817 cmpdi 0,r3,-2 /* Critical failure */
818 beq- ht64_pte_insert_failure
820 /* Now try secondary slot */
822 /* Phyical address in r5 */
823 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
824 sldi r5,r5,PAGE_SHIFT
826 /* Calculate secondary group hash */
827 andc r0,r27,r28
828 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
830 /* Call ppc_md.hpte_insert */
831 ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */
832 mr r4,r29 /* Retrieve va */
833 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
834 li r8,MMU_PAGE_64K
835 ld r9,STK_PARAM(R9)(r1) /* segment size */
836 _GLOBAL(ht64_call_hpte_insert2)
837 bl . /* patched by htab_finish_init() */
838 cmpdi 0,r3,0
839 bge+ ht64_pte_insert_ok /* Insertion successful */
840 cmpdi 0,r3,-2 /* Critical failure */
841 beq- ht64_pte_insert_failure
843 /* Both are full, we need to evict something */
844 mftb r0
845 /* Pick a random group based on TB */
846 andi. r0,r0,1
847 mr r5,r28
848 bne 2f
849 not r5,r5
850 2: and r0,r5,r27
851 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
852 /* Call ppc_md.hpte_remove */
853 _GLOBAL(ht64_call_hpte_remove)
854 bl . /* patched by htab_finish_init() */
856 /* Try all again */
857 b ht64_insert_pte
859 ht64_bail_ok:
860 li r3,0
861 b ht64_bail
863 ht64_pte_insert_ok:
864 /* Insert slot number & secondary bit in PTE */
865 rldimi r30,r3,12,63-15
867 /* Write out the PTE with a normal write
868 * (maybe add eieio may be good still ?)
869 */
870 ht64_write_out_pte:
871 ld r6,STK_PARAM(R6)(r1)
872 std r30,0(r6)
873 li r3, 0
874 ht64_bail:
875 ld r27,STK_REG(R27)(r1)
876 ld r28,STK_REG(R28)(r1)
877 ld r29,STK_REG(R29)(r1)
878 ld r30,STK_REG(R30)(r1)
879 ld r31,STK_REG(R31)(r1)
880 addi r1,r1,STACKFRAMESIZE
881 ld r0,16(r1)
882 mtlr r0
883 blr
885 ht64_modify_pte:
886 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
887 mr r4,r3
888 rlwinm r3,r31,32-12,29,31
890 /* Secondary group ? if yes, get a inverted hash value */
891 mr r5,r28
892 andi. r0,r31,_PAGE_F_SECOND
893 beq 1f
894 not r5,r5
895 1:
896 /* Calculate proper slot value for ppc_md.hpte_updatepp */
897 and r0,r5,r27
898 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
899 add r3,r0,r3 /* add slot idx */
901 /* Call ppc_md.hpte_updatepp */
902 mr r5,r29 /* va */
903 li r6,MMU_PAGE_64K
904 ld r7,STK_PARAM(R9)(r1) /* segment size */
905 ld r8,STK_PARAM(R8)(r1) /* get "local" param */
906 _GLOBAL(ht64_call_hpte_updatepp)
907 bl . /* patched by htab_finish_init() */
909 /* if we failed because typically the HPTE wasn't really here
910 * we try an insertion.
911 */
912 cmpdi 0,r3,-1
913 beq- ht64_insert_pte
915 /* Clear the BUSY bit and Write out the PTE */
916 li r0,_PAGE_BUSY
917 andc r30,r30,r0
918 b ht64_write_out_pte
920 ht64_wrong_access:
921 /* Bail out clearing reservation */
922 stdcx. r31,0,r6
923 li r3,1
924 b ht64_bail
926 ht64_pte_insert_failure:
927 /* Bail out restoring old PTE */
928 ld r6,STK_PARAM(R6)(r1)
929 std r31,0(r6)
930 li r3,-1
931 b ht64_bail
934 #endif /* CONFIG_PPC_HAS_HASH_64K */
937 /*****************************************************************************
938 * *
939 * Huge pages implementation is in hugetlbpage.c *
940 * *
941 *****************************************************************************/