| blob | 463174a4a64755e2d8d06d2f2b7e174d7297878d |
1 /*
2 * ppc64 MMU hashtable management routines
3 *
4 * (c) Copyright IBM Corp. 2003, 2005
5 *
6 * Maintained by: Benjamin Herrenschmidt
7 * <benh@kernel.crashing.org>
8 *
9 * This file is covered by the GNU Public Licence v2 as
10 * described in the kernel's COPYING file.
11 */
13 #include <asm/reg.h>
14 #include <asm/pgtable.h>
15 #include <asm/mmu.h>
16 #include <asm/page.h>
17 #include <asm/types.h>
18 #include <asm/ppc_asm.h>
19 #include <asm/asm-offsets.h>
20 #include <asm/cputable.h>
22 .text
24 /*
25 * Stackframe:
26 *
27 * +-> Back chain (SP + 256)
28 * | General register save area (SP + 112)
29 * | Parameter save area (SP + 48)
30 * | TOC save area (SP + 40)
31 * | link editor doubleword (SP + 32)
32 * | compiler doubleword (SP + 24)
33 * | LR save area (SP + 16)
34 * | CR save area (SP + 8)
35 * SP ---> +-- Back chain (SP + 0)
36 */
38 #ifndef CONFIG_PPC_64K_PAGES
40 /*****************************************************************************
41 * *
42 * 4K SW & 4K HW pages implementation *
43 * *
44 *****************************************************************************/
47 /*
48 * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
49 * pte_t *ptep, unsigned long trap, unsigned long flags,
50 * int ssize)
51 *
52 * Adds a 4K page to the hash table in a segment of 4K pages only
53 */
55 _GLOBAL(__hash_page_4K)
56 mflr r0
57 std r0,16(r1)
58 stdu r1,-STACKFRAMESIZE(r1)
59 /* Save all params that we need after a function call */
60 std r6,STK_PARAM(R6)(r1)
61 std r8,STK_PARAM(R8)(r1)
62 std r9,STK_PARAM(R9)(r1)
64 /* Save non-volatile registers.
65 * r31 will hold "old PTE"
66 * r30 is "new PTE"
67 * r29 is vpn
68 * r28 is a hash value
69 * r27 is hashtab mask (maybe dynamic patched instead ?)
70 */
71 std r27,STK_REG(R27)(r1)
72 std r28,STK_REG(R28)(r1)
73 std r29,STK_REG(R29)(r1)
74 std r30,STK_REG(R30)(r1)
75 std r31,STK_REG(R31)(r1)
77 /* Step 1:
78 *
79 * Check permissions, atomically mark the linux PTE busy
80 * and hashed.
81 */
82 1:
83 ldarx r31,0,r6
84 /* Check access rights (access & ~(pte_val(*ptep))) */
85 andc. r0,r4,r31
86 bne- htab_wrong_access
87 /* Check if PTE is busy */
88 andi. r0,r31,_PAGE_BUSY
89 /* If so, just bail out and refault if needed. Someone else
90 * is changing this PTE anyway and might hash it.
91 */
92 bne- htab_bail_ok
94 /* Prepare new PTE value (turn access RW into DIRTY, then
95 * add BUSY,HASHPTE and ACCESSED)
96 */
97 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
98 or r30,r30,r31
99 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
100 /* Write the linux PTE atomically (setting busy) */
101 stdcx. r30,0,r6
102 bne- 1b
103 isync
105 /* Step 2:
106 *
107 * Insert/Update the HPTE in the hash table. At this point,
108 * r4 (access) is re-useable, we use it for the new HPTE flags
109 */
111 BEGIN_FTR_SECTION
112 cmpdi r9,0 /* check segment size */
113 bne 3f
114 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
115 /* Calc vpn and put it in r29 */
116 sldi r29,r5,SID_SHIFT - VPN_SHIFT
117 rldicl r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT)
118 or r29,r28,r29
119 /*
120 * Calculate hash value for primary slot and store it in r28
121 * r3 = va, r5 = vsid
122 * r0 = (va >> 12) & ((1ul << (28 - 12)) -1)
123 */
124 rldicl r0,r3,64-12,48
125 xor r28,r5,r0 /* hash */
126 b 4f
128 3: /* Calc vpn and put it in r29 */
129 sldi r29,r5,SID_SHIFT_1T - VPN_SHIFT
130 rldicl r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT)
131 or r29,r28,r29
133 /*
134 * calculate hash value for primary slot and
135 * store it in r28 for 1T segment
136 * r3 = va, r5 = vsid
137 */
138 sldi r28,r5,25 /* vsid << 25 */
139 /* r0 = (va >> 12) & ((1ul << (40 - 12)) -1) */
140 rldicl r0,r3,64-12,36
141 xor r28,r28,r5 /* vsid ^ ( vsid << 25) */
142 xor r28,r28,r0 /* hash */
144 /* Convert linux PTE bits into HW equivalents */
145 4: andi. r3,r30,0x1fe /* Get basic set of flags */
146 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
147 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
148 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
149 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
150 andc r0,r30,r0 /* r0 = pte & ~r0 */
151 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
152 /*
153 * Always add "C" bit for perf. Memory coherence is always enabled
154 */
155 ori r3,r3,HPTE_R_C | HPTE_R_M
157 /* We eventually do the icache sync here (maybe inline that
158 * code rather than call a C function...)
159 */
160 BEGIN_FTR_SECTION
161 mr r4,r30
162 mr r5,r7
163 bl hash_page_do_lazy_icache
164 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
166 /* At this point, r3 contains new PP bits, save them in
167 * place of "access" in the param area (sic)
168 */
169 std r3,STK_PARAM(R4)(r1)
171 /* Get htab_hash_mask */
172 ld r4,htab_hash_mask@got(2)
173 ld r27,0(r4) /* htab_hash_mask -> r27 */
175 /* Check if we may already be in the hashtable, in this case, we
176 * go to out-of-line code to try to modify the HPTE
177 */
178 andi. r0,r31,_PAGE_HASHPTE
179 bne htab_modify_pte
181 htab_insert_pte:
182 /* Clear hpte bits in new pte (we also clear BUSY btw) and
183 * add _PAGE_HASHPTE
184 */
185 lis r0,_PAGE_HPTEFLAGS@h
186 ori r0,r0,_PAGE_HPTEFLAGS@l
187 andc r30,r30,r0
188 ori r30,r30,_PAGE_HASHPTE
190 /* physical address r5 */
191 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
192 sldi r5,r5,PAGE_SHIFT
194 /* Calculate primary group hash */
195 and r0,r28,r27
196 rldicr r3,r0,3,63-3 /* r3 = (hash & mask) << 3 */
198 /* Call ppc_md.hpte_insert */
199 ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */
200 mr r4,r29 /* Retrieve vpn */
201 li r7,0 /* !bolted, !secondary */
202 li r8,MMU_PAGE_4K /* page size */
203 li r9,MMU_PAGE_4K /* actual page size */
204 ld r10,STK_PARAM(R9)(r1) /* segment size */
205 .globl htab_call_hpte_insert1
206 htab_call_hpte_insert1:
207 bl . /* Patched by htab_finish_init() */
208 cmpdi 0,r3,0
209 bge htab_pte_insert_ok /* Insertion successful */
210 cmpdi 0,r3,-2 /* Critical failure */
211 beq- htab_pte_insert_failure
213 /* Now try secondary slot */
215 /* physical address r5 */
216 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
217 sldi r5,r5,PAGE_SHIFT
219 /* Calculate secondary group hash */
220 andc r0,r27,r28
221 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
223 /* Call ppc_md.hpte_insert */
224 ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */
225 mr r4,r29 /* Retrieve vpn */
226 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
227 li r8,MMU_PAGE_4K /* page size */
228 li r9,MMU_PAGE_4K /* actual page size */
229 ld r10,STK_PARAM(R9)(r1) /* segment size */
230 .globl htab_call_hpte_insert2
231 htab_call_hpte_insert2:
232 bl . /* Patched by htab_finish_init() */
233 cmpdi 0,r3,0
234 bge+ htab_pte_insert_ok /* Insertion successful */
235 cmpdi 0,r3,-2 /* Critical failure */
236 beq- htab_pte_insert_failure
238 /* Both are full, we need to evict something */
239 mftb r0
240 /* Pick a random group based on TB */
241 andi. r0,r0,1
242 mr r5,r28
243 bne 2f
244 not r5,r5
245 2: and r0,r5,r27
246 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
247 /* Call ppc_md.hpte_remove */
248 .globl htab_call_hpte_remove
249 htab_call_hpte_remove:
250 bl . /* Patched by htab_finish_init() */
252 /* Try all again */
253 b htab_insert_pte
255 htab_bail_ok:
256 li r3,0
257 b htab_bail
259 htab_pte_insert_ok:
260 /* Insert slot number & secondary bit in PTE */
261 rldimi r30,r3,12,63-15
263 /* Write out the PTE with a normal write
264 * (maybe add eieio may be good still ?)
265 */
266 htab_write_out_pte:
267 ld r6,STK_PARAM(R6)(r1)
268 std r30,0(r6)
269 li r3, 0
270 htab_bail:
271 ld r27,STK_REG(R27)(r1)
272 ld r28,STK_REG(R28)(r1)
273 ld r29,STK_REG(R29)(r1)
274 ld r30,STK_REG(R30)(r1)
275 ld r31,STK_REG(R31)(r1)
276 addi r1,r1,STACKFRAMESIZE
277 ld r0,16(r1)
278 mtlr r0
279 blr
281 htab_modify_pte:
282 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
283 mr r4,r3
284 rlwinm r3,r31,32-12,29,31
286 /* Secondary group ? if yes, get a inverted hash value */
287 mr r5,r28
288 andi. r0,r31,_PAGE_SECONDARY
289 beq 1f
290 not r5,r5
291 1:
292 /* Calculate proper slot value for ppc_md.hpte_updatepp */
293 and r0,r5,r27
294 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
295 add r3,r0,r3 /* add slot idx */
297 /* Call ppc_md.hpte_updatepp */
298 mr r5,r29 /* vpn */
299 li r6,MMU_PAGE_4K /* base page size */
300 li r7,MMU_PAGE_4K /* actual page size */
301 ld r8,STK_PARAM(R9)(r1) /* segment size */
302 ld r9,STK_PARAM(R8)(r1) /* get "flags" param */
303 .globl htab_call_hpte_updatepp
304 htab_call_hpte_updatepp:
305 bl . /* Patched by htab_finish_init() */
307 /* if we failed because typically the HPTE wasn't really here
308 * we try an insertion.
309 */
310 cmpdi 0,r3,-1
311 beq- htab_insert_pte
313 /* Clear the BUSY bit and Write out the PTE */
314 li r0,_PAGE_BUSY
315 andc r30,r30,r0
316 b htab_write_out_pte
318 htab_wrong_access:
319 /* Bail out clearing reservation */
320 stdcx. r31,0,r6
321 li r3,1
322 b htab_bail
324 htab_pte_insert_failure:
325 /* Bail out restoring old PTE */
326 ld r6,STK_PARAM(R6)(r1)
327 std r31,0(r6)
328 li r3,-1
329 b htab_bail
332 #else /* CONFIG_PPC_64K_PAGES */
335 /*****************************************************************************
336 * *
337 * 64K SW & 4K or 64K HW in a 4K segment pages implementation *
338 * *
339 *****************************************************************************/
341 /* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
342 * pte_t *ptep, unsigned long trap, unsigned local flags,
343 * int ssize, int subpg_prot)
344 */
346 /*
347 * For now, we do NOT implement Admixed pages
348 */
349 _GLOBAL(__hash_page_4K)
350 mflr r0
351 std r0,16(r1)
352 stdu r1,-STACKFRAMESIZE(r1)
353 /* Save all params that we need after a function call */
354 std r6,STK_PARAM(R6)(r1)
355 std r8,STK_PARAM(R8)(r1)
356 std r9,STK_PARAM(R9)(r1)
358 /* Save non-volatile registers.
359 * r31 will hold "old PTE"
360 * r30 is "new PTE"
361 * r29 is vpn
362 * r28 is a hash value
363 * r27 is hashtab mask (maybe dynamic patched instead ?)
364 * r26 is the hidx mask
365 * r25 is the index in combo page
366 */
367 std r25,STK_REG(R25)(r1)
368 std r26,STK_REG(R26)(r1)
369 std r27,STK_REG(R27)(r1)
370 std r28,STK_REG(R28)(r1)
371 std r29,STK_REG(R29)(r1)
372 std r30,STK_REG(R30)(r1)
373 std r31,STK_REG(R31)(r1)
375 /* Step 1:
376 *
377 * Check permissions, atomically mark the linux PTE busy
378 * and hashed.
379 */
380 1:
381 ldarx r31,0,r6
382 /* Check access rights (access & ~(pte_val(*ptep))) */
383 andc. r0,r4,r31
384 bne- htab_wrong_access
385 /* Check if PTE is busy */
386 andi. r0,r31,_PAGE_BUSY
387 /* If so, just bail out and refault if needed. Someone else
388 * is changing this PTE anyway and might hash it.
389 */
390 bne- htab_bail_ok
391 /* Prepare new PTE value (turn access RW into DIRTY, then
392 * add BUSY and ACCESSED)
393 */
394 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
395 or r30,r30,r31
396 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
397 oris r30,r30,_PAGE_COMBO@h
398 /* Write the linux PTE atomically (setting busy) */
399 stdcx. r30,0,r6
400 bne- 1b
401 isync
403 /* Step 2:
404 *
405 * Insert/Update the HPTE in the hash table. At this point,
406 * r4 (access) is re-useable, we use it for the new HPTE flags
407 */
409 /* Load the hidx index */
410 rldicl r25,r3,64-12,60
412 BEGIN_FTR_SECTION
413 cmpdi r9,0 /* check segment size */
414 bne 3f
415 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
416 /* Calc vpn and put it in r29 */
417 sldi r29,r5,SID_SHIFT - VPN_SHIFT
418 /*
419 * clrldi r3,r3,64 - SID_SHIFT --> ea & 0xfffffff
420 * srdi r28,r3,VPN_SHIFT
421 */
422 rldicl r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT)
423 or r29,r28,r29
424 /*
425 * Calculate hash value for primary slot and store it in r28
426 * r3 = va, r5 = vsid
427 * r0 = (va >> 12) & ((1ul << (28 - 12)) -1)
428 */
429 rldicl r0,r3,64-12,48
430 xor r28,r5,r0 /* hash */
431 b 4f
433 3: /* Calc vpn and put it in r29 */
434 sldi r29,r5,SID_SHIFT_1T - VPN_SHIFT
435 /*
436 * clrldi r3,r3,64 - SID_SHIFT_1T --> ea & 0xffffffffff
437 * srdi r28,r3,VPN_SHIFT
438 */
439 rldicl r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT)
440 or r29,r28,r29
442 /*
443 * Calculate hash value for primary slot and
444 * store it in r28 for 1T segment
445 * r3 = va, r5 = vsid
446 */
447 sldi r28,r5,25 /* vsid << 25 */
448 /* r0 = (va >> 12) & ((1ul << (40 - 12)) -1) */
449 rldicl r0,r3,64-12,36
450 xor r28,r28,r5 /* vsid ^ ( vsid << 25) */
451 xor r28,r28,r0 /* hash */
453 /* Convert linux PTE bits into HW equivalents */
454 4:
455 #ifdef CONFIG_PPC_SUBPAGE_PROT
456 andc r10,r30,r10
457 andi. r3,r10,0x1fe /* Get basic set of flags */
458 rlwinm r0,r10,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
459 #else
460 andi. r3,r30,0x1fe /* Get basic set of flags */
461 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
462 #endif
463 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
464 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
465 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
466 andc r0,r3,r0 /* r0 = pte & ~r0 */
467 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
468 /*
469 * Always add "C" bit for perf. Memory coherence is always enabled
470 */
471 ori r3,r3,HPTE_R_C | HPTE_R_M
473 /* We eventually do the icache sync here (maybe inline that
474 * code rather than call a C function...)
475 */
476 BEGIN_FTR_SECTION
477 mr r4,r30
478 mr r5,r7
479 bl hash_page_do_lazy_icache
480 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
482 /* At this point, r3 contains new PP bits, save them in
483 * place of "access" in the param area (sic)
484 */
485 std r3,STK_PARAM(R4)(r1)
487 /* Get htab_hash_mask */
488 ld r4,htab_hash_mask@got(2)
489 ld r27,0(r4) /* htab_hash_mask -> r27 */
491 /* Check if we may already be in the hashtable, in this case, we
492 * go to out-of-line code to try to modify the HPTE. We look for
493 * the bit at (1 >> (index + 32))
494 */
495 rldicl. r0,r31,64-12,48
496 li r26,0 /* Default hidx */
497 beq htab_insert_pte
499 /*
500 * Check if the pte was already inserted into the hash table
501 * as a 64k HW page, and invalidate the 64k HPTE if so.
502 */
503 andis. r0,r31,_PAGE_COMBO@h
504 beq htab_inval_old_hpte
506 ld r6,STK_PARAM(R6)(r1)
507 ori r26,r6,PTE_PAGE_HIDX_OFFSET /* Load the hidx mask. */
508 ld r26,0(r26)
509 addi r5,r25,36 /* Check actual HPTE_SUB bit, this */
510 rldcr. r0,r31,r5,0 /* must match pgtable.h definition */
511 bne htab_modify_pte
513 htab_insert_pte:
514 /* real page number in r5, PTE RPN value + index */
515 andis. r0,r31,_PAGE_4K_PFN@h
516 srdi r5,r31,PTE_RPN_SHIFT
517 bne- htab_special_pfn
518 sldi r5,r5,PAGE_FACTOR
519 add r5,r5,r25
520 htab_special_pfn:
521 sldi r5,r5,HW_PAGE_SHIFT
523 /* Calculate primary group hash */
524 and r0,r28,r27
525 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
527 /* Call ppc_md.hpte_insert */
528 ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */
529 mr r4,r29 /* Retrieve vpn */
530 li r7,0 /* !bolted, !secondary */
531 li r8,MMU_PAGE_4K /* page size */
532 li r9,MMU_PAGE_4K /* actual page size */
533 ld r10,STK_PARAM(R9)(r1) /* segment size */
534 .globl htab_call_hpte_insert1
535 htab_call_hpte_insert1:
536 bl . /* patched by htab_finish_init() */
537 cmpdi 0,r3,0
538 bge htab_pte_insert_ok /* Insertion successful */
539 cmpdi 0,r3,-2 /* Critical failure */
540 beq- htab_pte_insert_failure
542 /* Now try secondary slot */
544 /* real page number in r5, PTE RPN value + index */
545 andis. r0,r31,_PAGE_4K_PFN@h
546 srdi r5,r31,PTE_RPN_SHIFT
547 bne- 3f
548 sldi r5,r5,PAGE_FACTOR
549 add r5,r5,r25
550 3: sldi r5,r5,HW_PAGE_SHIFT
552 /* Calculate secondary group hash */
553 andc r0,r27,r28
554 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
556 /* Call ppc_md.hpte_insert */
557 ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */
558 mr r4,r29 /* Retrieve vpn */
559 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
560 li r8,MMU_PAGE_4K /* page size */
561 li r9,MMU_PAGE_4K /* actual page size */
562 ld r10,STK_PARAM(R9)(r1) /* segment size */
563 .globl htab_call_hpte_insert2
564 htab_call_hpte_insert2:
565 bl . /* patched by htab_finish_init() */
566 cmpdi 0,r3,0
567 bge+ htab_pte_insert_ok /* Insertion successful */
568 cmpdi 0,r3,-2 /* Critical failure */
569 beq- htab_pte_insert_failure
571 /* Both are full, we need to evict something */
572 mftb r0
573 /* Pick a random group based on TB */
574 andi. r0,r0,1
575 mr r5,r28
576 bne 2f
577 not r5,r5
578 2: and r0,r5,r27
579 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
580 /* Call ppc_md.hpte_remove */
581 .globl htab_call_hpte_remove
582 htab_call_hpte_remove:
583 bl . /* patched by htab_finish_init() */
585 /* Try all again */
586 b htab_insert_pte
588 /*
589 * Call out to C code to invalidate an 64k HW HPTE that is
590 * useless now that the segment has been switched to 4k pages.
591 */
592 htab_inval_old_hpte:
593 mr r3,r29 /* vpn */
594 mr r4,r31 /* PTE.pte */
595 li r5,0 /* PTE.hidx */
596 li r6,MMU_PAGE_64K /* psize */
597 ld r7,STK_PARAM(R9)(r1) /* ssize */
598 ld r8,STK_PARAM(R8)(r1) /* flags */
599 bl flush_hash_page
600 /* Clear out _PAGE_HPTE_SUB bits in the new linux PTE */
601 lis r0,_PAGE_HPTE_SUB@h
602 ori r0,r0,_PAGE_HPTE_SUB@l
603 andc r30,r30,r0
604 b htab_insert_pte
606 htab_bail_ok:
607 li r3,0
608 b htab_bail
610 htab_pte_insert_ok:
611 /* Insert slot number & secondary bit in PTE second half,
612 * clear _PAGE_BUSY and set approriate HPTE slot bit
613 */
614 ld r6,STK_PARAM(R6)(r1)
615 li r0,_PAGE_BUSY
616 andc r30,r30,r0
617 /* HPTE SUB bit */
618 li r0,1
619 subfic r5,r25,27 /* Must match bit position in */
620 sld r0,r0,r5 /* pgtable.h */
621 or r30,r30,r0
622 /* hindx */
623 sldi r5,r25,2
624 sld r3,r3,r5
625 li r4,0xf
626 sld r4,r4,r5
627 andc r26,r26,r4
628 or r26,r26,r3
629 ori r5,r6,PTE_PAGE_HIDX_OFFSET
630 std r26,0(r5)
631 lwsync
632 std r30,0(r6)
633 li r3, 0
634 htab_bail:
635 ld r25,STK_REG(R25)(r1)
636 ld r26,STK_REG(R26)(r1)
637 ld r27,STK_REG(R27)(r1)
638 ld r28,STK_REG(R28)(r1)
639 ld r29,STK_REG(R29)(r1)
640 ld r30,STK_REG(R30)(r1)
641 ld r31,STK_REG(R31)(r1)
642 addi r1,r1,STACKFRAMESIZE
643 ld r0,16(r1)
644 mtlr r0
645 blr
647 htab_modify_pte:
648 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
649 mr r4,r3
650 sldi r5,r25,2
651 srd r3,r26,r5
653 /* Secondary group ? if yes, get a inverted hash value */
654 mr r5,r28
655 andi. r0,r3,0x8 /* page secondary ? */
656 beq 1f
657 not r5,r5
658 1: andi. r3,r3,0x7 /* extract idx alone */
660 /* Calculate proper slot value for ppc_md.hpte_updatepp */
661 and r0,r5,r27
662 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
663 add r3,r0,r3 /* add slot idx */
665 /* Call ppc_md.hpte_updatepp */
666 mr r5,r29 /* vpn */
667 li r6,MMU_PAGE_4K /* base page size */
668 li r7,MMU_PAGE_4K /* actual page size */
669 ld r8,STK_PARAM(R9)(r1) /* segment size */
670 ld r9,STK_PARAM(R8)(r1) /* get "flags" param */
671 .globl htab_call_hpte_updatepp
672 htab_call_hpte_updatepp:
673 bl . /* patched by htab_finish_init() */
675 /* if we failed because typically the HPTE wasn't really here
676 * we try an insertion.
677 */
678 cmpdi 0,r3,-1
679 beq- htab_insert_pte
681 /* Clear the BUSY bit and Write out the PTE */
682 li r0,_PAGE_BUSY
683 andc r30,r30,r0
684 ld r6,STK_PARAM(R6)(r1)
685 std r30,0(r6)
686 li r3,0
687 b htab_bail
689 htab_wrong_access:
690 /* Bail out clearing reservation */
691 stdcx. r31,0,r6
692 li r3,1
693 b htab_bail
695 htab_pte_insert_failure:
696 /* Bail out restoring old PTE */
697 ld r6,STK_PARAM(R6)(r1)
698 std r31,0(r6)
699 li r3,-1
700 b htab_bail
702 #endif /* CONFIG_PPC_64K_PAGES */
704 #ifdef CONFIG_PPC_HAS_HASH_64K
706 /*****************************************************************************
707 * *
708 * 64K SW & 64K HW in a 64K segment pages implementation *
709 * *
710 *****************************************************************************/
712 _GLOBAL(__hash_page_64K)
713 mflr r0
714 std r0,16(r1)
715 stdu r1,-STACKFRAMESIZE(r1)
716 /* Save all params that we need after a function call */
717 std r6,STK_PARAM(R6)(r1)
718 std r8,STK_PARAM(R8)(r1)
719 std r9,STK_PARAM(R9)(r1)
721 /* Save non-volatile registers.
722 * r31 will hold "old PTE"
723 * r30 is "new PTE"
724 * r29 is vpn
725 * r28 is a hash value
726 * r27 is hashtab mask (maybe dynamic patched instead ?)
727 */
728 std r27,STK_REG(R27)(r1)
729 std r28,STK_REG(R28)(r1)
730 std r29,STK_REG(R29)(r1)
731 std r30,STK_REG(R30)(r1)
732 std r31,STK_REG(R31)(r1)
734 /* Step 1:
735 *
736 * Check permissions, atomically mark the linux PTE busy
737 * and hashed.
738 */
739 1:
740 ldarx r31,0,r6
741 /* Check access rights (access & ~(pte_val(*ptep))) */
742 andc. r0,r4,r31
743 bne- ht64_wrong_access
744 /* Check if PTE is busy */
745 andi. r0,r31,_PAGE_BUSY
746 /* If so, just bail out and refault if needed. Someone else
747 * is changing this PTE anyway and might hash it.
748 */
749 bne- ht64_bail_ok
750 BEGIN_FTR_SECTION
751 /* Check if PTE has the cache-inhibit bit set */
752 andi. r0,r31,_PAGE_NO_CACHE
753 /* If so, bail out and refault as a 4k page */
754 bne- ht64_bail_ok
755 END_MMU_FTR_SECTION_IFCLR(MMU_FTR_CI_LARGE_PAGE)
756 /* Prepare new PTE value (turn access RW into DIRTY, then
757 * add BUSY and ACCESSED)
758 */
759 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
760 or r30,r30,r31
761 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
762 /* Write the linux PTE atomically (setting busy) */
763 stdcx. r30,0,r6
764 bne- 1b
765 isync
767 /* Step 2:
768 *
769 * Insert/Update the HPTE in the hash table. At this point,
770 * r4 (access) is re-useable, we use it for the new HPTE flags
771 */
773 BEGIN_FTR_SECTION
774 cmpdi r9,0 /* check segment size */
775 bne 3f
776 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
777 /* Calc vpn and put it in r29 */
778 sldi r29,r5,SID_SHIFT - VPN_SHIFT
779 rldicl r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT)
780 or r29,r28,r29
782 /* Calculate hash value for primary slot and store it in r28
783 * r3 = va, r5 = vsid
784 * r0 = (va >> 16) & ((1ul << (28 - 16)) -1)
785 */
786 rldicl r0,r3,64-16,52
787 xor r28,r5,r0 /* hash */
788 b 4f
790 3: /* Calc vpn and put it in r29 */
791 sldi r29,r5,SID_SHIFT_1T - VPN_SHIFT
792 rldicl r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT)
793 or r29,r28,r29
794 /*
795 * calculate hash value for primary slot and
796 * store it in r28 for 1T segment
797 * r3 = va, r5 = vsid
798 */
799 sldi r28,r5,25 /* vsid << 25 */
800 /* r0 = (va >> 16) & ((1ul << (40 - 16)) -1) */
801 rldicl r0,r3,64-16,40
802 xor r28,r28,r5 /* vsid ^ ( vsid << 25) */
803 xor r28,r28,r0 /* hash */
805 /* Convert linux PTE bits into HW equivalents */
806 4: andi. r3,r30,0x1fe /* Get basic set of flags */
807 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
808 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
809 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
810 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
811 andc r0,r30,r0 /* r0 = pte & ~r0 */
812 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
813 /*
814 * Always add "C" bit for perf. Memory coherence is always enabled
815 */
816 ori r3,r3,HPTE_R_C | HPTE_R_M
818 /* We eventually do the icache sync here (maybe inline that
819 * code rather than call a C function...)
820 */
821 BEGIN_FTR_SECTION
822 mr r4,r30
823 mr r5,r7
824 bl hash_page_do_lazy_icache
825 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
827 /* At this point, r3 contains new PP bits, save them in
828 * place of "access" in the param area (sic)
829 */
830 std r3,STK_PARAM(R4)(r1)
832 /* Get htab_hash_mask */
833 ld r4,htab_hash_mask@got(2)
834 ld r27,0(r4) /* htab_hash_mask -> r27 */
836 /* Check if we may already be in the hashtable, in this case, we
837 * go to out-of-line code to try to modify the HPTE
838 */
839 rldicl. r0,r31,64-12,48
840 bne ht64_modify_pte
842 ht64_insert_pte:
843 /* Clear hpte bits in new pte (we also clear BUSY btw) and
844 * add _PAGE_HPTE_SUB0
845 */
846 lis r0,_PAGE_HPTEFLAGS@h
847 ori r0,r0,_PAGE_HPTEFLAGS@l
848 andc r30,r30,r0
849 #ifdef CONFIG_PPC_64K_PAGES
850 oris r30,r30,_PAGE_HPTE_SUB0@h
851 #else
852 ori r30,r30,_PAGE_HASHPTE
853 #endif
854 /* Phyical address in r5 */
855 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
856 sldi r5,r5,PAGE_SHIFT
858 /* Calculate primary group hash */
859 and r0,r28,r27
860 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
862 /* Call ppc_md.hpte_insert */
863 ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */
864 mr r4,r29 /* Retrieve vpn */
865 li r7,0 /* !bolted, !secondary */
866 li r8,MMU_PAGE_64K
867 li r9,MMU_PAGE_64K /* actual page size */
868 ld r10,STK_PARAM(R9)(r1) /* segment size */
869 .globl ht64_call_hpte_insert1
870 ht64_call_hpte_insert1:
871 bl . /* patched by htab_finish_init() */
872 cmpdi 0,r3,0
873 bge ht64_pte_insert_ok /* Insertion successful */
874 cmpdi 0,r3,-2 /* Critical failure */
875 beq- ht64_pte_insert_failure
877 /* Now try secondary slot */
879 /* Phyical address in r5 */
880 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
881 sldi r5,r5,PAGE_SHIFT
883 /* Calculate secondary group hash */
884 andc r0,r27,r28
885 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
887 /* Call ppc_md.hpte_insert */
888 ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */
889 mr r4,r29 /* Retrieve vpn */
890 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
891 li r8,MMU_PAGE_64K
892 li r9,MMU_PAGE_64K /* actual page size */
893 ld r10,STK_PARAM(R9)(r1) /* segment size */
894 .globl ht64_call_hpte_insert2
895 ht64_call_hpte_insert2:
896 bl . /* patched by htab_finish_init() */
897 cmpdi 0,r3,0
898 bge+ ht64_pte_insert_ok /* Insertion successful */
899 cmpdi 0,r3,-2 /* Critical failure */
900 beq- ht64_pte_insert_failure
902 /* Both are full, we need to evict something */
903 mftb r0
904 /* Pick a random group based on TB */
905 andi. r0,r0,1
906 mr r5,r28
907 bne 2f
908 not r5,r5
909 2: and r0,r5,r27
910 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
911 /* Call ppc_md.hpte_remove */
912 .globl ht64_call_hpte_remove
913 ht64_call_hpte_remove:
914 bl . /* patched by htab_finish_init() */
916 /* Try all again */
917 b ht64_insert_pte
919 ht64_bail_ok:
920 li r3,0
921 b ht64_bail
923 ht64_pte_insert_ok:
924 /* Insert slot number & secondary bit in PTE */
925 rldimi r30,r3,12,63-15
927 /* Write out the PTE with a normal write
928 * (maybe add eieio may be good still ?)
929 */
930 ht64_write_out_pte:
931 ld r6,STK_PARAM(R6)(r1)
932 std r30,0(r6)
933 li r3, 0
934 ht64_bail:
935 ld r27,STK_REG(R27)(r1)
936 ld r28,STK_REG(R28)(r1)
937 ld r29,STK_REG(R29)(r1)
938 ld r30,STK_REG(R30)(r1)
939 ld r31,STK_REG(R31)(r1)
940 addi r1,r1,STACKFRAMESIZE
941 ld r0,16(r1)
942 mtlr r0
943 blr
945 ht64_modify_pte:
946 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
947 mr r4,r3
948 rlwinm r3,r31,32-12,29,31
950 /* Secondary group ? if yes, get a inverted hash value */
951 mr r5,r28
952 andi. r0,r31,_PAGE_F_SECOND
953 beq 1f
954 not r5,r5
955 1:
956 /* Calculate proper slot value for ppc_md.hpte_updatepp */
957 and r0,r5,r27
958 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
959 add r3,r0,r3 /* add slot idx */
961 /* Call ppc_md.hpte_updatepp */
962 mr r5,r29 /* vpn */
963 li r6,MMU_PAGE_64K /* base page size */
964 li r7,MMU_PAGE_64K /* actual page size */
965 ld r8,STK_PARAM(R9)(r1) /* segment size */
966 ld r9,STK_PARAM(R8)(r1) /* get "flags" param */
967 .globl ht64_call_hpte_updatepp
968 ht64_call_hpte_updatepp:
969 bl . /* patched by htab_finish_init() */
971 /* if we failed because typically the HPTE wasn't really here
972 * we try an insertion.
973 */
974 cmpdi 0,r3,-1
975 beq- ht64_insert_pte
977 /* Clear the BUSY bit and Write out the PTE */
978 li r0,_PAGE_BUSY
979 andc r30,r30,r0
980 b ht64_write_out_pte
982 ht64_wrong_access:
983 /* Bail out clearing reservation */
984 stdcx. r31,0,r6
985 li r3,1
986 b ht64_bail
988 ht64_pte_insert_failure:
989 /* Bail out restoring old PTE */
990 ld r6,STK_PARAM(R6)(r1)
991 std r31,0(r6)
992 li r3,-1
993 b ht64_bail
996 #endif /* CONFIG_PPC_HAS_HASH_64K */
999 /*****************************************************************************
1000 * *
1001 * Huge pages implementation is in hugetlbpage.c *
1002 * *
1003 *****************************************************************************/