| blob | a719f53921a57c390503b708c38cb4e1146a176b |
1 /*
2 * ppc64 MMU hashtable management routines
3 *
4 * (c) Copyright IBM Corp. 2003, 2005
5 *
6 * Maintained by: Benjamin Herrenschmidt
7 * <benh@kernel.crashing.org>
8 *
9 * This file is covered by the GNU Public Licence v2 as
10 * described in the kernel's COPYING file.
11 */
13 #include <asm/reg.h>
14 #include <asm/pgtable.h>
15 #include <asm/mmu.h>
16 #include <asm/page.h>
17 #include <asm/types.h>
18 #include <asm/ppc_asm.h>
19 #include <asm/asm-offsets.h>
20 #include <asm/cputable.h>
22 .text
24 /*
25 * Stackframe:
26 *
27 * +-> Back chain (SP + 256)
28 * | General register save area (SP + 112)
29 * | Parameter save area (SP + 48)
30 * | TOC save area (SP + 40)
31 * | link editor doubleword (SP + 32)
32 * | compiler doubleword (SP + 24)
33 * | LR save area (SP + 16)
34 * | CR save area (SP + 8)
35 * SP ---> +-- Back chain (SP + 0)
36 */
37 #define STACKFRAMESIZE 256
39 /* Save parameters offsets */
40 #define STK_PARM(i) (STACKFRAMESIZE + 48 + ((i)-3)*8)
42 /* Save non-volatile offsets */
43 #define STK_REG(i) (112 + ((i)-14)*8)
46 #ifndef CONFIG_PPC_64K_PAGES
48 /*****************************************************************************
49 * *
50 * 4K SW & 4K HW pages implementation *
51 * *
52 *****************************************************************************/
55 /*
56 * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
57 * pte_t *ptep, unsigned long trap, int local, int ssize)
58 *
59 * Adds a 4K page to the hash table in a segment of 4K pages only
60 */
62 _GLOBAL(__hash_page_4K)
63 mflr r0
64 std r0,16(r1)
65 stdu r1,-STACKFRAMESIZE(r1)
66 /* Save all params that we need after a function call */
67 std r6,STK_PARM(r6)(r1)
68 std r8,STK_PARM(r8)(r1)
69 std r9,STK_PARM(r9)(r1)
71 /* Add _PAGE_PRESENT to access */
72 ori r4,r4,_PAGE_PRESENT
74 /* Save non-volatile registers.
75 * r31 will hold "old PTE"
76 * r30 is "new PTE"
77 * r29 is "va"
78 * r28 is a hash value
79 * r27 is hashtab mask (maybe dynamic patched instead ?)
80 */
81 std r27,STK_REG(r27)(r1)
82 std r28,STK_REG(r28)(r1)
83 std r29,STK_REG(r29)(r1)
84 std r30,STK_REG(r30)(r1)
85 std r31,STK_REG(r31)(r1)
87 /* Step 1:
88 *
89 * Check permissions, atomically mark the linux PTE busy
90 * and hashed.
91 */
92 1:
93 ldarx r31,0,r6
94 /* Check access rights (access & ~(pte_val(*ptep))) */
95 andc. r0,r4,r31
96 bne- htab_wrong_access
97 /* Check if PTE is busy */
98 andi. r0,r31,_PAGE_BUSY
99 /* If so, just bail out and refault if needed. Someone else
100 * is changing this PTE anyway and might hash it.
101 */
102 bne- htab_bail_ok
104 /* Prepare new PTE value (turn access RW into DIRTY, then
105 * add BUSY,HASHPTE and ACCESSED)
106 */
107 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
108 or r30,r30,r31
109 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
110 /* Write the linux PTE atomically (setting busy) */
111 stdcx. r30,0,r6
112 bne- 1b
113 isync
115 /* Step 2:
116 *
117 * Insert/Update the HPTE in the hash table. At this point,
118 * r4 (access) is re-useable, we use it for the new HPTE flags
119 */
121 BEGIN_FTR_SECTION
122 cmpdi r9,0 /* check segment size */
123 bne 3f
124 END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
125 /* Calc va and put it in r29 */
126 rldicr r29,r5,28,63-28
127 rldicl r3,r3,0,36
128 or r29,r3,r29
130 /* Calculate hash value for primary slot and store it in r28 */
131 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
132 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */
133 xor r28,r5,r0
134 b 4f
136 3: /* Calc VA and hash in r29 and r28 for 1T segment */
137 sldi r29,r5,40 /* vsid << 40 */
138 clrldi r3,r3,24 /* ea & 0xffffffffff */
139 rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */
140 clrldi r5,r5,40 /* vsid & 0xffffff */
141 rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */
142 xor r28,r28,r5
143 or r29,r3,r29 /* VA */
144 xor r28,r28,r0 /* hash */
146 /* Convert linux PTE bits into HW equivalents */
147 4: andi. r3,r30,0x1fe /* Get basic set of flags */
148 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
149 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
150 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
151 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
152 andc r0,r30,r0 /* r0 = pte & ~r0 */
153 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
154 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */
156 /* We eventually do the icache sync here (maybe inline that
157 * code rather than call a C function...)
158 */
159 BEGIN_FTR_SECTION
160 mr r4,r30
161 mr r5,r7
162 bl .hash_page_do_lazy_icache
163 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
165 /* At this point, r3 contains new PP bits, save them in
166 * place of "access" in the param area (sic)
167 */
168 std r3,STK_PARM(r4)(r1)
170 /* Get htab_hash_mask */
171 ld r4,htab_hash_mask@got(2)
172 ld r27,0(r4) /* htab_hash_mask -> r27 */
174 /* Check if we may already be in the hashtable, in this case, we
175 * go to out-of-line code to try to modify the HPTE
176 */
177 andi. r0,r31,_PAGE_HASHPTE
178 bne htab_modify_pte
180 htab_insert_pte:
181 /* Clear hpte bits in new pte (we also clear BUSY btw) and
182 * add _PAGE_HASHPTE
183 */
184 lis r0,_PAGE_HPTEFLAGS@h
185 ori r0,r0,_PAGE_HPTEFLAGS@l
186 andc r30,r30,r0
187 ori r30,r30,_PAGE_HASHPTE
189 /* physical address r5 */
190 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
191 sldi r5,r5,PAGE_SHIFT
193 /* Calculate primary group hash */
194 and r0,r28,r27
195 rldicr r3,r0,3,63-3 /* r3 = (hash & mask) << 3 */
197 /* Call ppc_md.hpte_insert */
198 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
199 mr r4,r29 /* Retreive va */
200 li r7,0 /* !bolted, !secondary */
201 li r8,MMU_PAGE_4K /* page size */
202 ld r9,STK_PARM(r9)(r1) /* segment size */
203 _GLOBAL(htab_call_hpte_insert1)
204 bl . /* Patched by htab_finish_init() */
205 cmpdi 0,r3,0
206 bge htab_pte_insert_ok /* Insertion successful */
207 cmpdi 0,r3,-2 /* Critical failure */
208 beq- htab_pte_insert_failure
210 /* Now try secondary slot */
212 /* physical address r5 */
213 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
214 sldi r5,r5,PAGE_SHIFT
216 /* Calculate secondary group hash */
217 andc r0,r27,r28
218 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
220 /* Call ppc_md.hpte_insert */
221 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
222 mr r4,r29 /* Retreive va */
223 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
224 li r8,MMU_PAGE_4K /* page size */
225 ld r9,STK_PARM(r9)(r1) /* segment size */
226 _GLOBAL(htab_call_hpte_insert2)
227 bl . /* Patched by htab_finish_init() */
228 cmpdi 0,r3,0
229 bge+ htab_pte_insert_ok /* Insertion successful */
230 cmpdi 0,r3,-2 /* Critical failure */
231 beq- htab_pte_insert_failure
233 /* Both are full, we need to evict something */
234 mftb r0
235 /* Pick a random group based on TB */
236 andi. r0,r0,1
237 mr r5,r28
238 bne 2f
239 not r5,r5
240 2: and r0,r5,r27
241 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
242 /* Call ppc_md.hpte_remove */
243 _GLOBAL(htab_call_hpte_remove)
244 bl . /* Patched by htab_finish_init() */
246 /* Try all again */
247 b htab_insert_pte
249 htab_bail_ok:
250 li r3,0
251 b htab_bail
253 htab_pte_insert_ok:
254 /* Insert slot number & secondary bit in PTE */
255 rldimi r30,r3,12,63-15
257 /* Write out the PTE with a normal write
258 * (maybe add eieio may be good still ?)
259 */
260 htab_write_out_pte:
261 ld r6,STK_PARM(r6)(r1)
262 std r30,0(r6)
263 li r3, 0
264 htab_bail:
265 ld r27,STK_REG(r27)(r1)
266 ld r28,STK_REG(r28)(r1)
267 ld r29,STK_REG(r29)(r1)
268 ld r30,STK_REG(r30)(r1)
269 ld r31,STK_REG(r31)(r1)
270 addi r1,r1,STACKFRAMESIZE
271 ld r0,16(r1)
272 mtlr r0
273 blr
275 htab_modify_pte:
276 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
277 mr r4,r3
278 rlwinm r3,r31,32-12,29,31
280 /* Secondary group ? if yes, get a inverted hash value */
281 mr r5,r28
282 andi. r0,r31,_PAGE_SECONDARY
283 beq 1f
284 not r5,r5
285 1:
286 /* Calculate proper slot value for ppc_md.hpte_updatepp */
287 and r0,r5,r27
288 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
289 add r3,r0,r3 /* add slot idx */
291 /* Call ppc_md.hpte_updatepp */
292 mr r5,r29 /* va */
293 li r6,MMU_PAGE_4K /* page size */
294 ld r7,STK_PARM(r9)(r1) /* segment size */
295 ld r8,STK_PARM(r8)(r1) /* get "local" param */
296 _GLOBAL(htab_call_hpte_updatepp)
297 bl . /* Patched by htab_finish_init() */
299 /* if we failed because typically the HPTE wasn't really here
300 * we try an insertion.
301 */
302 cmpdi 0,r3,-1
303 beq- htab_insert_pte
305 /* Clear the BUSY bit and Write out the PTE */
306 li r0,_PAGE_BUSY
307 andc r30,r30,r0
308 b htab_write_out_pte
310 htab_wrong_access:
311 /* Bail out clearing reservation */
312 stdcx. r31,0,r6
313 li r3,1
314 b htab_bail
316 htab_pte_insert_failure:
317 /* Bail out restoring old PTE */
318 ld r6,STK_PARM(r6)(r1)
319 std r31,0(r6)
320 li r3,-1
321 b htab_bail
324 #else /* CONFIG_PPC_64K_PAGES */
327 /*****************************************************************************
328 * *
329 * 64K SW & 4K or 64K HW in a 4K segment pages implementation *
330 * *
331 *****************************************************************************/
333 /* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
334 * pte_t *ptep, unsigned long trap, int local, int ssize,
335 * int subpg_prot)
336 */
338 /*
339 * For now, we do NOT implement Admixed pages
340 */
341 _GLOBAL(__hash_page_4K)
342 mflr r0
343 std r0,16(r1)
344 stdu r1,-STACKFRAMESIZE(r1)
345 /* Save all params that we need after a function call */
346 std r6,STK_PARM(r6)(r1)
347 std r8,STK_PARM(r8)(r1)
348 std r9,STK_PARM(r9)(r1)
350 /* Add _PAGE_PRESENT to access */
351 ori r4,r4,_PAGE_PRESENT
353 /* Save non-volatile registers.
354 * r31 will hold "old PTE"
355 * r30 is "new PTE"
356 * r29 is "va"
357 * r28 is a hash value
358 * r27 is hashtab mask (maybe dynamic patched instead ?)
359 * r26 is the hidx mask
360 * r25 is the index in combo page
361 */
362 std r25,STK_REG(r25)(r1)
363 std r26,STK_REG(r26)(r1)
364 std r27,STK_REG(r27)(r1)
365 std r28,STK_REG(r28)(r1)
366 std r29,STK_REG(r29)(r1)
367 std r30,STK_REG(r30)(r1)
368 std r31,STK_REG(r31)(r1)
370 /* Step 1:
371 *
372 * Check permissions, atomically mark the linux PTE busy
373 * and hashed.
374 */
375 1:
376 ldarx r31,0,r6
377 /* Check access rights (access & ~(pte_val(*ptep))) */
378 andc. r0,r4,r31
379 bne- htab_wrong_access
380 /* Check if PTE is busy */
381 andi. r0,r31,_PAGE_BUSY
382 /* If so, just bail out and refault if needed. Someone else
383 * is changing this PTE anyway and might hash it.
384 */
385 bne- htab_bail_ok
386 /* Prepare new PTE value (turn access RW into DIRTY, then
387 * add BUSY and ACCESSED)
388 */
389 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
390 or r30,r30,r31
391 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
392 oris r30,r30,_PAGE_COMBO@h
393 /* Write the linux PTE atomically (setting busy) */
394 stdcx. r30,0,r6
395 bne- 1b
396 isync
398 /* Step 2:
399 *
400 * Insert/Update the HPTE in the hash table. At this point,
401 * r4 (access) is re-useable, we use it for the new HPTE flags
402 */
404 /* Load the hidx index */
405 rldicl r25,r3,64-12,60
407 BEGIN_FTR_SECTION
408 cmpdi r9,0 /* check segment size */
409 bne 3f
410 END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
411 /* Calc va and put it in r29 */
412 rldicr r29,r5,28,63-28 /* r29 = (vsid << 28) */
413 rldicl r3,r3,0,36 /* r3 = (ea & 0x0fffffff) */
414 or r29,r3,r29 /* r29 = va */
416 /* Calculate hash value for primary slot and store it in r28 */
417 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
418 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */
419 xor r28,r5,r0
420 b 4f
422 3: /* Calc VA and hash in r29 and r28 for 1T segment */
423 sldi r29,r5,40 /* vsid << 40 */
424 clrldi r3,r3,24 /* ea & 0xffffffffff */
425 rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */
426 clrldi r5,r5,40 /* vsid & 0xffffff */
427 rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */
428 xor r28,r28,r5
429 or r29,r3,r29 /* VA */
430 xor r28,r28,r0 /* hash */
432 /* Convert linux PTE bits into HW equivalents */
433 4:
434 #ifdef CONFIG_PPC_SUBPAGE_PROT
435 andc r10,r30,r10
436 andi. r3,r10,0x1fe /* Get basic set of flags */
437 rlwinm r0,r10,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
438 #else
439 andi. r3,r30,0x1fe /* Get basic set of flags */
440 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
441 #endif
442 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
443 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
444 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
445 andc r0,r3,r0 /* r0 = pte & ~r0 */
446 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
447 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */
449 /* We eventually do the icache sync here (maybe inline that
450 * code rather than call a C function...)
451 */
452 BEGIN_FTR_SECTION
453 mr r4,r30
454 mr r5,r7
455 bl .hash_page_do_lazy_icache
456 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
458 /* At this point, r3 contains new PP bits, save them in
459 * place of "access" in the param area (sic)
460 */
461 std r3,STK_PARM(r4)(r1)
463 /* Get htab_hash_mask */
464 ld r4,htab_hash_mask@got(2)
465 ld r27,0(r4) /* htab_hash_mask -> r27 */
467 /* Check if we may already be in the hashtable, in this case, we
468 * go to out-of-line code to try to modify the HPTE. We look for
469 * the bit at (1 >> (index + 32))
470 */
471 rldicl. r0,r31,64-12,48
472 li r26,0 /* Default hidx */
473 beq htab_insert_pte
475 /*
476 * Check if the pte was already inserted into the hash table
477 * as a 64k HW page, and invalidate the 64k HPTE if so.
478 */
479 andis. r0,r31,_PAGE_COMBO@h
480 beq htab_inval_old_hpte
482 ld r6,STK_PARM(r6)(r1)
483 ori r26,r6,0x8000 /* Load the hidx mask */
484 ld r26,0(r26)
485 addi r5,r25,36 /* Check actual HPTE_SUB bit, this */
486 rldcr. r0,r31,r5,0 /* must match pgtable.h definition */
487 bne htab_modify_pte
489 htab_insert_pte:
490 /* real page number in r5, PTE RPN value + index */
491 andis. r0,r31,_PAGE_4K_PFN@h
492 srdi r5,r31,PTE_RPN_SHIFT
493 bne- htab_special_pfn
494 sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
495 add r5,r5,r25
496 htab_special_pfn:
497 sldi r5,r5,HW_PAGE_SHIFT
499 /* Calculate primary group hash */
500 and r0,r28,r27
501 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
503 /* Call ppc_md.hpte_insert */
504 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
505 mr r4,r29 /* Retreive va */
506 li r7,0 /* !bolted, !secondary */
507 li r8,MMU_PAGE_4K /* page size */
508 ld r9,STK_PARM(r9)(r1) /* segment size */
509 _GLOBAL(htab_call_hpte_insert1)
510 bl . /* patched by htab_finish_init() */
511 cmpdi 0,r3,0
512 bge htab_pte_insert_ok /* Insertion successful */
513 cmpdi 0,r3,-2 /* Critical failure */
514 beq- htab_pte_insert_failure
516 /* Now try secondary slot */
518 /* real page number in r5, PTE RPN value + index */
519 andis. r0,r31,_PAGE_4K_PFN@h
520 srdi r5,r31,PTE_RPN_SHIFT
521 bne- 3f
522 sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
523 add r5,r5,r25
524 3: sldi r5,r5,HW_PAGE_SHIFT
526 /* Calculate secondary group hash */
527 andc r0,r27,r28
528 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
530 /* Call ppc_md.hpte_insert */
531 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
532 mr r4,r29 /* Retreive va */
533 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
534 li r8,MMU_PAGE_4K /* page size */
535 ld r9,STK_PARM(r9)(r1) /* segment size */
536 _GLOBAL(htab_call_hpte_insert2)
537 bl . /* patched by htab_finish_init() */
538 cmpdi 0,r3,0
539 bge+ htab_pte_insert_ok /* Insertion successful */
540 cmpdi 0,r3,-2 /* Critical failure */
541 beq- htab_pte_insert_failure
543 /* Both are full, we need to evict something */
544 mftb r0
545 /* Pick a random group based on TB */
546 andi. r0,r0,1
547 mr r5,r28
548 bne 2f
549 not r5,r5
550 2: and r0,r5,r27
551 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
552 /* Call ppc_md.hpte_remove */
553 _GLOBAL(htab_call_hpte_remove)
554 bl . /* patched by htab_finish_init() */
556 /* Try all again */
557 b htab_insert_pte
559 /*
560 * Call out to C code to invalidate an 64k HW HPTE that is
561 * useless now that the segment has been switched to 4k pages.
562 */
563 htab_inval_old_hpte:
564 mr r3,r29 /* virtual addr */
565 mr r4,r31 /* PTE.pte */
566 li r5,0 /* PTE.hidx */
567 li r6,MMU_PAGE_64K /* psize */
568 ld r7,STK_PARM(r9)(r1) /* ssize */
569 ld r8,STK_PARM(r8)(r1) /* local */
570 bl .flush_hash_page
571 /* Clear out _PAGE_HPTE_SUB bits in the new linux PTE */
572 lis r0,_PAGE_HPTE_SUB@h
573 ori r0,r0,_PAGE_HPTE_SUB@l
574 andc r30,r30,r0
575 b htab_insert_pte
577 htab_bail_ok:
578 li r3,0
579 b htab_bail
581 htab_pte_insert_ok:
582 /* Insert slot number & secondary bit in PTE second half,
583 * clear _PAGE_BUSY and set approriate HPTE slot bit
584 */
585 ld r6,STK_PARM(r6)(r1)
586 li r0,_PAGE_BUSY
587 andc r30,r30,r0
588 /* HPTE SUB bit */
589 li r0,1
590 subfic r5,r25,27 /* Must match bit position in */
591 sld r0,r0,r5 /* pgtable.h */
592 or r30,r30,r0
593 /* hindx */
594 sldi r5,r25,2
595 sld r3,r3,r5
596 li r4,0xf
597 sld r4,r4,r5
598 andc r26,r26,r4
599 or r26,r26,r3
600 ori r5,r6,0x8000
601 std r26,0(r5)
602 lwsync
603 std r30,0(r6)
604 li r3, 0
605 htab_bail:
606 ld r25,STK_REG(r25)(r1)
607 ld r26,STK_REG(r26)(r1)
608 ld r27,STK_REG(r27)(r1)
609 ld r28,STK_REG(r28)(r1)
610 ld r29,STK_REG(r29)(r1)
611 ld r30,STK_REG(r30)(r1)
612 ld r31,STK_REG(r31)(r1)
613 addi r1,r1,STACKFRAMESIZE
614 ld r0,16(r1)
615 mtlr r0
616 blr
618 htab_modify_pte:
619 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
620 mr r4,r3
621 sldi r5,r25,2
622 srd r3,r26,r5
624 /* Secondary group ? if yes, get a inverted hash value */
625 mr r5,r28
626 andi. r0,r3,0x8 /* page secondary ? */
627 beq 1f
628 not r5,r5
629 1: andi. r3,r3,0x7 /* extract idx alone */
631 /* Calculate proper slot value for ppc_md.hpte_updatepp */
632 and r0,r5,r27
633 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
634 add r3,r0,r3 /* add slot idx */
636 /* Call ppc_md.hpte_updatepp */
637 mr r5,r29 /* va */
638 li r6,MMU_PAGE_4K /* page size */
639 ld r7,STK_PARM(r9)(r1) /* segment size */
640 ld r8,STK_PARM(r8)(r1) /* get "local" param */
641 _GLOBAL(htab_call_hpte_updatepp)
642 bl . /* patched by htab_finish_init() */
644 /* if we failed because typically the HPTE wasn't really here
645 * we try an insertion.
646 */
647 cmpdi 0,r3,-1
648 beq- htab_insert_pte
650 /* Clear the BUSY bit and Write out the PTE */
651 li r0,_PAGE_BUSY
652 andc r30,r30,r0
653 ld r6,STK_PARM(r6)(r1)
654 std r30,0(r6)
655 li r3,0
656 b htab_bail
658 htab_wrong_access:
659 /* Bail out clearing reservation */
660 stdcx. r31,0,r6
661 li r3,1
662 b htab_bail
664 htab_pte_insert_failure:
665 /* Bail out restoring old PTE */
666 ld r6,STK_PARM(r6)(r1)
667 std r31,0(r6)
668 li r3,-1
669 b htab_bail
671 #endif /* CONFIG_PPC_64K_PAGES */
673 #ifdef CONFIG_PPC_HAS_HASH_64K
675 /*****************************************************************************
676 * *
677 * 64K SW & 64K HW in a 64K segment pages implementation *
678 * *
679 *****************************************************************************/
681 _GLOBAL(__hash_page_64K)
682 mflr r0
683 std r0,16(r1)
684 stdu r1,-STACKFRAMESIZE(r1)
685 /* Save all params that we need after a function call */
686 std r6,STK_PARM(r6)(r1)
687 std r8,STK_PARM(r8)(r1)
688 std r9,STK_PARM(r9)(r1)
690 /* Add _PAGE_PRESENT to access */
691 ori r4,r4,_PAGE_PRESENT
693 /* Save non-volatile registers.
694 * r31 will hold "old PTE"
695 * r30 is "new PTE"
696 * r29 is "va"
697 * r28 is a hash value
698 * r27 is hashtab mask (maybe dynamic patched instead ?)
699 */
700 std r27,STK_REG(r27)(r1)
701 std r28,STK_REG(r28)(r1)
702 std r29,STK_REG(r29)(r1)
703 std r30,STK_REG(r30)(r1)
704 std r31,STK_REG(r31)(r1)
706 /* Step 1:
707 *
708 * Check permissions, atomically mark the linux PTE busy
709 * and hashed.
710 */
711 1:
712 ldarx r31,0,r6
713 /* Check access rights (access & ~(pte_val(*ptep))) */
714 andc. r0,r4,r31
715 bne- ht64_wrong_access
716 /* Check if PTE is busy */
717 andi. r0,r31,_PAGE_BUSY
718 /* If so, just bail out and refault if needed. Someone else
719 * is changing this PTE anyway and might hash it.
720 */
721 bne- ht64_bail_ok
722 BEGIN_FTR_SECTION
723 /* Check if PTE has the cache-inhibit bit set */
724 andi. r0,r31,_PAGE_NO_CACHE
725 /* If so, bail out and refault as a 4k page */
726 bne- ht64_bail_ok
727 END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE)
728 /* Prepare new PTE value (turn access RW into DIRTY, then
729 * add BUSY and ACCESSED)
730 */
731 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
732 or r30,r30,r31
733 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
734 /* Write the linux PTE atomically (setting busy) */
735 stdcx. r30,0,r6
736 bne- 1b
737 isync
739 /* Step 2:
740 *
741 * Insert/Update the HPTE in the hash table. At this point,
742 * r4 (access) is re-useable, we use it for the new HPTE flags
743 */
745 BEGIN_FTR_SECTION
746 cmpdi r9,0 /* check segment size */
747 bne 3f
748 END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
749 /* Calc va and put it in r29 */
750 rldicr r29,r5,28,63-28
751 rldicl r3,r3,0,36
752 or r29,r3,r29
754 /* Calculate hash value for primary slot and store it in r28 */
755 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
756 rldicl r0,r3,64-16,52 /* (ea >> 16) & 0xfff */
757 xor r28,r5,r0
758 b 4f
760 3: /* Calc VA and hash in r29 and r28 for 1T segment */
761 sldi r29,r5,40 /* vsid << 40 */
762 clrldi r3,r3,24 /* ea & 0xffffffffff */
763 rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */
764 clrldi r5,r5,40 /* vsid & 0xffffff */
765 rldicl r0,r3,64-16,40 /* (ea >> 16) & 0xffffff */
766 xor r28,r28,r5
767 or r29,r3,r29 /* VA */
768 xor r28,r28,r0 /* hash */
770 /* Convert linux PTE bits into HW equivalents */
771 4: andi. r3,r30,0x1fe /* Get basic set of flags */
772 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
773 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
774 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
775 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
776 andc r0,r30,r0 /* r0 = pte & ~r0 */
777 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
778 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */
780 /* We eventually do the icache sync here (maybe inline that
781 * code rather than call a C function...)
782 */
783 BEGIN_FTR_SECTION
784 mr r4,r30
785 mr r5,r7
786 bl .hash_page_do_lazy_icache
787 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
789 /* At this point, r3 contains new PP bits, save them in
790 * place of "access" in the param area (sic)
791 */
792 std r3,STK_PARM(r4)(r1)
794 /* Get htab_hash_mask */
795 ld r4,htab_hash_mask@got(2)
796 ld r27,0(r4) /* htab_hash_mask -> r27 */
798 /* Check if we may already be in the hashtable, in this case, we
799 * go to out-of-line code to try to modify the HPTE
800 */
801 rldicl. r0,r31,64-12,48
802 bne ht64_modify_pte
804 ht64_insert_pte:
805 /* Clear hpte bits in new pte (we also clear BUSY btw) and
806 * add _PAGE_HPTE_SUB0
807 */
808 lis r0,_PAGE_HPTEFLAGS@h
809 ori r0,r0,_PAGE_HPTEFLAGS@l
810 andc r30,r30,r0
811 #ifdef CONFIG_PPC_64K_PAGES
812 oris r30,r30,_PAGE_HPTE_SUB0@h
813 #else
814 ori r30,r30,_PAGE_HASHPTE
815 #endif
816 /* Phyical address in r5 */
817 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
818 sldi r5,r5,PAGE_SHIFT
820 /* Calculate primary group hash */
821 and r0,r28,r27
822 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
824 /* Call ppc_md.hpte_insert */
825 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
826 mr r4,r29 /* Retreive va */
827 li r7,0 /* !bolted, !secondary */
828 li r8,MMU_PAGE_64K
829 ld r9,STK_PARM(r9)(r1) /* segment size */
830 _GLOBAL(ht64_call_hpte_insert1)
831 bl . /* patched by htab_finish_init() */
832 cmpdi 0,r3,0
833 bge ht64_pte_insert_ok /* Insertion successful */
834 cmpdi 0,r3,-2 /* Critical failure */
835 beq- ht64_pte_insert_failure
837 /* Now try secondary slot */
839 /* Phyical address in r5 */
840 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
841 sldi r5,r5,PAGE_SHIFT
843 /* Calculate secondary group hash */
844 andc r0,r27,r28
845 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
847 /* Call ppc_md.hpte_insert */
848 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
849 mr r4,r29 /* Retreive va */
850 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
851 li r8,MMU_PAGE_64K
852 ld r9,STK_PARM(r9)(r1) /* segment size */
853 _GLOBAL(ht64_call_hpte_insert2)
854 bl . /* patched by htab_finish_init() */
855 cmpdi 0,r3,0
856 bge+ ht64_pte_insert_ok /* Insertion successful */
857 cmpdi 0,r3,-2 /* Critical failure */
858 beq- ht64_pte_insert_failure
860 /* Both are full, we need to evict something */
861 mftb r0
862 /* Pick a random group based on TB */
863 andi. r0,r0,1
864 mr r5,r28
865 bne 2f
866 not r5,r5
867 2: and r0,r5,r27
868 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
869 /* Call ppc_md.hpte_remove */
870 _GLOBAL(ht64_call_hpte_remove)
871 bl . /* patched by htab_finish_init() */
873 /* Try all again */
874 b ht64_insert_pte
876 ht64_bail_ok:
877 li r3,0
878 b ht64_bail
880 ht64_pte_insert_ok:
881 /* Insert slot number & secondary bit in PTE */
882 rldimi r30,r3,12,63-15
884 /* Write out the PTE with a normal write
885 * (maybe add eieio may be good still ?)
886 */
887 ht64_write_out_pte:
888 ld r6,STK_PARM(r6)(r1)
889 std r30,0(r6)
890 li r3, 0
891 ht64_bail:
892 ld r27,STK_REG(r27)(r1)
893 ld r28,STK_REG(r28)(r1)
894 ld r29,STK_REG(r29)(r1)
895 ld r30,STK_REG(r30)(r1)
896 ld r31,STK_REG(r31)(r1)
897 addi r1,r1,STACKFRAMESIZE
898 ld r0,16(r1)
899 mtlr r0
900 blr
902 ht64_modify_pte:
903 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
904 mr r4,r3
905 rlwinm r3,r31,32-12,29,31
907 /* Secondary group ? if yes, get a inverted hash value */
908 mr r5,r28
909 andi. r0,r31,_PAGE_F_SECOND
910 beq 1f
911 not r5,r5
912 1:
913 /* Calculate proper slot value for ppc_md.hpte_updatepp */
914 and r0,r5,r27
915 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
916 add r3,r0,r3 /* add slot idx */
918 /* Call ppc_md.hpte_updatepp */
919 mr r5,r29 /* va */
920 li r6,MMU_PAGE_64K
921 ld r7,STK_PARM(r9)(r1) /* segment size */
922 ld r8,STK_PARM(r8)(r1) /* get "local" param */
923 _GLOBAL(ht64_call_hpte_updatepp)
924 bl . /* patched by htab_finish_init() */
926 /* if we failed because typically the HPTE wasn't really here
927 * we try an insertion.
928 */
929 cmpdi 0,r3,-1
930 beq- ht64_insert_pte
932 /* Clear the BUSY bit and Write out the PTE */
933 li r0,_PAGE_BUSY
934 andc r30,r30,r0
935 b ht64_write_out_pte
937 ht64_wrong_access:
938 /* Bail out clearing reservation */
939 stdcx. r31,0,r6
940 li r3,1
941 b ht64_bail
943 ht64_pte_insert_failure:
944 /* Bail out restoring old PTE */
945 ld r6,STK_PARM(r6)(r1)
946 std r31,0(r6)
947 li r3,-1
948 b ht64_bail
951 #endif /* CONFIG_PPC_HAS_HASH_64K */
954 /*****************************************************************************
955 * *
956 * Huge pages implementation is in hugetlbpage.c *
957 * *
958 *****************************************************************************/