arch/powerpc/mm/hash_low_64.S

   1 /*
   2  * ppc64 MMU hashtable management routines
   3  *
   4  * (c) Copyright IBM Corp. 2003, 2005
   5  *
   6  * Maintained by: Benjamin Herrenschmidt
   7  *                <benh@kernel.crashing.org>
   8  *
   9  * This file is covered by the GNU Public Licence v2 as
  10  * described in the kernel's COPYING file.
  11  */
  12
  13 #include <asm/reg.h>
  14 #include <asm/pgtable.h>
  15 #include <asm/mmu.h>
  16 #include <asm/page.h>
  17 #include <asm/types.h>
  18 #include <asm/ppc_asm.h>
  19 #include <asm/asm-offsets.h>
  20 #include <asm/cputable.h>
  21
  22         .text
  23
  24 /*
  25  * Stackframe:
  26  *
  27  *         +-> Back chain                       (SP + 256)
  28  *         |   General register save area       (SP + 112)
  29  *         |   Parameter save area              (SP + 48)
  30  *         |   TOC save area                    (SP + 40)
  31  *         |   link editor doubleword           (SP + 32)
  32  *         |   compiler doubleword              (SP + 24)
  33  *         |   LR save area                     (SP + 16)
  34  *         |   CR save area                     (SP + 8)
  35  * SP ---> +-- Back chain                       (SP + 0)
  36  */
  37 #define STACKFRAMESIZE  256
  38
  39 /* Save parameters offsets */
  40 #define STK_PARM(i)     (STACKFRAMESIZE + 48 + ((i)-3)*8)
  41
  42 /* Save non-volatile offsets */
  43 #define STK_REG(i)      (112 + ((i)-14)*8)
  44
  45
  46 #ifndef CONFIG_PPC_64K_PAGES
  47
  48 /*****************************************************************************
  49  *                                                                           *
  50  *           4K SW & 4K HW pages implementation                              *
  51  *                                                                           *
  52  *****************************************************************************/
  53
  54
  55 /*
  56  * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
  57  *               pte_t *ptep, unsigned long trap, int local, int ssize)
  58  *
  59  * Adds a 4K page to the hash table in a segment of 4K pages only
  60  */
  61
  62 _GLOBAL(__hash_page_4K)
  63         mflr    r0
  64         std     r0,16(r1)
  65         stdu    r1,-STACKFRAMESIZE(r1)
  66         /* Save all params that we need after a function call */
  67         std     r6,STK_PARM(r6)(r1)
  68         std     r8,STK_PARM(r8)(r1)
  69         std     r9,STK_PARM(r9)(r1)
  70
  71         /* Add _PAGE_PRESENT to access */
  72         ori     r4,r4,_PAGE_PRESENT
  73
  74         /* Save non-volatile registers.
  75          * r31 will hold "old PTE"
  76          * r30 is "new PTE"
  77          * r29 is "va"
  78          * r28 is a hash value
  79          * r27 is hashtab mask (maybe dynamic patched instead ?)
  80          */
  81         std     r27,STK_REG(r27)(r1)
  82         std     r28,STK_REG(r28)(r1)
  83         std     r29,STK_REG(r29)(r1)
  84         std     r30,STK_REG(r30)(r1)
  85         std     r31,STK_REG(r31)(r1)
  86
  87         /* Step 1:
  88          *
  89          * Check permissions, atomically mark the linux PTE busy
  90          * and hashed.
  91          */
  92 1:
  93         ldarx   r31,0,r6
  94         /* Check access rights (access & ~(pte_val(*ptep))) */
  95         andc.   r0,r4,r31
  96         bne-    htab_wrong_access
  97         /* Check if PTE is busy */
  98         andi.   r0,r31,_PAGE_BUSY
  99         /* If so, just bail out and refault if needed. Someone else
 100          * is changing this PTE anyway and might hash it.
 101          */
 102         bne-    htab_bail_ok
 103
 104         /* Prepare new PTE value (turn access RW into DIRTY, then
 105          * add BUSY,HASHPTE and ACCESSED)
 106          */
 107         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
 108         or      r30,r30,r31
 109         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
 110         /* Write the linux PTE atomically (setting busy) */
 111         stdcx.  r30,0,r6
 112         bne-    1b
 113         isync
 114
 115         /* Step 2:
 116          *
 117          * Insert/Update the HPTE in the hash table. At this point,
 118          * r4 (access) is re-useable, we use it for the new HPTE flags
 119          */
 120
 121 BEGIN_FTR_SECTION
 122         cmpdi   r9,0                    /* check segment size */
 123         bne     3f
 124 END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
 125         /* Calc va and put it in r29 */
 126         rldicr  r29,r5,28,63-28
 127         rldicl  r3,r3,0,36
 128         or      r29,r3,r29
 129
 130         /* Calculate hash value for primary slot and store it in r28 */
 131         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
 132         rldicl  r0,r3,64-12,48          /* (ea >> 12) & 0xffff */
 133         xor     r28,r5,r0
 134         b       4f
 135
 136 3:      /* Calc VA and hash in r29 and r28 for 1T segment */
 137         sldi    r29,r5,40               /* vsid << 40 */
 138         clrldi  r3,r3,24                /* ea & 0xffffffffff */
 139         rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
 140         clrldi  r5,r5,40                /* vsid & 0xffffff */
 141         rldicl  r0,r3,64-12,36          /* (ea >> 12) & 0xfffffff */
 142         xor     r28,r28,r5
 143         or      r29,r3,r29              /* VA */
 144         xor     r28,r28,r0              /* hash */
 145
 146         /* Convert linux PTE bits into HW equivalents */
 147 4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
 148         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 149         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 150         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 151         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 152         andc    r0,r30,r0               /* r0 = pte & ~r0 */
 153         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 154         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
 155
 156         /* We eventually do the icache sync here (maybe inline that
 157          * code rather than call a C function...)
 158          */
 159 BEGIN_FTR_SECTION
 160         mr      r4,r30
 161         mr      r5,r7
 162         bl      .hash_page_do_lazy_icache
 163 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 164
 165         /* At this point, r3 contains new PP bits, save them in
 166          * place of "access" in the param area (sic)
 167          */
 168         std     r3,STK_PARM(r4)(r1)
 169
 170         /* Get htab_hash_mask */
 171         ld      r4,htab_hash_mask@got(2)
 172         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 173
 174         /* Check if we may already be in the hashtable, in this case, we
 175          * go to out-of-line code to try to modify the HPTE
 176          */
 177         andi.   r0,r31,_PAGE_HASHPTE
 178         bne     htab_modify_pte
 179
 180 htab_insert_pte:
 181         /* Clear hpte bits in new pte (we also clear BUSY btw) and
 182          * add _PAGE_HASHPTE
 183          */
 184         lis     r0,_PAGE_HPTEFLAGS@h
 185         ori     r0,r0,_PAGE_HPTEFLAGS@l
 186         andc    r30,r30,r0
 187         ori     r30,r30,_PAGE_HASHPTE
 188
 189         /* physical address r5 */
 190         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 191         sldi    r5,r5,PAGE_SHIFT
 192
 193         /* Calculate primary group hash */
 194         and     r0,r28,r27
 195         rldicr  r3,r0,3,63-3            /* r3 = (hash & mask) << 3 */
 196
 197         /* Call ppc_md.hpte_insert */
 198         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
 199         mr      r4,r29                  /* Retreive va */
 200         li      r7,0                    /* !bolted, !secondary */
 201         li      r8,MMU_PAGE_4K          /* page size */
 202         ld      r9,STK_PARM(r9)(r1)     /* segment size */
 203 _GLOBAL(htab_call_hpte_insert1)
 204         bl      .                       /* Patched by htab_finish_init() */
 205         cmpdi   0,r3,0
 206         bge     htab_pte_insert_ok      /* Insertion successful */
 207         cmpdi   0,r3,-2                 /* Critical failure */
 208         beq-    htab_pte_insert_failure
 209
 210         /* Now try secondary slot */
 211
 212         /* physical address r5 */
 213         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 214         sldi    r5,r5,PAGE_SHIFT
 215
 216         /* Calculate secondary group hash */
 217         andc    r0,r27,r28
 218         rldicr  r3,r0,3,63-3    /* r0 = (~hash & mask) << 3 */
 219
 220         /* Call ppc_md.hpte_insert */
 221         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
 222         mr      r4,r29                  /* Retreive va */
 223         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 224         li      r8,MMU_PAGE_4K          /* page size */
 225         ld      r9,STK_PARM(r9)(r1)     /* segment size */
 226 _GLOBAL(htab_call_hpte_insert2)
 227         bl      .                       /* Patched by htab_finish_init() */
 228         cmpdi   0,r3,0
 229         bge+    htab_pte_insert_ok      /* Insertion successful */
 230         cmpdi   0,r3,-2                 /* Critical failure */
 231         beq-    htab_pte_insert_failure
 232
 233         /* Both are full, we need to evict something */
 234         mftb    r0
 235         /* Pick a random group based on TB */
 236         andi.   r0,r0,1
 237         mr      r5,r28
 238         bne     2f
 239         not     r5,r5
 240 2:      and     r0,r5,r27
 241         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 242         /* Call ppc_md.hpte_remove */
 243 _GLOBAL(htab_call_hpte_remove)
 244         bl      .                       /* Patched by htab_finish_init() */
 245
 246         /* Try all again */
 247         b       htab_insert_pte
 248
 249 htab_bail_ok:
 250         li      r3,0
 251         b       htab_bail
 252
 253 htab_pte_insert_ok:
 254         /* Insert slot number & secondary bit in PTE */
 255         rldimi  r30,r3,12,63-15
 256
 257         /* Write out the PTE with a normal write
 258          * (maybe add eieio may be good still ?)
 259          */
 260 htab_write_out_pte:
 261         ld      r6,STK_PARM(r6)(r1)
 262         std     r30,0(r6)
 263         li      r3, 0
 264 htab_bail:
 265         ld      r27,STK_REG(r27)(r1)
 266         ld      r28,STK_REG(r28)(r1)
 267         ld      r29,STK_REG(r29)(r1)
 268         ld      r30,STK_REG(r30)(r1)
 269         ld      r31,STK_REG(r31)(r1)
 270         addi    r1,r1,STACKFRAMESIZE
 271         ld      r0,16(r1)
 272         mtlr    r0
 273         blr
 274
 275 htab_modify_pte:
 276         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 277         mr      r4,r3
 278         rlwinm  r3,r31,32-12,29,31
 279
 280         /* Secondary group ? if yes, get a inverted hash value */
 281         mr      r5,r28
 282         andi.   r0,r31,_PAGE_SECONDARY
 283         beq     1f
 284         not     r5,r5
 285 1:
 286         /* Calculate proper slot value for ppc_md.hpte_updatepp */
 287         and     r0,r5,r27
 288         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 289         add     r3,r0,r3        /* add slot idx */
 290
 291         /* Call ppc_md.hpte_updatepp */
 292         mr      r5,r29                  /* va */
 293         li      r6,MMU_PAGE_4K          /* page size */
 294         ld      r7,STK_PARM(r9)(r1)     /* segment size */
 295         ld      r8,STK_PARM(r8)(r1)     /* get "local" param */
 296 _GLOBAL(htab_call_hpte_updatepp)
 297         bl      .                       /* Patched by htab_finish_init() */
 298
 299         /* if we failed because typically the HPTE wasn't really here
 300          * we try an insertion.
 301          */
 302         cmpdi   0,r3,-1
 303         beq-    htab_insert_pte
 304
 305         /* Clear the BUSY bit and Write out the PTE */
 306         li      r0,_PAGE_BUSY
 307         andc    r30,r30,r0
 308         b       htab_write_out_pte
 309
 310 htab_wrong_access:
 311         /* Bail out clearing reservation */
 312         stdcx.  r31,0,r6
 313         li      r3,1
 314         b       htab_bail
 315
 316 htab_pte_insert_failure:
 317         /* Bail out restoring old PTE */
 318         ld      r6,STK_PARM(r6)(r1)
 319         std     r31,0(r6)
 320         li      r3,-1
 321         b       htab_bail
 322
 323
 324 #else /* CONFIG_PPC_64K_PAGES */
 325
 326
 327 /*****************************************************************************
 328  *                                                                           *
 329  *           64K SW & 4K or 64K HW in a 4K segment pages implementation      *
 330  *                                                                           *
 331  *****************************************************************************/
 332
 333 /* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
 334  *               pte_t *ptep, unsigned long trap, int local, int ssize,
 335  *               int subpg_prot)
 336  */
 337
 338 /*
 339  * For now, we do NOT implement Admixed pages
 340  */
 341 _GLOBAL(__hash_page_4K)
 342         mflr    r0
 343         std     r0,16(r1)
 344         stdu    r1,-STACKFRAMESIZE(r1)
 345         /* Save all params that we need after a function call */
 346         std     r6,STK_PARM(r6)(r1)
 347         std     r8,STK_PARM(r8)(r1)
 348         std     r9,STK_PARM(r9)(r1)
 349
 350         /* Add _PAGE_PRESENT to access */
 351         ori     r4,r4,_PAGE_PRESENT
 352
 353         /* Save non-volatile registers.
 354          * r31 will hold "old PTE"
 355          * r30 is "new PTE"
 356          * r29 is "va"
 357          * r28 is a hash value
 358          * r27 is hashtab mask (maybe dynamic patched instead ?)
 359          * r26 is the hidx mask
 360          * r25 is the index in combo page
 361          */
 362         std     r25,STK_REG(r25)(r1)
 363         std     r26,STK_REG(r26)(r1)
 364         std     r27,STK_REG(r27)(r1)
 365         std     r28,STK_REG(r28)(r1)
 366         std     r29,STK_REG(r29)(r1)
 367         std     r30,STK_REG(r30)(r1)
 368         std     r31,STK_REG(r31)(r1)
 369
 370         /* Step 1:
 371          *
 372          * Check permissions, atomically mark the linux PTE busy
 373          * and hashed.
 374          */
 375 1:
 376         ldarx   r31,0,r6
 377         /* Check access rights (access & ~(pte_val(*ptep))) */
 378         andc.   r0,r4,r31
 379         bne-    htab_wrong_access
 380         /* Check if PTE is busy */
 381         andi.   r0,r31,_PAGE_BUSY
 382         /* If so, just bail out and refault if needed. Someone else
 383          * is changing this PTE anyway and might hash it.
 384          */
 385         bne-    htab_bail_ok
 386         /* Prepare new PTE value (turn access RW into DIRTY, then
 387          * add BUSY and ACCESSED)
 388          */
 389         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
 390         or      r30,r30,r31
 391         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
 392         oris    r30,r30,_PAGE_COMBO@h
 393         /* Write the linux PTE atomically (setting busy) */
 394         stdcx.  r30,0,r6
 395         bne-    1b
 396         isync
 397
 398         /* Step 2:
 399          *
 400          * Insert/Update the HPTE in the hash table. At this point,
 401          * r4 (access) is re-useable, we use it for the new HPTE flags
 402          */
 403
 404         /* Load the hidx index */
 405         rldicl  r25,r3,64-12,60
 406
 407 BEGIN_FTR_SECTION
 408         cmpdi   r9,0                    /* check segment size */
 409         bne     3f
 410 END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
 411         /* Calc va and put it in r29 */
 412         rldicr  r29,r5,28,63-28         /* r29 = (vsid << 28) */
 413         rldicl  r3,r3,0,36              /* r3 = (ea & 0x0fffffff) */
 414         or      r29,r3,r29              /* r29 = va */
 415
 416         /* Calculate hash value for primary slot and store it in r28 */
 417         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
 418         rldicl  r0,r3,64-12,48          /* (ea >> 12) & 0xffff */
 419         xor     r28,r5,r0
 420         b       4f
 421
 422 3:      /* Calc VA and hash in r29 and r28 for 1T segment */
 423         sldi    r29,r5,40               /* vsid << 40 */
 424         clrldi  r3,r3,24                /* ea & 0xffffffffff */
 425         rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
 426         clrldi  r5,r5,40                /* vsid & 0xffffff */
 427         rldicl  r0,r3,64-12,36          /* (ea >> 12) & 0xfffffff */
 428         xor     r28,r28,r5
 429         or      r29,r3,r29              /* VA */
 430         xor     r28,r28,r0              /* hash */
 431
 432         /* Convert linux PTE bits into HW equivalents */
 433 4:
 434 #ifdef CONFIG_PPC_SUBPAGE_PROT
 435         andc    r10,r30,r10
 436         andi.   r3,r10,0x1fe            /* Get basic set of flags */
 437         rlwinm  r0,r10,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 438 #else
 439         andi.   r3,r30,0x1fe            /* Get basic set of flags */
 440         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 441 #endif
 442         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 443         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 444         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 445         andc    r0,r3,r0                /* r0 = pte & ~r0 */
 446         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 447         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
 448
 449         /* We eventually do the icache sync here (maybe inline that
 450          * code rather than call a C function...)
 451          */
 452 BEGIN_FTR_SECTION
 453         mr      r4,r30
 454         mr      r5,r7
 455         bl      .hash_page_do_lazy_icache
 456 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 457
 458         /* At this point, r3 contains new PP bits, save them in
 459          * place of "access" in the param area (sic)
 460          */
 461         std     r3,STK_PARM(r4)(r1)
 462
 463         /* Get htab_hash_mask */
 464         ld      r4,htab_hash_mask@got(2)
 465         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 466
 467         /* Check if we may already be in the hashtable, in this case, we
 468          * go to out-of-line code to try to modify the HPTE. We look for
 469          * the bit at (1 >> (index + 32))
 470          */
 471         rldicl. r0,r31,64-12,48
 472         li      r26,0                   /* Default hidx */
 473         beq     htab_insert_pte
 474
 475         /*
 476          * Check if the pte was already inserted into the hash table
 477          * as a 64k HW page, and invalidate the 64k HPTE if so.
 478          */
 479         andis.  r0,r31,_PAGE_COMBO@h
 480         beq     htab_inval_old_hpte
 481
 482         ld      r6,STK_PARM(r6)(r1)
 483         ori     r26,r6,0x8000           /* Load the hidx mask */
 484         ld      r26,0(r26)
 485         addi    r5,r25,36               /* Check actual HPTE_SUB bit, this */
 486         rldcr.  r0,r31,r5,0             /* must match pgtable.h definition */
 487         bne     htab_modify_pte
 488
 489 htab_insert_pte:
 490         /* real page number in r5, PTE RPN value + index */
 491         andis.  r0,r31,_PAGE_4K_PFN@h
 492         srdi    r5,r31,PTE_RPN_SHIFT
 493         bne-    htab_special_pfn
 494         sldi    r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
 495         add     r5,r5,r25
 496 htab_special_pfn:
 497         sldi    r5,r5,HW_PAGE_SHIFT
 498
 499         /* Calculate primary group hash */
 500         and     r0,r28,r27
 501         rldicr  r3,r0,3,63-3            /* r0 = (hash & mask) << 3 */
 502
 503         /* Call ppc_md.hpte_insert */
 504         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
 505         mr      r4,r29                  /* Retreive va */
 506         li      r7,0                    /* !bolted, !secondary */
 507         li      r8,MMU_PAGE_4K          /* page size */
 508         ld      r9,STK_PARM(r9)(r1)     /* segment size */
 509 _GLOBAL(htab_call_hpte_insert1)
 510         bl      .                       /* patched by htab_finish_init() */
 511         cmpdi   0,r3,0
 512         bge     htab_pte_insert_ok      /* Insertion successful */
 513         cmpdi   0,r3,-2                 /* Critical failure */
 514         beq-    htab_pte_insert_failure
 515
 516         /* Now try secondary slot */
 517
 518         /* real page number in r5, PTE RPN value + index */
 519         andis.  r0,r31,_PAGE_4K_PFN@h
 520         srdi    r5,r31,PTE_RPN_SHIFT
 521         bne-    3f
 522         sldi    r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
 523         add     r5,r5,r25
 524 3:      sldi    r5,r5,HW_PAGE_SHIFT
 525
 526         /* Calculate secondary group hash */
 527         andc    r0,r27,r28
 528         rldicr  r3,r0,3,63-3            /* r0 = (~hash & mask) << 3 */
 529
 530         /* Call ppc_md.hpte_insert */
 531         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
 532         mr      r4,r29                  /* Retreive va */
 533         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 534         li      r8,MMU_PAGE_4K          /* page size */
 535         ld      r9,STK_PARM(r9)(r1)     /* segment size */
 536 _GLOBAL(htab_call_hpte_insert2)
 537         bl      .                       /* patched by htab_finish_init() */
 538         cmpdi   0,r3,0
 539         bge+    htab_pte_insert_ok      /* Insertion successful */
 540         cmpdi   0,r3,-2                 /* Critical failure */
 541         beq-    htab_pte_insert_failure
 542
 543         /* Both are full, we need to evict something */
 544         mftb    r0
 545         /* Pick a random group based on TB */
 546         andi.   r0,r0,1
 547         mr      r5,r28
 548         bne     2f
 549         not     r5,r5
 550 2:      and     r0,r5,r27
 551         rldicr  r3,r0,3,63-3            /* r0 = (hash & mask) << 3 */
 552         /* Call ppc_md.hpte_remove */
 553 _GLOBAL(htab_call_hpte_remove)
 554         bl      .                       /* patched by htab_finish_init() */
 555
 556         /* Try all again */
 557         b       htab_insert_pte
 558
 559         /*
 560          * Call out to C code to invalidate an 64k HW HPTE that is
 561          * useless now that the segment has been switched to 4k pages.
 562          */
 563 htab_inval_old_hpte:
 564         mr      r3,r29                  /* virtual addr */
 565         mr      r4,r31                  /* PTE.pte */
 566         li      r5,0                    /* PTE.hidx */
 567         li      r6,MMU_PAGE_64K         /* psize */
 568         ld      r7,STK_PARM(r9)(r1)     /* ssize */
 569         ld      r8,STK_PARM(r8)(r1)     /* local */
 570         bl      .flush_hash_page
 571         /* Clear out _PAGE_HPTE_SUB bits in the new linux PTE */
 572         lis     r0,_PAGE_HPTE_SUB@h
 573         ori     r0,r0,_PAGE_HPTE_SUB@l
 574         andc    r30,r30,r0
 575         b       htab_insert_pte
 576
 577 htab_bail_ok:
 578         li      r3,0
 579         b       htab_bail
 580
 581 htab_pte_insert_ok:
 582         /* Insert slot number & secondary bit in PTE second half,
 583          * clear _PAGE_BUSY and set approriate HPTE slot bit
 584          */
 585         ld      r6,STK_PARM(r6)(r1)
 586         li      r0,_PAGE_BUSY
 587         andc    r30,r30,r0
 588         /* HPTE SUB bit */
 589         li      r0,1
 590         subfic  r5,r25,27               /* Must match bit position in */
 591         sld     r0,r0,r5                /* pgtable.h */
 592         or      r30,r30,r0
 593         /* hindx */
 594         sldi    r5,r25,2
 595         sld     r3,r3,r5
 596         li      r4,0xf
 597         sld     r4,r4,r5
 598         andc    r26,r26,r4
 599         or      r26,r26,r3
 600         ori     r5,r6,0x8000
 601         std     r26,0(r5)
 602         lwsync
 603         std     r30,0(r6)
 604         li      r3, 0
 605 htab_bail:
 606         ld      r25,STK_REG(r25)(r1)
 607         ld      r26,STK_REG(r26)(r1)
 608         ld      r27,STK_REG(r27)(r1)
 609         ld      r28,STK_REG(r28)(r1)
 610         ld      r29,STK_REG(r29)(r1)
 611         ld      r30,STK_REG(r30)(r1)
 612         ld      r31,STK_REG(r31)(r1)
 613         addi    r1,r1,STACKFRAMESIZE
 614         ld      r0,16(r1)
 615         mtlr    r0
 616         blr
 617
 618 htab_modify_pte:
 619         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 620         mr      r4,r3
 621         sldi    r5,r25,2
 622         srd     r3,r26,r5
 623
 624         /* Secondary group ? if yes, get a inverted hash value */
 625         mr      r5,r28
 626         andi.   r0,r3,0x8 /* page secondary ? */
 627         beq     1f
 628         not     r5,r5
 629 1:      andi.   r3,r3,0x7 /* extract idx alone */
 630
 631         /* Calculate proper slot value for ppc_md.hpte_updatepp */
 632         and     r0,r5,r27
 633         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 634         add     r3,r0,r3        /* add slot idx */
 635
 636         /* Call ppc_md.hpte_updatepp */
 637         mr      r5,r29                  /* va */
 638         li      r6,MMU_PAGE_4K          /* page size */
 639         ld      r7,STK_PARM(r9)(r1)     /* segment size */
 640         ld      r8,STK_PARM(r8)(r1)     /* get "local" param */
 641 _GLOBAL(htab_call_hpte_updatepp)
 642         bl      .                       /* patched by htab_finish_init() */
 643
 644         /* if we failed because typically the HPTE wasn't really here
 645          * we try an insertion.
 646          */
 647         cmpdi   0,r3,-1
 648         beq-    htab_insert_pte
 649
 650         /* Clear the BUSY bit and Write out the PTE */
 651         li      r0,_PAGE_BUSY
 652         andc    r30,r30,r0
 653         ld      r6,STK_PARM(r6)(r1)
 654         std     r30,0(r6)
 655         li      r3,0
 656         b       htab_bail
 657
 658 htab_wrong_access:
 659         /* Bail out clearing reservation */
 660         stdcx.  r31,0,r6
 661         li      r3,1
 662         b       htab_bail
 663
 664 htab_pte_insert_failure:
 665         /* Bail out restoring old PTE */
 666         ld      r6,STK_PARM(r6)(r1)
 667         std     r31,0(r6)
 668         li      r3,-1
 669         b       htab_bail
 670
 671 #endif /* CONFIG_PPC_64K_PAGES */
 672
 673 #ifdef CONFIG_PPC_HAS_HASH_64K
 674
 675 /*****************************************************************************
 676  *                                                                           *
 677  *           64K SW & 64K HW in a 64K segment pages implementation           *
 678  *                                                                           *
 679  *****************************************************************************/
 680
 681 _GLOBAL(__hash_page_64K)
 682         mflr    r0
 683         std     r0,16(r1)
 684         stdu    r1,-STACKFRAMESIZE(r1)
 685         /* Save all params that we need after a function call */
 686         std     r6,STK_PARM(r6)(r1)
 687         std     r8,STK_PARM(r8)(r1)
 688         std     r9,STK_PARM(r9)(r1)
 689
 690         /* Add _PAGE_PRESENT to access */
 691         ori     r4,r4,_PAGE_PRESENT
 692
 693         /* Save non-volatile registers.
 694          * r31 will hold "old PTE"
 695          * r30 is "new PTE"
 696          * r29 is "va"
 697          * r28 is a hash value
 698          * r27 is hashtab mask (maybe dynamic patched instead ?)
 699          */
 700         std     r27,STK_REG(r27)(r1)
 701         std     r28,STK_REG(r28)(r1)
 702         std     r29,STK_REG(r29)(r1)
 703         std     r30,STK_REG(r30)(r1)
 704         std     r31,STK_REG(r31)(r1)
 705
 706         /* Step 1:
 707          *
 708          * Check permissions, atomically mark the linux PTE busy
 709          * and hashed.
 710          */
 711 1:
 712         ldarx   r31,0,r6
 713         /* Check access rights (access & ~(pte_val(*ptep))) */
 714         andc.   r0,r4,r31
 715         bne-    ht64_wrong_access
 716         /* Check if PTE is busy */
 717         andi.   r0,r31,_PAGE_BUSY
 718         /* If so, just bail out and refault if needed. Someone else
 719          * is changing this PTE anyway and might hash it.
 720          */
 721         bne-    ht64_bail_ok
 722 BEGIN_FTR_SECTION
 723         /* Check if PTE has the cache-inhibit bit set */
 724         andi.   r0,r31,_PAGE_NO_CACHE
 725         /* If so, bail out and refault as a 4k page */
 726         bne-    ht64_bail_ok
 727 END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE)
 728         /* Prepare new PTE value (turn access RW into DIRTY, then
 729          * add BUSY and ACCESSED)
 730          */
 731         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
 732         or      r30,r30,r31
 733         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
 734         /* Write the linux PTE atomically (setting busy) */
 735         stdcx.  r30,0,r6
 736         bne-    1b
 737         isync
 738
 739         /* Step 2:
 740          *
 741          * Insert/Update the HPTE in the hash table. At this point,
 742          * r4 (access) is re-useable, we use it for the new HPTE flags
 743          */
 744
 745 BEGIN_FTR_SECTION
 746         cmpdi   r9,0                    /* check segment size */
 747         bne     3f
 748 END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
 749         /* Calc va and put it in r29 */
 750         rldicr  r29,r5,28,63-28
 751         rldicl  r3,r3,0,36
 752         or      r29,r3,r29
 753
 754         /* Calculate hash value for primary slot and store it in r28 */
 755         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
 756         rldicl  r0,r3,64-16,52          /* (ea >> 16) & 0xfff */
 757         xor     r28,r5,r0
 758         b       4f
 759
 760 3:      /* Calc VA and hash in r29 and r28 for 1T segment */
 761         sldi    r29,r5,40               /* vsid << 40 */
 762         clrldi  r3,r3,24                /* ea & 0xffffffffff */
 763         rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
 764         clrldi  r5,r5,40                /* vsid & 0xffffff */
 765         rldicl  r0,r3,64-16,40          /* (ea >> 16) & 0xffffff */
 766         xor     r28,r28,r5
 767         or      r29,r3,r29              /* VA */
 768         xor     r28,r28,r0              /* hash */
 769
 770         /* Convert linux PTE bits into HW equivalents */
 771 4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
 772         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 773         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 774         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 775         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 776         andc    r0,r30,r0               /* r0 = pte & ~r0 */
 777         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 778         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
 779
 780         /* We eventually do the icache sync here (maybe inline that
 781          * code rather than call a C function...)
 782          */
 783 BEGIN_FTR_SECTION
 784         mr      r4,r30
 785         mr      r5,r7
 786         bl      .hash_page_do_lazy_icache
 787 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 788
 789         /* At this point, r3 contains new PP bits, save them in
 790          * place of "access" in the param area (sic)
 791          */
 792         std     r3,STK_PARM(r4)(r1)
 793
 794         /* Get htab_hash_mask */
 795         ld      r4,htab_hash_mask@got(2)
 796         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 797
 798         /* Check if we may already be in the hashtable, in this case, we
 799          * go to out-of-line code to try to modify the HPTE
 800          */
 801         rldicl. r0,r31,64-12,48
 802         bne     ht64_modify_pte
 803
 804 ht64_insert_pte:
 805         /* Clear hpte bits in new pte (we also clear BUSY btw) and
 806          * add _PAGE_HPTE_SUB0
 807          */
 808         lis     r0,_PAGE_HPTEFLAGS@h
 809         ori     r0,r0,_PAGE_HPTEFLAGS@l
 810         andc    r30,r30,r0
 811 #ifdef CONFIG_PPC_64K_PAGES
 812         oris    r30,r30,_PAGE_HPTE_SUB0@h
 813 #else
 814         ori     r30,r30,_PAGE_HASHPTE
 815 #endif
 816         /* Phyical address in r5 */
 817         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 818         sldi    r5,r5,PAGE_SHIFT
 819
 820         /* Calculate primary group hash */
 821         and     r0,r28,r27
 822         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 823
 824         /* Call ppc_md.hpte_insert */
 825         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
 826         mr      r4,r29                  /* Retreive va */
 827         li      r7,0                    /* !bolted, !secondary */
 828         li      r8,MMU_PAGE_64K
 829         ld      r9,STK_PARM(r9)(r1)     /* segment size */
 830 _GLOBAL(ht64_call_hpte_insert1)
 831         bl      .                       /* patched by htab_finish_init() */
 832         cmpdi   0,r3,0
 833         bge     ht64_pte_insert_ok      /* Insertion successful */
 834         cmpdi   0,r3,-2                 /* Critical failure */
 835         beq-    ht64_pte_insert_failure
 836
 837         /* Now try secondary slot */
 838
 839         /* Phyical address in r5 */
 840         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 841         sldi    r5,r5,PAGE_SHIFT
 842
 843         /* Calculate secondary group hash */
 844         andc    r0,r27,r28
 845         rldicr  r3,r0,3,63-3    /* r0 = (~hash & mask) << 3 */
 846
 847         /* Call ppc_md.hpte_insert */
 848         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
 849         mr      r4,r29                  /* Retreive va */
 850         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 851         li      r8,MMU_PAGE_64K
 852         ld      r9,STK_PARM(r9)(r1)     /* segment size */
 853 _GLOBAL(ht64_call_hpte_insert2)
 854         bl      .                       /* patched by htab_finish_init() */
 855         cmpdi   0,r3,0
 856         bge+    ht64_pte_insert_ok      /* Insertion successful */
 857         cmpdi   0,r3,-2                 /* Critical failure */
 858         beq-    ht64_pte_insert_failure
 859
 860         /* Both are full, we need to evict something */
 861         mftb    r0
 862         /* Pick a random group based on TB */
 863         andi.   r0,r0,1
 864         mr      r5,r28
 865         bne     2f
 866         not     r5,r5
 867 2:      and     r0,r5,r27
 868         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 869         /* Call ppc_md.hpte_remove */
 870 _GLOBAL(ht64_call_hpte_remove)
 871         bl      .                       /* patched by htab_finish_init() */
 872
 873         /* Try all again */
 874         b       ht64_insert_pte
 875
 876 ht64_bail_ok:
 877         li      r3,0
 878         b       ht64_bail
 879
 880 ht64_pte_insert_ok:
 881         /* Insert slot number & secondary bit in PTE */
 882         rldimi  r30,r3,12,63-15
 883
 884         /* Write out the PTE with a normal write
 885          * (maybe add eieio may be good still ?)
 886          */
 887 ht64_write_out_pte:
 888         ld      r6,STK_PARM(r6)(r1)
 889         std     r30,0(r6)
 890         li      r3, 0
 891 ht64_bail:
 892         ld      r27,STK_REG(r27)(r1)
 893         ld      r28,STK_REG(r28)(r1)
 894         ld      r29,STK_REG(r29)(r1)
 895         ld      r30,STK_REG(r30)(r1)
 896         ld      r31,STK_REG(r31)(r1)
 897         addi    r1,r1,STACKFRAMESIZE
 898         ld      r0,16(r1)
 899         mtlr    r0
 900         blr
 901
 902 ht64_modify_pte:
 903         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 904         mr      r4,r3
 905         rlwinm  r3,r31,32-12,29,31
 906
 907         /* Secondary group ? if yes, get a inverted hash value */
 908         mr      r5,r28
 909         andi.   r0,r31,_PAGE_F_SECOND
 910         beq     1f
 911         not     r5,r5
 912 1:
 913         /* Calculate proper slot value for ppc_md.hpte_updatepp */
 914         and     r0,r5,r27
 915         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 916         add     r3,r0,r3        /* add slot idx */
 917
 918         /* Call ppc_md.hpte_updatepp */
 919         mr      r5,r29                  /* va */
 920         li      r6,MMU_PAGE_64K
 921         ld      r7,STK_PARM(r9)(r1)     /* segment size */
 922         ld      r8,STK_PARM(r8)(r1)     /* get "local" param */
 923 _GLOBAL(ht64_call_hpte_updatepp)
 924         bl      .                       /* patched by htab_finish_init() */
 925
 926         /* if we failed because typically the HPTE wasn't really here
 927          * we try an insertion.
 928          */
 929         cmpdi   0,r3,-1
 930         beq-    ht64_insert_pte
 931
 932         /* Clear the BUSY bit and Write out the PTE */
 933         li      r0,_PAGE_BUSY
 934         andc    r30,r30,r0
 935         b       ht64_write_out_pte
 936
 937 ht64_wrong_access:
 938         /* Bail out clearing reservation */
 939         stdcx.  r31,0,r6
 940         li      r3,1
 941         b       ht64_bail
 942
 943 ht64_pte_insert_failure:
 944         /* Bail out restoring old PTE */
 945         ld      r6,STK_PARM(r6)(r1)
 946         std     r31,0(r6)
 947         li      r3,-1
 948         b       ht64_bail
 949
 950
 951 #endif /* CONFIG_PPC_HAS_HASH_64K */
 952
 953
 954 /*****************************************************************************
 955  *                                                                           *
 956  *           Huge pages implementation is in hugetlbpage.c                   *
 957  *                                                                           *
 958  *****************************************************************************/