arch/powerpc/mm/hash_low_64.S

   1 /*
   2  * ppc64 MMU hashtable management routines
   3  *
   4  * (c) Copyright IBM Corp. 2003, 2005
   5  *
   6  * Maintained by: Benjamin Herrenschmidt
   7  *                <benh@kernel.crashing.org>
   8  *
   9  * This file is covered by the GNU Public Licence v2 as
  10  * described in the kernel's COPYING file.
  11  */
  12
  13 #include <asm/reg.h>
  14 #include <asm/pgtable.h>
  15 #include <asm/mmu.h>
  16 #include <asm/page.h>
  17 #include <asm/types.h>
  18 #include <asm/ppc_asm.h>
  19 #include <asm/asm-offsets.h>
  20 #include <asm/cputable.h>
  21
  22         .text
  23
  24 /*
  25  * Stackframe:
  26  *
  27  *         +-> Back chain                       (SP + 256)
  28  *         |   General register save area       (SP + 112)
  29  *         |   Parameter save area              (SP + 48)
  30  *         |   TOC save area                    (SP + 40)
  31  *         |   link editor doubleword           (SP + 32)
  32  *         |   compiler doubleword              (SP + 24)
  33  *         |   LR save area                     (SP + 16)
  34  *         |   CR save area                     (SP + 8)
  35  * SP ---> +-- Back chain                       (SP + 0)
  36  */
  37
  38 #ifndef CONFIG_PPC_64K_PAGES
  39
  40 /*****************************************************************************
  41  *                                                                           *
  42  *           4K SW & 4K HW pages implementation                              *
  43  *                                                                           *
  44  *****************************************************************************/
  45
  46
  47 /*
  48  * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
  49  *               pte_t *ptep, unsigned long trap, int local, int ssize)
  50  *
  51  * Adds a 4K page to the hash table in a segment of 4K pages only
  52  */
  53
  54 _GLOBAL(__hash_page_4K)
  55         mflr    r0
  56         std     r0,16(r1)
  57         stdu    r1,-STACKFRAMESIZE(r1)
  58         /* Save all params that we need after a function call */
  59         std     r6,STK_PARAM(R6)(r1)
  60         std     r8,STK_PARAM(R8)(r1)
  61         std     r9,STK_PARAM(R9)(r1)
  62
  63         /* Save non-volatile registers.
  64          * r31 will hold "old PTE"
  65          * r30 is "new PTE"
  66          * r29 is vpn
  67          * r28 is a hash value
  68          * r27 is hashtab mask (maybe dynamic patched instead ?)
  69          */
  70         std     r27,STK_REG(R27)(r1)
  71         std     r28,STK_REG(R28)(r1)
  72         std     r29,STK_REG(R29)(r1)
  73         std     r30,STK_REG(R30)(r1)
  74         std     r31,STK_REG(R31)(r1)
  75
  76         /* Step 1:
  77          *
  78          * Check permissions, atomically mark the linux PTE busy
  79          * and hashed.
  80          */
  81 1:
  82         ldarx   r31,0,r6
  83         /* Check access rights (access & ~(pte_val(*ptep))) */
  84         andc.   r0,r4,r31
  85         bne-    htab_wrong_access
  86         /* Check if PTE is busy */
  87         andi.   r0,r31,_PAGE_BUSY
  88         /* If so, just bail out and refault if needed. Someone else
  89          * is changing this PTE anyway and might hash it.
  90          */
  91         bne-    htab_bail_ok
  92
  93         /* Prepare new PTE value (turn access RW into DIRTY, then
  94          * add BUSY,HASHPTE and ACCESSED)
  95          */
  96         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
  97         or      r30,r30,r31
  98         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
  99         /* Write the linux PTE atomically (setting busy) */
 100         stdcx.  r30,0,r6
 101         bne-    1b
 102         isync
 103
 104         /* Step 2:
 105          *
 106          * Insert/Update the HPTE in the hash table. At this point,
 107          * r4 (access) is re-useable, we use it for the new HPTE flags
 108          */
 109
 110 BEGIN_FTR_SECTION
 111         cmpdi   r9,0                    /* check segment size */
 112         bne     3f
 113 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
 114         /* Calc vpn and put it in r29 */
 115         sldi    r29,r5,SID_SHIFT - VPN_SHIFT
 116         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT)
 117         or      r29,r28,r29
 118
 119         /* Calculate hash value for primary slot and store it in r28 */
 120         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
 121         rldicl  r0,r3,64-12,48          /* (ea >> 12) & 0xffff */
 122         xor     r28,r5,r0
 123         b       4f
 124
 125 3:      /* Calc vpn and put it in r29 */
 126         sldi    r29,r5,SID_SHIFT_1T - VPN_SHIFT
 127         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT)
 128         or      r29,r28,r29
 129
 130         /*
 131          * calculate hash value for primary slot and
 132          * store it in r28 for 1T segment
 133          */
 134         rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
 135         clrldi  r5,r5,40                /* vsid & 0xffffff */
 136         rldicl  r0,r3,64-12,36          /* (ea >> 12) & 0xfffffff */
 137         xor     r28,r28,r5
 138         xor     r28,r28,r0              /* hash */
 139
 140         /* Convert linux PTE bits into HW equivalents */
 141 4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
 142         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 143         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 144         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 145         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 146         andc    r0,r30,r0               /* r0 = pte & ~r0 */
 147         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 148         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
 149
 150         /* We eventually do the icache sync here (maybe inline that
 151          * code rather than call a C function...)
 152          */
 153 BEGIN_FTR_SECTION
 154         mr      r4,r30
 155         mr      r5,r7
 156         bl      .hash_page_do_lazy_icache
 157 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 158
 159         /* At this point, r3 contains new PP bits, save them in
 160          * place of "access" in the param area (sic)
 161          */
 162         std     r3,STK_PARAM(R4)(r1)
 163
 164         /* Get htab_hash_mask */
 165         ld      r4,htab_hash_mask@got(2)
 166         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 167
 168         /* Check if we may already be in the hashtable, in this case, we
 169          * go to out-of-line code to try to modify the HPTE
 170          */
 171         andi.   r0,r31,_PAGE_HASHPTE
 172         bne     htab_modify_pte
 173
 174 htab_insert_pte:
 175         /* Clear hpte bits in new pte (we also clear BUSY btw) and
 176          * add _PAGE_HASHPTE
 177          */
 178         lis     r0,_PAGE_HPTEFLAGS@h
 179         ori     r0,r0,_PAGE_HPTEFLAGS@l
 180         andc    r30,r30,r0
 181         ori     r30,r30,_PAGE_HASHPTE
 182
 183         /* physical address r5 */
 184         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 185         sldi    r5,r5,PAGE_SHIFT
 186
 187         /* Calculate primary group hash */
 188         and     r0,r28,r27
 189         rldicr  r3,r0,3,63-3            /* r3 = (hash & mask) << 3 */
 190
 191         /* Call ppc_md.hpte_insert */
 192         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 193         mr      r4,r29                  /* Retrieve vpn */
 194         li      r7,0                    /* !bolted, !secondary */
 195         li      r8,MMU_PAGE_4K          /* page size */
 196         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
 197 _GLOBAL(htab_call_hpte_insert1)
 198         bl      .                       /* Patched by htab_finish_init() */
 199         cmpdi   0,r3,0
 200         bge     htab_pte_insert_ok      /* Insertion successful */
 201         cmpdi   0,r3,-2                 /* Critical failure */
 202         beq-    htab_pte_insert_failure
 203
 204         /* Now try secondary slot */
 205
 206         /* physical address r5 */
 207         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 208         sldi    r5,r5,PAGE_SHIFT
 209
 210         /* Calculate secondary group hash */
 211         andc    r0,r27,r28
 212         rldicr  r3,r0,3,63-3    /* r0 = (~hash & mask) << 3 */
 213
 214         /* Call ppc_md.hpte_insert */
 215         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 216         mr      r4,r29                  /* Retrieve vpn */
 217         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 218         li      r8,MMU_PAGE_4K          /* page size */
 219         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
 220 _GLOBAL(htab_call_hpte_insert2)
 221         bl      .                       /* Patched by htab_finish_init() */
 222         cmpdi   0,r3,0
 223         bge+    htab_pte_insert_ok      /* Insertion successful */
 224         cmpdi   0,r3,-2                 /* Critical failure */
 225         beq-    htab_pte_insert_failure
 226
 227         /* Both are full, we need to evict something */
 228         mftb    r0
 229         /* Pick a random group based on TB */
 230         andi.   r0,r0,1
 231         mr      r5,r28
 232         bne     2f
 233         not     r5,r5
 234 2:      and     r0,r5,r27
 235         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 236         /* Call ppc_md.hpte_remove */
 237 _GLOBAL(htab_call_hpte_remove)
 238         bl      .                       /* Patched by htab_finish_init() */
 239
 240         /* Try all again */
 241         b       htab_insert_pte
 242
 243 htab_bail_ok:
 244         li      r3,0
 245         b       htab_bail
 246
 247 htab_pte_insert_ok:
 248         /* Insert slot number & secondary bit in PTE */
 249         rldimi  r30,r3,12,63-15
 250
 251         /* Write out the PTE with a normal write
 252          * (maybe add eieio may be good still ?)
 253          */
 254 htab_write_out_pte:
 255         ld      r6,STK_PARAM(R6)(r1)
 256         std     r30,0(r6)
 257         li      r3, 0
 258 htab_bail:
 259         ld      r27,STK_REG(R27)(r1)
 260         ld      r28,STK_REG(R28)(r1)
 261         ld      r29,STK_REG(R29)(r1)
 262         ld      r30,STK_REG(R30)(r1)
 263         ld      r31,STK_REG(R31)(r1)
 264         addi    r1,r1,STACKFRAMESIZE
 265         ld      r0,16(r1)
 266         mtlr    r0
 267         blr
 268
 269 htab_modify_pte:
 270         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 271         mr      r4,r3
 272         rlwinm  r3,r31,32-12,29,31
 273
 274         /* Secondary group ? if yes, get a inverted hash value */
 275         mr      r5,r28
 276         andi.   r0,r31,_PAGE_SECONDARY
 277         beq     1f
 278         not     r5,r5
 279 1:
 280         /* Calculate proper slot value for ppc_md.hpte_updatepp */
 281         and     r0,r5,r27
 282         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 283         add     r3,r0,r3        /* add slot idx */
 284
 285         /* Call ppc_md.hpte_updatepp */
 286         mr      r5,r29                  /* vpn */
 287         li      r6,MMU_PAGE_4K          /* page size */
 288         ld      r7,STK_PARAM(R9)(r1)    /* segment size */
 289         ld      r8,STK_PARAM(R8)(r1)    /* get "local" param */
 290 _GLOBAL(htab_call_hpte_updatepp)
 291         bl      .                       /* Patched by htab_finish_init() */
 292
 293         /* if we failed because typically the HPTE wasn't really here
 294          * we try an insertion.
 295          */
 296         cmpdi   0,r3,-1
 297         beq-    htab_insert_pte
 298
 299         /* Clear the BUSY bit and Write out the PTE */
 300         li      r0,_PAGE_BUSY
 301         andc    r30,r30,r0
 302         b       htab_write_out_pte
 303
 304 htab_wrong_access:
 305         /* Bail out clearing reservation */
 306         stdcx.  r31,0,r6
 307         li      r3,1
 308         b       htab_bail
 309
 310 htab_pte_insert_failure:
 311         /* Bail out restoring old PTE */
 312         ld      r6,STK_PARAM(R6)(r1)
 313         std     r31,0(r6)
 314         li      r3,-1
 315         b       htab_bail
 316
 317
 318 #else /* CONFIG_PPC_64K_PAGES */
 319
 320
 321 /*****************************************************************************
 322  *                                                                           *
 323  *           64K SW & 4K or 64K HW in a 4K segment pages implementation      *
 324  *                                                                           *
 325  *****************************************************************************/
 326
 327 /* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
 328  *               pte_t *ptep, unsigned long trap, int local, int ssize,
 329  *               int subpg_prot)
 330  */
 331
 332 /*
 333  * For now, we do NOT implement Admixed pages
 334  */
 335 _GLOBAL(__hash_page_4K)
 336         mflr    r0
 337         std     r0,16(r1)
 338         stdu    r1,-STACKFRAMESIZE(r1)
 339         /* Save all params that we need after a function call */
 340         std     r6,STK_PARAM(R6)(r1)
 341         std     r8,STK_PARAM(R8)(r1)
 342         std     r9,STK_PARAM(R9)(r1)
 343
 344         /* Save non-volatile registers.
 345          * r31 will hold "old PTE"
 346          * r30 is "new PTE"
 347          * r29 is vpn
 348          * r28 is a hash value
 349          * r27 is hashtab mask (maybe dynamic patched instead ?)
 350          * r26 is the hidx mask
 351          * r25 is the index in combo page
 352          */
 353         std     r25,STK_REG(R25)(r1)
 354         std     r26,STK_REG(R26)(r1)
 355         std     r27,STK_REG(R27)(r1)
 356         std     r28,STK_REG(R28)(r1)
 357         std     r29,STK_REG(R29)(r1)
 358         std     r30,STK_REG(R30)(r1)
 359         std     r31,STK_REG(R31)(r1)
 360
 361         /* Step 1:
 362          *
 363          * Check permissions, atomically mark the linux PTE busy
 364          * and hashed.
 365          */
 366 1:
 367         ldarx   r31,0,r6
 368         /* Check access rights (access & ~(pte_val(*ptep))) */
 369         andc.   r0,r4,r31
 370         bne-    htab_wrong_access
 371         /* Check if PTE is busy */
 372         andi.   r0,r31,_PAGE_BUSY
 373         /* If so, just bail out and refault if needed. Someone else
 374          * is changing this PTE anyway and might hash it.
 375          */
 376         bne-    htab_bail_ok
 377         /* Prepare new PTE value (turn access RW into DIRTY, then
 378          * add BUSY and ACCESSED)
 379          */
 380         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
 381         or      r30,r30,r31
 382         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
 383         oris    r30,r30,_PAGE_COMBO@h
 384         /* Write the linux PTE atomically (setting busy) */
 385         stdcx.  r30,0,r6
 386         bne-    1b
 387         isync
 388
 389         /* Step 2:
 390          *
 391          * Insert/Update the HPTE in the hash table. At this point,
 392          * r4 (access) is re-useable, we use it for the new HPTE flags
 393          */
 394
 395         /* Load the hidx index */
 396         rldicl  r25,r3,64-12,60
 397
 398 BEGIN_FTR_SECTION
 399         cmpdi   r9,0                    /* check segment size */
 400         bne     3f
 401 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
 402         /* Calc vpn and put it in r29 */
 403         sldi    r29,r5,SID_SHIFT - VPN_SHIFT
 404         /*
 405          * clrldi r3,r3,64 - SID_SHIFT -->  ea & 0xfffffff
 406          * srdi  r28,r3,VPN_SHIFT
 407          */
 408         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT)
 409         or      r29,r28,r29
 410
 411         /* Calculate hash value for primary slot and store it in r28 */
 412         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
 413         rldicl  r0,r3,64-12,48          /* (ea >> 12) & 0xffff */
 414         xor     r28,r5,r0
 415         b       4f
 416
 417 3:      /* Calc vpn and put it in r29 */
 418         sldi    r29,r5,SID_SHIFT_1T - VPN_SHIFT
 419         /*
 420          * clrldi r3,r3,64 - SID_SHIFT_1T -->  ea & 0xffffffffff
 421          * srdi r28,r3,VPN_SHIFT
 422          */
 423         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT)
 424         or      r29,r28,r29
 425
 426         /*
 427          * Calculate hash value for primary slot and
 428          * store it in r28  for 1T segment
 429          */
 430         rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
 431         clrldi  r5,r5,40                /* vsid & 0xffffff */
 432         rldicl  r0,r3,64-12,36          /* (ea >> 12) & 0xfffffff */
 433         xor     r28,r28,r5
 434         xor     r28,r28,r0              /* hash */
 435
 436         /* Convert linux PTE bits into HW equivalents */
 437 4:
 438 #ifdef CONFIG_PPC_SUBPAGE_PROT
 439         andc    r10,r30,r10
 440         andi.   r3,r10,0x1fe            /* Get basic set of flags */
 441         rlwinm  r0,r10,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 442 #else
 443         andi.   r3,r30,0x1fe            /* Get basic set of flags */
 444         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 445 #endif
 446         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 447         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 448         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 449         andc    r0,r3,r0                /* r0 = pte & ~r0 */
 450         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 451         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
 452
 453         /* We eventually do the icache sync here (maybe inline that
 454          * code rather than call a C function...)
 455          */
 456 BEGIN_FTR_SECTION
 457         mr      r4,r30
 458         mr      r5,r7
 459         bl      .hash_page_do_lazy_icache
 460 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 461
 462         /* At this point, r3 contains new PP bits, save them in
 463          * place of "access" in the param area (sic)
 464          */
 465         std     r3,STK_PARAM(R4)(r1)
 466
 467         /* Get htab_hash_mask */
 468         ld      r4,htab_hash_mask@got(2)
 469         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 470
 471         /* Check if we may already be in the hashtable, in this case, we
 472          * go to out-of-line code to try to modify the HPTE. We look for
 473          * the bit at (1 >> (index + 32))
 474          */
 475         rldicl. r0,r31,64-12,48
 476         li      r26,0                   /* Default hidx */
 477         beq     htab_insert_pte
 478
 479         /*
 480          * Check if the pte was already inserted into the hash table
 481          * as a 64k HW page, and invalidate the 64k HPTE if so.
 482          */
 483         andis.  r0,r31,_PAGE_COMBO@h
 484         beq     htab_inval_old_hpte
 485
 486         ld      r6,STK_PARAM(R6)(r1)
 487         ori     r26,r6,0x8000           /* Load the hidx mask */
 488         ld      r26,0(r26)
 489         addi    r5,r25,36               /* Check actual HPTE_SUB bit, this */
 490         rldcr.  r0,r31,r5,0             /* must match pgtable.h definition */
 491         bne     htab_modify_pte
 492
 493 htab_insert_pte:
 494         /* real page number in r5, PTE RPN value + index */
 495         andis.  r0,r31,_PAGE_4K_PFN@h
 496         srdi    r5,r31,PTE_RPN_SHIFT
 497         bne-    htab_special_pfn
 498         sldi    r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
 499         add     r5,r5,r25
 500 htab_special_pfn:
 501         sldi    r5,r5,HW_PAGE_SHIFT
 502
 503         /* Calculate primary group hash */
 504         and     r0,r28,r27
 505         rldicr  r3,r0,3,63-3            /* r0 = (hash & mask) << 3 */
 506
 507         /* Call ppc_md.hpte_insert */
 508         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 509         mr      r4,r29                  /* Retrieve vpn */
 510         li      r7,0                    /* !bolted, !secondary */
 511         li      r8,MMU_PAGE_4K          /* page size */
 512         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
 513 _GLOBAL(htab_call_hpte_insert1)
 514         bl      .                       /* patched by htab_finish_init() */
 515         cmpdi   0,r3,0
 516         bge     htab_pte_insert_ok      /* Insertion successful */
 517         cmpdi   0,r3,-2                 /* Critical failure */
 518         beq-    htab_pte_insert_failure
 519
 520         /* Now try secondary slot */
 521
 522         /* real page number in r5, PTE RPN value + index */
 523         andis.  r0,r31,_PAGE_4K_PFN@h
 524         srdi    r5,r31,PTE_RPN_SHIFT
 525         bne-    3f
 526         sldi    r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
 527         add     r5,r5,r25
 528 3:      sldi    r5,r5,HW_PAGE_SHIFT
 529
 530         /* Calculate secondary group hash */
 531         andc    r0,r27,r28
 532         rldicr  r3,r0,3,63-3            /* r0 = (~hash & mask) << 3 */
 533
 534         /* Call ppc_md.hpte_insert */
 535         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 536         mr      r4,r29                  /* Retrieve vpn */
 537         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 538         li      r8,MMU_PAGE_4K          /* page size */
 539         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
 540 _GLOBAL(htab_call_hpte_insert2)
 541         bl      .                       /* patched by htab_finish_init() */
 542         cmpdi   0,r3,0
 543         bge+    htab_pte_insert_ok      /* Insertion successful */
 544         cmpdi   0,r3,-2                 /* Critical failure */
 545         beq-    htab_pte_insert_failure
 546
 547         /* Both are full, we need to evict something */
 548         mftb    r0
 549         /* Pick a random group based on TB */
 550         andi.   r0,r0,1
 551         mr      r5,r28
 552         bne     2f
 553         not     r5,r5
 554 2:      and     r0,r5,r27
 555         rldicr  r3,r0,3,63-3            /* r0 = (hash & mask) << 3 */
 556         /* Call ppc_md.hpte_remove */
 557 _GLOBAL(htab_call_hpte_remove)
 558         bl      .                       /* patched by htab_finish_init() */
 559
 560         /* Try all again */
 561         b       htab_insert_pte
 562
 563         /*
 564          * Call out to C code to invalidate an 64k HW HPTE that is
 565          * useless now that the segment has been switched to 4k pages.
 566          */
 567 htab_inval_old_hpte:
 568         mr      r3,r29                  /* vpn */
 569         mr      r4,r31                  /* PTE.pte */
 570         li      r5,0                    /* PTE.hidx */
 571         li      r6,MMU_PAGE_64K         /* psize */
 572         ld      r7,STK_PARAM(R9)(r1)    /* ssize */
 573         ld      r8,STK_PARAM(R8)(r1)    /* local */
 574         bl      .flush_hash_page
 575         /* Clear out _PAGE_HPTE_SUB bits in the new linux PTE */
 576         lis     r0,_PAGE_HPTE_SUB@h
 577         ori     r0,r0,_PAGE_HPTE_SUB@l
 578         andc    r30,r30,r0
 579         b       htab_insert_pte
 580
 581 htab_bail_ok:
 582         li      r3,0
 583         b       htab_bail
 584
 585 htab_pte_insert_ok:
 586         /* Insert slot number & secondary bit in PTE second half,
 587          * clear _PAGE_BUSY and set approriate HPTE slot bit
 588          */
 589         ld      r6,STK_PARAM(R6)(r1)
 590         li      r0,_PAGE_BUSY
 591         andc    r30,r30,r0
 592         /* HPTE SUB bit */
 593         li      r0,1
 594         subfic  r5,r25,27               /* Must match bit position in */
 595         sld     r0,r0,r5                /* pgtable.h */
 596         or      r30,r30,r0
 597         /* hindx */
 598         sldi    r5,r25,2
 599         sld     r3,r3,r5
 600         li      r4,0xf
 601         sld     r4,r4,r5
 602         andc    r26,r26,r4
 603         or      r26,r26,r3
 604         ori     r5,r6,0x8000
 605         std     r26,0(r5)
 606         lwsync
 607         std     r30,0(r6)
 608         li      r3, 0
 609 htab_bail:
 610         ld      r25,STK_REG(R25)(r1)
 611         ld      r26,STK_REG(R26)(r1)
 612         ld      r27,STK_REG(R27)(r1)
 613         ld      r28,STK_REG(R28)(r1)
 614         ld      r29,STK_REG(R29)(r1)
 615         ld      r30,STK_REG(R30)(r1)
 616         ld      r31,STK_REG(R31)(r1)
 617         addi    r1,r1,STACKFRAMESIZE
 618         ld      r0,16(r1)
 619         mtlr    r0
 620         blr
 621
 622 htab_modify_pte:
 623         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 624         mr      r4,r3
 625         sldi    r5,r25,2
 626         srd     r3,r26,r5
 627
 628         /* Secondary group ? if yes, get a inverted hash value */
 629         mr      r5,r28
 630         andi.   r0,r3,0x8 /* page secondary ? */
 631         beq     1f
 632         not     r5,r5
 633 1:      andi.   r3,r3,0x7 /* extract idx alone */
 634
 635         /* Calculate proper slot value for ppc_md.hpte_updatepp */
 636         and     r0,r5,r27
 637         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 638         add     r3,r0,r3        /* add slot idx */
 639
 640         /* Call ppc_md.hpte_updatepp */
 641         mr      r5,r29                  /* vpn */
 642         li      r6,MMU_PAGE_4K          /* page size */
 643         ld      r7,STK_PARAM(R9)(r1)    /* segment size */
 644         ld      r8,STK_PARAM(R8)(r1)    /* get "local" param */
 645 _GLOBAL(htab_call_hpte_updatepp)
 646         bl      .                       /* patched by htab_finish_init() */
 647
 648         /* if we failed because typically the HPTE wasn't really here
 649          * we try an insertion.
 650          */
 651         cmpdi   0,r3,-1
 652         beq-    htab_insert_pte
 653
 654         /* Clear the BUSY bit and Write out the PTE */
 655         li      r0,_PAGE_BUSY
 656         andc    r30,r30,r0
 657         ld      r6,STK_PARAM(R6)(r1)
 658         std     r30,0(r6)
 659         li      r3,0
 660         b       htab_bail
 661
 662 htab_wrong_access:
 663         /* Bail out clearing reservation */
 664         stdcx.  r31,0,r6
 665         li      r3,1
 666         b       htab_bail
 667
 668 htab_pte_insert_failure:
 669         /* Bail out restoring old PTE */
 670         ld      r6,STK_PARAM(R6)(r1)
 671         std     r31,0(r6)
 672         li      r3,-1
 673         b       htab_bail
 674
 675 #endif /* CONFIG_PPC_64K_PAGES */
 676
 677 #ifdef CONFIG_PPC_HAS_HASH_64K
 678
 679 /*****************************************************************************
 680  *                                                                           *
 681  *           64K SW & 64K HW in a 64K segment pages implementation           *
 682  *                                                                           *
 683  *****************************************************************************/
 684
 685 _GLOBAL(__hash_page_64K)
 686         mflr    r0
 687         std     r0,16(r1)
 688         stdu    r1,-STACKFRAMESIZE(r1)
 689         /* Save all params that we need after a function call */
 690         std     r6,STK_PARAM(R6)(r1)
 691         std     r8,STK_PARAM(R8)(r1)
 692         std     r9,STK_PARAM(R9)(r1)
 693
 694         /* Save non-volatile registers.
 695          * r31 will hold "old PTE"
 696          * r30 is "new PTE"
 697          * r29 is vpn
 698          * r28 is a hash value
 699          * r27 is hashtab mask (maybe dynamic patched instead ?)
 700          */
 701         std     r27,STK_REG(R27)(r1)
 702         std     r28,STK_REG(R28)(r1)
 703         std     r29,STK_REG(R29)(r1)
 704         std     r30,STK_REG(R30)(r1)
 705         std     r31,STK_REG(R31)(r1)
 706
 707         /* Step 1:
 708          *
 709          * Check permissions, atomically mark the linux PTE busy
 710          * and hashed.
 711          */
 712 1:
 713         ldarx   r31,0,r6
 714         /* Check access rights (access & ~(pte_val(*ptep))) */
 715         andc.   r0,r4,r31
 716         bne-    ht64_wrong_access
 717         /* Check if PTE is busy */
 718         andi.   r0,r31,_PAGE_BUSY
 719         /* If so, just bail out and refault if needed. Someone else
 720          * is changing this PTE anyway and might hash it.
 721          */
 722         bne-    ht64_bail_ok
 723 BEGIN_FTR_SECTION
 724         /* Check if PTE has the cache-inhibit bit set */
 725         andi.   r0,r31,_PAGE_NO_CACHE
 726         /* If so, bail out and refault as a 4k page */
 727         bne-    ht64_bail_ok
 728 END_MMU_FTR_SECTION_IFCLR(MMU_FTR_CI_LARGE_PAGE)
 729         /* Prepare new PTE value (turn access RW into DIRTY, then
 730          * add BUSY and ACCESSED)
 731          */
 732         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
 733         or      r30,r30,r31
 734         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
 735         /* Write the linux PTE atomically (setting busy) */
 736         stdcx.  r30,0,r6
 737         bne-    1b
 738         isync
 739
 740         /* Step 2:
 741          *
 742          * Insert/Update the HPTE in the hash table. At this point,
 743          * r4 (access) is re-useable, we use it for the new HPTE flags
 744          */
 745
 746 BEGIN_FTR_SECTION
 747         cmpdi   r9,0                    /* check segment size */
 748         bne     3f
 749 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
 750         /* Calc vpn and put it in r29 */
 751         sldi    r29,r5,SID_SHIFT - VPN_SHIFT
 752         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT)
 753         or      r29,r28,r29
 754
 755         /* Calculate hash value for primary slot and store it in r28 */
 756         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
 757         rldicl  r0,r3,64-16,52          /* (ea >> 16) & 0xfff */
 758         xor     r28,r5,r0
 759         b       4f
 760
 761 3:      /* Calc vpn and put it in r29 */
 762         sldi    r29,r5,SID_SHIFT_1T - VPN_SHIFT
 763         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT)
 764         or      r29,r28,r29
 765
 766         /*
 767          * calculate hash value for primary slot and
 768          * store it in r28 for 1T segment
 769          */
 770         rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
 771         clrldi  r5,r5,40                /* vsid & 0xffffff */
 772         rldicl  r0,r3,64-16,40          /* (ea >> 16) & 0xffffff */
 773         xor     r28,r28,r5
 774         xor     r28,r28,r0              /* hash */
 775
 776         /* Convert linux PTE bits into HW equivalents */
 777 4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
 778         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 779         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 780         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 781         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 782         andc    r0,r30,r0               /* r0 = pte & ~r0 */
 783         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 784         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
 785
 786         /* We eventually do the icache sync here (maybe inline that
 787          * code rather than call a C function...)
 788          */
 789 BEGIN_FTR_SECTION
 790         mr      r4,r30
 791         mr      r5,r7
 792         bl      .hash_page_do_lazy_icache
 793 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 794
 795         /* At this point, r3 contains new PP bits, save them in
 796          * place of "access" in the param area (sic)
 797          */
 798         std     r3,STK_PARAM(R4)(r1)
 799
 800         /* Get htab_hash_mask */
 801         ld      r4,htab_hash_mask@got(2)
 802         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 803
 804         /* Check if we may already be in the hashtable, in this case, we
 805          * go to out-of-line code to try to modify the HPTE
 806          */
 807         rldicl. r0,r31,64-12,48
 808         bne     ht64_modify_pte
 809
 810 ht64_insert_pte:
 811         /* Clear hpte bits in new pte (we also clear BUSY btw) and
 812          * add _PAGE_HPTE_SUB0
 813          */
 814         lis     r0,_PAGE_HPTEFLAGS@h
 815         ori     r0,r0,_PAGE_HPTEFLAGS@l
 816         andc    r30,r30,r0
 817 #ifdef CONFIG_PPC_64K_PAGES
 818         oris    r30,r30,_PAGE_HPTE_SUB0@h
 819 #else
 820         ori     r30,r30,_PAGE_HASHPTE
 821 #endif
 822         /* Phyical address in r5 */
 823         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 824         sldi    r5,r5,PAGE_SHIFT
 825
 826         /* Calculate primary group hash */
 827         and     r0,r28,r27
 828         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 829
 830         /* Call ppc_md.hpte_insert */
 831         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 832         mr      r4,r29                  /* Retrieve vpn */
 833         li      r7,0                    /* !bolted, !secondary */
 834         li      r8,MMU_PAGE_64K
 835         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
 836 _GLOBAL(ht64_call_hpte_insert1)
 837         bl      .                       /* patched by htab_finish_init() */
 838         cmpdi   0,r3,0
 839         bge     ht64_pte_insert_ok      /* Insertion successful */
 840         cmpdi   0,r3,-2                 /* Critical failure */
 841         beq-    ht64_pte_insert_failure
 842
 843         /* Now try secondary slot */
 844
 845         /* Phyical address in r5 */
 846         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 847         sldi    r5,r5,PAGE_SHIFT
 848
 849         /* Calculate secondary group hash */
 850         andc    r0,r27,r28
 851         rldicr  r3,r0,3,63-3    /* r0 = (~hash & mask) << 3 */
 852
 853         /* Call ppc_md.hpte_insert */
 854         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 855         mr      r4,r29                  /* Retrieve vpn */
 856         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 857         li      r8,MMU_PAGE_64K
 858         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
 859 _GLOBAL(ht64_call_hpte_insert2)
 860         bl      .                       /* patched by htab_finish_init() */
 861         cmpdi   0,r3,0
 862         bge+    ht64_pte_insert_ok      /* Insertion successful */
 863         cmpdi   0,r3,-2                 /* Critical failure */
 864         beq-    ht64_pte_insert_failure
 865
 866         /* Both are full, we need to evict something */
 867         mftb    r0
 868         /* Pick a random group based on TB */
 869         andi.   r0,r0,1
 870         mr      r5,r28
 871         bne     2f
 872         not     r5,r5
 873 2:      and     r0,r5,r27
 874         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 875         /* Call ppc_md.hpte_remove */
 876 _GLOBAL(ht64_call_hpte_remove)
 877         bl      .                       /* patched by htab_finish_init() */
 878
 879         /* Try all again */
 880         b       ht64_insert_pte
 881
 882 ht64_bail_ok:
 883         li      r3,0
 884         b       ht64_bail
 885
 886 ht64_pte_insert_ok:
 887         /* Insert slot number & secondary bit in PTE */
 888         rldimi  r30,r3,12,63-15
 889
 890         /* Write out the PTE with a normal write
 891          * (maybe add eieio may be good still ?)
 892          */
 893 ht64_write_out_pte:
 894         ld      r6,STK_PARAM(R6)(r1)
 895         std     r30,0(r6)
 896         li      r3, 0
 897 ht64_bail:
 898         ld      r27,STK_REG(R27)(r1)
 899         ld      r28,STK_REG(R28)(r1)
 900         ld      r29,STK_REG(R29)(r1)
 901         ld      r30,STK_REG(R30)(r1)
 902         ld      r31,STK_REG(R31)(r1)
 903         addi    r1,r1,STACKFRAMESIZE
 904         ld      r0,16(r1)
 905         mtlr    r0
 906         blr
 907
 908 ht64_modify_pte:
 909         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 910         mr      r4,r3
 911         rlwinm  r3,r31,32-12,29,31
 912
 913         /* Secondary group ? if yes, get a inverted hash value */
 914         mr      r5,r28
 915         andi.   r0,r31,_PAGE_F_SECOND
 916         beq     1f
 917         not     r5,r5
 918 1:
 919         /* Calculate proper slot value for ppc_md.hpte_updatepp */
 920         and     r0,r5,r27
 921         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 922         add     r3,r0,r3        /* add slot idx */
 923
 924         /* Call ppc_md.hpte_updatepp */
 925         mr      r5,r29                  /* vpn */
 926         li      r6,MMU_PAGE_64K
 927         ld      r7,STK_PARAM(R9)(r1)    /* segment size */
 928         ld      r8,STK_PARAM(R8)(r1)    /* get "local" param */
 929 _GLOBAL(ht64_call_hpte_updatepp)
 930         bl      .                       /* patched by htab_finish_init() */
 931
 932         /* if we failed because typically the HPTE wasn't really here
 933          * we try an insertion.
 934          */
 935         cmpdi   0,r3,-1
 936         beq-    ht64_insert_pte
 937
 938         /* Clear the BUSY bit and Write out the PTE */
 939         li      r0,_PAGE_BUSY
 940         andc    r30,r30,r0
 941         b       ht64_write_out_pte
 942
 943 ht64_wrong_access:
 944         /* Bail out clearing reservation */
 945         stdcx.  r31,0,r6
 946         li      r3,1
 947         b       ht64_bail
 948
 949 ht64_pte_insert_failure:
 950         /* Bail out restoring old PTE */
 951         ld      r6,STK_PARAM(R6)(r1)
 952         std     r31,0(r6)
 953         li      r3,-1
 954         b       ht64_bail
 955
 956
 957 #endif /* CONFIG_PPC_HAS_HASH_64K */
 958
 959
 960 /*****************************************************************************
 961  *                                                                           *
 962  *           Huge pages implementation is in hugetlbpage.c                   *
 963  *                                                                           *
 964  *****************************************************************************/