2 * Copyright IBM Corporation, 2015
3 * Author Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of version 2 of the GNU Lesser General Public License
7 * as published by the Free Software Foundation.
9 * This program is distributed in the hope that it would be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
16 #include <asm/machdep.h>
19 int __hash_page_4K(unsigned long ea
, unsigned long access
, unsigned long vsid
,
20 pte_t
*ptep
, unsigned long trap
, unsigned long flags
,
21 int ssize
, int subpg_prot
)
23 unsigned long hpte_group
;
24 unsigned long rflags
, pa
;
25 unsigned long old_pte
, new_pte
;
26 unsigned long vpn
, hash
, slot
;
27 unsigned long shift
= mmu_psize_defs
[MMU_PAGE_4K
].shift
;
30 * atomically mark the linux large page PTE busy and dirty
33 pte_t pte
= READ_ONCE(*ptep
);
35 old_pte
= pte_val(pte
);
36 /* If PTE busy, retry the access */
37 if (unlikely(old_pte
& _PAGE_BUSY
))
39 /* If PTE permissions don't match, take page fault */
40 if (unlikely(access
& ~old_pte
))
43 * Try to lock the PTE, add ACCESSED and DIRTY if it was
44 * a write access. Since this is 4K insert of 64K page size
45 * also add _PAGE_COMBO
47 new_pte
= old_pte
| _PAGE_BUSY
| _PAGE_ACCESSED
;
48 if (access
& _PAGE_RW
)
49 new_pte
|= _PAGE_DIRTY
;
50 } while (old_pte
!= __cmpxchg_u64((unsigned long *)ptep
,
53 * PP bits. _PAGE_USER is already PP bit 0x2, so we only
54 * need to add in 0x1 if it's a read-only user page
56 rflags
= htab_convert_pte_flags(new_pte
);
58 if (!cpu_has_feature(CPU_FTR_NOEXECUTE
) &&
59 !cpu_has_feature(CPU_FTR_COHERENT_ICACHE
))
60 rflags
= hash_page_do_lazy_icache(rflags
, __pte(old_pte
), trap
);
62 vpn
= hpt_vpn(ea
, vsid
, ssize
);
63 if (unlikely(old_pte
& _PAGE_HASHPTE
)) {
65 * There MIGHT be an HPTE for this pte
67 hash
= hpt_hash(vpn
, shift
, ssize
);
68 if (old_pte
& _PAGE_F_SECOND
)
70 slot
= (hash
& htab_hash_mask
) * HPTES_PER_GROUP
;
71 slot
+= (old_pte
& _PAGE_F_GIX
) >> _PAGE_F_GIX_SHIFT
;
73 if (ppc_md
.hpte_updatepp(slot
, rflags
, vpn
, MMU_PAGE_4K
,
74 MMU_PAGE_4K
, ssize
, flags
) == -1)
75 old_pte
&= ~_PAGE_HPTEFLAGS
;
78 if (likely(!(old_pte
& _PAGE_HASHPTE
))) {
80 pa
= pte_pfn(__pte(old_pte
)) << PAGE_SHIFT
;
81 hash
= hpt_hash(vpn
, shift
, ssize
);
84 hpte_group
= ((hash
& htab_hash_mask
) * HPTES_PER_GROUP
) & ~0x7UL
;
86 /* Insert into the hash table, primary slot */
87 slot
= ppc_md
.hpte_insert(hpte_group
, vpn
, pa
, rflags
, 0,
88 MMU_PAGE_4K
, MMU_PAGE_4K
, ssize
);
90 * Primary is full, try the secondary
92 if (unlikely(slot
== -1)) {
93 hpte_group
= ((~hash
& htab_hash_mask
) * HPTES_PER_GROUP
) & ~0x7UL
;
94 slot
= ppc_md
.hpte_insert(hpte_group
, vpn
, pa
,
95 rflags
, HPTE_V_SECONDARY
,
96 MMU_PAGE_4K
, MMU_PAGE_4K
, ssize
);
99 hpte_group
= ((hash
& htab_hash_mask
) *
100 HPTES_PER_GROUP
) & ~0x7UL
;
101 ppc_md
.hpte_remove(hpte_group
);
103 * FIXME!! Should be try the group from which we removed ?
109 * Hypervisor failure. Restore old pte and return -1
110 * similar to __hash_page_*
112 if (unlikely(slot
== -2)) {
113 *ptep
= __pte(old_pte
);
114 hash_failure_debug(ea
, access
, vsid
, trap
, ssize
,
115 MMU_PAGE_4K
, MMU_PAGE_4K
, old_pte
);
118 new_pte
= (new_pte
& ~_PAGE_HPTEFLAGS
) | _PAGE_HASHPTE
;
119 new_pte
|= (slot
<< _PAGE_F_GIX_SHIFT
) & (_PAGE_F_SECOND
| _PAGE_F_GIX
);
121 *ptep
= __pte(new_pte
& ~_PAGE_BUSY
);