1 // SPDX-License-Identifier: GPL-2.0
2 /* arch/sparc64/mm/tlb.c
4 * Copyright (C) 2004 David S. Miller <davem@redhat.com>
7 #include <linux/kernel.h>
8 #include <linux/percpu.h>
10 #include <linux/swap.h>
11 #include <linux/preempt.h>
13 #include <asm/pgtable.h>
14 #include <asm/pgalloc.h>
15 #include <asm/tlbflush.h>
16 #include <asm/cacheflush.h>
17 #include <asm/mmu_context.h>
20 /* Heavily inspired by the ppc64 code. */
22 static DEFINE_PER_CPU(struct tlb_batch
, tlb_batch
);
24 void flush_tlb_pending(void)
26 struct tlb_batch
*tb
= &get_cpu_var(tlb_batch
);
27 struct mm_struct
*mm
= tb
->mm
;
34 if (CTX_VALID(mm
->context
)) {
35 if (tb
->tlb_nr
== 1) {
36 global_flush_tlb_page(mm
, tb
->vaddrs
[0]);
39 smp_flush_tlb_pending(tb
->mm
, tb
->tlb_nr
,
42 __flush_tlb_pending(CTX_HWBITS(tb
->mm
->context
),
43 tb
->tlb_nr
, &tb
->vaddrs
[0]);
51 put_cpu_var(tlb_batch
);
54 void arch_enter_lazy_mmu_mode(void)
56 struct tlb_batch
*tb
= this_cpu_ptr(&tlb_batch
);
61 void arch_leave_lazy_mmu_mode(void)
63 struct tlb_batch
*tb
= this_cpu_ptr(&tlb_batch
);
70 static void tlb_batch_add_one(struct mm_struct
*mm
, unsigned long vaddr
,
71 bool exec
, unsigned int hugepage_shift
)
73 struct tlb_batch
*tb
= &get_cpu_var(tlb_batch
);
82 if (unlikely(nr
!= 0 && mm
!= tb
->mm
)) {
88 flush_tsb_user_page(mm
, vaddr
, hugepage_shift
);
89 global_flush_tlb_page(mm
, vaddr
);
95 tb
->hugepage_shift
= hugepage_shift
;
98 if (tb
->hugepage_shift
!= hugepage_shift
) {
100 tb
->hugepage_shift
= hugepage_shift
;
104 tb
->vaddrs
[nr
] = vaddr
;
106 if (nr
>= TLB_BATCH_NR
)
110 put_cpu_var(tlb_batch
);
113 void tlb_batch_add(struct mm_struct
*mm
, unsigned long vaddr
,
114 pte_t
*ptep
, pte_t orig
, int fullmm
,
115 unsigned int hugepage_shift
)
117 if (tlb_type
!= hypervisor
&&
119 unsigned long paddr
, pfn
= pte_pfn(orig
);
120 struct address_space
*mapping
;
126 page
= pfn_to_page(pfn
);
127 if (PageReserved(page
))
130 /* A real file page? */
131 mapping
= page_mapping(page
);
135 paddr
= (unsigned long) page_address(page
);
136 if ((paddr
^ vaddr
) & (1 << 13))
137 flush_dcache_page_all(mm
, page
);
142 tlb_batch_add_one(mm
, vaddr
, pte_exec(orig
), hugepage_shift
);
145 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
146 static void tlb_batch_pmd_scan(struct mm_struct
*mm
, unsigned long vaddr
,
152 pte
= pte_offset_map(&pmd
, vaddr
);
153 end
= vaddr
+ HPAGE_SIZE
;
154 while (vaddr
< end
) {
155 if (pte_val(*pte
) & _PAGE_VALID
) {
156 bool exec
= pte_exec(*pte
);
158 tlb_batch_add_one(mm
, vaddr
, exec
, PAGE_SHIFT
);
167 static void __set_pmd_acct(struct mm_struct
*mm
, unsigned long addr
,
168 pmd_t orig
, pmd_t pmd
)
173 if ((pmd_val(pmd
) ^ pmd_val(orig
)) & _PAGE_PMD_HUGE
) {
175 * Note that this routine only sets pmds for THP pages.
176 * Hugetlb pages are handled elsewhere. We need to check
177 * for huge zero page. Huge zero pages are like hugetlb
178 * pages in that there is no RSS, but there is the need
179 * for TSB entries. So, huge zero page counts go into
182 if (pmd_val(pmd
) & _PAGE_PMD_HUGE
) {
183 if (is_huge_zero_page(pmd_page(pmd
)))
184 mm
->context
.hugetlb_pte_count
++;
186 mm
->context
.thp_pte_count
++;
188 if (is_huge_zero_page(pmd_page(orig
)))
189 mm
->context
.hugetlb_pte_count
--;
191 mm
->context
.thp_pte_count
--;
194 /* Do not try to allocate the TSB hash table if we
195 * don't have one already. We have various locks held
196 * and thus we'll end up doing a GFP_KERNEL allocation
197 * in an atomic context.
199 * Instead, we let the first TLB miss on a hugepage
204 if (!pmd_none(orig
)) {
206 if (pmd_trans_huge(orig
)) {
207 pte_t orig_pte
= __pte(pmd_val(orig
));
208 bool exec
= pte_exec(orig_pte
);
210 tlb_batch_add_one(mm
, addr
, exec
, REAL_HPAGE_SHIFT
);
211 tlb_batch_add_one(mm
, addr
+ REAL_HPAGE_SIZE
, exec
,
214 tlb_batch_pmd_scan(mm
, addr
, orig
);
219 void set_pmd_at(struct mm_struct
*mm
, unsigned long addr
,
220 pmd_t
*pmdp
, pmd_t pmd
)
225 __set_pmd_acct(mm
, addr
, orig
, pmd
);
228 static inline pmd_t
pmdp_establish(struct vm_area_struct
*vma
,
229 unsigned long address
, pmd_t
*pmdp
, pmd_t pmd
)
235 } while (cmpxchg64(&pmdp
->pmd
, old
.pmd
, pmd
.pmd
) != old
.pmd
);
236 __set_pmd_acct(vma
->vm_mm
, address
, old
, pmd
);
242 * This routine is only called when splitting a THP
244 pmd_t
pmdp_invalidate(struct vm_area_struct
*vma
, unsigned long address
,
249 entry
= __pmd(pmd_val(*pmdp
) & ~_PAGE_VALID
);
250 old
= pmdp_establish(vma
, address
, pmdp
, entry
);
251 flush_tlb_range(vma
, address
, address
+ HPAGE_PMD_SIZE
);
254 * set_pmd_at() will not be called in a way to decrement
255 * thp_pte_count when splitting a THP, so do it now.
256 * Sanity check pmd before doing the actual decrement.
258 if ((pmd_val(entry
) & _PAGE_PMD_HUGE
) &&
259 !is_huge_zero_page(pmd_page(entry
)))
260 (vma
->vm_mm
)->context
.thp_pte_count
--;
265 void pgtable_trans_huge_deposit(struct mm_struct
*mm
, pmd_t
*pmdp
,
268 struct list_head
*lh
= (struct list_head
*) pgtable
;
270 assert_spin_locked(&mm
->page_table_lock
);
273 if (!pmd_huge_pte(mm
, pmdp
))
276 list_add(lh
, (struct list_head
*) pmd_huge_pte(mm
, pmdp
));
277 pmd_huge_pte(mm
, pmdp
) = pgtable
;
280 pgtable_t
pgtable_trans_huge_withdraw(struct mm_struct
*mm
, pmd_t
*pmdp
)
282 struct list_head
*lh
;
285 assert_spin_locked(&mm
->page_table_lock
);
288 pgtable
= pmd_huge_pte(mm
, pmdp
);
289 lh
= (struct list_head
*) pgtable
;
291 pmd_huge_pte(mm
, pmdp
) = NULL
;
293 pmd_huge_pte(mm
, pmdp
) = (pgtable_t
) lh
->next
;
296 pte_val(pgtable
[0]) = 0;
297 pte_val(pgtable
[1]) = 0;
301 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */