2 * PARISC64 Huge TLB page support.
4 * This parisc implementation is heavily based on the SPARC and x86 code.
6 * Copyright (C) 2015 Helge Deller <deller@gmx.de>
11 #include <linux/hugetlb.h>
12 #include <linux/pagemap.h>
13 #include <linux/sysctl.h>
16 #include <asm/pgalloc.h>
18 #include <asm/tlbflush.h>
19 #include <asm/cacheflush.h>
20 #include <asm/mmu_context.h>
24 hugetlb_get_unmapped_area(struct file
*file
, unsigned long addr
,
25 unsigned long len
, unsigned long pgoff
, unsigned long flags
)
27 struct hstate
*h
= hstate_file(file
);
29 if (len
& ~huge_page_mask(h
))
34 if (flags
& MAP_FIXED
)
35 if (prepare_hugepage_range(file
, addr
, len
))
39 addr
= ALIGN(addr
, huge_page_size(h
));
41 /* we need to make sure the colouring is OK */
42 return arch_get_unmapped_area(file
, addr
, len
, pgoff
, flags
);
46 pte_t
*huge_pte_alloc(struct mm_struct
*mm
,
47 unsigned long addr
, unsigned long sz
)
54 /* We must align the address, because our caller will run
55 * set_huge_pte_at() on whatever we return, which writes out
56 * all of the sub-ptes for the hugepage range. So we have
57 * to give it the first such sub-pte.
61 pgd
= pgd_offset(mm
, addr
);
62 pud
= pud_alloc(mm
, pgd
, addr
);
64 pmd
= pmd_alloc(mm
, pud
, addr
);
66 pte
= pte_alloc_map(mm
, pmd
, addr
);
71 pte_t
*huge_pte_offset(struct mm_struct
*mm
, unsigned long addr
)
80 pgd
= pgd_offset(mm
, addr
);
81 if (!pgd_none(*pgd
)) {
82 pud
= pud_offset(pgd
, addr
);
83 if (!pud_none(*pud
)) {
84 pmd
= pmd_offset(pud
, addr
);
86 pte
= pte_offset_map(pmd
, addr
);
92 /* Purge data and instruction TLB entries. Must be called holding
93 * the pa_tlb_lock. The TLB purge instructions are slow on SMP
94 * machines since the purge must be broadcast to all CPUs.
96 static inline void purge_tlb_entries_huge(struct mm_struct
*mm
, unsigned long addr
)
100 /* We may use multiple physical huge pages (e.g. 2x1 MB) to emulate
101 * Linux standard huge pages (e.g. 2 MB) */
102 BUILD_BUG_ON(REAL_HPAGE_SHIFT
> HPAGE_SHIFT
);
105 addr
|= _HUGE_PAGE_SIZE_ENCODING_DEFAULT
;
107 for (i
= 0; i
< (1 << (HPAGE_SHIFT
-REAL_HPAGE_SHIFT
)); i
++) {
108 purge_tlb_entries(mm
, addr
);
109 addr
+= (1UL << REAL_HPAGE_SHIFT
);
113 /* __set_huge_pte_at() must be called holding the pa_tlb_lock. */
114 static void __set_huge_pte_at(struct mm_struct
*mm
, unsigned long addr
,
115 pte_t
*ptep
, pte_t entry
)
117 unsigned long addr_start
;
123 for (i
= 0; i
< (1 << HUGETLB_PAGE_ORDER
); i
++) {
124 set_pte(ptep
, entry
);
128 pte_val(entry
) += PAGE_SIZE
;
131 purge_tlb_entries_huge(mm
, addr_start
);
134 void set_huge_pte_at(struct mm_struct
*mm
, unsigned long addr
,
135 pte_t
*ptep
, pte_t entry
)
139 purge_tlb_start(flags
);
140 __set_huge_pte_at(mm
, addr
, ptep
, entry
);
141 purge_tlb_end(flags
);
145 pte_t
huge_ptep_get_and_clear(struct mm_struct
*mm
, unsigned long addr
,
151 purge_tlb_start(flags
);
153 __set_huge_pte_at(mm
, addr
, ptep
, __pte(0));
154 purge_tlb_end(flags
);
160 void huge_ptep_set_wrprotect(struct mm_struct
*mm
,
161 unsigned long addr
, pte_t
*ptep
)
166 purge_tlb_start(flags
);
168 __set_huge_pte_at(mm
, addr
, ptep
, pte_wrprotect(old_pte
));
169 purge_tlb_end(flags
);
172 int huge_ptep_set_access_flags(struct vm_area_struct
*vma
,
173 unsigned long addr
, pte_t
*ptep
,
174 pte_t pte
, int dirty
)
179 purge_tlb_start(flags
);
180 changed
= !pte_same(*ptep
, pte
);
182 __set_huge_pte_at(vma
->vm_mm
, addr
, ptep
, pte
);
184 purge_tlb_end(flags
);
189 int pmd_huge(pmd_t pmd
)
194 int pud_huge(pud_t pud
)