2 * IA-32 Huge TLB Page Support for Kernel.
4 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
7 #include <linux/init.h>
10 #include <linux/hugetlb.h>
11 #include <linux/pagemap.h>
12 #include <linux/err.h>
13 #include <linux/sysctl.h>
16 #include <asm/tlbflush.h>
17 #include <asm/pgalloc.h>
19 #if 0 /* This is just for testing */
21 follow_huge_addr(struct mm_struct
*mm
, unsigned long address
, int write
)
23 unsigned long start
= address
;
27 struct vm_area_struct
*vma
;
29 vma
= find_vma(mm
, addr
);
30 if (!vma
|| !is_vm_hugetlb_page(vma
))
31 return ERR_PTR(-EINVAL
);
33 pte
= huge_pte_offset(mm
, address
);
35 /* hugetlb should be locked, and hence, prefaulted */
36 WARN_ON(!pte
|| pte_none(*pte
));
38 page
= &pte_page(*pte
)[vpfn
% (HPAGE_SIZE
/PAGE_SIZE
)];
40 WARN_ON(!PageHead(page
));
45 int pmd_huge(pmd_t pmd
)
50 int pud_huge(pud_t pud
)
58 * pmd_huge() returns 1 if @pmd is hugetlb related entry, that is normal
59 * hugetlb entry or non-present (migration or hwpoisoned) hugetlb entry.
60 * Otherwise, returns 0.
62 int pmd_huge(pmd_t pmd
)
64 return !pmd_none(pmd
) &&
65 (pmd_val(pmd
) & (_PAGE_PRESENT
|_PAGE_PSE
)) != _PAGE_PRESENT
;
68 int pud_huge(pud_t pud
)
70 return !!(pud_val(pud
) & _PAGE_PSE
);
74 #ifdef CONFIG_HUGETLB_PAGE
75 static unsigned long hugetlb_get_unmapped_area_bottomup(struct file
*file
,
76 unsigned long addr
, unsigned long len
,
77 unsigned long pgoff
, unsigned long flags
)
79 struct hstate
*h
= hstate_file(file
);
80 struct vm_unmapped_area_info info
;
84 info
.low_limit
= current
->mm
->mmap_legacy_base
;
85 info
.high_limit
= TASK_SIZE
;
86 info
.align_mask
= PAGE_MASK
& ~huge_page_mask(h
);
87 info
.align_offset
= 0;
88 return vm_unmapped_area(&info
);
91 static unsigned long hugetlb_get_unmapped_area_topdown(struct file
*file
,
92 unsigned long addr0
, unsigned long len
,
93 unsigned long pgoff
, unsigned long flags
)
95 struct hstate
*h
= hstate_file(file
);
96 struct vm_unmapped_area_info info
;
99 info
.flags
= VM_UNMAPPED_AREA_TOPDOWN
;
101 info
.low_limit
= PAGE_SIZE
;
102 info
.high_limit
= current
->mm
->mmap_base
;
103 info
.align_mask
= PAGE_MASK
& ~huge_page_mask(h
);
104 info
.align_offset
= 0;
105 addr
= vm_unmapped_area(&info
);
108 * A failed mmap() very likely causes application failure,
109 * so fall back to the bottom-up function here. This scenario
110 * can happen with large stack limits and large mmap()
113 if (addr
& ~PAGE_MASK
) {
114 VM_BUG_ON(addr
!= -ENOMEM
);
116 info
.low_limit
= TASK_UNMAPPED_BASE
;
117 info
.high_limit
= TASK_SIZE
;
118 addr
= vm_unmapped_area(&info
);
125 hugetlb_get_unmapped_area(struct file
*file
, unsigned long addr
,
126 unsigned long len
, unsigned long pgoff
, unsigned long flags
)
128 struct hstate
*h
= hstate_file(file
);
129 struct mm_struct
*mm
= current
->mm
;
130 struct vm_area_struct
*vma
;
132 if (len
& ~huge_page_mask(h
))
137 if (flags
& MAP_FIXED
) {
138 if (prepare_hugepage_range(file
, addr
, len
))
144 addr
= ALIGN(addr
, huge_page_size(h
));
145 vma
= find_vma(mm
, addr
);
146 if (TASK_SIZE
- len
>= addr
&&
147 (!vma
|| addr
+ len
<= vma
->vm_start
))
150 if (mm
->get_unmapped_area
== arch_get_unmapped_area
)
151 return hugetlb_get_unmapped_area_bottomup(file
, addr
, len
,
154 return hugetlb_get_unmapped_area_topdown(file
, addr
, len
,
157 #endif /* CONFIG_HUGETLB_PAGE */
160 static __init
int setup_hugepagesz(char *opt
)
162 unsigned long ps
= memparse(opt
, &opt
);
163 if (ps
== PMD_SIZE
) {
164 hugetlb_add_hstate(PMD_SHIFT
- PAGE_SHIFT
);
165 } else if (ps
== PUD_SIZE
&& cpu_has_gbpages
) {
166 hugetlb_add_hstate(PUD_SHIFT
- PAGE_SHIFT
);
168 printk(KERN_ERR
"hugepagesz: Unsupported page size %lu M\n",
174 __setup("hugepagesz=", setup_hugepagesz
);
176 #if (defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || defined(CONFIG_CMA)
177 static __init
int gigantic_pages_init(void)
179 /* With compaction or CMA we can allocate gigantic pages at runtime */
180 if (cpu_has_gbpages
&& !size_to_hstate(1UL << PUD_SHIFT
))
181 hugetlb_add_hstate(PUD_SHIFT
- PAGE_SHIFT
);
184 arch_initcall(gigantic_pages_init
);