arch/x86/mm/hugetlbpage.c

   1 /*
   2  * IA-32 Huge TLB Page Support for Kernel.
   3  *
   4  * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
   5  */
   6
   7 #include <linux/init.h>
   8 #include <linux/fs.h>
   9 #include <linux/mm.h>
  10 #include <linux/hugetlb.h>
  11 #include <linux/pagemap.h>
  12 #include <linux/err.h>
  13 #include <linux/sysctl.h>
  14 #include <asm/mman.h>
  15 #include <asm/tlb.h>
  16 #include <asm/tlbflush.h>
  17 #include <asm/pgalloc.h>
  18
  19 #if 0   /* This is just for testing */
  20 struct page *
  21 follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)
  22 {
  23         unsigned long start = address;
  24         int length = 1;
  25         int nr;
  26         struct page *page;
  27         struct vm_area_struct *vma;
  28
  29         vma = find_vma(mm, addr);
  30         if (!vma || !is_vm_hugetlb_page(vma))
  31                 return ERR_PTR(-EINVAL);
  32
  33         pte = huge_pte_offset(mm, address);
  34
  35         /* hugetlb should be locked, and hence, prefaulted */
  36         WARN_ON(!pte || pte_none(*pte));
  37
  38         page = &pte_page(*pte)[vpfn % (HPAGE_SIZE/PAGE_SIZE)];
  39
  40         WARN_ON(!PageHead(page));
  41
  42         return page;
  43 }
  44
  45 int pmd_huge(pmd_t pmd)
  46 {
  47         return 0;
  48 }
  49
  50 int pud_huge(pud_t pud)
  51 {
  52         return 0;
  53 }
  54
  55 #else
  56
  57 /*
  58  * pmd_huge() returns 1 if @pmd is hugetlb related entry, that is normal
  59  * hugetlb entry or non-present (migration or hwpoisoned) hugetlb entry.
  60  * Otherwise, returns 0.
  61  */
  62 int pmd_huge(pmd_t pmd)
  63 {
  64         return !pmd_none(pmd) &&
  65                 (pmd_val(pmd) & (_PAGE_PRESENT|_PAGE_PSE)) != _PAGE_PRESENT;
  66 }
  67
  68 int pud_huge(pud_t pud)
  69 {
  70         return !!(pud_val(pud) & _PAGE_PSE);
  71 }
  72 #endif
  73
  74 /* x86_64 also uses this file */
  75
  76 #ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
  77 static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file,
  78                 unsigned long addr, unsigned long len,
  79                 unsigned long pgoff, unsigned long flags)
  80 {
  81         struct hstate *h = hstate_file(file);
  82         struct vm_unmapped_area_info info;
  83
  84         info.flags = 0;
  85         info.length = len;
  86         info.low_limit = TASK_UNMAPPED_BASE;
  87         info.high_limit = TASK_SIZE;
  88         info.align_mask = PAGE_MASK & ~huge_page_mask(h);
  89         info.align_offset = 0;
  90         return vm_unmapped_area(&info);
  91 }
  92
  93 static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file,
  94                 unsigned long addr0, unsigned long len,
  95                 unsigned long pgoff, unsigned long flags)
  96 {
  97         struct hstate *h = hstate_file(file);
  98         struct vm_unmapped_area_info info;
  99         unsigned long addr;
 100
 101         info.flags = VM_UNMAPPED_AREA_TOPDOWN;
 102         info.length = len;
 103         info.low_limit = PAGE_SIZE;
 104         info.high_limit = current->mm->mmap_base;
 105         info.align_mask = PAGE_MASK & ~huge_page_mask(h);
 106         info.align_offset = 0;
 107         addr = vm_unmapped_area(&info);
 108
 109         /*
 110          * A failed mmap() very likely causes application failure,
 111          * so fall back to the bottom-up function here. This scenario
 112          * can happen with large stack limits and large mmap()
 113          * allocations.
 114          */
 115         if (addr & ~PAGE_MASK) {
 116                 VM_BUG_ON(addr != -ENOMEM);
 117                 info.flags = 0;
 118                 info.low_limit = TASK_UNMAPPED_BASE;
 119                 info.high_limit = TASK_SIZE;
 120                 addr = vm_unmapped_area(&info);
 121         }
 122
 123         return addr;
 124 }
 125
 126 unsigned long
 127 hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
 128                 unsigned long len, unsigned long pgoff, unsigned long flags)
 129 {
 130         struct hstate *h = hstate_file(file);
 131         struct mm_struct *mm = current->mm;
 132         struct vm_area_struct *vma;
 133
 134         if (len & ~huge_page_mask(h))
 135                 return -EINVAL;
 136         if (len > TASK_SIZE)
 137                 return -ENOMEM;
 138
 139         if (flags & MAP_FIXED) {
 140                 if (prepare_hugepage_range(file, addr, len))
 141                         return -EINVAL;
 142                 return addr;
 143         }
 144
 145         if (addr) {
 146                 addr = ALIGN(addr, huge_page_size(h));
 147                 vma = find_vma(mm, addr);
 148                 if (TASK_SIZE - len >= addr &&
 149                     (!vma || addr + len <= vma->vm_start))
 150                         return addr;
 151         }
 152         if (mm->get_unmapped_area == arch_get_unmapped_area)
 153                 return hugetlb_get_unmapped_area_bottomup(file, addr, len,
 154                                 pgoff, flags);
 155         else
 156                 return hugetlb_get_unmapped_area_topdown(file, addr, len,
 157                                 pgoff, flags);
 158 }
 159
 160 #endif /*HAVE_ARCH_HUGETLB_UNMAPPED_AREA*/
 161
 162 #ifdef CONFIG_X86_64
 163 static __init int setup_hugepagesz(char *opt)
 164 {
 165         unsigned long ps = memparse(opt, &opt);
 166         if (ps == PMD_SIZE) {
 167                 hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
 168         } else if (ps == PUD_SIZE && cpu_has_gbpages) {
 169                 hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
 170         } else {
 171                 printk(KERN_ERR "hugepagesz: Unsupported page size %lu M\n",
 172                         ps >> 20);
 173                 return 0;
 174         }
 175         return 1;
 176 }
 177 __setup("hugepagesz=", setup_hugepagesz);
 178 #endif