arch/s390/mm/hugetlbpage.c

   1 /*
   2  *  IBM System z Huge TLB Page Support for Kernel.
   3  *
   4  *    Copyright IBM Corp. 2007
   5  *    Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com>
   6  */
   7
   8 #include <linux/mm.h>
   9 #include <linux/hugetlb.h>
  10
  11 static inline pmd_t __pte_to_pmd(pte_t pte)
  12 {
  13         pmd_t pmd;
  14
  15         /*
  16          * Convert encoding               pte bits         pmd bits
  17          *                              .IR...wrdytp    dy..R...I...wr
  18          * empty                        .10...000000 -> 00..0...1...00
  19          * prot-none, clean, old        .11...000001 -> 00..1...1...00
  20          * prot-none, clean, young      .11...000101 -> 01..1...1...00
  21          * prot-none, dirty, old        .10...001001 -> 10..1...1...00
  22          * prot-none, dirty, young      .10...001101 -> 11..1...1...00
  23          * read-only, clean, old        .11...010001 -> 00..1...1...01
  24          * read-only, clean, young      .01...010101 -> 01..1...0...01
  25          * read-only, dirty, old        .11...011001 -> 10..1...1...01
  26          * read-only, dirty, young      .01...011101 -> 11..1...0...01
  27          * read-write, clean, old       .11...110001 -> 00..0...1...11
  28          * read-write, clean, young     .01...110101 -> 01..0...0...11
  29          * read-write, dirty, old       .10...111001 -> 10..0...1...11
  30          * read-write, dirty, young     .00...111101 -> 11..0...0...11
  31          */
  32         if (pte_present(pte)) {
  33                 pmd_val(pmd) = pte_val(pte) & PAGE_MASK;
  34                 pmd_val(pmd) |= (pte_val(pte) & _PAGE_READ) >> 4;
  35                 pmd_val(pmd) |= (pte_val(pte) & _PAGE_WRITE) >> 4;
  36                 pmd_val(pmd) |= (pte_val(pte) & _PAGE_INVALID) >> 5;
  37                 pmd_val(pmd) |= (pte_val(pte) & _PAGE_PROTECT);
  38                 pmd_val(pmd) |= (pte_val(pte) & _PAGE_DIRTY) << 10;
  39                 pmd_val(pmd) |= (pte_val(pte) & _PAGE_YOUNG) << 10;
  40         } else
  41                 pmd_val(pmd) = _SEGMENT_ENTRY_INVALID;
  42         return pmd;
  43 }
  44
  45 static inline pte_t __pmd_to_pte(pmd_t pmd)
  46 {
  47         pte_t pte;
  48
  49         /*
  50          * Convert encoding                pmd bits         pte bits
  51          *                              dy..R...I...wr    .IR...wrdytp
  52          * empty                        00..0...1...00 -> .10...001100
  53          * prot-none, clean, old        00..0...1...00 -> .10...000001
  54          * prot-none, clean, young      01..0...1...00 -> .10...000101
  55          * prot-none, dirty, old        10..0...1...00 -> .10...001001
  56          * prot-none, dirty, young      11..0...1...00 -> .10...001101
  57          * read-only, clean, old        00..1...1...01 -> .11...010001
  58          * read-only, clean, young      01..1...1...01 -> .11...010101
  59          * read-only, dirty, old        10..1...1...01 -> .11...011001
  60          * read-only, dirty, young      11..1...1...01 -> .11...011101
  61          * read-write, clean, old       00..0...1...11 -> .10...110001
  62          * read-write, clean, young     01..0...1...11 -> .10...110101
  63          * read-write, dirty, old       10..0...1...11 -> .10...111001
  64          * read-write, dirty, young     11..0...1...11 -> .10...111101
  65          */
  66         if (pmd_present(pmd)) {
  67                 pte_val(pte) = pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN_LARGE;
  68                 pte_val(pte) |= _PAGE_LARGE | _PAGE_PRESENT;
  69                 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_READ) << 4;
  70                 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_WRITE) << 4;
  71                 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_INVALID) << 5;
  72                 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_PROTECT);
  73                 pmd_val(pmd) |= (pte_val(pte) & _PAGE_DIRTY) << 10;
  74                 pmd_val(pmd) |= (pte_val(pte) & _PAGE_YOUNG) << 10;
  75         } else
  76                 pte_val(pte) = _PAGE_INVALID;
  77         return pte;
  78 }
  79
  80 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
  81                      pte_t *ptep, pte_t pte)
  82 {
  83         pmd_t pmd;
  84
  85         pmd = __pte_to_pmd(pte);
  86         if (!MACHINE_HAS_HPAGE) {
  87                 /* Emulated huge ptes loose the dirty and young bit */
  88                 pmd_val(pmd) &= ~_SEGMENT_ENTRY_ORIGIN;
  89                 pmd_val(pmd) |= pte_page(pte)[1].index;
  90         } else
  91                 pmd_val(pmd) |= _SEGMENT_ENTRY_LARGE;
  92         *(pmd_t *) ptep = pmd;
  93 }
  94
  95 pte_t huge_ptep_get(pte_t *ptep)
  96 {
  97         unsigned long origin;
  98         pmd_t pmd;
  99
 100         pmd = *(pmd_t *) ptep;
 101         if (!MACHINE_HAS_HPAGE && pmd_present(pmd)) {
 102                 origin = pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN;
 103                 pmd_val(pmd) &= ~_SEGMENT_ENTRY_ORIGIN;
 104                 pmd_val(pmd) |= *(unsigned long *) origin;
 105                 /* Emulated huge ptes are young and dirty by definition */
 106                 pmd_val(pmd) |= _SEGMENT_ENTRY_YOUNG | _SEGMENT_ENTRY_DIRTY;
 107         }
 108         return __pmd_to_pte(pmd);
 109 }
 110
 111 pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
 112                               unsigned long addr, pte_t *ptep)
 113 {
 114         pmd_t *pmdp = (pmd_t *) ptep;
 115         pte_t pte = huge_ptep_get(ptep);
 116
 117         pmdp_flush_direct(mm, addr, pmdp);
 118         pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY;
 119         return pte;
 120 }
 121
 122 int arch_prepare_hugepage(struct page *page)
 123 {
 124         unsigned long addr = page_to_phys(page);
 125         pte_t pte;
 126         pte_t *ptep;
 127         int i;
 128
 129         if (MACHINE_HAS_HPAGE)
 130                 return 0;
 131
 132         ptep = (pte_t *) pte_alloc_one(&init_mm, addr);
 133         if (!ptep)
 134                 return -ENOMEM;
 135
 136         pte_val(pte) = addr;
 137         for (i = 0; i < PTRS_PER_PTE; i++) {
 138                 set_pte_at(&init_mm, addr + i * PAGE_SIZE, ptep + i, pte);
 139                 pte_val(pte) += PAGE_SIZE;
 140         }
 141         page[1].index = (unsigned long) ptep;
 142         return 0;
 143 }
 144
 145 void arch_release_hugepage(struct page *page)
 146 {
 147         pte_t *ptep;
 148
 149         if (MACHINE_HAS_HPAGE)
 150                 return;
 151
 152         ptep = (pte_t *) page[1].index;
 153         if (!ptep)
 154                 return;
 155         clear_table((unsigned long *) ptep, _PAGE_INVALID,
 156                     PTRS_PER_PTE * sizeof(pte_t));
 157         page_table_free(&init_mm, (unsigned long *) ptep);
 158         page[1].index = 0;
 159 }
 160
 161 pte_t *huge_pte_alloc(struct mm_struct *mm,
 162                         unsigned long addr, unsigned long sz)
 163 {
 164         pgd_t *pgdp;
 165         pud_t *pudp;
 166         pmd_t *pmdp = NULL;
 167
 168         pgdp = pgd_offset(mm, addr);
 169         pudp = pud_alloc(mm, pgdp, addr);
 170         if (pudp)
 171                 pmdp = pmd_alloc(mm, pudp, addr);
 172         return (pte_t *) pmdp;
 173 }
 174
 175 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
 176 {
 177         pgd_t *pgdp;
 178         pud_t *pudp;
 179         pmd_t *pmdp = NULL;
 180
 181         pgdp = pgd_offset(mm, addr);
 182         if (pgd_present(*pgdp)) {
 183                 pudp = pud_offset(pgdp, addr);
 184                 if (pud_present(*pudp))
 185                         pmdp = pmd_offset(pudp, addr);
 186         }
 187         return (pte_t *) pmdp;
 188 }
 189
 190 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
 191 {
 192         return 0;
 193 }
 194
 195 struct page *follow_huge_addr(struct mm_struct *mm, unsigned long address,
 196                               int write)
 197 {
 198         return ERR_PTR(-EINVAL);
 199 }
 200
 201 int pmd_huge(pmd_t pmd)
 202 {
 203         if (!MACHINE_HAS_HPAGE)
 204                 return 0;
 205
 206         return !!(pmd_val(pmd) & _SEGMENT_ENTRY_LARGE);
 207 }
 208
 209 int pud_huge(pud_t pud)
 210 {
 211         return 0;
 212 }
 213
 214 struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
 215                              pmd_t *pmdp, int write)
 216 {
 217         struct page *page;
 218
 219         if (!MACHINE_HAS_HPAGE)
 220                 return NULL;
 221
 222         page = pmd_page(*pmdp);
 223         if (page)
 224                 page += ((address & ~HPAGE_MASK) >> PAGE_SHIFT);
 225         return page;
 226 }