arch/powerpc/mm/pgtable-frag.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  *  Handling Page Tables through page fragments
   5  *
   6  */
   7
   8 #include <linux/kernel.h>
   9 #include <linux/gfp.h>
  10 #include <linux/mm.h>
  11 #include <linux/percpu.h>
  12 #include <linux/hardirq.h>
  13 #include <linux/hugetlb.h>
  14 #include <asm/pgalloc.h>
  15 #include <asm/tlbflush.h>
  16 #include <asm/tlb.h>
  17
  18 void pte_frag_destroy(void *pte_frag)
  19 {
  20         int count;
  21         struct ptdesc *ptdesc;
  22
  23         ptdesc = virt_to_ptdesc(pte_frag);
  24         /* drop all the pending references */
  25         count = ((unsigned long)pte_frag & ~PAGE_MASK) >> PTE_FRAG_SIZE_SHIFT;
  26         /* We allow PTE_FRAG_NR fragments from a PTE page */
  27         if (atomic_sub_and_test(PTE_FRAG_NR - count, &ptdesc->pt_frag_refcount)) {
  28                 pagetable_pte_dtor(ptdesc);
  29                 pagetable_free(ptdesc);
  30         }
  31 }
  32
  33 static pte_t *get_pte_from_cache(struct mm_struct *mm)
  34 {
  35         void *pte_frag, *ret;
  36
  37         if (PTE_FRAG_NR == 1)
  38                 return NULL;
  39
  40         spin_lock(&mm->page_table_lock);
  41         ret = pte_frag_get(&mm->context);
  42         if (ret) {
  43                 pte_frag = ret + PTE_FRAG_SIZE;
  44                 /*
  45                  * If we have taken up all the fragments mark PTE page NULL
  46                  */
  47                 if (((unsigned long)pte_frag & ~PAGE_MASK) == 0)
  48                         pte_frag = NULL;
  49                 pte_frag_set(&mm->context, pte_frag);
  50         }
  51         spin_unlock(&mm->page_table_lock);
  52         return (pte_t *)ret;
  53 }
  54
  55 static pte_t *__alloc_for_ptecache(struct mm_struct *mm, int kernel)
  56 {
  57         void *ret = NULL;
  58         struct ptdesc *ptdesc;
  59
  60         if (!kernel) {
  61                 ptdesc = pagetable_alloc(PGALLOC_GFP | __GFP_ACCOUNT, 0);
  62                 if (!ptdesc)
  63                         return NULL;
  64                 if (!pagetable_pte_ctor(ptdesc)) {
  65                         pagetable_free(ptdesc);
  66                         return NULL;
  67                 }
  68         } else {
  69                 ptdesc = pagetable_alloc(PGALLOC_GFP, 0);
  70                 if (!ptdesc)
  71                         return NULL;
  72         }
  73
  74         atomic_set(&ptdesc->pt_frag_refcount, 1);
  75
  76         ret = ptdesc_address(ptdesc);
  77         /*
  78          * if we support only one fragment just return the
  79          * allocated page.
  80          */
  81         if (PTE_FRAG_NR == 1)
  82                 return ret;
  83         spin_lock(&mm->page_table_lock);
  84         /*
  85          * If we find ptdesc_page set, we return
  86          * the allocated page with single fragment
  87          * count.
  88          */
  89         if (likely(!pte_frag_get(&mm->context))) {
  90                 atomic_set(&ptdesc->pt_frag_refcount, PTE_FRAG_NR);
  91                 pte_frag_set(&mm->context, ret + PTE_FRAG_SIZE);
  92         }
  93         spin_unlock(&mm->page_table_lock);
  94
  95         return (pte_t *)ret;
  96 }
  97
  98 pte_t *pte_fragment_alloc(struct mm_struct *mm, int kernel)
  99 {
 100         pte_t *pte;
 101
 102         pte = get_pte_from_cache(mm);
 103         if (pte)
 104                 return pte;
 105
 106         return __alloc_for_ptecache(mm, kernel);
 107 }
 108
 109 static void pte_free_now(struct rcu_head *head)
 110 {
 111         struct ptdesc *ptdesc;
 112
 113         ptdesc = container_of(head, struct ptdesc, pt_rcu_head);
 114         pagetable_pte_dtor(ptdesc);
 115         pagetable_free(ptdesc);
 116 }
 117
 118 void pte_fragment_free(unsigned long *table, int kernel)
 119 {
 120         struct ptdesc *ptdesc = virt_to_ptdesc(table);
 121
 122         if (pagetable_is_reserved(ptdesc))
 123                 return free_reserved_ptdesc(ptdesc);
 124
 125         BUG_ON(atomic_read(&ptdesc->pt_frag_refcount) <= 0);
 126         if (atomic_dec_and_test(&ptdesc->pt_frag_refcount)) {
 127                 if (kernel)
 128                         pagetable_free(ptdesc);
 129                 else if (folio_test_clear_active(ptdesc_folio(ptdesc)))
 130                         call_rcu(&ptdesc->pt_rcu_head, pte_free_now);
 131                 else
 132                         pte_free_now(&ptdesc->pt_rcu_head);
 133         }
 134 }
 135
 136 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 137 void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable)
 138 {
 139         struct folio *folio;
 140
 141         folio = virt_to_folio(pgtable);
 142         folio_set_active(folio);
 143         pte_fragment_free((unsigned long *)pgtable, 0);
 144 }
 145 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */