mm/ptdump.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include <linux/pagewalk.h>
   4 #include <linux/ptdump.h>
   5 #include <linux/kasan.h>
   6
   7 #ifdef CONFIG_KASAN
   8 /*
   9  * This is an optimization for KASAN=y case. Since all kasan page tables
  10  * eventually point to the kasan_early_shadow_page we could call note_page()
  11  * right away without walking through lower level page tables. This saves
  12  * us dozens of seconds (minutes for 5-level config) while checking for
  13  * W+X mapping or reading kernel_page_tables debugfs file.
  14  */
  15 static inline int note_kasan_page_table(struct mm_walk *walk,
  16                                         unsigned long addr)
  17 {
  18         struct ptdump_state *st = walk->private;
  19
  20         st->note_page(st, addr, 4, pte_val(kasan_early_shadow_pte[0]));
  21
  22         walk->action = ACTION_CONTINUE;
  23
  24         return 0;
  25 }
  26 #endif
  27
  28 static int ptdump_pgd_entry(pgd_t *pgd, unsigned long addr,
  29                             unsigned long next, struct mm_walk *walk)
  30 {
  31         struct ptdump_state *st = walk->private;
  32         pgd_t val = READ_ONCE(*pgd);
  33
  34 #if CONFIG_PGTABLE_LEVELS > 4 && defined(CONFIG_KASAN)
  35         if (pgd_page(val) == virt_to_page(lm_alias(kasan_early_shadow_p4d)))
  36                 return note_kasan_page_table(walk, addr);
  37 #endif
  38
  39         if (pgd_leaf(val))
  40                 st->note_page(st, addr, 0, pgd_val(val));
  41
  42         return 0;
  43 }
  44
  45 static int ptdump_p4d_entry(p4d_t *p4d, unsigned long addr,
  46                             unsigned long next, struct mm_walk *walk)
  47 {
  48         struct ptdump_state *st = walk->private;
  49         p4d_t val = READ_ONCE(*p4d);
  50
  51 #if CONFIG_PGTABLE_LEVELS > 3 && defined(CONFIG_KASAN)
  52         if (p4d_page(val) == virt_to_page(lm_alias(kasan_early_shadow_pud)))
  53                 return note_kasan_page_table(walk, addr);
  54 #endif
  55
  56         if (p4d_leaf(val))
  57                 st->note_page(st, addr, 1, p4d_val(val));
  58
  59         return 0;
  60 }
  61
  62 static int ptdump_pud_entry(pud_t *pud, unsigned long addr,
  63                             unsigned long next, struct mm_walk *walk)
  64 {
  65         struct ptdump_state *st = walk->private;
  66         pud_t val = READ_ONCE(*pud);
  67
  68 #if CONFIG_PGTABLE_LEVELS > 2 && defined(CONFIG_KASAN)
  69         if (pud_page(val) == virt_to_page(lm_alias(kasan_early_shadow_pmd)))
  70                 return note_kasan_page_table(walk, addr);
  71 #endif
  72
  73         if (pud_leaf(val))
  74                 st->note_page(st, addr, 2, pud_val(val));
  75
  76         return 0;
  77 }
  78
  79 static int ptdump_pmd_entry(pmd_t *pmd, unsigned long addr,
  80                             unsigned long next, struct mm_walk *walk)
  81 {
  82         struct ptdump_state *st = walk->private;
  83         pmd_t val = READ_ONCE(*pmd);
  84
  85 #if defined(CONFIG_KASAN)
  86         if (pmd_page(val) == virt_to_page(lm_alias(kasan_early_shadow_pte)))
  87                 return note_kasan_page_table(walk, addr);
  88 #endif
  89
  90         if (pmd_leaf(val))
  91                 st->note_page(st, addr, 3, pmd_val(val));
  92
  93         return 0;
  94 }
  95
  96 static int ptdump_pte_entry(pte_t *pte, unsigned long addr,
  97                             unsigned long next, struct mm_walk *walk)
  98 {
  99         struct ptdump_state *st = walk->private;
 100
 101         st->note_page(st, addr, 4, pte_val(READ_ONCE(*pte)));
 102
 103         return 0;
 104 }
 105
 106 static int ptdump_hole(unsigned long addr, unsigned long next,
 107                        int depth, struct mm_walk *walk)
 108 {
 109         struct ptdump_state *st = walk->private;
 110
 111         st->note_page(st, addr, depth, 0);
 112
 113         return 0;
 114 }
 115
 116 static const struct mm_walk_ops ptdump_ops = {
 117         .pgd_entry      = ptdump_pgd_entry,
 118         .p4d_entry      = ptdump_p4d_entry,
 119         .pud_entry      = ptdump_pud_entry,
 120         .pmd_entry      = ptdump_pmd_entry,
 121         .pte_entry      = ptdump_pte_entry,
 122         .pte_hole       = ptdump_hole,
 123 };
 124
 125 void ptdump_walk_pgd(struct ptdump_state *st, struct mm_struct *mm, pgd_t *pgd)
 126 {
 127         const struct ptdump_range *range = st->range;
 128
 129         down_read(&mm->mmap_sem);
 130         while (range->start != range->end) {
 131                 walk_page_range_novma(mm, range->start, range->end,
 132                                       &ptdump_ops, pgd, st);
 133                 range++;
 134         }
 135         up_read(&mm->mmap_sem);
 136
 137         /* Flush out the last page */
 138         st->note_page(st, 0, -1, 0);
 139 }