mm/kasan/kasan_init.c

   1 /*
   2  * This file contains some kasan initialization code.
   3  *
   4  * Copyright (c) 2015 Samsung Electronics Co., Ltd.
   5  * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  *
  11  */
  12
  13 #include <linux/bootmem.h>
  14 #include <linux/init.h>
  15 #include <linux/kasan.h>
  16 #include <linux/kernel.h>
  17 #include <linux/memblock.h>
  18 #include <linux/mm.h>
  19 #include <linux/pfn.h>
  20
  21 #include <asm/page.h>
  22 #include <asm/pgalloc.h>
  23
  24 /*
  25  * This page serves two purposes:
  26  *   - It used as early shadow memory. The entire shadow region populated
  27  *     with this page, before we will be able to setup normal shadow memory.
  28  *   - Latter it reused it as zero shadow to cover large ranges of memory
  29  *     that allowed to access, but not handled by kasan (vmalloc/vmemmap ...).
  30  */
  31 unsigned char kasan_zero_page[PAGE_SIZE] __page_aligned_bss;
  32
  33 #if CONFIG_PGTABLE_LEVELS > 4
  34 p4d_t kasan_zero_p4d[PTRS_PER_P4D] __page_aligned_bss;
  35 #endif
  36 #if CONFIG_PGTABLE_LEVELS > 3
  37 pud_t kasan_zero_pud[PTRS_PER_PUD] __page_aligned_bss;
  38 #endif
  39 #if CONFIG_PGTABLE_LEVELS > 2
  40 pmd_t kasan_zero_pmd[PTRS_PER_PMD] __page_aligned_bss;
  41 #endif
  42 pte_t kasan_zero_pte[PTRS_PER_PTE] __page_aligned_bss;
  43
  44 static __init void *early_alloc(size_t size, int node)
  45 {
  46         return memblock_virt_alloc_try_nid(size, size, __pa(MAX_DMA_ADDRESS),
  47                                         BOOTMEM_ALLOC_ACCESSIBLE, node);
  48 }
  49
  50 static void __init zero_pte_populate(pmd_t *pmd, unsigned long addr,
  51                                 unsigned long end)
  52 {
  53         pte_t *pte = pte_offset_kernel(pmd, addr);
  54         pte_t zero_pte;
  55
  56         zero_pte = pfn_pte(PFN_DOWN(__pa_symbol(kasan_zero_page)), PAGE_KERNEL);
  57         zero_pte = pte_wrprotect(zero_pte);
  58
  59         while (addr + PAGE_SIZE <= end) {
  60                 set_pte_at(&init_mm, addr, pte, zero_pte);
  61                 addr += PAGE_SIZE;
  62                 pte = pte_offset_kernel(pmd, addr);
  63         }
  64 }
  65
  66 static void __init zero_pmd_populate(pud_t *pud, unsigned long addr,
  67                                 unsigned long end)
  68 {
  69         pmd_t *pmd = pmd_offset(pud, addr);
  70         unsigned long next;
  71
  72         do {
  73                 next = pmd_addr_end(addr, end);
  74
  75                 if (IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) {
  76                         pmd_populate_kernel(&init_mm, pmd, lm_alias(kasan_zero_pte));
  77                         continue;
  78                 }
  79
  80                 if (pmd_none(*pmd)) {
  81                         pmd_populate_kernel(&init_mm, pmd,
  82                                         early_alloc(PAGE_SIZE, NUMA_NO_NODE));
  83                 }
  84                 zero_pte_populate(pmd, addr, next);
  85         } while (pmd++, addr = next, addr != end);
  86 }
  87
  88 static void __init zero_pud_populate(p4d_t *p4d, unsigned long addr,
  89                                 unsigned long end)
  90 {
  91         pud_t *pud = pud_offset(p4d, addr);
  92         unsigned long next;
  93
  94         do {
  95                 next = pud_addr_end(addr, end);
  96                 if (IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) {
  97                         pmd_t *pmd;
  98
  99                         pud_populate(&init_mm, pud, lm_alias(kasan_zero_pmd));
 100                         pmd = pmd_offset(pud, addr);
 101                         pmd_populate_kernel(&init_mm, pmd, lm_alias(kasan_zero_pte));
 102                         continue;
 103                 }
 104
 105                 if (pud_none(*pud)) {
 106                         pud_populate(&init_mm, pud,
 107                                 early_alloc(PAGE_SIZE, NUMA_NO_NODE));
 108                 }
 109                 zero_pmd_populate(pud, addr, next);
 110         } while (pud++, addr = next, addr != end);
 111 }
 112
 113 static void __init zero_p4d_populate(pgd_t *pgd, unsigned long addr,
 114                                 unsigned long end)
 115 {
 116         p4d_t *p4d = p4d_offset(pgd, addr);
 117         unsigned long next;
 118
 119         do {
 120                 next = p4d_addr_end(addr, end);
 121                 if (IS_ALIGNED(addr, P4D_SIZE) && end - addr >= P4D_SIZE) {
 122                         pud_t *pud;
 123                         pmd_t *pmd;
 124
 125                         p4d_populate(&init_mm, p4d, lm_alias(kasan_zero_pud));
 126                         pud = pud_offset(p4d, addr);
 127                         pud_populate(&init_mm, pud, lm_alias(kasan_zero_pmd));
 128                         pmd = pmd_offset(pud, addr);
 129                         pmd_populate_kernel(&init_mm, pmd,
 130                                                 lm_alias(kasan_zero_pte));
 131                         continue;
 132                 }
 133
 134                 if (p4d_none(*p4d)) {
 135                         p4d_populate(&init_mm, p4d,
 136                                 early_alloc(PAGE_SIZE, NUMA_NO_NODE));
 137                 }
 138                 zero_pud_populate(p4d, addr, next);
 139         } while (p4d++, addr = next, addr != end);
 140 }
 141
 142 /**
 143  * kasan_populate_zero_shadow - populate shadow memory region with
 144  *                               kasan_zero_page
 145  * @shadow_start - start of the memory range to populate
 146  * @shadow_end   - end of the memory range to populate
 147  */
 148 void __init kasan_populate_zero_shadow(const void *shadow_start,
 149                                 const void *shadow_end)
 150 {
 151         unsigned long addr = (unsigned long)shadow_start;
 152         unsigned long end = (unsigned long)shadow_end;
 153         pgd_t *pgd = pgd_offset_k(addr);
 154         unsigned long next;
 155
 156         do {
 157                 next = pgd_addr_end(addr, end);
 158
 159                 if (IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) {
 160                         p4d_t *p4d;
 161                         pud_t *pud;
 162                         pmd_t *pmd;
 163
 164                         /*
 165                          * kasan_zero_pud should be populated with pmds
 166                          * at this moment.
 167                          * [pud,pmd]_populate*() below needed only for
 168                          * 3,2 - level page tables where we don't have
 169                          * puds,pmds, so pgd_populate(), pud_populate()
 170                          * is noops.
 171                          *
 172                          * The ifndef is required to avoid build breakage.
 173                          *
 174                          * With 5level-fixup.h, pgd_populate() is not nop and
 175                          * we reference kasan_zero_p4d. It's not defined
 176                          * unless 5-level paging enabled.
 177                          *
 178                          * The ifndef can be dropped once all KASAN-enabled
 179                          * architectures will switch to pgtable-nop4d.h.
 180                          */
 181 #ifndef __ARCH_HAS_5LEVEL_HACK
 182                         pgd_populate(&init_mm, pgd, lm_alias(kasan_zero_p4d));
 183 #endif
 184                         p4d = p4d_offset(pgd, addr);
 185                         p4d_populate(&init_mm, p4d, lm_alias(kasan_zero_pud));
 186                         pud = pud_offset(p4d, addr);
 187                         pud_populate(&init_mm, pud, lm_alias(kasan_zero_pmd));
 188                         pmd = pmd_offset(pud, addr);
 189                         pmd_populate_kernel(&init_mm, pmd, lm_alias(kasan_zero_pte));
 190                         continue;
 191                 }
 192
 193                 if (pgd_none(*pgd)) {
 194                         pgd_populate(&init_mm, pgd,
 195                                 early_alloc(PAGE_SIZE, NUMA_NO_NODE));
 196                 }
 197                 zero_p4d_populate(pgd, addr, next);
 198         } while (pgd++, addr = next, addr != end);
 199 }