arch/arm/mm/kasan_init.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * This file contains kasan initialization code for ARM.
   4  *
   5  * Copyright (c) 2018 Samsung Electronics Co., Ltd.
   6  * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
   7  * Author: Linus Walleij <linus.walleij@linaro.org>
   8  */
   9
  10 #define pr_fmt(fmt) "kasan: " fmt
  11 #include <linux/kasan.h>
  12 #include <linux/kernel.h>
  13 #include <linux/memblock.h>
  14 #include <linux/sched/task.h>
  15 #include <linux/start_kernel.h>
  16 #include <linux/pgtable.h>
  17 #include <asm/cputype.h>
  18 #include <asm/highmem.h>
  19 #include <asm/mach/map.h>
  20 #include <asm/page.h>
  21 #include <asm/pgalloc.h>
  22 #include <asm/procinfo.h>
  23 #include <asm/proc-fns.h>
  24
  25 #include "mm.h"
  26
  27 static pgd_t tmp_pgd_table[PTRS_PER_PGD] __initdata __aligned(PGD_SIZE);
  28
  29 pmd_t tmp_pmd_table[PTRS_PER_PMD] __page_aligned_bss;
  30
  31 static __init void *kasan_alloc_block_raw(size_t size)
  32 {
  33         return memblock_alloc_try_nid_raw(size, size, __pa(MAX_DMA_ADDRESS),
  34                                       MEMBLOCK_ALLOC_NOLEAKTRACE, NUMA_NO_NODE);
  35 }
  36
  37 static __init void *kasan_alloc_block(size_t size)
  38 {
  39         return memblock_alloc_try_nid(size, size, __pa(MAX_DMA_ADDRESS),
  40                                       MEMBLOCK_ALLOC_NOLEAKTRACE, NUMA_NO_NODE);
  41 }
  42
  43 static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr,
  44                                       unsigned long end, bool early)
  45 {
  46         unsigned long next;
  47         pte_t *ptep = pte_offset_kernel(pmdp, addr);
  48
  49         do {
  50                 pte_t entry;
  51                 void *p;
  52
  53                 next = addr + PAGE_SIZE;
  54
  55                 if (!early) {
  56                         if (!pte_none(READ_ONCE(*ptep)))
  57                                 continue;
  58
  59                         p = kasan_alloc_block_raw(PAGE_SIZE);
  60                         if (!p) {
  61                                 panic("%s failed to allocate shadow page for address 0x%lx\n",
  62                                       __func__, addr);
  63                                 return;
  64                         }
  65                         memset(p, KASAN_SHADOW_INIT, PAGE_SIZE);
  66                         entry = pfn_pte(virt_to_pfn(p),
  67                                         __pgprot(pgprot_val(PAGE_KERNEL)));
  68                 } else if (pte_none(READ_ONCE(*ptep))) {
  69                         /*
  70                          * The early shadow memory is mapping all KASan
  71                          * operations to one and the same page in memory,
  72                          * "kasan_early_shadow_page" so that the instrumentation
  73                          * will work on a scratch area until we can set up the
  74                          * proper KASan shadow memory.
  75                          */
  76                         entry = pfn_pte(virt_to_pfn(kasan_early_shadow_page),
  77                                         __pgprot(_L_PTE_DEFAULT | L_PTE_DIRTY | L_PTE_XN));
  78                 } else {
  79                         /*
  80                          * Early shadow mappings are PMD_SIZE aligned, so if the
  81                          * first entry is already set, they must all be set.
  82                          */
  83                         return;
  84                 }
  85
  86                 set_pte_at(&init_mm, addr, ptep, entry);
  87         } while (ptep++, addr = next, addr != end);
  88 }
  89
  90 /*
  91  * The pmd (page middle directory) is only used on LPAE
  92  */
  93 static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr,
  94                                       unsigned long end, bool early)
  95 {
  96         unsigned long next;
  97         pmd_t *pmdp = pmd_offset(pudp, addr);
  98
  99         do {
 100                 if (pmd_none(*pmdp)) {
 101                         /*
 102                          * We attempt to allocate a shadow block for the PMDs
 103                          * used by the PTEs for this address if it isn't already
 104                          * allocated.
 105                          */
 106                         void *p = early ? kasan_early_shadow_pte :
 107                                 kasan_alloc_block(PAGE_SIZE);
 108
 109                         if (!p) {
 110                                 panic("%s failed to allocate shadow block for address 0x%lx\n",
 111                                       __func__, addr);
 112                                 return;
 113                         }
 114                         pmd_populate_kernel(&init_mm, pmdp, p);
 115                         flush_pmd_entry(pmdp);
 116                 }
 117
 118                 next = pmd_addr_end(addr, end);
 119                 kasan_pte_populate(pmdp, addr, next, early);
 120         } while (pmdp++, addr = next, addr != end);
 121 }
 122
 123 static void __init kasan_pgd_populate(unsigned long addr, unsigned long end,
 124                                       bool early)
 125 {
 126         unsigned long next;
 127         pgd_t *pgdp;
 128         p4d_t *p4dp;
 129         pud_t *pudp;
 130
 131         pgdp = pgd_offset_k(addr);
 132
 133         do {
 134                 /*
 135                  * Allocate and populate the shadow block of p4d folded into
 136                  * pud folded into pmd if it doesn't already exist
 137                  */
 138                 if (!early && pgd_none(*pgdp)) {
 139                         void *p = kasan_alloc_block(PAGE_SIZE);
 140
 141                         if (!p) {
 142                                 panic("%s failed to allocate shadow block for address 0x%lx\n",
 143                                       __func__, addr);
 144                                 return;
 145                         }
 146                         pgd_populate(&init_mm, pgdp, p);
 147                 }
 148
 149                 next = pgd_addr_end(addr, end);
 150                 /*
 151                  * We just immediately jump over the p4d and pud page
 152                  * directories since we believe ARM32 will never gain four
 153                  * nor five level page tables.
 154                  */
 155                 p4dp = p4d_offset(pgdp, addr);
 156                 pudp = pud_offset(p4dp, addr);
 157
 158                 kasan_pmd_populate(pudp, addr, next, early);
 159         } while (pgdp++, addr = next, addr != end);
 160 }
 161
 162 extern struct proc_info_list *lookup_processor_type(unsigned int);
 163
 164 void __init kasan_early_init(void)
 165 {
 166         struct proc_info_list *list;
 167
 168         /*
 169          * locate processor in the list of supported processor
 170          * types.  The linker builds this table for us from the
 171          * entries in arch/arm/mm/proc-*.S
 172          */
 173         list = lookup_processor_type(read_cpuid_id());
 174         if (list) {
 175 #ifdef MULTI_CPU
 176                 processor = *list->proc;
 177 #endif
 178         }
 179
 180         BUILD_BUG_ON((KASAN_SHADOW_END - (1UL << 29)) != KASAN_SHADOW_OFFSET);
 181         /*
 182          * We walk the page table and set all of the shadow memory to point
 183          * to the scratch page.
 184          */
 185         kasan_pgd_populate(KASAN_SHADOW_START, KASAN_SHADOW_END, true);
 186 }
 187
 188 static void __init clear_pgds(unsigned long start,
 189                         unsigned long end)
 190 {
 191         for (; start && start < end; start += PMD_SIZE)
 192                 pmd_clear(pmd_off_k(start));
 193 }
 194
 195 static int __init create_mapping(void *start, void *end)
 196 {
 197         void *shadow_start, *shadow_end;
 198
 199         shadow_start = kasan_mem_to_shadow(start);
 200         shadow_end = kasan_mem_to_shadow(end);
 201
 202         pr_info("Mapping kernel virtual memory block: %px-%px at shadow: %px-%px\n",
 203                 start, end, shadow_start, shadow_end);
 204
 205         kasan_pgd_populate((unsigned long)shadow_start & PAGE_MASK,
 206                            PAGE_ALIGN((unsigned long)shadow_end), false);
 207         return 0;
 208 }
 209
 210 void __init kasan_init(void)
 211 {
 212         phys_addr_t pa_start, pa_end;
 213         u64 i;
 214
 215         /*
 216          * We are going to perform proper setup of shadow memory.
 217          *
 218          * At first we should unmap early shadow (clear_pgds() call bellow).
 219          * However, instrumented code can't execute without shadow memory.
 220          *
 221          * To keep the early shadow memory MMU tables around while setting up
 222          * the proper shadow memory, we copy swapper_pg_dir (the initial page
 223          * table) to tmp_pgd_table and use that to keep the early shadow memory
 224          * mapped until the full shadow setup is finished. Then we swap back
 225          * to the proper swapper_pg_dir.
 226          */
 227
 228         memcpy(tmp_pgd_table, swapper_pg_dir, sizeof(tmp_pgd_table));
 229 #ifdef CONFIG_ARM_LPAE
 230         /* We need to be in the same PGD or this won't work */
 231         BUILD_BUG_ON(pgd_index(KASAN_SHADOW_START) !=
 232                      pgd_index(KASAN_SHADOW_END));
 233         memcpy(tmp_pmd_table,
 234                (void*)pgd_page_vaddr(*pgd_offset_k(KASAN_SHADOW_START)),
 235                sizeof(tmp_pmd_table));
 236         set_pgd(&tmp_pgd_table[pgd_index(KASAN_SHADOW_START)],
 237                 __pgd(__pa(tmp_pmd_table) | PMD_TYPE_TABLE | L_PGD_SWAPPER));
 238 #endif
 239         cpu_switch_mm(tmp_pgd_table, &init_mm);
 240         local_flush_tlb_all();
 241
 242         clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END);
 243
 244         if (!IS_ENABLED(CONFIG_KASAN_VMALLOC))
 245                 kasan_populate_early_shadow(kasan_mem_to_shadow((void *)VMALLOC_START),
 246                                             kasan_mem_to_shadow((void *)VMALLOC_END));
 247
 248         kasan_populate_early_shadow(kasan_mem_to_shadow((void *)VMALLOC_END),
 249                                     kasan_mem_to_shadow((void *)-1UL) + 1);
 250
 251         for_each_mem_range(i, &pa_start, &pa_end) {
 252                 void *start = __va(pa_start);
 253                 void *end = __va(pa_end);
 254
 255                 /* Do not attempt to shadow highmem */
 256                 if (pa_start >= arm_lowmem_limit) {
 257                         pr_info("Skip highmem block at %pa-%pa\n", &pa_start, &pa_end);
 258                         continue;
 259                 }
 260                 if (pa_end > arm_lowmem_limit) {
 261                         pr_info("Truncating shadow for memory block at %pa-%pa to lowmem region at %pa\n",
 262                                 &pa_start, &pa_end, &arm_lowmem_limit);
 263                         end = __va(arm_lowmem_limit);
 264                 }
 265                 if (start >= end) {
 266                         pr_info("Skipping invalid memory block %pa-%pa (virtual %p-%p)\n",
 267                                 &pa_start, &pa_end, start, end);
 268                         continue;
 269                 }
 270
 271                 create_mapping(start, end);
 272         }
 273
 274         /*
 275          * 1. The module global variables are in MODULES_VADDR ~ MODULES_END,
 276          *    so we need to map this area if CONFIG_KASAN_VMALLOC=n. With
 277          *    VMALLOC support KASAN will manage this region dynamically,
 278          *    refer to kasan_populate_vmalloc() and ARM's implementation of
 279          *    module_alloc().
 280          * 2. PKMAP_BASE ~ PKMAP_BASE+PMD_SIZE's shadow and MODULES_VADDR
 281          *    ~ MODULES_END's shadow is in the same PMD_SIZE, so we can't
 282          *    use kasan_populate_zero_shadow.
 283          */
 284         if (!IS_ENABLED(CONFIG_KASAN_VMALLOC) && IS_ENABLED(CONFIG_MODULES))
 285                 create_mapping((void *)MODULES_VADDR, (void *)(MODULES_END));
 286         create_mapping((void *)PKMAP_BASE, (void *)(PKMAP_BASE + PMD_SIZE));
 287
 288         /*
 289          * KAsan may reuse the contents of kasan_early_shadow_pte directly, so
 290          * we should make sure that it maps the zero page read-only.
 291          */
 292         for (i = 0; i < PTRS_PER_PTE; i++)
 293                 set_pte_at(&init_mm, KASAN_SHADOW_START + i*PAGE_SIZE,
 294                            &kasan_early_shadow_pte[i],
 295                            pfn_pte(virt_to_pfn(kasan_early_shadow_page),
 296                                 __pgprot(pgprot_val(PAGE_KERNEL)
 297                                          | L_PTE_RDONLY)));
 298
 299         cpu_switch_mm(swapper_pg_dir, &init_mm);
 300         local_flush_tlb_all();
 301
 302         memset(kasan_early_shadow_page, 0, PAGE_SIZE);
 303         pr_info("Kernel address sanitizer initialized\n");
 304         init_task.kasan_depth = 0;
 305 }