arch/s390/mm/kasan_init.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/kasan.h>
   3 #include <linux/sched/task.h>
   4 #include <linux/memblock.h>
   5 #include <linux/pgtable.h>
   6 #include <asm/pgalloc.h>
   7 #include <asm/kasan.h>
   8 #include <asm/mem_detect.h>
   9 #include <asm/processor.h>
  10 #include <asm/sclp.h>
  11 #include <asm/facility.h>
  12 #include <asm/sections.h>
  13 #include <asm/setup.h>
  14 #include <asm/uv.h>
  15
  16 unsigned long kasan_vmax;
  17 static unsigned long segment_pos __initdata;
  18 static unsigned long segment_low __initdata;
  19 static unsigned long pgalloc_pos __initdata;
  20 static unsigned long pgalloc_low __initdata;
  21 static unsigned long pgalloc_freeable __initdata;
  22 static bool has_edat __initdata;
  23 static bool has_nx __initdata;
  24
  25 #define __sha(x) ((unsigned long)kasan_mem_to_shadow((void *)x))
  26
  27 static pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE);
  28
  29 static void __init kasan_early_panic(const char *reason)
  30 {
  31         sclp_early_printk("The Linux kernel failed to boot with the KernelAddressSanitizer:\n");
  32         sclp_early_printk(reason);
  33         disabled_wait();
  34 }
  35
  36 static void * __init kasan_early_alloc_segment(void)
  37 {
  38         segment_pos -= _SEGMENT_SIZE;
  39
  40         if (segment_pos < segment_low)
  41                 kasan_early_panic("out of memory during initialisation\n");
  42
  43         return (void *)segment_pos;
  44 }
  45
  46 static void * __init kasan_early_alloc_pages(unsigned int order)
  47 {
  48         pgalloc_pos -= (PAGE_SIZE << order);
  49
  50         if (pgalloc_pos < pgalloc_low)
  51                 kasan_early_panic("out of memory during initialisation\n");
  52
  53         return (void *)pgalloc_pos;
  54 }
  55
  56 static void * __init kasan_early_crst_alloc(unsigned long val)
  57 {
  58         unsigned long *table;
  59
  60         table = kasan_early_alloc_pages(CRST_ALLOC_ORDER);
  61         if (table)
  62                 crst_table_init(table, val);
  63         return table;
  64 }
  65
  66 static pte_t * __init kasan_early_pte_alloc(void)
  67 {
  68         static void *pte_leftover;
  69         pte_t *pte;
  70
  71         BUILD_BUG_ON(_PAGE_TABLE_SIZE * 2 != PAGE_SIZE);
  72
  73         if (!pte_leftover) {
  74                 pte_leftover = kasan_early_alloc_pages(0);
  75                 pte = pte_leftover + _PAGE_TABLE_SIZE;
  76         } else {
  77                 pte = pte_leftover;
  78                 pte_leftover = NULL;
  79         }
  80         memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE);
  81         return pte;
  82 }
  83
  84 enum populate_mode {
  85         POPULATE_ONE2ONE,
  86         POPULATE_MAP,
  87         POPULATE_ZERO_SHADOW,
  88         POPULATE_SHALLOW
  89 };
  90 static void __init kasan_early_pgtable_populate(unsigned long address,
  91                                                 unsigned long end,
  92                                                 enum populate_mode mode)
  93 {
  94         unsigned long pgt_prot_zero, pgt_prot, sgt_prot;
  95         pgd_t *pg_dir;
  96         p4d_t *p4_dir;
  97         pud_t *pu_dir;
  98         pmd_t *pm_dir;
  99         pte_t *pt_dir;
 100
 101         pgt_prot_zero = pgprot_val(PAGE_KERNEL_RO);
 102         if (!has_nx)
 103                 pgt_prot_zero &= ~_PAGE_NOEXEC;
 104         pgt_prot = pgprot_val(PAGE_KERNEL);
 105         sgt_prot = pgprot_val(SEGMENT_KERNEL);
 106         if (!has_nx || mode == POPULATE_ONE2ONE) {
 107                 pgt_prot &= ~_PAGE_NOEXEC;
 108                 sgt_prot &= ~_SEGMENT_ENTRY_NOEXEC;
 109         }
 110
 111         while (address < end) {
 112                 pg_dir = pgd_offset_k(address);
 113                 if (pgd_none(*pg_dir)) {
 114                         if (mode == POPULATE_ZERO_SHADOW &&
 115                             IS_ALIGNED(address, PGDIR_SIZE) &&
 116                             end - address >= PGDIR_SIZE) {
 117                                 pgd_populate(&init_mm, pg_dir,
 118                                                 kasan_early_shadow_p4d);
 119                                 address = (address + PGDIR_SIZE) & PGDIR_MASK;
 120                                 continue;
 121                         }
 122                         p4_dir = kasan_early_crst_alloc(_REGION2_ENTRY_EMPTY);
 123                         pgd_populate(&init_mm, pg_dir, p4_dir);
 124                 }
 125
 126                 if (mode == POPULATE_SHALLOW) {
 127                         address = (address + P4D_SIZE) & P4D_MASK;
 128                         continue;
 129                 }
 130
 131                 p4_dir = p4d_offset(pg_dir, address);
 132                 if (p4d_none(*p4_dir)) {
 133                         if (mode == POPULATE_ZERO_SHADOW &&
 134                             IS_ALIGNED(address, P4D_SIZE) &&
 135                             end - address >= P4D_SIZE) {
 136                                 p4d_populate(&init_mm, p4_dir,
 137                                                 kasan_early_shadow_pud);
 138                                 address = (address + P4D_SIZE) & P4D_MASK;
 139                                 continue;
 140                         }
 141                         pu_dir = kasan_early_crst_alloc(_REGION3_ENTRY_EMPTY);
 142                         p4d_populate(&init_mm, p4_dir, pu_dir);
 143                 }
 144
 145                 pu_dir = pud_offset(p4_dir, address);
 146                 if (pud_none(*pu_dir)) {
 147                         if (mode == POPULATE_ZERO_SHADOW &&
 148                             IS_ALIGNED(address, PUD_SIZE) &&
 149                             end - address >= PUD_SIZE) {
 150                                 pud_populate(&init_mm, pu_dir,
 151                                                 kasan_early_shadow_pmd);
 152                                 address = (address + PUD_SIZE) & PUD_MASK;
 153                                 continue;
 154                         }
 155                         pm_dir = kasan_early_crst_alloc(_SEGMENT_ENTRY_EMPTY);
 156                         pud_populate(&init_mm, pu_dir, pm_dir);
 157                 }
 158
 159                 pm_dir = pmd_offset(pu_dir, address);
 160                 if (pmd_none(*pm_dir)) {
 161                         if (mode == POPULATE_ZERO_SHADOW &&
 162                             IS_ALIGNED(address, PMD_SIZE) &&
 163                             end - address >= PMD_SIZE) {
 164                                 pmd_populate(&init_mm, pm_dir,
 165                                                 kasan_early_shadow_pte);
 166                                 address = (address + PMD_SIZE) & PMD_MASK;
 167                                 continue;
 168                         }
 169                         /* the first megabyte of 1:1 is mapped with 4k pages */
 170                         if (has_edat && address && end - address >= PMD_SIZE &&
 171                             mode != POPULATE_ZERO_SHADOW) {
 172                                 void *page;
 173
 174                                 if (mode == POPULATE_ONE2ONE) {
 175                                         page = (void *)address;
 176                                 } else {
 177                                         page = kasan_early_alloc_segment();
 178                                         memset(page, 0, _SEGMENT_SIZE);
 179                                 }
 180                                 pmd_val(*pm_dir) = __pa(page) | sgt_prot;
 181                                 address = (address + PMD_SIZE) & PMD_MASK;
 182                                 continue;
 183                         }
 184
 185                         pt_dir = kasan_early_pte_alloc();
 186                         pmd_populate(&init_mm, pm_dir, pt_dir);
 187                 } else if (pmd_large(*pm_dir)) {
 188                         address = (address + PMD_SIZE) & PMD_MASK;
 189                         continue;
 190                 }
 191
 192                 pt_dir = pte_offset_kernel(pm_dir, address);
 193                 if (pte_none(*pt_dir)) {
 194                         void *page;
 195
 196                         switch (mode) {
 197                         case POPULATE_ONE2ONE:
 198                                 page = (void *)address;
 199                                 pte_val(*pt_dir) = __pa(page) | pgt_prot;
 200                                 break;
 201                         case POPULATE_MAP:
 202                                 page = kasan_early_alloc_pages(0);
 203                                 memset(page, 0, PAGE_SIZE);
 204                                 pte_val(*pt_dir) = __pa(page) | pgt_prot;
 205                                 break;
 206                         case POPULATE_ZERO_SHADOW:
 207                                 page = kasan_early_shadow_page;
 208                                 pte_val(*pt_dir) = __pa(page) | pgt_prot_zero;
 209                                 break;
 210                         case POPULATE_SHALLOW:
 211                                 /* should never happen */
 212                                 break;
 213                         }
 214                 }
 215                 address += PAGE_SIZE;
 216         }
 217 }
 218
 219 static void __init kasan_set_pgd(pgd_t *pgd, unsigned long asce_type)
 220 {
 221         unsigned long asce_bits;
 222
 223         asce_bits = asce_type | _ASCE_TABLE_LENGTH;
 224         S390_lowcore.kernel_asce = (__pa(pgd) & PAGE_MASK) | asce_bits;
 225         S390_lowcore.user_asce = S390_lowcore.kernel_asce;
 226
 227         __ctl_load(S390_lowcore.kernel_asce, 1, 1);
 228         __ctl_load(S390_lowcore.kernel_asce, 7, 7);
 229         __ctl_load(S390_lowcore.kernel_asce, 13, 13);
 230 }
 231
 232 static void __init kasan_enable_dat(void)
 233 {
 234         psw_t psw;
 235
 236         psw.mask = __extract_psw();
 237         psw_bits(psw).dat = 1;
 238         psw_bits(psw).as = PSW_BITS_AS_HOME;
 239         __load_psw_mask(psw.mask);
 240 }
 241
 242 static void __init kasan_early_detect_facilities(void)
 243 {
 244         if (test_facility(8)) {
 245                 has_edat = true;
 246                 __ctl_set_bit(0, 23);
 247         }
 248         if (!noexec_disabled && test_facility(130)) {
 249                 has_nx = true;
 250                 __ctl_set_bit(0, 20);
 251         }
 252 }
 253
 254 static bool __init has_uv_sec_stor_limit(void)
 255 {
 256         /*
 257          * keep these conditions in line with setup_uv()
 258          */
 259         if (!is_prot_virt_host())
 260                 return false;
 261
 262         if (is_prot_virt_guest())
 263                 return false;
 264
 265         if (!test_facility(158))
 266                 return false;
 267
 268         return !!uv_info.max_sec_stor_addr;
 269 }
 270
 271 void __init kasan_early_init(void)
 272 {
 273         unsigned long untracked_mem_end;
 274         unsigned long shadow_alloc_size;
 275         unsigned long vmax_unlimited;
 276         unsigned long initrd_end;
 277         unsigned long memsize;
 278         unsigned long pgt_prot = pgprot_val(PAGE_KERNEL_RO);
 279         pte_t pte_z;
 280         pmd_t pmd_z = __pmd(__pa(kasan_early_shadow_pte) | _SEGMENT_ENTRY);
 281         pud_t pud_z = __pud(__pa(kasan_early_shadow_pmd) | _REGION3_ENTRY);
 282         p4d_t p4d_z = __p4d(__pa(kasan_early_shadow_pud) | _REGION2_ENTRY);
 283
 284         kasan_early_detect_facilities();
 285         if (!has_nx)
 286                 pgt_prot &= ~_PAGE_NOEXEC;
 287         pte_z = __pte(__pa(kasan_early_shadow_page) | pgt_prot);
 288
 289         memsize = get_mem_detect_end();
 290         if (!memsize)
 291                 kasan_early_panic("cannot detect physical memory size\n");
 292         /*
 293          * Kasan currently supports standby memory but only if it follows
 294          * online memory (default allocation), i.e. no memory holes.
 295          * - memsize represents end of online memory
 296          * - ident_map_size represents online + standby and memory limits
 297          *   accounted.
 298          * Kasan maps "memsize" right away.
 299          * [0, memsize]                 - as identity mapping
 300          * [__sha(0), __sha(memsize)]   - shadow memory for identity mapping
 301          * The rest [memsize, ident_map_size] if memsize < ident_map_size
 302          * could be mapped/unmapped dynamically later during memory hotplug.
 303          */
 304         memsize = min(memsize, ident_map_size);
 305
 306         BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, P4D_SIZE));
 307         BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, P4D_SIZE));
 308         crst_table_init((unsigned long *)early_pg_dir, _REGION2_ENTRY_EMPTY);
 309         untracked_mem_end = kasan_vmax = vmax_unlimited = _REGION1_SIZE;
 310         if (has_uv_sec_stor_limit())
 311                 kasan_vmax = min(vmax_unlimited, uv_info.max_sec_stor_addr);
 312
 313         /* init kasan zero shadow */
 314         crst_table_init((unsigned long *)kasan_early_shadow_p4d,
 315                                 p4d_val(p4d_z));
 316         crst_table_init((unsigned long *)kasan_early_shadow_pud,
 317                                 pud_val(pud_z));
 318         crst_table_init((unsigned long *)kasan_early_shadow_pmd,
 319                                 pmd_val(pmd_z));
 320         memset64((u64 *)kasan_early_shadow_pte, pte_val(pte_z), PTRS_PER_PTE);
 321
 322         shadow_alloc_size = memsize >> KASAN_SHADOW_SCALE_SHIFT;
 323         pgalloc_low = round_up((unsigned long)_end, _SEGMENT_SIZE);
 324         if (IS_ENABLED(CONFIG_BLK_DEV_INITRD)) {
 325                 initrd_end =
 326                     round_up(INITRD_START + INITRD_SIZE, _SEGMENT_SIZE);
 327                 pgalloc_low = max(pgalloc_low, initrd_end);
 328         }
 329
 330         if (pgalloc_low + shadow_alloc_size > memsize)
 331                 kasan_early_panic("out of memory during initialisation\n");
 332
 333         if (has_edat) {
 334                 segment_pos = round_down(memsize, _SEGMENT_SIZE);
 335                 segment_low = segment_pos - shadow_alloc_size;
 336                 pgalloc_pos = segment_low;
 337         } else {
 338                 pgalloc_pos = memsize;
 339         }
 340         init_mm.pgd = early_pg_dir;
 341         /*
 342          * Current memory layout:
 343          * +- 0 -------------+     +- shadow start -+
 344          * | 1:1 ram mapping |    /| 1/8 ram        |
 345          * |                 |   / |                |
 346          * +- end of ram ----+  /  +----------------+
 347          * | ... gap ...     | /   |                |
 348          * |                 |/    |    kasan       |
 349          * +- shadow start --+     |    zero        |
 350          * | 1/8 addr space  |     |    page        |
 351          * +- shadow end    -+     |    mapping     |
 352          * | ... gap ...     |\    |  (untracked)   |
 353          * +- vmalloc area  -+ \   |                |
 354          * | vmalloc_size    |  \  |                |
 355          * +- modules vaddr -+   \ +----------------+
 356          * | 2Gb             |    \|      unmapped  | allocated per module
 357          * +-----------------+     +- shadow end ---+
 358          *
 359          * Current memory layout (KASAN_VMALLOC):
 360          * +- 0 -------------+     +- shadow start -+
 361          * | 1:1 ram mapping |    /| 1/8 ram        |
 362          * |                 |   / |                |
 363          * +- end of ram ----+  /  +----------------+
 364          * | ... gap ...     | /   |    kasan       |
 365          * |                 |/    |    zero        |
 366          * +- shadow start --+     |    page        |
 367          * | 1/8 addr space  |     |    mapping     |
 368          * +- shadow end    -+     |  (untracked)   |
 369          * | ... gap ...     |\    |                |
 370          * +- vmalloc area  -+ \   +- vmalloc area -+
 371          * | vmalloc_size    |  \  |shallow populate|
 372          * +- modules vaddr -+   \ +- modules area -+
 373          * | 2Gb             |    \|shallow populate|
 374          * +-----------------+     +- shadow end ---+
 375          */
 376         /* populate kasan shadow (for identity mapping and zero page mapping) */
 377         kasan_early_pgtable_populate(__sha(0), __sha(memsize), POPULATE_MAP);
 378         if (IS_ENABLED(CONFIG_MODULES))
 379                 untracked_mem_end = kasan_vmax - MODULES_LEN;
 380         if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) {
 381                 untracked_mem_end = kasan_vmax - vmalloc_size - MODULES_LEN;
 382                 /* shallowly populate kasan shadow for vmalloc and modules */
 383                 kasan_early_pgtable_populate(__sha(untracked_mem_end), __sha(kasan_vmax),
 384                                              POPULATE_SHALLOW);
 385         }
 386         /* populate kasan shadow for untracked memory */
 387         kasan_early_pgtable_populate(__sha(ident_map_size), __sha(untracked_mem_end),
 388                                      POPULATE_ZERO_SHADOW);
 389         kasan_early_pgtable_populate(__sha(kasan_vmax), __sha(vmax_unlimited),
 390                                      POPULATE_ZERO_SHADOW);
 391         /* memory allocated for identity mapping structs will be freed later */
 392         pgalloc_freeable = pgalloc_pos;
 393         /* populate identity mapping */
 394         kasan_early_pgtable_populate(0, memsize, POPULATE_ONE2ONE);
 395         kasan_set_pgd(early_pg_dir, _ASCE_TYPE_REGION2);
 396         kasan_enable_dat();
 397         /* enable kasan */
 398         init_task.kasan_depth = 0;
 399         memblock_reserve(pgalloc_pos, memsize - pgalloc_pos);
 400         sclp_early_printk("KernelAddressSanitizer initialized\n");
 401 }
 402
 403 void __init kasan_copy_shadow_mapping(void)
 404 {
 405         /*
 406          * At this point we are still running on early pages setup early_pg_dir,
 407          * while swapper_pg_dir has just been initialized with identity mapping.
 408          * Carry over shadow memory region from early_pg_dir to swapper_pg_dir.
 409          */
 410
 411         pgd_t *pg_dir_src;
 412         pgd_t *pg_dir_dst;
 413         p4d_t *p4_dir_src;
 414         p4d_t *p4_dir_dst;
 415
 416         pg_dir_src = pgd_offset_raw(early_pg_dir, KASAN_SHADOW_START);
 417         pg_dir_dst = pgd_offset_raw(init_mm.pgd, KASAN_SHADOW_START);
 418         p4_dir_src = p4d_offset(pg_dir_src, KASAN_SHADOW_START);
 419         p4_dir_dst = p4d_offset(pg_dir_dst, KASAN_SHADOW_START);
 420         memcpy(p4_dir_dst, p4_dir_src,
 421                (KASAN_SHADOW_SIZE >> P4D_SHIFT) * sizeof(p4d_t));
 422 }
 423
 424 void __init kasan_free_early_identity(void)
 425 {
 426         memblock_free(pgalloc_pos, pgalloc_freeable - pgalloc_pos);
 427 }