arch/riscv/mm/pageattr.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2019 SiFive
   4  */
   5
   6 #include <linux/pagewalk.h>
   7 #include <linux/pgtable.h>
   8 #include <linux/vmalloc.h>
   9 #include <asm/tlbflush.h>
  10 #include <asm/bitops.h>
  11 #include <asm/set_memory.h>
  12
  13 struct pageattr_masks {
  14         pgprot_t set_mask;
  15         pgprot_t clear_mask;
  16 };
  17
  18 static unsigned long set_pageattr_masks(unsigned long val, struct mm_walk *walk)
  19 {
  20         struct pageattr_masks *masks = walk->private;
  21         unsigned long new_val = val;
  22
  23         new_val &= ~(pgprot_val(masks->clear_mask));
  24         new_val |= (pgprot_val(masks->set_mask));
  25
  26         return new_val;
  27 }
  28
  29 static int pageattr_p4d_entry(p4d_t *p4d, unsigned long addr,
  30                               unsigned long next, struct mm_walk *walk)
  31 {
  32         p4d_t val = p4dp_get(p4d);
  33
  34         if (p4d_leaf(val)) {
  35                 val = __p4d(set_pageattr_masks(p4d_val(val), walk));
  36                 set_p4d(p4d, val);
  37         }
  38
  39         return 0;
  40 }
  41
  42 static int pageattr_pud_entry(pud_t *pud, unsigned long addr,
  43                               unsigned long next, struct mm_walk *walk)
  44 {
  45         pud_t val = pudp_get(pud);
  46
  47         if (pud_leaf(val)) {
  48                 val = __pud(set_pageattr_masks(pud_val(val), walk));
  49                 set_pud(pud, val);
  50         }
  51
  52         return 0;
  53 }
  54
  55 static int pageattr_pmd_entry(pmd_t *pmd, unsigned long addr,
  56                               unsigned long next, struct mm_walk *walk)
  57 {
  58         pmd_t val = pmdp_get(pmd);
  59
  60         if (pmd_leaf(val)) {
  61                 val = __pmd(set_pageattr_masks(pmd_val(val), walk));
  62                 set_pmd(pmd, val);
  63         }
  64
  65         return 0;
  66 }
  67
  68 static int pageattr_pte_entry(pte_t *pte, unsigned long addr,
  69                               unsigned long next, struct mm_walk *walk)
  70 {
  71         pte_t val = ptep_get(pte);
  72
  73         val = __pte(set_pageattr_masks(pte_val(val), walk));
  74         set_pte(pte, val);
  75
  76         return 0;
  77 }
  78
  79 static int pageattr_pte_hole(unsigned long addr, unsigned long next,
  80                              int depth, struct mm_walk *walk)
  81 {
  82         /* Nothing to do here */
  83         return 0;
  84 }
  85
  86 static const struct mm_walk_ops pageattr_ops = {
  87         .p4d_entry = pageattr_p4d_entry,
  88         .pud_entry = pageattr_pud_entry,
  89         .pmd_entry = pageattr_pmd_entry,
  90         .pte_entry = pageattr_pte_entry,
  91         .pte_hole = pageattr_pte_hole,
  92         .walk_lock = PGWALK_RDLOCK,
  93 };
  94
  95 #ifdef CONFIG_64BIT
  96 static int __split_linear_mapping_pmd(pud_t *pudp,
  97                                       unsigned long vaddr, unsigned long end)
  98 {
  99         pmd_t *pmdp;
 100         unsigned long next;
 101
 102         pmdp = pmd_offset(pudp, vaddr);
 103
 104         do {
 105                 next = pmd_addr_end(vaddr, end);
 106
 107                 if (next - vaddr >= PMD_SIZE &&
 108                     vaddr <= (vaddr & PMD_MASK) && end >= next)
 109                         continue;
 110
 111                 if (pmd_leaf(pmdp_get(pmdp))) {
 112                         struct page *pte_page;
 113                         unsigned long pfn = _pmd_pfn(pmdp_get(pmdp));
 114                         pgprot_t prot = __pgprot(pmd_val(pmdp_get(pmdp)) & ~_PAGE_PFN_MASK);
 115                         pte_t *ptep_new;
 116                         int i;
 117
 118                         pte_page = alloc_page(GFP_KERNEL);
 119                         if (!pte_page)
 120                                 return -ENOMEM;
 121
 122                         ptep_new = (pte_t *)page_address(pte_page);
 123                         for (i = 0; i < PTRS_PER_PTE; ++i, ++ptep_new)
 124                                 set_pte(ptep_new, pfn_pte(pfn + i, prot));
 125
 126                         smp_wmb();
 127
 128                         set_pmd(pmdp, pfn_pmd(page_to_pfn(pte_page), PAGE_TABLE));
 129                 }
 130         } while (pmdp++, vaddr = next, vaddr != end);
 131
 132         return 0;
 133 }
 134
 135 static int __split_linear_mapping_pud(p4d_t *p4dp,
 136                                       unsigned long vaddr, unsigned long end)
 137 {
 138         pud_t *pudp;
 139         unsigned long next;
 140         int ret;
 141
 142         pudp = pud_offset(p4dp, vaddr);
 143
 144         do {
 145                 next = pud_addr_end(vaddr, end);
 146
 147                 if (next - vaddr >= PUD_SIZE &&
 148                     vaddr <= (vaddr & PUD_MASK) && end >= next)
 149                         continue;
 150
 151                 if (pud_leaf(pudp_get(pudp))) {
 152                         struct page *pmd_page;
 153                         unsigned long pfn = _pud_pfn(pudp_get(pudp));
 154                         pgprot_t prot = __pgprot(pud_val(pudp_get(pudp)) & ~_PAGE_PFN_MASK);
 155                         pmd_t *pmdp_new;
 156                         int i;
 157
 158                         pmd_page = alloc_page(GFP_KERNEL);
 159                         if (!pmd_page)
 160                                 return -ENOMEM;
 161
 162                         pmdp_new = (pmd_t *)page_address(pmd_page);
 163                         for (i = 0; i < PTRS_PER_PMD; ++i, ++pmdp_new)
 164                                 set_pmd(pmdp_new,
 165                                         pfn_pmd(pfn + ((i * PMD_SIZE) >> PAGE_SHIFT), prot));
 166
 167                         smp_wmb();
 168
 169                         set_pud(pudp, pfn_pud(page_to_pfn(pmd_page), PAGE_TABLE));
 170                 }
 171
 172                 ret = __split_linear_mapping_pmd(pudp, vaddr, next);
 173                 if (ret)
 174                         return ret;
 175         } while (pudp++, vaddr = next, vaddr != end);
 176
 177         return 0;
 178 }
 179
 180 static int __split_linear_mapping_p4d(pgd_t *pgdp,
 181                                       unsigned long vaddr, unsigned long end)
 182 {
 183         p4d_t *p4dp;
 184         unsigned long next;
 185         int ret;
 186
 187         p4dp = p4d_offset(pgdp, vaddr);
 188
 189         do {
 190                 next = p4d_addr_end(vaddr, end);
 191
 192                 /*
 193                  * If [vaddr; end] contains [vaddr & P4D_MASK; next], we don't
 194                  * need to split, we'll change the protections on the whole P4D.
 195                  */
 196                 if (next - vaddr >= P4D_SIZE &&
 197                     vaddr <= (vaddr & P4D_MASK) && end >= next)
 198                         continue;
 199
 200                 if (p4d_leaf(p4dp_get(p4dp))) {
 201                         struct page *pud_page;
 202                         unsigned long pfn = _p4d_pfn(p4dp_get(p4dp));
 203                         pgprot_t prot = __pgprot(p4d_val(p4dp_get(p4dp)) & ~_PAGE_PFN_MASK);
 204                         pud_t *pudp_new;
 205                         int i;
 206
 207                         pud_page = alloc_page(GFP_KERNEL);
 208                         if (!pud_page)
 209                                 return -ENOMEM;
 210
 211                         /*
 212                          * Fill the pud level with leaf puds that have the same
 213                          * protections as the leaf p4d.
 214                          */
 215                         pudp_new = (pud_t *)page_address(pud_page);
 216                         for (i = 0; i < PTRS_PER_PUD; ++i, ++pudp_new)
 217                                 set_pud(pudp_new,
 218                                         pfn_pud(pfn + ((i * PUD_SIZE) >> PAGE_SHIFT), prot));
 219
 220                         /*
 221                          * Make sure the pud filling is not reordered with the
 222                          * p4d store which could result in seeing a partially
 223                          * filled pud level.
 224                          */
 225                         smp_wmb();
 226
 227                         set_p4d(p4dp, pfn_p4d(page_to_pfn(pud_page), PAGE_TABLE));
 228                 }
 229
 230                 ret = __split_linear_mapping_pud(p4dp, vaddr, next);
 231                 if (ret)
 232                         return ret;
 233         } while (p4dp++, vaddr = next, vaddr != end);
 234
 235         return 0;
 236 }
 237
 238 static int __split_linear_mapping_pgd(pgd_t *pgdp,
 239                                       unsigned long vaddr,
 240                                       unsigned long end)
 241 {
 242         unsigned long next;
 243         int ret;
 244
 245         do {
 246                 next = pgd_addr_end(vaddr, end);
 247                 /* We never use PGD mappings for the linear mapping */
 248                 ret = __split_linear_mapping_p4d(pgdp, vaddr, next);
 249                 if (ret)
 250                         return ret;
 251         } while (pgdp++, vaddr = next, vaddr != end);
 252
 253         return 0;
 254 }
 255
 256 static int split_linear_mapping(unsigned long start, unsigned long end)
 257 {
 258         return __split_linear_mapping_pgd(pgd_offset_k(start), start, end);
 259 }
 260 #endif  /* CONFIG_64BIT */
 261
 262 static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask,
 263                         pgprot_t clear_mask)
 264 {
 265         int ret;
 266         unsigned long start = addr;
 267         unsigned long end = start + PAGE_SIZE * numpages;
 268         unsigned long __maybe_unused lm_start;
 269         unsigned long __maybe_unused lm_end;
 270         struct pageattr_masks masks = {
 271                 .set_mask = set_mask,
 272                 .clear_mask = clear_mask
 273         };
 274
 275         if (!numpages)
 276                 return 0;
 277
 278         mmap_write_lock(&init_mm);
 279
 280 #ifdef CONFIG_64BIT
 281         /*
 282          * We are about to change the permissions of a kernel mapping, we must
 283          * apply the same changes to its linear mapping alias, which may imply
 284          * splitting a huge mapping.
 285          */
 286
 287         if (is_vmalloc_or_module_addr((void *)start)) {
 288                 struct vm_struct *area = NULL;
 289                 int i, page_start;
 290
 291                 area = find_vm_area((void *)start);
 292                 page_start = (start - (unsigned long)area->addr) >> PAGE_SHIFT;
 293
 294                 for (i = page_start; i < page_start + numpages; ++i) {
 295                         lm_start = (unsigned long)page_address(area->pages[i]);
 296                         lm_end = lm_start + PAGE_SIZE;
 297
 298                         ret = split_linear_mapping(lm_start, lm_end);
 299                         if (ret)
 300                                 goto unlock;
 301
 302                         ret = walk_page_range_novma(&init_mm, lm_start, lm_end,
 303                                                     &pageattr_ops, NULL, &masks);
 304                         if (ret)
 305                                 goto unlock;
 306                 }
 307         } else if (is_kernel_mapping(start) || is_linear_mapping(start)) {
 308                 if (is_kernel_mapping(start)) {
 309                         lm_start = (unsigned long)lm_alias(start);
 310                         lm_end = (unsigned long)lm_alias(end);
 311                 } else {
 312                         lm_start = start;
 313                         lm_end = end;
 314                 }
 315
 316                 ret = split_linear_mapping(lm_start, lm_end);
 317                 if (ret)
 318                         goto unlock;
 319
 320                 ret = walk_page_range_novma(&init_mm, lm_start, lm_end,
 321                                             &pageattr_ops, NULL, &masks);
 322                 if (ret)
 323                         goto unlock;
 324         }
 325
 326         ret =  walk_page_range_novma(&init_mm, start, end, &pageattr_ops, NULL,
 327                                      &masks);
 328
 329 unlock:
 330         mmap_write_unlock(&init_mm);
 331
 332         /*
 333          * We can't use flush_tlb_kernel_range() here as we may have split a
 334          * hugepage that is larger than that, so let's flush everything.
 335          */
 336         flush_tlb_all();
 337 #else
 338         ret =  walk_page_range_novma(&init_mm, start, end, &pageattr_ops, NULL,
 339                                      &masks);
 340
 341         mmap_write_unlock(&init_mm);
 342
 343         flush_tlb_kernel_range(start, end);
 344 #endif
 345
 346         return ret;
 347 }
 348
 349 int set_memory_rw_nx(unsigned long addr, int numpages)
 350 {
 351         return __set_memory(addr, numpages, __pgprot(_PAGE_READ | _PAGE_WRITE),
 352                             __pgprot(_PAGE_EXEC));
 353 }
 354
 355 int set_memory_ro(unsigned long addr, int numpages)
 356 {
 357         return __set_memory(addr, numpages, __pgprot(_PAGE_READ),
 358                             __pgprot(_PAGE_WRITE));
 359 }
 360
 361 int set_memory_rw(unsigned long addr, int numpages)
 362 {
 363         return __set_memory(addr, numpages, __pgprot(_PAGE_READ | _PAGE_WRITE),
 364                             __pgprot(0));
 365 }
 366
 367 int set_memory_x(unsigned long addr, int numpages)
 368 {
 369         return __set_memory(addr, numpages, __pgprot(_PAGE_EXEC), __pgprot(0));
 370 }
 371
 372 int set_memory_nx(unsigned long addr, int numpages)
 373 {
 374         return __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_EXEC));
 375 }
 376
 377 int set_direct_map_invalid_noflush(struct page *page)
 378 {
 379         return __set_memory((unsigned long)page_address(page), 1,
 380                             __pgprot(0), __pgprot(_PAGE_PRESENT));
 381 }
 382
 383 int set_direct_map_default_noflush(struct page *page)
 384 {
 385         return __set_memory((unsigned long)page_address(page), 1,
 386                             PAGE_KERNEL, __pgprot(_PAGE_EXEC));
 387 }
 388
 389 #ifdef CONFIG_DEBUG_PAGEALLOC
 390 static int debug_pagealloc_set_page(pte_t *pte, unsigned long addr, void *data)
 391 {
 392         int enable = *(int *)data;
 393
 394         unsigned long val = pte_val(ptep_get(pte));
 395
 396         if (enable)
 397                 val |= _PAGE_PRESENT;
 398         else
 399                 val &= ~_PAGE_PRESENT;
 400
 401         set_pte(pte, __pte(val));
 402
 403         return 0;
 404 }
 405
 406 void __kernel_map_pages(struct page *page, int numpages, int enable)
 407 {
 408         if (!debug_pagealloc_enabled())
 409                 return;
 410
 411         unsigned long start = (unsigned long)page_address(page);
 412         unsigned long size = PAGE_SIZE * numpages;
 413
 414         apply_to_existing_page_range(&init_mm, start, size, debug_pagealloc_set_page, &enable);
 415
 416         flush_tlb_kernel_range(start, start + size);
 417 }
 418 #endif
 419
 420 bool kernel_page_present(struct page *page)
 421 {
 422         unsigned long addr = (unsigned long)page_address(page);
 423         pgd_t *pgd;
 424         pud_t *pud;
 425         p4d_t *p4d;
 426         pmd_t *pmd;
 427         pte_t *pte;
 428
 429         pgd = pgd_offset_k(addr);
 430         if (!pgd_present(pgdp_get(pgd)))
 431                 return false;
 432         if (pgd_leaf(pgdp_get(pgd)))
 433                 return true;
 434
 435         p4d = p4d_offset(pgd, addr);
 436         if (!p4d_present(p4dp_get(p4d)))
 437                 return false;
 438         if (p4d_leaf(p4dp_get(p4d)))
 439                 return true;
 440
 441         pud = pud_offset(p4d, addr);
 442         if (!pud_present(pudp_get(pud)))
 443                 return false;
 444         if (pud_leaf(pudp_get(pud)))
 445                 return true;
 446
 447         pmd = pmd_offset(pud, addr);
 448         if (!pmd_present(pmdp_get(pmd)))
 449                 return false;
 450         if (pmd_leaf(pmdp_get(pmd)))
 451                 return true;
 452
 453         pte = pte_offset_kernel(pmd, addr);
 454         return pte_present(ptep_get(pte));
 455 }