arch/x86/include/asm/pgtable.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _ASM_X86_PGTABLE_H
   3 #define _ASM_X86_PGTABLE_H
   4
   5 #include <linux/mem_encrypt.h>
   6 #include <asm/page.h>
   7 #include <asm/pgtable_types.h>
   8
   9 /*
  10  * Macro to mark a page protection value as UC-
  11  */
  12 #define pgprot_noncached(prot)                                          \
  13         ((boot_cpu_data.x86 > 3)                                        \
  14          ? (__pgprot(pgprot_val(prot) |                                 \
  15                      cachemode2protval(_PAGE_CACHE_MODE_UC_MINUS)))     \
  16          : (prot))
  17
  18 /*
  19  * Macros to add or remove encryption attribute
  20  */
  21 #define pgprot_encrypted(prot)  __pgprot(__sme_set(pgprot_val(prot)))
  22 #define pgprot_decrypted(prot)  __pgprot(__sme_clr(pgprot_val(prot)))
  23
  24 #ifndef __ASSEMBLY__
  25 #include <asm/x86_init.h>
  26 #include <asm/fpu/xstate.h>
  27 #include <asm/fpu/api.h>
  28 #include <asm-generic/pgtable_uffd.h>
  29
  30 extern pgd_t early_top_pgt[PTRS_PER_PGD];
  31 int __init __early_make_pgtable(unsigned long address, pmdval_t pmd);
  32
  33 void ptdump_walk_pgd_level(struct seq_file *m, struct mm_struct *mm);
  34 void ptdump_walk_pgd_level_debugfs(struct seq_file *m, struct mm_struct *mm,
  35                                    bool user);
  36 void ptdump_walk_pgd_level_checkwx(void);
  37 void ptdump_walk_user_pgd_level_checkwx(void);
  38
  39 #ifdef CONFIG_DEBUG_WX
  40 #define debug_checkwx()         ptdump_walk_pgd_level_checkwx()
  41 #define debug_checkwx_user()    ptdump_walk_user_pgd_level_checkwx()
  42 #else
  43 #define debug_checkwx()         do { } while (0)
  44 #define debug_checkwx_user()    do { } while (0)
  45 #endif
  46
  47 /*
  48  * ZERO_PAGE is a global shared page that is always zero: used
  49  * for zero-mapped memory areas etc..
  50  */
  51 extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]
  52         __visible;
  53 #define ZERO_PAGE(vaddr) ((void)(vaddr),virt_to_page(empty_zero_page))
  54
  55 extern spinlock_t pgd_lock;
  56 extern struct list_head pgd_list;
  57
  58 extern struct mm_struct *pgd_page_get_mm(struct page *page);
  59
  60 extern pmdval_t early_pmd_flags;
  61
  62 #ifdef CONFIG_PARAVIRT_XXL
  63 #include <asm/paravirt.h>
  64 #else  /* !CONFIG_PARAVIRT_XXL */
  65 #define set_pte(ptep, pte)              native_set_pte(ptep, pte)
  66 #define set_pte_at(mm, addr, ptep, pte) native_set_pte_at(mm, addr, ptep, pte)
  67
  68 #define set_pte_atomic(ptep, pte)                                       \
  69         native_set_pte_atomic(ptep, pte)
  70
  71 #define set_pmd(pmdp, pmd)              native_set_pmd(pmdp, pmd)
  72
  73 #ifndef __PAGETABLE_P4D_FOLDED
  74 #define set_pgd(pgdp, pgd)              native_set_pgd(pgdp, pgd)
  75 #define pgd_clear(pgd)                  (pgtable_l5_enabled() ? native_pgd_clear(pgd) : 0)
  76 #endif
  77
  78 #ifndef set_p4d
  79 # define set_p4d(p4dp, p4d)             native_set_p4d(p4dp, p4d)
  80 #endif
  81
  82 #ifndef __PAGETABLE_PUD_FOLDED
  83 #define p4d_clear(p4d)                  native_p4d_clear(p4d)
  84 #endif
  85
  86 #ifndef set_pud
  87 # define set_pud(pudp, pud)             native_set_pud(pudp, pud)
  88 #endif
  89
  90 #ifndef __PAGETABLE_PUD_FOLDED
  91 #define pud_clear(pud)                  native_pud_clear(pud)
  92 #endif
  93
  94 #define pte_clear(mm, addr, ptep)       native_pte_clear(mm, addr, ptep)
  95 #define pmd_clear(pmd)                  native_pmd_clear(pmd)
  96
  97 #define pgd_val(x)      native_pgd_val(x)
  98 #define __pgd(x)        native_make_pgd(x)
  99
 100 #ifndef __PAGETABLE_P4D_FOLDED
 101 #define p4d_val(x)      native_p4d_val(x)
 102 #define __p4d(x)        native_make_p4d(x)
 103 #endif
 104
 105 #ifndef __PAGETABLE_PUD_FOLDED
 106 #define pud_val(x)      native_pud_val(x)
 107 #define __pud(x)        native_make_pud(x)
 108 #endif
 109
 110 #ifndef __PAGETABLE_PMD_FOLDED
 111 #define pmd_val(x)      native_pmd_val(x)
 112 #define __pmd(x)        native_make_pmd(x)
 113 #endif
 114
 115 #define pte_val(x)      native_pte_val(x)
 116 #define __pte(x)        native_make_pte(x)
 117
 118 #define arch_end_context_switch(prev)   do {} while(0)
 119 #endif  /* CONFIG_PARAVIRT_XXL */
 120
 121 /*
 122  * The following only work if pte_present() is true.
 123  * Undefined behaviour if not..
 124  */
 125 static inline int pte_dirty(pte_t pte)
 126 {
 127         return pte_flags(pte) & _PAGE_DIRTY;
 128 }
 129
 130
 131 static inline u32 read_pkru(void)
 132 {
 133         if (boot_cpu_has(X86_FEATURE_OSPKE))
 134                 return rdpkru();
 135         return 0;
 136 }
 137
 138 static inline void write_pkru(u32 pkru)
 139 {
 140         struct pkru_state *pk;
 141
 142         if (!boot_cpu_has(X86_FEATURE_OSPKE))
 143                 return;
 144
 145         pk = get_xsave_addr(&current->thread.fpu.state.xsave, XFEATURE_PKRU);
 146
 147         /*
 148          * The PKRU value in xstate needs to be in sync with the value that is
 149          * written to the CPU. The FPU restore on return to userland would
 150          * otherwise load the previous value again.
 151          */
 152         fpregs_lock();
 153         if (pk)
 154                 pk->pkru = pkru;
 155         __write_pkru(pkru);
 156         fpregs_unlock();
 157 }
 158
 159 static inline int pte_young(pte_t pte)
 160 {
 161         return pte_flags(pte) & _PAGE_ACCESSED;
 162 }
 163
 164 static inline int pmd_dirty(pmd_t pmd)
 165 {
 166         return pmd_flags(pmd) & _PAGE_DIRTY;
 167 }
 168
 169 static inline int pmd_young(pmd_t pmd)
 170 {
 171         return pmd_flags(pmd) & _PAGE_ACCESSED;
 172 }
 173
 174 static inline int pud_dirty(pud_t pud)
 175 {
 176         return pud_flags(pud) & _PAGE_DIRTY;
 177 }
 178
 179 static inline int pud_young(pud_t pud)
 180 {
 181         return pud_flags(pud) & _PAGE_ACCESSED;
 182 }
 183
 184 static inline int pte_write(pte_t pte)
 185 {
 186         return pte_flags(pte) & _PAGE_RW;
 187 }
 188
 189 static inline int pte_huge(pte_t pte)
 190 {
 191         return pte_flags(pte) & _PAGE_PSE;
 192 }
 193
 194 static inline int pte_global(pte_t pte)
 195 {
 196         return pte_flags(pte) & _PAGE_GLOBAL;
 197 }
 198
 199 static inline int pte_exec(pte_t pte)
 200 {
 201         return !(pte_flags(pte) & _PAGE_NX);
 202 }
 203
 204 static inline int pte_special(pte_t pte)
 205 {
 206         return pte_flags(pte) & _PAGE_SPECIAL;
 207 }
 208
 209 /* Entries that were set to PROT_NONE are inverted */
 210
 211 static inline u64 protnone_mask(u64 val);
 212
 213 static inline unsigned long pte_pfn(pte_t pte)
 214 {
 215         phys_addr_t pfn = pte_val(pte);
 216         pfn ^= protnone_mask(pfn);
 217         return (pfn & PTE_PFN_MASK) >> PAGE_SHIFT;
 218 }
 219
 220 static inline unsigned long pmd_pfn(pmd_t pmd)
 221 {
 222         phys_addr_t pfn = pmd_val(pmd);
 223         pfn ^= protnone_mask(pfn);
 224         return (pfn & pmd_pfn_mask(pmd)) >> PAGE_SHIFT;
 225 }
 226
 227 static inline unsigned long pud_pfn(pud_t pud)
 228 {
 229         phys_addr_t pfn = pud_val(pud);
 230         pfn ^= protnone_mask(pfn);
 231         return (pfn & pud_pfn_mask(pud)) >> PAGE_SHIFT;
 232 }
 233
 234 static inline unsigned long p4d_pfn(p4d_t p4d)
 235 {
 236         return (p4d_val(p4d) & p4d_pfn_mask(p4d)) >> PAGE_SHIFT;
 237 }
 238
 239 static inline unsigned long pgd_pfn(pgd_t pgd)
 240 {
 241         return (pgd_val(pgd) & PTE_PFN_MASK) >> PAGE_SHIFT;
 242 }
 243
 244 #define p4d_leaf        p4d_large
 245 static inline int p4d_large(p4d_t p4d)
 246 {
 247         /* No 512 GiB pages yet */
 248         return 0;
 249 }
 250
 251 #define pte_page(pte)   pfn_to_page(pte_pfn(pte))
 252
 253 #define pmd_leaf        pmd_large
 254 static inline int pmd_large(pmd_t pte)
 255 {
 256         return pmd_flags(pte) & _PAGE_PSE;
 257 }
 258
 259 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 260 /* NOTE: when predicate huge page, consider also pmd_devmap, or use pmd_large */
 261 static inline int pmd_trans_huge(pmd_t pmd)
 262 {
 263         return (pmd_val(pmd) & (_PAGE_PSE|_PAGE_DEVMAP)) == _PAGE_PSE;
 264 }
 265
 266 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
 267 static inline int pud_trans_huge(pud_t pud)
 268 {
 269         return (pud_val(pud) & (_PAGE_PSE|_PAGE_DEVMAP)) == _PAGE_PSE;
 270 }
 271 #endif
 272
 273 #define has_transparent_hugepage has_transparent_hugepage
 274 static inline int has_transparent_hugepage(void)
 275 {
 276         return boot_cpu_has(X86_FEATURE_PSE);
 277 }
 278
 279 #ifdef CONFIG_ARCH_HAS_PTE_DEVMAP
 280 static inline int pmd_devmap(pmd_t pmd)
 281 {
 282         return !!(pmd_val(pmd) & _PAGE_DEVMAP);
 283 }
 284
 285 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
 286 static inline int pud_devmap(pud_t pud)
 287 {
 288         return !!(pud_val(pud) & _PAGE_DEVMAP);
 289 }
 290 #else
 291 static inline int pud_devmap(pud_t pud)
 292 {
 293         return 0;
 294 }
 295 #endif
 296
 297 static inline int pgd_devmap(pgd_t pgd)
 298 {
 299         return 0;
 300 }
 301 #endif
 302 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 303
 304 static inline pte_t pte_set_flags(pte_t pte, pteval_t set)
 305 {
 306         pteval_t v = native_pte_val(pte);
 307
 308         return native_make_pte(v | set);
 309 }
 310
 311 static inline pte_t pte_clear_flags(pte_t pte, pteval_t clear)
 312 {
 313         pteval_t v = native_pte_val(pte);
 314
 315         return native_make_pte(v & ~clear);
 316 }
 317
 318 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
 319 static inline int pte_uffd_wp(pte_t pte)
 320 {
 321         return pte_flags(pte) & _PAGE_UFFD_WP;
 322 }
 323
 324 static inline pte_t pte_mkuffd_wp(pte_t pte)
 325 {
 326         return pte_set_flags(pte, _PAGE_UFFD_WP);
 327 }
 328
 329 static inline pte_t pte_clear_uffd_wp(pte_t pte)
 330 {
 331         return pte_clear_flags(pte, _PAGE_UFFD_WP);
 332 }
 333 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
 334
 335 static inline pte_t pte_mkclean(pte_t pte)
 336 {
 337         return pte_clear_flags(pte, _PAGE_DIRTY);
 338 }
 339
 340 static inline pte_t pte_mkold(pte_t pte)
 341 {
 342         return pte_clear_flags(pte, _PAGE_ACCESSED);
 343 }
 344
 345 static inline pte_t pte_wrprotect(pte_t pte)
 346 {
 347         return pte_clear_flags(pte, _PAGE_RW);
 348 }
 349
 350 static inline pte_t pte_mkexec(pte_t pte)
 351 {
 352         return pte_clear_flags(pte, _PAGE_NX);
 353 }
 354
 355 static inline pte_t pte_mkdirty(pte_t pte)
 356 {
 357         return pte_set_flags(pte, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
 358 }
 359
 360 static inline pte_t pte_mkyoung(pte_t pte)
 361 {
 362         return pte_set_flags(pte, _PAGE_ACCESSED);
 363 }
 364
 365 static inline pte_t pte_mkwrite(pte_t pte)
 366 {
 367         return pte_set_flags(pte, _PAGE_RW);
 368 }
 369
 370 static inline pte_t pte_mkhuge(pte_t pte)
 371 {
 372         return pte_set_flags(pte, _PAGE_PSE);
 373 }
 374
 375 static inline pte_t pte_clrhuge(pte_t pte)
 376 {
 377         return pte_clear_flags(pte, _PAGE_PSE);
 378 }
 379
 380 static inline pte_t pte_mkglobal(pte_t pte)
 381 {
 382         return pte_set_flags(pte, _PAGE_GLOBAL);
 383 }
 384
 385 static inline pte_t pte_clrglobal(pte_t pte)
 386 {
 387         return pte_clear_flags(pte, _PAGE_GLOBAL);
 388 }
 389
 390 static inline pte_t pte_mkspecial(pte_t pte)
 391 {
 392         return pte_set_flags(pte, _PAGE_SPECIAL);
 393 }
 394
 395 static inline pte_t pte_mkdevmap(pte_t pte)
 396 {
 397         return pte_set_flags(pte, _PAGE_SPECIAL|_PAGE_DEVMAP);
 398 }
 399
 400 static inline pmd_t pmd_set_flags(pmd_t pmd, pmdval_t set)
 401 {
 402         pmdval_t v = native_pmd_val(pmd);
 403
 404         return native_make_pmd(v | set);
 405 }
 406
 407 static inline pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear)
 408 {
 409         pmdval_t v = native_pmd_val(pmd);
 410
 411         return native_make_pmd(v & ~clear);
 412 }
 413
 414 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
 415 static inline int pmd_uffd_wp(pmd_t pmd)
 416 {
 417         return pmd_flags(pmd) & _PAGE_UFFD_WP;
 418 }
 419
 420 static inline pmd_t pmd_mkuffd_wp(pmd_t pmd)
 421 {
 422         return pmd_set_flags(pmd, _PAGE_UFFD_WP);
 423 }
 424
 425 static inline pmd_t pmd_clear_uffd_wp(pmd_t pmd)
 426 {
 427         return pmd_clear_flags(pmd, _PAGE_UFFD_WP);
 428 }
 429 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
 430
 431 static inline pmd_t pmd_mkold(pmd_t pmd)
 432 {
 433         return pmd_clear_flags(pmd, _PAGE_ACCESSED);
 434 }
 435
 436 static inline pmd_t pmd_mkclean(pmd_t pmd)
 437 {
 438         return pmd_clear_flags(pmd, _PAGE_DIRTY);
 439 }
 440
 441 static inline pmd_t pmd_wrprotect(pmd_t pmd)
 442 {
 443         return pmd_clear_flags(pmd, _PAGE_RW);
 444 }
 445
 446 static inline pmd_t pmd_mkdirty(pmd_t pmd)
 447 {
 448         return pmd_set_flags(pmd, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
 449 }
 450
 451 static inline pmd_t pmd_mkdevmap(pmd_t pmd)
 452 {
 453         return pmd_set_flags(pmd, _PAGE_DEVMAP);
 454 }
 455
 456 static inline pmd_t pmd_mkhuge(pmd_t pmd)
 457 {
 458         return pmd_set_flags(pmd, _PAGE_PSE);
 459 }
 460
 461 static inline pmd_t pmd_mkyoung(pmd_t pmd)
 462 {
 463         return pmd_set_flags(pmd, _PAGE_ACCESSED);
 464 }
 465
 466 static inline pmd_t pmd_mkwrite(pmd_t pmd)
 467 {
 468         return pmd_set_flags(pmd, _PAGE_RW);
 469 }
 470
 471 static inline pud_t pud_set_flags(pud_t pud, pudval_t set)
 472 {
 473         pudval_t v = native_pud_val(pud);
 474
 475         return native_make_pud(v | set);
 476 }
 477
 478 static inline pud_t pud_clear_flags(pud_t pud, pudval_t clear)
 479 {
 480         pudval_t v = native_pud_val(pud);
 481
 482         return native_make_pud(v & ~clear);
 483 }
 484
 485 static inline pud_t pud_mkold(pud_t pud)
 486 {
 487         return pud_clear_flags(pud, _PAGE_ACCESSED);
 488 }
 489
 490 static inline pud_t pud_mkclean(pud_t pud)
 491 {
 492         return pud_clear_flags(pud, _PAGE_DIRTY);
 493 }
 494
 495 static inline pud_t pud_wrprotect(pud_t pud)
 496 {
 497         return pud_clear_flags(pud, _PAGE_RW);
 498 }
 499
 500 static inline pud_t pud_mkdirty(pud_t pud)
 501 {
 502         return pud_set_flags(pud, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
 503 }
 504
 505 static inline pud_t pud_mkdevmap(pud_t pud)
 506 {
 507         return pud_set_flags(pud, _PAGE_DEVMAP);
 508 }
 509
 510 static inline pud_t pud_mkhuge(pud_t pud)
 511 {
 512         return pud_set_flags(pud, _PAGE_PSE);
 513 }
 514
 515 static inline pud_t pud_mkyoung(pud_t pud)
 516 {
 517         return pud_set_flags(pud, _PAGE_ACCESSED);
 518 }
 519
 520 static inline pud_t pud_mkwrite(pud_t pud)
 521 {
 522         return pud_set_flags(pud, _PAGE_RW);
 523 }
 524
 525 #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
 526 static inline int pte_soft_dirty(pte_t pte)
 527 {
 528         return pte_flags(pte) & _PAGE_SOFT_DIRTY;
 529 }
 530
 531 static inline int pmd_soft_dirty(pmd_t pmd)
 532 {
 533         return pmd_flags(pmd) & _PAGE_SOFT_DIRTY;
 534 }
 535
 536 static inline int pud_soft_dirty(pud_t pud)
 537 {
 538         return pud_flags(pud) & _PAGE_SOFT_DIRTY;
 539 }
 540
 541 static inline pte_t pte_mksoft_dirty(pte_t pte)
 542 {
 543         return pte_set_flags(pte, _PAGE_SOFT_DIRTY);
 544 }
 545
 546 static inline pmd_t pmd_mksoft_dirty(pmd_t pmd)
 547 {
 548         return pmd_set_flags(pmd, _PAGE_SOFT_DIRTY);
 549 }
 550
 551 static inline pud_t pud_mksoft_dirty(pud_t pud)
 552 {
 553         return pud_set_flags(pud, _PAGE_SOFT_DIRTY);
 554 }
 555
 556 static inline pte_t pte_clear_soft_dirty(pte_t pte)
 557 {
 558         return pte_clear_flags(pte, _PAGE_SOFT_DIRTY);
 559 }
 560
 561 static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd)
 562 {
 563         return pmd_clear_flags(pmd, _PAGE_SOFT_DIRTY);
 564 }
 565
 566 static inline pud_t pud_clear_soft_dirty(pud_t pud)
 567 {
 568         return pud_clear_flags(pud, _PAGE_SOFT_DIRTY);
 569 }
 570
 571 #endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */
 572
 573 /*
 574  * Mask out unsupported bits in a present pgprot.  Non-present pgprots
 575  * can use those bits for other purposes, so leave them be.
 576  */
 577 static inline pgprotval_t massage_pgprot(pgprot_t pgprot)
 578 {
 579         pgprotval_t protval = pgprot_val(pgprot);
 580
 581         if (protval & _PAGE_PRESENT)
 582                 protval &= __supported_pte_mask;
 583
 584         return protval;
 585 }
 586
 587 static inline pgprotval_t check_pgprot(pgprot_t pgprot)
 588 {
 589         pgprotval_t massaged_val = massage_pgprot(pgprot);
 590
 591         /* mmdebug.h can not be included here because of dependencies */
 592 #ifdef CONFIG_DEBUG_VM
 593         WARN_ONCE(pgprot_val(pgprot) != massaged_val,
 594                   "attempted to set unsupported pgprot: %016llx "
 595                   "bits: %016llx supported: %016llx\n",
 596                   (u64)pgprot_val(pgprot),
 597                   (u64)pgprot_val(pgprot) ^ massaged_val,
 598                   (u64)__supported_pte_mask);
 599 #endif
 600
 601         return massaged_val;
 602 }
 603
 604 static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot)
 605 {
 606         phys_addr_t pfn = (phys_addr_t)page_nr << PAGE_SHIFT;
 607         pfn ^= protnone_mask(pgprot_val(pgprot));
 608         pfn &= PTE_PFN_MASK;
 609         return __pte(pfn | check_pgprot(pgprot));
 610 }
 611
 612 static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
 613 {
 614         phys_addr_t pfn = (phys_addr_t)page_nr << PAGE_SHIFT;
 615         pfn ^= protnone_mask(pgprot_val(pgprot));
 616         pfn &= PHYSICAL_PMD_PAGE_MASK;
 617         return __pmd(pfn | check_pgprot(pgprot));
 618 }
 619
 620 static inline pud_t pfn_pud(unsigned long page_nr, pgprot_t pgprot)
 621 {
 622         phys_addr_t pfn = (phys_addr_t)page_nr << PAGE_SHIFT;
 623         pfn ^= protnone_mask(pgprot_val(pgprot));
 624         pfn &= PHYSICAL_PUD_PAGE_MASK;
 625         return __pud(pfn | check_pgprot(pgprot));
 626 }
 627
 628 static inline pmd_t pmd_mknotpresent(pmd_t pmd)
 629 {
 630         return pfn_pmd(pmd_pfn(pmd),
 631                       __pgprot(pmd_flags(pmd) & ~(_PAGE_PRESENT|_PAGE_PROTNONE)));
 632 }
 633
 634 static inline u64 flip_protnone_guard(u64 oldval, u64 val, u64 mask);
 635
 636 static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
 637 {
 638         pteval_t val = pte_val(pte), oldval = val;
 639
 640         /*
 641          * Chop off the NX bit (if present), and add the NX portion of
 642          * the newprot (if present):
 643          */
 644         val &= _PAGE_CHG_MASK;
 645         val |= check_pgprot(newprot) & ~_PAGE_CHG_MASK;
 646         val = flip_protnone_guard(oldval, val, PTE_PFN_MASK);
 647         return __pte(val);
 648 }
 649
 650 static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
 651 {
 652         pmdval_t val = pmd_val(pmd), oldval = val;
 653
 654         val &= _HPAGE_CHG_MASK;
 655         val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK;
 656         val = flip_protnone_guard(oldval, val, PHYSICAL_PMD_PAGE_MASK);
 657         return __pmd(val);
 658 }
 659
 660 /*
 661  * mprotect needs to preserve PAT and encryption bits when updating
 662  * vm_page_prot
 663  */
 664 #define pgprot_modify pgprot_modify
 665 static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
 666 {
 667         pgprotval_t preservebits = pgprot_val(oldprot) & _PAGE_CHG_MASK;
 668         pgprotval_t addbits = pgprot_val(newprot) & ~_PAGE_CHG_MASK;
 669         return __pgprot(preservebits | addbits);
 670 }
 671
 672 #define pte_pgprot(x) __pgprot(pte_flags(x))
 673 #define pmd_pgprot(x) __pgprot(pmd_flags(x))
 674 #define pud_pgprot(x) __pgprot(pud_flags(x))
 675 #define p4d_pgprot(x) __pgprot(p4d_flags(x))
 676
 677 #define canon_pgprot(p) __pgprot(massage_pgprot(p))
 678
 679 static inline pgprot_t arch_filter_pgprot(pgprot_t prot)
 680 {
 681         return canon_pgprot(prot);
 682 }
 683
 684 static inline int is_new_memtype_allowed(u64 paddr, unsigned long size,
 685                                          enum page_cache_mode pcm,
 686                                          enum page_cache_mode new_pcm)
 687 {
 688         /*
 689          * PAT type is always WB for untracked ranges, so no need to check.
 690          */
 691         if (x86_platform.is_untracked_pat_range(paddr, paddr + size))
 692                 return 1;
 693
 694         /*
 695          * Certain new memtypes are not allowed with certain
 696          * requested memtype:
 697          * - request is uncached, return cannot be write-back
 698          * - request is write-combine, return cannot be write-back
 699          * - request is write-through, return cannot be write-back
 700          * - request is write-through, return cannot be write-combine
 701          */
 702         if ((pcm == _PAGE_CACHE_MODE_UC_MINUS &&
 703              new_pcm == _PAGE_CACHE_MODE_WB) ||
 704             (pcm == _PAGE_CACHE_MODE_WC &&
 705              new_pcm == _PAGE_CACHE_MODE_WB) ||
 706             (pcm == _PAGE_CACHE_MODE_WT &&
 707              new_pcm == _PAGE_CACHE_MODE_WB) ||
 708             (pcm == _PAGE_CACHE_MODE_WT &&
 709              new_pcm == _PAGE_CACHE_MODE_WC)) {
 710                 return 0;
 711         }
 712
 713         return 1;
 714 }
 715
 716 pmd_t *populate_extra_pmd(unsigned long vaddr);
 717 pte_t *populate_extra_pte(unsigned long vaddr);
 718
 719 #ifdef CONFIG_PAGE_TABLE_ISOLATION
 720 pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd);
 721
 722 /*
 723  * Take a PGD location (pgdp) and a pgd value that needs to be set there.
 724  * Populates the user and returns the resulting PGD that must be set in
 725  * the kernel copy of the page tables.
 726  */
 727 static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd)
 728 {
 729         if (!static_cpu_has(X86_FEATURE_PTI))
 730                 return pgd;
 731         return __pti_set_user_pgtbl(pgdp, pgd);
 732 }
 733 #else   /* CONFIG_PAGE_TABLE_ISOLATION */
 734 static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd)
 735 {
 736         return pgd;
 737 }
 738 #endif  /* CONFIG_PAGE_TABLE_ISOLATION */
 739
 740 #endif  /* __ASSEMBLY__ */
 741
 742
 743 #ifdef CONFIG_X86_32
 744 # include <asm/pgtable_32.h>
 745 #else
 746 # include <asm/pgtable_64.h>
 747 #endif
 748
 749 #ifndef __ASSEMBLY__
 750 #include <linux/mm_types.h>
 751 #include <linux/mmdebug.h>
 752 #include <linux/log2.h>
 753 #include <asm/fixmap.h>
 754
 755 static inline int pte_none(pte_t pte)
 756 {
 757         return !(pte.pte & ~(_PAGE_KNL_ERRATUM_MASK));
 758 }
 759
 760 #define __HAVE_ARCH_PTE_SAME
 761 static inline int pte_same(pte_t a, pte_t b)
 762 {
 763         return a.pte == b.pte;
 764 }
 765
 766 static inline int pte_present(pte_t a)
 767 {
 768         return pte_flags(a) & (_PAGE_PRESENT | _PAGE_PROTNONE);
 769 }
 770
 771 #ifdef CONFIG_ARCH_HAS_PTE_DEVMAP
 772 static inline int pte_devmap(pte_t a)
 773 {
 774         return (pte_flags(a) & _PAGE_DEVMAP) == _PAGE_DEVMAP;
 775 }
 776 #endif
 777
 778 #define pte_accessible pte_accessible
 779 static inline bool pte_accessible(struct mm_struct *mm, pte_t a)
 780 {
 781         if (pte_flags(a) & _PAGE_PRESENT)
 782                 return true;
 783
 784         if ((pte_flags(a) & _PAGE_PROTNONE) &&
 785                         mm_tlb_flush_pending(mm))
 786                 return true;
 787
 788         return false;
 789 }
 790
 791 static inline int pmd_present(pmd_t pmd)
 792 {
 793         /*
 794          * Checking for _PAGE_PSE is needed too because
 795          * split_huge_page will temporarily clear the present bit (but
 796          * the _PAGE_PSE flag will remain set at all times while the
 797          * _PAGE_PRESENT bit is clear).
 798          */
 799         return pmd_flags(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE | _PAGE_PSE);
 800 }
 801
 802 #ifdef CONFIG_NUMA_BALANCING
 803 /*
 804  * These work without NUMA balancing but the kernel does not care. See the
 805  * comment in include/asm-generic/pgtable.h
 806  */
 807 static inline int pte_protnone(pte_t pte)
 808 {
 809         return (pte_flags(pte) & (_PAGE_PROTNONE | _PAGE_PRESENT))
 810                 == _PAGE_PROTNONE;
 811 }
 812
 813 static inline int pmd_protnone(pmd_t pmd)
 814 {
 815         return (pmd_flags(pmd) & (_PAGE_PROTNONE | _PAGE_PRESENT))
 816                 == _PAGE_PROTNONE;
 817 }
 818 #endif /* CONFIG_NUMA_BALANCING */
 819
 820 static inline int pmd_none(pmd_t pmd)
 821 {
 822         /* Only check low word on 32-bit platforms, since it might be
 823            out of sync with upper half. */
 824         unsigned long val = native_pmd_val(pmd);
 825         return (val & ~_PAGE_KNL_ERRATUM_MASK) == 0;
 826 }
 827
 828 static inline unsigned long pmd_page_vaddr(pmd_t pmd)
 829 {
 830         return (unsigned long)__va(pmd_val(pmd) & pmd_pfn_mask(pmd));
 831 }
 832
 833 /*
 834  * Currently stuck as a macro due to indirect forward reference to
 835  * linux/mmzone.h's __section_mem_map_addr() definition:
 836  */
 837 #define pmd_page(pmd)   pfn_to_page(pmd_pfn(pmd))
 838
 839 /*
 840  * the pmd page can be thought of an array like this: pmd_t[PTRS_PER_PMD]
 841  *
 842  * this macro returns the index of the entry in the pmd page which would
 843  * control the given virtual address
 844  */
 845 static inline unsigned long pmd_index(unsigned long address)
 846 {
 847         return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1);
 848 }
 849
 850 /*
 851  * Conversion functions: convert a page and protection to a page entry,
 852  * and a page entry and page directory to the page they refer to.
 853  *
 854  * (Currently stuck as a macro because of indirect forward reference
 855  * to linux/mm.h:page_to_nid())
 856  */
 857 #define mk_pte(page, pgprot)   pfn_pte(page_to_pfn(page), (pgprot))
 858
 859 /*
 860  * the pte page can be thought of an array like this: pte_t[PTRS_PER_PTE]
 861  *
 862  * this function returns the index of the entry in the pte page which would
 863  * control the given virtual address
 864  *
 865  * Also define macro so we can test if pte_index is defined for arch.
 866  */
 867 #define pte_index pte_index
 868 static inline unsigned long pte_index(unsigned long address)
 869 {
 870         return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);
 871 }
 872
 873 static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address)
 874 {
 875         return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address);
 876 }
 877
 878 static inline int pmd_bad(pmd_t pmd)
 879 {
 880         return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE;
 881 }
 882
 883 static inline unsigned long pages_to_mb(unsigned long npg)
 884 {
 885         return npg >> (20 - PAGE_SHIFT);
 886 }
 887
 888 #if CONFIG_PGTABLE_LEVELS > 2
 889 static inline int pud_none(pud_t pud)
 890 {
 891         return (native_pud_val(pud) & ~(_PAGE_KNL_ERRATUM_MASK)) == 0;
 892 }
 893
 894 static inline int pud_present(pud_t pud)
 895 {
 896         return pud_flags(pud) & _PAGE_PRESENT;
 897 }
 898
 899 static inline unsigned long pud_page_vaddr(pud_t pud)
 900 {
 901         return (unsigned long)__va(pud_val(pud) & pud_pfn_mask(pud));
 902 }
 903
 904 /*
 905  * Currently stuck as a macro due to indirect forward reference to
 906  * linux/mmzone.h's __section_mem_map_addr() definition:
 907  */
 908 #define pud_page(pud)   pfn_to_page(pud_pfn(pud))
 909
 910 /* Find an entry in the second-level page table.. */
 911 static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
 912 {
 913         return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(address);
 914 }
 915
 916 #define pud_leaf        pud_large
 917 static inline int pud_large(pud_t pud)
 918 {
 919         return (pud_val(pud) & (_PAGE_PSE | _PAGE_PRESENT)) ==
 920                 (_PAGE_PSE | _PAGE_PRESENT);
 921 }
 922
 923 static inline int pud_bad(pud_t pud)
 924 {
 925         return (pud_flags(pud) & ~(_KERNPG_TABLE | _PAGE_USER)) != 0;
 926 }
 927 #else
 928 #define pud_leaf        pud_large
 929 static inline int pud_large(pud_t pud)
 930 {
 931         return 0;
 932 }
 933 #endif  /* CONFIG_PGTABLE_LEVELS > 2 */
 934
 935 static inline unsigned long pud_index(unsigned long address)
 936 {
 937         return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1);
 938 }
 939
 940 #if CONFIG_PGTABLE_LEVELS > 3
 941 static inline int p4d_none(p4d_t p4d)
 942 {
 943         return (native_p4d_val(p4d) & ~(_PAGE_KNL_ERRATUM_MASK)) == 0;
 944 }
 945
 946 static inline int p4d_present(p4d_t p4d)
 947 {
 948         return p4d_flags(p4d) & _PAGE_PRESENT;
 949 }
 950
 951 static inline unsigned long p4d_page_vaddr(p4d_t p4d)
 952 {
 953         return (unsigned long)__va(p4d_val(p4d) & p4d_pfn_mask(p4d));
 954 }
 955
 956 /*
 957  * Currently stuck as a macro due to indirect forward reference to
 958  * linux/mmzone.h's __section_mem_map_addr() definition:
 959  */
 960 #define p4d_page(p4d)   pfn_to_page(p4d_pfn(p4d))
 961
 962 /* Find an entry in the third-level page table.. */
 963 static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)
 964 {
 965         return (pud_t *)p4d_page_vaddr(*p4d) + pud_index(address);
 966 }
 967
 968 static inline int p4d_bad(p4d_t p4d)
 969 {
 970         unsigned long ignore_flags = _KERNPG_TABLE | _PAGE_USER;
 971
 972         if (IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION))
 973                 ignore_flags |= _PAGE_NX;
 974
 975         return (p4d_flags(p4d) & ~ignore_flags) != 0;
 976 }
 977 #endif  /* CONFIG_PGTABLE_LEVELS > 3 */
 978
 979 static inline unsigned long p4d_index(unsigned long address)
 980 {
 981         return (address >> P4D_SHIFT) & (PTRS_PER_P4D - 1);
 982 }
 983
 984 #if CONFIG_PGTABLE_LEVELS > 4
 985 static inline int pgd_present(pgd_t pgd)
 986 {
 987         if (!pgtable_l5_enabled())
 988                 return 1;
 989         return pgd_flags(pgd) & _PAGE_PRESENT;
 990 }
 991
 992 static inline unsigned long pgd_page_vaddr(pgd_t pgd)
 993 {
 994         return (unsigned long)__va((unsigned long)pgd_val(pgd) & PTE_PFN_MASK);
 995 }
 996
 997 /*
 998  * Currently stuck as a macro due to indirect forward reference to
 999  * linux/mmzone.h's __section_mem_map_addr() definition:
1000  */
1001 #define pgd_page(pgd)   pfn_to_page(pgd_pfn(pgd))
1002
1003 /* to find an entry in a page-table-directory. */
1004 static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address)
1005 {
1006         if (!pgtable_l5_enabled())
1007                 return (p4d_t *)pgd;
1008         return (p4d_t *)pgd_page_vaddr(*pgd) + p4d_index(address);
1009 }
1010
1011 static inline int pgd_bad(pgd_t pgd)
1012 {
1013         unsigned long ignore_flags = _PAGE_USER;
1014
1015         if (!pgtable_l5_enabled())
1016                 return 0;
1017
1018         if (IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION))
1019                 ignore_flags |= _PAGE_NX;
1020
1021         return (pgd_flags(pgd) & ~ignore_flags) != _KERNPG_TABLE;
1022 }
1023
1024 static inline int pgd_none(pgd_t pgd)
1025 {
1026         if (!pgtable_l5_enabled())
1027                 return 0;
1028         /*
1029          * There is no need to do a workaround for the KNL stray
1030          * A/D bit erratum here.  PGDs only point to page tables
1031          * except on 32-bit non-PAE which is not supported on
1032          * KNL.
1033          */
1034         return !native_pgd_val(pgd);
1035 }
1036 #endif  /* CONFIG_PGTABLE_LEVELS > 4 */
1037
1038 #endif  /* __ASSEMBLY__ */
1039
1040 /*
1041  * the pgd page can be thought of an array like this: pgd_t[PTRS_PER_PGD]
1042  *
1043  * this macro returns the index of the entry in the pgd page which would
1044  * control the given virtual address
1045  */
1046 #define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
1047
1048 /*
1049  * pgd_offset() returns a (pgd_t *)
1050  * pgd_index() is used get the offset into the pgd page's array of pgd_t's;
1051  */
1052 #define pgd_offset_pgd(pgd, address) (pgd + pgd_index((address)))
1053 /*
1054  * a shortcut to get a pgd_t in a given mm
1055  */
1056 #define pgd_offset(mm, address) pgd_offset_pgd((mm)->pgd, (address))
1057 /*
1058  * a shortcut which implies the use of the kernel's pgd, instead
1059  * of a process's
1060  */
1061 #define pgd_offset_k(address) pgd_offset(&init_mm, (address))
1062
1063
1064 #define KERNEL_PGD_BOUNDARY     pgd_index(PAGE_OFFSET)
1065 #define KERNEL_PGD_PTRS         (PTRS_PER_PGD - KERNEL_PGD_BOUNDARY)
1066
1067 #ifndef __ASSEMBLY__
1068
1069 extern int direct_gbpages;
1070 void init_mem_mapping(void);
1071 void early_alloc_pgt_buf(void);
1072 extern void memblock_find_dma_reserve(void);
1073
1074 #ifdef CONFIG_X86_64
1075 /* Realmode trampoline initialization. */
1076 extern pgd_t trampoline_pgd_entry;
1077 static inline void __meminit init_trampoline_default(void)
1078 {
1079         /* Default trampoline pgd value */
1080         trampoline_pgd_entry = init_top_pgt[pgd_index(__PAGE_OFFSET)];
1081 }
1082
1083 void __init poking_init(void);
1084
1085 unsigned long init_memory_mapping(unsigned long start,
1086                                   unsigned long end, pgprot_t prot);
1087
1088 # ifdef CONFIG_RANDOMIZE_MEMORY
1089 void __meminit init_trampoline(void);
1090 # else
1091 #  define init_trampoline init_trampoline_default
1092 # endif
1093 #else
1094 static inline void init_trampoline(void) { }
1095 #endif
1096
1097 /* local pte updates need not use xchg for locking */
1098 static inline pte_t native_local_ptep_get_and_clear(pte_t *ptep)
1099 {
1100         pte_t res = *ptep;
1101
1102         /* Pure native function needs no input for mm, addr */
1103         native_pte_clear(NULL, 0, ptep);
1104         return res;
1105 }
1106
1107 static inline pmd_t native_local_pmdp_get_and_clear(pmd_t *pmdp)
1108 {
1109         pmd_t res = *pmdp;
1110
1111         native_pmd_clear(pmdp);
1112         return res;
1113 }
1114
1115 static inline pud_t native_local_pudp_get_and_clear(pud_t *pudp)
1116 {
1117         pud_t res = *pudp;
1118
1119         native_pud_clear(pudp);
1120         return res;
1121 }
1122
1123 static inline void native_set_pte_at(struct mm_struct *mm, unsigned long addr,
1124                                      pte_t *ptep , pte_t pte)
1125 {
1126         native_set_pte(ptep, pte);
1127 }
1128
1129 static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
1130                               pmd_t *pmdp, pmd_t pmd)
1131 {
1132         set_pmd(pmdp, pmd);
1133 }
1134
1135 static inline void set_pud_at(struct mm_struct *mm, unsigned long addr,
1136                               pud_t *pudp, pud_t pud)
1137 {
1138         native_set_pud(pudp, pud);
1139 }
1140
1141 /*
1142  * We only update the dirty/accessed state if we set
1143  * the dirty bit by hand in the kernel, since the hardware
1144  * will do the accessed bit for us, and we don't want to
1145  * race with other CPU's that might be updating the dirty
1146  * bit at the same time.
1147  */
1148 struct vm_area_struct;
1149
1150 #define  __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
1151 extern int ptep_set_access_flags(struct vm_area_struct *vma,
1152                                  unsigned long address, pte_t *ptep,
1153                                  pte_t entry, int dirty);
1154
1155 #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
1156 extern int ptep_test_and_clear_young(struct vm_area_struct *vma,
1157                                      unsigned long addr, pte_t *ptep);
1158
1159 #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
1160 extern int ptep_clear_flush_young(struct vm_area_struct *vma,
1161                                   unsigned long address, pte_t *ptep);
1162
1163 #define __HAVE_ARCH_PTEP_GET_AND_CLEAR
1164 static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
1165                                        pte_t *ptep)
1166 {
1167         pte_t pte = native_ptep_get_and_clear(ptep);
1168         return pte;
1169 }
1170
1171 #define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
1172 static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
1173                                             unsigned long addr, pte_t *ptep,
1174                                             int full)
1175 {
1176         pte_t pte;
1177         if (full) {
1178                 /*
1179                  * Full address destruction in progress; paravirt does not
1180                  * care about updates and native needs no locking
1181                  */
1182                 pte = native_local_ptep_get_and_clear(ptep);
1183         } else {
1184                 pte = ptep_get_and_clear(mm, addr, ptep);
1185         }
1186         return pte;
1187 }
1188
1189 #define __HAVE_ARCH_PTEP_SET_WRPROTECT
1190 static inline void ptep_set_wrprotect(struct mm_struct *mm,
1191                                       unsigned long addr, pte_t *ptep)
1192 {
1193         clear_bit(_PAGE_BIT_RW, (unsigned long *)&ptep->pte);
1194 }
1195
1196 #define flush_tlb_fix_spurious_fault(vma, address) do { } while (0)
1197
1198 #define mk_pmd(page, pgprot)   pfn_pmd(page_to_pfn(page), (pgprot))
1199
1200 #define  __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
1201 extern int pmdp_set_access_flags(struct vm_area_struct *vma,
1202                                  unsigned long address, pmd_t *pmdp,
1203                                  pmd_t entry, int dirty);
1204 extern int pudp_set_access_flags(struct vm_area_struct *vma,
1205                                  unsigned long address, pud_t *pudp,
1206                                  pud_t entry, int dirty);
1207
1208 #define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
1209 extern int pmdp_test_and_clear_young(struct vm_area_struct *vma,
1210                                      unsigned long addr, pmd_t *pmdp);
1211 extern int pudp_test_and_clear_young(struct vm_area_struct *vma,
1212                                      unsigned long addr, pud_t *pudp);
1213
1214 #define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
1215 extern int pmdp_clear_flush_young(struct vm_area_struct *vma,
1216                                   unsigned long address, pmd_t *pmdp);
1217
1218
1219 #define pmd_write pmd_write
1220 static inline int pmd_write(pmd_t pmd)
1221 {
1222         return pmd_flags(pmd) & _PAGE_RW;
1223 }
1224
1225 #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
1226 static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr,
1227                                        pmd_t *pmdp)
1228 {
1229         return native_pmdp_get_and_clear(pmdp);
1230 }
1231
1232 #define __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR
1233 static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm,
1234                                         unsigned long addr, pud_t *pudp)
1235 {
1236         return native_pudp_get_and_clear(pudp);
1237 }
1238
1239 #define __HAVE_ARCH_PMDP_SET_WRPROTECT
1240 static inline void pmdp_set_wrprotect(struct mm_struct *mm,
1241                                       unsigned long addr, pmd_t *pmdp)
1242 {
1243         clear_bit(_PAGE_BIT_RW, (unsigned long *)pmdp);
1244 }
1245
1246 #define pud_write pud_write
1247 static inline int pud_write(pud_t pud)
1248 {
1249         return pud_flags(pud) & _PAGE_RW;
1250 }
1251
1252 #ifndef pmdp_establish
1253 #define pmdp_establish pmdp_establish
1254 static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
1255                 unsigned long address, pmd_t *pmdp, pmd_t pmd)
1256 {
1257         if (IS_ENABLED(CONFIG_SMP)) {
1258                 return xchg(pmdp, pmd);
1259         } else {
1260                 pmd_t old = *pmdp;
1261                 WRITE_ONCE(*pmdp, pmd);
1262                 return old;
1263         }
1264 }
1265 #endif
1266 /*
1267  * Page table pages are page-aligned.  The lower half of the top
1268  * level is used for userspace and the top half for the kernel.
1269  *
1270  * Returns true for parts of the PGD that map userspace and
1271  * false for the parts that map the kernel.
1272  */
1273 static inline bool pgdp_maps_userspace(void *__ptr)
1274 {
1275         unsigned long ptr = (unsigned long)__ptr;
1276
1277         return (((ptr & ~PAGE_MASK) / sizeof(pgd_t)) < PGD_KERNEL_START);
1278 }
1279
1280 #define pgd_leaf        pgd_large
1281 static inline int pgd_large(pgd_t pgd) { return 0; }
1282
1283 #ifdef CONFIG_PAGE_TABLE_ISOLATION
1284 /*
1285  * All top-level PAGE_TABLE_ISOLATION page tables are order-1 pages
1286  * (8k-aligned and 8k in size).  The kernel one is at the beginning 4k and
1287  * the user one is in the last 4k.  To switch between them, you
1288  * just need to flip the 12th bit in their addresses.
1289  */
1290 #define PTI_PGTABLE_SWITCH_BIT  PAGE_SHIFT
1291
1292 /*
1293  * This generates better code than the inline assembly in
1294  * __set_bit().
1295  */
1296 static inline void *ptr_set_bit(void *ptr, int bit)
1297 {
1298         unsigned long __ptr = (unsigned long)ptr;
1299
1300         __ptr |= BIT(bit);
1301         return (void *)__ptr;
1302 }
1303 static inline void *ptr_clear_bit(void *ptr, int bit)
1304 {
1305         unsigned long __ptr = (unsigned long)ptr;
1306
1307         __ptr &= ~BIT(bit);
1308         return (void *)__ptr;
1309 }
1310
1311 static inline pgd_t *kernel_to_user_pgdp(pgd_t *pgdp)
1312 {
1313         return ptr_set_bit(pgdp, PTI_PGTABLE_SWITCH_BIT);
1314 }
1315
1316 static inline pgd_t *user_to_kernel_pgdp(pgd_t *pgdp)
1317 {
1318         return ptr_clear_bit(pgdp, PTI_PGTABLE_SWITCH_BIT);
1319 }
1320
1321 static inline p4d_t *kernel_to_user_p4dp(p4d_t *p4dp)
1322 {
1323         return ptr_set_bit(p4dp, PTI_PGTABLE_SWITCH_BIT);
1324 }
1325
1326 static inline p4d_t *user_to_kernel_p4dp(p4d_t *p4dp)
1327 {
1328         return ptr_clear_bit(p4dp, PTI_PGTABLE_SWITCH_BIT);
1329 }
1330 #endif /* CONFIG_PAGE_TABLE_ISOLATION */
1331
1332 /*
1333  * clone_pgd_range(pgd_t *dst, pgd_t *src, int count);
1334  *
1335  *  dst - pointer to pgd range anwhere on a pgd page
1336  *  src - ""
1337  *  count - the number of pgds to copy.
1338  *
1339  * dst and src can be on the same page, but the range must not overlap,
1340  * and must not cross a page boundary.
1341  */
1342 static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count)
1343 {
1344         memcpy(dst, src, count * sizeof(pgd_t));
1345 #ifdef CONFIG_PAGE_TABLE_ISOLATION
1346         if (!static_cpu_has(X86_FEATURE_PTI))
1347                 return;
1348         /* Clone the user space pgd as well */
1349         memcpy(kernel_to_user_pgdp(dst), kernel_to_user_pgdp(src),
1350                count * sizeof(pgd_t));
1351 #endif
1352 }
1353
1354 #define PTE_SHIFT ilog2(PTRS_PER_PTE)
1355 static inline int page_level_shift(enum pg_level level)
1356 {
1357         return (PAGE_SHIFT - PTE_SHIFT) + level * PTE_SHIFT;
1358 }
1359 static inline unsigned long page_level_size(enum pg_level level)
1360 {
1361         return 1UL << page_level_shift(level);
1362 }
1363 static inline unsigned long page_level_mask(enum pg_level level)
1364 {
1365         return ~(page_level_size(level) - 1);
1366 }
1367
1368 /*
1369  * The x86 doesn't have any external MMU info: the kernel page
1370  * tables contain all the necessary information.
1371  */
1372 static inline void update_mmu_cache(struct vm_area_struct *vma,
1373                 unsigned long addr, pte_t *ptep)
1374 {
1375 }
1376 static inline void update_mmu_cache_pmd(struct vm_area_struct *vma,
1377                 unsigned long addr, pmd_t *pmd)
1378 {
1379 }
1380 static inline void update_mmu_cache_pud(struct vm_area_struct *vma,
1381                 unsigned long addr, pud_t *pud)
1382 {
1383 }
1384
1385 #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
1386 static inline pte_t pte_swp_mksoft_dirty(pte_t pte)
1387 {
1388         return pte_set_flags(pte, _PAGE_SWP_SOFT_DIRTY);
1389 }
1390
1391 static inline int pte_swp_soft_dirty(pte_t pte)
1392 {
1393         return pte_flags(pte) & _PAGE_SWP_SOFT_DIRTY;
1394 }
1395
1396 static inline pte_t pte_swp_clear_soft_dirty(pte_t pte)
1397 {
1398         return pte_clear_flags(pte, _PAGE_SWP_SOFT_DIRTY);
1399 }
1400
1401 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
1402 static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd)
1403 {
1404         return pmd_set_flags(pmd, _PAGE_SWP_SOFT_DIRTY);
1405 }
1406
1407 static inline int pmd_swp_soft_dirty(pmd_t pmd)
1408 {
1409         return pmd_flags(pmd) & _PAGE_SWP_SOFT_DIRTY;
1410 }
1411
1412 static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd)
1413 {
1414         return pmd_clear_flags(pmd, _PAGE_SWP_SOFT_DIRTY);
1415 }
1416 #endif
1417 #endif
1418
1419 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
1420 static inline pte_t pte_swp_mkuffd_wp(pte_t pte)
1421 {
1422         return pte_set_flags(pte, _PAGE_SWP_UFFD_WP);
1423 }
1424
1425 static inline int pte_swp_uffd_wp(pte_t pte)
1426 {
1427         return pte_flags(pte) & _PAGE_SWP_UFFD_WP;
1428 }
1429
1430 static inline pte_t pte_swp_clear_uffd_wp(pte_t pte)
1431 {
1432         return pte_clear_flags(pte, _PAGE_SWP_UFFD_WP);
1433 }
1434
1435 static inline pmd_t pmd_swp_mkuffd_wp(pmd_t pmd)
1436 {
1437         return pmd_set_flags(pmd, _PAGE_SWP_UFFD_WP);
1438 }
1439
1440 static inline int pmd_swp_uffd_wp(pmd_t pmd)
1441 {
1442         return pmd_flags(pmd) & _PAGE_SWP_UFFD_WP;
1443 }
1444
1445 static inline pmd_t pmd_swp_clear_uffd_wp(pmd_t pmd)
1446 {
1447         return pmd_clear_flags(pmd, _PAGE_SWP_UFFD_WP);
1448 }
1449 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
1450
1451 #define PKRU_AD_BIT 0x1
1452 #define PKRU_WD_BIT 0x2
1453 #define PKRU_BITS_PER_PKEY 2
1454
1455 #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
1456 extern u32 init_pkru_value;
1457 #else
1458 #define init_pkru_value 0
1459 #endif
1460
1461 static inline bool __pkru_allows_read(u32 pkru, u16 pkey)
1462 {
1463         int pkru_pkey_bits = pkey * PKRU_BITS_PER_PKEY;
1464         return !(pkru & (PKRU_AD_BIT << pkru_pkey_bits));
1465 }
1466
1467 static inline bool __pkru_allows_write(u32 pkru, u16 pkey)
1468 {
1469         int pkru_pkey_bits = pkey * PKRU_BITS_PER_PKEY;
1470         /*
1471          * Access-disable disables writes too so we need to check
1472          * both bits here.
1473          */
1474         return !(pkru & ((PKRU_AD_BIT|PKRU_WD_BIT) << pkru_pkey_bits));
1475 }
1476
1477 static inline u16 pte_flags_pkey(unsigned long pte_flags)
1478 {
1479 #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
1480         /* ifdef to avoid doing 59-bit shift on 32-bit values */
1481         return (pte_flags & _PAGE_PKEY_MASK) >> _PAGE_BIT_PKEY_BIT0;
1482 #else
1483         return 0;
1484 #endif
1485 }
1486
1487 static inline bool __pkru_allows_pkey(u16 pkey, bool write)
1488 {
1489         u32 pkru = read_pkru();
1490
1491         if (!__pkru_allows_read(pkru, pkey))
1492                 return false;
1493         if (write && !__pkru_allows_write(pkru, pkey))
1494                 return false;
1495
1496         return true;
1497 }
1498
1499 /*
1500  * 'pteval' can come from a PTE, PMD or PUD.  We only check
1501  * _PAGE_PRESENT, _PAGE_USER, and _PAGE_RW in here which are the
1502  * same value on all 3 types.
1503  */
1504 static inline bool __pte_access_permitted(unsigned long pteval, bool write)
1505 {
1506         unsigned long need_pte_bits = _PAGE_PRESENT|_PAGE_USER;
1507
1508         if (write)
1509                 need_pte_bits |= _PAGE_RW;
1510
1511         if ((pteval & need_pte_bits) != need_pte_bits)
1512                 return 0;
1513
1514         return __pkru_allows_pkey(pte_flags_pkey(pteval), write);
1515 }
1516
1517 #define pte_access_permitted pte_access_permitted
1518 static inline bool pte_access_permitted(pte_t pte, bool write)
1519 {
1520         return __pte_access_permitted(pte_val(pte), write);
1521 }
1522
1523 #define pmd_access_permitted pmd_access_permitted
1524 static inline bool pmd_access_permitted(pmd_t pmd, bool write)
1525 {
1526         return __pte_access_permitted(pmd_val(pmd), write);
1527 }
1528
1529 #define pud_access_permitted pud_access_permitted
1530 static inline bool pud_access_permitted(pud_t pud, bool write)
1531 {
1532         return __pte_access_permitted(pud_val(pud), write);
1533 }
1534
1535 #define __HAVE_ARCH_PFN_MODIFY_ALLOWED 1
1536 extern bool pfn_modify_allowed(unsigned long pfn, pgprot_t prot);
1537
1538 static inline bool arch_has_pfn_modify_check(void)
1539 {
1540         return boot_cpu_has_bug(X86_BUG_L1TF);
1541 }
1542
1543 #define arch_faults_on_old_pte arch_faults_on_old_pte
1544 static inline bool arch_faults_on_old_pte(void)
1545 {
1546         return false;
1547 }
1548
1549 #include <asm-generic/pgtable.h>
1550 #endif  /* __ASSEMBLY__ */
1551
1552 #endif /* _ASM_X86_PGTABLE_H */