arch/riscv/include/asm/pgtable-64.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * Copyright (C) 2012 Regents of the University of California
   4  */
   5
   6 #ifndef _ASM_RISCV_PGTABLE_64_H
   7 #define _ASM_RISCV_PGTABLE_64_H
   8
   9 #include <linux/bits.h>
  10 #include <linux/const.h>
  11 #include <asm/errata_list.h>
  12
  13 extern bool pgtable_l4_enabled;
  14 extern bool pgtable_l5_enabled;
  15
  16 #define PGDIR_SHIFT_L3  30
  17 #define PGDIR_SHIFT_L4  39
  18 #define PGDIR_SHIFT_L5  48
  19 #define PGDIR_SHIFT     (pgtable_l5_enabled ? PGDIR_SHIFT_L5 : \
  20                 (pgtable_l4_enabled ? PGDIR_SHIFT_L4 : PGDIR_SHIFT_L3))
  21 /* Size of region mapped by a page global directory */
  22 #define PGDIR_SIZE      (_AC(1, UL) << PGDIR_SHIFT)
  23 #define PGDIR_MASK      (~(PGDIR_SIZE - 1))
  24
  25 /* p4d is folded into pgd in case of 4-level page table */
  26 #define P4D_SHIFT_L3   30
  27 #define P4D_SHIFT_L4   39
  28 #define P4D_SHIFT_L5   39
  29 #define P4D_SHIFT      (pgtable_l5_enabled ? P4D_SHIFT_L5 : \
  30                 (pgtable_l4_enabled ? P4D_SHIFT_L4 : P4D_SHIFT_L3))
  31 #define P4D_SIZE       (_AC(1, UL) << P4D_SHIFT)
  32 #define P4D_MASK       (~(P4D_SIZE - 1))
  33
  34 /* pud is folded into pgd in case of 3-level page table */
  35 #define PUD_SHIFT      30
  36 #define PUD_SIZE       (_AC(1, UL) << PUD_SHIFT)
  37 #define PUD_MASK       (~(PUD_SIZE - 1))
  38
  39 #define PMD_SHIFT       21
  40 /* Size of region mapped by a page middle directory */
  41 #define PMD_SIZE        (_AC(1, UL) << PMD_SHIFT)
  42 #define PMD_MASK        (~(PMD_SIZE - 1))
  43
  44 /* Page 4th Directory entry */
  45 typedef struct {
  46         unsigned long p4d;
  47 } p4d_t;
  48
  49 #define p4d_val(x)      ((x).p4d)
  50 #define __p4d(x)        ((p4d_t) { (x) })
  51 #define PTRS_PER_P4D    (PAGE_SIZE / sizeof(p4d_t))
  52
  53 /* Page Upper Directory entry */
  54 typedef struct {
  55         unsigned long pud;
  56 } pud_t;
  57
  58 #define pud_val(x)      ((x).pud)
  59 #define __pud(x)        ((pud_t) { (x) })
  60 #define PTRS_PER_PUD    (PAGE_SIZE / sizeof(pud_t))
  61
  62 /* Page Middle Directory entry */
  63 typedef struct {
  64         unsigned long pmd;
  65 } pmd_t;
  66
  67 #define pmd_val(x)      ((x).pmd)
  68 #define __pmd(x)        ((pmd_t) { (x) })
  69
  70 #define PTRS_PER_PMD    (PAGE_SIZE / sizeof(pmd_t))
  71
  72 /*
  73  * rv64 PTE format:
  74  * | 63 | 62 61 | 60 54 | 53  10 | 9             8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0
  75  *   N      MT     RSV    PFN      reserved for SW   D   A   G   U   X   W   R   V
  76  */
  77 #define _PAGE_PFN_MASK  GENMASK(53, 10)
  78
  79 /*
  80  * [63] Svnapot definitions:
  81  * 0 Svnapot disabled
  82  * 1 Svnapot enabled
  83  */
  84 #define _PAGE_NAPOT_SHIFT       63
  85 #define _PAGE_NAPOT             BIT(_PAGE_NAPOT_SHIFT)
  86 /*
  87  * Only 64KB (order 4) napot ptes supported.
  88  */
  89 #define NAPOT_CONT_ORDER_BASE 4
  90 enum napot_cont_order {
  91         NAPOT_CONT64KB_ORDER = NAPOT_CONT_ORDER_BASE,
  92         NAPOT_ORDER_MAX,
  93 };
  94
  95 #define for_each_napot_order(order)                                             \
  96         for (order = NAPOT_CONT_ORDER_BASE; order < NAPOT_ORDER_MAX; order++)
  97 #define for_each_napot_order_rev(order)                                         \
  98         for (order = NAPOT_ORDER_MAX - 1;                                       \
  99              order >= NAPOT_CONT_ORDER_BASE; order--)
 100 #define napot_cont_order(val)   (__builtin_ctzl((val.pte >> _PAGE_PFN_SHIFT) << 1))
 101
 102 #define napot_cont_shift(order) ((order) + PAGE_SHIFT)
 103 #define napot_cont_size(order)  BIT(napot_cont_shift(order))
 104 #define napot_cont_mask(order)  (~(napot_cont_size(order) - 1UL))
 105 #define napot_pte_num(order)    BIT(order)
 106
 107 #ifdef CONFIG_RISCV_ISA_SVNAPOT
 108 #define HUGE_MAX_HSTATE         (2 + (NAPOT_ORDER_MAX - NAPOT_CONT_ORDER_BASE))
 109 #else
 110 #define HUGE_MAX_HSTATE         2
 111 #endif
 112
 113 /*
 114  * [62:61] Svpbmt Memory Type definitions:
 115  *
 116  *  00 - PMA    Normal Cacheable, No change to implied PMA memory type
 117  *  01 - NC     Non-cacheable, idempotent, weakly-ordered Main Memory
 118  *  10 - IO     Non-cacheable, non-idempotent, strongly-ordered I/O memory
 119  *  11 - Rsvd   Reserved for future standard use
 120  */
 121 #define _PAGE_NOCACHE_SVPBMT    (1UL << 61)
 122 #define _PAGE_IO_SVPBMT         (1UL << 62)
 123 #define _PAGE_MTMASK_SVPBMT     (_PAGE_NOCACHE_SVPBMT | _PAGE_IO_SVPBMT)
 124
 125 /*
 126  * [63:59] T-Head Memory Type definitions:
 127  * bit[63] SO - Strong Order
 128  * bit[62] C - Cacheable
 129  * bit[61] B - Bufferable
 130  * bit[60] SH - Shareable
 131  * bit[59] Sec - Trustable
 132  * 00110 - NC   Weakly-ordered, Non-cacheable, Bufferable, Shareable, Non-trustable
 133  * 01110 - PMA  Weakly-ordered, Cacheable, Bufferable, Shareable, Non-trustable
 134  * 10010 - IO   Strongly-ordered, Non-cacheable, Non-bufferable, Shareable, Non-trustable
 135  */
 136 #define _PAGE_PMA_THEAD         ((1UL << 62) | (1UL << 61) | (1UL << 60))
 137 #define _PAGE_NOCACHE_THEAD     ((1UL << 61) | (1UL << 60))
 138 #define _PAGE_IO_THEAD          ((1UL << 63) | (1UL << 60))
 139 #define _PAGE_MTMASK_THEAD      (_PAGE_PMA_THEAD | _PAGE_IO_THEAD | (1UL << 59))
 140
 141 static inline u64 riscv_page_mtmask(void)
 142 {
 143         u64 val;
 144
 145         ALT_SVPBMT(val, _PAGE_MTMASK);
 146         return val;
 147 }
 148
 149 static inline u64 riscv_page_nocache(void)
 150 {
 151         u64 val;
 152
 153         ALT_SVPBMT(val, _PAGE_NOCACHE);
 154         return val;
 155 }
 156
 157 static inline u64 riscv_page_io(void)
 158 {
 159         u64 val;
 160
 161         ALT_SVPBMT(val, _PAGE_IO);
 162         return val;
 163 }
 164
 165 #define _PAGE_NOCACHE           riscv_page_nocache()
 166 #define _PAGE_IO                riscv_page_io()
 167 #define _PAGE_MTMASK            riscv_page_mtmask()
 168
 169 /* Set of bits to preserve across pte_modify() */
 170 #define _PAGE_CHG_MASK  (~(unsigned long)(_PAGE_PRESENT | _PAGE_READ |  \
 171                                           _PAGE_WRITE | _PAGE_EXEC |    \
 172                                           _PAGE_USER | _PAGE_GLOBAL |   \
 173                                           _PAGE_MTMASK))
 174
 175 static inline int pud_present(pud_t pud)
 176 {
 177         return (pud_val(pud) & _PAGE_PRESENT);
 178 }
 179
 180 static inline int pud_none(pud_t pud)
 181 {
 182         return (pud_val(pud) == 0);
 183 }
 184
 185 static inline int pud_bad(pud_t pud)
 186 {
 187         return !pud_present(pud);
 188 }
 189
 190 #define pud_leaf        pud_leaf
 191 static inline bool pud_leaf(pud_t pud)
 192 {
 193         return pud_present(pud) && (pud_val(pud) & _PAGE_LEAF);
 194 }
 195
 196 static inline int pud_user(pud_t pud)
 197 {
 198         return pud_val(pud) & _PAGE_USER;
 199 }
 200
 201 static inline void set_pud(pud_t *pudp, pud_t pud)
 202 {
 203         WRITE_ONCE(*pudp, pud);
 204 }
 205
 206 static inline void pud_clear(pud_t *pudp)
 207 {
 208         set_pud(pudp, __pud(0));
 209 }
 210
 211 static inline pud_t pfn_pud(unsigned long pfn, pgprot_t prot)
 212 {
 213         return __pud((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot));
 214 }
 215
 216 static inline unsigned long _pud_pfn(pud_t pud)
 217 {
 218         return __page_val_to_pfn(pud_val(pud));
 219 }
 220
 221 static inline pmd_t *pud_pgtable(pud_t pud)
 222 {
 223         return (pmd_t *)pfn_to_virt(__page_val_to_pfn(pud_val(pud)));
 224 }
 225
 226 static inline struct page *pud_page(pud_t pud)
 227 {
 228         return pfn_to_page(__page_val_to_pfn(pud_val(pud)));
 229 }
 230
 231 #define mm_p4d_folded  mm_p4d_folded
 232 static inline bool mm_p4d_folded(struct mm_struct *mm)
 233 {
 234         if (pgtable_l5_enabled)
 235                 return false;
 236
 237         return true;
 238 }
 239
 240 #define mm_pud_folded  mm_pud_folded
 241 static inline bool mm_pud_folded(struct mm_struct *mm)
 242 {
 243         if (pgtable_l4_enabled)
 244                 return false;
 245
 246         return true;
 247 }
 248
 249 #define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))
 250
 251 static inline pmd_t pfn_pmd(unsigned long pfn, pgprot_t prot)
 252 {
 253         unsigned long prot_val = pgprot_val(prot);
 254
 255         ALT_THEAD_PMA(prot_val);
 256
 257         return __pmd((pfn << _PAGE_PFN_SHIFT) | prot_val);
 258 }
 259
 260 static inline unsigned long _pmd_pfn(pmd_t pmd)
 261 {
 262         return __page_val_to_pfn(pmd_val(pmd));
 263 }
 264
 265 #define mk_pmd(page, prot)    pfn_pmd(page_to_pfn(page), prot)
 266
 267 #define pmd_ERROR(e) \
 268         pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e))
 269
 270 #define pud_ERROR(e)   \
 271         pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e))
 272
 273 #define p4d_ERROR(e)   \
 274         pr_err("%s:%d: bad p4d %016lx.\n", __FILE__, __LINE__, p4d_val(e))
 275
 276 static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)
 277 {
 278         if (pgtable_l4_enabled)
 279                 WRITE_ONCE(*p4dp, p4d);
 280         else
 281                 set_pud((pud_t *)p4dp, (pud_t){ p4d_val(p4d) });
 282 }
 283
 284 static inline int p4d_none(p4d_t p4d)
 285 {
 286         if (pgtable_l4_enabled)
 287                 return (p4d_val(p4d) == 0);
 288
 289         return 0;
 290 }
 291
 292 static inline int p4d_present(p4d_t p4d)
 293 {
 294         if (pgtable_l4_enabled)
 295                 return (p4d_val(p4d) & _PAGE_PRESENT);
 296
 297         return 1;
 298 }
 299
 300 static inline int p4d_bad(p4d_t p4d)
 301 {
 302         if (pgtable_l4_enabled)
 303                 return !p4d_present(p4d);
 304
 305         return 0;
 306 }
 307
 308 static inline void p4d_clear(p4d_t *p4d)
 309 {
 310         if (pgtable_l4_enabled)
 311                 set_p4d(p4d, __p4d(0));
 312 }
 313
 314 static inline p4d_t pfn_p4d(unsigned long pfn, pgprot_t prot)
 315 {
 316         return __p4d((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot));
 317 }
 318
 319 static inline unsigned long _p4d_pfn(p4d_t p4d)
 320 {
 321         return __page_val_to_pfn(p4d_val(p4d));
 322 }
 323
 324 static inline pud_t *p4d_pgtable(p4d_t p4d)
 325 {
 326         if (pgtable_l4_enabled)
 327                 return (pud_t *)pfn_to_virt(__page_val_to_pfn(p4d_val(p4d)));
 328
 329         return (pud_t *)pud_pgtable((pud_t) { p4d_val(p4d) });
 330 }
 331 #define p4d_page_vaddr(p4d)     ((unsigned long)p4d_pgtable(p4d))
 332
 333 static inline struct page *p4d_page(p4d_t p4d)
 334 {
 335         return pfn_to_page(__page_val_to_pfn(p4d_val(p4d)));
 336 }
 337
 338 #define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
 339
 340 #define pud_offset pud_offset
 341 pud_t *pud_offset(p4d_t *p4d, unsigned long address);
 342
 343 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
 344 {
 345         if (pgtable_l5_enabled)
 346                 WRITE_ONCE(*pgdp, pgd);
 347         else
 348                 set_p4d((p4d_t *)pgdp, (p4d_t){ pgd_val(pgd) });
 349 }
 350
 351 static inline int pgd_none(pgd_t pgd)
 352 {
 353         if (pgtable_l5_enabled)
 354                 return (pgd_val(pgd) == 0);
 355
 356         return 0;
 357 }
 358
 359 static inline int pgd_present(pgd_t pgd)
 360 {
 361         if (pgtable_l5_enabled)
 362                 return (pgd_val(pgd) & _PAGE_PRESENT);
 363
 364         return 1;
 365 }
 366
 367 static inline int pgd_bad(pgd_t pgd)
 368 {
 369         if (pgtable_l5_enabled)
 370                 return !pgd_present(pgd);
 371
 372         return 0;
 373 }
 374
 375 static inline void pgd_clear(pgd_t *pgd)
 376 {
 377         if (pgtable_l5_enabled)
 378                 set_pgd(pgd, __pgd(0));
 379 }
 380
 381 static inline p4d_t *pgd_pgtable(pgd_t pgd)
 382 {
 383         if (pgtable_l5_enabled)
 384                 return (p4d_t *)pfn_to_virt(__page_val_to_pfn(pgd_val(pgd)));
 385
 386         return (p4d_t *)p4d_pgtable((p4d_t) { pgd_val(pgd) });
 387 }
 388 #define pgd_page_vaddr(pgd)     ((unsigned long)pgd_pgtable(pgd))
 389
 390 static inline struct page *pgd_page(pgd_t pgd)
 391 {
 392         return pfn_to_page(__page_val_to_pfn(pgd_val(pgd)));
 393 }
 394 #define pgd_page(pgd)   pgd_page(pgd)
 395
 396 #define p4d_index(addr) (((addr) >> P4D_SHIFT) & (PTRS_PER_P4D - 1))
 397
 398 #define p4d_offset p4d_offset
 399 p4d_t *p4d_offset(pgd_t *pgd, unsigned long address);
 400
 401 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 402 static inline int pte_devmap(pte_t pte);
 403 static inline pte_t pmd_pte(pmd_t pmd);
 404
 405 static inline int pmd_devmap(pmd_t pmd)
 406 {
 407         return pte_devmap(pmd_pte(pmd));
 408 }
 409
 410 static inline int pud_devmap(pud_t pud)
 411 {
 412         return 0;
 413 }
 414
 415 static inline int pgd_devmap(pgd_t pgd)
 416 {
 417         return 0;
 418 }
 419 #endif
 420
 421 #endif /* _ASM_RISCV_PGTABLE_64_H */