arch/powerpc/mm/pkeys.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * PowerPC Memory Protection Keys management
   4  *
   5  * Copyright 2017, Ram Pai, IBM Corporation.
   6  */
   7
   8 #include <asm/mman.h>
   9 #include <asm/setup.h>
  10 #include <linux/pkeys.h>
  11 #include <linux/of_device.h>
  12
  13 DEFINE_STATIC_KEY_TRUE(pkey_disabled);
  14 bool pkey_execute_disable_supported;
  15 int  pkeys_total;               /* Total pkeys as per device tree */
  16 bool pkeys_devtree_defined;     /* pkey property exported by device tree */
  17 u32  initial_allocation_mask;   /* Bits set for reserved keys */
  18 u64  pkey_amr_uamor_mask;       /* Bits in AMR/UMOR not to be touched */
  19 u64  pkey_iamr_mask;            /* Bits in AMR not to be touched */
  20
  21 #define AMR_BITS_PER_PKEY 2
  22 #define AMR_RD_BIT 0x1UL
  23 #define AMR_WR_BIT 0x2UL
  24 #define IAMR_EX_BIT 0x1UL
  25 #define PKEY_REG_BITS (sizeof(u64)*8)
  26 #define pkeyshift(pkey) (PKEY_REG_BITS - ((pkey+1) * AMR_BITS_PER_PKEY))
  27
  28 static void scan_pkey_feature(void)
  29 {
  30         u32 vals[2];
  31         struct device_node *cpu;
  32
  33         cpu = of_find_node_by_type(NULL, "cpu");
  34         if (!cpu)
  35                 return;
  36
  37         if (of_property_read_u32_array(cpu,
  38                         "ibm,processor-storage-keys", vals, 2))
  39                 return;
  40
  41         /*
  42          * Since any pkey can be used for data or execute, we will just treat
  43          * all keys as equal and track them as one entity.
  44          */
  45         pkeys_total = be32_to_cpu(vals[0]);
  46         pkeys_devtree_defined = true;
  47 }
  48
  49 static inline bool pkey_mmu_enabled(void)
  50 {
  51         if (firmware_has_feature(FW_FEATURE_LPAR))
  52                 return pkeys_total;
  53         else
  54                 return cpu_has_feature(CPU_FTR_PKEY);
  55 }
  56
  57 int pkey_initialize(void)
  58 {
  59         int os_reserved, i;
  60
  61         /*
  62          * We define PKEY_DISABLE_EXECUTE in addition to the arch-neutral
  63          * generic defines for PKEY_DISABLE_ACCESS and PKEY_DISABLE_WRITE.
  64          * Ensure that the bits a distinct.
  65          */
  66         BUILD_BUG_ON(PKEY_DISABLE_EXECUTE &
  67                      (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE));
  68
  69         /*
  70          * pkey_to_vmflag_bits() assumes that the pkey bits are contiguous
  71          * in the vmaflag. Make sure that is really the case.
  72          */
  73         BUILD_BUG_ON(__builtin_clzl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) +
  74                      __builtin_popcountl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT)
  75                                 != (sizeof(u64) * BITS_PER_BYTE));
  76
  77         /* scan the device tree for pkey feature */
  78         scan_pkey_feature();
  79
  80         /*
  81          * Let's assume 32 pkeys on P8 bare metal, if its not defined by device
  82          * tree. We make this exception since skiboot forgot to expose this
  83          * property on power8.
  84          */
  85         if (!pkeys_devtree_defined && !firmware_has_feature(FW_FEATURE_LPAR) &&
  86                         cpu_has_feature(CPU_FTRS_POWER8))
  87                 pkeys_total = 32;
  88
  89         /*
  90          * Adjust the upper limit, based on the number of bits supported by
  91          * arch-neutral code.
  92          */
  93         pkeys_total = min_t(int, pkeys_total,
  94                         (ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT));
  95
  96         if (!pkey_mmu_enabled() || radix_enabled() || !pkeys_total)
  97                 static_branch_enable(&pkey_disabled);
  98         else
  99                 static_branch_disable(&pkey_disabled);
 100
 101         if (static_branch_likely(&pkey_disabled))
 102                 return 0;
 103
 104         /*
 105          * The device tree cannot be relied to indicate support for
 106          * execute_disable support. Instead we use a PVR check.
 107          */
 108         if (pvr_version_is(PVR_POWER7) || pvr_version_is(PVR_POWER7p))
 109                 pkey_execute_disable_supported = false;
 110         else
 111                 pkey_execute_disable_supported = true;
 112
 113 #ifdef CONFIG_PPC_4K_PAGES
 114         /*
 115          * The OS can manage only 8 pkeys due to its inability to represent them
 116          * in the Linux 4K PTE.
 117          */
 118         os_reserved = pkeys_total - 8;
 119 #else
 120         os_reserved = 0;
 121 #endif
 122         /*
 123          * Bits are in LE format. NOTE: 1, 0 are reserved.
 124          * key 0 is the default key, which allows read/write/execute.
 125          * key 1 is recommended not to be used. PowerISA(3.0) page 1015,
 126          * programming note.
 127          */
 128         initial_allocation_mask = ~0x0;
 129
 130         /* register mask is in BE format */
 131         pkey_amr_uamor_mask = ~0x0ul;
 132         pkey_iamr_mask = ~0x0ul;
 133
 134         for (i = 2; i < (pkeys_total - os_reserved); i++) {
 135                 initial_allocation_mask &= ~(0x1 << i);
 136                 pkey_amr_uamor_mask &= ~(0x3ul << pkeyshift(i));
 137                 pkey_iamr_mask &= ~(0x1ul << pkeyshift(i));
 138         }
 139         return 0;
 140 }
 141
 142 arch_initcall(pkey_initialize);
 143
 144 void pkey_mm_init(struct mm_struct *mm)
 145 {
 146         if (static_branch_likely(&pkey_disabled))
 147                 return;
 148         mm_pkey_allocation_map(mm) = initial_allocation_mask;
 149         /* -1 means unallocated or invalid */
 150         mm->context.execute_only_pkey = -1;
 151 }
 152
 153 static inline u64 read_amr(void)
 154 {
 155         return mfspr(SPRN_AMR);
 156 }
 157
 158 static inline void write_amr(u64 value)
 159 {
 160         mtspr(SPRN_AMR, value);
 161 }
 162
 163 static inline u64 read_iamr(void)
 164 {
 165         if (!likely(pkey_execute_disable_supported))
 166                 return 0x0UL;
 167
 168         return mfspr(SPRN_IAMR);
 169 }
 170
 171 static inline void write_iamr(u64 value)
 172 {
 173         if (!likely(pkey_execute_disable_supported))
 174                 return;
 175
 176         mtspr(SPRN_IAMR, value);
 177 }
 178
 179 static inline u64 read_uamor(void)
 180 {
 181         return mfspr(SPRN_UAMOR);
 182 }
 183
 184 static inline void write_uamor(u64 value)
 185 {
 186         mtspr(SPRN_UAMOR, value);
 187 }
 188
 189 static bool is_pkey_enabled(int pkey)
 190 {
 191         u64 uamor = read_uamor();
 192         u64 pkey_bits = 0x3ul << pkeyshift(pkey);
 193         u64 uamor_pkey_bits = (uamor & pkey_bits);
 194
 195         /*
 196          * Both the bits in UAMOR corresponding to the key should be set or
 197          * reset.
 198          */
 199         WARN_ON(uamor_pkey_bits && (uamor_pkey_bits != pkey_bits));
 200         return !!(uamor_pkey_bits);
 201 }
 202
 203 static inline void init_amr(int pkey, u8 init_bits)
 204 {
 205         u64 new_amr_bits = (((u64)init_bits & 0x3UL) << pkeyshift(pkey));
 206         u64 old_amr = read_amr() & ~((u64)(0x3ul) << pkeyshift(pkey));
 207
 208         write_amr(old_amr | new_amr_bits);
 209 }
 210
 211 static inline void init_iamr(int pkey, u8 init_bits)
 212 {
 213         u64 new_iamr_bits = (((u64)init_bits & 0x1UL) << pkeyshift(pkey));
 214         u64 old_iamr = read_iamr() & ~((u64)(0x1ul) << pkeyshift(pkey));
 215
 216         write_iamr(old_iamr | new_iamr_bits);
 217 }
 218
 219 static void pkey_status_change(int pkey, bool enable)
 220 {
 221         u64 old_uamor;
 222
 223         /* Reset the AMR and IAMR bits for this key */
 224         init_amr(pkey, 0x0);
 225         init_iamr(pkey, 0x0);
 226
 227         /* Enable/disable key */
 228         old_uamor = read_uamor();
 229         if (enable)
 230                 old_uamor |= (0x3ul << pkeyshift(pkey));
 231         else
 232                 old_uamor &= ~(0x3ul << pkeyshift(pkey));
 233         write_uamor(old_uamor);
 234 }
 235
 236 void __arch_activate_pkey(int pkey)
 237 {
 238         pkey_status_change(pkey, true);
 239 }
 240
 241 void __arch_deactivate_pkey(int pkey)
 242 {
 243         pkey_status_change(pkey, false);
 244 }
 245
 246 /*
 247  * Set the access rights in AMR IAMR and UAMOR registers for @pkey to that
 248  * specified in @init_val.
 249  */
 250 int __arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
 251                                 unsigned long init_val)
 252 {
 253         u64 new_amr_bits = 0x0ul;
 254         u64 new_iamr_bits = 0x0ul;
 255
 256         if (!is_pkey_enabled(pkey))
 257                 return -EINVAL;
 258
 259         if (init_val & PKEY_DISABLE_EXECUTE) {
 260                 if (!pkey_execute_disable_supported)
 261                         return -EINVAL;
 262                 new_iamr_bits |= IAMR_EX_BIT;
 263         }
 264         init_iamr(pkey, new_iamr_bits);
 265
 266         /* Set the bits we need in AMR: */
 267         if (init_val & PKEY_DISABLE_ACCESS)
 268                 new_amr_bits |= AMR_RD_BIT | AMR_WR_BIT;
 269         else if (init_val & PKEY_DISABLE_WRITE)
 270                 new_amr_bits |= AMR_WR_BIT;
 271
 272         init_amr(pkey, new_amr_bits);
 273         return 0;
 274 }
 275
 276 void thread_pkey_regs_save(struct thread_struct *thread)
 277 {
 278         if (static_branch_likely(&pkey_disabled))
 279                 return;
 280
 281         /*
 282          * TODO: Skip saving registers if @thread hasn't used any keys yet.
 283          */
 284         thread->amr = read_amr();
 285         thread->iamr = read_iamr();
 286         thread->uamor = read_uamor();
 287 }
 288
 289 void thread_pkey_regs_restore(struct thread_struct *new_thread,
 290                               struct thread_struct *old_thread)
 291 {
 292         if (static_branch_likely(&pkey_disabled))
 293                 return;
 294
 295         /*
 296          * TODO: Just set UAMOR to zero if @new_thread hasn't used any keys yet.
 297          */
 298         if (old_thread->amr != new_thread->amr)
 299                 write_amr(new_thread->amr);
 300         if (old_thread->iamr != new_thread->iamr)
 301                 write_iamr(new_thread->iamr);
 302         if (old_thread->uamor != new_thread->uamor)
 303                 write_uamor(new_thread->uamor);
 304 }
 305
 306 void thread_pkey_regs_init(struct thread_struct *thread)
 307 {
 308         if (static_branch_likely(&pkey_disabled))
 309                 return;
 310
 311         write_amr(read_amr() & pkey_amr_uamor_mask);
 312         write_iamr(read_iamr() & pkey_iamr_mask);
 313         write_uamor(read_uamor() & pkey_amr_uamor_mask);
 314 }
 315
 316 static inline bool pkey_allows_readwrite(int pkey)
 317 {
 318         int pkey_shift = pkeyshift(pkey);
 319
 320         if (!is_pkey_enabled(pkey))
 321                 return true;
 322
 323         return !(read_amr() & ((AMR_RD_BIT|AMR_WR_BIT) << pkey_shift));
 324 }
 325
 326 int __execute_only_pkey(struct mm_struct *mm)
 327 {
 328         bool need_to_set_mm_pkey = false;
 329         int execute_only_pkey = mm->context.execute_only_pkey;
 330         int ret;
 331
 332         /* Do we need to assign a pkey for mm's execute-only maps? */
 333         if (execute_only_pkey == -1) {
 334                 /* Go allocate one to use, which might fail */
 335                 execute_only_pkey = mm_pkey_alloc(mm);
 336                 if (execute_only_pkey < 0)
 337                         return -1;
 338                 need_to_set_mm_pkey = true;
 339         }
 340
 341         /*
 342          * We do not want to go through the relatively costly dance to set AMR
 343          * if we do not need to. Check it first and assume that if the
 344          * execute-only pkey is readwrite-disabled than we do not have to set it
 345          * ourselves.
 346          */
 347         if (!need_to_set_mm_pkey && !pkey_allows_readwrite(execute_only_pkey))
 348                 return execute_only_pkey;
 349
 350         /*
 351          * Set up AMR so that it denies access for everything other than
 352          * execution.
 353          */
 354         ret = __arch_set_user_pkey_access(current, execute_only_pkey,
 355                                           PKEY_DISABLE_ACCESS |
 356                                           PKEY_DISABLE_WRITE);
 357         /*
 358          * If the AMR-set operation failed somehow, just return 0 and
 359          * effectively disable execute-only support.
 360          */
 361         if (ret) {
 362                 mm_pkey_free(mm, execute_only_pkey);
 363                 return -1;
 364         }
 365
 366         /* We got one, store it and use it from here on out */
 367         if (need_to_set_mm_pkey)
 368                 mm->context.execute_only_pkey = execute_only_pkey;
 369         return execute_only_pkey;
 370 }
 371
 372 static inline bool vma_is_pkey_exec_only(struct vm_area_struct *vma)
 373 {
 374         /* Do this check first since the vm_flags should be hot */
 375         if ((vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC)) != VM_EXEC)
 376                 return false;
 377
 378         return (vma_pkey(vma) == vma->vm_mm->context.execute_only_pkey);
 379 }
 380
 381 /*
 382  * This should only be called for *plain* mprotect calls.
 383  */
 384 int __arch_override_mprotect_pkey(struct vm_area_struct *vma, int prot,
 385                                   int pkey)
 386 {
 387         /*
 388          * If the currently associated pkey is execute-only, but the requested
 389          * protection requires read or write, move it back to the default pkey.
 390          */
 391         if (vma_is_pkey_exec_only(vma) && (prot & (PROT_READ | PROT_WRITE)))
 392                 return 0;
 393
 394         /*
 395          * The requested protection is execute-only. Hence let's use an
 396          * execute-only pkey.
 397          */
 398         if (prot == PROT_EXEC) {
 399                 pkey = execute_only_pkey(vma->vm_mm);
 400                 if (pkey > 0)
 401                         return pkey;
 402         }
 403
 404         /* Nothing to override. */
 405         return vma_pkey(vma);
 406 }
 407
 408 static bool pkey_access_permitted(int pkey, bool write, bool execute)
 409 {
 410         int pkey_shift;
 411         u64 amr;
 412
 413         if (!pkey)
 414                 return true;
 415
 416         if (!is_pkey_enabled(pkey))
 417                 return true;
 418
 419         pkey_shift = pkeyshift(pkey);
 420         if (execute && !(read_iamr() & (IAMR_EX_BIT << pkey_shift)))
 421                 return true;
 422
 423         amr = read_amr(); /* Delay reading amr until absolutely needed */
 424         return ((!write && !(amr & (AMR_RD_BIT << pkey_shift))) ||
 425                 (write &&  !(amr & (AMR_WR_BIT << pkey_shift))));
 426 }
 427
 428 bool arch_pte_access_permitted(u64 pte, bool write, bool execute)
 429 {
 430         if (static_branch_likely(&pkey_disabled))
 431                 return true;
 432
 433         return pkey_access_permitted(pte_to_pkey_bits(pte), write, execute);
 434 }
 435
 436 /*
 437  * We only want to enforce protection keys on the current thread because we
 438  * effectively have no access to AMR/IAMR for other threads or any way to tell
 439  * which AMR/IAMR in a threaded process we could use.
 440  *
 441  * So do not enforce things if the VMA is not from the current mm, or if we are
 442  * in a kernel thread.
 443  */
 444 static inline bool vma_is_foreign(struct vm_area_struct *vma)
 445 {
 446         if (!current->mm)
 447                 return true;
 448
 449         /* if it is not our ->mm, it has to be foreign */
 450         if (current->mm != vma->vm_mm)
 451                 return true;
 452
 453         return false;
 454 }
 455
 456 bool arch_vma_access_permitted(struct vm_area_struct *vma, bool write,
 457                                bool execute, bool foreign)
 458 {
 459         if (static_branch_likely(&pkey_disabled))
 460                 return true;
 461         /*
 462          * Do not enforce our key-permissions on a foreign vma.
 463          */
 464         if (foreign || vma_is_foreign(vma))
 465                 return true;
 466
 467         return pkey_access_permitted(vma_pkey(vma), write, execute);
 468 }