arch/riscv/kernel/sys_hwprobe.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * The hwprobe interface, for allowing userspace to probe to see which features
   4  * are supported by the hardware.  See Documentation/arch/riscv/hwprobe.rst for
   5  * more details.
   6  */
   7 #include <linux/syscalls.h>
   8 #include <asm/cacheflush.h>
   9 #include <asm/cpufeature.h>
  10 #include <asm/hwprobe.h>
  11 #include <asm/processor.h>
  12 #include <asm/delay.h>
  13 #include <asm/sbi.h>
  14 #include <asm/switch_to.h>
  15 #include <asm/uaccess.h>
  16 #include <asm/unistd.h>
  17 #include <asm/vector.h>
  18 #include <vdso/vsyscall.h>
  19
  20
  21 static void hwprobe_arch_id(struct riscv_hwprobe *pair,
  22                             const struct cpumask *cpus)
  23 {
  24         u64 id = -1ULL;
  25         bool first = true;
  26         int cpu;
  27
  28         for_each_cpu(cpu, cpus) {
  29                 u64 cpu_id;
  30
  31                 switch (pair->key) {
  32                 case RISCV_HWPROBE_KEY_MVENDORID:
  33                         cpu_id = riscv_cached_mvendorid(cpu);
  34                         break;
  35                 case RISCV_HWPROBE_KEY_MIMPID:
  36                         cpu_id = riscv_cached_mimpid(cpu);
  37                         break;
  38                 case RISCV_HWPROBE_KEY_MARCHID:
  39                         cpu_id = riscv_cached_marchid(cpu);
  40                         break;
  41                 }
  42
  43                 if (first) {
  44                         id = cpu_id;
  45                         first = false;
  46                 }
  47
  48                 /*
  49                  * If there's a mismatch for the given set, return -1 in the
  50                  * value.
  51                  */
  52                 if (id != cpu_id) {
  53                         id = -1ULL;
  54                         break;
  55                 }
  56         }
  57
  58         pair->value = id;
  59 }
  60
  61 static void hwprobe_isa_ext0(struct riscv_hwprobe *pair,
  62                              const struct cpumask *cpus)
  63 {
  64         int cpu;
  65         u64 missing = 0;
  66
  67         pair->value = 0;
  68         if (has_fpu())
  69                 pair->value |= RISCV_HWPROBE_IMA_FD;
  70
  71         if (riscv_isa_extension_available(NULL, c))
  72                 pair->value |= RISCV_HWPROBE_IMA_C;
  73
  74         if (has_vector() && riscv_isa_extension_available(NULL, v))
  75                 pair->value |= RISCV_HWPROBE_IMA_V;
  76
  77         /*
  78          * Loop through and record extensions that 1) anyone has, and 2) anyone
  79          * doesn't have.
  80          */
  81         for_each_cpu(cpu, cpus) {
  82                 struct riscv_isainfo *isainfo = &hart_isa[cpu];
  83
  84 #define EXT_KEY(ext)                                                                    \
  85         do {                                                                            \
  86                 if (__riscv_isa_extension_available(isainfo->isa, RISCV_ISA_EXT_##ext)) \
  87                         pair->value |= RISCV_HWPROBE_EXT_##ext;                         \
  88                 else                                                                    \
  89                         missing |= RISCV_HWPROBE_EXT_##ext;                             \
  90         } while (false)
  91
  92                 /*
  93                  * Only use EXT_KEY() for extensions which can be exposed to userspace,
  94                  * regardless of the kernel's configuration, as no other checks, besides
  95                  * presence in the hart_isa bitmap, are made.
  96                  */
  97                 EXT_KEY(ZACAS);
  98                 EXT_KEY(ZAWRS);
  99                 EXT_KEY(ZBA);
 100                 EXT_KEY(ZBB);
 101                 EXT_KEY(ZBC);
 102                 EXT_KEY(ZBKB);
 103                 EXT_KEY(ZBKC);
 104                 EXT_KEY(ZBKX);
 105                 EXT_KEY(ZBS);
 106                 EXT_KEY(ZCA);
 107                 EXT_KEY(ZCB);
 108                 EXT_KEY(ZCMOP);
 109                 EXT_KEY(ZICBOZ);
 110                 EXT_KEY(ZICOND);
 111                 EXT_KEY(ZIHINTNTL);
 112                 EXT_KEY(ZIHINTPAUSE);
 113                 EXT_KEY(ZIMOP);
 114                 EXT_KEY(ZKND);
 115                 EXT_KEY(ZKNE);
 116                 EXT_KEY(ZKNH);
 117                 EXT_KEY(ZKSED);
 118                 EXT_KEY(ZKSH);
 119                 EXT_KEY(ZKT);
 120                 EXT_KEY(ZTSO);
 121
 122                 /*
 123                  * All the following extensions must depend on the kernel
 124                  * support of V.
 125                  */
 126                 if (has_vector()) {
 127                         EXT_KEY(ZVBB);
 128                         EXT_KEY(ZVBC);
 129                         EXT_KEY(ZVE32F);
 130                         EXT_KEY(ZVE32X);
 131                         EXT_KEY(ZVE64D);
 132                         EXT_KEY(ZVE64F);
 133                         EXT_KEY(ZVE64X);
 134                         EXT_KEY(ZVFH);
 135                         EXT_KEY(ZVFHMIN);
 136                         EXT_KEY(ZVKB);
 137                         EXT_KEY(ZVKG);
 138                         EXT_KEY(ZVKNED);
 139                         EXT_KEY(ZVKNHA);
 140                         EXT_KEY(ZVKNHB);
 141                         EXT_KEY(ZVKSED);
 142                         EXT_KEY(ZVKSH);
 143                         EXT_KEY(ZVKT);
 144                 }
 145
 146                 if (has_fpu()) {
 147                         EXT_KEY(ZCD);
 148                         EXT_KEY(ZCF);
 149                         EXT_KEY(ZFA);
 150                         EXT_KEY(ZFH);
 151                         EXT_KEY(ZFHMIN);
 152                 }
 153
 154                 if (IS_ENABLED(CONFIG_RISCV_ISA_SUPM))
 155                         EXT_KEY(SUPM);
 156 #undef EXT_KEY
 157         }
 158
 159         /* Now turn off reporting features if any CPU is missing it. */
 160         pair->value &= ~missing;
 161 }
 162
 163 static bool hwprobe_ext0_has(const struct cpumask *cpus, unsigned long ext)
 164 {
 165         struct riscv_hwprobe pair;
 166
 167         hwprobe_isa_ext0(&pair, cpus);
 168         return (pair.value & ext);
 169 }
 170
 171 #if defined(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS)
 172 static u64 hwprobe_misaligned(const struct cpumask *cpus)
 173 {
 174         int cpu;
 175         u64 perf = -1ULL;
 176
 177         for_each_cpu(cpu, cpus) {
 178                 int this_perf = per_cpu(misaligned_access_speed, cpu);
 179
 180                 if (perf == -1ULL)
 181                         perf = this_perf;
 182
 183                 if (perf != this_perf) {
 184                         perf = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
 185                         break;
 186                 }
 187         }
 188
 189         if (perf == -1ULL)
 190                 return RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
 191
 192         return perf;
 193 }
 194 #else
 195 static u64 hwprobe_misaligned(const struct cpumask *cpus)
 196 {
 197         if (IS_ENABLED(CONFIG_RISCV_EFFICIENT_UNALIGNED_ACCESS))
 198                 return RISCV_HWPROBE_MISALIGNED_SCALAR_FAST;
 199
 200         if (IS_ENABLED(CONFIG_RISCV_EMULATED_UNALIGNED_ACCESS) && unaligned_ctl_available())
 201                 return RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED;
 202
 203         return RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW;
 204 }
 205 #endif
 206
 207 #ifdef CONFIG_RISCV_VECTOR_MISALIGNED
 208 static u64 hwprobe_vec_misaligned(const struct cpumask *cpus)
 209 {
 210         int cpu;
 211         u64 perf = -1ULL;
 212
 213         /* Return if supported or not even if speed wasn't probed */
 214         for_each_cpu(cpu, cpus) {
 215                 int this_perf = per_cpu(vector_misaligned_access, cpu);
 216
 217                 if (perf == -1ULL)
 218                         perf = this_perf;
 219
 220                 if (perf != this_perf) {
 221                         perf = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
 222                         break;
 223                 }
 224         }
 225
 226         if (perf == -1ULL)
 227                 return RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
 228
 229         return perf;
 230 }
 231 #else
 232 static u64 hwprobe_vec_misaligned(const struct cpumask *cpus)
 233 {
 234         if (IS_ENABLED(CONFIG_RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS))
 235                 return RISCV_HWPROBE_MISALIGNED_VECTOR_FAST;
 236
 237         if (IS_ENABLED(CONFIG_RISCV_SLOW_VECTOR_UNALIGNED_ACCESS))
 238                 return RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW;
 239
 240         return RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
 241 }
 242 #endif
 243
 244 static void hwprobe_one_pair(struct riscv_hwprobe *pair,
 245                              const struct cpumask *cpus)
 246 {
 247         switch (pair->key) {
 248         case RISCV_HWPROBE_KEY_MVENDORID:
 249         case RISCV_HWPROBE_KEY_MARCHID:
 250         case RISCV_HWPROBE_KEY_MIMPID:
 251                 hwprobe_arch_id(pair, cpus);
 252                 break;
 253         /*
 254          * The kernel already assumes that the base single-letter ISA
 255          * extensions are supported on all harts, and only supports the
 256          * IMA base, so just cheat a bit here and tell that to
 257          * userspace.
 258          */
 259         case RISCV_HWPROBE_KEY_BASE_BEHAVIOR:
 260                 pair->value = RISCV_HWPROBE_BASE_BEHAVIOR_IMA;
 261                 break;
 262
 263         case RISCV_HWPROBE_KEY_IMA_EXT_0:
 264                 hwprobe_isa_ext0(pair, cpus);
 265                 break;
 266
 267         case RISCV_HWPROBE_KEY_CPUPERF_0:
 268         case RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF:
 269                 pair->value = hwprobe_misaligned(cpus);
 270                 break;
 271
 272         case RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF:
 273                 pair->value = hwprobe_vec_misaligned(cpus);
 274                 break;
 275
 276         case RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE:
 277                 pair->value = 0;
 278                 if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ))
 279                         pair->value = riscv_cboz_block_size;
 280                 break;
 281         case RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS:
 282                 pair->value = user_max_virt_addr();
 283                 break;
 284
 285         case RISCV_HWPROBE_KEY_TIME_CSR_FREQ:
 286                 pair->value = riscv_timebase;
 287                 break;
 288
 289         /*
 290          * For forward compatibility, unknown keys don't fail the whole
 291          * call, but get their element key set to -1 and value set to 0
 292          * indicating they're unrecognized.
 293          */
 294         default:
 295                 pair->key = -1;
 296                 pair->value = 0;
 297                 break;
 298         }
 299 }
 300
 301 static int hwprobe_get_values(struct riscv_hwprobe __user *pairs,
 302                               size_t pair_count, size_t cpusetsize,
 303                               unsigned long __user *cpus_user,
 304                               unsigned int flags)
 305 {
 306         size_t out;
 307         int ret;
 308         cpumask_t cpus;
 309
 310         /* Check the reserved flags. */
 311         if (flags != 0)
 312                 return -EINVAL;
 313
 314         /*
 315          * The interface supports taking in a CPU mask, and returns values that
 316          * are consistent across that mask. Allow userspace to specify NULL and
 317          * 0 as a shortcut to all online CPUs.
 318          */
 319         cpumask_clear(&cpus);
 320         if (!cpusetsize && !cpus_user) {
 321                 cpumask_copy(&cpus, cpu_online_mask);
 322         } else {
 323                 if (cpusetsize > cpumask_size())
 324                         cpusetsize = cpumask_size();
 325
 326                 ret = copy_from_user(&cpus, cpus_user, cpusetsize);
 327                 if (ret)
 328                         return -EFAULT;
 329
 330                 /*
 331                  * Userspace must provide at least one online CPU, without that
 332                  * there's no way to define what is supported.
 333                  */
 334                 cpumask_and(&cpus, &cpus, cpu_online_mask);
 335                 if (cpumask_empty(&cpus))
 336                         return -EINVAL;
 337         }
 338
 339         for (out = 0; out < pair_count; out++, pairs++) {
 340                 struct riscv_hwprobe pair;
 341
 342                 if (get_user(pair.key, &pairs->key))
 343                         return -EFAULT;
 344
 345                 pair.value = 0;
 346                 hwprobe_one_pair(&pair, &cpus);
 347                 ret = put_user(pair.key, &pairs->key);
 348                 if (ret == 0)
 349                         ret = put_user(pair.value, &pairs->value);
 350
 351                 if (ret)
 352                         return -EFAULT;
 353         }
 354
 355         return 0;
 356 }
 357
 358 static int hwprobe_get_cpus(struct riscv_hwprobe __user *pairs,
 359                             size_t pair_count, size_t cpusetsize,
 360                             unsigned long __user *cpus_user,
 361                             unsigned int flags)
 362 {
 363         cpumask_t cpus, one_cpu;
 364         bool clear_all = false;
 365         size_t i;
 366         int ret;
 367
 368         if (flags != RISCV_HWPROBE_WHICH_CPUS)
 369                 return -EINVAL;
 370
 371         if (!cpusetsize || !cpus_user)
 372                 return -EINVAL;
 373
 374         if (cpusetsize > cpumask_size())
 375                 cpusetsize = cpumask_size();
 376
 377         ret = copy_from_user(&cpus, cpus_user, cpusetsize);
 378         if (ret)
 379                 return -EFAULT;
 380
 381         if (cpumask_empty(&cpus))
 382                 cpumask_copy(&cpus, cpu_online_mask);
 383
 384         cpumask_and(&cpus, &cpus, cpu_online_mask);
 385
 386         cpumask_clear(&one_cpu);
 387
 388         for (i = 0; i < pair_count; i++) {
 389                 struct riscv_hwprobe pair, tmp;
 390                 int cpu;
 391
 392                 ret = copy_from_user(&pair, &pairs[i], sizeof(pair));
 393                 if (ret)
 394                         return -EFAULT;
 395
 396                 if (!riscv_hwprobe_key_is_valid(pair.key)) {
 397                         clear_all = true;
 398                         pair = (struct riscv_hwprobe){ .key = -1, };
 399                         ret = copy_to_user(&pairs[i], &pair, sizeof(pair));
 400                         if (ret)
 401                                 return -EFAULT;
 402                 }
 403
 404                 if (clear_all)
 405                         continue;
 406
 407                 tmp = (struct riscv_hwprobe){ .key = pair.key, };
 408
 409                 for_each_cpu(cpu, &cpus) {
 410                         cpumask_set_cpu(cpu, &one_cpu);
 411
 412                         hwprobe_one_pair(&tmp, &one_cpu);
 413
 414                         if (!riscv_hwprobe_pair_cmp(&tmp, &pair))
 415                                 cpumask_clear_cpu(cpu, &cpus);
 416
 417                         cpumask_clear_cpu(cpu, &one_cpu);
 418                 }
 419         }
 420
 421         if (clear_all)
 422                 cpumask_clear(&cpus);
 423
 424         ret = copy_to_user(cpus_user, &cpus, cpusetsize);
 425         if (ret)
 426                 return -EFAULT;
 427
 428         return 0;
 429 }
 430
 431 static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs,
 432                             size_t pair_count, size_t cpusetsize,
 433                             unsigned long __user *cpus_user,
 434                             unsigned int flags)
 435 {
 436         if (flags & RISCV_HWPROBE_WHICH_CPUS)
 437                 return hwprobe_get_cpus(pairs, pair_count, cpusetsize,
 438                                         cpus_user, flags);
 439
 440         return hwprobe_get_values(pairs, pair_count, cpusetsize,
 441                                   cpus_user, flags);
 442 }
 443
 444 #ifdef CONFIG_MMU
 445
 446 static int __init init_hwprobe_vdso_data(void)
 447 {
 448         struct vdso_data *vd = __arch_get_k_vdso_data();
 449         struct arch_vdso_time_data *avd = &vd->arch_data;
 450         u64 id_bitsmash = 0;
 451         struct riscv_hwprobe pair;
 452         int key;
 453
 454         /*
 455          * Initialize vDSO data with the answers for the "all CPUs" case, to
 456          * save a syscall in the common case.
 457          */
 458         for (key = 0; key <= RISCV_HWPROBE_MAX_KEY; key++) {
 459                 pair.key = key;
 460                 hwprobe_one_pair(&pair, cpu_online_mask);
 461
 462                 WARN_ON_ONCE(pair.key < 0);
 463
 464                 avd->all_cpu_hwprobe_values[key] = pair.value;
 465                 /*
 466                  * Smash together the vendor, arch, and impl IDs to see if
 467                  * they're all 0 or any negative.
 468                  */
 469                 if (key <= RISCV_HWPROBE_KEY_MIMPID)
 470                         id_bitsmash |= pair.value;
 471         }
 472
 473         /*
 474          * If the arch, vendor, and implementation ID are all the same across
 475          * all harts, then assume all CPUs are the same, and allow the vDSO to
 476          * answer queries for arbitrary masks. However if all values are 0 (not
 477          * populated) or any value returns -1 (varies across CPUs), then the
 478          * vDSO should defer to the kernel for exotic cpu masks.
 479          */
 480         avd->homogeneous_cpus = id_bitsmash != 0 && id_bitsmash != -1;
 481         return 0;
 482 }
 483
 484 arch_initcall_sync(init_hwprobe_vdso_data);
 485
 486 #endif /* CONFIG_MMU */
 487
 488 SYSCALL_DEFINE5(riscv_hwprobe, struct riscv_hwprobe __user *, pairs,
 489                 size_t, pair_count, size_t, cpusetsize, unsigned long __user *,
 490                 cpus, unsigned int, flags)
 491 {
 492         return do_riscv_hwprobe(pairs, pair_count, cpusetsize,
 493                                 cpus, flags);
 494 }