arch/arm64/kernel/cpu_errata.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Contains CPU specific errata definitions
   4  *
   5  * Copyright (C) 2014 ARM Ltd.
   6  */
   7
   8 #include <linux/arm-smccc.h>
   9 #include <linux/types.h>
  10 #include <linux/cpu.h>
  11 #include <asm/cpu.h>
  12 #include <asm/cputype.h>
  13 #include <asm/cpufeature.h>
  14 #include <asm/smp_plat.h>
  15
  16 static bool __maybe_unused
  17 is_affected_midr_range(const struct arm64_cpu_capabilities *entry, int scope)
  18 {
  19         const struct arm64_midr_revidr *fix;
  20         u32 midr = read_cpuid_id(), revidr;
  21
  22         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
  23         if (!is_midr_in_range(midr, &entry->midr_range))
  24                 return false;
  25
  26         midr &= MIDR_REVISION_MASK | MIDR_VARIANT_MASK;
  27         revidr = read_cpuid(REVIDR_EL1);
  28         for (fix = entry->fixed_revs; fix && fix->revidr_mask; fix++)
  29                 if (midr == fix->midr_rv && (revidr & fix->revidr_mask))
  30                         return false;
  31
  32         return true;
  33 }
  34
  35 static bool __maybe_unused
  36 is_affected_midr_range_list(const struct arm64_cpu_capabilities *entry,
  37                             int scope)
  38 {
  39         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
  40         return is_midr_in_range_list(read_cpuid_id(), entry->midr_range_list);
  41 }
  42
  43 static bool __maybe_unused
  44 is_kryo_midr(const struct arm64_cpu_capabilities *entry, int scope)
  45 {
  46         u32 model;
  47
  48         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
  49
  50         model = read_cpuid_id();
  51         model &= MIDR_IMPLEMENTOR_MASK | (0xf00 << MIDR_PARTNUM_SHIFT) |
  52                  MIDR_ARCHITECTURE_MASK;
  53
  54         return model == entry->midr_range.model;
  55 }
  56
  57 static bool
  58 has_mismatched_cache_type(const struct arm64_cpu_capabilities *entry,
  59                           int scope)
  60 {
  61         u64 mask = arm64_ftr_reg_ctrel0.strict_mask;
  62         u64 sys = arm64_ftr_reg_ctrel0.sys_val & mask;
  63         u64 ctr_raw, ctr_real;
  64
  65         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
  66
  67         /*
  68          * We want to make sure that all the CPUs in the system expose
  69          * a consistent CTR_EL0 to make sure that applications behaves
  70          * correctly with migration.
  71          *
  72          * If a CPU has CTR_EL0.IDC but does not advertise it via CTR_EL0 :
  73          *
  74          * 1) It is safe if the system doesn't support IDC, as CPU anyway
  75          *    reports IDC = 0, consistent with the rest.
  76          *
  77          * 2) If the system has IDC, it is still safe as we trap CTR_EL0
  78          *    access on this CPU via the ARM64_HAS_CACHE_IDC capability.
  79          *
  80          * So, we need to make sure either the raw CTR_EL0 or the effective
  81          * CTR_EL0 matches the system's copy to allow a secondary CPU to boot.
  82          */
  83         ctr_raw = read_cpuid_cachetype() & mask;
  84         ctr_real = read_cpuid_effective_cachetype() & mask;
  85
  86         return (ctr_real != sys) && (ctr_raw != sys);
  87 }
  88
  89 static void
  90 cpu_enable_trap_ctr_access(const struct arm64_cpu_capabilities *cap)
  91 {
  92         u64 mask = arm64_ftr_reg_ctrel0.strict_mask;
  93         bool enable_uct_trap = false;
  94
  95         /* Trap CTR_EL0 access on this CPU, only if it has a mismatch */
  96         if ((read_cpuid_cachetype() & mask) !=
  97             (arm64_ftr_reg_ctrel0.sys_val & mask))
  98                 enable_uct_trap = true;
  99
 100         /* ... or if the system is affected by an erratum */
 101         if (cap->capability == ARM64_WORKAROUND_1542419)
 102                 enable_uct_trap = true;
 103
 104         if (enable_uct_trap)
 105                 sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0);
 106 }
 107
 108 atomic_t arm64_el2_vector_last_slot = ATOMIC_INIT(-1);
 109
 110 #include <asm/mmu_context.h>
 111 #include <asm/cacheflush.h>
 112
 113 DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data);
 114
 115 #ifdef CONFIG_KVM_INDIRECT_VECTORS
 116 extern char __smccc_workaround_1_smc_start[];
 117 extern char __smccc_workaround_1_smc_end[];
 118
 119 static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start,
 120                                 const char *hyp_vecs_end)
 121 {
 122         void *dst = lm_alias(__bp_harden_hyp_vecs_start + slot * SZ_2K);
 123         int i;
 124
 125         for (i = 0; i < SZ_2K; i += 0x80)
 126                 memcpy(dst + i, hyp_vecs_start, hyp_vecs_end - hyp_vecs_start);
 127
 128         __flush_icache_range((uintptr_t)dst, (uintptr_t)dst + SZ_2K);
 129 }
 130
 131 static void install_bp_hardening_cb(bp_hardening_cb_t fn,
 132                                     const char *hyp_vecs_start,
 133                                     const char *hyp_vecs_end)
 134 {
 135         static DEFINE_RAW_SPINLOCK(bp_lock);
 136         int cpu, slot = -1;
 137
 138         /*
 139          * detect_harden_bp_fw() passes NULL for the hyp_vecs start/end if
 140          * we're a guest. Skip the hyp-vectors work.
 141          */
 142         if (!hyp_vecs_start) {
 143                 __this_cpu_write(bp_hardening_data.fn, fn);
 144                 return;
 145         }
 146
 147         raw_spin_lock(&bp_lock);
 148         for_each_possible_cpu(cpu) {
 149                 if (per_cpu(bp_hardening_data.fn, cpu) == fn) {
 150                         slot = per_cpu(bp_hardening_data.hyp_vectors_slot, cpu);
 151                         break;
 152                 }
 153         }
 154
 155         if (slot == -1) {
 156                 slot = atomic_inc_return(&arm64_el2_vector_last_slot);
 157                 BUG_ON(slot >= BP_HARDEN_EL2_SLOTS);
 158                 __copy_hyp_vect_bpi(slot, hyp_vecs_start, hyp_vecs_end);
 159         }
 160
 161         __this_cpu_write(bp_hardening_data.hyp_vectors_slot, slot);
 162         __this_cpu_write(bp_hardening_data.fn, fn);
 163         raw_spin_unlock(&bp_lock);
 164 }
 165 #else
 166 #define __smccc_workaround_1_smc_start          NULL
 167 #define __smccc_workaround_1_smc_end            NULL
 168
 169 static void install_bp_hardening_cb(bp_hardening_cb_t fn,
 170                                       const char *hyp_vecs_start,
 171                                       const char *hyp_vecs_end)
 172 {
 173         __this_cpu_write(bp_hardening_data.fn, fn);
 174 }
 175 #endif  /* CONFIG_KVM_INDIRECT_VECTORS */
 176
 177 #include <linux/arm-smccc.h>
 178
 179 static void call_smc_arch_workaround_1(void)
 180 {
 181         arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
 182 }
 183
 184 static void call_hvc_arch_workaround_1(void)
 185 {
 186         arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
 187 }
 188
 189 static void qcom_link_stack_sanitization(void)
 190 {
 191         u64 tmp;
 192
 193         asm volatile("mov       %0, x30         \n"
 194                      ".rept     16              \n"
 195                      "bl        . + 4           \n"
 196                      ".endr                     \n"
 197                      "mov       x30, %0         \n"
 198                      : "=&r" (tmp));
 199 }
 200
 201 static bool __nospectre_v2;
 202 static int __init parse_nospectre_v2(char *str)
 203 {
 204         __nospectre_v2 = true;
 205         return 0;
 206 }
 207 early_param("nospectre_v2", parse_nospectre_v2);
 208
 209 /*
 210  * -1: No workaround
 211  *  0: No workaround required
 212  *  1: Workaround installed
 213  */
 214 static int detect_harden_bp_fw(void)
 215 {
 216         bp_hardening_cb_t cb;
 217         void *smccc_start, *smccc_end;
 218         struct arm_smccc_res res;
 219         u32 midr = read_cpuid_id();
 220
 221         arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
 222                              ARM_SMCCC_ARCH_WORKAROUND_1, &res);
 223
 224         switch ((int)res.a0) {
 225         case 1:
 226                 /* Firmware says we're just fine */
 227                 return 0;
 228         case 0:
 229                 break;
 230         default:
 231                 return -1;
 232         }
 233
 234         switch (arm_smccc_1_1_get_conduit()) {
 235         case SMCCC_CONDUIT_HVC:
 236                 cb = call_hvc_arch_workaround_1;
 237                 /* This is a guest, no need to patch KVM vectors */
 238                 smccc_start = NULL;
 239                 smccc_end = NULL;
 240                 break;
 241
 242         case SMCCC_CONDUIT_SMC:
 243                 cb = call_smc_arch_workaround_1;
 244                 smccc_start = __smccc_workaround_1_smc_start;
 245                 smccc_end = __smccc_workaround_1_smc_end;
 246                 break;
 247
 248         default:
 249                 return -1;
 250         }
 251
 252         if (((midr & MIDR_CPU_MODEL_MASK) == MIDR_QCOM_FALKOR) ||
 253             ((midr & MIDR_CPU_MODEL_MASK) == MIDR_QCOM_FALKOR_V1))
 254                 cb = qcom_link_stack_sanitization;
 255
 256         if (IS_ENABLED(CONFIG_HARDEN_BRANCH_PREDICTOR))
 257                 install_bp_hardening_cb(cb, smccc_start, smccc_end);
 258
 259         return 1;
 260 }
 261
 262 DEFINE_PER_CPU_READ_MOSTLY(u64, arm64_ssbd_callback_required);
 263
 264 int ssbd_state __read_mostly = ARM64_SSBD_KERNEL;
 265 static bool __ssb_safe = true;
 266
 267 static const struct ssbd_options {
 268         const char      *str;
 269         int             state;
 270 } ssbd_options[] = {
 271         { "force-on",   ARM64_SSBD_FORCE_ENABLE, },
 272         { "force-off",  ARM64_SSBD_FORCE_DISABLE, },
 273         { "kernel",     ARM64_SSBD_KERNEL, },
 274 };
 275
 276 static int __init ssbd_cfg(char *buf)
 277 {
 278         int i;
 279
 280         if (!buf || !buf[0])
 281                 return -EINVAL;
 282
 283         for (i = 0; i < ARRAY_SIZE(ssbd_options); i++) {
 284                 int len = strlen(ssbd_options[i].str);
 285
 286                 if (strncmp(buf, ssbd_options[i].str, len))
 287                         continue;
 288
 289                 ssbd_state = ssbd_options[i].state;
 290                 return 0;
 291         }
 292
 293         return -EINVAL;
 294 }
 295 early_param("ssbd", ssbd_cfg);
 296
 297 void __init arm64_update_smccc_conduit(struct alt_instr *alt,
 298                                        __le32 *origptr, __le32 *updptr,
 299                                        int nr_inst)
 300 {
 301         u32 insn;
 302
 303         BUG_ON(nr_inst != 1);
 304
 305         switch (arm_smccc_1_1_get_conduit()) {
 306         case SMCCC_CONDUIT_HVC:
 307                 insn = aarch64_insn_get_hvc_value();
 308                 break;
 309         case SMCCC_CONDUIT_SMC:
 310                 insn = aarch64_insn_get_smc_value();
 311                 break;
 312         default:
 313                 return;
 314         }
 315
 316         *updptr = cpu_to_le32(insn);
 317 }
 318
 319 void __init arm64_enable_wa2_handling(struct alt_instr *alt,
 320                                       __le32 *origptr, __le32 *updptr,
 321                                       int nr_inst)
 322 {
 323         BUG_ON(nr_inst != 1);
 324         /*
 325          * Only allow mitigation on EL1 entry/exit and guest
 326          * ARCH_WORKAROUND_2 handling if the SSBD state allows it to
 327          * be flipped.
 328          */
 329         if (arm64_get_ssbd_state() == ARM64_SSBD_KERNEL)
 330                 *updptr = cpu_to_le32(aarch64_insn_gen_nop());
 331 }
 332
 333 void arm64_set_ssbd_mitigation(bool state)
 334 {
 335         int conduit;
 336
 337         if (!IS_ENABLED(CONFIG_ARM64_SSBD)) {
 338                 pr_info_once("SSBD disabled by kernel configuration\n");
 339                 return;
 340         }
 341
 342         if (this_cpu_has_cap(ARM64_SSBS)) {
 343                 if (state)
 344                         asm volatile(SET_PSTATE_SSBS(0));
 345                 else
 346                         asm volatile(SET_PSTATE_SSBS(1));
 347                 return;
 348         }
 349
 350         conduit = arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_WORKAROUND_2, state,
 351                                        NULL);
 352
 353         WARN_ON_ONCE(conduit == SMCCC_CONDUIT_NONE);
 354 }
 355
 356 static bool has_ssbd_mitigation(const struct arm64_cpu_capabilities *entry,
 357                                     int scope)
 358 {
 359         struct arm_smccc_res res;
 360         bool required = true;
 361         s32 val;
 362         bool this_cpu_safe = false;
 363         int conduit;
 364
 365         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
 366
 367         if (cpu_mitigations_off())
 368                 ssbd_state = ARM64_SSBD_FORCE_DISABLE;
 369
 370         /* delay setting __ssb_safe until we get a firmware response */
 371         if (is_midr_in_range_list(read_cpuid_id(), entry->midr_range_list))
 372                 this_cpu_safe = true;
 373
 374         if (this_cpu_has_cap(ARM64_SSBS)) {
 375                 if (!this_cpu_safe)
 376                         __ssb_safe = false;
 377                 required = false;
 378                 goto out_printmsg;
 379         }
 380
 381         conduit = arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
 382                                        ARM_SMCCC_ARCH_WORKAROUND_2, &res);
 383
 384         if (conduit == SMCCC_CONDUIT_NONE) {
 385                 ssbd_state = ARM64_SSBD_UNKNOWN;
 386                 if (!this_cpu_safe)
 387                         __ssb_safe = false;
 388                 return false;
 389         }
 390
 391         val = (s32)res.a0;
 392
 393         switch (val) {
 394         case SMCCC_RET_NOT_SUPPORTED:
 395                 ssbd_state = ARM64_SSBD_UNKNOWN;
 396                 if (!this_cpu_safe)
 397                         __ssb_safe = false;
 398                 return false;
 399
 400         /* machines with mixed mitigation requirements must not return this */
 401         case SMCCC_RET_NOT_REQUIRED:
 402                 pr_info_once("%s mitigation not required\n", entry->desc);
 403                 ssbd_state = ARM64_SSBD_MITIGATED;
 404                 return false;
 405
 406         case SMCCC_RET_SUCCESS:
 407                 __ssb_safe = false;
 408                 required = true;
 409                 break;
 410
 411         case 1: /* Mitigation not required on this CPU */
 412                 required = false;
 413                 break;
 414
 415         default:
 416                 WARN_ON(1);
 417                 if (!this_cpu_safe)
 418                         __ssb_safe = false;
 419                 return false;
 420         }
 421
 422         switch (ssbd_state) {
 423         case ARM64_SSBD_FORCE_DISABLE:
 424                 arm64_set_ssbd_mitigation(false);
 425                 required = false;
 426                 break;
 427
 428         case ARM64_SSBD_KERNEL:
 429                 if (required) {
 430                         __this_cpu_write(arm64_ssbd_callback_required, 1);
 431                         arm64_set_ssbd_mitigation(true);
 432                 }
 433                 break;
 434
 435         case ARM64_SSBD_FORCE_ENABLE:
 436                 arm64_set_ssbd_mitigation(true);
 437                 required = true;
 438                 break;
 439
 440         default:
 441                 WARN_ON(1);
 442                 break;
 443         }
 444
 445 out_printmsg:
 446         switch (ssbd_state) {
 447         case ARM64_SSBD_FORCE_DISABLE:
 448                 pr_info_once("%s disabled from command-line\n", entry->desc);
 449                 break;
 450
 451         case ARM64_SSBD_FORCE_ENABLE:
 452                 pr_info_once("%s forced from command-line\n", entry->desc);
 453                 break;
 454         }
 455
 456         return required;
 457 }
 458
 459 /* known invulnerable cores */
 460 static const struct midr_range arm64_ssb_cpus[] = {
 461         MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
 462         MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
 463         MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
 464         MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53),
 465         {},
 466 };
 467
 468 #ifdef CONFIG_ARM64_ERRATUM_1463225
 469 DEFINE_PER_CPU(int, __in_cortex_a76_erratum_1463225_wa);
 470
 471 static bool
 472 has_cortex_a76_erratum_1463225(const struct arm64_cpu_capabilities *entry,
 473                                int scope)
 474 {
 475         u32 midr = read_cpuid_id();
 476         /* Cortex-A76 r0p0 - r3p1 */
 477         struct midr_range range = MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 3, 1);
 478
 479         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
 480         return is_midr_in_range(midr, &range) && is_kernel_in_hyp_mode();
 481 }
 482 #endif
 483
 484 static void __maybe_unused
 485 cpu_enable_cache_maint_trap(const struct arm64_cpu_capabilities *__unused)
 486 {
 487         sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCI, 0);
 488 }
 489
 490 #define CAP_MIDR_RANGE(model, v_min, r_min, v_max, r_max)       \
 491         .matches = is_affected_midr_range,                      \
 492         .midr_range = MIDR_RANGE(model, v_min, r_min, v_max, r_max)
 493
 494 #define CAP_MIDR_ALL_VERSIONS(model)                                    \
 495         .matches = is_affected_midr_range,                              \
 496         .midr_range = MIDR_ALL_VERSIONS(model)
 497
 498 #define MIDR_FIXED(rev, revidr_mask) \
 499         .fixed_revs = (struct arm64_midr_revidr[]){{ (rev), (revidr_mask) }, {}}
 500
 501 #define ERRATA_MIDR_RANGE(model, v_min, r_min, v_max, r_max)            \
 502         .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,                         \
 503         CAP_MIDR_RANGE(model, v_min, r_min, v_max, r_max)
 504
 505 #define CAP_MIDR_RANGE_LIST(list)                               \
 506         .matches = is_affected_midr_range_list,                 \
 507         .midr_range_list = list
 508
 509 /* Errata affecting a range of revisions of  given model variant */
 510 #define ERRATA_MIDR_REV_RANGE(m, var, r_min, r_max)      \
 511         ERRATA_MIDR_RANGE(m, var, r_min, var, r_max)
 512
 513 /* Errata affecting a single variant/revision of a model */
 514 #define ERRATA_MIDR_REV(model, var, rev)        \
 515         ERRATA_MIDR_RANGE(model, var, rev, var, rev)
 516
 517 /* Errata affecting all variants/revisions of a given a model */
 518 #define ERRATA_MIDR_ALL_VERSIONS(model)                         \
 519         .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,                 \
 520         CAP_MIDR_ALL_VERSIONS(model)
 521
 522 /* Errata affecting a list of midr ranges, with same work around */
 523 #define ERRATA_MIDR_RANGE_LIST(midr_list)                       \
 524         .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,                 \
 525         CAP_MIDR_RANGE_LIST(midr_list)
 526
 527 /* Track overall mitigation state. We are only mitigated if all cores are ok */
 528 static bool __hardenbp_enab = true;
 529 static bool __spectrev2_safe = true;
 530
 531 int get_spectre_v2_workaround_state(void)
 532 {
 533         if (__spectrev2_safe)
 534                 return ARM64_BP_HARDEN_NOT_REQUIRED;
 535
 536         if (!__hardenbp_enab)
 537                 return ARM64_BP_HARDEN_UNKNOWN;
 538
 539         return ARM64_BP_HARDEN_WA_NEEDED;
 540 }
 541
 542 /*
 543  * List of CPUs that do not need any Spectre-v2 mitigation at all.
 544  */
 545 static const struct midr_range spectre_v2_safe_list[] = {
 546         MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
 547         MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
 548         MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
 549         MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53),
 550         MIDR_ALL_VERSIONS(MIDR_HISI_TSV110),
 551         MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_3XX_SILVER),
 552         MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_SILVER),
 553         { /* sentinel */ }
 554 };
 555
 556 /*
 557  * Track overall bp hardening for all heterogeneous cores in the machine.
 558  * We are only considered "safe" if all booted cores are known safe.
 559  */
 560 static bool __maybe_unused
 561 check_branch_predictor(const struct arm64_cpu_capabilities *entry, int scope)
 562 {
 563         int need_wa;
 564
 565         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
 566
 567         /* If the CPU has CSV2 set, we're safe */
 568         if (cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64PFR0_EL1),
 569                                                  ID_AA64PFR0_CSV2_SHIFT))
 570                 return false;
 571
 572         /* Alternatively, we have a list of unaffected CPUs */
 573         if (is_midr_in_range_list(read_cpuid_id(), spectre_v2_safe_list))
 574                 return false;
 575
 576         /* Fallback to firmware detection */
 577         need_wa = detect_harden_bp_fw();
 578         if (!need_wa)
 579                 return false;
 580
 581         __spectrev2_safe = false;
 582
 583         if (!IS_ENABLED(CONFIG_HARDEN_BRANCH_PREDICTOR)) {
 584                 pr_warn_once("spectrev2 mitigation disabled by kernel configuration\n");
 585                 __hardenbp_enab = false;
 586                 return false;
 587         }
 588
 589         /* forced off */
 590         if (__nospectre_v2 || cpu_mitigations_off()) {
 591                 pr_info_once("spectrev2 mitigation disabled by command line option\n");
 592                 __hardenbp_enab = false;
 593                 return false;
 594         }
 595
 596         if (need_wa < 0) {
 597                 pr_warn_once("ARM_SMCCC_ARCH_WORKAROUND_1 missing from firmware\n");
 598                 __hardenbp_enab = false;
 599         }
 600
 601         return (need_wa > 0);
 602 }
 603
 604 static const __maybe_unused struct midr_range tx2_family_cpus[] = {
 605         MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN),
 606         MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2),
 607         {},
 608 };
 609
 610 static bool __maybe_unused
 611 needs_tx2_tvm_workaround(const struct arm64_cpu_capabilities *entry,
 612                          int scope)
 613 {
 614         int i;
 615
 616         if (!is_affected_midr_range_list(entry, scope) ||
 617             !is_hyp_mode_available())
 618                 return false;
 619
 620         for_each_possible_cpu(i) {
 621                 if (MPIDR_AFFINITY_LEVEL(cpu_logical_map(i), 0) != 0)
 622                         return true;
 623         }
 624
 625         return false;
 626 }
 627
 628 static bool __maybe_unused
 629 has_neoverse_n1_erratum_1542419(const struct arm64_cpu_capabilities *entry,
 630                                 int scope)
 631 {
 632         u32 midr = read_cpuid_id();
 633         bool has_dic = read_cpuid_cachetype() & BIT(CTR_DIC_SHIFT);
 634         const struct midr_range range = MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1);
 635
 636         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
 637         return is_midr_in_range(midr, &range) && has_dic;
 638 }
 639
 640 #if defined(CONFIG_HARDEN_EL2_VECTORS) || defined(CONFIG_ARM64_ERRATUM_1319367)
 641
 642 static const struct midr_range ca57_a72[] = {
 643         MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
 644         MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
 645         {},
 646 };
 647
 648 #endif
 649
 650 #ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI
 651 static const struct arm64_cpu_capabilities arm64_repeat_tlbi_list[] = {
 652 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1009
 653         {
 654                 ERRATA_MIDR_REV(MIDR_QCOM_FALKOR_V1, 0, 0)
 655         },
 656         {
 657                 .midr_range.model = MIDR_QCOM_KRYO,
 658                 .matches = is_kryo_midr,
 659         },
 660 #endif
 661 #ifdef CONFIG_ARM64_ERRATUM_1286807
 662         {
 663                 ERRATA_MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 3, 0),
 664         },
 665 #endif
 666         {},
 667 };
 668 #endif
 669
 670 #ifdef CONFIG_CAVIUM_ERRATUM_27456
 671 const struct midr_range cavium_erratum_27456_cpus[] = {
 672         /* Cavium ThunderX, T88 pass 1.x - 2.1 */
 673         MIDR_RANGE(MIDR_THUNDERX, 0, 0, 1, 1),
 674         /* Cavium ThunderX, T81 pass 1.0 */
 675         MIDR_REV(MIDR_THUNDERX_81XX, 0, 0),
 676         {},
 677 };
 678 #endif
 679
 680 #ifdef CONFIG_CAVIUM_ERRATUM_30115
 681 static const struct midr_range cavium_erratum_30115_cpus[] = {
 682         /* Cavium ThunderX, T88 pass 1.x - 2.2 */
 683         MIDR_RANGE(MIDR_THUNDERX, 0, 0, 1, 2),
 684         /* Cavium ThunderX, T81 pass 1.0 - 1.2 */
 685         MIDR_REV_RANGE(MIDR_THUNDERX_81XX, 0, 0, 2),
 686         /* Cavium ThunderX, T83 pass 1.0 */
 687         MIDR_REV(MIDR_THUNDERX_83XX, 0, 0),
 688         {},
 689 };
 690 #endif
 691
 692 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003
 693 static const struct arm64_cpu_capabilities qcom_erratum_1003_list[] = {
 694         {
 695                 ERRATA_MIDR_REV(MIDR_QCOM_FALKOR_V1, 0, 0),
 696         },
 697         {
 698                 .midr_range.model = MIDR_QCOM_KRYO,
 699                 .matches = is_kryo_midr,
 700         },
 701         {},
 702 };
 703 #endif
 704
 705 #ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE
 706 static const struct midr_range workaround_clean_cache[] = {
 707 #if     defined(CONFIG_ARM64_ERRATUM_826319) || \
 708         defined(CONFIG_ARM64_ERRATUM_827319) || \
 709         defined(CONFIG_ARM64_ERRATUM_824069)
 710         /* Cortex-A53 r0p[012]: ARM errata 826319, 827319, 824069 */
 711         MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 2),
 712 #endif
 713 #ifdef  CONFIG_ARM64_ERRATUM_819472
 714         /* Cortex-A53 r0p[01] : ARM errata 819472 */
 715         MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 1),
 716 #endif
 717         {},
 718 };
 719 #endif
 720
 721 #ifdef CONFIG_ARM64_ERRATUM_1418040
 722 /*
 723  * - 1188873 affects r0p0 to r2p0
 724  * - 1418040 affects r0p0 to r3p1
 725  */
 726 static const struct midr_range erratum_1418040_list[] = {
 727         /* Cortex-A76 r0p0 to r3p1 */
 728         MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 3, 1),
 729         /* Neoverse-N1 r0p0 to r3p1 */
 730         MIDR_RANGE(MIDR_NEOVERSE_N1, 0, 0, 3, 1),
 731         {},
 732 };
 733 #endif
 734
 735 #ifdef CONFIG_ARM64_ERRATUM_845719
 736 static const struct midr_range erratum_845719_list[] = {
 737         /* Cortex-A53 r0p[01234] */
 738         MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 4),
 739         /* Brahma-B53 r0p[0] */
 740         MIDR_REV(MIDR_BRAHMA_B53, 0, 0),
 741         {},
 742 };
 743 #endif
 744
 745 #ifdef CONFIG_ARM64_ERRATUM_843419
 746 static const struct arm64_cpu_capabilities erratum_843419_list[] = {
 747         {
 748                 /* Cortex-A53 r0p[01234] */
 749                 .matches = is_affected_midr_range,
 750                 ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 4),
 751                 MIDR_FIXED(0x4, BIT(8)),
 752         },
 753         {
 754                 /* Brahma-B53 r0p[0] */
 755                 .matches = is_affected_midr_range,
 756                 ERRATA_MIDR_REV(MIDR_BRAHMA_B53, 0, 0),
 757         },
 758         {},
 759 };
 760 #endif
 761
 762 #ifdef CONFIG_ARM64_WORKAROUND_SPECULATIVE_AT_VHE
 763 static const struct midr_range erratum_speculative_at_vhe_list[] = {
 764 #ifdef CONFIG_ARM64_ERRATUM_1165522
 765         /* Cortex A76 r0p0 to r2p0 */
 766         MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 2, 0),
 767 #endif
 768 #ifdef CONFIG_ARM64_ERRATUM_1530923
 769         /* Cortex A55 r0p0 to r2p0 */
 770         MIDR_RANGE(MIDR_CORTEX_A55, 0, 0, 2, 0),
 771 #endif
 772         {},
 773 };
 774 #endif
 775
 776 const struct arm64_cpu_capabilities arm64_errata[] = {
 777 #ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE
 778         {
 779                 .desc = "ARM errata 826319, 827319, 824069, 819472",
 780                 .capability = ARM64_WORKAROUND_CLEAN_CACHE,
 781                 ERRATA_MIDR_RANGE_LIST(workaround_clean_cache),
 782                 .cpu_enable = cpu_enable_cache_maint_trap,
 783         },
 784 #endif
 785 #ifdef CONFIG_ARM64_ERRATUM_832075
 786         {
 787         /* Cortex-A57 r0p0 - r1p2 */
 788                 .desc = "ARM erratum 832075",
 789                 .capability = ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE,
 790                 ERRATA_MIDR_RANGE(MIDR_CORTEX_A57,
 791                                   0, 0,
 792                                   1, 2),
 793         },
 794 #endif
 795 #ifdef CONFIG_ARM64_ERRATUM_834220
 796         {
 797         /* Cortex-A57 r0p0 - r1p2 */
 798                 .desc = "ARM erratum 834220",
 799                 .capability = ARM64_WORKAROUND_834220,
 800                 ERRATA_MIDR_RANGE(MIDR_CORTEX_A57,
 801                                   0, 0,
 802                                   1, 2),
 803         },
 804 #endif
 805 #ifdef CONFIG_ARM64_ERRATUM_843419
 806         {
 807                 .desc = "ARM erratum 843419",
 808                 .capability = ARM64_WORKAROUND_843419,
 809                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
 810                 .matches = cpucap_multi_entry_cap_matches,
 811                 .match_list = erratum_843419_list,
 812         },
 813 #endif
 814 #ifdef CONFIG_ARM64_ERRATUM_845719
 815         {
 816                 .desc = "ARM erratum 845719",
 817                 .capability = ARM64_WORKAROUND_845719,
 818                 ERRATA_MIDR_RANGE_LIST(erratum_845719_list),
 819         },
 820 #endif
 821 #ifdef CONFIG_CAVIUM_ERRATUM_23154
 822         {
 823         /* Cavium ThunderX, pass 1.x */
 824                 .desc = "Cavium erratum 23154",
 825                 .capability = ARM64_WORKAROUND_CAVIUM_23154,
 826                 ERRATA_MIDR_REV_RANGE(MIDR_THUNDERX, 0, 0, 1),
 827         },
 828 #endif
 829 #ifdef CONFIG_CAVIUM_ERRATUM_27456
 830         {
 831                 .desc = "Cavium erratum 27456",
 832                 .capability = ARM64_WORKAROUND_CAVIUM_27456,
 833                 ERRATA_MIDR_RANGE_LIST(cavium_erratum_27456_cpus),
 834         },
 835 #endif
 836 #ifdef CONFIG_CAVIUM_ERRATUM_30115
 837         {
 838                 .desc = "Cavium erratum 30115",
 839                 .capability = ARM64_WORKAROUND_CAVIUM_30115,
 840                 ERRATA_MIDR_RANGE_LIST(cavium_erratum_30115_cpus),
 841         },
 842 #endif
 843         {
 844                 .desc = "Mismatched cache type (CTR_EL0)",
 845                 .capability = ARM64_MISMATCHED_CACHE_TYPE,
 846                 .matches = has_mismatched_cache_type,
 847                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
 848                 .cpu_enable = cpu_enable_trap_ctr_access,
 849         },
 850 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003
 851         {
 852                 .desc = "Qualcomm Technologies Falkor/Kryo erratum 1003",
 853                 .capability = ARM64_WORKAROUND_QCOM_FALKOR_E1003,
 854                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
 855                 .matches = cpucap_multi_entry_cap_matches,
 856                 .match_list = qcom_erratum_1003_list,
 857         },
 858 #endif
 859 #ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI
 860         {
 861                 .desc = "Qualcomm erratum 1009, ARM erratum 1286807",
 862                 .capability = ARM64_WORKAROUND_REPEAT_TLBI,
 863                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
 864                 .matches = cpucap_multi_entry_cap_matches,
 865                 .match_list = arm64_repeat_tlbi_list,
 866         },
 867 #endif
 868 #ifdef CONFIG_ARM64_ERRATUM_858921
 869         {
 870         /* Cortex-A73 all versions */
 871                 .desc = "ARM erratum 858921",
 872                 .capability = ARM64_WORKAROUND_858921,
 873                 ERRATA_MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
 874         },
 875 #endif
 876         {
 877                 .capability = ARM64_HARDEN_BRANCH_PREDICTOR,
 878                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
 879                 .matches = check_branch_predictor,
 880         },
 881 #ifdef CONFIG_HARDEN_EL2_VECTORS
 882         {
 883                 .desc = "EL2 vector hardening",
 884                 .capability = ARM64_HARDEN_EL2_VECTORS,
 885                 ERRATA_MIDR_RANGE_LIST(ca57_a72),
 886         },
 887 #endif
 888         {
 889                 .desc = "Speculative Store Bypass Disable",
 890                 .capability = ARM64_SSBD,
 891                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
 892                 .matches = has_ssbd_mitigation,
 893                 .midr_range_list = arm64_ssb_cpus,
 894         },
 895 #ifdef CONFIG_ARM64_ERRATUM_1418040
 896         {
 897                 .desc = "ARM erratum 1418040",
 898                 .capability = ARM64_WORKAROUND_1418040,
 899                 ERRATA_MIDR_RANGE_LIST(erratum_1418040_list),
 900         },
 901 #endif
 902 #ifdef CONFIG_ARM64_WORKAROUND_SPECULATIVE_AT_VHE
 903         {
 904                 .desc = "ARM errata 1165522, 1530923",
 905                 .capability = ARM64_WORKAROUND_SPECULATIVE_AT_VHE,
 906                 ERRATA_MIDR_RANGE_LIST(erratum_speculative_at_vhe_list),
 907         },
 908 #endif
 909 #ifdef CONFIG_ARM64_ERRATUM_1463225
 910         {
 911                 .desc = "ARM erratum 1463225",
 912                 .capability = ARM64_WORKAROUND_1463225,
 913                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
 914                 .matches = has_cortex_a76_erratum_1463225,
 915         },
 916 #endif
 917 #ifdef CONFIG_CAVIUM_TX2_ERRATUM_219
 918         {
 919                 .desc = "Cavium ThunderX2 erratum 219 (KVM guest sysreg trapping)",
 920                 .capability = ARM64_WORKAROUND_CAVIUM_TX2_219_TVM,
 921                 ERRATA_MIDR_RANGE_LIST(tx2_family_cpus),
 922                 .matches = needs_tx2_tvm_workaround,
 923         },
 924         {
 925                 .desc = "Cavium ThunderX2 erratum 219 (PRFM removal)",
 926                 .capability = ARM64_WORKAROUND_CAVIUM_TX2_219_PRFM,
 927                 ERRATA_MIDR_RANGE_LIST(tx2_family_cpus),
 928         },
 929 #endif
 930 #ifdef CONFIG_ARM64_ERRATUM_1542419
 931         {
 932                 /* we depend on the firmware portion for correctness */
 933                 .desc = "ARM erratum 1542419 (kernel portion)",
 934                 .capability = ARM64_WORKAROUND_1542419,
 935                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
 936                 .matches = has_neoverse_n1_erratum_1542419,
 937                 .cpu_enable = cpu_enable_trap_ctr_access,
 938         },
 939 #endif
 940 #ifdef CONFIG_ARM64_ERRATUM_1319367
 941         {
 942                 .desc = "ARM erratum 1319367",
 943                 .capability = ARM64_WORKAROUND_SPECULATIVE_AT_NVHE,
 944                 ERRATA_MIDR_RANGE_LIST(ca57_a72),
 945         },
 946 #endif
 947         {
 948         }
 949 };
 950
 951 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr,
 952                             char *buf)
 953 {
 954         return sprintf(buf, "Mitigation: __user pointer sanitization\n");
 955 }
 956
 957 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr,
 958                 char *buf)
 959 {
 960         switch (get_spectre_v2_workaround_state()) {
 961         case ARM64_BP_HARDEN_NOT_REQUIRED:
 962                 return sprintf(buf, "Not affected\n");
 963         case ARM64_BP_HARDEN_WA_NEEDED:
 964                 return sprintf(buf, "Mitigation: Branch predictor hardening\n");
 965         case ARM64_BP_HARDEN_UNKNOWN:
 966         default:
 967                 return sprintf(buf, "Vulnerable\n");
 968         }
 969 }
 970
 971 ssize_t cpu_show_spec_store_bypass(struct device *dev,
 972                 struct device_attribute *attr, char *buf)
 973 {
 974         if (__ssb_safe)
 975                 return sprintf(buf, "Not affected\n");
 976
 977         switch (ssbd_state) {
 978         case ARM64_SSBD_KERNEL:
 979         case ARM64_SSBD_FORCE_ENABLE:
 980                 if (IS_ENABLED(CONFIG_ARM64_SSBD))
 981                         return sprintf(buf,
 982                             "Mitigation: Speculative Store Bypass disabled via prctl\n");
 983         }
 984
 985         return sprintf(buf, "Vulnerable\n");
 986 }