arch/x86/kernel/cpu/bugs.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *  Copyright (C) 1994  Linus Torvalds
   4  *
   5  *  Cyrix stuff, June 1998 by:
   6  *      - Rafael R. Reilova (moved everything from head.S),
   7  *        <rreilova@ececs.uc.edu>
   8  *      - Channing Corn (tests & fixes),
   9  *      - Andrew D. Balsa (code cleanup).
  10  */
  11 #include <linux/init.h>
  12 #include <linux/utsname.h>
  13 #include <linux/cpu.h>
  14 #include <linux/module.h>
  15 #include <linux/nospec.h>
  16 #include <linux/prctl.h>
  17
  18 #include <asm/spec-ctrl.h>
  19 #include <asm/cmdline.h>
  20 #include <asm/bugs.h>
  21 #include <asm/processor.h>
  22 #include <asm/processor-flags.h>
  23 #include <asm/fpu/internal.h>
  24 #include <asm/msr.h>
  25 #include <asm/vmx.h>
  26 #include <asm/paravirt.h>
  27 #include <asm/alternative.h>
  28 #include <asm/pgtable.h>
  29 #include <asm/set_memory.h>
  30 #include <asm/intel-family.h>
  31 #include <asm/hypervisor.h>
  32 #include <asm/e820/api.h>
  33
  34 static void __init spectre_v2_select_mitigation(void);
  35 static void __init ssb_select_mitigation(void);
  36 static void __init l1tf_select_mitigation(void);
  37
  38 /*
  39  * Our boot-time value of the SPEC_CTRL MSR. We read it once so that any
  40  * writes to SPEC_CTRL contain whatever reserved bits have been set.
  41  */
  42 u64 __ro_after_init x86_spec_ctrl_base;
  43 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base);
  44
  45 /*
  46  * The vendor and possibly platform specific bits which can be modified in
  47  * x86_spec_ctrl_base.
  48  */
  49 static u64 __ro_after_init x86_spec_ctrl_mask = SPEC_CTRL_IBRS;
  50
  51 /*
  52  * AMD specific MSR info for Speculative Store Bypass control.
  53  * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu().
  54  */
  55 u64 __ro_after_init x86_amd_ls_cfg_base;
  56 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask;
  57
  58 void __init check_bugs(void)
  59 {
  60         identify_boot_cpu();
  61
  62         /*
  63          * identify_boot_cpu() initialized SMT support information, let the
  64          * core code know.
  65          */
  66         cpu_smt_check_topology_early();
  67
  68         if (!IS_ENABLED(CONFIG_SMP)) {
  69                 pr_info("CPU: ");
  70                 print_cpu_info(&boot_cpu_data);
  71         }
  72
  73         /*
  74          * Read the SPEC_CTRL MSR to account for reserved bits which may
  75          * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD
  76          * init code as it is not enumerated and depends on the family.
  77          */
  78         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
  79                 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
  80
  81         /* Allow STIBP in MSR_SPEC_CTRL if supported */
  82         if (boot_cpu_has(X86_FEATURE_STIBP))
  83                 x86_spec_ctrl_mask |= SPEC_CTRL_STIBP;
  84
  85         /* Select the proper spectre mitigation before patching alternatives */
  86         spectre_v2_select_mitigation();
  87
  88         /*
  89          * Select proper mitigation for any exposure to the Speculative Store
  90          * Bypass vulnerability.
  91          */
  92         ssb_select_mitigation();
  93
  94         l1tf_select_mitigation();
  95
  96 #ifdef CONFIG_X86_32
  97         /*
  98          * Check whether we are able to run this kernel safely on SMP.
  99          *
 100          * - i386 is no longer supported.
 101          * - In order to run on anything without a TSC, we need to be
 102          *   compiled for a i486.
 103          */
 104         if (boot_cpu_data.x86 < 4)
 105                 panic("Kernel requires i486+ for 'invlpg' and other features");
 106
 107         init_utsname()->machine[1] =
 108                 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
 109         alternative_instructions();
 110
 111         fpu__init_check_bugs();
 112 #else /* CONFIG_X86_64 */
 113         alternative_instructions();
 114
 115         /*
 116          * Make sure the first 2MB area is not mapped by huge pages
 117          * There are typically fixed size MTRRs in there and overlapping
 118          * MTRRs into large pages causes slow downs.
 119          *
 120          * Right now we don't do that with gbpages because there seems
 121          * very little benefit for that case.
 122          */
 123         if (!direct_gbpages)
 124                 set_memory_4k((unsigned long)__va(0), 1);
 125 #endif
 126 }
 127
 128 /* The kernel command line selection */
 129 enum spectre_v2_mitigation_cmd {
 130         SPECTRE_V2_CMD_NONE,
 131         SPECTRE_V2_CMD_AUTO,
 132         SPECTRE_V2_CMD_FORCE,
 133         SPECTRE_V2_CMD_RETPOLINE,
 134         SPECTRE_V2_CMD_RETPOLINE_GENERIC,
 135         SPECTRE_V2_CMD_RETPOLINE_AMD,
 136 };
 137
 138 static const char *spectre_v2_strings[] = {
 139         [SPECTRE_V2_NONE]                       = "Vulnerable",
 140         [SPECTRE_V2_RETPOLINE_MINIMAL]          = "Vulnerable: Minimal generic ASM retpoline",
 141         [SPECTRE_V2_RETPOLINE_MINIMAL_AMD]      = "Vulnerable: Minimal AMD ASM retpoline",
 142         [SPECTRE_V2_RETPOLINE_GENERIC]          = "Mitigation: Full generic retpoline",
 143         [SPECTRE_V2_RETPOLINE_AMD]              = "Mitigation: Full AMD retpoline",
 144 };
 145
 146 #undef pr_fmt
 147 #define pr_fmt(fmt)     "Spectre V2 : " fmt
 148
 149 static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init =
 150         SPECTRE_V2_NONE;
 151
 152 void
 153 x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)
 154 {
 155         u64 msrval, guestval, hostval = x86_spec_ctrl_base;
 156         struct thread_info *ti = current_thread_info();
 157
 158         /* Is MSR_SPEC_CTRL implemented ? */
 159         if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) {
 160                 /*
 161                  * Restrict guest_spec_ctrl to supported values. Clear the
 162                  * modifiable bits in the host base value and or the
 163                  * modifiable bits from the guest value.
 164                  */
 165                 guestval = hostval & ~x86_spec_ctrl_mask;
 166                 guestval |= guest_spec_ctrl & x86_spec_ctrl_mask;
 167
 168                 /* SSBD controlled in MSR_SPEC_CTRL */
 169                 if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
 170                     static_cpu_has(X86_FEATURE_AMD_SSBD))
 171                         hostval |= ssbd_tif_to_spec_ctrl(ti->flags);
 172
 173                 if (hostval != guestval) {
 174                         msrval = setguest ? guestval : hostval;
 175                         wrmsrl(MSR_IA32_SPEC_CTRL, msrval);
 176                 }
 177         }
 178
 179         /*
 180          * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
 181          * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.
 182          */
 183         if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
 184             !static_cpu_has(X86_FEATURE_VIRT_SSBD))
 185                 return;
 186
 187         /*
 188          * If the host has SSBD mitigation enabled, force it in the host's
 189          * virtual MSR value. If its not permanently enabled, evaluate
 190          * current's TIF_SSBD thread flag.
 191          */
 192         if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE))
 193                 hostval = SPEC_CTRL_SSBD;
 194         else
 195                 hostval = ssbd_tif_to_spec_ctrl(ti->flags);
 196
 197         /* Sanitize the guest value */
 198         guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD;
 199
 200         if (hostval != guestval) {
 201                 unsigned long tif;
 202
 203                 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) :
 204                                  ssbd_spec_ctrl_to_tif(hostval);
 205
 206                 speculative_store_bypass_update(tif);
 207         }
 208 }
 209 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl);
 210
 211 static void x86_amd_ssb_disable(void)
 212 {
 213         u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask;
 214
 215         if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
 216                 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD);
 217         else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD))
 218                 wrmsrl(MSR_AMD64_LS_CFG, msrval);
 219 }
 220
 221 #ifdef RETPOLINE
 222 static bool spectre_v2_bad_module;
 223
 224 bool retpoline_module_ok(bool has_retpoline)
 225 {
 226         if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline)
 227                 return true;
 228
 229         pr_err("System may be vulnerable to spectre v2\n");
 230         spectre_v2_bad_module = true;
 231         return false;
 232 }
 233
 234 static inline const char *spectre_v2_module_string(void)
 235 {
 236         return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
 237 }
 238 #else
 239 static inline const char *spectre_v2_module_string(void) { return ""; }
 240 #endif
 241
 242 static void __init spec2_print_if_insecure(const char *reason)
 243 {
 244         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
 245                 pr_info("%s selected on command line.\n", reason);
 246 }
 247
 248 static void __init spec2_print_if_secure(const char *reason)
 249 {
 250         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
 251                 pr_info("%s selected on command line.\n", reason);
 252 }
 253
 254 static inline bool retp_compiler(void)
 255 {
 256         return __is_defined(RETPOLINE);
 257 }
 258
 259 static inline bool match_option(const char *arg, int arglen, const char *opt)
 260 {
 261         int len = strlen(opt);
 262
 263         return len == arglen && !strncmp(arg, opt, len);
 264 }
 265
 266 static const struct {
 267         const char *option;
 268         enum spectre_v2_mitigation_cmd cmd;
 269         bool secure;
 270 } mitigation_options[] = {
 271         { "off",               SPECTRE_V2_CMD_NONE,              false },
 272         { "on",                SPECTRE_V2_CMD_FORCE,             true },
 273         { "retpoline",         SPECTRE_V2_CMD_RETPOLINE,         false },
 274         { "retpoline,amd",     SPECTRE_V2_CMD_RETPOLINE_AMD,     false },
 275         { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
 276         { "auto",              SPECTRE_V2_CMD_AUTO,              false },
 277 };
 278
 279 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
 280 {
 281         char arg[20];
 282         int ret, i;
 283         enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO;
 284
 285         if (cmdline_find_option_bool(boot_command_line, "nospectre_v2"))
 286                 return SPECTRE_V2_CMD_NONE;
 287         else {
 288                 ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg));
 289                 if (ret < 0)
 290                         return SPECTRE_V2_CMD_AUTO;
 291
 292                 for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) {
 293                         if (!match_option(arg, ret, mitigation_options[i].option))
 294                                 continue;
 295                         cmd = mitigation_options[i].cmd;
 296                         break;
 297                 }
 298
 299                 if (i >= ARRAY_SIZE(mitigation_options)) {
 300                         pr_err("unknown option (%s). Switching to AUTO select\n", arg);
 301                         return SPECTRE_V2_CMD_AUTO;
 302                 }
 303         }
 304
 305         if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
 306              cmd == SPECTRE_V2_CMD_RETPOLINE_AMD ||
 307              cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) &&
 308             !IS_ENABLED(CONFIG_RETPOLINE)) {
 309                 pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option);
 310                 return SPECTRE_V2_CMD_AUTO;
 311         }
 312
 313         if (cmd == SPECTRE_V2_CMD_RETPOLINE_AMD &&
 314             boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
 315                 pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n");
 316                 return SPECTRE_V2_CMD_AUTO;
 317         }
 318
 319         if (mitigation_options[i].secure)
 320                 spec2_print_if_secure(mitigation_options[i].option);
 321         else
 322                 spec2_print_if_insecure(mitigation_options[i].option);
 323
 324         return cmd;
 325 }
 326
 327 static void __init spectre_v2_select_mitigation(void)
 328 {
 329         enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
 330         enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
 331
 332         /*
 333          * If the CPU is not affected and the command line mode is NONE or AUTO
 334          * then nothing to do.
 335          */
 336         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
 337             (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
 338                 return;
 339
 340         switch (cmd) {
 341         case SPECTRE_V2_CMD_NONE:
 342                 return;
 343
 344         case SPECTRE_V2_CMD_FORCE:
 345         case SPECTRE_V2_CMD_AUTO:
 346                 if (IS_ENABLED(CONFIG_RETPOLINE))
 347                         goto retpoline_auto;
 348                 break;
 349         case SPECTRE_V2_CMD_RETPOLINE_AMD:
 350                 if (IS_ENABLED(CONFIG_RETPOLINE))
 351                         goto retpoline_amd;
 352                 break;
 353         case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
 354                 if (IS_ENABLED(CONFIG_RETPOLINE))
 355                         goto retpoline_generic;
 356                 break;
 357         case SPECTRE_V2_CMD_RETPOLINE:
 358                 if (IS_ENABLED(CONFIG_RETPOLINE))
 359                         goto retpoline_auto;
 360                 break;
 361         }
 362         pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!");
 363         return;
 364
 365 retpoline_auto:
 366         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
 367         retpoline_amd:
 368                 if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
 369                         pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n");
 370                         goto retpoline_generic;
 371                 }
 372                 mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_AMD :
 373                                          SPECTRE_V2_RETPOLINE_MINIMAL_AMD;
 374                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
 375                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 376         } else {
 377         retpoline_generic:
 378                 mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_GENERIC :
 379                                          SPECTRE_V2_RETPOLINE_MINIMAL;
 380                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 381         }
 382
 383         spectre_v2_enabled = mode;
 384         pr_info("%s\n", spectre_v2_strings[mode]);
 385
 386         /*
 387          * If spectre v2 protection has been enabled, unconditionally fill
 388          * RSB during a context switch; this protects against two independent
 389          * issues:
 390          *
 391          *      - RSB underflow (and switch to BTB) on Skylake+
 392          *      - SpectreRSB variant of spectre v2 on X86_BUG_SPECTRE_V2 CPUs
 393          */
 394         setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
 395         pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n");
 396
 397         /* Initialize Indirect Branch Prediction Barrier if supported */
 398         if (boot_cpu_has(X86_FEATURE_IBPB)) {
 399                 setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
 400                 pr_info("Spectre v2 mitigation: Enabling Indirect Branch Prediction Barrier\n");
 401         }
 402
 403         /*
 404          * Retpoline means the kernel is safe because it has no indirect
 405          * branches. But firmware isn't, so use IBRS to protect that.
 406          */
 407         if (boot_cpu_has(X86_FEATURE_IBRS)) {
 408                 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
 409                 pr_info("Enabling Restricted Speculation for firmware calls\n");
 410         }
 411 }
 412
 413 #undef pr_fmt
 414 #define pr_fmt(fmt)     "Speculative Store Bypass: " fmt
 415
 416 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE;
 417
 418 /* The kernel command line selection */
 419 enum ssb_mitigation_cmd {
 420         SPEC_STORE_BYPASS_CMD_NONE,
 421         SPEC_STORE_BYPASS_CMD_AUTO,
 422         SPEC_STORE_BYPASS_CMD_ON,
 423         SPEC_STORE_BYPASS_CMD_PRCTL,
 424         SPEC_STORE_BYPASS_CMD_SECCOMP,
 425 };
 426
 427 static const char *ssb_strings[] = {
 428         [SPEC_STORE_BYPASS_NONE]        = "Vulnerable",
 429         [SPEC_STORE_BYPASS_DISABLE]     = "Mitigation: Speculative Store Bypass disabled",
 430         [SPEC_STORE_BYPASS_PRCTL]       = "Mitigation: Speculative Store Bypass disabled via prctl",
 431         [SPEC_STORE_BYPASS_SECCOMP]     = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp",
 432 };
 433
 434 static const struct {
 435         const char *option;
 436         enum ssb_mitigation_cmd cmd;
 437 } ssb_mitigation_options[] = {
 438         { "auto",       SPEC_STORE_BYPASS_CMD_AUTO },    /* Platform decides */
 439         { "on",         SPEC_STORE_BYPASS_CMD_ON },      /* Disable Speculative Store Bypass */
 440         { "off",        SPEC_STORE_BYPASS_CMD_NONE },    /* Don't touch Speculative Store Bypass */
 441         { "prctl",      SPEC_STORE_BYPASS_CMD_PRCTL },   /* Disable Speculative Store Bypass via prctl */
 442         { "seccomp",    SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */
 443 };
 444
 445 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void)
 446 {
 447         enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO;
 448         char arg[20];
 449         int ret, i;
 450
 451         if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable")) {
 452                 return SPEC_STORE_BYPASS_CMD_NONE;
 453         } else {
 454                 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable",
 455                                           arg, sizeof(arg));
 456                 if (ret < 0)
 457                         return SPEC_STORE_BYPASS_CMD_AUTO;
 458
 459                 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) {
 460                         if (!match_option(arg, ret, ssb_mitigation_options[i].option))
 461                                 continue;
 462
 463                         cmd = ssb_mitigation_options[i].cmd;
 464                         break;
 465                 }
 466
 467                 if (i >= ARRAY_SIZE(ssb_mitigation_options)) {
 468                         pr_err("unknown option (%s). Switching to AUTO select\n", arg);
 469                         return SPEC_STORE_BYPASS_CMD_AUTO;
 470                 }
 471         }
 472
 473         return cmd;
 474 }
 475
 476 static enum ssb_mitigation __init __ssb_select_mitigation(void)
 477 {
 478         enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE;
 479         enum ssb_mitigation_cmd cmd;
 480
 481         if (!boot_cpu_has(X86_FEATURE_SSBD))
 482                 return mode;
 483
 484         cmd = ssb_parse_cmdline();
 485         if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) &&
 486             (cmd == SPEC_STORE_BYPASS_CMD_NONE ||
 487              cmd == SPEC_STORE_BYPASS_CMD_AUTO))
 488                 return mode;
 489
 490         switch (cmd) {
 491         case SPEC_STORE_BYPASS_CMD_AUTO:
 492         case SPEC_STORE_BYPASS_CMD_SECCOMP:
 493                 /*
 494                  * Choose prctl+seccomp as the default mode if seccomp is
 495                  * enabled.
 496                  */
 497                 if (IS_ENABLED(CONFIG_SECCOMP))
 498                         mode = SPEC_STORE_BYPASS_SECCOMP;
 499                 else
 500                         mode = SPEC_STORE_BYPASS_PRCTL;
 501                 break;
 502         case SPEC_STORE_BYPASS_CMD_ON:
 503                 mode = SPEC_STORE_BYPASS_DISABLE;
 504                 break;
 505         case SPEC_STORE_BYPASS_CMD_PRCTL:
 506                 mode = SPEC_STORE_BYPASS_PRCTL;
 507                 break;
 508         case SPEC_STORE_BYPASS_CMD_NONE:
 509                 break;
 510         }
 511
 512         /*
 513          * We have three CPU feature flags that are in play here:
 514          *  - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
 515          *  - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass
 516          *  - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation
 517          */
 518         if (mode == SPEC_STORE_BYPASS_DISABLE) {
 519                 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE);
 520                 /*
 521                  * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may
 522                  * use a completely different MSR and bit dependent on family.
 523                  */
 524                 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) &&
 525                     !static_cpu_has(X86_FEATURE_AMD_SSBD)) {
 526                         x86_amd_ssb_disable();
 527                 } else {
 528                         x86_spec_ctrl_base |= SPEC_CTRL_SSBD;
 529                         x86_spec_ctrl_mask |= SPEC_CTRL_SSBD;
 530                         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 531                 }
 532         }
 533
 534         return mode;
 535 }
 536
 537 static void ssb_select_mitigation(void)
 538 {
 539         ssb_mode = __ssb_select_mitigation();
 540
 541         if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
 542                 pr_info("%s\n", ssb_strings[ssb_mode]);
 543 }
 544
 545 #undef pr_fmt
 546 #define pr_fmt(fmt)     "Speculation prctl: " fmt
 547
 548 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl)
 549 {
 550         bool update;
 551
 552         if (ssb_mode != SPEC_STORE_BYPASS_PRCTL &&
 553             ssb_mode != SPEC_STORE_BYPASS_SECCOMP)
 554                 return -ENXIO;
 555
 556         switch (ctrl) {
 557         case PR_SPEC_ENABLE:
 558                 /* If speculation is force disabled, enable is not allowed */
 559                 if (task_spec_ssb_force_disable(task))
 560                         return -EPERM;
 561                 task_clear_spec_ssb_disable(task);
 562                 update = test_and_clear_tsk_thread_flag(task, TIF_SSBD);
 563                 break;
 564         case PR_SPEC_DISABLE:
 565                 task_set_spec_ssb_disable(task);
 566                 update = !test_and_set_tsk_thread_flag(task, TIF_SSBD);
 567                 break;
 568         case PR_SPEC_FORCE_DISABLE:
 569                 task_set_spec_ssb_disable(task);
 570                 task_set_spec_ssb_force_disable(task);
 571                 update = !test_and_set_tsk_thread_flag(task, TIF_SSBD);
 572                 break;
 573         default:
 574                 return -ERANGE;
 575         }
 576
 577         /*
 578          * If being set on non-current task, delay setting the CPU
 579          * mitigation until it is next scheduled.
 580          */
 581         if (task == current && update)
 582                 speculative_store_bypass_update_current();
 583
 584         return 0;
 585 }
 586
 587 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
 588                              unsigned long ctrl)
 589 {
 590         switch (which) {
 591         case PR_SPEC_STORE_BYPASS:
 592                 return ssb_prctl_set(task, ctrl);
 593         default:
 594                 return -ENODEV;
 595         }
 596 }
 597
 598 #ifdef CONFIG_SECCOMP
 599 void arch_seccomp_spec_mitigate(struct task_struct *task)
 600 {
 601         if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP)
 602                 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE);
 603 }
 604 #endif
 605
 606 static int ssb_prctl_get(struct task_struct *task)
 607 {
 608         switch (ssb_mode) {
 609         case SPEC_STORE_BYPASS_DISABLE:
 610                 return PR_SPEC_DISABLE;
 611         case SPEC_STORE_BYPASS_SECCOMP:
 612         case SPEC_STORE_BYPASS_PRCTL:
 613                 if (task_spec_ssb_force_disable(task))
 614                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
 615                 if (task_spec_ssb_disable(task))
 616                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
 617                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
 618         default:
 619                 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
 620                         return PR_SPEC_ENABLE;
 621                 return PR_SPEC_NOT_AFFECTED;
 622         }
 623 }
 624
 625 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
 626 {
 627         switch (which) {
 628         case PR_SPEC_STORE_BYPASS:
 629                 return ssb_prctl_get(task);
 630         default:
 631                 return -ENODEV;
 632         }
 633 }
 634
 635 void x86_spec_ctrl_setup_ap(void)
 636 {
 637         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
 638                 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 639
 640         if (ssb_mode == SPEC_STORE_BYPASS_DISABLE)
 641                 x86_amd_ssb_disable();
 642 }
 643
 644 #undef pr_fmt
 645 #define pr_fmt(fmt)     "L1TF: " fmt
 646
 647 /* Default mitigation for L1TF-affected CPUs */
 648 enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH;
 649 #if IS_ENABLED(CONFIG_KVM_INTEL)
 650 EXPORT_SYMBOL_GPL(l1tf_mitigation);
 651 #endif
 652 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
 653 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation);
 654
 655 /*
 656  * These CPUs all support 44bits physical address space internally in the
 657  * cache but CPUID can report a smaller number of physical address bits.
 658  *
 659  * The L1TF mitigation uses the top most address bit for the inversion of
 660  * non present PTEs. When the installed memory reaches into the top most
 661  * address bit due to memory holes, which has been observed on machines
 662  * which report 36bits physical address bits and have 32G RAM installed,
 663  * then the mitigation range check in l1tf_select_mitigation() triggers.
 664  * This is a false positive because the mitigation is still possible due to
 665  * the fact that the cache uses 44bit internally. Use the cache bits
 666  * instead of the reported physical bits and adjust them on the affected
 667  * machines to 44bit if the reported bits are less than 44.
 668  */
 669 static void override_cache_bits(struct cpuinfo_x86 *c)
 670 {
 671         if (c->x86 != 6)
 672                 return;
 673
 674         switch (c->x86_model) {
 675         case INTEL_FAM6_NEHALEM:
 676         case INTEL_FAM6_WESTMERE:
 677         case INTEL_FAM6_SANDYBRIDGE:
 678         case INTEL_FAM6_IVYBRIDGE:
 679         case INTEL_FAM6_HASWELL_CORE:
 680         case INTEL_FAM6_HASWELL_ULT:
 681         case INTEL_FAM6_HASWELL_GT3E:
 682         case INTEL_FAM6_BROADWELL_CORE:
 683         case INTEL_FAM6_BROADWELL_GT3E:
 684         case INTEL_FAM6_SKYLAKE_MOBILE:
 685         case INTEL_FAM6_SKYLAKE_DESKTOP:
 686         case INTEL_FAM6_KABYLAKE_MOBILE:
 687         case INTEL_FAM6_KABYLAKE_DESKTOP:
 688                 if (c->x86_cache_bits < 44)
 689                         c->x86_cache_bits = 44;
 690                 break;
 691         }
 692 }
 693
 694 static void __init l1tf_select_mitigation(void)
 695 {
 696         u64 half_pa;
 697
 698         if (!boot_cpu_has_bug(X86_BUG_L1TF))
 699                 return;
 700
 701         override_cache_bits(&boot_cpu_data);
 702
 703         switch (l1tf_mitigation) {
 704         case L1TF_MITIGATION_OFF:
 705         case L1TF_MITIGATION_FLUSH_NOWARN:
 706         case L1TF_MITIGATION_FLUSH:
 707                 break;
 708         case L1TF_MITIGATION_FLUSH_NOSMT:
 709         case L1TF_MITIGATION_FULL:
 710                 cpu_smt_disable(false);
 711                 break;
 712         case L1TF_MITIGATION_FULL_FORCE:
 713                 cpu_smt_disable(true);
 714                 break;
 715         }
 716
 717 #if CONFIG_PGTABLE_LEVELS == 2
 718         pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n");
 719         return;
 720 #endif
 721
 722         half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT;
 723         if (e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) {
 724                 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n");
 725                 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n",
 726                                 half_pa);
 727                 pr_info("However, doing so will make a part of your RAM unusable.\n");
 728                 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/l1tf.html might help you decide.\n");
 729                 return;
 730         }
 731
 732         setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV);
 733 }
 734
 735 static int __init l1tf_cmdline(char *str)
 736 {
 737         if (!boot_cpu_has_bug(X86_BUG_L1TF))
 738                 return 0;
 739
 740         if (!str)
 741                 return -EINVAL;
 742
 743         if (!strcmp(str, "off"))
 744                 l1tf_mitigation = L1TF_MITIGATION_OFF;
 745         else if (!strcmp(str, "flush,nowarn"))
 746                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN;
 747         else if (!strcmp(str, "flush"))
 748                 l1tf_mitigation = L1TF_MITIGATION_FLUSH;
 749         else if (!strcmp(str, "flush,nosmt"))
 750                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
 751         else if (!strcmp(str, "full"))
 752                 l1tf_mitigation = L1TF_MITIGATION_FULL;
 753         else if (!strcmp(str, "full,force"))
 754                 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE;
 755
 756         return 0;
 757 }
 758 early_param("l1tf", l1tf_cmdline);
 759
 760 #undef pr_fmt
 761
 762 #ifdef CONFIG_SYSFS
 763
 764 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion"
 765
 766 #if IS_ENABLED(CONFIG_KVM_INTEL)
 767 static const char *l1tf_vmx_states[] = {
 768         [VMENTER_L1D_FLUSH_AUTO]                = "auto",
 769         [VMENTER_L1D_FLUSH_NEVER]               = "vulnerable",
 770         [VMENTER_L1D_FLUSH_COND]                = "conditional cache flushes",
 771         [VMENTER_L1D_FLUSH_ALWAYS]              = "cache flushes",
 772         [VMENTER_L1D_FLUSH_EPT_DISABLED]        = "EPT disabled",
 773         [VMENTER_L1D_FLUSH_NOT_REQUIRED]        = "flush not necessary"
 774 };
 775
 776 static ssize_t l1tf_show_state(char *buf)
 777 {
 778         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO)
 779                 return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
 780
 781         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED ||
 782             (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER &&
 783              cpu_smt_control == CPU_SMT_ENABLED))
 784                 return sprintf(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG,
 785                                l1tf_vmx_states[l1tf_vmx_mitigation]);
 786
 787         return sprintf(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG,
 788                        l1tf_vmx_states[l1tf_vmx_mitigation],
 789                        cpu_smt_control == CPU_SMT_ENABLED ? "vulnerable" : "disabled");
 790 }
 791 #else
 792 static ssize_t l1tf_show_state(char *buf)
 793 {
 794         return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
 795 }
 796 #endif
 797
 798 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
 799                                char *buf, unsigned int bug)
 800 {
 801         if (!boot_cpu_has_bug(bug))
 802                 return sprintf(buf, "Not affected\n");
 803
 804         switch (bug) {
 805         case X86_BUG_CPU_MELTDOWN:
 806                 if (boot_cpu_has(X86_FEATURE_PTI))
 807                         return sprintf(buf, "Mitigation: PTI\n");
 808
 809                 if (hypervisor_is_type(X86_HYPER_XEN_PV))
 810                         return sprintf(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n");
 811
 812                 break;
 813
 814         case X86_BUG_SPECTRE_V1:
 815                 return sprintf(buf, "Mitigation: __user pointer sanitization\n");
 816
 817         case X86_BUG_SPECTRE_V2:
 818                 return sprintf(buf, "%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
 819                                boot_cpu_has(X86_FEATURE_USE_IBPB) ? ", IBPB" : "",
 820                                boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
 821                                spectre_v2_module_string());
 822
 823         case X86_BUG_SPEC_STORE_BYPASS:
 824                 return sprintf(buf, "%s\n", ssb_strings[ssb_mode]);
 825
 826         case X86_BUG_L1TF:
 827                 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV))
 828                         return l1tf_show_state(buf);
 829                 break;
 830         default:
 831                 break;
 832         }
 833
 834         return sprintf(buf, "Vulnerable\n");
 835 }
 836
 837 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
 838 {
 839         return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN);
 840 }
 841
 842 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
 843 {
 844         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1);
 845 }
 846
 847 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
 848 {
 849         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2);
 850 }
 851
 852 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)
 853 {
 854         return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS);
 855 }
 856
 857 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf)
 858 {
 859         return cpu_show_common(dev, attr, buf, X86_BUG_L1TF);
 860 }
 861 #endif