usr/src/uts/i86pc/os/cmi.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26
  27 /*
  28  * Public interface to routines implemented by CPU modules
  29  */
  30
  31 #include <sys/types.h>
  32 #include <sys/atomic.h>
  33 #include <sys/x86_archext.h>
  34 #include <sys/cpu_module_impl.h>
  35 #include <sys/cpu_module_ms.h>
  36 #include <sys/fm/util.h>
  37 #include <sys/reboot.h>
  38 #include <sys/modctl.h>
  39 #include <sys/param.h>
  40 #include <sys/cmn_err.h>
  41 #include <sys/systm.h>
  42 #include <sys/fm/protocol.h>
  43 #include <sys/pcb.h>
  44 #include <sys/ontrap.h>
  45 #include <sys/psw.h>
  46 #include <sys/privregs.h>
  47 #include <sys/machsystm.h>
  48
  49 /*
  50  * Set to force cmi_init to fail.
  51  */
  52 int cmi_no_init = 0;
  53
  54 /*
  55  * Set to avoid MCA initialization.
  56  */
  57 int cmi_no_mca_init = 0;
  58
  59 /*
  60  * If cleared for debugging we will not attempt to load a model-specific
  61  * cpu module but will load the generic cpu module instead.
  62  */
  63 int cmi_force_generic = 0;
  64
  65 /*
  66  * If cleared for debugging, we will suppress panicking on fatal hardware
  67  * errors.  This should *only* be used for debugging; it use can and will
  68  * cause data corruption if actual hardware errors are detected by the system.
  69  */
  70 int cmi_panic_on_uncorrectable_error = 1;
  71
  72 #ifndef __xpv
  73 /*
  74  * Set to indicate whether we are able to enable cmci interrupt.
  75  */
  76 int cmi_enable_cmci = 0;
  77 #endif
  78
  79 /*
  80  * Subdirectory (relative to the module search path) in which we will
  81  * look for cpu modules.
  82  */
  83 #define CPUMOD_SUBDIR   "cpu"
  84
  85 /*
  86  * CPU modules have a filenames such as "cpu.AuthenticAMD.15" and
  87  * "cpu.generic" - the "cpu" prefix is specified by the following.
  88  */
  89 #define CPUMOD_PREFIX   "cpu"
  90
  91 /*
  92  * Structure used to keep track of cpu modules we have loaded and their ops
  93  */
  94 typedef struct cmi {
  95         struct cmi *cmi_next;
  96         struct cmi *cmi_prev;
  97         const cmi_ops_t *cmi_ops;
  98         struct modctl *cmi_modp;
  99         uint_t cmi_refcnt;
 100 } cmi_t;
 101
 102 static cmi_t *cmi_list;
 103 static const cmi_mc_ops_t *cmi_mc_global_ops;
 104 static void *cmi_mc_global_data;
 105 static kmutex_t cmi_load_lock;
 106
 107 /*
 108  * Functions we need from cmi_hw.c that are not part of the cpu_module.h
 109  * interface.
 110  */
 111 extern cmi_hdl_t cmi_hdl_create(enum cmi_hdl_class, uint_t, uint_t, uint_t);
 112 extern void cmi_hdl_destroy(cmi_hdl_t ophdl);
 113 extern void cmi_hdl_setcmi(cmi_hdl_t, void *, void *);
 114 extern void *cmi_hdl_getcmi(cmi_hdl_t);
 115 extern void cmi_hdl_setmc(cmi_hdl_t, const struct cmi_mc_ops *, void *);
 116 extern void cmi_hdl_inj_begin(cmi_hdl_t);
 117 extern void cmi_hdl_inj_end(cmi_hdl_t);
 118 extern void cmi_read_smbios(cmi_hdl_t);
 119
 120 #define HDL2CMI(hdl)            cmi_hdl_getcmi(hdl)
 121
 122 #define CMI_OPS(cmi)            (cmi)->cmi_ops
 123 #define CMI_OP_PRESENT(cmi, op) ((cmi) && CMI_OPS(cmi)->op != NULL)
 124
 125 #define CMI_MATCH_VENDOR        0       /* Just match on vendor */
 126 #define CMI_MATCH_FAMILY        1       /* Match down to family */
 127 #define CMI_MATCH_MODEL         2       /* Match down to model */
 128 #define CMI_MATCH_STEPPING      3       /* Match down to stepping */
 129
 130 static void
 131 cmi_link(cmi_t *cmi)
 132 {
 133         ASSERT(MUTEX_HELD(&cmi_load_lock));
 134
 135         cmi->cmi_prev = NULL;
 136         cmi->cmi_next = cmi_list;
 137         if (cmi_list != NULL)
 138                 cmi_list->cmi_prev = cmi;
 139         cmi_list = cmi;
 140 }
 141
 142 static void
 143 cmi_unlink(cmi_t *cmi)
 144 {
 145         ASSERT(MUTEX_HELD(&cmi_load_lock));
 146         ASSERT(cmi->cmi_refcnt == 0);
 147
 148         if (cmi->cmi_prev != NULL)
 149                 cmi->cmi_prev = cmi->cmi_next;
 150
 151         if (cmi->cmi_next != NULL)
 152                 cmi->cmi_next->cmi_prev = cmi->cmi_prev;
 153
 154         if (cmi_list == cmi)
 155                 cmi_list = cmi->cmi_next;
 156 }
 157
 158 /*
 159  * Hold the module in memory.  We call to CPU modules without using the
 160  * stubs mechanism, so these modules must be manually held in memory.
 161  * The mod_ref acts as if another loaded module has a dependency on us.
 162  */
 163 static void
 164 cmi_hold(cmi_t *cmi)
 165 {
 166         ASSERT(MUTEX_HELD(&cmi_load_lock));
 167
 168         mutex_enter(&mod_lock);
 169         cmi->cmi_modp->mod_ref++;
 170         mutex_exit(&mod_lock);
 171         cmi->cmi_refcnt++;
 172 }
 173
 174 static void
 175 cmi_rele(cmi_t *cmi)
 176 {
 177         ASSERT(MUTEX_HELD(&cmi_load_lock));
 178
 179         mutex_enter(&mod_lock);
 180         cmi->cmi_modp->mod_ref--;
 181         mutex_exit(&mod_lock);
 182
 183         if (--cmi->cmi_refcnt == 0) {
 184                 cmi_unlink(cmi);
 185                 kmem_free(cmi, sizeof (cmi_t));
 186         }
 187 }
 188
 189 static cmi_ops_t *
 190 cmi_getops(modctl_t *modp)
 191 {
 192         cmi_ops_t *ops;
 193
 194         if ((ops = (cmi_ops_t *)modlookup_by_modctl(modp, "_cmi_ops")) ==
 195             NULL) {
 196                 cmn_err(CE_WARN, "cpu module '%s' is invalid: no _cmi_ops "
 197                     "found", modp->mod_modname);
 198                 return (NULL);
 199         }
 200
 201         if (ops->cmi_init == NULL) {
 202                 cmn_err(CE_WARN, "cpu module '%s' is invalid: no cmi_init "
 203                     "entry point", modp->mod_modname);
 204                 return (NULL);
 205         }
 206
 207         return (ops);
 208 }
 209
 210 static cmi_t *
 211 cmi_load_modctl(modctl_t *modp)
 212 {
 213         cmi_ops_t *ops;
 214         uintptr_t ver;
 215         cmi_t *cmi;
 216         cmi_api_ver_t apiver;
 217
 218         ASSERT(MUTEX_HELD(&cmi_load_lock));
 219
 220         for (cmi = cmi_list; cmi != NULL; cmi = cmi->cmi_next) {
 221                 if (cmi->cmi_modp == modp)
 222                         return (cmi);
 223         }
 224
 225         if ((ver = modlookup_by_modctl(modp, "_cmi_api_version")) == NULL) {
 226                 /*
 227                  * Apparently a cpu module before versioning was introduced -
 228                  * we call this version 0.
 229                  */
 230                 apiver = CMI_API_VERSION_0;
 231         } else {
 232                 apiver = *((cmi_api_ver_t *)ver);
 233                 if (!CMI_API_VERSION_CHKMAGIC(apiver)) {
 234                         cmn_err(CE_WARN, "cpu module '%s' is invalid: "
 235                             "_cmi_api_version 0x%x has bad magic",
 236                             modp->mod_modname, apiver);
 237                         return (NULL);
 238                 }
 239         }
 240
 241         if (apiver != CMI_API_VERSION) {
 242                 cmn_err(CE_WARN, "cpu module '%s' has API version %d, "
 243                     "kernel requires API version %d", modp->mod_modname,
 244                     CMI_API_VERSION_TOPRINT(apiver),
 245                     CMI_API_VERSION_TOPRINT(CMI_API_VERSION));
 246                 return (NULL);
 247         }
 248
 249         if ((ops = cmi_getops(modp)) == NULL)
 250                 return (NULL);
 251
 252         cmi = kmem_zalloc(sizeof (*cmi), KM_SLEEP);
 253         cmi->cmi_ops = ops;
 254         cmi->cmi_modp = modp;
 255
 256         cmi_link(cmi);
 257
 258         return (cmi);
 259 }
 260
 261 static int
 262 cmi_cpu_match(cmi_hdl_t hdl1, cmi_hdl_t hdl2, int match)
 263 {
 264         if (match >= CMI_MATCH_VENDOR &&
 265             cmi_hdl_vendor(hdl1) != cmi_hdl_vendor(hdl2))
 266                 return (0);
 267
 268         if (match >= CMI_MATCH_FAMILY &&
 269             cmi_hdl_family(hdl1) != cmi_hdl_family(hdl2))
 270                 return (0);
 271
 272         if (match >= CMI_MATCH_MODEL &&
 273             cmi_hdl_model(hdl1) != cmi_hdl_model(hdl2))
 274                 return (0);
 275
 276         if (match >= CMI_MATCH_STEPPING &&
 277             cmi_hdl_stepping(hdl1) != cmi_hdl_stepping(hdl2))
 278                 return (0);
 279
 280         return (1);
 281 }
 282
 283 static int
 284 cmi_search_list_cb(cmi_hdl_t whdl, void *arg1, void *arg2, void *arg3)
 285 {
 286         cmi_hdl_t thdl = (cmi_hdl_t)arg1;
 287         int match = *((int *)arg2);
 288         cmi_hdl_t *rsltp = (cmi_hdl_t *)arg3;
 289
 290         if (cmi_cpu_match(thdl, whdl, match)) {
 291                 cmi_hdl_hold(whdl);     /* short-term hold */
 292                 *rsltp = whdl;
 293                 return (CMI_HDL_WALK_DONE);
 294         } else {
 295                 return (CMI_HDL_WALK_NEXT);
 296         }
 297 }
 298
 299 static cmi_t *
 300 cmi_search_list(cmi_hdl_t hdl, int match)
 301 {
 302         cmi_hdl_t dhdl = NULL;
 303         cmi_t *cmi = NULL;
 304
 305         ASSERT(MUTEX_HELD(&cmi_load_lock));
 306
 307         cmi_hdl_walk(cmi_search_list_cb, (void *)hdl, (void *)&match, &dhdl);
 308         if (dhdl) {
 309                 cmi = HDL2CMI(dhdl);
 310                 cmi_hdl_rele(dhdl);     /* held in cmi_search_list_cb */
 311         }
 312
 313         return (cmi);
 314 }
 315
 316 static cmi_t *
 317 cmi_load_module(cmi_hdl_t hdl, int match, int *chosenp)
 318 {
 319         modctl_t *modp;
 320         cmi_t *cmi;
 321         int modid;
 322         uint_t s[3];
 323
 324         ASSERT(MUTEX_HELD(&cmi_load_lock));
 325         ASSERT(match == CMI_MATCH_STEPPING || match == CMI_MATCH_MODEL ||
 326             match == CMI_MATCH_FAMILY || match == CMI_MATCH_VENDOR);
 327
 328         /*
 329          * Have we already loaded a module for a cpu with the same
 330          * vendor/family/model/stepping?
 331          */
 332         if ((cmi = cmi_search_list(hdl, match)) != NULL) {
 333                 cmi_hold(cmi);
 334                 return (cmi);
 335         }
 336
 337         s[0] = cmi_hdl_family(hdl);
 338         s[1] = cmi_hdl_model(hdl);
 339         s[2] = cmi_hdl_stepping(hdl);
 340         modid = modload_qualified(CPUMOD_SUBDIR, CPUMOD_PREFIX,
 341             cmi_hdl_vendorstr(hdl), ".", s, match, chosenp);
 342
 343         if (modid == -1)
 344                 return (NULL);
 345
 346         modp = mod_hold_by_id(modid);
 347         cmi = cmi_load_modctl(modp);
 348         if (cmi)
 349                 cmi_hold(cmi);
 350         mod_release_mod(modp);
 351
 352         return (cmi);
 353 }
 354
 355 /*
 356  * Try to load a cpu module with specific support for this chip type.
 357  */
 358 static cmi_t *
 359 cmi_load_specific(cmi_hdl_t hdl, void **datap)
 360 {
 361         cmi_t *cmi;
 362         int err;
 363         int i;
 364
 365         ASSERT(MUTEX_HELD(&cmi_load_lock));
 366
 367         for (i = CMI_MATCH_STEPPING; i >= CMI_MATCH_VENDOR; i--) {
 368                 int suffixlevel;
 369
 370                 if ((cmi = cmi_load_module(hdl, i, &suffixlevel)) == NULL)
 371                         return (NULL);
 372
 373                 /*
 374                  * A module has loaded and has a _cmi_ops structure, and the
 375                  * module has been held for this instance.  Call its cmi_init
 376                  * entry point - we expect success (0) or ENOTSUP.
 377                  */
 378                 if ((err = cmi->cmi_ops->cmi_init(hdl, datap)) == 0) {
 379                         if (boothowto & RB_VERBOSE) {
 380                                 printf("initialized cpu module '%s' on "
 381                                     "chip %d core %d strand %d\n",
 382                                     cmi->cmi_modp->mod_modname,
 383                                     cmi_hdl_chipid(hdl), cmi_hdl_coreid(hdl),
 384                                     cmi_hdl_strandid(hdl));
 385                         }
 386                         return (cmi);
 387                 } else if (err != ENOTSUP) {
 388                         cmn_err(CE_WARN, "failed to init cpu module '%s' on "
 389                             "chip %d core %d strand %d: err=%d\n",
 390                             cmi->cmi_modp->mod_modname,
 391                             cmi_hdl_chipid(hdl), cmi_hdl_coreid(hdl),
 392                             cmi_hdl_strandid(hdl), err);
 393                 }
 394
 395                 /*
 396                  * The module failed or declined to init, so release
 397                  * it and update i to be equal to the number
 398                  * of suffices actually used in the last module path.
 399                  */
 400                 cmi_rele(cmi);
 401                 i = suffixlevel;
 402         }
 403
 404         return (NULL);
 405 }
 406
 407 /*
 408  * Load the generic IA32 MCA cpu module, which may still supplement
 409  * itself with model-specific support through cpu model-specific modules.
 410  */
 411 static cmi_t *
 412 cmi_load_generic(cmi_hdl_t hdl, void **datap)
 413 {
 414         modctl_t *modp;
 415         cmi_t *cmi;
 416         int modid;
 417         int err;
 418
 419         ASSERT(MUTEX_HELD(&cmi_load_lock));
 420
 421         if ((modid = modload(CPUMOD_SUBDIR, CPUMOD_PREFIX ".generic")) == -1)
 422                 return (NULL);
 423
 424         modp = mod_hold_by_id(modid);
 425         cmi = cmi_load_modctl(modp);
 426         if (cmi)
 427                 cmi_hold(cmi);
 428         mod_release_mod(modp);
 429
 430         if (cmi == NULL)
 431                 return (NULL);
 432
 433         if ((err = cmi->cmi_ops->cmi_init(hdl, datap)) != 0) {
 434                 if (err != ENOTSUP)
 435                         cmn_err(CE_WARN, CPUMOD_PREFIX ".generic failed to "
 436                             "init: err=%d", err);
 437                 cmi_rele(cmi);
 438                 return (NULL);
 439         }
 440
 441         return (cmi);
 442 }
 443
 444 cmi_hdl_t
 445 cmi_init(enum cmi_hdl_class class, uint_t chipid, uint_t coreid,
 446     uint_t strandid)
 447 {
 448         cmi_t *cmi = NULL;
 449         cmi_hdl_t hdl;
 450         void *data;
 451
 452         if (cmi_no_init) {
 453                 cmi_no_mca_init = 1;
 454                 return (NULL);
 455         }
 456
 457         mutex_enter(&cmi_load_lock);
 458
 459         if ((hdl = cmi_hdl_create(class, chipid, coreid, strandid)) == NULL) {
 460                 mutex_exit(&cmi_load_lock);
 461                 cmn_err(CE_WARN, "There will be no MCA support on chip %d "
 462                     "core %d strand %d (cmi_hdl_create returned NULL)\n",
 463                     chipid, coreid, strandid);
 464                 return (NULL);
 465         }
 466
 467         if (!cmi_force_generic)
 468                 cmi = cmi_load_specific(hdl, &data);
 469
 470         if (cmi == NULL && (cmi = cmi_load_generic(hdl, &data)) == NULL) {
 471                 cmn_err(CE_WARN, "There will be no MCA support on chip %d "
 472                     "core %d strand %d\n", chipid, coreid, strandid);
 473                 cmi_hdl_rele(hdl);
 474                 mutex_exit(&cmi_load_lock);
 475                 return (NULL);
 476         }
 477
 478         cmi_hdl_setcmi(hdl, cmi, data);
 479
 480         cms_init(hdl);
 481
 482         cmi_read_smbios(hdl);
 483
 484         mutex_exit(&cmi_load_lock);
 485
 486         return (hdl);
 487 }
 488
 489 /*
 490  * cmi_fini is called on DR deconfigure of a cpu resource.
 491  * It should not be called at simple offline of a cpu.
 492  */
 493 void
 494 cmi_fini(cmi_hdl_t hdl)
 495 {
 496         cmi_t *cmi = HDL2CMI(hdl);
 497
 498         if (cms_present(hdl))
 499                 cms_fini(hdl);
 500
 501         if (CMI_OP_PRESENT(cmi, cmi_fini))
 502                 CMI_OPS(cmi)->cmi_fini(hdl);
 503
 504         cmi_hdl_destroy(hdl);
 505 }
 506
 507 /*
 508  * cmi_post_startup is called from post_startup for the boot cpu only (no
 509  * other cpus are started yet).
 510  */
 511 void
 512 cmi_post_startup(void)
 513 {
 514         cmi_hdl_t hdl;
 515         cmi_t *cmi;
 516
 517         if (cmi_no_mca_init != 0 ||
 518             (hdl = cmi_hdl_any()) == NULL)      /* short-term hold */
 519                 return;
 520
 521         cmi = HDL2CMI(hdl);
 522
 523         if (CMI_OP_PRESENT(cmi, cmi_post_startup))
 524                 CMI_OPS(cmi)->cmi_post_startup(hdl);
 525
 526         cmi_hdl_rele(hdl);
 527 }
 528
 529 /*
 530  * Called just once from start_other_cpus when all processors are started.
 531  * This will not be called for each cpu, so the registered op must not
 532  * assume it is called as such.  We are not necessarily executing on
 533  * the boot cpu.
 534  */
 535 void
 536 cmi_post_mpstartup(void)
 537 {
 538         cmi_hdl_t hdl;
 539         cmi_t *cmi;
 540
 541         if (cmi_no_mca_init != 0 ||
 542             (hdl = cmi_hdl_any()) == NULL)      /* short-term hold */
 543                 return;
 544
 545         cmi = HDL2CMI(hdl);
 546
 547         if (CMI_OP_PRESENT(cmi, cmi_post_mpstartup))
 548                 CMI_OPS(cmi)->cmi_post_mpstartup(hdl);
 549
 550         cmi_hdl_rele(hdl);
 551 }
 552
 553 void
 554 cmi_faulted_enter(cmi_hdl_t hdl)
 555 {
 556         cmi_t *cmi = HDL2CMI(hdl);
 557
 558         if (cmi_no_mca_init != 0)
 559                 return;
 560
 561         if (CMI_OP_PRESENT(cmi, cmi_faulted_enter))
 562                 CMI_OPS(cmi)->cmi_faulted_enter(hdl);
 563 }
 564
 565 void
 566 cmi_faulted_exit(cmi_hdl_t hdl)
 567 {
 568         cmi_t *cmi = HDL2CMI(hdl);
 569
 570         if (cmi_no_mca_init != 0)
 571                 return;
 572
 573         if (CMI_OP_PRESENT(cmi, cmi_faulted_exit))
 574                 CMI_OPS(cmi)->cmi_faulted_exit(hdl);
 575 }
 576
 577 void
 578 cmi_mca_init(cmi_hdl_t hdl)
 579 {
 580         cmi_t *cmi;
 581
 582         if (cmi_no_mca_init != 0)
 583                 return;
 584
 585         cmi = HDL2CMI(hdl);
 586
 587         if (CMI_OP_PRESENT(cmi, cmi_mca_init))
 588                 CMI_OPS(cmi)->cmi_mca_init(hdl);
 589 }
 590
 591 #define CMI_RESPONSE_PANIC              0x0     /* panic must have value 0 */
 592 #define CMI_RESPONSE_NONE               0x1
 593 #define CMI_RESPONSE_CKILL              0x2
 594 #define CMI_RESPONSE_REBOOT             0x3     /* not implemented */
 595 #define CMI_RESPONSE_ONTRAP_PROT        0x4
 596 #define CMI_RESPONSE_LOFAULT_PROT       0x5
 597
 598 /*
 599  * Return 0 if we will panic in response to this machine check, otherwise
 600  * non-zero.  If the caller is cmi_mca_trap in this file then the nonzero
 601  * return values are to be interpreted from CMI_RESPONSE_* above.
 602  *
 603  * This function must just return what will be done without actually
 604  * doing anything; this includes not changing the regs.
 605  */
 606 int
 607 cmi_mce_response(struct regs *rp, uint64_t disp)
 608 {
 609         int panicrsp = cmi_panic_on_uncorrectable_error ? CMI_RESPONSE_PANIC :
 610             CMI_RESPONSE_NONE;
 611         on_trap_data_t *otp;
 612
 613         ASSERT(rp != NULL);     /* don't call for polling, only on #MC */
 614
 615         /*
 616          * If no bits are set in the disposition then there is nothing to
 617          * worry about and we do not need to trampoline to ontrap or
 618          * lofault handlers.
 619          */
 620         if (disp == 0)
 621                 return (CMI_RESPONSE_NONE);
 622
 623         /*
 624          * Unconstrained errors cannot be forgiven, even by ontrap or
 625          * lofault protection.  The data is not poisoned and may not
 626          * even belong to the trapped context - eg a writeback of
 627          * data that is found to be bad.
 628          */
 629         if (disp & CMI_ERRDISP_UC_UNCONSTRAINED)
 630                 return (panicrsp);
 631
 632         /*
 633          * ontrap OT_DATA_EC and lofault protection forgive any disposition
 634          * other than unconstrained, even those normally forced fatal.
 635          */
 636         if ((otp = curthread->t_ontrap) != NULL && otp->ot_prot & OT_DATA_EC)
 637                 return (CMI_RESPONSE_ONTRAP_PROT);
 638         else if (curthread->t_lofault)
 639                 return (CMI_RESPONSE_LOFAULT_PROT);
 640
 641         /*
 642          * Forced-fatal errors are terminal even in user mode.
 643          */
 644         if (disp & CMI_ERRDISP_FORCEFATAL)
 645                 return (panicrsp);
 646
 647         /*
 648          * If the trapped context is corrupt or we have no instruction pointer
 649          * to resume at (and aren't trampolining to a fault handler)
 650          * then in the kernel case we must panic and in usermode we
 651          * kill the affected contract.
 652          */
 653         if (disp & (CMI_ERRDISP_CURCTXBAD | CMI_ERRDISP_RIPV_INVALID))
 654                 return (USERMODE(rp->r_cs) ?  CMI_RESPONSE_CKILL : panicrsp);
 655
 656         /*
 657          * Anything else is harmless
 658          */
 659         return (CMI_RESPONSE_NONE);
 660 }
 661
 662 int cma_mca_trap_panic_suppressed = 0;
 663
 664 static void
 665 cmi_mca_panic(void)
 666 {
 667         if (cmi_panic_on_uncorrectable_error) {
 668                 fm_panic("Unrecoverable Machine-Check Exception");
 669         } else {
 670                 cmn_err(CE_WARN, "suppressing panic from fatal #mc");
 671                 cma_mca_trap_panic_suppressed++;
 672         }
 673 }
 674
 675
 676 int cma_mca_trap_contract_kills = 0;
 677 int cma_mca_trap_ontrap_forgiven = 0;
 678 int cma_mca_trap_lofault_forgiven = 0;
 679
 680 /*
 681  * Native #MC handler - we branch to here from mcetrap
 682  */
 683 /*ARGSUSED*/
 684 void
 685 cmi_mca_trap(struct regs *rp)
 686 {
 687 #ifndef __xpv
 688         cmi_hdl_t hdl = NULL;
 689         uint64_t disp;
 690         cmi_t *cmi;
 691         int s;
 692
 693         if (cmi_no_mca_init != 0)
 694                 return;
 695
 696         /*
 697          * This function can call cmn_err, and the cpu module cmi_mca_trap
 698          * entry point may also elect to call cmn_err (e.g., if it can't
 699          * log the error onto an errorq, say very early in boot).
 700          * We need to let cprintf know that we must not block.
 701          */
 702         s = spl8();
 703
 704         if ((hdl = cmi_hdl_lookup(CMI_HDL_NATIVE, cmi_ntv_hwchipid(CPU),
 705             cmi_ntv_hwcoreid(CPU), cmi_ntv_hwstrandid(CPU))) == NULL ||
 706             (cmi = HDL2CMI(hdl)) == NULL ||
 707             !CMI_OP_PRESENT(cmi, cmi_mca_trap)) {
 708
 709                 cmn_err(CE_WARN, "#MC exception on cpuid %d: %s",
 710                     CPU->cpu_id,
 711                     hdl ? "handle lookup ok but no #MC handler found" :
 712                     "handle lookup failed");
 713
 714                 if (hdl != NULL)
 715                         cmi_hdl_rele(hdl);
 716
 717                 splx(s);
 718                 return;
 719         }
 720
 721         disp = CMI_OPS(cmi)->cmi_mca_trap(hdl, rp);
 722
 723         switch (cmi_mce_response(rp, disp)) {
 724         default:
 725                 cmn_err(CE_WARN, "Invalid response from cmi_mce_response");
 726                 /*FALLTHRU*/
 727
 728         case CMI_RESPONSE_PANIC:
 729                 cmi_mca_panic();
 730                 break;
 731
 732         case CMI_RESPONSE_NONE:
 733                 break;
 734
 735         case CMI_RESPONSE_CKILL:
 736                 ttolwp(curthread)->lwp_pcb.pcb_flags |= ASYNC_HWERR;
 737                 aston(curthread);
 738                 cma_mca_trap_contract_kills++;
 739                 break;
 740
 741         case CMI_RESPONSE_ONTRAP_PROT: {
 742                 on_trap_data_t *otp = curthread->t_ontrap;
 743                 otp->ot_trap = OT_DATA_EC;
 744                 rp->r_pc = otp->ot_trampoline;
 745                 cma_mca_trap_ontrap_forgiven++;
 746                 break;
 747         }
 748
 749         case CMI_RESPONSE_LOFAULT_PROT:
 750                 rp->r_r0 = EFAULT;
 751                 rp->r_pc = curthread->t_lofault;
 752                 cma_mca_trap_lofault_forgiven++;
 753                 break;
 754         }
 755
 756         cmi_hdl_rele(hdl);
 757         splx(s);
 758 #endif  /* __xpv */
 759 }
 760
 761 void
 762 cmi_hdl_poke(cmi_hdl_t hdl)
 763 {
 764         cmi_t *cmi = HDL2CMI(hdl);
 765
 766         if (!CMI_OP_PRESENT(cmi, cmi_hdl_poke))
 767                 return;
 768
 769         CMI_OPS(cmi)->cmi_hdl_poke(hdl);
 770 }
 771
 772 #ifndef __xpv
 773 void
 774 cmi_cmci_trap()
 775 {
 776         cmi_hdl_t hdl = NULL;
 777         cmi_t *cmi;
 778
 779         if (cmi_no_mca_init != 0)
 780                 return;
 781
 782         if ((hdl = cmi_hdl_lookup(CMI_HDL_NATIVE, cmi_ntv_hwchipid(CPU),
 783             cmi_ntv_hwcoreid(CPU), cmi_ntv_hwstrandid(CPU))) == NULL ||
 784             (cmi = HDL2CMI(hdl)) == NULL ||
 785             !CMI_OP_PRESENT(cmi, cmi_cmci_trap)) {
 786
 787                 cmn_err(CE_WARN, "CMCI interrupt on cpuid %d: %s",
 788                     CPU->cpu_id,
 789                     hdl ? "handle lookup ok but no CMCI handler found" :
 790                     "handle lookup failed");
 791
 792                 if (hdl != NULL)
 793                         cmi_hdl_rele(hdl);
 794
 795                 return;
 796         }
 797
 798         CMI_OPS(cmi)->cmi_cmci_trap(hdl);
 799
 800         cmi_hdl_rele(hdl);
 801 }
 802 #endif  /* __xpv */
 803
 804 void
 805 cmi_mc_register(cmi_hdl_t hdl, const cmi_mc_ops_t *mcops, void *mcdata)
 806 {
 807         if (!cmi_no_mca_init)
 808                 cmi_hdl_setmc(hdl, mcops, mcdata);
 809 }
 810
 811 cmi_errno_t
 812 cmi_mc_register_global(const cmi_mc_ops_t *mcops, void *mcdata)
 813 {
 814         if (!cmi_no_mca_init) {
 815                 if (cmi_mc_global_ops != NULL || cmi_mc_global_data != NULL ||
 816                     mcops == NULL || mcops->cmi_mc_patounum == NULL ||
 817                     mcops->cmi_mc_unumtopa == NULL) {
 818                         return (CMIERR_UNKNOWN);
 819                 }
 820                 cmi_mc_global_data = mcdata;
 821                 cmi_mc_global_ops = mcops;
 822         }
 823         return (CMI_SUCCESS);
 824 }
 825
 826 void
 827 cmi_mc_sw_memscrub_disable(void)
 828 {
 829         memscrub_disable();
 830 }
 831
 832 cmi_errno_t
 833 cmi_mc_patounum(uint64_t pa, uint8_t valid_hi, uint8_t valid_lo, uint32_t synd,
 834     int syndtype, mc_unum_t *up)
 835 {
 836         const struct cmi_mc_ops *mcops;
 837         cmi_hdl_t hdl;
 838         cmi_errno_t rv;
 839
 840         if (cmi_no_mca_init)
 841                 return (CMIERR_MC_ABSENT);
 842
 843         if (cmi_mc_global_ops != NULL) {
 844                 if (cmi_mc_global_ops->cmi_mc_patounum == NULL)
 845                         return (CMIERR_MC_NOTSUP);
 846                 return (cmi_mc_global_ops->cmi_mc_patounum(cmi_mc_global_data,
 847                     pa, valid_hi, valid_lo, synd, syndtype, up));
 848         }
 849
 850         if ((hdl = cmi_hdl_any()) == NULL)      /* short-term hold */
 851                 return (CMIERR_MC_ABSENT);
 852
 853         if ((mcops = cmi_hdl_getmcops(hdl)) == NULL ||
 854             mcops->cmi_mc_patounum == NULL) {
 855                 cmi_hdl_rele(hdl);
 856                 return (CMIERR_MC_NOTSUP);
 857         }
 858
 859         rv = mcops->cmi_mc_patounum(cmi_hdl_getmcdata(hdl), pa, valid_hi,
 860             valid_lo, synd, syndtype, up);
 861
 862         cmi_hdl_rele(hdl);
 863
 864         return (rv);
 865 }
 866
 867 cmi_errno_t
 868 cmi_mc_unumtopa(mc_unum_t *up, nvlist_t *nvl, uint64_t *pap)
 869 {
 870         const struct cmi_mc_ops *mcops;
 871         cmi_hdl_t hdl;
 872         cmi_errno_t rv;
 873         nvlist_t *hcsp;
 874
 875         if (up != NULL && nvl != NULL)
 876                 return (CMIERR_API);    /* convert from just one form */
 877
 878         if (cmi_no_mca_init)
 879                 return (CMIERR_MC_ABSENT);
 880
 881         if (cmi_mc_global_ops != NULL) {
 882                 if (cmi_mc_global_ops->cmi_mc_unumtopa == NULL)
 883                         return (CMIERR_MC_NOTSUP);
 884                 return (cmi_mc_global_ops->cmi_mc_unumtopa(cmi_mc_global_data,
 885                     up, nvl, pap));
 886         }
 887
 888         if ((hdl = cmi_hdl_any()) == NULL)      /* short-term hold */
 889                 return (CMIERR_MC_ABSENT);
 890
 891         if ((mcops = cmi_hdl_getmcops(hdl)) == NULL ||
 892             mcops->cmi_mc_unumtopa == NULL) {
 893                 cmi_hdl_rele(hdl);
 894
 895                 if (nvl != NULL && nvlist_lookup_nvlist(nvl,
 896                     FM_FMRI_HC_SPECIFIC, &hcsp) == 0 &&
 897                     (nvlist_lookup_uint64(hcsp,
 898                     "asru-" FM_FMRI_HC_SPECIFIC_PHYSADDR, pap) == 0 ||
 899                     nvlist_lookup_uint64(hcsp, FM_FMRI_HC_SPECIFIC_PHYSADDR,
 900                     pap) == 0)) {
 901                         return (CMIERR_MC_PARTIALUNUMTOPA);
 902                 } else {
 903                         return (mcops && mcops->cmi_mc_unumtopa == NULL ?
 904                             CMIERR_MC_NOTSUP : CMIERR_MC_ABSENT);
 905                 }
 906         }
 907
 908         rv = mcops->cmi_mc_unumtopa(cmi_hdl_getmcdata(hdl), up, nvl, pap);
 909
 910         cmi_hdl_rele(hdl);
 911
 912         return (rv);
 913 }
 914
 915 void
 916 cmi_mc_logout(cmi_hdl_t hdl, boolean_t ismc, boolean_t sync)
 917 {
 918         const struct cmi_mc_ops *mcops;
 919
 920         if (cmi_no_mca_init)
 921                 return;
 922
 923         if (cmi_mc_global_ops != NULL)
 924                 mcops = cmi_mc_global_ops;
 925         else
 926                 mcops = cmi_hdl_getmcops(hdl);
 927
 928         if (mcops != NULL && mcops->cmi_mc_logout != NULL)
 929                 mcops->cmi_mc_logout(hdl, ismc, sync);
 930 }
 931
 932 cmi_errno_t
 933 cmi_hdl_msrinject(cmi_hdl_t hdl, cmi_mca_regs_t *regs, uint_t nregs,
 934     int force)
 935 {
 936         cmi_t *cmi = cmi_hdl_getcmi(hdl);
 937         cmi_errno_t rc;
 938
 939         if (!CMI_OP_PRESENT(cmi, cmi_msrinject))
 940                 return (CMIERR_NOTSUP);
 941
 942         cmi_hdl_inj_begin(hdl);
 943         rc = CMI_OPS(cmi)->cmi_msrinject(hdl, regs, nregs, force);
 944         cmi_hdl_inj_end(hdl);
 945
 946         return (rc);
 947 }
 948
 949 boolean_t
 950 cmi_panic_on_ue(void)
 951 {
 952         return (cmi_panic_on_uncorrectable_error ? B_TRUE : B_FALSE);
 953 }
 954
 955 void
 956 cmi_panic_callback(void)
 957 {
 958         cmi_hdl_t hdl;
 959         cmi_t *cmi;
 960
 961         if (cmi_no_mca_init || (hdl = cmi_hdl_any()) == NULL)
 962                 return;
 963
 964         cmi = cmi_hdl_getcmi(hdl);
 965         if (CMI_OP_PRESENT(cmi, cmi_panic_callback))
 966                 CMI_OPS(cmi)->cmi_panic_callback();
 967
 968         cmi_hdl_rele(hdl);
 969 }