usr/src/uts/i86pc/os/cms.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2016 by Delphix. All rights reserved.
  25  */
  26 /*
  27  * Copyright (c) 2010, Intel Corporation.
  28  * All rights reserved.
  29  */
  30
  31 #include <sys/types.h>
  32 #include <sys/cpu_module_ms_impl.h>
  33 #include <sys/cpuvar.h>
  34 #include <sys/ksynch.h>
  35 #include <sys/modctl.h>
  36 #include <sys/x86_archext.h>
  37 #include <sys/systm.h>
  38 #include <sys/cmn_err.h>
  39 #include <sys/param.h>
  40 #include <sys/reboot.h>
  41
  42 /*
  43  * Set to prevent model-specific support from initialising.
  44  */
  45 int cms_no_model_specific = 0;
  46
  47 /*
  48  * Subdirectory (relative to the module search path) in which we will
  49  * look for model-specific modules.
  50  */
  51 #define CPUMOD_MS_SUBDIR        "cpu"
  52
  53 /*
  54  * Cpu model-specific modules have filenames beginning with the following.
  55  */
  56 #define CPUMOD_MS_PREFIX        "cpu_ms"
  57
  58 #define HDL2CMS(hdl)            cms_hdl_getcms(hdl)
  59
  60 #define CMS_OPS(cms)            (cms)->cms_ops
  61 #define CMS_OP_PRESENT(cms, op) ((cms) && CMS_OPS(cms)->op != NULL)
  62
  63 struct cms_cpuid {
  64         const char *vendor;
  65         uint_t family;
  66         uint_t model;
  67         uint_t stepping;
  68 };
  69
  70 #define CMS_MATCH_VENDOR        0       /* Just match on vendor */
  71 #define CMS_MATCH_FAMILY        1       /* Match down to family */
  72 #define CMS_MATCH_MODEL         2       /* Match down to model */
  73 #define CMS_MATCH_STEPPING      3       /* Match down to stepping */
  74
  75 /*
  76  * Structure used to keep track of modules we have loaded.
  77  */
  78 typedef struct cms {
  79         struct cms *cms_next;
  80         struct cms *cms_prev;
  81         const cms_ops_t *cms_ops;
  82         struct modctl *cms_modp;
  83         uint_t cms_refcnt;
  84 } cms_t;
  85
  86 static cms_t *cms_list;
  87 static kmutex_t cms_load_lock;
  88
  89 /*
  90  * We stash a cms_t and associated private data via cmi_hdl_setspecific.
  91  */
  92 struct cms_ctl {
  93         cms_t *cs_cms;
  94         void *cs_cmsdata;
  95 };
  96
  97 static cms_t *
  98 cms_hdl_getcms(cmi_hdl_t hdl)
  99 {
 100         struct cms_ctl *cdp = cmi_hdl_getspecific(hdl);
 101
 102         return (cdp != NULL ? cdp->cs_cms : NULL);
 103 }
 104
 105 void *
 106 cms_hdl_getcmsdata(cmi_hdl_t hdl)
 107 {
 108         struct cms_ctl *cdp = cmi_hdl_getspecific(hdl);
 109
 110         return (cdp != NULL ? cdp->cs_cmsdata : NULL);
 111 }
 112
 113 static void
 114 cms_link(cms_t *cms)
 115 {
 116         ASSERT(MUTEX_HELD(&cms_load_lock));
 117
 118         cms->cms_prev = NULL;
 119         cms->cms_next = cms_list;
 120         if (cms_list != NULL)
 121                 cms_list->cms_prev = cms;
 122         cms_list = cms;
 123 }
 124
 125 static void
 126 cms_unlink(cms_t *cms)
 127 {
 128         ASSERT(MUTEX_HELD(&cms_load_lock));
 129         ASSERT(cms->cms_refcnt == 0);
 130
 131         if (cms->cms_prev != NULL)
 132                 cms->cms_prev->cms_next = cms->cms_next;
 133
 134         if (cms->cms_next != NULL)
 135                 cms->cms_next->cms_prev = cms->cms_prev;
 136
 137         if (cms_list == cms)
 138                 cms_list = cms->cms_next;
 139 }
 140
 141 /*
 142  * Hold the module in memory.  We call to CPU modules without using the
 143  * stubs mechanism, so these modules must be manually held in memory.
 144  * The mod_ref acts as if another loaded module has a dependency on us.
 145  */
 146 static void
 147 cms_hold(cms_t *cms)
 148 {
 149         ASSERT(MUTEX_HELD(&cms_load_lock));
 150
 151         mutex_enter(&mod_lock);
 152         cms->cms_modp->mod_ref++;
 153         mutex_exit(&mod_lock);
 154         cms->cms_refcnt++;
 155 }
 156
 157 static void
 158 cms_rele(cms_t *cms)
 159 {
 160         ASSERT(MUTEX_HELD(&cms_load_lock));
 161
 162         mutex_enter(&mod_lock);
 163         cms->cms_modp->mod_ref--;
 164         mutex_exit(&mod_lock);
 165
 166         if (--cms->cms_refcnt == 0) {
 167                 cms_unlink(cms);
 168                 kmem_free(cms, sizeof (cms_t));
 169         }
 170 }
 171
 172 static cms_ops_t *
 173 cms_getops(modctl_t *modp)
 174 {
 175         cms_ops_t *ops;
 176
 177         if ((ops = (cms_ops_t *)modlookup_by_modctl(modp, "_cms_ops")) ==
 178             NULL) {
 179                 cmn_err(CE_WARN, "cpu_ms module '%s' is invalid: no _cms_ops "
 180                     "found", modp->mod_modname);
 181                 return (NULL);
 182         }
 183
 184         if (ops->cms_init == NULL) {
 185                 cmn_err(CE_WARN, "cpu_ms module '%s' is invalid: no cms_init "
 186                     "entry point", modp->mod_modname);
 187                 return (NULL);
 188         }
 189
 190         return (ops);
 191 }
 192
 193 static cms_t *
 194 cms_load_modctl(modctl_t *modp)
 195 {
 196         cms_ops_t *ops;
 197         uintptr_t ver;
 198         cms_t *cms;
 199         cms_api_ver_t apiver;
 200
 201         ASSERT(MUTEX_HELD(&cms_load_lock));
 202
 203         for (cms = cms_list; cms != NULL; cms = cms->cms_next) {
 204                 if (cms->cms_modp == modp)
 205                         return (cms);
 206         }
 207
 208         if ((ver = modlookup_by_modctl(modp, "_cms_api_version")) == NULL) {
 209                 cmn_err(CE_WARN, "cpu model-specific module '%s' is invalid:  "
 210                     "no _cms_api_version", modp->mod_modname);
 211                 return (NULL);
 212         } else {
 213                 apiver = *((cms_api_ver_t *)ver);
 214                 if (!CMS_API_VERSION_CHKMAGIC(apiver)) {
 215                         cmn_err(CE_WARN, "cpu model-specific module '%s' is "
 216                             "invalid: _cms_api_version 0x%x has bad magic",
 217                             modp->mod_modname, apiver);
 218                         return (NULL);
 219                 }
 220         }
 221
 222         if (apiver != CMS_API_VERSION) {
 223                 cmn_err(CE_WARN, "cpu model-specific module '%s' has API "
 224                     "version %d, kernel requires API version %d",
 225                     modp->mod_modname, CMS_API_VERSION_TOPRINT(apiver),
 226                     CMS_API_VERSION_TOPRINT(CMS_API_VERSION));
 227         return (NULL);
 228         }
 229
 230         if ((ops = cms_getops(modp)) == NULL)
 231                 return (NULL);
 232
 233         cms = kmem_zalloc(sizeof (cms_t), KM_SLEEP);
 234         cms->cms_ops = ops;
 235         cms->cms_modp = modp;
 236
 237         cms_link(cms);
 238
 239         return (cms);
 240 }
 241
 242 static int
 243 cms_cpu_match(cmi_hdl_t hdl1, cmi_hdl_t hdl2, int match)
 244 {
 245         if (match >= CMS_MATCH_VENDOR &&
 246             cmi_hdl_vendor(hdl1) != cmi_hdl_vendor(hdl2))
 247                 return (0);
 248
 249         if (match >= CMS_MATCH_FAMILY &&
 250             cmi_hdl_family(hdl1) != cmi_hdl_family(hdl2))
 251                 return (0);
 252
 253         if (match >= CMS_MATCH_MODEL &&
 254             cmi_hdl_model(hdl1) != cmi_hdl_model(hdl2))
 255                 return (0);
 256
 257         if (match >= CMS_MATCH_STEPPING &&
 258             cmi_hdl_stepping(hdl1) != cmi_hdl_stepping(hdl2))
 259                 return (0);
 260
 261         return (1);
 262 }
 263
 264 static int
 265 cms_search_list_cb(cmi_hdl_t whdl, void *arg1, void *arg2, void *arg3)
 266 {
 267         cmi_hdl_t thdl = (cmi_hdl_t)arg1;
 268         int match = *((int *)arg2);
 269         cmi_hdl_t *rsltp = (cmi_hdl_t *)arg3;
 270
 271         if (cms_cpu_match(thdl, whdl, match)) {
 272                 cmi_hdl_hold(whdl);     /* short-term hold */
 273                 *rsltp = whdl;
 274                 return (CMI_HDL_WALK_DONE);
 275         } else {
 276                 return (CMI_HDL_WALK_NEXT);
 277         }
 278 }
 279
 280 /*
 281  * Look to see if we've already got a module loaded for a CPU just
 282  * like this one.  If we do, then we'll re-use it.
 283  */
 284 static cms_t *
 285 cms_search_list(cmi_hdl_t hdl, int match)
 286 {
 287         cmi_hdl_t dhdl = NULL;
 288         cms_t *cms = NULL;
 289
 290         ASSERT(MUTEX_HELD(&cms_load_lock));
 291
 292         cmi_hdl_walk(cms_search_list_cb, (void *)hdl, (void *)&match, &dhdl);
 293         if (dhdl) {
 294                 cms = HDL2CMS(dhdl);
 295                 cmi_hdl_rele(dhdl);     /* held in cms_search_list_cb */
 296         }
 297
 298         return (cms);
 299 }
 300
 301 /*
 302  * Try to find or load a module that offers model-specific support for
 303  * this vendor/family/model/stepping combination.  When attempting to load
 304  * a module we look in CPUMOD_MS_SUBDIR first for a match on
 305  * vendor/family/model/stepping, then on vendor/family/model (ignoring
 306  * stepping), then on vendor/family (ignoring model and stepping), then
 307  * on vendor alone.
 308  */
 309 static cms_t *
 310 cms_load_module(cmi_hdl_t hdl, int match, int *chosenp)
 311 {
 312         modctl_t *modp;
 313         cms_t *cms;
 314         int modid;
 315         uint_t s[3];
 316
 317         ASSERT(MUTEX_HELD(&cms_load_lock));
 318         ASSERT(match == CMS_MATCH_STEPPING || match == CMS_MATCH_MODEL ||
 319             match == CMS_MATCH_FAMILY || match == CMS_MATCH_VENDOR);
 320
 321         s[0] = cmi_hdl_family(hdl);
 322         s[1] = cmi_hdl_model(hdl);
 323         s[2] = cmi_hdl_stepping(hdl);
 324
 325         /*
 326          * Have we already loaded a module for a cpu with the same
 327          * vendor/family/model/stepping?
 328          */
 329         if ((cms = cms_search_list(hdl, match)) != NULL) {
 330                 cms_hold(cms);
 331                 return (cms);
 332         }
 333
 334         modid = modload_qualified(CPUMOD_MS_SUBDIR, CPUMOD_MS_PREFIX,
 335             cmi_hdl_vendorstr(hdl), ".", s, match, chosenp);
 336
 337         if (modid == -1)
 338                 return (NULL);
 339
 340         modp = mod_hold_by_id(modid);
 341         cms = cms_load_modctl(modp);
 342         if (cms)
 343                 cms_hold(cms);
 344         mod_release_mod(modp);
 345
 346         return (cms);
 347 }
 348
 349 static cms_t *
 350 cms_load_specific(cmi_hdl_t hdl, void **datap)
 351 {
 352         cms_t *cms;
 353         int err;
 354         int i;
 355
 356         ASSERT(MUTEX_HELD(&cms_load_lock));
 357
 358         for (i = CMS_MATCH_STEPPING; i >= CMS_MATCH_VENDOR; i--) {
 359                 int suffixlevel;
 360
 361                 if ((cms = cms_load_module(hdl, i, &suffixlevel)) == NULL)
 362                         return (NULL);
 363
 364                 /*
 365                  * A module has loaded and has a _cms_ops structure, and the
 366                  * module has been held for this instance.  Call the cms_init
 367                  * entry point - we expect success (0) or ENOTSUP.
 368                  */
 369                 if ((err = cms->cms_ops->cms_init(hdl, datap)) == 0) {
 370                         if (boothowto & RB_VERBOSE) {
 371                                 printf("initialized model-specific "
 372                                     "module '%s' on chip %d core %d "
 373                                     "strand %d\n",
 374                                     cms->cms_modp->mod_modname,
 375                                     cmi_hdl_chipid(hdl), cmi_hdl_coreid(hdl),
 376                                     cmi_hdl_strandid(hdl));
 377                         }
 378                         return (cms);
 379                 } else if (err != ENOTSUP) {
 380                         cmn_err(CE_WARN, "failed to init model-specific "
 381                             "module '%s' on chip %d core %d strand %d: err=%d",
 382                             cms->cms_modp->mod_modname,
 383                             cmi_hdl_chipid(hdl), cmi_hdl_coreid(hdl),
 384                             cmi_hdl_strandid(hdl), err);
 385                 }
 386
 387                 /*
 388                  * The module failed or declined to init, so release
 389                  * it and potentially change i to be equal to the number
 390                  * of suffices actually used in the last module path.
 391                  */
 392                 cms_rele(cms);
 393                 i = suffixlevel;
 394         }
 395
 396         return (NULL);
 397 }
 398
 399 void
 400 cms_init(cmi_hdl_t hdl)
 401 {
 402         cms_t *cms;
 403         void *data;
 404
 405         if (cms_no_model_specific != 0)
 406                 return;
 407
 408         mutex_enter(&cms_load_lock);
 409
 410         if ((cms = cms_load_specific(hdl, &data)) != NULL) {
 411                 struct cms_ctl *cdp;
 412
 413                 ASSERT(cmi_hdl_getspecific(hdl) == NULL);
 414
 415                 cdp = kmem_alloc(sizeof (*cdp), KM_SLEEP);
 416                 cdp->cs_cms = cms;
 417                 cdp->cs_cmsdata = data;
 418                 cmi_hdl_setspecific(hdl, cdp);
 419         }
 420
 421         mutex_exit(&cms_load_lock);
 422 }
 423
 424 void
 425 cms_fini(cmi_hdl_t hdl)
 426 {
 427         cms_t *cms = HDL2CMS(hdl);
 428         struct cms_ctl *cdp;
 429
 430         if (CMS_OP_PRESENT(cms, cms_fini))
 431                 CMS_OPS(cms)->cms_fini(hdl);
 432
 433         mutex_enter(&cms_load_lock);
 434         cdp = (struct cms_ctl *)cmi_hdl_getspecific(hdl);
 435         if (cdp != NULL) {
 436                 if (cdp->cs_cms != NULL)
 437                         cms_rele(cdp->cs_cms);
 438                 kmem_free(cdp, sizeof (*cdp));
 439         }
 440         mutex_exit(&cms_load_lock);
 441 }
 442
 443 boolean_t
 444 cms_present(cmi_hdl_t hdl)
 445 {
 446         return (HDL2CMS(hdl) != NULL ? B_TRUE : B_FALSE);
 447 }
 448
 449 void
 450 cms_post_startup(cmi_hdl_t hdl)
 451 {
 452         cms_t *cms = HDL2CMS(hdl);
 453
 454         if (CMS_OP_PRESENT(cms, cms_post_startup))
 455                 CMS_OPS(cms)->cms_post_startup(hdl);
 456 }
 457
 458 void
 459 cms_post_mpstartup(cmi_hdl_t hdl)
 460 {
 461         cms_t *cms = HDL2CMS(hdl);
 462
 463         if (CMS_OP_PRESENT(cms, cms_post_mpstartup))
 464                 CMS_OPS(cms)->cms_post_mpstartup(hdl);
 465 }
 466
 467 size_t
 468 cms_logout_size(cmi_hdl_t hdl)
 469 {
 470         cms_t *cms = HDL2CMS(hdl);
 471
 472         if (!CMS_OP_PRESENT(cms, cms_logout_size))
 473                 return (0);
 474
 475         return (CMS_OPS(cms)->cms_logout_size(hdl));
 476 }
 477
 478 uint64_t
 479 cms_mcgctl_val(cmi_hdl_t hdl, int nbanks, uint64_t def)
 480 {
 481         cms_t *cms = HDL2CMS(hdl);
 482
 483         if (!CMS_OP_PRESENT(cms, cms_mcgctl_val))
 484                 return (def);
 485
 486         return (CMS_OPS(cms)->cms_mcgctl_val(hdl, nbanks, def));
 487 }
 488
 489 boolean_t
 490 cms_bankctl_skipinit(cmi_hdl_t hdl, int banknum)
 491 {
 492         cms_t *cms = HDL2CMS(hdl);
 493
 494         if (!CMS_OP_PRESENT(cms, cms_bankctl_skipinit))
 495                 return (B_FALSE);
 496
 497         return (CMS_OPS(cms)->cms_bankctl_skipinit(hdl, banknum));
 498 }
 499
 500 uint64_t
 501 cms_bankctl_val(cmi_hdl_t hdl, int banknum, uint64_t def)
 502 {
 503         cms_t *cms = HDL2CMS(hdl);
 504
 505         if (!CMS_OP_PRESENT(cms, cms_bankctl_val))
 506                 return (def);
 507
 508         return (CMS_OPS(cms)->cms_bankctl_val(hdl, banknum, def));
 509 }
 510
 511 boolean_t
 512 cms_bankstatus_skipinit(cmi_hdl_t hdl, int banknum)
 513 {
 514         cms_t *cms = HDL2CMS(hdl);
 515
 516         if (!CMS_OP_PRESENT(cms, cms_bankstatus_skipinit))
 517                 return (B_FALSE);
 518
 519         return (CMS_OPS(cms)->cms_bankstatus_skipinit(hdl, banknum));
 520 }
 521
 522 uint64_t
 523 cms_bankstatus_val(cmi_hdl_t hdl, int banknum, uint64_t def)
 524 {
 525         cms_t *cms = HDL2CMS(hdl);
 526
 527         if (!CMS_OP_PRESENT(cms, cms_bankstatus_val))
 528                 return (def);
 529
 530         return (CMS_OPS(cms)->cms_bankstatus_val(hdl, banknum, def));
 531 }
 532
 533 void
 534 cms_mca_init(cmi_hdl_t hdl, int nbanks)
 535 {
 536         cms_t *cms = HDL2CMS(hdl);
 537
 538         if (CMS_OP_PRESENT(cms, cms_mca_init))
 539                 CMS_OPS(cms)->cms_mca_init(hdl, nbanks);
 540 }
 541
 542 uint64_t
 543 cms_poll_ownermask(cmi_hdl_t hdl, hrtime_t poll_interval)
 544 {
 545         cms_t *cms = HDL2CMS(hdl);
 546
 547         if (CMS_OP_PRESENT(cms, cms_poll_ownermask))
 548                 return (CMS_OPS(cms)->cms_poll_ownermask(hdl, poll_interval));
 549         else
 550                 return (-1ULL);         /* poll all banks by default */
 551 }
 552
 553 void
 554 cms_bank_logout(cmi_hdl_t hdl, int banknum, uint64_t status, uint64_t addr,
 555     uint64_t misc, void *mslogout)
 556 {
 557         cms_t *cms = HDL2CMS(hdl);
 558
 559         if (mslogout != NULL && CMS_OP_PRESENT(cms, cms_bank_logout))
 560                 CMS_OPS(cms)->cms_bank_logout(hdl, banknum, status, addr,
 561                     misc, mslogout);
 562 }
 563
 564 cms_errno_t
 565 cms_msrinject(cmi_hdl_t hdl, uint_t msr, uint64_t val)
 566 {
 567         cms_t *cms = HDL2CMS(hdl);
 568
 569         if (CMS_OP_PRESENT(cms, cms_msrinject))
 570                 return (CMS_OPS(cms)->cms_msrinject(hdl, msr, val));
 571         else
 572                 return (CMSERR_NOTSUP);
 573 }
 574
 575 uint32_t
 576 cms_error_action(cmi_hdl_t hdl, int ismc, int banknum, uint64_t status,
 577     uint64_t addr, uint64_t misc, void *mslogout)
 578 {
 579         cms_t *cms = HDL2CMS(hdl);
 580
 581         if (CMS_OP_PRESENT(cms, cms_error_action))
 582                 return (CMS_OPS(cms)->cms_error_action(hdl, ismc, banknum,
 583                     status, addr, misc, mslogout));
 584         else
 585                 return (0);
 586 }
 587
 588 cms_cookie_t
 589 cms_disp_match(cmi_hdl_t hdl, int ismc, int banknum, uint64_t status,
 590     uint64_t addr, uint64_t misc, void *mslogout)
 591 {
 592         cms_t *cms = HDL2CMS(hdl);
 593
 594         if (CMS_OP_PRESENT(cms, cms_disp_match))
 595                 return (CMS_OPS(cms)->cms_disp_match(hdl, ismc, banknum,
 596                     status, addr, misc, mslogout));
 597         else
 598                 return (NULL);
 599
 600 }
 601
 602 void
 603 cms_ereport_class(cmi_hdl_t hdl, cms_cookie_t mscookie, const char **cpuclsp,
 604     const char **leafclsp)
 605 {
 606         cms_t *cms = HDL2CMS(hdl);
 607
 608         if (cpuclsp == NULL || leafclsp == NULL)
 609                 return;
 610
 611         *cpuclsp = *leafclsp = NULL;
 612         if (CMS_OP_PRESENT(cms, cms_ereport_class)) {
 613                 CMS_OPS(cms)->cms_ereport_class(hdl, mscookie, cpuclsp,
 614                     leafclsp);
 615         }
 616 }
 617
 618 nvlist_t *
 619 cms_ereport_detector(cmi_hdl_t hdl, int bankno, cms_cookie_t mscookie,
 620     nv_alloc_t *nva)
 621 {
 622         cms_t *cms = HDL2CMS(hdl);
 623
 624         if (CMS_OP_PRESENT(cms, cms_ereport_detector))
 625                 return (CMS_OPS(cms)->cms_ereport_detector(hdl, bankno,
 626                     mscookie, nva));
 627         else
 628                 return (NULL);
 629
 630 }
 631
 632 boolean_t
 633 cms_ereport_includestack(cmi_hdl_t hdl, cms_cookie_t mscookie)
 634 {
 635         cms_t *cms = HDL2CMS(hdl);
 636
 637         if (CMS_OP_PRESENT(cms, cms_ereport_includestack)) {
 638                 return (CMS_OPS(cms)->cms_ereport_includestack(hdl, mscookie));
 639         } else {
 640                 return (B_FALSE);
 641         }
 642 }
 643
 644 void
 645 cms_ereport_add_logout(cmi_hdl_t hdl, nvlist_t *nvl, nv_alloc_t *nva,
 646     int banknum, uint64_t status, uint64_t addr, uint64_t misc, void *mslogout,
 647     cms_cookie_t mscookie)
 648 {
 649         cms_t *cms = HDL2CMS(hdl);
 650
 651         if (CMS_OP_PRESENT(cms, cms_ereport_add_logout))
 652                 CMS_OPS(cms)->cms_ereport_add_logout(hdl, nvl, nva, banknum,
 653                     status, addr, misc, mslogout, mscookie);
 654
 655 }