usr/src/uts/common/krtld/kobj.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 /*
  26  * Copyright 2011 Bayard G. Bell <buffer.g.overflow@gmail.com>.
  27  * All rights reserved. Use is subject to license terms.
  28  */
  29
  30 /*
  31  * Kernel's linker/loader
  32  */
  33
  34 #include <sys/types.h>
  35 #include <sys/param.h>
  36 #include <sys/sysmacros.h>
  37 #include <sys/systm.h>
  38 #include <sys/user.h>
  39 #include <sys/kmem.h>
  40 #include <sys/reboot.h>
  41 #include <sys/bootconf.h>
  42 #include <sys/debug.h>
  43 #include <sys/uio.h>
  44 #include <sys/file.h>
  45 #include <sys/vnode.h>
  46 #include <sys/user.h>
  47 #include <sys/mman.h>
  48 #include <vm/as.h>
  49 #include <vm/seg_kp.h>
  50 #include <vm/seg_kmem.h>
  51 #include <sys/elf.h>
  52 #include <sys/elf_notes.h>
  53 #include <sys/vmsystm.h>
  54 #include <sys/kdi.h>
  55 #include <sys/atomic.h>
  56 #include <sys/kmdb.h>
  57
  58 #include <sys/link.h>
  59 #include <sys/kobj.h>
  60 #include <sys/ksyms.h>
  61 #include <sys/disp.h>
  62 #include <sys/modctl.h>
  63 #include <sys/varargs.h>
  64 #include <sys/kstat.h>
  65 #include <sys/kobj_impl.h>
  66 #include <sys/fs/decomp.h>
  67 #include <sys/callb.h>
  68 #include <sys/cmn_err.h>
  69 #include <sys/tnf_probe.h>
  70 #include <sys/zmod.h>
  71
  72 #include <krtld/reloc.h>
  73 #include <krtld/kobj_kdi.h>
  74 #include <sys/sha1.h>
  75 #include <sys/crypto/elfsign.h>
  76
  77 #if !defined(_OBP)
  78 #include <sys/bootvfs.h>
  79 #endif
  80
  81 /*
  82  * do_symbols() error codes
  83  */
  84 #define DOSYM_UNDEF             -1      /* undefined symbol */
  85 #define DOSYM_UNSAFE            -2      /* MT-unsafe driver symbol */
  86
  87 #if !defined(_OBP)
  88 static void synthetic_bootaux(char *, val_t *);
  89 #endif
  90
  91 static struct module *load_exec(val_t *, char *);
  92 static void load_linker(val_t *);
  93 static struct modctl *add_primary(const char *filename, int);
  94 static int bind_primary(val_t *, int);
  95 static int load_primary(struct module *, int);
  96 static int load_kmdb(val_t *);
  97 static int get_progbits(struct module *, struct _buf *);
  98 static int get_syms(struct module *, struct _buf *);
  99 static int get_ctf(struct module *, struct _buf *);
 100 static void get_signature(struct module *, struct _buf *);
 101 static int do_common(struct module *);
 102 static void add_dependent(struct module *, struct module *);
 103 static int do_dependents(struct modctl *, char *, size_t);
 104 static int do_symbols(struct module *, Elf64_Addr);
 105 static void module_assign(struct modctl *, struct module *);
 106 static void free_module_data(struct module *);
 107 static char *depends_on(struct module *);
 108 static char *getmodpath(const char *);
 109 static char *basename(char *);
 110 static void attr_val(val_t *);
 111 static char *find_libmacro(char *);
 112 static char *expand_libmacro(char *, char *, char *);
 113 static int read_bootflags(void);
 114 static int kobj_comp_setup(struct _buf *, struct compinfo *);
 115 static int kobj_uncomp_blk(struct _buf *, caddr_t, uint_t);
 116 static int kobj_read_blks(struct _buf *, caddr_t, uint_t, uint_t);
 117 static int kobj_boot_open(char *, int);
 118 static int kobj_boot_close(int);
 119 static int kobj_boot_seek(int, off_t, off_t);
 120 static int kobj_boot_read(int, caddr_t, size_t);
 121 static int kobj_boot_fstat(int, struct bootstat *);
 122 static int kobj_boot_compinfo(int, struct compinfo *);
 123
 124 static Sym *lookup_one(struct module *, const char *);
 125 static void sym_insert(struct module *, char *, symid_t);
 126 static Sym *sym_lookup(struct module *, Sym *);
 127
 128 static struct kobjopen_tctl *kobjopen_alloc(char *filename);
 129 static void kobjopen_free(struct kobjopen_tctl *ltp);
 130 static void kobjopen_thread(struct kobjopen_tctl *ltp);
 131 static int kobj_is_compressed(intptr_t);
 132
 133 extern int kcopy(const void *, void *, size_t);
 134 extern int elf_mach_ok(Ehdr *);
 135 extern int alloc_gottable(struct module *, caddr_t *, caddr_t *);
 136
 137 #if !defined(_OBP)
 138 extern int kobj_boot_mountroot(void);
 139 #endif
 140
 141 static void tnf_unsplice_probes(uint_t, struct modctl *);
 142 extern tnf_probe_control_t *__tnf_probe_list_head;
 143 extern tnf_tag_data_t *__tnf_tag_list_head;
 144
 145 extern int modrootloaded;
 146 extern int swaploaded;
 147 extern int bop_io_quiesced;
 148 extern int last_module_id;
 149
 150 extern char stubs_base[];
 151 extern char stubs_end[];
 152
 153 #ifdef KOBJ_DEBUG
 154 /*
 155  * Values that can be or'd in to kobj_debug and their effects:
 156  *
 157  *      D_DEBUG         - misc. debugging information.
 158  *      D_SYMBOLS       - list symbols and their values as they are entered
 159  *                        into the hash table
 160  *      D_RELOCATIONS   - display relocation processing information
 161  *      D_LOADING       - display information about each module as it
 162  *                        is loaded.
 163  */
 164 int kobj_debug = 0;
 165
 166 #define KOBJ_MARK(s)    if (kobj_debug & D_DEBUG)       \
 167         (_kobj_printf(ops, "%d", __LINE__), _kobj_printf(ops, ": %s\n", s))
 168 #else
 169 #define KOBJ_MARK(s)    /* discard */
 170 #endif
 171
 172 #define MODPATH_PROPNAME        "module-path"
 173
 174 #ifdef MODDIR_SUFFIX
 175 static char slash_moddir_suffix_slash[] = MODDIR_SUFFIX "/";
 176 #else
 177 #define slash_moddir_suffix_slash       ""
 178 #endif
 179
 180 #define _moddebug       get_weakish_int(&moddebug)
 181 #define _modrootloaded  get_weakish_int(&modrootloaded)
 182 #define _swaploaded     get_weakish_int(&swaploaded)
 183 #define _ioquiesced     get_weakish_int(&bop_io_quiesced)
 184
 185 #define mod(X)          (struct module *)((X)->modl_modp->mod_mp)
 186
 187 void    *romp;          /* rom vector (opaque to us) */
 188 struct bootops *ops;    /* bootops vector */
 189 void *dbvec;            /* debug vector */
 190
 191 /*
 192  * kobjopen thread control structure
 193  */
 194 struct kobjopen_tctl {
 195         ksema_t         sema;
 196         char            *name;          /* name of file */
 197         struct vnode    *vp;            /* vnode return from vn_open() */
 198         int             Errno;          /* error return from vnopen    */
 199 };
 200
 201 /*
 202  * Structure for defining dynamically expandable library macros
 203  */
 204
 205 struct lib_macro_info {
 206         char    *lmi_list;              /* ptr to list of possible choices */
 207         char    *lmi_macroname;         /* pointer to macro name */
 208         ushort_t lmi_ba_index;          /* index into bootaux vector */
 209         ushort_t lmi_macrolen;          /* macro length */
 210 } libmacros[] = {
 211         { NULL, "CPU", BA_CPU, 0 },
 212         { NULL, "MMU", BA_MMU, 0 }
 213 };
 214
 215 #define NLIBMACROS      sizeof (libmacros) / sizeof (struct lib_macro_info)
 216
 217 char *boot_cpu_compatible_list;                 /* make $CPU available */
 218
 219 char *kobj_module_path;                         /* module search path */
 220 vmem_t  *text_arena;                            /* module text arena */
 221 static vmem_t *data_arena;                      /* module data & bss arena */
 222 static vmem_t *ctf_arena;                       /* CTF debug data arena */
 223 static struct modctl *kobj_modules = NULL;      /* modules loaded */
 224 int kobj_mmu_pagesize;                          /* system pagesize */
 225 static int lg_pagesize;                         /* "large" pagesize */
 226 static int kobj_last_module_id = 0;             /* id assignment */
 227 static kmutex_t kobj_lock;                      /* protects mach memory list */
 228
 229 /*
 230  * The following functions have been implemented by the kernel.
 231  * However, many 3rd party drivers provide their own implementations
 232  * of these functions.  When such drivers are loaded, messages
 233  * indicating that these symbols have been multiply defined will be
 234  * emitted to the console.  To avoid alarming customers for no good
 235  * reason, we simply suppress such warnings for the following set of
 236  * functions.
 237  */
 238 static char *suppress_sym_list[] =
 239 {
 240         "strstr",
 241         "strncat",
 242         "strlcat",
 243         "strlcpy",
 244         "strspn",
 245         "memcpy",
 246         "memset",
 247         "memmove",
 248         "memcmp",
 249         "memchr",
 250         "__udivdi3",
 251         "__divdi3",
 252         "__umoddi3",
 253         "__moddi3",
 254         NULL            /* This entry must exist */
 255 };
 256
 257 /* indexed by KOBJ_NOTIFY_* */
 258 static kobj_notify_list_t *kobj_notifiers[KOBJ_NOTIFY_MAX + 1];
 259
 260 /*
 261  * TNF probe management globals
 262  */
 263 tnf_probe_control_t     *__tnf_probe_list_head = NULL;
 264 tnf_tag_data_t          *__tnf_tag_list_head = NULL;
 265 int                     tnf_changed_probe_list = 0;
 266
 267 /*
 268  * Prefix for statically defined tracing (SDT) DTrace probes.
 269  */
 270 const char              *sdt_prefix = "__dtrace_probe_";
 271
 272 /*
 273  * Beginning and end of the kernel's dynamic text/data segments.
 274  */
 275 static caddr_t _text;
 276 static caddr_t _etext;
 277 static caddr_t _data;
 278
 279 /*
 280  * The sparc linker doesn't create a memory location
 281  * for a variable named _edata, so _edata can only be
 282  * referred to, not modified.  krtld needs a static
 283  * variable to modify it - within krtld, of course -
 284  * outside of krtld, e_data is used in all kernels.
 285  */
 286 #if defined(__sparc)
 287 static caddr_t _edata;
 288 #else
 289 extern caddr_t _edata;
 290 #endif
 291
 292 Addr dynseg = 0;        /* load address of "dynamic" segment */
 293 size_t dynsize;         /* "dynamic" segment size */
 294
 295
 296 int standalone = 1;                     /* an unwholey kernel? */
 297 int use_iflush;                         /* iflush after relocations */
 298
 299 /*
 300  * _kobj_printf()
 301  *
 302  * Common printf function pointer. Can handle only one conversion
 303  * specification in the format string. Some of the functions invoked
 304  * through this function pointer cannot handle more that one conversion
 305  * specification in the format string.
 306  */
 307 void (*_kobj_printf)(void *, const char *, ...);        /* printf routine */
 308
 309 /*
 310  * Standalone function pointers for use within krtld.
 311  * Many platforms implement optimized platmod versions of
 312  * utilities such as bcopy and any such are not yet available
 313  * until the kernel is more completely stitched together.
 314  * See kobj_impl.h
 315  */
 316 void (*kobj_bcopy)(const void *, void *, size_t);
 317 void (*kobj_bzero)(void *, size_t);
 318 size_t (*kobj_strlcat)(char *, const char *, size_t);
 319
 320 static kobj_stat_t kobj_stat;
 321
 322 #define MINALIGN        8       /* at least a double-word */
 323
 324 int
 325 get_weakish_int(int *ip)
 326 {
 327         if (standalone)
 328                 return (0);
 329         return (ip == NULL ? 0 : *ip);
 330 }
 331
 332 static void *
 333 get_weakish_pointer(void **ptrp)
 334 {
 335         if (standalone)
 336                 return (0);
 337         return (ptrp == NULL ? 0 : *ptrp);
 338 }
 339
 340 /*
 341  * XXX fix dependencies on "kernel"; this should work
 342  * for other standalone binaries as well.
 343  *
 344  * XXX Fix hashing code to use one pointer to
 345  * hash entries.
 346  *      |----------|
 347  *      | nbuckets |
 348  *      |----------|
 349  *      | nchains  |
 350  *      |----------|
 351  *      | bucket[] |
 352  *      |----------|
 353  *      | chain[]  |
 354  *      |----------|
 355  */
 356
 357 /*
 358  * Load, bind and relocate all modules that
 359  * form the primary kernel. At this point, our
 360  * externals have not been relocated.
 361  */
 362 void
 363 kobj_init(
 364         void *romvec,
 365         void *dvec,
 366         struct bootops *bootvec,
 367         val_t *bootaux)
 368 {
 369         struct module *mp;
 370         struct modctl *modp;
 371         Addr entry;
 372         char filename[MAXPATHLEN];
 373
 374         /*
 375          * Save these to pass on to
 376          * the booted standalone.
 377          */
 378         romp = romvec;
 379         dbvec = dvec;
 380
 381         ops = bootvec;
 382         kobj_setup_standalone_vectors();
 383
 384         KOBJ_MARK("Entered kobj_init()");
 385
 386         (void) BOP_GETPROP(ops, "whoami", filename);
 387
 388         /*
 389          * We don't support standalone debuggers anymore.  The use of kadb
 390          * will interfere with the later use of kmdb.  Let the user mend
 391          * their ways now.  Users will reach this message if they still
 392          * have the kadb binary on their system (perhaps they used an old
 393          * bfu, or maybe they intentionally copied it there) and have
 394          * specified its use in a way that eluded our checking in the boot
 395          * program.
 396          */
 397         if (dvec != NULL) {
 398                 _kobj_printf(ops, "\nWARNING: Standalone debuggers such as "
 399                     "kadb are no longer supported\n\n");
 400                 goto fail;
 401         }
 402
 403 #if defined(_OBP)
 404         /*
 405          * OBP allows us to read both the ramdisk and
 406          * the underlying root fs when root is a disk.
 407          * This can lower incidences of unbootable systems
 408          * when the archive is out-of-date with the /etc
 409          * state files.
 410          */
 411         if (BOP_MOUNTROOT() != BOOT_SVC_OK) {
 412                 _kobj_printf(ops, "can't mount boot fs\n");
 413                 goto fail;
 414         }
 415 #else
 416         {
 417                 /* on x86, we always boot with a ramdisk */
 418                 (void) kobj_boot_mountroot();
 419
 420                 /*
 421                  * Now that the ramdisk is mounted, finish boot property
 422                  * initialization.
 423                  */
 424                 boot_prop_finish();
 425         }
 426
 427 #if !defined(_UNIX_KRTLD)
 428         /*
 429          * 'unix' is linked together with 'krtld' into one executable and
 430          * the early boot code does -not- hand us any of the dynamic metadata
 431          * about the executable. In particular, it does not read in, map or
 432          * otherwise look at the program headers. We fake all that up now.
 433          *
 434          * We do this early as DTrace static probes and tnf probes both call
 435          * undefined references.  We have to process those relocations before
 436          * calling any of them.
 437          *
 438          * OBP tells kobj_start() where the ELF image is in memory, so it
 439          * synthesized bootaux before kobj_init() was called
 440          */
 441         if (bootaux[BA_PHDR].ba_ptr == NULL)
 442                 synthetic_bootaux(filename, bootaux);
 443
 444 #endif  /* !_UNIX_KRTLD */
 445 #endif  /* _OBP */
 446
 447         /*
 448          * Save the interesting attribute-values
 449          * (scanned by kobj_boot).
 450          */
 451         attr_val(bootaux);
 452
 453         /*
 454          * Set the module search path.
 455          */
 456         kobj_module_path = getmodpath(filename);
 457
 458         boot_cpu_compatible_list = find_libmacro("CPU");
 459
 460         /*
 461          * These two modules have actually been
 462          * loaded by boot, but we finish the job
 463          * by introducing them into the world of
 464          * loadable modules.
 465          */
 466
 467         mp = load_exec(bootaux, filename);
 468         load_linker(bootaux);
 469
 470         /*
 471          * Load all the primary dependent modules.
 472          */
 473         if (load_primary(mp, KOBJ_LM_PRIMARY) == -1)
 474                 goto fail;
 475
 476         /*
 477          * Glue it together.
 478          */
 479         if (bind_primary(bootaux, KOBJ_LM_PRIMARY) == -1)
 480                 goto fail;
 481
 482         entry = bootaux[BA_ENTRY].ba_val;
 483
 484         /*
 485          * Get the boot flags
 486          */
 487         bootflags(ops);
 488
 489         if (boothowto & RB_VERBOSE)
 490                 kobj_lm_dump(KOBJ_LM_PRIMARY);
 491
 492         kobj_kdi_init();
 493
 494         if (boothowto & RB_KMDB) {
 495                 if (load_kmdb(bootaux) < 0)
 496                         goto fail;
 497         }
 498
 499         /*
 500          * Post setup.
 501          */
 502         s_text = _text;
 503         e_text = _etext;
 504         s_data = _data;
 505         e_data = _edata;
 506
 507         kobj_sync_instruction_memory(s_text, e_text - s_text);
 508
 509 #ifdef  KOBJ_DEBUG
 510         if (kobj_debug & D_DEBUG)
 511                 _kobj_printf(ops,
 512                     "krtld: transferring control to: 0x%p\n", entry);
 513 #endif
 514
 515         /*
 516          * Make sure the mod system knows about the modules already loaded.
 517          */
 518         last_module_id = kobj_last_module_id;
 519         bcopy(kobj_modules, &modules, sizeof (modules));
 520         modp = &modules;
 521         do {
 522                 if (modp->mod_next == kobj_modules)
 523                         modp->mod_next = &modules;
 524                 if (modp->mod_prev == kobj_modules)
 525                         modp->mod_prev = &modules;
 526         } while ((modp = modp->mod_next) != &modules);
 527
 528         standalone = 0;
 529
 530 #ifdef  KOBJ_DEBUG
 531         if (kobj_debug & D_DEBUG)
 532                 _kobj_printf(ops,
 533                     "krtld: really transferring control to: 0x%p\n", entry);
 534 #endif
 535
 536         /* restore printf/bcopy/bzero vectors before returning */
 537         kobj_restore_vectors();
 538
 539 #if defined(_DBOOT)
 540         /*
 541          * krtld was called from a dboot ELF section, the embedded
 542          * dboot code contains the real entry via bootaux
 543          */
 544         exitto((caddr_t)entry);
 545 #else
 546         /*
 547          * krtld was directly called from startup
 548          */
 549         return;
 550 #endif
 551
 552 fail:
 553
 554         _kobj_printf(ops, "krtld: error during initial load/link phase\n");
 555
 556 #if !defined(_UNIX_KRTLD)
 557         _kobj_printf(ops, "\n");
 558         _kobj_printf(ops, "krtld could neither locate nor resolve symbols"
 559             " for:\n");
 560         _kobj_printf(ops, "    %s\n", filename);
 561         _kobj_printf(ops, "in the boot archive. Please verify that this"
 562             " file\n");
 563         _kobj_printf(ops, "matches what is found in the boot archive.\n");
 564         _kobj_printf(ops, "You may need to boot using the Solaris failsafe to"
 565             " fix this.\n");
 566         bop_panic("Unable to boot");
 567 #endif
 568 }
 569
 570 #if !defined(_UNIX_KRTLD) && !defined(_OBP)
 571 /*
 572  * Synthesize additional metadata that describes the executable if
 573  * krtld's caller didn't do it.
 574  *
 575  * (When the dynamic executable has an interpreter, the boot program
 576  * does all this for us.  Where we don't have an interpreter, (or a
 577  * even a boot program, perhaps) we have to do this for ourselves.)
 578  */
 579 static void
 580 synthetic_bootaux(char *filename, val_t *bootaux)
 581 {
 582         Ehdr ehdr;
 583         caddr_t phdrbase;
 584         struct _buf *file;
 585         int i, n;
 586
 587         /*
 588          * Elf header
 589          */
 590         KOBJ_MARK("synthetic_bootaux()");
 591         KOBJ_MARK(filename);
 592         file = kobj_open_file(filename);
 593         if (file == (struct _buf *)-1) {
 594                 _kobj_printf(ops, "krtld: failed to open '%s'\n", filename);
 595                 return;
 596         }
 597         KOBJ_MARK("reading program headers");
 598         if (kobj_read_file(file, (char *)&ehdr, sizeof (ehdr), 0) < 0) {
 599                 _kobj_printf(ops, "krtld: %s: failed to read ehder\n",
 600                     filename);
 601                 return;
 602         }
 603
 604         /*
 605          * Program headers
 606          */
 607         bootaux[BA_PHNUM].ba_val = ehdr.e_phnum;
 608         bootaux[BA_PHENT].ba_val = ehdr.e_phentsize;
 609         n = ehdr.e_phentsize * ehdr.e_phnum;
 610
 611         phdrbase = kobj_alloc(n, KM_WAIT | KM_TMP);
 612
 613         if (kobj_read_file(file, phdrbase, n, ehdr.e_phoff) < 0) {
 614                 _kobj_printf(ops, "krtld: %s: failed to read phdrs\n",
 615                     filename);
 616                 return;
 617         }
 618         bootaux[BA_PHDR].ba_ptr = phdrbase;
 619         kobj_close_file(file);
 620         KOBJ_MARK("closed file");
 621
 622         /*
 623          * Find the dynamic section address
 624          */
 625         for (i = 0; i < ehdr.e_phnum; i++) {
 626                 Phdr *phdr = (Phdr *)(phdrbase + ehdr.e_phentsize * i);
 627
 628                 if (phdr->p_type == PT_DYNAMIC) {
 629                         bootaux[BA_DYNAMIC].ba_ptr = (void *)phdr->p_vaddr;
 630                         break;
 631                 }
 632         }
 633         KOBJ_MARK("synthetic_bootaux() done");
 634 }
 635 #endif  /* !_UNIX_KRTLD && !_OBP */
 636
 637 /*
 638  * Set up any global information derived
 639  * from attribute/values in the boot or
 640  * aux vector.
 641  */
 642 static void
 643 attr_val(val_t *bootaux)
 644 {
 645         Phdr *phdr;
 646         int phnum, phsize;
 647         int i;
 648
 649         KOBJ_MARK("attr_val()");
 650         kobj_mmu_pagesize = bootaux[BA_PAGESZ].ba_val;
 651         lg_pagesize = bootaux[BA_LPAGESZ].ba_val;
 652         use_iflush = bootaux[BA_IFLUSH].ba_val;
 653
 654         phdr = (Phdr *)bootaux[BA_PHDR].ba_ptr;
 655         phnum = bootaux[BA_PHNUM].ba_val;
 656         phsize = bootaux[BA_PHENT].ba_val;
 657         for (i = 0; i < phnum; i++) {
 658                 phdr = (Phdr *)(bootaux[BA_PHDR].ba_val + i * phsize);
 659
 660                 if (phdr->p_type != PT_LOAD) {
 661                         continue;
 662                 }
 663                 /*
 664                  * Bounds of the various segments.
 665                  */
 666                 if (!(phdr->p_flags & PF_X)) {
 667 #if defined(_RELSEG)
 668                         /*
 669                          * sparc kernel puts the dynamic info
 670                          * into a separate segment, which is
 671                          * free'd in bop_fini()
 672                          */
 673                         ASSERT(phdr->p_vaddr != 0);
 674                         dynseg = phdr->p_vaddr;
 675                         dynsize = phdr->p_memsz;
 676 #else
 677                         ASSERT(phdr->p_vaddr == 0);
 678 #endif
 679                 } else {
 680                         if (phdr->p_flags & PF_W) {
 681                                 _data = (caddr_t)phdr->p_vaddr;
 682                                 _edata = _data + phdr->p_memsz;
 683                         } else {
 684                                 _text = (caddr_t)phdr->p_vaddr;
 685                                 _etext = _text + phdr->p_memsz;
 686                         }
 687                 }
 688         }
 689
 690         /* To do the kobj_alloc, _edata needs to be set. */
 691         for (i = 0; i < NLIBMACROS; i++) {
 692                 if (bootaux[libmacros[i].lmi_ba_index].ba_ptr != NULL) {
 693                         libmacros[i].lmi_list = kobj_alloc(
 694                             strlen(bootaux[libmacros[i].lmi_ba_index].ba_ptr) +
 695                             1, KM_WAIT);
 696                         (void) strcpy(libmacros[i].lmi_list,
 697                             bootaux[libmacros[i].lmi_ba_index].ba_ptr);
 698                 }
 699                 libmacros[i].lmi_macrolen = strlen(libmacros[i].lmi_macroname);
 700         }
 701 }
 702
 703 /*
 704  * Set up the booted executable.
 705  */
 706 static struct module *
 707 load_exec(val_t *bootaux, char *filename)
 708 {
 709         struct modctl *cp;
 710         struct module *mp;
 711         Dyn *dyn;
 712         Sym *sp;
 713         int i, lsize, osize, nsize, allocsize;
 714         char *libname, *tmp;
 715         char path[MAXPATHLEN];
 716
 717 #ifdef KOBJ_DEBUG
 718         if (kobj_debug & D_DEBUG)
 719                 _kobj_printf(ops, "module path '%s'\n", kobj_module_path);
 720 #endif
 721
 722         KOBJ_MARK("add_primary");
 723         cp = add_primary(filename, KOBJ_LM_PRIMARY);
 724
 725         KOBJ_MARK("struct module");
 726         mp = kobj_zalloc(sizeof (struct module), KM_WAIT);
 727         cp->mod_mp = mp;
 728
 729         /*
 730          * We don't have the following information
 731          * since this module is an executable and not
 732          * a relocatable .o.
 733          */
 734         mp->symtbl_section = 0;
 735         mp->shdrs = NULL;
 736         mp->strhdr = NULL;
 737
 738         /*
 739          * Since this module is the only exception,
 740          * we cons up some section headers.
 741          */
 742         KOBJ_MARK("symhdr");
 743         mp->symhdr = kobj_zalloc(sizeof (Shdr), KM_WAIT);
 744
 745         KOBJ_MARK("strhdr");
 746         mp->strhdr = kobj_zalloc(sizeof (Shdr), KM_WAIT);
 747
 748         mp->symhdr->sh_type = SHT_SYMTAB;
 749         mp->strhdr->sh_type = SHT_STRTAB;
 750         /*
 751          * Scan the dynamic structure.
 752          */
 753         for (dyn = (Dyn *) bootaux[BA_DYNAMIC].ba_ptr;
 754             dyn->d_tag != DT_NULL; dyn++) {
 755                 switch (dyn->d_tag) {
 756                 case DT_SYMTAB:
 757                         mp->symspace = mp->symtbl = (char *)dyn->d_un.d_ptr;
 758                         mp->symhdr->sh_addr = dyn->d_un.d_ptr;
 759                         break;
 760                 case DT_HASH:
 761                         mp->nsyms = *((uint_t *)dyn->d_un.d_ptr + 1);
 762                         mp->hashsize = *(uint_t *)dyn->d_un.d_ptr;
 763                         break;
 764                 case DT_STRTAB:
 765                         mp->strings = (char *)dyn->d_un.d_ptr;
 766                         mp->strhdr->sh_addr = dyn->d_un.d_ptr;
 767                         break;
 768                 case DT_STRSZ:
 769                         mp->strhdr->sh_size = dyn->d_un.d_val;
 770                         break;
 771                 case DT_SYMENT:
 772                         mp->symhdr->sh_entsize = dyn->d_un.d_val;
 773                         break;
 774                 }
 775         }
 776
 777         /*
 778          * Collapse any DT_NEEDED entries into one string.
 779          */
 780         nsize = osize = 0;
 781         allocsize = MAXPATHLEN;
 782
 783         KOBJ_MARK("depends_on");
 784         mp->depends_on = kobj_alloc(allocsize, KM_WAIT);
 785
 786         for (dyn = (Dyn *) bootaux[BA_DYNAMIC].ba_ptr;
 787             dyn->d_tag != DT_NULL; dyn++)
 788                 if (dyn->d_tag == DT_NEEDED) {
 789                         char *_lib;
 790
 791                         libname = mp->strings + dyn->d_un.d_val;
 792                         if (strchr(libname, '$') != NULL) {
 793                                 if ((_lib = expand_libmacro(libname,
 794                                     path, path)) != NULL)
 795                                         libname = _lib;
 796                                 else
 797                                         _kobj_printf(ops, "krtld: "
 798                                             "load_exec: fail to "
 799                                             "expand %s\n", libname);
 800                         }
 801                         lsize = strlen(libname);
 802                         nsize += lsize;
 803                         if (nsize + 1 > allocsize) {
 804                                 KOBJ_MARK("grow depends_on");
 805                                 tmp = kobj_alloc(allocsize + MAXPATHLEN,
 806                                     KM_WAIT);
 807                                 bcopy(mp->depends_on, tmp, osize);
 808                                 kobj_free(mp->depends_on, allocsize);
 809                                 mp->depends_on = tmp;
 810                                 allocsize += MAXPATHLEN;
 811                         }
 812                         bcopy(libname, mp->depends_on + osize, lsize);
 813                         *(mp->depends_on + nsize) = ' '; /* separate */
 814                         nsize++;
 815                         osize = nsize;
 816                 }
 817         if (nsize) {
 818                 mp->depends_on[nsize - 1] = '\0'; /* terminate the string */
 819                 /*
 820                  * alloc with exact size and copy whatever it got over
 821                  */
 822                 KOBJ_MARK("realloc depends_on");
 823                 tmp = kobj_alloc(nsize, KM_WAIT);
 824                 bcopy(mp->depends_on, tmp, nsize);
 825                 kobj_free(mp->depends_on, allocsize);
 826                 mp->depends_on = tmp;
 827         } else {
 828                 kobj_free(mp->depends_on, allocsize);
 829                 mp->depends_on = NULL;
 830         }
 831
 832         mp->flags = KOBJ_EXEC|KOBJ_PRIM;        /* NOT a relocatable .o */
 833         mp->symhdr->sh_size = mp->nsyms * mp->symhdr->sh_entsize;
 834         /*
 835          * We allocate our own table since we don't
 836          * hash undefined references.
 837          */
 838         KOBJ_MARK("chains");
 839         mp->chains = kobj_zalloc(mp->nsyms * sizeof (symid_t), KM_WAIT);
 840         KOBJ_MARK("buckets");
 841         mp->buckets = kobj_zalloc(mp->hashsize * sizeof (symid_t), KM_WAIT);
 842
 843         mp->text = _text;
 844         mp->data = _data;
 845
 846         mp->text_size = _etext - _text;
 847         mp->data_size = _edata - _data;
 848
 849         cp->mod_text = mp->text;
 850         cp->mod_text_size = mp->text_size;
 851
 852         mp->filename = cp->mod_filename;
 853
 854 #ifdef  KOBJ_DEBUG
 855         if (kobj_debug & D_LOADING) {
 856                 _kobj_printf(ops, "krtld: file=%s\n", mp->filename);
 857                 _kobj_printf(ops, "\ttext: 0x%p", mp->text);
 858                 _kobj_printf(ops, " size: 0x%x\n", mp->text_size);
 859                 _kobj_printf(ops, "\tdata: 0x%p", mp->data);
 860                 _kobj_printf(ops, " dsize: 0x%x\n", mp->data_size);
 861         }
 862 #endif /* KOBJ_DEBUG */
 863
 864         /*
 865          * Insert symbols into the hash table.
 866          */
 867         for (i = 0; i < mp->nsyms; i++) {
 868                 sp = (Sym *)(mp->symtbl + i * mp->symhdr->sh_entsize);
 869
 870                 if (sp->st_name == 0 || sp->st_shndx == SHN_UNDEF)
 871                         continue;
 872 #if defined(__sparc)
 873                 /*
 874                  * Register symbols are ignored in the kernel
 875                  */
 876                 if (ELF_ST_TYPE(sp->st_info) == STT_SPARC_REGISTER)
 877                         continue;
 878 #endif  /* __sparc */
 879
 880                 sym_insert(mp, mp->strings + sp->st_name, i);
 881         }
 882
 883         KOBJ_MARK("load_exec done");
 884         return (mp);
 885 }
 886
 887 /*
 888  * Set up the linker module (if it's compiled in, LDNAME is NULL)
 889  */
 890 static void
 891 load_linker(val_t *bootaux)
 892 {
 893         struct module *kmp = (struct module *)kobj_modules->mod_mp;
 894         struct module *mp;
 895         struct modctl *cp;
 896         int i;
 897         Shdr *shp;
 898         Sym *sp;
 899         int shsize;
 900         char *dlname = (char *)bootaux[BA_LDNAME].ba_ptr;
 901
 902         /*
 903          * On some architectures, krtld is compiled into the kernel.
 904          */
 905         if (dlname == NULL)
 906                 return;
 907
 908         cp = add_primary(dlname, KOBJ_LM_PRIMARY);
 909
 910         mp = kobj_zalloc(sizeof (struct module), KM_WAIT);
 911
 912         cp->mod_mp = mp;
 913         mp->hdr = *(Ehdr *)bootaux[BA_LDELF].ba_ptr;
 914         shsize = mp->hdr.e_shentsize * mp->hdr.e_shnum;
 915         mp->shdrs = kobj_alloc(shsize, KM_WAIT);
 916         bcopy(bootaux[BA_LDSHDR].ba_ptr, mp->shdrs, shsize);
 917
 918         for (i = 1; i < (int)mp->hdr.e_shnum; i++) {
 919                 shp = (Shdr *)(mp->shdrs + (i * mp->hdr.e_shentsize));
 920
 921                 if (shp->sh_flags & SHF_ALLOC) {
 922                         if (shp->sh_flags & SHF_WRITE) {
 923                                 if (mp->data == NULL)
 924                                         mp->data = (char *)shp->sh_addr;
 925                         } else if (mp->text == NULL) {
 926                                 mp->text = (char *)shp->sh_addr;
 927                         }
 928                 }
 929                 if (shp->sh_type == SHT_SYMTAB) {
 930                         mp->symtbl_section = i;
 931                         mp->symhdr = shp;
 932                         mp->symspace = mp->symtbl = (char *)shp->sh_addr;
 933                 }
 934         }
 935         mp->nsyms = mp->symhdr->sh_size / mp->symhdr->sh_entsize;
 936         mp->flags = KOBJ_INTERP|KOBJ_PRIM;
 937         mp->strhdr = (Shdr *)
 938             (mp->shdrs + mp->symhdr->sh_link * mp->hdr.e_shentsize);
 939         mp->strings = (char *)mp->strhdr->sh_addr;
 940         mp->hashsize = kobj_gethashsize(mp->nsyms);
 941
 942         mp->symsize = mp->symhdr->sh_size + mp->strhdr->sh_size + sizeof (int) +
 943             (mp->hashsize + mp->nsyms) * sizeof (symid_t);
 944
 945         mp->chains = kobj_zalloc(mp->nsyms * sizeof (symid_t), KM_WAIT);
 946         mp->buckets = kobj_zalloc(mp->hashsize * sizeof (symid_t), KM_WAIT);
 947
 948         mp->bss = bootaux[BA_BSS].ba_val;
 949         mp->bss_align = 0;      /* pre-aligned during allocation */
 950         mp->bss_size = (uintptr_t)_edata - mp->bss;
 951         mp->text_size = _etext - mp->text;
 952         mp->data_size = _edata - mp->data;
 953         mp->filename = cp->mod_filename;
 954         cp->mod_text = mp->text;
 955         cp->mod_text_size = mp->text_size;
 956
 957         /*
 958          * Now that we've figured out where the linker is,
 959          * set the limits for the booted object.
 960          */
 961         kmp->text_size = (size_t)(mp->text - kmp->text);
 962         kmp->data_size = (size_t)(mp->data - kmp->data);
 963         kobj_modules->mod_text_size = kmp->text_size;
 964
 965 #ifdef  KOBJ_DEBUG
 966         if (kobj_debug & D_LOADING) {
 967                 _kobj_printf(ops, "krtld: file=%s\n", mp->filename);
 968                 _kobj_printf(ops, "\ttext:0x%p", mp->text);
 969                 _kobj_printf(ops, " size: 0x%x\n", mp->text_size);
 970                 _kobj_printf(ops, "\tdata:0x%p", mp->data);
 971                 _kobj_printf(ops, " dsize: 0x%x\n", mp->data_size);
 972         }
 973 #endif /* KOBJ_DEBUG */
 974
 975         /*
 976          * Insert the symbols into the hash table.
 977          */
 978         for (i = 0; i < mp->nsyms; i++) {
 979                 sp = (Sym *)(mp->symtbl + i * mp->symhdr->sh_entsize);
 980
 981                 if (sp->st_name == 0 || sp->st_shndx == SHN_UNDEF)
 982                         continue;
 983                 if (ELF_ST_BIND(sp->st_info) == STB_GLOBAL) {
 984                         if (sp->st_shndx == SHN_COMMON)
 985                                 sp->st_shndx = SHN_ABS;
 986                 }
 987                 sym_insert(mp, mp->strings + sp->st_name, i);
 988         }
 989
 990 }
 991
 992 static kobj_notify_list_t **
 993 kobj_notify_lookup(uint_t type)
 994 {
 995         ASSERT(type != 0 && type < sizeof (kobj_notifiers) /
 996             sizeof (kobj_notify_list_t *));
 997
 998         return (&kobj_notifiers[type]);
 999 }
1000
1001 int
1002 kobj_notify_add(kobj_notify_list_t *knp)
1003 {
1004         kobj_notify_list_t **knl;
1005
1006         knl = kobj_notify_lookup(knp->kn_type);
1007
1008         knp->kn_next = NULL;
1009         knp->kn_prev = NULL;
1010
1011         mutex_enter(&kobj_lock);
1012
1013         if (*knl != NULL) {
1014                 (*knl)->kn_prev = knp;
1015                 knp->kn_next = *knl;
1016         }
1017         (*knl) = knp;
1018
1019         mutex_exit(&kobj_lock);
1020         return (0);
1021 }
1022
1023 int
1024 kobj_notify_remove(kobj_notify_list_t *knp)
1025 {
1026         kobj_notify_list_t **knl = kobj_notify_lookup(knp->kn_type);
1027         kobj_notify_list_t *tknp;
1028
1029         mutex_enter(&kobj_lock);
1030
1031         /* LINTED */
1032         if (tknp = knp->kn_next)
1033                 tknp->kn_prev = knp->kn_prev;
1034
1035         /* LINTED */
1036         if (tknp = knp->kn_prev)
1037                 tknp->kn_next = knp->kn_next;
1038         else
1039                 *knl = knp->kn_next;
1040
1041         mutex_exit(&kobj_lock);
1042
1043         return (0);
1044 }
1045
1046 /*
1047  * Notify all interested callbacks of a specified change in module state.
1048  */
1049 static void
1050 kobj_notify(int type, struct modctl *modp)
1051 {
1052         kobj_notify_list_t *knp;
1053
1054         if (modp->mod_loadflags & MOD_NONOTIFY || standalone)
1055                 return;
1056
1057         mutex_enter(&kobj_lock);
1058
1059         for (knp = *(kobj_notify_lookup(type)); knp != NULL; knp = knp->kn_next)
1060                 knp->kn_func(type, modp);
1061
1062         /*
1063          * KDI notification must be last (it has to allow for work done by the
1064          * other notification callbacks), so we call it manually.
1065          */
1066         kobj_kdi_mod_notify(type, modp);
1067
1068         mutex_exit(&kobj_lock);
1069 }
1070
1071 /*
1072  * Create the module path.
1073  */
1074 static char *
1075 getmodpath(const char *filename)
1076 {
1077         char *path = kobj_zalloc(MAXPATHLEN, KM_WAIT);
1078
1079         /*
1080          * Platform code gets first crack, then add
1081          * the default components
1082          */
1083         mach_modpath(path, filename);
1084         if (*path != '\0')
1085                 (void) strcat(path, " ");
1086         return (strcat(path, MOD_DEFPATH));
1087 }
1088
1089 static struct modctl *
1090 add_primary(const char *filename, int lmid)
1091 {
1092         struct modctl *cp;
1093
1094         cp = kobj_zalloc(sizeof (struct modctl), KM_WAIT);
1095
1096         cp->mod_filename = kobj_alloc(strlen(filename) + 1, KM_WAIT);
1097
1098         /*
1099          * For symbol lookup, we assemble our own
1100          * modctl list of the primary modules.
1101          */
1102
1103         (void) strcpy(cp->mod_filename, filename);
1104         cp->mod_modname = basename(cp->mod_filename);
1105
1106         /* set values for modinfo assuming that the load will work */
1107         cp->mod_prim = 1;
1108         cp->mod_loaded = 1;
1109         cp->mod_installed = 1;
1110         cp->mod_loadcnt = 1;
1111         cp->mod_loadflags = MOD_NOAUTOUNLOAD;
1112
1113         cp->mod_id = kobj_last_module_id++;
1114
1115         /*
1116          * Link the module in. We'll pass this info on
1117          * to the mod squad later.
1118          */
1119         if (kobj_modules == NULL) {
1120                 kobj_modules = cp;
1121                 cp->mod_prev = cp->mod_next = cp;
1122         } else {
1123                 cp->mod_prev = kobj_modules->mod_prev;
1124                 cp->mod_next = kobj_modules;
1125                 kobj_modules->mod_prev->mod_next = cp;
1126                 kobj_modules->mod_prev = cp;
1127         }
1128
1129         kobj_lm_append(lmid, cp);
1130
1131         return (cp);
1132 }
1133
1134 static int
1135 bind_primary(val_t *bootaux, int lmid)
1136 {
1137         struct modctl_list *linkmap = kobj_lm_lookup(lmid);
1138         struct modctl_list *lp;
1139         struct module *mp;
1140
1141         /*
1142          * Do common symbols.
1143          */
1144         for (lp = linkmap; lp; lp = lp->modl_next) {
1145                 mp = mod(lp);
1146
1147                 /*
1148                  * Don't do common section relocations for modules that
1149                  * don't need it.
1150                  */
1151                 if (mp->flags & (KOBJ_EXEC|KOBJ_INTERP))
1152                         continue;
1153
1154                 if (do_common(mp) < 0)
1155                         return (-1);
1156         }
1157
1158         /*
1159          * Resolve symbols.
1160          */
1161         for (lp = linkmap; lp; lp = lp->modl_next) {
1162                 mp = mod(lp);
1163
1164                 if (do_symbols(mp, 0) < 0)
1165                         return (-1);
1166         }
1167
1168         /*
1169          * Do relocations.
1170          */
1171         for (lp = linkmap; lp; lp = lp->modl_next) {
1172                 mp = mod(lp);
1173
1174                 if (mp->flags & KOBJ_EXEC) {
1175                         Dyn *dyn;
1176                         Word relasz = 0, relaent = 0;
1177                         Word shtype;
1178                         char *rela = NULL;
1179
1180                         for (dyn = (Dyn *)bootaux[BA_DYNAMIC].ba_ptr;
1181                             dyn->d_tag != DT_NULL; dyn++) {
1182                                 switch (dyn->d_tag) {
1183                                 case DT_RELASZ:
1184                                 case DT_RELSZ:
1185                                         relasz = dyn->d_un.d_val;
1186                                         break;
1187                                 case DT_RELAENT:
1188                                 case DT_RELENT:
1189                                         relaent = dyn->d_un.d_val;
1190                                         break;
1191                                 case DT_RELA:
1192                                         shtype = SHT_RELA;
1193                                         rela = (char *)dyn->d_un.d_ptr;
1194                                         break;
1195                                 case DT_REL:
1196                                         shtype = SHT_REL;
1197                                         rela = (char *)dyn->d_un.d_ptr;
1198                                         break;
1199                                 }
1200                         }
1201                         if (relasz == 0 ||
1202                             relaent == 0 || rela == NULL) {
1203                                 _kobj_printf(ops, "krtld: bind_primary(): "
1204                                     "no relocation information found for "
1205                                     "module %s\n", mp->filename);
1206                                 return (-1);
1207                         }
1208 #ifdef  KOBJ_DEBUG
1209                         if (kobj_debug & D_RELOCATIONS)
1210                                 _kobj_printf(ops, "krtld: relocating: file=%s "
1211                                     "KOBJ_EXEC\n", mp->filename);
1212 #endif
1213                         if (do_relocate(mp, rela, shtype, relasz/relaent,
1214                             relaent, (Addr)mp->text) < 0)
1215                                 return (-1);
1216                 } else {
1217                         if (do_relocations(mp) < 0)
1218                                 return (-1);
1219                 }
1220
1221                 kobj_sync_instruction_memory(mp->text, mp->text_size);
1222         }
1223
1224         for (lp = linkmap; lp; lp = lp->modl_next) {
1225                 mp = mod(lp);
1226
1227                 /*
1228                  * We need to re-read the full symbol table for the boot file,
1229                  * since we couldn't use the full one before.  We also need to
1230                  * load the CTF sections of both the boot file and the
1231                  * interpreter (us).
1232                  */
1233                 if (mp->flags & KOBJ_EXEC) {
1234                         struct _buf *file;
1235                         int n;
1236
1237                         file = kobj_open_file(mp->filename);
1238                         if (file == (struct _buf *)-1)
1239                                 return (-1);
1240                         if (kobj_read_file(file, (char *)&mp->hdr,
1241                             sizeof (mp->hdr), 0) < 0)
1242                                 return (-1);
1243                         n = mp->hdr.e_shentsize * mp->hdr.e_shnum;
1244                         mp->shdrs = kobj_alloc(n, KM_WAIT);
1245                         if (kobj_read_file(file, mp->shdrs, n,
1246                             mp->hdr.e_shoff) < 0)
1247                                 return (-1);
1248                         if (get_syms(mp, file) < 0)
1249                                 return (-1);
1250                         if (get_ctf(mp, file) < 0)
1251                                 return (-1);
1252                         kobj_close_file(file);
1253                         mp->flags |= KOBJ_RELOCATED;
1254
1255                 } else if (mp->flags & KOBJ_INTERP) {
1256                         struct _buf *file;
1257
1258                         /*
1259                          * The interpreter path fragment in mp->filename
1260                          * will already have the module directory suffix
1261                          * in it (if appropriate).
1262                          */
1263                         file = kobj_open_path(mp->filename, 1, 0);
1264                         if (file == (struct _buf *)-1)
1265                                 return (-1);
1266                         if (get_ctf(mp, file) < 0)
1267                                 return (-1);
1268                         kobj_close_file(file);
1269                         mp->flags |= KOBJ_RELOCATED;
1270                 }
1271         }
1272
1273         return (0);
1274 }
1275
1276 static struct modctl *
1277 mod_already_loaded(char *modname)
1278 {
1279         struct modctl *mctl = kobj_modules;
1280
1281         do {
1282                 if (strcmp(modname, mctl->mod_filename) == 0)
1283                         return (mctl);
1284                 mctl = mctl->mod_next;
1285
1286         } while (mctl != kobj_modules);
1287
1288         return (NULL);
1289 }
1290
1291 /*
1292  * Load all the primary dependent modules.
1293  */
1294 static int
1295 load_primary(struct module *mp, int lmid)
1296 {
1297         struct modctl *cp;
1298         struct module *dmp;
1299         char *p, *q;
1300         char modname[MODMAXNAMELEN];
1301
1302         if ((p = mp->depends_on) == NULL)
1303                 return (0);
1304
1305         /* CONSTANTCONDITION */
1306         while (1) {
1307                 /*
1308                  * Skip space.
1309                  */
1310                 while (*p && (*p == ' ' || *p == '\t'))
1311                         p++;
1312                 /*
1313                  * Get module name.
1314                  */
1315                 q = modname;
1316                 while (*p && *p != ' ' && *p != '\t')
1317                         *q++ = *p++;
1318
1319                 if (q == modname)
1320                         break;
1321
1322                 *q = '\0';
1323                 /*
1324                  * Check for dup dependencies.
1325                  */
1326                 if (strcmp(modname, "dtracestubs") == 0 ||
1327                     mod_already_loaded(modname) != NULL)
1328                         continue;
1329
1330                 cp = add_primary(modname, lmid);
1331                 cp->mod_busy = 1;
1332                 /*
1333                  * Load it.
1334                  */
1335                 (void) kobj_load_module(cp, 1);
1336                 cp->mod_busy = 0;
1337
1338                 if ((dmp = cp->mod_mp) == NULL) {
1339                         cp->mod_loaded = 0;
1340                         cp->mod_installed = 0;
1341                         cp->mod_loadcnt = 0;
1342                         return (-1);
1343                 }
1344
1345                 add_dependent(mp, dmp);
1346                 dmp->flags |= KOBJ_PRIM;
1347
1348                 /*
1349                  * Recurse.
1350                  */
1351                 if (load_primary(dmp, lmid) == -1) {
1352                         cp->mod_loaded = 0;
1353                         cp->mod_installed = 0;
1354                         cp->mod_loadcnt = 0;
1355                         return (-1);
1356                 }
1357         }
1358         return (0);
1359 }
1360
1361 static int
1362 console_is_usb_serial(void)
1363 {
1364         char *console;
1365         int len, ret;
1366
1367         if ((len = BOP_GETPROPLEN(ops, "console")) == -1)
1368                 return (0);
1369
1370         console = kobj_zalloc(len, KM_WAIT|KM_TMP);
1371         (void) BOP_GETPROP(ops, "console", console);
1372         ret = (strcmp(console, "usb-serial") == 0);
1373         kobj_free(console, len);
1374
1375         return (ret);
1376 }
1377
1378 static int
1379 load_kmdb(val_t *bootaux)
1380 {
1381         struct modctl *mctl;
1382         struct module *mp;
1383         Sym *sym;
1384
1385         if (console_is_usb_serial()) {
1386                 _kobj_printf(ops, "kmdb not loaded "
1387                     "(unsupported on usb serial console)\n");
1388                 return (0);
1389         }
1390
1391         _kobj_printf(ops, "Loading kmdb...\n");
1392
1393         if ((mctl = add_primary("misc/kmdbmod", KOBJ_LM_DEBUGGER)) == NULL)
1394                 return (-1);
1395
1396         mctl->mod_busy = 1;
1397         (void) kobj_load_module(mctl, 1);
1398         mctl->mod_busy = 0;
1399
1400         if ((mp = mctl->mod_mp) == NULL)
1401                 return (-1);
1402
1403         mp->flags |= KOBJ_PRIM;
1404
1405         if (load_primary(mp, KOBJ_LM_DEBUGGER) < 0)
1406                 return (-1);
1407
1408         if (boothowto & RB_VERBOSE)
1409                 kobj_lm_dump(KOBJ_LM_DEBUGGER);
1410
1411         if (bind_primary(bootaux, KOBJ_LM_DEBUGGER) < 0)
1412                 return (-1);
1413
1414         if ((sym = lookup_one(mctl->mod_mp, "kctl_boot_activate")) == NULL)
1415                 return (-1);
1416
1417 #ifdef  KOBJ_DEBUG
1418         if (kobj_debug & D_DEBUG) {
1419                 _kobj_printf(ops, "calling kctl_boot_activate() @ 0x%lx\n",
1420                     sym->st_value);
1421                 _kobj_printf(ops, "\tops 0x%p\n", ops);
1422                 _kobj_printf(ops, "\tromp 0x%p\n", romp);
1423         }
1424 #endif
1425
1426         if (((kctl_boot_activate_f *)sym->st_value)(ops, romp, 0,
1427             (const char **)kobj_kmdb_argv) < 0)
1428                 return (-1);
1429
1430         return (0);
1431 }
1432
1433 /*
1434  * Return a string listing module dependencies.
1435  */
1436 static char *
1437 depends_on(struct module *mp)
1438 {
1439         Sym *sp;
1440         char *depstr, *q;
1441
1442         /*
1443          * The module doesn't have a depends_on value, so let's try it the
1444          * old-fashioned way - via "_depends_on"
1445          */
1446         if ((sp = lookup_one(mp, "_depends_on")) == NULL)
1447                 return (NULL);
1448
1449         q = (char *)sp->st_value;
1450
1451 #ifdef KOBJ_DEBUG
1452         /*
1453          * _depends_on is a deprecated interface, so we warn about its use
1454          * irrespective of subsequent processing errors. How else are we going
1455          * to be able to deco this interface completely?
1456          * Changes initially limited to DEBUG because third-party modules
1457          * should be flagged to developers before general use base.
1458          */
1459         _kobj_printf(ops,
1460             "Warning: %s uses deprecated _depends_on interface.\n",
1461             mp->filename);
1462         _kobj_printf(ops, "Please notify module developer or vendor.\n");
1463 #endif
1464
1465         /*
1466          * Idiot checks. Make sure it's
1467          * in-bounds and NULL terminated.
1468          */
1469         if (kobj_addrcheck(mp, q) || q[sp->st_size - 1] != '\0') {
1470                 _kobj_printf(ops, "Error processing dependency for %s\n",
1471                     mp->filename);
1472                 return (NULL);
1473         }
1474
1475         depstr = (char *)kobj_alloc(strlen(q) + 1, KM_WAIT);
1476         (void) strcpy(depstr, q);
1477
1478         return (depstr);
1479 }
1480
1481 void
1482 kobj_getmodinfo(void *xmp, struct modinfo *modinfo)
1483 {
1484         struct module *mp;
1485         mp = (struct module *)xmp;
1486
1487         modinfo->mi_base = mp->text;
1488         modinfo->mi_size = mp->text_size + mp->data_size;
1489 }
1490
1491 /*
1492  * kobj_export_ksyms() performs the following services:
1493  *
1494  * (1) Migrates the symbol table from boot/kobj memory to the ksyms arena.
1495  * (2) Removes unneeded symbols to save space.
1496  * (3) Reduces memory footprint by using VM_BESTFIT allocations.
1497  * (4) Makes the symbol table visible to /dev/ksyms.
1498  */
1499 static void
1500 kobj_export_ksyms(struct module *mp)
1501 {
1502         Sym *esp = (Sym *)(mp->symtbl + mp->symhdr->sh_size);
1503         Sym *sp, *osp;
1504         char *name;
1505         size_t namelen;
1506         struct module *omp;
1507         uint_t nsyms;
1508         size_t symsize = mp->symhdr->sh_entsize;
1509         size_t locals = 1;
1510         size_t strsize;
1511
1512         /*
1513          * Make a copy of the original module structure.
1514          */
1515         omp = kobj_alloc(sizeof (struct module), KM_WAIT);
1516         bcopy(mp, omp, sizeof (struct module));
1517
1518         /*
1519          * Compute the sizes of the new symbol table sections.
1520          */
1521         for (nsyms = strsize = 1, osp = (Sym *)omp->symtbl; osp < esp; osp++) {
1522                 if (osp->st_value == 0)
1523                         continue;
1524                 if (sym_lookup(omp, osp) == NULL)
1525                         continue;
1526                 name = omp->strings + osp->st_name;
1527                 namelen = strlen(name);
1528                 if (ELF_ST_BIND(osp->st_info) == STB_LOCAL)
1529                         locals++;
1530                 nsyms++;
1531                 strsize += namelen + 1;
1532         }
1533
1534         mp->nsyms = nsyms;
1535         mp->hashsize = kobj_gethashsize(mp->nsyms);
1536
1537         /*
1538          * ksyms_lock must be held as writer during any operation that
1539          * modifies ksyms_arena, including allocation from same, and
1540          * must not be dropped until the arena is vmem_walk()able.
1541          */
1542         rw_enter(&ksyms_lock, RW_WRITER);
1543
1544         /*
1545          * Allocate space for the new section headers (symtab and strtab),
1546          * symbol table, buckets, chains, and strings.
1547          */
1548         mp->symsize = (2 * sizeof (Shdr)) + (nsyms * symsize) +
1549             (mp->hashsize + mp->nsyms) * sizeof (symid_t) + strsize;
1550
1551         if (mp->flags & KOBJ_NOKSYMS) {
1552                 mp->symspace = kobj_alloc(mp->symsize, KM_WAIT);
1553         } else {
1554                 mp->symspace = vmem_alloc(ksyms_arena, mp->symsize,
1555                     VM_BESTFIT | VM_SLEEP);
1556         }
1557         bzero(mp->symspace, mp->symsize);
1558
1559         /*
1560          * Divvy up symspace.
1561          */
1562         mp->shdrs = mp->symspace;
1563         mp->symhdr = (Shdr *)mp->shdrs;
1564         mp->strhdr = (Shdr *)(mp->symhdr + 1);
1565         mp->symtbl = (char *)(mp->strhdr + 1);
1566         mp->buckets = (symid_t *)(mp->symtbl + (nsyms * symsize));
1567         mp->chains = (symid_t *)(mp->buckets + mp->hashsize);
1568         mp->strings = (char *)(mp->chains + nsyms);
1569
1570         /*
1571          * Fill in the new section headers (symtab and strtab).
1572          */
1573         mp->hdr.e_shnum = 2;
1574         mp->symtbl_section = 0;
1575
1576         mp->symhdr->sh_type = SHT_SYMTAB;
1577         mp->symhdr->sh_addr = (Addr)mp->symtbl;
1578         mp->symhdr->sh_size = nsyms * symsize;
1579         mp->symhdr->sh_link = 1;
1580         mp->symhdr->sh_info = locals;
1581         mp->symhdr->sh_addralign = sizeof (Addr);
1582         mp->symhdr->sh_entsize = symsize;
1583
1584         mp->strhdr->sh_type = SHT_STRTAB;
1585         mp->strhdr->sh_addr = (Addr)mp->strings;
1586         mp->strhdr->sh_size = strsize;
1587         mp->strhdr->sh_addralign = 1;
1588
1589         /*
1590          * Construct the new symbol table.
1591          */
1592         for (nsyms = strsize = 1, osp = (Sym *)omp->symtbl; osp < esp; osp++) {
1593                 if (osp->st_value == 0)
1594                         continue;
1595                 if (sym_lookup(omp, osp) == NULL)
1596                         continue;
1597                 name = omp->strings + osp->st_name;
1598                 namelen = strlen(name);
1599                 sp = (Sym *)(mp->symtbl + symsize * nsyms);
1600                 bcopy(osp, sp, symsize);
1601                 bcopy(name, mp->strings + strsize, namelen);
1602                 sp->st_name = strsize;
1603                 sym_insert(mp, name, nsyms);
1604                 nsyms++;
1605                 strsize += namelen + 1;
1606         }
1607
1608         rw_exit(&ksyms_lock);
1609
1610         /*
1611          * Free the old section headers -- we'll never need them again.
1612          */
1613         if (!(mp->flags & KOBJ_PRIM)) {
1614                 uint_t  shn;
1615                 Shdr    *shp;
1616
1617                 for (shn = 1; shn < omp->hdr.e_shnum; shn++) {
1618                         shp = (Shdr *)(omp->shdrs + shn * omp->hdr.e_shentsize);
1619                         switch (shp->sh_type) {
1620                         case SHT_RELA:
1621                         case SHT_REL:
1622                                 if (shp->sh_addr != 0) {
1623                                         kobj_free((void *)shp->sh_addr,
1624                                             shp->sh_size);
1625                                 }
1626                                 break;
1627                         }
1628                 }
1629                 kobj_free(omp->shdrs, omp->hdr.e_shentsize * omp->hdr.e_shnum);
1630         }
1631         /*
1632          * Discard the old symbol table and our copy of the module strucure.
1633          */
1634         if (!(mp->flags & KOBJ_PRIM))
1635                 kobj_free(omp->symspace, omp->symsize);
1636         kobj_free(omp, sizeof (struct module));
1637 }
1638
1639 static void
1640 kobj_export_ctf(struct module *mp)
1641 {
1642         char *data = mp->ctfdata;
1643         size_t size = mp->ctfsize;
1644
1645         if (data != NULL) {
1646                 if (_moddebug & MODDEBUG_NOCTF) {
1647                         mp->ctfdata = NULL;
1648                         mp->ctfsize = 0;
1649                 } else {
1650                         mp->ctfdata = vmem_alloc(ctf_arena, size,
1651                             VM_BESTFIT | VM_SLEEP);
1652                         bcopy(data, mp->ctfdata, size);
1653                 }
1654
1655                 if (!(mp->flags & KOBJ_PRIM))
1656                         kobj_free(data, size);
1657         }
1658 }
1659
1660 void
1661 kobj_export_module(struct module *mp)
1662 {
1663         kobj_export_ksyms(mp);
1664         kobj_export_ctf(mp);
1665
1666         mp->flags |= KOBJ_EXPORTED;
1667 }
1668
1669 static int
1670 process_dynamic(struct module *mp, char *dyndata, char *strdata)
1671 {
1672         char *path = NULL, *depstr = NULL;
1673         int allocsize = 0, osize = 0, nsize = 0;
1674         char *libname, *tmp;
1675         int lsize;
1676         Dyn *dynp;
1677
1678         for (dynp = (Dyn *)dyndata; dynp && dynp->d_tag != DT_NULL; dynp++) {
1679                 switch (dynp->d_tag) {
1680                 case DT_NEEDED:
1681                         /*
1682                          * Read the DT_NEEDED entries, expanding the macros they
1683                          * contain (if any), and concatenating them into a
1684                          * single space-separated dependency list.
1685                          */
1686                         libname = (ulong_t)dynp->d_un.d_ptr + strdata;
1687
1688                         if (strchr(libname, '$') != NULL) {
1689                                 char *_lib;
1690
1691                                 if (path == NULL)
1692                                         path = kobj_alloc(MAXPATHLEN, KM_WAIT);
1693                                 if ((_lib = expand_libmacro(libname, path,
1694                                     path)) != NULL)
1695                                         libname = _lib;
1696                                 else {
1697                                         _kobj_printf(ops, "krtld: "
1698                                             "process_dynamic: failed to expand "
1699                                             "%s\n", libname);
1700                                 }
1701                         }
1702
1703                         lsize = strlen(libname);
1704                         nsize += lsize;
1705                         if (nsize + 1 > allocsize) {
1706                                 tmp = kobj_alloc(allocsize + MAXPATHLEN,
1707                                     KM_WAIT);
1708                                 if (depstr != NULL) {
1709                                         bcopy(depstr, tmp, osize);
1710                                         kobj_free(depstr, allocsize);
1711                                 }
1712                                 depstr = tmp;
1713                                 allocsize += MAXPATHLEN;
1714                         }
1715                         bcopy(libname, depstr + osize, lsize);
1716                         *(depstr + nsize) = ' '; /* separator */
1717                         nsize++;
1718                         osize = nsize;
1719                         break;
1720
1721                 case DT_FLAGS_1:
1722                         if (dynp->d_un.d_val & DF_1_IGNMULDEF)
1723                                 mp->flags |= KOBJ_IGNMULDEF;
1724                         if (dynp->d_un.d_val & DF_1_NOKSYMS)
1725                                 mp->flags |= KOBJ_NOKSYMS;
1726
1727                         break;
1728                 }
1729         }
1730
1731         /*
1732          * finish up the depends string (if any)
1733          */
1734         if (depstr != NULL) {
1735                 *(depstr + nsize - 1) = '\0'; /* overwrite separator w/term */
1736                 if (path != NULL)
1737                         kobj_free(path, MAXPATHLEN);
1738
1739                 tmp = kobj_alloc(nsize, KM_WAIT);
1740                 bcopy(depstr, tmp, nsize);
1741                 kobj_free(depstr, allocsize);
1742                 depstr = tmp;
1743
1744                 mp->depends_on = depstr;
1745         }
1746
1747         return (0);
1748 }
1749
1750 static int
1751 do_dynamic(struct module *mp, struct _buf *file)
1752 {
1753         Shdr *dshp, *dstrp, *shp;
1754         char *dyndata, *dstrdata;
1755         int dshn, shn, rc;
1756
1757         /* find and validate the dynamic section (if any) */
1758
1759         for (dshp = NULL, shn = 1; shn < mp->hdr.e_shnum; shn++) {
1760                 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
1761                 switch (shp->sh_type) {
1762                 case SHT_DYNAMIC:
1763                         if (dshp != NULL) {
1764                                 _kobj_printf(ops, "krtld: get_dynamic: %s, ",
1765                                     mp->filename);
1766                                 _kobj_printf(ops,
1767                                     "multiple dynamic sections\n");
1768                                 return (-1);
1769                         } else {
1770                                 dshp = shp;
1771                                 dshn = shn;
1772                         }
1773                         break;
1774                 }
1775         }
1776
1777         if (dshp == NULL)
1778                 return (0);
1779
1780         if (dshp->sh_link > mp->hdr.e_shnum) {
1781                 _kobj_printf(ops, "krtld: get_dynamic: %s, ", mp->filename);
1782                 _kobj_printf(ops, "no section for sh_link %d\n", dshp->sh_link);
1783                 return (-1);
1784         }
1785         dstrp = (Shdr *)(mp->shdrs + dshp->sh_link * mp->hdr.e_shentsize);
1786
1787         if (dstrp->sh_type != SHT_STRTAB) {
1788                 _kobj_printf(ops, "krtld: get_dynamic: %s, ", mp->filename);
1789                 _kobj_printf(ops, "sh_link not a string table for section %d\n",
1790                     dshn);
1791                 return (-1);
1792         }
1793
1794         /* read it from disk */
1795
1796         dyndata = kobj_alloc(dshp->sh_size, KM_WAIT|KM_TMP);
1797         if (kobj_read_file(file, dyndata, dshp->sh_size, dshp->sh_offset) < 0) {
1798                 _kobj_printf(ops, "krtld: get_dynamic: %s, ", mp->filename);
1799                 _kobj_printf(ops, "error reading section %d\n", dshn);
1800
1801                 kobj_free(dyndata, dshp->sh_size);
1802                 return (-1);
1803         }
1804
1805         dstrdata = kobj_alloc(dstrp->sh_size, KM_WAIT|KM_TMP);
1806         if (kobj_read_file(file, dstrdata, dstrp->sh_size,
1807             dstrp->sh_offset) < 0) {
1808                 _kobj_printf(ops, "krtld: get_dynamic: %s, ", mp->filename);
1809                 _kobj_printf(ops, "error reading section %d\n", dshp->sh_link);
1810
1811                 kobj_free(dyndata, dshp->sh_size);
1812                 kobj_free(dstrdata, dstrp->sh_size);
1813                 return (-1);
1814         }
1815
1816         /* pull the interesting pieces out */
1817
1818         rc = process_dynamic(mp, dyndata, dstrdata);
1819
1820         kobj_free(dyndata, dshp->sh_size);
1821         kobj_free(dstrdata, dstrp->sh_size);
1822
1823         return (rc);
1824 }
1825
1826 void
1827 kobj_set_ctf(struct module *mp, caddr_t data, size_t size)
1828 {
1829         if (!standalone) {
1830                 if (mp->ctfdata != NULL) {
1831                         if (vmem_contains(ctf_arena, mp->ctfdata,
1832                             mp->ctfsize)) {
1833                                 vmem_free(ctf_arena, mp->ctfdata, mp->ctfsize);
1834                         } else {
1835                                 kobj_free(mp->ctfdata, mp->ctfsize);
1836                         }
1837                 }
1838         }
1839
1840         /*
1841          * The order is very important here.  We need to make sure that
1842          * consumers, at any given instant, see a consistent state.  We'd
1843          * rather they see no CTF data than the address of one buffer and the
1844          * size of another.
1845          */
1846         mp->ctfdata = NULL;
1847         membar_producer();
1848         mp->ctfsize = size;
1849         mp->ctfdata = data;
1850         membar_producer();
1851 }
1852
1853 int
1854 kobj_load_module(struct modctl *modp, int use_path)
1855 {
1856         char *filename = modp->mod_filename;
1857         char *modname = modp->mod_modname;
1858         int i;
1859         int n;
1860         struct _buf *file;
1861         struct module *mp = NULL;
1862 #ifdef MODDIR_SUFFIX
1863         int no_suffixdir_drv = 0;
1864 #endif
1865
1866         mp = kobj_zalloc(sizeof (struct module), KM_WAIT);
1867
1868         /*
1869          * We need to prevent kmdb's symbols from leaking into /dev/ksyms.
1870          * kmdb contains a bunch of symbols with well-known names, symbols
1871          * which will mask the real versions, thus causing no end of trouble
1872          * for mdb.
1873          */
1874         if (strcmp(modp->mod_modname, "kmdbmod") == 0)
1875                 mp->flags |= KOBJ_NOKSYMS;
1876
1877         file = kobj_open_path(filename, use_path, 1);
1878         if (file == (struct _buf *)-1) {
1879 #ifdef MODDIR_SUFFIX
1880                 file = kobj_open_path(filename, use_path, 0);
1881 #endif
1882                 if (file == (struct _buf *)-1) {
1883                         kobj_free(mp, sizeof (*mp));
1884                         goto bad;
1885                 }
1886 #ifdef MODDIR_SUFFIX
1887                 /*
1888                  * There is no driver module in the ISA specific (suffix)
1889                  * subdirectory but there is a module in the parent directory.
1890                  */
1891                 if (strncmp(filename, "drv/", 4) == 0) {
1892                         no_suffixdir_drv = 1;
1893                 }
1894 #endif
1895         }
1896
1897         mp->filename = kobj_alloc(strlen(file->_name) + 1, KM_WAIT);
1898         (void) strcpy(mp->filename, file->_name);
1899
1900         if (kobj_read_file(file, (char *)&mp->hdr, sizeof (mp->hdr), 0) < 0) {
1901                 _kobj_printf(ops, "kobj_load_module: %s read header failed\n",
1902                     modname);
1903                 kobj_free(mp->filename, strlen(file->_name) + 1);
1904                 kobj_free(mp, sizeof (*mp));
1905                 goto bad;
1906         }
1907         for (i = 0; i < SELFMAG; i++) {
1908                 if (mp->hdr.e_ident[i] != ELFMAG[i]) {
1909                         if (_moddebug & MODDEBUG_ERRMSG)
1910                                 _kobj_printf(ops, "%s not an elf module\n",
1911                                     modname);
1912                         kobj_free(mp->filename, strlen(file->_name) + 1);
1913                         kobj_free(mp, sizeof (*mp));
1914                         goto bad;
1915                 }
1916         }
1917         /*
1918          * It's ELF, but is it our ISA?  Interpreting the header
1919          * from a file for a byte-swapped ISA could cause a huge
1920          * and unsatisfiable value to be passed to kobj_alloc below
1921          * and therefore hang booting.
1922          */
1923         if (!elf_mach_ok(&mp->hdr)) {
1924                 if (_moddebug & MODDEBUG_ERRMSG)
1925                         _kobj_printf(ops, "%s not an elf module for this ISA\n",
1926                             modname);
1927                 kobj_free(mp->filename, strlen(file->_name) + 1);
1928                 kobj_free(mp, sizeof (*mp));
1929 #ifdef MODDIR_SUFFIX
1930                 /*
1931                  * The driver mod is not in the ISA specific subdirectory
1932                  * and the module in the parent directory is not our ISA.
1933                  * If it is our ISA, for now we will silently succeed.
1934                  */
1935                 if (no_suffixdir_drv == 1) {
1936                         cmn_err(CE_CONT, "?NOTICE: %s: 64-bit driver module"
1937                             " not found\n", modname);
1938                 }
1939 #endif
1940                 goto bad;
1941         }
1942
1943         /*
1944          * All modules, save for unix, should be relocatable (as opposed to
1945          * dynamic).  Dynamic modules come with PLTs and GOTs, which can't
1946          * currently be processed by krtld.
1947          */
1948         if (mp->hdr.e_type != ET_REL) {
1949                 if (_moddebug & MODDEBUG_ERRMSG)
1950                         _kobj_printf(ops, "%s isn't a relocatable (ET_REL) "
1951                             "module\n", modname);
1952                 kobj_free(mp->filename, strlen(file->_name) + 1);
1953                 kobj_free(mp, sizeof (*mp));
1954                 goto bad;
1955         }
1956
1957         n = mp->hdr.e_shentsize * mp->hdr.e_shnum;
1958         mp->shdrs = kobj_alloc(n, KM_WAIT);
1959
1960         if (kobj_read_file(file, mp->shdrs, n, mp->hdr.e_shoff) < 0) {
1961                 _kobj_printf(ops, "kobj_load_module: %s error reading "
1962                     "section headers\n", modname);
1963                 kobj_free(mp->shdrs, n);
1964                 kobj_free(mp->filename, strlen(file->_name) + 1);
1965                 kobj_free(mp, sizeof (*mp));
1966                 goto bad;
1967         }
1968
1969         kobj_notify(KOBJ_NOTIFY_MODLOADING, modp);
1970         module_assign(modp, mp);
1971
1972         /* read in sections */
1973         if (get_progbits(mp, file) < 0) {
1974                 _kobj_printf(ops, "%s error reading sections\n", modname);
1975                 goto bad;
1976         }
1977
1978         if (do_dynamic(mp, file) < 0) {
1979                 _kobj_printf(ops, "%s error reading dynamic section\n",
1980                     modname);
1981                 goto bad;
1982         }
1983
1984         modp->mod_text = mp->text;
1985         modp->mod_text_size = mp->text_size;
1986
1987         /* read in symbols; adjust values for each section's real address */
1988         if (get_syms(mp, file) < 0) {
1989                 _kobj_printf(ops, "%s error reading symbols\n",
1990                     modname);
1991                 goto bad;
1992         }
1993
1994         /*
1995          * If we didn't dependency information from the dynamic section, look
1996          * for it the old-fashioned way.
1997          */
1998         if (mp->depends_on == NULL)
1999                 mp->depends_on = depends_on(mp);
2000
2001         if (get_ctf(mp, file) < 0) {
2002                 _kobj_printf(ops, "%s debug information will not "
2003                     "be available\n", modname);
2004         }
2005
2006         /* primary kernel modules do not have a signature section */
2007         if (!(mp->flags & KOBJ_PRIM))
2008                 get_signature(mp, file);
2009
2010 #ifdef  KOBJ_DEBUG
2011         if (kobj_debug & D_LOADING) {
2012                 _kobj_printf(ops, "krtld: file=%s\n", mp->filename);
2013                 _kobj_printf(ops, "\ttext:0x%p", mp->text);
2014                 _kobj_printf(ops, " size: 0x%x\n", mp->text_size);
2015                 _kobj_printf(ops, "\tdata:0x%p", mp->data);
2016                 _kobj_printf(ops, " dsize: 0x%x\n", mp->data_size);
2017         }
2018 #endif /* KOBJ_DEBUG */
2019
2020         /*
2021          * For primary kernel modules, we defer
2022          * symbol resolution and relocation until
2023          * all primary objects have been loaded.
2024          */
2025         if (!standalone) {
2026                 int ddrval, dcrval;
2027                 char *dependent_modname;
2028                 /* load all dependents */
2029                 dependent_modname = kobj_zalloc(MODMAXNAMELEN, KM_WAIT);
2030                 ddrval = do_dependents(modp, dependent_modname, MODMAXNAMELEN);
2031
2032                 /*
2033                  * resolve undefined and common symbols,
2034                  * also allocates common space
2035                  */
2036                 if ((dcrval = do_common(mp)) < 0) {
2037                         switch (dcrval) {
2038                         case DOSYM_UNSAFE:
2039                                 _kobj_printf(ops, "WARNING: mod_load: "
2040                                     "MT-unsafe module '%s' rejected\n",
2041                                     modname);
2042                                 break;
2043                         case DOSYM_UNDEF:
2044                                 _kobj_printf(ops, "WARNING: mod_load: "
2045                                     "cannot load module '%s'\n",
2046                                     modname);
2047                                 if (ddrval == -1) {
2048                                         _kobj_printf(ops, "WARNING: %s: ",
2049                                             modname);
2050                                         _kobj_printf(ops,
2051                                             "unable to resolve dependency, "
2052                                             "module '%s' not found\n",
2053                                             dependent_modname);
2054                                 }
2055                                 break;
2056                         }
2057                 }
2058                 kobj_free(dependent_modname, MODMAXNAMELEN);
2059                 if (dcrval < 0)
2060                         goto bad;
2061
2062                 /* process relocation tables */
2063                 if (do_relocations(mp) < 0) {
2064                         _kobj_printf(ops, "%s error doing relocations\n",
2065                             modname);
2066                         goto bad;
2067                 }
2068
2069                 if (mp->destination) {
2070                         off_t   off = (uintptr_t)mp->destination & PAGEOFFSET;
2071                         caddr_t base = (caddr_t)mp->destination - off;
2072                         size_t  size = P2ROUNDUP(mp->text_size + off, PAGESIZE);
2073
2074                         hat_unload(kas.a_hat, base, size, HAT_UNLOAD_UNLOCK);
2075                         vmem_free(heap_arena, base, size);
2076                 }
2077
2078                 /* sync_instruction_memory */
2079                 kobj_sync_instruction_memory(mp->text, mp->text_size);
2080                 kobj_export_module(mp);
2081                 kobj_notify(KOBJ_NOTIFY_MODLOADED, modp);
2082         }
2083         kobj_close_file(file);
2084         return (0);
2085 bad:
2086         if (file != (struct _buf *)-1)
2087                 kobj_close_file(file);
2088         if (modp->mod_mp != NULL)
2089                 free_module_data(modp->mod_mp);
2090
2091         module_assign(modp, NULL);
2092         return ((file == (struct _buf *)-1) ? ENOENT : EINVAL);
2093 }
2094
2095 int
2096 kobj_load_primary_module(struct modctl *modp)
2097 {
2098         struct modctl *dep;
2099         struct module *mp;
2100
2101         if (kobj_load_module(modp, 0) != 0)
2102                 return (-1);
2103
2104         mp = modp->mod_mp;
2105         mp->flags |= KOBJ_PRIM;
2106
2107         /* Bind new module to its dependents */
2108         if (mp->depends_on != NULL && (dep =
2109             mod_already_loaded(mp->depends_on)) == NULL) {
2110 #ifdef  KOBJ_DEBUG
2111                 if (kobj_debug & D_DEBUG) {
2112                         _kobj_printf(ops, "krtld: failed to resolve deps "
2113                             "for primary %s\n", modp->mod_modname);
2114                 }
2115 #endif
2116                 return (-1);
2117         }
2118
2119         add_dependent(mp, dep->mod_mp);
2120
2121         /*
2122          * Relocate it.  This module may not be part of a link map, so we
2123          * can't use bind_primary.
2124          */
2125         if (do_common(mp) < 0 || do_symbols(mp, 0) < 0 ||
2126             do_relocations(mp) < 0) {
2127 #ifdef  KOBJ_DEBUG
2128                 if (kobj_debug & D_DEBUG) {
2129                         _kobj_printf(ops, "krtld: failed to relocate "
2130                             "primary %s\n", modp->mod_modname);
2131                 }
2132 #endif
2133                 return (-1);
2134         }
2135
2136         return (0);
2137 }
2138
2139 static void
2140 module_assign(struct modctl *cp, struct module *mp)
2141 {
2142         if (standalone) {
2143                 cp->mod_mp = mp;
2144                 return;
2145         }
2146         mutex_enter(&mod_lock);
2147         cp->mod_mp = mp;
2148         cp->mod_gencount++;
2149         mutex_exit(&mod_lock);
2150 }
2151
2152 void
2153 kobj_unload_module(struct modctl *modp)
2154 {
2155         struct module *mp = modp->mod_mp;
2156
2157         if ((_moddebug & MODDEBUG_KEEPTEXT) && mp) {
2158                 _kobj_printf(ops, "text for %s ", mp->filename);
2159                 _kobj_printf(ops, "was at %p\n", mp->text);
2160                 mp->text = NULL;        /* don't actually free it */
2161         }
2162
2163         kobj_notify(KOBJ_NOTIFY_MODUNLOADING, modp);
2164
2165         /*
2166          * Null out mod_mp first, so consumers (debuggers) know not to look
2167          * at the module structure any more.
2168          */
2169         mutex_enter(&mod_lock);
2170         modp->mod_mp = NULL;
2171         mutex_exit(&mod_lock);
2172
2173         kobj_notify(KOBJ_NOTIFY_MODUNLOADED, modp);
2174         free_module_data(mp);
2175 }
2176
2177 static void
2178 free_module_data(struct module *mp)
2179 {
2180         struct module_list *lp, *tmp;
2181         int ksyms_exported = 0;
2182
2183         lp = mp->head;
2184         while (lp) {
2185                 tmp = lp;
2186                 lp = lp->next;
2187                 kobj_free((char *)tmp, sizeof (*tmp));
2188         }
2189
2190         rw_enter(&ksyms_lock, RW_WRITER);
2191         if (mp->symspace) {
2192                 if (vmem_contains(ksyms_arena, mp->symspace, mp->symsize)) {
2193                         vmem_free(ksyms_arena, mp->symspace, mp->symsize);
2194                         ksyms_exported = 1;
2195                 } else {
2196                         if (mp->flags & KOBJ_NOKSYMS)
2197                                 ksyms_exported = 1;
2198                         kobj_free(mp->symspace, mp->symsize);
2199                 }
2200         }
2201         rw_exit(&ksyms_lock);
2202
2203         if (mp->ctfdata) {
2204                 if (vmem_contains(ctf_arena, mp->ctfdata, mp->ctfsize))
2205                         vmem_free(ctf_arena, mp->ctfdata, mp->ctfsize);
2206                 else
2207                         kobj_free(mp->ctfdata, mp->ctfsize);
2208         }
2209
2210         if (mp->sigdata)
2211                 kobj_free(mp->sigdata, mp->sigsize);
2212
2213         /*
2214          * We did not get far enough into kobj_export_ksyms() to free allocated
2215          * buffers because we encounted error conditions. Free the buffers.
2216          */
2217         if ((ksyms_exported == 0) && (mp->shdrs != NULL)) {
2218                 uint_t shn;
2219                 Shdr *shp;
2220
2221                 for (shn = 1; shn < mp->hdr.e_shnum; shn++) {
2222                         shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
2223                         switch (shp->sh_type) {
2224                         case SHT_RELA:
2225                         case SHT_REL:
2226                                 if (shp->sh_addr != 0)
2227                                         kobj_free((void *)shp->sh_addr,
2228                                             shp->sh_size);
2229                                 break;
2230                         }
2231                 }
2232 err_free_done:
2233                 if (!(mp->flags & KOBJ_PRIM)) {
2234                         kobj_free(mp->shdrs,
2235                             mp->hdr.e_shentsize * mp->hdr.e_shnum);
2236                 }
2237         }
2238
2239         if (mp->bss)
2240                 vmem_free(data_arena, (void *)mp->bss, mp->bss_size);
2241
2242         if (mp->fbt_tab)
2243                 kobj_texthole_free(mp->fbt_tab, mp->fbt_size);
2244
2245         if (mp->textwin_base)
2246                 kobj_textwin_free(mp);
2247
2248         if (mp->sdt_probes != NULL) {
2249                 sdt_probedesc_t *sdp = mp->sdt_probes, *next;
2250
2251                 while (sdp != NULL) {
2252                         next = sdp->sdpd_next;
2253                         kobj_free(sdp->sdpd_name, strlen(sdp->sdpd_name) + 1);
2254                         kobj_free(sdp, sizeof (sdt_probedesc_t));
2255                         sdp = next;
2256                 }
2257         }
2258
2259         if (mp->sdt_tab)
2260                 kobj_texthole_free(mp->sdt_tab, mp->sdt_size);
2261         if (mp->text)
2262                 vmem_free(text_arena, mp->text, mp->text_size);
2263         if (mp->data)
2264                 vmem_free(data_arena, mp->data, mp->data_size);
2265         if (mp->depends_on)
2266                 kobj_free(mp->depends_on, strlen(mp->depends_on)+1);
2267         if (mp->filename)
2268                 kobj_free(mp->filename, strlen(mp->filename)+1);
2269
2270         kobj_free((char *)mp, sizeof (*mp));
2271 }
2272
2273 static int
2274 get_progbits(struct module *mp, struct _buf *file)
2275 {
2276         struct proginfo *tp, *dp, *sdp;
2277         Shdr *shp;
2278         reloc_dest_t dest = NULL;
2279         uintptr_t bits_ptr;
2280         uintptr_t text = 0, data, textptr;
2281         uint_t shn;
2282         int err = -1;
2283
2284         tp = kobj_zalloc(sizeof (struct proginfo), KM_WAIT|KM_TMP);
2285         dp = kobj_zalloc(sizeof (struct proginfo), KM_WAIT|KM_TMP);
2286         sdp = kobj_zalloc(sizeof (struct proginfo), KM_WAIT|KM_TMP);
2287         /*
2288          * loop through sections to find out how much space we need
2289          * for text, data, (also bss that is already assigned)
2290          */
2291         if (get_progbits_size(mp, tp, dp, sdp) < 0)
2292                 goto done;
2293
2294         mp->text_size = tp->size;
2295         mp->data_size = dp->size;
2296
2297         if (standalone) {
2298                 caddr_t limit = _data;
2299
2300                 if (lg_pagesize && _text + lg_pagesize < limit)
2301                         limit = _text + lg_pagesize;
2302
2303                 mp->text = kobj_segbrk(&_etext, mp->text_size,
2304                     tp->align, limit);
2305                 /*
2306                  * If we can't grow the text segment, try the
2307                  * data segment before failing.
2308                  */
2309                 if (mp->text == NULL) {
2310                         mp->text = kobj_segbrk(&_edata, mp->text_size,
2311                             tp->align, 0);
2312                 }
2313
2314                 mp->data = kobj_segbrk(&_edata, mp->data_size, dp->align, 0);
2315
2316                 if (mp->text == NULL || mp->data == NULL)
2317                         goto done;
2318
2319         } else {
2320                 if (text_arena == NULL)
2321                         kobj_vmem_init(&text_arena, &data_arena);
2322
2323                 /*
2324                  * some architectures may want to load the module on a
2325                  * page that is currently read only. It may not be
2326                  * possible for those architectures to remap their page
2327                  * on the fly. So we provide a facility for them to hang
2328                  * a private hook where the memory they assign the module
2329                  * is not the actual place where the module loads.
2330                  *
2331                  * In this case there are two addresses that deal with the
2332                  * modload.
2333                  * 1) the final destination of the module
2334                  * 2) the address that is used to view the newly
2335                  * loaded module until all the relocations relative to 1
2336                  * above are completed.
2337                  *
2338                  * That is what dest is used for below.
2339                  */
2340                 mp->text_size += tp->align;
2341                 mp->data_size += dp->align;
2342
2343                 mp->text = kobj_text_alloc(text_arena, mp->text_size);
2344
2345                 /*
2346                  * a remap is taking place. Align the text ptr relative
2347                  * to the secondary mapping. That is where the bits will
2348                  * be read in.
2349                  */
2350                 if (kvseg.s_base != NULL && !vmem_contains(heaptext_arena,
2351                     mp->text, mp->text_size)) {
2352                         off_t   off = (uintptr_t)mp->text & PAGEOFFSET;
2353                         size_t  size = P2ROUNDUP(mp->text_size + off, PAGESIZE);
2354                         caddr_t map = vmem_alloc(heap_arena, size, VM_SLEEP);
2355                         caddr_t orig = mp->text - off;
2356                         pgcnt_t pages = size / PAGESIZE;
2357
2358                         dest = (reloc_dest_t)(map + off);
2359                         text = ALIGN((uintptr_t)dest, tp->align);
2360
2361                         while (pages--) {
2362                                 hat_devload(kas.a_hat, map, PAGESIZE,
2363                                     hat_getpfnum(kas.a_hat, orig),
2364                                     PROT_READ | PROT_WRITE | PROT_EXEC,
2365                                     HAT_LOAD_NOCONSIST | HAT_LOAD_LOCK);
2366                                 map += PAGESIZE;
2367                                 orig += PAGESIZE;
2368                         }
2369                         /*
2370                          * Since we set up a non-cacheable mapping, we need
2371                          * to flush any old entries in the cache that might
2372                          * be left around from the read-only mapping.
2373                          */
2374                         dcache_flushall();
2375                 }
2376                 if (mp->data_size)
2377                         mp->data = vmem_alloc(data_arena, mp->data_size,
2378                             VM_SLEEP | VM_BESTFIT);
2379         }
2380         textptr = (uintptr_t)mp->text;
2381         textptr = ALIGN(textptr, tp->align);
2382         mp->destination = dest;
2383
2384         /*
2385          * This is the case where a remap is not being done.
2386          */
2387         if (text == 0)
2388                 text = ALIGN((uintptr_t)mp->text, tp->align);
2389         data = ALIGN((uintptr_t)mp->data, dp->align);
2390
2391         /* now loop though sections assigning addresses and loading the data */
2392         for (shn = 1; shn < mp->hdr.e_shnum; shn++) {
2393                 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
2394                 if (!(shp->sh_flags & SHF_ALLOC))
2395                         continue;
2396
2397                 if ((shp->sh_flags & SHF_WRITE) == 0)
2398                         bits_ptr = text;
2399                 else
2400                         bits_ptr = data;
2401
2402                 bits_ptr = ALIGN(bits_ptr, shp->sh_addralign);
2403
2404                 if (shp->sh_type == SHT_NOBITS) {
2405                         /*
2406                          * Zero bss.
2407                          */
2408                         bzero((caddr_t)bits_ptr, shp->sh_size);
2409                         shp->sh_type = SHT_PROGBITS;
2410                 } else {
2411                         if (kobj_read_file(file, (char *)bits_ptr,
2412                             shp->sh_size, shp->sh_offset) < 0)
2413                                 goto done;
2414                 }
2415
2416                 if (shp->sh_flags & SHF_WRITE) {
2417                         shp->sh_addr = bits_ptr;
2418                 } else {
2419                         textptr = ALIGN(textptr, shp->sh_addralign);
2420                         shp->sh_addr = textptr;
2421                         textptr += shp->sh_size;
2422                 }
2423
2424                 bits_ptr += shp->sh_size;
2425                 if ((shp->sh_flags & SHF_WRITE) == 0)
2426                         text = bits_ptr;
2427                 else
2428                         data = bits_ptr;
2429         }
2430
2431         err = 0;
2432 done:
2433         /*
2434          * Free and mark as freed the section headers here so that
2435          * free_module_data() does not have to worry about this buffer.
2436          *
2437          * This buffer is freed here because one of the possible reasons
2438          * for error is a section with non-zero sh_addr and in that case
2439          * free_module_data() would have no way of recognizing that this
2440          * buffer was unallocated.
2441          */
2442         if (err != 0) {
2443                 kobj_free(mp->shdrs, mp->hdr.e_shentsize * mp->hdr.e_shnum);
2444                 mp->shdrs = NULL;
2445         }
2446
2447         (void) kobj_free(tp, sizeof (struct proginfo));
2448         (void) kobj_free(dp, sizeof (struct proginfo));
2449         (void) kobj_free(sdp, sizeof (struct proginfo));
2450
2451         return (err);
2452 }
2453
2454 /*
2455  * Go through suppress_sym_list to see if "multiply defined"
2456  * warning of this symbol should be suppressed.  Return 1 if
2457  * warning should be suppressed, 0 otherwise.
2458  */
2459 static int
2460 kobj_suppress_warning(char *symname)
2461 {
2462         int     i;
2463
2464         for (i = 0; suppress_sym_list[i] != NULL; i++) {
2465                 if (strcmp(suppress_sym_list[i], symname) == 0)
2466                         return (1);
2467         }
2468
2469         return (0);
2470 }
2471
2472 static int
2473 get_syms(struct module *mp, struct _buf *file)
2474 {
2475         uint_t          shn;
2476         Shdr    *shp;
2477         uint_t          i;
2478         Sym     *sp, *ksp;
2479         char            *symname;
2480         int             dosymtab = 0;
2481
2482         /*
2483          * Find the interesting sections.
2484          */
2485         for (shn = 1; shn < mp->hdr.e_shnum; shn++) {
2486                 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
2487                 switch (shp->sh_type) {
2488                 case SHT_SYMTAB:
2489                         mp->symtbl_section = shn;
2490                         mp->symhdr = shp;
2491                         dosymtab++;
2492                         break;
2493
2494                 case SHT_RELA:
2495                 case SHT_REL:
2496                         /*
2497                          * Already loaded.
2498                          */
2499                         if (shp->sh_addr)
2500                                 continue;
2501
2502                         /* KM_TMP since kobj_free'd in do_relocations */
2503                         shp->sh_addr = (Addr)
2504                             kobj_alloc(shp->sh_size, KM_WAIT|KM_TMP);
2505
2506                         if (kobj_read_file(file, (char *)shp->sh_addr,
2507                             shp->sh_size, shp->sh_offset) < 0) {
2508                                 _kobj_printf(ops, "krtld: get_syms: %s, ",
2509                                     mp->filename);
2510                                 _kobj_printf(ops, "error reading section %d\n",
2511                                     shn);
2512                                 return (-1);
2513                         }
2514                         break;
2515                 }
2516         }
2517
2518         /*
2519          * This is true for a stripped executable.  In the case of
2520          * 'unix' it can be stripped but it still contains the SHT_DYNSYM,
2521          * and since that symbol information is still present everything
2522          * is just fine.
2523          */
2524         if (!dosymtab) {
2525                 if (mp->flags & KOBJ_EXEC)
2526                         return (0);
2527                 _kobj_printf(ops, "krtld: get_syms: %s ",
2528                     mp->filename);
2529                 _kobj_printf(ops, "no SHT_SYMTAB symbol table found\n");
2530                 return (-1);
2531         }
2532
2533         /*
2534          * get the associated string table header
2535          */
2536         if ((mp->symhdr == 0) || (mp->symhdr->sh_link >= mp->hdr.e_shnum))
2537                 return (-1);
2538         mp->strhdr = (Shdr *)
2539             (mp->shdrs + mp->symhdr->sh_link * mp->hdr.e_shentsize);
2540
2541         mp->nsyms = mp->symhdr->sh_size / mp->symhdr->sh_entsize;
2542         mp->hashsize = kobj_gethashsize(mp->nsyms);
2543
2544         /*
2545          * Allocate space for the symbol table, buckets, chains, and strings.
2546          */
2547         mp->symsize = mp->symhdr->sh_size +
2548             (mp->hashsize + mp->nsyms) * sizeof (symid_t) + mp->strhdr->sh_size;
2549         mp->symspace = kobj_zalloc(mp->symsize, KM_WAIT|KM_SCRATCH);
2550
2551         mp->symtbl = mp->symspace;
2552         mp->buckets = (symid_t *)(mp->symtbl + mp->symhdr->sh_size);
2553         mp->chains = mp->buckets + mp->hashsize;
2554         mp->strings = (char *)(mp->chains + mp->nsyms);
2555
2556         if (kobj_read_file(file, mp->symtbl,
2557             mp->symhdr->sh_size, mp->symhdr->sh_offset) < 0 ||
2558             kobj_read_file(file, mp->strings,
2559             mp->strhdr->sh_size, mp->strhdr->sh_offset) < 0)
2560                 return (-1);
2561
2562         /*
2563          * loop through the symbol table adjusting values to account
2564          * for where each section got loaded into memory.  Also
2565          * fill in the hash table.
2566          */
2567         for (i = 1; i < mp->nsyms; i++) {
2568                 sp = (Sym *)(mp->symtbl + i * mp->symhdr->sh_entsize);
2569                 if (sp->st_shndx < SHN_LORESERVE) {
2570                         if (sp->st_shndx >= mp->hdr.e_shnum) {
2571                                 _kobj_printf(ops, "%s bad shndx ",
2572                                     file->_name);
2573                                 _kobj_printf(ops, "in symbol %d\n", i);
2574                                 return (-1);
2575                         }
2576                         shp = (Shdr *)
2577                             (mp->shdrs +
2578                             sp->st_shndx * mp->hdr.e_shentsize);
2579                         if (!(mp->flags & KOBJ_EXEC))
2580                                 sp->st_value += shp->sh_addr;
2581                 }
2582
2583                 if (sp->st_name == 0 || sp->st_shndx == SHN_UNDEF)
2584                         continue;
2585                 if (sp->st_name >= mp->strhdr->sh_size)
2586                         return (-1);
2587
2588                 symname = mp->strings + sp->st_name;
2589
2590                 if (!(mp->flags & KOBJ_EXEC) &&
2591                     ELF_ST_BIND(sp->st_info) == STB_GLOBAL) {
2592                         ksp = kobj_lookup_all(mp, symname, 0);
2593
2594                         if (ksp && ELF_ST_BIND(ksp->st_info) == STB_GLOBAL &&
2595                             !kobj_suppress_warning(symname) &&
2596                             sp->st_shndx != SHN_UNDEF &&
2597                             sp->st_shndx != SHN_COMMON &&
2598                             ksp->st_shndx != SHN_UNDEF &&
2599                             ksp->st_shndx != SHN_COMMON) {
2600                                 /*
2601                                  * Unless this symbol is a stub, it's multiply
2602                                  * defined.  Multiply-defined symbols are
2603                                  * usually bad, but some objects (kmdb) have
2604                                  * a legitimate need to have their own
2605                                  * copies of common functions.
2606                                  */
2607                                 if ((standalone ||
2608                                     ksp->st_value < (uintptr_t)stubs_base ||
2609                                     ksp->st_value >= (uintptr_t)stubs_end) &&
2610                                     !(mp->flags & KOBJ_IGNMULDEF)) {
2611                                         _kobj_printf(ops,
2612                                             "%s symbol ", file->_name);
2613                                         _kobj_printf(ops,
2614                                             "%s multiply defined\n", symname);
2615                                 }
2616                         }
2617                 }
2618
2619                 sym_insert(mp, symname, i);
2620         }
2621
2622         return (0);
2623 }
2624
2625 static int
2626 get_ctf(struct module *mp, struct _buf *file)
2627 {
2628         char *shstrtab, *ctfdata;
2629         size_t shstrlen;
2630         Shdr *shp;
2631         uint_t i;
2632
2633         if (_moddebug & MODDEBUG_NOCTF)
2634                 return (0); /* do not attempt to even load CTF data */
2635
2636         if (mp->hdr.e_shstrndx >= mp->hdr.e_shnum) {
2637                 _kobj_printf(ops, "krtld: get_ctf: %s, ",
2638                     mp->filename);
2639                 _kobj_printf(ops, "corrupt e_shstrndx %u\n",
2640                     mp->hdr.e_shstrndx);
2641                 return (-1);
2642         }
2643
2644         shp = (Shdr *)(mp->shdrs + mp->hdr.e_shstrndx * mp->hdr.e_shentsize);
2645         shstrlen = shp->sh_size;
2646         shstrtab = kobj_alloc(shstrlen, KM_WAIT|KM_TMP);
2647
2648         if (kobj_read_file(file, shstrtab, shstrlen, shp->sh_offset) < 0) {
2649                 _kobj_printf(ops, "krtld: get_ctf: %s, ",
2650                     mp->filename);
2651                 _kobj_printf(ops, "error reading section %u\n",
2652                     mp->hdr.e_shstrndx);
2653                 kobj_free(shstrtab, shstrlen);
2654                 return (-1);
2655         }
2656
2657         for (i = 0; i < mp->hdr.e_shnum; i++) {
2658                 shp = (Shdr *)(mp->shdrs + i * mp->hdr.e_shentsize);
2659
2660                 if (shp->sh_size != 0 && shp->sh_name < shstrlen &&
2661                     strcmp(shstrtab + shp->sh_name, ".SUNW_ctf") == 0) {
2662                         ctfdata = kobj_alloc(shp->sh_size, KM_WAIT|KM_SCRATCH);
2663
2664                         if (kobj_read_file(file, ctfdata, shp->sh_size,
2665                             shp->sh_offset) < 0) {
2666                                 _kobj_printf(ops, "krtld: get_ctf: %s, error "
2667                                     "reading .SUNW_ctf data\n", mp->filename);
2668                                 kobj_free(ctfdata, shp->sh_size);
2669                                 kobj_free(shstrtab, shstrlen);
2670                                 return (-1);
2671                         }
2672
2673                         mp->ctfdata = ctfdata;
2674                         mp->ctfsize = shp->sh_size;
2675                         break;
2676                 }
2677         }
2678
2679         kobj_free(shstrtab, shstrlen);
2680         return (0);
2681 }
2682
2683 #define SHA1_DIGEST_LENGTH      20      /* SHA1 digest length in bytes */
2684
2685 /*
2686  * Return the hash of the ELF sections that are memory resident.
2687  * i.e. text and data.  We skip a SHT_NOBITS section since it occupies
2688  * no space in the file. We use SHA1 here since libelfsign uses
2689  * it and both places need to use the same algorithm.
2690  */
2691 static void
2692 crypto_es_hash(struct module *mp, char *hash, char *shstrtab)
2693 {
2694         uint_t shn;
2695         Shdr *shp;
2696         SHA1_CTX ctx;
2697
2698         SHA1Init(&ctx);
2699
2700         for (shn = 1; shn < mp->hdr.e_shnum; shn++) {
2701                 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
2702                 if (!(shp->sh_flags & SHF_ALLOC) || shp->sh_size == 0)
2703                         continue;
2704
2705                 /*
2706                  * The check should ideally be shp->sh_type == SHT_NOBITS.
2707                  * However, we can't do that check here as get_progbits()
2708                  * resets the type.
2709                  */
2710                 if (strcmp(shstrtab + shp->sh_name, ".bss") == 0)
2711                         continue;
2712 #ifdef  KOBJ_DEBUG
2713                 if (kobj_debug & D_DEBUG)
2714                         _kobj_printf(ops,
2715                             "krtld: crypto_es_hash: updating hash with"
2716                             " %s data size=%d\n", shstrtab + shp->sh_name,
2717                             shp->sh_size);
2718 #endif
2719                 ASSERT(shp->sh_addr != NULL);
2720                 SHA1Update(&ctx, (const uint8_t *)shp->sh_addr, shp->sh_size);
2721         }
2722
2723         SHA1Final((uchar_t *)hash, &ctx);
2724 }
2725
2726 /*
2727  * Get the .SUNW_signature section for the module, it it exists.
2728  *
2729  * This section exists only for crypto modules. None of the
2730  * primary modules have this section currently.
2731  */
2732 static void
2733 get_signature(struct module *mp, struct _buf *file)
2734 {
2735         char *shstrtab, *sigdata = NULL;
2736         size_t shstrlen;
2737         Shdr *shp;
2738         uint_t i;
2739
2740         if (mp->hdr.e_shstrndx >= mp->hdr.e_shnum) {
2741                 _kobj_printf(ops, "krtld: get_signature: %s, ",
2742                     mp->filename);
2743                 _kobj_printf(ops, "corrupt e_shstrndx %u\n",
2744                     mp->hdr.e_shstrndx);
2745                 return;
2746         }
2747
2748         shp = (Shdr *)(mp->shdrs + mp->hdr.e_shstrndx * mp->hdr.e_shentsize);
2749         shstrlen = shp->sh_size;
2750         shstrtab = kobj_alloc(shstrlen, KM_WAIT|KM_TMP);
2751
2752         if (kobj_read_file(file, shstrtab, shstrlen, shp->sh_offset) < 0) {
2753                 _kobj_printf(ops, "krtld: get_signature: %s, ",
2754                     mp->filename);
2755                 _kobj_printf(ops, "error reading section %u\n",
2756                     mp->hdr.e_shstrndx);
2757                 kobj_free(shstrtab, shstrlen);
2758                 return;
2759         }
2760
2761         for (i = 0; i < mp->hdr.e_shnum; i++) {
2762                 shp = (Shdr *)(mp->shdrs + i * mp->hdr.e_shentsize);
2763                 if (shp->sh_size != 0 && shp->sh_name < shstrlen &&
2764                     strcmp(shstrtab + shp->sh_name,
2765                     ELF_SIGNATURE_SECTION) == 0) {
2766                         filesig_vers_t filesig_version;
2767                         size_t sigsize = shp->sh_size + SHA1_DIGEST_LENGTH;
2768                         sigdata = kobj_alloc(sigsize, KM_WAIT|KM_SCRATCH);
2769
2770                         if (kobj_read_file(file, sigdata, shp->sh_size,
2771                             shp->sh_offset) < 0) {
2772                                 _kobj_printf(ops, "krtld: get_signature: %s,"
2773                                     " error reading .SUNW_signature data\n",
2774                                     mp->filename);
2775                                 kobj_free(sigdata, sigsize);
2776                                 kobj_free(shstrtab, shstrlen);
2777                                 return;
2778                         }
2779                         filesig_version = ((struct filesignatures *)sigdata)->
2780                             filesig_sig.filesig_version;
2781                         if (!(filesig_version == FILESIG_VERSION1 ||
2782                             filesig_version == FILESIG_VERSION3)) {
2783                                 /* skip versions we don't understand */
2784                                 kobj_free(sigdata, sigsize);
2785                                 kobj_free(shstrtab, shstrlen);
2786                                 return;
2787                         }
2788
2789                         mp->sigdata = sigdata;
2790                         mp->sigsize = sigsize;
2791                         break;
2792                 }
2793         }
2794
2795         if (sigdata != NULL) {
2796                 crypto_es_hash(mp, sigdata + shp->sh_size, shstrtab);
2797         }
2798
2799         kobj_free(shstrtab, shstrlen);
2800 }
2801
2802 static void
2803 add_dependent(struct module *mp, struct module *dep)
2804 {
2805         struct module_list *lp;
2806
2807         for (lp = mp->head; lp; lp = lp->next) {
2808                 if (lp->mp == dep)
2809                         return; /* already on the list */
2810         }
2811
2812         if (lp == NULL) {
2813                 lp = kobj_zalloc(sizeof (*lp), KM_WAIT);
2814
2815                 lp->mp = dep;
2816                 lp->next = NULL;
2817                 if (mp->tail)
2818                         mp->tail->next = lp;
2819                 else
2820                         mp->head = lp;
2821                 mp->tail = lp;
2822         }
2823 }
2824
2825 static int
2826 do_dependents(struct modctl *modp, char *modname, size_t modnamelen)
2827 {
2828         struct module *mp;
2829         struct modctl *req;
2830         char *d, *p, *q;
2831         int c;
2832         char *err_modname = NULL;
2833
2834         mp = modp->mod_mp;
2835
2836         if ((p = mp->depends_on) == NULL)
2837                 return (0);
2838
2839         for (;;) {
2840                 /*
2841                  * Skip space.
2842                  */
2843                 while (*p && (*p == ' ' || *p == '\t'))
2844                         p++;
2845                 /*
2846                  * Get module name.
2847                  */
2848                 d = p;
2849                 q = modname;
2850                 c = 0;
2851                 while (*p && *p != ' ' && *p != '\t') {
2852                         if (c < modnamelen - 1) {
2853                                 *q++ = *p;
2854                                 c++;
2855                         }
2856                         p++;
2857                 }
2858
2859                 if (q == modname)
2860                         break;
2861
2862                 if (c == modnamelen - 1) {
2863                         char *dep = kobj_alloc(p - d + 1, KM_WAIT|KM_TMP);
2864
2865                         (void) strncpy(dep, d,  p - d + 1);
2866                         dep[p - d] = '\0';
2867
2868                         _kobj_printf(ops, "%s: dependency ", modp->mod_modname);
2869                         _kobj_printf(ops, "'%s' too long ", dep);
2870                         _kobj_printf(ops, "(max %d chars)\n", modnamelen);
2871
2872                         kobj_free(dep, p - d + 1);
2873
2874                         return (-1);
2875                 }
2876
2877                 *q = '\0';
2878                 if ((req = mod_load_requisite(modp, modname)) == NULL) {
2879 #ifndef KOBJ_DEBUG
2880                         if (_moddebug & MODDEBUG_LOADMSG) {
2881 #endif  /* KOBJ_DEBUG */
2882                                 _kobj_printf(ops,
2883                                     "%s: unable to resolve dependency, ",
2884                                     modp->mod_modname);
2885                                 _kobj_printf(ops, "cannot load module '%s'\n",
2886                                     modname);
2887 #ifndef KOBJ_DEBUG
2888                         }
2889 #endif  /* KOBJ_DEBUG */
2890                         if (err_modname == NULL) {
2891                                 /*
2892                                  * This must be the same size as the modname
2893                                  * one.
2894                                  */
2895                                 err_modname = kobj_zalloc(MODMAXNAMELEN,
2896                                     KM_WAIT);
2897
2898                                 /*
2899                                  * We can use strcpy() here without fearing
2900                                  * the NULL terminator because the size of
2901                                  * err_modname is the same as one of modname,
2902                                  * and it's filled with zeros.
2903                                  */
2904                                 (void) strcpy(err_modname, modname);
2905                         }
2906                         continue;
2907                 }
2908
2909                 add_dependent(mp, req->mod_mp);
2910                 mod_release_mod(req);
2911
2912         }
2913
2914         if (err_modname != NULL) {
2915                 /*
2916                  * Copy the first module name where you detect an error to keep
2917                  * its behavior the same as before.
2918                  * This way keeps minimizing the memory use for error
2919                  * modules, and this might be important at boot time because
2920                  * the memory usage is a crucial factor for booting in most
2921                  * cases. You can expect more verbose messages when using
2922                  * a debug kernel or setting a bit in moddebug.
2923                  */
2924                 bzero(modname, MODMAXNAMELEN);
2925                 (void) strcpy(modname, err_modname);
2926                 kobj_free(err_modname, MODMAXNAMELEN);
2927                 return (-1);
2928         }
2929
2930         return (0);
2931 }
2932
2933 static int
2934 do_common(struct module *mp)
2935 {
2936         int err;
2937
2938         /*
2939          * first time through, assign all symbols defined in other
2940          * modules, and count up how much common space will be needed
2941          * (bss_size and bss_align)
2942          */
2943         if ((err = do_symbols(mp, 0)) < 0)
2944                 return (err);
2945         /*
2946          * increase bss_size by the maximum delta that could be
2947          * computed by the ALIGN below
2948          */
2949         mp->bss_size += mp->bss_align;
2950         if (mp->bss_size) {
2951                 if (standalone)
2952                         mp->bss = (uintptr_t)kobj_segbrk(&_edata, mp->bss_size,
2953                             MINALIGN, 0);
2954                 else
2955                         mp->bss = (uintptr_t)vmem_alloc(data_arena,
2956                             mp->bss_size, VM_SLEEP | VM_BESTFIT);
2957                 bzero((void *)mp->bss, mp->bss_size);
2958                 /* now assign addresses to all common symbols */
2959                 if ((err = do_symbols(mp, ALIGN(mp->bss, mp->bss_align))) < 0)
2960                         return (err);
2961         }
2962         return (0);
2963 }
2964
2965 static int
2966 do_symbols(struct module *mp, Elf64_Addr bss_base)
2967 {
2968         int bss_align;
2969         uintptr_t bss_ptr;
2970         int err;
2971         int i;
2972         Sym *sp, *sp1;
2973         char *name;
2974         int assign;
2975         int resolved = 1;
2976
2977         /*
2978          * Nothing left to do (optimization).
2979          */
2980         if (mp->flags & KOBJ_RESOLVED)
2981                 return (0);
2982
2983         assign = (bss_base) ? 1 : 0;
2984         bss_ptr = bss_base;
2985         bss_align = 0;
2986         err = 0;
2987
2988         for (i = 1; i < mp->nsyms; i++) {
2989                 sp = (Sym *)(mp->symtbl + mp->symhdr->sh_entsize * i);
2990                 /*
2991                  * we know that st_name is in bounds, since get_sections
2992                  * has already checked all of the symbols
2993                  */
2994                 name = mp->strings + sp->st_name;
2995                 if (sp->st_shndx != SHN_UNDEF && sp->st_shndx != SHN_COMMON)
2996                         continue;
2997 #if defined(__sparc)
2998                 /*
2999                  * Register symbols are ignored in the kernel
3000                  */
3001                 if (ELF_ST_TYPE(sp->st_info) == STT_SPARC_REGISTER) {
3002                         if (*name != '\0') {
3003                                 _kobj_printf(ops, "%s: named REGISTER symbol ",
3004                                     mp->filename);
3005                                 _kobj_printf(ops, "not supported '%s'\n",
3006                                     name);
3007                                 err = DOSYM_UNDEF;
3008                         }
3009                         continue;
3010                 }
3011 #endif  /* __sparc */
3012                 /*
3013                  * TLS symbols are ignored in the kernel
3014                  */
3015                 if (ELF_ST_TYPE(sp->st_info) == STT_TLS) {
3016                         _kobj_printf(ops, "%s: TLS symbol ",
3017                             mp->filename);
3018                         _kobj_printf(ops, "not supported '%s'\n",
3019                             name);
3020                         err = DOSYM_UNDEF;
3021                         continue;
3022                 }
3023
3024                 if (ELF_ST_BIND(sp->st_info) != STB_LOCAL) {
3025                         if ((sp1 = kobj_lookup_all(mp, name, 0)) != NULL) {
3026                                 sp->st_shndx = SHN_ABS;
3027                                 sp->st_value = sp1->st_value;
3028                                 continue;
3029                         }
3030                 }
3031
3032                 if (sp->st_shndx == SHN_UNDEF) {
3033                         resolved = 0;
3034
3035                         if (strncmp(name, sdt_prefix, strlen(sdt_prefix)) == 0)
3036                                 continue;
3037
3038                         /*
3039                          * If it's not a weak reference and it's
3040                          * not a primary object, it's an error.
3041                          * (Primary objects may take more than
3042                          * one pass to resolve)
3043                          */
3044                         if (!(mp->flags & KOBJ_PRIM) &&
3045                             ELF_ST_BIND(sp->st_info) != STB_WEAK) {
3046                                 _kobj_printf(ops, "%s: undefined symbol",
3047                                     mp->filename);
3048                                 _kobj_printf(ops, " '%s'\n", name);
3049                                 /*
3050                                  * Try to determine whether this symbol
3051                                  * represents a dependency on obsolete
3052                                  * unsafe driver support.  This is just
3053                                  * to make the warning more informative.
3054                                  */
3055                                 if (strcmp(name, "sleep") == 0 ||
3056                                     strcmp(name, "unsleep") == 0 ||
3057                                     strcmp(name, "wakeup") == 0 ||
3058                                     strcmp(name, "bsd_compat_ioctl") == 0 ||
3059                                     strcmp(name, "unsafe_driver") == 0 ||
3060                                     strncmp(name, "spl", 3) == 0 ||
3061                                     strncmp(name, "i_ddi_spl", 9) == 0)
3062                                         err = DOSYM_UNSAFE;
3063                                 if (err == 0)
3064                                         err = DOSYM_UNDEF;
3065                         }
3066                         continue;
3067                 }
3068                 /*
3069                  * It's a common symbol - st_value is the
3070                  * required alignment.
3071                  */
3072                 if (sp->st_value > bss_align)
3073                         bss_align = sp->st_value;
3074                 bss_ptr = ALIGN(bss_ptr, sp->st_value);
3075                 if (assign) {
3076                         sp->st_shndx = SHN_ABS;
3077                         sp->st_value = bss_ptr;
3078                 }
3079                 bss_ptr += sp->st_size;
3080         }
3081         if (err)
3082                 return (err);
3083         if (assign == 0 && mp->bss == NULL) {
3084                 mp->bss_align = bss_align;
3085                 mp->bss_size = bss_ptr;
3086         } else if (resolved) {
3087                 mp->flags |= KOBJ_RESOLVED;
3088         }
3089
3090         return (0);
3091 }
3092
3093 uint_t
3094 kobj_hash_name(const char *p)
3095 {
3096         uint_t g;
3097         uint_t hval;
3098
3099         hval = 0;
3100         while (*p) {
3101                 hval = (hval << 4) + *p++;
3102                 if ((g = (hval & 0xf0000000)) != 0)
3103                         hval ^= g >> 24;
3104                 hval &= ~g;
3105         }
3106         return (hval);
3107 }
3108
3109 /* look for name in all modules */
3110 uintptr_t
3111 kobj_getsymvalue(char *name, int kernelonly)
3112 {
3113         Sym             *sp;
3114         struct modctl   *modp;
3115         struct module   *mp;
3116         uintptr_t       value = 0;
3117
3118         if ((sp = kobj_lookup_kernel(name)) != NULL)
3119                 return ((uintptr_t)sp->st_value);
3120
3121         if (kernelonly)
3122                 return (0);     /* didn't find it in the kernel so give up */
3123
3124         mutex_enter(&mod_lock);
3125         modp = &modules;
3126         do {
3127                 mp = (struct module *)modp->mod_mp;
3128                 if (mp && !(mp->flags & KOBJ_PRIM) && modp->mod_loaded &&
3129                     (sp = lookup_one(mp, name))) {
3130                         value = (uintptr_t)sp->st_value;
3131                         break;
3132                 }
3133         } while ((modp = modp->mod_next) != &modules);
3134         mutex_exit(&mod_lock);
3135         return (value);
3136 }
3137
3138 /* look for a symbol near value. */
3139 char *
3140 kobj_getsymname(uintptr_t value, ulong_t *offset)
3141 {
3142         char *name = NULL;
3143         struct modctl *modp;
3144
3145         struct modctl_list *lp;
3146         struct module *mp;
3147
3148         /*
3149          * Loop through the primary kernel modules.
3150          */
3151         for (lp = kobj_lm_lookup(KOBJ_LM_PRIMARY); lp; lp = lp->modl_next) {
3152                 mp = mod(lp);
3153
3154                 if ((name = kobj_searchsym(mp, value, offset)) != NULL)
3155                         return (name);
3156         }
3157
3158         mutex_enter(&mod_lock);
3159         modp = &modules;
3160         do {
3161                 mp = (struct module *)modp->mod_mp;
3162                 if (mp && !(mp->flags & KOBJ_PRIM) && modp->mod_loaded &&
3163                     (name = kobj_searchsym(mp, value, offset)))
3164                         break;
3165         } while ((modp = modp->mod_next) != &modules);
3166         mutex_exit(&mod_lock);
3167         return (name);
3168 }
3169
3170 /* return address of symbol and size */
3171
3172 uintptr_t
3173 kobj_getelfsym(char *name, void *mp, int *size)
3174 {
3175         Sym *sp;
3176
3177         if (mp == NULL)
3178                 sp = kobj_lookup_kernel(name);
3179         else
3180                 sp = lookup_one(mp, name);
3181
3182         if (sp == NULL)
3183                 return (0);
3184
3185         *size = (int)sp->st_size;
3186         return ((uintptr_t)sp->st_value);
3187 }
3188
3189 uintptr_t
3190 kobj_lookup(struct module *mod, const char *name)
3191 {
3192         Sym *sp;
3193
3194         sp = lookup_one(mod, name);
3195
3196         if (sp == NULL)
3197                 return (0);
3198
3199         return ((uintptr_t)sp->st_value);
3200 }
3201
3202 char *
3203 kobj_searchsym(struct module *mp, uintptr_t value, ulong_t *offset)
3204 {
3205         Sym *symtabptr;
3206         char *strtabptr;
3207         int symnum;
3208         Sym *sym;
3209         Sym *cursym;
3210         uintptr_t curval;
3211
3212         *offset = (ulong_t)-1l;         /* assume not found */
3213         cursym  = NULL;
3214
3215         if (kobj_addrcheck(mp, (void *)value) != 0)
3216                 return (NULL);          /* not in this module */
3217
3218         strtabptr  = mp->strings;
3219         symtabptr  = (Sym *)mp->symtbl;
3220
3221         /*
3222          * Scan the module's symbol table for a symbol <= value
3223          */
3224         for (symnum = 1, sym = symtabptr + 1;
3225             symnum < mp->nsyms; symnum++, sym = (Sym *)
3226             ((uintptr_t)sym + mp->symhdr->sh_entsize)) {
3227                 if (ELF_ST_BIND(sym->st_info) != STB_GLOBAL) {
3228                         if (ELF_ST_BIND(sym->st_info) != STB_LOCAL)
3229                                 continue;
3230                         if (ELF_ST_TYPE(sym->st_info) != STT_OBJECT &&
3231                             ELF_ST_TYPE(sym->st_info) != STT_FUNC)
3232                                 continue;
3233                 }
3234
3235                 curval = (uintptr_t)sym->st_value;
3236
3237                 if (curval > value)
3238                         continue;
3239
3240                 /*
3241                  * If one or both are functions...
3242                  */
3243                 if (ELF_ST_TYPE(sym->st_info) == STT_FUNC || (cursym != NULL &&
3244                     ELF_ST_TYPE(cursym->st_info) == STT_FUNC)) {
3245                         /* Ignore if the address is out of the bounds */
3246                         if (value - sym->st_value >= sym->st_size)
3247                                 continue;
3248
3249                         if (cursym != NULL &&
3250                             ELF_ST_TYPE(cursym->st_info) == STT_FUNC) {
3251                                 /* Prefer the function to the non-function */
3252                                 if (ELF_ST_TYPE(sym->st_info) != STT_FUNC)
3253                                         continue;
3254
3255                                 /* Prefer the larger of the two functions */
3256                                 if (sym->st_size <= cursym->st_size)
3257                                         continue;
3258                         }
3259                 } else if (value - curval >= *offset) {
3260                         continue;
3261                 }
3262
3263                 *offset = (ulong_t)(value - curval);
3264                 cursym = sym;
3265         }
3266         if (cursym == NULL)
3267                 return (NULL);
3268
3269         return (strtabptr + cursym->st_name);
3270 }
3271
3272 Sym *
3273 kobj_lookup_all(struct module *mp, char *name, int include_self)
3274 {
3275         Sym *sp;
3276         struct module_list *mlp;
3277         struct modctl_list *clp;
3278         struct module *mmp;
3279
3280         if (include_self && (sp = lookup_one(mp, name)) != NULL)
3281                 return (sp);
3282
3283         for (mlp = mp->head; mlp; mlp = mlp->next) {
3284                 if ((sp = lookup_one(mlp->mp, name)) != NULL &&
3285                     ELF_ST_BIND(sp->st_info) != STB_LOCAL)
3286                         return (sp);
3287         }
3288
3289         /*
3290          * Loop through the primary kernel modules.
3291          */
3292         for (clp = kobj_lm_lookup(KOBJ_LM_PRIMARY); clp; clp = clp->modl_next) {
3293                 mmp = mod(clp);
3294
3295                 if (mmp == NULL || mp == mmp)
3296                         continue;
3297
3298                 if ((sp = lookup_one(mmp, name)) != NULL &&
3299                     ELF_ST_BIND(sp->st_info) != STB_LOCAL)
3300                         return (sp);
3301         }
3302         return (NULL);
3303 }
3304
3305 Sym *
3306 kobj_lookup_kernel(const char *name)
3307 {
3308         struct modctl_list *lp;
3309         struct module *mp;
3310         Sym *sp;
3311
3312         /*
3313          * Loop through the primary kernel modules.
3314          */
3315         for (lp = kobj_lm_lookup(KOBJ_LM_PRIMARY); lp; lp = lp->modl_next) {
3316                 mp = mod(lp);
3317
3318                 if (mp == NULL)
3319                         continue;
3320
3321                 if ((sp = lookup_one(mp, name)) != NULL)
3322                         return (sp);
3323         }
3324         return (NULL);
3325 }
3326
3327 static Sym *
3328 lookup_one(struct module *mp, const char *name)
3329 {
3330         symid_t *ip;
3331         char *name1;
3332         Sym *sp;
3333
3334         for (ip = &mp->buckets[kobj_hash_name(name) % mp->hashsize]; *ip;
3335             ip = &mp->chains[*ip]) {
3336                 sp = (Sym *)(mp->symtbl +
3337                     mp->symhdr->sh_entsize * *ip);
3338                 name1 = mp->strings + sp->st_name;
3339                 if (strcmp(name, name1) == 0 &&
3340                     ELF_ST_TYPE(sp->st_info) != STT_FILE &&
3341                     sp->st_shndx != SHN_UNDEF &&
3342                     sp->st_shndx != SHN_COMMON)
3343                         return (sp);
3344         }
3345         return (NULL);
3346 }
3347
3348 /*
3349  * Lookup a given symbol pointer in the module's symbol hash.  If the symbol
3350  * is hashed, return the symbol pointer; otherwise return NULL.
3351  */
3352 static Sym *
3353 sym_lookup(struct module *mp, Sym *ksp)
3354 {
3355         char *name = mp->strings + ksp->st_name;
3356         symid_t *ip;
3357         Sym *sp;
3358
3359         for (ip = &mp->buckets[kobj_hash_name(name) % mp->hashsize]; *ip;
3360             ip = &mp->chains[*ip]) {
3361                 sp = (Sym *)(mp->symtbl + mp->symhdr->sh_entsize * *ip);
3362                 if (sp == ksp)
3363                         return (ksp);
3364         }
3365         return (NULL);
3366 }
3367
3368 static void
3369 sym_insert(struct module *mp, char *name, symid_t index)
3370 {
3371         symid_t *ip;
3372
3373 #ifdef KOBJ_DEBUG
3374                 if (kobj_debug & D_SYMBOLS) {
3375                         static struct module *lastmp = NULL;
3376                         Sym *sp;
3377                         if (lastmp != mp) {
3378                                 _kobj_printf(ops,
3379                                     "krtld: symbol entry: file=%s\n",
3380                                     mp->filename);
3381                                 _kobj_printf(ops,
3382                                     "krtld:\tsymndx\tvalue\t\t"
3383                                     "symbol name\n");
3384                                 lastmp = mp;
3385                         }
3386                         sp = (Sym *)(mp->symtbl +
3387                             index * mp->symhdr->sh_entsize);
3388                         _kobj_printf(ops, "krtld:\t[%3d]", index);
3389                         _kobj_printf(ops, "\t0x%lx", sp->st_value);
3390                         _kobj_printf(ops, "\t%s\n", name);
3391                 }
3392
3393 #endif
3394         for (ip = &mp->buckets[kobj_hash_name(name) % mp->hashsize]; *ip;
3395             ip = &mp->chains[*ip]) {
3396                 ;
3397         }
3398         *ip = index;
3399 }
3400
3401 struct modctl *
3402 kobj_boot_mod_lookup(const char *modname)
3403 {
3404         struct modctl *mctl = kobj_modules;
3405
3406         do {
3407                 if (strcmp(modname, mctl->mod_modname) == 0)
3408                         return (mctl);
3409         } while ((mctl = mctl->mod_next) != kobj_modules);
3410
3411         return (NULL);
3412 }
3413
3414 /*
3415  * Determine if the module exists.
3416  */
3417 int
3418 kobj_path_exists(char *name, int use_path)
3419 {
3420         struct _buf *file;
3421
3422         file = kobj_open_path(name, use_path, 1);
3423 #ifdef  MODDIR_SUFFIX
3424         if (file == (struct _buf *)-1)
3425                 file = kobj_open_path(name, use_path, 0);
3426 #endif  /* MODDIR_SUFFIX */
3427         if (file == (struct _buf *)-1)
3428                 return (0);
3429         kobj_close_file(file);
3430         return (1);
3431 }
3432
3433 /*
3434  * fullname is dynamically allocated to be able to hold the
3435  * maximum size string that can be constructed from name.
3436  * path is exactly like the shell PATH variable.
3437  */
3438 struct _buf *
3439 kobj_open_path(char *name, int use_path, int use_moddir_suffix)
3440 {
3441         char *p, *q;
3442         char *pathp;
3443         char *pathpsave;
3444         char *fullname;
3445         int maxpathlen;
3446         struct _buf *file;
3447
3448 #if !defined(MODDIR_SUFFIX)
3449         use_moddir_suffix = B_FALSE;
3450 #endif
3451
3452         if (!use_path)
3453                 pathp = "";             /* use name as specified */
3454         else
3455                 pathp = kobj_module_path;
3456                                         /* use configured default path */
3457
3458         pathpsave = pathp;              /* keep this for error reporting */
3459
3460         /*
3461          * Allocate enough space for the largest possible fullname.
3462          * since path is of the form <directory> : <directory> : ...
3463          * we're potentially allocating a little more than we need to
3464          * but we'll allocate the exact amount when we find the right directory.
3465          * (The + 3 below is one for NULL terminator and one for the '/'
3466          * we might have to add at the beginning of path and one for
3467          * the '/' between path and name.)
3468          */
3469         maxpathlen = strlen(pathp) + strlen(name) + 3;
3470         /* sizeof includes null */
3471         maxpathlen += sizeof (slash_moddir_suffix_slash) - 1;
3472         fullname = kobj_zalloc(maxpathlen, KM_WAIT);
3473
3474         for (;;) {
3475                 p = fullname;
3476                 if (*pathp != '\0' && *pathp != '/')
3477                         *p++ = '/';     /* path must start with '/' */
3478                 while (*pathp && *pathp != ':' && *pathp != ' ')
3479                         *p++ = *pathp++;
3480                 if (p != fullname && p[-1] != '/')
3481                         *p++ = '/';
3482                 if (use_moddir_suffix) {
3483                         char *b = basename(name);
3484                         char *s;
3485
3486                         /* copy everything up to the base name */
3487                         q = name;
3488                         while (q != b && *q)
3489                                 *p++ = *q++;
3490                         s = slash_moddir_suffix_slash;
3491                         while (*s)
3492                                 *p++ = *s++;
3493                         /* copy the rest */
3494                         while (*b)
3495                                 *p++ = *b++;
3496                 } else {
3497                         q = name;
3498                         while (*q)
3499                                 *p++ = *q++;
3500                 }
3501                 *p = 0;
3502                 if ((file = kobj_open_file(fullname)) != (struct _buf *)-1) {
3503                         kobj_free(fullname, maxpathlen);
3504                         return (file);
3505                 }
3506                 while (*pathp == ' ' || *pathp == ':')
3507                         pathp++;
3508                 if (*pathp == 0)
3509                         break;
3510
3511         }
3512         kobj_free(fullname, maxpathlen);
3513         if (_moddebug & MODDEBUG_ERRMSG) {
3514                 _kobj_printf(ops, "can't open %s,", name);
3515                 _kobj_printf(ops, " path is %s\n", pathpsave);
3516         }
3517         return ((struct _buf *)-1);
3518 }
3519
3520 intptr_t
3521 kobj_open(char *filename)
3522 {
3523         struct vnode *vp;
3524         int fd;
3525
3526         if (_modrootloaded) {
3527                 struct kobjopen_tctl *ltp = kobjopen_alloc(filename);
3528                 int Errno;
3529
3530                 /*
3531                  * Hand off the open to a thread who has a
3532                  * stack size capable handling the request.
3533                  */
3534                 if (curthread != &t0) {
3535                         (void) thread_create(NULL, DEFAULTSTKSZ * 2,
3536                             kobjopen_thread, ltp, 0, &p0, TS_RUN, maxclsyspri);
3537                         sema_p(&ltp->sema);
3538                         Errno = ltp->Errno;
3539                         vp = ltp->vp;
3540                 } else {
3541                         /*
3542                          * 1098067: module creds should not be those of the
3543                          * caller
3544                          */
3545                         cred_t *saved_cred = curthread->t_cred;
3546                         curthread->t_cred = kcred;
3547                         Errno = vn_openat(filename, UIO_SYSSPACE, FREAD, 0, &vp,
3548                             0, 0, rootdir, -1);
3549                         curthread->t_cred = saved_cred;
3550                 }
3551                 kobjopen_free(ltp);
3552
3553                 if (Errno) {
3554                         if (_moddebug & MODDEBUG_ERRMSG) {
3555                                 _kobj_printf(ops,
3556                                     "kobj_open: vn_open of %s fails, ",
3557                                     filename);
3558                                 _kobj_printf(ops, "Errno = %d\n", Errno);
3559                         }
3560                         return (-1);
3561                 } else {
3562                         if (_moddebug & MODDEBUG_ERRMSG) {
3563                                 _kobj_printf(ops, "kobj_open: '%s'", filename);
3564                                 _kobj_printf(ops, " vp = %p\n", vp);
3565                         }
3566                         return ((intptr_t)vp);
3567                 }
3568         } else {
3569                 fd = kobj_boot_open(filename, 0);
3570
3571                 if (_moddebug & MODDEBUG_ERRMSG) {
3572                         if (fd < 0)
3573                                 _kobj_printf(ops,
3574                                     "kobj_open: can't open %s\n", filename);
3575                         else {
3576                                 _kobj_printf(ops, "kobj_open: '%s'", filename);
3577                                 _kobj_printf(ops, " descr = 0x%x\n", fd);
3578                         }
3579                 }
3580                 return ((intptr_t)fd);
3581         }
3582 }
3583
3584 /*
3585  * Calls to kobj_open() are handled off to this routine as a separate thread.
3586  */
3587 static void
3588 kobjopen_thread(struct kobjopen_tctl *ltp)
3589 {
3590         kmutex_t        cpr_lk;
3591         callb_cpr_t     cpr_i;
3592
3593         mutex_init(&cpr_lk, NULL, MUTEX_DEFAULT, NULL);
3594         CALLB_CPR_INIT(&cpr_i, &cpr_lk, callb_generic_cpr, "kobjopen");
3595         ltp->Errno = vn_open(ltp->name, UIO_SYSSPACE, FREAD, 0, &(ltp->vp),
3596             0, 0);
3597         sema_v(&ltp->sema);
3598         mutex_enter(&cpr_lk);
3599         CALLB_CPR_EXIT(&cpr_i);
3600         mutex_destroy(&cpr_lk);
3601         thread_exit();
3602 }
3603
3604 /*
3605  * allocate and initialize a kobjopen thread structure
3606  */
3607 static struct kobjopen_tctl *
3608 kobjopen_alloc(char *filename)
3609 {
3610         struct kobjopen_tctl *ltp = kmem_zalloc(sizeof (*ltp), KM_SLEEP);
3611
3612         ASSERT(filename != NULL);
3613
3614         ltp->name = kmem_alloc(strlen(filename) + 1, KM_SLEEP);
3615         bcopy(filename, ltp->name, strlen(filename) + 1);
3616         sema_init(&ltp->sema, 0, NULL, SEMA_DEFAULT, NULL);
3617         return (ltp);
3618 }
3619
3620 /*
3621  * free a kobjopen thread control structure
3622  */
3623 static void
3624 kobjopen_free(struct kobjopen_tctl *ltp)
3625 {
3626         sema_destroy(&ltp->sema);
3627         kmem_free(ltp->name, strlen(ltp->name) + 1);
3628         kmem_free(ltp, sizeof (*ltp));
3629 }
3630
3631 int
3632 kobj_read(intptr_t descr, char *buf, uint_t size, uint_t offset)
3633 {
3634         int stat;
3635         ssize_t resid;
3636
3637         if (_modrootloaded) {
3638                 if ((stat = vn_rdwr(UIO_READ, (struct vnode *)descr, buf, size,
3639                     (offset_t)offset, UIO_SYSSPACE, 0, (rlim64_t)0, CRED(),
3640                     &resid)) != 0) {
3641                         _kobj_printf(ops,
3642                             "vn_rdwr failed with error 0x%x\n", stat);
3643                         return (-1);
3644                 }
3645                 return (size - resid);
3646         } else {
3647                 int count = 0;
3648
3649                 if (kobj_boot_seek((int)descr, (off_t)0, offset) != 0) {
3650                         _kobj_printf(ops,
3651                             "kobj_read: seek 0x%x failed\n", offset);
3652                         return (-1);
3653                 }
3654
3655                 count = kobj_boot_read((int)descr, buf, size);
3656                 if (count < size) {
3657                         if (_moddebug & MODDEBUG_ERRMSG) {
3658                                 _kobj_printf(ops,
3659                                     "kobj_read: req %d bytes, ", size);
3660                                 _kobj_printf(ops, "got %d\n", count);
3661                         }
3662                 }
3663                 return (count);
3664         }
3665 }
3666
3667 void
3668 kobj_close(intptr_t descr)
3669 {
3670         if (_moddebug & MODDEBUG_ERRMSG)
3671                 _kobj_printf(ops, "kobj_close: 0x%lx\n", descr);
3672
3673         if (_modrootloaded) {
3674                 struct vnode *vp = (struct vnode *)descr;
3675                 (void) VOP_CLOSE(vp, FREAD, 1, (offset_t)0, CRED(), NULL);
3676                 VN_RELE(vp);
3677         } else
3678                 (void) kobj_boot_close((int)descr);
3679 }
3680
3681 int
3682 kobj_fstat(intptr_t descr, struct bootstat *buf)
3683 {
3684         if (buf == NULL)
3685                 return (-1);
3686
3687         if (_modrootloaded) {
3688                 vattr_t vattr;
3689                 struct vnode *vp = (struct vnode *)descr;
3690                 if (VOP_GETATTR(vp, &vattr, 0, kcred, NULL) != 0)
3691                         return (-1);
3692
3693                 /*
3694                  * The vattr and bootstat structures are similar, but not
3695                  * identical.  We do our best to fill in the bootstat structure
3696                  * from the contents of vattr (transfering only the ones that
3697                  * are obvious.
3698                  */
3699
3700                 buf->st_mode = (uint32_t)vattr.va_mode;
3701                 buf->st_nlink = (uint32_t)vattr.va_nlink;
3702                 buf->st_uid = (int32_t)vattr.va_uid;
3703                 buf->st_gid = (int32_t)vattr.va_gid;
3704                 buf->st_rdev = (uint64_t)vattr.va_rdev;
3705                 buf->st_size = (uint64_t)vattr.va_size;
3706                 buf->st_atim.tv_sec = (int64_t)vattr.va_atime.tv_sec;
3707                 buf->st_atim.tv_nsec = (int64_t)vattr.va_atime.tv_nsec;
3708                 buf->st_mtim.tv_sec = (int64_t)vattr.va_mtime.tv_sec;
3709                 buf->st_mtim.tv_nsec = (int64_t)vattr.va_mtime.tv_nsec;
3710                 buf->st_ctim.tv_sec = (int64_t)vattr.va_ctime.tv_sec;
3711                 buf->st_ctim.tv_nsec = (int64_t)vattr.va_ctime.tv_nsec;
3712                 buf->st_blksize = (int32_t)vattr.va_blksize;
3713                 buf->st_blocks = (int64_t)vattr.va_nblocks;
3714
3715                 return (0);
3716         }
3717
3718         return (kobj_boot_fstat((int)descr, buf));
3719 }
3720
3721
3722 struct _buf *
3723 kobj_open_file(char *name)
3724 {
3725         struct _buf *file;
3726         struct compinfo cbuf;
3727         intptr_t fd;
3728
3729         if ((fd = kobj_open(name)) == -1) {
3730                 return ((struct _buf *)-1);
3731         }
3732
3733         file = kobj_zalloc(sizeof (struct _buf), KM_WAIT|KM_TMP);
3734         file->_fd = fd;
3735         file->_name = kobj_alloc(strlen(name)+1, KM_WAIT|KM_TMP);
3736         file->_cnt = file->_size = file->_off = 0;
3737         file->_ln = 1;
3738         file->_ptr = file->_base;
3739         (void) strcpy(file->_name, name);
3740
3741         /*
3742          * Before root is mounted, we must check
3743          * for a compressed file and do our own
3744          * buffering.
3745          */
3746         if (_modrootloaded) {
3747                 file->_base = kobj_zalloc(MAXBSIZE, KM_WAIT);
3748                 file->_bsize = MAXBSIZE;
3749
3750                 /* Check if the file is compressed */
3751                 file->_iscmp = kobj_is_compressed(fd);
3752         } else {
3753                 if (kobj_boot_compinfo(fd, &cbuf) != 0) {
3754                         kobj_close_file(file);
3755                         return ((struct _buf *)-1);
3756                 }
3757                 file->_iscmp = cbuf.iscmp;
3758                 if (file->_iscmp) {
3759                         if (kobj_comp_setup(file, &cbuf) != 0) {
3760                                 kobj_close_file(file);
3761                                 return ((struct _buf *)-1);
3762                         }
3763                 } else {
3764                         file->_base = kobj_zalloc(cbuf.blksize, KM_WAIT|KM_TMP);
3765                         file->_bsize = cbuf.blksize;
3766                 }
3767         }
3768         return (file);
3769 }
3770
3771 static int
3772 kobj_comp_setup(struct _buf *file, struct compinfo *cip)
3773 {
3774         struct comphdr *hdr;
3775
3776         /*
3777          * read the compressed image into memory,
3778          * so we can deompress from there
3779          */
3780         file->_dsize = cip->fsize;
3781         file->_dbuf = kobj_alloc(cip->fsize, KM_WAIT|KM_TMP);
3782         if (kobj_read(file->_fd, file->_dbuf, cip->fsize, 0) != cip->fsize) {
3783                 kobj_free(file->_dbuf, cip->fsize);
3784                 return (-1);
3785         }
3786
3787         hdr = kobj_comphdr(file);
3788         if (hdr->ch_magic != CH_MAGIC_ZLIB || hdr->ch_version != CH_VERSION ||
3789             hdr->ch_algorithm != CH_ALG_ZLIB || hdr->ch_fsize == 0 ||
3790             !ISP2(hdr->ch_blksize)) {
3791                 kobj_free(file->_dbuf, cip->fsize);
3792                 return (-1);
3793         }
3794         file->_base = kobj_alloc(hdr->ch_blksize, KM_WAIT|KM_TMP);
3795         file->_bsize = hdr->ch_blksize;
3796         return (0);
3797 }
3798
3799 void
3800 kobj_close_file(struct _buf *file)
3801 {
3802         kobj_close(file->_fd);
3803         if (file->_base != NULL)
3804                 kobj_free(file->_base, file->_bsize);
3805         if (file->_dbuf != NULL)
3806                 kobj_free(file->_dbuf, file->_dsize);
3807         kobj_free(file->_name, strlen(file->_name)+1);
3808         kobj_free(file, sizeof (struct _buf));
3809 }
3810
3811 int
3812 kobj_read_file(struct _buf *file, char *buf, uint_t size, uint_t off)
3813 {
3814         int b_size, c_size;
3815         int b_off;      /* Offset into buffer for start of bcopy */
3816         int count = 0;
3817         int page_addr;
3818
3819         if (_moddebug & MODDEBUG_ERRMSG) {
3820                 _kobj_printf(ops, "kobj_read_file: size=%x,", size);
3821                 _kobj_printf(ops, " offset=%x at", off);
3822                 _kobj_printf(ops, " buf=%x\n", buf);
3823         }
3824
3825         /*
3826          * Handle compressed (gzip for now) file here. First get the
3827          * compressed size, then read the image into memory and finally
3828          * call zlib to decompress the image at the supplied memory buffer.
3829          */
3830         if (file->_iscmp == CH_MAGIC_GZIP) {
3831                 ulong_t dlen;
3832                 vattr_t vattr;
3833                 struct vnode *vp = (struct vnode *)file->_fd;
3834                 ssize_t resid;
3835                 int err = 0;
3836
3837                 if (VOP_GETATTR(vp, &vattr, 0, kcred, NULL) != 0)
3838                         return (-1);
3839
3840                 file->_dbuf = kobj_alloc(vattr.va_size, KM_WAIT|KM_TMP);
3841                 file->_dsize = vattr.va_size;
3842
3843                 /* Read the compressed file into memory */
3844                 if ((err = vn_rdwr(UIO_READ, vp, file->_dbuf, vattr.va_size,
3845                     (offset_t)(0), UIO_SYSSPACE, 0, (rlim64_t)0, CRED(),
3846                     &resid)) != 0) {
3847
3848                         _kobj_printf(ops, "kobj_read_file :vn_rdwr() failed, "
3849                             "error code 0x%x\n", err);
3850                         return (-1);
3851                 }
3852
3853                 dlen = size;
3854
3855                 /* Decompress the image at the supplied memory buffer */
3856                 if ((err = z_uncompress(buf, &dlen, file->_dbuf,
3857                     vattr.va_size)) != Z_OK) {
3858                         _kobj_printf(ops, "kobj_read_file: z_uncompress "
3859                             "failed, error code : 0x%x\n", err);
3860                         return (-1);
3861                 }
3862
3863                 if (dlen != size) {
3864                         _kobj_printf(ops, "kobj_read_file: z_uncompress "
3865                             "failed to uncompress (size returned 0x%x , "
3866                             "expected size: 0x%x)\n", dlen, size);
3867                         return (-1);
3868                 }
3869
3870                 return (0);
3871         }
3872
3873         while (size) {
3874                 page_addr = F_PAGE(file, off);
3875                 b_size = file->_size;
3876                 /*
3877                  * If we have the filesystem page the caller's referring to
3878                  * and we have something in the buffer,
3879                  * satisfy as much of the request from the buffer as we can.
3880                  */
3881                 if (page_addr == file->_off && b_size > 0) {
3882                         b_off = B_OFFSET(file, off);
3883                         c_size = b_size - b_off;
3884                         /*
3885                          * If there's nothing to copy, we're at EOF.
3886                          */
3887                         if (c_size <= 0)
3888                                 break;
3889                         if (c_size > size)
3890                                 c_size = size;
3891                         if (buf) {
3892                                 if (_moddebug & MODDEBUG_ERRMSG)
3893                                         _kobj_printf(ops, "copying %x bytes\n",
3894                                             c_size);
3895                                 bcopy(file->_base+b_off, buf, c_size);
3896                                 size -= c_size;
3897                                 off += c_size;
3898                                 buf += c_size;
3899                                 count += c_size;
3900                         } else {
3901                                 _kobj_printf(ops, "kobj_read: system error");
3902                                 count = -1;
3903                                 break;
3904                         }
3905                 } else {
3906                         /*
3907                          * If the caller's offset is page aligned and
3908                          * the caller want's at least a filesystem page and
3909                          * the caller provided a buffer,
3910                          * read directly into the caller's buffer.
3911                          */
3912                         if (page_addr == off &&
3913                             (c_size = F_BLKS(file, size)) && buf) {
3914                                 c_size = kobj_read_blks(file, buf, c_size,
3915                                     page_addr);
3916                                 if (c_size < 0) {
3917                                         count = -1;
3918                                         break;
3919                                 }
3920                                 count += c_size;
3921                                 if (c_size != F_BLKS(file, size))
3922                                         break;
3923                                 size -= c_size;
3924                                 off += c_size;
3925                                 buf += c_size;
3926                         /*
3927                          * Otherwise, read into our buffer and copy next time
3928                          * around the loop.
3929                          */
3930                         } else {
3931                                 file->_off = page_addr;
3932                                 c_size = kobj_read_blks(file, file->_base,
3933                                     file->_bsize, page_addr);
3934                                 file->_ptr = file->_base;
3935                                 file->_cnt = c_size;
3936                                 file->_size = c_size;
3937                                 /*
3938                                  * If a _filbuf call or nothing read, break.
3939                                  */
3940                                 if (buf == NULL || c_size <= 0) {
3941                                         count = c_size;
3942                                         break;
3943                                 }
3944                         }
3945                         if (_moddebug & MODDEBUG_ERRMSG)
3946                                 _kobj_printf(ops, "read %x bytes\n", c_size);
3947                 }
3948         }
3949         if (_moddebug & MODDEBUG_ERRMSG)
3950                 _kobj_printf(ops, "count = %x\n", count);
3951
3952         return (count);
3953 }
3954
3955 static int
3956 kobj_read_blks(struct _buf *file, char *buf, uint_t size, uint_t off)
3957 {
3958         int ret;
3959
3960         ASSERT(B_OFFSET(file, size) == 0 && B_OFFSET(file, off) == 0);
3961         if (file->_iscmp) {
3962                 uint_t blks;
3963                 int nret;
3964
3965                 ret = 0;
3966                 for (blks = size / file->_bsize; blks != 0; blks--) {
3967                         nret = kobj_uncomp_blk(file, buf, off);
3968                         if (nret == -1)
3969                                 return (-1);
3970                         buf += nret;
3971                         off += nret;
3972                         ret += nret;
3973                         if (nret < file->_bsize)
3974                                 break;
3975                 }
3976         } else
3977                 ret = kobj_read(file->_fd, buf, size, off);
3978         return (ret);
3979 }
3980
3981 static int
3982 kobj_uncomp_blk(struct _buf *file, char *buf, uint_t off)
3983 {
3984         struct comphdr *hdr = kobj_comphdr(file);
3985         ulong_t dlen, slen;
3986         caddr_t src;
3987         int i;
3988
3989         dlen = file->_bsize;
3990         i = off / file->_bsize;
3991         src = file->_dbuf + hdr->ch_blkmap[i];
3992         if (i == hdr->ch_fsize / file->_bsize)
3993                 slen = file->_dsize - hdr->ch_blkmap[i];
3994         else
3995                 slen = hdr->ch_blkmap[i + 1] - hdr->ch_blkmap[i];
3996         if (z_uncompress(buf, &dlen, src, slen) != Z_OK)
3997                 return (-1);
3998         return (dlen);
3999 }
4000
4001 int
4002 kobj_filbuf(struct _buf *f)
4003 {
4004         if (kobj_read_file(f, NULL, f->_bsize, f->_off + f->_size) > 0)
4005                 return (kobj_getc(f));
4006         return (-1);
4007 }
4008
4009 void
4010 kobj_free(void *address, size_t size)
4011 {
4012         if (standalone)
4013                 return;
4014
4015         kmem_free(address, size);
4016         kobj_stat.nfree_calls++;
4017         kobj_stat.nfree += size;
4018 }
4019
4020 void *
4021 kobj_zalloc(size_t size, int flag)
4022 {
4023         void *v;
4024
4025         if ((v = kobj_alloc(size, flag)) != 0) {
4026                 bzero(v, size);
4027         }
4028
4029         return (v);
4030 }
4031
4032 void *
4033 kobj_alloc(size_t size, int flag)
4034 {
4035         /*
4036          * If we are running standalone in the
4037          * linker, we ask boot for memory.
4038          * Either it's temporary memory that we lose
4039          * once boot is mapped out or we allocate it
4040          * permanently using the dynamic data segment.
4041          */
4042         if (standalone) {
4043 #if defined(_OBP)
4044                 if (flag & (KM_TMP | KM_SCRATCH))
4045                         return (bop_temp_alloc(size, MINALIGN));
4046 #else
4047                 if (flag & (KM_TMP | KM_SCRATCH))
4048                         return (BOP_ALLOC(ops, 0, size, MINALIGN));
4049 #endif
4050                 return (kobj_segbrk(&_edata, size, MINALIGN, 0));
4051         }
4052
4053         kobj_stat.nalloc_calls++;
4054         kobj_stat.nalloc += size;
4055
4056         return (kmem_alloc(size, (flag & KM_NOWAIT) ? KM_NOSLEEP : KM_SLEEP));
4057 }
4058
4059 /*
4060  * Allow the "mod" system to sync up with the work
4061  * already done by kobj during the initial loading
4062  * of the kernel.  This also gives us a chance
4063  * to reallocate memory that belongs to boot.
4064  */
4065 void
4066 kobj_sync(void)
4067 {
4068         struct modctl_list *lp, **lpp;
4069
4070         /*
4071          * The module path can be set in /etc/system via 'moddir' commands
4072          */
4073         if (default_path != NULL)
4074                 kobj_module_path = default_path;
4075         else
4076                 default_path = kobj_module_path;
4077
4078         ksyms_arena = vmem_create("ksyms", NULL, 0, sizeof (uint64_t),
4079             segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP);
4080
4081         ctf_arena = vmem_create("ctf", NULL, 0, sizeof (uint_t),
4082             segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP);
4083
4084         /*
4085          * Move symbol tables from boot memory to ksyms_arena.
4086          */
4087         for (lpp = kobj_linkmaps; *lpp != NULL; lpp++) {
4088                 for (lp = *lpp; lp != NULL; lp = lp->modl_next)
4089                         kobj_export_module(mod(lp));
4090         }
4091 }
4092
4093 caddr_t
4094 kobj_segbrk(caddr_t *spp, size_t size, size_t align, caddr_t limit)
4095 {
4096         uintptr_t va, pva;
4097         size_t alloc_pgsz = kobj_mmu_pagesize;
4098         size_t alloc_align = BO_NO_ALIGN;
4099         size_t alloc_size;
4100
4101         /*
4102          * If we are using "large" mappings for the kernel,
4103          * request aligned memory from boot using the
4104          * "large" pagesize.
4105          */
4106         if (lg_pagesize) {
4107                 alloc_align = lg_pagesize;
4108                 alloc_pgsz = lg_pagesize;
4109         }
4110
4111 #if defined(__sparc)
4112         /* account for redzone */
4113         if (limit)
4114                 limit -= alloc_pgsz;
4115 #endif  /* __sparc */
4116
4117         va = ALIGN((uintptr_t)*spp, align);
4118         pva = P2ROUNDUP((uintptr_t)*spp, alloc_pgsz);
4119         /*
4120          * Need more pages?
4121          */
4122         if (va + size > pva) {
4123                 uintptr_t npva;
4124
4125                 alloc_size = P2ROUNDUP(size - (pva - va), alloc_pgsz);
4126                 /*
4127                  * Check for overlapping segments.
4128                  */
4129                 if (limit && limit <= *spp + alloc_size) {
4130                         return ((caddr_t)0);
4131                 }
4132
4133                 npva = (uintptr_t)BOP_ALLOC(ops, (caddr_t)pva,
4134                     alloc_size, alloc_align);
4135
4136                 if (npva == NULL) {
4137                         _kobj_printf(ops, "BOP_ALLOC failed, 0x%lx bytes",
4138                             alloc_size);
4139                         _kobj_printf(ops, " aligned %lx", alloc_align);
4140                         _kobj_printf(ops, " at 0x%lx\n", pva);
4141                         return (NULL);
4142                 }
4143         }
4144         *spp = (caddr_t)(va + size);
4145
4146         return ((caddr_t)va);
4147 }
4148
4149 /*
4150  * Calculate the number of output hash buckets.
4151  * We use the next prime larger than n / 4,
4152  * so the average hash chain is about 4 entries.
4153  * More buckets would just be a waste of memory.
4154  */
4155 uint_t
4156 kobj_gethashsize(uint_t n)
4157 {
4158         int f;
4159         int hsize = MAX(n / 4, 2);
4160
4161         for (f = 2; f * f <= hsize; f++)
4162                 if (hsize % f == 0)
4163                         hsize += f = 1;
4164
4165         return (hsize);
4166 }
4167
4168 /*
4169  * Get the file size.
4170  *
4171  * Before root is mounted, files are compressed in the boot_archive ramdisk
4172  * (in the memory). kobj_fstat would return the compressed file size.
4173  * In order to get the uncompressed file size, read the file to the end and
4174  * count its size.
4175  */
4176 int
4177 kobj_get_filesize(struct _buf *file, uint64_t *size)
4178 {
4179         int err = 0;
4180         ssize_t resid;
4181         uint32_t buf;
4182
4183         if (_modrootloaded) {
4184                 struct bootstat bst;
4185
4186                 if (kobj_fstat(file->_fd, &bst) != 0)
4187                         return (EIO);
4188                 *size = bst.st_size;
4189
4190                 if (file->_iscmp == CH_MAGIC_GZIP) {
4191                         /*
4192                          * Read the last 4 bytes of the compressed (gzip)
4193                          * image to get the size of its uncompressed
4194                          * version.
4195                          */
4196                         if ((err = vn_rdwr(UIO_READ, (struct vnode *)file->_fd,
4197                             (char *)(&buf), 4, (offset_t)(*size - 4),
4198                             UIO_SYSSPACE, 0, (rlim64_t)0, CRED(), &resid))
4199                             != 0) {
4200                                 _kobj_printf(ops, "kobj_get_filesize: "
4201                                     "vn_rdwr() failed with error 0x%x\n", err);
4202                                 return (-1);
4203                         }
4204
4205                         *size =  (uint64_t)buf;
4206                 }
4207         } else {
4208
4209 #if defined(_OBP)
4210                 struct bootstat bsb;
4211
4212                 if (file->_iscmp) {
4213                         struct comphdr *hdr = kobj_comphdr(file);
4214
4215                         *size = hdr->ch_fsize;
4216                 } else if (kobj_boot_fstat(file->_fd, &bsb) != 0)
4217                         return (EIO);
4218                 else
4219                         *size = bsb.st_size;
4220 #else
4221                 char *buf;
4222                 int count;
4223                 uint64_t offset = 0;
4224
4225                 buf = kmem_alloc(MAXBSIZE, KM_SLEEP);
4226                 do {
4227                         count = kobj_read_file(file, buf, MAXBSIZE, offset);
4228                         if (count < 0) {
4229                                 kmem_free(buf, MAXBSIZE);
4230                                 return (EIO);
4231                         }
4232                         offset += count;
4233                 } while (count == MAXBSIZE);
4234                 kmem_free(buf, MAXBSIZE);
4235
4236                 *size = offset;
4237 #endif
4238         }
4239
4240         return (0);
4241 }
4242
4243 static char *
4244 basename(char *s)
4245 {
4246         char *p, *q;
4247
4248         q = NULL;
4249         p = s;
4250         do {
4251                 if (*p == '/')
4252                         q = p;
4253         } while (*p++);
4254         return (q ? q + 1 : s);
4255 }
4256
4257 void
4258 kobj_stat_get(kobj_stat_t *kp)
4259 {
4260         *kp = kobj_stat;
4261 }
4262
4263 int
4264 kobj_getpagesize()
4265 {
4266         return (lg_pagesize);
4267 }
4268
4269 void
4270 kobj_textwin_alloc(struct module *mp)
4271 {
4272         ASSERT(MUTEX_HELD(&mod_lock));
4273
4274         if (mp->textwin != NULL)
4275                 return;
4276
4277         /*
4278          * If the text is not contained in the heap, then it is not contained
4279          * by a writable mapping.  (Specifically, it's on the nucleus page.)
4280          * We allocate a read/write mapping for this module's text to allow
4281          * the text to be patched without calling hot_patch_kernel_text()
4282          * (which is quite slow).
4283          */
4284         if (!vmem_contains(heaptext_arena, mp->text, mp->text_size)) {
4285                 uintptr_t text = (uintptr_t)mp->text;
4286                 uintptr_t size = (uintptr_t)mp->text_size;
4287                 uintptr_t i;
4288                 caddr_t va;
4289                 size_t sz = ((text + size + PAGESIZE - 1) & PAGEMASK) -
4290                     (text & PAGEMASK);
4291
4292                 va = mp->textwin_base = vmem_alloc(heap_arena, sz, VM_SLEEP);
4293
4294                 for (i = text & PAGEMASK; i < text + size; i += PAGESIZE) {
4295                         hat_devload(kas.a_hat, va, PAGESIZE,
4296                             hat_getpfnum(kas.a_hat, (caddr_t)i),
4297                             PROT_READ | PROT_WRITE,
4298                             HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST);
4299                         va += PAGESIZE;
4300                 }
4301
4302                 mp->textwin = mp->textwin_base + (text & PAGEOFFSET);
4303         } else {
4304                 mp->textwin = mp->text;
4305         }
4306 }
4307
4308 void
4309 kobj_textwin_free(struct module *mp)
4310 {
4311         uintptr_t text = (uintptr_t)mp->text;
4312         uintptr_t tsize = (uintptr_t)mp->text_size;
4313         size_t size = (((text + tsize + PAGESIZE - 1) & PAGEMASK) -
4314             (text & PAGEMASK));
4315
4316         mp->textwin = NULL;
4317
4318         if (mp->textwin_base == NULL)
4319                 return;
4320
4321         hat_unload(kas.a_hat, mp->textwin_base, size, HAT_UNLOAD_UNLOCK);
4322         vmem_free(heap_arena, mp->textwin_base, size);
4323         mp->textwin_base = NULL;
4324 }
4325
4326 static char *
4327 find_libmacro(char *name)
4328 {
4329         int lmi;
4330
4331         for (lmi = 0; lmi < NLIBMACROS; lmi++) {
4332                 if (strcmp(name, libmacros[lmi].lmi_macroname) == 0)
4333                         return (libmacros[lmi].lmi_list);
4334         }
4335         return (NULL);
4336 }
4337
4338 /*
4339  * Check for $MACRO in tail (string to expand) and expand it in path at pathend
4340  * returns path if successful, else NULL
4341  * Support multiple $MACROs expansion and the first valid path will be returned
4342  * Caller's responsibility to provide enough space in path to expand
4343  */
4344 char *
4345 expand_libmacro(char *tail, char *path, char *pathend)
4346 {
4347         char c, *p, *p1, *p2, *path2, *endp;
4348         int diff, lmi, macrolen, valid_macro, more_macro;
4349         struct _buf *file;
4350
4351         /*
4352          * check for $MACROS between nulls or slashes
4353          */
4354         p = strchr(tail, '$');
4355         if (p == NULL)
4356                 return (NULL);
4357         for (lmi = 0; lmi < NLIBMACROS; lmi++) {
4358                 macrolen = libmacros[lmi].lmi_macrolen;
4359                 if (strncmp(p + 1, libmacros[lmi].lmi_macroname, macrolen) == 0)
4360                         break;
4361         }
4362
4363         valid_macro = 0;
4364         if (lmi < NLIBMACROS) {
4365                 /*
4366                  * The following checks are used to restrict expansion of
4367                  * macros to those that form a full directory/file name
4368                  * and to keep the behavior same as before.  If this
4369                  * restriction is removed or no longer valid in the future,
4370                  * the checks below can be deleted.
4371                  */
4372                 if ((p == tail) || (*(p - 1) == '/')) {
4373                         c = *(p + macrolen + 1);
4374                         if (c == '/' || c == '\0')
4375                                 valid_macro = 1;
4376                 }
4377         }
4378
4379         if (!valid_macro) {
4380                 p2 = strchr(p, '/');
4381                 /*
4382                  * if no more macro to expand, then just copy whatever left
4383                  * and check whether it exists
4384                  */
4385                 if (p2 == NULL || strchr(p2, '$') == NULL) {
4386                         (void) strcpy(pathend, tail);
4387                         if ((file = kobj_open_path(path, 1, 1)) !=
4388                             (struct _buf *)-1) {
4389                                 kobj_close_file(file);
4390                                 return (path);
4391                         } else
4392                                 return (NULL);
4393                 } else {
4394                         /*
4395                          * copy all chars before '/' and call expand_libmacro()
4396                          * again
4397                          */
4398                         diff = p2 - tail;
4399                         bcopy(tail, pathend, diff);
4400                         pathend += diff;
4401                         *(pathend) = '\0';
4402                         return (expand_libmacro(p2, path, pathend));
4403                 }
4404         }
4405
4406         more_macro = 0;
4407         if (c != '\0') {
4408                 endp = p + macrolen + 1;
4409                 if (strchr(endp, '$') != NULL)
4410                         more_macro = 1;
4411         } else
4412                 endp = NULL;
4413
4414         /*
4415          * copy lmi_list and split it into components.
4416          * then put the part of tail before $MACRO into path
4417          * at pathend
4418          */
4419         diff = p - tail;
4420         if (diff > 0)
4421                 bcopy(tail, pathend, diff);
4422         path2 = pathend + diff;
4423         p1 = libmacros[lmi].lmi_list;
4424         while (p1 && (*p1 != '\0')) {
4425                 p2 = strchr(p1, ':');
4426                 if (p2) {
4427                         diff = p2 - p1;
4428                         bcopy(p1, path2, diff);
4429                         *(path2 + diff) = '\0';
4430                 } else {
4431                         diff = strlen(p1);
4432                         bcopy(p1, path2, diff + 1);
4433                 }
4434                 /* copy endp only if there isn't any more macro to expand */
4435                 if (!more_macro && (endp != NULL))
4436                         (void) strcat(path2, endp);
4437                 file = kobj_open_path(path, 1, 1);
4438                 if (file != (struct _buf *)-1) {
4439                         kobj_close_file(file);
4440                         /*
4441                          * if more macros to expand then call expand_libmacro(),
4442                          * else return path which has the whole path
4443                          */
4444                         if (!more_macro || (expand_libmacro(endp, path,
4445                             path2 + diff) != NULL)) {
4446                                 return (path);
4447                         }
4448                 }
4449                 if (p2)
4450                         p1 = ++p2;
4451                 else
4452                         return (NULL);
4453         }
4454         return (NULL);
4455 }
4456
4457 static void
4458 tnf_add_notifyunload(kobj_notify_f *fp)
4459 {
4460         kobj_notify_list_t *entry;
4461
4462         entry = kobj_alloc(sizeof (kobj_notify_list_t), KM_WAIT);
4463         entry->kn_type = KOBJ_NOTIFY_MODUNLOADING;
4464         entry->kn_func = fp;
4465         (void) kobj_notify_add(entry);
4466 }
4467
4468 /* ARGSUSED */
4469 static void
4470 tnf_unsplice_probes(uint_t what, struct modctl *mod)
4471 {
4472         tnf_probe_control_t **p;
4473         tnf_tag_data_t **q;
4474         struct module *mp = mod->mod_mp;
4475
4476         if (!(mp->flags & KOBJ_TNF_PROBE))
4477                 return;
4478
4479         for (p = &__tnf_probe_list_head; *p; )
4480                 if (kobj_addrcheck(mp, (char *)*p) == 0)
4481                         *p = (*p)->next;
4482                 else
4483                         p = &(*p)->next;
4484
4485         for (q = &__tnf_tag_list_head; *q; )
4486                 if (kobj_addrcheck(mp, (char *)*q) == 0)
4487                         *q = (tnf_tag_data_t *)(*q)->tag_version;
4488                 else
4489                         q = (tnf_tag_data_t **)&(*q)->tag_version;
4490
4491         tnf_changed_probe_list = 1;
4492 }
4493
4494 int
4495 tnf_splice_probes(int boot_load, tnf_probe_control_t *plist,
4496     tnf_tag_data_t *tlist)
4497 {
4498         int result = 0;
4499         static int add_notify = 1;
4500
4501         if (plist) {
4502                 tnf_probe_control_t *pl;
4503
4504                 for (pl = plist; pl->next; )
4505                         pl = pl->next;
4506
4507                 if (!boot_load)
4508                         mutex_enter(&mod_lock);
4509                 tnf_changed_probe_list = 1;
4510                 pl->next = __tnf_probe_list_head;
4511                 __tnf_probe_list_head = plist;
4512                 if (!boot_load)
4513                         mutex_exit(&mod_lock);
4514                 result = 1;
4515         }
4516
4517         if (tlist) {
4518                 tnf_tag_data_t *tl;
4519
4520                 for (tl = tlist; tl->tag_version; )
4521                         tl = (tnf_tag_data_t *)tl->tag_version;
4522
4523                 if (!boot_load)
4524                         mutex_enter(&mod_lock);
4525                 tl->tag_version = (tnf_tag_version_t *)__tnf_tag_list_head;
4526                 __tnf_tag_list_head = tlist;
4527                 if (!boot_load)
4528                         mutex_exit(&mod_lock);
4529                 result = 1;
4530         }
4531         if (!boot_load && result && add_notify) {
4532                 tnf_add_notifyunload(tnf_unsplice_probes);
4533                 add_notify = 0;
4534         }
4535         return (result);
4536 }
4537
4538 char *kobj_file_buf;
4539 int kobj_file_bufsize;
4540
4541 /*
4542  * This code is for the purpose of manually recording which files
4543  * needs to go into the boot archive on any given system.
4544  *
4545  * To enable the code, set kobj_file_bufsize in /etc/system
4546  * and reboot the system, then use mdb to look at kobj_file_buf.
4547  */
4548 static void
4549 kobj_record_file(char *filename)
4550 {
4551         static char *buf;
4552         static int size = 0;
4553         int n;
4554
4555         if (kobj_file_bufsize == 0)     /* don't bother */
4556                 return;
4557
4558         if (kobj_file_buf == NULL) {    /* allocate buffer */
4559                 size = kobj_file_bufsize;
4560                 buf = kobj_file_buf = kobj_alloc(size, KM_WAIT|KM_TMP);
4561         }
4562
4563         n = snprintf(buf, size, "%s\n", filename);
4564         if (n > size)
4565                 n = size;
4566         size -= n;
4567         buf += n;
4568 }
4569
4570 static int
4571 kobj_boot_fstat(int fd, struct bootstat *stp)
4572 {
4573 #if defined(_OBP)
4574         if (!standalone && _ioquiesced)
4575                 return (-1);
4576         return (BOP_FSTAT(ops, fd, stp));
4577 #else
4578         return (BRD_FSTAT(bfs_ops, fd, stp));
4579 #endif
4580 }
4581
4582 static int
4583 kobj_boot_open(char *filename, int flags)
4584 {
4585 #if defined(_OBP)
4586
4587         /*
4588          * If io via bootops is quiesced, it means boot is no longer
4589          * available to us.  We make it look as if we can't open the
4590          * named file - which is reasonably accurate.
4591          */
4592         if (!standalone && _ioquiesced)
4593                 return (-1);
4594
4595         kobj_record_file(filename);
4596         return (BOP_OPEN(filename, flags));
4597 #else /* x86 */
4598         kobj_record_file(filename);
4599         return (BRD_OPEN(bfs_ops, filename, flags));
4600 #endif
4601 }
4602
4603 static int
4604 kobj_boot_close(int fd)
4605 {
4606 #if defined(_OBP)
4607         if (!standalone && _ioquiesced)
4608                 return (-1);
4609
4610         return (BOP_CLOSE(fd));
4611 #else /* x86 */
4612         return (BRD_CLOSE(bfs_ops, fd));
4613 #endif
4614 }
4615
4616 /*ARGSUSED*/
4617 static int
4618 kobj_boot_seek(int fd, off_t hi, off_t lo)
4619 {
4620 #if defined(_OBP)
4621         return (BOP_SEEK(fd, lo) == -1 ? -1 : 0);
4622 #else
4623         return (BRD_SEEK(bfs_ops, fd, lo, SEEK_SET));
4624 #endif
4625 }
4626
4627 static int
4628 kobj_boot_read(int fd, caddr_t buf, size_t size)
4629 {
4630 #if defined(_OBP)
4631         return (BOP_READ(fd, buf, size));
4632 #else
4633         return (BRD_READ(bfs_ops, fd, buf, size));
4634 #endif
4635 }
4636
4637 static int
4638 kobj_boot_compinfo(int fd, struct compinfo *cb)
4639 {
4640         return (boot_compinfo(fd, cb));
4641 }
4642
4643 /*
4644  * Check if the file is compressed (for now we handle only gzip).
4645  * It returns CH_MAGIC_GZIP if the file is compressed and 0 otherwise.
4646  */
4647 static int
4648 kobj_is_compressed(intptr_t fd)
4649 {
4650         struct vnode *vp = (struct vnode *)fd;
4651         ssize_t resid;
4652         uint16_t magic_buf;
4653         int err = 0;
4654
4655         if ((err = vn_rdwr(UIO_READ, vp, (caddr_t)((intptr_t)&magic_buf),
4656             sizeof (magic_buf), (offset_t)(0),
4657             UIO_SYSSPACE, 0, (rlim64_t)0, CRED(), &resid)) != 0) {
4658
4659                 _kobj_printf(ops, "kobj_is_compressed: vn_rdwr() failed, "
4660                     "error code 0x%x\n", err);
4661                 return (0);
4662         }
4663
4664         if (magic_buf == CH_MAGIC_GZIP)
4665                 return (CH_MAGIC_GZIP);
4666
4667         return (0);
4668 }