lib/libzpool/kernel.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  23  * Copyright (c) 2012, 2018 by Delphix. All rights reserved.
  24  * Copyright (c) 2016 Actifio, Inc. All rights reserved.
  25  */
  26
  27 #include <assert.h>
  28 #include <fcntl.h>
  29 #include <libgen.h>
  30 #include <poll.h>
  31 #include <stdio.h>
  32 #include <stdlib.h>
  33 #include <string.h>
  34 #include <libzutil.h>
  35 #include <sys/crypto/icp.h>
  36 #include <sys/processor.h>
  37 #include <sys/rrwlock.h>
  38 #include <sys/spa.h>
  39 #include <sys/stat.h>
  40 #include <sys/systeminfo.h>
  41 #include <sys/time.h>
  42 #include <sys/utsname.h>
  43 #include <sys/zfs_context.h>
  44 #include <sys/zfs_onexit.h>
  45 #include <sys/zfs_vfsops.h>
  46 #include <sys/zstd/zstd.h>
  47 #include <sys/zvol.h>
  48 #include <zfs_fletcher.h>
  49 #include <zlib.h>
  50
  51 /*
  52  * Emulation of kernel services in userland.
  53  */
  54
  55 uint64_t physmem;
  56 char hw_serial[HW_HOSTID_LEN];
  57 struct utsname hw_utsname;
  58
  59 /* If set, all blocks read will be copied to the specified directory. */
  60 char *vn_dumpdir = NULL;
  61
  62 /* this only exists to have its address taken */
  63 struct proc p0;
  64
  65 /*
  66  * =========================================================================
  67  * threads
  68  * =========================================================================
  69  *
  70  * TS_STACK_MIN is dictated by the minimum allowed pthread stack size.  While
  71  * TS_STACK_MAX is somewhat arbitrary, it was selected to be large enough for
  72  * the expected stack depth while small enough to avoid exhausting address
  73  * space with high thread counts.
  74  */
  75 #define TS_STACK_MIN    MAX(PTHREAD_STACK_MIN, 32768)
  76 #define TS_STACK_MAX    (256 * 1024)
  77
  78 struct zk_thread_wrapper {
  79         void (*func)(void *);
  80         void *arg;
  81 };
  82
  83 static void *
  84 zk_thread_wrapper(void *arg)
  85 {
  86         struct zk_thread_wrapper ztw;
  87         memcpy(&ztw, arg, sizeof (ztw));
  88         free(arg);
  89         ztw.func(ztw.arg);
  90         return (NULL);
  91 }
  92
  93 kthread_t *
  94 zk_thread_create(void (*func)(void *), void *arg, size_t stksize, int state)
  95 {
  96         pthread_attr_t attr;
  97         pthread_t tid;
  98         char *stkstr;
  99         struct zk_thread_wrapper *ztw;
 100         int detachstate = PTHREAD_CREATE_DETACHED;
 101
 102         VERIFY0(pthread_attr_init(&attr));
 103
 104         if (state & TS_JOINABLE)
 105                 detachstate = PTHREAD_CREATE_JOINABLE;
 106
 107         VERIFY0(pthread_attr_setdetachstate(&attr, detachstate));
 108
 109         /*
 110          * We allow the default stack size in user space to be specified by
 111          * setting the ZFS_STACK_SIZE environment variable.  This allows us
 112          * the convenience of observing and debugging stack overruns in
 113          * user space.  Explicitly specified stack sizes will be honored.
 114          * The usage of ZFS_STACK_SIZE is discussed further in the
 115          * ENVIRONMENT VARIABLES sections of the ztest(1) man page.
 116          */
 117         if (stksize == 0) {
 118                 stkstr = getenv("ZFS_STACK_SIZE");
 119
 120                 if (stkstr == NULL)
 121                         stksize = TS_STACK_MAX;
 122                 else
 123                         stksize = MAX(atoi(stkstr), TS_STACK_MIN);
 124         }
 125
 126         VERIFY3S(stksize, >, 0);
 127         stksize = P2ROUNDUP(MAX(stksize, TS_STACK_MIN), PAGESIZE);
 128
 129         /*
 130          * If this ever fails, it may be because the stack size is not a
 131          * multiple of system page size.
 132          */
 133         VERIFY0(pthread_attr_setstacksize(&attr, stksize));
 134         VERIFY0(pthread_attr_setguardsize(&attr, PAGESIZE));
 135
 136         VERIFY(ztw = malloc(sizeof (*ztw)));
 137         ztw->func = func;
 138         ztw->arg = arg;
 139         VERIFY0(pthread_create(&tid, &attr, zk_thread_wrapper, ztw));
 140         VERIFY0(pthread_attr_destroy(&attr));
 141
 142         return ((void *)(uintptr_t)tid);
 143 }
 144
 145 /*
 146  * =========================================================================
 147  * kstats
 148  * =========================================================================
 149  */
 150 kstat_t *
 151 kstat_create(const char *module, int instance, const char *name,
 152     const char *class, uchar_t type, ulong_t ndata, uchar_t ks_flag)
 153 {
 154         (void) module, (void) instance, (void) name, (void) class, (void) type,
 155             (void) ndata, (void) ks_flag;
 156         return (NULL);
 157 }
 158
 159 void
 160 kstat_install(kstat_t *ksp)
 161 {
 162         (void) ksp;
 163 }
 164
 165 void
 166 kstat_delete(kstat_t *ksp)
 167 {
 168         (void) ksp;
 169 }
 170
 171 void
 172 kstat_set_raw_ops(kstat_t *ksp,
 173     int (*headers)(char *buf, size_t size),
 174     int (*data)(char *buf, size_t size, void *data),
 175     void *(*addr)(kstat_t *ksp, loff_t index))
 176 {
 177         (void) ksp, (void) headers, (void) data, (void) addr;
 178 }
 179
 180 /*
 181  * =========================================================================
 182  * mutexes
 183  * =========================================================================
 184  */
 185
 186 void
 187 mutex_init(kmutex_t *mp, char *name, int type, void *cookie)
 188 {
 189         (void) name, (void) type, (void) cookie;
 190         VERIFY0(pthread_mutex_init(&mp->m_lock, NULL));
 191         memset(&mp->m_owner, 0, sizeof (pthread_t));
 192 }
 193
 194 void
 195 mutex_destroy(kmutex_t *mp)
 196 {
 197         VERIFY0(pthread_mutex_destroy(&mp->m_lock));
 198 }
 199
 200 void
 201 mutex_enter(kmutex_t *mp)
 202 {
 203         VERIFY0(pthread_mutex_lock(&mp->m_lock));
 204         mp->m_owner = pthread_self();
 205 }
 206
 207 int
 208 mutex_tryenter(kmutex_t *mp)
 209 {
 210         int error = pthread_mutex_trylock(&mp->m_lock);
 211         if (error == 0) {
 212                 mp->m_owner = pthread_self();
 213                 return (1);
 214         } else {
 215                 VERIFY3S(error, ==, EBUSY);
 216                 return (0);
 217         }
 218 }
 219
 220 void
 221 mutex_exit(kmutex_t *mp)
 222 {
 223         memset(&mp->m_owner, 0, sizeof (pthread_t));
 224         VERIFY0(pthread_mutex_unlock(&mp->m_lock));
 225 }
 226
 227 /*
 228  * =========================================================================
 229  * rwlocks
 230  * =========================================================================
 231  */
 232
 233 void
 234 rw_init(krwlock_t *rwlp, char *name, int type, void *arg)
 235 {
 236         (void) name, (void) type, (void) arg;
 237         VERIFY0(pthread_rwlock_init(&rwlp->rw_lock, NULL));
 238         rwlp->rw_readers = 0;
 239         rwlp->rw_owner = 0;
 240 }
 241
 242 void
 243 rw_destroy(krwlock_t *rwlp)
 244 {
 245         VERIFY0(pthread_rwlock_destroy(&rwlp->rw_lock));
 246 }
 247
 248 void
 249 rw_enter(krwlock_t *rwlp, krw_t rw)
 250 {
 251         if (rw == RW_READER) {
 252                 VERIFY0(pthread_rwlock_rdlock(&rwlp->rw_lock));
 253                 atomic_inc_uint(&rwlp->rw_readers);
 254         } else {
 255                 VERIFY0(pthread_rwlock_wrlock(&rwlp->rw_lock));
 256                 rwlp->rw_owner = pthread_self();
 257         }
 258 }
 259
 260 void
 261 rw_exit(krwlock_t *rwlp)
 262 {
 263         if (RW_READ_HELD(rwlp))
 264                 atomic_dec_uint(&rwlp->rw_readers);
 265         else
 266                 rwlp->rw_owner = 0;
 267
 268         VERIFY0(pthread_rwlock_unlock(&rwlp->rw_lock));
 269 }
 270
 271 int
 272 rw_tryenter(krwlock_t *rwlp, krw_t rw)
 273 {
 274         int error;
 275
 276         if (rw == RW_READER)
 277                 error = pthread_rwlock_tryrdlock(&rwlp->rw_lock);
 278         else
 279                 error = pthread_rwlock_trywrlock(&rwlp->rw_lock);
 280
 281         if (error == 0) {
 282                 if (rw == RW_READER)
 283                         atomic_inc_uint(&rwlp->rw_readers);
 284                 else
 285                         rwlp->rw_owner = pthread_self();
 286
 287                 return (1);
 288         }
 289
 290         VERIFY3S(error, ==, EBUSY);
 291
 292         return (0);
 293 }
 294
 295 uint32_t
 296 zone_get_hostid(void *zonep)
 297 {
 298         /*
 299          * We're emulating the system's hostid in userland.
 300          */
 301         (void) zonep;
 302         return (strtoul(hw_serial, NULL, 10));
 303 }
 304
 305 int
 306 rw_tryupgrade(krwlock_t *rwlp)
 307 {
 308         (void) rwlp;
 309         return (0);
 310 }
 311
 312 /*
 313  * =========================================================================
 314  * condition variables
 315  * =========================================================================
 316  */
 317
 318 void
 319 cv_init(kcondvar_t *cv, char *name, int type, void *arg)
 320 {
 321         (void) name, (void) type, (void) arg;
 322         VERIFY0(pthread_cond_init(cv, NULL));
 323 }
 324
 325 void
 326 cv_destroy(kcondvar_t *cv)
 327 {
 328         VERIFY0(pthread_cond_destroy(cv));
 329 }
 330
 331 void
 332 cv_wait(kcondvar_t *cv, kmutex_t *mp)
 333 {
 334         memset(&mp->m_owner, 0, sizeof (pthread_t));
 335         VERIFY0(pthread_cond_wait(cv, &mp->m_lock));
 336         mp->m_owner = pthread_self();
 337 }
 338
 339 int
 340 cv_wait_sig(kcondvar_t *cv, kmutex_t *mp)
 341 {
 342         cv_wait(cv, mp);
 343         return (1);
 344 }
 345
 346 int
 347 cv_timedwait(kcondvar_t *cv, kmutex_t *mp, clock_t abstime)
 348 {
 349         int error;
 350         struct timeval tv;
 351         struct timespec ts;
 352         clock_t delta;
 353
 354         delta = abstime - ddi_get_lbolt();
 355         if (delta <= 0)
 356                 return (-1);
 357
 358         VERIFY(gettimeofday(&tv, NULL) == 0);
 359
 360         ts.tv_sec = tv.tv_sec + delta / hz;
 361         ts.tv_nsec = tv.tv_usec * NSEC_PER_USEC + (delta % hz) * (NANOSEC / hz);
 362         if (ts.tv_nsec >= NANOSEC) {
 363                 ts.tv_sec++;
 364                 ts.tv_nsec -= NANOSEC;
 365         }
 366
 367         memset(&mp->m_owner, 0, sizeof (pthread_t));
 368         error = pthread_cond_timedwait(cv, &mp->m_lock, &ts);
 369         mp->m_owner = pthread_self();
 370
 371         if (error == ETIMEDOUT)
 372                 return (-1);
 373
 374         VERIFY0(error);
 375
 376         return (1);
 377 }
 378
 379 int
 380 cv_timedwait_hires(kcondvar_t *cv, kmutex_t *mp, hrtime_t tim, hrtime_t res,
 381     int flag)
 382 {
 383         (void) res;
 384         int error;
 385         struct timeval tv;
 386         struct timespec ts;
 387         hrtime_t delta;
 388
 389         ASSERT(flag == 0 || flag == CALLOUT_FLAG_ABSOLUTE);
 390
 391         delta = tim;
 392         if (flag & CALLOUT_FLAG_ABSOLUTE)
 393                 delta -= gethrtime();
 394
 395         if (delta <= 0)
 396                 return (-1);
 397
 398         VERIFY0(gettimeofday(&tv, NULL));
 399
 400         ts.tv_sec = tv.tv_sec + delta / NANOSEC;
 401         ts.tv_nsec = tv.tv_usec * NSEC_PER_USEC + (delta % NANOSEC);
 402         if (ts.tv_nsec >= NANOSEC) {
 403                 ts.tv_sec++;
 404                 ts.tv_nsec -= NANOSEC;
 405         }
 406
 407         memset(&mp->m_owner, 0, sizeof (pthread_t));
 408         error = pthread_cond_timedwait(cv, &mp->m_lock, &ts);
 409         mp->m_owner = pthread_self();
 410
 411         if (error == ETIMEDOUT)
 412                 return (-1);
 413
 414         VERIFY0(error);
 415
 416         return (1);
 417 }
 418
 419 void
 420 cv_signal(kcondvar_t *cv)
 421 {
 422         VERIFY0(pthread_cond_signal(cv));
 423 }
 424
 425 void
 426 cv_broadcast(kcondvar_t *cv)
 427 {
 428         VERIFY0(pthread_cond_broadcast(cv));
 429 }
 430
 431 /*
 432  * =========================================================================
 433  * procfs list
 434  * =========================================================================
 435  */
 436
 437 void
 438 seq_printf(struct seq_file *m, const char *fmt, ...)
 439 {
 440         (void) m, (void) fmt;
 441 }
 442
 443 void
 444 procfs_list_install(const char *module,
 445     const char *submodule,
 446     const char *name,
 447     mode_t mode,
 448     procfs_list_t *procfs_list,
 449     int (*show)(struct seq_file *f, void *p),
 450     int (*show_header)(struct seq_file *f),
 451     int (*clear)(procfs_list_t *procfs_list),
 452     size_t procfs_list_node_off)
 453 {
 454         (void) module, (void) submodule, (void) name, (void) mode, (void) show,
 455             (void) show_header, (void) clear;
 456         mutex_init(&procfs_list->pl_lock, NULL, MUTEX_DEFAULT, NULL);
 457         list_create(&procfs_list->pl_list,
 458             procfs_list_node_off + sizeof (procfs_list_node_t),
 459             procfs_list_node_off + offsetof(procfs_list_node_t, pln_link));
 460         procfs_list->pl_next_id = 1;
 461         procfs_list->pl_node_offset = procfs_list_node_off;
 462 }
 463
 464 void
 465 procfs_list_uninstall(procfs_list_t *procfs_list)
 466 {
 467         (void) procfs_list;
 468 }
 469
 470 void
 471 procfs_list_destroy(procfs_list_t *procfs_list)
 472 {
 473         ASSERT(list_is_empty(&procfs_list->pl_list));
 474         list_destroy(&procfs_list->pl_list);
 475         mutex_destroy(&procfs_list->pl_lock);
 476 }
 477
 478 #define NODE_ID(procfs_list, obj) \
 479                 (((procfs_list_node_t *)(((char *)obj) + \
 480                 (procfs_list)->pl_node_offset))->pln_id)
 481
 482 void
 483 procfs_list_add(procfs_list_t *procfs_list, void *p)
 484 {
 485         ASSERT(MUTEX_HELD(&procfs_list->pl_lock));
 486         NODE_ID(procfs_list, p) = procfs_list->pl_next_id++;
 487         list_insert_tail(&procfs_list->pl_list, p);
 488 }
 489
 490 /*
 491  * =========================================================================
 492  * vnode operations
 493  * =========================================================================
 494  */
 495
 496 /*
 497  * =========================================================================
 498  * Figure out which debugging statements to print
 499  * =========================================================================
 500  */
 501
 502 static char *dprintf_string;
 503 static int dprintf_print_all;
 504
 505 int
 506 dprintf_find_string(const char *string)
 507 {
 508         char *tmp_str = dprintf_string;
 509         int len = strlen(string);
 510
 511         /*
 512          * Find out if this is a string we want to print.
 513          * String format: file1.c,function_name1,file2.c,file3.c
 514          */
 515
 516         while (tmp_str != NULL) {
 517                 if (strncmp(tmp_str, string, len) == 0 &&
 518                     (tmp_str[len] == ',' || tmp_str[len] == '\0'))
 519                         return (1);
 520                 tmp_str = strchr(tmp_str, ',');
 521                 if (tmp_str != NULL)
 522                         tmp_str++; /* Get rid of , */
 523         }
 524         return (0);
 525 }
 526
 527 void
 528 dprintf_setup(int *argc, char **argv)
 529 {
 530         int i, j;
 531
 532         /*
 533          * Debugging can be specified two ways: by setting the
 534          * environment variable ZFS_DEBUG, or by including a
 535          * "debug=..."  argument on the command line.  The command
 536          * line setting overrides the environment variable.
 537          */
 538
 539         for (i = 1; i < *argc; i++) {
 540                 int len = strlen("debug=");
 541                 /* First look for a command line argument */
 542                 if (strncmp("debug=", argv[i], len) == 0) {
 543                         dprintf_string = argv[i] + len;
 544                         /* Remove from args */
 545                         for (j = i; j < *argc; j++)
 546                                 argv[j] = argv[j+1];
 547                         argv[j] = NULL;
 548                         (*argc)--;
 549                 }
 550         }
 551
 552         if (dprintf_string == NULL) {
 553                 /* Look for ZFS_DEBUG environment variable */
 554                 dprintf_string = getenv("ZFS_DEBUG");
 555         }
 556
 557         /*
 558          * Are we just turning on all debugging?
 559          */
 560         if (dprintf_find_string("on"))
 561                 dprintf_print_all = 1;
 562
 563         if (dprintf_string != NULL)
 564                 zfs_flags |= ZFS_DEBUG_DPRINTF;
 565 }
 566
 567 /*
 568  * =========================================================================
 569  * debug printfs
 570  * =========================================================================
 571  */
 572 void
 573 __dprintf(boolean_t dprint, const char *file, const char *func,
 574     int line, const char *fmt, ...)
 575 {
 576         /* Get rid of annoying "../common/" prefix to filename. */
 577         const char *newfile = zfs_basename(file);
 578
 579         va_list adx;
 580         if (dprint) {
 581                 /* dprintf messages are printed immediately */
 582
 583                 if (!dprintf_print_all &&
 584                     !dprintf_find_string(newfile) &&
 585                     !dprintf_find_string(func))
 586                         return;
 587
 588                 /* Print out just the function name if requested */
 589                 flockfile(stdout);
 590                 if (dprintf_find_string("pid"))
 591                         (void) printf("%d ", getpid());
 592                 if (dprintf_find_string("tid"))
 593                         (void) printf("%ju ",
 594                             (uintmax_t)(uintptr_t)pthread_self());
 595                 if (dprintf_find_string("cpu"))
 596                         (void) printf("%u ", getcpuid());
 597                 if (dprintf_find_string("time"))
 598                         (void) printf("%llu ", gethrtime());
 599                 if (dprintf_find_string("long"))
 600                         (void) printf("%s, line %d: ", newfile, line);
 601                 (void) printf("dprintf: %s: ", func);
 602                 va_start(adx, fmt);
 603                 (void) vprintf(fmt, adx);
 604                 va_end(adx);
 605                 funlockfile(stdout);
 606         } else {
 607                 /* zfs_dbgmsg is logged for dumping later */
 608                 size_t size;
 609                 char *buf;
 610                 int i;
 611
 612                 size = 1024;
 613                 buf = umem_alloc(size, UMEM_NOFAIL);
 614                 i = snprintf(buf, size, "%s:%d:%s(): ", newfile, line, func);
 615
 616                 if (i < size) {
 617                         va_start(adx, fmt);
 618                         (void) vsnprintf(buf + i, size - i, fmt, adx);
 619                         va_end(adx);
 620                 }
 621
 622                 __zfs_dbgmsg(buf);
 623
 624                 umem_free(buf, size);
 625         }
 626 }
 627
 628 /*
 629  * =========================================================================
 630  * cmn_err() and panic()
 631  * =========================================================================
 632  */
 633 static char ce_prefix[CE_IGNORE][10] = { "", "NOTICE: ", "WARNING: ", "" };
 634 static char ce_suffix[CE_IGNORE][2] = { "", "\n", "\n", "" };
 635
 636 __attribute__((noreturn)) void
 637 vpanic(const char *fmt, va_list adx)
 638 {
 639         (void) fprintf(stderr, "error: ");
 640         (void) vfprintf(stderr, fmt, adx);
 641         (void) fprintf(stderr, "\n");
 642
 643         abort();        /* think of it as a "user-level crash dump" */
 644 }
 645
 646 __attribute__((noreturn)) void
 647 panic(const char *fmt, ...)
 648 {
 649         va_list adx;
 650
 651         va_start(adx, fmt);
 652         vpanic(fmt, adx);
 653         va_end(adx);
 654 }
 655
 656 void
 657 vcmn_err(int ce, const char *fmt, va_list adx)
 658 {
 659         if (ce == CE_PANIC)
 660                 vpanic(fmt, adx);
 661         if (ce != CE_NOTE) {    /* suppress noise in userland stress testing */
 662                 (void) fprintf(stderr, "%s", ce_prefix[ce]);
 663                 (void) vfprintf(stderr, fmt, adx);
 664                 (void) fprintf(stderr, "%s", ce_suffix[ce]);
 665         }
 666 }
 667
 668 void
 669 cmn_err(int ce, const char *fmt, ...)
 670 {
 671         va_list adx;
 672
 673         va_start(adx, fmt);
 674         vcmn_err(ce, fmt, adx);
 675         va_end(adx);
 676 }
 677
 678 /*
 679  * =========================================================================
 680  * misc routines
 681  * =========================================================================
 682  */
 683
 684 void
 685 delay(clock_t ticks)
 686 {
 687         (void) poll(0, 0, ticks * (1000 / hz));
 688 }
 689
 690 /*
 691  * Find highest one bit set.
 692  * Returns bit number + 1 of highest bit that is set, otherwise returns 0.
 693  * The __builtin_clzll() function is supported by both GCC and Clang.
 694  */
 695 int
 696 highbit64(uint64_t i)
 697 {
 698         if (i == 0)
 699         return (0);
 700
 701         return (NBBY * sizeof (uint64_t) - __builtin_clzll(i));
 702 }
 703
 704 /*
 705  * Find lowest one bit set.
 706  * Returns bit number + 1 of lowest bit that is set, otherwise returns 0.
 707  * The __builtin_ffsll() function is supported by both GCC and Clang.
 708  */
 709 int
 710 lowbit64(uint64_t i)
 711 {
 712         if (i == 0)
 713                 return (0);
 714
 715         return (__builtin_ffsll(i));
 716 }
 717
 718 const char *random_path = "/dev/random";
 719 const char *urandom_path = "/dev/urandom";
 720 static int random_fd = -1, urandom_fd = -1;
 721
 722 void
 723 random_init(void)
 724 {
 725         VERIFY((random_fd = open(random_path, O_RDONLY | O_CLOEXEC)) != -1);
 726         VERIFY((urandom_fd = open(urandom_path, O_RDONLY | O_CLOEXEC)) != -1);
 727 }
 728
 729 void
 730 random_fini(void)
 731 {
 732         close(random_fd);
 733         close(urandom_fd);
 734
 735         random_fd = -1;
 736         urandom_fd = -1;
 737 }
 738
 739 static int
 740 random_get_bytes_common(uint8_t *ptr, size_t len, int fd)
 741 {
 742         size_t resid = len;
 743         ssize_t bytes;
 744
 745         ASSERT(fd != -1);
 746
 747         while (resid != 0) {
 748                 bytes = read(fd, ptr, resid);
 749                 ASSERT3S(bytes, >=, 0);
 750                 ptr += bytes;
 751                 resid -= bytes;
 752         }
 753
 754         return (0);
 755 }
 756
 757 int
 758 random_get_bytes(uint8_t *ptr, size_t len)
 759 {
 760         return (random_get_bytes_common(ptr, len, random_fd));
 761 }
 762
 763 int
 764 random_get_pseudo_bytes(uint8_t *ptr, size_t len)
 765 {
 766         return (random_get_bytes_common(ptr, len, urandom_fd));
 767 }
 768
 769 int
 770 ddi_strtoul(const char *hw_serial, char **nptr, int base, unsigned long *result)
 771 {
 772         (void) nptr;
 773         char *end;
 774
 775         *result = strtoul(hw_serial, &end, base);
 776         if (*result == 0)
 777                 return (errno);
 778         return (0);
 779 }
 780
 781 int
 782 ddi_strtoull(const char *str, char **nptr, int base, u_longlong_t *result)
 783 {
 784         (void) nptr;
 785         char *end;
 786
 787         *result = strtoull(str, &end, base);
 788         if (*result == 0)
 789                 return (errno);
 790         return (0);
 791 }
 792
 793 utsname_t *
 794 utsname(void)
 795 {
 796         return (&hw_utsname);
 797 }
 798
 799 /*
 800  * =========================================================================
 801  * kernel emulation setup & teardown
 802  * =========================================================================
 803  */
 804 static int
 805 umem_out_of_memory(void)
 806 {
 807         char errmsg[] = "out of memory -- generating core dump\n";
 808
 809         (void) fprintf(stderr, "%s", errmsg);
 810         abort();
 811         return (0);
 812 }
 813
 814 void
 815 kernel_init(int mode)
 816 {
 817         extern uint_t rrw_tsd_key;
 818
 819         umem_nofail_callback(umem_out_of_memory);
 820
 821         physmem = sysconf(_SC_PHYS_PAGES);
 822
 823         dprintf("physmem = %llu pages (%.2f GB)\n", (u_longlong_t)physmem,
 824             (double)physmem * sysconf(_SC_PAGE_SIZE) / (1ULL << 30));
 825
 826         (void) snprintf(hw_serial, sizeof (hw_serial), "%ld",
 827             (mode & SPA_MODE_WRITE) ? get_system_hostid() : 0);
 828
 829         random_init();
 830
 831         VERIFY0(uname(&hw_utsname));
 832
 833         system_taskq_init();
 834         icp_init();
 835
 836         zstd_init();
 837
 838         spa_init((spa_mode_t)mode);
 839
 840         fletcher_4_init();
 841
 842         tsd_create(&rrw_tsd_key, rrw_tsd_destroy);
 843 }
 844
 845 void
 846 kernel_fini(void)
 847 {
 848         fletcher_4_fini();
 849         spa_fini();
 850
 851         zstd_fini();
 852
 853         icp_fini();
 854         system_taskq_fini();
 855
 856         random_fini();
 857 }
 858
 859 uid_t
 860 crgetuid(cred_t *cr)
 861 {
 862         (void) cr;
 863         return (0);
 864 }
 865
 866 uid_t
 867 crgetruid(cred_t *cr)
 868 {
 869         (void) cr;
 870         return (0);
 871 }
 872
 873 gid_t
 874 crgetgid(cred_t *cr)
 875 {
 876         (void) cr;
 877         return (0);
 878 }
 879
 880 int
 881 crgetngroups(cred_t *cr)
 882 {
 883         (void) cr;
 884         return (0);
 885 }
 886
 887 gid_t *
 888 crgetgroups(cred_t *cr)
 889 {
 890         (void) cr;
 891         return (NULL);
 892 }
 893
 894 int
 895 zfs_secpolicy_snapshot_perms(const char *name, cred_t *cr)
 896 {
 897         (void) name, (void) cr;
 898         return (0);
 899 }
 900
 901 int
 902 zfs_secpolicy_rename_perms(const char *from, const char *to, cred_t *cr)
 903 {
 904         (void) from, (void) to, (void) cr;
 905         return (0);
 906 }
 907
 908 int
 909 zfs_secpolicy_destroy_perms(const char *name, cred_t *cr)
 910 {
 911         (void) name, (void) cr;
 912         return (0);
 913 }
 914
 915 int
 916 secpolicy_zfs(const cred_t *cr)
 917 {
 918         (void) cr;
 919         return (0);
 920 }
 921
 922 int
 923 secpolicy_zfs_proc(const cred_t *cr, proc_t *proc)
 924 {
 925         (void) cr, (void) proc;
 926         return (0);
 927 }
 928
 929 ksiddomain_t *
 930 ksid_lookupdomain(const char *dom)
 931 {
 932         ksiddomain_t *kd;
 933
 934         kd = umem_zalloc(sizeof (ksiddomain_t), UMEM_NOFAIL);
 935         kd->kd_name = spa_strdup(dom);
 936         return (kd);
 937 }
 938
 939 void
 940 ksiddomain_rele(ksiddomain_t *ksid)
 941 {
 942         spa_strfree(ksid->kd_name);
 943         umem_free(ksid, sizeof (ksiddomain_t));
 944 }
 945
 946 char *
 947 kmem_vasprintf(const char *fmt, va_list adx)
 948 {
 949         char *buf = NULL;
 950         va_list adx_copy;
 951
 952         va_copy(adx_copy, adx);
 953         VERIFY(vasprintf(&buf, fmt, adx_copy) != -1);
 954         va_end(adx_copy);
 955
 956         return (buf);
 957 }
 958
 959 char *
 960 kmem_asprintf(const char *fmt, ...)
 961 {
 962         char *buf = NULL;
 963         va_list adx;
 964
 965         va_start(adx, fmt);
 966         VERIFY(vasprintf(&buf, fmt, adx) != -1);
 967         va_end(adx);
 968
 969         return (buf);
 970 }
 971
 972 zfs_file_t *
 973 zfs_onexit_fd_hold(int fd, minor_t *minorp)
 974 {
 975         (void) fd;
 976         *minorp = 0;
 977         return (NULL);
 978 }
 979
 980 void
 981 zfs_onexit_fd_rele(zfs_file_t *fp)
 982 {
 983         (void) fp;
 984 }
 985
 986 int
 987 zfs_onexit_add_cb(minor_t minor, void (*func)(void *), void *data,
 988     uint64_t *action_handle)
 989 {
 990         (void) minor, (void) func, (void) data, (void) action_handle;
 991         return (0);
 992 }
 993
 994 fstrans_cookie_t
 995 spl_fstrans_mark(void)
 996 {
 997         return ((fstrans_cookie_t)0);
 998 }
 999
1000 void
1001 spl_fstrans_unmark(fstrans_cookie_t cookie)
1002 {
1003         (void) cookie;
1004 }
1005
1006 int
1007 __spl_pf_fstrans_check(void)
1008 {
1009         return (0);
1010 }
1011
1012 int
1013 kmem_cache_reap_active(void)
1014 {
1015         return (0);
1016 }
1017
1018 void *zvol_tag = "zvol_tag";
1019
1020 void
1021 zvol_create_minor(const char *name)
1022 {
1023         (void) name;
1024 }
1025
1026 void
1027 zvol_create_minors_recursive(const char *name)
1028 {
1029         (void) name;
1030 }
1031
1032 void
1033 zvol_remove_minors(spa_t *spa, const char *name, boolean_t async)
1034 {
1035         (void) spa, (void) name, (void) async;
1036 }
1037
1038 void
1039 zvol_rename_minors(spa_t *spa, const char *oldname, const char *newname,
1040     boolean_t async)
1041 {
1042         (void) spa, (void) oldname, (void) newname, (void) async;
1043 }
1044
1045 /*
1046  * Open file
1047  *
1048  * path - fully qualified path to file
1049  * flags - file attributes O_READ / O_WRITE / O_EXCL
1050  * fpp - pointer to return file pointer
1051  *
1052  * Returns 0 on success underlying error on failure.
1053  */
1054 int
1055 zfs_file_open(const char *path, int flags, int mode, zfs_file_t **fpp)
1056 {
1057         int fd = -1;
1058         int dump_fd = -1;
1059         int err;
1060         int old_umask = 0;
1061         zfs_file_t *fp;
1062         struct stat64 st;
1063
1064         if (!(flags & O_CREAT) && stat64(path, &st) == -1)
1065                 return (errno);
1066
1067         if (!(flags & O_CREAT) && S_ISBLK(st.st_mode))
1068                 flags |= O_DIRECT;
1069
1070         if (flags & O_CREAT)
1071                 old_umask = umask(0);
1072
1073         fd = open64(path, flags, mode);
1074         if (fd == -1)
1075                 return (errno);
1076
1077         if (flags & O_CREAT)
1078                 (void) umask(old_umask);
1079
1080         if (vn_dumpdir != NULL) {
1081                 char *dumppath = umem_zalloc(MAXPATHLEN, UMEM_NOFAIL);
1082                 const char *inpath = zfs_basename(path);
1083
1084                 (void) snprintf(dumppath, MAXPATHLEN,
1085                     "%s/%s", vn_dumpdir, inpath);
1086                 dump_fd = open64(dumppath, O_CREAT | O_WRONLY, 0666);
1087                 umem_free(dumppath, MAXPATHLEN);
1088                 if (dump_fd == -1) {
1089                         err = errno;
1090                         close(fd);
1091                         return (err);
1092                 }
1093         } else {
1094                 dump_fd = -1;
1095         }
1096
1097         (void) fcntl(fd, F_SETFD, FD_CLOEXEC);
1098
1099         fp = umem_zalloc(sizeof (zfs_file_t), UMEM_NOFAIL);
1100         fp->f_fd = fd;
1101         fp->f_dump_fd = dump_fd;
1102         *fpp = fp;
1103
1104         return (0);
1105 }
1106
1107 void
1108 zfs_file_close(zfs_file_t *fp)
1109 {
1110         close(fp->f_fd);
1111         if (fp->f_dump_fd != -1)
1112                 close(fp->f_dump_fd);
1113
1114         umem_free(fp, sizeof (zfs_file_t));
1115 }
1116
1117 /*
1118  * Stateful write - use os internal file pointer to determine where to
1119  * write and update on successful completion.
1120  *
1121  * fp -  pointer to file (pipe, socket, etc) to write to
1122  * buf - buffer to write
1123  * count - # of bytes to write
1124  * resid -  pointer to count of unwritten bytes  (if short write)
1125  *
1126  * Returns 0 on success errno on failure.
1127  */
1128 int
1129 zfs_file_write(zfs_file_t *fp, const void *buf, size_t count, ssize_t *resid)
1130 {
1131         ssize_t rc;
1132
1133         rc = write(fp->f_fd, buf, count);
1134         if (rc < 0)
1135                 return (errno);
1136
1137         if (resid) {
1138                 *resid = count - rc;
1139         } else if (rc != count) {
1140                 return (EIO);
1141         }
1142
1143         return (0);
1144 }
1145
1146 /*
1147  * Stateless write - os internal file pointer is not updated.
1148  *
1149  * fp -  pointer to file (pipe, socket, etc) to write to
1150  * buf - buffer to write
1151  * count - # of bytes to write
1152  * off - file offset to write to (only valid for seekable types)
1153  * resid -  pointer to count of unwritten bytes
1154  *
1155  * Returns 0 on success errno on failure.
1156  */
1157 int
1158 zfs_file_pwrite(zfs_file_t *fp, const void *buf,
1159     size_t count, loff_t pos, ssize_t *resid)
1160 {
1161         ssize_t rc, split, done;
1162         int sectors;
1163
1164         /*
1165          * To simulate partial disk writes, we split writes into two
1166          * system calls so that the process can be killed in between.
1167          * This is used by ztest to simulate realistic failure modes.
1168          */
1169         sectors = count >> SPA_MINBLOCKSHIFT;
1170         split = (sectors > 0 ? rand() % sectors : 0) << SPA_MINBLOCKSHIFT;
1171         rc = pwrite64(fp->f_fd, buf, split, pos);
1172         if (rc != -1) {
1173                 done = rc;
1174                 rc = pwrite64(fp->f_fd, (char *)buf + split,
1175                     count - split, pos + split);
1176         }
1177 #ifdef __linux__
1178         if (rc == -1 && errno == EINVAL) {
1179                 /*
1180                  * Under Linux, this most likely means an alignment issue
1181                  * (memory or disk) due to O_DIRECT, so we abort() in order
1182                  * to catch the offender.
1183                  */
1184                 abort();
1185         }
1186 #endif
1187
1188         if (rc < 0)
1189                 return (errno);
1190
1191         done += rc;
1192
1193         if (resid) {
1194                 *resid = count - done;
1195         } else if (done != count) {
1196                 return (EIO);
1197         }
1198
1199         return (0);
1200 }
1201
1202 /*
1203  * Stateful read - use os internal file pointer to determine where to
1204  * read and update on successful completion.
1205  *
1206  * fp -  pointer to file (pipe, socket, etc) to read from
1207  * buf - buffer to write
1208  * count - # of bytes to read
1209  * resid -  pointer to count of unread bytes (if short read)
1210  *
1211  * Returns 0 on success errno on failure.
1212  */
1213 int
1214 zfs_file_read(zfs_file_t *fp, void *buf, size_t count, ssize_t *resid)
1215 {
1216         int rc;
1217
1218         rc = read(fp->f_fd, buf, count);
1219         if (rc < 0)
1220                 return (errno);
1221
1222         if (resid) {
1223                 *resid = count - rc;
1224         } else if (rc != count) {
1225                 return (EIO);
1226         }
1227
1228         return (0);
1229 }
1230
1231 /*
1232  * Stateless read - os internal file pointer is not updated.
1233  *
1234  * fp -  pointer to file (pipe, socket, etc) to read from
1235  * buf - buffer to write
1236  * count - # of bytes to write
1237  * off - file offset to read from (only valid for seekable types)
1238  * resid -  pointer to count of unwritten bytes (if short write)
1239  *
1240  * Returns 0 on success errno on failure.
1241  */
1242 int
1243 zfs_file_pread(zfs_file_t *fp, void *buf, size_t count, loff_t off,
1244     ssize_t *resid)
1245 {
1246         ssize_t rc;
1247
1248         rc = pread64(fp->f_fd, buf, count, off);
1249         if (rc < 0) {
1250 #ifdef __linux__
1251                 /*
1252                  * Under Linux, this most likely means an alignment issue
1253                  * (memory or disk) due to O_DIRECT, so we abort() in order to
1254                  * catch the offender.
1255                  */
1256                 if (errno == EINVAL)
1257                         abort();
1258 #endif
1259                 return (errno);
1260         }
1261
1262         if (fp->f_dump_fd != -1) {
1263                 int status;
1264
1265                 status = pwrite64(fp->f_dump_fd, buf, rc, off);
1266                 ASSERT(status != -1);
1267         }
1268
1269         if (resid) {
1270                 *resid = count - rc;
1271         } else if (rc != count) {
1272                 return (EIO);
1273         }
1274
1275         return (0);
1276 }
1277
1278 /*
1279  * lseek - set / get file pointer
1280  *
1281  * fp -  pointer to file (pipe, socket, etc) to read from
1282  * offp - value to seek to, returns current value plus passed offset
1283  * whence - see man pages for standard lseek whence values
1284  *
1285  * Returns 0 on success errno on failure (ESPIPE for non seekable types)
1286  */
1287 int
1288 zfs_file_seek(zfs_file_t *fp, loff_t *offp, int whence)
1289 {
1290         loff_t rc;
1291
1292         rc = lseek(fp->f_fd, *offp, whence);
1293         if (rc < 0)
1294                 return (errno);
1295
1296         *offp = rc;
1297
1298         return (0);
1299 }
1300
1301 /*
1302  * Get file attributes
1303  *
1304  * filp - file pointer
1305  * zfattr - pointer to file attr structure
1306  *
1307  * Currently only used for fetching size and file mode
1308  *
1309  * Returns 0 on success or error code of underlying getattr call on failure.
1310  */
1311 int
1312 zfs_file_getattr(zfs_file_t *fp, zfs_file_attr_t *zfattr)
1313 {
1314         struct stat64 st;
1315
1316         if (fstat64_blk(fp->f_fd, &st) == -1)
1317                 return (errno);
1318
1319         zfattr->zfa_size = st.st_size;
1320         zfattr->zfa_mode = st.st_mode;
1321
1322         return (0);
1323 }
1324
1325 /*
1326  * Sync file to disk
1327  *
1328  * filp - file pointer
1329  * flags - O_SYNC and or O_DSYNC
1330  *
1331  * Returns 0 on success or error code of underlying sync call on failure.
1332  */
1333 int
1334 zfs_file_fsync(zfs_file_t *fp, int flags)
1335 {
1336         (void) flags;
1337
1338         if (fsync(fp->f_fd) < 0)
1339                 return (errno);
1340
1341         return (0);
1342 }
1343
1344 /*
1345  * fallocate - allocate or free space on disk
1346  *
1347  * fp - file pointer
1348  * mode (non-standard options for hole punching etc)
1349  * offset - offset to start allocating or freeing from
1350  * len - length to free / allocate
1351  *
1352  * OPTIONAL
1353  */
1354 int
1355 zfs_file_fallocate(zfs_file_t *fp, int mode, loff_t offset, loff_t len)
1356 {
1357 #ifdef __linux__
1358         return (fallocate(fp->f_fd, mode, offset, len));
1359 #else
1360         (void) fp, (void) mode, (void) offset, (void) len;
1361         return (EOPNOTSUPP);
1362 #endif
1363 }
1364
1365 /*
1366  * Request current file pointer offset
1367  *
1368  * fp - pointer to file
1369  *
1370  * Returns current file offset.
1371  */
1372 loff_t
1373 zfs_file_off(zfs_file_t *fp)
1374 {
1375         return (lseek(fp->f_fd, SEEK_CUR, 0));
1376 }
1377
1378 /*
1379  * unlink file
1380  *
1381  * path - fully qualified file path
1382  *
1383  * Returns 0 on success.
1384  *
1385  * OPTIONAL
1386  */
1387 int
1388 zfs_file_unlink(const char *path)
1389 {
1390         return (remove(path));
1391 }
1392
1393 /*
1394  * Get reference to file pointer
1395  *
1396  * fd - input file descriptor
1397  *
1398  * Returns pointer to file struct or NULL.
1399  * Unsupported in user space.
1400  */
1401 zfs_file_t *
1402 zfs_file_get(int fd)
1403 {
1404         (void) fd;
1405         abort();
1406         return (NULL);
1407 }
1408 /*
1409  * Drop reference to file pointer
1410  *
1411  * fp - pointer to file struct
1412  *
1413  * Unsupported in user space.
1414  */
1415 void
1416 zfs_file_put(zfs_file_t *fp)
1417 {
1418         abort();
1419         (void) fp;
1420 }
1421
1422 void
1423 zfsvfs_update_fromname(const char *oldname, const char *newname)
1424 {
1425         (void) oldname, (void) newname;
1426 }
1427
1428 void
1429 spa_import_os(spa_t *spa)
1430 {
1431         (void) spa;
1432 }
1433
1434 void
1435 spa_export_os(spa_t *spa)
1436 {
1437         (void) spa;
1438 }
1439
1440 void
1441 spa_activate_os(spa_t *spa)
1442 {
1443         (void) spa;
1444 }
1445
1446 void
1447 spa_deactivate_os(spa_t *spa)
1448 {
1449         (void) spa;
1450 }