scripts/zfs-tests.sh

   1 #!/usr/bin/env bash
   2 #
   3 # CDDL HEADER START
   4 #
   5 # The contents of this file are subject to the terms of the
   6 # Common Development and Distribution License, Version 1.0 only
   7 # (the "License").  You may not use this file except in compliance
   8 # with the License.
   9 #
  10 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  11 # or http://www.opensolaris.org/os/licensing.
  12 # See the License for the specific language governing permissions
  13 # and limitations under the License.
  14 #
  15 # When distributing Covered Code, include this CDDL HEADER in each
  16 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  17 # If applicable, add the following below this CDDL HEADER, with the
  18 # fields enclosed by brackets "[]" replaced with your own identifying
  19 # information: Portions Copyright [yyyy] [name of copyright owner]
  20 #
  21 # CDDL HEADER END
  22 #
  23
  24 BASE_DIR=$(dirname "$0")
  25 SCRIPT_COMMON=common.sh
  26 if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then
  27 . "${BASE_DIR}/${SCRIPT_COMMON}"
  28 else
  29 echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
  30 fi
  31
  32 PROG=zfs-tests.sh
  33 VERBOSE="no"
  34 QUIET=""
  35 CLEANUP="yes"
  36 CLEANUPALL="no"
  37 LOOPBACK="yes"
  38 STACK_TRACER="no"
  39 FILESIZE="4G"
  40 DEFAULT_RUNFILES="common.run,$(uname | tr '[:upper:]' '[:lower:]').run"
  41 RUNFILES=${RUNFILES:-$DEFAULT_RUNFILES}
  42 FILEDIR=${FILEDIR:-/var/tmp}
  43 DISKS=${DISKS:-""}
  44 SINGLETEST=()
  45 SINGLETESTUSER="root"
  46 TAGS=""
  47 ITERATIONS=1
  48 ZFS_DBGMSG="$STF_SUITE/callbacks/zfs_dbgmsg.ksh"
  49 ZFS_DMESG="$STF_SUITE/callbacks/zfs_dmesg.ksh"
  50 UNAME=$(uname -s)
  51
  52 # Override some defaults if on FreeBSD
  53 if [ "$UNAME" = "FreeBSD" ] ; then
  54         TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DMESG"}
  55         LOSETUP=/sbin/mdconfig
  56         DMSETUP=/sbin/gpart
  57 else
  58         ZFS_MMP="$STF_SUITE/callbacks/zfs_mmp.ksh"
  59         TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DBGMSG:$ZFS_DMESG:$ZFS_MMP"}
  60         LOSETUP=${LOSETUP:-/sbin/losetup}
  61         DMSETUP=${DMSETUP:-/sbin/dmsetup}
  62 fi
  63
  64 #
  65 # Log an informational message when additional verbosity is enabled.
  66 #
  67 msg() {
  68         if [ "$VERBOSE" = "yes" ]; then
  69                 echo "$@"
  70         fi
  71 }
  72
  73 #
  74 # Log a failure message, cleanup, and return an error.
  75 #
  76 fail() {
  77         echo -e "$PROG: $1" >&2
  78         cleanup
  79         exit 1
  80 }
  81
  82 cleanup_freebsd_loopback() {
  83         for TEST_LOOPBACK in ${LOOPBACKS}; do
  84                 if [ -c "/dev/${TEST_LOOPBACK}" ]; then
  85                         sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}" ||
  86                             echo "Failed to destroy: ${TEST_LOOPBACK}"
  87                 fi
  88         done
  89 }
  90
  91 cleanup_linux_loopback() {
  92         for TEST_LOOPBACK in ${LOOPBACKS}; do
  93                 LOOP_DEV=$(basename "$TEST_LOOPBACK")
  94                 DM_DEV=$(sudo "${DMSETUP}" ls 2>/dev/null | \
  95                     grep "${LOOP_DEV}" | cut -f1)
  96
  97                 if [ -n "$DM_DEV" ]; then
  98                         sudo "${DMSETUP}" remove "${DM_DEV}" ||
  99                             echo "Failed to remove: ${DM_DEV}"
 100                 fi
 101
 102                 if [ -n "${TEST_LOOPBACK}" ]; then
 103                         sudo "${LOSETUP}" -d "${TEST_LOOPBACK}" ||
 104                             echo "Failed to remove: ${TEST_LOOPBACK}"
 105                 fi
 106         done
 107 }
 108
 109 #
 110 # Attempt to remove loopback devices and files which where created earlier
 111 # by this script to run the test framework.  The '-k' option may be passed
 112 # to the script to suppress cleanup for debugging purposes.
 113 #
 114 cleanup() {
 115         if [ "$CLEANUP" = "no" ]; then
 116                 return 0
 117         fi
 118
 119
 120         if [ "$LOOPBACK" = "yes" ]; then
 121                 if [ "$UNAME" = "FreeBSD" ] ; then
 122                         cleanup_freebsd_loopback
 123                 else
 124                         cleanup_linux_loopback
 125                 fi
 126         fi
 127
 128         for TEST_FILE in ${FILES}; do
 129                 rm -f "${TEST_FILE}" &>/dev/null
 130         done
 131
 132         if [ "$STF_PATH_REMOVE" = "yes" ] && [ -d "$STF_PATH" ]; then
 133                 rm -Rf "$STF_PATH"
 134         fi
 135 }
 136 trap cleanup EXIT
 137
 138 #
 139 # Attempt to remove all testpools (testpool.XXX), unopened dm devices,
 140 # loopback devices, and files.  This is a useful way to cleanup a previous
 141 # test run failure which has left the system in an unknown state.  This can
 142 # be dangerous and should only be used in a dedicated test environment.
 143 #
 144 cleanup_all() {
 145         local TEST_POOLS
 146         TEST_POOLS=$(sudo "$ZPOOL" list -H -o name | grep testpool)
 147         local TEST_LOOPBACKS
 148         if [ "$UNAME" = "FreeBSD" ] ; then
 149                 TEST_LOOPBACKS=$(sudo "${LOSETUP}" -l)
 150         else
 151                 TEST_LOOPBACKS=$(sudo "${LOSETUP}" -a|grep file-vdev|cut -f1 -d:)
 152         fi
 153         local TEST_FILES
 154         TEST_FILES=$(ls /var/tmp/file-vdev* 2>/dev/null)
 155
 156         msg
 157         msg "--- Cleanup ---"
 158         msg "Removing pool(s):     $(echo "${TEST_POOLS}" | tr '\n' ' ')"
 159         for TEST_POOL in $TEST_POOLS; do
 160                 sudo "$ZPOOL" destroy "${TEST_POOL}"
 161         done
 162
 163         if [ "$UNAME" != "FreeBSD" ] ; then
 164                 msg "Removing dm(s):       $(sudo "${DMSETUP}" ls |
 165                     grep loop | tr '\n' ' ')"
 166                 sudo "${DMSETUP}" remove_all
 167         fi
 168
 169         msg "Removing loopback(s): $(echo "${TEST_LOOPBACKS}" | tr '\n' ' ')"
 170         for TEST_LOOPBACK in $TEST_LOOPBACKS; do
 171                 if [ "$UNAME" = "FreeBSD" ] ; then
 172                         sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}"
 173                 else
 174                         sudo "${LOSETUP}" -d "${TEST_LOOPBACK}"
 175                 fi
 176         done
 177
 178         msg "Removing files(s):    $(echo "${TEST_FILES}" | tr '\n' ' ')"
 179         for TEST_FILE in $TEST_FILES; do
 180                 sudo rm -f "${TEST_FILE}"
 181         done
 182 }
 183
 184 #
 185 # Takes a name as the only arguments and looks for the following variations
 186 # on that name.  If one is found it is returned.
 187 #
 188 # $RUNFILE_DIR/<name>
 189 # $RUNFILE_DIR/<name>.run
 190 # <name>
 191 # <name>.run
 192 #
 193 find_runfile() {
 194         local NAME=$1
 195         local RESULT=""
 196
 197         if [ -f "$RUNFILE_DIR/$NAME" ]; then
 198                 RESULT="$RUNFILE_DIR/$NAME"
 199         elif [ -f "$RUNFILE_DIR/$NAME.run" ]; then
 200                 RESULT="$RUNFILE_DIR/$NAME.run"
 201         elif [ -f "$NAME" ]; then
 202                 RESULT="$NAME"
 203         elif [ -f "$NAME.run" ]; then
 204                 RESULT="$NAME.run"
 205         fi
 206
 207         echo "$RESULT"
 208 }
 209
 210 #
 211 # Symlink file if it appears under any of the given paths.
 212 #
 213 create_links() {
 214         local dir_list="$1"
 215         local file_list="$2"
 216
 217         [ -n "$STF_PATH" ] || fail "STF_PATH wasn't correctly set"
 218
 219         for i in $file_list; do
 220                 for j in $dir_list; do
 221                         [ ! -e "$STF_PATH/$i" ] || continue
 222
 223                         if [ ! -d "$j/$i" ] && [ -e "$j/$i" ]; then
 224                                 ln -sf "$j/$i" "$STF_PATH/$i" || \
 225                                     fail "Couldn't link $i"
 226                                 break
 227                         fi
 228                 done
 229
 230                 [ ! -e "$STF_PATH/$i" ] && STF_MISSING_BIN="$STF_MISSING_BIN$i "
 231         done
 232 }
 233
 234 #
 235 # Constrain the path to limit the available binaries to a known set.
 236 # When running in-tree a top level ./bin/ directory is created for
 237 # convenience, otherwise a temporary directory is used.
 238 #
 239 constrain_path() {
 240         . "$STF_SUITE/include/commands.cfg"
 241
 242         # On FreeBSD, base system zfs utils are in /sbin and OpenZFS utils
 243         # install to /usr/local/sbin. To avoid testing the wrong utils we
 244         # need /usr/local to come before / in the path search order.
 245         SYSTEM_DIRS="/usr/local/bin /usr/local/sbin"
 246         SYSTEM_DIRS+=" /usr/bin /usr/sbin /bin /sbin"
 247
 248         if [ "$INTREE" = "yes" ]; then
 249                 # Constrained path set to ./zfs/bin/
 250                 STF_PATH="$BIN_DIR"
 251                 STF_PATH_REMOVE="no"
 252                 STF_MISSING_BIN=""
 253                 if [ ! -d "$STF_PATH" ]; then
 254                         mkdir "$STF_PATH"
 255                         chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
 256                 fi
 257
 258                 # Special case links for standard zfs utilities
 259                 DIRS="$(find "$CMD_DIR" -type d \( ! -name .deps -a \
 260                     ! -name .libs \) -print | tr '\n' ' ')"
 261                 create_links "$DIRS" "$ZFS_FILES"
 262
 263                 # Special case links for zfs test suite utilities
 264                 DIRS="$(find "$STF_SUITE" -type d \( ! -name .deps -a \
 265                     ! -name .libs \) -print | tr '\n' ' ')"
 266                 create_links "$DIRS" "$ZFSTEST_FILES"
 267         else
 268                 # Constrained path set to /var/tmp/constrained_path.*
 269                 SYSTEMDIR=${SYSTEMDIR:-/var/tmp/constrained_path.XXXX}
 270                 STF_PATH=$(mktemp -d "$SYSTEMDIR")
 271                 STF_PATH_REMOVE="yes"
 272                 STF_MISSING_BIN=""
 273
 274                 chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
 275
 276                 # Special case links for standard zfs utilities
 277                 create_links "$SYSTEM_DIRS" "$ZFS_FILES"
 278
 279                 # Special case links for zfs test suite utilities
 280                 create_links "$STF_SUITE/bin" "$ZFSTEST_FILES"
 281         fi
 282
 283         # Standard system utilities
 284         SYSTEM_FILES="$SYSTEM_FILES_COMMON"
 285         if [ "$UNAME" = "FreeBSD" ] ; then
 286                 SYSTEM_FILES+=" $SYSTEM_FILES_FREEBSD"
 287         else
 288                 SYSTEM_FILES+=" $SYSTEM_FILES_LINUX"
 289         fi
 290         create_links "$SYSTEM_DIRS" "$SYSTEM_FILES"
 291
 292         # Exceptions
 293         ln -fs "$STF_PATH/awk" "$STF_PATH/nawk"
 294         if [ "$UNAME" = "Linux" ] ; then
 295                 ln -fs /sbin/fsck.ext4 "$STF_PATH/fsck"
 296                 ln -fs /sbin/mkfs.ext4 "$STF_PATH/newfs"
 297                 ln -fs "$STF_PATH/gzip" "$STF_PATH/compress"
 298                 ln -fs "$STF_PATH/gunzip" "$STF_PATH/uncompress"
 299                 ln -fs "$STF_PATH/exportfs" "$STF_PATH/share"
 300                 ln -fs "$STF_PATH/exportfs" "$STF_PATH/unshare"
 301         elif [ "$UNAME" = "FreeBSD" ] ; then
 302                 ln -fs /usr/local/bin/ksh93 "$STF_PATH/ksh"
 303         fi
 304
 305         if [ -L "$STF_PATH/arc_summary3" ]; then
 306                 ln -fs "$STF_PATH/arc_summary3" "$STF_PATH/arc_summary"
 307         fi
 308 }
 309
 310 #
 311 # Output a useful usage message.
 312 #
 313 usage() {
 314 cat << EOF
 315 USAGE:
 316 $0 [hvqxkfS] [-s SIZE] [-r RUNFILES] [-t PATH] [-u USER]
 317
 318 DESCRIPTION:
 319         ZFS Test Suite launch script
 320
 321 OPTIONS:
 322         -h          Show this message
 323         -v          Verbose zfs-tests.sh output
 324         -q          Quiet test-runner output
 325         -x          Remove all testpools, dm, lo, and files (unsafe)
 326         -k          Disable cleanup after test failure
 327         -f          Use files only, disables block device tests
 328         -S          Enable stack tracer (negative performance impact)
 329         -c          Only create and populate constrained path
 330         -n NFSFILE  Use the nfsfile to determine the NFS configuration
 331         -I NUM      Number of iterations
 332         -d DIR      Use DIR for files and loopback devices
 333         -s SIZE     Use vdevs of SIZE (default: 4G)
 334         -r RUNFILES Run tests in RUNFILES (default: ${DEFAULT_RUNFILES})
 335         -t PATH     Run single test at PATH relative to test suite
 336         -T TAGS     Comma separated list of tags (default: 'functional')
 337         -u USER     Run single test as USER (default: root)
 338
 339 EXAMPLES:
 340 # Run the default (linux) suite of tests and output the configuration used.
 341 $0 -v
 342
 343 # Run a smaller suite of tests designed to run more quickly.
 344 $0 -r linux-fast
 345
 346 # Run a single test
 347 $0 -t tests/functional/cli_root/zfs_bookmark/zfs_bookmark_cliargs.ksh
 348
 349 # Cleanup a previous run of the test suite prior to testing, run the
 350 # default (linux) suite of tests and perform no cleanup on exit.
 351 $0 -x
 352
 353 EOF
 354 }
 355
 356 while getopts 'hvqxkfScn:d:s:r:?t:T:u:I:' OPTION; do
 357         case $OPTION in
 358         h)
 359                 usage
 360                 exit 1
 361                 ;;
 362         v)
 363                 # shellcheck disable=SC2034
 364                 VERBOSE="yes"
 365                 ;;
 366         q)
 367                 QUIET="yes"
 368                 ;;
 369         x)
 370                 CLEANUPALL="yes"
 371                 ;;
 372         k)
 373                 CLEANUP="no"
 374                 ;;
 375         f)
 376                 LOOPBACK="no"
 377                 ;;
 378         S)
 379                 STACK_TRACER="yes"
 380                 ;;
 381         c)
 382                 constrain_path
 383                 exit
 384                 ;;
 385         n)
 386                 nfsfile=$OPTARG
 387                 [[ -f $nfsfile ]] || fail "Cannot read file: $nfsfile"
 388                 export NFS=1
 389                 . "$nfsfile"
 390                 ;;
 391         d)
 392                 FILEDIR="$OPTARG"
 393                 ;;
 394         I)
 395                 ITERATIONS="$OPTARG"
 396                 if [ "$ITERATIONS" -le 0 ]; then
 397                         fail "Iterations must be greater than 0."
 398                 fi
 399                 ;;
 400         s)
 401                 FILESIZE="$OPTARG"
 402                 ;;
 403         r)
 404                 RUNFILES="$OPTARG"
 405                 ;;
 406         t)
 407                 if [ ${#SINGLETEST[@]} -ne 0 ]; then
 408                         fail "-t can only be provided once."
 409                 fi
 410                 SINGLETEST+=("$OPTARG")
 411                 ;;
 412         T)
 413                 TAGS="$OPTARG"
 414                 ;;
 415         u)
 416                 SINGLETESTUSER="$OPTARG"
 417                 ;;
 418         ?)
 419                 usage
 420                 exit
 421                 ;;
 422         esac
 423 done
 424
 425 shift $((OPTIND-1))
 426
 427 FILES=${FILES:-"$FILEDIR/file-vdev0 $FILEDIR/file-vdev1 $FILEDIR/file-vdev2"}
 428 LOOPBACKS=${LOOPBACKS:-""}
 429
 430 if [ ${#SINGLETEST[@]} -ne 0 ]; then
 431         if [ -n "$TAGS" ]; then
 432                 fail "-t and -T are mutually exclusive."
 433         fi
 434         RUNFILE_DIR="/var/tmp"
 435         RUNFILES="zfs-tests.$$.run"
 436         SINGLEQUIET="False"
 437
 438         if [ -n "$QUIET" ]; then
 439                 SINGLEQUIET="True"
 440         fi
 441
 442         cat >$RUNFILE_DIR/$RUNFILES << EOF
 443 [DEFAULT]
 444 pre =
 445 quiet = $SINGLEQUIET
 446 pre_user = root
 447 user = $SINGLETESTUSER
 448 timeout = 600
 449 post_user = root
 450 post =
 451 outputdir = /var/tmp/test_results
 452 EOF
 453         for t in "${SINGLETEST[@]}"
 454         do
 455                 SINGLETESTDIR=$(dirname "$t")
 456                 SINGLETESTFILE=$(basename "$t")
 457                 SETUPSCRIPT=
 458                 CLEANUPSCRIPT=
 459
 460                 if [ -f "$STF_SUITE/$SINGLETESTDIR/setup.ksh" ]; then
 461                         SETUPSCRIPT="setup"
 462                 fi
 463
 464                 if [ -f "$STF_SUITE/$SINGLETESTDIR/cleanup.ksh" ]; then
 465                         CLEANUPSCRIPT="cleanup"
 466                 fi
 467
 468                 cat >>$RUNFILE_DIR/$RUNFILES << EOF
 469
 470 [$SINGLETESTDIR]
 471 tests = ['$SINGLETESTFILE']
 472 pre = $SETUPSCRIPT
 473 post = $CLEANUPSCRIPT
 474 tags = ['functional']
 475 EOF
 476         done
 477 fi
 478
 479 #
 480 # Use default tag if none was specified
 481 #
 482 TAGS=${TAGS:='functional'}
 483
 484 #
 485 # Attempt to locate the runfiles describing the test workload.
 486 #
 487 R=""
 488 IFS=,
 489 for RUNFILE in $RUNFILES; do
 490         if [ -n "$RUNFILE" ]; then
 491                 SAVED_RUNFILE="$RUNFILE"
 492                 RUNFILE=$(find_runfile "$RUNFILE")
 493                 [ -z "$RUNFILE" ] && fail "Cannot find runfile: $SAVED_RUNFILE"
 494                 R+="${R:+,}${RUNFILE}"
 495         fi
 496
 497         if [ ! -r "$RUNFILE" ]; then
 498                 fail "Cannot read runfile: $RUNFILE"
 499         fi
 500 done
 501 unset IFS
 502 RUNFILES=$R
 503
 504 #
 505 # This script should not be run as root.  Instead the test user, which may
 506 # be a normal user account, needs to be configured such that it can
 507 # run commands via sudo passwordlessly.
 508 #
 509 if [ "$(id -u)" = "0" ]; then
 510         fail "This script must not be run as root."
 511 fi
 512
 513 if [ "$(sudo whoami)" != "root" ]; then
 514         fail "Passwordless sudo access required."
 515 fi
 516
 517 #
 518 # Constrain the available binaries to a known set.
 519 #
 520 constrain_path
 521
 522 #
 523 # Check if ksh exists
 524 #
 525 if [ "$UNAME" = "FreeBSD" ]; then
 526         sudo ln -fs /usr/local/bin/ksh93 /bin/ksh
 527 fi
 528 [ -e "$STF_PATH/ksh" ] || fail "This test suite requires ksh."
 529 [ -e "$STF_SUITE/include/default.cfg" ] || fail \
 530     "Missing $STF_SUITE/include/default.cfg file."
 531
 532 #
 533 # Verify the ZFS module stack is loaded.
 534 #
 535 if [ "$STACK_TRACER" = "yes" ]; then
 536         sudo "${ZFS_SH}" -S &>/dev/null
 537 else
 538         sudo "${ZFS_SH}" &>/dev/null
 539 fi
 540
 541 #
 542 # Attempt to cleanup all previous state for a new test run.
 543 #
 544 if [ "$CLEANUPALL" = "yes" ]; then
 545         cleanup_all
 546 fi
 547
 548 #
 549 # By default preserve any existing pools
 550 # NOTE: Since 'zpool list' outputs a newline-delimited list convert $KEEP from
 551 # space-delimited to newline-delimited.
 552 #
 553 if [ -z "${KEEP}" ]; then
 554         KEEP="$(sudo "$ZPOOL" list -H -o name)"
 555         if [ -z "${KEEP}" ]; then
 556                 KEEP="rpool"
 557         fi
 558 else
 559         KEEP="$(echo -e "${KEEP//[[:blank:]]/\n}")"
 560 fi
 561
 562 #
 563 # NOTE: The following environment variables are undocumented
 564 # and should be used for testing purposes only:
 565 #
 566 # __ZFS_POOL_EXCLUDE - don't iterate over the pools it lists
 567 # __ZFS_POOL_RESTRICT - iterate only over the pools it lists
 568 #
 569 # See libzfs/libzfs_config.c for more information.
 570 #
 571 if [ "$UNAME" = "FreeBSD" ] ; then
 572         __ZFS_POOL_EXCLUDE="$(echo "$KEEP" | tr -s '\n' ' ')"
 573 else
 574         __ZFS_POOL_EXCLUDE="$(echo "$KEEP" | sed ':a;N;s/\n/ /g;ba')"
 575 fi
 576
 577 . "$STF_SUITE/include/default.cfg"
 578
 579 msg
 580 msg "--- Configuration ---"
 581 msg "Runfiles:        $RUNFILES"
 582 msg "STF_TOOLS:       $STF_TOOLS"
 583 msg "STF_SUITE:       $STF_SUITE"
 584 msg "STF_PATH:        $STF_PATH"
 585
 586 #
 587 # No DISKS have been provided so a basic file or loopback based devices
 588 # must be created for the test suite to use.
 589 #
 590 if [ -z "${DISKS}" ]; then
 591         #
 592         # Create sparse files for the test suite.  These may be used
 593         # directory or have loopback devices layered on them.
 594         #
 595         for TEST_FILE in ${FILES}; do
 596                 [ -f "$TEST_FILE" ] && fail "Failed file exists: ${TEST_FILE}"
 597                 truncate -s "${FILESIZE}" "${TEST_FILE}" ||
 598                     fail "Failed creating: ${TEST_FILE} ($?)"
 599                 if [[ "$DISKS" ]]; then
 600                         DISKS="$DISKS $TEST_FILE"
 601                 else
 602                         DISKS="$TEST_FILE"
 603                 fi
 604         done
 605
 606         #
 607         # If requested setup loopback devices backed by the sparse files.
 608         #
 609         if [ "$LOOPBACK" = "yes" ]; then
 610                 DISKS=""
 611
 612                 test -x "$LOSETUP" || fail "$LOSETUP utility must be installed"
 613
 614                 for TEST_FILE in ${FILES}; do
 615                         if [ "$UNAME" = "FreeBSD" ] ; then
 616                                 MDDEVICE=$(sudo "${LOSETUP}" -a -t vnode -f "${TEST_FILE}")
 617                                 if [ -z "$MDDEVICE" ] ; then
 618                                         fail "Failed: ${TEST_FILE} -> loopback"
 619                                 fi
 620                                 LOOPBACKS="${LOOPBACKS}${MDDEVICE} "
 621                                 if [[ "$DISKS" ]]; then
 622                                         DISKS="$DISKS $MDDEVICE"
 623                                 else
 624                                         DISKS="$MDDEVICE"
 625                                 fi
 626                         else
 627                                 TEST_LOOPBACK=$(sudo "${LOSETUP}" -f)
 628                                 sudo "${LOSETUP}" "${TEST_LOOPBACK}" "${TEST_FILE}" ||
 629                                     fail "Failed: ${TEST_FILE} -> ${TEST_LOOPBACK}"
 630                                 LOOPBACKS="${LOOPBACKS}${TEST_LOOPBACK} "
 631                                 BASELOOPBACKS=$(basename "$TEST_LOOPBACK")
 632                                 if [[ "$DISKS" ]]; then
 633                                         DISKS="$DISKS $BASELOOPBACKS"
 634                                 else
 635                                         DISKS="$BASELOOPBACKS"
 636                                 fi
 637                         fi
 638                 done
 639         fi
 640 fi
 641
 642 NUM_DISKS=$(echo "${DISKS}" | awk '{print NF}')
 643 [ "$NUM_DISKS" -lt 3 ] && fail "Not enough disks ($NUM_DISKS/3 minimum)"
 644
 645 #
 646 # Disable SELinux until the ZFS Test Suite has been updated accordingly.
 647 #
 648 if [ -x "$STF_PATH/setenforce" ]; then
 649         sudo setenforce permissive &>/dev/null
 650 fi
 651
 652 #
 653 # Enable internal ZFS debug log and clear it.
 654 #
 655 if [ -e /sys/module/zfs/parameters/zfs_dbgmsg_enable ]; then
 656         sudo /bin/sh -c "echo 1 >/sys/module/zfs/parameters/zfs_dbgmsg_enable"
 657         sudo /bin/sh -c "echo 0 >/proc/spl/kstat/zfs/dbgmsg"
 658 fi
 659
 660 msg "FILEDIR:         $FILEDIR"
 661 msg "FILES:           $FILES"
 662 msg "LOOPBACKS:       $LOOPBACKS"
 663 msg "DISKS:           $DISKS"
 664 msg "NUM_DISKS:       $NUM_DISKS"
 665 msg "FILESIZE:        $FILESIZE"
 666 msg "ITERATIONS:      $ITERATIONS"
 667 msg "TAGS:            $TAGS"
 668 msg "STACK_TRACER:    $STACK_TRACER"
 669 msg "Keep pool(s):    $KEEP"
 670 msg "Missing util(s): $STF_MISSING_BIN"
 671 msg ""
 672
 673 export STF_TOOLS
 674 export STF_SUITE
 675 export STF_PATH
 676 export DISKS
 677 export FILEDIR
 678 export KEEP
 679 export __ZFS_POOL_EXCLUDE
 680 export TESTFAIL_CALLBACKS
 681 export PATH=$STF_PATH
 682
 683 if [ "$UNAME" = "FreeBSD" ] ; then
 684         mkdir -p "$FILEDIR" || true
 685         RESULTS_FILE=$(mktemp -u "${FILEDIR}/zts-results.XXXX")
 686         REPORT_FILE=$(mktemp -u "${FILEDIR}/zts-report.XXXX")
 687 else
 688         RESULTS_FILE=$(mktemp -u -t zts-results.XXXX -p "$FILEDIR")
 689         REPORT_FILE=$(mktemp -u -t zts-report.XXXX -p "$FILEDIR")
 690 fi
 691
 692 #
 693 # Run all the tests as specified.
 694 #
 695 msg "${TEST_RUNNER} ${QUIET:+-q}" \
 696     "-c \"${RUNFILES}\"" \
 697     "-T \"${TAGS}\"" \
 698     "-i \"${STF_SUITE}\"" \
 699     "-I \"${ITERATIONS}\""
 700 ${TEST_RUNNER} ${QUIET:+-q} \
 701     -c "${RUNFILES}" \
 702     -T "${TAGS}" \
 703     -i "${STF_SUITE}" \
 704     -I "${ITERATIONS}" \
 705     2>&1 | tee "$RESULTS_FILE"
 706
 707 #
 708 # Analyze the results.
 709 #
 710 set -o pipefail
 711 ${ZTS_REPORT} "$RESULTS_FILE" | tee "$REPORT_FILE"
 712 RESULT=$?
 713 set +o pipefail
 714
 715 RESULTS_DIR=$(awk '/^Log directory/ { print $3 }' "$RESULTS_FILE")
 716 if [ -d "$RESULTS_DIR" ]; then
 717         cat "$RESULTS_FILE" "$REPORT_FILE" >"$RESULTS_DIR/results"
 718 fi
 719
 720 rm -f "$RESULTS_FILE" "$REPORT_FILE"
 721
 722 if [ ${#SINGLETEST[@]} -ne 0 ]; then
 723         rm -f "$RUNFILES" &>/dev/null
 724 fi
 725
 726 exit ${RESULT}