scripts/zfs-tests.sh

   1 #!/bin/sh
   2 # shellcheck disable=SC2154
   3 #
   4 # CDDL HEADER START
   5 #
   6 # The contents of this file are subject to the terms of the
   7 # Common Development and Distribution License, Version 1.0 only
   8 # (the "License").  You may not use this file except in compliance
   9 # with the License.
  10 #
  11 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  12 # or http://www.opensolaris.org/os/licensing.
  13 # See the License for the specific language governing permissions
  14 # and limitations under the License.
  15 #
  16 # When distributing Covered Code, include this CDDL HEADER in each
  17 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  18 # If applicable, add the following below this CDDL HEADER, with the
  19 # fields enclosed by brackets "[]" replaced with your own identifying
  20 # information: Portions Copyright [yyyy] [name of copyright owner]
  21 #
  22 # CDDL HEADER END
  23 #
  24
  25 #
  26 # Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
  27 #
  28
  29 SCRIPT_COMMON=${SCRIPT_COMMON:-${0%/*}/common.sh}
  30 . "${SCRIPT_COMMON}" || exit
  31
  32 PROG=zfs-tests.sh
  33 VERBOSE="no"
  34 QUIET=""
  35 CLEANUP="yes"
  36 CLEANUPALL="no"
  37 KMSG=""
  38 LOOPBACK="yes"
  39 STACK_TRACER="no"
  40 FILESIZE="4G"
  41 DEFAULT_RUNFILES="common.run,$(uname | tr '[:upper:]' '[:lower:]').run"
  42 RUNFILES=${RUNFILES:-$DEFAULT_RUNFILES}
  43 FILEDIR=${FILEDIR:-/var/tmp}
  44 DISKS=${DISKS:-""}
  45 SINGLETEST=""
  46 SINGLETESTUSER="root"
  47 TAGS=""
  48 ITERATIONS=1
  49 ZFS_DBGMSG="$STF_SUITE/callbacks/zfs_dbgmsg.ksh"
  50 ZFS_DMESG="$STF_SUITE/callbacks/zfs_dmesg.ksh"
  51 UNAME=$(uname)
  52 RERUN=""
  53 KMEMLEAK=""
  54
  55 # Override some defaults if on FreeBSD
  56 if [ "$UNAME" = "FreeBSD" ] ; then
  57         TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DMESG"}
  58         LOSETUP=/sbin/mdconfig
  59         DMSETUP=/sbin/gpart
  60 else
  61         ZFS_MMP="$STF_SUITE/callbacks/zfs_mmp.ksh"
  62         TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DBGMSG:$ZFS_DMESG:$ZFS_MMP"}
  63         LOSETUP=${LOSETUP:-/sbin/losetup}
  64         DMSETUP=${DMSETUP:-/sbin/dmsetup}
  65 fi
  66
  67 #
  68 # Log an informational message when additional verbosity is enabled.
  69 #
  70 msg() {
  71         if [ "$VERBOSE" = "yes" ]; then
  72                 echo "$@"
  73         fi
  74 }
  75
  76 #
  77 # Log a failure message, cleanup, and return an error.
  78 #
  79 fail() {
  80         echo "$PROG: $1" >&2
  81         cleanup
  82         exit 1
  83 }
  84
  85 cleanup_freebsd_loopback() {
  86         for TEST_LOOPBACK in ${LOOPBACKS}; do
  87                 if [ -c "/dev/${TEST_LOOPBACK}" ]; then
  88                         sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}" ||
  89                             echo "Failed to destroy: ${TEST_LOOPBACK}"
  90                 fi
  91         done
  92 }
  93
  94 cleanup_linux_loopback() {
  95         for TEST_LOOPBACK in ${LOOPBACKS}; do
  96                 LOOP_DEV="${TEST_LOOPBACK##*/}"
  97                 DM_DEV=$(sudo "${DMSETUP}" ls 2>/dev/null | \
  98                     awk -v l="${LOOP_DEV}" '$0 ~ l {print $1}')
  99
 100                 if [ -n "$DM_DEV" ]; then
 101                         sudo "${DMSETUP}" remove "${DM_DEV}" ||
 102                             echo "Failed to remove: ${DM_DEV}"
 103                 fi
 104
 105                 if [ -n "${TEST_LOOPBACK}" ]; then
 106                         sudo "${LOSETUP}" -d "${TEST_LOOPBACK}" ||
 107                             echo "Failed to remove: ${TEST_LOOPBACK}"
 108                 fi
 109         done
 110 }
 111
 112 #
 113 # Attempt to remove loopback devices and files which where created earlier
 114 # by this script to run the test framework.  The '-k' option may be passed
 115 # to the script to suppress cleanup for debugging purposes.
 116 #
 117 cleanup() {
 118         if [ "$CLEANUP" = "no" ]; then
 119                 return 0
 120         fi
 121
 122
 123         if [ "$LOOPBACK" = "yes" ]; then
 124                 if [ "$UNAME" = "FreeBSD" ] ; then
 125                         cleanup_freebsd_loopback
 126                 else
 127                         cleanup_linux_loopback
 128                 fi
 129         fi
 130
 131         # shellcheck disable=SC2086
 132         rm -f ${FILES} >/dev/null 2>&1
 133
 134         if [ "$STF_PATH_REMOVE" = "yes" ] && [ -d "$STF_PATH" ]; then
 135                 rm -Rf "$STF_PATH"
 136         fi
 137 }
 138 trap cleanup EXIT
 139
 140 #
 141 # Attempt to remove all testpools (testpool.XXX), unopened dm devices,
 142 # loopback devices, and files.  This is a useful way to cleanup a previous
 143 # test run failure which has left the system in an unknown state.  This can
 144 # be dangerous and should only be used in a dedicated test environment.
 145 #
 146 cleanup_all() {
 147         TEST_POOLS=$(ASAN_OPTIONS=detect_leaks=false "$ZPOOL" list -Ho name | grep testpool)
 148         if [ "$UNAME" = "FreeBSD" ] ; then
 149                 TEST_LOOPBACKS=$(sudo "${LOSETUP}" -l)
 150         else
 151                 TEST_LOOPBACKS=$("${LOSETUP}" -a | awk -F: '/file-vdev/ {print $1}')
 152         fi
 153         TEST_FILES=$(ls "${FILEDIR}"/file-vdev* /var/tmp/file-vdev* 2>/dev/null)
 154
 155         msg
 156         msg "--- Cleanup ---"
 157         # shellcheck disable=2116,2086
 158         msg "Removing pool(s):     $(echo ${TEST_POOLS})"
 159         for TEST_POOL in $TEST_POOLS; do
 160                 sudo env ASAN_OPTIONS=detect_leaks=false "$ZPOOL" destroy "${TEST_POOL}"
 161         done
 162
 163         if [ "$UNAME" != "FreeBSD" ] ; then
 164                 msg "Removing all dm(s):   $(sudo "${DMSETUP}" ls |
 165                     grep loop | tr '\n' ' ')"
 166                 sudo "${DMSETUP}" remove_all
 167         fi
 168
 169         # shellcheck disable=2116,2086
 170         msg "Removing loopback(s): $(echo ${TEST_LOOPBACKS})"
 171         for TEST_LOOPBACK in $TEST_LOOPBACKS; do
 172                 if [ "$UNAME" = "FreeBSD" ] ; then
 173                         sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}"
 174                 else
 175                         sudo "${LOSETUP}" -d "${TEST_LOOPBACK}"
 176                 fi
 177         done
 178
 179         # shellcheck disable=2116,2086
 180         msg "Removing files(s):    $(echo ${TEST_FILES})"
 181         # shellcheck disable=2086
 182         sudo rm -f ${TEST_FILES}
 183 }
 184
 185 #
 186 # Takes a name as the only arguments and looks for the following variations
 187 # on that name.  If one is found it is returned.
 188 #
 189 # $RUNFILE_DIR/<name>
 190 # $RUNFILE_DIR/<name>.run
 191 # <name>
 192 # <name>.run
 193 #
 194 find_runfile() {
 195         NAME=$1
 196
 197         if [ -f "$RUNFILE_DIR/$NAME" ]; then
 198                 echo "$RUNFILE_DIR/$NAME"
 199         elif [ -f "$RUNFILE_DIR/$NAME.run" ]; then
 200                 echo "$RUNFILE_DIR/$NAME.run"
 201         elif [ -f "$NAME" ]; then
 202                 echo "$NAME"
 203         elif [ -f "$NAME.run" ]; then
 204                 echo "$NAME.run"
 205         else
 206                 return 1
 207         fi
 208 }
 209
 210 #
 211 # Symlink file if it appears under any of the given paths.
 212 #
 213 create_links() {
 214         dir_list="$1"
 215         file_list="$2"
 216
 217         [ -n "$STF_PATH" ] || fail "STF_PATH wasn't correctly set"
 218
 219         for i in $file_list; do
 220                 for j in $dir_list; do
 221                         [ ! -e "$STF_PATH/$i" ] || continue
 222
 223                         if [ ! -d "$j/$i" ] && [ -e "$j/$i" ]; then
 224                                 ln -sf "$j/$i" "$STF_PATH/$i" || \
 225                                     fail "Couldn't link $i"
 226                                 break
 227                         fi
 228                 done
 229
 230                 [ ! -e "$STF_PATH/$i" ] && \
 231                     STF_MISSING_BIN="$STF_MISSING_BIN $i"
 232         done
 233         STF_MISSING_BIN=${STF_MISSING_BIN# }
 234 }
 235
 236 #
 237 # Constrain the path to limit the available binaries to a known set.
 238 # When running in-tree a top level ./bin/ directory is created for
 239 # convenience, otherwise a temporary directory is used.
 240 #
 241 constrain_path() {
 242         . "$STF_SUITE/include/commands.cfg"
 243
 244         # On FreeBSD, base system zfs utils are in /sbin and OpenZFS utils
 245         # install to /usr/local/sbin. To avoid testing the wrong utils we
 246         # need /usr/local to come before / in the path search order.
 247         SYSTEM_DIRS="/usr/local/bin /usr/local/sbin"
 248         SYSTEM_DIRS="$SYSTEM_DIRS /usr/bin /usr/sbin /bin /sbin $LIBEXEC_DIR"
 249
 250         if [ "$INTREE" = "yes" ]; then
 251                 # Constrained path set to $(top_builddir)/tests/zfs-tests/bin
 252                 STF_PATH="$BIN_DIR"
 253                 STF_PATH_REMOVE="no"
 254                 STF_MISSING_BIN=""
 255                 if [ ! -d "$STF_PATH" ]; then
 256                         mkdir "$STF_PATH"
 257                         chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
 258                 fi
 259
 260                 # Special case links for standard zfs utilities
 261                 create_links "$CMD_DIR" "$ZFS_FILES"
 262
 263                 # Special case links for zfs test suite utilities
 264                 create_links "$CMD_DIR/tests/zfs-tests/cmd" "$ZFSTEST_FILES"
 265         else
 266                 # Constrained path set to /var/tmp/constrained_path.*
 267                 SYSTEMDIR=${SYSTEMDIR:-/var/tmp/constrained_path.XXXXXX}
 268                 STF_PATH=$(mktemp -d "$SYSTEMDIR")
 269                 STF_PATH_REMOVE="yes"
 270                 STF_MISSING_BIN=""
 271
 272                 chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
 273
 274                 # Special case links for standard zfs utilities
 275                 create_links "$SYSTEM_DIRS" "$ZFS_FILES"
 276
 277                 # Special case links for zfs test suite utilities
 278                 create_links "$STF_SUITE/bin" "$ZFSTEST_FILES"
 279         fi
 280
 281         # Standard system utilities
 282         SYSTEM_FILES="$SYSTEM_FILES_COMMON"
 283         if [ "$UNAME" = "FreeBSD" ] ; then
 284                 SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_FREEBSD"
 285         else
 286                 SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_LINUX"
 287         fi
 288         create_links "$SYSTEM_DIRS" "$SYSTEM_FILES"
 289
 290         # Exceptions
 291         if [ "$UNAME" = "Linux" ] ; then
 292                 ln -fs /sbin/fsck.ext4 "$STF_PATH/fsck"
 293                 ln -fs /sbin/mkfs.ext4 "$STF_PATH/newfs"
 294                 ln -fs "$STF_PATH/gzip" "$STF_PATH/compress"
 295                 ln -fs "$STF_PATH/gunzip" "$STF_PATH/uncompress"
 296         elif [ "$UNAME" = "FreeBSD" ] ; then
 297                 ln -fs /usr/local/bin/ksh93 "$STF_PATH/ksh"
 298         fi
 299 }
 300
 301 #
 302 # Output a useful usage message.
 303 #
 304 usage() {
 305 cat << EOF
 306 USAGE:
 307 $0 [-hvqxkfS] [-s SIZE] [-r RUNFILES] [-t PATH] [-u USER]
 308
 309 DESCRIPTION:
 310         ZFS Test Suite launch script
 311
 312 OPTIONS:
 313         -h          Show this message
 314         -v          Verbose zfs-tests.sh output
 315         -q          Quiet test-runner output
 316         -x          Remove all testpools, dm, lo, and files (unsafe)
 317         -k          Disable cleanup after test failure
 318         -K          Log test names to /dev/kmsg
 319         -f          Use files only, disables block device tests
 320         -S          Enable stack tracer (negative performance impact)
 321         -c          Only create and populate constrained path
 322         -R          Automatically rerun failing tests
 323         -m          Enable kmemleak reporting (Linux only)
 324         -n NFSFILE  Use the nfsfile to determine the NFS configuration
 325         -I NUM      Number of iterations
 326         -d DIR      Use world-writable DIR for files and loopback devices
 327         -s SIZE     Use vdevs of SIZE (default: 4G)
 328         -r RUNFILES Run tests in RUNFILES (default: ${DEFAULT_RUNFILES})
 329         -t PATH     Run single test at PATH relative to test suite
 330         -T TAGS     Comma separated list of tags (default: 'functional')
 331         -u USER     Run single test as USER (default: root)
 332
 333 EXAMPLES:
 334 # Run the default ($(echo "${DEFAULT_RUNFILES}" | sed 's/\.run//')) suite of tests and output the configuration used.
 335 $0 -v
 336
 337 # Run a smaller suite of tests designed to run more quickly.
 338 $0 -r linux-fast
 339
 340 # Run a single test
 341 $0 -t tests/functional/cli_root/zfs_bookmark/zfs_bookmark_cliargs.ksh
 342
 343 # Cleanup a previous run of the test suite prior to testing, run the
 344 # default ($(echo "${DEFAULT_RUNFILES}" | sed 's/\.run//')) suite of tests and perform no cleanup on exit.
 345 $0 -x
 346
 347 EOF
 348 }
 349
 350 while getopts 'hvqxkKfScRmn:d:s:r:?t:T:u:I:' OPTION; do
 351         case $OPTION in
 352         h)
 353                 usage
 354                 exit 1
 355                 ;;
 356         v)
 357                 VERBOSE="yes"
 358                 ;;
 359         q)
 360                 QUIET="yes"
 361                 ;;
 362         x)
 363                 CLEANUPALL="yes"
 364                 ;;
 365         k)
 366                 CLEANUP="no"
 367                 ;;
 368         K)
 369                 KMSG="yes"
 370                 ;;
 371         f)
 372                 LOOPBACK="no"
 373                 ;;
 374         S)
 375                 STACK_TRACER="yes"
 376                 ;;
 377         c)
 378                 constrain_path
 379                 exit
 380                 ;;
 381         R)
 382                 RERUN="yes"
 383                 ;;
 384         m)
 385                 KMEMLEAK="yes"
 386                 ;;
 387         n)
 388                 nfsfile=$OPTARG
 389                 [ -f "$nfsfile" ] || fail "Cannot read file: $nfsfile"
 390                 export NFS=1
 391                 . "$nfsfile"
 392                 ;;
 393         d)
 394                 FILEDIR="$OPTARG"
 395                 ;;
 396         I)
 397                 ITERATIONS="$OPTARG"
 398                 if [ "$ITERATIONS" -le 0 ]; then
 399                         fail "Iterations must be greater than 0."
 400                 fi
 401                 ;;
 402         s)
 403                 FILESIZE="$OPTARG"
 404                 ;;
 405         r)
 406                 RUNFILES="$OPTARG"
 407                 ;;
 408         t)
 409                 if [ -n "$SINGLETEST" ]; then
 410                         fail "-t can only be provided once."
 411                 fi
 412                 SINGLETEST="$OPTARG"
 413                 ;;
 414         T)
 415                 TAGS="$OPTARG"
 416                 ;;
 417         u)
 418                 SINGLETESTUSER="$OPTARG"
 419                 ;;
 420         ?)
 421                 usage
 422                 exit
 423                 ;;
 424         *)
 425                 ;;
 426         esac
 427 done
 428
 429 shift $((OPTIND-1))
 430
 431 FILES=${FILES:-"$FILEDIR/file-vdev0 $FILEDIR/file-vdev1 $FILEDIR/file-vdev2"}
 432 LOOPBACKS=${LOOPBACKS:-""}
 433
 434 if [ -n "$SINGLETEST" ]; then
 435         if [ -n "$TAGS" ]; then
 436                 fail "-t and -T are mutually exclusive."
 437         fi
 438         RUNFILE_DIR="/var/tmp"
 439         RUNFILES="zfs-tests.$$.run"
 440         [ -n "$QUIET" ] && SINGLEQUIET="True" || SINGLEQUIET="False"
 441
 442         cat >"${RUNFILE_DIR}/${RUNFILES}" << EOF
 443 [DEFAULT]
 444 pre =
 445 quiet = $SINGLEQUIET
 446 pre_user = root
 447 user = $SINGLETESTUSER
 448 timeout = 600
 449 post_user = root
 450 post =
 451 outputdir = /var/tmp/test_results
 452 EOF
 453         SINGLETESTDIR="${SINGLETEST%/*}"
 454
 455         SETUPDIR="$SINGLETESTDIR"
 456         [ "${SETUPDIR#/}" = "$SETUPDIR" ] && SETUPDIR="$STF_SUITE/$SINGLETESTDIR"
 457         [ -x "$SETUPDIR/setup.ksh"   ] && SETUPSCRIPT="setup"     || SETUPSCRIPT=
 458         [ -x "$SETUPDIR/cleanup.ksh" ] && CLEANUPSCRIPT="cleanup" || CLEANUPSCRIPT=
 459
 460         SINGLETESTFILE="${SINGLETEST##*/}"
 461         cat >>"${RUNFILE_DIR}/${RUNFILES}" << EOF
 462
 463 [$SINGLETESTDIR]
 464 tests = ['$SINGLETESTFILE']
 465 pre = $SETUPSCRIPT
 466 post = $CLEANUPSCRIPT
 467 tags = ['functional']
 468 EOF
 469 fi
 470
 471 #
 472 # Use default tag if none was specified
 473 #
 474 TAGS=${TAGS:='functional'}
 475
 476 #
 477 # Attempt to locate the runfiles describing the test workload.
 478 #
 479 R=""
 480 IFS=,
 481 for RUNFILE in $RUNFILES; do
 482         if [ -n "$RUNFILE" ]; then
 483                 SAVED_RUNFILE="$RUNFILE"
 484                 RUNFILE=$(find_runfile "$RUNFILE") ||
 485                         fail "Cannot find runfile: $SAVED_RUNFILE"
 486                 R="$R,$RUNFILE"
 487         fi
 488
 489         if [ ! -r "$RUNFILE" ]; then
 490                 fail "Cannot read runfile: $RUNFILE"
 491         fi
 492 done
 493 unset IFS
 494 RUNFILES=${R#,}
 495
 496 #
 497 # This script should not be run as root.  Instead the test user, which may
 498 # be a normal user account, needs to be configured such that it can
 499 # run commands via sudo passwordlessly.
 500 #
 501 if [ "$(id -u)" = "0" ]; then
 502         fail "This script must not be run as root."
 503 fi
 504
 505 if [ "$(sudo id -un)" != "root" ]; then
 506         fail "Passwordless sudo access required."
 507 fi
 508
 509 #
 510 # Constrain the available binaries to a known set.
 511 #
 512 constrain_path
 513
 514 #
 515 # Check if ksh exists
 516 #
 517 if [ "$UNAME" = "FreeBSD" ]; then
 518         sudo ln -fs /usr/local/bin/ksh93 /bin/ksh
 519 fi
 520 [ -e "$STF_PATH/ksh" ] || fail "This test suite requires ksh."
 521 [ -e "$STF_SUITE/include/default.cfg" ] || fail \
 522     "Missing $STF_SUITE/include/default.cfg file."
 523
 524 #
 525 # Verify the ZFS module stack is loaded.
 526 #
 527 if [ "$STACK_TRACER" = "yes" ]; then
 528         sudo "${ZFS_SH}" -S >/dev/null 2>&1
 529 else
 530         sudo "${ZFS_SH}" >/dev/null 2>&1
 531 fi
 532
 533 #
 534 # Attempt to cleanup all previous state for a new test run.
 535 #
 536 if [ "$CLEANUPALL" = "yes" ]; then
 537         cleanup_all
 538 fi
 539
 540 #
 541 # By default preserve any existing pools
 542 #
 543 if [ -z "${KEEP}" ]; then
 544         KEEP="$(ASAN_OPTIONS=detect_leaks=false "$ZPOOL" list -Ho name | tr -s '[:space:]' ' ')"
 545         if [ -z "${KEEP}" ]; then
 546                 KEEP="rpool"
 547         fi
 548 else
 549         KEEP="$(echo "$KEEP" | tr -s '[:space:]' ' ')"
 550 fi
 551
 552 #
 553 # NOTE: The following environment variables are undocumented
 554 # and should be used for testing purposes only:
 555 #
 556 # __ZFS_POOL_EXCLUDE - don't iterate over the pools it lists
 557 # __ZFS_POOL_RESTRICT - iterate only over the pools it lists
 558 #
 559 # See libzfs/libzfs_config.c for more information.
 560 #
 561 __ZFS_POOL_EXCLUDE="$KEEP"
 562
 563 . "$STF_SUITE/include/default.cfg"
 564
 565 #
 566 # No DISKS have been provided so a basic file or loopback based devices
 567 # must be created for the test suite to use.
 568 #
 569 if [ -z "${DISKS}" ]; then
 570         #
 571         # If this is a performance run, prevent accidental use of
 572         # loopback devices.
 573         #
 574         [ "$TAGS" = "perf" ] && fail "Running perf tests without disks."
 575
 576         #
 577         # Create sparse files for the test suite.  These may be used
 578         # directory or have loopback devices layered on them.
 579         #
 580         for TEST_FILE in ${FILES}; do
 581                 [ -f "$TEST_FILE" ] && fail "Failed file exists: ${TEST_FILE}"
 582                 truncate -s "${FILESIZE}" "${TEST_FILE}" ||
 583                     fail "Failed creating: ${TEST_FILE} ($?)"
 584         done
 585
 586         #
 587         # If requested setup loopback devices backed by the sparse files.
 588         #
 589         if [ "$LOOPBACK" = "yes" ]; then
 590                 test -x "$LOSETUP" || fail "$LOSETUP utility must be installed"
 591
 592                 for TEST_FILE in ${FILES}; do
 593                         if [ "$UNAME" = "FreeBSD" ] ; then
 594                                 MDDEVICE=$(sudo "${LOSETUP}" -a -t vnode -f "${TEST_FILE}")
 595                                 if [ -z "$MDDEVICE" ] ; then
 596                                         fail "Failed: ${TEST_FILE} -> loopback"
 597                                 fi
 598                                 DISKS="$DISKS $MDDEVICE"
 599                                 LOOPBACKS="$LOOPBACKS $MDDEVICE"
 600                         else
 601                                 TEST_LOOPBACK=$(sudo "${LOSETUP}" --show -f "${TEST_FILE}") ||
 602                                     fail "Failed: ${TEST_FILE} -> ${TEST_LOOPBACK}"
 603                                 BASELOOPBACK="${TEST_LOOPBACK##*/}"
 604                                 DISKS="$DISKS $BASELOOPBACK"
 605                                 LOOPBACKS="$LOOPBACKS $TEST_LOOPBACK"
 606                         fi
 607                 done
 608                 DISKS=${DISKS# }
 609                 LOOPBACKS=${LOOPBACKS# }
 610         else
 611                 DISKS="$FILES"
 612         fi
 613 fi
 614
 615 #
 616 # It may be desirable to test with fewer disks than the default when running
 617 # the performance tests, but the functional tests require at least three.
 618 #
 619 NUM_DISKS=$(echo "${DISKS}" | awk '{print NF}')
 620 if [ "$TAGS" != "perf" ]; then
 621         [ "$NUM_DISKS" -lt 3 ] && fail "Not enough disks ($NUM_DISKS/3 minimum)"
 622 fi
 623
 624 #
 625 # Disable SELinux until the ZFS Test Suite has been updated accordingly.
 626 #
 627 if command -v setenforce >/dev/null; then
 628         sudo setenforce permissive >/dev/null 2>&1
 629 fi
 630
 631 #
 632 # Enable internal ZFS debug log and clear it.
 633 #
 634 if [ -e /sys/module/zfs/parameters/zfs_dbgmsg_enable ]; then
 635         sudo sh -c "echo 1 >/sys/module/zfs/parameters/zfs_dbgmsg_enable"
 636         sudo sh -c "echo 0 >/proc/spl/kstat/zfs/dbgmsg"
 637 fi
 638
 639 msg
 640 msg "--- Configuration ---"
 641 msg "Runfiles:        $RUNFILES"
 642 msg "STF_TOOLS:       $STF_TOOLS"
 643 msg "STF_SUITE:       $STF_SUITE"
 644 msg "STF_PATH:        $STF_PATH"
 645 msg "FILEDIR:         $FILEDIR"
 646 msg "FILES:           $FILES"
 647 msg "LOOPBACKS:       $LOOPBACKS"
 648 msg "DISKS:           $DISKS"
 649 msg "NUM_DISKS:       $NUM_DISKS"
 650 msg "FILESIZE:        $FILESIZE"
 651 msg "ITERATIONS:      $ITERATIONS"
 652 msg "TAGS:            $TAGS"
 653 msg "STACK_TRACER:    $STACK_TRACER"
 654 msg "Keep pool(s):    $KEEP"
 655 msg "Missing util(s): $STF_MISSING_BIN"
 656 msg ""
 657
 658 export STF_TOOLS
 659 export STF_SUITE
 660 export STF_PATH
 661 export DISKS
 662 export FILEDIR
 663 export KEEP
 664 export __ZFS_POOL_EXCLUDE
 665 export TESTFAIL_CALLBACKS
 666
 667 mktemp_file() {
 668         if [ "$UNAME" = "FreeBSD" ]; then
 669                 mktemp -u "${FILEDIR}/$1.XXXXXX"
 670         else
 671                 mktemp -ut "$1.XXXXXX" -p "$FILEDIR"
 672         fi
 673 }
 674 mkdir -p "$FILEDIR" || :
 675 RESULTS_FILE=$(mktemp_file zts-results)
 676 REPORT_FILE=$(mktemp_file zts-report)
 677
 678 #
 679 # Run all the tests as specified.
 680 #
 681 msg "${TEST_RUNNER}" \
 682     "${QUIET:+-q}" \
 683     "${KMEMLEAK:+-m}" \
 684     "${KMSG:+-K}" \
 685     "-c \"${RUNFILES}\"" \
 686     "-T \"${TAGS}\"" \
 687     "-i \"${STF_SUITE}\"" \
 688     "-I \"${ITERATIONS}\""
 689 { PATH=$STF_PATH \
 690     ${TEST_RUNNER} \
 691     ${QUIET:+-q} \
 692     ${KMEMLEAK:+-m} \
 693     ${KMSG:+-K} \
 694     -c "${RUNFILES}" \
 695     -T "${TAGS}" \
 696     -i "${STF_SUITE}" \
 697     -I "${ITERATIONS}" \
 698     2>&1; echo $? >"$REPORT_FILE"; } | tee "$RESULTS_FILE"
 699 read -r RUNRESULT <"$REPORT_FILE"
 700
 701 #
 702 # Analyze the results.
 703 #
 704 ${ZTS_REPORT} ${RERUN:+--no-maybes} "$RESULTS_FILE" >"$REPORT_FILE"
 705 RESULT=$?
 706
 707 if [ "$RESULT" -eq "2" ] && [ -n "$RERUN" ]; then
 708         MAYBES="$($ZTS_REPORT --list-maybes)"
 709         TEMP_RESULTS_FILE=$(mktemp_file zts-results-tmp)
 710         TEST_LIST=$(mktemp_file test-list)
 711         grep "^Test:.*\[FAIL\]" "$RESULTS_FILE" >"$TEMP_RESULTS_FILE"
 712         for test_name in $MAYBES; do
 713                 grep "$test_name " "$TEMP_RESULTS_FILE" >>"$TEST_LIST"
 714         done
 715         { PATH=$STF_PATH \
 716             ${TEST_RUNNER} \
 717                 ${QUIET:+-q} \
 718                 ${KMEMLEAK:+-m} \
 719             -c "${RUNFILES}" \
 720             -T "${TAGS}" \
 721             -i "${STF_SUITE}" \
 722             -I "${ITERATIONS}" \
 723             -l "${TEST_LIST}" \
 724             2>&1; echo $? >"$REPORT_FILE"; } | tee "$RESULTS_FILE"
 725         read -r RUNRESULT <"$REPORT_FILE"
 726         #
 727         # Analyze the results.
 728         #
 729         ${ZTS_REPORT} --no-maybes "$RESULTS_FILE" >"$REPORT_FILE"
 730         RESULT=$?
 731 fi
 732
 733
 734 cat "$REPORT_FILE"
 735
 736 RESULTS_DIR=$(awk '/^Log directory/ { print $3 }' "$RESULTS_FILE")
 737 if [ -d "$RESULTS_DIR" ]; then
 738         cat "$RESULTS_FILE" "$REPORT_FILE" >"$RESULTS_DIR/results"
 739 fi
 740
 741 rm -f "$RESULTS_FILE" "$REPORT_FILE" "$TEST_LIST" "$TEMP_RESULTS_FILE"
 742
 743 if [ -n "$SINGLETEST" ]; then
 744         rm -f "$RUNFILES" >/dev/null 2>&1
 745 fi
 746
 747 [ "$RUNRESULT" -gt 3 ] && exit "$RUNRESULT" || exit "$RESULT"