scripts/zfs-tests.sh

   1 #!/bin/sh
   2 # shellcheck disable=SC2154
   3 #
   4 # CDDL HEADER START
   5 #
   6 # The contents of this file are subject to the terms of the
   7 # Common Development and Distribution License, Version 1.0 only
   8 # (the "License").  You may not use this file except in compliance
   9 # with the License.
  10 #
  11 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  12 # or http://www.opensolaris.org/os/licensing.
  13 # See the License for the specific language governing permissions
  14 # and limitations under the License.
  15 #
  16 # When distributing Covered Code, include this CDDL HEADER in each
  17 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  18 # If applicable, add the following below this CDDL HEADER, with the
  19 # fields enclosed by brackets "[]" replaced with your own identifying
  20 # information: Portions Copyright [yyyy] [name of copyright owner]
  21 #
  22 # CDDL HEADER END
  23 #
  24
  25 #
  26 # Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
  27 #
  28
  29 BASE_DIR=$(dirname "$0")
  30 SCRIPT_COMMON=common.sh
  31 if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then
  32 . "${BASE_DIR}/${SCRIPT_COMMON}"
  33 else
  34 echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
  35 fi
  36
  37 PROG=zfs-tests.sh
  38 VERBOSE="no"
  39 QUIET=""
  40 CLEANUP="yes"
  41 CLEANUPALL="no"
  42 LOOPBACK="yes"
  43 STACK_TRACER="no"
  44 FILESIZE="4G"
  45 DEFAULT_RUNFILES="common.run,$(uname | tr '[:upper:]' '[:lower:]').run"
  46 RUNFILES=${RUNFILES:-$DEFAULT_RUNFILES}
  47 FILEDIR=${FILEDIR:-/var/tmp}
  48 DISKS=${DISKS:-""}
  49 SINGLETEST=""
  50 SINGLETESTUSER="root"
  51 TAGS=""
  52 ITERATIONS=1
  53 ZFS_DBGMSG="$STF_SUITE/callbacks/zfs_dbgmsg.ksh"
  54 ZFS_DMESG="$STF_SUITE/callbacks/zfs_dmesg.ksh"
  55 UNAME=$(uname -s)
  56 RERUN=""
  57 KMEMLEAK=""
  58
  59 # Override some defaults if on FreeBSD
  60 if [ "$UNAME" = "FreeBSD" ] ; then
  61         TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DMESG"}
  62         LOSETUP=/sbin/mdconfig
  63         DMSETUP=/sbin/gpart
  64 else
  65         ZFS_MMP="$STF_SUITE/callbacks/zfs_mmp.ksh"
  66         TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DBGMSG:$ZFS_DMESG:$ZFS_MMP"}
  67         LOSETUP=${LOSETUP:-/sbin/losetup}
  68         DMSETUP=${DMSETUP:-/sbin/dmsetup}
  69 fi
  70
  71 #
  72 # Log an informational message when additional verbosity is enabled.
  73 #
  74 msg() {
  75         if [ "$VERBOSE" = "yes" ]; then
  76                 echo "$@"
  77         fi
  78 }
  79
  80 #
  81 # Log a failure message, cleanup, and return an error.
  82 #
  83 fail() {
  84         echo "$PROG: $1" >&2
  85         cleanup
  86         exit 1
  87 }
  88
  89 cleanup_freebsd_loopback() {
  90         for TEST_LOOPBACK in ${LOOPBACKS}; do
  91                 if [ -c "/dev/${TEST_LOOPBACK}" ]; then
  92                         sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}" ||
  93                             echo "Failed to destroy: ${TEST_LOOPBACK}"
  94                 fi
  95         done
  96 }
  97
  98 cleanup_linux_loopback() {
  99         for TEST_LOOPBACK in ${LOOPBACKS}; do
 100                 LOOP_DEV="${TEST_LOOPBACK##*/}"
 101                 DM_DEV=$(sudo "${DMSETUP}" ls 2>/dev/null | \
 102                     grep "${LOOP_DEV}" | cut -f1)
 103
 104                 if [ -n "$DM_DEV" ]; then
 105                         sudo "${DMSETUP}" remove "${DM_DEV}" ||
 106                             echo "Failed to remove: ${DM_DEV}"
 107                 fi
 108
 109                 if [ -n "${TEST_LOOPBACK}" ]; then
 110                         sudo "${LOSETUP}" -d "${TEST_LOOPBACK}" ||
 111                             echo "Failed to remove: ${TEST_LOOPBACK}"
 112                 fi
 113         done
 114 }
 115
 116 #
 117 # Attempt to remove loopback devices and files which where created earlier
 118 # by this script to run the test framework.  The '-k' option may be passed
 119 # to the script to suppress cleanup for debugging purposes.
 120 #
 121 cleanup() {
 122         if [ "$CLEANUP" = "no" ]; then
 123                 return 0
 124         fi
 125
 126
 127         if [ "$LOOPBACK" = "yes" ]; then
 128                 if [ "$UNAME" = "FreeBSD" ] ; then
 129                         cleanup_freebsd_loopback
 130                 else
 131                         cleanup_linux_loopback
 132                 fi
 133         fi
 134
 135         for TEST_FILE in ${FILES}; do
 136                 rm -f "${TEST_FILE}" >/dev/null 2>&1
 137         done
 138
 139         if [ "$STF_PATH_REMOVE" = "yes" ] && [ -d "$STF_PATH" ]; then
 140                 rm -Rf "$STF_PATH"
 141         fi
 142 }
 143 trap cleanup EXIT
 144
 145 #
 146 # Attempt to remove all testpools (testpool.XXX), unopened dm devices,
 147 # loopback devices, and files.  This is a useful way to cleanup a previous
 148 # test run failure which has left the system in an unknown state.  This can
 149 # be dangerous and should only be used in a dedicated test environment.
 150 #
 151 cleanup_all() {
 152         TEST_POOLS=$(sudo env ASAN_OPTIONS=detect_leaks=false "$ZPOOL" list -H -o name | grep testpool)
 153         if [ "$UNAME" = "FreeBSD" ] ; then
 154                 TEST_LOOPBACKS=$(sudo "${LOSETUP}" -l)
 155         else
 156                 TEST_LOOPBACKS=$(sudo "${LOSETUP}" -a|grep file-vdev|cut -f1 -d:)
 157         fi
 158         TEST_FILES=$(ls /var/tmp/file-vdev* 2>/dev/null)
 159
 160         msg
 161         msg "--- Cleanup ---"
 162         msg "Removing pool(s):     $(echo "${TEST_POOLS}" | tr '\n' ' ')"
 163         for TEST_POOL in $TEST_POOLS; do
 164                 sudo env ASAN_OPTIONS=detect_leaks=false "$ZPOOL" destroy "${TEST_POOL}"
 165         done
 166
 167         if [ "$UNAME" != "FreeBSD" ] ; then
 168                 msg "Removing dm(s):       $(sudo "${DMSETUP}" ls |
 169                     grep loop | tr '\n' ' ')"
 170                 sudo "${DMSETUP}" remove_all
 171         fi
 172
 173         msg "Removing loopback(s): $(echo "${TEST_LOOPBACKS}" | tr '\n' ' ')"
 174         for TEST_LOOPBACK in $TEST_LOOPBACKS; do
 175                 if [ "$UNAME" = "FreeBSD" ] ; then
 176                         sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}"
 177                 else
 178                         sudo "${LOSETUP}" -d "${TEST_LOOPBACK}"
 179                 fi
 180         done
 181
 182         msg "Removing files(s):    $(echo "${TEST_FILES}" | tr '\n' ' ')"
 183         for TEST_FILE in $TEST_FILES; do
 184                 sudo rm -f "${TEST_FILE}"
 185         done
 186 }
 187
 188 #
 189 # Takes a name as the only arguments and looks for the following variations
 190 # on that name.  If one is found it is returned.
 191 #
 192 # $RUNFILE_DIR/<name>
 193 # $RUNFILE_DIR/<name>.run
 194 # <name>
 195 # <name>.run
 196 #
 197 find_runfile() {
 198         NAME=$1
 199         RESULT=""
 200
 201         if [ -f "$RUNFILE_DIR/$NAME" ]; then
 202                 RESULT="$RUNFILE_DIR/$NAME"
 203         elif [ -f "$RUNFILE_DIR/$NAME.run" ]; then
 204                 RESULT="$RUNFILE_DIR/$NAME.run"
 205         elif [ -f "$NAME" ]; then
 206                 RESULT="$NAME"
 207         elif [ -f "$NAME.run" ]; then
 208                 RESULT="$NAME.run"
 209         fi
 210
 211         echo "$RESULT"
 212 }
 213
 214 #
 215 # Symlink file if it appears under any of the given paths.
 216 #
 217 create_links() {
 218         dir_list="$1"
 219         file_list="$2"
 220
 221         [ -n "$STF_PATH" ] || fail "STF_PATH wasn't correctly set"
 222
 223         for i in $file_list; do
 224                 for j in $dir_list; do
 225                         [ ! -e "$STF_PATH/$i" ] || continue
 226
 227                         if [ ! -d "$j/$i" ] && [ -e "$j/$i" ]; then
 228                                 ln -sf "$j/$i" "$STF_PATH/$i" || \
 229                                     fail "Couldn't link $i"
 230                                 break
 231                         fi
 232                 done
 233
 234                 [ ! -e "$STF_PATH/$i" ] && \
 235                     STF_MISSING_BIN="$STF_MISSING_BIN $i"
 236         done
 237         STF_MISSING_BIN=${STF_MISSING_BIN# }
 238 }
 239
 240 #
 241 # Constrain the path to limit the available binaries to a known set.
 242 # When running in-tree a top level ./bin/ directory is created for
 243 # convenience, otherwise a temporary directory is used.
 244 #
 245 constrain_path() {
 246         . "$STF_SUITE/include/commands.cfg"
 247
 248         # On FreeBSD, base system zfs utils are in /sbin and OpenZFS utils
 249         # install to /usr/local/sbin. To avoid testing the wrong utils we
 250         # need /usr/local to come before / in the path search order.
 251         SYSTEM_DIRS="/usr/local/bin /usr/local/sbin"
 252         SYSTEM_DIRS="$SYSTEM_DIRS /usr/bin /usr/sbin /bin /sbin $LIBEXEC_DIR"
 253
 254         if [ "$INTREE" = "yes" ]; then
 255                 # Constrained path set to ./zfs/bin/
 256                 STF_PATH="$BIN_DIR"
 257                 STF_PATH_REMOVE="no"
 258                 STF_MISSING_BIN=""
 259                 if [ ! -d "$STF_PATH" ]; then
 260                         mkdir "$STF_PATH"
 261                         chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
 262                 fi
 263
 264                 # Special case links for standard zfs utilities
 265                 DIRS="$(find "$CMD_DIR" -type d \( ! -name .deps -a \
 266                     ! -name .libs \) -print | tr '\n' ' ')"
 267                 create_links "$DIRS" "$ZFS_FILES"
 268
 269                 # Special case links for zfs test suite utilities
 270                 DIRS="$(find "$STF_SUITE" -type d \( ! -name .deps -a \
 271                     ! -name .libs \) -print | tr '\n' ' ')"
 272                 create_links "$DIRS" "$ZFSTEST_FILES"
 273         else
 274                 # Constrained path set to /var/tmp/constrained_path.*
 275                 SYSTEMDIR=${SYSTEMDIR:-/var/tmp/constrained_path.XXXXXX}
 276                 STF_PATH=$(mktemp -d "$SYSTEMDIR")
 277                 STF_PATH_REMOVE="yes"
 278                 STF_MISSING_BIN=""
 279
 280                 chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
 281
 282                 # Special case links for standard zfs utilities
 283                 create_links "$SYSTEM_DIRS" "$ZFS_FILES"
 284
 285                 # Special case links for zfs test suite utilities
 286                 create_links "$STF_SUITE/bin" "$ZFSTEST_FILES"
 287         fi
 288
 289         # Standard system utilities
 290         SYSTEM_FILES="$SYSTEM_FILES_COMMON"
 291         if [ "$UNAME" = "FreeBSD" ] ; then
 292                 SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_FREEBSD"
 293         else
 294                 SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_LINUX"
 295         fi
 296         create_links "$SYSTEM_DIRS" "$SYSTEM_FILES"
 297
 298         # Exceptions
 299         ln -fs "$STF_PATH/awk" "$STF_PATH/nawk"
 300         if [ "$UNAME" = "Linux" ] ; then
 301                 ln -fs /sbin/fsck.ext4 "$STF_PATH/fsck"
 302                 ln -fs /sbin/mkfs.ext4 "$STF_PATH/newfs"
 303                 ln -fs "$STF_PATH/gzip" "$STF_PATH/compress"
 304                 ln -fs "$STF_PATH/gunzip" "$STF_PATH/uncompress"
 305                 ln -fs "$STF_PATH/exportfs" "$STF_PATH/share"
 306                 ln -fs "$STF_PATH/exportfs" "$STF_PATH/unshare"
 307         elif [ "$UNAME" = "FreeBSD" ] ; then
 308                 ln -fs /usr/local/bin/ksh93 "$STF_PATH/ksh"
 309         fi
 310 }
 311
 312 #
 313 # Output a useful usage message.
 314 #
 315 usage() {
 316 cat << EOF
 317 USAGE:
 318 $0 [-hvqxkfS] [-s SIZE] [-r RUNFILES] [-t PATH] [-u USER]
 319
 320 DESCRIPTION:
 321         ZFS Test Suite launch script
 322
 323 OPTIONS:
 324         -h          Show this message
 325         -v          Verbose zfs-tests.sh output
 326         -q          Quiet test-runner output
 327         -x          Remove all testpools, dm, lo, and files (unsafe)
 328         -k          Disable cleanup after test failure
 329         -f          Use files only, disables block device tests
 330         -S          Enable stack tracer (negative performance impact)
 331         -c          Only create and populate constrained path
 332         -R          Automatically rerun failing tests
 333         -m          Enable kmemleak reporting (Linux only)
 334         -n NFSFILE  Use the nfsfile to determine the NFS configuration
 335         -I NUM      Number of iterations
 336         -d DIR      Use DIR for files and loopback devices
 337         -s SIZE     Use vdevs of SIZE (default: 4G)
 338         -r RUNFILES Run tests in RUNFILES (default: ${DEFAULT_RUNFILES})
 339         -t PATH     Run single test at PATH relative to test suite
 340         -T TAGS     Comma separated list of tags (default: 'functional')
 341         -u USER     Run single test as USER (default: root)
 342
 343 EXAMPLES:
 344 # Run the default (linux) suite of tests and output the configuration used.
 345 $0 -v
 346
 347 # Run a smaller suite of tests designed to run more quickly.
 348 $0 -r linux-fast
 349
 350 # Run a single test
 351 $0 -t tests/functional/cli_root/zfs_bookmark/zfs_bookmark_cliargs.ksh
 352
 353 # Cleanup a previous run of the test suite prior to testing, run the
 354 # default (linux) suite of tests and perform no cleanup on exit.
 355 $0 -x
 356
 357 EOF
 358 }
 359
 360 while getopts 'hvqxkfScRmn:d:s:r:?t:T:u:I:' OPTION; do
 361         case $OPTION in
 362         h)
 363                 usage
 364                 exit 1
 365                 ;;
 366         v)
 367                 VERBOSE="yes"
 368                 ;;
 369         q)
 370                 QUIET="yes"
 371                 ;;
 372         x)
 373                 CLEANUPALL="yes"
 374                 ;;
 375         k)
 376                 CLEANUP="no"
 377                 ;;
 378         f)
 379                 LOOPBACK="no"
 380                 ;;
 381         S)
 382                 STACK_TRACER="yes"
 383                 ;;
 384         c)
 385                 constrain_path
 386                 exit
 387                 ;;
 388         R)
 389                 RERUN="yes"
 390                 ;;
 391         m)
 392                 KMEMLEAK="yes"
 393                 ;;
 394         n)
 395                 nfsfile=$OPTARG
 396                 [ -f "$nfsfile" ] || fail "Cannot read file: $nfsfile"
 397                 export NFS=1
 398                 . "$nfsfile"
 399                 ;;
 400         d)
 401                 FILEDIR="$OPTARG"
 402                 ;;
 403         I)
 404                 ITERATIONS="$OPTARG"
 405                 if [ "$ITERATIONS" -le 0 ]; then
 406                         fail "Iterations must be greater than 0."
 407                 fi
 408                 ;;
 409         s)
 410                 FILESIZE="$OPTARG"
 411                 ;;
 412         r)
 413                 RUNFILES="$OPTARG"
 414                 ;;
 415         t)
 416                 if [ -n "$SINGLETEST" ]; then
 417                         fail "-t can only be provided once."
 418                 fi
 419                 SINGLETEST="$OPTARG"
 420                 ;;
 421         T)
 422                 TAGS="$OPTARG"
 423                 ;;
 424         u)
 425                 SINGLETESTUSER="$OPTARG"
 426                 ;;
 427         ?)
 428                 usage
 429                 exit
 430                 ;;
 431         *)
 432                 ;;
 433         esac
 434 done
 435
 436 shift $((OPTIND-1))
 437
 438 FILES=${FILES:-"$FILEDIR/file-vdev0 $FILEDIR/file-vdev1 $FILEDIR/file-vdev2"}
 439 LOOPBACKS=${LOOPBACKS:-""}
 440
 441 if [ -n "$SINGLETEST" ]; then
 442         if [ -n "$TAGS" ]; then
 443                 fail "-t and -T are mutually exclusive."
 444         fi
 445         RUNFILE_DIR="/var/tmp"
 446         RUNFILES="zfs-tests.$$.run"
 447         SINGLEQUIET="False"
 448
 449         if [ -n "$QUIET" ]; then
 450                 SINGLEQUIET="True"
 451         fi
 452
 453         cat >"${RUNFILE_DIR}/${RUNFILES}" << EOF
 454 [DEFAULT]
 455 pre =
 456 quiet = $SINGLEQUIET
 457 pre_user = root
 458 user = $SINGLETESTUSER
 459 timeout = 600
 460 post_user = root
 461 post =
 462 outputdir = /var/tmp/test_results
 463 EOF
 464         SINGLETESTDIR=$(dirname "$SINGLETEST")
 465         SINGLETESTFILE=$(basename "$SINGLETEST")
 466         SETUPSCRIPT=
 467         CLEANUPSCRIPT=
 468
 469         if [ -f "$STF_SUITE/$SINGLETESTDIR/setup.ksh" ]; then
 470                 SETUPSCRIPT="setup"
 471         fi
 472
 473         if [ -f "$STF_SUITE/$SINGLETESTDIR/cleanup.ksh" ]; then
 474                 CLEANUPSCRIPT="cleanup"
 475         fi
 476
 477         cat >>"${RUNFILE_DIR}/${RUNFILES}" << EOF
 478
 479 [$SINGLETESTDIR]
 480 tests = ['$SINGLETESTFILE']
 481 pre = $SETUPSCRIPT
 482 post = $CLEANUPSCRIPT
 483 tags = ['functional']
 484 EOF
 485 fi
 486
 487 #
 488 # Use default tag if none was specified
 489 #
 490 TAGS=${TAGS:='functional'}
 491
 492 #
 493 # Attempt to locate the runfiles describing the test workload.
 494 #
 495 R=""
 496 IFS=,
 497 for RUNFILE in $RUNFILES; do
 498         if [ -n "$RUNFILE" ]; then
 499                 SAVED_RUNFILE="$RUNFILE"
 500                 RUNFILE=$(find_runfile "$RUNFILE")
 501                 [ -z "$RUNFILE" ] && fail "Cannot find runfile: $SAVED_RUNFILE"
 502                 R="$R,$RUNFILE"
 503         fi
 504
 505         if [ ! -r "$RUNFILE" ]; then
 506                 fail "Cannot read runfile: $RUNFILE"
 507         fi
 508 done
 509 unset IFS
 510 RUNFILES=${R#,}
 511
 512 #
 513 # This script should not be run as root.  Instead the test user, which may
 514 # be a normal user account, needs to be configured such that it can
 515 # run commands via sudo passwordlessly.
 516 #
 517 if [ "$(id -u)" = "0" ]; then
 518         fail "This script must not be run as root."
 519 fi
 520
 521 if [ "$(sudo whoami)" != "root" ]; then
 522         fail "Passwordless sudo access required."
 523 fi
 524
 525 #
 526 # Constrain the available binaries to a known set.
 527 #
 528 constrain_path
 529
 530 #
 531 # Check if ksh exists
 532 #
 533 if [ "$UNAME" = "FreeBSD" ]; then
 534         sudo ln -fs /usr/local/bin/ksh93 /bin/ksh
 535 fi
 536 [ -e "$STF_PATH/ksh" ] || fail "This test suite requires ksh."
 537 [ -e "$STF_SUITE/include/default.cfg" ] || fail \
 538     "Missing $STF_SUITE/include/default.cfg file."
 539
 540 #
 541 # Verify the ZFS module stack is loaded.
 542 #
 543 if [ "$STACK_TRACER" = "yes" ]; then
 544         sudo "${ZFS_SH}" -S >/dev/null 2>&1
 545 else
 546         sudo "${ZFS_SH}" >/dev/null 2>&1
 547 fi
 548
 549 #
 550 # Attempt to cleanup all previous state for a new test run.
 551 #
 552 if [ "$CLEANUPALL" = "yes" ]; then
 553         cleanup_all
 554 fi
 555
 556 #
 557 # By default preserve any existing pools
 558 # NOTE: Since 'zpool list' outputs a newline-delimited list convert $KEEP from
 559 # space-delimited to newline-delimited.
 560 #
 561 if [ -z "${KEEP}" ]; then
 562         KEEP="$(sudo env ASAN_OPTIONS=detect_leaks=false "$ZPOOL" list -H -o name)"
 563         if [ -z "${KEEP}" ]; then
 564                 KEEP="rpool"
 565         fi
 566 else
 567         KEEP="$(echo "$KEEP" | tr '[:blank:]' '\n')"
 568 fi
 569
 570 #
 571 # NOTE: The following environment variables are undocumented
 572 # and should be used for testing purposes only:
 573 #
 574 # __ZFS_POOL_EXCLUDE - don't iterate over the pools it lists
 575 # __ZFS_POOL_RESTRICT - iterate only over the pools it lists
 576 #
 577 # See libzfs/libzfs_config.c for more information.
 578 #
 579 if [ "$UNAME" = "FreeBSD" ] ; then
 580         __ZFS_POOL_EXCLUDE="$(echo "$KEEP" | tr -s '\n' ' ')"
 581 else
 582         __ZFS_POOL_EXCLUDE="$(echo "$KEEP" | sed ':a;N;s/\n/ /g;ba')"
 583 fi
 584
 585 . "$STF_SUITE/include/default.cfg"
 586
 587 #
 588 # No DISKS have been provided so a basic file or loopback based devices
 589 # must be created for the test suite to use.
 590 #
 591 if [ -z "${DISKS}" ]; then
 592         #
 593         # If this is a performance run, prevent accidental use of
 594         # loopback devices.
 595         #
 596         [ "$TAGS" = "perf" ] && fail "Running perf tests without disks."
 597
 598         #
 599         # Create sparse files for the test suite.  These may be used
 600         # directory or have loopback devices layered on them.
 601         #
 602         for TEST_FILE in ${FILES}; do
 603                 [ -f "$TEST_FILE" ] && fail "Failed file exists: ${TEST_FILE}"
 604                 truncate -s "${FILESIZE}" "${TEST_FILE}" ||
 605                     fail "Failed creating: ${TEST_FILE} ($?)"
 606         done
 607
 608         #
 609         # If requested setup loopback devices backed by the sparse files.
 610         #
 611         if [ "$LOOPBACK" = "yes" ]; then
 612                 test -x "$LOSETUP" || fail "$LOSETUP utility must be installed"
 613
 614                 for TEST_FILE in ${FILES}; do
 615                         if [ "$UNAME" = "FreeBSD" ] ; then
 616                                 MDDEVICE=$(sudo "${LOSETUP}" -a -t vnode -f "${TEST_FILE}")
 617                                 if [ -z "$MDDEVICE" ] ; then
 618                                         fail "Failed: ${TEST_FILE} -> loopback"
 619                                 fi
 620                                 DISKS="$DISKS $MDDEVICE"
 621                                 LOOPBACKS="$LOOPBACKS $MDDEVICE"
 622                         else
 623                                 TEST_LOOPBACK=$(sudo "${LOSETUP}" -f)
 624                                 sudo "${LOSETUP}" "${TEST_LOOPBACK}" "${TEST_FILE}" ||
 625                                     fail "Failed: ${TEST_FILE} -> ${TEST_LOOPBACK}"
 626                                 BASELOOPBACK="${TEST_LOOPBACK##*/}"
 627                                 DISKS="$DISKS $BASELOOPBACK"
 628                                 LOOPBACKS="$LOOPBACKS $TEST_LOOPBACK"
 629                         fi
 630                 done
 631                 DISKS=${DISKS# }
 632                 LOOPBACKS=${LOOPBACKS# }
 633         else
 634                 DISKS="$FILES"
 635         fi
 636 fi
 637
 638 #
 639 # It may be desirable to test with fewer disks than the default when running
 640 # the performance tests, but the functional tests require at least three.
 641 #
 642 NUM_DISKS=$(echo "${DISKS}" | awk '{print NF}')
 643 if [ "$TAGS" != "perf" ]; then
 644         [ "$NUM_DISKS" -lt 3 ] && fail "Not enough disks ($NUM_DISKS/3 minimum)"
 645 fi
 646
 647 #
 648 # Disable SELinux until the ZFS Test Suite has been updated accordingly.
 649 #
 650 if [ -x "$STF_PATH/setenforce" ]; then
 651         sudo setenforce permissive >/dev/null 2>&1
 652 fi
 653
 654 #
 655 # Enable internal ZFS debug log and clear it.
 656 #
 657 if [ -e /sys/module/zfs/parameters/zfs_dbgmsg_enable ]; then
 658         sudo /bin/sh -c "echo 1 >/sys/module/zfs/parameters/zfs_dbgmsg_enable"
 659         sudo /bin/sh -c "echo 0 >/proc/spl/kstat/zfs/dbgmsg"
 660 fi
 661
 662 msg
 663 msg "--- Configuration ---"
 664 msg "Runfiles:        $RUNFILES"
 665 msg "STF_TOOLS:       $STF_TOOLS"
 666 msg "STF_SUITE:       $STF_SUITE"
 667 msg "STF_PATH:        $STF_PATH"
 668 msg "FILEDIR:         $FILEDIR"
 669 msg "FILES:           $FILES"
 670 msg "LOOPBACKS:       $LOOPBACKS"
 671 msg "DISKS:           $DISKS"
 672 msg "NUM_DISKS:       $NUM_DISKS"
 673 msg "FILESIZE:        $FILESIZE"
 674 msg "ITERATIONS:      $ITERATIONS"
 675 msg "TAGS:            $TAGS"
 676 msg "STACK_TRACER:    $STACK_TRACER"
 677 msg "Keep pool(s):    $KEEP"
 678 msg "Missing util(s): $STF_MISSING_BIN"
 679 msg ""
 680
 681 export STF_TOOLS
 682 export STF_SUITE
 683 export STF_PATH
 684 export DISKS
 685 export FILEDIR
 686 export KEEP
 687 export __ZFS_POOL_EXCLUDE
 688 export TESTFAIL_CALLBACKS
 689 export PATH=$STF_PATH
 690
 691 mktemp_file() {
 692         if [ "$UNAME" = "FreeBSD" ]; then
 693                 mktemp -u "${FILEDIR}/$1.XXXXXX"
 694         else
 695                 mktemp -ut "$1.XXXXXX" -p "$FILEDIR"
 696         fi
 697 }
 698 mkdir -p "$FILEDIR" || :
 699 RESULTS_FILE=$(mktemp_file zts-results)
 700 REPORT_FILE=$(mktemp_file zts-report)
 701
 702 #
 703 # Run all the tests as specified.
 704 #
 705 msg "${TEST_RUNNER}" \
 706     "${QUIET:+-q}" \
 707     "${KMEMLEAK:+-m}" \
 708     "-c \"${RUNFILES}\"" \
 709     "-T \"${TAGS}\"" \
 710     "-i \"${STF_SUITE}\"" \
 711     "-I \"${ITERATIONS}\""
 712 { ${TEST_RUNNER} \
 713     ${QUIET:+-q} \
 714     ${KMEMLEAK:+-m} \
 715     -c "${RUNFILES}" \
 716     -T "${TAGS}" \
 717     -i "${STF_SUITE}" \
 718     -I "${ITERATIONS}" \
 719     2>&1; echo $? >"$REPORT_FILE"; } | tee "$RESULTS_FILE"
 720 read -r RUNRESULT <"$REPORT_FILE"
 721
 722 #
 723 # Analyze the results.
 724 #
 725 ${ZTS_REPORT} ${RERUN:+--no-maybes} "$RESULTS_FILE" >"$REPORT_FILE"
 726 RESULT=$?
 727
 728 if [ "$RESULT" -eq "2" ] && [ -n "$RERUN" ]; then
 729         MAYBES="$($ZTS_REPORT --list-maybes)"
 730         TEMP_RESULTS_FILE=$(mktemp_file zts-results-tmp)
 731         TEST_LIST=$(mktemp_file test-list)
 732         grep "^Test:.*\[FAIL\]" "$RESULTS_FILE" >"$TEMP_RESULTS_FILE"
 733         for test_name in $MAYBES; do
 734                 grep "$test_name " "$TEMP_RESULTS_FILE" >>"$TEST_LIST"
 735         done
 736         { ${TEST_RUNNER} \
 737             ${QUIET:+-q} \
 738             ${KMEMLEAK:+-m} \
 739             -c "${RUNFILES}" \
 740             -T "${TAGS}" \
 741             -i "${STF_SUITE}" \
 742             -I "${ITERATIONS}" \
 743             -l "${TEST_LIST}" \
 744             2>&1; echo $? >"$REPORT_FILE"; } | tee "$RESULTS_FILE"
 745         read -r RUNRESULT <"$REPORT_FILE"
 746         #
 747         # Analyze the results.
 748         #
 749         ${ZTS_REPORT} --no-maybes "$RESULTS_FILE" >"$REPORT_FILE"
 750         RESULT=$?
 751 fi
 752
 753
 754 cat "$REPORT_FILE"
 755
 756 RESULTS_DIR=$(awk '/^Log directory/ { print $3 }' "$RESULTS_FILE")
 757 if [ -d "$RESULTS_DIR" ]; then
 758         cat "$RESULTS_FILE" "$REPORT_FILE" >"$RESULTS_DIR/results"
 759 fi
 760
 761 rm -f "$RESULTS_FILE" "$REPORT_FILE"
 762
 763 if [ -n "$SINGLETEST" ]; then
 764         rm -f "$RUNFILES" >/dev/null 2>&1
 765 fi
 766
 767 [ "$RUNRESULT" -gt 3 ] && exit "$RUNRESULT" || exit "$RESULT"