scripts/zfs-tests.sh

   1 #!/usr/bin/env bash
   2 # shellcheck disable=SC2154
   3 # shellcheck disable=SC2292
   4 #
   5 # CDDL HEADER START
   6 #
   7 # The contents of this file are subject to the terms of the
   8 # Common Development and Distribution License, Version 1.0 only
   9 # (the "License").  You may not use this file except in compliance
  10 # with the License.
  11 #
  12 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  13 # or https://opensource.org/licenses/CDDL-1.0.
  14 # See the License for the specific language governing permissions
  15 # and limitations under the License.
  16 #
  17 # When distributing Covered Code, include this CDDL HEADER in each
  18 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  19 # If applicable, add the following below this CDDL HEADER, with the
  20 # fields enclosed by brackets "[]" replaced with your own identifying
  21 # information: Portions Copyright [yyyy] [name of copyright owner]
  22 #
  23 # CDDL HEADER END
  24 #
  25
  26 #
  27 # Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
  28 #
  29
  30 SCRIPT_COMMON=${SCRIPT_COMMON:-${0%/*}/common.sh}
  31 . "${SCRIPT_COMMON}" || exit
  32
  33 PROG=zfs-tests.sh
  34 VERBOSE="no"
  35 QUIET=""
  36 DEBUG=""
  37 CLEANUP="yes"
  38 CLEANUPALL="no"
  39 KMSG=""
  40 LOOPBACK="yes"
  41 STACK_TRACER="no"
  42 FILESIZE="4G"
  43 DEFAULT_RUNFILES="common.run,$(uname | tr '[:upper:]' '[:lower:]').run"
  44 RUNFILES=${RUNFILES:-$DEFAULT_RUNFILES}
  45 FILEDIR=${FILEDIR:-/var/tmp}
  46 DISKS=${DISKS:-""}
  47 SINGLETEST=""
  48 SINGLETESTUSER="root"
  49 TAGS=""
  50 ITERATIONS=1
  51 ZFS_DBGMSG="$STF_SUITE/callbacks/zfs_dbgmsg.ksh"
  52 ZFS_DMESG="$STF_SUITE/callbacks/zfs_dmesg.ksh"
  53 UNAME=$(uname)
  54 RERUN=""
  55 KMEMLEAK=""
  56
  57 # Override some defaults if on FreeBSD
  58 if [ "$UNAME" = "FreeBSD" ] ; then
  59         TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DMESG"}
  60         LOSETUP=/sbin/mdconfig
  61         DMSETUP=/sbin/gpart
  62 else
  63         ZFS_MMP="$STF_SUITE/callbacks/zfs_mmp.ksh"
  64         TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DBGMSG:$ZFS_DMESG:$ZFS_MMP"}
  65         LOSETUP=${LOSETUP:-/sbin/losetup}
  66         DMSETUP=${DMSETUP:-/sbin/dmsetup}
  67 fi
  68
  69 #
  70 # Log an informational message when additional verbosity is enabled.
  71 #
  72 msg() {
  73         if [ "$VERBOSE" = "yes" ]; then
  74                 echo "$@"
  75         fi
  76 }
  77
  78 #
  79 # Log a failure message, cleanup, and return an error.
  80 #
  81 fail() {
  82         echo "$PROG: $1" >&2
  83         cleanup
  84         exit 1
  85 }
  86
  87 cleanup_freebsd_loopback() {
  88         for TEST_LOOPBACK in ${LOOPBACKS}; do
  89                 if [ -c "/dev/${TEST_LOOPBACK}" ]; then
  90                         sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}" ||
  91                             echo "Failed to destroy: ${TEST_LOOPBACK}"
  92                 fi
  93         done
  94 }
  95
  96 cleanup_linux_loopback() {
  97         for TEST_LOOPBACK in ${LOOPBACKS}; do
  98                 LOOP_DEV="${TEST_LOOPBACK##*/}"
  99                 DM_DEV=$(sudo "${DMSETUP}" ls 2>/dev/null | \
 100                     awk -v l="${LOOP_DEV}" '$0 ~ l {print $1}')
 101
 102                 if [ -n "$DM_DEV" ]; then
 103                         sudo "${DMSETUP}" remove "${DM_DEV}" ||
 104                             echo "Failed to remove: ${DM_DEV}"
 105                 fi
 106
 107                 if [ -n "${TEST_LOOPBACK}" ]; then
 108                         sudo "${LOSETUP}" -d "${TEST_LOOPBACK}" ||
 109                             echo "Failed to remove: ${TEST_LOOPBACK}"
 110                 fi
 111         done
 112 }
 113
 114 #
 115 # Attempt to remove loopback devices and files which where created earlier
 116 # by this script to run the test framework.  The '-k' option may be passed
 117 # to the script to suppress cleanup for debugging purposes.
 118 #
 119 cleanup() {
 120         if [ "$CLEANUP" = "no" ]; then
 121                 return 0
 122         fi
 123
 124
 125         if [ "$LOOPBACK" = "yes" ]; then
 126                 if [ "$UNAME" = "FreeBSD" ] ; then
 127                         cleanup_freebsd_loopback
 128                 else
 129                         cleanup_linux_loopback
 130                 fi
 131         fi
 132
 133         # shellcheck disable=SC2086
 134         rm -f ${FILES} >/dev/null 2>&1
 135
 136         if [ "$STF_PATH_REMOVE" = "yes" ] && [ -d "$STF_PATH" ]; then
 137                 rm -Rf "$STF_PATH"
 138         fi
 139 }
 140 trap cleanup EXIT
 141
 142 #
 143 # Attempt to remove all testpools (testpool.XXX), unopened dm devices,
 144 # loopback devices, and files.  This is a useful way to cleanup a previous
 145 # test run failure which has left the system in an unknown state.  This can
 146 # be dangerous and should only be used in a dedicated test environment.
 147 #
 148 cleanup_all() {
 149         TEST_POOLS=$(ASAN_OPTIONS=detect_leaks=false "$ZPOOL" list -Ho name | grep testpool)
 150         if [ "$UNAME" = "FreeBSD" ] ; then
 151                 TEST_LOOPBACKS=$(sudo "${LOSETUP}" -l)
 152         else
 153                 TEST_LOOPBACKS=$("${LOSETUP}" -a | awk -F: '/file-vdev/ {print $1}')
 154         fi
 155         TEST_FILES=$(ls "${FILEDIR}"/file-vdev* /var/tmp/file-vdev* 2>/dev/null)
 156
 157         msg
 158         msg "--- Cleanup ---"
 159         # shellcheck disable=2116,2086
 160         msg "Removing pool(s):     $(echo ${TEST_POOLS})"
 161         for TEST_POOL in $TEST_POOLS; do
 162                 sudo env ASAN_OPTIONS=detect_leaks=false "$ZPOOL" destroy "${TEST_POOL}"
 163         done
 164
 165         if [ "$UNAME" != "FreeBSD" ] ; then
 166                 msg "Removing all dm(s):   $(sudo "${DMSETUP}" ls |
 167                     grep loop | tr '\n' ' ')"
 168                 sudo "${DMSETUP}" remove_all
 169         fi
 170
 171         # shellcheck disable=2116,2086
 172         msg "Removing loopback(s): $(echo ${TEST_LOOPBACKS})"
 173         for TEST_LOOPBACK in $TEST_LOOPBACKS; do
 174                 if [ "$UNAME" = "FreeBSD" ] ; then
 175                         sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}"
 176                 else
 177                         sudo "${LOSETUP}" -d "${TEST_LOOPBACK}"
 178                 fi
 179         done
 180
 181         # shellcheck disable=2116,2086
 182         msg "Removing files(s):    $(echo ${TEST_FILES})"
 183         # shellcheck disable=2086
 184         sudo rm -f ${TEST_FILES}
 185 }
 186
 187 #
 188 # Takes a name as the only arguments and looks for the following variations
 189 # on that name.  If one is found it is returned.
 190 #
 191 # $RUNFILE_DIR/<name>
 192 # $RUNFILE_DIR/<name>.run
 193 # <name>
 194 # <name>.run
 195 #
 196 find_runfile() {
 197         NAME=$1
 198
 199         if [ -f "$RUNFILE_DIR/$NAME" ]; then
 200                 echo "$RUNFILE_DIR/$NAME"
 201         elif [ -f "$RUNFILE_DIR/$NAME.run" ]; then
 202                 echo "$RUNFILE_DIR/$NAME.run"
 203         elif [ -f "$NAME" ]; then
 204                 echo "$NAME"
 205         elif [ -f "$NAME.run" ]; then
 206                 echo "$NAME.run"
 207         else
 208                 return 1
 209         fi
 210 }
 211
 212 # Given a TAGS with a format like "1/3" or "2/3" then divide up the test list
 213 # into portions and print that portion.  So "1/3" for "the first third of the
 214 # test tags".
 215 #
 216 #
 217 split_tags() {
 218         # Get numerator and denominator
 219         NUM=$(echo "$TAGS" | cut -d/ -f1)
 220         DEN=$(echo "$TAGS" | cut -d/ -f2)
 221         # At the point this is called, RUNFILES will contain a comma separated
 222         # list of full paths to the runfiles, like:
 223         #
 224         # "/home/hutter/qemu/tests/runfiles/common.run,/home/hutter/qemu/tests/runfiles/linux.run"
 225         #
 226         # So to get tags for our selected tests we do:
 227         #
 228         # 1. Remove unneeded chars: [],\
 229         # 2. Print out the last field of each tag line.  This will be the tag
 230         #    for the test (like 'zpool_add').
 231         # 3. Remove duplicates between the runfiles.  If the same tag is defined
 232         #    in multiple runfiles, then when you do '-T <tag>' ZTS is smart
 233         #    enough to know to run the tag in each runfile.  So '-T zpool_add'
 234         #    will run the zpool_add from common.run and linux.run.
 235         # 4. Ignore the 'functional' tag since we only want individual tests
 236         # 5. Print out the tests in our faction of all tests.  This uses modulus
 237         #    so "1/3" will run tests 1,3,6,9 etc.  That way the tests are
 238         #    interleaved so, say, "3/4" isn't running all the zpool_* tests that
 239         #    appear alphabetically at the end.
 240         # 6. Remove trailing comma from list
 241         #
 242         # TAGS will then look like:
 243         #
 244         # "append,atime,bootfs,cachefile,checksum,cp_files,deadman,dos_attributes, ..."
 245
 246         # Change the comma to a space for easy processing
 247         _RUNFILES=${RUNFILES//","/" "}
 248         # shellcheck disable=SC2002,SC2086
 249         cat $_RUNFILES | tr -d "[],\'" | awk '/tags = /{print $NF}' | sort | \
 250                 uniq | grep -v functional | \
 251                 awk -v num="$NUM" -v den="$DEN" '{ if(NR % den == (num - 1)) {printf "%s,",$0}}' | \
 252                 sed -E 's/,$//'
 253 }
 254
 255 #
 256 # Symlink file if it appears under any of the given paths.
 257 #
 258 create_links() {
 259         dir_list="$1"
 260         file_list="$2"
 261
 262         [ -n "$STF_PATH" ] || fail "STF_PATH wasn't correctly set"
 263
 264         for i in $file_list; do
 265                 for j in $dir_list; do
 266                         [ ! -e "$STF_PATH/$i" ] || continue
 267
 268                         if [ ! -d "$j/$i" ] && [ -e "$j/$i" ]; then
 269                                 ln -sf "$j/$i" "$STF_PATH/$i" || \
 270                                     fail "Couldn't link $i"
 271                                 break
 272                         fi
 273                 done
 274
 275                 [ ! -e "$STF_PATH/$i" ] && \
 276                     STF_MISSING_BIN="$STF_MISSING_BIN $i"
 277         done
 278         STF_MISSING_BIN=${STF_MISSING_BIN# }
 279 }
 280
 281 #
 282 # Constrain the path to limit the available binaries to a known set.
 283 # When running in-tree a top level ./bin/ directory is created for
 284 # convenience, otherwise a temporary directory is used.
 285 #
 286 constrain_path() {
 287         . "$STF_SUITE/include/commands.cfg"
 288
 289         # On FreeBSD, base system zfs utils are in /sbin and OpenZFS utils
 290         # install to /usr/local/sbin. To avoid testing the wrong utils we
 291         # need /usr/local to come before / in the path search order.
 292         SYSTEM_DIRS="/usr/local/bin /usr/local/sbin"
 293         SYSTEM_DIRS="$SYSTEM_DIRS /usr/bin /usr/sbin /bin /sbin $LIBEXEC_DIR"
 294
 295         if [ "$INTREE" = "yes" ]; then
 296                 # Constrained path set to $(top_builddir)/tests/zfs-tests/bin
 297                 STF_PATH="$BIN_DIR"
 298                 STF_PATH_REMOVE="no"
 299                 STF_MISSING_BIN=""
 300                 if [ ! -d "$STF_PATH" ]; then
 301                         mkdir "$STF_PATH"
 302                         chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
 303                 fi
 304
 305                 # Special case links for standard zfs utilities
 306                 create_links "$CMD_DIR" "$ZFS_FILES"
 307
 308                 # Special case links for zfs test suite utilities
 309                 create_links "$CMD_DIR/tests/zfs-tests/cmd" "$ZFSTEST_FILES"
 310         else
 311                 # Constrained path set to /var/tmp/constrained_path.*
 312                 SYSTEMDIR=${SYSTEMDIR:-/var/tmp/constrained_path.XXXXXX}
 313                 STF_PATH=$(mktemp -d "$SYSTEMDIR")
 314                 STF_PATH_REMOVE="yes"
 315                 STF_MISSING_BIN=""
 316
 317                 chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
 318
 319                 # Special case links for standard zfs utilities
 320                 create_links "$SYSTEM_DIRS" "$ZFS_FILES"
 321
 322                 # Special case links for zfs test suite utilities
 323                 create_links "$STF_SUITE/bin" "$ZFSTEST_FILES"
 324         fi
 325
 326         # Standard system utilities
 327         SYSTEM_FILES="$SYSTEM_FILES_COMMON"
 328         if [ "$UNAME" = "FreeBSD" ] ; then
 329                 SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_FREEBSD"
 330         else
 331                 SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_LINUX"
 332         fi
 333         create_links "$SYSTEM_DIRS" "$SYSTEM_FILES"
 334
 335         # Exceptions
 336         if [ "$UNAME" = "Linux" ] ; then
 337                 ln -fs /sbin/fsck.ext4 "$STF_PATH/fsck"
 338                 ln -fs /sbin/mkfs.ext4 "$STF_PATH/newfs"
 339                 ln -fs "$STF_PATH/gzip" "$STF_PATH/compress"
 340                 ln -fs "$STF_PATH/gunzip" "$STF_PATH/uncompress"
 341         elif [ "$UNAME" = "FreeBSD" ] ; then
 342                 ln -fs /usr/local/bin/ksh93 "$STF_PATH/ksh"
 343         fi
 344 }
 345
 346 #
 347 # Output a useful usage message.
 348 #
 349 usage() {
 350 cat << EOF
 351 USAGE:
 352 $0 [-hvqxkfS] [-s SIZE] [-r RUNFILES] [-t PATH] [-u USER]
 353
 354 DESCRIPTION:
 355         ZFS Test Suite launch script
 356
 357 OPTIONS:
 358         -h          Show this message
 359         -v          Verbose zfs-tests.sh output
 360         -q          Quiet test-runner output
 361         -D          Debug; show all test output immediately (noisy)
 362         -x          Remove all testpools, dm, lo, and files (unsafe)
 363         -k          Disable cleanup after test failure
 364         -K          Log test names to /dev/kmsg
 365         -f          Use files only, disables block device tests
 366         -S          Enable stack tracer (negative performance impact)
 367         -c          Only create and populate constrained path
 368         -R          Automatically rerun failing tests
 369         -m          Enable kmemleak reporting (Linux only)
 370         -n NFSFILE  Use the nfsfile to determine the NFS configuration
 371         -I NUM      Number of iterations
 372         -d DIR      Use world-writable DIR for files and loopback devices
 373         -s SIZE     Use vdevs of SIZE (default: 4G)
 374         -r RUNFILES Run tests in RUNFILES (default: ${DEFAULT_RUNFILES})
 375         -t PATH|NAME  Run single test at PATH relative to test suite,
 376                         or search for test by NAME
 377         -T TAGS     Comma separated list of tags (default: 'functional')
 378                     Alternately, specify a fraction like "1/3" or "2/3" to
 379                      run the first third of tests or 2nd third of the tests.  This
 380                      is useful for splitting up the test amongst different
 381                      runners.
 382         -u USER     Run single test as USER (default: root)
 383
 384 EXAMPLES:
 385 # Run the default ${DEFAULT_RUNFILES//\.run/} suite of tests and output the configuration used.
 386 $0 -v
 387
 388 # Run a smaller suite of tests designed to run more quickly.
 389 $0 -r linux-fast
 390
 391 # Run a single test
 392 $0 -t tests/functional/cli_root/zfs_bookmark/zfs_bookmark_cliargs.ksh
 393
 394 # Run a single test by name
 395 $0 -t zfs_bookmark_cliargs
 396
 397 # Cleanup a previous run of the test suite prior to testing, run the
 398 # default ${DEFAULT_RUNFILES//\.run//} suite of tests and perform no cleanup on exit.
 399 $0 -x
 400
 401 EOF
 402 }
 403
 404 while getopts 'hvqxkKfScRmn:d:Ds:r:?t:T:u:I:' OPTION; do
 405         case $OPTION in
 406         h)
 407                 usage
 408                 exit 1
 409                 ;;
 410         v)
 411                 VERBOSE="yes"
 412                 ;;
 413         q)
 414                 QUIET="yes"
 415                 ;;
 416         x)
 417                 CLEANUPALL="yes"
 418                 ;;
 419         k)
 420                 CLEANUP="no"
 421                 ;;
 422         K)
 423                 KMSG="yes"
 424                 ;;
 425         f)
 426                 LOOPBACK="no"
 427                 ;;
 428         S)
 429                 STACK_TRACER="yes"
 430                 ;;
 431         c)
 432                 constrain_path
 433                 exit
 434                 ;;
 435         R)
 436                 RERUN="yes"
 437                 ;;
 438         m)
 439                 KMEMLEAK="yes"
 440                 ;;
 441         n)
 442                 nfsfile=$OPTARG
 443                 [ -f "$nfsfile" ] || fail "Cannot read file: $nfsfile"
 444                 export NFS=1
 445                 . "$nfsfile"
 446                 ;;
 447         d)
 448                 FILEDIR="$OPTARG"
 449                 ;;
 450         D)
 451                 DEBUG="yes"
 452                 ;;
 453         I)
 454                 ITERATIONS="$OPTARG"
 455                 if [ "$ITERATIONS" -le 0 ]; then
 456                         fail "Iterations must be greater than 0."
 457                 fi
 458                 ;;
 459         s)
 460                 FILESIZE="$OPTARG"
 461                 ;;
 462         r)
 463                 RUNFILES="$OPTARG"
 464                 ;;
 465         t)
 466                 if [ -n "$SINGLETEST" ]; then
 467                         fail "-t can only be provided once."
 468                 fi
 469                 SINGLETEST="$OPTARG"
 470                 ;;
 471         T)
 472                 TAGS="$OPTARG"
 473                 ;;
 474         u)
 475                 SINGLETESTUSER="$OPTARG"
 476                 ;;
 477         ?)
 478                 usage
 479                 exit
 480                 ;;
 481         *)
 482                 ;;
 483         esac
 484 done
 485
 486 shift $((OPTIND-1))
 487
 488 FILES=${FILES:-"$FILEDIR/file-vdev0 $FILEDIR/file-vdev1 $FILEDIR/file-vdev2"}
 489 LOOPBACKS=${LOOPBACKS:-""}
 490
 491 if [ -n "$SINGLETEST" ]; then
 492         if [ -n "$TAGS" ]; then
 493                 fail "-t and -T are mutually exclusive."
 494         fi
 495         RUNFILE_DIR="/var/tmp"
 496         RUNFILES="zfs-tests.$$.run"
 497         [ -n "$QUIET" ] && SINGLEQUIET="True" || SINGLEQUIET="False"
 498
 499         cat >"${RUNFILE_DIR}/${RUNFILES}" << EOF
 500 [DEFAULT]
 501 pre =
 502 quiet = $SINGLEQUIET
 503 pre_user = root
 504 user = $SINGLETESTUSER
 505 timeout = 600
 506 post_user = root
 507 post =
 508 outputdir = /var/tmp/test_results
 509 EOF
 510         if [ "$SINGLETEST" = "${SINGLETEST%/*}" ] ; then
 511                 NEWSINGLETEST=$(find "$STF_SUITE" -name "$SINGLETEST*" -print -quit)
 512                 if [ -z "$NEWSINGLETEST" ] ; then
 513                         fail "couldn't find test matching '$SINGLETEST'"
 514                 fi
 515                 SINGLETEST=$NEWSINGLETEST
 516         fi
 517
 518         SINGLETESTDIR="${SINGLETEST%/*}"
 519         SETUPDIR="$SINGLETESTDIR"
 520         [ "${SETUPDIR#/}" = "$SETUPDIR" ] && SETUPDIR="$STF_SUITE/$SINGLETESTDIR"
 521         [ -x "$SETUPDIR/setup.ksh"   ] && SETUPSCRIPT="setup"     || SETUPSCRIPT=
 522         [ -x "$SETUPDIR/cleanup.ksh" ] && CLEANUPSCRIPT="cleanup" || CLEANUPSCRIPT=
 523
 524         SINGLETESTFILE="${SINGLETEST##*/}"
 525         cat >>"${RUNFILE_DIR}/${RUNFILES}" << EOF
 526
 527 [$SINGLETESTDIR]
 528 tests = ['$SINGLETESTFILE']
 529 pre = $SETUPSCRIPT
 530 post = $CLEANUPSCRIPT
 531 tags = ['functional']
 532 EOF
 533 fi
 534
 535 #
 536 # Use default tag if none was specified
 537 #
 538 TAGS=${TAGS:='functional'}
 539
 540
 541
 542 #
 543 # Attempt to locate the runfiles describing the test workload.
 544 #
 545 R=""
 546 IFS=,
 547 for RUNFILE in $RUNFILES; do
 548         if [ -n "$RUNFILE" ]; then
 549                 SAVED_RUNFILE="$RUNFILE"
 550                 RUNFILE=$(find_runfile "$RUNFILE") ||
 551                         fail "Cannot find runfile: $SAVED_RUNFILE"
 552                 R="$R,$RUNFILE"
 553         fi
 554
 555         if [ ! -r "$RUNFILE" ]; then
 556                 fail "Cannot read runfile: $RUNFILE"
 557         fi
 558 done
 559 unset IFS
 560 RUNFILES=${R#,}
 561
 562 # The tag can be a fraction to indicate which portion of ZTS to run, Like
 563 #
 564 #       "1/3": Run first one third of all tests in runfiles
 565 #       "2/3": Run second one third of all test in runfiles
 566 #       "6/10": Run 6th tenth of all tests in runfiles
 567 #
 568 # This is useful for splitting up the test across multiple runners.
 569 #
 570 # After this code block, TAGS will be transformed from something like
 571 # "1/3" to a comma separate taglist, like:
 572 #
 573 # "append,atime,bootfs,cachefile,checksum,cp_files,deadman,dos_attributes, ..."
 574 #
 575 if echo "$TAGS" | grep -Eq '^[0-9]+/[0-9]+$' ; then
 576         TAGS=$(split_tags)
 577 fi
 578
 579 #
 580 # This script should not be run as root.  Instead the test user, which may
 581 # be a normal user account, needs to be configured such that it can
 582 # run commands via sudo passwordlessly.
 583 #
 584 if [ "$(id -u)" = "0" ]; then
 585         fail "This script must not be run as root."
 586 fi
 587
 588 if [ "$(sudo id -un)" != "root" ]; then
 589         fail "Passwordless sudo access required."
 590 fi
 591
 592 #
 593 # Constrain the available binaries to a known set.
 594 #
 595 constrain_path
 596
 597 #
 598 # Check if ksh exists
 599 #
 600 if [ "$UNAME" = "FreeBSD" ]; then
 601         sudo ln -fs /usr/local/bin/ksh93 /bin/ksh
 602 fi
 603 [ -e "$STF_PATH/ksh" ] || fail "This test suite requires ksh."
 604 [ -e "$STF_SUITE/include/default.cfg" ] || fail \
 605     "Missing $STF_SUITE/include/default.cfg file."
 606
 607 #
 608 # Verify the ZFS module stack is loaded.
 609 #
 610 if [ "$STACK_TRACER" = "yes" ]; then
 611         sudo "${ZFS_SH}" -S >/dev/null 2>&1
 612 else
 613         sudo "${ZFS_SH}" >/dev/null 2>&1
 614 fi
 615
 616 #
 617 # Attempt to cleanup all previous state for a new test run.
 618 #
 619 if [ "$CLEANUPALL" = "yes" ]; then
 620         cleanup_all
 621 fi
 622
 623 #
 624 # By default preserve any existing pools
 625 #
 626 if [ -z "${KEEP}" ]; then
 627         KEEP="$(ASAN_OPTIONS=detect_leaks=false "$ZPOOL" list -Ho name | tr -s '[:space:]' ' ')"
 628         if [ -z "${KEEP}" ]; then
 629                 KEEP="rpool"
 630         fi
 631 else
 632         KEEP="$(echo "$KEEP" | tr -s '[:space:]' ' ')"
 633 fi
 634
 635 #
 636 # NOTE: The following environment variables are undocumented
 637 # and should be used for testing purposes only:
 638 #
 639 # __ZFS_POOL_EXCLUDE - don't iterate over the pools it lists
 640 # __ZFS_POOL_RESTRICT - iterate only over the pools it lists
 641 #
 642 # See libzfs/libzfs_config.c for more information.
 643 #
 644 __ZFS_POOL_EXCLUDE="$KEEP"
 645
 646 . "$STF_SUITE/include/default.cfg"
 647
 648 #
 649 # No DISKS have been provided so a basic file or loopback based devices
 650 # must be created for the test suite to use.
 651 #
 652 if [ -z "${DISKS}" ]; then
 653         #
 654         # If this is a performance run, prevent accidental use of
 655         # loopback devices.
 656         #
 657         [ "$TAGS" = "perf" ] && fail "Running perf tests without disks."
 658
 659         #
 660         # Create sparse files for the test suite.  These may be used
 661         # directory or have loopback devices layered on them.
 662         #
 663         for TEST_FILE in ${FILES}; do
 664                 [ -f "$TEST_FILE" ] && fail "Failed file exists: ${TEST_FILE}"
 665                 truncate -s "${FILESIZE}" "${TEST_FILE}" ||
 666                     fail "Failed creating: ${TEST_FILE} ($?)"
 667         done
 668
 669         #
 670         # If requested setup loopback devices backed by the sparse files.
 671         #
 672         if [ "$LOOPBACK" = "yes" ]; then
 673                 test -x "$LOSETUP" || fail "$LOSETUP utility must be installed"
 674
 675                 for TEST_FILE in ${FILES}; do
 676                         if [ "$UNAME" = "FreeBSD" ] ; then
 677                                 MDDEVICE=$(sudo "${LOSETUP}" -a -t vnode -f "${TEST_FILE}")
 678                                 if [ -z "$MDDEVICE" ] ; then
 679                                         fail "Failed: ${TEST_FILE} -> loopback"
 680                                 fi
 681                                 DISKS="$DISKS $MDDEVICE"
 682                                 LOOPBACKS="$LOOPBACKS $MDDEVICE"
 683                         else
 684                                 TEST_LOOPBACK=$(sudo "${LOSETUP}" --show -f "${TEST_FILE}") ||
 685                                     fail "Failed: ${TEST_FILE} -> ${TEST_LOOPBACK}"
 686                                 BASELOOPBACK="${TEST_LOOPBACK##*/}"
 687                                 DISKS="$DISKS $BASELOOPBACK"
 688                                 LOOPBACKS="$LOOPBACKS $TEST_LOOPBACK"
 689                         fi
 690                 done
 691                 DISKS=${DISKS# }
 692                 LOOPBACKS=${LOOPBACKS# }
 693         else
 694                 DISKS="$FILES"
 695         fi
 696 fi
 697
 698 #
 699 # It may be desirable to test with fewer disks than the default when running
 700 # the performance tests, but the functional tests require at least three.
 701 #
 702 NUM_DISKS=$(echo "${DISKS}" | awk '{print NF}')
 703 if [ "$TAGS" != "perf" ]; then
 704         [ "$NUM_DISKS" -lt 3 ] && fail "Not enough disks ($NUM_DISKS/3 minimum)"
 705 fi
 706
 707 #
 708 # Disable SELinux until the ZFS Test Suite has been updated accordingly.
 709 #
 710 if command -v setenforce >/dev/null; then
 711         sudo setenforce permissive >/dev/null 2>&1
 712 fi
 713
 714 #
 715 # Enable internal ZFS debug log and clear it.
 716 #
 717 if [ -e /sys/module/zfs/parameters/zfs_dbgmsg_enable ]; then
 718         sudo sh -c "echo 1 >/sys/module/zfs/parameters/zfs_dbgmsg_enable"
 719         sudo sh -c "echo 0 >/proc/spl/kstat/zfs/dbgmsg"
 720 fi
 721
 722 msg
 723 msg "--- Configuration ---"
 724 msg "Runfiles:        $RUNFILES"
 725 msg "STF_TOOLS:       $STF_TOOLS"
 726 msg "STF_SUITE:       $STF_SUITE"
 727 msg "STF_PATH:        $STF_PATH"
 728 msg "FILEDIR:         $FILEDIR"
 729 msg "FILES:           $FILES"
 730 msg "LOOPBACKS:       $LOOPBACKS"
 731 msg "DISKS:           $DISKS"
 732 msg "NUM_DISKS:       $NUM_DISKS"
 733 msg "FILESIZE:        $FILESIZE"
 734 msg "ITERATIONS:      $ITERATIONS"
 735 msg "TAGS:            $TAGS"
 736 msg "STACK_TRACER:    $STACK_TRACER"
 737 msg "Keep pool(s):    $KEEP"
 738 msg "Missing util(s): $STF_MISSING_BIN"
 739 msg ""
 740
 741 export STF_TOOLS
 742 export STF_SUITE
 743 export STF_PATH
 744 export DISKS
 745 export FILEDIR
 746 export KEEP
 747 export __ZFS_POOL_EXCLUDE
 748 export TESTFAIL_CALLBACKS
 749
 750 mktemp_file() {
 751         if [ "$UNAME" = "FreeBSD" ]; then
 752                 mktemp -u "${FILEDIR}/$1.XXXXXX"
 753         else
 754                 mktemp -ut "$1.XXXXXX" -p "$FILEDIR"
 755         fi
 756 }
 757 mkdir -p "$FILEDIR" || :
 758 RESULTS_FILE=$(mktemp_file zts-results)
 759 REPORT_FILE=$(mktemp_file zts-report)
 760
 761 #
 762 # Run all the tests as specified.
 763 #
 764 msg "${TEST_RUNNER}" \
 765     "${QUIET:+-q}" \
 766     "${DEBUG:+-D}" \
 767     "${KMEMLEAK:+-m}" \
 768     "${KMSG:+-K}" \
 769     "-c \"${RUNFILES}\"" \
 770     "-T \"${TAGS}\"" \
 771     "-i \"${STF_SUITE}\"" \
 772     "-I \"${ITERATIONS}\""
 773 { PATH=$STF_PATH \
 774     ${TEST_RUNNER} \
 775     ${QUIET:+-q} \
 776     ${DEBUG:+-D} \
 777     ${KMEMLEAK:+-m} \
 778     ${KMSG:+-K} \
 779     -c "${RUNFILES}" \
 780     -T "${TAGS}" \
 781     -i "${STF_SUITE}" \
 782     -I "${ITERATIONS}" \
 783     2>&1; echo $? >"$REPORT_FILE"; } | tee "$RESULTS_FILE"
 784 read -r RUNRESULT <"$REPORT_FILE"
 785
 786 #
 787 # Analyze the results.
 788 #
 789 ${ZTS_REPORT} ${RERUN:+--no-maybes} "$RESULTS_FILE" >"$REPORT_FILE"
 790 RESULT=$?
 791
 792 if [ "$RESULT" -eq "2" ] && [ -n "$RERUN" ]; then
 793         MAYBES="$($ZTS_REPORT --list-maybes)"
 794         TEMP_RESULTS_FILE=$(mktemp_file zts-results-tmp)
 795         TEST_LIST=$(mktemp_file test-list)
 796         grep "^Test:.*\[FAIL\]" "$RESULTS_FILE" >"$TEMP_RESULTS_FILE"
 797         for test_name in $MAYBES; do
 798                 grep "$test_name " "$TEMP_RESULTS_FILE" >>"$TEST_LIST"
 799         done
 800         { PATH=$STF_PATH \
 801             ${TEST_RUNNER} \
 802                 ${QUIET:+-q} \
 803                 ${DEBUG:+-D} \
 804                 ${KMEMLEAK:+-m} \
 805             -c "${RUNFILES}" \
 806             -T "${TAGS}" \
 807             -i "${STF_SUITE}" \
 808             -I "${ITERATIONS}" \
 809             -l "${TEST_LIST}" \
 810             2>&1; echo $? >"$REPORT_FILE"; } | tee "$RESULTS_FILE"
 811         read -r RUNRESULT <"$REPORT_FILE"
 812         #
 813         # Analyze the results.
 814         #
 815         ${ZTS_REPORT} --no-maybes "$RESULTS_FILE" >"$REPORT_FILE"
 816         RESULT=$?
 817 fi
 818
 819
 820 cat "$REPORT_FILE"
 821
 822 RESULTS_DIR=$(awk '/^Log directory/ { print $3 }' "$RESULTS_FILE")
 823 if [ -d "$RESULTS_DIR" ]; then
 824         cat "$RESULTS_FILE" "$REPORT_FILE" >"$RESULTS_DIR/results"
 825 fi
 826
 827 rm -f "$RESULTS_FILE" "$REPORT_FILE" "$TEST_LIST" "$TEMP_RESULTS_FILE"
 828
 829 if [ -n "$SINGLETEST" ]; then
 830         rm -f "$RUNFILES" >/dev/null 2>&1
 831 fi
 832
 833 [ "$RUNRESULT" -gt 3 ] && exit "$RUNRESULT" || exit "$RESULT"