scripts/zimport.sh

   1 #!/usr/bin/env bash
   2 #
   3 # Verify that an assortment of known good reference pools can be imported
   4 # using different versions of the ZoL code.
   5 #
   6 # By default references pools for the major ZFS implementation will be
   7 # checked against the most recent ZoL tags and the master development branch.
   8 # Alternate tags or branches may be verified with the '-s <src-tag> option.
   9 # Passing the keyword "installed" will instruct the script to test whatever
  10 # version is installed.
  11 #
  12 # Preferentially a reference pool is used for all tests.  However, if one
  13 # does not exist and the pool-tag matches one of the src-tags then a new
  14 # reference pool will be created using binaries from that source build.
  15 # This is particularly useful when you need to test your changes before
  16 # opening a pull request.  The keyword 'all' can be used as short hand
  17 # refer to all available reference pools.
  18 #
  19 # New reference pools may be added by placing a bzip2 compressed tarball
  20 # of the pool in the scripts/zfs-images directory and then passing
  21 # the -p <pool-tag> option.  To increase the test coverage reference pools
  22 # should be collected for all the major ZFS implementations.  Having these
  23 # pools easily available is also helpful to the developers.
  24 #
  25 # Care should be taken to run these tests with a kernel supported by all
  26 # the listed tags.  Otherwise build failure will cause false positives.
  27 #
  28 #
  29 # EXAMPLES:
  30 #
  31 # The following example will verify the zfs-0.6.2 tag, the master branch,
  32 # and the installed zfs version can correctly import the listed pools.
  33 # Note there is no reference pool available for master and installed but
  34 # because binaries are available one is automatically constructed.  The
  35 # working directory is also preserved between runs (-k) preventing the
  36 # need to rebuild from source for multiple runs.
  37 #
  38 #  zimport.sh -k -f /var/tmp/zimport \
  39 #      -s "zfs-0.6.2 master installed" \
  40 #      -p "zevo-1.1.1 zol-0.6.2 zol-0.6.2-173 master installed"
  41 #
  42 # ------------------------ OpenZFS Source Versions ----------------
  43 #                 zfs-0.6.2       master          0.6.2-175_g36eb554
  44 # -----------------------------------------------------------------
  45 # Clone ZFS       Local         Local           Skip
  46 # Build ZFS       Pass          Pass            Skip
  47 # -----------------------------------------------------------------
  48 # zevo-1.1.1      Pass          Pass            Pass
  49 # zol-0.6.2       Pass          Pass            Pass
  50 # zol-0.6.2-173   Fail          Pass            Pass
  51 # master          Pass          Pass            Pass
  52 # installed       Pass          Pass            Pass
  53 #
  54
  55 BASE_DIR=$(dirname "$0")
  56 SCRIPT_COMMON=common.sh
  57 if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then
  58         . "${BASE_DIR}/${SCRIPT_COMMON}"
  59 else
  60         echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
  61 fi
  62
  63 PROG=zimport.sh
  64 SRC_TAGS="zfs-0.6.5.11 master"
  65 POOL_TAGS="all master"
  66 POOL_CREATE_OPTIONS=
  67 TEST_DIR=$(mktemp -u -d -p /var/tmp zimport.XXXXXXXX)
  68 KEEP="no"
  69 VERBOSE="no"
  70 COLOR="yes"
  71 REPO="https://github.com/openzfs"
  72 IMAGES_DIR="$SCRIPTDIR/zfs-images/"
  73 IMAGES_TAR="https://github.com/openzfs/zfs-images/tarball/master"
  74 ERROR=0
  75
  76 CONFIG_LOG="configure.log"
  77 CONFIG_OPTIONS=${CONFIG_OPTIONS:-""}
  78 MAKE_LOG="make.log"
  79 MAKE_OPTIONS=${MAKE_OPTIONS:-"-s -j$(nproc)"}
  80
  81 COLOR_GREEN="\033[0;32m"
  82 COLOR_RED="\033[0;31m"
  83 COLOR_BROWN="\033[0;33m"
  84 COLOR_RESET="\033[0m"
  85
  86 usage() {
  87 cat << EOF
  88 USAGE:
  89 zimport.sh [hvl] [-r repo] [-s src-tag] [-i pool-dir] [-p pool-tag]
  90     [-f path] [-o options]
  91
  92 DESCRIPTION:
  93         ZPOOL import verification tests
  94
  95 OPTIONS:
  96         -h                Show this message
  97         -v                Verbose
  98         -c                No color
  99         -k                Keep temporary directory
 100         -r <repo>         Source repository ($REPO)
 101         -s <src-tag>...   Verify ZoL versions with the listed tags
 102         -i <pool-dir>     Pool image directory
 103         -p <pool-tag>...  Verify pools created with the listed tags
 104         -f <path>         Temporary directory to use
 105         -o <options>      Additional options to pass to 'zpool create'
 106
 107 EOF
 108 }
 109
 110 while getopts 'hvckr:s:i:p:f:o:?' OPTION; do
 111         case $OPTION in
 112         h)
 113                 usage
 114                 exit 1
 115                 ;;
 116         v)
 117                 VERBOSE="yes"
 118                 ;;
 119         c)
 120                 COLOR="no"
 121                 ;;
 122         k)
 123                 KEEP="yes"
 124                 ;;
 125         r)
 126                 REPO="$OPTARG"
 127                 ;;
 128         s)
 129                 SRC_TAGS="$OPTARG"
 130                 ;;
 131         i)
 132                 IMAGES_DIR="$OPTARG"
 133                 ;;
 134         p)
 135                 POOL_TAGS="$OPTARG"
 136                 ;;
 137         f)
 138                 TEST_DIR="$OPTARG"
 139                 ;;
 140         o)
 141                 POOL_CREATE_OPTIONS="$OPTARG"
 142                 ;;
 143         ?)
 144                 usage
 145                 exit 1
 146                 ;;
 147         esac
 148 done
 149
 150 #
 151 # Verify the module start is not loaded
 152 #
 153 if lsmod | grep zfs >/dev/null; then
 154         echo "ZFS modules must be unloaded"
 155         exit 1
 156 fi
 157
 158 #
 159 # Create a random directory tree of files and sub-directories to
 160 # to act as a copy source for the various regression tests.
 161 #
 162 populate() {
 163         local ROOT=$1
 164         local MAX_DIR_SIZE=$2
 165         local MAX_FILE_SIZE=$3
 166
 167         # shellcheck disable=SC2086
 168         mkdir -p $ROOT/{a,b,c,d,e,f,g}/{h,i}
 169         DIRS=$(find "$ROOT")
 170
 171         for DIR in $DIRS; do
 172                 COUNT=$((RANDOM % MAX_DIR_SIZE))
 173
 174                 # shellcheck disable=SC2034
 175                 for i in $(seq $COUNT); do
 176                         FILE=$(mktemp -p "$DIR")
 177                         SIZE=$((RANDOM % MAX_FILE_SIZE))
 178                         dd if=/dev/urandom of="$FILE" bs=1k \
 179                             count="$SIZE" &>/dev/null
 180                 done
 181         done
 182
 183         return 0
 184 }
 185
 186 SRC_DIR=$(mktemp -d -p /var/tmp/ zfs.src.XXXXXXXX)
 187 trap 'rm -Rf "$SRC_DIR"' INT TERM EXIT
 188 populate "$SRC_DIR" 10 100
 189
 190 SRC_DIR="$TEST_DIR/src"
 191 SRC_DIR_ZFS="$SRC_DIR/zfs"
 192
 193 if [ "$COLOR" = "no" ]; then
 194         COLOR_GREEN=""
 195         COLOR_BROWN=""
 196         COLOR_RED=""
 197         COLOR_RESET=""
 198 fi
 199
 200 pass_nonewline() {
 201         echo -n -e "${COLOR_GREEN}Pass${COLOR_RESET}\t\t"
 202 }
 203
 204 skip_nonewline() {
 205         echo -n -e "${COLOR_BROWN}Skip${COLOR_RESET}\t\t"
 206 }
 207
 208 fail_nonewline() {
 209         echo -n -e "${COLOR_RED}Fail${COLOR_RESET}\t\t"
 210 }
 211
 212 #
 213 # Log a failure message, cleanup, and return an error.
 214 #
 215 fail() {
 216         echo -e "$PROG: $1" >&2
 217         $ZFS_SH -u >/dev/null 2>&1
 218         exit 1
 219 }
 220
 221 #
 222 # Set several helper variables which are derived from a source tag.
 223 #
 224 # ZFS_TAG - The passed zfs-x.y.z tag
 225 # ZFS_DIR - The zfs directory name
 226 # ZFS_URL - The zfs github URL to fetch the tarball
 227 #
 228 src_set_vars() {
 229         local TAG=$1
 230
 231         ZFS_TAG="$TAG"
 232         ZFS_DIR="$SRC_DIR_ZFS/$ZFS_TAG"
 233         ZFS_URL="$REPO/zfs/tarball/$ZFS_TAG"
 234
 235         if [ "$TAG" = "installed" ]; then
 236                 ZPOOL_CMD=$(command -v zpool)
 237                 ZFS_CMD=$(command -v zfs)
 238                 ZFS_SH="/usr/share/zfs/zfs.sh"
 239         else
 240                 ZPOOL_CMD="./cmd/zpool/zpool"
 241                 ZFS_CMD="./cmd/zfs/zfs"
 242                 ZFS_SH="./scripts/zfs.sh"
 243         fi
 244 }
 245
 246 #
 247 # Set several helper variables which are derived from a pool name such
 248 # as zol-0.6.x, zevo-1.1.1, etc.  These refer to example pools from various
 249 # ZFS implementations which are used to verify compatibility.
 250 #
 251 # POOL_TAG          - The example pools name in scripts/zfs-images/.
 252 # POOL_BZIP         - The full path to the example bzip2 compressed pool.
 253 # POOL_DIR          - The top level test path for this pool.
 254 # POOL_DIR_PRISTINE - The directory containing a pristine version of the pool.
 255 # POOL_DIR_COPY     - The directory containing a working copy of the pool.
 256 # POOL_DIR_SRC      - Location of a source build if it exists for this pool.
 257 #
 258 pool_set_vars() {
 259         local TAG=$1
 260
 261         POOL_TAG=$TAG
 262         POOL_BZIP=$IMAGES_DIR/$POOL_TAG.tar.bz2
 263         POOL_DIR=$TEST_DIR/pools/$POOL_TAG
 264         POOL_DIR_PRISTINE=$POOL_DIR/pristine
 265         POOL_DIR_COPY=$POOL_DIR/copy
 266         POOL_DIR_SRC="$SRC_DIR_ZFS/${POOL_TAG//zol/zfs}"
 267 }
 268
 269 #
 270 # Construct a non-trivial pool given a specific version of the source.  More
 271 # interesting pools provide better test coverage so this function should
 272 # extended as needed to create more realistic pools.
 273 #
 274 pool_create() {
 275         pool_set_vars "$1"
 276         src_set_vars "$1"
 277
 278         if [ "$POOL_TAG" != "installed" ]; then
 279                 cd "$POOL_DIR_SRC" || fail "Failed 'cd $POOL_DIR_SRC'"
 280         fi
 281
 282         $ZFS_SH zfs="spa_config_path=$POOL_DIR_PRISTINE" || \
 283             fail "Failed to load kmods"
 284
 285         # Create a file vdev RAIDZ pool.
 286         truncate -s 1G \
 287             "$POOL_DIR_PRISTINE/vdev1" "$POOL_DIR_PRISTINE/vdev2" \
 288             "$POOL_DIR_PRISTINE/vdev3" "$POOL_DIR_PRISTINE/vdev4" || \
 289             fail "Failed 'truncate -s 1G ...'"
 290         # shellcheck disable=SC2086
 291         $ZPOOL_CMD create $POOL_CREATE_OPTIONS "$POOL_TAG" raidz \
 292             "$POOL_DIR_PRISTINE/vdev1" "$POOL_DIR_PRISTINE/vdev2" \
 293             "$POOL_DIR_PRISTINE/vdev3" "$POOL_DIR_PRISTINE/vdev4" || \
 294             fail "Failed '$ZPOOL_CMD create $POOL_CREATE_OPTIONS $POOL_TAG ...'"
 295
 296         # Create a pool/fs filesystem with some random contents.
 297         $ZFS_CMD create "$POOL_TAG/fs" || \
 298             fail "Failed '$ZFS_CMD create $POOL_TAG/fs'"
 299         populate "/$POOL_TAG/fs/" 10 100
 300
 301         # Snapshot that filesystem, clone it, remove the files/dirs,
 302         # replace them with new files/dirs.
 303         $ZFS_CMD snap "$POOL_TAG/fs@snap" || \
 304             fail "Failed '$ZFS_CMD snap $POOL_TAG/fs@snap'"
 305         $ZFS_CMD clone "$POOL_TAG/fs@snap" "$POOL_TAG/clone" || \
 306             fail "Failed '$ZFS_CMD clone $POOL_TAG/fs@snap $POOL_TAG/clone'"
 307         # shellcheck disable=SC2086
 308         rm -Rf /$POOL_TAG/clone/*
 309         populate "/$POOL_TAG/clone/" 10 100
 310
 311         # Scrub the pool, delay slightly, then export it.  It is now
 312         # somewhat interesting for testing purposes.
 313         $ZPOOL_CMD scrub "$POOL_TAG" || \
 314             fail "Failed '$ZPOOL_CMD scrub $POOL_TAG'"
 315         sleep 10
 316         $ZPOOL_CMD export "$POOL_TAG" || \
 317             fail "Failed '$ZPOOL_CMD export $POOL_TAG'"
 318
 319         $ZFS_SH -u || fail "Failed to unload kmods"
 320 }
 321
 322 # If the zfs-images directory doesn't exist fetch a copy from Github then
 323 # cache it in the $TEST_DIR and update $IMAGES_DIR.
 324 if [ ! -d "$IMAGES_DIR" ]; then
 325         IMAGES_DIR="$TEST_DIR/zfs-images"
 326         mkdir -p "$IMAGES_DIR"
 327         curl -sL "$IMAGES_TAR" | \
 328             tar -xz -C "$IMAGES_DIR" --strip-components=1 || \
 329             fail "Failed to download pool images"
 330 fi
 331
 332 # Given the available images in the zfs-images directory substitute the
 333 # list of available images for the reserved keyword 'all'.
 334 for TAG in $POOL_TAGS; do
 335
 336         if  [ "$TAG" = "all" ]; then
 337                 # shellcheck disable=SC2010
 338                 ALL_TAGS=$(ls "$IMAGES_DIR" | grep "tar.bz2" | \
 339                     sed 's/.tar.bz2//' | tr '\n' ' ')
 340                 NEW_TAGS="$NEW_TAGS $ALL_TAGS"
 341         else
 342                 NEW_TAGS="$NEW_TAGS $TAG"
 343         fi
 344 done
 345 POOL_TAGS="$NEW_TAGS"
 346
 347 if [ "$VERBOSE" = "yes" ]; then
 348         echo "---------------------------- Options ----------------------------"
 349         echo "VERBOSE=$VERBOSE"
 350         echo "KEEP=$KEEP"
 351         echo "REPO=$REPO"
 352         echo "SRC_TAGS=$SRC_TAGS"
 353         echo "POOL_TAGS=$POOL_TAGS"
 354         echo "PATH=$TEST_DIR"
 355         echo "POOL_CREATE_OPTIONS=$POOL_CREATE_OPTIONS"
 356         echo
 357 fi
 358
 359 if [ ! -d "$TEST_DIR" ]; then
 360         mkdir -p "$TEST_DIR"
 361 fi
 362
 363 if [ ! -d "$SRC_DIR" ]; then
 364         mkdir -p "$SRC_DIR"
 365 fi
 366
 367 # Print a header for all tags which are being tested.
 368 echo "------------------------ OpenZFS Source Versions ----------------"
 369 printf "%-16s" " "
 370 for TAG in $SRC_TAGS; do
 371         src_set_vars "$TAG"
 372
 373         if [ "$TAG" = "installed" ]; then
 374                 ZFS_VERSION=$(modinfo zfs | awk '/version:/ { print $2; exit }')
 375                 if [ -n "$ZFS_VERSION" ]; then
 376                         printf "%-16s" "$ZFS_VERSION"
 377                 else
 378                         fail "ZFS is not installed"
 379                 fi
 380         else
 381                 printf "%-16s" "$TAG"
 382         fi
 383 done
 384 echo -e "\n-----------------------------------------------------------------"
 385
 386 #
 387 # Attempt to generate the tarball from your local git repository, if that
 388 # fails then attempt to download the tarball from Github.
 389 #
 390 printf "%-16s" "Clone ZFS"
 391 for TAG in $SRC_TAGS; do
 392         src_set_vars "$TAG"
 393
 394         if [ -d "$ZFS_DIR" ]; then
 395                 skip_nonewline
 396         elif  [ "$ZFS_TAG" = "installed" ]; then
 397                 skip_nonewline
 398         else
 399                 cd "$SRC_DIR" || fail "Failed 'cd $SRC_DIR'"
 400
 401                 if [ ! -d "$SRC_DIR_ZFS" ]; then
 402                         mkdir -p "$SRC_DIR_ZFS"
 403                 fi
 404
 405                 git archive --format=tar --prefix="$ZFS_TAG/ $ZFS_TAG" \
 406                     -o "$SRC_DIR_ZFS/$ZFS_TAG.tar" &>/dev/null || \
 407                     rm "$SRC_DIR_ZFS/$ZFS_TAG.tar"
 408                 if [ -s "$SRC_DIR_ZFS/$ZFS_TAG.tar" ]; then
 409                         tar -xf "$SRC_DIR_ZFS/$ZFS_TAG.tar" -C "$SRC_DIR_ZFS"
 410                         rm "$SRC_DIR_ZFS/$ZFS_TAG.tar"
 411                         echo -n -e "${COLOR_GREEN}Local${COLOR_RESET}\t\t"
 412                 else
 413                         mkdir -p "$ZFS_DIR" || fail "Failed to create $ZFS_DIR"
 414                         curl -sL "$ZFS_URL" | tar -xz -C "$ZFS_DIR" \
 415                             --strip-components=1 || \
 416                             fail "Failed to download $ZFS_URL"
 417                         echo -n -e "${COLOR_GREEN}Remote${COLOR_RESET}\t\t"
 418                 fi
 419         fi
 420 done
 421 printf "\n"
 422
 423 # Build the listed tags
 424 printf "%-16s" "Build ZFS"
 425 for TAG in $SRC_TAGS; do
 426         src_set_vars "$TAG"
 427
 428         if [ -f "$ZFS_DIR/module/zfs/zfs.ko" ]; then
 429                 skip_nonewline
 430         elif  [ "$ZFS_TAG" = "installed" ]; then
 431                 skip_nonewline
 432         else
 433                 cd "$ZFS_DIR" || fail "Failed 'cd $ZFS_DIR'"
 434                 make distclean &>/dev/null
 435                 ./autogen.sh >>"$CONFIG_LOG" 2>&1 || \
 436                     fail "Failed ZFS 'autogen.sh'"
 437                 # shellcheck disable=SC2086
 438                 ./configure $CONFIG_OPTIONS >>"$CONFIG_LOG" 2>&1 || \
 439                     fail "Failed ZFS 'configure $CONFIG_OPTIONS'"
 440                 # shellcheck disable=SC2086
 441                 make $MAKE_OPTIONS >>"$MAKE_LOG" 2>&1 || \
 442                     fail "Failed ZFS 'make $MAKE_OPTIONS'"
 443                 pass_nonewline
 444         fi
 445 done
 446 printf "\n"
 447 echo "-----------------------------------------------------------------"
 448
 449 # Either create a new pool using 'zpool create', or alternately restore an
 450 # existing pool from another ZFS implementation for compatibility testing.
 451 for TAG in $POOL_TAGS; do
 452         pool_set_vars "$TAG"
 453         SKIP=0
 454
 455         printf "%-16s" "$POOL_TAG"
 456         rm -Rf "$POOL_DIR"
 457         mkdir -p "$POOL_DIR_PRISTINE"
 458
 459         # Use the existing compressed image if available.
 460         if [ -f "$POOL_BZIP" ]; then
 461                 tar -xjf "$POOL_BZIP" -C "$POOL_DIR_PRISTINE" \
 462                     --strip-components=1 || \
 463                     fail "Failed 'tar -xjf $POOL_BZIP"
 464         # Use the installed version to create the pool.
 465         elif  [ "$TAG" = "installed" ]; then
 466                 pool_create "$TAG"
 467         # A source build is available to create the pool.
 468         elif [ -d "$POOL_DIR_SRC" ]; then
 469                 pool_create "$TAG"
 470         else
 471                 SKIP=1
 472         fi
 473
 474         # Verify 'zpool import' works for all listed source versions.
 475         for SRC_TAG in $SRC_TAGS; do
 476
 477                 if [ $SKIP -eq 1 ]; then
 478                         skip_nonewline
 479                         continue
 480                 fi
 481
 482                 src_set_vars "$SRC_TAG"
 483                 if [ "$SRC_TAG" != "installed" ]; then
 484                         cd "$ZFS_DIR" || fail "Failed 'cd $ZFS_DIR'"
 485                 fi
 486                 $ZFS_SH zfs="spa_config_path=$POOL_DIR_COPY"
 487
 488                 cp -a --sparse=always "$POOL_DIR_PRISTINE" \
 489                     "$POOL_DIR_COPY" || \
 490                     fail "Failed to copy $POOL_DIR_PRISTINE to $POOL_DIR_COPY"
 491                 POOL_NAME=$($ZPOOL_CMD import -d "$POOL_DIR_COPY" | \
 492                     awk '/pool:/ { print $2; exit 0 }')
 493
 494                 $ZPOOL_CMD import -N -d "$POOL_DIR_COPY" \
 495                    "$POOL_NAME" &>/dev/null
 496                 # shellcheck disable=SC2181
 497                 if [ $? -ne 0 ]; then
 498                         fail_nonewline
 499                         ERROR=1
 500                 else
 501                         $ZPOOL_CMD export "$POOL_NAME" || \
 502                             fail "Failed to export pool"
 503                         pass_nonewline
 504                 fi
 505
 506                 rm -Rf "$POOL_DIR_COPY"
 507
 508                 $ZFS_SH -u || fail "Failed to unload kmods"
 509         done
 510         printf "\n"
 511 done
 512
 513 if [ "$KEEP" = "no" ]; then
 514         rm -Rf "$TEST_DIR"
 515 fi
 516
 517 exit $ERROR