scripts/zimport.sh

   1 #!/usr/bin/env bash
   2 #
   3 # Verify that an assortment of known good reference pools can be imported
   4 # using different versions of OpenZFS code.
   5 #
   6 # By default references pools for the major ZFS implementation will be
   7 # checked against the most recent OpenZFS tags and the master development branch.
   8 # Alternate tags or branches may be verified with the '-s <src-tag> option.
   9 # Passing the keyword "installed" will instruct the script to test whatever
  10 # version is installed.
  11 #
  12 # Preferentially a reference pool is used for all tests.  However, if one
  13 # does not exist and the pool-tag matches one of the src-tags then a new
  14 # reference pool will be created using binaries from that source build.
  15 # This is particularly useful when you need to test your changes before
  16 # opening a pull request.  The keyword 'all' can be used as short hand
  17 # refer to all available reference pools.
  18 #
  19 # New reference pools may be added by placing a bzip2 compressed tarball
  20 # of the pool in the scripts/zfs-images directory and then passing
  21 # the -p <pool-tag> option.  To increase the test coverage reference pools
  22 # should be collected for all the major ZFS implementations.  Having these
  23 # pools easily available is also helpful to the developers.
  24 #
  25 # Care should be taken to run these tests with a kernel supported by all
  26 # the listed tags.  Otherwise build failure will cause false positives.
  27 #
  28 #
  29 # EXAMPLES:
  30 #
  31 # The following example will verify the zfs-0.6.2 tag, the master branch,
  32 # and the installed zfs version can correctly import the listed pools.
  33 # Note there is no reference pool available for master and installed but
  34 # because binaries are available one is automatically constructed.  The
  35 # working directory is also preserved between runs (-k) preventing the
  36 # need to rebuild from source for multiple runs.
  37 #
  38 #  zimport.sh -k -f /var/tmp/zimport \
  39 #      -s "zfs-0.6.2 master installed" \
  40 #      -p "zevo-1.1.1 zol-0.6.2 zol-0.6.2-173 master installed"
  41 #
  42 # ------------------------ OpenZFS Source Versions ----------------
  43 #                 zfs-0.6.2       master          0.6.2-175_g36eb554
  44 # -----------------------------------------------------------------
  45 # Clone ZFS       Local         Local           Skip
  46 # Build ZFS       Pass          Pass            Skip
  47 # -----------------------------------------------------------------
  48 # zevo-1.1.1      Pass          Pass            Pass
  49 # zol-0.6.2       Pass          Pass            Pass
  50 # zol-0.6.2-173   Fail          Pass            Pass
  51 # master          Pass          Pass            Pass
  52 # installed       Pass          Pass            Pass
  53 #
  54
  55 BASE_DIR=$(dirname "$0")
  56 SCRIPT_COMMON=common.sh
  57 if [[ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]]; then
  58         . "${BASE_DIR}/${SCRIPT_COMMON}"
  59 else
  60         echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
  61 fi
  62
  63 PROG=zimport.sh
  64 SRC_TAGS="zfs-0.6.5.11 master"
  65 POOL_TAGS="all master"
  66 POOL_CREATE_OPTIONS=
  67 TEST_DIR=$(mktemp -u -d -p /var/tmp zimport.XXXXXXXX)
  68 KEEP="no"
  69 VERBOSE="no"
  70 COLOR="yes"
  71 REPO="https://github.com/openzfs"
  72 IMAGES_DIR="${BASE_DIR}/zfs-images/"
  73 IMAGES_TAR="https://github.com/openzfs/zfs-images/tarball/master"
  74 ERROR=0
  75
  76 CONFIG_LOG="configure.log"
  77 CONFIG_OPTIONS=${CONFIG_OPTIONS:-""}
  78 MAKE_LOG="make.log"
  79 MAKE_OPTIONS=${MAKE_OPTIONS:-"-s -j$(nproc)"}
  80
  81 COLOR_GREEN="\033[0;32m"
  82 COLOR_RED="\033[0;31m"
  83 COLOR_BROWN="\033[0;33m"
  84 COLOR_RESET="\033[0m"
  85
  86 usage() {
  87 cat << EOF
  88 USAGE:
  89 zimport.sh [hvl] [-r repo] [-s src-tag] [-i pool-dir] [-p pool-tag]
  90     [-f path] [-o options]
  91
  92 DESCRIPTION:
  93         ZPOOL import verification tests
  94
  95 OPTIONS:
  96         -h                Show this message
  97         -v                Verbose
  98         -c                No color
  99         -k                Keep temporary directory
 100         -r <repo>         Source repository ($REPO)
 101         -s <src-tag>...   Verify OpenZFS versions with the listed tags
 102         -i <pool-dir>     Pool image directory
 103         -p <pool-tag>...  Verify pools created with the listed tags
 104         -f <path>         Temporary directory to use
 105         -o <options>      Additional options to pass to 'zpool create'
 106
 107 EOF
 108 }
 109
 110 while getopts 'hvckr:s:i:p:f:o:?' OPTION; do
 111         case $OPTION in
 112         h)
 113                 usage
 114                 exit 1
 115                 ;;
 116         v)
 117                 VERBOSE="yes"
 118                 ;;
 119         c)
 120                 COLOR="no"
 121                 ;;
 122         k)
 123                 KEEP="yes"
 124                 ;;
 125         r)
 126                 REPO="$OPTARG"
 127                 ;;
 128         s)
 129                 SRC_TAGS="$OPTARG"
 130                 ;;
 131         i)
 132                 IMAGES_DIR="$OPTARG"
 133                 ;;
 134         p)
 135                 POOL_TAGS="$OPTARG"
 136                 ;;
 137         f)
 138                 TEST_DIR="$OPTARG"
 139                 ;;
 140         o)
 141                 POOL_CREATE_OPTIONS="$OPTARG"
 142                 ;;
 143         *)
 144                 usage
 145                 exit 1
 146                 ;;
 147         esac
 148 done
 149
 150 #
 151 # Verify the module start is not loaded
 152 #
 153 if lsmod | grep zfs >/dev/null; then
 154         echo "ZFS modules must be unloaded"
 155         exit 1
 156 fi
 157
 158 #
 159 # Create a random directory tree of files and sub-directories to
 160 # to act as a copy source for the various regression tests.
 161 #
 162 populate() {
 163         local ROOT=$1
 164         local MAX_DIR_SIZE=$2
 165         local MAX_FILE_SIZE=$3
 166
 167         mkdir -p "$ROOT"/{a,b,c,d,e,f,g}/{h,i}
 168         DIRS=$(find "$ROOT")
 169
 170         for DIR in $DIRS; do
 171                 COUNT=$((RANDOM % MAX_DIR_SIZE))
 172
 173                 for _ in $(seq "$COUNT"); do
 174                         FILE=$(mktemp -p "$DIR")
 175                         SIZE=$((RANDOM % MAX_FILE_SIZE))
 176                         dd if=/dev/urandom of="$FILE" bs=1k \
 177                             count="$SIZE" &>/dev/null
 178                 done
 179         done
 180
 181         return 0
 182 }
 183
 184 SRC_DIR=$(mktemp -d -p /var/tmp/ zfs.src.XXXXXXXX)
 185 trap 'rm -Rf "$SRC_DIR"' INT TERM EXIT
 186 populate "$SRC_DIR" 10 100
 187
 188 SRC_DIR="$TEST_DIR/src"
 189 SRC_DIR_ZFS="$SRC_DIR/zfs"
 190
 191 if [[ "$COLOR" = "no" ]]; then
 192         COLOR_GREEN=""
 193         COLOR_BROWN=""
 194         COLOR_RED=""
 195         COLOR_RESET=""
 196 fi
 197
 198 pass_nonewline() {
 199         echo -n -e "${COLOR_GREEN}Pass${COLOR_RESET}\t\t"
 200 }
 201
 202 skip_nonewline() {
 203         echo -n -e "${COLOR_BROWN}Skip${COLOR_RESET}\t\t"
 204 }
 205
 206 fail_nonewline() {
 207         echo -n -e "${COLOR_RED}Fail${COLOR_RESET}\t\t"
 208 }
 209
 210 #
 211 # Log a failure message, cleanup, and return an error.
 212 #
 213 fail() {
 214         echo -e "$PROG: $1" >&2
 215         $ZFS_SH -u >/dev/null 2>&1
 216         exit 1
 217 }
 218
 219 #
 220 # Set several helper variables which are derived from a source tag.
 221 #
 222 # ZFS_TAG - The passed zfs-x.y.z tag
 223 # ZFS_DIR - The zfs directory name
 224 # ZFS_URL - The zfs github URL to fetch the tarball
 225 #
 226 src_set_vars() {
 227         local TAG=$1
 228
 229         ZFS_TAG="$TAG"
 230         ZFS_DIR="$SRC_DIR_ZFS/$ZFS_TAG"
 231         ZFS_URL="$REPO/zfs/tarball/$ZFS_TAG"
 232
 233         if [[ "$TAG" = "installed" ]]; then
 234                 ZPOOL_CMD=$(command -v zpool)
 235                 ZFS_CMD=$(command -v zfs)
 236                 ZFS_SH="/usr/share/zfs/zfs.sh"
 237         else
 238                 ZPOOL_CMD="./zpool"
 239                 ZFS_CMD="./zfs"
 240                 ZFS_SH="./scripts/zfs.sh"
 241         fi
 242 }
 243
 244 #
 245 # Set several helper variables which are derived from a pool name such
 246 # as zol-0.6.x, zevo-1.1.1, etc.  These refer to example pools from various
 247 # ZFS implementations which are used to verify compatibility.
 248 #
 249 # POOL_TAG          - The example pools name in scripts/zfs-images/.
 250 # POOL_BZIP         - The full path to the example bzip2 compressed pool.
 251 # POOL_DIR          - The top level test path for this pool.
 252 # POOL_DIR_PRISTINE - The directory containing a pristine version of the pool.
 253 # POOL_DIR_COPY     - The directory containing a working copy of the pool.
 254 # POOL_DIR_SRC      - Location of a source build if it exists for this pool.
 255 #
 256 pool_set_vars() {
 257         local TAG=$1
 258
 259         POOL_TAG=$TAG
 260         POOL_BZIP=$IMAGES_DIR/$POOL_TAG.tar.bz2
 261         POOL_DIR=$TEST_DIR/pools/$POOL_TAG
 262         POOL_DIR_PRISTINE=$POOL_DIR/pristine
 263         POOL_DIR_COPY=$POOL_DIR/copy
 264         POOL_DIR_SRC="$SRC_DIR_ZFS/${POOL_TAG//zol/zfs}"
 265 }
 266
 267 #
 268 # Construct a non-trivial pool given a specific version of the source.  More
 269 # interesting pools provide better test coverage so this function should
 270 # extended as needed to create more realistic pools.
 271 #
 272 pool_create() {
 273         pool_set_vars "$1"
 274         src_set_vars "$1"
 275
 276         if [[ "$POOL_TAG" != "installed" ]]; then
 277                 cd "$POOL_DIR_SRC" || fail "Failed 'cd $POOL_DIR_SRC'"
 278         fi
 279
 280         $ZFS_SH zfs="spa_config_path=$POOL_DIR_PRISTINE" || \
 281             fail "Failed to load kmods"
 282
 283         # Create a file vdev RAIDZ pool.
 284         truncate -s 1G \
 285             "$POOL_DIR_PRISTINE/vdev1" "$POOL_DIR_PRISTINE/vdev2" \
 286             "$POOL_DIR_PRISTINE/vdev3" "$POOL_DIR_PRISTINE/vdev4" || \
 287             fail "Failed 'truncate -s 1G ...'"
 288         # shellcheck disable=SC2086
 289         $ZPOOL_CMD create $POOL_CREATE_OPTIONS "$POOL_TAG" raidz \
 290             "$POOL_DIR_PRISTINE/vdev1" "$POOL_DIR_PRISTINE/vdev2" \
 291             "$POOL_DIR_PRISTINE/vdev3" "$POOL_DIR_PRISTINE/vdev4" || \
 292             fail "Failed '$ZPOOL_CMD create $POOL_CREATE_OPTIONS $POOL_TAG ...'"
 293
 294         # Create a pool/fs filesystem with some random contents.
 295         $ZFS_CMD create "$POOL_TAG/fs" || \
 296             fail "Failed '$ZFS_CMD create $POOL_TAG/fs'"
 297         populate "/$POOL_TAG/fs/" 10 100
 298
 299         # Snapshot that filesystem, clone it, remove the files/dirs,
 300         # replace them with new files/dirs.
 301         $ZFS_CMD snap "$POOL_TAG/fs@snap" || \
 302             fail "Failed '$ZFS_CMD snap $POOL_TAG/fs@snap'"
 303         $ZFS_CMD clone "$POOL_TAG/fs@snap" "$POOL_TAG/clone" || \
 304             fail "Failed '$ZFS_CMD clone $POOL_TAG/fs@snap $POOL_TAG/clone'"
 305         # shellcheck disable=SC2086
 306         rm -Rf /$POOL_TAG/clone/*
 307         populate "/$POOL_TAG/clone/" 10 100
 308
 309         # Scrub the pool, delay slightly, then export it.  It is now
 310         # somewhat interesting for testing purposes.
 311         $ZPOOL_CMD scrub "$POOL_TAG" || \
 312             fail "Failed '$ZPOOL_CMD scrub $POOL_TAG'"
 313         sleep 10
 314         $ZPOOL_CMD export "$POOL_TAG" || \
 315             fail "Failed '$ZPOOL_CMD export $POOL_TAG'"
 316
 317         $ZFS_SH -u || fail "Failed to unload kmods"
 318 }
 319
 320 # If the zfs-images directory doesn't exist fetch a copy from Github then
 321 # cache it in the $TEST_DIR and update $IMAGES_DIR.
 322 if [[ ! -d "$IMAGES_DIR" ]]; then
 323         IMAGES_DIR="$TEST_DIR/zfs-images"
 324         mkdir -p "$IMAGES_DIR"
 325         curl -sL "$IMAGES_TAR" | \
 326             tar -xz -C "$IMAGES_DIR" --strip-components=1 || \
 327             fail "Failed to download pool images"
 328 fi
 329
 330 # Given the available images in the zfs-images directory substitute the
 331 # list of available images for the reserved keyword 'all'.
 332 for TAG in $POOL_TAGS; do
 333
 334         if  [[ "$TAG" = "all" ]]; then
 335                 ALL_TAGS=$(echo "$IMAGES_DIR"/*.tar.bz2 | \
 336                     sed "s|$IMAGES_DIR/||g;s|.tar.bz2||g")
 337                 NEW_TAGS="$NEW_TAGS $ALL_TAGS"
 338         else
 339                 NEW_TAGS="$NEW_TAGS $TAG"
 340         fi
 341 done
 342 POOL_TAGS="$NEW_TAGS"
 343
 344 if [[ "$VERBOSE" = "yes" ]]; then
 345         echo "---------------------------- Options ----------------------------"
 346         echo "VERBOSE=$VERBOSE"
 347         echo "KEEP=$KEEP"
 348         echo "REPO=$REPO"
 349         echo "SRC_TAGS=$SRC_TAGS"
 350         echo "POOL_TAGS=$POOL_TAGS"
 351         echo "PATH=$TEST_DIR"
 352         echo "POOL_CREATE_OPTIONS=$POOL_CREATE_OPTIONS"
 353         echo
 354 fi
 355
 356 if [[ ! -d "$TEST_DIR" ]]; then
 357         mkdir -p "$TEST_DIR"
 358 fi
 359
 360 if [[ ! -d "$SRC_DIR" ]]; then
 361         mkdir -p "$SRC_DIR"
 362 fi
 363
 364 # Print a header for all tags which are being tested.
 365 echo "------------------------ OpenZFS Source Versions ----------------"
 366 printf "%-16s" " "
 367 for TAG in $SRC_TAGS; do
 368         src_set_vars "$TAG"
 369
 370         if [[ "$TAG" = "installed" ]]; then
 371                 ZFS_VERSION=$(modinfo zfs | awk '/version:/ { print $2; exit }')
 372                 if [[ -n "$ZFS_VERSION" ]]; then
 373                         printf "%-16s" "$ZFS_VERSION"
 374                 else
 375                         fail "ZFS is not installed"
 376                 fi
 377         else
 378                 printf "%-16s" "$TAG"
 379         fi
 380 done
 381 echo -e "\n-----------------------------------------------------------------"
 382
 383 #
 384 # Attempt to generate the tarball from your local git repository, if that
 385 # fails then attempt to download the tarball from Github.
 386 #
 387 printf "%-16s" "Clone ZFS"
 388 for TAG in $SRC_TAGS; do
 389         src_set_vars "$TAG"
 390
 391         if [[ -d "$ZFS_DIR" ]]; then
 392                 skip_nonewline
 393         elif  [[ "$ZFS_TAG" = "installed" ]]; then
 394                 skip_nonewline
 395         else
 396                 cd "$SRC_DIR" || fail "Failed 'cd $SRC_DIR'"
 397
 398                 if [[ ! -d "$SRC_DIR_ZFS" ]]; then
 399                         mkdir -p "$SRC_DIR_ZFS"
 400                 fi
 401
 402                 git archive --format=tar --prefix="$ZFS_TAG/ $ZFS_TAG" \
 403                     -o "$SRC_DIR_ZFS/$ZFS_TAG.tar" &>/dev/null || \
 404                     rm "$SRC_DIR_ZFS/$ZFS_TAG.tar"
 405                 if [[ -s "$SRC_DIR_ZFS/$ZFS_TAG.tar" ]]; then
 406                         tar -xf "$SRC_DIR_ZFS/$ZFS_TAG.tar" -C "$SRC_DIR_ZFS"
 407                         rm "$SRC_DIR_ZFS/$ZFS_TAG.tar"
 408                         echo -n -e "${COLOR_GREEN}Local${COLOR_RESET}\t\t"
 409                 else
 410                         mkdir -p "$ZFS_DIR" || fail "Failed to create $ZFS_DIR"
 411                         curl -sL "$ZFS_URL" | tar -xz -C "$ZFS_DIR" \
 412                             --strip-components=1 || \
 413                             fail "Failed to download $ZFS_URL"
 414                         echo -n -e "${COLOR_GREEN}Remote${COLOR_RESET}\t\t"
 415                 fi
 416         fi
 417 done
 418 printf "\n"
 419
 420 # Build the listed tags
 421 printf "%-16s" "Build ZFS"
 422 for TAG in $SRC_TAGS; do
 423         src_set_vars "$TAG"
 424
 425         if [[ -f "$ZFS_DIR/module/zfs/zfs.ko" ]]; then
 426                 skip_nonewline
 427         elif  [[ "$ZFS_TAG" = "installed" ]]; then
 428                 skip_nonewline
 429         else
 430                 cd "$ZFS_DIR" || fail "Failed 'cd $ZFS_DIR'"
 431                 make distclean &>/dev/null
 432                 ./autogen.sh >>"$CONFIG_LOG" 2>&1 || \
 433                     fail "Failed ZFS 'autogen.sh'"
 434                 # shellcheck disable=SC2086
 435                 ./configure $CONFIG_OPTIONS >>"$CONFIG_LOG" 2>&1 || \
 436                     fail "Failed ZFS 'configure $CONFIG_OPTIONS'"
 437                 # shellcheck disable=SC2086
 438                 make $MAKE_OPTIONS >>"$MAKE_LOG" 2>&1 || \
 439                     fail "Failed ZFS 'make $MAKE_OPTIONS'"
 440                 pass_nonewline
 441         fi
 442 done
 443 printf "\n"
 444 echo "-----------------------------------------------------------------"
 445
 446 # Either create a new pool using 'zpool create', or alternately restore an
 447 # existing pool from another ZFS implementation for compatibility testing.
 448 for TAG in $POOL_TAGS; do
 449         pool_set_vars "$TAG"
 450         SKIP=0
 451
 452         printf "%-16s" "$POOL_TAG"
 453         rm -Rf "$POOL_DIR"
 454         mkdir -p "$POOL_DIR_PRISTINE"
 455
 456         # Use the existing compressed image if available.
 457         if [[ -f "$POOL_BZIP" ]]; then
 458                 tar -xjf "$POOL_BZIP" -C "$POOL_DIR_PRISTINE" \
 459                     --strip-components=1 || \
 460                     fail "Failed 'tar -xjf $POOL_BZIP"
 461         # Use the installed version to create the pool.
 462         elif  [[ "$TAG" = "installed" ]]; then
 463                 pool_create "$TAG"
 464         # A source build is available to create the pool.
 465         elif [[ -d "$POOL_DIR_SRC" ]]; then
 466                 pool_create "$TAG"
 467         else
 468                 SKIP=1
 469         fi
 470
 471         # Verify 'zpool import' works for all listed source versions.
 472         for SRC_TAG in $SRC_TAGS; do
 473
 474                 if [[ "$SKIP" -eq 1 ]]; then
 475                         skip_nonewline
 476                         continue
 477                 fi
 478
 479                 src_set_vars "$SRC_TAG"
 480                 if [[ "$SRC_TAG" != "installed" ]]; then
 481                         cd "$ZFS_DIR" || fail "Failed 'cd $ZFS_DIR'"
 482                 fi
 483                 $ZFS_SH zfs="spa_config_path=$POOL_DIR_COPY"
 484
 485                 cp -a --sparse=always "$POOL_DIR_PRISTINE" \
 486                     "$POOL_DIR_COPY" || \
 487                     fail "Failed to copy $POOL_DIR_PRISTINE to $POOL_DIR_COPY"
 488                 POOL_NAME=$($ZPOOL_CMD import -d "$POOL_DIR_COPY" | \
 489                     awk '/pool:/ { print $2; exit }')
 490
 491                 if ! $ZPOOL_CMD import -N -d "$POOL_DIR_COPY"
 492                     "$POOL_NAME" &>/dev/null; then
 493                         fail_nonewline
 494                         ERROR=1
 495                 else
 496                         $ZPOOL_CMD export "$POOL_NAME" || \
 497                             fail "Failed to export pool"
 498                         pass_nonewline
 499                 fi
 500
 501                 rm -Rf "$POOL_DIR_COPY"
 502
 503                 $ZFS_SH -u || fail "Failed to unload kmods"
 504         done
 505         printf "\n"
 506 done
 507
 508 if [[ "$KEEP" = "no" ]]; then
 509         rm -Rf "$TEST_DIR"
 510 fi
 511
 512 exit "$ERROR"