scripts/zloop.sh

   1 #!/usr/bin/env bash
   2
   3 #
   4 # CDDL HEADER START
   5 #
   6 # This file and its contents are supplied under the terms of the
   7 # Common Development and Distribution License ("CDDL"), version 1.0.
   8 # You may only use this file in accordance with the terms of version
   9 # 1.0 of the CDDL.
  10 #
  11 # A full copy of the text of the CDDL should have accompanied this
  12 # source.  A copy of the CDDL is also available via the Internet at
  13 # http://www.illumos.org/license/CDDL.
  14 #
  15 # CDDL HEADER END
  16 #
  17
  18 #
  19 # Copyright (c) 2015 by Delphix. All rights reserved.
  20 # Copyright (C) 2016 Lawrence Livermore National Security, LLC.
  21 # Copyright (c) 2017, Intel Corporation.
  22 #
  23
  24 BASE_DIR=${0%/*}
  25 SCRIPT_COMMON=common.sh
  26 if [[ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]]; then
  27         . "${BASE_DIR}/${SCRIPT_COMMON}"
  28 else
  29         echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
  30 fi
  31
  32 # shellcheck disable=SC2034
  33 PROG=zloop.sh
  34 GDB=${GDB:-gdb}
  35
  36 DEFAULTWORKDIR=/var/tmp
  37 DEFAULTCOREDIR=/var/tmp/zloop
  38
  39 function usage
  40 {
  41         cat >&2 <<EOF
  42
  43 $0 [-hl] [-c <dump directory>] [-f <vdev directory>]
  44   [-m <max core dumps>] [-s <vdev size>] [-t <timeout>]
  45   [-I <max iterations>] [-- [extra ztest parameters]]
  46
  47   This script runs ztest repeatedly with randomized arguments.
  48   If a crash is encountered, the ztest logs, any associated
  49   vdev files, and core file (if one exists) are moved to the
  50   output directory ($DEFAULTCOREDIR by default). Any options
  51   after the -- end-of-options marker will be passed to ztest.
  52
  53   Options:
  54     -c  Specify a core dump directory to use.
  55     -f  Specify working directory for ztest vdev files.
  56     -h  Print this help message.
  57     -l  Create 'ztest.core.N' symlink to core directory.
  58     -m  Max number of core dumps to allow before exiting.
  59     -s  Size of vdev devices.
  60     -t  Total time to loop for, in seconds. If not provided,
  61         zloop runs forever.
  62     -I  Max number of iterations to loop before exiting.
  63
  64 EOF
  65 }
  66
  67 function or_die
  68 {
  69         if ! "$@"; then
  70                 echo "Command failed: $*"
  71                 exit 1
  72         fi
  73 }
  74
  75 case $(uname) in
  76 FreeBSD)
  77         coreglob="z*.core"
  78         ;;
  79 Linux)
  80         # core file helpers
  81         read -r origcorepattern </proc/sys/kernel/core_pattern
  82         coreglob="$(grep -E -o '^([^|%[:space:]]*)' /proc/sys/kernel/core_pattern)*"
  83
  84         if [[ $coreglob = "*" ]]; then
  85                 echo "Setting core file pattern..."
  86                 echo "core" > /proc/sys/kernel/core_pattern
  87                 coreglob="$(grep -E -o '^([^|%[:space:]]*)' \
  88                     /proc/sys/kernel/core_pattern)*"
  89         fi
  90         ;;
  91 *)
  92         exit 1
  93         ;;
  94 esac
  95
  96 function core_file
  97 {
  98         # shellcheck disable=SC2012,SC2086
  99         ls -tr1 $coreglob 2>/dev/null | head -1
 100 }
 101
 102 function core_prog
 103 {
 104         # shellcheck disable=SC2154
 105         prog=$ZTEST
 106         core_id=$($GDB --batch -c "$1" | grep "Core was generated by" | \
 107             tr  \' ' ')
 108         if [[ "$core_id" == *"zdb "* ]]; then
 109                 # shellcheck disable=SC2154
 110                 prog=$ZDB
 111         fi
 112         printf "%s" "$prog"
 113 }
 114
 115 function store_core
 116 {
 117         core="$(core_file)"
 118         if [[ $ztrc -ne 0 ]] || [[ -f "$core" ]]; then
 119                 df -h "$workdir" >>ztest.out
 120                 coreid=$(date "+zloop-%y%m%d-%H%M%S")
 121                 foundcrashes=$((foundcrashes + 1))
 122
 123                 # zdb debugging
 124                 zdbcmd="$ZDB -U "$workdir/zpool.cache" -dddMmDDG ztest"
 125                 zdbdebug=$($zdbcmd 2>&1)
 126                 echo -e "$zdbcmd\n" >>ztest.zdb
 127                 echo "$zdbdebug" >>ztest.zdb
 128
 129                 dest=$coredir/$coreid
 130                 or_die mkdir -p "$dest/vdev"
 131
 132                 if [[ $symlink -ne 0 ]]; then
 133                         or_die ln -sf "$dest" "ztest.core.${foundcrashes}"
 134                 fi
 135
 136                 echo "*** ztest crash found - moving logs to $dest"
 137
 138                 or_die mv ztest.history ztest.zdb ztest.out "$dest/"
 139                 or_die mv "$workdir/"ztest* "$dest/vdev/"
 140
 141                 if [[ -e "$workdir/zpool.cache" ]]; then
 142                         or_die mv "$workdir/zpool.cache" "$dest/vdev/"
 143                 fi
 144
 145                 # check for core
 146                 if [[ -f "$core" ]]; then
 147                         coreprog=$(core_prog "$core")
 148                         coredebug=$($GDB --batch --quiet \
 149                             -ex "set print thread-events off" \
 150                             -ex "printf \"*\n* Backtrace \n*\n\"" \
 151                             -ex "bt" \
 152                             -ex "printf \"*\n* Libraries \n*\n\"" \
 153                             -ex "info sharedlib" \
 154                             -ex "printf \"*\n* Threads (full) \n*\n\"" \
 155                             -ex "info threads" \
 156                             -ex "printf \"*\n* Backtraces \n*\n\"" \
 157                             -ex "thread apply all bt" \
 158                             -ex "printf \"*\n* Backtraces (full) \n*\n\"" \
 159                             -ex "thread apply all bt full" \
 160                             -ex "quit" "$coreprog" "$core" 2>&1 | \
 161                             grep -v "New LWP")
 162
 163                         # Dump core + logs to stored directory
 164                         echo "$coredebug" >>"$dest/ztest.gdb"
 165                         or_die mv "$core" "$dest/"
 166
 167                         # Record info in cores logfile
 168                         echo "*** core @ $coredir/$coreid/$core:" | \
 169                             tee -a ztest.cores
 170                 fi
 171
 172                 if [[ $coremax -gt 0 ]] &&
 173                    [[ $foundcrashes -ge $coremax ]]; then
 174                         echo "exiting... max $coremax allowed cores"
 175                         exit 1
 176                 else
 177                         echo "continuing..."
 178                 fi
 179         fi
 180 }
 181
 182 # parse arguments
 183 # expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args]
 184 coredir=$DEFAULTCOREDIR
 185 basedir=$DEFAULTWORKDIR
 186 rundir="zloop-run"
 187 timeout=0
 188 size="512m"
 189 coremax=0
 190 symlink=0
 191 iterations=0
 192 while getopts ":ht:m:I:s:c:f:l" opt; do
 193         case $opt in
 194                 t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;;
 195                 m ) [[ $OPTARG -gt 0 ]] && coremax=$OPTARG ;;
 196                 I ) [[ -n $OPTARG ]] && iterations=$OPTARG ;;
 197                 s ) [[ -n $OPTARG ]] && size=$OPTARG ;;
 198                 c ) [[ -n $OPTARG ]] && coredir=$OPTARG ;;
 199                 f ) [[ -n $OPTARG ]] && basedir=$(readlink -f "$OPTARG") ;;
 200                 l ) symlink=1 ;;
 201                 h ) usage
 202                     exit 2
 203                     ;;
 204                 * ) echo "Invalid argument: -$OPTARG";
 205                     usage
 206                     exit 1
 207         esac
 208 done
 209 # pass remaining arguments on to ztest
 210 shift $((OPTIND - 1))
 211
 212 # enable core dumps
 213 ulimit -c unlimited
 214 export ASAN_OPTIONS=abort_on_error=true:halt_on_error=true:allocator_may_return_null=true:disable_coredump=false:detect_stack_use_after_return=true
 215 export UBSAN_OPTIONS=abort_on_error=true:halt_on_error=true:print_stacktrace=true
 216
 217 if [[ -f "$(core_file)" ]]; then
 218         echo -n "There's a core dump here you might want to look at first... "
 219         core_file
 220         echo
 221         exit 1
 222 fi
 223
 224 if [[ ! -d $coredir ]]; then
 225         echo "core dump directory ($coredir) does not exist, creating it."
 226         or_die mkdir -p "$coredir"
 227 fi
 228
 229 if [[ ! -w $coredir ]]; then
 230         echo "core dump directory ($coredir) is not writable."
 231         exit 1
 232 fi
 233
 234 or_die rm -f ztest.history ztest.zdb ztest.cores
 235
 236 ztrc=0          # ztest return value
 237 foundcrashes=0  # number of crashes found so far
 238 starttime=$(date +%s)
 239 curtime=$starttime
 240 iteration=0
 241
 242 # if no timeout was specified, loop forever.
 243 while (( timeout == 0 )) || (( curtime <= (starttime + timeout) )); do
 244         if (( iterations > 0 )) && (( iteration++ == iterations )); then
 245                 break
 246         fi
 247
 248         zopt="-G -VVVVV"
 249
 250         # start each run with an empty directory
 251         workdir="$basedir/$rundir"
 252         or_die rm -rf "$workdir"
 253         or_die mkdir "$workdir"
 254
 255         # ashift range 9 - 15
 256         align=$(((RANDOM % 2) * 3 + 9))
 257
 258         # choose parity value
 259         parity=$(((RANDOM % 3) + 1))
 260
 261         draid_data=0
 262         draid_spares=0
 263
 264         # randomly use special classes
 265         class="special=random"
 266
 267         # choose between four types of configs
 268         # (basic, raidz mix, raidz expansion, and draid mix)
 269         case $((RANDOM % 4)) in
 270
 271         # basic mirror configuration
 272         0)      parity=1
 273                 mirrors=2
 274                 raid_children=0
 275                 vdevs=2
 276                 raid_type="raidz"
 277                 ;;
 278
 279         # fully randomized mirror/raidz (sans dRAID)
 280         1)      mirrors=$(((RANDOM % 3) * 1))
 281                 raid_children=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2)))
 282                 vdevs=$(((RANDOM % 3) + 3))
 283                 raid_type="raidz"
 284                 ;;
 285
 286         # randomized raidz expansion (one top-level raidz vdev)
 287         2)      mirrors=0
 288                 vdevs=1
 289                 # derive initial raidz disk count based on parity choice
 290                 #   P1: 3 - 7 disks
 291                 #   P2: 5 - 9 disks
 292                 #   P3: 7 - 11 disks
 293                 raid_children=$(((RANDOM % 5) + (parity * 2) + 1))
 294
 295                 # 1/3 of the time use a dedicated '-X' raidz expansion test
 296                 if [[ $((RANDOM % 3)) -eq 0 ]]; then
 297                         zopt="$zopt -X -t 16"
 298                         raid_type="raidz"
 299                 else
 300                         raid_type="eraidz"
 301                 fi
 302                 ;;
 303
 304         # fully randomized dRAID (sans mirror/raidz)
 305         3)      mirrors=0
 306                 draid_data=$(((RANDOM % 8) + 3))
 307                 draid_spares=$(((RANDOM % 2) + parity))
 308                 stripe=$((draid_data + parity))
 309                 extra=$((draid_spares + (RANDOM % 4)))
 310                 raid_children=$(((((RANDOM % 4) + 1) * stripe) + extra))
 311                 vdevs=$((RANDOM % 3))
 312                 raid_type="draid"
 313                 ;;
 314         *)
 315                 # avoid shellcheck SC2249
 316                 ;;
 317         esac
 318
 319         zopt="$zopt -K $raid_type"
 320         zopt="$zopt -m $mirrors"
 321         zopt="$zopt -r $raid_children"
 322         zopt="$zopt -D $draid_data"
 323         zopt="$zopt -S $draid_spares"
 324         zopt="$zopt -R $parity"
 325         zopt="$zopt -v $vdevs"
 326         zopt="$zopt -a $align"
 327         zopt="$zopt -C $class"
 328         zopt="$zopt -s $size"
 329         zopt="$zopt -f $workdir"
 330
 331         cmd="$ZTEST $zopt $*"
 332         echo "$(date '+%m/%d %T') $cmd" | tee -a ztest.history ztest.out
 333         $cmd >>ztest.out 2>&1
 334         ztrc=$?
 335         grep -E '===|WARNING' ztest.out >>ztest.history
 336
 337         store_core
 338
 339         curtime=$(date +%s)
 340 done
 341
 342 echo "zloop finished, $foundcrashes crashes found"
 343
 344 # restore core pattern.
 345 case $(uname) in
 346 Linux)
 347         echo "$origcorepattern" > /proc/sys/kernel/core_pattern
 348         ;;
 349 *)
 350         ;;
 351 esac
 352
 353 uptime >>ztest.out
 354
 355 if [[ $foundcrashes -gt 0 ]]; then
 356         exit 1
 357 fi