ctdb/tests/UNIT/eventscripts/scripts/60.nfs.sh

   1 setup()
   2 {
   3         setup_public_addresses
   4         setup_shares
   5
   6         # shellcheck disable=SC2034
   7         # Used in expected output
   8         service_name="nfs"
   9
  10         if [ -z "$CTDB_NFS_DISTRO_STYLE" ]; then
  11                 # Currently supported: sysvinit-redhat, systemd-redhat
  12                 CTDB_NFS_DISTRO_STYLE="systemd-redhat"
  13         fi
  14
  15         export FAKE_RPCINFO_SERVICES=""
  16
  17         setup_script_options <<EOF
  18 CTDB_NFS_SKIP_SHARE_CHECK="no"
  19 # This doesn't even need to exist
  20 CTDB_NFS_EXPORTS_FILE="${CTDB_TEST_TMP_DIR}/etc-exports"
  21 EOF
  22
  23         export RPCNFSDCOUNT
  24
  25         if [ "$1" != "down" ]; then
  26                 debug <<EOF
  27 Setting up NFS environment: all RPC services up, NFS managed by CTDB
  28 EOF
  29
  30                 case "$CTDB_NFS_DISTRO_STYLE" in
  31                 sysvinit-*)
  32                         service "nfs" force-started
  33                         service "nfslock" force-started
  34                         ;;
  35                 systemd-*)
  36                         service "nfs-service" force-started
  37                         service "nfs-mountd" force-started
  38                         service "rpc-rquotad" force-started
  39                         service "rpc-statd" force-started
  40                         ;;
  41                 esac
  42
  43                 rpc_services_up \
  44                         "portmapper" "nfs" "mountd" "rquotad" \
  45                         "nlockmgr" "status"
  46
  47                 nfs_setup_fake_threads "nfsd"
  48                 nfs_setup_fake_threads "rpc.foobar" # Set the variable to empty
  49         else
  50                 debug <<EOF
  51 Setting up NFS environment: all RPC services down, NFS not managed by CTDB
  52 EOF
  53
  54                 case "$CTDB_NFS_DISTRO_STYLE" in
  55                 sysvinit-*)
  56                         service "nfs" force-stopped
  57                         service "nfslock" force-stopped
  58                         service "nfs-kernel-server" force-stopped
  59                         ;;
  60                 systemd-*)
  61                         service "nfs-server" force-stopped
  62                         service "nfs-mountd" force-stopped
  63                         service "rpc-quotad" force-stopped
  64                         service "rpc-statd" force-stopped
  65                         ;;
  66                 esac
  67         fi
  68 }
  69
  70 rpc_services_down()
  71 {
  72         _out=""
  73         for _s in $FAKE_RPCINFO_SERVICES; do
  74                 for _i; do
  75                         if [ "$_i" = "${_s%%:*}" ]; then
  76                                 debug "Marking RPC service \"${_i}\" as UNAVAILABLE"
  77                                 continue 2
  78                         fi
  79                 done
  80                 _out="${_out}${_out:+ }${_s}"
  81         done
  82         FAKE_RPCINFO_SERVICES="$_out"
  83 }
  84
  85 rpc_services_up()
  86 {
  87         _out="$FAKE_RPCINFO_SERVICES"
  88         for _i; do
  89                 debug "Marking RPC service \"${_i}\" as available"
  90                 case "$_i" in
  91                 portmapper) _t="2:4" ;;
  92                 nfs) _t="2:3" ;;
  93                 mountd) _t="1:3" ;;
  94                 rquotad) _t="1:2" ;;
  95                 nlockmgr) _t="3:4" ;;
  96                 status) _t="1:1" ;;
  97                 *) die "Internal error - unsupported RPC service \"${_i}\"" ;;
  98                 esac
  99
 100                 _out="${_out}${_out:+ }${_i}:${_t}"
 101         done
 102         export FAKE_RPCINFO_SERVICES="$_out"
 103 }
 104
 105 nfs_setup_fake_threads()
 106 {
 107         _prog="$1"
 108         shift
 109
 110         case "$_prog" in
 111         nfsd)
 112                 export PROCFS_PATH="${CTDB_TEST_TMP_DIR}/proc"
 113                 _threads="${PROCFS_PATH}/fs/nfsd/threads"
 114                 mkdir -p "$(dirname "$_threads")"
 115                 echo $# >"$_threads"
 116                 export FAKE_NFSD_THREAD_PIDS="$*"
 117                 ;;
 118         *)
 119                 export FAKE_RPC_THREAD_PIDS="$*"
 120                 ;;
 121         esac
 122 }
 123
 124 nfs_stats_set_changed()
 125 {
 126         FAKE_NFS_STATS_CHANGED=" $* "
 127 }
 128
 129 nfs_stats_check_changed()
 130 {
 131         _rpc_service="$1"
 132         _iteration="$2"
 133
 134         _t="$FAKE_NFS_STATS_CHANGED"
 135         if [ -z "$_t" ]; then
 136                 return 1
 137         fi
 138         if [ "${_t#* "${_rpc_service}"}" != "$_t" ]; then
 139                 return 0
 140         fi
 141         # Statistics always change on the first iteration
 142         if [ "$_iteration" -eq 1 ]; then
 143                 return 0
 144         fi
 145
 146         return 1
 147 }
 148
 149 guess_output()
 150 {
 151         case "$1" in
 152         "${CTDB_NFS_CALLOUT} start nlockmgr")
 153                 case "$CTDB_NFS_DISTRO_STYLE" in
 154                 sysvinit-redhat)
 155                         echo "&Starting nfslock: OK"
 156                         ;;
 157                 sysvinit-debian)
 158                         cat <<EOF
 159 &Starting nfs-kernel-server: OK
 160 EOF
 161                         ;;
 162                 systemd-*)
 163                         echo "&Starting rpc-statd: OK"
 164                         ;;
 165                 esac
 166                 ;;
 167         "${CTDB_NFS_CALLOUT} start nfs")
 168                 case "$CTDB_NFS_DISTRO_STYLE" in
 169                 sysvinit-redhat)
 170                         cat <<EOF
 171 &Starting nfslock: OK
 172 &Starting nfs: OK
 173 EOF
 174                         ;;
 175                 sysvinit-debian)
 176                         cat <<EOF
 177 &Starting nfs-kernel-server: OK
 178 EOF
 179                         ;;
 180                 systemd-redhat)
 181                         cat <<EOF
 182 &Starting rpc-statd: OK
 183 &Starting nfs-server: OK
 184 &Starting rpc-rquotad: OK
 185 EOF
 186                         ;;
 187                 systemd-debian)
 188                         cat <<EOF
 189 &Starting rpc-statd: OK
 190 &Starting nfs-server: OK
 191 &Starting quotarpc: OK
 192 EOF
 193                         ;;
 194                 esac
 195                 ;;
 196         "${CTDB_NFS_CALLOUT} stop mountd")
 197                 case "$CTDB_NFS_DISTRO_STYLE" in
 198                 systemd-*)
 199                         echo "Stopping nfs-mountd: OK"
 200                         ;;
 201                 esac
 202                 ;;
 203         "${CTDB_NFS_CALLOUT} stop rquotad")
 204                 case "$CTDB_NFS_DISTRO_STYLE" in
 205                 systemd-redhat)
 206                         echo "Stopping rpc-rquotad: OK"
 207                         ;;
 208                 systemd-debian)
 209                         if service "quotarpc" status >/dev/null; then
 210                                 echo "Stopping quotarpc: OK"
 211                         else
 212                                 echo "service: can't stop quotarpc - not running"
 213                         fi
 214                         ;;
 215                 esac
 216                 ;;
 217         "${CTDB_NFS_CALLOUT} stop status")
 218                 case "$CTDB_NFS_DISTRO_STYLE" in
 219                 systemd-*)
 220                         echo "Stopping rpc-statd: OK"
 221                         ;;
 222                 esac
 223                 ;;
 224         "${CTDB_NFS_CALLOUT} start mountd")
 225                 case "$CTDB_NFS_DISTRO_STYLE" in
 226                 systemd-*)
 227                         echo "&Starting nfs-mountd: OK"
 228                         ;;
 229                 esac
 230                 ;;
 231         "${CTDB_NFS_CALLOUT} start rquotad")
 232                 case "$CTDB_NFS_DISTRO_STYLE" in
 233                 systemd-redhat)
 234                         echo "&Starting rpc-rquotad: OK"
 235                         ;;
 236                 systemd-debian)
 237                         echo "&Starting quotarpc: OK"
 238                         ;;
 239                 esac
 240                 ;;
 241         "${CTDB_NFS_CALLOUT} start status")
 242                 case "$CTDB_NFS_DISTRO_STYLE" in
 243                 systemd-*)
 244                         echo "&Starting rpc-statd: OK"
 245                         ;;
 246                 esac
 247                 ;;
 248         *)
 249                 : # Nothing
 250                 ;;
 251         esac
 252 }
 253
 254 # Set the required result for a particular RPC program having failed
 255 # for a certain number of iterations.  This is probably still a work
 256 # in progress.  Note that we could hook aggressively
 257 # nfs_check_rpc_service() to try to implement this but we're better
 258 # off testing nfs_check_rpc_service() using independent code...  even
 259 # if it is incomplete and hacky.  So, if the 60.nfs eventscript
 260 # changes and the tests start to fail then it may be due to this
 261 # function being incomplete.
 262 rpc_set_service_failure_response()
 263 {
 264         _rpc_service="$1"
 265         _numfails="${2:-1}" # default 1
 266
 267         # Default
 268         ok_null
 269         if [ "$_numfails" -eq 0 ]; then
 270                 return
 271         fi
 272
 273         nfs_load_config
 274
 275         # A handy newline.  :-)
 276         _nl="
 277 "
 278
 279         _dir="${CTDB_NFS_CHECKS_DIR:-${CTDB_BASE}/nfs-checks.d}"
 280
 281         _file=$(ls "$_dir"/[0-9][0-9]."${_rpc_service}.check")
 282         [ -r "$_file" ] ||
 283                 die "RPC check file \"$_file\" does not exist or is not unique"
 284
 285         _out="${CTDB_TEST_TMP_DIR}/rpc_failure_output"
 286         : >"$_out"
 287         _rc_file="${CTDB_TEST_TMP_DIR}/rpc_result"
 288
 289         (
 290                 # Subshell to restrict scope variables...
 291
 292                 # Defaults
 293                 # shellcheck disable=SC2034
 294                 # Unused, but for completeness, possible future use
 295                 family="tcp"
 296                 version=""
 297                 unhealthy_after=1
 298                 restart_every=0
 299                 service_stop_cmd=""
 300                 service_start_cmd=""
 301                 # shellcheck disable=SC2034
 302                 # Unused, but for completeness, possible future use
 303                 service_check_cmd=""
 304                 service_debug_cmd=""
 305
 306                 # Don't bother syntax checking, eventscript does that...
 307                 . "$_file"
 308
 309                 # Just use the first version, or use default.  This is
 310                 # dumb but handles all the cases that we care about
 311                 # now...
 312                 if [ -n "$version" ]; then
 313                         _ver="${version%% *}"
 314                 else
 315                         case "$_rpc_service" in
 316                         portmapper) _ver="" ;;
 317                         *) _ver=1 ;;
 318                         esac
 319                 fi
 320                 _rpc_check_out="\
 321 $_rpc_service failed RPC check:
 322 rpcinfo: RPC: Program not registered
 323 program $_rpc_service${_ver:+ version }${_ver} is not available"
 324
 325                 if [ "$_numfails" -eq -1 ]; then
 326                         _unhealthy=false
 327                         echo 0 >"$_rc_file"
 328                         printf 'WARNING: statistics changed but %s\n' \
 329                                 "$_rpc_check_out" >>"$_out"
 330                 elif [ $unhealthy_after -gt 0 ] &&
 331                         [ "$_numfails" -ge $unhealthy_after ]; then
 332                         _unhealthy=true
 333                         echo 1 >"$_rc_file"
 334                         echo "ERROR: ${_rpc_check_out}" >>"$_out"
 335                 else
 336                         _unhealthy=false
 337                         echo 0 >"$_rc_file"
 338                 fi
 339
 340                 if [ $restart_every -gt 0 ] &&
 341                         [ $((_numfails % restart_every)) -eq 0 ]; then
 342                         if ! $_unhealthy; then
 343                                 echo "WARNING: ${_rpc_check_out}" >>"$_out"
 344                         fi
 345
 346                         echo "Trying to restart service \"${_rpc_service}\"..." \
 347                                 >>"$_out"
 348
 349                         guess_output "$service_stop_cmd" >>"$_out"
 350
 351                         if [ -n "$service_debug_cmd" ]; then
 352                                 $service_debug_cmd >>"$_out" 2>&1
 353                         fi
 354
 355                         guess_output "$service_start_cmd" >>"$_out"
 356                 fi
 357         )
 358
 359         read -r _rc <"$_rc_file"
 360         required_result "$_rc" <"$_out"
 361
 362         rm -f "$_out" "$_rc_file"
 363 }
 364
 365 program_stack_traces()
 366 {
 367         _prog="$1"
 368         _max="${2:-1}"
 369
 370         _count=1
 371         if [ "$_prog" = "nfsd" ]; then
 372                 _pids="$FAKE_NFSD_THREAD_PIDS"
 373         else
 374                 _pids="$FAKE_RPC_THREAD_PIDS"
 375         fi
 376         for _pid in $_pids; do
 377                 [ $_count -le "$_max" ] || break
 378
 379                 program_stack_trace "$_prog" "$_pid"
 380                 _count=$((_count + 1))
 381         done
 382 }
 383
 384 # Run an NFS eventscript iteratively.
 385 #
 386 # - 1st argument is the number of iterations.
 387 #
 388 # - 2nd argument is the NFS/RPC service being tested
 389 #
 390 #   This service is marked down before the 1st iteration.
 391 #
 392 #   rpcinfo is then used on each iteration to test the availability of
 393 #   the service.
 394 #
 395 #   If this is not set or null it is assumed all services are healthy
 396 #   and no output or non-zero return codes are generated.  This is
 397 #   useful in baseline tests to confirm that the eventscript and test
 398 #   infrastructure is working correctly.
 399 #
 400 # - 3rd argument is optional iteration on which to bring the RPC
 401 #   service back up
 402 #
 403 nfs_iterate_test()
 404 {
 405         _repeats="$1"
 406         _rpc_service="$2"
 407         _up_iteration="${3:--1}"
 408         if [ -n "$2" ]; then
 409                 shift 2
 410         else
 411                 shift
 412         fi
 413
 414         if [ -n "$_rpc_service" ]; then
 415                 debug <<EOF
 416 --------------------------------------------------
 417 EOF
 418                 rpc_services_down "$_rpc_service"
 419         fi
 420
 421         debug <<EOF
 422 --------------------------------------------------
 423 EOF
 424         # shellcheck disable=SC2154
 425         # Variables defined in define_test()
 426         echo "Running $_repeats iterations of \"$script $event\" $args"
 427
 428         _iterate_failcount=0
 429         for _iteration in $(seq 1 "$_repeats"); do
 430                 if [ -n "$_rpc_service" ]; then
 431                         if [ "$_iteration" = "$_up_iteration" ]; then
 432                                 debug <<EOF
 433 --------------------------------------------------
 434 EOF
 435                                 rpc_services_up "$_rpc_service"
 436                         fi
 437
 438                         if rpcinfo -T tcp localhost "$_rpc_service" \
 439                                 >/dev/null 2>&1; then
 440                                 _iterate_failcount=0
 441                         elif nfs_stats_check_changed \
 442                                 "$_rpc_service" "$_iteration"; then
 443                                 _iterate_failcount=-1
 444                         else
 445                                 # -1 above is a special case of 0:
 446                                 # hack, unhack ;-)
 447                                 if [ $_iterate_failcount -eq -1 ]; then
 448                                         _iterate_failcount=0
 449                                 fi
 450                                 _iterate_failcount=$((_iterate_failcount + 1))
 451                         fi
 452                         rpc_set_service_failure_response \
 453                                 "$_rpc_service" $_iterate_failcount
 454                 fi
 455                 _out=$(simple_test 2>&1)
 456                 _ret=$?
 457                 if "$CTDB_TEST_VERBOSE" || [ $_ret -ne 0 ]; then
 458                         cat <<EOF
 459 ##################################################
 460 Iteration ${_iteration}:
 461 $_out
 462 EOF
 463                 fi
 464                 if [ $_ret -ne 0 ]; then
 465                         exit $_ret
 466                 fi
 467         done
 468 }