ctdb/tools/onnode

   1 #!/usr/bin/env bash
   2
   3 # Run commands on CTDB nodes.
   4
   5 # See http://ctdb.samba.org/ for more information about CTDB.
   6
   7 # Copyright (C) Martin Schwenke  2008
   8
   9 # Based on an earlier script by Andrew Tridgell and Ronnie Sahlberg.
  10
  11 # Copyright (C) Andrew Tridgell  2007
  12
  13 # This program is free software; you can redistribute it and/or modify
  14 # it under the terms of the GNU General Public License as published by
  15 # the Free Software Foundation; either version 3 of the License, or
  16 # (at your option) any later version.
  17
  18 # This program is distributed in the hope that it will be useful,
  19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21 # GNU General Public License for more details.
  22
  23 # You should have received a copy of the GNU General Public License
  24 # along with this program; if not, see <http://www.gnu.org/licenses/>.
  25
  26 prog=$(basename "$0")
  27
  28 usage ()
  29 {
  30     cat >&2 <<EOF
  31 Usage: onnode [OPTION] ... <NODES> <COMMAND> ...
  32   options:
  33     -c          Run in current working directory on specified nodes.
  34     -f          Specify nodes file, overriding default.
  35     -i          Keep standard input open - the default is to close it.
  36     -n          Allow nodes to be specified by name.
  37     -p          Run command in parallel on specified nodes.
  38     -P          Push given files to nodes instead of running commands.
  39     -q          Do not print node addresses (overrides -v).
  40     -v          Print node address even for a single node.
  41   <NODES>       "all", "any", "ok" (or "healthy"), "con" (or "connected") ; or
  42                 a node number (0 base); or
  43                 a hostname (if -n is specified); or
  44                 list (comma separated) of <NODES>; or
  45                 range (hyphen separated) of node numbers.
  46 EOF
  47     exit 1
  48
  49 }
  50
  51 invalid_nodespec ()
  52 {
  53     echo "Invalid <nodespec>" >&2 ; echo >&2
  54     usage
  55 }
  56
  57 # Defaults.
  58 current=false
  59 ctdb_nodes_file=""
  60 parallel=false
  61 verbose=false
  62 quiet=false
  63 names_ok=false
  64 push=false
  65 stdin=false
  66
  67 if [ -z "$CTDB_BASE" ] ; then
  68     CTDB_BASE="/usr/local/etc/ctdb"
  69 fi
  70
  71 parse_options ()
  72 {
  73         local opt
  74
  75         while getopts "cf:hnpqvPi?" opt ; do
  76                 case "$opt" in
  77                 c) current=true ;;
  78                 f) ctdb_nodes_file="$OPTARG" ;;
  79                 n) names_ok=true ;;
  80                 p) parallel=true ;;
  81                 q) quiet=true ;;
  82                 v) verbose=true ;;
  83                 P) push=true ;;
  84                 i) stdin=true ;;
  85                 \?|h) usage ;;
  86                 esac
  87         done
  88         shift $((OPTIND - 1))
  89
  90         if [ $# -lt 2 ] ; then
  91                 usage
  92         fi
  93
  94         nodespec="$1" ; shift
  95         command="$*"
  96 }
  97
  98 echo_nth ()
  99 {
 100     local n="$1" ; shift
 101
 102     # Note that this is 0-based
 103     local node=""
 104     if [ "$n" -le $# ] ; then
 105             shift "$n"
 106             node="$1"
 107     fi
 108
 109     if [ -n "$node" ] && [ "$node" != "#DEAD" ] ; then
 110         echo "$node"
 111     else
 112         echo "${prog}: \"node ${n}\" does not exist" >&2
 113         exit 1
 114     fi
 115 }
 116
 117 parse_nodespec ()
 118 {
 119     # Subshell avoids hacks to restore $IFS.
 120     (
 121         IFS=","
 122         for i in $1 ; do
 123             case "$i" in
 124                 *-*) seq "${i%-*}" "${i#*-}" 2>/dev/null || invalid_nodespec ;;
 125                 all|any|ok|healthy|con|connected) echo "$i" ;;
 126                 *)
 127                     [ "$i" -gt -1 ] 2>/dev/null || $names_ok || invalid_nodespec
 128                     echo "$i"
 129             esac
 130         done
 131     )
 132 }
 133
 134 ctdb_status_output="" # cache
 135 get_nodes_with_status ()
 136 {
 137     local all_nodes="$1"
 138     local status="$2"
 139
 140     if [ -z "$ctdb_status_output" ] ; then
 141         ctdb_status_output=$(ctdb -X status 2>&1)
 142         # No! Checking the exit code afterwards is actually clearer...
 143         # shellcheck disable=SC2181
 144         if [ $? -ne 0 ] ; then
 145             echo "${prog}: unable to get status of CTDB nodes" >&2
 146             echo "$ctdb_status_output" >&2
 147             exit 1
 148         fi
 149         local nl="
 150 "
 151         ctdb_status_output="${ctdb_status_output#*"${nl}"}"
 152     fi
 153
 154     (
 155         local i
 156         IFS="${IFS}|"
 157         while IFS="" read -r i ; do
 158
 159             # Intentional word splitting
 160             # shellcheck disable=SC2086
 161             set -- $i # split line on colons
 162             shift     # line starts with : so 1st field is empty
 163             local pnn="$1" ; shift
 164             shift # ignore IP address but need status bits below
 165
 166             case "$status" in
 167                 healthy)
 168                     # If any bit is 1, don't match this address.
 169                     local s
 170                     for s ; do
 171                         [ "$s" != "1" ] || continue 2
 172                     done
 173                     ;;
 174                 connected)
 175                     # If disconnected bit is not 0, don't match this address.
 176                     [ "$1" = "0" ] || continue
 177                     ;;
 178                 *)
 179                     invalid_nodespec
 180             esac
 181
 182             # Intentional multi-word expansion
 183             # shellcheck disable=SC2086
 184             echo_nth "$pnn" $all_nodes
 185         done <<<"$ctdb_status_output"
 186     )
 187 }
 188
 189 get_any_available_node ()
 190 {
 191     local all_nodes="$1"
 192
 193     # We do a recursive onnode to find which nodes are up and running.
 194     local out line
 195     out=$("$0" -pq all ctdb pnn 2>&1)
 196     while read -r line ; do
 197         if [[ "$line" =~ ^[0-9]+$ ]] ; then
 198             local pnn="$line"
 199             # Intentional multi-word expansion
 200             # shellcheck disable=SC2086
 201             echo_nth "$pnn" $all_nodes
 202             return 0
 203         fi
 204         # Else must be an error message from a down node.
 205     done <<<"$out"
 206     return 1
 207 }
 208
 209 get_nodes ()
 210 {
 211         local all_nodes
 212
 213         local f="${CTDB_BASE}/nodes"
 214         if [ -n "$ctdb_nodes_file" ] ; then
 215                 f="$ctdb_nodes_file"
 216                 if [ ! -e "$f" ] && [ "${f#/}" = "$f" ] ; then
 217                         # $f is relative, try in $CTDB_BASE
 218                         f="${CTDB_BASE}/${f}"
 219                 fi
 220         fi
 221
 222         if [ ! -r "$f" ] ; then
 223                 echo "${prog}: unable to open nodes file  \"${f}\"" >&2
 224                 exit 1
 225         fi
 226
 227         all_nodes=$(sed -e 's@#.*@@g' -e 's@ *@@g' -e 's@^$@#DEAD@' "$f")
 228
 229         local n nodes
 230         nodes=$(parse_nodespec "$1") || exit $?
 231         for n in $nodes ; do
 232                 case "$n" in
 233                 all)
 234                         echo "${all_nodes//#DEAD/}"
 235                         ;;
 236                 any)
 237                         get_any_available_node "$all_nodes" || exit 1
 238                         ;;
 239                 ok|healthy)
 240                         get_nodes_with_status "$all_nodes" "healthy" || exit 1
 241                         ;;
 242                 con|connected)
 243                         get_nodes_with_status "$all_nodes" "connected" || exit 1
 244                         ;;
 245                 [0-9]|[0-9][0-9]|[0-9][0-9][0-9])
 246                         # Intentional multi-word expansion
 247                         # shellcheck disable=SC2086
 248                         echo_nth "$n" $all_nodes
 249                         ;;
 250                 *)
 251                         $names_ok || invalid_nodespec
 252                         echo "$n"
 253                 esac
 254         done
 255 }
 256
 257 # shellcheck disable=SC2317
 258 # push() called indirectly via $ONNODE_SSH
 259 push ()
 260 {
 261         local host="$1"
 262         local files="$2"
 263
 264         local f
 265         for f in $files ; do
 266                 $verbose && echo "Pushing $f"
 267                 case "$f" in
 268                 /*) rsync "$f" "[${host}]:${f}" ;;
 269                 *)  rsync "${PWD}/${f}" "[${host}]:${PWD}/${f}" ;;
 270                 esac
 271         done
 272 }
 273
 274 ######################################################################
 275
 276 parse_options "$@"
 277
 278 ssh_opts=
 279 if $push ; then
 280         if [ -n "$ONNODE_SSH" ] ; then
 281                 export RSYNC_RSH="$ONNODE_SSH"
 282         fi
 283         ONNODE_SSH=push
 284 else
 285         $current && command="cd $PWD && $command"
 286
 287         # Could "2>/dev/null || true" but want to see errors from typos in file.
 288         [ -r "${CTDB_BASE}/onnode.conf" ] && . "${CTDB_BASE}/onnode.conf"
 289         [ -n "$ONNODE_SSH" ] || ONNODE_SSH=ssh
 290         # $ONNODE_SSH must accept the -n option - it can be ignored!
 291         if $parallel || ! $stdin ; then
 292                 ssh_opts="-n"
 293         fi
 294 fi
 295
 296 ######################################################################
 297
 298 nodes=$(get_nodes "$nodespec") || exit $?
 299
 300 if $quiet ; then
 301     verbose=false
 302 else
 303     # If $nodes contains a space or a newline then assume multiple nodes.
 304     nl="
 305 "
 306     [ "$nodes" != "${nodes%[ "${nl}"]*}" ] && verbose=true
 307 fi
 308
 309 pids=""
 310 # Intentional multi-word expansion
 311 # shellcheck disable=SC2086
 312 trap 'kill -TERM $pids 2>/dev/null' INT TERM
 313 # There's a small race here where the kill can fail if no processes
 314 # have been added to $pids and the script is interrupted.  However,
 315 # the part of the window where it matter is very small.
 316 retcode=0
 317 for n in $nodes ; do
 318         set -o pipefail 2>/dev/null
 319
 320         ssh_cmd="$ONNODE_SSH $ssh_opts"
 321         if $parallel ; then
 322                 if $verbose ; then
 323                         $ssh_cmd "$n" "$command" 2>&1 | sed -e "s@^@[$n] @"
 324                 else
 325                         $ssh_cmd "$n" "$command"
 326                 fi &
 327                 pids="${pids} $!"
 328         else
 329                 if $verbose ; then
 330                         echo >&2 ; echo ">> NODE: $n <<" >&2
 331                 fi
 332                 {
 333                         $ssh_cmd "$n" "$command"
 334                 } || retcode=$?
 335         fi
 336 done
 337
 338 if $parallel ; then
 339         for p in $pids; do
 340                 wait "$p" || retcode=$?
 341         done
 342 fi
 343
 344 exit $retcode