wmake/wmakeSchedulerUptime

   1 #!/bin/bash
   2 #------------------------------------------------------------------------------
   3 # =========                 |
   4 # \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
   5 #  \\    /   O peration     |
   6 #   \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
   7 #    \\/     M anipulation  |
   8 #-------------------------------------------------------------------------------
   9 # License
  10 #     This file is part of OpenFOAM.
  11 #
  12 #     OpenFOAM is free software: you can redistribute it and/or modify it
  13 #     under the terms of the GNU General Public License as published by
  14 #     the Free Software Foundation, either version 3 of the License, or
  15 #     (at your option) any later version.
  16 #
  17 #     OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
  18 #     ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  19 #     FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  20 #     for more details.
  21 #
  22 #     You should have received a copy of the GNU General Public License
  23 #     along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
  24 #
  25 # Script
  26 #     wmakeSchedulerUptime
  27 #
  28 # Description
  29 #     Scheduler for network distributed compilations using wmake.
  30 #     - WM_HOSTS contains a list of hosts and number of concurrent processes
  31 #     eg,
  32 #        export WM_HOSTS="hostA:1 hostB:2 hostC:1"
  33 #     - WM_COLOURS contains a list of colours to cycle through
  34 #        export WM_COLOURS="black blue green cyan red magenta yellow"
  35 #
  36 #     Sources the relevant cshrc/bashrc if not set.
  37 #
  38 #     WM_PROJECT_DIR, WM_PROJECT and WM_PROJECT_VERSION will have been set
  39 #     before calling this routine.
  40 #     FOAM_INST_DIR may possibly have been set (to find installation)
  41 #
  42 # Usage
  43 #     wmakeSchedulerUptime COMMAND
  44 #         run 'COMMAND' on one of the slots listed in WM_HOSTS
  45 #
  46 #     wmakeScheduler -count
  47 #         count the total number of slots available in WM_HOSTS
  48 #         eg,  export WM_NCOMPPROCS=$(wmakeScheduler -count)
  49 #
  50 #-------------------------------------------------------------------------------
  51
  52 # csh sets HOST, bash sets HOSTNAME
  53 : ${HOST:=$HOSTNAME}
  54
  55
  56 lockDir=$HOME/.$WM_PROJECT/.wmake
  57 # fallback - 1 core on current host
  58 : ${WM_HOSTS:=$HOST:1}
  59
  60 # count the total number of slots available and exit
  61 if [ "$1" = "-count" ]
  62 then
  63     expr $(
  64         for slotGroup in $WM_HOSTS
  65         do
  66             n=${slotGroup##*:}
  67             [ "$n" = "${slotGroup%%:*}" ] && n=1  # missing ':'
  68             echo "+ ${n:-1}"
  69         done
  70     )
  71     exit 0
  72 fi
  73
  74 # where to source WM_PROJECT settings in a remote shell
  75 # This code tries to figure out which cshrc or bashrc to execute.
  76 # !! Assumes remote computer running same shell and startup files
  77 # in same location
  78
  79 sourceFoam=false    # fallback command
  80 case $SHELL in
  81 */csh | */tcsh )    # [t]csh vs bash|ksh|sh
  82     shellRc=cshrc
  83     ;;
  84 *)
  85     shellRc=bashrc
  86     ;;
  87 esac
  88
  89 # check ~/.$WM_PROJECT/$WM_PROJECT_VERSION/
  90 # check ~/.$WM_PROJECT/
  91 # check <installedProject>/etc/
  92 if [ "$WM_PROJECT" ]
  93 then
  94     for i in \
  95         $HOME/.$WM_PROJECT/$WM_PROJECT_VERSION \
  96         $HOME/.$WM_PROJECT \
  97         $WM_PROJECT_DIR/etc \
  98         ;
  99     do
 100         if [ -f "$i/$shellRc" ]
 101         then
 102             sourceFoam="$i/$shellRc"
 103             break
 104         fi
 105     done
 106 fi
 107
 108 # Construct test string for remote execution.
 109 # Source WM_PROJECT settings if WM_PROJECT environment not set.
 110 # attempt to preserve the installation directory 'FOAM_INST_DIR'
 111 # use FOAM_SETTINGS to pass command-line settings
 112 case $sourceFoam in
 113 */bashrc)
 114     if [ "$FOAM_INST_DIR" ]
 115     then
 116         sourceFoam='[ "$WM_PROJECT" ] || '"FOAM_INST_DIR=$FOAM_INST_DIR . $sourceFoam $FOAM_SETTINGS"
 117     else
 118         sourceFoam='[ "$WM_PROJECT" ] || '". $sourceFoam $FOAM_SETTINGS"
 119     fi
 120     ;;
 121
 122 */cshrc)
 123     # TODO: csh equivalent to bash code (preserving FOAM_INST_DIR)
 124     sourceFoam='if ( ! $?WM_PROJECT ) source '"$sourceFoam $FOAM_SETTINGS"
 125     ;;
 126 esac
 127
 128 # quote double-quotes for remote command line
 129 rcmd=$(echo $* | sed -e s/\"/\'\"\'/g)
 130 ## the same, without forking (not ksh, maybe not /bin/sh either)
 131 # rcmd=$(while [ "$#" -gt 0 ]; do echo "${1//\"/'\"'}"; shift; done)
 132
 133
 134 # Convert WM_COLOURS into an array
 135 declare colourList
 136 nColours=0
 137 for col in $WM_COLOURS
 138 do
 139     colourList[$nColours]=$col
 140     ((nColours = $nColours + 1))
 141 done
 142
 143 # Bashism: make pipe fail early.
 144 # This ensures the return value of the command is returned and not of the
 145 # colouring pipe etc.
 146 set -o pipefail
 147
 148
 149 #
 150 # colour output by argument 1
 151 #
 152 colourPipe()
 153 {
 154     if [ "$1" ]
 155     then
 156     (
 157         while read line
 158         do
 159             setterm -foreground $1
 160             echo "$line"
 161         done
 162         setterm -foreground default
 163     )
 164     else
 165         cat
 166     fi
 167 }
 168
 169
 170
 171 # parse options
 172 nprocs=1
 173 while [ "$#" -gt 0 ]
 174 do
 175     case "$1" in
 176     -np)
 177         shift
 178         nprocs=$1
 179         shift
 180         ;;
 181     -*)
 182         usage "unknown option: '$*'"
 183         ;;
 184     *)
 185         break
 186         ;;
 187     esac
 188 done
 189
 190
 191 colourIndex=0
 192
 193 while :
 194 do
 195     for slotGroup in $WM_HOSTS
 196     do
 197         # split 'host:N', but catch 'host:' and 'host' too
 198         host=${slotGroup%%:*}
 199         n=${slotGroup##*:}
 200         [ "$n" = "$host" ] && n=1  # missing ':'
 201         : ${n:=1}
 202
 203
 204         # Determine load
 205         if [ "$host" = "$HOST" ]; then
 206             stat=`uptime`
 207         else
 208             stat=`ssh $host uptime`
 209         fi
 210         load=`echo "$stat" | sed -e 's/.*average:[^0-9.]*\([0-9.]*\).*/\1/'`
 211
 212
 213         #echo "Machine:$host  load:$load  allowed:$WM_NCOMPPROCS  nprocs:$nprocs"
 214
 215
 216         # Check if adding nprocs to load causes overload
 217         if (echo '' | awk "{if ($load + $nprocs > $WM_NCOMPPROCS) {exit 1}}")
 218         then
 219             if [ "$nColours" -gt 0 ]
 220             then
 221                 # Set colour
 222                 colour="${colourList[$colourIndex]}"
 223                 echo "Machine:$host  Starting:$*"
 224
 225                 if [ "$host" = "$HOST" ]; then
 226                     eval $* 2>&1 | colourPipe "$colour"
 227                 else
 228                     ssh $host "$sourceFoam 2>/dev/null; cd $PWD && $rcmd" 2>&1 | colourPipe "$colour"
 229                 fi
 230                 retval=$?
 231             else
 232                 echo "Machine:$host  Starting:$*"
 233                 if [ "$host" = "$HOST" ]; then
 234                     eval $* 2>&1
 235                 else
 236                     ssh $host "$sourceFoam 2>/dev/null; cd $PWD && $rcmd" 2>&1
 237                 fi
 238                 retval=$?
 239             fi
 240             exit $retval
 241         fi
 242
 243
 244         # Cycle through colours. Note: outside lock clause!
 245         colourIndex=$(expr $colourIndex + 1)
 246         [ "$colourIndex" -lt "$nColours" ] || colourIndex=0
 247
 248     done
 249
 250     # Did not find any free machines. Rest a bit.
 251     #echo "Too much load. Waiting"
 252     sleep 1
 253 done
 254
 255 #------------------------------------------------------------------------------