sdcc-cf/bin/dcf.sh

   1 #!/bin/sh
   2
   3 # dcf.sh - Distributed Compile Farm Mediator
   4 #
   5 # Copyright (c) 2007-2011, Borut Razem <borut dot razem at gmail dot com>
   6 #
   7 # This file is part of sdcc.
   8 #
   9 #  This software is provided 'as-is', without any express or implied
  10 #  warranty.  In no event will the authors be held liable for any damages
  11 #  arising from the use of this software.
  12 #
  13 #  Permission is granted to anyone to use this software for any purpose,
  14 #  including commercial applications, and to alter it and redistribute it
  15 #  freely, subject to the following restrictions:
  16 #
  17 #  1. The origin of this software must not be misrepresented; you must not
  18 #     claim that you wrote the original software. If you use this software
  19 #     in a product, an acknowledgment in the product documentation would be
  20 #     appreciated but is not required.
  21 #  2. Altered source versions must be plainly marked as such, and must not be
  22 #     misrepresented as being the original software.
  23 #  3. This notice may not be removed or altered from any source distribution.
  24
  25 LOG_LINES=1000  # max number of lines in the log file
  26 BWLIMIT=21      # bandwith limit in KiB for rsync
  27
  28 ETC_DIR=$HOME/etc
  29 LOG_DIR=$HOME/log
  30 LOCK_DIR=$HOME/lock
  31
  32 DCF_BUILDER_LIST_FILE=$ETC_DIR/dcf_list
  33 DCF_LOG=$LOG_DIR/dcf.log
  34 DCF_LOCK=$LOCK_DIR/dcf.lock
  35
  36 TREE_FILE=$HOME/tmp/tree.txt
  37
  38 mkdir -p $LOG_DIR $LOCK_DIR
  39
  40 WEBHOST=web.sourceforge.net
  41 WEBUSER=sdcc-builder
  42 WEBHTDOCSDIR=/home/project-web/sdcc/htdocs
  43
  44 FRSHOST=frs.sourceforge.net
  45 FRSUSER=$WEBUSER
  46 FRSDIR=/home/frs/project/sdcc/snapshot_builds
  47
  48
  49 # debugging: print
  50 debug_print ()
  51 # $*: text to print
  52 {
  53   test -n "${DEBUG}" && echo =DBG= $*
  54 }
  55
  56 # debugging: print the command and execute it
  57 debug_exec ()
  58 # $*: command to debug & execute
  59 {
  60   debug_print $*
  61   $*
  62 }
  63
  64 # substring
  65 substr ()
  66 # $1: string
  67 # $2: offset
  68 # $3: length
  69 {
  70   if test "$3" = ""
  71   then
  72     echo "$1" | awk "{print substr(\$0, $2)}"
  73   else
  74     echo "$1" | awk "{print substr(\$0, $2, $3)}"
  75   fi
  76 }
  77
  78
  79 # file type
  80 file_type ()
  81 # $1: premissions string
  82 {
  83   expr "$1" : "^\(.\)[-rwxXst]*$"
  84 }
  85
  86
  87 # generate file tree list
  88 tree ()
  89 # $1: user
  90 # $2: host
  91 # $3: root
  92 # $4: subdir
  93 {
  94   local old_ifs files line file type subdir
  95
  96   files=$(echo "ls -lt" | sftp -b- "$1@$2:$3/$4" | sed -e '/^sftp> /d')
  97   old_ifs=$IFS
  98   IFS='
  99 '
 100   for line in ${files}
 101   do
 102     # get file name from ls -l line
 103     # NOTE: this works only for file names without spaces!
 104     file=$(expr "${line}" : ".*[ ]\([^ ][^ ]*\)$")
 105     type=$(file_type $(substr "${line}" 0 8))
 106     case "${type}" in
 107     d)
 108       if test -z "$4"
 109       then
 110         subdir="${file}"
 111       else
 112         subdir="$4${file}"
 113       fi
 114       echo "${line}/$4"
 115       tree "$1" "$2" "$3" "${subdir}/"
 116       ;;
 117     [-l])
 118       echo "${line}"
 119       ;;
 120     esac
 121   done
 122   IFS="${old_ifs}"
 123 }
 124
 125
 126 # remove files & directories specified in arguments
 127 # relays on find -depth
 128 rm_list ()
 129 {
 130   local file
 131
 132   for file in $*
 133   do
 134     if test -d $file
 135     then
 136       rmdir $file
 137     else
 138       rm -f $file
 139     fi
 140   done
 141 }
 142
 143
 144 # list files & directories specified in arguments
 145 # each file in a new line
 146 list_files ()
 147 {
 148   local file
 149
 150   for file in $*
 151   do
 152     echo "  $file"
 153   done
 154 }
 155
 156
 157 # write the standard input to beginning of the log file
 158 # and truncate it to $LOG_LINES lines
 159 log_it ()
 160 {
 161   local LOG=$(cat)
 162
 163   if test -n "$LOG"
 164   then
 165     if test -e $DCF_LOG -a $(echo "$LOG" | wc -l) -lt $LOG_LINES
 166     then
 167       LOG=$(echo -e "$LOG\n" | cat - $DCF_LOG | head -n $LOG_LINES)
 168     fi
 169     echo "$LOG" > $DCF_LOG
 170   fi
 171 }
 172
 173
 174 # remove more than 7 files in dir from fsr server
 175 rm_old_versions ()
 176 {
 177   local i j k
 178
 179   for i in ${FRSDIR}
 180   do
 181     for j in $(echo "ls -1t $i" | sftp -b- ${FRSUSER}@${FRSHOST} | sed -e '/^sftp> /d')
 182     do
 183       for k in $(echo "ls -1t $j" | sftp -b- ${FRSUSER}@${FRSHOST} | sed -e '/^sftp> /d' | sed -e '1,7d')
 184       do
 185         if [ -n "$k" ]; then echo "removing $k"; echo "rm $k" | sftp -b- ${FRSUSER}@${FRSHOST}; fi
 186       done
 187     done
 188   done
 189
 190   for i in ${WEBHTDOCSDIR}/regression_test_results
 191   do
 192     for j in $(echo "ls -1t $i" | sftp -b- ${WEBUSER}@${WEBHOST} | sed -e '/^sftp> /d')
 193     do
 194       for k in $(echo "ls -1t $j" | sftp -b- ${WEBUSER}@${WEBHOST} | sed -e '/^sftp> /d' | sed -e '1,7d')
 195       do
 196         if [ -n "$k" ]; then echo "removing $k"; echo "rm $k" | sftp -b- ${WEBUSER}@${WEBHOST}; fi
 197       done
 198     done
 199   done
 200
 201   for k in $(echo "ls -1t ${WEBHTDOCSDIR}/changelog_heads" | sftp -b- ${WEBUSER}@${WEBHOST} | sed -e '/^sftp> /d' | sed -e '1,7d')
 202   do
 203     if [ -n "$k" ]; then echo "removing $k"; echo "rm $k" | sftp -b- ${WEBUSER}@${WEBHOST}; fi
 204   done
 205 }
 206
 207
 208 # cleanup the lock file
 209 cleanup ()
 210 {
 211   rm -f $DCF_LOCK
 212 }
 213
 214
 215 # synchronise directory
 216 sync_dir ()
 217 # $1: source directory to sync
 218 # $2: traget machine/directory to sync
 219 # $3: exclude source directories from sync
 220 {
 221   local ret=1 excl file_list
 222
 223   if test -d $1 && pushd $1 > /dev/null
 224   then
 225     if test -z "$3"
 226     then
 227       file_list=$(find * -depth -print 2>/dev/null)
 228     else
 229       file_list=$(find * -depth -print 2>/dev/null | grep -v -e "^$3")
 230     fi
 231     if test -n "${file_list}"
 232     then
 233       echo "+++ start: $(date)"
 234       echo "=== files in $1:"
 235       list_files ${file_list}
 236
 237       excl=""
 238       echo "=== rsyncing..."
 239       if test -n "$3"
 240       then
 241         for dir in $3
 242         do
 243           excl=${excl}" --exclude "${dir}
 244         done
 245       fi
 246       debug_print "current directory: $(pwd)"
 247       debug_exec rsync $RSYNC_OPTS --relative --recursive --include='*.exe' ${excl} -e ssh --size-only * $2 2>&1 | grep -v -e "skipping directory"
 248
 249       echo "=== removing..."
 250       rm_list ${file_list}
 251
 252       echo "=== removing old versions..."
 253       rm_old_versions
 254
 255       echo "--- end: $(date)"
 256
 257       ret=0
 258     fi
 259     popd > /dev/null
 260   fi
 261
 262   return $ret
 263 }
 264
 265 # main procedure
 266 {
 267   trap 'echo dcf.sh caught signal ; cleanup ; exit 1' 1 2 3 13 15
 268
 269   if test -e ${DCF_BUILDER_LIST_FILE}
 270   then
 271     lockfile -r 0 ${DCF_LOCK} || exit 1
 272
 273     test "${BWLIMIT}" != "" && RSYNC_OPTS="${RSYNC_OPTS} --bwlimit=${BWLIMIT}"
 274
 275     rm -f ${TREE_FILE}
 276
 277     {
 278       while read -r builder
 279       do
 280         export builder
 281         export TREE_FILE
 282         (
 283           builder=$(echo ${builder} | sed -e "s/^\(.*\)#.*$/\1/" -e "s/[ \t]*$//")
 284           if test ! -z "${builder}"
 285           then
 286             if sync_dir "/home/${builder}/htdocs/snapshots" ${FRSUSER}@${FRSHOST}:${FRSDIR}/
 287             then
 288               if test ! -e ${TREE_FILE}
 289               then
 290                 tree ${FRSUSER} ${FRSHOST} ${FRSDIR} > ${TREE_FILE}
 291               fi
 292               sync_dir "/home/${builder}/htdocs" ${WEBUSER}@${WEBHOST}:${WEBHTDOCSDIR}/ "snapshots"
 293             fi
 294           fi
 295         )
 296       done
 297     } < ${DCF_BUILDER_LIST_FILE}
 298
 299     if test -e ${TREE_FILE}
 300     then
 301       # upload the new version of ${TREE_FILE}, needed by snap.php to create sdcc snapshots web page
 302       echo "put ${TREE_FILE} ${WEBHTDOCSDIR}/$(basename ${TREE_FILE})" | sftp -b- ${WEBUSER}@${WEBHOST}
 303     fi
 304   fi
 305
 306   cleanup
 307 } 2>&1 | log_it
 308
 309 exit 0