contrib/server-side/svnmirror.sh

   1 #!/bin/sh
   2 #######################################################################
   3 #
   4 # svnmirror.sh
   5 #
   6 VERSION="0.0.7"
   7 #
   8 # This script syncs changes from a developer repository to a
   9 #             _READONLY_public_repository_
  10 #
  11 # It supports pushing or pulling the changes via ssh and svn tools.
  12 # It is intended to be run manually or from cron.
  13 #
  14 #######################################################################
  15 #
  16 # Things you have to take care of:
  17 #
  18 # 1. You need write access to the directory structure on both boxes
  19 #    for more see the warning at:
  20 #    http://svnbook.red-bean.com/html-chunk/ch06s03.html
  21 #
  22 # 2. For running it from cron i suggest the use of the ssh agent e.g
  23 #    via keychain <http://www.gentoo.org/proj/en/keychain.xml>
  24 #    in general read "man ssh-agent" and "man ssh-keygen".
  25 #
  26 # 3. Do NOT run it from post commit scripts it can lead to broken public
  27 #    repositories!
  28 #
  29 # 4. You do not have to be afraid of the public repos. If the pipe is
  30 #    broken nothing will be committed to it.
  31 #    21:40:47 <@sussman> tberman:  welcome to atomic commits.
  32 #
  33 # 5. For local syncing use "svnadmin hotcopy"
  34 #    see: "svnadmin help hotcopy"
  35 #
  36 #######################################################################
  37 #
  38 # Authors:
  39 #    - Martin Furter <mf@rola.ch>
  40 #    - Marcus Rückert <darix@irssi.org>
  41 #    - Joerg Sonnenberger <joerg@bec.de>
  42 #
  43 #    with suggestions from:
  44 #       - tberman (#svn at freenode)
  45 #         - he actually needed such a script :)
  46 #       - Erik Huelsmann <e.huelsmann@gmx.net>
  47 #         - the little if for remote version :)
  48 #         - status reply
  49 #       - Ben Collins-Sussman <sussman@collab.net>
  50 #         - for some help with the svn commands
  51 #       - Bjørn Magnus Mathisen <epic@generation.no>
  52 #         - for pointing out i forgot to replace one ssh with $LSSH
  53 #       - John Belmonte <john@neggie.net>
  54 #         - for pointing out that we use stderr for a non-error message
  55 #       - Loic Calvez <l.calvez@openchange.org>
  56 #         - filtering
  57 #
  58 # Users:
  59 #    our biggest users atm are Mono Project and MonoDevelop
  60 #    Mono Team currently mirrors the repos every 30 minutes!:)
  61 #    see http://www.mono-project.com/contributing/anonsvn.html
  62 #
  63 #    Openchange.org team is using this script with an excluding filter
  64 #    to synchronise private and public svn server once a day.
  65 #
  66 # License:
  67 #    The same as svn itself. for latest version check:
  68 #    http://svn.collab.net/repos/svn/trunk/subversion/LICENSE
  69 #
  70 # Thanks to the subversion team for their great work.
  71 #
  72 # Links:
  73 #    If you do not like our solution check:
  74 #       - svnpush
  75 #         + http://svn.collab.net/repos/svn/trunk/contrib/client-side/svn-push/svn-push.c
  76 #       - svn replicate
  77 #         + https://open.datacore.ch/read-only/
  78 #       - SVN::Mirror and SVN::Web
  79 #         + http://svn.elixus.org/repos/member/clkao/
  80 #         + http://svn.elixus.org/svnweb/repos/browse/member/clkao/
  81 #
  82 # Changes:
  83 #  0.0.7
  84 #    + allow optional filtering using svndumpfilter
  85 #
  86 #  0.0.6
  87 #    + print a non-error message to stdout instead of stderr.
  88 #  0.0.5
  89 #    + added DUMPPARAMS to specify the params for the svnadmin dump call
  90 #    + added note about svnadmin 1.1 and "--deltas" cmdline option
  91 #    * made the script POSIX sh-compatible
  92 #      (zsh in native-mode does not work atm!)
  93 #    * changed handling of default settings
  94 #    * created config file has all lines commented out now
  95 #    + check if necessary values are set
  96 #    + added note about current users
  97 #    * fixed documentation in the default config
  98 #
  99 #  0.0.4
 100 #    + added comandline switch -v which shows the version.
 101 #    + using markers now to find the config so it's not twice in this script
 102 #    + added/changed some more documentation
 103 #
 104 #  0.0.3
 105 #    + added comandline switches: -C (create config) -f (read config file)
 106 #      -h (help) -q (quiet).
 107 #
 108 #  0.0.2
 109 #    * initial version
 110 #
 111 #######################################################################
 112 #
 113 # uncomment this line for debugging
 114 # set -x
 115 #
 116 #_CONFIG_START_MARKER
 117 #######################################################################
 118 #
 119 # svnmirror default config
 120 #
 121 # Everything starting with "R" refers to the remote host
 122 # with "L" to the local host
 123 #
 124 # the required variables are LREPOS, RREPOS and RHOST
 125 #
 126 #######################################################################
 127 #
 128 # Mode:
 129 #
 130 # push == Mirror from local repository to remote (readonly) repository
 131 # pull == Mirror from remote repository to local (readonly) repository
 132 #
 133 DFLT_MODE="push"
 134
 135 #
 136 # keychain = path to keychain sh file
 137 # If the variable is not set, the script tries to read
 138 # "$HOME/.keychain/`uname -n`-sh", if it is readable.
 139 #
 140 DFLT_KEYCHAIN=""
 141
 142 #
 143 # DUMPPARMS
 144 #
 145 # see "svnadmin help dump" for it.
 146 # default is "--incremental"
 147 #
 148 # if you use svn 1.1 on both you should add "--deltas"
 149 # it should speed up the transfer on slow lines.
 150 #
 151 DFLT_DUMPPARAMS="--incremental"
 152
 153 # Absolute path of svnlook on the local host.
 154 DFLT_LSVNLOOK="/usr/bin/svnlook"
 155
 156 # Absolute path of svnadmin on the local host.
 157 DFLT_LSVNADMIN="/usr/bin/svnadmin"
 158
 159 # Absolute path of svndumpfilter on the local machine.
 160 DFLT_LSVNDUMPFILTER="/usr/bin/svndumpfilter"
 161
 162 # Absolute path to the repository on the local host.
 163 # REQUIRED
 164 DFLT_LREPOS="/path/to/repos"
 165
 166 # Absolute path of ssh on the local host.
 167 #   '-c blowfish -C' to speed up the transfer a bit :)
 168 DFLT_LSSH="/usr/bin/ssh -c blowfish -C"
 169
 170 # Name or IP of the remote host.
 171 # REQUIRED
 172 DFLT_RHOST="host"
 173
 174 # UNIX username on the remote host.
 175 # defaults to the local username.
 176 DFLT_RUSER="user"
 177
 178 # Absolute path of svnlook on the remote host.
 179 DFLT_RSVNLOOK="/usr/bin/svnlook"
 180
 181 # Absolute path of svnadmin on the remote host.
 182 DFLT_RSVNADMIN="/usr/bin/svnadmin"
 183
 184 # Absolute path of svndumpfilter on the local machine.
 185 DFLT_RSVNDUMPFILTER="/usr/bin/svndumpfilter"
 186
 187 # Absolute path to the repository on the remote host.
 188 # REQUIRED
 189 DFLT_RREPOS="/path/to/repos"
 190
 191 # Additional commands you want to run before loading the dump in to the repos
 192 # e.g. setting umask or change group via newgrp.
 193 #
 194 # Make sure the last char is a ";".
 195 #
 196 DFLT_LADDITIONAL=""
 197 DFLT_RADDITIONAL=""
 198
 199 #
 200 # with the filter directive you can optionally include a svndumpfilter
 201 # into the pipe. see svndumpfilter help for more.
 202 #
 203 # examples:
 204 #
 205 # this will skip trunk/foo.conf from syncing
 206 #
 207 # FILTER="exclude trunk/foo.conf"
 208 #
 209 # this filter will only sync trunk
 210 #
 211 # FILTER="include trunk/"
 212 #
 213 #
 214 DFLT_FILTER=""
 215 #_CONFIG_END_MARKER
 216 #######################################################################
 217 #
 218 # create a config file
 219 #
 220 #######################################################################
 221
 222 create_config()
 223 {
 224     CFGFILE="$1"
 225     if [ -f "${CFGFILE}" ]; then
 226         echo "config file '${CFGFILE}' already exists." >&2
 227         exit 1
 228     fi
 229     SVNMIRROR="$0"
 230     if [ ! -f "$0" ]; then
 231         SVNMIRROR=`which "$0"`
 232         if [ $? -ne 0 ]; then
 233             echo "could not locate $0" >&2
 234             exit 1
 235         fi
 236     fi
 237     STARTLINE=`grep -n "^#_CONFIG_START_MARKER" "$SVNMIRROR" | sed 's/:.*$//'`
 238     ENDLINE=`grep -n "^#_CONFIG_END_MARKER" "$SVNMIRROR" | sed 's/:.*$//'`
 239     ENDLINE=`expr ${ENDLINE} - 1`
 240     LINECOUNT=`expr ${ENDLINE} - ${STARTLINE}`
 241     head -n ${ENDLINE} "$SVNMIRROR" | tail -$LINECOUNT | sed -e 's/^#/##/' -e 's/^DFLT_/# /g' | \
 242         sed 's/default config/config/' > "$CFGFILE"
 243 }
 244
 245 #######################################################################
 246 #
 247 # parse the commandline
 248 #
 249 #######################################################################
 250 #
 251 #
 252
 253 show_help()
 254 {
 255     echo ""
 256     echo "usage:"
 257     echo "  svnmirror.sh [-f configfile] [-h] [-q] [-v]"
 258     echo "  svnmirror.sh -C configfile"
 259     echo ""
 260     echo "  -C configfile   create a config file"
 261     echo "  -f configfile   read config from the specified file"
 262     echo "  -h              show this help"
 263     echo "  -q              quiet (-q -q for really quiet)"
 264     echo "  -v              show version"
 265     echo ""
 266     echo "Note: For -f, configfile is relative to the current PATH settings"
 267     echo ""
 268 }
 269
 270 CONFIGREAD=false
 271 QUIET=""
 272 VERBOSE=true
 273 while getopts C:f:hqv OPTION; do
 274     case "$OPTION" in
 275         C)
 276             create_config "${OPTARG}"
 277             exit 0
 278             ;;
 279         f)
 280             if [ ${CONFIGREAD} = true ]; then
 281                 echo "config already read" >&2
 282                 exit 1
 283             elif [ ! -r "${OPTARG}" ]; then
 284                 echo "cannot read config file '${OPTARG}'" >&2
 285                 exit 1
 286             else
 287                 . ${OPTARG}
 288             fi
 289             ;;
 290         h)
 291             show_help
 292             exit 0
 293             ;;
 294         q)
 295             if [ -z "${QUIET}" ]; then
 296                 QUIET="-q"
 297             else
 298                 VERBOSE=false
 299             fi
 300             ;;
 301         v)
 302             echo "svnmirror.sh $VERSION"
 303             exit 0
 304             ;;
 305         \?)
 306             echo "  for help use $0 -h"
 307             exit 1
 308             ;;
 309     esac
 310 done
 311
 312 #######################################################################
 313 #
 314 # add default values
 315 #
 316 #######################################################################
 317 MODE="${MODE:-$DFLT_MODE}"
 318 DUMPPARAMS="${DUMPPARAMS:-$DFLT_DUMPPARAMS}"
 319 LSVNLOOK="${LSVNLOOK:-$DFLT_LSVNLOOK}"
 320 LSVNADMIN="${LSVNADMIN:-$DFLT_LSVNADMIN}"
 321 LSVNDUMPFILTER="${LSVNDUMPFILTER:-$DFLT_LSVNDUMPFILTER}"
 322 LSSH="${LSSH:-$DFLT_LSSH}"
 323 RUSER="${RUSER:+$RUSER@}"
 324 RSVNLOOK="${RSVNLOOK:-$DFLT_RSVNLOOK}"
 325 RSVNADMIN="${RSVNADMIN:-$DFLT_RSVNADMIN}"
 326 RSVNDUMPFILTER="${RSVNDUMPFILTER:-$DFLT_RSVNDUMPFILTER}"
 327 LADDITIONAL="${LADDITIONAL:-$DFLT_LADDITIONAL}"
 328 RADDITIONAL="${RADDITIONAL:-$DFLT_RADDITIONAL}"
 329
 330 echo "${LREPOS:?Required variable LREPOS is not set in config file}" > /dev/null
 331 echo "${RREPOS:?Required variable RREPOS is not set in config file}" > /dev/null
 332 echo "${RHOST:?Required variable RHOST is not set in config file}" > /dev/null
 333
 334 KEYCHAIN="${KEYCHAIN:-$HOME/.keychain/`uname -n`-sh}"
 335 [ -n "${KEYCHAIN}" -a -r "${KEYCHAIN}" ] && . ${KEYCHAIN}
 336
 337 #######################################################################
 338 #
 339 # the actual script
 340 #
 341 #######################################################################
 342 #
 343
 344 #
 345 # getting version of the remote repository
 346 #
 347 RVERSION=`${LSSH} ${RUSER}${RHOST} ${RSVNLOOK} youngest ${RREPOS}`
 348 if [ -z "${RVERSION}" ] ; then
 349     echo "getting version of remote repository failed" >&2
 350     exit 1
 351 fi
 352
 353 #
 354 # getting version of the local repository
 355 #
 356 ${LADDITIONAL}
 357 LVERSION=`${LSVNLOOK} youngest ${LREPOS}`
 358 if [ -z "${LVERSION}" ] ; then
 359     echo "getting version of local repository failed" >&2
 360     exit 1
 361 fi
 362
 363 #
 364 # compare revision numbers
 365 #
 366 if [ ${RVERSION} -eq ${LVERSION} ] ; then
 367     [ ${VERBOSE} = true ] && \
 368         echo "both repositories are already at ${LVERSION}"
 369     exit 0
 370 fi
 371
 372 #
 373 # syncing
 374 #
 375 RC=0
 376 if [ ${MODE} = "push" ] ; then
 377     if [ ${RVERSION} -gt ${LVERSION} ] ; then
 378         echo "revision of remote repos is higher than local one" >&2
 379         exit 1
 380     fi
 381     DUMPFILTER="cat"
 382     if [ -n "$FILTER" ] ; then
 383         DUMPFILTER="${LSVNDUMPFILTER} ${FILTER}"
 384         [ ${VERBOSE} = true ] && echo "using filter 'svndumpfilter ${FILTER}"
 385     fi
 386     REVRANGE="`expr ${RVERSION} + 1`:${LVERSION}"
 387     [ ${VERBOSE} = true ] && \
 388         echo -n "syncing r${REVRANGE} to ${RHOST} ";
 389     ${LSVNADMIN} dump ${QUIET} ${DUMPPARAMS} -r${REVRANGE} ${LREPOS} | ${DUMPFILTER} | \
 390     ${LSSH} ${RUSER}${RHOST} "${RADDITIONAL} ${RSVNADMIN} load ${QUIET} ${RREPOS}" || \
 391     RC=1
 392 elif [ ${MODE} = "pull" ] ; then
 393     if [ ${LVERSION} -gt ${RVERSION} ] ; then
 394         echo "revision of local repos is higher than remote one" >&2
 395         exit 1
 396     fi
 397     DUMPFILTER=""
 398     if [ -n "$FILTER" ] ; then
 399         DUMPFILTER="| ${LSVNDUMPFILTER} ${FILTER}"
 400         [ ${VERBOSE} = true ] && echo "using filter 'svndumpfilter ${FILTER}"
 401     fi
 402     REVRANGE="`expr ${LVERSION} + 1`:${RVERSION}"
 403     [ ${VERBOSE} = true ] && \
 404         echo -n "syncing r${REVRANGE} from ${RHOST} ";
 405     ${LSSH} ${RUSER}${RHOST} \
 406     "${RADDITIONAL} ${RSVNADMIN} dump ${QUIET} ${DUMPPARAMS} -r${REVRANGE} ${RREPOS} ${DUMPFILTER}" | \
 407     ${LSVNADMIN} load ${QUIET} ${LREPOS} || \
 408     RC=1
 409 else
 410     echo "invalid mode \"${MODE}\" specified!" >&2
 411     exit 1
 412 fi
 413 if [ ${RC} -ne 0 ]; then
 414     echo "failed" >&2
 415 else
 416     [ ${VERBOSE} = true ] && \
 417         echo "successfull completed."
 418 fi
 419 exit ${RC}
 420