pd_fetch

   1 #!/usr/bin/env bash
   2 #
   3 # Copyright (c) 2007-2009 The PureDarwin Project.
   4 # All rights reserved.
   5 #
   6 # @LICENSE_HEADER_START@
   7 #
   8 # Redistribution and use in source and binary forms, with or without
   9 # modification, are permitted provided that the following conditions
  10 # are met:
  11 # 1. Redistributions of source code must retain the above copyright
  12 #    notice, this list of conditions and the following disclaimer.
  13 # 2. Redistributions in binary form must reproduce the above copyright
  14 #    notice, this list of conditions and the following disclaimer in the
  15 #    documentation and/or other materials provided with the distribution.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
  18 # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  19 # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  20 # PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  21 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  22 # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  23 # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  24 # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  25 # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  26 # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  27 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 #
  29 # @LICENSE_HEADER_END@
  30 #
  31
  32 #
  33 # probono <probono@puredarwin.org>
  34 # aladin   <aladin@puredarwin.org>
  35 #
  36
  37 #
  38 # Downloads and check "vital" BinaryRoots and BinaryDrivers from DarwinBuild website
  39 #
  40
  41 #
  42 # Changelog (deprecated)
  43 #
  44 # 20090326 - Please, see `hg log' or `svn log' from now (Mercurial) - aladin
  45 # 20081219 - Added 9G55 and 9G55pd1 (experimental patched packages) - aladin
  46 # 20081210 - Added 9F33pd1 (experimental packages, our patched binaryroots) - aladin
  47 # 20081209 - License header update - aladin
  48 # 20081115 - Minor updates - aladin
  49 # 20080921 - SHA-1 checksum added - aladin
  50 # 20080831 - Minor updates and fixes - aladin
  51 # 20080606 - Cleaned up - aladin
  52 # 20080528 - Now used in order to download binary roots and drivers - aladin
  53 # 20080527 - Fixed download order, curl and minor bugs - aladin
  54 # 20080305 - Recursive {search|fetch} of binary files from DarwinBuild - aladin
  55 # 20080107 - Initial release - probono
  56 #
  57
  58 ################################################################################
  59 # GLOBAL VARIABLES
  60 ################################################################################
  61
  62 # Find out where this script lives
  63 DIRNAME=$(cd $(dirname $0) && pwd)
  64
  65 # Reading global variables in pd_config file
  66 . "$DIRNAME"/pd_config
  67
  68 ################################################################################
  69 # INITIAL CONDITION
  70 ################################################################################
  71
  72 # Autorun
  73 if [ "$1" = "" ]; then
  74         echo
  75         echo "Fetch "vital" binary files (roots and drivers)."
  76         echo
  77         echo "Legend:"
  78         echo " . project.root.tar.gz\tThe archive has already been retrieved"
  79         echo " + project.root.tar.gz\tThe archive comes from a local darwinbuild package"
  80         echo " * project.root.tar.gz\tThe archive comes from a remote repository"
  81         echo " ? project.root.tar.gz\tThe archive is missing"
  82         #echo " ! project.root.tar.gz\tThe archive is \"corrupted\""
  83
  84         echo
  85         echo "Retrieving binaryroots, please wait..."
  86         echo
  87
  88         # Clone the PureDarwin binary roots repository (Mercurial)
  89         # Source: http://code.google.com/p/puredarwin/source/list?repo=roots
  90         if [ ! -e $BINARYROOTS_DIR ]; then
  91                 hg clone https://roots.puredarwin.googlecode.com/hg/ $TEMPDIR #--debug
  92         else
  93                 cd $TEMPDIR
  94                         hg pull
  95                         hg update #--debug
  96                 cd - > /dev/null
  97         fi
  98
  99         # Recursive call of itself
 100         $0 0 `cat $BINARYROOTS_LIST_FILE $BINARYROOTS_EXTRA_LIST_FILE`
 101         exit 0
 102 fi
 103
 104 ################################################################################
 105 # OTHER VARIABLES
 106 ################################################################################
 107
 108 # Recursive level found in arg1 (loop counter)
 109 LEVEL=$1
 110 # Current build version target taken in BINVERSION array from pd_config
 111 BUILD_VERSION=${BINVERSION[$LEVEL]}
 112 # Verbose mode (or not) for curl
 113 CURL_VERBOSE_FLAG="-s"
 114 #CURL_VERBOSE_FLAG="--progress-bar"
 115 #CURL_VERBOSE_FLAG=""
 116
 117 # Eating args $1
 118 #
 119 #       Shifting LEVEL
 120 #       $# (arg0..argn) is now the list of missing binary files (from pd_roots file)
 121 #
 122 #       At the beginning, all the files are missing
 123 #       At the end, no file should missing
 124 shift
 125
 126 ################################################################################
 127 # RECURSIVE CONDITIONS
 128 ################################################################################
 129
 130 # Break condition: no more missing files found in args, success.
 131 if [ $# -eq 0 ]; then
 132         echo
 133         echo "Fetching binary files complete ($MISSING_LIST_INIT files)"
 134         if [ "$BAD_SHA1_LIST" ]; then
 135                 echo "WARNING! SHA-1 checksum failed on:$BAD_SHA1_LIST"
 136         fi
 137         exit 0
 138 fi
 139
 140 # Break condition: no more level to go in, end of array, some missing files could remain..
 141 if [ $LEVEL -gt $[${#BINVERSION[@]} - 1] ]; then
 142         echo
 143         echo "Fetching binary files uncomplete"
 144         echo "Missing $[$MISSING_LIST_INIT - $[$MISSING_LIST_INIT - $#]] files: $*"
 145         if [ "$BAD_SHA1_LIST" ]; then
 146                 echo "WARNING! SHA-1 checksum failed on:$BAD_SHA1_LIST"
 147         fi
 148         exit -1
 149 fi
 150
 151 # Initialization, level 0 is the beginning
 152 if [ $LEVEL -eq 0 ]; then
 153         # As `cat pd_roots | wc -l` (because all files are missing)
 154         MISSING_LIST_INIT=$#
 155         export BAD_SHA1_LIST=""
 156 fi
 157
 158 ###############################################################################
 159 # MAIN
 160 ###############################################################################
 161
 162 # Matching condition: current build version does belong to the target darwin release
 163 MAJOR_BUILD_VERSION=`echo $BUILD_VERSION | cut -b1-"$(printf "$DARWIN_RELEASE" | wc -c | sed 's/ //g')"`
 164 # Faking major build version for specific build version "pd" (mostly binaryroots for Darwin 9)
 165 if [ "$BUILD_VERSION" = "pd" ]; then
 166         MAJOR_BUILD_VERSION="$DARWIN_RELEASE"
 167 fi
 168 if [ "$DARWIN_RELEASE" =  "$MAJOR_BUILD_VERSION" ]; then
 169
 170         echo
 171         echo "BuildVersion: $BUILD_VERSION ($[$MISSING_LIST_INIT - $#]/$MISSING_LIST_INIT files)"
 172
 173         for x in $*; do
 174
 175                 BINARYROOT_FOUND=""
 176
 177                 ###############################################################
 178                 # LOCAL BINARYROOT
 179                 ###############################################################
 180                 if [ "$BINARYROOT_FOUND" = "" ]; then
 181                         # Test missing binary file existence in the local target dir ($BINARYROOTS_DEFAULT_DIR)
 182                         if [ -e "$BINARYROOTS_DEFAULT_DIR/$BUILD_VERSION/${x}.root.tar.gz" ]; then
 183                                 echo " . ${x}.root.tar.gz"
 184                                 BINARYROOT_FOUND="$BINARYROOTS_DEFAULT_DIR/$BUILD_VERSION/${x}.root.tar.gz"
 185
 186                                 # Verifying SHA1 checksum
 187                                 #T_SHA1=`openssl dgst -sha1 $BINARYROOTS_DEFAULT_DIR/$BUILD_VERSION/${x}.root.tar.gz`
 188                                 #grep "$T_SHA1" pd_sha1  > /dev/null 2> /dev/null
 189                                 #if [ $? -eq 0 ]; then
 190                                         # Found, kept and checked!
 191                                 #       printf " . ${x}.root.tar.gz"
 192                                 #else
 193                                         # Bad checksum or not up to date or unknow!
 194                                 #       echo " ! ${x}.root.tar.gz";
 195                                         # Adding binary to bad SHA-1 binary list
 196                                 #       BAD_SHA1_LIST="$BAD_SHA1_LIST $x"
 197                                 #fi
 198                         fi
 199                 fi
 200
 201                 ###############################################################
 202                 # LOCAL EXTRA BINARYROOT
 203                 ###############################################################
 204                 if [ "$BINARYROOT_FOUND" = "" ]; then
 205                         # Test missing binary file existence in the local build directories
 206                         z=0
 207                         while [ $z -lt $[${#BINARYROOTS_DIR[@]}] ]; do
 208                                 BINARY_DIR=${BINARYROOTS_DIR[$z]}
 209                                 if [ -e "$BINARY_DIR/$BUILD_VERSION/${x}.root.tar.gz" ]; then
 210                                         echo " . ${x}.root.tar.gz"
 211                                         BINARYROOT_FOUND="$BINARY_DIR/$BUILD_VERSION/${x}.root.tar.gz"
 212                                         break # FIXME while stop. / stop at first one found
 213                                 fi
 214                         z=$[$z + 1]
 215                         done # End of WHILE through BINARYROOTS_DIR
 216                 fi
 217
 218                 ###############################################################
 219                 # LOCAL DARWINBUILD BINARYROOT
 220                 ###############################################################
 221                 if [ "$BINARYROOT_FOUND" = "" ]; then
 222                         # Test missing binary file existence in the darwinbuild build directory
 223                         DARWINBUILD_BUILD_DIR_FOUND=""
 224                         for DARWINBUILD_BUILD_DIR in `ls $DARWINBUILD_DIR`; do
 225                                 if [ -e "$DARWINBUILD_DIR/$DARWINBUILD_BUILD_DIR/.build/build" ]; then
 226                                         DARWINBUILD_BUILD_DIR_BUILDVERSION=`cat $DARWINBUILD_DIR/$DARWINBUILD_BUILD_DIR/.build/build 2> /dev/null`
 227                                         if [ "${BINVERSION[$LEVEL]}"  = "$DARWINBUILD_BUILD_DIR_BUILDVERSION" ]; then
 228                                                 # Take the first one # FIXME
 229                                                 DARWINBUILD_BUILD_DIR_FOUND=$DARWINBUILD_DIR/$DARWINBUILD_BUILD_DIR
 230                                                 break
 231                                         fi
 232                                 fi
 233                         done
 234                         if [ ! "$DARWINBUILD_BUILD_DIR_FOUND" = "" ]; then
 235                                 DARWINBUILD_BINARYROOT_FOUND=""
 236                                 if [ -e $DARWINBUILD_BUILD_DIR_FOUND/Roots/.DownloadCache/${x}.root.tar.gz ]; then
 237                                         # Took first from ../Roots/.DownloadCache/.. which content is == fetching it remotely in case this step fail to found
 238                                         DARWINBUILD_BINARYROOT_FOUND="$DARWINBUILD_BUILD_DIR_FOUND/Roots/.DownloadCache/${x}.root.tar.gz"
 239                                 else
 240                                         # Then took from ../Packages/..
 241                                         if [ -e $DARWINBUILD_BUILD_DIR_FOUND/Packages/${x}.root.tar.gz ]; then
 242                                                 DARWINBUILD_BINARYROOT_FOUND="$DARWINBUILD_BUILD_DIR_FOUND/Packages/${x}.root.tar.gz"
 243                                         fi
 244                                 fi
 245                         fi
 246                         if [ ! $DARWINBUILD_BINARYROOT_FOUND = "" ]; then
 247                                 echo " + ${x}.root.tar.gz"
 248                                 BINARYROOT_FOUND=$DARWINBUILD_BINARYROOT_FOUND
 249                         fi
 250                 fi
 251
 252                 ###############################################################
 253                 # REMOTE (DARWINBUILD) BINARYROOT
 254                 ###############################################################
 255                 if [ "$BINARYROOT_FOUND" = "" ]; then
 256                         # Iterative loop in BINARYROOTS_HTTP_PATH
 257                         y=0
 258                         while [ $y -lt $[${#BINARYROOTS_HTTP_PATH[@]}] ]; do
 259                                 BINARY_HTTP_PATH=${BINARYROOTS_HTTP_PATH[$y]}
 260
 261                                 # PSEUDO PREFETCH
 262                                 # FIXME add switch or rm to flush the cache or.. test how it behaves before and after an updated page..
 263                                 PSEUDOCACHE_FILE="$TEMPDIR/$(basename $BINARY_HTTP_PATH)/cache-$BUILD_VERSION"
 264                                 if [ ! -e $PSEUDOCACHE_FILE ]; then
 265                                         curl -f $CURL_VERBOSE_FLAG --create-dir --output "$PSEUDOCACHE_FILE" -C -  "$BINARY_HTTP_PATH/$BUILD_VERSION/"
 266                                         if [ ! $? -eq 0 ]; then
 267                                                 touch $PSEUDOCACHE_FILE # create an empty one for next time..
 268                                         fi
 269                                 fi
 270
 271                                 # Test missing binary file existence in the local binary file list cache
 272                                 grep "\"${x}.root.tar.gz\"" $PSEUDOCACHE_FILE > /dev/null 2> /dev/null
 273                                 if [ $? -eq 0 ]; then
 274                                         # Fetching missing or corrupted binary file into $BINARYROOTS_DEFAULT_DIR
 275                                         curl -C -f --create-dir $CURL_VERBOSE_FLAG --output "$BINARYROOTS_DEFAULT_DIR/$BUILD_VERSION/${x}.root.tar.gz" -C - "$BINARY_HTTP_PATH/$BUILD_VERSION/${x}.root.tar.gz"
 276                                         if [ $? -eq 0 ]; then
 277                                                 # Found remote archive
 278
 279                                                 # Verifying SHA1 checksum
 280                                                 #T_SHA1=`openssl dgst -sha1 $BINARYROOTS_DEFAULT_DIR/$BUILD_VERSION/${x}.root.tar.gz`
 281                                                 #grep "$T_SHA1" pd_sha1  > /dev/null 2> /dev/null
 282                                                 #if [ $? -eq 0 ]; then
 283                                                         # Found, kept and checked!
 284                                                         echo " * ${x}.root.tar.gz" # ($y)
 285                 #                                       # Removing old version since a new one has been fetch
 286                 #                                       T_LEVEL=$[$LEVEL + 1]
 287                 #                                       while [ $T_LEVEL -lt $[${#BINVERSION[@]} - 1] ]; do
 288                 #                                               ####rm -f $BINARYROOTS_DEFAULT_DIR/${BINVERSION[$T_LEVEL]}/${x}.root.tar.gz
 289                 #                                               T_LEVEL=$[$T_LEVEL + 1]
 290                 #                                       done
 291                 #                               else
 292                                                         # Bad checksum!
 293                 #                                       echo " ! ${x}.root.tar.gz";
 294                                                         # Adding binary to bad SHA-1 binary list
 295                 #                                       BAD_SHA1_LIST=$x
 296                                                 #fi
 297
 298                                                 BINARYROOT_FOUND="$BINARYROOTS_DEFAULT_DIR/$BUILD_VERSION/${x}.root.tar.gz"
 299                                                 break # This breaks the WHILE through BINARYROOTS_HTTP_PATH
 300                                         fi
 301                                 fi
 302                                 y=$[$y + 1] # next element in BINARYROOTS_HTTP_PATH
 303                         done # End of WHILE through BINARYROOTS_HTTP_PATH
 304                 fi
 305
 306                 ###############################################################
 307                 # MISSING BINARYROOT
 308                 ###############################################################
 309                 if [ "$BINARYROOT_FOUND" = "" ]; then
 310                         echo " ? ${x}.root.tar.gz";
 311                         #####rm -f $BINARYROOTS_DEFAULT_DIR/$BUILD_VERSION/${x}.root.tar.gz
 312                         # Adding missing binary to missing binary list
 313                         MISSING_LIST="${MISSING_LIST} ${x}"
 314                 fi
 315
 316         done # End of FOR through missing binaryroots list (MISSING_LIST)
 317 else
 318         #echo "Skipping. Major build version ($MAJOR_BUILD_VERSION (build ver: $BUILD_VERSION)) does not match Major Darwin release ($DARWIN_RELEASE)."
 319         # Adding initial missing binary to missing binary list since no current BuildVersion directory has been found
 320         MISSING_LIST="$*"
 321 fi
 322
 323 ################################################################################
 324 # RECURSIVE CALL
 325 ################################################################################
 326
 327 # Recursive call + propagating MISSING_LIST_INIT in env
 328 MISSING_LIST_INIT="$MISSING_LIST_INIT" $0 $[$LEVEL + 1] $MISSING_LIST
 329
 330 exit 0