bin/update/make_incremental_update.sh

   1 #!/bin/bash
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this
   4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 #
   7 # This tool generates incremental update packages for the update system.
   8 # Author: Darin Fisher
   9 #
  10
  11 . $(dirname "$0")/common.sh
  12
  13 # -----------------------------------------------------------------------------
  14
  15 print_usage() {
  16   notice "Usage: $(basename $0) [OPTIONS] ARCHIVE FROMDIR TODIR"
  17   notice ""
  18   notice "The differences between FROMDIR and TODIR will be stored in ARCHIVE."
  19   notice ""
  20   notice "Options:"
  21   notice "  -h  show this help text"
  22   notice "  -f  clobber this file in the installation"
  23   notice "      Must be a path to a file to clobber in the partial update."
  24   notice ""
  25 }
  26
  27 check_for_forced_update() {
  28   force_list="$1"
  29   forced_file_chk="$2"
  30
  31   local f
  32
  33   if [ "$forced_file_chk" = "precomplete" ]; then
  34     ## "true" *giggle*
  35     return 0;
  36   fi
  37
  38   if [ "$forced_file_chk" = "Contents/Resources/precomplete" ]; then
  39     ## "true" *giggle*
  40     return 0;
  41   fi
  42
  43   if [ "$forced_file_chk" = "removed-files" ]; then
  44     ## "true" *giggle*
  45     return 0;
  46   fi
  47
  48   if [ "$forced_file_chk" = "Contents/Resources/removed-files" ]; then
  49     ## "true" *giggle*
  50     return 0;
  51   fi
  52
  53   if [ "${forced_file_chk##*.}" = "chk" ]; then
  54     ## "true" *giggle*
  55     return 0;
  56   fi
  57
  58   for f in $force_list; do
  59     #echo comparing $forced_file_chk to $f
  60     if [ "$forced_file_chk" = "$f" ]; then
  61       ## "true" *giggle*
  62       return 0;
  63     fi
  64   done
  65   ## 'false'... because this is bash. Oh yay!
  66   return 1;
  67 }
  68
  69 if [ $# = 0 ]; then
  70   print_usage
  71   exit 1
  72 fi
  73
  74 requested_forced_updates='Contents/MacOS/firefox'
  75
  76 while getopts "hf:" flag
  77 do
  78    case "$flag" in
  79       h) print_usage; exit 0
  80       ;;
  81       f) requested_forced_updates="$requested_forced_updates $OPTARG"
  82       ;;
  83       ?) print_usage; exit 1
  84       ;;
  85    esac
  86 done
  87
  88 # -----------------------------------------------------------------------------
  89
  90 let arg_start=$OPTIND-1
  91 shift $arg_start
  92
  93 archive="$1"
  94 olddir="$2"
  95 newdir="$3"
  96 # Prevent the workdir from being inside the targetdir so it isn't included in
  97 # the update mar.
  98 if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then
  99   # Remove the /
 100   newdir=$(echo "$newdir" | sed -e 's:\/$::')
 101 fi
 102 workdir="$newdir.work"
 103 updatemanifestv2="$workdir/updatev2.manifest"
 104 updatemanifestv3="$workdir/updatev3.manifest"
 105
 106 mkdir -p "$workdir"
 107 echo "updatev2.manifest" >> $workdir/files.txt
 108 echo "updatev3.manifest" >> $workdir/files.txt
 109
 110 # Generate a list of all files in the target directory.
 111 pushd "$olddir"
 112 if test $? -ne 0 ; then
 113   exit 1
 114 fi
 115
 116 list_files oldfiles
 117 list_dirs olddirs
 118
 119 popd
 120
 121 pushd "$newdir"
 122 if test $? -ne 0 ; then
 123   exit 1
 124 fi
 125
 126 # if [ ! -f "precomplete" ]; then
 127 #   if [ ! -f "Contents/Resources/precomplete" ]; then
 128 #     notice "precomplete file is missing!"
 129 #     exit 1
 130 #   fi
 131 # fi
 132
 133 list_dirs newdirs
 134 list_files newfiles
 135
 136 popd
 137
 138 # Add the type of update to the beginning of the update manifests.
 139 notice ""
 140 notice "Adding type instruction to update manifests"
 141 > $updatemanifestv2
 142 > $updatemanifestv3
 143 notice "       type partial"
 144 echo "type \"partial\"" >> $updatemanifestv2
 145 echo "type \"partial\"" >> $updatemanifestv3
 146
 147 notice ""
 148 notice "Adding file patch and add instructions to update manifests"
 149
 150 num_oldfiles=${#oldfiles[*]}
 151 remove_array=
 152 num_removes=0
 153
 154 for ((i=0; $i<$num_oldfiles; i=$i+1)); do
 155   f="${oldfiles[$i]}"
 156
 157   # If this file exists in the new directory as well, then check if it differs.
 158   if [ -f "$newdir/$f" ]; then
 159
 160     if check_for_add_if_not_update "$f"; then
 161       # The full workdir may not exist yet, so create it if necessary.
 162       mkdir -p `dirname "$workdir/$f"`
 163       $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
 164       copy_perm "$newdir/$f" "$workdir/$f"
 165       make_add_if_not_instruction "$f" "$updatemanifestv3"
 166       echo $f >> $workdir/files.txt
 167       continue 1
 168     fi
 169
 170     if check_for_forced_update "$requested_forced_updates" "$f"; then
 171       # The full workdir may not exist yet, so create it if necessary.
 172       mkdir -p `dirname "$workdir/$f"`
 173       $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
 174       copy_perm "$newdir/$f" "$workdir/$f"
 175       make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1
 176       echo $f >> $workdir/files.txt
 177       continue 1
 178     fi
 179
 180     if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then
 181       # Compute both the compressed binary diff and the compressed file, and
 182       # compare the sizes.  Then choose the smaller of the two to package.
 183       dir=$(dirname "$workdir/$f")
 184       mkdir -p "$dir"
 185       notice "diffing \"$f\""
 186       # MBSDIFF_HOOK represents the communication interface with funsize and,
 187       # if enabled, caches the intermediate patches for future use and
 188       # compute avoidance
 189       #
 190       # An example of MBSDIFF_HOOK env variable could look like this:
 191       # export MBSDIFF_HOOK="myscript.sh -A https://funsize/api -c /home/user"
 192       # where myscript.sh has the following usage:
 193       # myscript.sh -A SERVER-URL [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] \
 194       #   PATH-FROM-URL PATH-TO-URL PATH-PATCH SERVER-URL
 195       #
 196       # Note: patches are bzipped stashed in funsize to gain more speed
 197
 198       # if service is not enabled then default to old behavior
 199       if [ -z "$MBSDIFF_HOOK" ]; then
 200         $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
 201         $BZIP2 -z9 "$workdir/$f.patch"
 202       else
 203         # if service enabled then check patch existence for retrieval
 204         if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then
 205           notice "file \"$f\" found in funsize, diffing skipped"
 206         else
 207           # if not found already - compute it and cache it for future use
 208           $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
 209           $BZIP2 -z9 "$workdir/$f.patch"
 210           $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"
 211         fi
 212       fi
 213       $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
 214       copy_perm "$newdir/$f" "$workdir/$f"
 215       patchfile="$workdir/$f.patch.bz2"
 216       patchsize=$(get_file_size "$patchfile")
 217       fullsize=$(get_file_size "$workdir/$f")
 218
 219       if [ $patchsize -lt $fullsize ]; then
 220         make_patch_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
 221         mv -f "$patchfile" "$workdir/$f.patch"
 222         rm -f "$workdir/$f"
 223         echo $f.patch >> $workdir/files.txt
 224       else
 225         make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
 226         rm -f "$patchfile"
 227         echo $f >> $workdir/files.txt
 228       fi
 229     fi
 230   else
 231     # remove instructions are added after add / patch instructions for
 232     # consistency with make_incremental_updates.py
 233     remove_array[$num_removes]=$f
 234     (( num_removes++ ))
 235   fi
 236 done
 237
 238 # Newly added files
 239 notice ""
 240 notice "Adding file add instructions to update manifests"
 241 num_newfiles=${#newfiles[*]}
 242
 243 for ((i=0; $i<$num_newfiles; i=$i+1)); do
 244   f="${newfiles[$i]}"
 245
 246   # If we've already tested this file, then skip it
 247   for ((j=0; $j<$num_oldfiles; j=$j+1)); do
 248     if [ "$f" = "${oldfiles[j]}" ]; then
 249       continue 2
 250     fi
 251   done
 252
 253   dir=$(dirname "$workdir/$f")
 254   mkdir -p "$dir"
 255
 256   $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
 257   copy_perm "$newdir/$f" "$workdir/$f"
 258
 259   if check_for_add_if_not_update "$f"; then
 260     make_add_if_not_instruction "$f" "$updatemanifestv3"
 261   else
 262     make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
 263   fi
 264
 265
 266   echo $f >> $workdir/files.txt
 267 done
 268
 269 notice ""
 270 notice "Adding file remove instructions to update manifests"
 271 for ((i=0; $i<$num_removes; i=$i+1)); do
 272   f="${remove_array[$i]}"
 273   notice "     remove \"$f\""
 274   echo "remove \"$f\"" >> $updatemanifestv2
 275   echo "remove \"$f\"" >> $updatemanifestv3
 276 done
 277
 278 # Add remove instructions for any dead files.
 279 notice ""
 280 notice "Adding file and directory remove instructions from file 'removed-files'"
 281 append_remove_instructions "$newdir" "$updatemanifestv2" "$updatemanifestv3"
 282
 283 notice ""
 284 notice "Adding directory remove instructions for directories that no longer exist"
 285 num_olddirs=${#olddirs[*]}
 286
 287 for ((i=0; $i<$num_olddirs; i=$i+1)); do
 288   f="${olddirs[$i]}"
 289   # If this dir doesn't exist in the new directory remove it.
 290   if [ ! -d "$newdir/$f" ]; then
 291     notice "      rmdir $f/"
 292     echo "rmdir \"$f/\"" >> $updatemanifestv2
 293     echo "rmdir \"$f/\"" >> $updatemanifestv3
 294   fi
 295 done
 296
 297 $BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
 298 $BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
 299
 300 mar_command="$MAR"
 301 if [[ -n $PRODUCT_VERSION ]]
 302 then
 303   mar_command="$mar_command -V $PRODUCT_VERSION"
 304 fi
 305 if [[ -n $CHANNEL_ID ]]
 306 then
 307   mar_command="$mar_command -H $CHANNEL_ID"
 308 fi
 309 mar_command="$mar_command -C \"$workdir\" -c output.mar -f $workdir/files.txt"
 310 eval "$mar_command"
 311 mv -f "$workdir/output.mar" "$archive"
 312
 313 # cleanup
 314 rm -fr "$workdir"
 315
 316 notice ""
 317 notice "Finished"
 318 notice ""