tools/resources/optimize-png-files.sh

   1 #!/bin/bash -i
   2 # Copyright 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 # The optimization code is based on pngslim (http://goo.gl/a0XHg)
   7 # and executes a similar pipleline to optimize the png file size.
   8 # The steps that require pngoptimizercl/pngrewrite/deflopt are omitted,
   9 # but this runs all other processes, including:
  10 # 1) various color-dependent optimizations using optipng.
  11 # 2) optimize the number of huffman blocks.
  12 # 3) randomize the huffman table.
  13 # 4) Further optimize using optipng and advdef (zlib stream).
  14 # Due to the step 3), each run may produce slightly different results.
  15 #
  16 # Note(oshima): In my experiment, advdef didn't reduce much. I'm keeping it
  17 # for now as it does not take much time to run.
  18
  19 readonly ALL_DIRS="
  20 ash/resources
  21 chrome/android/java/res
  22 chrome/app/theme
  23 chrome/browser/resources
  24 chrome/renderer/resources
  25 component/resources
  26 content/public/android/java/res
  27 content/app/resources
  28 content/renderer/resources
  29 content/shell/resources
  30 remoting/resources
  31 ui/resources
  32 ui/chromeos/resources
  33 ui/webui/resources/images
  34 win8/metro_driver/resources
  35 "
  36
  37 # Files larger than this file size (in bytes) will
  38 # use the optimization parameters tailored for large files.
  39 LARGE_FILE_THRESHOLD=3000
  40
  41 # Constants used for optimization
  42 readonly DEFAULT_MIN_BLOCK_SIZE=128
  43 readonly DEFAULT_LIMIT_BLOCKS=256
  44 readonly DEFAULT_RANDOM_TRIALS=100
  45 # Taken from the recommendation in the pngslim's readme.txt.
  46 readonly LARGE_MIN_BLOCK_SIZE=1
  47 readonly LARGE_LIMIT_BLOCKS=2
  48 readonly LARGE_RANDOM_TRIALS=1
  49
  50 # Global variables for stats
  51 TOTAL_OLD_BYTES=0
  52 TOTAL_NEW_BYTES=0
  53 TOTAL_FILE=0
  54 CORRUPTED_FILE=0
  55 PROCESSED_FILE=0
  56
  57 declare -a THROBBER_STR=('-' '\\' '|' '/')
  58 THROBBER_COUNT=0
  59
  60 VERBOSE=false
  61
  62 # Echo only if verbose option is set.
  63 function info {
  64   if $VERBOSE ; then
  65     echo $@
  66   fi
  67 }
  68
  69 # Show throbber character at current cursor position.
  70 function throbber {
  71   info -ne "${THROBBER_STR[$THROBBER_COUNT]}\b"
  72   let THROBBER_COUNT=$THROBBER_COUNT+1
  73   let THROBBER_COUNT=$THROBBER_COUNT%4
  74 }
  75
  76 # Usage: pngout_loop <file> <png_out_options> ...
  77 # Optimize the png file using pngout with the given options
  78 # using various block split thresholds and filter types.
  79 function pngout_loop {
  80   local file=$1
  81   shift
  82   local opts=$*
  83   if [ $OPTIMIZE_LEVEL == 1 ]; then
  84     for j in $(eval echo {0..5}); do
  85       throbber
  86       pngout -q -k1 -s1 -f$j $opts $file
  87     done
  88   else
  89     for i in 0 128 256 512; do
  90       for j in $(eval echo {0..5}); do
  91         throbber
  92         pngout -q -k1 -s1 -b$i -f$j $opts $file
  93       done
  94     done
  95   fi
  96 }
  97
  98 # Usage: get_color_depth_list
  99 # Returns the list of color depth options for current optimization level.
 100 function get_color_depth_list {
 101   if [ $OPTIMIZE_LEVEL == 1 ]; then
 102     echo "-d0"
 103   else
 104     echo "-d1 -d2 -d4 -d8"
 105   fi
 106 }
 107
 108 # Usage: process_grayscale <file>
 109 # Optimize grayscale images for all color bit depths.
 110 #
 111 # TODO(oshima): Experiment with -d0 w/o -c0.
 112 function process_grayscale {
 113   info -ne "\b\b\b\b\b\b\b\bgray...."
 114   for opt in $(get_color_depth_list); do
 115     pngout_loop $file -c0 $opt
 116   done
 117 }
 118
 119 # Usage: process_grayscale_alpha <file>
 120 # Optimize grayscale images with alpha for all color bit depths.
 121 function process_grayscale_alpha {
 122   info -ne "\b\b\b\b\b\b\b\bgray-a.."
 123   pngout_loop $file -c4
 124   for opt in $(get_color_depth_list); do
 125     pngout_loop $file -c3 $opt
 126   done
 127 }
 128
 129 # Usage: process_rgb <file>
 130 # Optimize rgb images with or without alpha for all color bit depths.
 131 function process_rgb {
 132   info -ne "\b\b\b\b\b\b\b\brgb....."
 133   for opt in $(get_color_depth_list); do
 134     pngout_loop $file -c3 $opt
 135   done
 136   pngout_loop $file -c2
 137   pngout_loop $file -c6
 138 }
 139
 140 # Usage: huffman_blocks <file>
 141 # Optimize the huffman blocks.
 142 function huffman_blocks {
 143   info -ne "\b\b\b\b\b\b\b\bhuffman."
 144   local file=$1
 145   local size=$(stat -c%s $file)
 146   local min_block_size=$DEFAULT_MIN_BLOCK_SIZE
 147   local limit_blocks=$DEFAULT_LIMIT_BLOCKS
 148
 149   if [ $size -gt $LARGE_FILE_THRESHOLD ]; then
 150     min_block_size=$LARGE_MIN_BLOCK_SIZE
 151     limit_blocks=$LARGE_LIMIT_BLOCKS
 152   fi
 153   let max_blocks=$size/$min_block_size
 154   if [ $max_blocks -gt $limit_blocks ]; then
 155     max_blocks=$limit_blocks
 156   fi
 157
 158   for i in $(eval echo {2..$max_blocks}); do
 159     throbber
 160     pngout -q -k1 -ks -s1 -n$i $file
 161   done
 162 }
 163
 164 # Usage: random_huffman_table_trial <file>
 165 # Try compressing by randomizing the initial huffman table.
 166 #
 167 # TODO(oshima): Try adjusting different parameters for large files to
 168 # reduce runtime.
 169 function random_huffman_table_trial {
 170   info -ne "\b\b\b\b\b\b\b\brandom.."
 171   local file=$1
 172   local old_size=$(stat -c%s $file)
 173   local trials_count=$DEFAULT_RANDOM_TRIALS
 174
 175   if [ $old_size -gt $LARGE_FILE_THRESHOLD ]; then
 176     trials_count=$LARGE_RANDOM_TRIALS
 177   fi
 178   for i in $(eval echo {1..$trials_count}); do
 179     throbber
 180     pngout -q -k1 -ks -s0 -r $file
 181   done
 182   local new_size=$(stat -c%s $file)
 183   if [ $new_size -lt $old_size ]; then
 184     random_huffman_table_trial $file
 185   fi
 186 }
 187
 188 # Usage: final_comprssion <file>
 189 # Further compress using optipng and advdef.
 190 # TODO(oshima): Experiment with 256.
 191 function final_compression {
 192   info -ne "\b\b\b\b\b\b\b\bfinal..."
 193   local file=$1
 194   if [ $OPTIMIZE_LEVEL == 2 ]; then
 195     for i in 32k 16k 8k 4k 2k 1k 512; do
 196       throbber
 197       optipng -q -nb -nc -zw$i -zc1-9 -zm1-9 -zs0-3 -f0-5 $file
 198     done
 199   fi
 200   for i in $(eval echo {1..4}); do
 201     throbber
 202     advdef -q -z -$i $file
 203   done
 204
 205   # Clear the current line.
 206   if $VERBOSE ; then
 207     printf "\033[0G\033[K"
 208   fi
 209 }
 210
 211 # Usage: get_color_type <file>
 212 # Returns the color type name of the png file. Here is the list of names
 213 # for each color type codes.
 214 # 0: grayscale
 215 # 2: RGB
 216 # 3: colormap
 217 # 4: gray+alpha
 218 # 6: RGBA
 219 # See http://en.wikipedia.org/wiki/Portable_Network_Graphics#Color_depth
 220 # for details about the color type code.
 221 function get_color_type {
 222   local file=$1
 223   echo $(file $file | awk -F, '{print $3}' | awk '{print $2}')
 224 }
 225
 226 # Usage: optimize_size <file>
 227 # Performs png file optimization.
 228 function optimize_size {
 229   # Print filename, trimmed to ensure it + status don't take more than 1 line
 230   local filename_length=${#file}
 231   local -i allowed_length=$COLUMNS-11
 232   local -i trimmed_length=$filename_length-$COLUMNS+14
 233   if [ "$filename_length" -lt "$allowed_length" ]; then
 234     info -n "$file|........"
 235   else
 236     info -n "...${file:$trimmed_length}|........"
 237   fi
 238
 239   local file=$1
 240
 241   advdef -q -z -4 $file
 242
 243   pngout -q -s4 -c0 -force $file $file.tmp.png
 244   if [ -f $file.tmp.png ]; then
 245     rm $file.tmp.png
 246     process_grayscale $file
 247     process_grayscale_alpha $file
 248   else
 249     pngout -q -s4 -c4 -force $file $file.tmp.png
 250     if [ -f $file.tmp.png ]; then
 251       rm $file.tmp.png
 252       process_grayscale_alpha $file
 253     else
 254       process_rgb $file
 255     fi
 256   fi
 257
 258   info -ne "\b\b\b\b\b\b\b\bfilter.."
 259   local old_color_type=$(get_color_type $file)
 260   optipng -q -zc9 -zm8 -zs0-3 -f0-5 $file -out $file.tmp.png
 261   local new_color_type=$(get_color_type $file.tmp.png)
 262   # optipng may corrupt a png file when reducing the color type
 263   # to grayscale/grayscale+alpha. Just skip such cases until
 264   # the bug is fixed. See crbug.com/174505, crbug.com/174084.
 265   # The issue is reported in
 266   # https://sourceforge.net/tracker/?func=detail&aid=3603630&group_id=151404&atid=780913
 267   if [[ $old_color_type == "RGBA" && $new_color_type == gray* ]] ; then
 268     rm $file.tmp.png
 269   else
 270     mv $file.tmp.png $file
 271   fi
 272   pngout -q -k1 -s1 $file
 273
 274   huffman_blocks $file
 275
 276   # TODO(oshima): Experiment with strategy 1.
 277   info -ne "\b\b\b\b\b\b\b\bstrategy"
 278   if [ $OPTIMIZE_LEVEL == 2 ]; then
 279     for i in 3 2 0; do
 280       pngout -q -k1 -ks -s$i $file
 281     done
 282   else
 283     pngout -q -k1 -ks -s1 $file
 284   fi
 285
 286   if [ $OPTIMIZE_LEVEL == 2 ]; then
 287     random_huffman_table_trial $file
 288   fi
 289
 290   final_compression $file
 291 }
 292
 293 # Usage: process_file <file>
 294 function process_file {
 295   local file=$1
 296   local name=$(basename $file)
 297   # -rem alla removes all ancillary chunks except for tRNS
 298   pngcrush -d $TMP_DIR -brute -reduce -rem alla $file > /dev/null 2>&1
 299
 300   if [ -f $TMP_DIR/$name -a $OPTIMIZE_LEVEL != 0 ]; then
 301     optimize_size $TMP_DIR/$name
 302   fi
 303 }
 304
 305 # Usage: optimize_file <file>
 306 function optimize_file {
 307   local file=$1
 308   if $using_cygwin ; then
 309     file=$(cygpath -w $file)
 310   fi
 311
 312   local name=$(basename $file)
 313   local old=$(stat -c%s $file)
 314   local tmp_file=$TMP_DIR/$name
 315   let TOTAL_FILE+=1
 316
 317   process_file $file
 318
 319   if [ ! -e $tmp_file ] ; then
 320     let CORRUPTED_FILE+=1
 321     echo "$file may be corrupted; skipping\n"
 322     return
 323   fi
 324
 325   local new=$(stat -c%s $tmp_file)
 326   let diff=$old-$new
 327   let percent=$diff*100
 328   let percent=$percent/$old
 329
 330   if [ $new -lt $old ]; then
 331     info "$file: $old => $new ($diff bytes: $percent%)"
 332     cp "$tmp_file" "$file"
 333     let TOTAL_OLD_BYTES+=$old
 334     let TOTAL_NEW_BYTES+=$new
 335     let PROCESSED_FILE+=1
 336   else
 337     if [ $OPTIMIZE_LEVEL == 0 ]; then
 338       info "$file: Skipped"
 339     else
 340       info "$file: Unable to reduce size"
 341     fi
 342     rm $tmp_file
 343   fi
 344 }
 345
 346 function optimize_dir {
 347   local dir=$1
 348   if $using_cygwin ; then
 349     dir=$(cygpath -w $dir)
 350   fi
 351
 352   for f in $(find $dir -name "*.png"); do
 353     optimize_file $f
 354   done
 355 }
 356
 357 function install_if_not_installed {
 358   local program=$1
 359   local package=$2
 360   which $program > /dev/null 2>&1
 361   if [ "$?" != "0" ]; then
 362     if $using_cygwin ; then
 363       echo "Couldn't find $program. " \
 364            "Please run cygwin's setup.exe and install the $package package."
 365       exit 1
 366     else
 367       read -p "Couldn't find $program. Do you want to install? (y/n)"
 368       [ "$REPLY" == "y" ] && sudo apt-get install $package
 369       [ "$REPLY" == "y" ] || exit
 370     fi
 371   fi
 372 }
 373
 374 function fail_if_not_installed {
 375   local program=$1
 376   local url=$2
 377   which $program > /dev/null 2>&1
 378   if [ $? != 0 ]; then
 379     echo "Couldn't find $program. Please download and install it from $url ."
 380     exit 1
 381   fi
 382 }
 383
 384 # Check pngcrush version and exit if the version is in bad range.
 385 # See crbug.com/404893.
 386 function exit_if_bad_pngcrush_version {
 387   local version=$(pngcrush -v | awk "/pngcrush 1.7./ {print \$3}")
 388   local version_num=$(echo $version | sed "s/\.//g")
 389   if [[ (1748 -lt $version_num && $version_num -lt 1773) ]] ; then
 390     echo "Your pngcrush ($version) has a bug that exists from " \
 391          "1.7.49 to 1.7.72  (see crbug.com/404893 for details)."
 392     echo "Please upgrade pngcrush and try again"
 393     exit 1;
 394   fi
 395 }
 396
 397 function show_help {
 398   local program=$(basename $0)
 399   echo \
 400 "Usage: $program [options] <dir> ...
 401
 402 $program is a utility to reduce the size of png files by removing
 403 unnecessary chunks and compressing the image.
 404
 405 Options:
 406   -o<optimize_level>  Specify optimization level: (default is 1)
 407       0  Just run pngcrush. It removes unnecessary chunks and perform basic
 408          optimization on the encoded data.
 409       1  Optimize png files using pngout/optipng and advdef. This can further
 410          reduce addtional 5~30%. This is the default level.
 411       2  Aggressively optimize the size of png files. This may produce
 412          addtional 1%~5% reduction.  Warning: this is *VERY*
 413          slow and can take hours to process all files.
 414   -r<revision> If this is specified, the script processes only png files
 415                changed since this revision. The <dir> options will be used
 416                to narrow down the files under specific directories.
 417   -v  Shows optimization process for each file.
 418   -h  Print this help text."
 419   exit 1
 420 }
 421
 422 if [ ! -e ../.gclient ]; then
 423   echo "$0 must be run in src directory"
 424   exit 1
 425 fi
 426
 427 if [ "$(expr substr $(uname -s) 1 6)" == "CYGWIN" ]; then
 428   using_cygwin=true
 429 else
 430   using_cygwin=false
 431 fi
 432
 433 # The -i in the shebang line should result in $COLUMNS being set on newer
 434 # versions of bash.  If it's not set yet, attempt to set it.
 435 if [ -z $COLUMNS ]; then
 436   which tput > /dev/null 2>&1
 437   if [ "$?" == "0" ]; then
 438     COLUMNS=$(tput cols)
 439   else
 440     # No tput either... give up and just guess 80 columns.
 441     COLUMNS=80
 442   fi
 443   export COLUMNS
 444 fi
 445
 446 OPTIMIZE_LEVEL=1
 447 # Parse options
 448 while getopts o:r:h:v opts
 449 do
 450   case $opts in
 451     r)
 452       COMMIT=$(git svn find-rev r$OPTARG | tail -1) || exit
 453       if [ -z "$COMMIT" ] ; then
 454         echo "Revision $OPTARG not found"
 455         show_help
 456       fi
 457       ;;
 458     o)
 459       if [[ "$OPTARG" != 0 && "$OPTARG" != 1 && "$OPTARG" != 2 ]] ; then
 460         show_help
 461       fi
 462       OPTIMIZE_LEVEL=$OPTARG
 463       ;;
 464     v)
 465       VERBOSE=true
 466       ;;
 467     [h?])
 468       show_help;;
 469   esac
 470 done
 471
 472 # Remove options from argument list.
 473 shift $(($OPTIND -1))
 474
 475 # Make sure we have all necessary commands installed.
 476 install_if_not_installed pngcrush pngcrush
 477 exit_if_bad_pngcrush_version
 478
 479 if [ $OPTIMIZE_LEVEL -ge 1 ]; then
 480   install_if_not_installed optipng optipng
 481
 482   if $using_cygwin ; then
 483     fail_if_not_installed advdef "http://advancemame.sourceforge.net/comp-readme.html"
 484   else
 485     install_if_not_installed advdef advancecomp
 486   fi
 487
 488   if $using_cygwin ; then
 489     pngout_url="http://www.advsys.net/ken/utils.htm"
 490   else
 491     pngout_url="http://www.jonof.id.au/kenutils"
 492   fi
 493   fail_if_not_installed pngout $pngout_url
 494 fi
 495
 496 # Create tmp directory for crushed png file.
 497 TMP_DIR=$(mktemp -d)
 498 if $using_cygwin ; then
 499   TMP_DIR=$(cygpath -w $TMP_DIR)
 500 fi
 501
 502 # Make sure we cleanup temp dir
 503 #trap "rm -rf $TMP_DIR" EXIT
 504
 505 # If no directories are specified, optimize all directories.
 506 DIRS=$@
 507 set ${DIRS:=$ALL_DIRS}
 508
 509 info "Optimize level=$OPTIMIZE_LEVEL"
 510
 511 if [ -n "$COMMIT" ] ; then
 512  ALL_FILES=$(git diff --name-only $COMMIT HEAD $DIRS | grep "png$")
 513  ALL_FILES_LIST=( $ALL_FILES )
 514  echo "Processing ${#ALL_FILES_LIST[*]} files"
 515  for f in $ALL_FILES; do
 516    if [ -f $f ] ; then
 517      optimize_file $f
 518    else
 519      echo "Skipping deleted file: $f";
 520    fi
 521  done
 522 else
 523   for d in $DIRS; do
 524     if [ -d $d ] ; then
 525       info "Optimizing png files in $d"
 526       optimize_dir $d
 527       info ""
 528     elif [ -f $d ] ; then
 529       optimize_file $d
 530     else
 531       echo "Not a file or directory: $d";
 532     fi
 533   done
 534 fi
 535
 536 # Print the results.
 537 echo "Optimized $PROCESSED_FILE/$TOTAL_FILE files in" \
 538      "$(date -d "0 + $SECONDS sec" +%Ts)"
 539 if [ $PROCESSED_FILE != 0 ]; then
 540   let diff=$TOTAL_OLD_BYTES-$TOTAL_NEW_BYTES
 541   let percent=$diff*100/$TOTAL_OLD_BYTES
 542   echo "Result: $TOTAL_OLD_BYTES => $TOTAL_NEW_BYTES bytes" \
 543        "($diff bytes: $percent%)"
 544 fi
 545 if [ $CORRUPTED_FILE != 0 ]; then
 546   echo "Warning: corrupted files found: $CORRUPTED_FILE"
 547   echo "Please contact the author of the CL that landed corrupted png files"
 548 fi