test/run_tests.sh

   1 #!/bin/sh
   2 #
   3 # $Id$
   4 #
   5 # Copyright (c) 2003, 2004, 2005, 2006, 2007 Dan McMahill
   6
   7 #  This program is free software; you can redistribute it and/or modify
   8 #  it under the terms of version 2 of the GNU General Public License as
   9 #  published by the Free Software Foundation
  10 #
  11 #  This program is distributed in the hope that it will be useful,
  12 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 #  GNU General Public License for more details.
  15 #
  16 #  You should have received a copy of the GNU General Public License
  17 #  along with this program; if not, write to the Free Software
  18 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111 USA
  19 # All rights reserved.
  20 #
  21 # This code was derived from code written by Dan McMahill as part of the
  22 # latex-mk testsuite.  The original code was covered by a BSD license
  23 # but the copyright holder is releasing the version for gerbv under the GPL.
  24
  25 magic_test_skip=${GERBV_MAGIC_TEST_SKIP:-no}
  26
  27 if test "x${magic_test_skip}" = "xyes" ; then
  28         cat << EOF
  29
  30 The environment variable GERBV_MAGIC_TEST_SKIP is set to yes.
  31 This causes the testsuite to skip all tests and report no errors.
  32 This is used for certain debugging *only*.  The primary use is to
  33 allow testing of the 'distcheck' target without including the effects
  34 of the testsuite. The reason this is useful is that due to minor differences
  35 in cairo versions and perhaps roundoff in different CPU's, the testsuite
  36 may falsely report failures on some systems.  These reported failures
  37 prevent using 'distcheck' for verifying the rest of the build system.
  38
  39 EOF
  40
  41         exit 0
  42 fi
  43
  44 regen=no
  45
  46 usage() {
  47 cat <<EOF
  48
  49 ******* WARNING **** WARNING *******
  50 This script is still in progress and
  51 may not be functional at all.  Please
  52 stay tuned and hold off on any bug
  53 reports until this message is gone
  54 ******* WARNING **** WARNING *******
  55
  56 $0 -- Run gerbv regression tests
  57
  58 $0 -h|--help
  59 $0 [-g | --golden dir] [-r|--regen] [testname1 [testname2[ ...]]]
  60
  61 OVERVIEW
  62
  63 The $0 script is used both for running the gerbv regression testsuite
  64 as well as maintaining it.  The way the test suite works is a number
  65 of different layouts are exported to portable network graphics (PNG)
  66 files.  The resulting PNG files are then compared on a pixel by pixel
  67 basis to a set of reference PNG files.
  68
  69 Current versions of gerbv use cairo for rendering.  Besides the
  70 ability to render to the screen, cairo can render to PNG files.  Since
  71 the same rendering engine is used for screen and PNG rendering and the
  72 parser is common, verification via PNG files is fairly accurate.
  73
  74 OPTIONS
  75
  76 -g | --golden <dir>    :  Specifies that <dir> should be used for the
  77                           reference files.
  78
  79 LIMITATIONS
  80
  81 The GUI interface is not checked via the regression testsuite.
  82 However, the RS274-X and drill file parsers along with the rendering
  83 can be considered the core of the program.
  84
  85
  86 EOF
  87 }
  88
  89 show_sep() {
  90     echo "----------------------------------------------------------------------"
  91 }
  92
  93 all_tests=""
  94 while test -n "$1"
  95   do
  96   case "$1"
  97       in
  98
  99       -h|--help)
 100           usage
 101           exit 0
 102           ;;
 103
 104       -g|--golden)
 105         # set the 'golden' directory.
 106           REFDIR="$2"
 107           shift 2
 108           ;;
 109
 110       -r|--regen)
 111         # regenerate the 'golden' output files.  Use this with caution.
 112         # In particular, all differences should be noted and understood.
 113           regen=yes
 114           shift
 115           ;;
 116
 117       -*)
 118           echo "unknown option: $1"
 119           exit 1
 120           ;;
 121
 122       *)
 123           break
 124           ;;
 125
 126   esac
 127 done
 128
 129 all_tests="$*"
 130
 131 # The gerbv executible
 132 GERBV=${GERBV:-../src/run_gerbv --}
 133 GERBV_DEFAULT_FLAGS=${GERBV_DEFAULT_FLAGS:---export=png --window=640x480}
 134
 135 # Source directory
 136 srcdir=${srcdir:-.}
 137
 138 # various ImageMagick tools
 139 IM_ANIMATE=${IM_ANIMATE:-animate}
 140 IM_COMPARE=${IM_COMPARE:-compare}
 141 IM_COMPOSITE=${IM_COMPOSITE:-composite}
 142 IM_CONVERT=${IM_CONVERT:-convert}
 143 IM_DISPLAY=${IM_DISPLAY:-display}
 144 IM_MONTAGE=${IM_MONTAGE:-montage}
 145
 146 # golden directories
 147 INDIR=${INDIR:-${srcdir}/inputs}
 148 OUTDIR=outputs
 149 REFDIR=${REFDIR:-${srcdir}/golden}
 150 ERRDIR=mismatch
 151
 152 # some system tools
 153 AWK=${AWK:-awk}
 154
 155 # the list of tests to run
 156 TESTLIST=${srcdir}/tests.list
 157
 158 if test "X$regen" = "Xyes" ; then
 159     OUTDIR="${REFDIR}"
 160 fi
 161
 162 # create output directory
 163 if test ! -d $OUTDIR ; then
 164     mkdir -p $OUTDIR
 165     if test $? -ne 0 ; then
 166         echo "Failed to create output directory ${OUTDIR}"
 167         exit 1
 168     fi
 169 fi
 170
 171 if test -z "$all_tests" ; then
 172     if test ! -f ${TESTLIST} ; then
 173         echo "ERROR: ($0)  Test list $TESTLIST does not exist"
 174         exit 1
 175     fi
 176     all_tests=`${AWK} 'BEGIN{FS="|"} /^#/{next} {print $1}' ${TESTLIST} | sed 's; ;;g'`
 177 fi
 178
 179 if test -z "${all_tests}" ; then
 180     echo "$0:  No tests specified"
 181     exit 0
 182 fi
 183
 184
 185 # fail/pass/total counts
 186 fail=0
 187 pass=0
 188 skip=0
 189 tot=0
 190
 191 cat << EOF
 192
 193 srcdir                ${srcdir}
 194 top_srcdir            ${top_srcdir}
 195
 196 AWK                   ${AWK}
 197 ERRDIR                ${ERRDIR}
 198 GERBV                 ${GERBV}
 199 GERBV_DEFAULT_FLAGS   ${GERBV_DEFAULT_FLAGS}
 200 INDIR                 ${INDIR}
 201 OUTDIR                ${OUTDIR}
 202 REFDIR                ${REFDIR}
 203 TESTLIST              ${TESTLIST}
 204
 205 ImageMagick Tools:
 206
 207 IM_ANIMATE               ${IM_ANIMATE}
 208 IM_COMPARE               ${IM_COMPARE}
 209 IM_COMPOSITE             ${IM_COMPOSITE}
 210 IM_CONVERT               ${IM_CONVERT}
 211 IM_DISPLAY               ${IM_DISPLAY}
 212 IM_MONTAGE               ${IM_MONTAGE}
 213
 214 EOF
 215
 216 for t in $all_tests ; do
 217     show_sep
 218     echo "Test:  $t"
 219
 220     tot=`expr $tot + 1`
 221
 222     ######################################################################
 223     #
 224     # extract the details for the test
 225     #
 226
 227     gerbv_flags="${GERBV_DEFAULT_FLAGS}"
 228
 229     refpng="${REFDIR}/${t}.png"
 230     outpng="${OUTDIR}/${t}.png"
 231     errdir="${ERRDIR}/${t}"
 232
 233     # test_name | layout file(s) | [optional arguments to gerbv] | [mismatch]
 234     name=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {print $1}'`
 235     files=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {print $2}'`
 236     args=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {print $3}'`
 237     mismatch=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {if($2 == "mismatch"){print "yes"}else{print "no"}}'`
 238
 239     if test "X${name}" = "X" ; then
 240         echo "ERROR:  Specified test ${t} does not appear to exist"
 241         skip=`expr $skip + 1`
 242         continue
 243     fi
 244
 245     if test "X${args}" != "X" ; then
 246         gerbv_flags="${args}"
 247     fi
 248
 249     ######################################################################
 250     #
 251     # check to see if the files we need exist
 252     #
 253
 254     missing_files=no
 255     path_files=""
 256     for f in $files ; do
 257         if test ! -f ${INDIR}/${f} ; then
 258             echo "ERROR:  File $f specified as part of the $t test does not exist"
 259             missing_files=yes
 260         else
 261             path_files="${path_files} ${INDIR}/${f}"
 262         fi
 263     done
 264     if test "$missing_files" = "yes" ; then
 265         echo "${t} had missing input files.  Skipping test"
 266         skip=`expr $skip + 1`
 267         continue
 268     fi
 269
 270     ######################################################################
 271     #
 272     # export the layout to PNG
 273     #
 274
 275     echo "${GERBV} ${gerbv_flags} --output=${outpng} ${path_files}"
 276     ${GERBV} ${gerbv_flags} --output=${outpng} ${path_files}
 277
 278     ######################################################################
 279     #
 280     # compare to the golden PNG file
 281     #
 282
 283     if test "X$regen" != "Xyes" ; then
 284         if test -f ${REFDIR}/${t}.png ; then
 285             same=`${IM_COMPARE} -metric MAE $refpng $outpng  null: 2>&1 | \
 286                 ${AWK} '{if($1 == 0){print "yes"} else {print "no"}}'`
 287             if test "$same" = yes ; then
 288                 echo "PASS"
 289                 pass=`expr $pass + 1`
 290             else
 291                 echo "FAILED:  See ${errdir}"
 292                 mkdir -p ${errdir}
 293                 ${IM_COMPARE} ${refpng} ${outpng} ${errdir}/compare.png
 294                 ${IM_COMPOSITE} ${refpng} ${outpng} -compose difference ${errdir}/composite.png
 295                 ${IM_CONVERT} ${refpng} ${outpng} -compose difference -composite  -colorspace gray   ${errdir}/gray.png
 296 cat > ${errdir}/animate.sh << EOF
 297 #!/bin/sh
 298 ${IM_CONVERT} -label "%f" ${refpng} ${outpng} miff:- | \
 299 ${IM_MONTAGE} - -geometry +0+0 -tile 1x1 miff:- | \
 300 ${IM_ANIMATE} -delay 0.5 -loop 0 -
 301 EOF
 302                 chmod a+x ${errdir}/animate.sh
 303                 fail=`expr $fail + 1`
 304             fi
 305         else
 306             echo "No reference file ${REFDIR}/${t}.png.  Skipping test"
 307             skip=`expr $skip + 1`
 308         fi
 309     else
 310         echo "Regenerated"
 311     fi
 312
 313
 314 done
 315
 316 show_sep
 317 echo "Passed $pass, failed $fail, skipped $skip out of $tot tests."
 318
 319 rc=0
 320 if test $pass -ne $tot ; then
 321     rc=1
 322 fi
 323
 324 exit $rc
 325