contrib/compare_tests

   1 #!/bin/sh
   2 # This script automatically test the given tool with the tool's test cases,
   3 # reporting anything of interest.
   4
   5 # Written by Mike Stump <mrs@cygnus.com>
   6 # Subdir comparison added by Quentin Neill <quentin.neill@amd.com>
   7
   8 usage()
   9 {
  10         if [ -n "$1" ] ; then
  11                 echo "$0: Error: $1" >&2
  12                 echo >&2
  13         fi
  14         cat >&2 <<EOUSAGE
  15 Usage: $0 [-strict] PREVIOUS CURRENT
  16
  17 Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest.
  18
  19         If PREVIOUS and CURRENT are directories, find and compare any *.sum files.
  20
  21         Unless -strict is given, these discrepancies are not counted as errors:
  22                 missing/extra .sum files when comparing directories
  23                 tests that failed in PREVIOUS but pass in CURRENT
  24                 tests that were not in PREVIOUS but appear in CURRENT
  25                 tests in PREVIOUS that are missing in CURRENT
  26
  27         Exit with the following values:
  28                 0 if there is nothing of interest
  29                 1 if there are errors when comparing single test case files
  30                 N for the number of errors found when comparing directories
  31 EOUSAGE
  32         exit 2
  33 }
  34
  35 export LC_ALL=C
  36
  37 tool=gxx
  38
  39 TMPDIR=${TMPDIR:-/tmp}
  40 tmp1=$TMPDIR/$tool-testing.$$a
  41 tmp2=$TMPDIR/$tool-testing.$$b
  42 now_s=$TMPDIR/$tool-testing.$$d
  43 before_s=$TMPDIR/$tool-testing.$$e
  44 lst1=$TMPDIR/$tool-lst1.$$
  45 lst2=$TMPDIR/$tool-lst2.$$
  46 lst3=$TMPDIR/$tool-lst3.$$
  47 lst4=$TMPDIR/$tool-lst4.$$
  48 lst5=$TMPDIR/$tool-lst5.$$
  49 sum1=$TMPDIR/$tool-sum1.$$
  50 sum2=$TMPDIR/$tool-sum2.$$
  51 tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2"
  52
  53 [ "$1" = "-strict" ] && strict=$1 && shift
  54 [ "$1" = "-?" ] && usage
  55 [ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"
  56
  57 trap "rm -f $tmps" 0 1 2 3 5 9 13 15
  58 exit_status=0
  59
  60 if [ -d "$1" -a -d "$2" ] ; then
  61         find "$1/" -name '*.sum' >$lst1
  62         find "$2/" -name '*.sum' >$lst2
  63         echo "# Comparing directories"
  64         echo "## Dir1=$1: `cat $lst1 | wc -l` sum files"
  65         echo "## Dir2=$2: `cat $lst2 | wc -l` sum files"
  66         echo
  67         # remove leading directory components to compare
  68         sed -e "s|^$1[/]*||" $lst1 | sort >$lst3
  69         sed -e "s|^$2[/]*||" $lst2 | sort >$lst4
  70         comm -23 $lst3 $lst4 >$lst5
  71         if [ -s $lst5 ] ; then
  72                 echo "# Extra sum files in Dir1=$1"
  73                 sed -e "s|^|< $1/|" $lst5
  74                 echo
  75                 [ -n "$strict" ] && exit_status=`expr $exit_status + 1`
  76         fi
  77         comm -13 $lst3 $lst4 >$lst5
  78         if [ -s $lst5 ] ; then
  79                 echo "# Extra sum files in Dir2=$2"
  80                 sed -e "s|^|> $2/|" $lst5
  81                 echo
  82                 [ -n "$strict" ] && exit_status=`expr $exit_status + 1`
  83         fi
  84         comm -12 $lst3 $lst4 | sort -u >$lst5
  85         if [ ! -s $lst5 ] ; then
  86                 echo "# No common sum files"
  87                 exit_status=`expr $exit_status + 1`
  88                 exit $exit_status
  89         fi
  90         cmnsums=`cat $lst5 | wc -l`
  91         echo "# Comparing $cmnsums common sum files"
  92         ( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1
  93         ( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2
  94         echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2"
  95         ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2
  96         ret=$?
  97         if [ $ret -ne 0 ]; then
  98                 exit_status=`expr $exit_status + 1`
  99                 echo "## Differences found"
 100         fi
 101         if [ $exit_status -ne 0 ]; then
 102                 echo "# $exit_status differences in $cmnsums common sum files found"
 103         else
 104                 echo "# No differences found in $cmnsums common sum files"
 105         fi
 106         exit $exit_status
 107 elif [ -d "$1" -o -d "$2" ] ; then
 108         usage "Must specify either two directories or two files"
 109 fi
 110
 111 sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^[   ]*=== [^ ]* tests ===$/ {tool = $2} /^Running target / {target = $3} { if (tool != "") { sub(/:[       ]/, "&"tool": " ); }; if (target != "unix") { sub(/:[   ]/, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp1
 112 sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$2" | awk '/^[   ]*=== [^ ]* tests ===$/ {tool = $2} /^Running target / {target = $3} { if (tool != "") { sub(/:[       ]/, "&"tool": " ); }; if (target != "unix") { sub(/:[   ]/, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp2
 113
 114 before=$tmp1
 115 now=$tmp2
 116
 117
 118 if sort -k 2 </dev/null >/dev/null 2>&1; then
 119   skip1='-k 2'
 120 else
 121   skip1='+1'
 122 fi
 123
 124 sort -t ':' $skip1 "$now" > "$now_s"
 125 sort -t ':' $skip1 "$before" > "$before_s"
 126
 127 grep '^FAIL:' "$now_s" | sed 's/^[^:]*:[        ]//' >$tmp1
 128 grep '^PASS' "$before_s" | sed 's/^[^:]*:[      ]//' | comm -12 $tmp1 - >$tmp2
 129
 130 grep -s . $tmp2 >/dev/null
 131 if [ $? = 0 ]; then
 132         num=`cat $tmp2 | wc -l`
 133         echo "Tests that now fail, but worked before ($num tests):"
 134         echo
 135         cat $tmp2
 136         echo
 137         exit_status=1
 138 fi
 139
 140 grep '^PASS' "$now_s" | sed 's/^[^:]*:[         ]//' >$tmp1
 141 grep '^FAIL' "$before_s" | sed 's/^[^:]*:[      ]//' | comm -12 $tmp1 - >$tmp2
 142
 143 grep -s . $tmp2 >/dev/null
 144 if [ $? = 0 ]; then
 145         num=`cat $tmp2 | wc -l`
 146         echo "Tests that now work, but didn't before ($num tests):"
 147         echo
 148         cat $tmp2
 149         [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 150         echo
 151 fi
 152
 153 grep '^FAIL' "$now_s" | sed 's/^[^:]*:[         ]//' >$tmp1
 154 grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[     ]//' | comm -23 $tmp1 - >$tmp2
 155
 156 grep -s . $tmp2 >/dev/null
 157 if [ $? = 0 ]; then
 158         num=`cat $tmp2 | wc -l`
 159         echo "New tests that FAIL ($num tests):"
 160         echo
 161         cat $tmp2
 162         echo
 163         exit_status=1
 164 fi
 165
 166 grep '^PASS' "$now_s" | sed 's/^[^:]*:[         ]//' >$tmp1
 167 grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[     ]//' | comm -23 $tmp1 - >$tmp2
 168
 169 grep -s . $tmp2 >/dev/null
 170 if [ $? = 0 ]; then
 171         num=`cat $tmp2 | wc -l`
 172         echo "New tests that PASS ($num tests):"
 173         echo
 174         cat $tmp2
 175         [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 176         echo
 177 fi
 178
 179 grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[        ]//' >$tmp1
 180 grep '^PASS' "$before_s" | sed 's/^[^:]*:[      ]//' | comm -13 $tmp1 - >$tmp2
 181
 182 grep -s . $tmp2 >/dev/null
 183 if [ $? = 0 ]; then
 184         num=`cat $tmp2 | wc -l`
 185         echo "Old tests that passed, that have disappeared ($num tests): (Eeek!)"
 186         echo
 187         cat $tmp2
 188         [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 189         echo
 190 fi
 191
 192 grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[        ]//' >$tmp1
 193 grep '^FAIL' "$before_s" | sed 's/^[^:]*:[      ]//' | comm -13 $tmp1 - >$tmp2
 194
 195 grep -s . $tmp2 >/dev/null
 196 if [ $? = 0 ]; then
 197         num=`cat $tmp2 | wc -l`
 198         echo "Old tests that failed, that have disappeared ($num tests): (Eeek!)"
 199         echo
 200         cat $tmp2
 201         [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 202         echo
 203 fi
 204
 205 exit $exit_status