js/tests/runtests.sh

   1 #!/bin/bash -e
   2 # -*- Mode: Shell-script; tab-width: 4; indent-tabs-mode: nil; -*-
   3
   4 # ***** BEGIN LICENSE BLOCK *****
   5 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   6 #
   7 # The contents of this file are subject to the Mozilla Public License Version
   8 # 1.1 (the "License"); you may not use this file except in compliance with
   9 # the License. You may obtain a copy of the License at
  10 # http://www.mozilla.org/MPL/
  11 #
  12 # Software distributed under the License is distributed on an "AS IS" basis,
  13 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  14 # for the specific language governing rights and limitations under the
  15 # License.
  16 #
  17 # The Original Code is Mozilla JavaScript Testing Utilities
  18 #
  19 # The Initial Developer of the Original Code is
  20 # Mozilla Corporation.
  21 # Portions created by the Initial Developer are Copyright (C) 2007
  22 # the Initial Developer. All Rights Reserved.
  23 #
  24 # Contributor(s): Bob Clary <bclary@bclary.com>
  25 #
  26 # Alternatively, the contents of this file may be used under the terms of
  27 # either the GNU General Public License Version 2 or later (the "GPL"), or
  28 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  29 # in which case the provisions of the GPL or the LGPL are applicable instead
  30 # of those above. If you wish to allow use of your version of this file only
  31 # under the terms of either the GPL or the LGPL, and not to allow others to
  32 # use your version of this file under the terms of the MPL, indicate your
  33 # decision by deleting the provisions above and replace them with the notice
  34 # and other provisions required by the GPL or the LGPL. If you do not delete
  35 # the provisions above, a recipient may use your version of this file under
  36 # the terms of any one of the MPL, the GPL or the LGPL.
  37 #
  38 # ***** END LICENSE BLOCK *****
  39
  40 if [[ -z "$TEST_DIR" ]]; then
  41     cat <<EOF
  42 `basename $0`: error
  43
  44 TEST_DIR, the location of the Sisyphus framework,
  45 is required to be set prior to calling this script.
  46 EOF
  47     exit 2
  48 fi
  49
  50 if [[ ! -e $TEST_DIR/bin/library.sh ]]; then
  51     echo "TEST_DIR=$TEST_DIR"
  52     echo ""
  53     echo "This script requires the Sisyphus testing framework. Please "
  54     echo "cvs check out the Sisyphys framework from mozilla/testing/sisyphus"
  55     echo "and set the environment variable TEST_DIR to the directory where it"
  56     echo "located."
  57     echo ""
  58
  59     exit 2
  60 fi
  61
  62 source $TEST_DIR/bin/library.sh
  63
  64 TEST_JSDIR=${TEST_JSDIR:-$TEST_DIR/tests/mozilla.org/js}
  65
  66 usage()
  67 {
  68     cat <<EOF
  69 usage: runtests.sh -p products -b branches -e extra\\
  70                    -T  buildtypes -B buildcommands  \\
  71                    [-v] [-S] [-X excludetests] [-I includetests] [-c] [-t] \\
  72                    [-J javascriptoptions]
  73
  74 variable            description
  75 ===============     ============================================================
  76 -p products         space separated list of js, firefox, fennec
  77 -b branches         space separated list of branches 1.8.0, 1.8.1, 1.9.0, 1.9.1
  78 -e extra            optional. extra qualifier to pick build tree and mozconfig.
  79 -T buildtypes       space separated list of build types opt debug
  80 -B buildcommands    optional space separated list of build commands
  81                     clean, checkout, build. If not specified, defaults to
  82                     'clean checkout build'.
  83                     If you wish to skip any build steps, simply specify a value
  84                     not containing any of the build commands, e.g. 'none'.
  85 -v                  optional. verbose - copies log file output to stdout.
  86 -S                  optional. summary - output tailered for use with
  87                     Buildbot|Tinderbox
  88 -X excludetests     optional. By default the test will exclude the
  89                     tests listed in spidermonkey-n-\$branch.tests,
  90                     performance-\$branch.tests. excludetests is a list of either
  91                     individual tests, manifest files or sub-directories which
  92                     will override the default exclusion list.
  93 -I includetests     optional. By default the test will include the
  94                     JavaScript tests appropriate for the branch. includetests is a
  95                     list of either individual tests, manifest files or
  96                     sub-directories which will override the default inclusion
  97                     list.
  98 -c                  optional. By default the test will exclude tests
  99                     which crash on this branch, test type, build type and
 100                     operating system. -c will include tests which crash.
 101                     Typically this should only be used in combination with -R.
 102                     This has no effect on shell based tests which execute crash
 103                     tests regardless.
 104 -t                  optional. By default the test will exclude tests
 105                     which time out on this branch, test type, build type and
 106                     operating system. -t will include tests which timeout.
 107 -J jsoptions        optional. Set JavaScript options:
 108                     -Z n Set gczeal to n. Currently, only valid for
 109                     debug builds of Gecko 1.8.1.15, 1.9.0 and later.
 110                     -z optional. use split objects in the shell.
 111                     -j optional. use JIT in the shell. Only available on 1.9.1 and later
 112
 113 if an argument contains more than one value, it must be quoted.
 114 EOF
 115     exit 2
 116 }
 117
 118 verbose=0
 119
 120 while getopts "p:b:T:B:e:X:I:J:vSct" optname;
 121 do
 122     case $optname in
 123         p) products=$OPTARG;;
 124         b) branches=$OPTARG;;
 125         T) buildtypes=$OPTARG;;
 126         e) extra="$OPTARG"
 127             extraflag="-e $OPTARG";;
 128         B) buildcommands=$OPTARG;;
 129         v) verbose=1
 130             verboseflag="-v";;
 131         S) summary=1;;
 132         X) excludetests=$OPTARG;;
 133         I) includetests=$OPTARG;;
 134         J) javascriptoptions=$OPTARG;;
 135         c) crashes=1;;
 136         t) timeouts=1;;
 137     esac
 138 done
 139
 140  # javascriptoptions will be passed by environment to test.sh
 141
 142 if [[ -z "$products" || -z "$branches" || -z "$buildtypes" ]]; then
 143     usage
 144 fi
 145
 146 if [[ -z "$buildcommands" ]]; then
 147     buildcommands="clean checkout build"
 148 fi
 149
 150 case $buildtypes in
 151     nightly)
 152         buildtypes="nightly-$OSID"
 153         ;;
 154     opt|debug|opt*debug)
 155         if [[ -n "$buildcommands" ]]; then
 156             builder.sh -p "$products" -b "$branches" $extraflag -B "$buildcommands" -T "$buildtypes" "$verboseflag"
 157         fi
 158         ;;
 159 esac
 160
 161 testlogfilelist=`mktemp /tmp/TESTLOGFILES.XXXX`
 162 trap "_exit; rm -f $testlogfilelist" EXIT
 163
 164 export testlogfiles
 165 export testlogfile
 166
 167 # because without pipefail, the pipe will not return a non-zero
 168 # exit code, we must pipe stderr from tester.sh to stdout and then
 169 # look into the testlogfilelist for the error
 170
 171 branchesextra=`combo.sh -d - "$branches" "$extra"`
 172
 173 # can't test tester.sh's exit code to see if there was
 174 # an error since we are piping it and can't count on pipefail
 175 tester.sh -t "$TEST_JSDIR/test.sh" $verboseflag "$products" "$branchesextra" "$buildtypes" 2>&1 | tee -a $testlogfilelist
 176 testlogfiles="`grep '^log:' $testlogfilelist|sed 's|^log: ||'`"
 177
 178 fatalerrors=`grep 'FATAL ERROR' $testlogfiles | cat`
 179 if [[ -n "$fatalerrors" ]]; then
 180     testlogarray=( $testlogfiles )
 181     let itestlog=${#testlogarray[*]}-1
 182     error "`tail -n 20 ${testlogarray[$itestlog]}`" $LINENO
 183 fi
 184
 185 for testlogfile in $testlogfiles; do
 186
 187     if [[ -n "$DEBUG" ]]; then
 188         dumpvars testlogfile
 189     fi
 190
 191     case "$testlogfile" in
 192         *,js,*) testtype=shell;;
 193         *,firefox,*) testtype=browser;;
 194         *,fennec,*) testtype=browser;;
 195         *) error "unknown testtype in logfile $testlogfile" $LINENO;;
 196     esac
 197
 198     case "$testlogfile" in
 199         *,opt,*) buildtype=opt;;
 200         *,debug,*) buildtype=debug;;
 201         *,nightly*) buildtype=opt;;
 202         *) error "unknown buildtype in logfile $testlogfile" $LINENO;;
 203     esac
 204
 205     case "$testlogfile" in
 206         *,1.8.0*) branch=1.8.0;;
 207         *,1.8.1*) branch=1.8.1;;
 208         *,1.9.0*) branch=1.9.0;;
 209         *,1.9.1*) branch=1.9.1;;
 210         *) error "unknown branch in logfile $testlogfile" $LINENO;;
 211     esac
 212
 213     repo=`grep -m 1 '^environment: TEST_MOZILLA_HG=' $testlogfile | sed 's|.*TEST_MOZILLA_HG=http://hg.mozilla.org/\(.*\)|\1|'`
 214     if [[ -z "$repo" ]]; then
 215         repo=CVS
 216     fi
 217     debug "repo=$repo"
 218
 219     outputprefix=$testlogfile
 220
 221     if [[ -n "$DEBUG" ]]; then
 222         dumpvars branch buildtype testtype OSID testlogfile TEST_PROCESSORTYPE TEST_KERNEL outputprefix
 223     fi
 224
 225     if ! $TEST_DIR/tests/mozilla.org/js/known-failures.pl \
 226         -b $branch \
 227         -T $buildtype \
 228         -R $repo \
 229         -t $testtype \
 230         -o "$OSID" \
 231         -K "$TEST_KERNEL" \
 232         -A "$TEST_PROCESSORTYPE" \
 233         -M "$TEST_MEMORY" \
 234         -S "$TEST_CPUSPEED" \
 235         -z `date +%z` \
 236         -l $testlogfile \
 237         -r $TEST_JSDIR/failures.txt \
 238         -O $outputprefix; then
 239         error "known-failures.pl" $LINENO
 240     fi
 241
 242     if [[ -n "$summary" ]]; then
 243
 244         # use let to work around mac problem where numbers were
 245         # output with leading characters.
 246         # if let's arg evaluates to 0, let will return 1
 247         # so we need to test
 248
 249         if let npass="`grep TEST_RESULT=PASSED ${outputprefix}-results-all.log | wc -l`"; then true; fi
 250         if let nfail="`cat ${outputprefix}-results-failures.log | wc -l`"; then true; fi
 251         if let nfixes="`cat ${outputprefix}-results-possible-fixes.log | wc -l`"; then true; fi
 252         if let nregressions="`cat ${outputprefix}-results-possible-regressions.log | wc -l`"; then true; fi
 253
 254         echo -e "\nJavaScript Tests $branch $buildtype $testtype\n"
 255         echo -e "\nFailures:\n"
 256         cat "${outputprefix}-results-failures.log"
 257         echo -e "\nPossible Fixes:\n"
 258         cat "${outputprefix}-results-possible-fixes.log"
 259         echo -e "\nPossible Regressions:\n"
 260         cat "${outputprefix}-results-possible-regressions.log"
 261         echo -e "\nTinderboxPrint:<div title=\"$testlogfile\">\n"
 262         echo -e "\nTinderboxPrint:js tests<br/>$branch $buildtype $testtype<br/>$npass/$nfail<br/>F:$nfixes R:$nregressions"
 263         echo -e "\nTinderboxPrint:</div>\n"
 264     fi
 265
 266 done