js/tests/test.sh

   1 #!/bin/bash -e
   2 # -*- Mode: Shell-script; tab-width: 4; indent-tabs-mode: nil; -*-
   3
   4 # ***** BEGIN LICENSE BLOCK *****
   5 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   6 #
   7 # The contents of this file are subject to the Mozilla Public License Version
   8 # 1.1 (the "License"); you may not use this file except in compliance with
   9 # the License. You may obtain a copy of the License at
  10 # http://www.mozilla.org/MPL/
  11 #
  12 # Software distributed under the License is distributed on an "AS IS" basis,
  13 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  14 # for the specific language governing rights and limitations under the
  15 # License.
  16 #
  17 # The Original Code is Mozilla JavaScript Testing Utilities
  18 #
  19 # The Initial Developer of the Original Code is
  20 # Mozilla Corporation.
  21 # Portions created by the Initial Developer are Copyright (C) 2007
  22 # the Initial Developer. All Rights Reserved.
  23 #
  24 # Contributor(s): Bob Clary <bclary@bclary.com>
  25 #
  26 # Alternatively, the contents of this file may be used under the terms of
  27 # either the GNU General Public License Version 2 or later (the "GPL"), or
  28 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  29 # in which case the provisions of the GPL or the LGPL are applicable instead
  30 # of those above. If you wish to allow use of your version of this file only
  31 # under the terms of either the GPL or the LGPL, and not to allow others to
  32 # use your version of this file under the terms of the MPL, indicate your
  33 # decision by deleting the provisions above and replace them with the notice
  34 # and other provisions required by the GPL or the LGPL. If you do not delete
  35 # the provisions above, a recipient may use your version of this file under
  36 # the terms of any one of the MPL, the GPL or the LGPL.
  37 #
  38 # ***** END LICENSE BLOCK *****
  39
  40 if [[ -z "$TEST_DIR" ]]; then
  41     cat <<EOF
  42 `basename $0`: error
  43
  44 TEST_DIR, the location of the Sisyphus framework,
  45 is required to be set prior to calling this script.
  46 EOF
  47     exit 2
  48 fi
  49
  50 if [[ ! -e $TEST_DIR/bin/library.sh ]]; then
  51     echo "TEST_DIR=$TEST_DIR"
  52     echo ""
  53     echo "This script requires the Sisyphus testing framework. Please "
  54     echo "cvs check out the Sisyphys framework from mozilla/testing/sisyphus"
  55     echo "and set the environment variable TEST_DIR to the directory where it"
  56     echo "located."
  57     echo ""
  58
  59     exit 2
  60 fi
  61
  62 source $TEST_DIR/bin/library.sh
  63
  64 TEST_JSDIR=${TEST_JSDIR:-$TEST_DIR/tests/mozilla.org/js}
  65
  66 TEST_JSSHELL_TIMEOUT=${TEST_JSSHELL_TIMEOUT:-480}
  67 TEST_JSEACH_TIMEOUT=${TEST_JSEACH_TIMEOUT:-485}
  68 TEST_JSEACH_PAGE_TIMEOUT=${TEST_JSEACH_PAGE_TIMEOUT:-480}
  69 TEST_JSALL_TIMEOUT=${TEST_JSALL_TIMEOUT:-21600}
  70 TEST_WWW_JS=`echo $TEST_JSDIR|sed "s|$TEST_DIR||"`
  71
  72 #
  73 # options processing
  74 #
  75 usage()
  76 {
  77     cat <<EOF
  78 usage: test.sh -p product -b branch -T buildtype -x executablepath -N profilename \\
  79                [-X excludetests] [-I includetests] [-c] [-t] [-F] [-d datafiles]
  80
  81 variable            description
  82 ===============     ============================================================
  83 -p product          required. firefox|thunderbird|js|fennec
  84 -b branch           required. 1.8.0|1.8.1|1.9.0|1.9.1
  85 -s jsshellsourcepath       required for shell. path to js shell source directory mozilla/js/src
  86 -T buildtype        required. one of opt debug
  87 -x executablepath   required for browser. directory-tree containing executable 'product'
  88 -N profilename      required for browser. profile name
  89 -X excludetests     optional. By default the test will exclude the
  90                     tests listed in spidermonkey-n-\$branch.tests,
  91                     performance-\$branch.tests. excludetests is a list of either
  92                     individual tests, manifest files or sub-directories which
  93                     will override the default exclusion list.
  94 -I includetests     optional. By default the test will include the
  95                     JavaScript tests appropriate for the branch. includetests is a
  96                     list of either individual tests, manifest files or
  97                     sub-directories which will override the default inclusion
  98                     list.
  99 -c                  optional. By default the test will exclude tests
 100                     which crash on this branch, test type, build type and
 101                     operating system. -c will include tests which crash.
 102                     Typically this should only be used in combination with -R.
 103                     This has no effect on shell based tests which execute crash
 104                     tests regardless.
 105 -t                  optional. By default the test will exclude tests
 106                     which time out on this branch, test type, build type and
 107                     operating system. -t will include tests which timeout.
 108 -J jsoptions        optional. Set JavaScript options:
 109                     -Z n Set gczeal to n. Currently, only valid for
 110                     debug builds of Gecko 1.8.1.15, 1.9.0 and later.
 111                     -z optional. use split objects in the shell.
 112                     -j optional. use JIT in the shell. Only available on 1.9.1 and later
 113 -F                  optional. Just generate file lists without running any tests.
 114 -d datafiles        optional. one or more filenames of files containing
 115                     environment variable definitions to be included.
 116
 117                     note that the environment variables should have the same
 118                     names as in the "variable" column.
 119
 120 if an argument contains more than one value, it must be quoted.
 121 EOF
 122     exit 2
 123 }
 124
 125 while getopts "p:b:s:T:x:N:d:X:I:J:RctF" optname
 126 do
 127     case $optname in
 128         p)
 129             product=$OPTARG;;
 130         b)
 131             branch=$OPTARG;;
 132         T)
 133             buildtype=$OPTARG;;
 134         s)
 135             jsshellsourcepath=$OPTARG;;
 136         N)
 137             profilename=$OPTARG;;
 138         x)
 139             executablepath=$OPTARG;;
 140         X)
 141             excludetests=$OPTARG;;
 142         I)
 143             includetests=$OPTARG;;
 144         c)
 145             crashes=1;;
 146         t)
 147             timeouts=1;;
 148         F)
 149             filesonly=1;;
 150         J)
 151             javascriptoptions=$OPTARG
 152             ;;
 153         d) datafiles=$OPTARG;;
 154     esac
 155 done
 156
 157 if [[ -n "$javascriptoptions" ]]; then
 158     unset OPTIND
 159     while getopts "Z:jz" optname $javascriptoptions; do
 160         case $optname in
 161             Z)
 162                 gczealshell="-Z $OPTARG"
 163                 gczealbrowser=";gczeal=$OPTARG"
 164                 ;;
 165             z)
 166                 splitobjects="-z"
 167                 ;;
 168             j)
 169                 jitshell="-j"
 170                 jitbrowser=";jit"
 171                 ;;
 172         esac
 173     done
 174 fi
 175
 176 # include environment variables
 177 if [[ -n "$datafiles" ]]; then
 178     for datafile in $datafiles; do
 179         source $datafile
 180     done
 181 fi
 182
 183 if [[ -n "$gczeal" && "$buildtype" != "debug" ]]; then
 184     error "gczeal is supported for buildtype debug and not $buildtype"
 185 fi
 186
 187 dumpvars product branch buildtype jsshellsourcepath profilename executablepath excludetests includetests crashes timeouts filesonly javascriptoptions datafiles | sed "s|^|arguments: |"
 188
 189 pushd $TEST_JSDIR
 190
 191 case $product in
 192     js)
 193         if [[ -z "$branch" || -z "$buildtype"  || -z "$jsshellsourcepath" ]]; then
 194             usage
 195         fi
 196         source config.sh
 197         testtype=shell
 198         executable="$jsshellsourcepath/$JS_OBJDIR/js$EXE_EXT"
 199         ;;
 200
 201     firefox|thunderbird|fennec)
 202         if [[ -z "$branch" || -z "$buildtype"  || -z "$executablepath" || -z "$profilename" ]]; then
 203             usage
 204         fi
 205         testtype=browser
 206         executable=`get_executable $product $branch $executablepath`
 207
 208         ;;
 209     *)
 210         echo "Unknown product: $product"
 211         usage
 212         ;;
 213 esac
 214
 215 function shellfileversion()
 216 {
 217     local jsfile=$1
 218     local version
 219
 220     case $jsfile in
 221         ecma/*)
 222             version=150;;
 223         ecma_2/*)
 224             version=150;;
 225         ecma_3/*)
 226             version=150;;
 227         ecma_3_1/*)
 228             version=180;;
 229         js1_1/*)
 230             version=150;;
 231         js1_2/*)
 232             version=150;;
 233         js1_3/*)
 234             version=150;;
 235         js1_4/*)
 236             version=150;;
 237         js1_5/*)
 238             version=150;;
 239         js1_6/*)
 240             version=160;;
 241         js1_7/*)
 242             version=170;;
 243         js1_8/*)
 244             version=180;;
 245         js1_8_1/*)
 246             version=180;;
 247         js1_9/*)
 248             version=190;;
 249         js2_0/*)
 250             version=200;;
 251         *)
 252             version=150;;
 253     esac
 254     echo $version
 255 }
 256
 257 function browserfileversion()
 258 {
 259     local jsfile=$1
 260     local version
 261
 262     case $jsfile in
 263         ecma/*)
 264             version=1.5;;
 265         ecma_2/*)
 266             version=1.5;;
 267         ecma_3/*)
 268             version=1.5;;
 269         ecma_3_1/*)
 270             version=1.8;;
 271         js1_1/*)
 272             version=1.1;;
 273         js1_2/*)
 274             version=1.5;;
 275         js1_3/*)
 276             version=1.5;;
 277         js1_4/*)
 278             version=1.5;;
 279         js1_5/*)
 280             version=1.5;;
 281         js1_6/*)
 282             version=1.6;;
 283         js1_7/*)
 284             version=1.7;;
 285         js1_8/*)
 286             version=1.8;;
 287         js1_8_1/*)
 288             version=1.8;;
 289         js1_9/*)
 290             version=1.9;;
 291         js2_0/*)
 292             version=2.0;;
 293         *)
 294             version=1.5;;
 295     esac
 296     echo $version
 297 }
 298
 299 rm -f finished-$branch-$testtype-$buildtype
 300
 301 if ! make failures.txt; then
 302     error "during make failures.txt" $LINENO
 303 fi
 304
 305 includetestsfile="included-$branch-$testtype-$buildtype.tests"
 306 rm -f $includetestsfile
 307 touch $includetestsfile
 308
 309 if [[ -z "$includetests" ]]; then
 310     # by default include tests appropriate for the branch
 311     includetests="e4x ecma ecma_2 ecma_3 js1_1 js1_2 js1_3 js1_4 js1_5 js1_6"
 312
 313     case "$branch" in
 314         1.8.0)
 315             ;;
 316         1.8.1)
 317             includetests="$includetests js1_7"
 318             ;;
 319         1.9.0)
 320             includetests="$includetests js1_7 js1_8"
 321             ;;
 322         1.9.1)
 323             includetests="$includetests js1_7 js1_8 ecma_3_1 js1_8_1"
 324             ;;
 325     esac
 326 fi
 327
 328 for i in $includetests; do
 329     if [[ -f "$i" ]]; then
 330         echo "# including $i" >> $includetestsfile
 331         if echo $i | grep -q '\.js$'; then
 332             echo $i >> $includetestsfile
 333         else
 334             cat $i >> $includetestsfile
 335         fi
 336     elif [[ -d "$i" ]]; then
 337         find $i -name '*.js' -print | egrep -v '(shell|browser|template|jsref|userhook.*|\.#.*)\.js' | sed 's/^\.\///' | sort >> $includetestsfile
 338     fi
 339 done
 340
 341 excludetestsfile="excluded-$branch-$testtype-$buildtype.tests"
 342 rm -f $excludetestsfile
 343 touch $excludetestsfile
 344
 345 if [[ -z "$excludetests" ]]; then
 346     excludetests="spidermonkey-n-$branch.tests performance-$branch.tests"
 347 fi
 348
 349 for e in $excludetests; do
 350     if [[ -f "$e" ]]; then
 351         echo "# excluding $e" >> $excludetestsfile
 352         if echo $e | grep -q '\.js$'; then
 353             echo $e >> $excludetestsfile
 354         else
 355             cat $e >> $excludetestsfile
 356         fi
 357     elif [[ -d "$e" ]]; then
 358         find $e -name '*.js' -print | egrep -v '(shell|browser|template|jsref|userhook.*|\.#.*)\.js' | sed 's/^\.\///' | sort >> $excludetestsfile
 359     fi
 360 done
 361
 362 # convert the numeric speed rating to a prose value
 363 if [[ $TEST_CPUSPEED -lt 4 ]]; then
 364     TEST_CPUSPEED=slow
 365 elif [[ $TEST_CPUSPEED -lt 9 ]]; then
 366     TEST_CPUSPEED=medium
 367 else
 368     TEST_CPUSPEED=fast
 369 fi
 370
 371 if [[ -z "$TEST_MOZILLA_HG" ]]; then
 372     repo=CVS
 373 else
 374     repo=`basename $TEST_MOZILLA_HG`
 375 fi
 376 debug "repo=$repo"
 377
 378 pattern="TEST_BRANCH=($branch|[.][*]), TEST_REPO=($repo|[.][*]), TEST_BUILDTYPE=($buildtype|[.][*]), TEST_TYPE=($testtype|[.][*]), TEST_OS=($OSID|[.][*]), TEST_KERNEL=($TEST_KERNEL|[.][*]), TEST_PROCESSORTYPE=($TEST_PROCESSORTYPE|[.][*]), TEST_MEMORY=($TEST_MEMORY|[.][*]), TEST_CPUSPEED=($TEST_CPUSPEED|[.][*]),"
 379
 380 if [[ -z "$timeouts" ]]; then
 381     echo "# exclude tests that time out" >> $excludetestsfile
 382 #    echo "$pattern .*TEST_EXITSTATUS=TIMED OUT," >> $excludetestsfile
 383     egrep "$pattern .*TEST_EXITSTATUS=TIMED OUT," failures.txt | \
 384         sed 's/.*TEST_ID=\([^,]*\),.*/\1/' | sort -u >> $excludetestsfile
 385 fi
 386
 387 if [[ -z "$crashes" ]]; then
 388     echo "# exclude tests that crash" >> $excludetestsfile
 389 #    echo "$pattern .*TEST_EXITSTATUS=(CRASHED|ABNORMAL)" >> $excludetestsfile
 390     egrep "$pattern .*TEST_EXITSTATUS=(CRASHED|ABNORMAL)" failures.txt  | \
 391         sed 's/.*TEST_ID=\([^,]*\),.*/\1/' | sort -u >> $excludetestsfile
 392
 393 fi
 394
 395 cat $includetestsfile | sed 's|^|include: |'
 396 cat $excludetestsfile | sed 's|^|exclude: |'
 397
 398 case $testtype in
 399     shell)
 400         echo "JavaScriptTest: Begin Run"
 401         cat $includetestsfile | while read jsfile
 402         do
 403             if echo $jsfile | grep -q '^#'; then
 404                 continue
 405             fi
 406
 407             if ! grep -q $jsfile $excludetestsfile; then
 408
 409                 version=`shellfileversion $jsfile`
 410
 411                 subsuitetestdir=`dirname $jsfile`
 412                 suitetestdir=`dirname $subsuitetestdir`
 413                 echo "JavaScriptTest: Begin Test $jsfile"
 414                 if eval $TIMECOMMAND timed_run.py $TEST_JSEACH_TIMEOUT \"$jsfile\" \
 415                     $EXECUTABLE_DRIVER \
 416                     $executable -v $version \
 417                     -S 524288 \
 418                     $gczealshell \
 419                     $splitobjects \
 420                     $jitshell \
 421                     -f ./shell.js \
 422                     -f $suitetestdir/shell.js \
 423                     -f $subsuitetestdir/shell.js \
 424                     -f ./$jsfile \
 425                     -f ./js-test-driver-end.js; then
 426                     true
 427                 else
 428                     rc=$?
 429                 fi
 430                 if [[ $rc == 99 ]]; then
 431                     # note that this loop is executing in a sub-process
 432                     # error will terminate the sub-process but will transfer
 433                     # control to the next statement following the loop which
 434                     # in this case is the "End Run" output which incorrectly
 435                     # labels the test run as completed.
 436                     error "User Interrupt"
 437                 fi
 438                 echo "JavaScriptTest: End Test $jsfile"
 439             fi
 440         done
 441         rc=$?
 442         if [[ $rc != 0 ]]; then
 443             error ""
 444         fi
 445         echo "JavaScriptTest: End Run"
 446         ;;
 447
 448     browser)
 449         urllist="urllist-$branch-$testtype-$buildtype.tests"
 450         urlhtml="urllist-$branch-$testtype-$buildtype.html"
 451
 452         rm -f $urllist $urlhtml
 453
 454         cat > $urlhtml <<EOF
 455 <html xmlns="http://www.w3.org/1999/xhtml">
 456 <head>
 457 <title>JavaScript Tests</title>
 458 </head>
 459 <body>
 460 <ul>
 461 EOF
 462
 463         cat $includetestsfile | while read jsfile
 464         do
 465             if echo $jsfile | grep -q '^#'; then
 466                 continue
 467             fi
 468
 469             if ! grep -q $jsfile $excludetestsfile; then
 470
 471                 version=";version=`browserfileversion $jsfile`"
 472
 473                 echo "http://$TEST_HTTP/$TEST_WWW_JS/js-test-driver-standards.html?test=$jsfile;language=type;text/javascript$version$gczealbrowser$jitbrowser" >> $urllist
 474                 echo "<li><a href='http://$TEST_HTTP/$TEST_WWW_JS/js-test-driver-standards.html?test=$jsfile;language=type;text/javascript$version$gczealbrowser$jitbrowser'>$jsfile</a></li>" >> $urlhtml
 475             fi
 476         done
 477
 478         cat >> $urlhtml <<EOF
 479 </ul>
 480 </body>
 481 </html>
 482 EOF
 483
 484         chmod a+r $urlhtml
 485
 486         if [[ -z "$filesonly" ]]; then
 487             echo "JavaScriptTest: Begin Run"
 488             cat "$urllist" | while read url;
 489             do
 490                 edit-talkback.sh -p "$product" -b "$branch" -x "$executablepath" -i "$url"
 491                 jsfile=`echo $url | sed "s|http://$TEST_HTTP/$TEST_WWW_JS/js-test-driver-standards.html?test=\([^;]*\);.*|\1|"`
 492                 echo "JavaScriptTest: Begin Test $jsfile"
 493                 if eval $TIMECOMMAND timed_run.py $TEST_JSEACH_TIMEOUT \"$jsfile\" \
 494                     $EXECUTABLE_DRIVER \
 495                     \"$executable\" -P \"$profilename\" \
 496                     -spider -start -quit \
 497                     -uri \"$url\" \
 498                     -depth 0 -timeout \"$TEST_JSEACH_PAGE_TIMEOUT\" \
 499                     -hook \"http://$TEST_HTTP/$TEST_WWW_JS/userhookeach.js\"; then
 500                     true
 501                 else
 502                     rc=$?
 503                 fi
 504                 if [[ $rc == 99 ]]; then
 505                     error "User Interrupt"
 506                 fi
 507                 echo "JavaScriptTest: End Test $jsfile"
 508             done
 509             echo "JavaScriptTest: End Run"
 510         fi
 511         ;;
 512     *)
 513         ;;
 514 esac
 515
 516 popd