js/tests/test.sh

   1 #!/bin/bash -e
   2 # -*- Mode: Shell-script; tab-width: 4; indent-tabs-mode: nil; -*-
   3
   4 # ***** BEGIN LICENSE BLOCK *****
   5 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   6 #
   7 # The contents of this file are subject to the Mozilla Public License Version
   8 # 1.1 (the "License"); you may not use this file except in compliance with
   9 # the License. You may obtain a copy of the License at
  10 # http://www.mozilla.org/MPL/
  11 #
  12 # Software distributed under the License is distributed on an "AS IS" basis,
  13 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  14 # for the specific language governing rights and limitations under the
  15 # License.
  16 #
  17 # The Original Code is Mozilla JavaScript Testing Utilities
  18 #
  19 # The Initial Developer of the Original Code is
  20 # Mozilla Corporation.
  21 # Portions created by the Initial Developer are Copyright (C) 2007
  22 # the Initial Developer. All Rights Reserved.
  23 #
  24 # Contributor(s): Bob Clary <bclary@bclary.com>
  25 #
  26 # Alternatively, the contents of this file may be used under the terms of
  27 # either the GNU General Public License Version 2 or later (the "GPL"), or
  28 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  29 # in which case the provisions of the GPL or the LGPL are applicable instead
  30 # of those above. If you wish to allow use of your version of this file only
  31 # under the terms of either the GPL or the LGPL, and not to allow others to
  32 # use your version of this file under the terms of the MPL, indicate your
  33 # decision by deleting the provisions above and replace them with the notice
  34 # and other provisions required by the GPL or the LGPL. If you do not delete
  35 # the provisions above, a recipient may use your version of this file under
  36 # the terms of any one of the MPL, the GPL or the LGPL.
  37 #
  38 # ***** END LICENSE BLOCK *****
  39
  40 if [[ -z "$TEST_DIR" ]]; then
  41     cat <<EOF
  42 `basename $0`: error
  43
  44 TEST_DIR, the location of the Sisyphus framework,
  45 is required to be set prior to calling this script.
  46 EOF
  47     exit 2
  48 fi
  49
  50 if [[ ! -e $TEST_DIR/bin/library.sh ]]; then
  51     echo "TEST_DIR=$TEST_DIR"
  52     echo ""
  53     echo "This script requires the Sisyphus testing framework. Please "
  54     echo "cvs check out the Sisyphys framework from mozilla/testing/sisyphus"
  55     echo "and set the environment variable TEST_DIR to the directory where it"
  56     echo "located."
  57     echo ""
  58
  59     exit 2
  60 fi
  61
  62 source $TEST_DIR/bin/library.sh
  63
  64 TEST_JSDIR=${TEST_JSDIR:-$TEST_DIR/tests/mozilla.org/js}
  65
  66 TEST_JSSHELL_TIMEOUT=${TEST_JSSHELL_TIMEOUT:-480}
  67 TEST_JSEACH_TIMEOUT=${TEST_JSEACH_TIMEOUT:-485}
  68 TEST_JSEACH_PAGE_TIMEOUT=${TEST_JSEACH_PAGE_TIMEOUT:-480}
  69 TEST_JSALL_TIMEOUT=${TEST_JSALL_TIMEOUT:-21600}
  70 TEST_WWW_JS=`echo $TEST_JSDIR|sed "s|$TEST_DIR||"`
  71
  72 #
  73 # options processing
  74 #
  75 usage()
  76 {
  77     cat <<EOF
  78 usage: test.sh -p product -b branch -T buildtype -x executablepath -N profilename \\
  79                [-X excludetests] [-I includetests] [-c] [-t] [-F] [-d datafiles]
  80
  81 variable            description
  82 ===============     ============================================================
  83 -p product          required. firefox|thunderbird|js
  84 -b branch           required. 1.8.0|1.8.1|1.9.0|1.9.1
  85 -s jsshellsourcepath       required for shell. path to js shell source directory mozilla/js/src
  86 -T buildtype        required. one of opt debug
  87 -x executablepath   required for browser. directory-tree containing executable 'product'
  88 -N profilename      required for browser. profile name
  89 -X excludetests     optional. By default the test will exclude the
  90                     tests listed in spidermonkey-n-\$branch.tests,
  91                     performance-\$branch.tests. excludetests is a list of either
  92                     individual tests, manifest files or sub-directories which
  93                     will override the default exclusion list.
  94 -I includetests     optional. By default the test will include the
  95                     JavaScript tests appropriate for the branch. includetests is a
  96                     list of either individual tests, manifest files or
  97                     sub-directories which will override the default inclusion
  98                     list.
  99 -c                  optional. By default the test will exclude tests
 100                     which crash on this branch, test type, build type and
 101                     operating system. -c will include tests which crash.
 102                     Typically this should only be used in combination with -R.
 103                     This has no effect on shell based tests which execute crash
 104                     tests regardless.
 105 -t                  optional. By default the test will exclude tests
 106                     which time out on this branch, test type, build type and
 107                     operating system. -t will include tests which timeout.
 108 -J jsoptions        optional. Set JavaScript options:
 109                     -Z n Set gczeal to n. Currently, only valid for
 110                     debug builds of Gecko 1.8.1.15, 1.9.0 and later.
 111                     -z optional. use split objects in the shell.
 112                     -j optional. use JIT in the shell. Only available on 1.9.1 and later
 113 -F                  optional. Just generate file lists without running any tests.
 114 -d datafiles        optional. one or more filenames of files containing
 115                     environment variable definitions to be included.
 116
 117                     note that the environment variables should have the same
 118                     names as in the "variable" column.
 119
 120 if an argument contains more than one value, it must be quoted.
 121 EOF
 122     exit 2
 123 }
 124
 125 while getopts "p:b:s:T:x:N:d:X:I:J:RctF" optname
 126 do
 127     case $optname in
 128         p)
 129             product=$OPTARG;;
 130         b)
 131             branch=$OPTARG;;
 132         T)
 133             buildtype=$OPTARG;;
 134         s)
 135             jsshellsourcepath=$OPTARG;;
 136         N)
 137             profilename=$OPTARG;;
 138         x)
 139             executablepath=$OPTARG;;
 140         X)
 141             excludetests=$OPTARG;;
 142         I)
 143             includetests=$OPTARG;;
 144         c)
 145             crashes=1;;
 146         t)
 147             timeouts=1;;
 148         F)
 149             filesonly=1;;
 150         J)
 151             javascriptoptions=$OPTARG
 152             ;;
 153         d) datafiles=$OPTARG;;
 154     esac
 155 done
 156
 157 if [[ -n "$javascriptoptions" ]]; then
 158     unset OPTIND
 159     while getopts "Z:jz" optname $javascriptoptions; do
 160         case $optname in
 161             Z)
 162                 gczealshell="-Z $OPTARG"
 163                 gczealbrowser=";gczeal=$OPTARG"
 164                 ;;
 165             z)
 166                 splitobjects="-z"
 167                 ;;
 168             j)
 169                 jit="-j"
 170                 ;;
 171         esac
 172     done
 173 fi
 174
 175 # include environment variables
 176 if [[ -n "$datafiles" ]]; then
 177     for datafile in $datafiles; do
 178         source $datafile
 179     done
 180 fi
 181
 182 if [[ -n "$gczeal" && "$buildtype" != "debug" ]]; then
 183     error "gczeal is supported for buildtype debug and not $buildtype"
 184 fi
 185
 186 dumpvars product branch buildtype jsshellsourcepath profilename executablepath excludetests includetests crashes timeouts filesonly javascriptoptions datafiles | sed "s|^|arguments: |"
 187
 188 pushd $TEST_JSDIR
 189
 190 case $product in
 191     js)
 192         if [[ -z "$branch" || -z "$buildtype"  || -z "$jsshellsourcepath" ]]; then
 193             usage
 194         fi
 195         source config.sh
 196         testtype=shell
 197         executable="$jsshellsourcepath/$JS_OBJDIR/js$EXE_EXT"
 198         ;;
 199
 200     firefox|thunderbird)
 201         if [[ -z "$branch" || -z "$buildtype"  || -z "$executablepath" || -z "$profilename" ]]; then
 202             usage
 203         fi
 204         testtype=browser
 205         executable=`get_executable $product $branch $executablepath`
 206
 207         ;;
 208     *)
 209         echo "Unknown product: $product"
 210         usage
 211         ;;
 212 esac
 213
 214 function shellfileversion()
 215 {
 216     local jsfile=$1
 217     local version
 218
 219     case $jsfile in
 220         ecma/*)
 221             version=150;;
 222         ecma_2/*)
 223             version=150;;
 224         ecma_3/*)
 225             version=150;;
 226         ecma_3_1/*)
 227             version=180;;
 228         js1_1/*)
 229             version=150;;
 230         js1_2/*)
 231             version=150;;
 232         js1_3/*)
 233             version=150;;
 234         js1_4/*)
 235             version=150;;
 236         js1_5/*)
 237             version=150;;
 238         js1_6/*)
 239             version=160;;
 240         js1_7/*)
 241             version=170;;
 242         js1_8/*)
 243             version=180;;
 244         js1_8_1/*)
 245             version=180;;
 246         js1_9/*)
 247             version=190;;
 248         js2_0/*)
 249             version=200;;
 250         *)
 251             version=150;;
 252     esac
 253     echo $version
 254 }
 255
 256 function browserfileversion()
 257 {
 258     local jsfile=$1
 259     local version
 260
 261     case $jsfile in
 262         ecma/*)
 263             version=1.5;;
 264         ecma_2/*)
 265             version=1.5;;
 266         ecma_3/*)
 267             version=1.5;;
 268         ecma_3_1/*)
 269             version=1.8;;
 270         js1_1/*)
 271             version=1.1;;
 272         js1_2/*)
 273             version=1.5;;
 274         js1_3/*)
 275             version=1.5;;
 276         js1_4/*)
 277             version=1.5;;
 278         js1_5/*)
 279             version=1.5;;
 280         js1_6/*)
 281             version=1.6;;
 282         js1_7/*)
 283             version=1.7;;
 284         js1_8/*)
 285             version=1.8;;
 286         js1_8_1/*)
 287             version=1.8;;
 288         js1_9/*)
 289             version=1.9;;
 290         js2_0/*)
 291             version=2.0;;
 292         *)
 293             version=1.5;;
 294     esac
 295     echo $version
 296 }
 297
 298 rm -f finished-$branch-$testtype-$buildtype
 299
 300 if ! make failures.txt; then
 301     error "during make failures.txt" $LINENO
 302 fi
 303
 304 includetestsfile="included-$branch-$testtype-$buildtype.tests"
 305 rm -f $includetestsfile
 306 touch $includetestsfile
 307
 308 if [[ -z "$includetests" ]]; then
 309     # by default include tests appropriate for the branch
 310     includetests="e4x ecma ecma_2 ecma_3 js1_1 js1_2 js1_3 js1_4 js1_5 js1_6"
 311
 312     case "$branch" in
 313         1.8.0)
 314             ;;
 315         1.8.1)
 316             includetests="$includetests js1_7"
 317             ;;
 318         1.9.0)
 319             includetests="$includetests js1_7 js1_8"
 320             ;;
 321         1.9.1)
 322             includetests="$includetests js1_7 js1_8 ecma_3_1 js1_8_1"
 323             ;;
 324     esac
 325 fi
 326
 327 for i in $includetests; do
 328     if [[ -f "$i" ]]; then
 329         echo "# including $i" >> $includetestsfile
 330         if echo $i | grep -q '\.js$'; then
 331             echo $i >> $includetestsfile
 332         else
 333             cat $i >> $includetestsfile
 334         fi
 335     elif [[ -d "$i" ]]; then
 336         find $i -name '*.js' -print | egrep -v '(shell|browser|template|jsref|userhook.*|\.#.*)\.js' | sed 's/^\.\///' | sort >> $includetestsfile
 337     fi
 338 done
 339
 340 excludetestsfile="excluded-$branch-$testtype-$buildtype.tests"
 341 rm -f $excludetestsfile
 342 touch $excludetestsfile
 343
 344 if [[ -z "$excludetests" ]]; then
 345     excludetests="spidermonkey-n-$branch.tests performance-$branch.tests"
 346 fi
 347
 348 for e in $excludetests; do
 349     if [[ -f "$e" ]]; then
 350         echo "# excluding $e" >> $excludetestsfile
 351         if echo $e | grep -q '\.js$'; then
 352             echo $e >> $excludetestsfile
 353         else
 354             cat $e >> $excludetestsfile
 355         fi
 356     elif [[ -d "$e" ]]; then
 357         find $e -name '*.js' -print | egrep -v '(shell|browser|template|jsref|userhook.*|\.#.*)\.js' | sed 's/^\.\///' | sort >> $excludetestsfile
 358     fi
 359 done
 360
 361 # convert the numeric speed rating to a prose value
 362 if [[ $TEST_CPUSPEED -lt 4 ]]; then
 363     TEST_CPUSPEED=slow
 364 elif [[ $TEST_CPUSPEED -lt 9 ]]; then
 365     TEST_CPUSPEED=medium
 366 else
 367     TEST_CPUSPEED=fast
 368 fi
 369
 370 if [[ -z "$TEST_MOZILLA_HG" ]]; then
 371     repo=CVS
 372 else
 373     repo=`basename $TEST_MOZILLA_HG`
 374 fi
 375 debug "repo=$repo"
 376
 377 pattern="TEST_BRANCH=($branch|[.][*]), TEST_REPO=($repo|[.][*]), TEST_BUILDTYPE=($buildtype|[.][*]), TEST_TYPE=($testtype|[.][*]), TEST_OS=($OSID|[.][*]), TEST_KERNEL=($TEST_KERNEL|[.][*]), TEST_PROCESSORTYPE=($TEST_PROCESSORTYPE|[.][*]), TEST_MEMORY=($TEST_MEMORY|[.][*]), TEST_CPUSPEED=($TEST_CPUSPEED|[.][*]),"
 378
 379 if [[ -z "$timeouts" ]]; then
 380     echo "# exclude tests that time out" >> $excludetestsfile
 381     echo "$pattern .*TEST_EXITSTATUS=TIMED OUT," >> $excludetestsfile
 382     egrep "$pattern .*TEST_EXITSTATUS=TIMED OUT," failures.txt | \
 383         sed 's/.*TEST_ID=\([^,]*\),.*/\1/' | sort -u >> $excludetestsfile
 384 fi
 385
 386 if [[ -z "$crashes" ]]; then
 387     echo "# exclude tests that crash" >> $excludetestsfile
 388     echo "$pattern .*TEST_EXITSTATUS=(CRASHED|ABNORMAL)" >> $excludetestsfile
 389     egrep "$pattern .*TEST_EXITSTATUS=(CRASHED|ABNORMAL)" failures.txt  | \
 390         sed 's/.*TEST_ID=\([^,]*\),.*/\1/' | sort -u >> $excludetestsfile
 391
 392 fi
 393
 394 cat $includetestsfile | sed 's|^|include: |'
 395 cat $excludetestsfile | sed 's|^|exclude: |'
 396
 397 case $testtype in
 398     shell)
 399         echo "JavaScriptTest: Begin Run"
 400         cat $includetestsfile | while read jsfile
 401         do
 402             if echo $jsfile | grep -q '^#'; then
 403                 continue
 404             fi
 405
 406             if ! grep -q $jsfile $excludetestsfile; then
 407
 408                 version=`shellfileversion $jsfile`
 409
 410                 subsuitetestdir=`dirname $jsfile`
 411                 suitetestdir=`dirname $subsuitetestdir`
 412                 echo "JavaScriptTest: Begin Test $jsfile"
 413                 if eval $TIMECOMMAND timed_run.py $TEST_JSEACH_TIMEOUT \"$jsfile\" \
 414                     $EXECUTABLE_DRIVER \
 415                     $executable -v $version \
 416                     -S 524288 \
 417                     $gczealshell \
 418                     $splitobjects \
 419                     $jit \
 420                     -f ./shell.js \
 421                     -f $suitetestdir/shell.js \
 422                     -f $subsuitetestdir/shell.js \
 423                     -f ./$jsfile \
 424                     -f ./js-test-driver-end.js; then
 425                     true
 426                 else
 427                     rc=$?
 428                 fi
 429                 if [[ $rc == 99 ]]; then
 430                     # note that this loop is executing in a sub-process
 431                     # error will terminate the sub-process but will transfer
 432                     # control to the next statement following the loop which
 433                     # in this case is the "End Run" output which incorrectly
 434                     # labels the test run as completed.
 435                     error "User Interrupt"
 436                 fi
 437                 echo "JavaScriptTest: End Test $jsfile"
 438             fi
 439         done
 440         rc=$?
 441         if [[ $rc != 0 ]]; then
 442             error ""
 443         fi
 444         echo "JavaScriptTest: End Run"
 445         ;;
 446
 447     browser)
 448         urllist="urllist-$branch-$testtype-$buildtype.tests"
 449         urlhtml="urllist-$branch-$testtype-$buildtype.html"
 450
 451         rm -f $urllist $urlhtml
 452
 453         cat > $urlhtml <<EOF
 454 <html xmlns="http://www.w3.org/1999/xhtml">
 455 <head>
 456 <title>JavaScript Tests</title>
 457 </head>
 458 <body>
 459 <ul>
 460 EOF
 461
 462         cat $includetestsfile | while read jsfile
 463         do
 464             if echo $jsfile | grep -q '^#'; then
 465                 continue
 466             fi
 467
 468             if ! grep -q $jsfile $excludetestsfile; then
 469
 470                 version=";version=`browserfileversion $jsfile`"
 471
 472                 echo "http://$TEST_HTTP/$TEST_WWW_JS/js-test-driver-standards.html?test=$jsfile;language=type;text/javascript$version$gczealbrowser" >> $urllist
 473                 echo "<li><a href='http://$TEST_HTTP/$TEST_WWW_JS/js-test-driver-standards.html?test=$jsfile;language=type;text/javascript$version$gczealbrowser'>$jsfile</a></li>" >> $urlhtml
 474             fi
 475         done
 476
 477         cat >> $urlhtml <<EOF
 478 </ul>
 479 </body>
 480 </html>
 481 EOF
 482
 483         chmod a+r $urlhtml
 484
 485         if [[ -z "$filesonly" ]]; then
 486             echo "JavaScriptTest: Begin Run"
 487             cat "$urllist" | while read url;
 488             do
 489                 edit-talkback.sh -p "$product" -b "$branch" -x "$executablepath" -i "$url"
 490                 jsfile=`echo $url | sed "s|http://$TEST_HTTP/$TEST_WWW_JS/js-test-driver-standards.html?test=\([^;]*\);.*|\1|"`
 491                 echo "JavaScriptTest: Begin Test $jsfile"
 492                 if eval $TIMECOMMAND timed_run.py $TEST_JSEACH_TIMEOUT \"$jsfile\" \
 493                     $EXECUTABLE_DRIVER \
 494                     \"$executable\" -P \"$profilename\" \
 495                     -spider -start -quit \
 496                     -uri \"$url\" \
 497                     -depth 0 -timeout \"$TEST_JSEACH_PAGE_TIMEOUT\" \
 498                     -hook \"http://$TEST_HTTP/$TEST_WWW_JS/userhookeach.js\"; then
 499                     true
 500                 else
 501                     rc=$?
 502                 fi
 503                 if [[ $rc == 99 ]]; then
 504                     error "User Interrupt"
 505                 fi
 506                 echo "JavaScriptTest: End Test $jsfile"
 507             done
 508             echo "JavaScriptTest: End Run"
 509         fi
 510         ;;
 511     *)
 512         ;;
 513 esac
 514
 515 popd