js/tests/bisect.sh

   1 #!/bin/bash -e
   2 # -*- Mode: Shell-script; tab-width: 4; indent-tabs-mode: nil; -*-
   3
   4 # ***** BEGIN LICENSE BLOCK *****
   5 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   6 #
   7 # The contents of this file are subject to the Mozilla Public License Version
   8 # 1.1 (the "License"); you may not use this file except in compliance with
   9 # the License. You may obtain a copy of the License at
  10 # http://www.mozilla.org/MPL/
  11 #
  12 # Software distributed under the License is distributed on an "AS IS" basis,
  13 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  14 # for the specific language governing rights and limitations under the
  15 # License.
  16 #
  17 # The Original Code is Mozilla JavaScript Testing Utilities
  18 #
  19 # The Initial Developer of the Original Code is
  20 # Mozilla Corporation.
  21 # Portions created by the Initial Developer are Copyright (C) 2008
  22 # the Initial Developer. All Rights Reserved.
  23 #
  24 # Contributor(s): Bob Clary <bclary@bclary.com>
  25 #
  26 # Alternatively, the contents of this file may be used under the terms of
  27 # either the GNU General Public License Version 2 or later (the "GPL"), or
  28 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  29 # in which case the provisions of the GPL or the LGPL are applicable instead
  30 # of those above. If you wish to allow use of your version of this file only
  31 # under the terms of either the GPL or the LGPL, and not to allow others to
  32 # use your version of this file under the terms of the MPL, indicate your
  33 # decision by deleting the provisions above and replace them with the notice
  34 # and other provisions required by the GPL or the LGPL. If you do not delete
  35 # the provisions above, a recipient may use your version of this file under
  36 # the terms of any one of the MPL, the GPL or the LGPL.
  37 #
  38 # ***** END LICENSE BLOCK *****
  39
  40 if [[ -z "$TEST_DIR" ]]; then
  41     cat <<EOF
  42 `basename $0`: error
  43
  44 TEST_DIR, the location of the Sisyphus framework,
  45 is required to be set prior to calling this script.
  46
  47 EOF
  48     exit 2
  49 fi
  50
  51 if [[ ! -e $TEST_DIR/bin/library.sh ]]; then
  52     cat <<EOF
  53 TEST_DIR=$TEST_DIR"
  54
  55 This script requires the Sisyphus testing framework. Please
  56 cvs check out the Sisyphys framework from mozilla/testing/sisyphus
  57 and set the environment variable TEST_DIR to the directory where it
  58 located.
  59
  60 EOF
  61     exit 2
  62 fi
  63
  64 source $TEST_DIR/bin/library.sh
  65
  66 usage()
  67 {
  68     cat <<EOF
  69 usage: bisect.sh -p product -b branch -e extra\\
  70                    -T  buildtype \\
  71                    -t test \\
  72                    -S string \\
  73                    -G good -B bad \\
  74                    [-J javascriptoptions]
  75
  76     variable            description
  77     ===============     ============================================================
  78     -p bisect_product     one of js, firefox
  79     -b bisect_branches    one of branches 1.8.0, 1.8.1, 1.9.0, 1.9.1
  80     -e bisect_extra       optional. extra qualifier to pick build tree and mozconfig.
  81     -T bisect_buildtype   one of build types opt debug
  82     -t bisect_test        Test to be bisected.
  83     -S bisect_string      optional. String containing a regular expression which
  84                           can be used to distinguish different failures for the given
  85                           test. The script will search the failure log for the pattern
  86                           "\$bisect_test.*\$string" to determine if a test failure
  87                           matches the expected failure.
  88                           bisect_string can contain any extended regular expressions
  89                           supported by egrep.
  90     -G bisect_good        For branches 1.8.0, 1.8.1, 1.9.0, date test passed
  91                           For branch 1.9.1, revision test passed
  92     -B bisect_bad         For branches, 1.8.0, 1.8.1, 1.9.0 date test failed
  93                           For branch 1.9.1, revision test failed.
  94
  95        If the good revision (test passed) occurred  prior to the bad revision
  96        (test failed), the script will search for the first bad revision which
  97        caused the test to regress.
  98
  99        If the bad revision (test failed) occurred prior to the good revision
 100        (test passed), the script will search for the first good revision which
 101        fixed the failing test.
 102
 103     -J javascriptoptions  optional. Set JavaScript options:
 104          -Z n Set gczeal to n. Currently, only valid for
 105               debug builds of Gecko 1.8.1.15, 1.9.0 and later.
 106          -z optional. use split objects in the shell.
 107          -j optional. use JIT in the shell. Only available on 1.9.1 and later
 108 EOF
 109
 110     exit 2
 111 }
 112
 113 verbose=0
 114
 115 while getopts "p:b:T:e:t:S:G:B:J:" optname;
 116 do
 117     case $optname in
 118         p) bisect_product=$OPTARG;;
 119         b) bisect_branch=$OPTARG;;
 120         T) bisect_buildtype=$OPTARG;;
 121         e) bisect_extra="$OPTARG"
 122             bisect_extraflag="-e $OPTARG";;
 123         t) bisect_test="$OPTARG";;
 124         S) bisect_string="$OPTARG";;
 125         G) bisect_good="$OPTARG";;
 126         B) bisect_bad="$OPTARG";;
 127         J) javascriptoptions=$OPTARG;;
 128     esac
 129 done
 130
 131 # javascriptoptions will be passed by environment to runtests.sh
 132
 133 if [[ -z "$bisect_product" || -z "$bisect_branch" || -z "$bisect_buildtype" || -z "$bisect_test" || -z "$bisect_good" || -z "$bisect_bad" ]]; then
 134     echo "bisect_product: $bisect_product, bisect_branch: $bisect_branch, bisect_buildtype: $bisect_buildtype, bisect_test: $bisect_test, bisect_good: $bisect_good, bisect_bad: $bisect_bad"
 135     usage
 136 fi
 137
 138 OPTIND=1
 139
 140 eval source $TEST_DIR/bin/set-build-env.sh -p $bisect_product -b $bisect_branch $bisect_extraflag -T $bisect_buildtype > /dev/null
 141
 142 # TEST_JSDIR must be set after set-build-env.sh is called
 143 # on Windows since TEST_DIR can be modified in set-build-env.sh
 144 # from the pure cygwin path /work/... to a the cygwin windows path
 145 # /c/work/...
 146 TEST_JSDIR=${TEST_JSDIR:-$TEST_DIR/tests/mozilla.org/js}
 147
 148 case $bisect_branch in
 149     1.8.*|1.9.0)
 150
 151         #
 152         # binary search using CVS
 153         #
 154
 155         # convert dates to seconds for ordering
 156         localgood=`dateparse.pl $bisect_good`
 157         localbad=`dateparse.pl $bisect_bad`
 158
 159         # if good < bad, then we are searching for a regression,
 160         # i.e. the first bad changeset
 161         # if bad < good, then we are searching for a fix.
 162         # i.e. the first good changeset. so we reverse the nature
 163         # of good and bad.
 164
 165         if (( $localgood < $localbad )); then
 166             cat <<EOF
 167
 168 searching for a regression between $bisect_good and $bisect_bad
 169 the bisection is searching for the transition from test failure not found, to test failure found.
 170
 171 EOF
 172
 173             searchtype="regression"
 174             bisect_start=$bisect_good
 175             bisect_stop=$bisect_bad
 176         else
 177             cat <<EOF
 178
 179 searching for a fix between $bisect_bad and $bisect_good
 180 the bisection is searching for the transition from test failure found to test failure not found.
 181
 182 EOF
 183
 184             searchtype="fix"
 185             bisect_start=$bisect_bad
 186             bisect_stop=$bisect_good
 187         fi
 188
 189         let seconds_start="`dateparse.pl $bisect_start`"
 190         let seconds_stop="`dateparse.pl $bisect_stop`"
 191
 192         echo "checking that the test fails in the bad revision $bisect_bad"
 193         eval $TEST_DIR/bin/builder.sh -p $bisect_product -b $bisect_branch $bisect_extraflag -T $bisect_buildtype -B "clobber" > /dev/null
 194         export MOZ_CO_DATE="$bisect_bad"
 195         eval $TEST_DIR/bin/builder.sh -p $bisect_product -b $bisect_branch $bisect_extraflag -T $bisect_buildtype -B "checkout" > /dev/null
 196         bisect_log=`eval $TEST_JSDIR/runtests.sh -p $bisect_product -b $bisect_branch $bisect_extraflag -T $bisect_buildtype -I $bisect_test -B "build" -c -t -X /dev/null 2>&1 | grep '_js.log $' | sed 's|log: \([^ ]*\) |\1|'`
 197         if [[ -z "$bisect_log" ]]; then
 198             echo "test $bisect_test not run."
 199         else
 200             if egrep -q "$bisect_test.*$bisect_string" ${bisect_log}-results-failures.log; then
 201                 echo "test failure $bisect_test.*$bisect_string found, bad revision $bisect_bad confirmed"
 202                 bad_confirmed=1
 203             else
 204                 echo "test failure $bisect_test.*$bisect_string not found, bad revision $bisect_bad *not* confirmed"
 205             fi
 206         fi
 207
 208         if [[ "$bad_confirmed" != "1" ]]; then
 209             error "bad revision not confirmed";
 210         fi
 211
 212         echo "checking that the test passes in the good revision $bisect_good"
 213         eval $TEST_DIR/bin/builder.sh -p $bisect_product -b $bisect_branch $bisect_extraflag -T $bisect_buildtype -B "clobber" > /dev/null
 214         export MOZ_CO_DATE="$bisect_good"
 215         eval $TEST_DIR/bin/builder.sh -p $bisect_product -b $bisect_branch $bisect_extraflag -T $bisect_buildtype -B "checkout" > /dev/null
 216
 217         bisect_log=`eval $TEST_JSDIR/runtests.sh -p $bisect_product -b $bisect_branch $bisect_extraflag -T $bisect_buildtype -I $bisect_test -B "build" -c -t -X /dev/null 2>&1 | grep '_js.log $' | sed 's|log: \([^ ]*\) |\1|'`
 218         if [[ -z "$bisect_log" ]]; then
 219             echo "test $bisect_test not run."
 220         else
 221             if egrep -q "$bisect_test.*$bisect_string" ${bisect_log}-results-failures.log; then
 222                 echo "test failure $bisect_test.*$bisect_string found, good revision $bisect_good *not* confirmed"
 223             else
 224                 echo "test failure $bisect_test.*$bisect_string not found, good revision $bisect_good confirmed"
 225                 good_confirmed=1
 226             fi
 227         fi
 228
 229         if [[ "$good_confirmed" != "1" ]]; then
 230             error "good revision not confirmed";
 231         fi
 232
 233         echo "bisecting $bisect_start to $bisect_stop"
 234
 235         #
 236         # place an array of dates of checkins into an array and
 237         # perform a binary search on those dates.
 238         #
 239         declare -a seconds_array date_array
 240
 241         # load the cvs checkin dates into an array. the array will look like
 242         # date_array[i] date value
 243         # date_array[i+1] time value
 244
 245         pushd $BUILDTREE/mozilla
 246         date_array=(`cvs -q -z3 log -N -d "$bisect_start<$bisect_stop" | grep "^date: " | sed 's|^date: \([^;]*\).*|\1|' | sort -u`)
 247         popd
 248
 249         let seconds_index=0 1
 250         let date_index=0 1
 251
 252         while (( $date_index < ${#date_array[@]} )); do
 253             seconds_array[$seconds_index]=`dateparse.pl "${date_array[$date_index]} ${date_array[$date_index+1]} UTC"`
 254             let seconds_index=$seconds_index+1
 255             let date_index=$date_index+2
 256         done
 257
 258         let seconds_index_start=0 1
 259         let seconds_index_stop=${#seconds_array[@]}
 260
 261         while true; do
 262
 263             if (( $seconds_index_start+1 >= $seconds_index_stop )); then
 264                 echo "*** date `date -r ${seconds_array[$seconds_index_stop]}` found ***"
 265                 break;
 266             fi
 267
 268             unset result
 269
 270             # clobber before setting new changeset.
 271             eval $TEST_DIR/bin/builder.sh -p $bisect_product -b $bisect_branch $bisect_extraflag -T $bisect_buildtype -B "clobber" > /dev/null
 272
 273             let seconds_index_middle="($seconds_index_start + $seconds_index_stop)/2"
 274             let seconds_middle="${seconds_array[$seconds_index_middle]}"
 275
 276             bisect_middle="`date -r $seconds_middle`"
 277             export MOZ_CO_DATE="$bisect_middle"
 278             echo "testing $MOZ_CO_DATE"
 279
 280             eval $TEST_DIR/bin/builder.sh -p $bisect_product -b $bisect_branch $bisect_extraflag -T $bisect_buildtype -B "checkout" > /dev/null
 281
 282             bisect_log=`eval $TEST_JSDIR/runtests.sh -p $bisect_product -b $bisect_branch $bisect_extraflag -T $bisect_buildtype -I $bisect_test -B "build" -c -t -X /dev/null 2>&1 | grep '_js.log $' | sed 's|log: \([^ ]*\) |\1|'`
 283             if [[ -z "$bisect_log" ]]; then
 284                 echo "test $bisect_test not run. Skipping changeset"
 285                 let seconds_index_start=$seconds_index_start+1
 286             else
 287                 if [[ "$searchtype" == "regression" ]]; then
 288                     # searching for a regression, pass -> fail
 289                     if egrep -q "$bisect_test.*$bisect_string" ${bisect_log}-results-failures.log; then
 290                         echo "test failure $bisect_test.*$bisect_string found"
 291                         let seconds_index_stop=$seconds_index_middle;
 292                     else
 293                         echo "test failure $bisect_test.*$bisect_string not found"
 294                         let seconds_index_start=$seconds_index_middle
 295                     fi
 296                 else
 297                     # searching for a fix, fail -> pass
 298                     if egrep -q "$bisect_test.*$bisect_string" ${bisect_log}-results-failures.log; then
 299                         echo "test failure $bisect_test.*$bisect_string found"
 300                         let seconds_index_start=$seconds_index_middle
 301                     else
 302                         echo "test failure $bisect_test.*$bisect_string not found"
 303                         let seconds_index_stop=$seconds_index_middle
 304                     fi
 305                 fi
 306             fi
 307
 308         done
 309         ;;
 310
 311     1.9.1)
 312         #
 313         # binary search using mercurial
 314         #
 315
 316         TEST_MOZILLA_HG_LOCAL=${TEST_MOZILLA_HG_LOCAL:-$BUILDDIR/hg.mozilla.org/`basename $TEST_MOZILLA_HG`}
 317         hg -R $TEST_MOZILLA_HG_LOCAL pull -r tip
 318
 319         REPO=$BUILDTREE/mozilla
 320         hg -R $REPO pull -r tip
 321
 322         # convert revision numbers to local revision numbers for ordering
 323         localgood=`hg -R $REPO id -n -r $bisect_good`
 324         localbad=`hg -R $REPO id -n -r $bisect_bad`
 325
 326         # if good < bad, then we are searching for a regression,
 327         # i.e. the first bad changeset
 328         # if bad < good, then we are searching for a fix.
 329         # i.e. the first good changeset. so we reverse the nature
 330         # of good and bad.
 331
 332         if (( $localgood < $localbad )); then
 333             cat <<EOF
 334
 335 searching for a regression between $localgood:$bisect_good and $localbad:$bisect_bad
 336 the result is considered good when the test result does not appear in the failure log, bad otherwise.
 337 the bisection is searching for the transition from test failure not found, to test failure found.
 338
 339 EOF
 340
 341             searchtype="regression"
 342             bisect_start=$bisect_good
 343             bisect_stop=$bisect_bad
 344         else
 345             cat <<EOF
 346
 347 searching for a fix between $localbad:$bisect_bad and $localgood:$bisect_good
 348 the result is considered good when the test result does appear in the failure log, bad otherwise.
 349 the bisection is searching for the transition from test failure found to test failure not found.
 350
 351 EOF
 352
 353             searchtype="fix"
 354             bisect_start=$bisect_bad
 355             bisect_stop=$bisect_good
 356         fi
 357
 358         echo "checking that the test fails in the bad revision $localbad:$bisect_bad"
 359         eval $TEST_DIR/bin/builder.sh -p $bisect_product -b $bisect_branch $bisect_extraflag -T $bisect_buildtype -B "clobber" > /dev/null
 360         hg -R $REPO update -C -r $bisect_bad
 361         bisect_log=`eval $TEST_JSDIR/runtests.sh -p $bisect_product -b $bisect_branch $bisect_extraflag -T $bisect_buildtype -I $bisect_test -B "build" -c -t -X /dev/null 2>&1 | grep '_js.log $' | sed 's|log: \([^ ]*\) |\1|'`
 362         if [[ -z "$bisect_log" ]]; then
 363             echo "test $bisect_test not run."
 364         else
 365             if egrep -q "$bisect_test.*$bisect_string" ${bisect_log}-results-failures.log; then
 366                 echo "test failure $bisect_test.*$bisect_string found, bad revision $localbad:$bisect_bad confirmed"
 367                 bad_confirmed=1
 368             else
 369                 echo "test failure $bisect_test.*$bisect_string not found, bad revision $localbad:$bisect_bad *not* confirmed"
 370             fi
 371         fi
 372
 373         if [[ "$bad_confirmed" != "1" ]]; then
 374             error "bad revision not confirmed";
 375         fi
 376
 377         echo "checking that the test passes in the good revision $localgood:$bisect_good"
 378         eval $TEST_DIR/bin/builder.sh -p $bisect_product -b $bisect_branch $bisect_extraflag -T $bisect_buildtype -B "clobber" > /dev/null
 379         hg -R $REPO update -C -r $bisect_good
 380         bisect_log=`eval $TEST_JSDIR/runtests.sh -p $bisect_product -b $bisect_branch $bisect_extraflag -T $bisect_buildtype -I $bisect_test -B "build" -c -t -X /dev/null 2>&1 | grep '_js.log $' | sed 's|log: \([^ ]*\) |\1|'`
 381         if [[ -z "$bisect_log" ]]; then
 382             echo "test $bisect_test not run."
 383         else
 384             if egrep -q "$bisect_test.*$bisect_string" ${bisect_log}-results-failures.log; then
 385                 echo "test failure $bisect_test.*$bisect_string found, good revision $localgood:$bisect_good *not* confirmed"
 386             else
 387                 echo "test failure $bisect_test.*$bisect_string not found, good revision $localgood:$bisect_good confirmed"
 388                 good_confirmed=1
 389             fi
 390         fi
 391
 392         if [[ "$good_confirmed" != "1" ]]; then
 393             error "good revision not confirmed";
 394         fi
 395
 396         echo "bisecting $REPO $bisect_start to $bisect_stop"
 397         hg -q -R $REPO bisect --reset
 398         hg -q -R $REPO bisect --good $bisect_start
 399         hg -q -R $REPO bisect --bad $bisect_stop
 400
 401         while true; do
 402             unset result
 403
 404             # clobber before setting new changeset.
 405             eval $TEST_DIR/bin/builder.sh -p $bisect_product -b $bisect_branch $bisect_extraflag -T $bisect_buildtype -B "clobber" > /dev/null
 406
 407             hg -R $REPO bisect
 408             bisect_log=`eval $TEST_JSDIR/runtests.sh -p $bisect_product -b $bisect_branch $bisect_extraflag -T $bisect_buildtype -I $bisect_test -B "build" -c -t -X /dev/null 2>&1 | grep '_js.log $' | sed 's|log: \([^ ]*\) |\1|'`
 409             if [[ -z "$bisect_log" ]]; then
 410                 echo "test $bisect_test not run. Skipping changeset"
 411                 hg -R $REPO bisect --skip
 412             else
 413                 if [[ "$searchtype" == "regression" ]]; then
 414                     # searching for a regression
 415                     # the result is considered good when the test does not appear in the failure log
 416                     if egrep -q "$bisect_test.*$bisect_string" ${bisect_log}-results-failures.log; then
 417                         echo "test failure $bisect_test.*$bisect_string found, marking revision bad"
 418                         result=`hg -R $REPO bisect --bad 2>&1`
 419                     else
 420                         echo "test failure $bisect_test.*$bisect_string not found, marking revision good"
 421                         result=`hg -R $REPO bisect --good 2>&1`
 422                     fi
 423                 else
 424                     # searching for a fix
 425                     # the result is considered good when the test does appear in the failure log
 426                     if egrep -q "$bisect_test.*$bisect_string" ${bisect_log}-results-failures.log; then
 427                         echo "test failure $bisect_test.*$bisect_string found, marking revision good"
 428                         result=`hg -R $REPO bisect --good 2>&1`
 429                     else
 430                         echo "test failure $bisect_test.*$bisect_string not found, marking revision bad"
 431                         result=`hg -R $REPO bisect --bad 2>&1`
 432                     fi
 433                 fi
 434             fi
 435
 436             if echo $result | egrep -q "The first (good|bad) revision is:"; then
 437                 result_revision=`echo $result | sed "s|The first .* revision is:.*changeset: [0-9]*:\([^ ]*\).*|\1|"`
 438                 echo $result | sed "s|The first .* revision is:|$searchtype|"
 439                 echo "*** revision $result_revision found ***"
 440                 break
 441             fi
 442
 443         done
 444         ;;
 445 esac