tools/valgrind/shard-all-tests.sh

   1 #!/bin/sh
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 # Script to run tests under tools/valgrind/chrome_tests.sh
   7 # in a loop looking for rare/flaky valgrind warnings, and
   8 # generate suppressions for them, to be later filed as bugs
   9 # and added to our suppressions file.
  10 #
  11 # FIXME: Layout tests are a bit funny - they have their own
  12 # sharding control, and should probably be tweaked to obey
  13 # GTEST_SHARD_INDEX/GTEST_TOTAL_SHARDS like the rest,
  14 # but they take days and days to run, so they are left
  15 # out of this script.
  16
  17 if test ! -d chrome
  18 then
  19   echo "Please run from parent directory of chrome and build directories"
  20   exit 1
  21 fi
  22
  23 if test "$1" = ""
  24 then
  25   echo "Usage: shard-all-tests.sh [BUILDTYPE=Release] target [target ...]"
  26   echo "Example: shard-all-tests.sh ui_tests"
  27   exit 1
  28 fi
  29
  30 set -x
  31 set -e
  32
  33 # Regexp to match any valgrind error
  34 PATTERN="ERROR SUMMARY: [^0]|are definitely|uninitialised|Unhandled exception|\
  35 Invalid read|Invalid write|Invalid free|Source and desti|Mismatched free|\
  36 unaddressable byte|vex x86|impossible|Assertion|INTERNAL ERROR|finish writing|OUCH"
  37
  38 BUILDTYPE=Debug
  39 case "$1" in
  40   BUILDTYPE=Debug) BUILDTYPE=Debug ; shift ;;
  41   BUILDTYPE=Release) BUILDTYPE=Release ; shift ;;
  42   BUILDTYPE=*) echo "unknown build type $1"; exit 1;;
  43   *) ;;
  44 esac
  45 TESTS="$@"
  46
  47 what_to_build() {
  48   echo $TESTS | tr ' ' '\012' | grep -v layout_tests || true
  49   echo $TESTS | grep -q layout_tests && echo test_shell || true
  50   echo $TESTS | grep -q ui_tests && echo chrome || true
  51 }
  52
  53 # Wrap xcodebuild to take same arguments as our make, more or less
  54 xcodemake() {
  55   for target in $*
  56   do
  57     case $target in
  58     chrome)         xcodebuild -configuration $BUILDTYPE -project chrome/chrome.xcodeproj -target chrome ;;
  59     ui_tests)       xcodebuild -configuration $BUILDTYPE -project chrome/chrome.xcodeproj -target ui_tests ;;
  60     base_unittests) xcodebuild -configuration $BUILDTYPE -project base/base.xcodeproj     -target base_unittests ;;
  61     net_unittests)  xcodebuild -configuration $BUILDTYPE -project net/net.xcodeproj       -target net_unittests ;;
  62     *) echo "dunno how to build $target yet"; exit 1 ;;
  63     esac
  64   done
  65 }
  66
  67 build_tests() {
  68   buildtype=$1
  69   shift
  70
  71   OS=`uname`
  72   case $OS in
  73   Linux)
  74     # Lame way to autodetect whether 'make' or 'hammer' is in use
  75     if test -d out
  76     then
  77       make -j4 BUILDTYPE=$1 $@
  78     else
  79       # fixme: obey buildtype
  80       hammer $@
  81     fi
  82     ;;
  83   Darwin)
  84     xcodemake $@
  85     ;;
  86   *) echo "don't know how to build on os $OS"
  87     ;;
  88   esac
  89 }
  90
  91 TESTS_BUILDABLE=`what_to_build`
  92 echo building $TESTS_BUILDABLE
  93 build_tests $BUILDTYPE $TESTS_BUILDABLE
  94
  95 # Divide each test suite up into 100 shards, as first step
  96 # in tracking down exact source of errors.
  97 export GTEST_TOTAL_SHARDS=100
  98
  99 rm -rf *.vlog *.vtmp || true
 100
 101 iter=0
 102 while test $iter -lt 1000
 103 do
 104   for testname in $TESTS
 105   do
 106     export GTEST_SHARD_INDEX=0
 107     while test $GTEST_SHARD_INDEX -lt $GTEST_TOTAL_SHARDS
 108     do
 109       i=$GTEST_SHARD_INDEX
 110       sh tools/valgrind/chrome_tests.sh -b xcodebuild/$BUILDTYPE -t ${testname} --tool_flags="--nocleanup_on_exit" > ${testname}_$i.vlog 2>&1 || true
 111       mv valgrind.tmp ${testname}_$i.vtmp
 112       GTEST_SHARD_INDEX=`expr $GTEST_SHARD_INDEX + 1`
 113     done
 114   done
 115
 116   # Save any interesting log files from this iteration
 117   # Also show interesting lines on stdout, to make tail -f more interesting
 118   if egrep "$PATTERN" *.vlog
 119   then
 120     mkdir -p shard-results/$iter
 121     mv `egrep -l "$PATTERN" *.vlog` shard-results/$iter
 122     # ideally we'd only save the .vtmp's corresponding to the .vlogs we saved
 123     mv *.vtmp shard-results/$iter
 124   fi
 125
 126   rm -rf *.vlog *.vtmp || true
 127   iter=`expr $iter + 1`
 128 done