tools/valgrind/waterfall.sh

   1 #!/bin/bash
   2
   3 # Copyright (c) 2011 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 # This script can be used by waterfall sheriffs to fetch the status
   8 # of Valgrind bots on the memory waterfall and test if their local
   9 # suppressions match the reports on the waterfall.
  10
  11 set -e
  12
  13 THISDIR=$(dirname "${0}")
  14 LOGS_DIR=$THISDIR/waterfall.tmp
  15 WATERFALL_PAGE="http://build.chromium.org/p/chromium.memory/builders"
  16 WATERFALL_FYI_PAGE="http://build.chromium.org/p/chromium.memory.fyi/builders"
  17
  18 download() {
  19   # Download a file.
  20   # $1 = URL to download
  21   # $2 = Path to the output file
  22   # {{{1
  23   if [ "$(which curl)" != "" ]
  24   then
  25     if ! curl -s -o "$2" "$1"
  26     then
  27       echo
  28       echo "Failed to download '$1'... aborting"
  29       exit 1
  30     fi
  31   elif [ "$(which wget)" != "" ]
  32   then
  33     if ! wget "$1" -O "$2" -q
  34     then
  35       echo
  36       echo "Failed to download '$1'... aborting"
  37       exit 1
  38     fi
  39   else
  40     echo "Need either curl or wget to download stuff... aborting"
  41     exit 1
  42   fi
  43   # }}}
  44 }
  45
  46 fetch_logs() {
  47   # Fetch Valgrind logs from the waterfall {{{1
  48
  49   # TODO(timurrrr,maruel): use JSON, see
  50   # http://build.chromium.org/p/chromium.memory/json/help
  51
  52   rm -rf "$LOGS_DIR" # Delete old logs
  53   mkdir "$LOGS_DIR"
  54
  55   echo "Fetching the list of builders..."
  56   download $1 "$LOGS_DIR/builders"
  57   SLAVES=$(grep "<a href=\"builders\/" "$LOGS_DIR/builders" | \
  58            grep 'td class="box"' | \
  59            sed "s/.*<a href=\"builders\///" | sed "s/\".*//" | \
  60            sort | uniq)
  61
  62   for S in $SLAVES
  63   do
  64     SLAVE_URL=$1/$S
  65     SLAVE_NAME=$(echo $S | sed -e "s/%20/ /g" -e "s/%28/(/g" -e "s/%29/)/g")
  66     echo -n "Fetching builds by slave '${SLAVE_NAME}'"
  67     download $SLAVE_URL?numbuilds=${NUMBUILDS} "$LOGS_DIR/slave_${S}"
  68
  69     # We speed up the 'fetch' step by skipping the builds/tests which succeeded.
  70     # TODO(timurrrr): OTOH, we won't be able to check
  71     # if some suppression is not used anymore.
  72     #
  73     # The awk script here joins the lines ending with </td> to make it possible
  74     # to find the failed builds.
  75     LIST_OF_BUILDS=$(cat "$LOGS_DIR/slave_$S" | \
  76                      awk 'BEGIN { buf = "" }
  77                           {
  78                             if ($0 ~ /<\/td>/) { buf = (buf $0); }
  79                             else {
  80                               if (buf) { print buf; buf="" }
  81                               print $0
  82                             }
  83                           }
  84                           END {if (buf) print buf}' | \
  85                      grep "success\|failure" | \
  86                      head -n $NUMBUILDS | \
  87                      grep "failure" | \
  88                      grep -v "failed compile" | \
  89                      sed "s/.*\/builds\///" | sed "s/\".*//")
  90
  91     for BUILD in $LIST_OF_BUILDS
  92     do
  93       # We'll fetch a few tiny URLs now, let's use a temp file.
  94       TMPFILE=$(mktemp -t memory_waterfall.XXXXXX)
  95       download $SLAVE_URL/builds/$BUILD "$TMPFILE"
  96
  97       REPORT_FILE="$LOGS_DIR/report_${S}_${BUILD}"
  98       rm -f $REPORT_FILE 2>/dev/null || true  # make sure it doesn't exist
  99
 100       REPORT_URLS=$(grep -o "[0-9]\+/steps/memory.*/logs/[0-9A-F]\{16\}" \
 101                     "$TMPFILE" \
 102                     || true)  # `true` is to succeed on empty output
 103       FAILED_TESTS=$(grep -o "[0-9]\+/steps/memory.*/logs/[A-Za-z0-9_.]\+" \
 104                      "$TMPFILE" | grep -v "[0-9A-F]\{16\}" \
 105                      | grep -v "stdio" || true)
 106
 107       for REPORT in $REPORT_URLS
 108       do
 109         download "$SLAVE_URL/builds/$REPORT/text" "$TMPFILE"
 110         echo "" >> "$TMPFILE"  # Add a newline at the end
 111         cat "$TMPFILE" | tr -d '\r' >> "$REPORT_FILE"
 112       done
 113
 114       for FAILURE in $FAILED_TESTS
 115       do
 116         echo -n "FAILED:" >> "$REPORT_FILE"
 117         echo "$FAILURE" | sed -e "s/.*\/logs\///" -e "s/\/.*//" \
 118           >> "$REPORT_FILE"
 119       done
 120
 121       rm "$TMPFILE"
 122       echo $SLAVE_URL/builds/$BUILD >> "$REPORT_FILE"
 123     done
 124     echo " DONE"
 125   done
 126   # }}}
 127 }
 128
 129 match_suppressions() {
 130   PYTHONPATH=$THISDIR/../python/google \
 131              python "$THISDIR/test_suppressions.py" $@ "$LOGS_DIR/report_"*
 132 }
 133
 134 match_gtest_excludes() {
 135   for PLATFORM in "Linux" "Chromium%20Mac" "Chromium%20OS"
 136   do
 137     echo
 138     echo "Test failures on ${PLATFORM}:" | sed "s/%20/ /"
 139     grep -h -o "^FAILED:.*" -R "$LOGS_DIR"/*${PLATFORM}* | \
 140          grep -v "FAILS\|FLAKY" | sort | uniq | \
 141          sed -e "s/^FAILED://" -e "s/^/  /"
 142     # Don't put any operators between "grep | sed" and "RESULT=$PIPESTATUS"
 143     RESULT=$PIPESTATUS
 144
 145     if [ "$RESULT" == 1 ]
 146     then
 147       echo "  None!"
 148     else
 149       echo
 150       echo "  Note: we don't check for failures already excluded locally yet"
 151       echo "  TODO(timurrrr): don't list tests we've already excluded locally"
 152     fi
 153   done
 154   echo
 155   echo "Note: we don't print FAILS/FLAKY tests and 1200s-timeout failures"
 156 }
 157
 158 usage() {
 159   cat <<EOF
 160 usage: $0 fetch|match options
 161
 162 This script can be used by waterfall sheriffs to fetch the status
 163 of Valgrind bots on the memory waterfall and test if their local
 164 suppressions match the reports on the waterfall.
 165
 166 OPTIONS:
 167    -h      Show this message
 168    -n N    Fetch N builds from each slave.
 169
 170 COMMANDS:
 171   fetch    Fetch Valgrind logs from the memory waterfall
 172   match    Test the local suppression files against the downloaded logs
 173
 174 EOF
 175 }
 176
 177 NUMBUILDS=3
 178
 179 CMD=$1
 180 if [ $# != 0 ]; then
 181   shift
 182 fi
 183
 184 # Arguments for "match" are handled in match_suppressions
 185 if [ "$CMD" != "match" ]; then
 186   while getopts “hn:” OPTION
 187   do
 188     case $OPTION in
 189       h)
 190         usage
 191         exit
 192         ;;
 193       n)
 194         NUMBUILDS=$OPTARG
 195         ;;
 196       ?)
 197         usage
 198         exit
 199         ;;
 200     esac
 201   done
 202   shift $((OPTIND-1))
 203   if [ $# != 0 ]; then
 204     usage
 205     exit 1
 206   fi
 207 fi
 208
 209 if [ "$CMD" = "fetch" ]; then
 210   echo "Fetching $NUMBUILDS builds"
 211   fetch_logs $WATERFALL_PAGE
 212   fetch_logs $WATERFALL_FYI_PAGE
 213 elif [ "$CMD" = "match" ]; then
 214   match_suppressions $@
 215   match_gtest_excludes
 216 elif [ "$CMD" = "blame" ]; then
 217   echo The blame command died of bitrot. If you need it, please reimplement it.
 218   echo Reimplementation is blocked on http://crbug.com/82688
 219 else
 220   usage
 221   exit 1
 222 fi