webkit/tools/layout_tests/canary-webkit-revisions.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2011 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Retrieve passing and failing WebKit revision numbers from canaries.
   7
   8 From each canary,
   9 - the last WebKit revision number for which all the tests have passed,
  10 - the last WebKit revision number for which the tests were run, and
  11 - the names of failing layout tests
  12 are retrieved and printed.
  13 """
  14
  15
  16 import json
  17 import optparse
  18 import re
  19 import sys
  20 import urllib2
  21
  22 _WEBKIT_REVISION_IN_DEPS_RE = re.compile(r'"webkit_revision"\s*:\s*"(\d+)"')
  23 _DEPS_FILE_URL = "http://src.chromium.org/viewvc/chrome/trunk/src/DEPS"
  24 _DEFAULT_BUILDERS = [
  25   "Webkit Win",
  26   "Webkit Vista",
  27   "Webkit Win7",
  28   "Webkit Win (dbg)(1)",
  29   "Webkit Win (dbg)(2)",
  30   "Webkit Mac10.5 (CG)",
  31   "Webkit Mac10.6 (CG)",
  32   "Webkit Mac10.5 (CG)(dbg)(1)",
  33   "Webkit Mac10.5 (CG)(dbg)(2)",
  34   "Webkit Mac10.6 (CG)(dbg)",
  35   "Webkit Linux",
  36   "Webkit Linux 32",
  37   "Webkit Linux (dbg)(1)",
  38   "Webkit Linux (dbg)(2)",
  39 ]
  40 _DEFAULT_MAX_BUILDS = 10
  41 _TEST_PREFIX = "&tests="
  42 _TEST_SUFFIX = '">'
  43 _WEBKIT_TESTS = "webkit_tests"
  44
  45
  46 def _OpenUrl(url):
  47   """Opens a URL.
  48
  49   Returns:
  50       A file-like object in case of success, an empty list otherwise.
  51   """
  52   try:
  53     return urllib2.urlopen(url)
  54   except urllib2.URLError, url_error:
  55     message = ""
  56     # Surprisingly, urllib2.URLError has different attributes based on the
  57     # kinds of errors -- "code" for HTTP-level errors, "reason" for others.
  58     if hasattr(url_error, "code"):
  59       message = "Status code: %d" % url_error.code
  60     if hasattr(url_error, "reason"):
  61       message = url_error.reason
  62     print >>sys.stderr, "Failed to open %s: %s" % (url, message)
  63   return []
  64
  65
  66 def _WebkitRevisionInDeps():
  67   """Returns the WebKit revision specified in DEPS file.
  68
  69   Returns:
  70       Revision number as int. -1 in case of error.
  71   """
  72   for line in _OpenUrl(_DEPS_FILE_URL):
  73     match = _WEBKIT_REVISION_IN_DEPS_RE.search(line)
  74     if match:
  75       return int(match.group(1))
  76   return -1
  77
  78
  79 class _BuildResult(object):
  80   """Build result for a builder.
  81
  82   Holds builder name, the last passing revision, the last run revision, and
  83   a list of names of failing tests. Revision nubmer 0 is used to represent
  84   that the revision doesn't exist.
  85   """
  86   def __init__(self, builder, last_passing_revision, last_run_revision,
  87                failing_tests):
  88     """Constructs build results."""
  89     self.builder = builder
  90     self.last_passing_revision = last_passing_revision
  91     self.last_run_revision = last_run_revision
  92     self.failing_tests = failing_tests
  93
  94
  95 def _BuilderUrlFor(builder, max_builds):
  96   """Constructs the URL for a builder to retrieve the last results."""
  97   url = ("http://build.chromium.org/p/chromium.webkit/json/builders/%s/builds" %
  98          urllib2.quote(builder))
  99   if max_builds == -1:
 100     return url + "/_all?as_text=1"
 101   return (url + "?as_text=1&" +
 102           '&'.join(["select=%d" % -i for i in range(1, 1 + max_builds)]))
 103
 104
 105 def _ExtractFailingTests(build):
 106   """Extracts failing test names from a build result entry JSON object."""
 107   failing_tests = []
 108   for step in build["steps"]:
 109     if step["name"] == _WEBKIT_TESTS:
 110       for text in step["text"]:
 111         prefix = text.find(_TEST_PREFIX)
 112         suffix = text.find(_TEST_SUFFIX)
 113         if prefix != -1 and suffix != -1:
 114           failing_tests += sorted(
 115               text[prefix + len(_TEST_PREFIX): suffix].split(","))
 116     elif "results" in step:
 117       # Existence of "results" entry seems to mean failure.
 118       failing_tests.append(" ".join(step["text"]))
 119   return failing_tests
 120
 121
 122 def _RetrieveBuildResult(builder, max_builds, oldest_revision_to_check):
 123   """Retrieves build results for a builder.
 124
 125   Checks the last passing revision, the last run revision, and failing tests
 126   for the last builds of a builder.
 127
 128   Args:
 129       builder: Builder name.
 130       max_builds: Maximum number of builds to check.
 131       oldest_revision_to_check: Oldest WebKit revision to check.
 132
 133   Returns:
 134       _BuildResult instance.
 135   """
 136   last_run_revision = 0
 137   failing_tests = []
 138   succeeded = False
 139   builds_json = _OpenUrl(_BuilderUrlFor(builder, max_builds))
 140   if not builds_json:
 141     return _BuildResult(builder, 0, 0, failing_tests)
 142   builds = [(int(value["number"]), value) for unused_key, value
 143             in json.loads(''.join(builds_json)).items()
 144             if value.has_key("number")]
 145   builds.sort()
 146   builds.reverse()
 147   for unused_key, build in builds:
 148     if not build.has_key("text"):
 149       continue
 150     if len(build["text"]) < 2:
 151       continue
 152     if not build.has_key("sourceStamp"):
 153       continue
 154     if build["text"][1] == "successful":
 155       succeeded = True
 156     elif not failing_tests:
 157       failing_tests = _ExtractFailingTests(build)
 158     revision = 0
 159     if build["sourceStamp"]["branch"] == "trunk":
 160       revision = int(build["sourceStamp"]["changes"][-1]["revision"])
 161     if revision and not last_run_revision:
 162       last_run_revision = revision
 163     if revision and revision < oldest_revision_to_check:
 164       break
 165     if not succeeded or not revision:
 166       continue
 167     return _BuildResult(builder, revision, last_run_revision, failing_tests)
 168   return _BuildResult(builder, 0, last_run_revision, failing_tests)
 169
 170
 171 def _PrintPassingRevisions(results, unused_verbose):
 172   """Prints passing revisions and the range of such revisions.
 173
 174   Args:
 175       results: A list of build results.
 176   """
 177   print "**** Passing revisions *****"
 178   min_passing_revision = sys.maxint
 179   max_passing_revision = 0
 180   for result in results:
 181     if result.last_passing_revision:
 182       min_passing_revision = min(min_passing_revision,
 183                                  result.last_passing_revision)
 184       max_passing_revision = max(max_passing_revision,
 185                                  result.last_passing_revision)
 186       print 'The last passing run was at r%d on "%s"' % (
 187           result.last_passing_revision, result.builder)
 188     else:
 189       print 'No passing runs on "%s"' % result.builder
 190   if max_passing_revision:
 191     print "Passing revision range: r%d - r%d" % (
 192           min_passing_revision, max_passing_revision)
 193
 194
 195 def _PrintFailingRevisions(results, verbose):
 196   """Prints failing revisions and the failing tests.
 197
 198   Args:
 199       results: A list of build results.
 200   """
 201   failing_test_to_builders = {}
 202   print "**** Failing revisions *****"
 203   for result in results:
 204     if result.last_run_revision and result.failing_tests:
 205       print ('The last run was at r%d on "%s" and the following %d tests'
 206              ' failed' % (result.last_run_revision, result.builder,
 207                           len(result.failing_tests)))
 208       for test in result.failing_tests:
 209         print "  " + test
 210         failing_test_to_builders.setdefault(test, set()).add(result.builder)
 211   if verbose:
 212     _PrintFailingTestsForBuilderSubsets(failing_test_to_builders)
 213
 214
 215 class _FailingTestsForBuilderSubset(object):
 216   def __init__(self, subset_size):
 217     self._subset_size = subset_size
 218     self._tests = []
 219
 220   def SubsetSize(self):
 221     return self._subset_size
 222
 223   def Tests(self):
 224     return self._tests
 225
 226
 227 def _PrintFailingTestsForBuilderSubsets(failing_test_to_builders):
 228   """Prints failing test for builder subsets.
 229
 230   Prints failing tests for each subset of builders, in descending order of the
 231   set size.
 232   """
 233   print "**** Failing tests ****"
 234   builders_to_tests = {}
 235   for test in failing_test_to_builders:
 236     builders = sorted(failing_test_to_builders[test])
 237     subset_name = ", ".join(builders)
 238     tests = builders_to_tests.setdefault(
 239         subset_name, _FailingTestsForBuilderSubset(len(builders))).Tests()
 240     tests.append(test)
 241   # Sort subsets in descending order of size and then name.
 242   builder_subsets = [(builders_to_tests[subset_name].SubsetSize(), subset_name)
 243                      for subset_name in builders_to_tests]
 244   for subset_size, subset_name in reversed(sorted(builder_subsets)):
 245     print "** Tests failing for %d builders: %s **" % (subset_size,
 246                                                        subset_name)
 247     for test in sorted(builders_to_tests[subset_name].Tests()):
 248       print test
 249
 250
 251 def _ParseOptions():
 252   """Parses command-line options."""
 253   parser = optparse.OptionParser(usage="%prog [options] [builders]")
 254   parser.add_option("-m", "--max_builds", type="int",
 255                     default=-1,
 256                     help="Maximum number of builds to check for each builder."
 257                          " Defaults to all builds for which record is"
 258                          " available. Checking is ended either when the maximum"
 259                          " number is reached, the remaining builds are older"
 260                          " than the DEPS WebKit revision, or a passing"
 261                          " revision is found.")
 262   parser.add_option("-v", "--verbose", action="store_true", default=False,
 263                     dest="verbose")
 264   return parser.parse_args()
 265
 266
 267 def _Main():
 268   """The main function."""
 269   options, builders = _ParseOptions()
 270   if not builders:
 271     builders = _DEFAULT_BUILDERS
 272   oldest_revision_to_check = _WebkitRevisionInDeps()
 273   if options.max_builds == -1 and oldest_revision_to_check == -1:
 274     options.max_builds = _DEFAULT_MAX_BUILDS
 275   if options.max_builds != -1:
 276     print "Maxium number of builds to check: %d" % options.max_builds
 277   if oldest_revision_to_check != -1:
 278     print "Oldest revision to check: %d" % oldest_revision_to_check
 279   sys.stdout.flush()
 280   results = []
 281   for builder in builders:
 282     print '"%s"' % builder
 283     sys.stdout.flush()
 284     results.append(_RetrieveBuildResult(
 285         builder, options.max_builds, oldest_revision_to_check))
 286   _PrintFailingRevisions(results, options.verbose)
 287   _PrintPassingRevisions(results, options.verbose)
 288
 289
 290 if __name__ == "__main__":
 291   _Main()