chrome/test/chromedriver/run_buildbot_steps.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Runs all the buildbot steps for ChromeDriver except for update/compile."""
   7
   8 import bisect
   9 import csv
  10 import datetime
  11 import glob
  12 import json
  13 import optparse
  14 import os
  15 import platform as platform_module
  16 import re
  17 import shutil
  18 import StringIO
  19 import sys
  20 import tempfile
  21 import time
  22 import urllib2
  23
  24 _THIS_DIR = os.path.abspath(os.path.dirname(__file__))
  25 GS_CHROMEDRIVER_BUCKET = 'gs://chromedriver'
  26 GS_CHROMEDRIVER_DATA_BUCKET = 'gs://chromedriver-data'
  27 GS_CHROMEDRIVER_RELEASE_URL = 'http://chromedriver.storage.googleapis.com'
  28 GS_CONTINUOUS_URL = GS_CHROMEDRIVER_DATA_BUCKET + '/continuous'
  29 GS_PREBUILTS_URL = GS_CHROMEDRIVER_DATA_BUCKET + '/prebuilts'
  30 GS_SERVER_LOGS_URL = GS_CHROMEDRIVER_DATA_BUCKET + '/server_logs'
  31 SERVER_LOGS_LINK = (
  32     'http://chromedriver-data.storage.googleapis.com/server_logs')
  33 TEST_LOG_FORMAT = '%s_log.json'
  34 GS_GIT_LOG_URL = (
  35     'https://chromium.googlesource.com/chromium/src/+/%s?format=json')
  36 GS_SEARCH_PATTERN = (
  37     r'Cr-Commit-Position: refs/heads/master@{#(\d+)}')
  38 CR_REV_URL = 'https://cr-rev.appspot.com/_ah/api/crrev/v1/redirect/%s'
  39
  40 SCRIPT_DIR = os.path.join(_THIS_DIR, os.pardir, os.pardir, os.pardir, os.pardir,
  41                           os.pardir, os.pardir, os.pardir, 'scripts')
  42 SITE_CONFIG_DIR = os.path.join(_THIS_DIR, os.pardir, os.pardir, os.pardir,
  43                                os.pardir, os.pardir, os.pardir, os.pardir,
  44                                'site_config')
  45 sys.path.append(SCRIPT_DIR)
  46 sys.path.append(SITE_CONFIG_DIR)
  47
  48 import archive
  49 import chrome_paths
  50 from slave import gsutil_download
  51 from slave import slave_utils
  52 import util
  53
  54
  55 def _ArchivePrebuilts(revision):
  56   """Uploads the prebuilts to google storage."""
  57   util.MarkBuildStepStart('archive prebuilts')
  58   zip_path = util.Zip(os.path.join(chrome_paths.GetBuildDir(['chromedriver']),
  59                                    'chromedriver'))
  60   if slave_utils.GSUtilCopy(
  61       zip_path,
  62       '%s/%s' % (GS_PREBUILTS_URL, 'r%s.zip' % revision)):
  63     util.MarkBuildStepError()
  64
  65
  66 def _ArchiveServerLogs():
  67   """Uploads chromedriver server logs to google storage."""
  68   util.MarkBuildStepStart('archive chromedriver server logs')
  69   for server_log in glob.glob(os.path.join(tempfile.gettempdir(),
  70                                            'chromedriver_*')):
  71     base_name = os.path.basename(server_log)
  72     util.AddLink(base_name, '%s/%s' % (SERVER_LOGS_LINK, base_name))
  73     slave_utils.GSUtilCopy(
  74         server_log,
  75         '%s/%s' % (GS_SERVER_LOGS_URL, base_name),
  76         mimetype='text/plain')
  77
  78
  79 def _DownloadPrebuilts():
  80   """Downloads the most recent prebuilts from google storage."""
  81   util.MarkBuildStepStart('Download latest chromedriver')
  82
  83   zip_path = os.path.join(util.MakeTempDir(), 'build.zip')
  84   if gsutil_download.DownloadLatestFile(GS_PREBUILTS_URL,
  85                                         GS_PREBUILTS_URL + '/r',
  86                                         zip_path):
  87     util.MarkBuildStepError()
  88
  89   util.Unzip(zip_path, chrome_paths.GetBuildDir(['host_forwarder']))
  90
  91
  92 def _GetTestResultsLog(platform):
  93   """Gets the test results log for the given platform.
  94
  95   Args:
  96     platform: The platform that the test results log is for.
  97
  98   Returns:
  99     A dictionary where the keys are SVN revisions and the values are booleans
 100     indicating whether the tests passed.
 101   """
 102   temp_log = tempfile.mkstemp()[1]
 103   log_name = TEST_LOG_FORMAT % platform
 104   result = slave_utils.GSUtilDownloadFile(
 105       '%s/%s' % (GS_CHROMEDRIVER_DATA_BUCKET, log_name), temp_log)
 106   if result:
 107     return {}
 108   with open(temp_log, 'rb') as log_file:
 109     json_dict = json.load(log_file)
 110   # Workaround for json encoding dictionary keys as strings.
 111   return dict([(int(v[0]), v[1]) for v in json_dict.items()])
 112
 113
 114 def _PutTestResultsLog(platform, test_results_log):
 115   """Pushes the given test results log to google storage."""
 116   temp_dir = util.MakeTempDir()
 117   log_name = TEST_LOG_FORMAT % platform
 118   log_path = os.path.join(temp_dir, log_name)
 119   with open(log_path, 'wb') as log_file:
 120     json.dump(test_results_log, log_file)
 121   if slave_utils.GSUtilCopyFile(log_path, GS_CHROMEDRIVER_DATA_BUCKET):
 122     raise Exception('Failed to upload test results log to google storage')
 123
 124
 125 def _UpdateTestResultsLog(platform, revision, passed):
 126   """Updates the test results log for the given platform.
 127
 128   Args:
 129     platform: The platform name.
 130     revision: The SVN revision number.
 131     passed: Boolean indicating whether the tests passed at this revision.
 132   """
 133   assert isinstance(revision, int), 'The revision must be an integer'
 134   log = _GetTestResultsLog(platform)
 135   if len(log) > 500:
 136     del log[min(log.keys())]
 137   assert revision not in log, 'Results already exist for revision %s' % revision
 138   log[revision] = bool(passed)
 139   _PutTestResultsLog(platform, log)
 140
 141
 142 def _GetVersion():
 143   """Get the current chromedriver version."""
 144   with open(os.path.join(_THIS_DIR, 'VERSION'), 'r') as f:
 145     return f.read().strip()
 146
 147
 148 def _GetSupportedChromeVersions():
 149   """Get the minimum and maximum supported Chrome versions.
 150
 151   Returns:
 152     A tuple of the form (min_version, max_version).
 153   """
 154   # Minimum supported Chrome version is embedded as:
 155   # const int kMinimumSupportedChromeVersion[] = {27, 0, 1453, 0};
 156   with open(os.path.join(_THIS_DIR, 'chrome', 'version.cc'), 'r') as f:
 157     lines = f.readlines()
 158     chrome_min_version_line = [
 159         x for x in lines if 'kMinimumSupportedChromeVersion' in x]
 160   chrome_min_version = chrome_min_version_line[0].split('{')[1].split(',')[0]
 161   with open(os.path.join(chrome_paths.GetSrc(), 'chrome', 'VERSION'), 'r') as f:
 162     chrome_max_version = f.readlines()[0].split('=')[1].strip()
 163   return (chrome_min_version, chrome_max_version)
 164
 165
 166 def _RevisionState(test_results_log, revision):
 167   """Check the state of tests at a given SVN revision.
 168
 169   Considers tests as having passed at a revision if they passed at revisons both
 170   before and after.
 171
 172   Args:
 173     test_results_log: A test results log dictionary from _GetTestResultsLog().
 174     revision: The revision to check at.
 175
 176   Returns:
 177     'passed', 'failed', or 'unknown'
 178   """
 179   assert isinstance(revision, int), 'The revision must be an integer'
 180   keys = sorted(test_results_log.keys())
 181   # Return passed if the exact revision passed on Android.
 182   if revision in test_results_log:
 183     return 'passed' if test_results_log[revision] else 'failed'
 184   # Tests were not run on this exact revision on Android.
 185   index = bisect.bisect_right(keys, revision)
 186   # Tests have not yet run on Android at or above this revision.
 187   if index == len(test_results_log):
 188     return 'unknown'
 189   # No log exists for any prior revision, assume it failed.
 190   if index == 0:
 191     return 'failed'
 192   # Return passed if the revisions on both sides passed.
 193   if test_results_log[keys[index]] and test_results_log[keys[index - 1]]:
 194     return 'passed'
 195   return 'failed'
 196
 197
 198 def _ArchiveGoodBuild(platform, revision):
 199   """Archive chromedriver binary if the build is green."""
 200   assert platform != 'android'
 201   util.MarkBuildStepStart('archive build')
 202
 203   server_name = 'chromedriver'
 204   if util.IsWindows():
 205     server_name += '.exe'
 206   zip_path = util.Zip(os.path.join(chrome_paths.GetBuildDir([server_name]),
 207                                    server_name))
 208
 209   build_name = 'chromedriver_%s_%s.%s.zip' % (
 210       platform, _GetVersion(), revision)
 211   build_url = '%s/%s' % (GS_CONTINUOUS_URL, build_name)
 212   if slave_utils.GSUtilCopy(zip_path, build_url):
 213     util.MarkBuildStepError()
 214
 215   (latest_fd, latest_file) = tempfile.mkstemp()
 216   os.write(latest_fd, build_name)
 217   os.close(latest_fd)
 218   latest_url = '%s/latest_%s' % (GS_CONTINUOUS_URL, platform)
 219   if slave_utils.GSUtilCopy(latest_file, latest_url, mimetype='text/plain'):
 220     util.MarkBuildStepError()
 221   os.remove(latest_file)
 222
 223
 224 def _WasReleased(version, platform):
 225   """Check if the specified version is released for the given platform."""
 226   result, _ = slave_utils.GSUtilListBucket(
 227       '%s/%s/chromedriver_%s.zip' % (GS_CHROMEDRIVER_BUCKET, version, platform),
 228       [])
 229   return result == 0
 230
 231
 232 def _MaybeRelease(platform):
 233   """Releases a release candidate if conditions are right."""
 234   assert platform != 'android'
 235
 236   version = _GetVersion()
 237
 238   # Check if the current version has already been released.
 239   if _WasReleased(version, platform):
 240     return
 241
 242   # Fetch Android test results.
 243   android_test_results = _GetTestResultsLog('android')
 244
 245   # Fetch release candidates.
 246   result, output = slave_utils.GSUtilListBucket(
 247       '%s/chromedriver_%s_%s*' % (
 248           GS_CONTINUOUS_URL, platform, version),
 249       [])
 250   assert result == 0 and output, 'No release candidates found'
 251   candidate_pattern = re.compile(
 252       r'.*/chromedriver_%s_%s\.(\d+)\.zip$' % (platform, version))
 253   candidates = []
 254   for line in output.strip().split('\n'):
 255     result = candidate_pattern.match(line)
 256     if not result:
 257       print 'Ignored line "%s"' % line
 258       continue
 259     candidates.append(int(result.group(1)))
 260
 261   # Release the latest candidate build that passed Android, if any.
 262   # In this way, if a hot fix is needed, we can delete the release from
 263   # the chromedriver bucket instead of bumping up the release version number.
 264   candidates.sort(reverse=True)
 265   for revision in candidates:
 266     android_result = _RevisionState(android_test_results, revision)
 267     if android_result == 'failed':
 268       print 'Android tests did not pass at revision', revision
 269     elif android_result == 'passed':
 270       print 'Android tests passed at revision', revision
 271       candidate = 'chromedriver_%s_%s.%s.zip' % (platform, version, revision)
 272       _Release('%s/%s' % (GS_CONTINUOUS_URL, candidate), version, platform)
 273       break
 274     else:
 275       print 'Android tests have not run at a revision as recent as', revision
 276
 277
 278 def _Release(build, version, platform):
 279   """Releases the given candidate build."""
 280   release_name = 'chromedriver_%s.zip' % platform
 281   util.MarkBuildStepStart('releasing %s' % release_name)
 282   temp_dir = util.MakeTempDir()
 283   slave_utils.GSUtilCopy(build, temp_dir)
 284   zip_path = os.path.join(temp_dir, os.path.basename(build))
 285
 286   if util.IsLinux():
 287     util.Unzip(zip_path, temp_dir)
 288     server_path = os.path.join(temp_dir, 'chromedriver')
 289     util.RunCommand(['strip', server_path])
 290     zip_path = util.Zip(server_path)
 291
 292   slave_utils.GSUtilCopy(
 293       zip_path, '%s/%s/%s' % (GS_CHROMEDRIVER_BUCKET, version, release_name))
 294
 295   _MaybeUploadReleaseNotes(version)
 296   _MaybeUpdateLatestRelease(version)
 297
 298
 299 def _GetWebPageContent(url):
 300   """Return the content of the web page specified by the given url."""
 301   return urllib2.urlopen(url).read()
 302
 303
 304 def _MaybeUploadReleaseNotes(version):
 305   """Upload release notes if conditions are right."""
 306   # Check if the current version has already been released.
 307   notes_name = 'notes.txt'
 308   notes_url = '%s/%s/%s' % (GS_CHROMEDRIVER_BUCKET, version, notes_name)
 309   prev_version = '.'.join([version.split('.')[0],
 310                            str(int(version.split('.')[1]) - 1)])
 311   prev_notes_url = '%s/%s/%s' % (
 312       GS_CHROMEDRIVER_BUCKET, prev_version, notes_name)
 313
 314   result, _ = slave_utils.GSUtilListBucket(notes_url, [])
 315   if result == 0:
 316     return
 317
 318   fixed_issues = []
 319   query = ('https://code.google.com/p/chromedriver/issues/csv?'
 320            'q=status%3AToBeReleased&colspec=ID%20Summary')
 321   issues = StringIO.StringIO(_GetWebPageContent(query).split('\n', 1)[1])
 322   for issue in csv.reader(issues):
 323     if not issue:
 324       continue
 325     issue_id = issue[0]
 326     desc = issue[1]
 327     labels = issue[2]
 328     fixed_issues += ['Resolved issue %s: %s [%s]' % (issue_id, desc, labels)]
 329
 330   old_notes = ''
 331   temp_notes_fname = tempfile.mkstemp()[1]
 332   if not slave_utils.GSUtilDownloadFile(prev_notes_url, temp_notes_fname):
 333     with open(temp_notes_fname, 'rb') as f:
 334       old_notes = f.read()
 335
 336   new_notes = '----------ChromeDriver v%s (%s)----------\n%s\n%s\n\n%s' % (
 337       version, datetime.date.today().isoformat(),
 338       'Supports Chrome v%s-%s' % _GetSupportedChromeVersions(),
 339       '\n'.join(fixed_issues),
 340       old_notes)
 341   with open(temp_notes_fname, 'w') as f:
 342     f.write(new_notes)
 343
 344   if slave_utils.GSUtilCopy(temp_notes_fname, notes_url, mimetype='text/plain'):
 345     util.MarkBuildStepError()
 346
 347
 348 def _MaybeUpdateLatestRelease(version):
 349   """Update the file LATEST_RELEASE with the latest release version number."""
 350   latest_release_fname = 'LATEST_RELEASE'
 351   latest_release_url = '%s/%s' % (GS_CHROMEDRIVER_BUCKET, latest_release_fname)
 352
 353   # Check if LATEST_RELEASE is up-to-date.
 354   latest_released_version = _GetWebPageContent(
 355       '%s/%s' % (GS_CHROMEDRIVER_RELEASE_URL, latest_release_fname))
 356   if version == latest_released_version:
 357     return
 358
 359   # Check if chromedriver was released on all supported platforms.
 360   supported_platforms = ['linux32', 'linux64', 'mac32', 'win32']
 361   for platform in supported_platforms:
 362     if not _WasReleased(version, platform):
 363       return
 364
 365   util.MarkBuildStepStart('updating LATEST_RELEASE to %s' % version)
 366
 367   temp_latest_release_fname = tempfile.mkstemp()[1]
 368   with open(temp_latest_release_fname, 'w') as f:
 369     f.write(version)
 370   if slave_utils.GSUtilCopy(temp_latest_release_fname, latest_release_url,
 371                             mimetype='text/plain'):
 372     util.MarkBuildStepError()
 373
 374
 375 def _CleanTmpDir():
 376   tmp_dir = tempfile.gettempdir()
 377   print 'cleaning temp directory:', tmp_dir
 378   for file_name in os.listdir(tmp_dir):
 379     file_path = os.path.join(tmp_dir, file_name)
 380     if os.path.isdir(file_path):
 381       print 'deleting sub-directory', file_path
 382       shutil.rmtree(file_path, True)
 383     if file_name.startswith('chromedriver_'):
 384       print 'deleting file', file_path
 385       os.remove(file_path)
 386
 387
 388 def _GetCommitPositionFromGitHash(snapshot_hashcode):
 389   json_url = GS_GIT_LOG_URL % snapshot_hashcode
 390   try:
 391     response = urllib2.urlopen(json_url)
 392   except urllib2.HTTPError as error:
 393     util.PrintAndFlush('HTTP Error %d' % error.getcode())
 394     return None
 395   except urllib2.URLError as error:
 396     util.PrintAndFlush('URL Error %s' % error.message)
 397     return None
 398   data = json.loads(response.read()[4:])
 399   if 'message' in data:
 400     message = data['message'].split('\n')
 401     message = [line for line in message if line.strip()]
 402     search_pattern = re.compile(GS_SEARCH_PATTERN)
 403     result = search_pattern.search(message[len(message)-1])
 404     if result:
 405       return result.group(1)
 406   util.PrintAndFlush('Failed to get svn revision number for %s' %
 407                      snapshot_hashcode)
 408   return None
 409
 410
 411 def _GetGitHashFromCommitPosition(commit_position):
 412   json_url = CR_REV_URL % commit_position
 413   try:
 414     response = urllib2.urlopen(json_url)
 415   except urllib2.HTTPError as error:
 416     util.PrintAndFlush('HTTP Error %d' % error.getcode())
 417     return None
 418   except urllib2.URLError as error:
 419     util.PrintAndFlush('URL Error %s' % error.message)
 420     return None
 421   data = json.loads(response.read())
 422   if 'git_sha' in data:
 423     return data['git_sha']
 424   util.PrintAndFlush('Failed to get git hash for %s' % commit_position)
 425   return None
 426
 427
 428 def _WaitForLatestSnapshot(revision):
 429   util.MarkBuildStepStart('wait_for_snapshot')
 430   def _IsRevisionNumber(revision):
 431     if isinstance(revision, int):
 432       return True
 433     else:
 434       return revision.isdigit()
 435   while True:
 436     snapshot_revision = archive.GetLatestSnapshotVersion()
 437     if not _IsRevisionNumber(snapshot_revision):
 438       snapshot_revision = _GetCommitPositionFromGitHash(snapshot_revision)
 439     if revision is not None and snapshot_revision is not None:
 440       if int(snapshot_revision) >= int(revision):
 441         break
 442       util.PrintAndFlush('Waiting for snapshot >= %s, found %s' %
 443                          (revision, snapshot_revision))
 444     time.sleep(60)
 445   util.PrintAndFlush('Got snapshot revision %s' % snapshot_revision)
 446
 447
 448 def _AddToolsToPath(platform_name):
 449   """Add some tools like Ant and Java to PATH for testing steps to use."""
 450   paths = []
 451   error_message = ''
 452   if platform_name == 'win32':
 453     paths = [
 454         # Path to Ant and Java, required for the java acceptance tests.
 455         'C:\\Program Files (x86)\\Java\\ant\\bin',
 456         'C:\\Program Files (x86)\\Java\\jre\\bin',
 457     ]
 458     error_message = ('Java test steps will fail as expected and '
 459                      'they can be ignored.\n'
 460                      'Ant, Java or others might not be installed on bot.\n'
 461                      'Please refer to page "WATERFALL" on site '
 462                      'go/chromedriver.')
 463   if paths:
 464     util.MarkBuildStepStart('Add tools to PATH')
 465     path_missing = False
 466     for path in paths:
 467       if not os.path.isdir(path) or not os.listdir(path):
 468         print 'Directory "%s" is not found or empty.' % path
 469         path_missing = True
 470     if path_missing:
 471       print error_message
 472       util.MarkBuildStepError()
 473       return
 474     os.environ['PATH'] += os.pathsep + os.pathsep.join(paths)
 475
 476
 477 def main():
 478   parser = optparse.OptionParser()
 479   parser.add_option(
 480       '', '--android-packages',
 481       help=('Comma separated list of application package names, '
 482             'if running tests on Android.'))
 483   parser.add_option(
 484       '-r', '--revision', help='Chromium revision')
 485   parser.add_option(
 486       '', '--update-log', action='store_true',
 487       help='Update the test results log (only applicable to Android)')
 488   options, _ = parser.parse_args()
 489
 490   bitness = '32'
 491   if util.IsLinux() and platform_module.architecture()[0] == '64bit':
 492     bitness = '64'
 493   platform = '%s%s' % (util.GetPlatformName(), bitness)
 494   if options.android_packages:
 495     platform = 'android'
 496
 497   _CleanTmpDir()
 498
 499   if not options.revision:
 500     commit_position = None
 501   elif options.revision.isdigit():
 502     commit_position = options.revision
 503   else:
 504     commit_position = _GetCommitPositionFromGitHash(options.revision)
 505
 506   if platform == 'android':
 507     if not options.revision and options.update_log:
 508       parser.error('Must supply a --revision with --update-log')
 509     _DownloadPrebuilts()
 510   else:
 511     if not options.revision:
 512       parser.error('Must supply a --revision')
 513     if platform == 'linux64':
 514       _ArchivePrebuilts(commit_position)
 515     _WaitForLatestSnapshot(commit_position)
 516
 517   _AddToolsToPath(platform)
 518
 519   cmd = [
 520       sys.executable,
 521       os.path.join(_THIS_DIR, 'test', 'run_all_tests.py'),
 522   ]
 523   if platform == 'android':
 524     cmd.append('--android-packages=' + options.android_packages)
 525
 526   passed = (util.RunCommand(cmd) == 0)
 527
 528   _ArchiveServerLogs()
 529
 530   if platform == 'android':
 531     if options.update_log:
 532       util.MarkBuildStepStart('update test result log')
 533       _UpdateTestResultsLog(platform, commit_position, passed)
 534   elif passed:
 535     _ArchiveGoodBuild(platform, commit_position)
 536     _MaybeRelease(platform)
 537
 538   if not passed:
 539     # Make sure the build is red if there is some uncaught exception during
 540     # running run_all_tests.py.
 541     util.MarkBuildStepStart('run_all_tests.py')
 542     util.MarkBuildStepError()
 543
 544   # Add a "cleanup" step so that errors from runtest.py or bb_device_steps.py
 545   # (which invoke this script) are kept in thier own build step.
 546   util.MarkBuildStepStart('cleanup')
 547
 548
 549 if __name__ == '__main__':
 550   main()