chrome/test/chromedriver/run_buildbot_steps.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Runs all the buildbot steps for ChromeDriver except for update/compile."""
   7
   8 import bisect
   9 import csv
  10 import datetime
  11 import glob
  12 import json
  13 import optparse
  14 import os
  15 import platform as platform_module
  16 import re
  17 import shutil
  18 import StringIO
  19 import sys
  20 import tempfile
  21 import time
  22 import urllib2
  23
  24 _THIS_DIR = os.path.abspath(os.path.dirname(__file__))
  25 GS_CHROMEDRIVER_BUCKET = 'gs://chromedriver'
  26 GS_CHROMEDRIVER_DATA_BUCKET = 'gs://chromedriver-data'
  27 GS_CHROMEDRIVER_RELEASE_URL = 'http://chromedriver.storage.googleapis.com'
  28 GS_CONTINUOUS_URL = GS_CHROMEDRIVER_DATA_BUCKET + '/continuous'
  29 GS_PREBUILTS_URL = GS_CHROMEDRIVER_DATA_BUCKET + '/prebuilts'
  30 GS_SERVER_LOGS_URL = GS_CHROMEDRIVER_DATA_BUCKET + '/server_logs'
  31 SERVER_LOGS_LINK = (
  32     'http://chromedriver-data.storage.googleapis.com/server_logs')
  33 TEST_LOG_FORMAT = '%s_log.json'
  34 GS_GIT_LOG_URL = (
  35     'https://chromium.googlesource.com/chromium/src/+/%s?format=json')
  36 GS_SEARCH_PATTERN = (
  37     r'Cr-Commit-Position: refs/heads/master@{#(\d+)}')
  38 CR_REV_URL = 'https://cr-rev.appspot.com/_ah/api/crrev/v1/redirect/%s'
  39
  40 SCRIPT_DIR = os.path.join(_THIS_DIR, os.pardir, os.pardir, os.pardir, os.pardir,
  41                           os.pardir, os.pardir, os.pardir, 'scripts')
  42 SITE_CONFIG_DIR = os.path.join(_THIS_DIR, os.pardir, os.pardir, os.pardir,
  43                                os.pardir, os.pardir, os.pardir, os.pardir,
  44                                'site_config')
  45 sys.path.append(SCRIPT_DIR)
  46 sys.path.append(SITE_CONFIG_DIR)
  47
  48 import archive
  49 import chrome_paths
  50 from slave import gsutil_download
  51 from slave import slave_utils
  52 import util
  53
  54
  55 def _ArchivePrebuilts(commit_position):
  56   """Uploads the prebuilts to google storage."""
  57   util.MarkBuildStepStart('archive prebuilts')
  58   zip_path = util.Zip(os.path.join(chrome_paths.GetBuildDir(['chromedriver']),
  59                                    'chromedriver'))
  60   if slave_utils.GSUtilCopy(
  61       zip_path,
  62       '%s/%s' % (GS_PREBUILTS_URL, 'r%s.zip' % commit_position)):
  63     util.MarkBuildStepError()
  64
  65
  66 def _ArchiveServerLogs():
  67   """Uploads chromedriver server logs to google storage."""
  68   util.MarkBuildStepStart('archive chromedriver server logs')
  69   for server_log in glob.glob(os.path.join(tempfile.gettempdir(),
  70                                            'chromedriver_*')):
  71     base_name = os.path.basename(server_log)
  72     util.AddLink(base_name, '%s/%s' % (SERVER_LOGS_LINK, base_name))
  73     slave_utils.GSUtilCopy(
  74         server_log,
  75         '%s/%s' % (GS_SERVER_LOGS_URL, base_name),
  76         mimetype='text/plain')
  77
  78
  79 def _DownloadPrebuilts():
  80   """Downloads the most recent prebuilts from google storage."""
  81   util.MarkBuildStepStart('Download latest chromedriver')
  82
  83   zip_path = os.path.join(util.MakeTempDir(), 'build.zip')
  84   if gsutil_download.DownloadLatestFile(GS_PREBUILTS_URL,
  85                                         GS_PREBUILTS_URL + '/r',
  86                                         zip_path):
  87     util.MarkBuildStepError()
  88
  89   util.Unzip(zip_path, chrome_paths.GetBuildDir(['host_forwarder']))
  90
  91
  92 def _GetTestResultsLog(platform):
  93   """Gets the test results log for the given platform.
  94
  95   Args:
  96     platform: The platform that the test results log is for.
  97
  98   Returns:
  99     A dictionary where the keys are commit positions and the values are booleans
 100     indicating whether the tests passed.
 101   """
 102   temp_log = tempfile.mkstemp()[1]
 103   log_name = TEST_LOG_FORMAT % platform
 104   result = slave_utils.GSUtilDownloadFile(
 105       '%s/%s' % (GS_CHROMEDRIVER_DATA_BUCKET, log_name), temp_log)
 106   if result:
 107     return {}
 108   with open(temp_log, 'rb') as log_file:
 109     json_dict = json.load(log_file)
 110   # Workaround for json encoding dictionary keys as strings.
 111   return dict([(int(v[0]), v[1]) for v in json_dict.items()])
 112
 113
 114 def _PutTestResultsLog(platform, test_results_log):
 115   """Pushes the given test results log to google storage."""
 116   temp_dir = util.MakeTempDir()
 117   log_name = TEST_LOG_FORMAT % platform
 118   log_path = os.path.join(temp_dir, log_name)
 119   with open(log_path, 'wb') as log_file:
 120     json.dump(test_results_log, log_file)
 121   if slave_utils.GSUtilCopyFile(log_path, GS_CHROMEDRIVER_DATA_BUCKET):
 122     raise Exception('Failed to upload test results log to google storage')
 123
 124
 125 def _UpdateTestResultsLog(platform, commit_position, passed):
 126   """Updates the test results log for the given platform.
 127
 128   Args:
 129     platform: The platform name.
 130     commit_position: The commit position number.
 131     passed: Boolean indicating whether the tests passed at this commit position.
 132   """
 133
 134   assert commit_position.isdigit(), 'The commit position must be a number'
 135   commit_position = int(commit_position)
 136   log = _GetTestResultsLog(platform)
 137   if len(log) > 500:
 138     del log[min(log.keys())]
 139   assert commit_position not in log, \
 140       'Results already exist for commit position %s' % commit_position
 141   log[commit_position] = bool(passed)
 142   _PutTestResultsLog(platform, log)
 143
 144
 145 def _GetVersion():
 146   """Get the current chromedriver version."""
 147   with open(os.path.join(_THIS_DIR, 'VERSION'), 'r') as f:
 148     return f.read().strip()
 149
 150
 151 def _GetSupportedChromeVersions():
 152   """Get the minimum and maximum supported Chrome versions.
 153
 154   Returns:
 155     A tuple of the form (min_version, max_version).
 156   """
 157   # Minimum supported Chrome version is embedded as:
 158   # const int kMinimumSupportedChromeVersion[] = {27, 0, 1453, 0};
 159   with open(os.path.join(_THIS_DIR, 'chrome', 'version.cc'), 'r') as f:
 160     lines = f.readlines()
 161     chrome_min_version_line = [
 162         x for x in lines if 'kMinimumSupportedChromeVersion' in x]
 163   chrome_min_version = chrome_min_version_line[0].split('{')[1].split(',')[0]
 164   with open(os.path.join(chrome_paths.GetSrc(), 'chrome', 'VERSION'), 'r') as f:
 165     chrome_max_version = f.readlines()[0].split('=')[1].strip()
 166   return (chrome_min_version, chrome_max_version)
 167
 168
 169 def _CommitPositionState(test_results_log, commit_position):
 170   """Check the state of tests at a given commit position.
 171
 172   Considers tests as having passed at a commit position if they passed at
 173   revisons both before and after.
 174
 175   Args:
 176     test_results_log: A test results log dictionary from _GetTestResultsLog().
 177     commit_position: The commit position to check at.
 178
 179   Returns:
 180     'passed', 'failed', or 'unknown'
 181   """
 182   assert isinstance(commit_position, int), 'The commit position must be an int'
 183   keys = sorted(test_results_log.keys())
 184   # Return passed if the exact commit position passed on Android.
 185   if commit_position in test_results_log:
 186     return 'passed' if test_results_log[commit_position] else 'failed'
 187   # Tests were not run on this exact commit position on Android.
 188   index = bisect.bisect_right(keys, commit_position)
 189   # Tests have not yet run on Android at or above this commit position.
 190   if index == len(test_results_log):
 191     return 'unknown'
 192   # No log exists for any prior commit position, assume it failed.
 193   if index == 0:
 194     return 'failed'
 195   # Return passed if the commit position on both sides passed.
 196   if test_results_log[keys[index]] and test_results_log[keys[index - 1]]:
 197     return 'passed'
 198   return 'failed'
 199
 200
 201 def _ArchiveGoodBuild(platform, commit_position):
 202   """Archive chromedriver binary if the build is green."""
 203   assert platform != 'android'
 204   util.MarkBuildStepStart('archive build')
 205
 206   server_name = 'chromedriver'
 207   if util.IsWindows():
 208     server_name += '.exe'
 209   zip_path = util.Zip(os.path.join(chrome_paths.GetBuildDir([server_name]),
 210                                    server_name))
 211
 212   build_name = 'chromedriver_%s_%s.%s.zip' % (
 213       platform, _GetVersion(), commit_position)
 214   build_url = '%s/%s' % (GS_CONTINUOUS_URL, build_name)
 215   if slave_utils.GSUtilCopy(zip_path, build_url):
 216     util.MarkBuildStepError()
 217
 218   (latest_fd, latest_file) = tempfile.mkstemp()
 219   os.write(latest_fd, build_name)
 220   os.close(latest_fd)
 221   latest_url = '%s/latest_%s' % (GS_CONTINUOUS_URL, platform)
 222   if slave_utils.GSUtilCopy(latest_file, latest_url, mimetype='text/plain'):
 223     util.MarkBuildStepError()
 224   os.remove(latest_file)
 225
 226
 227 def _WasReleased(version, platform):
 228   """Check if the specified version is released for the given platform."""
 229   result, _ = slave_utils.GSUtilListBucket(
 230       '%s/%s/chromedriver_%s.zip' % (GS_CHROMEDRIVER_BUCKET, version, platform),
 231       [])
 232   return result == 0
 233
 234
 235 def _MaybeRelease(platform):
 236   """Releases a release candidate if conditions are right."""
 237   assert platform != 'android'
 238
 239   version = _GetVersion()
 240
 241   # Check if the current version has already been released.
 242   if _WasReleased(version, platform):
 243     return
 244
 245   # Fetch Android test results.
 246   android_test_results = _GetTestResultsLog('android')
 247
 248   # Fetch release candidates.
 249   result, output = slave_utils.GSUtilListBucket(
 250       '%s/chromedriver_%s_%s*' % (
 251           GS_CONTINUOUS_URL, platform, version),
 252       [])
 253   assert result == 0 and output, 'No release candidates found'
 254   candidate_pattern = re.compile(
 255       r'.*/chromedriver_%s_%s\.(\d+)\.zip$' % (platform, version))
 256   candidates = []
 257   for line in output.strip().split('\n'):
 258     result = candidate_pattern.match(line)
 259     if not result:
 260       print 'Ignored line "%s"' % line
 261       continue
 262     candidates.append(int(result.group(1)))
 263
 264   # Release the latest candidate build that passed Android, if any.
 265   # In this way, if a hot fix is needed, we can delete the release from
 266   # the chromedriver bucket instead of bumping up the release version number.
 267   candidates.sort(reverse=True)
 268   for commit_position in candidates:
 269     android_result = _CommitPositionState(android_test_results, commit_position)
 270     if android_result == 'failed':
 271       print 'Android tests did not pass at commit position', commit_position
 272     elif android_result == 'passed':
 273       print 'Android tests passed at commit position', commit_position
 274       candidate = 'chromedriver_%s_%s.%s.zip' % (
 275           platform, version, commit_position)
 276       _Release('%s/%s' % (GS_CONTINUOUS_URL, candidate), version, platform)
 277       break
 278     else:
 279       print 'Android tests have not run at a commit position as recent as', \
 280           commit_position
 281
 282
 283 def _Release(build, version, platform):
 284   """Releases the given candidate build."""
 285   release_name = 'chromedriver_%s.zip' % platform
 286   util.MarkBuildStepStart('releasing %s' % release_name)
 287   temp_dir = util.MakeTempDir()
 288   slave_utils.GSUtilCopy(build, temp_dir)
 289   zip_path = os.path.join(temp_dir, os.path.basename(build))
 290
 291   if util.IsLinux():
 292     util.Unzip(zip_path, temp_dir)
 293     server_path = os.path.join(temp_dir, 'chromedriver')
 294     util.RunCommand(['strip', server_path])
 295     zip_path = util.Zip(server_path)
 296
 297   slave_utils.GSUtilCopy(
 298       zip_path, '%s/%s/%s' % (GS_CHROMEDRIVER_BUCKET, version, release_name))
 299
 300   _MaybeUploadReleaseNotes(version)
 301   _MaybeUpdateLatestRelease(version)
 302
 303
 304 def _GetWebPageContent(url):
 305   """Return the content of the web page specified by the given url."""
 306   return urllib2.urlopen(url).read()
 307
 308
 309 def _MaybeUploadReleaseNotes(version):
 310   """Upload release notes if conditions are right."""
 311   # Check if the current version has already been released.
 312   notes_name = 'notes.txt'
 313   notes_url = '%s/%s/%s' % (GS_CHROMEDRIVER_BUCKET, version, notes_name)
 314   prev_version = '.'.join([version.split('.')[0],
 315                            str(int(version.split('.')[1]) - 1)])
 316   prev_notes_url = '%s/%s/%s' % (
 317       GS_CHROMEDRIVER_BUCKET, prev_version, notes_name)
 318
 319   result, _ = slave_utils.GSUtilListBucket(notes_url, [])
 320   if result == 0:
 321     return
 322
 323   fixed_issues = []
 324   query = ('https://code.google.com/p/chromedriver/issues/csv?'
 325            'q=status%3AToBeReleased&colspec=ID%20Summary')
 326   issues = StringIO.StringIO(_GetWebPageContent(query).split('\n', 1)[1])
 327   for issue in csv.reader(issues):
 328     if not issue:
 329       continue
 330     issue_id = issue[0]
 331     desc = issue[1]
 332     labels = issue[2]
 333     fixed_issues += ['Resolved issue %s: %s [%s]' % (issue_id, desc, labels)]
 334
 335   old_notes = ''
 336   temp_notes_fname = tempfile.mkstemp()[1]
 337   if not slave_utils.GSUtilDownloadFile(prev_notes_url, temp_notes_fname):
 338     with open(temp_notes_fname, 'rb') as f:
 339       old_notes = f.read()
 340
 341   new_notes = '----------ChromeDriver v%s (%s)----------\n%s\n%s\n\n%s' % (
 342       version, datetime.date.today().isoformat(),
 343       'Supports Chrome v%s-%s' % _GetSupportedChromeVersions(),
 344       '\n'.join(fixed_issues),
 345       old_notes)
 346   with open(temp_notes_fname, 'w') as f:
 347     f.write(new_notes)
 348
 349   if slave_utils.GSUtilCopy(temp_notes_fname, notes_url, mimetype='text/plain'):
 350     util.MarkBuildStepError()
 351
 352
 353 def _MaybeUpdateLatestRelease(version):
 354   """Update the file LATEST_RELEASE with the latest release version number."""
 355   latest_release_fname = 'LATEST_RELEASE'
 356   latest_release_url = '%s/%s' % (GS_CHROMEDRIVER_BUCKET, latest_release_fname)
 357
 358   # Check if LATEST_RELEASE is up-to-date.
 359   latest_released_version = _GetWebPageContent(
 360       '%s/%s' % (GS_CHROMEDRIVER_RELEASE_URL, latest_release_fname))
 361   if version == latest_released_version:
 362     return
 363
 364   # Check if chromedriver was released on all supported platforms.
 365   supported_platforms = ['linux32', 'linux64', 'mac32', 'win32']
 366   for platform in supported_platforms:
 367     if not _WasReleased(version, platform):
 368       return
 369
 370   util.MarkBuildStepStart('updating LATEST_RELEASE to %s' % version)
 371
 372   temp_latest_release_fname = tempfile.mkstemp()[1]
 373   with open(temp_latest_release_fname, 'w') as f:
 374     f.write(version)
 375   if slave_utils.GSUtilCopy(temp_latest_release_fname, latest_release_url,
 376                             mimetype='text/plain'):
 377     util.MarkBuildStepError()
 378
 379
 380 def _CleanTmpDir():
 381   tmp_dir = tempfile.gettempdir()
 382   print 'cleaning temp directory:', tmp_dir
 383   for file_name in os.listdir(tmp_dir):
 384     file_path = os.path.join(tmp_dir, file_name)
 385     if os.path.isdir(file_path):
 386       print 'deleting sub-directory', file_path
 387       shutil.rmtree(file_path, True)
 388     if file_name.startswith('chromedriver_'):
 389       print 'deleting file', file_path
 390       os.remove(file_path)
 391
 392
 393 def _GetCommitPositionFromGitHash(snapshot_hashcode):
 394   json_url = GS_GIT_LOG_URL % snapshot_hashcode
 395   try:
 396     response = urllib2.urlopen(json_url)
 397   except urllib2.HTTPError as error:
 398     util.PrintAndFlush('HTTP Error %d' % error.getcode())
 399     return None
 400   except urllib2.URLError as error:
 401     util.PrintAndFlush('URL Error %s' % error.message)
 402     return None
 403   data = json.loads(response.read()[4:])
 404   if 'message' in data:
 405     message = data['message'].split('\n')
 406     message = [line for line in message if line.strip()]
 407     search_pattern = re.compile(GS_SEARCH_PATTERN)
 408     result = search_pattern.search(message[len(message)-1])
 409     if result:
 410       return result.group(1)
 411   util.PrintAndFlush('Failed to get commit position number for %s' %
 412                      snapshot_hashcode)
 413   return None
 414
 415
 416 def _GetGitHashFromCommitPosition(commit_position):
 417   json_url = CR_REV_URL % commit_position
 418   try:
 419     response = urllib2.urlopen(json_url)
 420   except urllib2.HTTPError as error:
 421     util.PrintAndFlush('HTTP Error %d' % error.getcode())
 422     return None
 423   except urllib2.URLError as error:
 424     util.PrintAndFlush('URL Error %s' % error.message)
 425     return None
 426   data = json.loads(response.read())
 427   if 'git_sha' in data:
 428     return data['git_sha']
 429   util.PrintAndFlush('Failed to get git hash for %s' % commit_position)
 430   return None
 431
 432
 433 def _WaitForLatestSnapshot(commit_position):
 434   util.MarkBuildStepStart('wait_for_snapshot')
 435   while True:
 436     snapshot_position = archive.GetLatestSnapshotVersion()
 437     if commit_position is not None and snapshot_position is not None:
 438       if int(snapshot_position) >= int(commit_position):
 439         break
 440       util.PrintAndFlush('Waiting for snapshot >= %s, found %s' %
 441                          (commit_position, snapshot_position))
 442     time.sleep(60)
 443   util.PrintAndFlush('Got snapshot commit position %s' % snapshot_position)
 444
 445
 446 def _AddToolsToPath(platform_name):
 447   """Add some tools like Ant and Java to PATH for testing steps to use."""
 448   paths = []
 449   error_message = ''
 450   if platform_name == 'win32':
 451     paths = [
 452         # Path to Ant and Java, required for the java acceptance tests.
 453         'C:\\Program Files (x86)\\Java\\ant\\bin',
 454         'C:\\Program Files (x86)\\Java\\jre\\bin',
 455     ]
 456     error_message = ('Java test steps will fail as expected and '
 457                      'they can be ignored.\n'
 458                      'Ant, Java or others might not be installed on bot.\n'
 459                      'Please refer to page "WATERFALL" on site '
 460                      'go/chromedriver.')
 461   if paths:
 462     util.MarkBuildStepStart('Add tools to PATH')
 463     path_missing = False
 464     for path in paths:
 465       if not os.path.isdir(path) or not os.listdir(path):
 466         print 'Directory "%s" is not found or empty.' % path
 467         path_missing = True
 468     if path_missing:
 469       print error_message
 470       util.MarkBuildStepError()
 471       return
 472     os.environ['PATH'] += os.pathsep + os.pathsep.join(paths)
 473
 474
 475 def main():
 476   parser = optparse.OptionParser()
 477   parser.add_option(
 478       '', '--android-packages',
 479       help=('Comma separated list of application package names, '
 480             'if running tests on Android.'))
 481   parser.add_option(
 482       '-r', '--revision', help='Chromium git revision hash')
 483   parser.add_option(
 484       '', '--update-log', action='store_true',
 485       help='Update the test results log (only applicable to Android)')
 486   options, _ = parser.parse_args()
 487
 488   bitness = '32'
 489   if util.IsLinux() and platform_module.architecture()[0] == '64bit':
 490     bitness = '64'
 491   platform = '%s%s' % (util.GetPlatformName(), bitness)
 492   if options.android_packages:
 493     platform = 'android'
 494
 495   _CleanTmpDir()
 496
 497   if not options.revision:
 498     commit_position = None
 499   else:
 500     commit_position = _GetCommitPositionFromGitHash(options.revision)
 501
 502   if platform == 'android':
 503     if not options.revision and options.update_log:
 504       parser.error('Must supply a --revision with --update-log')
 505     _DownloadPrebuilts()
 506   else:
 507     if not options.revision:
 508       parser.error('Must supply a --revision')
 509     if platform == 'linux64':
 510       _ArchivePrebuilts(commit_position)
 511     _WaitForLatestSnapshot(commit_position)
 512
 513   _AddToolsToPath(platform)
 514
 515   cmd = [
 516       sys.executable,
 517       os.path.join(_THIS_DIR, 'test', 'run_all_tests.py'),
 518   ]
 519   if platform == 'android':
 520     cmd.append('--android-packages=' + options.android_packages)
 521
 522   passed = (util.RunCommand(cmd) == 0)
 523
 524   _ArchiveServerLogs()
 525
 526   if platform == 'android':
 527     if options.update_log:
 528       util.MarkBuildStepStart('update test result log')
 529       _UpdateTestResultsLog(platform, commit_position, passed)
 530   elif passed:
 531     _ArchiveGoodBuild(platform, commit_position)
 532     _MaybeRelease(platform)
 533
 534   if not passed:
 535     # Make sure the build is red if there is some uncaught exception during
 536     # running run_all_tests.py.
 537     util.MarkBuildStepStart('run_all_tests.py')
 538     util.MarkBuildStepError()
 539
 540   # Add a "cleanup" step so that errors from runtest.py or bb_device_steps.py
 541   # (which invoke this script) are kept in thier own build step.
 542   util.MarkBuildStepStart('cleanup')
 543
 544
 545 if __name__ == '__main__':
 546   main()