chrome/test/chromedriver/run_buildbot_steps.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Runs all the buildbot steps for ChromeDriver except for update/compile."""
   7
   8 import bisect
   9 import csv
  10 import datetime
  11 import glob
  12 import json
  13 import optparse
  14 import os
  15 import platform as platform_module
  16 import re
  17 import shutil
  18 import StringIO
  19 import sys
  20 import tempfile
  21 import time
  22 import urllib2
  23
  24 _THIS_DIR = os.path.abspath(os.path.dirname(__file__))
  25 GS_CHROMEDRIVER_BUCKET = 'gs://chromedriver'
  26 GS_CHROMEDRIVER_DATA_BUCKET = 'gs://chromedriver-data'
  27 GS_CHROMEDRIVER_RELEASE_URL = 'http://chromedriver.storage.googleapis.com'
  28 GS_CONTINUOUS_URL = GS_CHROMEDRIVER_DATA_BUCKET + '/continuous'
  29 GS_PREBUILTS_URL = GS_CHROMEDRIVER_DATA_BUCKET + '/prebuilts'
  30 GS_SERVER_LOGS_URL = GS_CHROMEDRIVER_DATA_BUCKET + '/server_logs'
  31 SERVER_LOGS_LINK = (
  32     'http://chromedriver-data.storage.googleapis.com/server_logs')
  33 TEST_LOG_FORMAT = '%s_log.json'
  34 GS_GIT_LOG_URL = (
  35     'https://chromium.googlesource.com/chromium/src/+/%s?format=json')
  36 GS_SEARCH_PATTERN = (
  37     r'Cr-Commit-Position: refs/heads/master@{#(\d+)}')
  38 CR_REV_URL = 'https://cr-rev.appspot.com/_ah/api/crrev/v1/redirect/%s'
  39
  40 SCRIPT_DIR = os.path.join(_THIS_DIR, os.pardir, os.pardir, os.pardir, os.pardir,
  41                           os.pardir, os.pardir, os.pardir, 'scripts')
  42 SITE_CONFIG_DIR = os.path.join(_THIS_DIR, os.pardir, os.pardir, os.pardir,
  43                                os.pardir, os.pardir, os.pardir, os.pardir,
  44                                'site_config')
  45 sys.path.append(SCRIPT_DIR)
  46 sys.path.append(SITE_CONFIG_DIR)
  47
  48 import archive
  49 import chrome_paths
  50 from slave import gsutil_download
  51 from slave import slave_utils
  52 import util
  53
  54
  55 def _ArchivePrebuilts(commit_position):
  56   """Uploads the prebuilts to google storage."""
  57   util.MarkBuildStepStart('archive prebuilts')
  58   zip_path = util.Zip(os.path.join(chrome_paths.GetBuildDir(['chromedriver']),
  59                                    'chromedriver'))
  60   if slave_utils.GSUtilCopy(
  61       zip_path,
  62       '%s/%s' % (GS_PREBUILTS_URL, 'r%s.zip' % commit_position)):
  63     util.MarkBuildStepError()
  64
  65
  66 def _ArchiveServerLogs():
  67   """Uploads chromedriver server logs to google storage."""
  68   util.MarkBuildStepStart('archive chromedriver server logs')
  69   for server_log in glob.glob(os.path.join(tempfile.gettempdir(),
  70                                            'chromedriver_*')):
  71     base_name = os.path.basename(server_log)
  72     util.AddLink(base_name, '%s/%s' % (SERVER_LOGS_LINK, base_name))
  73     slave_utils.GSUtilCopy(
  74         server_log,
  75         '%s/%s' % (GS_SERVER_LOGS_URL, base_name),
  76         mimetype='text/plain')
  77
  78
  79 def _DownloadPrebuilts():
  80   """Downloads the most recent prebuilts from google storage."""
  81   util.MarkBuildStepStart('Download latest chromedriver')
  82
  83   zip_path = os.path.join(util.MakeTempDir(), 'build.zip')
  84   if gsutil_download.DownloadLatestFile(GS_PREBUILTS_URL,
  85                                         GS_PREBUILTS_URL + '/r',
  86                                         zip_path):
  87     util.MarkBuildStepError()
  88
  89   util.Unzip(zip_path, chrome_paths.GetBuildDir(['host_forwarder']))
  90
  91
  92 def _GetTestResultsLog(platform):
  93   """Gets the test results log for the given platform.
  94
  95   Args:
  96     platform: The platform that the test results log is for.
  97
  98   Returns:
  99     A dictionary where the keys are commit positions and the values are booleans
 100     indicating whether the tests passed.
 101   """
 102   temp_log = tempfile.mkstemp()[1]
 103   log_name = TEST_LOG_FORMAT % platform
 104   result = slave_utils.GSUtilDownloadFile(
 105       '%s/%s' % (GS_CHROMEDRIVER_DATA_BUCKET, log_name), temp_log)
 106   if result:
 107     return {}
 108   with open(temp_log, 'rb') as log_file:
 109     json_dict = json.load(log_file)
 110   # Workaround for json encoding dictionary keys as strings.
 111   return dict([(int(v[0]), v[1]) for v in json_dict.items()])
 112
 113
 114 def _PutTestResultsLog(platform, test_results_log):
 115   """Pushes the given test results log to google storage."""
 116   temp_dir = util.MakeTempDir()
 117   log_name = TEST_LOG_FORMAT % platform
 118   log_path = os.path.join(temp_dir, log_name)
 119   with open(log_path, 'wb') as log_file:
 120     json.dump(test_results_log, log_file)
 121   if slave_utils.GSUtilCopyFile(log_path, GS_CHROMEDRIVER_DATA_BUCKET):
 122     raise Exception('Failed to upload test results log to google storage')
 123
 124
 125 def _UpdateTestResultsLog(platform, commit_position, passed):
 126   """Updates the test results log for the given platform.
 127
 128   Args:
 129     platform: The platform name.
 130     commit_position: The commit position number.
 131     passed: Boolean indicating whether the tests passed at this commit position.
 132   """
 133
 134   assert commit_position.isdigit(), 'The commit position must be a number'
 135   commit_position = int(commit_position)
 136   log = _GetTestResultsLog(platform)
 137   if len(log) > 500:
 138     del log[min(log.keys())]
 139   assert commit_position not in log, \
 140       'Results already exist for commit position %s' % commit_position
 141   log[commit_position] = bool(passed)
 142   _PutTestResultsLog(platform, log)
 143
 144
 145 def _GetVersion():
 146   """Get the current chromedriver version."""
 147   with open(os.path.join(_THIS_DIR, 'VERSION'), 'r') as f:
 148     return f.read().strip()
 149
 150
 151 def _GetSupportedChromeVersions():
 152   """Get the minimum and maximum supported Chrome versions.
 153
 154   Returns:
 155     A tuple of the form (min_version, max_version).
 156   """
 157   # Minimum supported Chrome version is embedded as:
 158   # const int kMinimumSupportedChromeVersion[] = {27, 0, 1453, 0};
 159   with open(os.path.join(_THIS_DIR, 'chrome', 'version.cc'), 'r') as f:
 160     lines = f.readlines()
 161     chrome_min_version_line = [
 162         x for x in lines if 'kMinimumSupportedChromeVersion' in x]
 163   chrome_min_version = chrome_min_version_line[0].split('{')[1].split(',')[0]
 164   with open(os.path.join(chrome_paths.GetSrc(), 'chrome', 'VERSION'), 'r') as f:
 165     chrome_max_version = f.readlines()[0].split('=')[1].strip()
 166   return (chrome_min_version, chrome_max_version)
 167
 168
 169 def _CommitPositionState(test_results_log, commit_position):
 170   """Check the state of tests at a given commit position.
 171
 172   Considers tests as having passed at a commit position if they passed at
 173   revisons both before and after.
 174
 175   Args:
 176     test_results_log: A test results log dictionary from _GetTestResultsLog().
 177     commit_position: The commit position to check at.
 178
 179   Returns:
 180     'passed', 'failed', or 'unknown'
 181   """
 182   assert isinstance(commit_position, int), 'The commit position must be an int'
 183   keys = sorted(test_results_log.keys())
 184   # Return passed if the exact commit position passed on Android.
 185   if commit_position in test_results_log:
 186     return 'passed' if test_results_log[commit_position] else 'failed'
 187   # Tests were not run on this exact commit position on Android.
 188   index = bisect.bisect_right(keys, commit_position)
 189   # Tests have not yet run on Android at or above this commit position.
 190   if index == len(test_results_log):
 191     return 'unknown'
 192   # No log exists for any prior commit position, assume it failed.
 193   if index == 0:
 194     return 'failed'
 195   # Return passed if the commit position on both sides passed.
 196   if test_results_log[keys[index]] and test_results_log[keys[index - 1]]:
 197     return 'passed'
 198   return 'failed'
 199
 200
 201 def _ArchiveGoodBuild(platform, commit_position):
 202   """Archive chromedriver binary if the build is green."""
 203   assert platform != 'android'
 204   util.MarkBuildStepStart('archive build')
 205
 206   server_name = 'chromedriver'
 207   if util.IsWindows():
 208     server_name += '.exe'
 209   zip_path = util.Zip(os.path.join(chrome_paths.GetBuildDir([server_name]),
 210                                    server_name))
 211
 212   build_name = 'chromedriver_%s_%s.%s.zip' % (
 213       platform, _GetVersion(), commit_position)
 214   build_url = '%s/%s' % (GS_CONTINUOUS_URL, build_name)
 215   if slave_utils.GSUtilCopy(zip_path, build_url):
 216     util.MarkBuildStepError()
 217
 218   (latest_fd, latest_file) = tempfile.mkstemp()
 219   os.write(latest_fd, build_name)
 220   os.close(latest_fd)
 221   latest_url = '%s/latest_%s' % (GS_CONTINUOUS_URL, platform)
 222   if slave_utils.GSUtilCopy(latest_file, latest_url, mimetype='text/plain'):
 223     util.MarkBuildStepError()
 224   os.remove(latest_file)
 225
 226
 227 def _WasReleased(version, platform):
 228   """Check if the specified version is released for the given platform."""
 229   result, _ = slave_utils.GSUtilListBucket(
 230       '%s/%s/chromedriver_%s.zip' % (GS_CHROMEDRIVER_BUCKET, version, platform),
 231       [])
 232   return result == 0
 233
 234
 235 def _MaybeRelease(platform):
 236   """Releases a release candidate if conditions are right."""
 237   assert platform != 'android'
 238
 239   version = _GetVersion()
 240
 241   # Check if the current version has already been released.
 242   if _WasReleased(version, platform):
 243     return
 244
 245   # Fetch Android test results.
 246   android_test_results = _GetTestResultsLog('android')
 247
 248   # Fetch release candidates.
 249   result, output = slave_utils.GSUtilListBucket(
 250       '%s/chromedriver_%s_%s*' % (
 251           GS_CONTINUOUS_URL, platform, version),
 252       [])
 253   assert result == 0 and output, 'No release candidates found'
 254   candidate_pattern = re.compile(
 255       r'.*/chromedriver_%s_%s\.(\d+)\.zip$' % (platform, version))
 256   candidates = []
 257   for line in output.strip().split('\n'):
 258     result = candidate_pattern.match(line)
 259     if not result:
 260       print 'Ignored line "%s"' % line
 261       continue
 262     candidates.append(int(result.group(1)))
 263
 264   # Release the latest candidate build that passed Android, if any.
 265   # In this way, if a hot fix is needed, we can delete the release from
 266   # the chromedriver bucket instead of bumping up the release version number.
 267   candidates.sort(reverse=True)
 268   for commit_position in candidates:
 269     android_result = _CommitPositionState(android_test_results, commit_position)
 270     if android_result == 'failed':
 271       print 'Android tests did not pass at commit position', commit_position
 272     elif android_result == 'passed':
 273       print 'Android tests passed at commit position', commit_position
 274       candidate = 'chromedriver_%s_%s.%s.zip' % (
 275           platform, version, commit_position)
 276       _Release('%s/%s' % (GS_CONTINUOUS_URL, candidate), version, platform)
 277       break
 278     else:
 279       print 'Android tests have not run at a commit position as recent as', \
 280           commit_position
 281
 282
 283 def _Release(build, version, platform):
 284   """Releases the given candidate build."""
 285   release_name = 'chromedriver_%s.zip' % platform
 286   util.MarkBuildStepStart('releasing %s' % release_name)
 287   temp_dir = util.MakeTempDir()
 288   slave_utils.GSUtilCopy(build, temp_dir)
 289   zip_path = os.path.join(temp_dir, os.path.basename(build))
 290
 291   if util.IsLinux():
 292     util.Unzip(zip_path, temp_dir)
 293     server_path = os.path.join(temp_dir, 'chromedriver')
 294     util.RunCommand(['strip', server_path])
 295     zip_path = util.Zip(server_path)
 296
 297   slave_utils.GSUtilCopy(
 298       zip_path, '%s/%s/%s' % (GS_CHROMEDRIVER_BUCKET, version, release_name))
 299
 300   _MaybeUploadReleaseNotes(version)
 301   _MaybeUpdateLatestRelease(version)
 302
 303
 304 def _GetWebPageContent(url):
 305   """Return the content of the web page specified by the given url."""
 306   return urllib2.urlopen(url).read()
 307
 308
 309 def _MaybeUploadReleaseNotes(version):
 310   """Upload release notes if conditions are right."""
 311   # Check if the current version has already been released.
 312   notes_name = 'notes.txt'
 313   notes_url = '%s/%s/%s' % (GS_CHROMEDRIVER_BUCKET, version, notes_name)
 314   prev_version = '.'.join([version.split('.')[0],
 315                            str(int(version.split('.')[1]) - 1)])
 316   prev_notes_url = '%s/%s/%s' % (
 317       GS_CHROMEDRIVER_BUCKET, prev_version, notes_name)
 318
 319   result, _ = slave_utils.GSUtilListBucket(notes_url, [])
 320   if result == 0:
 321     return
 322
 323   fixed_issues = []
 324   query = ('https://code.google.com/p/chromedriver/issues/csv?'
 325            'can=1&q=label%%3AChromeDriver-%s&colspec=ID%%20Summary' % version)
 326   issues = StringIO.StringIO(_GetWebPageContent(query).split('\n', 1)[1])
 327   for issue in csv.reader(issues):
 328     if not issue:
 329       continue
 330     issue_id = issue[0]
 331     desc = issue[1]
 332     labels = issue[2].split(', ')
 333     labels.remove('ChromeDriver-%s' % version)
 334     if 'Hotlist-GoodFirstBug' in labels:
 335       labels.remove('Hotlist-GoodFirstBug')
 336     fixed_issues += ['Resolved issue %s: %s [%s]' % (issue_id, desc, labels)]
 337
 338   old_notes = ''
 339   temp_notes_fname = tempfile.mkstemp()[1]
 340   if not slave_utils.GSUtilDownloadFile(prev_notes_url, temp_notes_fname):
 341     with open(temp_notes_fname, 'rb') as f:
 342       old_notes = f.read()
 343
 344   new_notes = '----------ChromeDriver v%s (%s)----------\n%s\n%s\n\n%s' % (
 345       version, datetime.date.today().isoformat(),
 346       'Supports Chrome v%s-%s' % _GetSupportedChromeVersions(),
 347       '\n'.join(fixed_issues),
 348       old_notes)
 349   with open(temp_notes_fname, 'w') as f:
 350     f.write(new_notes)
 351
 352   if slave_utils.GSUtilCopy(temp_notes_fname, notes_url, mimetype='text/plain'):
 353     util.MarkBuildStepError()
 354
 355
 356 def _MaybeUpdateLatestRelease(version):
 357   """Update the file LATEST_RELEASE with the latest release version number."""
 358   latest_release_fname = 'LATEST_RELEASE'
 359   latest_release_url = '%s/%s' % (GS_CHROMEDRIVER_BUCKET, latest_release_fname)
 360
 361   # Check if LATEST_RELEASE is up-to-date.
 362   latest_released_version = _GetWebPageContent(
 363       '%s/%s' % (GS_CHROMEDRIVER_RELEASE_URL, latest_release_fname))
 364   if version == latest_released_version:
 365     return
 366
 367   # Check if chromedriver was released on all supported platforms.
 368   supported_platforms = ['linux32', 'linux64', 'mac32', 'win32']
 369   for platform in supported_platforms:
 370     if not _WasReleased(version, platform):
 371       return
 372
 373   util.MarkBuildStepStart('updating LATEST_RELEASE to %s' % version)
 374
 375   temp_latest_release_fname = tempfile.mkstemp()[1]
 376   with open(temp_latest_release_fname, 'w') as f:
 377     f.write(version)
 378   if slave_utils.GSUtilCopy(temp_latest_release_fname, latest_release_url,
 379                             mimetype='text/plain'):
 380     util.MarkBuildStepError()
 381
 382
 383 def _CleanTmpDir():
 384   tmp_dir = tempfile.gettempdir()
 385   print 'cleaning temp directory:', tmp_dir
 386   for file_name in os.listdir(tmp_dir):
 387     file_path = os.path.join(tmp_dir, file_name)
 388     if os.path.isdir(file_path):
 389       print 'deleting sub-directory', file_path
 390       shutil.rmtree(file_path, True)
 391     if file_name.startswith('chromedriver_'):
 392       print 'deleting file', file_path
 393       os.remove(file_path)
 394
 395
 396 def _GetCommitPositionFromGitHash(snapshot_hashcode):
 397   json_url = GS_GIT_LOG_URL % snapshot_hashcode
 398   try:
 399     response = urllib2.urlopen(json_url)
 400   except urllib2.HTTPError as error:
 401     util.PrintAndFlush('HTTP Error %d' % error.getcode())
 402     return None
 403   except urllib2.URLError as error:
 404     util.PrintAndFlush('URL Error %s' % error.message)
 405     return None
 406   data = json.loads(response.read()[4:])
 407   if 'message' in data:
 408     message = data['message'].split('\n')
 409     message = [line for line in message if line.strip()]
 410     search_pattern = re.compile(GS_SEARCH_PATTERN)
 411     result = search_pattern.search(message[len(message)-1])
 412     if result:
 413       return result.group(1)
 414   util.PrintAndFlush('Failed to get commit position number for %s' %
 415                      snapshot_hashcode)
 416   return None
 417
 418
 419 def _GetGitHashFromCommitPosition(commit_position):
 420   json_url = CR_REV_URL % commit_position
 421   try:
 422     response = urllib2.urlopen(json_url)
 423   except urllib2.HTTPError as error:
 424     util.PrintAndFlush('HTTP Error %d' % error.getcode())
 425     return None
 426   except urllib2.URLError as error:
 427     util.PrintAndFlush('URL Error %s' % error.message)
 428     return None
 429   data = json.loads(response.read())
 430   if 'git_sha' in data:
 431     return data['git_sha']
 432   util.PrintAndFlush('Failed to get git hash for %s' % commit_position)
 433   return None
 434
 435
 436 def _WaitForLatestSnapshot(commit_position):
 437   util.MarkBuildStepStart('wait_for_snapshot')
 438   while True:
 439     snapshot_position = archive.GetLatestSnapshotVersion()
 440     if commit_position is not None and snapshot_position is not None:
 441       if int(snapshot_position) >= int(commit_position):
 442         break
 443       util.PrintAndFlush('Waiting for snapshot >= %s, found %s' %
 444                          (commit_position, snapshot_position))
 445     time.sleep(60)
 446   util.PrintAndFlush('Got snapshot commit position %s' % snapshot_position)
 447
 448
 449 def _AddToolsToPath(platform_name):
 450   """Add some tools like Ant and Java to PATH for testing steps to use."""
 451   paths = []
 452   error_message = ''
 453   if platform_name == 'win32':
 454     paths = [
 455         # Path to Ant and Java, required for the java acceptance tests.
 456         'C:\\Program Files (x86)\\Java\\ant\\bin',
 457         'C:\\Program Files (x86)\\Java\\jre\\bin',
 458     ]
 459     error_message = ('Java test steps will fail as expected and '
 460                      'they can be ignored.\n'
 461                      'Ant, Java or others might not be installed on bot.\n'
 462                      'Please refer to page "WATERFALL" on site '
 463                      'go/chromedriver.')
 464   if paths:
 465     util.MarkBuildStepStart('Add tools to PATH')
 466     path_missing = False
 467     for path in paths:
 468       if not os.path.isdir(path) or not os.listdir(path):
 469         print 'Directory "%s" is not found or empty.' % path
 470         path_missing = True
 471     if path_missing:
 472       print error_message
 473       util.MarkBuildStepError()
 474       return
 475     os.environ['PATH'] += os.pathsep + os.pathsep.join(paths)
 476
 477
 478 def main():
 479   parser = optparse.OptionParser()
 480   parser.add_option(
 481       '', '--android-packages',
 482       help=('Comma separated list of application package names, '
 483             'if running tests on Android.'))
 484   parser.add_option(
 485       '-r', '--revision', help='Chromium git revision hash')
 486   parser.add_option(
 487       '', '--update-log', action='store_true',
 488       help='Update the test results log (only applicable to Android)')
 489   options, _ = parser.parse_args()
 490
 491   bitness = '32'
 492   if util.IsLinux() and platform_module.architecture()[0] == '64bit':
 493     bitness = '64'
 494   platform = '%s%s' % (util.GetPlatformName(), bitness)
 495   if options.android_packages:
 496     platform = 'android'
 497
 498   _CleanTmpDir()
 499
 500   if not options.revision:
 501     commit_position = None
 502   else:
 503     commit_position = _GetCommitPositionFromGitHash(options.revision)
 504
 505   if platform == 'android':
 506     if not options.revision and options.update_log:
 507       parser.error('Must supply a --revision with --update-log')
 508     _DownloadPrebuilts()
 509   else:
 510     if not options.revision:
 511       parser.error('Must supply a --revision')
 512     if platform == 'linux64':
 513       _ArchivePrebuilts(commit_position)
 514     _WaitForLatestSnapshot(commit_position)
 515
 516   _AddToolsToPath(platform)
 517
 518   cmd = [
 519       sys.executable,
 520       os.path.join(_THIS_DIR, 'test', 'run_all_tests.py'),
 521   ]
 522   if platform == 'android':
 523     cmd.append('--android-packages=' + options.android_packages)
 524
 525   passed = (util.RunCommand(cmd) == 0)
 526
 527   _ArchiveServerLogs()
 528
 529   if platform == 'android':
 530     if options.update_log:
 531       util.MarkBuildStepStart('update test result log')
 532       _UpdateTestResultsLog(platform, commit_position, passed)
 533   elif passed:
 534     _ArchiveGoodBuild(platform, commit_position)
 535     _MaybeRelease(platform)
 536
 537   if not passed:
 538     # Make sure the build is red if there is some uncaught exception during
 539     # running run_all_tests.py.
 540     util.MarkBuildStepStart('run_all_tests.py')
 541     util.MarkBuildStepError()
 542
 543   # Add a "cleanup" step so that errors from runtest.py or bb_device_steps.py
 544   # (which invoke this script) are kept in thier own build step.
 545   util.MarkBuildStepStart('cleanup')
 546
 547
 548 if __name__ == '__main__':
 549   main()