chrome/test/chromedriver/run_buildbot_steps.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Runs all the buildbot steps for ChromeDriver except for update/compile."""
   7
   8 import bisect
   9 import csv
  10 import datetime
  11 import glob
  12 import json
  13 import optparse
  14 import os
  15 import platform as platform_module
  16 import re
  17 import shutil
  18 import StringIO
  19 import subprocess
  20 import sys
  21 import tempfile
  22 import time
  23 import urllib2
  24
  25 import archive
  26 import chrome_paths
  27 import util
  28
  29 _THIS_DIR = os.path.abspath(os.path.dirname(__file__))
  30 GS_CHROMEDRIVER_BUCKET = 'gs://chromedriver'
  31 GS_CHROMEDRIVER_DATA_BUCKET = 'gs://chromedriver-data'
  32 GS_CONTINUOUS_URL = GS_CHROMEDRIVER_DATA_BUCKET + '/continuous'
  33 GS_PREBUILTS_URL = GS_CHROMEDRIVER_DATA_BUCKET + '/prebuilts'
  34 GS_SERVER_LOGS_URL = GS_CHROMEDRIVER_DATA_BUCKET + '/server_logs'
  35 SERVER_LOGS_LINK = (
  36   'http://chromedriver-data.storage.googleapis.com/server_logs')
  37 TEST_LOG_FORMAT = '%s_log.json'
  38
  39 SCRIPT_DIR = os.path.join(_THIS_DIR, os.pardir, os.pardir, os.pardir, os.pardir,
  40                           os.pardir, os.pardir, os.pardir, 'scripts')
  41 SITE_CONFIG_DIR = os.path.join(_THIS_DIR, os.pardir, os.pardir, os.pardir,
  42                                os.pardir, os.pardir, os.pardir, os.pardir,
  43                                'site_config')
  44 sys.path.append(SCRIPT_DIR)
  45 sys.path.append(SITE_CONFIG_DIR)
  46 from slave import gsutil_download
  47 from slave import slave_utils
  48
  49
  50 def _ArchivePrebuilts(revision):
  51   """Uploads the prebuilts to google storage."""
  52   util.MarkBuildStepStart('archive prebuilts')
  53   zip_path = util.Zip(os.path.join(chrome_paths.GetBuildDir(['chromedriver']),
  54                                    'chromedriver'))
  55   if slave_utils.GSUtilCopy(
  56       zip_path,
  57       '%s/%s' % (GS_PREBUILTS_URL, 'r%s.zip' % revision)):
  58     util.MarkBuildStepError()
  59
  60
  61 def _ArchiveServerLogs():
  62   """Uploads chromedriver server logs to google storage."""
  63   util.MarkBuildStepStart('archive chromedriver server logs')
  64   for server_log in glob.glob(os.path.join(tempfile.gettempdir(),
  65                                            'chromedriver_*')):
  66     base_name = os.path.basename(server_log)
  67     util.AddLink(base_name, '%s/%s' % (SERVER_LOGS_LINK, base_name))
  68     slave_utils.GSUtilCopy(
  69         server_log,
  70         '%s/%s' % (GS_SERVER_LOGS_URL, base_name),
  71         mimetype='text/plain')
  72
  73
  74 def _DownloadPrebuilts():
  75   """Downloads the most recent prebuilts from google storage."""
  76   util.MarkBuildStepStart('Download latest chromedriver')
  77
  78   zip_path = os.path.join(util.MakeTempDir(), 'build.zip')
  79   if gsutil_download.DownloadLatestFile(GS_PREBUILTS_URL, 'r', zip_path):
  80     util.MarkBuildStepError()
  81
  82   util.Unzip(zip_path, chrome_paths.GetBuildDir(['host_forwarder']))
  83
  84
  85 def _GetTestResultsLog(platform):
  86   """Gets the test results log for the given platform.
  87
  88   Returns:
  89     A dictionary where the keys are SVN revisions and the values are booleans
  90     indicating whether the tests passed.
  91   """
  92   temp_log = tempfile.mkstemp()[1]
  93   log_name = TEST_LOG_FORMAT % platform
  94   result = slave_utils.GSUtilDownloadFile(
  95       '%s/%s' % (GS_CHROMEDRIVER_DATA_BUCKET, log_name), temp_log)
  96   if result:
  97     return {}
  98   with open(temp_log, 'rb') as log_file:
  99     json_dict = json.load(log_file)
 100   # Workaround for json encoding dictionary keys as strings.
 101   return dict([(int(v[0]), v[1]) for v in json_dict.items()])
 102
 103
 104 def _PutTestResultsLog(platform, test_results_log):
 105   """Pushes the given test results log to google storage."""
 106   temp_dir = util.MakeTempDir()
 107   log_name = TEST_LOG_FORMAT % platform
 108   log_path = os.path.join(temp_dir, log_name)
 109   with open(log_path, 'wb') as log_file:
 110     json.dump(test_results_log, log_file)
 111   if slave_utils.GSUtilCopyFile(log_path, GS_CHROMEDRIVER_DATA_BUCKET):
 112     raise Exception('Failed to upload test results log to google storage')
 113
 114
 115 def _UpdateTestResultsLog(platform, revision, passed):
 116   """Updates the test results log for the given platform.
 117
 118   Args:
 119     platform: The platform name.
 120     revision: The SVN revision number.
 121     passed: Boolean indicating whether the tests passed at this revision.
 122   """
 123   assert isinstance(revision, int), 'The revision must be an integer'
 124   log = _GetTestResultsLog(platform)
 125   if len(log) > 500:
 126     del log[min(log.keys())]
 127   assert revision not in log, 'Results already exist for revision %s' % revision
 128   log[revision] = bool(passed)
 129   _PutTestResultsLog(platform, log)
 130
 131
 132 def _GetVersion():
 133   """Get the current chromedriver version."""
 134   with open(os.path.join(_THIS_DIR, 'VERSION'), 'r') as f:
 135     return f.read().strip()
 136
 137
 138 def _GetSupportedChromeVersions():
 139   """Get the minimum and maximum supported Chrome versions.
 140
 141   Returns:
 142     A tuple of the form (min_version, max_version).
 143   """
 144   # Minimum supported Chrome version is embedded as:
 145   # const int kMinimumSupportedChromeVersion[] = {27, 0, 1453, 0};
 146   with open(os.path.join(_THIS_DIR, 'chrome', 'version.cc'), 'r') as f:
 147     lines = f.readlines()
 148     chrome_min_version_line = filter(
 149         lambda x: 'kMinimumSupportedChromeVersion' in x, lines)
 150   chrome_min_version = chrome_min_version_line[0].split('{')[1].split(',')[0]
 151   with open(os.path.join(chrome_paths.GetSrc(), 'chrome', 'VERSION'), 'r') as f:
 152     chrome_max_version = f.readlines()[0].split('=')[1].strip()
 153   return (chrome_min_version, chrome_max_version)
 154
 155
 156 def _RevisionState(test_results_log, revision):
 157   """Check the state of tests at a given SVN revision.
 158
 159   Considers tests as having passed at a revision if they passed at revisons both
 160   before and after.
 161
 162   Args:
 163     test_results_log: A test results log dictionary from _GetTestResultsLog().
 164     revision: The revision to check at.
 165
 166   Returns:
 167     'passed', 'failed', or 'unknown'
 168   """
 169   assert isinstance(revision, int), 'The revision must be an integer'
 170   keys = sorted(test_results_log.keys())
 171   # Return passed if the exact revision passed on Android.
 172   if revision in test_results_log:
 173     return 'passed' if test_results_log[revision] else 'failed'
 174   # Tests were not run on this exact revision on Android.
 175   index = bisect.bisect_right(keys, revision)
 176   # Tests have not yet run on Android at or above this revision.
 177   if index == len(test_results_log):
 178     return 'unknown'
 179   # No log exists for any prior revision, assume it failed.
 180   if index == 0:
 181     return 'failed'
 182   # Return passed if the revisions on both sides passed.
 183   if test_results_log[keys[index]] and test_results_log[keys[index - 1]]:
 184     return 'passed'
 185   return 'failed'
 186
 187
 188 def _ArchiveGoodBuild(platform, revision):
 189   assert platform != 'android'
 190   util.MarkBuildStepStart('archive build')
 191
 192   server_name = 'chromedriver'
 193   if util.IsWindows():
 194     server_name += '.exe'
 195   zip_path = util.Zip(os.path.join(chrome_paths.GetBuildDir([server_name]),
 196                                    server_name))
 197
 198   build_name = 'chromedriver_%s_%s.%s.zip' % (
 199       platform, _GetVersion(), revision)
 200   build_url = '%s/%s' % (GS_CONTINUOUS_URL, build_name)
 201   if slave_utils.GSUtilCopy(zip_path, build_url):
 202     util.MarkBuildStepError()
 203
 204   (latest_fd, latest_file) = tempfile.mkstemp()
 205   os.write(latest_fd, build_name)
 206   os.close(latest_fd)
 207   latest_url = '%s/latest_%s' % (GS_CONTINUOUS_URL, platform)
 208   if slave_utils.GSUtilCopy(latest_file, latest_url, mimetype='text/plain'):
 209     util.MarkBuildStepError()
 210   os.remove(latest_file)
 211
 212
 213 def _MaybeRelease(platform):
 214   """Releases a release candidate if conditions are right."""
 215   assert platform != 'android'
 216
 217   # Check if the current version has already been released.
 218   result, _ = slave_utils.GSUtilListBucket(
 219       '%s/%s/chromedriver_%s*' % (
 220           GS_CHROMEDRIVER_BUCKET, _GetVersion(), platform),
 221       [])
 222   if result == 0:
 223     return
 224
 225   # Fetch Android test results.
 226   android_test_results = _GetTestResultsLog('android')
 227
 228   # Fetch release candidates.
 229   result, output = slave_utils.GSUtilListBucket(
 230       '%s/chromedriver_%s_%s*' % (
 231           GS_CONTINUOUS_URL, platform, _GetVersion()),
 232       [])
 233   assert result == 0 and output, 'No release candidates found'
 234   candidates = [b.split('/')[-1] for b in output.strip().split('\n')]
 235   candidate_pattern = re.compile('chromedriver_%s_%s\.\d+\.zip'
 236       % (platform, _GetVersion()))
 237
 238   # Release the first candidate build that passed Android, if any.
 239   for candidate in candidates:
 240     if not candidate_pattern.match(candidate):
 241       print 'Ignored candidate "%s"' % candidate
 242       continue
 243     revision = candidate.split('.')[-2]
 244     android_result = _RevisionState(android_test_results, int(revision))
 245     if android_result == 'failed':
 246       print 'Android tests did not pass at revision', revision
 247     elif android_result == 'passed':
 248       print 'Android tests passed at revision', revision
 249       _Release('%s/%s' % (GS_CONTINUOUS_URL, candidate), platform)
 250       break
 251     else:
 252       print 'Android tests have not run at a revision as recent as', revision
 253
 254
 255 def _Release(build, platform):
 256   """Releases the given candidate build."""
 257   release_name = 'chromedriver_%s.zip' % platform
 258   util.MarkBuildStepStart('releasing %s' % release_name)
 259   slave_utils.GSUtilCopy(
 260       build, '%s/%s/%s' % (GS_CHROMEDRIVER_BUCKET, _GetVersion(), release_name))
 261
 262   _MaybeUploadReleaseNotes()
 263
 264
 265 def _MaybeUploadReleaseNotes():
 266   """Upload release notes if conditions are right."""
 267   # Check if the current version has already been released.
 268   version = _GetVersion()
 269   notes_name = 'notes.txt'
 270   notes_url = '%s/%s/%s' % (GS_CHROMEDRIVER_BUCKET, version, notes_name)
 271   prev_version = '.'.join([version.split('.')[0],
 272                           str(int(version.split('.')[1]) - 1)])
 273   prev_notes_url = '%s/%s/%s' % (
 274       GS_CHROMEDRIVER_BUCKET, prev_version, notes_name)
 275
 276   result, _ = slave_utils.GSUtilListBucket(notes_url, [])
 277   if result == 0:
 278     return
 279
 280   fixed_issues = []
 281   query = ('https://code.google.com/p/chromedriver/issues/csv?'
 282            'q=status%3AToBeReleased&colspec=ID%20Summary')
 283   issues = StringIO.StringIO(urllib2.urlopen(query).read().split('\n', 1)[1])
 284   for issue in csv.reader(issues):
 285     if not issue:
 286       continue
 287     id = issue[0]
 288     desc = issue[1]
 289     labels = issue[2]
 290     fixed_issues += ['Resolved issue %s: %s [%s]' % (id, desc, labels)]
 291
 292   old_notes = ''
 293   temp_notes_fname = tempfile.mkstemp()[1]
 294   if not slave_utils.GSUtilDownloadFile(prev_notes_url, temp_notes_fname):
 295     with open(temp_notes_fname, 'rb') as f:
 296       old_notes = f.read()
 297
 298   new_notes = '----------ChromeDriver v%s (%s)----------\n%s\n%s\n\n%s' % (
 299       version, datetime.date.today().isoformat(),
 300       'Supports Chrome v%s-%s' % _GetSupportedChromeVersions(),
 301       '\n'.join(fixed_issues),
 302       old_notes)
 303   with open(temp_notes_fname, 'w') as f:
 304     f.write(new_notes)
 305
 306   if slave_utils.GSUtilCopy(temp_notes_fname, notes_url, mimetype='text/plain'):
 307     util.MarkBuildStepError()
 308
 309
 310 def _KillChromes():
 311   chrome_map = {
 312       'win': 'chrome.exe',
 313       'mac': 'Chromium',
 314       'linux': 'chrome',
 315   }
 316   if util.IsWindows():
 317     cmd = ['taskkill', '/F', '/IM']
 318   else:
 319     cmd = ['killall', '-9']
 320   cmd.append(chrome_map[util.GetPlatformName()])
 321   util.RunCommand(cmd)
 322
 323
 324 def _CleanTmpDir():
 325   tmp_dir = tempfile.gettempdir()
 326   print 'cleaning temp directory:', tmp_dir
 327   for file_name in os.listdir(tmp_dir):
 328     file_path = os.path.join(tmp_dir, file_name)
 329     if os.path.isdir(file_path):
 330       print 'deleting sub-directory', file_path
 331       shutil.rmtree(file_path, True)
 332     if file_name.startswith('chromedriver_'):
 333       print 'deleting file', file_path
 334       os.remove(file_path)
 335
 336
 337 def _WaitForLatestSnapshot(revision):
 338   util.MarkBuildStepStart('wait_for_snapshot')
 339   while True:
 340     snapshot_revision = archive.GetLatestRevision(archive.Site.SNAPSHOT)
 341     if int(snapshot_revision) >= int(revision):
 342       break
 343     util.PrintAndFlush('Waiting for snapshot >= %s, found %s' %
 344                        (revision, snapshot_revision))
 345     time.sleep(60)
 346   util.PrintAndFlush('Got snapshot revision %s' % snapshot_revision)
 347
 348
 349 def _AddToolsToPath(platform_name):
 350   """Add some tools like Ant and Java to PATH for testing steps to use."""
 351   paths = []
 352   error_message = ''
 353   if platform_name == 'win32':
 354     paths = [
 355         # Path to Ant and Java, required for the java acceptance tests.
 356         'C:\\Program Files (x86)\\Java\\ant\\bin',
 357         'C:\\Program Files (x86)\\Java\\jre\\bin',
 358     ]
 359     error_message = ('Java test steps will fail as expected and '
 360                      'they can be ignored.\n'
 361                      'Ant, Java or others might not be installed on bot.\n'
 362                      'Please refer to page "WATERFALL" on site '
 363                      'go/chromedriver.')
 364   if paths:
 365     util.MarkBuildStepStart('Add tools to PATH')
 366     path_missing = False
 367     for path in paths:
 368       if not os.path.isdir(path) or not os.listdir(path):
 369         print 'Directory "%s" is not found or empty.' % path
 370         path_missing = True
 371     if path_missing:
 372       print error_message
 373       util.MarkBuildStepError()
 374       return
 375     os.environ['PATH'] += os.pathsep + os.pathsep.join(paths)
 376
 377
 378 def main():
 379   parser = optparse.OptionParser()
 380   parser.add_option(
 381       '', '--android-packages',
 382       help='Comma separated list of application package names, '
 383            'if running tests on Android.')
 384   parser.add_option(
 385       '-r', '--revision', type='int', help='Chromium revision')
 386   parser.add_option('', '--update-log', action='store_true',
 387       help='Update the test results log (only applicable to Android)')
 388   options, _ = parser.parse_args()
 389
 390   bitness = '32'
 391   if util.IsLinux() and platform_module.architecture()[0] == '64bit':
 392     bitness = '64'
 393   platform = '%s%s' % (util.GetPlatformName(), bitness)
 394   if options.android_packages:
 395     platform = 'android'
 396
 397   if platform != 'android':
 398     _KillChromes()
 399   _CleanTmpDir()
 400
 401   if platform == 'android':
 402     if not options.revision and options.update_log:
 403       parser.error('Must supply a --revision with --update-log')
 404     _DownloadPrebuilts()
 405   else:
 406     if not options.revision:
 407       parser.error('Must supply a --revision')
 408     if platform == 'linux64':
 409       _ArchivePrebuilts(options.revision)
 410     _WaitForLatestSnapshot(options.revision)
 411
 412   _AddToolsToPath(platform)
 413
 414   cmd = [
 415       sys.executable,
 416       os.path.join(_THIS_DIR, 'test', 'run_all_tests.py'),
 417   ]
 418   if platform == 'android':
 419     cmd.append('--android-packages=' + options.android_packages)
 420
 421   passed = (util.RunCommand(cmd) == 0)
 422
 423   _ArchiveServerLogs()
 424
 425   if platform == 'android':
 426     if options.update_log:
 427       util.MarkBuildStepStart('update test result log')
 428       _UpdateTestResultsLog(platform, options.revision, passed)
 429   elif passed:
 430     _ArchiveGoodBuild(platform, options.revision)
 431     _MaybeRelease(platform)
 432
 433   if not passed:
 434     # Make sure the build is red if there is some uncaught exception during
 435     # running run_all_tests.py.
 436     util.MarkBuildStepStart('run_all_tests.py')
 437     util.MarkBuildStepError()
 438
 439   # Add a "cleanup" step so that errors from runtest.py or bb_device_steps.py
 440   # (which invoke this script) are kept in thier own build step.
 441   util.MarkBuildStepStart('cleanup')
 442
 443
 444 if __name__ == '__main__':
 445   main()