tools/run-bisect-perf-regression.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Run Performance Test Bisect Tool
   7
   8 This script is used by a trybot to run the src/tools/bisect-perf-regression.py
   9 script with the parameters specified in run-bisect-perf-regression.cfg. It will
  10 check out a copy of the depot in a subdirectory 'bisect' of the working
  11 directory provided, and run the bisect-perf-regression.py script there.
  12
  13 """
  14
  15 import imp
  16 import optparse
  17 import os
  18 import subprocess
  19 import sys
  20 import traceback
  21
  22 import bisect_utils
  23 bisect = imp.load_source('bisect-perf-regression',
  24     os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])),
  25         'bisect-perf-regression.py'))
  26
  27
  28 CROS_BOARD_ENV = 'BISECT_CROS_BOARD'
  29 CROS_IP_ENV = 'BISECT_CROS_IP'
  30
  31
  32 class Goma(object):
  33
  34   def __init__(self, path_to_goma):
  35     self._abs_path_to_goma = None
  36     self._abs_path_to_goma_file = None
  37     if path_to_goma:
  38       self._abs_path_to_goma = os.path.abspath(path_to_goma)
  39       self._abs_path_to_goma_file = self._GetExecutablePath(
  40           self._abs_path_to_goma)
  41
  42   def __enter__(self):
  43     if self._HasGOMAPath():
  44       self._SetupAndStart()
  45     return self
  46
  47   def __exit__(self, *_):
  48     if self._HasGOMAPath():
  49       self._Stop()
  50
  51   def _HasGOMAPath(self):
  52     return bool(self._abs_path_to_goma)
  53
  54   def _GetExecutablePath(self, path_to_goma):
  55     if os.name == 'nt':
  56       return os.path.join(path_to_goma, 'goma_ctl.bat')
  57     else:
  58       return os.path.join(path_to_goma, 'goma_ctl.sh')
  59
  60   def _SetupEnvVars(self):
  61     if os.name == 'nt':
  62       os.environ['CC'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
  63           ' cl.exe')
  64       os.environ['CXX'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
  65           ' cl.exe')
  66     else:
  67       os.environ['PATH'] = os.pathsep.join([self._abs_path_to_goma,
  68           os.environ['PATH']])
  69
  70   def _SetupAndStart(self):
  71     """Sets up GOMA and launches it.
  72
  73     Args:
  74       path_to_goma: Path to goma directory.
  75
  76     Returns:
  77       True if successful."""
  78     self._SetupEnvVars()
  79
  80     # Sometimes goma is lingering around if something went bad on a previous
  81     # run. Stop it before starting a new process. Can ignore the return code
  82     # since it will return an error if it wasn't running.
  83     self._Stop()
  84
  85     if subprocess.call([self._abs_path_to_goma_file, 'start']):
  86       raise RuntimeError('GOMA failed to start.')
  87
  88   def _Stop(self):
  89     subprocess.call([self._abs_path_to_goma_file, 'stop'])
  90
  91
  92
  93 def _LoadConfigFile(path_to_file):
  94   """Attempts to load the specified config file as a module
  95   and grab the global config dict.
  96
  97   Args:
  98     path_to_file: Path to the file.
  99
 100   Returns:
 101     The config dict which should be formatted as follows:
 102     {'command': string, 'good_revision': string, 'bad_revision': string
 103      'metric': string, etc...}.
 104     Returns None on failure.
 105   """
 106   try:
 107     local_vars = {}
 108     execfile(path_to_file, local_vars)
 109
 110     return local_vars['config']
 111   except:
 112     print
 113     traceback.print_exc()
 114     print
 115     return {}
 116
 117
 118 def _ValidateConfigFile(config_contents, valid_parameters):
 119   """Validates the config file contents, checking whether all values are
 120   non-empty.
 121
 122   Args:
 123     config_contents: Contents of the config file passed from _LoadConfigFile.
 124     valid_parameters: A list of parameters to check for.
 125
 126   Returns:
 127     True if valid.
 128   """
 129   try:
 130     [config_contents[current_parameter]
 131         for current_parameter in valid_parameters]
 132     config_has_values = [v and type(v) is str
 133         for v in config_contents.values() if v]
 134     return config_has_values
 135   except KeyError:
 136     return False
 137
 138
 139 def _ValidatePerfConfigFile(config_contents):
 140   """Validates that the perf config file contents. This is used when we're
 141   doing a perf try job, rather than a bisect. The file is called
 142   run-perf-test.cfg by default.
 143
 144   The parameters checked are the required parameters; any additional optional
 145   parameters won't be checked and validation will still pass.
 146
 147   Args:
 148     config_contents: Contents of the config file passed from _LoadConfigFile.
 149
 150   Returns:
 151     True if valid.
 152   """
 153   valid_parameters = ['command', 'metric', 'repeat_count', 'truncate_percent',
 154       'max_time_minutes']
 155   return _ValidateConfigFile(config_contents, valid_parameters)
 156
 157
 158 def _ValidateBisectConfigFile(config_contents):
 159   """Validates that the bisect config file contents. The parameters checked are
 160   the required parameters; any additional optional parameters won't be checked
 161   and validation will still pass.
 162
 163   Args:
 164     config_contents: Contents of the config file passed from _LoadConfigFile.
 165
 166   Returns:
 167     True if valid.
 168   """
 169   valid_params = ['command', 'good_revision', 'bad_revision', 'metric',
 170       'repeat_count', 'truncate_percent', 'max_time_minutes']
 171   return _ValidateConfigFile(config_contents, valid_params)
 172
 173
 174 def _OutputFailedResults(text_to_print):
 175   bisect_utils.OutputAnnotationStepStart('Results - Failed')
 176   print
 177   print text_to_print
 178   print
 179   bisect_utils.OutputAnnotationStepClosed()
 180
 181
 182 def _CreateBisectOptionsFromConfig(config):
 183   opts_dict = {}
 184   opts_dict['command'] = config['command']
 185   opts_dict['metric'] = config['metric']
 186
 187   if config['repeat_count']:
 188     opts_dict['repeat_test_count'] = int(config['repeat_count'])
 189
 190   if config['truncate_percent']:
 191     opts_dict['truncate_percent'] = int(config['truncate_percent'])
 192
 193   if config['max_time_minutes']:
 194     opts_dict['max_time_minutes'] = int(config['max_time_minutes'])
 195
 196   if config.has_key('use_goma'):
 197     opts_dict['use_goma'] = config['use_goma']
 198
 199   opts_dict['build_preference'] = 'ninja'
 200   opts_dict['output_buildbot_annotations'] = True
 201
 202   if '--browser=cros' in config['command']:
 203     opts_dict['target_platform'] = 'cros'
 204
 205     if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
 206       opts_dict['cros_board'] = os.environ[CROS_BOARD_ENV]
 207       opts_dict['cros_remote_ip'] = os.environ[CROS_IP_ENV]
 208     else:
 209       raise RuntimeError('Cros build selected, but BISECT_CROS_IP or'
 210           'BISECT_CROS_BOARD undefined.')
 211   elif 'android' in config['command']:
 212     if 'android-chrome' in config['command']:
 213       opts_dict['target_platform'] = 'android-chrome'
 214     else:
 215       opts_dict['target_platform'] = 'android'
 216
 217   return bisect.BisectOptions.FromDict(opts_dict)
 218
 219
 220 def _RunPerformanceTest(config, path_to_file):
 221   # Bisect script expects to be run from src
 222   os.chdir(os.path.join(path_to_file, '..'))
 223
 224   bisect_utils.OutputAnnotationStepStart('Building With Patch')
 225
 226   opts = _CreateBisectOptionsFromConfig(config)
 227   b = bisect.BisectPerformanceMetrics(None, opts)
 228
 229   if bisect_utils.RunGClient(['runhooks']):
 230     raise RuntimeError('Failed to run gclient runhooks')
 231
 232   if not b.BuildCurrentRevision('chromium'):
 233     raise RuntimeError('Patched version failed to build.')
 234
 235   bisect_utils.OutputAnnotationStepClosed()
 236   bisect_utils.OutputAnnotationStepStart('Running With Patch')
 237
 238   results_with_patch = b.RunPerformanceTestAndParseResults(
 239       opts.command, opts.metric, reset_on_first_run=True, results_label='Patch')
 240
 241   if results_with_patch[1]:
 242     raise RuntimeError('Patched version failed to run performance test.')
 243
 244   bisect_utils.OutputAnnotationStepClosed()
 245
 246   bisect_utils.OutputAnnotationStepStart('Reverting Patch')
 247   if bisect_utils.RunGClient(['revert']):
 248     raise RuntimeError('Failed to run gclient runhooks')
 249   bisect_utils.OutputAnnotationStepClosed()
 250
 251   bisect_utils.OutputAnnotationStepStart('Building Without Patch')
 252
 253   if bisect_utils.RunGClient(['runhooks']):
 254     raise RuntimeError('Failed to run gclient runhooks')
 255
 256   if not b.BuildCurrentRevision('chromium'):
 257     raise RuntimeError('Unpatched version failed to build.')
 258
 259   bisect_utils.OutputAnnotationStepClosed()
 260   bisect_utils.OutputAnnotationStepStart('Running Without Patch')
 261
 262   results_without_patch = b.RunPerformanceTestAndParseResults(
 263       opts.command, opts.metric, upload_on_last_run=True, results_label='ToT')
 264
 265   if results_without_patch[1]:
 266     raise RuntimeError('Unpatched version failed to run performance test.')
 267
 268   # Find the link to the cloud stored results file.
 269   output = results_without_patch[2]
 270   cloud_file_link = [t for t in output.splitlines()
 271       if 'storage.googleapis.com/chromium-telemetry/html-results/' in t]
 272   if cloud_file_link:
 273     # What we're getting here is basically "View online at http://..." so parse
 274     # out just the url portion.
 275     cloud_file_link = cloud_file_link[0]
 276     cloud_file_link = [t for t in cloud_file_link.split(' ')
 277         if 'storage.googleapis.com/chromium-telemetry/html-results/' in t]
 278     assert cloud_file_link, "Couldn't parse url from output."
 279     cloud_file_link = cloud_file_link[0]
 280   else:
 281     cloud_file_link = ''
 282
 283   # Calculate the % difference in the means of the 2 runs.
 284   percent_diff_in_means = (results_with_patch[0]['mean'] /
 285       max(0.0001, results_without_patch[0]['mean'])) * 100.0 - 100.0
 286   std_err = bisect.CalculatePooledStandardError(
 287       [results_with_patch[0]['values'], results_without_patch[0]['values']])
 288
 289   bisect_utils.OutputAnnotationStepClosed()
 290   bisect_utils.OutputAnnotationStepStart('Results - %.02f +- %0.02f delta' %
 291       (percent_diff_in_means, std_err))
 292   print ' %s %s %s' % (''.center(10, ' '), 'Mean'.center(20, ' '),
 293       'Std. Error'.center(20, ' '))
 294   print ' %s %s %s' % ('Patch'.center(10, ' '),
 295       ('%.02f' % results_with_patch[0]['mean']).center(20, ' '),
 296       ('%.02f' % results_with_patch[0]['std_err']).center(20, ' '))
 297   print ' %s %s %s' % ('No Patch'.center(10, ' '),
 298       ('%.02f' % results_without_patch[0]['mean']).center(20, ' '),
 299       ('%.02f' % results_without_patch[0]['std_err']).center(20, ' '))
 300   if cloud_file_link:
 301     bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
 302   bisect_utils.OutputAnnotationStepClosed()
 303
 304
 305 def _SetupAndRunPerformanceTest(config, path_to_file, path_to_goma):
 306   """Attempts to build and run the current revision with and without the
 307   current patch, with the parameters passed in.
 308
 309   Args:
 310     config: The config read from run-perf-test.cfg.
 311     path_to_file: Path to the bisect-perf-regression.py script.
 312     path_to_goma: Path to goma directory.
 313
 314   Returns:
 315     0 on success, otherwise 1.
 316   """
 317   try:
 318     with Goma(path_to_goma) as goma:
 319       config['use_goma'] = bool(path_to_goma)
 320       _RunPerformanceTest(config, path_to_file)
 321     return 0
 322   except RuntimeError, e:
 323     bisect_utils.OutputAnnotationStepClosed()
 324     _OutputFailedResults('Error: %s' % e.message)
 325     return 1
 326
 327
 328 def _RunBisectionScript(config, working_directory, path_to_file, path_to_goma,
 329     path_to_extra_src, dry_run):
 330   """Attempts to execute src/tools/bisect-perf-regression.py with the parameters
 331   passed in.
 332
 333   Args:
 334     config: A dict containing the parameters to pass to the script.
 335     working_directory: A working directory to provide to the
 336       bisect-perf-regression.py script, where it will store it's own copy of
 337       the depot.
 338     path_to_file: Path to the bisect-perf-regression.py script.
 339     path_to_goma: Path to goma directory.
 340     path_to_extra_src: Path to extra source file.
 341     dry_run: Do a dry run, skipping sync, build, and performance testing steps.
 342
 343   Returns:
 344     0 on success, otherwise 1.
 345   """
 346   bisect_utils.OutputAnnotationStepStart('Config')
 347   print
 348   for k, v in config.iteritems():
 349     print '  %s : %s' % (k, v)
 350   print
 351   bisect_utils.OutputAnnotationStepClosed()
 352
 353   cmd = ['python', os.path.join(path_to_file, 'bisect-perf-regression.py'),
 354          '-c', config['command'],
 355          '-g', config['good_revision'],
 356          '-b', config['bad_revision'],
 357          '-m', config['metric'],
 358          '--working_directory', working_directory,
 359          '--output_buildbot_annotations']
 360
 361   if config['repeat_count']:
 362     cmd.extend(['-r', config['repeat_count']])
 363
 364   if config['truncate_percent']:
 365     cmd.extend(['-t', config['truncate_percent']])
 366
 367   if config['max_time_minutes']:
 368     cmd.extend(['--max_time_minutes', config['max_time_minutes']])
 369
 370   cmd.extend(['--build_preference', 'ninja'])
 371
 372   if '--browser=cros' in config['command']:
 373     cmd.extend(['--target_platform', 'cros'])
 374
 375     if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
 376       cmd.extend(['--cros_board', os.environ[CROS_BOARD_ENV]])
 377       cmd.extend(['--cros_remote_ip', os.environ[CROS_IP_ENV]])
 378     else:
 379       print 'Error: Cros build selected, but BISECT_CROS_IP or'\
 380             'BISECT_CROS_BOARD undefined.'
 381       print
 382       return 1
 383
 384   if 'android' in config['command']:
 385     if 'android-chrome' in config['command']:
 386       cmd.extend(['--target_platform', 'android-chrome'])
 387     else:
 388       cmd.extend(['--target_platform', 'android'])
 389
 390   if path_to_goma:
 391     cmd.append('--use_goma')
 392
 393   if path_to_extra_src:
 394     cmd.extend(['--extra_src', path_to_extra_src])
 395
 396   if dry_run:
 397     cmd.extend(['--debug_ignore_build', '--debug_ignore_sync',
 398         '--debug_ignore_perf_test'])
 399   cmd = [str(c) for c in cmd]
 400
 401   with Goma(path_to_goma) as goma:
 402     return_code = subprocess.call(cmd)
 403
 404   if return_code:
 405     print 'Error: bisect-perf-regression.py returned with error %d' %\
 406         return_code
 407     print
 408
 409   return return_code
 410
 411
 412 def main():
 413
 414   usage = ('%prog [options] [-- chromium-options]\n'
 415            'Used by a trybot to run the bisection script using the parameters'
 416            ' provided in the run-bisect-perf-regression.cfg file.')
 417
 418   parser = optparse.OptionParser(usage=usage)
 419   parser.add_option('-w', '--working_directory',
 420                     type='str',
 421                     help='A working directory to supply to the bisection '
 422                     'script, which will use it as the location to checkout '
 423                     'a copy of the chromium depot.')
 424   parser.add_option('-p', '--path_to_goma',
 425                     type='str',
 426                     help='Path to goma directory. If this is supplied, goma '
 427                     'builds will be enabled.')
 428   parser.add_option('--path_to_config',
 429                     type='str',
 430                     help='Path to the config file to use. If this is supplied, '
 431                     'the bisect script will use this to override the default '
 432                     'config file path. The script will attempt to load it '
 433                     'as a bisect config first, then a perf config.')
 434   parser.add_option('--extra_src',
 435                     type='str',
 436                     help='Path to extra source file. If this is supplied, '
 437                     'bisect script will use this to override default behavior.')
 438   parser.add_option('--dry_run',
 439                     action="store_true",
 440                     help='The script will perform the full bisect, but '
 441                     'without syncing, building, or running the performance '
 442                     'tests.')
 443   (opts, args) = parser.parse_args()
 444
 445   path_to_current_directory = os.path.abspath(os.path.dirname(sys.argv[0]))
 446
 447   # If they've specified their own config file, use that instead.
 448   if opts.path_to_config:
 449     path_to_bisect_cfg = opts.path_to_config
 450   else:
 451     path_to_bisect_cfg = os.path.join(path_to_current_directory,
 452         'run-bisect-perf-regression.cfg')
 453
 454   config = _LoadConfigFile(path_to_bisect_cfg)
 455
 456   # Check if the config is valid.
 457   config_is_valid = _ValidateBisectConfigFile(config)
 458
 459   if config and config_is_valid:
 460     if not opts.working_directory:
 461       print 'Error: missing required parameter: --working_directory'
 462       print
 463       parser.print_help()
 464       return 1
 465
 466     return _RunBisectionScript(config, opts.working_directory,
 467         path_to_current_directory, opts.path_to_goma, opts.extra_src,
 468         opts.dry_run)
 469   else:
 470     perf_cfg_files = ['run-perf-test.cfg', os.path.join('..', 'third_party',
 471         'WebKit', 'Tools', 'run-perf-test.cfg')]
 472
 473     for current_perf_cfg_file in perf_cfg_files:
 474       if opts.path_to_config:
 475         path_to_perf_cfg = opts.path_to_config
 476       else:
 477         path_to_perf_cfg = os.path.join(
 478             os.path.abspath(os.path.dirname(sys.argv[0])),
 479             current_perf_cfg_file)
 480
 481       config = _LoadConfigFile(path_to_perf_cfg)
 482       config_is_valid = _ValidatePerfConfigFile(config)
 483
 484       if config and config_is_valid:
 485         return _SetupAndRunPerformanceTest(config, path_to_current_directory,
 486             opts.path_to_goma)
 487
 488     print 'Error: Could not load config file. Double check your changes to '\
 489           'run-bisect-perf-regression.cfg/run-perf-test.cfg for syntax errors.'
 490     print
 491     return 1
 492
 493
 494 if __name__ == '__main__':
 495   sys.exit(main())