tools/run-bisect-perf-regression.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Run Performance Test Bisect Tool
   7
   8 This script is used by a try bot to run the src/tools/bisect-perf-regression.py
   9 script with the parameters specified in run-bisect-perf-regression.cfg. It will
  10 check out a copy of the depot in a subdirectory 'bisect' of the working
  11 directory provided, and run the bisect-perf-regression.py script there.
  12
  13 """
  14
  15 import imp
  16 import optparse
  17 import os
  18 import platform
  19 import subprocess
  20 import sys
  21 import traceback
  22
  23 from auto_bisect import bisect_utils
  24 from auto_bisect import math_utils
  25
  26 bisect = imp.load_source('bisect-perf-regression',
  27     os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])),
  28         'bisect-perf-regression.py'))
  29
  30
  31 CROS_BOARD_ENV = 'BISECT_CROS_BOARD'
  32 CROS_IP_ENV = 'BISECT_CROS_IP'
  33
  34 # Default config file names.
  35 BISECT_REGRESSION_CONFIG = 'run-bisect-perf-regression.cfg'
  36 RUN_TEST_CONFIG = 'run-perf-test.cfg'
  37 WEBKIT_RUN_TEST_CONFIG = os.path.join(
  38     '..', 'third_party', 'WebKit', 'Tools', 'run-perf-test.cfg')
  39
  40 class Goma(object):
  41
  42   def __init__(self, path_to_goma):
  43     self._abs_path_to_goma = None
  44     self._abs_path_to_goma_file = None
  45     if not path_to_goma:
  46       return
  47     self._abs_path_to_goma = os.path.abspath(path_to_goma)
  48     filename = 'goma_ctl.bat' if os.name == 'nt' else 'goma_ctl.sh'
  49     self._abs_path_to_goma_file = os.path.join(self._abs_path_to_goma, filename)
  50
  51   def __enter__(self):
  52     if self._HasGomaPath():
  53       self._SetupAndStart()
  54     return self
  55
  56   def __exit__(self, *_):
  57     if self._HasGomaPath():
  58       self._Stop()
  59
  60   def _HasGomaPath(self):
  61     return bool(self._abs_path_to_goma)
  62
  63   def _SetupEnvVars(self):
  64     if os.name == 'nt':
  65       os.environ['CC'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
  66           ' cl.exe')
  67       os.environ['CXX'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
  68           ' cl.exe')
  69     else:
  70       os.environ['PATH'] = os.pathsep.join([self._abs_path_to_goma,
  71           os.environ['PATH']])
  72
  73   def _SetupAndStart(self):
  74     """Sets up goma and launches it.
  75
  76     Args:
  77       path_to_goma: Path to goma directory.
  78
  79     Returns:
  80       True if successful."""
  81     self._SetupEnvVars()
  82
  83     # Sometimes goma is lingering around if something went bad on a previous
  84     # run. Stop it before starting a new process. Can ignore the return code
  85     # since it will return an error if it wasn't running.
  86     self._Stop()
  87
  88     if subprocess.call([self._abs_path_to_goma_file, 'start']):
  89       raise RuntimeError('Goma failed to start.')
  90
  91   def _Stop(self):
  92     subprocess.call([self._abs_path_to_goma_file, 'stop'])
  93
  94
  95 def _LoadConfigFile(config_file_path):
  96   """Attempts to load the specified config file as a module
  97   and grab the global config dict.
  98
  99   Args:
 100     config_file_path: Path to the config file.
 101
 102   Returns:
 103     If successful, returns the config dict loaded from the file. If no
 104     such dictionary could be loaded, returns the empty dictionary.
 105   """
 106   try:
 107     local_vars = {}
 108     execfile(config_file_path, local_vars)
 109     return local_vars['config']
 110   except Exception:
 111     print
 112     traceback.print_exc()
 113     print
 114     return {}
 115
 116
 117 def _ValidateConfigFile(config_contents, valid_parameters):
 118   """Validates the config file contents, checking whether all values are
 119   non-empty.
 120
 121   Args:
 122     config_contents: A config dictionary.
 123     valid_parameters: A list of parameters to check for.
 124
 125   Returns:
 126     True if valid.
 127   """
 128   for parameter in valid_parameters:
 129     if parameter not in config_contents:
 130       return False
 131     value = config_contents[parameter]
 132     if not value or type(value) is not str:
 133       return False
 134   return True
 135
 136
 137 def _ValidatePerfConfigFile(config_contents):
 138   """Validates the perf config file contents.
 139
 140   This is used when we're doing a perf try job, rather than a bisect.
 141   The config file is called run-perf-test.cfg by default.
 142
 143   The parameters checked are the required parameters; any additional optional
 144   parameters won't be checked and validation will still pass.
 145
 146   Args:
 147     config_contents: A config dictionary.
 148
 149   Returns:
 150     True if valid.
 151   """
 152   valid_parameters = [
 153       'command',
 154       'repeat_count',
 155       'truncate_percent',
 156       'max_time_minutes',
 157   ]
 158   return _ValidateConfigFile(config_contents, valid_parameters)
 159
 160
 161 def _ValidateBisectConfigFile(config_contents):
 162   """Validates the bisect config file contents.
 163
 164   The parameters checked are the required parameters; any additional optional
 165   parameters won't be checked and validation will still pass.
 166
 167   Args:
 168     config_contents: A config dictionary.
 169
 170   Returns:
 171     True if valid.
 172   """
 173   valid_params = [
 174       'command',
 175       'good_revision',
 176       'bad_revision',
 177       'metric',
 178       'repeat_count',
 179       'truncate_percent',
 180       'max_time_minutes',
 181   ]
 182   return _ValidateConfigFile(config_contents, valid_params)
 183
 184
 185 def _OutputFailedResults(text_to_print):
 186   bisect_utils.OutputAnnotationStepStart('Results - Failed')
 187   print
 188   print text_to_print
 189   print
 190   bisect_utils.OutputAnnotationStepClosed()
 191
 192
 193 def _CreateBisectOptionsFromConfig(config):
 194   print config['command']
 195   opts_dict = {}
 196   opts_dict['command'] = config['command']
 197   opts_dict['metric'] = config.get('metric')
 198
 199   if config['repeat_count']:
 200     opts_dict['repeat_test_count'] = int(config['repeat_count'])
 201
 202   if config['truncate_percent']:
 203     opts_dict['truncate_percent'] = int(config['truncate_percent'])
 204
 205   if config['max_time_minutes']:
 206     opts_dict['max_time_minutes'] = int(config['max_time_minutes'])
 207
 208   if config.has_key('use_goma'):
 209     opts_dict['use_goma'] = config['use_goma']
 210   if config.has_key('goma_dir'):
 211     opts_dict['goma_dir'] = config['goma_dir']
 212
 213   opts_dict['build_preference'] = 'ninja'
 214   opts_dict['output_buildbot_annotations'] = True
 215
 216   if '--browser=cros' in config['command']:
 217     opts_dict['target_platform'] = 'cros'
 218
 219     if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
 220       opts_dict['cros_board'] = os.environ[CROS_BOARD_ENV]
 221       opts_dict['cros_remote_ip'] = os.environ[CROS_IP_ENV]
 222     else:
 223       raise RuntimeError('CrOS build selected, but BISECT_CROS_IP or'
 224           'BISECT_CROS_BOARD undefined.')
 225   elif 'android' in config['command']:
 226     if 'android-chrome-shell' in config['command']:
 227       opts_dict['target_platform'] = 'android'
 228     elif 'android-chrome' in config['command']:
 229       opts_dict['target_platform'] = 'android-chrome'
 230     else:
 231       opts_dict['target_platform'] = 'android'
 232
 233   return bisect.BisectOptions.FromDict(opts_dict)
 234
 235
 236 def _RunPerformanceTest(config, path_to_file):
 237   """Runs a performance test with and without the current patch.
 238
 239   Args:
 240     config: Contents of the config file, a dictionary.
 241     path_to_file: Path to the bisect-perf-regression.py script.
 242
 243   Attempts to build and run the current revision with and without the
 244   current patch, with the parameters passed in.
 245   """
 246   # Bisect script expects to be run from the src directory
 247   os.chdir(os.path.join(path_to_file, '..'))
 248
 249   bisect_utils.OutputAnnotationStepStart('Building With Patch')
 250
 251   opts = _CreateBisectOptionsFromConfig(config)
 252   b = bisect.BisectPerformanceMetrics(None, opts)
 253
 254   if bisect_utils.RunGClient(['runhooks']):
 255     raise RuntimeError('Failed to run gclient runhooks')
 256
 257   if not b.BuildCurrentRevision('chromium'):
 258     raise RuntimeError('Patched version failed to build.')
 259
 260   bisect_utils.OutputAnnotationStepClosed()
 261   bisect_utils.OutputAnnotationStepStart('Running With Patch')
 262
 263   results_with_patch = b.RunPerformanceTestAndParseResults(
 264       opts.command, opts.metric, reset_on_first_run=True, results_label='Patch')
 265
 266   if results_with_patch[1]:
 267     raise RuntimeError('Patched version failed to run performance test.')
 268
 269   bisect_utils.OutputAnnotationStepClosed()
 270
 271   bisect_utils.OutputAnnotationStepStart('Reverting Patch')
 272   # TODO: When this is re-written to recipes, this should use bot_update's
 273   # revert mechanism to fully revert the client. But for now, since we know that
 274   # the perf try bot currently only supports src/ and src/third_party/WebKit, we
 275   # simply reset those two directories.
 276   bisect_utils.CheckRunGit(['reset', '--hard'])
 277   bisect_utils.CheckRunGit(['reset', '--hard'],
 278                            os.path.join('third_party', 'WebKit'))
 279   bisect_utils.OutputAnnotationStepClosed()
 280
 281   bisect_utils.OutputAnnotationStepStart('Building Without Patch')
 282
 283   if bisect_utils.RunGClient(['runhooks']):
 284     raise RuntimeError('Failed to run gclient runhooks')
 285
 286   if not b.BuildCurrentRevision('chromium'):
 287     raise RuntimeError('Unpatched version failed to build.')
 288
 289   bisect_utils.OutputAnnotationStepClosed()
 290   bisect_utils.OutputAnnotationStepStart('Running Without Patch')
 291
 292   results_without_patch = b.RunPerformanceTestAndParseResults(
 293       opts.command, opts.metric, upload_on_last_run=True, results_label='ToT')
 294
 295   if results_without_patch[1]:
 296     raise RuntimeError('Unpatched version failed to run performance test.')
 297
 298   # Find the link to the cloud stored results file.
 299   output = results_without_patch[2]
 300   cloud_file_link = [t for t in output.splitlines()
 301       if 'storage.googleapis.com/chromium-telemetry/html-results/' in t]
 302   if cloud_file_link:
 303     # What we're getting here is basically "View online at http://..." so parse
 304     # out just the URL portion.
 305     cloud_file_link = cloud_file_link[0]
 306     cloud_file_link = [t for t in cloud_file_link.split(' ')
 307         if 'storage.googleapis.com/chromium-telemetry/html-results/' in t]
 308     assert cloud_file_link, 'Couldn\'t parse URL from output.'
 309     cloud_file_link = cloud_file_link[0]
 310   else:
 311     cloud_file_link = ''
 312
 313   # Calculate the % difference in the means of the 2 runs.
 314   percent_diff_in_means = None
 315   std_err = None
 316   if (results_with_patch[0].has_key('mean') and
 317       results_with_patch[0].has_key('values')):
 318     percent_diff_in_means = (results_with_patch[0]['mean'] /
 319         max(0.0001, results_without_patch[0]['mean'])) * 100.0 - 100.0
 320     std_err = math_utils.PooledStandardError(
 321         [results_with_patch[0]['values'], results_without_patch[0]['values']])
 322
 323   bisect_utils.OutputAnnotationStepClosed()
 324   if percent_diff_in_means is not None and std_err is not None:
 325     bisect_utils.OutputAnnotationStepStart('Results - %.02f +- %0.02f delta' %
 326         (percent_diff_in_means, std_err))
 327     print ' %s %s %s' % (''.center(10, ' '), 'Mean'.center(20, ' '),
 328         'Std. Error'.center(20, ' '))
 329     print ' %s %s %s' % ('Patch'.center(10, ' '),
 330         ('%.02f' % results_with_patch[0]['mean']).center(20, ' '),
 331         ('%.02f' % results_with_patch[0]['std_err']).center(20, ' '))
 332     print ' %s %s %s' % ('No Patch'.center(10, ' '),
 333         ('%.02f' % results_without_patch[0]['mean']).center(20, ' '),
 334         ('%.02f' % results_without_patch[0]['std_err']).center(20, ' '))
 335     if cloud_file_link:
 336       bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
 337     bisect_utils.OutputAnnotationStepClosed()
 338   elif cloud_file_link:
 339     bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
 340
 341
 342 def _SetupAndRunPerformanceTest(config, path_to_file, path_to_goma):
 343   """Attempts to build and run the current revision with and without the
 344   current patch, with the parameters passed in.
 345
 346   Args:
 347     config: The config read from run-perf-test.cfg.
 348     path_to_file: Path to the bisect-perf-regression.py script.
 349     path_to_goma: Path to goma directory.
 350
 351   Returns:
 352     The exit code of bisect-perf-regression.py: 0 on success, otherwise 1.
 353   """
 354   try:
 355     with Goma(path_to_goma) as _:
 356       config['use_goma'] = bool(path_to_goma)
 357       if config['use_goma']:
 358         config['goma_dir'] = os.path.abspath(path_to_goma)
 359       _RunPerformanceTest(config, path_to_file)
 360     return 0
 361   except RuntimeError, e:
 362     bisect_utils.OutputAnnotationStepClosed()
 363     _OutputFailedResults('Error: %s' % e.message)
 364     return 1
 365
 366
 367 def _RunBisectionScript(
 368     config, working_directory, path_to_file, path_to_goma, path_to_extra_src,
 369     dry_run):
 370   """Attempts to execute bisect-perf-regression.py with the given parameters.
 371
 372   Args:
 373     config: A dict containing the parameters to pass to the script.
 374     working_directory: A working directory to provide to the
 375       bisect-perf-regression.py script, where it will store it's own copy of
 376       the depot.
 377     path_to_file: Path to the bisect-perf-regression.py script.
 378     path_to_goma: Path to goma directory.
 379     path_to_extra_src: Path to extra source file.
 380     dry_run: Do a dry run, skipping sync, build, and performance testing steps.
 381
 382   Returns:
 383     An exit status code: 0 on success, otherwise 1.
 384   """
 385   _PrintConfigStep(config)
 386
 387   cmd = ['python', os.path.join(path_to_file, 'bisect-perf-regression.py'),
 388          '-c', config['command'],
 389          '-g', config['good_revision'],
 390          '-b', config['bad_revision'],
 391          '-m', config['metric'],
 392          '--working_directory', working_directory,
 393          '--output_buildbot_annotations']
 394
 395   if config.get('metric'):
 396     cmd.extend(['-m', config['metric']])
 397
 398   if config['repeat_count']:
 399     cmd.extend(['-r', config['repeat_count']])
 400
 401   if config['truncate_percent']:
 402     cmd.extend(['-t', config['truncate_percent']])
 403
 404   if config['max_time_minutes']:
 405     cmd.extend(['--max_time_minutes', config['max_time_minutes']])
 406
 407   if config.has_key('bisect_mode'):
 408     cmd.extend(['--bisect_mode', config['bisect_mode']])
 409
 410   cmd.extend(['--build_preference', 'ninja'])
 411
 412   if '--browser=cros' in config['command']:
 413     cmd.extend(['--target_platform', 'cros'])
 414
 415     if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
 416       cmd.extend(['--cros_board', os.environ[CROS_BOARD_ENV]])
 417       cmd.extend(['--cros_remote_ip', os.environ[CROS_IP_ENV]])
 418     else:
 419       print ('Error: Cros build selected, but BISECT_CROS_IP or'
 420              'BISECT_CROS_BOARD undefined.\n')
 421       return 1
 422
 423   if 'android' in config['command']:
 424     if 'android-chrome-shell' in config['command']:
 425       cmd.extend(['--target_platform', 'android'])
 426     elif 'android-chrome' in config['command']:
 427       cmd.extend(['--target_platform', 'android-chrome'])
 428     else:
 429       cmd.extend(['--target_platform', 'android'])
 430
 431   if path_to_goma:
 432     # For Windows XP platforms, goma service is not supported.
 433     # Moreover we don't compile chrome when gs_bucket flag is set instead
 434     # use builds archives, therefore ignore goma service for Windows XP.
 435     # See http://crbug.com/330900.
 436     if config.get('gs_bucket') and platform.release() == 'XP':
 437       print ('Goma doesn\'t have a win32 binary, therefore it is not supported '
 438              'on Windows XP platform. Please refer to crbug.com/330900.')
 439       path_to_goma = None
 440     cmd.append('--use_goma')
 441
 442   if path_to_extra_src:
 443     cmd.extend(['--extra_src', path_to_extra_src])
 444
 445   # These flags are used to download build archives from cloud storage if
 446   # available, otherwise will post a try_job_http request to build it on the
 447   # try server.
 448   if config.get('gs_bucket'):
 449     if config.get('builder_host') and config.get('builder_port'):
 450       cmd.extend(['--gs_bucket', config['gs_bucket'],
 451                   '--builder_host', config['builder_host'],
 452                   '--builder_port', config['builder_port']
 453                  ])
 454     else:
 455       print ('Error: Specified gs_bucket, but missing builder_host or '
 456              'builder_port information in config.')
 457       return 1
 458
 459   if dry_run:
 460     cmd.extend(['--debug_ignore_build', '--debug_ignore_sync',
 461         '--debug_ignore_perf_test'])
 462   cmd = [str(c) for c in cmd]
 463
 464   with Goma(path_to_goma) as _:
 465     return_code = subprocess.call(cmd)
 466
 467   if return_code:
 468     print ('Error: bisect-perf-regression.py returned with error %d\n'
 469            % return_code)
 470
 471   return return_code
 472
 473
 474 def _PrintConfigStep(config):
 475   """Prints out the given config, along with Buildbot annotations."""
 476   bisect_utils.OutputAnnotationStepStart('Config')
 477   print
 478   for k, v in config.iteritems():
 479     print '  %s : %s' % (k, v)
 480   print
 481   bisect_utils.OutputAnnotationStepClosed()
 482
 483
 484 def _OptionParser():
 485   """Returns the options parser for run-bisect-perf-regression.py."""
 486   usage = ('%prog [options] [-- chromium-options]\n'
 487            'Used by a try bot to run the bisection script using the parameters'
 488            ' provided in the run-bisect-perf-regression.cfg file.')
 489   parser = optparse.OptionParser(usage=usage)
 490   parser.add_option('-w', '--working_directory',
 491                     type='str',
 492                     help='A working directory to supply to the bisection '
 493                     'script, which will use it as the location to checkout '
 494                     'a copy of the chromium depot.')
 495   parser.add_option('-p', '--path_to_goma',
 496                     type='str',
 497                     help='Path to goma directory. If this is supplied, goma '
 498                     'builds will be enabled.')
 499   parser.add_option('--path_to_config',
 500                     type='str',
 501                     help='Path to the config file to use. If this is supplied, '
 502                     'the bisect script will use this to override the default '
 503                     'config file path. The script will attempt to load it '
 504                     'as a bisect config first, then a perf config.')
 505   parser.add_option('--extra_src',
 506                     type='str',
 507                     help='Path to extra source file. If this is supplied, '
 508                     'bisect script will use this to override default behavior.')
 509   parser.add_option('--dry_run',
 510                     action="store_true",
 511                     help='The script will perform the full bisect, but '
 512                     'without syncing, building, or running the performance '
 513                     'tests.')
 514   return parser
 515
 516
 517 def main():
 518   """Entry point for run-bisect-perf-regression.py.
 519
 520   Reads the config file, and then tries to either bisect a regression or
 521   just run a performance test, depending on the particular config parameters
 522   specified in the config file.
 523   """
 524
 525   parser = _OptionParser()
 526   opts, _ = parser.parse_args()
 527
 528   current_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
 529
 530   # Use the default config file path unless one was specified.
 531   config_path = os.path.join(current_dir, BISECT_REGRESSION_CONFIG)
 532   if opts.path_to_config:
 533     config_path = opts.path_to_config
 534   config = _LoadConfigFile(config_path)
 535
 536   # Check if the config is valid for running bisect job.
 537   config_is_valid = _ValidateBisectConfigFile(config)
 538
 539   if config and config_is_valid:
 540     if not opts.working_directory:
 541       print 'Error: missing required parameter: --working_directory\n'
 542       parser.print_help()
 543       return 1
 544
 545     return _RunBisectionScript(
 546         config, opts.working_directory, current_dir,
 547         opts.path_to_goma, opts.extra_src, opts.dry_run)
 548
 549   # If it wasn't valid for running a bisect, then maybe the user wanted
 550   # to run a perf test instead of a bisect job. Try reading any possible
 551   # perf test config files.
 552   perf_cfg_files = [RUN_TEST_CONFIG, WEBKIT_RUN_TEST_CONFIG]
 553   for current_perf_cfg_file in perf_cfg_files:
 554     if opts.path_to_config:
 555       path_to_perf_cfg = opts.path_to_config
 556     else:
 557       path_to_perf_cfg = os.path.join(
 558           os.path.abspath(os.path.dirname(sys.argv[0])),
 559           current_perf_cfg_file)
 560
 561     config = _LoadConfigFile(path_to_perf_cfg)
 562     config_is_valid = _ValidatePerfConfigFile(config)
 563
 564     if config and config_is_valid:
 565       return _SetupAndRunPerformanceTest(
 566           config, current_dir, opts.path_to_goma)
 567
 568   print ('Error: Could not load config file. Double check your changes to '
 569          'run-bisect-perf-regression.cfg or run-perf-test.cfg for syntax '
 570          'errors.\n')
 571   return 1
 572
 573
 574 if __name__ == '__main__':
 575   sys.exit(main())