tools/run-bisect-perf-regression.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Run Performance Test Bisect Tool
   7
   8 This script is used by a try bot to run the src/tools/bisect-perf-regression.py
   9 script with the parameters specified in src/tools/auto_bisect/bisect.cfg.
  10 It will check out a copy of the depot in a subdirectory 'bisect' of the working
  11 directory provided, and run the bisect-perf-regression.py script there.
  12 """
  13
  14 import imp
  15 import optparse
  16 import os
  17 import platform
  18 import subprocess
  19 import sys
  20 import traceback
  21
  22 from auto_bisect import bisect_utils
  23 from auto_bisect import math_utils
  24
  25 bisect = imp.load_source('bisect-perf-regression',
  26     os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])),
  27         'bisect-perf-regression.py'))
  28
  29
  30 CROS_BOARD_ENV = 'BISECT_CROS_BOARD'
  31 CROS_IP_ENV = 'BISECT_CROS_IP'
  32
  33 # Default config file paths, relative to this script.
  34 BISECT_REGRESSION_CONFIG = os.path.join('auto_bisect', 'bisect.cfg')
  35 RUN_TEST_CONFIG = 'run-perf-test.cfg'
  36 WEBKIT_RUN_TEST_CONFIG = os.path.join(
  37     '..', 'third_party', 'WebKit', 'Tools', 'run-perf-test.cfg')
  38
  39
  40 class Goma(object):
  41
  42   def __init__(self, path_to_goma):
  43     self._abs_path_to_goma = None
  44     self._abs_path_to_goma_file = None
  45     if not path_to_goma:
  46       return
  47     self._abs_path_to_goma = os.path.abspath(path_to_goma)
  48     filename = 'goma_ctl.bat' if os.name == 'nt' else 'goma_ctl.sh'
  49     self._abs_path_to_goma_file = os.path.join(self._abs_path_to_goma, filename)
  50
  51   def __enter__(self):
  52     if self._HasGomaPath():
  53       self._SetupAndStart()
  54     return self
  55
  56   def __exit__(self, *_):
  57     if self._HasGomaPath():
  58       self._Stop()
  59
  60   def _HasGomaPath(self):
  61     return bool(self._abs_path_to_goma)
  62
  63   def _SetupEnvVars(self):
  64     if os.name == 'nt':
  65       os.environ['CC'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
  66           ' cl.exe')
  67       os.environ['CXX'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
  68           ' cl.exe')
  69     else:
  70       os.environ['PATH'] = os.pathsep.join([self._abs_path_to_goma,
  71           os.environ['PATH']])
  72
  73   def _SetupAndStart(self):
  74     """Sets up goma and launches it.
  75
  76     Args:
  77       path_to_goma: Path to goma directory.
  78
  79     Returns:
  80       True if successful."""
  81     self._SetupEnvVars()
  82
  83     # Sometimes goma is lingering around if something went bad on a previous
  84     # run. Stop it before starting a new process. Can ignore the return code
  85     # since it will return an error if it wasn't running.
  86     self._Stop()
  87
  88     if subprocess.call([self._abs_path_to_goma_file, 'start']):
  89       raise RuntimeError('Goma failed to start.')
  90
  91   def _Stop(self):
  92     subprocess.call([self._abs_path_to_goma_file, 'stop'])
  93
  94
  95 def _LoadConfigFile(config_file_path):
  96   """Attempts to load the specified config file as a module
  97   and grab the global config dict.
  98
  99   Args:
 100     config_file_path: Path to the config file.
 101
 102   Returns:
 103     If successful, returns the config dict loaded from the file. If no
 104     such dictionary could be loaded, returns the empty dictionary.
 105   """
 106   try:
 107     local_vars = {}
 108     execfile(config_file_path, local_vars)
 109     return local_vars['config']
 110   except Exception:
 111     print
 112     traceback.print_exc()
 113     print
 114     return {}
 115
 116
 117 def _ValidateConfigFile(config_contents, valid_parameters):
 118   """Validates the config file contents, checking whether all values are
 119   non-empty.
 120
 121   Args:
 122     config_contents: A config dictionary.
 123     valid_parameters: A list of parameters to check for.
 124
 125   Returns:
 126     True if valid.
 127   """
 128   for parameter in valid_parameters:
 129     if parameter not in config_contents:
 130       return False
 131     value = config_contents[parameter]
 132     if not value or type(value) is not str:
 133       return False
 134   return True
 135
 136
 137 def _ValidatePerfConfigFile(config_contents):
 138   """Validates the perf config file contents.
 139
 140   This is used when we're doing a perf try job, rather than a bisect.
 141   The config file is called run-perf-test.cfg by default.
 142
 143   The parameters checked are the required parameters; any additional optional
 144   parameters won't be checked and validation will still pass.
 145
 146   Args:
 147     config_contents: A config dictionary.
 148
 149   Returns:
 150     True if valid.
 151   """
 152   valid_parameters = [
 153       'command',
 154       'repeat_count',
 155       'truncate_percent',
 156       'max_time_minutes',
 157   ]
 158   return _ValidateConfigFile(config_contents, valid_parameters)
 159
 160
 161 def _ValidateBisectConfigFile(config_contents):
 162   """Validates the bisect config file contents.
 163
 164   The parameters checked are the required parameters; any additional optional
 165   parameters won't be checked and validation will still pass.
 166
 167   Args:
 168     config_contents: A config dictionary.
 169
 170   Returns:
 171     True if valid.
 172   """
 173   valid_params = [
 174       'command',
 175       'good_revision',
 176       'bad_revision',
 177       'metric',
 178       'repeat_count',
 179       'truncate_percent',
 180       'max_time_minutes',
 181   ]
 182   return _ValidateConfigFile(config_contents, valid_params)
 183
 184
 185 def _OutputFailedResults(text_to_print):
 186   bisect_utils.OutputAnnotationStepStart('Results - Failed')
 187   print
 188   print text_to_print
 189   print
 190   bisect_utils.OutputAnnotationStepClosed()
 191
 192
 193 def _CreateBisectOptionsFromConfig(config):
 194   print config['command']
 195   opts_dict = {}
 196   opts_dict['command'] = config['command']
 197   opts_dict['metric'] = config.get('metric')
 198
 199   if config['repeat_count']:
 200     opts_dict['repeat_test_count'] = int(config['repeat_count'])
 201
 202   if config['truncate_percent']:
 203     opts_dict['truncate_percent'] = int(config['truncate_percent'])
 204
 205   if config['max_time_minutes']:
 206     opts_dict['max_time_minutes'] = int(config['max_time_minutes'])
 207
 208   if config.has_key('use_goma'):
 209     opts_dict['use_goma'] = config['use_goma']
 210   if config.has_key('goma_dir'):
 211     opts_dict['goma_dir'] = config['goma_dir']
 212
 213   opts_dict['build_preference'] = 'ninja'
 214   opts_dict['output_buildbot_annotations'] = True
 215
 216   if '--browser=cros' in config['command']:
 217     opts_dict['target_platform'] = 'cros'
 218
 219     if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
 220       opts_dict['cros_board'] = os.environ[CROS_BOARD_ENV]
 221       opts_dict['cros_remote_ip'] = os.environ[CROS_IP_ENV]
 222     else:
 223       raise RuntimeError('CrOS build selected, but BISECT_CROS_IP or'
 224           'BISECT_CROS_BOARD undefined.')
 225   elif 'android' in config['command']:
 226     if 'android-chrome-shell' in config['command']:
 227       opts_dict['target_platform'] = 'android'
 228     elif 'android-chrome' in config['command']:
 229       opts_dict['target_platform'] = 'android-chrome'
 230     else:
 231       opts_dict['target_platform'] = 'android'
 232
 233   return bisect.BisectOptions.FromDict(opts_dict)
 234
 235
 236 def _RunPerformanceTest(config, path_to_file):
 237   """Runs a performance test with and without the current patch.
 238
 239   Args:
 240     config: Contents of the config file, a dictionary.
 241     path_to_file: Path to the bisect-perf-regression.py script.
 242
 243   Attempts to build and run the current revision with and without the
 244   current patch, with the parameters passed in.
 245   """
 246   # Bisect script expects to be run from the src directory
 247   os.chdir(os.path.join(path_to_file, '..'))
 248
 249   bisect_utils.OutputAnnotationStepStart('Building With Patch')
 250
 251   opts = _CreateBisectOptionsFromConfig(config)
 252   b = bisect.BisectPerformanceMetrics(None, opts)
 253
 254   if bisect_utils.RunGClient(['runhooks']):
 255     raise RuntimeError('Failed to run gclient runhooks')
 256
 257   if not b.BuildCurrentRevision('chromium'):
 258     raise RuntimeError('Patched version failed to build.')
 259
 260   bisect_utils.OutputAnnotationStepClosed()
 261   bisect_utils.OutputAnnotationStepStart('Running With Patch')
 262
 263   results_with_patch = b.RunPerformanceTestAndParseResults(
 264       opts.command, opts.metric, reset_on_first_run=True, results_label='Patch')
 265
 266   if results_with_patch[1]:
 267     raise RuntimeError('Patched version failed to run performance test.')
 268
 269   bisect_utils.OutputAnnotationStepClosed()
 270
 271   bisect_utils.OutputAnnotationStepStart('Reverting Patch')
 272   # TODO: When this is re-written to recipes, this should use bot_update's
 273   # revert mechanism to fully revert the client. But for now, since we know that
 274   # the perf try bot currently only supports src/ and src/third_party/WebKit, we
 275   # simply reset those two directories.
 276   bisect_utils.CheckRunGit(['reset', '--hard'])
 277   bisect_utils.CheckRunGit(['reset', '--hard'],
 278                            os.path.join('third_party', 'WebKit'))
 279   bisect_utils.OutputAnnotationStepClosed()
 280
 281   bisect_utils.OutputAnnotationStepStart('Building Without Patch')
 282
 283   if bisect_utils.RunGClient(['runhooks']):
 284     raise RuntimeError('Failed to run gclient runhooks')
 285
 286   if not b.BuildCurrentRevision('chromium'):
 287     raise RuntimeError('Unpatched version failed to build.')
 288
 289   bisect_utils.OutputAnnotationStepClosed()
 290   bisect_utils.OutputAnnotationStepStart('Running Without Patch')
 291
 292   results_without_patch = b.RunPerformanceTestAndParseResults(
 293       opts.command, opts.metric, upload_on_last_run=True, results_label='ToT')
 294
 295   if results_without_patch[1]:
 296     raise RuntimeError('Unpatched version failed to run performance test.')
 297
 298   # Find the link to the cloud stored results file.
 299   output = results_without_patch[2]
 300   cloud_file_link = [t for t in output.splitlines()
 301       if 'storage.googleapis.com/chromium-telemetry/html-results/' in t]
 302   if cloud_file_link:
 303     # What we're getting here is basically "View online at http://..." so parse
 304     # out just the URL portion.
 305     cloud_file_link = cloud_file_link[0]
 306     cloud_file_link = [t for t in cloud_file_link.split(' ')
 307         if 'storage.googleapis.com/chromium-telemetry/html-results/' in t]
 308     assert cloud_file_link, 'Couldn\'t parse URL from output.'
 309     cloud_file_link = cloud_file_link[0]
 310   else:
 311     cloud_file_link = ''
 312
 313   # Calculate the % difference in the means of the 2 runs.
 314   percent_diff_in_means = None
 315   std_err = None
 316   if (results_with_patch[0].has_key('mean') and
 317       results_with_patch[0].has_key('values')):
 318     percent_diff_in_means = (results_with_patch[0]['mean'] /
 319         max(0.0001, results_without_patch[0]['mean'])) * 100.0 - 100.0
 320     std_err = math_utils.PooledStandardError(
 321         [results_with_patch[0]['values'], results_without_patch[0]['values']])
 322
 323   bisect_utils.OutputAnnotationStepClosed()
 324   if percent_diff_in_means is not None and std_err is not None:
 325     bisect_utils.OutputAnnotationStepStart('Results - %.02f +- %0.02f delta' %
 326         (percent_diff_in_means, std_err))
 327     print ' %s %s %s' % (''.center(10, ' '), 'Mean'.center(20, ' '),
 328         'Std. Error'.center(20, ' '))
 329     print ' %s %s %s' % ('Patch'.center(10, ' '),
 330         ('%.02f' % results_with_patch[0]['mean']).center(20, ' '),
 331         ('%.02f' % results_with_patch[0]['std_err']).center(20, ' '))
 332     print ' %s %s %s' % ('No Patch'.center(10, ' '),
 333         ('%.02f' % results_without_patch[0]['mean']).center(20, ' '),
 334         ('%.02f' % results_without_patch[0]['std_err']).center(20, ' '))
 335     if cloud_file_link:
 336       bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
 337     bisect_utils.OutputAnnotationStepClosed()
 338   elif cloud_file_link:
 339     bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
 340
 341
 342 def _SetupAndRunPerformanceTest(config, path_to_file, path_to_goma):
 343   """Attempts to build and run the current revision with and without the
 344   current patch, with the parameters passed in.
 345
 346   Args:
 347     config: The config read from run-perf-test.cfg.
 348     path_to_file: Path to the bisect-perf-regression.py script.
 349     path_to_goma: Path to goma directory.
 350
 351   Returns:
 352     The exit code of bisect-perf-regression.py: 0 on success, otherwise 1.
 353   """
 354   try:
 355     with Goma(path_to_goma) as _:
 356       config['use_goma'] = bool(path_to_goma)
 357       if config['use_goma']:
 358         config['goma_dir'] = os.path.abspath(path_to_goma)
 359       _RunPerformanceTest(config, path_to_file)
 360     return 0
 361   except RuntimeError, e:
 362     bisect_utils.OutputAnnotationStepClosed()
 363     _OutputFailedResults('Error: %s' % e.message)
 364     return 1
 365
 366
 367 def _RunBisectionScript(
 368     config, working_directory, path_to_file, path_to_goma, path_to_extra_src,
 369     dry_run):
 370   """Attempts to execute bisect-perf-regression.py with the given parameters.
 371
 372   Args:
 373     config: A dict containing the parameters to pass to the script.
 374     working_directory: A working directory to provide to the
 375       bisect-perf-regression.py script, where it will store it's own copy of
 376       the depot.
 377     path_to_file: Path to the bisect-perf-regression.py script.
 378     path_to_goma: Path to goma directory.
 379     path_to_extra_src: Path to extra source file.
 380     dry_run: Do a dry run, skipping sync, build, and performance testing steps.
 381
 382   Returns:
 383     An exit status code: 0 on success, otherwise 1.
 384   """
 385   _PrintConfigStep(config)
 386
 387   cmd = ['python', os.path.join(path_to_file, 'bisect-perf-regression.py'),
 388          '-c', config['command'],
 389          '-g', config['good_revision'],
 390          '-b', config['bad_revision'],
 391          '-m', config['metric'],
 392          '--working_directory', working_directory,
 393          '--output_buildbot_annotations']
 394
 395   if config.get('metric'):
 396     cmd.extend(['-m', config['metric']])
 397
 398   if config['repeat_count']:
 399     cmd.extend(['-r', config['repeat_count']])
 400
 401   if config['truncate_percent']:
 402     cmd.extend(['-t', config['truncate_percent']])
 403
 404   if config['max_time_minutes']:
 405     cmd.extend(['--max_time_minutes', config['max_time_minutes']])
 406
 407   if config.has_key('bisect_mode'):
 408     cmd.extend(['--bisect_mode', config['bisect_mode']])
 409
 410   cmd.extend(['--build_preference', 'ninja'])
 411
 412   if '--browser=cros' in config['command']:
 413     cmd.extend(['--target_platform', 'cros'])
 414
 415     if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
 416       cmd.extend(['--cros_board', os.environ[CROS_BOARD_ENV]])
 417       cmd.extend(['--cros_remote_ip', os.environ[CROS_IP_ENV]])
 418     else:
 419       print ('Error: Cros build selected, but BISECT_CROS_IP or'
 420              'BISECT_CROS_BOARD undefined.\n')
 421       return 1
 422
 423   if 'android' in config['command']:
 424     if 'android-chrome-shell' in config['command']:
 425       cmd.extend(['--target_platform', 'android'])
 426     elif 'android-chrome' in config['command']:
 427       cmd.extend(['--target_platform', 'android-chrome'])
 428     else:
 429       cmd.extend(['--target_platform', 'android'])
 430
 431   if path_to_goma:
 432     # For Windows XP platforms, goma service is not supported.
 433     # Moreover we don't compile chrome when gs_bucket flag is set instead
 434     # use builds archives, therefore ignore goma service for Windows XP.
 435     # See http://crbug.com/330900.
 436     if config.get('gs_bucket') and platform.release() == 'XP':
 437       print ('Goma doesn\'t have a win32 binary, therefore it is not supported '
 438              'on Windows XP platform. Please refer to crbug.com/330900.')
 439       path_to_goma = None
 440     cmd.append('--use_goma')
 441
 442   if path_to_extra_src:
 443     cmd.extend(['--extra_src', path_to_extra_src])
 444
 445   # These flags are used to download build archives from cloud storage if
 446   # available, otherwise will post a try_job_http request to build it on the
 447   # try server.
 448   if config.get('gs_bucket'):
 449     if config.get('builder_host') and config.get('builder_port'):
 450       cmd.extend(['--gs_bucket', config['gs_bucket'],
 451                   '--builder_host', config['builder_host'],
 452                   '--builder_port', config['builder_port']
 453                  ])
 454     else:
 455       print ('Error: Specified gs_bucket, but missing builder_host or '
 456              'builder_port information in config.')
 457       return 1
 458
 459   if dry_run:
 460     cmd.extend(['--debug_ignore_build', '--debug_ignore_sync',
 461         '--debug_ignore_perf_test'])
 462   cmd = [str(c) for c in cmd]
 463
 464   with Goma(path_to_goma) as _:
 465     return_code = subprocess.call(cmd)
 466
 467   if return_code:
 468     print ('Error: bisect-perf-regression.py returned with error %d\n'
 469            % return_code)
 470
 471   return return_code
 472
 473
 474 def _PrintConfigStep(config):
 475   """Prints out the given config, along with Buildbot annotations."""
 476   bisect_utils.OutputAnnotationStepStart('Config')
 477   print
 478   for k, v in config.iteritems():
 479     print '  %s : %s' % (k, v)
 480   print
 481   bisect_utils.OutputAnnotationStepClosed()
 482
 483
 484 def _OptionParser():
 485   """Returns the options parser for run-bisect-perf-regression.py."""
 486   usage = ('%prog [options] [-- chromium-options]\n'
 487            'Used by a try bot to run the bisection script using the parameters'
 488            ' provided in the auto_bisect/bisect.cfg file.')
 489   parser = optparse.OptionParser(usage=usage)
 490   parser.add_option('-w', '--working_directory',
 491                     type='str',
 492                     help='A working directory to supply to the bisection '
 493                     'script, which will use it as the location to checkout '
 494                     'a copy of the chromium depot.')
 495   parser.add_option('-p', '--path_to_goma',
 496                     type='str',
 497                     help='Path to goma directory. If this is supplied, goma '
 498                     'builds will be enabled.')
 499   parser.add_option('--path_to_config',
 500                     type='str',
 501                     help='Path to the config file to use. If this is supplied, '
 502                     'the bisect script will use this to override the default '
 503                     'config file path. The script will attempt to load it '
 504                     'as a bisect config first, then a perf config.')
 505   parser.add_option('--extra_src',
 506                     type='str',
 507                     help='Path to extra source file. If this is supplied, '
 508                     'bisect script will use this to override default behavior.')
 509   parser.add_option('--dry_run',
 510                     action="store_true",
 511                     help='The script will perform the full bisect, but '
 512                     'without syncing, building, or running the performance '
 513                     'tests.')
 514   return parser
 515
 516
 517 def main():
 518   """Entry point for run-bisect-perf-regression.py.
 519
 520   Reads the config file, and then tries to either bisect a regression or
 521   just run a performance test, depending on the particular config parameters
 522   specified in the config file.
 523   """
 524
 525   parser = _OptionParser()
 526   opts, _ = parser.parse_args()
 527
 528   current_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
 529
 530   # Use the default config file path unless one was specified.
 531   config_path = os.path.join(current_dir, BISECT_REGRESSION_CONFIG)
 532   if opts.path_to_config:
 533     config_path = opts.path_to_config
 534   config = _LoadConfigFile(config_path)
 535
 536   # Check if the config is valid for running bisect job.
 537   config_is_valid = _ValidateBisectConfigFile(config)
 538
 539   if config and config_is_valid:
 540     if not opts.working_directory:
 541       print 'Error: missing required parameter: --working_directory\n'
 542       parser.print_help()
 543       return 1
 544
 545     return _RunBisectionScript(
 546         config, opts.working_directory, current_dir,
 547         opts.path_to_goma, opts.extra_src, opts.dry_run)
 548
 549   # If it wasn't valid for running a bisect, then maybe the user wanted
 550   # to run a perf test instead of a bisect job. Try reading any possible
 551   # perf test config files.
 552   perf_cfg_files = [RUN_TEST_CONFIG, WEBKIT_RUN_TEST_CONFIG]
 553   for current_perf_cfg_file in perf_cfg_files:
 554     if opts.path_to_config:
 555       path_to_perf_cfg = opts.path_to_config
 556     else:
 557       path_to_perf_cfg = os.path.join(
 558           os.path.abspath(os.path.dirname(sys.argv[0])),
 559           current_perf_cfg_file)
 560
 561     config = _LoadConfigFile(path_to_perf_cfg)
 562     config_is_valid = _ValidatePerfConfigFile(config)
 563
 564     if config and config_is_valid:
 565       return _SetupAndRunPerformanceTest(
 566           config, current_dir, opts.path_to_goma)
 567
 568   print ('Error: Could not load config file. Double check your changes to '
 569          'auto_bisect/bisect.cfg or run-perf-test.cfg for syntax '
 570          'errors.\n')
 571   return 1
 572
 573
 574 if __name__ == '__main__':
 575   sys.exit(main())