tools/run-bisect-perf-regression.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Run Performance Test Bisect Tool
   7
   8 This script is used by a try bot to run the bisect script with the parameters
   9 specified in the bisect config file. It checks out a copy of the depot in
  10 a subdirectory 'bisect' of the working directory provided, annd runs the
  11 bisect scrip there.
  12 """
  13
  14 import optparse
  15 import os
  16 import platform
  17 import subprocess
  18 import sys
  19 import traceback
  20
  21 from auto_bisect import bisect_perf_regression
  22 from auto_bisect import bisect_utils
  23 from auto_bisect import math_utils
  24
  25 CROS_BOARD_ENV = 'BISECT_CROS_BOARD'
  26 CROS_IP_ENV = 'BISECT_CROS_IP'
  27
  28 SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
  29 SRC_DIR = os.path.join(SCRIPT_DIR, os.path.pardir)
  30 BISECT_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'auto_bisect', 'bisect.cfg')
  31 RUN_TEST_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'run-perf-test.cfg')
  32 WEBKIT_RUN_TEST_CONFIG_PATH = os.path.join(
  33     SRC_DIR, 'third_party', 'WebKit', 'Tools', 'run-perf-test.cfg')
  34 BISECT_SCRIPT_DIR = os.path.join(SCRIPT_DIR, 'auto_bisect')
  35
  36
  37 class Goma(object):
  38
  39   def __init__(self, path_to_goma):
  40     self._abs_path_to_goma = None
  41     self._abs_path_to_goma_file = None
  42     if not path_to_goma:
  43       return
  44     self._abs_path_to_goma = os.path.abspath(path_to_goma)
  45     filename = 'goma_ctl.bat' if os.name == 'nt' else 'goma_ctl.sh'
  46     self._abs_path_to_goma_file = os.path.join(self._abs_path_to_goma, filename)
  47
  48   def __enter__(self):
  49     if self._HasGomaPath():
  50       self._SetupAndStart()
  51     return self
  52
  53   def __exit__(self, *_):
  54     if self._HasGomaPath():
  55       self._Stop()
  56
  57   def _HasGomaPath(self):
  58     return bool(self._abs_path_to_goma)
  59
  60   def _SetupEnvVars(self):
  61     if os.name == 'nt':
  62       os.environ['CC'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
  63           ' cl.exe')
  64       os.environ['CXX'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
  65           ' cl.exe')
  66     else:
  67       os.environ['PATH'] = os.pathsep.join([self._abs_path_to_goma,
  68           os.environ['PATH']])
  69
  70   def _SetupAndStart(self):
  71     """Sets up goma and launches it.
  72
  73     Args:
  74       path_to_goma: Path to goma directory.
  75
  76     Returns:
  77       True if successful."""
  78     self._SetupEnvVars()
  79
  80     # Sometimes goma is lingering around if something went bad on a previous
  81     # run. Stop it before starting a new process. Can ignore the return code
  82     # since it will return an error if it wasn't running.
  83     self._Stop()
  84
  85     if subprocess.call([self._abs_path_to_goma_file, 'start']):
  86       raise RuntimeError('Goma failed to start.')
  87
  88   def _Stop(self):
  89     subprocess.call([self._abs_path_to_goma_file, 'stop'])
  90
  91
  92 def _LoadConfigFile(config_file_path):
  93   """Attempts to load the specified config file as a module
  94   and grab the global config dict.
  95
  96   Args:
  97     config_file_path: Path to the config file.
  98
  99   Returns:
 100     If successful, returns the config dict loaded from the file. If no
 101     such dictionary could be loaded, returns the empty dictionary.
 102   """
 103   try:
 104     local_vars = {}
 105     execfile(config_file_path, local_vars)
 106     return local_vars['config']
 107   except Exception:
 108     print
 109     traceback.print_exc()
 110     print
 111     return {}
 112
 113
 114 def _ValidateConfigFile(config_contents, valid_parameters):
 115   """Validates the config file contents, checking whether all values are
 116   non-empty.
 117
 118   Args:
 119     config_contents: A config dictionary.
 120     valid_parameters: A list of parameters to check for.
 121
 122   Returns:
 123     True if valid.
 124   """
 125   for parameter in valid_parameters:
 126     if parameter not in config_contents:
 127       return False
 128     value = config_contents[parameter]
 129     if not value or type(value) is not str:
 130       return False
 131   return True
 132
 133
 134 def _ValidatePerfConfigFile(config_contents):
 135   """Validates the perf config file contents.
 136
 137   This is used when we're doing a perf try job, rather than a bisect.
 138   The config file is called run-perf-test.cfg by default.
 139
 140   The parameters checked are the required parameters; any additional optional
 141   parameters won't be checked and validation will still pass.
 142
 143   Args:
 144     config_contents: A config dictionary.
 145
 146   Returns:
 147     True if valid.
 148   """
 149   valid_parameters = [
 150       'command',
 151       'repeat_count',
 152       'truncate_percent',
 153       'max_time_minutes',
 154   ]
 155   return _ValidateConfigFile(config_contents, valid_parameters)
 156
 157
 158 def _ValidateBisectConfigFile(config_contents):
 159   """Validates the bisect config file contents.
 160
 161   The parameters checked are the required parameters; any additional optional
 162   parameters won't be checked and validation will still pass.
 163
 164   Args:
 165     config_contents: A config dictionary.
 166
 167   Returns:
 168     True if valid.
 169   """
 170   valid_params = [
 171       'command',
 172       'good_revision',
 173       'bad_revision',
 174       'metric',
 175       'repeat_count',
 176       'truncate_percent',
 177       'max_time_minutes',
 178   ]
 179   return _ValidateConfigFile(config_contents, valid_params)
 180
 181
 182 def _OutputFailedResults(text_to_print):
 183   bisect_utils.OutputAnnotationStepStart('Results - Failed')
 184   print
 185   print text_to_print
 186   print
 187   bisect_utils.OutputAnnotationStepClosed()
 188
 189
 190 def _CreateBisectOptionsFromConfig(config):
 191   print config['command']
 192   opts_dict = {}
 193   opts_dict['command'] = config['command']
 194   opts_dict['metric'] = config.get('metric')
 195
 196   if config['repeat_count']:
 197     opts_dict['repeat_test_count'] = int(config['repeat_count'])
 198
 199   if config['truncate_percent']:
 200     opts_dict['truncate_percent'] = int(config['truncate_percent'])
 201
 202   if config['max_time_minutes']:
 203     opts_dict['max_time_minutes'] = int(config['max_time_minutes'])
 204
 205   if config.has_key('use_goma'):
 206     opts_dict['use_goma'] = config['use_goma']
 207   if config.has_key('goma_dir'):
 208     opts_dict['goma_dir'] = config['goma_dir']
 209
 210   if config.has_key('improvement_direction'):
 211     opts_dict['improvement_direction'] = int(config['improvement_direction'])
 212
 213   opts_dict['build_preference'] = 'ninja'
 214   opts_dict['output_buildbot_annotations'] = True
 215
 216   if '--browser=cros' in config['command']:
 217     opts_dict['target_platform'] = 'cros'
 218
 219     if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
 220       opts_dict['cros_board'] = os.environ[CROS_BOARD_ENV]
 221       opts_dict['cros_remote_ip'] = os.environ[CROS_IP_ENV]
 222     else:
 223       raise RuntimeError('CrOS build selected, but BISECT_CROS_IP or'
 224           'BISECT_CROS_BOARD undefined.')
 225   elif 'android' in config['command']:
 226     if 'android-chrome-shell' in config['command']:
 227       opts_dict['target_platform'] = 'android'
 228     elif 'android-chrome' in config['command']:
 229       opts_dict['target_platform'] = 'android-chrome'
 230     else:
 231       opts_dict['target_platform'] = 'android'
 232
 233   return bisect_perf_regression.BisectOptions.FromDict(opts_dict)
 234
 235
 236 def _RunPerformanceTest(config):
 237   """Runs a performance test with and without the current patch.
 238
 239   Args:
 240     config: Contents of the config file, a dictionary.
 241
 242   Attempts to build and run the current revision with and without the
 243   current patch, with the parameters passed in.
 244   """
 245   # Bisect script expects to be run from the src directory
 246   os.chdir(SRC_DIR)
 247
 248   bisect_utils.OutputAnnotationStepStart('Building With Patch')
 249
 250   opts = _CreateBisectOptionsFromConfig(config)
 251   b = bisect_perf_regression.BisectPerformanceMetrics(opts)
 252
 253   if bisect_utils.RunGClient(['runhooks']):
 254     raise RuntimeError('Failed to run gclient runhooks')
 255
 256   if not b.BuildCurrentRevision('chromium'):
 257     raise RuntimeError('Patched version failed to build.')
 258
 259   bisect_utils.OutputAnnotationStepClosed()
 260   bisect_utils.OutputAnnotationStepStart('Running With Patch')
 261
 262   results_with_patch = b.RunPerformanceTestAndParseResults(
 263       opts.command, opts.metric, reset_on_first_run=True, results_label='Patch')
 264
 265   if results_with_patch[1]:
 266     raise RuntimeError('Patched version failed to run performance test.')
 267
 268   bisect_utils.OutputAnnotationStepClosed()
 269
 270   bisect_utils.OutputAnnotationStepStart('Reverting Patch')
 271   # TODO: When this is re-written to recipes, this should use bot_update's
 272   # revert mechanism to fully revert the client. But for now, since we know that
 273   # the perf try bot currently only supports src/ and src/third_party/WebKit, we
 274   # simply reset those two directories.
 275   bisect_utils.CheckRunGit(['reset', '--hard'])
 276   bisect_utils.CheckRunGit(['reset', '--hard'],
 277                            os.path.join('third_party', 'WebKit'))
 278   bisect_utils.OutputAnnotationStepClosed()
 279
 280   bisect_utils.OutputAnnotationStepStart('Building Without Patch')
 281
 282   if bisect_utils.RunGClient(['runhooks']):
 283     raise RuntimeError('Failed to run gclient runhooks')
 284
 285   if not b.BuildCurrentRevision('chromium'):
 286     raise RuntimeError('Unpatched version failed to build.')
 287
 288   bisect_utils.OutputAnnotationStepClosed()
 289   bisect_utils.OutputAnnotationStepStart('Running Without Patch')
 290
 291   results_without_patch = b.RunPerformanceTestAndParseResults(
 292       opts.command, opts.metric, upload_on_last_run=True, results_label='ToT')
 293
 294   if results_without_patch[1]:
 295     raise RuntimeError('Unpatched version failed to run performance test.')
 296
 297   # Find the link to the cloud stored results file.
 298   output = results_without_patch[2]
 299   cloud_file_link = [t for t in output.splitlines()
 300       if 'storage.googleapis.com/chromium-telemetry/html-results/' in t]
 301   if cloud_file_link:
 302     # What we're getting here is basically "View online at http://..." so parse
 303     # out just the URL portion.
 304     cloud_file_link = cloud_file_link[0]
 305     cloud_file_link = [t for t in cloud_file_link.split(' ')
 306         if 'storage.googleapis.com/chromium-telemetry/html-results/' in t]
 307     assert cloud_file_link, 'Couldn\'t parse URL from output.'
 308     cloud_file_link = cloud_file_link[0]
 309   else:
 310     cloud_file_link = ''
 311
 312   # Calculate the % difference in the means of the 2 runs.
 313   percent_diff_in_means = None
 314   std_err = None
 315   if (results_with_patch[0].has_key('mean') and
 316       results_with_patch[0].has_key('values')):
 317     percent_diff_in_means = (results_with_patch[0]['mean'] /
 318         max(0.0001, results_without_patch[0]['mean'])) * 100.0 - 100.0
 319     std_err = math_utils.PooledStandardError(
 320         [results_with_patch[0]['values'], results_without_patch[0]['values']])
 321
 322   bisect_utils.OutputAnnotationStepClosed()
 323   if percent_diff_in_means is not None and std_err is not None:
 324     bisect_utils.OutputAnnotationStepStart('Results - %.02f +- %0.02f delta' %
 325         (percent_diff_in_means, std_err))
 326     print ' %s %s %s' % (''.center(10, ' '), 'Mean'.center(20, ' '),
 327         'Std. Error'.center(20, ' '))
 328     print ' %s %s %s' % ('Patch'.center(10, ' '),
 329         ('%.02f' % results_with_patch[0]['mean']).center(20, ' '),
 330         ('%.02f' % results_with_patch[0]['std_err']).center(20, ' '))
 331     print ' %s %s %s' % ('No Patch'.center(10, ' '),
 332         ('%.02f' % results_without_patch[0]['mean']).center(20, ' '),
 333         ('%.02f' % results_without_patch[0]['std_err']).center(20, ' '))
 334     if cloud_file_link:
 335       bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
 336     bisect_utils.OutputAnnotationStepClosed()
 337   elif cloud_file_link:
 338     bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
 339
 340
 341 def _SetupAndRunPerformanceTest(config, path_to_goma):
 342   """Attempts to build and run the current revision with and without the
 343   current patch, with the parameters passed in.
 344
 345   Args:
 346     config: The config read from run-perf-test.cfg.
 347     path_to_goma: Path to goma directory.
 348
 349   Returns:
 350     An exit code: 0 on success, otherwise 1.
 351   """
 352   if platform.release() == 'XP':
 353     print 'Windows XP is not supported for perf try jobs because it lacks '
 354     print 'goma support. Please refer to crbug.com/330900.'
 355     return 1
 356   try:
 357     with Goma(path_to_goma) as _:
 358       config['use_goma'] = bool(path_to_goma)
 359       if config['use_goma']:
 360         config['goma_dir'] = os.path.abspath(path_to_goma)
 361       _RunPerformanceTest(config)
 362     return 0
 363   except RuntimeError, e:
 364     bisect_utils.OutputAnnotationStepClosed()
 365     _OutputFailedResults('Error: %s' % e.message)
 366     return 1
 367
 368
 369 def _RunBisectionScript(
 370     config, working_directory, path_to_goma, path_to_extra_src, dry_run):
 371   """Attempts to execute the bisect script with the given parameters.
 372
 373   Args:
 374     config: A dict containing the parameters to pass to the script.
 375     working_directory: A working directory to provide to the bisect script,
 376       where it will store it's own copy of the depot.
 377     path_to_goma: Path to goma directory.
 378     path_to_extra_src: Path to extra source file.
 379     dry_run: Do a dry run, skipping sync, build, and performance testing steps.
 380
 381   Returns:
 382     An exit status code: 0 on success, otherwise 1.
 383   """
 384   _PrintConfigStep(config)
 385
 386   cmd = ['python', os.path.join(BISECT_SCRIPT_DIR, 'bisect_perf_regression.py'),
 387          '-c', config['command'],
 388          '-g', config['good_revision'],
 389          '-b', config['bad_revision'],
 390          '-m', config['metric'],
 391          '--working_directory', working_directory,
 392          '--output_buildbot_annotations']
 393
 394   if config.get('metric'):
 395     cmd.extend(['-m', config['metric']])
 396
 397   if config['repeat_count']:
 398     cmd.extend(['-r', config['repeat_count']])
 399
 400   if config['truncate_percent']:
 401     cmd.extend(['-t', config['truncate_percent']])
 402
 403   if config['max_time_minutes']:
 404     cmd.extend(['--max_time_minutes', config['max_time_minutes']])
 405
 406   if config.has_key('bisect_mode'):
 407     cmd.extend(['--bisect_mode', config['bisect_mode']])
 408
 409   if config.has_key('improvement_direction'):
 410     cmd.extend(['-d', config['improvement_direction']])
 411
 412   cmd.extend(['--build_preference', 'ninja'])
 413
 414   if '--browser=cros' in config['command']:
 415     cmd.extend(['--target_platform', 'cros'])
 416
 417     if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
 418       cmd.extend(['--cros_board', os.environ[CROS_BOARD_ENV]])
 419       cmd.extend(['--cros_remote_ip', os.environ[CROS_IP_ENV]])
 420     else:
 421       print ('Error: Cros build selected, but BISECT_CROS_IP or'
 422              'BISECT_CROS_BOARD undefined.\n')
 423       return 1
 424
 425   if 'android' in config['command']:
 426     if 'android-chrome-shell' in config['command']:
 427       cmd.extend(['--target_platform', 'android'])
 428     elif 'android-chrome' in config['command']:
 429       cmd.extend(['--target_platform', 'android-chrome'])
 430     else:
 431       cmd.extend(['--target_platform', 'android'])
 432
 433   if path_to_goma:
 434     # For Windows XP platforms, goma service is not supported.
 435     # Moreover we don't compile chrome when gs_bucket flag is set instead
 436     # use builds archives, therefore ignore goma service for Windows XP.
 437     # See http://crbug.com/330900.
 438     if config.get('gs_bucket') and platform.release() == 'XP':
 439       print ('Goma doesn\'t have a win32 binary, therefore it is not supported '
 440              'on Windows XP platform. Please refer to crbug.com/330900.')
 441       path_to_goma = None
 442     cmd.append('--use_goma')
 443
 444   if path_to_extra_src:
 445     cmd.extend(['--extra_src', path_to_extra_src])
 446
 447   # These flags are used to download build archives from cloud storage if
 448   # available, otherwise will post a try_job_http request to build it on the
 449   # try server.
 450   if config.get('gs_bucket'):
 451     if config.get('builder_host') and config.get('builder_port'):
 452       cmd.extend(['--gs_bucket', config['gs_bucket'],
 453                   '--builder_host', config['builder_host'],
 454                   '--builder_port', config['builder_port']
 455                  ])
 456     else:
 457       print ('Error: Specified gs_bucket, but missing builder_host or '
 458              'builder_port information in config.')
 459       return 1
 460
 461   if dry_run:
 462     cmd.extend(['--debug_ignore_build', '--debug_ignore_sync',
 463         '--debug_ignore_perf_test'])
 464   cmd = [str(c) for c in cmd]
 465
 466   with Goma(path_to_goma) as _:
 467     return_code = subprocess.call(cmd)
 468
 469   if return_code:
 470     print ('Error: bisect_perf_regression.py returned with error %d\n'
 471            % return_code)
 472
 473   return return_code
 474
 475
 476 def _PrintConfigStep(config):
 477   """Prints out the given config, along with Buildbot annotations."""
 478   bisect_utils.OutputAnnotationStepStart('Config')
 479   print
 480   for k, v in config.iteritems():
 481     print '  %s : %s' % (k, v)
 482   print
 483   bisect_utils.OutputAnnotationStepClosed()
 484
 485
 486 def _OptionParser():
 487   """Returns the options parser for run-bisect-perf-regression.py."""
 488   usage = ('%prog [options] [-- chromium-options]\n'
 489            'Used by a try bot to run the bisection script using the parameters'
 490            ' provided in the auto_bisect/bisect.cfg file.')
 491   parser = optparse.OptionParser(usage=usage)
 492   parser.add_option('-w', '--working_directory',
 493                     type='str',
 494                     help='A working directory to supply to the bisection '
 495                     'script, which will use it as the location to checkout '
 496                     'a copy of the chromium depot.')
 497   parser.add_option('-p', '--path_to_goma',
 498                     type='str',
 499                     help='Path to goma directory. If this is supplied, goma '
 500                     'builds will be enabled.')
 501   parser.add_option('--path_to_config',
 502                     type='str',
 503                     help='Path to the config file to use. If this is supplied, '
 504                     'the bisect script will use this to override the default '
 505                     'config file path. The script will attempt to load it '
 506                     'as a bisect config first, then a perf config.')
 507   parser.add_option('--extra_src',
 508                     type='str',
 509                     help='Path to extra source file. If this is supplied, '
 510                     'bisect script will use this to override default behavior.')
 511   parser.add_option('--dry_run',
 512                     action="store_true",
 513                     help='The script will perform the full bisect, but '
 514                     'without syncing, building, or running the performance '
 515                     'tests.')
 516   return parser
 517
 518
 519 def main():
 520   """Entry point for run-bisect-perf-regression.py.
 521
 522   Reads the config file, and then tries to either bisect a regression or
 523   just run a performance test, depending on the particular config parameters
 524   specified in the config file.
 525   """
 526   parser = _OptionParser()
 527   opts, _ = parser.parse_args()
 528
 529   # Use the default config file path unless one was specified.
 530   config_path = BISECT_CONFIG_PATH
 531   if opts.path_to_config:
 532     config_path = opts.path_to_config
 533   config = _LoadConfigFile(config_path)
 534
 535   # Check if the config is valid for running bisect job.
 536   config_is_valid = _ValidateBisectConfigFile(config)
 537
 538   if config and config_is_valid:
 539     if not opts.working_directory:
 540       print 'Error: missing required parameter: --working_directory\n'
 541       parser.print_help()
 542       return 1
 543
 544     return _RunBisectionScript(
 545         config, opts.working_directory, opts.path_to_goma, opts.extra_src,
 546         opts.dry_run)
 547
 548   # If it wasn't valid for running a bisect, then maybe the user wanted
 549   # to run a perf test instead of a bisect job. Try reading any possible
 550   # perf test config files.
 551   perf_cfg_files = [RUN_TEST_CONFIG_PATH, WEBKIT_RUN_TEST_CONFIG_PATH]
 552   for current_perf_cfg_file in perf_cfg_files:
 553     if opts.path_to_config:
 554       path_to_perf_cfg = opts.path_to_config
 555     else:
 556       path_to_perf_cfg = os.path.join(
 557           os.path.abspath(os.path.dirname(sys.argv[0])),
 558           current_perf_cfg_file)
 559
 560     config = _LoadConfigFile(path_to_perf_cfg)
 561     config_is_valid = _ValidatePerfConfigFile(config)
 562
 563     if config and config_is_valid:
 564       return _SetupAndRunPerformanceTest(config, opts.path_to_goma)
 565
 566   print ('Error: Could not load config file. Double check your changes to '
 567          'auto_bisect/bisect.cfg or run-perf-test.cfg for syntax errors.\n')
 568   return 1
 569
 570
 571 if __name__ == '__main__':
 572   sys.exit(main())