tools/run-bisect-perf-regression.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Run Performance Test Bisect Tool
   7
   8 This script is used by a try bot to run the bisect script with the parameters
   9 specified in the bisect config file. It checks out a copy of the depot in
  10 a subdirectory 'bisect' of the working directory provided, annd runs the
  11 bisect scrip there.
  12 """
  13
  14 import optparse
  15 import os
  16 import platform
  17 import subprocess
  18 import sys
  19 import traceback
  20
  21 from auto_bisect import bisect_perf_regression
  22 from auto_bisect import bisect_utils
  23 from auto_bisect import math_utils
  24
  25 CROS_BOARD_ENV = 'BISECT_CROS_BOARD'
  26 CROS_IP_ENV = 'BISECT_CROS_IP'
  27
  28 SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
  29 SRC_DIR = os.path.join(SCRIPT_DIR, os.path.pardir)
  30 BISECT_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'auto_bisect', 'bisect.cfg')
  31 RUN_TEST_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'run-perf-test.cfg')
  32 WEBKIT_RUN_TEST_CONFIG_PATH = os.path.join(
  33     SRC_DIR, 'third_party', 'WebKit', 'Tools', 'run-perf-test.cfg')
  34 BISECT_SCRIPT_DIR = os.path.join(SCRIPT_DIR, 'auto_bisect')
  35
  36
  37 class Goma(object):
  38
  39   def __init__(self, path_to_goma):
  40     self._abs_path_to_goma = None
  41     self._abs_path_to_goma_file = None
  42     if not path_to_goma:
  43       return
  44     self._abs_path_to_goma = os.path.abspath(path_to_goma)
  45     filename = 'goma_ctl.bat' if os.name == 'nt' else 'goma_ctl.sh'
  46     self._abs_path_to_goma_file = os.path.join(self._abs_path_to_goma, filename)
  47
  48   def __enter__(self):
  49     if self._HasGomaPath():
  50       self._SetupAndStart()
  51     return self
  52
  53   def __exit__(self, *_):
  54     if self._HasGomaPath():
  55       self._Stop()
  56
  57   def _HasGomaPath(self):
  58     return bool(self._abs_path_to_goma)
  59
  60   def _SetupEnvVars(self):
  61     if os.name == 'nt':
  62       os.environ['CC'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
  63           ' cl.exe')
  64       os.environ['CXX'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
  65           ' cl.exe')
  66     else:
  67       os.environ['PATH'] = os.pathsep.join([self._abs_path_to_goma,
  68           os.environ['PATH']])
  69
  70   def _SetupAndStart(self):
  71     """Sets up goma and launches it.
  72
  73     Args:
  74       path_to_goma: Path to goma directory.
  75
  76     Returns:
  77       True if successful."""
  78     self._SetupEnvVars()
  79
  80     # Sometimes goma is lingering around if something went bad on a previous
  81     # run. Stop it before starting a new process. Can ignore the return code
  82     # since it will return an error if it wasn't running.
  83     self._Stop()
  84
  85     if subprocess.call([self._abs_path_to_goma_file, 'start']):
  86       raise RuntimeError('Goma failed to start.')
  87
  88   def _Stop(self):
  89     subprocess.call([self._abs_path_to_goma_file, 'stop'])
  90
  91
  92 def _LoadConfigFile(config_file_path):
  93   """Attempts to load the specified config file as a module
  94   and grab the global config dict.
  95
  96   Args:
  97     config_file_path: Path to the config file.
  98
  99   Returns:
 100     If successful, returns the config dict loaded from the file. If no
 101     such dictionary could be loaded, returns the empty dictionary.
 102   """
 103   try:
 104     local_vars = {}
 105     execfile(config_file_path, local_vars)
 106     return local_vars['config']
 107   except Exception:
 108     print
 109     traceback.print_exc()
 110     print
 111     return {}
 112
 113
 114 def _ValidateConfigFile(config_contents, required_parameters):
 115   """Validates the config file contents, checking whether all values are
 116   non-empty.
 117
 118   Args:
 119     config_contents: A config dictionary.
 120     required_parameters: A list of parameters to check for.
 121
 122   Returns:
 123     True if valid.
 124   """
 125   for parameter in required_parameters:
 126     if parameter not in config_contents:
 127       return False
 128     value = config_contents[parameter]
 129     if not value or type(value) is not str:
 130       return False
 131   return True
 132
 133
 134 def _ValidatePerfConfigFile(config_contents):
 135   """Validates the perf config file contents.
 136
 137   This is used when we're doing a perf try job, rather than a bisect.
 138   The config file is called run-perf-test.cfg by default.
 139
 140   The parameters checked are the required parameters; any additional optional
 141   parameters won't be checked and validation will still pass.
 142
 143   Args:
 144     config_contents: A config dictionary.
 145
 146   Returns:
 147     True if valid.
 148   """
 149   required_parameters = [
 150       'command',
 151       'repeat_count',
 152       'truncate_percent',
 153       'max_time_minutes',
 154   ]
 155   return _ValidateConfigFile(config_contents, required_parameters)
 156
 157
 158 def _ValidateBisectConfigFile(config_contents):
 159   """Validates the bisect config file contents.
 160
 161   The parameters checked are the required parameters; any additional optional
 162   parameters won't be checked and validation will still pass.
 163
 164   Args:
 165     config_contents: A config dictionary.
 166
 167   Returns:
 168     True if valid.
 169   """
 170   required_params = [
 171       'command',
 172       'good_revision',
 173       'bad_revision',
 174       'metric',
 175       'repeat_count',
 176       'truncate_percent',
 177       'max_time_minutes',
 178   ]
 179   return _ValidateConfigFile(config_contents, required_params)
 180
 181
 182 def _OutputFailedResults(text_to_print):
 183   bisect_utils.OutputAnnotationStepStart('Results - Failed')
 184   print
 185   print text_to_print
 186   print
 187   bisect_utils.OutputAnnotationStepClosed()
 188
 189
 190 def _CreateBisectOptionsFromConfig(config):
 191   print config['command']
 192   opts_dict = {}
 193   opts_dict['command'] = config['command']
 194   opts_dict['metric'] = config.get('metric')
 195
 196   if config['repeat_count']:
 197     opts_dict['repeat_test_count'] = int(config['repeat_count'])
 198
 199   if config['truncate_percent']:
 200     opts_dict['truncate_percent'] = int(config['truncate_percent'])
 201
 202   if config['max_time_minutes']:
 203     opts_dict['max_time_minutes'] = int(config['max_time_minutes'])
 204
 205   if config.has_key('use_goma'):
 206     opts_dict['use_goma'] = config['use_goma']
 207   if config.has_key('goma_dir'):
 208     opts_dict['goma_dir'] = config['goma_dir']
 209
 210   if config.has_key('improvement_direction'):
 211     opts_dict['improvement_direction'] = int(config['improvement_direction'])
 212
 213   if config.has_key('bug_id') and str(config['bug_id']).isdigit():
 214     opts_dict['bug_id'] = config['bug_id']
 215
 216   opts_dict['build_preference'] = 'ninja'
 217   opts_dict['output_buildbot_annotations'] = True
 218
 219   if '--browser=cros' in config['command']:
 220     opts_dict['target_platform'] = 'cros'
 221
 222     if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
 223       opts_dict['cros_board'] = os.environ[CROS_BOARD_ENV]
 224       opts_dict['cros_remote_ip'] = os.environ[CROS_IP_ENV]
 225     else:
 226       raise RuntimeError('CrOS build selected, but BISECT_CROS_IP or'
 227           'BISECT_CROS_BOARD undefined.')
 228   elif 'android' in config['command']:
 229     if 'android-chrome-shell' in config['command']:
 230       opts_dict['target_platform'] = 'android'
 231     elif 'android-chrome' in config['command']:
 232       opts_dict['target_platform'] = 'android-chrome'
 233     else:
 234       opts_dict['target_platform'] = 'android'
 235
 236   return bisect_perf_regression.BisectOptions.FromDict(opts_dict)
 237
 238
 239 def _ParseCloudLinksFromOutput(output, bucket):
 240   cloud_file_links = [t for t in output.splitlines()
 241       if 'storage.googleapis.com/chromium-telemetry/%s/' % bucket in t]
 242
 243   # What we're getting here is basically "View online at http://..." so parse
 244   # out just the URL portion.
 245   for i in xrange(len(cloud_file_links)):
 246     cloud_file_link = cloud_file_links[i]
 247     cloud_file_link = [t for t in cloud_file_link.split(' ')
 248         if 'storage.googleapis.com/chromium-telemetry/%s/' % bucket in t]
 249     assert cloud_file_link, 'Couldn\'t parse URL from output.'
 250     cloud_file_links[i] = cloud_file_link[0]
 251   return cloud_file_links
 252
 253
 254 def _RunPerformanceTest(config):
 255   """Runs a performance test with and without the current patch.
 256
 257   Args:
 258     config: Contents of the config file, a dictionary.
 259
 260   Attempts to build and run the current revision with and without the
 261   current patch, with the parameters passed in.
 262   """
 263   # Bisect script expects to be run from the src directory
 264   os.chdir(SRC_DIR)
 265
 266   bisect_utils.OutputAnnotationStepStart('Building With Patch')
 267
 268   opts = _CreateBisectOptionsFromConfig(config)
 269   b = bisect_perf_regression.BisectPerformanceMetrics(opts, os.getcwd())
 270
 271   if bisect_utils.RunGClient(['runhooks']):
 272     raise RuntimeError('Failed to run gclient runhooks')
 273
 274   if not b.ObtainBuild('chromium'):
 275     raise RuntimeError('Patched version failed to build.')
 276
 277   bisect_utils.OutputAnnotationStepClosed()
 278   bisect_utils.OutputAnnotationStepStart('Running With Patch')
 279
 280   results_with_patch = b.RunPerformanceTestAndParseResults(
 281       opts.command,
 282       opts.metric,
 283       reset_on_first_run=True,
 284       upload_on_last_run=True,
 285       results_label='Patch')
 286
 287   if results_with_patch[1]:
 288     raise RuntimeError('Patched version failed to run performance test.')
 289
 290   bisect_utils.OutputAnnotationStepClosed()
 291
 292   bisect_utils.OutputAnnotationStepStart('Reverting Patch')
 293   # TODO: When this is re-written to recipes, this should use bot_update's
 294   # revert mechanism to fully revert the client. But for now, since we know that
 295   # the perf try bot currently only supports src/ and src/third_party/WebKit, we
 296   # simply reset those two directories.
 297   bisect_utils.CheckRunGit(['reset', '--hard'])
 298   bisect_utils.CheckRunGit(['reset', '--hard'],
 299                            os.path.join('third_party', 'WebKit'))
 300   bisect_utils.OutputAnnotationStepClosed()
 301
 302   bisect_utils.OutputAnnotationStepStart('Building Without Patch')
 303
 304   if bisect_utils.RunGClient(['runhooks']):
 305     raise RuntimeError('Failed to run gclient runhooks')
 306
 307   if not b.ObtainBuild('chromium'):
 308     raise RuntimeError('Unpatched version failed to build.')
 309
 310   bisect_utils.OutputAnnotationStepClosed()
 311   bisect_utils.OutputAnnotationStepStart('Running Without Patch')
 312
 313   results_without_patch = b.RunPerformanceTestAndParseResults(
 314       opts.command, opts.metric, upload_on_last_run=True, results_label='ToT')
 315
 316   if results_without_patch[1]:
 317     raise RuntimeError('Unpatched version failed to run performance test.')
 318
 319   # Find the link to the cloud stored results file.
 320   cloud_file_link = _ParseCloudLinksFromOutput(results_without_patch[2],
 321                                                'html-results')
 322   output = results_without_patch[2]
 323   if cloud_file_link:
 324     cloud_file_link = cloud_file_link[0]
 325   else:
 326     cloud_file_link = ''
 327
 328   profiler_file_links_with_patch = _ParseCloudLinksFromOutput(
 329       results_with_patch[2], 'profiling-results')
 330
 331   profiler_file_links_without_patch = _ParseCloudLinksFromOutput(
 332       results_without_patch[2], 'profiling-results')
 333
 334   # Calculate the % difference in the means of the 2 runs.
 335   percent_diff_in_means = None
 336   std_err = None
 337   if (results_with_patch[0].has_key('mean') and
 338       results_with_patch[0].has_key('values')):
 339     percent_diff_in_means = (results_with_patch[0]['mean'] /
 340         max(0.0001, results_without_patch[0]['mean'])) * 100.0 - 100.0
 341     std_err = math_utils.PooledStandardError(
 342         [results_with_patch[0]['values'], results_without_patch[0]['values']])
 343
 344   bisect_utils.OutputAnnotationStepClosed()
 345   if percent_diff_in_means is not None and std_err is not None:
 346     bisect_utils.OutputAnnotationStepStart('Results - %.02f +- %0.02f delta' %
 347         (percent_diff_in_means, std_err))
 348     print ' %s %s %s' % (''.center(10, ' '), 'Mean'.center(20, ' '),
 349         'Std. Error'.center(20, ' '))
 350     print ' %s %s %s' % ('Patch'.center(10, ' '),
 351         ('%.02f' % results_with_patch[0]['mean']).center(20, ' '),
 352         ('%.02f' % results_with_patch[0]['std_err']).center(20, ' '))
 353     print ' %s %s %s' % ('No Patch'.center(10, ' '),
 354         ('%.02f' % results_without_patch[0]['mean']).center(20, ' '),
 355         ('%.02f' % results_without_patch[0]['std_err']).center(20, ' '))
 356     if cloud_file_link:
 357       bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
 358     bisect_utils.OutputAnnotationStepClosed()
 359   elif cloud_file_link:
 360     bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
 361
 362   if profiler_file_links_with_patch and profiler_file_links_without_patch:
 363     for i in xrange(len(profiler_file_links_with_patch)):
 364       bisect_utils.OutputAnnotationStepLink(
 365           'With Patch - Profiler Data[%d]' % i,
 366           profiler_file_links_with_patch[i])
 367     for i in xrange(len(profiler_file_links_without_patch)):
 368       bisect_utils.OutputAnnotationStepLink(
 369           'Without Patch - Profiler Data[%d]' % i,
 370           profiler_file_links_without_patch[i])
 371
 372
 373 def _SetupAndRunPerformanceTest(config, path_to_goma):
 374   """Attempts to build and run the current revision with and without the
 375   current patch, with the parameters passed in.
 376
 377   Args:
 378     config: The config read from run-perf-test.cfg.
 379     path_to_goma: Path to goma directory.
 380
 381   Returns:
 382     An exit code: 0 on success, otherwise 1.
 383   """
 384   if platform.release() == 'XP':
 385     print 'Windows XP is not supported for perf try jobs because it lacks '
 386     print 'goma support. Please refer to crbug.com/330900.'
 387     return 1
 388   try:
 389     with Goma(path_to_goma) as _:
 390       config['use_goma'] = bool(path_to_goma)
 391       if config['use_goma']:
 392         config['goma_dir'] = os.path.abspath(path_to_goma)
 393       _RunPerformanceTest(config)
 394     return 0
 395   except RuntimeError, e:
 396     bisect_utils.OutputAnnotationStepClosed()
 397     _OutputFailedResults('Error: %s' % e.message)
 398     return 1
 399
 400
 401 def _RunBisectionScript(
 402     config, working_directory, path_to_goma, path_to_extra_src, dry_run):
 403   """Attempts to execute the bisect script with the given parameters.
 404
 405   Args:
 406     config: A dict containing the parameters to pass to the script.
 407     working_directory: A working directory to provide to the bisect script,
 408       where it will store it's own copy of the depot.
 409     path_to_goma: Path to goma directory.
 410     path_to_extra_src: Path to extra source file.
 411     dry_run: Do a dry run, skipping sync, build, and performance testing steps.
 412
 413   Returns:
 414     An exit status code: 0 on success, otherwise 1.
 415   """
 416   _PrintConfigStep(config)
 417
 418   cmd = ['python', os.path.join(BISECT_SCRIPT_DIR, 'bisect_perf_regression.py'),
 419          '-c', config['command'],
 420          '-g', config['good_revision'],
 421          '-b', config['bad_revision'],
 422          '-m', config['metric'],
 423          '--working_directory', working_directory,
 424          '--output_buildbot_annotations']
 425
 426   if config.get('metric'):
 427     cmd.extend(['-m', config['metric']])
 428
 429   if config['repeat_count']:
 430     cmd.extend(['-r', config['repeat_count']])
 431
 432   if config['truncate_percent']:
 433     cmd.extend(['-t', config['truncate_percent']])
 434
 435   if config['max_time_minutes']:
 436     cmd.extend(['--max_time_minutes', config['max_time_minutes']])
 437
 438   if config.has_key('bisect_mode'):
 439     cmd.extend(['--bisect_mode', config['bisect_mode']])
 440
 441   if config.has_key('improvement_direction'):
 442     cmd.extend(['-d', config['improvement_direction']])
 443
 444   if config.has_key('bug_id'):
 445     cmd.extend(['--bug_id', config['bug_id']])
 446
 447   cmd.extend(['--build_preference', 'ninja'])
 448
 449   if '--browser=cros' in config['command']:
 450     cmd.extend(['--target_platform', 'cros'])
 451
 452     if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
 453       cmd.extend(['--cros_board', os.environ[CROS_BOARD_ENV]])
 454       cmd.extend(['--cros_remote_ip', os.environ[CROS_IP_ENV]])
 455     else:
 456       print ('Error: Cros build selected, but BISECT_CROS_IP or'
 457              'BISECT_CROS_BOARD undefined.\n')
 458       return 1
 459
 460   if 'android' in config['command']:
 461     if 'android-chrome-shell' in config['command']:
 462       cmd.extend(['--target_platform', 'android'])
 463     elif 'android-chrome' in config['command']:
 464       cmd.extend(['--target_platform', 'android-chrome'])
 465     else:
 466       cmd.extend(['--target_platform', 'android'])
 467
 468   if path_to_goma:
 469     # For Windows XP platforms, goma service is not supported.
 470     # Moreover we don't compile chrome when gs_bucket flag is set instead
 471     # use builds archives, therefore ignore goma service for Windows XP.
 472     # See http://crbug.com/330900.
 473     if config.get('gs_bucket') and platform.release() == 'XP':
 474       print ('Goma doesn\'t have a win32 binary, therefore it is not supported '
 475              'on Windows XP platform. Please refer to crbug.com/330900.')
 476       path_to_goma = None
 477     cmd.append('--use_goma')
 478
 479   if path_to_extra_src:
 480     cmd.extend(['--extra_src', path_to_extra_src])
 481
 482   # These flags are used to download build archives from cloud storage if
 483   # available, otherwise will post a try_job_http request to build it on the
 484   # try server.
 485   if config.get('gs_bucket'):
 486     if config.get('builder_host') and config.get('builder_port'):
 487       cmd.extend(['--gs_bucket', config['gs_bucket'],
 488                   '--builder_host', config['builder_host'],
 489                   '--builder_port', config['builder_port']
 490                  ])
 491     else:
 492       print ('Error: Specified gs_bucket, but missing builder_host or '
 493              'builder_port information in config.')
 494       return 1
 495
 496   if dry_run:
 497     cmd.extend(['--debug_ignore_build', '--debug_ignore_sync',
 498         '--debug_ignore_perf_test'])
 499   cmd = [str(c) for c in cmd]
 500
 501   with Goma(path_to_goma) as _:
 502     return_code = subprocess.call(cmd)
 503
 504   if return_code:
 505     print ('Error: bisect_perf_regression.py returned with error %d\n'
 506            % return_code)
 507
 508   return return_code
 509
 510
 511 def _PrintConfigStep(config):
 512   """Prints out the given config, along with Buildbot annotations."""
 513   bisect_utils.OutputAnnotationStepStart('Config')
 514   print
 515   for k, v in config.iteritems():
 516     print '  %s : %s' % (k, v)
 517   print
 518   bisect_utils.OutputAnnotationStepClosed()
 519
 520
 521 def _OptionParser():
 522   """Returns the options parser for run-bisect-perf-regression.py."""
 523   usage = ('%prog [options] [-- chromium-options]\n'
 524            'Used by a try bot to run the bisection script using the parameters'
 525            ' provided in the auto_bisect/bisect.cfg file.')
 526   parser = optparse.OptionParser(usage=usage)
 527   parser.add_option('-w', '--working_directory',
 528                     type='str',
 529                     help='A working directory to supply to the bisection '
 530                     'script, which will use it as the location to checkout '
 531                     'a copy of the chromium depot.')
 532   parser.add_option('-p', '--path_to_goma',
 533                     type='str',
 534                     help='Path to goma directory. If this is supplied, goma '
 535                     'builds will be enabled.')
 536   parser.add_option('--path_to_config',
 537                     type='str',
 538                     help='Path to the config file to use. If this is supplied, '
 539                     'the bisect script will use this to override the default '
 540                     'config file path. The script will attempt to load it '
 541                     'as a bisect config first, then a perf config.')
 542   parser.add_option('--extra_src',
 543                     type='str',
 544                     help='Path to extra source file. If this is supplied, '
 545                     'bisect script will use this to override default behavior.')
 546   parser.add_option('--dry_run',
 547                     action="store_true",
 548                     help='The script will perform the full bisect, but '
 549                     'without syncing, building, or running the performance '
 550                     'tests.')
 551   return parser
 552
 553
 554 def main():
 555   """Entry point for run-bisect-perf-regression.py.
 556
 557   Reads the config file, and then tries to either bisect a regression or
 558   just run a performance test, depending on the particular config parameters
 559   specified in the config file.
 560   """
 561   parser = _OptionParser()
 562   opts, _ = parser.parse_args()
 563
 564   # Use the default config file path unless one was specified.
 565   config_path = BISECT_CONFIG_PATH
 566   if opts.path_to_config:
 567     config_path = opts.path_to_config
 568   config = _LoadConfigFile(config_path)
 569
 570   # Check if the config is valid for running bisect job.
 571   config_is_valid = _ValidateBisectConfigFile(config)
 572
 573   if config and config_is_valid:
 574     if not opts.working_directory:
 575       print 'Error: missing required parameter: --working_directory\n'
 576       parser.print_help()
 577       return 1
 578
 579     return _RunBisectionScript(
 580         config, opts.working_directory, opts.path_to_goma, opts.extra_src,
 581         opts.dry_run)
 582
 583   # If it wasn't valid for running a bisect, then maybe the user wanted
 584   # to run a perf test instead of a bisect job. Try reading any possible
 585   # perf test config files.
 586   perf_cfg_files = [RUN_TEST_CONFIG_PATH, WEBKIT_RUN_TEST_CONFIG_PATH]
 587   for current_perf_cfg_file in perf_cfg_files:
 588     if opts.path_to_config:
 589       path_to_perf_cfg = opts.path_to_config
 590     else:
 591       path_to_perf_cfg = os.path.join(
 592           os.path.abspath(os.path.dirname(sys.argv[0])),
 593           current_perf_cfg_file)
 594
 595     config = _LoadConfigFile(path_to_perf_cfg)
 596     config_is_valid = _ValidatePerfConfigFile(config)
 597
 598     if config and config_is_valid:
 599       return _SetupAndRunPerformanceTest(config, opts.path_to_goma)
 600
 601   print ('Error: Could not load config file. Double check your changes to '
 602          'auto_bisect/bisect.cfg or run-perf-test.cfg for syntax errors.\n')
 603   return 1
 604
 605
 606 if __name__ == '__main__':
 607   sys.exit(main())