tools/run-bisect-perf-regression.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Run Performance Test Bisect Tool
   7
   8 This script is used by a try bot to run the bisect script with the parameters
   9 specified in the bisect config file. It checks out a copy of the depot in
  10 a subdirectory 'bisect' of the working directory provided, annd runs the
  11 bisect scrip there.
  12 """
  13
  14 import optparse
  15 import os
  16 import platform
  17 import re
  18 import subprocess
  19 import sys
  20 import traceback
  21
  22 from auto_bisect import bisect_perf_regression
  23 from auto_bisect import bisect_utils
  24 from auto_bisect import math_utils
  25 from auto_bisect import source_control
  26
  27 CROS_BOARD_ENV = 'BISECT_CROS_BOARD'
  28 CROS_IP_ENV = 'BISECT_CROS_IP'
  29
  30 SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
  31 SRC_DIR = os.path.join(SCRIPT_DIR, os.path.pardir)
  32 BISECT_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'auto_bisect', 'bisect.cfg')
  33 RUN_TEST_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'run-perf-test.cfg')
  34 WEBKIT_RUN_TEST_CONFIG_PATH = os.path.join(
  35     SRC_DIR, 'third_party', 'WebKit', 'Tools', 'run-perf-test.cfg')
  36 BISECT_SCRIPT_DIR = os.path.join(SCRIPT_DIR, 'auto_bisect')
  37
  38
  39 class Goma(object):
  40
  41   def __init__(self, path_to_goma):
  42     self._abs_path_to_goma = None
  43     self._abs_path_to_goma_file = None
  44     if not path_to_goma:
  45       return
  46     self._abs_path_to_goma = os.path.abspath(path_to_goma)
  47     filename = 'goma_ctl.bat' if os.name == 'nt' else 'goma_ctl.sh'
  48     self._abs_path_to_goma_file = os.path.join(self._abs_path_to_goma, filename)
  49
  50   def __enter__(self):
  51     if self._HasGomaPath():
  52       self._SetupAndStart()
  53     return self
  54
  55   def __exit__(self, *_):
  56     if self._HasGomaPath():
  57       self._Stop()
  58
  59   def _HasGomaPath(self):
  60     return bool(self._abs_path_to_goma)
  61
  62   def _SetupEnvVars(self):
  63     if os.name == 'nt':
  64       os.environ['CC'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
  65           ' cl.exe')
  66       os.environ['CXX'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
  67           ' cl.exe')
  68     else:
  69       os.environ['PATH'] = os.pathsep.join([self._abs_path_to_goma,
  70           os.environ['PATH']])
  71
  72   def _SetupAndStart(self):
  73     """Sets up goma and launches it.
  74
  75     Args:
  76       path_to_goma: Path to goma directory.
  77
  78     Returns:
  79       True if successful."""
  80     self._SetupEnvVars()
  81
  82     # Sometimes goma is lingering around if something went bad on a previous
  83     # run. Stop it before starting a new process. Can ignore the return code
  84     # since it will return an error if it wasn't running.
  85     self._Stop()
  86
  87     if subprocess.call([self._abs_path_to_goma_file, 'start']):
  88       raise RuntimeError('Goma failed to start.')
  89
  90   def _Stop(self):
  91     subprocess.call([self._abs_path_to_goma_file, 'stop'])
  92
  93
  94 def _LoadConfigFile(config_file_path):
  95   """Attempts to load the specified config file as a module
  96   and grab the global config dict.
  97
  98   Args:
  99     config_file_path: Path to the config file.
 100
 101   Returns:
 102     If successful, returns the config dict loaded from the file. If no
 103     such dictionary could be loaded, returns the empty dictionary.
 104   """
 105   try:
 106     local_vars = {}
 107     execfile(config_file_path, local_vars)
 108     return local_vars['config']
 109   except Exception:
 110     print
 111     traceback.print_exc()
 112     print
 113     return {}
 114
 115
 116 def _ValidateConfigFile(config_contents, required_parameters):
 117   """Validates the config file contents, checking whether all values are
 118   non-empty.
 119
 120   Args:
 121     config_contents: A config dictionary.
 122     required_parameters: A list of parameters to check for.
 123
 124   Returns:
 125     True if valid.
 126   """
 127   for parameter in required_parameters:
 128     if parameter not in config_contents:
 129       return False
 130     value = config_contents[parameter]
 131     if not value or type(value) is not str:
 132       return False
 133   return True
 134
 135
 136 def _ValidatePerfConfigFile(config_contents):
 137   """Validates the perf config file contents.
 138
 139   This is used when we're doing a perf try job, rather than a bisect.
 140   The config file is called run-perf-test.cfg by default.
 141
 142   The parameters checked are the required parameters; any additional optional
 143   parameters won't be checked and validation will still pass.
 144
 145   Args:
 146     config_contents: A config dictionary.
 147
 148   Returns:
 149     True if valid.
 150   """
 151   required_parameters = [
 152       'command',
 153       'repeat_count',
 154       'truncate_percent',
 155       'max_time_minutes',
 156   ]
 157   return _ValidateConfigFile(config_contents, required_parameters)
 158
 159
 160 def _ValidateBisectConfigFile(config_contents):
 161   """Validates the bisect config file contents.
 162
 163   The parameters checked are the required parameters; any additional optional
 164   parameters won't be checked and validation will still pass.
 165
 166   Args:
 167     config_contents: A config dictionary.
 168
 169   Returns:
 170     True if valid.
 171   """
 172   required_params = [
 173       'command',
 174       'good_revision',
 175       'bad_revision',
 176       'metric',
 177       'repeat_count',
 178       'truncate_percent',
 179       'max_time_minutes',
 180   ]
 181   return _ValidateConfigFile(config_contents, required_params)
 182
 183
 184 def _OutputFailedResults(text_to_print):
 185   bisect_utils.OutputAnnotationStepStart('Results - Failed')
 186   print
 187   print text_to_print
 188   print
 189   bisect_utils.OutputAnnotationStepClosed()
 190
 191
 192 def _CreateBisectOptionsFromConfig(config):
 193   print config['command']
 194   opts_dict = {}
 195   opts_dict['command'] = config['command']
 196   opts_dict['metric'] = config.get('metric')
 197
 198   if config['repeat_count']:
 199     opts_dict['repeat_test_count'] = int(config['repeat_count'])
 200
 201   if config['truncate_percent']:
 202     opts_dict['truncate_percent'] = int(config['truncate_percent'])
 203
 204   if config['max_time_minutes']:
 205     opts_dict['max_time_minutes'] = int(config['max_time_minutes'])
 206
 207   if config.has_key('use_goma'):
 208     opts_dict['use_goma'] = config['use_goma']
 209   if config.has_key('goma_dir'):
 210     opts_dict['goma_dir'] = config['goma_dir']
 211
 212   if config.has_key('improvement_direction'):
 213     opts_dict['improvement_direction'] = int(config['improvement_direction'])
 214
 215   if config.has_key('bug_id') and str(config['bug_id']).isdigit():
 216     opts_dict['bug_id'] = config['bug_id']
 217
 218   opts_dict['build_preference'] = 'ninja'
 219   opts_dict['output_buildbot_annotations'] = True
 220
 221   if '--browser=cros' in config['command']:
 222     opts_dict['target_platform'] = 'cros'
 223
 224     if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
 225       opts_dict['cros_board'] = os.environ[CROS_BOARD_ENV]
 226       opts_dict['cros_remote_ip'] = os.environ[CROS_IP_ENV]
 227     else:
 228       raise RuntimeError('CrOS build selected, but BISECT_CROS_IP or'
 229           'BISECT_CROS_BOARD undefined.')
 230   elif 'android' in config['command']:
 231     if 'android-chrome-shell' in config['command']:
 232       opts_dict['target_platform'] = 'android'
 233     elif 'android-chrome' in config['command']:
 234       opts_dict['target_platform'] = 'android-chrome'
 235     else:
 236       opts_dict['target_platform'] = 'android'
 237
 238   return bisect_perf_regression.BisectOptions.FromDict(opts_dict)
 239
 240
 241 def _ParseCloudLinksFromOutput(output):
 242   html_results_pattern = re.compile(
 243       r'\s(?P<VALUES>http://storage.googleapis.com/' +
 244           'chromium-telemetry/html-results/results-[a-z0-9-_]+)\s',
 245       re.MULTILINE)
 246   profiler_pattern = re.compile(
 247       r'\s(?P<VALUES>https://console.developers.google.com/' +
 248           'm/cloudstorage/b/[a-z-]+/o/profiler-[a-z0-9-_.]+)\s',
 249       re.MULTILINE)
 250
 251   results = {
 252       'html-results': html_results_pattern.findall(output),
 253       'profiler': profiler_pattern.findall(output),
 254   }
 255
 256   return results
 257
 258
 259 def _ParseAndOutputCloudLinks(results_without_patch, results_with_patch):
 260   cloud_links_without_patch = _ParseCloudLinksFromOutput(
 261       results_without_patch[2])
 262   cloud_links_with_patch = _ParseCloudLinksFromOutput(
 263       results_with_patch[2])
 264
 265   cloud_file_link = (cloud_links_without_patch['html-results'][0]
 266       if cloud_links_without_patch['html-results'] else '')
 267
 268   profiler_file_links_with_patch = cloud_links_with_patch['profiler']
 269   profiler_file_links_without_patch = cloud_links_without_patch['profiler']
 270
 271   # Calculate the % difference in the means of the 2 runs.
 272   percent_diff_in_means = None
 273   std_err = None
 274   if (results_with_patch[0].has_key('mean') and
 275       results_with_patch[0].has_key('values')):
 276     percent_diff_in_means = (results_with_patch[0]['mean'] /
 277         max(0.0001, results_without_patch[0]['mean'])) * 100.0 - 100.0
 278     std_err = math_utils.PooledStandardError(
 279         [results_with_patch[0]['values'], results_without_patch[0]['values']])
 280
 281   if percent_diff_in_means is not None and std_err is not None:
 282     bisect_utils.OutputAnnotationStepStart('Results - %.02f +- %0.02f delta' %
 283         (percent_diff_in_means, std_err))
 284     print ' %s %s %s' % (''.center(10, ' '), 'Mean'.center(20, ' '),
 285         'Std. Error'.center(20, ' '))
 286     print ' %s %s %s' % ('Patch'.center(10, ' '),
 287         ('%.02f' % results_with_patch[0]['mean']).center(20, ' '),
 288         ('%.02f' % results_with_patch[0]['std_err']).center(20, ' '))
 289     print ' %s %s %s' % ('No Patch'.center(10, ' '),
 290         ('%.02f' % results_without_patch[0]['mean']).center(20, ' '),
 291         ('%.02f' % results_without_patch[0]['std_err']).center(20, ' '))
 292     if cloud_file_link:
 293       bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
 294     bisect_utils.OutputAnnotationStepClosed()
 295   elif cloud_file_link:
 296     bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
 297
 298   if profiler_file_links_with_patch and profiler_file_links_without_patch:
 299     for i in xrange(len(profiler_file_links_with_patch)):
 300       bisect_utils.OutputAnnotationStepLink(
 301           'With Patch - Profiler Data[%d]' % i,
 302           profiler_file_links_with_patch[i])
 303     for i in xrange(len(profiler_file_links_without_patch)):
 304       bisect_utils.OutputAnnotationStepLink(
 305           'Without Patch - Profiler Data[%d]' % i,
 306           profiler_file_links_without_patch[i])
 307
 308
 309 def _ResolveRevisionsFromConfig(config):
 310   if not 'good_revision' in config and not 'bad_revision' in config:
 311     return (None, None)
 312
 313   bad_revision = source_control.ResolveToRevision(
 314       config['bad_revision'], 'chromium', bisect_utils.DEPOT_DEPS_NAME, 100)
 315   if not bad_revision:
 316     raise RuntimeError('Failed to resolve [%s] to git hash.',
 317         config['bad_revision'])
 318   good_revision = source_control.ResolveToRevision(
 319       config['good_revision'], 'chromium', bisect_utils.DEPOT_DEPS_NAME, -100)
 320   if not good_revision:
 321     raise RuntimeError('Failed to resolve [%s] to git hash.',
 322         config['good_revision'])
 323
 324   return (good_revision, bad_revision)
 325
 326
 327 def _GetStepAnnotationStringsDict(config):
 328   if 'good_revision' in config and 'bad_revision' in config:
 329     return {
 330         'build1': 'Building [%s]' % config['good_revision'],
 331         'build2': 'Building [%s]' % config['bad_revision'],
 332         'run1': 'Running [%s]' % config['good_revision'],
 333         'run2': 'Running [%s]' % config['bad_revision'],
 334         'sync1': 'Syncing [%s]' % config['good_revision'],
 335         'sync2': 'Syncing [%s]' % config['bad_revision'],
 336         'results_label1': config['good_revision'],
 337         'results_label2': config['bad_revision'],
 338     }
 339   else:
 340     return {
 341         'build1': 'Building With Patch',
 342         'build2': 'Building Without Patch',
 343         'run1': 'Running With Patch',
 344         'run2': 'Running Without Patch',
 345         'results_label1': 'Patch',
 346         'results_label2': 'ToT',
 347     }
 348
 349
 350 def _RunBuildStepForPerformanceTest(bisect_instance,
 351                                     build_string,
 352                                     sync_string,
 353                                     revision):
 354   if revision:
 355     bisect_utils.OutputAnnotationStepStart(sync_string)
 356     if not source_control.SyncToRevision(revision, 'gclient'):
 357       raise RuntimeError('Failed [%s].' % sync_string)
 358     bisect_utils.OutputAnnotationStepClosed()
 359
 360   bisect_utils.OutputAnnotationStepStart(build_string)
 361
 362   if bisect_utils.RunGClient(['runhooks']):
 363     raise RuntimeError('Failed to run gclient runhooks')
 364
 365   if not bisect_instance.ObtainBuild('chromium'):
 366     raise RuntimeError('Patched version failed to build.')
 367
 368   bisect_utils.OutputAnnotationStepClosed()
 369
 370
 371 def _RunCommandStepForPerformanceTest(bisect_instance,
 372                                       opts,
 373                                       reset_on_first_run,
 374                                       upload_on_last_run,
 375                                       results_label,
 376                                       run_string):
 377   bisect_utils.OutputAnnotationStepStart(run_string)
 378
 379   results = bisect_instance.RunPerformanceTestAndParseResults(
 380       opts.command,
 381       opts.metric,
 382       reset_on_first_run=reset_on_first_run,
 383       upload_on_last_run=upload_on_last_run,
 384       results_label=results_label)
 385
 386   if results[1]:
 387     raise RuntimeError('Patched version failed to run performance test.')
 388
 389   bisect_utils.OutputAnnotationStepClosed()
 390
 391   return results
 392
 393
 394 def _RunPerformanceTest(config):
 395   """Runs a performance test with and without the current patch.
 396
 397   Args:
 398     config: Contents of the config file, a dictionary.
 399
 400   Attempts to build and run the current revision with and without the
 401   current patch, with the parameters passed in.
 402   """
 403   # Bisect script expects to be run from the src directory
 404   os.chdir(SRC_DIR)
 405
 406   opts = _CreateBisectOptionsFromConfig(config)
 407   revisions = _ResolveRevisionsFromConfig(config)
 408   annotations_dict = _GetStepAnnotationStringsDict(config)
 409   b = bisect_perf_regression.BisectPerformanceMetrics(opts, os.getcwd())
 410
 411   _RunBuildStepForPerformanceTest(b,
 412                                   annotations_dict.get('build1'),
 413                                   annotations_dict.get('sync1'),
 414                                   revisions[0])
 415
 416   results_with_patch = _RunCommandStepForPerformanceTest(
 417       b, opts, True, True, annotations_dict['results_label1'],
 418       annotations_dict['run1'])
 419
 420   bisect_utils.OutputAnnotationStepStart('Reverting Patch')
 421   # TODO: When this is re-written to recipes, this should use bot_update's
 422   # revert mechanism to fully revert the client. But for now, since we know that
 423   # the perf try bot currently only supports src/ and src/third_party/WebKit, we
 424   # simply reset those two directories.
 425   bisect_utils.CheckRunGit(['reset', '--hard'])
 426   bisect_utils.CheckRunGit(['reset', '--hard'],
 427                            os.path.join('third_party', 'WebKit'))
 428   bisect_utils.OutputAnnotationStepClosed()
 429
 430   _RunBuildStepForPerformanceTest(b,
 431                                   annotations_dict.get('build2'),
 432                                   annotations_dict.get('sync2'),
 433                                   revisions[1])
 434
 435   results_without_patch = _RunCommandStepForPerformanceTest(
 436       b, opts, False, True, annotations_dict['results_label2'],
 437       annotations_dict['run2'])
 438
 439   # Find the link to the cloud stored results file.
 440   _ParseAndOutputCloudLinks(results_without_patch, results_with_patch)
 441
 442
 443 def _SetupAndRunPerformanceTest(config, path_to_goma):
 444   """Attempts to build and run the current revision with and without the
 445   current patch, with the parameters passed in.
 446
 447   Args:
 448     config: The config read from run-perf-test.cfg.
 449     path_to_goma: Path to goma directory.
 450
 451   Returns:
 452     An exit code: 0 on success, otherwise 1.
 453   """
 454   if platform.release() == 'XP':
 455     print 'Windows XP is not supported for perf try jobs because it lacks '
 456     print 'goma support. Please refer to crbug.com/330900.'
 457     return 1
 458   try:
 459     with Goma(path_to_goma) as _:
 460       config['use_goma'] = bool(path_to_goma)
 461       if config['use_goma']:
 462         config['goma_dir'] = os.path.abspath(path_to_goma)
 463       _RunPerformanceTest(config)
 464     return 0
 465   except RuntimeError, e:
 466     bisect_utils.OutputAnnotationStepClosed()
 467     _OutputFailedResults('Error: %s' % e.message)
 468     return 1
 469
 470
 471 def _RunBisectionScript(
 472     config, working_directory, path_to_goma, path_to_extra_src, dry_run):
 473   """Attempts to execute the bisect script with the given parameters.
 474
 475   Args:
 476     config: A dict containing the parameters to pass to the script.
 477     working_directory: A working directory to provide to the bisect script,
 478       where it will store it's own copy of the depot.
 479     path_to_goma: Path to goma directory.
 480     path_to_extra_src: Path to extra source file.
 481     dry_run: Do a dry run, skipping sync, build, and performance testing steps.
 482
 483   Returns:
 484     An exit status code: 0 on success, otherwise 1.
 485   """
 486   _PrintConfigStep(config)
 487
 488   # Construct the basic command with all necessary arguments.
 489   cmd = [
 490       'python',
 491       os.path.join(BISECT_SCRIPT_DIR, 'bisect_perf_regression.py'),
 492       '--command', config['command'],
 493       '--good_revision', config['good_revision'],
 494       '--bad_revision', config['bad_revision'],
 495       '--metric', config['metric'],
 496       '--working_directory', working_directory,
 497       '--output_buildbot_annotations'
 498   ]
 499
 500   # Add flags for any optional config parameters if given in the config.
 501   options = [
 502       ('repeat_count', '--repeat_test_count'),
 503       ('truncate_percent', '--truncate_percent'),
 504       ('max_time_minutes', '--max_time_minutes'),
 505       ('bisect_mode', '--bisect_mode'),
 506       ('improvement_direction', '--improvement_direction'),
 507       ('bug_id', '--bug_id'),
 508       ('builder_host', '--builder_host'),
 509       ('builder_port', '--builder_port'),
 510   ]
 511   for config_key, flag in options:
 512     if config.has_key(config_key):
 513       cmd.extend([flag, config[config_key]])
 514
 515   cmd.extend(['--build_preference', 'ninja'])
 516
 517   # Possibly set the target platform name based on the browser name in a
 518   # Telemetry command.
 519   if 'android-chrome-shell' in config['command']:
 520     cmd.extend(['--target_platform', 'android'])
 521   elif 'android-chrome' in config['command']:
 522     cmd.extend(['--target_platform', 'android-chrome'])
 523   elif 'android' in config['command']:
 524     cmd.extend(['--target_platform', 'android'])
 525
 526   if path_to_goma:
 527     # For Windows XP platforms, goma service is not supported.
 528     # Moreover we don't compile chrome when gs_bucket flag is set instead
 529     # use builds archives, therefore ignore goma service for Windows XP.
 530     # See http://crbug.com/330900.
 531     if platform.release() == 'XP':
 532       print ('Goma doesn\'t have a win32 binary, therefore it is not supported '
 533              'on Windows XP platform. Please refer to crbug.com/330900.')
 534       path_to_goma = None
 535     cmd.append('--use_goma')
 536
 537   if path_to_extra_src:
 538     cmd.extend(['--extra_src', path_to_extra_src])
 539
 540   if dry_run:
 541     cmd.extend([
 542         '--debug_ignore_build',
 543         '--debug_ignore_sync',
 544         '--debug_ignore_perf_test'
 545     ])
 546
 547   cmd = [str(c) for c in cmd]
 548
 549   with Goma(path_to_goma) as _:
 550     return_code = subprocess.call(cmd)
 551
 552   if return_code:
 553     print ('Error: bisect_perf_regression.py returned with error %d\n'
 554            % return_code)
 555
 556   return return_code
 557
 558
 559 def _PrintConfigStep(config):
 560   """Prints out the given config, along with Buildbot annotations."""
 561   bisect_utils.OutputAnnotationStepStart('Config')
 562   print
 563   for k, v in config.iteritems():
 564     print '  %s : %s' % (k, v)
 565   print
 566   bisect_utils.OutputAnnotationStepClosed()
 567
 568
 569 def _OptionParser():
 570   """Returns the options parser for run-bisect-perf-regression.py."""
 571   usage = ('%prog [options] [-- chromium-options]\n'
 572            'Used by a try bot to run the bisection script using the parameters'
 573            ' provided in the auto_bisect/bisect.cfg file.')
 574   parser = optparse.OptionParser(usage=usage)
 575   parser.add_option('-w', '--working_directory',
 576                     type='str',
 577                     help='A working directory to supply to the bisection '
 578                     'script, which will use it as the location to checkout '
 579                     'a copy of the chromium depot.')
 580   parser.add_option('-p', '--path_to_goma',
 581                     type='str',
 582                     help='Path to goma directory. If this is supplied, goma '
 583                     'builds will be enabled.')
 584   parser.add_option('--path_to_config',
 585                     type='str',
 586                     help='Path to the config file to use. If this is supplied, '
 587                     'the bisect script will use this to override the default '
 588                     'config file path. The script will attempt to load it '
 589                     'as a bisect config first, then a perf config.')
 590   parser.add_option('--extra_src',
 591                     type='str',
 592                     help='Path to extra source file. If this is supplied, '
 593                     'bisect script will use this to override default behavior.')
 594   parser.add_option('--dry_run',
 595                     action="store_true",
 596                     help='The script will perform the full bisect, but '
 597                     'without syncing, building, or running the performance '
 598                     'tests.')
 599   return parser
 600
 601
 602 def main():
 603   """Entry point for run-bisect-perf-regression.py.
 604
 605   Reads the config file, and then tries to either bisect a regression or
 606   just run a performance test, depending on the particular config parameters
 607   specified in the config file.
 608   """
 609   parser = _OptionParser()
 610   opts, _ = parser.parse_args()
 611
 612   # Use the default config file path unless one was specified.
 613   config_path = BISECT_CONFIG_PATH
 614   if opts.path_to_config:
 615     config_path = opts.path_to_config
 616   config = _LoadConfigFile(config_path)
 617
 618   # Check if the config is valid for running bisect job.
 619   config_is_valid = _ValidateBisectConfigFile(config)
 620
 621   if config and config_is_valid:
 622     if not opts.working_directory:
 623       print 'Error: missing required parameter: --working_directory\n'
 624       parser.print_help()
 625       return 1
 626
 627     return _RunBisectionScript(
 628         config, opts.working_directory, opts.path_to_goma, opts.extra_src,
 629         opts.dry_run)
 630
 631   # If it wasn't valid for running a bisect, then maybe the user wanted
 632   # to run a perf test instead of a bisect job. Try reading any possible
 633   # perf test config files.
 634   perf_cfg_files = [RUN_TEST_CONFIG_PATH, WEBKIT_RUN_TEST_CONFIG_PATH]
 635   for current_perf_cfg_file in perf_cfg_files:
 636     if opts.path_to_config:
 637       path_to_perf_cfg = opts.path_to_config
 638     else:
 639       path_to_perf_cfg = os.path.join(
 640           os.path.abspath(os.path.dirname(sys.argv[0])),
 641           current_perf_cfg_file)
 642
 643     config = _LoadConfigFile(path_to_perf_cfg)
 644     config_is_valid = _ValidatePerfConfigFile(config)
 645
 646     if config and config_is_valid:
 647       return _SetupAndRunPerformanceTest(config, opts.path_to_goma)
 648
 649   print ('Error: Could not load config file. Double check your changes to '
 650          'auto_bisect/bisect.cfg or run-perf-test.cfg for syntax errors.\n')
 651   return 1
 652
 653
 654 if __name__ == '__main__':
 655   sys.exit(main())