tools/auto_bisect/bisect_perf_regression_test.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import os
   6 import re
   7 import shutil
   8 import sys
   9 import unittest
  10
  11 SRC = os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir)
  12 sys.path.append(os.path.join(SRC, 'third_party', 'pymock'))
  13
  14 import bisect_perf_regression
  15 import bisect_utils
  16 import fetch_build
  17 import mock
  18 import source_control
  19
  20
  21 # Regression confidence: 0%
  22 CLEAR_NON_REGRESSION = [
  23     # Mean: 30.223 Std. Dev.: 11.383
  24     [[16.886], [16.909], [16.99], [17.723], [17.952], [18.118], [19.028],
  25      [19.552], [21.954], [38.573], [38.839], [38.965], [40.007], [40.572],
  26      [41.491], [42.002], [42.33], [43.109], [43.238]],
  27     # Mean: 34.76 Std. Dev.: 11.516
  28     [[16.426], [17.347], [20.593], [21.177], [22.791], [27.843], [28.383],
  29      [28.46], [29.143], [40.058], [40.303], [40.558], [41.918], [42.44],
  30      [45.223], [46.494], [50.002], [50.625], [50.839]]
  31 ]
  32
  33 # Regression confidence: ~ 90%
  34 ALMOST_REGRESSION = [
  35     # Mean: 30.042 Std. Dev.: 2.002
  36     [[26.146], [28.04], [28.053], [28.074], [28.168], [28.209], [28.471],
  37      [28.652], [28.664], [30.862], [30.973], [31.002], [31.897], [31.929],
  38      [31.99], [32.214], [32.323], [32.452], [32.696]],
  39     # Mean: 33.008 Std. Dev.: 4.265
  40     [[34.963], [30.741], [39.677], [39.512], [34.314], [31.39], [34.361],
  41      [25.2], [30.489], [29.434]]
  42 ]
  43
  44 # Regression confidence: ~ 98%
  45 BARELY_REGRESSION = [
  46     # Mean: 28.828 Std. Dev.: 1.993
  47     [[26.96], [27.605], [27.768], [27.829], [28.006], [28.206], [28.393],
  48      [28.911], [28.933], [30.38], [30.462], [30.808], [31.74], [31.805],
  49      [31.899], [32.077], [32.454], [32.597], [33.155]],
  50     # Mean: 31.156 Std. Dev.: 1.980
  51     [[28.729], [29.112], [29.258], [29.454], [29.789], [30.036], [30.098],
  52      [30.174], [30.534], [32.285], [32.295], [32.552], [32.572], [32.967],
  53      [33.165], [33.403], [33.588], [33.744], [34.147], [35.84]]
  54 ]
  55
  56 # Regression confidence: 99.5%
  57 CLEAR_REGRESSION = [
  58     # Mean: 30.254 Std. Dev.: 2.987
  59     [[26.494], [26.621], [26.701], [26.997], [26.997], [27.05], [27.37],
  60      [27.488], [27.556], [31.846], [32.192], [32.21], [32.586], [32.596],
  61      [32.618], [32.95], [32.979], [33.421], [33.457], [34.97]],
  62     # Mean: 33.190 Std. Dev.: 2.972
  63     [[29.547], [29.713], [29.835], [30.132], [30.132], [30.33], [30.406],
  64      [30.592], [30.72], [34.486], [35.247], [35.253], [35.335], [35.378],
  65      [35.934], [36.233], [36.41], [36.947], [37.982]]
  66 ]
  67
  68 # Regression confidence > 95%, taken from: crbug.com/434318
  69 # Specifically from Builder android_nexus10_perf_bisect Build #1198
  70 MULTIPLE_VALUES = [
  71     [
  72         [18.916, 22.371, 8.527, 5.877, 5.407, 9.476, 8.100, 5.334,
  73         4.507, 4.842, 8.485, 8.308, 27.490, 4.560, 4.804, 23.068, 17.577,
  74         17.346, 26.738, 60.330, 32.307, 5.468, 27.803, 27.373, 17.823,
  75         5.158, 27.439, 5.236, 11.413],
  76         [18.999, 22.642, 8.158, 5.995, 5.495, 9.499, 8.092, 5.324,
  77         4.468, 4.788, 8.248, 7.853, 27.533, 4.410, 4.622, 22.341, 22.313,
  78         17.072, 26.731, 57.513, 33.001, 5.500, 28.297, 27.277, 26.462,
  79         5.009, 27.361, 5.130, 10.955]
  80     ],
  81     [
  82         [18.238, 22.365, 8.555, 5.939, 5.437, 9.463, 7.047, 5.345, 4.517,
  83         4.796, 8.593, 7.901, 27.499, 4.378, 5.040, 4.904, 4.816, 4.828,
  84         4.853, 57.363, 34.184, 5.482, 28.190, 27.290, 26.694, 5.099,
  85         4.905, 5.290, 4.813],
  86         [18.301, 22.522, 8.035, 6.021, 5.565, 9.037, 6.998, 5.321, 4.485,
  87         4.768, 8.397, 7.865, 27.636, 4.640, 5.015, 4.962, 4.933, 4.977,
  88         4.961, 60.648, 34.593, 5.538, 28.454, 27.297, 26.490, 5.099, 5,
  89         5.247, 4.945],
  90         [18.907, 23.368, 8.100, 6.169, 5.621, 9.971, 8.161, 5.331, 4.513,
  91         4.837, 8.255, 7.852, 26.209, 4.388, 5.045, 5.029, 5.032, 4.946,
  92         4.973, 60.334, 33.377, 5.499, 28.275, 27.550, 26.103, 5.108,
  93         4.951, 5.285, 4.910],
  94         [18.715, 23.748, 8.128, 6.148, 5.691, 9.361, 8.106, 5.334, 4.528,
  95         4.965, 8.261, 7.851, 27.282, 4.391, 4.949, 4.981, 4.964, 4.935,
  96         4.933, 60.231, 33.361, 5.489, 28.106, 27.457, 26.648, 5.108,
  97         4.963, 5.272, 4.954]
  98     ]
  99 ]
 100
 101 # Default options for the dry run
 102 DEFAULT_OPTIONS = {
 103     'debug_ignore_build': True,
 104     'debug_ignore_sync': True,
 105     'debug_ignore_perf_test': True,
 106     'debug_ignore_regression_confidence': True,
 107     'command': 'fake_command',
 108     'metric': 'fake/metric',
 109     'good_revision': 280000,
 110     'bad_revision': 280005,
 111 }
 112
 113 # This global is a placeholder for a generator to be defined by the test cases
 114 # that use _MockRunTests.
 115 _MockResultsGenerator = (x for x in [])
 116
 117
 118 def _MockRunTests(*args, **kwargs):
 119   _, _ = args, kwargs
 120   return _FakeTestResult(_MockResultsGenerator.next())
 121
 122
 123 def _FakeTestResult(values):
 124   result_dict = {'mean': 0.0, 'std_err': 0.0, 'std_dev': 0.0, 'values': values}
 125   success_code = 0
 126   return (result_dict, success_code)
 127
 128
 129 def _GetBisectPerformanceMetricsInstance(options_dict):
 130   """Returns an instance of the BisectPerformanceMetrics class."""
 131   opts = bisect_perf_regression.BisectOptions.FromDict(options_dict)
 132   return bisect_perf_regression.BisectPerformanceMetrics(opts, os.getcwd())
 133
 134
 135 def _GetExtendedOptions(improvement_dir, fake_first, ignore_confidence=True):
 136   """Returns the a copy of the default options dict plus some options."""
 137   result = dict(DEFAULT_OPTIONS)
 138   result.update({
 139       'improvement_direction': improvement_dir,
 140       'debug_fake_first_test_mean': fake_first,
 141       'debug_ignore_regression_confidence': ignore_confidence})
 142   return result
 143
 144
 145 def _GenericDryRun(options, print_results=False):
 146   """Performs a dry run of the bisector.
 147
 148   Args:
 149     options: Dictionary containing the options for the bisect instance.
 150     print_results: Boolean telling whether to call FormatAndPrintResults.
 151
 152   Returns:
 153     The results dictionary as returned by the bisect Run method.
 154   """
 155   _AbortIfThereAreStagedChanges()
 156   # Disable rmtree to avoid deleting local trees.
 157   old_rmtree = shutil.rmtree
 158   shutil.rmtree = lambda path, on_error: None
 159   # git reset HEAD may be run during the dry run, which removes staged changes.
 160   try:
 161     bisect_instance = _GetBisectPerformanceMetricsInstance(options)
 162     results = bisect_instance.Run(
 163         bisect_instance.opts.command, bisect_instance.opts.bad_revision,
 164         bisect_instance.opts.good_revision, bisect_instance.opts.metric)
 165
 166     if print_results:
 167       bisect_instance.printer.FormatAndPrintResults(results)
 168
 169     return results
 170   finally:
 171     shutil.rmtree = old_rmtree
 172
 173
 174 def _AbortIfThereAreStagedChanges():
 175   """Exits the test prematurely if there are staged changes."""
 176   # The output of "git status --short" will be an empty string if there are
 177   # no staged changes in the current branch. Untracked files are ignored
 178   # because when running the presubmit on the trybot there are sometimes
 179   # untracked changes to the run-perf-test.cfg and bisect.cfg files.
 180   status_output = bisect_utils.CheckRunGit(
 181       ['status', '--short', '--untracked-files=no'])
 182   if status_output:
 183     print 'There are un-committed changes in the current branch.'
 184     print 'Aborting the tests to avoid destroying local changes. Changes:'
 185     print status_output
 186     sys.exit(1)
 187
 188
 189 class BisectPerfRegressionTest(unittest.TestCase):
 190   """Test case for other functions and classes in bisect-perf-regression.py."""
 191
 192   def setUp(self):
 193     self.cwd = os.getcwd()
 194     os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__),
 195                                           os.path.pardir, os.path.pardir)))
 196
 197   def tearDown(self):
 198     os.chdir(self.cwd)
 199
 200   def testBisectOptionsCanPrintHelp(self):
 201     """Tests that the argument parser can be made and can print help."""
 202     bisect_options = bisect_perf_regression.BisectOptions()
 203     parser = bisect_options._CreateCommandLineParser()
 204     parser.format_help()
 205
 206   def testParseDEPSStringManually(self):
 207     """Tests DEPS parsing."""
 208     deps_file_contents = """
 209     vars = {
 210         'ffmpeg_hash':
 211              '@ac4a9f31fe2610bd146857bbd55d7a260003a888',
 212         'webkit_url':
 213              'https://chromium.googlesource.com/chromium/blink.git',
 214         'git_url':
 215              'https://chromium.googlesource.com',
 216         'webkit_rev':
 217              '@e01ac0a267d1017288bc67fa3c366b10469d8a24',
 218         'angle_revision':
 219              '74697cf2064c0a2c0d7e1b1b28db439286766a05'
 220     }"""
 221
 222     # Should only expect SVN/git revisions to come through, and URLs should be
 223     # filtered out.
 224     expected_vars_dict = {
 225         'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888',
 226         'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24',
 227         'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05'
 228     }
 229     # Testing private function.
 230     # pylint: disable=W0212
 231     vars_dict = bisect_perf_regression._ParseRevisionsFromDEPSFileManually(
 232         deps_file_contents)
 233     self.assertEqual(vars_dict, expected_vars_dict)
 234
 235   def _AssertParseResult(self, expected_values, result_string):
 236     """Asserts some values are parsed from a RESULT line."""
 237     results_template = ('RESULT other_chart: other_trace= 123 count\n'
 238                         'RESULT my_chart: my_trace= %(value)s\n')
 239     results = results_template % {'value': result_string}
 240     metric = ['my_chart', 'my_trace']
 241     # Testing private function.
 242     # pylint: disable=W0212
 243     values = bisect_perf_regression._TryParseResultValuesFromOutput(
 244         metric, results)
 245     self.assertEqual(expected_values, values)
 246
 247   def testTryParseResultValuesFromOutput_WithSingleValue(self):
 248     """Tests result pattern <*>RESULT <graph>: <trace>= <value>"""
 249     self._AssertParseResult([66.88], '66.88 kb')
 250     self._AssertParseResult([66.88], '66.88 ')
 251     self._AssertParseResult([-66.88], '-66.88 kb')
 252     self._AssertParseResult([66], '66 kb')
 253     self._AssertParseResult([0.66], '.66 kb')
 254     self._AssertParseResult([], '. kb')
 255     self._AssertParseResult([], 'aaa kb')
 256
 257   def testTryParseResultValuesFromOutput_WithMultiValue(self):
 258     """Tests result pattern <*>RESULT <graph>: <trace>= [<value>,<value>, ..]"""
 259     self._AssertParseResult([66.88], '[66.88] kb')
 260     self._AssertParseResult([66.88, 99.44], '[66.88, 99.44]kb')
 261     self._AssertParseResult([66.88, 99.44], '[ 66.88, 99.44 ]')
 262     self._AssertParseResult([-66.88, 99.44], '[-66.88, 99.44] kb')
 263     self._AssertParseResult([-66, 99], '[-66,99] kb')
 264     self._AssertParseResult([-66, 99], '[-66,99,] kb')
 265     self._AssertParseResult([-66, 0.99], '[-66,.99] kb')
 266     self._AssertParseResult([], '[] kb')
 267     self._AssertParseResult([], '[-66,abc] kb')
 268
 269   def testTryParseResultValuesFromOutputWithMeanStd(self):
 270     """Tests result pattern <*>RESULT <graph>: <trace>= {<mean, std}"""
 271     self._AssertParseResult([33.22], '{33.22, 3.6} kb')
 272     self._AssertParseResult([33.22], '{33.22, 3.6} kb')
 273     self._AssertParseResult([33.22], '{33.22,3.6}kb')
 274     self._AssertParseResult([33.22], '{33.22,3.6} kb')
 275     self._AssertParseResult([33.22], '{ 33.22,3.6 }kb')
 276     self._AssertParseResult([-33.22], '{-33.22,3.6}kb')
 277     self._AssertParseResult([22], '{22,6}kb')
 278     self._AssertParseResult([.22], '{.22,6}kb')
 279     self._AssertParseResult([], '{.22,6, 44}kb')
 280     self._AssertParseResult([], '{}kb')
 281     self._AssertParseResult([], '{XYZ}kb')
 282
 283   def _AssertCompatibleCommand(
 284       self, expected_command, original_command, revision, target_platform):
 285     """Tests the modification of the command that might be done.
 286
 287     This modification to the command is done in order to get a Telemetry
 288     command that works; before some revisions, the browser name that Telemetry
 289     expects is different in some cases, but we want it to work anyway.
 290
 291     Specifically, only for android:
 292       After r276628, only android-chrome-shell works.
 293       Prior to r274857, only android-chromium-testshell works.
 294       In the range [274857, 276628], both work.
 295     """
 296     bisect_options = bisect_perf_regression.BisectOptions()
 297     bisect_options.output_buildbot_annotations = None
 298     bisect_instance = bisect_perf_regression.BisectPerformanceMetrics(
 299         bisect_options, os.getcwd())
 300     bisect_instance.opts.target_platform = target_platform
 301     git_revision = source_control.ResolveToRevision(
 302         revision, 'chromium', bisect_utils.DEPOT_DEPS_NAME, 100)
 303     depot = 'chromium'
 304     command = bisect_instance.GetCompatibleCommand(
 305         original_command, git_revision, depot)
 306     self.assertEqual(expected_command, command)
 307
 308   def testGetCompatibleCommand_ChangeToTestShell(self):
 309     # For revisions <= r274857, only android-chromium-testshell is used.
 310     self._AssertCompatibleCommand(
 311         'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo',
 312         'tools/perf/run_benchmark -v --browser=android-chrome-shell foo',
 313         274857, 'android')
 314
 315   def testGetCompatibleCommand_ChangeToShell(self):
 316     # For revisions >= r276728, only android-chrome-shell can be used.
 317     self._AssertCompatibleCommand(
 318         'tools/perf/run_benchmark -v --browser=android-chrome-shell foo',
 319         'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo',
 320         276628, 'android')
 321
 322   def testGetCompatibleCommand_NoChange(self):
 323     # For revisions < r276728, android-chromium-testshell can be used.
 324     self._AssertCompatibleCommand(
 325         'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo',
 326         'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo',
 327         274858, 'android')
 328     # For revisions > r274857, android-chrome-shell can be used.
 329     self._AssertCompatibleCommand(
 330         'tools/perf/run_benchmark -v --browser=android-chrome-shell foo',
 331         'tools/perf/run_benchmark -v --browser=android-chrome-shell foo',
 332         274858, 'android')
 333
 334   def testGetCompatibleCommand_NonAndroidPlatform(self):
 335     # In most cases, there's no need to change Telemetry command.
 336     # For revisions >= r276728, only android-chrome-shell can be used.
 337     self._AssertCompatibleCommand(
 338         'tools/perf/run_benchmark -v --browser=release foo',
 339         'tools/perf/run_benchmark -v --browser=release foo',
 340         276628, 'chromium')
 341
 342   # This method doesn't reference self; it fails if an error is thrown.
 343   # pylint: disable=R0201
 344   def testDryRun(self):
 345     """Does a dry run of the bisect script.
 346
 347     This serves as a smoke test to catch errors in the basic execution of the
 348     script.
 349     """
 350     _GenericDryRun(DEFAULT_OPTIONS, True)
 351
 352   def testBisectImprovementDirectionFails(self):
 353     """Dry run of a bisect with an improvement instead of regression."""
 354     # Test result goes from 0 to 100 where higher is better
 355     results = _GenericDryRun(_GetExtendedOptions(1, 100))
 356     self.assertIsNotNone(results.error)
 357     self.assertIn('not a regression', results.error)
 358
 359     # Test result goes from 0 to -100 where lower is better
 360     results = _GenericDryRun(_GetExtendedOptions(-1, -100))
 361     self.assertIsNotNone(results.error)
 362     self.assertIn('not a regression', results.error)
 363
 364   def testBisectImprovementDirectionSucceeds(self):
 365     """Bisects with improvement direction matching regression range."""
 366     # Test result goes from 0 to 100 where lower is better
 367     results = _GenericDryRun(_GetExtendedOptions(-1, 100))
 368     self.assertIsNone(results.error)
 369     # Test result goes from 0 to -100 where higher is better
 370     results = _GenericDryRun(_GetExtendedOptions(1, -100))
 371     self.assertIsNone(results.error)
 372
 373   def _CheckAbortsEarly(self, results):
 374     """Returns True if the bisect job would abort early."""
 375     global _MockResultsGenerator
 376     _MockResultsGenerator = (r for r in results)
 377     bisect_class = bisect_perf_regression.BisectPerformanceMetrics
 378     original_run_tests = bisect_class.RunPerformanceTestAndParseResults
 379     bisect_class.RunPerformanceTestAndParseResults = _MockRunTests
 380
 381     try:
 382       _GenericDryRun(_GetExtendedOptions(0, 0, False))
 383     except StopIteration:
 384       # If StopIteration was raised, that means that the next value after
 385       # the first two values was requested, so the job was not aborted.
 386       return False
 387     finally:
 388       bisect_class.RunPerformanceTestAndParseResults = original_run_tests
 389
 390     # If the job was aborted, there should be a warning about it.
 391     assert [w for w in results.warnings
 392             if 'could not reproduce the regression' in w]
 393     return True
 394
 395   def testBisectStopsOnClearUnclearRegression(self):
 396     self.assertTrue(self._CheckAbortsEarly(CLEAR_NON_REGRESSION))
 397
 398   def testBisectStopsOnClearUnclearRegression(self):
 399     self.assertFalse(self._CheckAbortsEarly(ALMOST_REGRESSION))
 400
 401   def testBisectStopsOnClearUnclearRegression(self):
 402     self.assertFalse(self._CheckAbortsEarly(CLEAR_REGRESSION))
 403
 404   def testBisectStopsOnClearUnclearRegression(self):
 405     self.assertFalse(self._CheckAbortsEarly(BARELY_REGRESSION))
 406
 407   def testBisectStopsOnClearUnclearRegression(self):
 408     self.assertFalse(self._CheckAbortsEarly(MULTIPLE_VALUES))
 409
 410   def testGetCommitPosition(self):
 411     cp_git_rev = '7017a81991de983e12ab50dfc071c70e06979531'
 412     self.assertEqual(291765, source_control.GetCommitPosition(cp_git_rev))
 413
 414     svn_git_rev = 'e6db23a037cad47299a94b155b95eebd1ee61a58'
 415     self.assertEqual(291467, source_control.GetCommitPosition(svn_git_rev))
 416
 417   def testGetCommitPositionForV8(self):
 418     bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS)
 419     v8_rev = '21d700eedcdd6570eff22ece724b63a5eefe78cb'
 420     depot_path = os.path.join(bisect_instance.src_cwd, 'v8')
 421     self.assertEqual(
 422         23634, source_control.GetCommitPosition(v8_rev, depot_path))
 423
 424   def testGetCommitPositionForWebKit(self):
 425     bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS)
 426     wk_rev = 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61'
 427     depot_path = os.path.join(bisect_instance.src_cwd, 'third_party', 'WebKit')
 428     self.assertEqual(
 429         181660, source_control.GetCommitPosition(wk_rev, depot_path))
 430
 431   def testGetCommitPositionForSkia(self):
 432     bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS)
 433     skia_rev = 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61'
 434     depot_path = os.path.join(bisect_instance.src_cwd, 'third_party', 'skia')
 435     # Skia doesn't use commit positions, and GetCommitPosition should
 436     # return None for repos that don't use commit positions.
 437     self.assertIsNone(source_control.GetCommitPosition(skia_rev, depot_path))
 438
 439   def testUpdateDepsContent(self):
 440     bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS)
 441     deps_file = 'DEPS'
 442     # We are intentionally reading DEPS file contents instead of string literal
 443     # with few lines from DEPS because to check if the format we are expecting
 444     # to search is not changed in DEPS content.
 445     # TODO (prasadv): Add a separate test to validate the DEPS contents with the
 446     # format that bisect script expects.
 447     deps_contents = bisect_perf_regression.ReadStringFromFile(deps_file)
 448     deps_key = 'v8_revision'
 449     depot = 'v8'
 450     git_revision = 'a12345789a23456789a123456789a123456789'
 451     updated_content = bisect_instance.UpdateDepsContents(
 452         deps_contents, depot, git_revision, deps_key)
 453     self.assertIsNotNone(updated_content)
 454     ss = re.compile('["\']%s["\']: ["\']%s["\']' % (deps_key, git_revision))
 455     self.assertIsNotNone(re.search(ss, updated_content))
 456
 457   @mock.patch('bisect_utils.RunGClient')
 458   def testSyncToRevisionForChromium(self, mock_RunGClient):
 459     bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS)
 460     mock_RunGClient.return_value = 0
 461     bisect_instance._SyncRevision(
 462         'chromium', 'e6db23a037cad47299a94b155b95eebd1ee61a58', 'gclient')
 463     expected_params = [
 464         'sync',
 465         '--verbose',
 466         '--nohooks',
 467         '--force',
 468         '--delete_unversioned_trees',
 469         '--revision',
 470         'src@e6db23a037cad47299a94b155b95eebd1ee61a58'
 471         ]
 472
 473     mock_RunGClient.assert_called_with(expected_params, cwd=None)
 474
 475   @mock.patch('bisect_utils.RunGit')
 476   def testSyncToRevisionForWebKit(self, mock_RunGit):
 477     bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS)
 478     mock_RunGit.return_value = None, None
 479     bisect_instance._SyncRevision(
 480         'webkit', 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61' , None)
 481     expected_params = ['checkout', 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61']
 482     mock_RunGit.assert_called_with(expected_params)
 483
 484
 485 class DepotDirectoryRegistryTest(unittest.TestCase):
 486
 487   def setUp(self):
 488     self.old_chdir = os.chdir
 489     os.chdir = self.mockChdir
 490     self.old_depot_names = bisect_utils.DEPOT_NAMES
 491     bisect_utils.DEPOT_NAMES = ['mock_depot']
 492     self.old_depot_deps_name = bisect_utils.DEPOT_DEPS_NAME
 493     bisect_utils.DEPOT_DEPS_NAME = {'mock_depot': {'src': 'src/foo'}}
 494
 495     self.registry = bisect_perf_regression.DepotDirectoryRegistry('/mock/src')
 496     self.cur_dir = None
 497
 498   def tearDown(self):
 499     os.chdir = self.old_chdir
 500     bisect_utils.DEPOT_NAMES = self.old_depot_names
 501     bisect_utils.DEPOT_DEPS_NAME = self.old_depot_deps_name
 502
 503   def mockChdir(self, new_dir):
 504     self.cur_dir = new_dir
 505
 506   def testReturnsCorrectResultForChrome(self):
 507     self.assertEqual(self.registry.GetDepotDir('chromium'), '/mock/src')
 508
 509   def testUsesDepotSpecToInitializeRegistry(self):
 510     self.assertEqual(self.registry.GetDepotDir('mock_depot'), '/mock/src/foo')
 511
 512   def testChangedTheDirectory(self):
 513     self.registry.ChangeToDepotDir('mock_depot')
 514     self.assertEqual(self.cur_dir, '/mock/src/foo')
 515
 516
 517 # The tests below test private functions (W0212).
 518 # pylint: disable=W0212
 519 class GitTryJobTestCases(unittest.TestCase):
 520
 521   """Test case for bisect try job."""
 522   def setUp(self):
 523     bisect_utils_patcher = mock.patch('bisect_perf_regression.bisect_utils')
 524     self.mock_bisect_utils = bisect_utils_patcher.start()
 525     self.addCleanup(bisect_utils_patcher.stop)
 526
 527   def _SetupRunGitMock(self, git_cmds):
 528     """Setup RunGit mock with expected output for given git command."""
 529     def side_effect(git_cmd_args):
 530       for val in git_cmds:
 531         if set(val[0]) == set(git_cmd_args):
 532           return val[1]
 533     self.mock_bisect_utils.RunGit = mock.Mock(side_effect=side_effect)
 534
 535   def _AssertRunGitExceptions(self, git_cmds, func, *args):
 536     """Setup RunGit mock and tests RunGitException.
 537
 538     Args:
 539       git_cmds: List of tuples with git command and expected output.
 540       func: Callback function to be executed.
 541       args: List of arguments to be passed to the function.
 542     """
 543     self._SetupRunGitMock(git_cmds)
 544     self.assertRaises(bisect_perf_regression.RunGitError,
 545                       func,
 546                       *args)
 547
 548   def testNotGitRepo(self):
 549     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 550     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 551     cmds = [(['rev-parse', '--abbrev-ref', 'HEAD'], (None, 128))]
 552     self._AssertRunGitExceptions(cmds,
 553                                  bisect_perf_regression._PrepareBisectBranch,
 554                                  parent_branch, new_branch)
 555
 556   def testFailedCheckoutMaster(self):
 557     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 558     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 559     cmds = [
 560         (['rev-parse', '--abbrev-ref', 'HEAD'], (new_branch, 0)),
 561         (['checkout', '-f', parent_branch], ('Checkout Failed', 1)),
 562     ]
 563     self._AssertRunGitExceptions(cmds,
 564                                  bisect_perf_regression._PrepareBisectBranch,
 565                                  parent_branch, new_branch)
 566
 567   def testDeleteBisectBranchIfExists(self):
 568     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 569     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 570     cmds = [
 571         (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
 572         (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)),
 573         (['branch', '-D', new_branch], ('Failed to delete branch', 128)),
 574     ]
 575     self._AssertRunGitExceptions(cmds,
 576                                  bisect_perf_regression._PrepareBisectBranch,
 577                                  parent_branch, new_branch)
 578
 579   def testCreatNewBranchFails(self):
 580     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 581     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 582     cmds = [
 583         (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
 584         (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)),
 585         (['branch', '-D', new_branch], ('None', 0)),
 586         (['update-index', '--refresh', '-q'], (None, 0)),
 587         (['diff-index', 'HEAD'], (None, 0)),
 588         (['checkout', '-b', new_branch], ('Failed to create branch', 128)),
 589     ]
 590     self._AssertRunGitExceptions(cmds,
 591                                  bisect_perf_regression._PrepareBisectBranch,
 592                                  parent_branch, new_branch)
 593
 594   def testSetUpstreamToFails(self):
 595     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 596     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 597     cmds = [
 598         (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
 599         (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)),
 600         (['branch', '-D', new_branch], ('None', 0)),
 601         (['update-index', '--refresh', '-q'], (None, 0)),
 602         (['diff-index', 'HEAD'], (None, 0)),
 603         (['checkout', '-b', new_branch], ('None', 0)),
 604         (['branch', '--set-upstream-to', parent_branch],
 605          ('Setuptream fails', 1)),
 606     ]
 607     self._AssertRunGitExceptions(cmds,
 608                                  bisect_perf_regression._PrepareBisectBranch,
 609                                  parent_branch, new_branch)
 610
 611   def testStartBuilderTryJobForException(self):
 612     git_revision = 'ac4a9f31fe2610bd146857bbd55d7a260003a888'
 613     bot_name = 'linux_perf_bisect_builder'
 614     bisect_job_name = 'testBisectJobname'
 615     patch = None
 616     patch_content = '/dev/null'
 617     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 618     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 619     try_cmd = [
 620         (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
 621         (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)),
 622         (['branch', '-D', new_branch], ('None', 0)),
 623         (['update-index', '--refresh', '-q'], (None, 0)),
 624         (['diff-index', 'HEAD'], (None, 0)),
 625         (['checkout', '-b', new_branch], ('None', 0)),
 626         (['branch', '--set-upstream-to', parent_branch],
 627          ('Setuptream fails', 0)),
 628         (['try',
 629           '--bot=%s' % bot_name,
 630           '--revision=%s' % git_revision,
 631           '--name=%s' % bisect_job_name,
 632           '--svn_repo=%s' % bisect_perf_regression.PERF_SVN_REPO_URL,
 633           '--diff=%s' % patch_content
 634          ], (None, 1)),
 635     ]
 636     self._AssertRunGitExceptions(
 637         try_cmd, bisect_perf_regression._StartBuilderTryJob,
 638         fetch_build.PERF_BUILDER, git_revision, bot_name, bisect_job_name,
 639         patch)
 640
 641   def testBuilderTryJob(self):
 642     git_revision = 'ac4a9f31fe2610bd146857bbd55d7a260003a888'
 643     bot_name = 'linux_perf_bisect_builder'
 644     bisect_job_name = 'testBisectJobname'
 645     patch = None
 646     patch_content = '/dev/null'
 647     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 648     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 649     try_cmd = [
 650         (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
 651         (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)),
 652         (['branch', '-D', new_branch], ('None', 0)),
 653         (['update-index', '--refresh', '-q'], (None, 0)),
 654         (['diff-index', 'HEAD'], (None, 0)),
 655         (['checkout', '-b', new_branch], ('None', 0)),
 656         (['branch', '--set-upstream-to', parent_branch],
 657          ('Setuptream fails', 0)),
 658         (['try',
 659           '--bot=%s' % bot_name,
 660           '--revision=%s' % git_revision,
 661           '--name=%s' % bisect_job_name,
 662           '--svn_repo=%s' % bisect_perf_regression.PERF_SVN_REPO_URL,
 663           '--diff=%s' % patch_content
 664          ], (None, 0)),
 665     ]
 666     self._SetupRunGitMock(try_cmd)
 667     bisect_perf_regression._StartBuilderTryJob(
 668         fetch_build.PERF_BUILDER, git_revision, bot_name, bisect_job_name,
 669         patch)
 670
 671
 672 if __name__ == '__main__':
 673   unittest.main()