tools/auto_bisect/bisect_perf_regression_test.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import os
   6 import re
   7 import shutil
   8 import sys
   9 import unittest
  10
  11 SRC = os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir)
  12 sys.path.append(os.path.join(SRC, 'third_party', 'pymock'))
  13
  14 import bisect_perf_regression
  15 import bisect_utils
  16 import mock
  17 import source_control
  18
  19
  20 # Regression confidence: 0%
  21 CLEAR_NON_REGRESSION = [
  22     # Mean: 30.223 Std. Dev.: 11.383
  23     [[16.886], [16.909], [16.99], [17.723], [17.952], [18.118], [19.028],
  24      [19.552], [21.954], [38.573], [38.839], [38.965], [40.007], [40.572],
  25      [41.491], [42.002], [42.33], [43.109], [43.238]],
  26     # Mean: 34.76 Std. Dev.: 11.516
  27     [[16.426], [17.347], [20.593], [21.177], [22.791], [27.843], [28.383],
  28      [28.46], [29.143], [40.058], [40.303], [40.558], [41.918], [42.44],
  29      [45.223], [46.494], [50.002], [50.625], [50.839]]
  30 ]
  31 # Regression confidence: ~ 90%
  32 ALMOST_REGRESSION = [
  33     # Mean: 30.042 Std. Dev.: 2.002
  34     [[26.146], [28.04], [28.053], [28.074], [28.168], [28.209], [28.471],
  35      [28.652], [28.664], [30.862], [30.973], [31.002], [31.897], [31.929],
  36      [31.99], [32.214], [32.323], [32.452], [32.696]],
  37     # Mean: 33.008 Std. Dev.: 4.265
  38     [[34.963], [30.741], [39.677], [39.512], [34.314], [31.39], [34.361],
  39      [25.2], [30.489], [29.434]]
  40 ]
  41 # Regression confidence: ~ 98%
  42 BARELY_REGRESSION = [
  43     # Mean: 28.828 Std. Dev.: 1.993
  44     [[26.96], [27.605], [27.768], [27.829], [28.006], [28.206], [28.393],
  45      [28.911], [28.933], [30.38], [30.462], [30.808], [31.74], [31.805],
  46      [31.899], [32.077], [32.454], [32.597], [33.155]],
  47     # Mean: 31.156 Std. Dev.: 1.980
  48     [[28.729], [29.112], [29.258], [29.454], [29.789], [30.036], [30.098],
  49      [30.174], [30.534], [32.285], [32.295], [32.552], [32.572], [32.967],
  50      [33.165], [33.403], [33.588], [33.744], [34.147], [35.84]]
  51 ]
  52 # Regression confidence: 99.5%
  53 CLEAR_REGRESSION = [
  54     # Mean: 30.254 Std. Dev.: 2.987
  55     [[26.494], [26.621], [26.701], [26.997], [26.997], [27.05], [27.37],
  56      [27.488], [27.556], [31.846], [32.192], [32.21], [32.586], [32.596],
  57      [32.618], [32.95], [32.979], [33.421], [33.457], [34.97]],
  58     # Mean: 33.190 Std. Dev.: 2.972
  59     [[29.547], [29.713], [29.835], [30.132], [30.132], [30.33], [30.406],
  60      [30.592], [30.72], [34.486], [35.247], [35.253], [35.335], [35.378],
  61      [35.934], [36.233], [36.41], [36.947], [37.982]]
  62 ]
  63 # Default options for the dry run
  64 DEFAULT_OPTIONS = {
  65     'debug_ignore_build': True,
  66     'debug_ignore_sync': True,
  67     'debug_ignore_perf_test': True,
  68     'debug_ignore_regression_confidence': True,
  69     'command': 'fake_command',
  70     'metric': 'fake/metric',
  71     'good_revision': 280000,
  72     'bad_revision': 280005,
  73 }
  74
  75 # This global is a placeholder for a generator to be defined by the testcases
  76 # that use _MockRunTest
  77 _MockResultsGenerator = (x for x in [])
  78
  79 def _FakeTestResult(values):
  80   result_dict = {'mean': 0.0, 'std_err': 0.0, 'std_dev': 0.0, 'values': values}
  81   success_code = 0
  82   return (result_dict, success_code)
  83
  84
  85 def _MockRunTests(*args, **kwargs):
  86   _, _ = args, kwargs
  87   return _FakeTestResult(_MockResultsGenerator.next())
  88
  89
  90 def _GetBisectPerformanceMetricsInstance(options_dict):
  91   """Returns an instance of the BisectPerformanceMetrics class."""
  92   opts = bisect_perf_regression.BisectOptions.FromDict(options_dict)
  93   return bisect_perf_regression.BisectPerformanceMetrics(opts, os.getcwd())
  94
  95
  96 def _GetExtendedOptions(improvement_dir, fake_first, ignore_confidence=True):
  97   """Returns the a copy of the default options dict plus some options."""
  98   result = dict(DEFAULT_OPTIONS)
  99   result.update({
 100       'improvement_direction': improvement_dir,
 101       'debug_fake_first_test_mean': fake_first,
 102       'debug_ignore_regression_confidence': ignore_confidence})
 103   return result
 104
 105
 106 def _GenericDryRun(options, print_results=False):
 107   """Performs a dry run of the bisector.
 108
 109   Args:
 110     options: Dictionary containing the options for the bisect instance.
 111     print_results: Boolean telling whether to call FormatAndPrintResults.
 112
 113   Returns:
 114     The results dictionary as returned by the bisect Run method.
 115   """
 116   # Disable rmtree to avoid deleting local trees.
 117   old_rmtree = shutil.rmtree
 118   try:
 119     shutil.rmtree = lambda path, onerror: None
 120     bisect_instance = _GetBisectPerformanceMetricsInstance(options)
 121     results = bisect_instance.Run(
 122         bisect_instance.opts.command, bisect_instance.opts.bad_revision,
 123         bisect_instance.opts.good_revision, bisect_instance.opts.metric)
 124
 125     if print_results:
 126       bisect_instance.printer.FormatAndPrintResults(results)
 127
 128     return results
 129   finally:
 130     shutil.rmtree = old_rmtree
 131
 132
 133 class BisectPerfRegressionTest(unittest.TestCase):
 134   """Test case for other functions and classes in bisect-perf-regression.py."""
 135
 136   def setUp(self):
 137     self.cwd = os.getcwd()
 138     os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__),
 139                                           os.path.pardir, os.path.pardir)))
 140
 141   def tearDown(self):
 142     os.chdir(self.cwd)
 143
 144   def testParseDEPSStringManually(self):
 145     """Tests DEPS parsing."""
 146     deps_file_contents = """
 147     vars = {
 148         'ffmpeg_hash':
 149              '@ac4a9f31fe2610bd146857bbd55d7a260003a888',
 150         'webkit_url':
 151              'https://chromium.googlesource.com/chromium/blink.git',
 152         'git_url':
 153              'https://chromium.googlesource.com',
 154         'webkit_rev':
 155              '@e01ac0a267d1017288bc67fa3c366b10469d8a24',
 156         'angle_revision':
 157              '74697cf2064c0a2c0d7e1b1b28db439286766a05'
 158     }"""
 159
 160     # Should only expect SVN/git revisions to come through, and URLs should be
 161     # filtered out.
 162     expected_vars_dict = {
 163         'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888',
 164         'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24',
 165         'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05'
 166     }
 167     # Testing private function.
 168     # pylint: disable=W0212
 169     vars_dict = bisect_perf_regression._ParseRevisionsFromDEPSFileManually(
 170         deps_file_contents)
 171     self.assertEqual(vars_dict, expected_vars_dict)
 172
 173   def _AssertParseResult(self, expected_values, result_string):
 174     """Asserts some values are parsed from a RESULT line."""
 175     results_template = ('RESULT other_chart: other_trace= 123 count\n'
 176                         'RESULT my_chart: my_trace= %(value)s\n')
 177     results = results_template % {'value': result_string}
 178     metric = ['my_chart', 'my_trace']
 179     # Testing private function.
 180     # pylint: disable=W0212
 181     values = bisect_perf_regression._TryParseResultValuesFromOutput(
 182         metric, results)
 183     self.assertEqual(expected_values, values)
 184
 185   def testTryParseResultValuesFromOutput_WithSingleValue(self):
 186     """Tests result pattern <*>RESULT <graph>: <trace>= <value>"""
 187     self._AssertParseResult([66.88], '66.88 kb')
 188     self._AssertParseResult([66.88], '66.88 ')
 189     self._AssertParseResult([-66.88], '-66.88 kb')
 190     self._AssertParseResult([66], '66 kb')
 191     self._AssertParseResult([0.66], '.66 kb')
 192     self._AssertParseResult([], '. kb')
 193     self._AssertParseResult([], 'aaa kb')
 194
 195   def testTryParseResultValuesFromOutput_WithMultiValue(self):
 196     """Tests result pattern <*>RESULT <graph>: <trace>= [<value>,<value>, ..]"""
 197     self._AssertParseResult([66.88], '[66.88] kb')
 198     self._AssertParseResult([66.88, 99.44], '[66.88, 99.44]kb')
 199     self._AssertParseResult([66.88, 99.44], '[ 66.88, 99.44 ]')
 200     self._AssertParseResult([-66.88, 99.44], '[-66.88, 99.44] kb')
 201     self._AssertParseResult([-66, 99], '[-66,99] kb')
 202     self._AssertParseResult([-66, 99], '[-66,99,] kb')
 203     self._AssertParseResult([-66, 0.99], '[-66,.99] kb')
 204     self._AssertParseResult([], '[] kb')
 205     self._AssertParseResult([], '[-66,abc] kb')
 206
 207   def testTryParseResultValuesFromOutputWithMeanStd(self):
 208     """Tests result pattern <*>RESULT <graph>: <trace>= {<mean, std}"""
 209     self._AssertParseResult([33.22], '{33.22, 3.6} kb')
 210     self._AssertParseResult([33.22], '{33.22, 3.6} kb')
 211     self._AssertParseResult([33.22], '{33.22,3.6}kb')
 212     self._AssertParseResult([33.22], '{33.22,3.6} kb')
 213     self._AssertParseResult([33.22], '{ 33.22,3.6 }kb')
 214     self._AssertParseResult([-33.22], '{-33.22,3.6}kb')
 215     self._AssertParseResult([22], '{22,6}kb')
 216     self._AssertParseResult([.22], '{.22,6}kb')
 217     self._AssertParseResult([], '{.22,6, 44}kb')
 218     self._AssertParseResult([], '{}kb')
 219     self._AssertParseResult([], '{XYZ}kb')
 220
 221   def _AssertCompatibleCommand(
 222       self, expected_command, original_command, revision, target_platform):
 223     """Tests the modification of the command that might be done.
 224
 225     This modification to the command is done in order to get a Telemetry
 226     command that works; before some revisions, the browser name that Telemetry
 227     expects is different in some cases, but we want it to work anyway.
 228
 229     Specifically, only for android:
 230       After r276628, only android-chrome-shell works.
 231       Prior to r274857, only android-chromium-testshell works.
 232       In the range [274857, 276628], both work.
 233     """
 234     bisect_options = bisect_perf_regression.BisectOptions()
 235     bisect_options.output_buildbot_annotations = None
 236     bisect_instance = bisect_perf_regression.BisectPerformanceMetrics(
 237         bisect_options, os.getcwd())
 238     bisect_instance.opts.target_platform = target_platform
 239     git_revision = source_control.ResolveToRevision(
 240         revision, 'chromium', bisect_utils.DEPOT_DEPS_NAME, 100)
 241     depot = 'chromium'
 242     command = bisect_instance.GetCompatibleCommand(
 243         original_command, git_revision, depot)
 244     self.assertEqual(expected_command, command)
 245
 246   def testGetCompatibleCommand_ChangeToTestShell(self):
 247     # For revisions <= r274857, only android-chromium-testshell is used.
 248     self._AssertCompatibleCommand(
 249         'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo',
 250         'tools/perf/run_benchmark -v --browser=android-chrome-shell foo',
 251         274857, 'android')
 252
 253   def testGetCompatibleCommand_ChangeToShell(self):
 254     # For revisions >= r276728, only android-chrome-shell can be used.
 255     self._AssertCompatibleCommand(
 256         'tools/perf/run_benchmark -v --browser=android-chrome-shell foo',
 257         'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo',
 258         276628, 'android')
 259
 260   def testGetCompatibleCommand_NoChange(self):
 261     # For revisions < r276728, android-chromium-testshell can be used.
 262     self._AssertCompatibleCommand(
 263         'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo',
 264         'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo',
 265         274858, 'android')
 266     # For revisions > r274857, android-chrome-shell can be used.
 267     self._AssertCompatibleCommand(
 268         'tools/perf/run_benchmark -v --browser=android-chrome-shell foo',
 269         'tools/perf/run_benchmark -v --browser=android-chrome-shell foo',
 270         274858, 'android')
 271
 272   def testGetCompatibleCommand_NonAndroidPlatform(self):
 273     # In most cases, there's no need to change Telemetry command.
 274     # For revisions >= r276728, only android-chrome-shell can be used.
 275     self._AssertCompatibleCommand(
 276         'tools/perf/run_benchmark -v --browser=release foo',
 277         'tools/perf/run_benchmark -v --browser=release foo',
 278         276628, 'chromium')
 279
 280   # This method doesn't reference self; it fails if an error is thrown.
 281   # pylint: disable=R0201
 282   def testDryRun(self):
 283     """Does a dry run of the bisect script.
 284
 285     This serves as a smoke test to catch errors in the basic execution of the
 286     script.
 287     """
 288     _GenericDryRun(DEFAULT_OPTIONS, True)
 289
 290   def testBisectImprovementDirectionFails(self):
 291     """Dry run of a bisect with an improvement instead of regression."""
 292     # Test result goes from 0 to 100 where higher is better
 293     results = _GenericDryRun(_GetExtendedOptions(1, 100))
 294     self.assertIsNotNone(results.error)
 295     self.assertIn('not a regression', results.error)
 296
 297     # Test result goes from 0 to -100 where lower is better
 298     results = _GenericDryRun(_GetExtendedOptions(-1, -100))
 299     self.assertIsNotNone(results.error)
 300     self.assertIn('not a regression', results.error)
 301
 302   def testBisectImprovementDirectionSucceeds(self):
 303     """Bisects with improvement direction matching regression range."""
 304     # Test result goes from 0 to 100 where lower is better
 305     results = _GenericDryRun(_GetExtendedOptions(-1, 100))
 306     self.assertIsNone(results.error)
 307     # Test result goes from 0 to -100 where higher is better
 308     results = _GenericDryRun(_GetExtendedOptions(1, -100))
 309     self.assertIsNone(results.error)
 310
 311   @mock.patch('bisect_perf_regression.BisectPerformanceMetrics.'
 312               'RunPerformanceTestAndParseResults', _MockRunTests)
 313   def testBisectStopsOnDoubtfulRegression(self):
 314     global _MockResultsGenerator
 315     _MockResultsGenerator = (rs for rs in CLEAR_NON_REGRESSION)
 316     results = _GenericDryRun(_GetExtendedOptions(0, 0, False))
 317     confidence_warnings = [x for x in results.warnings if x.startswith(
 318         '\nWe could not reproduce the regression')]
 319     self.assertGreater(len(confidence_warnings), 0)
 320
 321     _MockResultsGenerator = (rs for rs in ALMOST_REGRESSION)
 322     results = _GenericDryRun(_GetExtendedOptions(0, 0, False))
 323     confidence_warnings = [x for x in results.warnings if x.startswith(
 324         '\nWe could not reproduce the regression')]
 325     self.assertGreater(len(confidence_warnings), 0)
 326
 327   @mock.patch('bisect_perf_regression.BisectPerformanceMetrics.'
 328               'RunPerformanceTestAndParseResults', _MockRunTests)
 329   def testBisectContinuesOnClearRegression(self):
 330     global _MockResultsGenerator
 331     _MockResultsGenerator = (rs for rs in CLEAR_REGRESSION)
 332     with self.assertRaises(StopIteration):
 333       _GenericDryRun(_GetExtendedOptions(0, 0, False))
 334
 335     _MockResultsGenerator = (rs for rs in BARELY_REGRESSION)
 336     with self.assertRaises(StopIteration):
 337       _GenericDryRun(_GetExtendedOptions(0, 0, False))
 338
 339   def testGetCommitPosition(self):
 340     cp_git_rev = '7017a81991de983e12ab50dfc071c70e06979531'
 341     self.assertEqual(291765, source_control.GetCommitPosition(cp_git_rev))
 342
 343     svn_git_rev = 'e6db23a037cad47299a94b155b95eebd1ee61a58'
 344     self.assertEqual(291467, source_control.GetCommitPosition(svn_git_rev))
 345
 346   def testGetCommitPositionForV8(self):
 347     bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS)
 348     v8_rev = '21d700eedcdd6570eff22ece724b63a5eefe78cb'
 349     depot_path = os.path.join(bisect_instance.src_cwd, 'v8')
 350     self.assertEqual(
 351         23634, source_control.GetCommitPosition(v8_rev, depot_path))
 352
 353   def testGetCommitPositionForWebKit(self):
 354     bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS)
 355     wk_rev = 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61'
 356     depot_path = os.path.join(bisect_instance.src_cwd, 'third_party', 'WebKit')
 357     self.assertEqual(
 358         181660, source_control.GetCommitPosition(wk_rev, depot_path))
 359
 360   def testUpdateDepsContent(self):
 361     bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS)
 362     deps_file = 'DEPS'
 363     # We are intentionally reading DEPS file contents instead of string literal
 364     # with few lines from DEPS because to check if the format we are expecting
 365     # to search is not changed in DEPS content.
 366     # TODO (prasadv): Add a separate test to validate the DEPS contents with the
 367     # format that bisect script expects.
 368     deps_contents = bisect_perf_regression.ReadStringFromFile(deps_file)
 369     deps_key = 'v8_revision'
 370     depot = 'v8'
 371     git_revision = 'a12345789a23456789a123456789a123456789'
 372     updated_content = bisect_instance.UpdateDepsContents(
 373         deps_contents, depot, git_revision, deps_key)
 374     self.assertIsNotNone(updated_content)
 375     ss = re.compile('["\']%s["\']: ["\']%s["\']' % (deps_key, git_revision))
 376     self.assertIsNotNone(re.search(ss, updated_content))
 377
 378
 379 class DepotDirectoryRegistryTest(unittest.TestCase):
 380
 381   def setUp(self):
 382     self.old_chdir = os.chdir
 383     os.chdir = self.mockChdir
 384     self.old_depot_names = bisect_utils.DEPOT_NAMES
 385     bisect_utils.DEPOT_NAMES = ['mock_depot']
 386     self.old_depot_deps_name = bisect_utils.DEPOT_DEPS_NAME
 387     bisect_utils.DEPOT_DEPS_NAME = {'mock_depot': {'src': 'src/foo'}}
 388
 389     self.registry = bisect_perf_regression.DepotDirectoryRegistry('/mock/src')
 390     self.cur_dir = None
 391
 392   def tearDown(self):
 393     os.chdir = self.old_chdir
 394     bisect_utils.DEPOT_NAMES = self.old_depot_names
 395     bisect_utils.DEPOT_DEPS_NAME = self.old_depot_deps_name
 396
 397   def mockChdir(self, new_dir):
 398     self.cur_dir = new_dir
 399
 400   def testReturnsCorrectResultForChrome(self):
 401     self.assertEqual(self.registry.GetDepotDir('chromium'), '/mock/src')
 402
 403   def testUsesDepotSpecToInitializeRegistry(self):
 404     self.assertEqual(self.registry.GetDepotDir('mock_depot'), '/mock/src/foo')
 405
 406   def testChangedTheDirectory(self):
 407     self.registry.ChangeToDepotDir('mock_depot')
 408     self.assertEqual(self.cur_dir, '/mock/src/foo')
 409
 410
 411 # The tests below test private functions (W0212).
 412 # pylint: disable=W0212
 413 class GitTryJobTestCases(unittest.TestCase):
 414
 415   """Test case for bisect try job."""
 416   def setUp(self):
 417     bisect_utils_patcher = mock.patch('bisect_perf_regression.bisect_utils')
 418     self.mock_bisect_utils = bisect_utils_patcher.start()
 419     self.addCleanup(bisect_utils_patcher.stop)
 420
 421   def _SetupRunGitMock(self, git_cmds):
 422     """Setup RunGit mock with expected output for given git command."""
 423     def side_effect(git_cmd_args):
 424       for val in git_cmds:
 425         if set(val[0]) == set(git_cmd_args):
 426           return val[1]
 427     self.mock_bisect_utils.RunGit = mock.Mock(side_effect=side_effect)
 428
 429   def _AssertRunGitExceptions(self, git_cmds, func, *args):
 430     """Setup RunGit mock and tests RunGitException.
 431
 432     Args:
 433       git_cmds: List of tuples with git command and expected output.
 434       func: Callback function to be executed.
 435       args: List of arguments to be passed to the function.
 436     """
 437     self._SetupRunGitMock(git_cmds)
 438     self.assertRaises(bisect_perf_regression.RunGitError,
 439                       func,
 440                       *args)
 441
 442   def testNotGitRepo(self):
 443     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 444     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 445     cmds = [(['rev-parse', '--abbrev-ref', 'HEAD'], (None, 128))]
 446     self._AssertRunGitExceptions(cmds,
 447                                  bisect_perf_regression._PrepareBisectBranch,
 448                                  parent_branch, new_branch)
 449
 450   def testFailedCheckoutMaster(self):
 451     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 452     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 453     cmds = [
 454         (['rev-parse', '--abbrev-ref', 'HEAD'], (new_branch, 0)),
 455         (['checkout', '-f', parent_branch], ('Checkout Failed', 1)),
 456     ]
 457     self._AssertRunGitExceptions(cmds,
 458                                  bisect_perf_regression._PrepareBisectBranch,
 459                                  parent_branch, new_branch)
 460
 461   def testDeleteBisectBranchIfExists(self):
 462     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 463     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 464     cmds = [
 465         (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
 466         (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)),
 467         (['branch', '-D', new_branch], ('Failed to delete branch', 128)),
 468     ]
 469     self._AssertRunGitExceptions(cmds,
 470                                  bisect_perf_regression._PrepareBisectBranch,
 471                                  parent_branch, new_branch)
 472
 473   def testCreatNewBranchFails(self):
 474     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 475     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 476     cmds = [
 477         (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
 478         (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)),
 479         (['branch', '-D', new_branch], ('None', 0)),
 480         (['update-index', '--refresh', '-q'], (None, 0)),
 481         (['diff-index', 'HEAD'], (None, 0)),
 482         (['checkout', '-b', new_branch], ('Failed to create branch', 128)),
 483     ]
 484     self._AssertRunGitExceptions(cmds,
 485                                  bisect_perf_regression._PrepareBisectBranch,
 486                                  parent_branch, new_branch)
 487
 488   def testSetUpstreamToFails(self):
 489     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 490     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 491     cmds = [
 492         (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
 493         (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)),
 494         (['branch', '-D', new_branch], ('None', 0)),
 495         (['update-index', '--refresh', '-q'], (None, 0)),
 496         (['diff-index', 'HEAD'], (None, 0)),
 497         (['checkout', '-b', new_branch], ('None', 0)),
 498         (['branch', '--set-upstream-to', parent_branch],
 499          ('Setuptream fails', 1)),
 500     ]
 501     self._AssertRunGitExceptions(cmds,
 502                                  bisect_perf_regression._PrepareBisectBranch,
 503                                  parent_branch, new_branch)
 504
 505   def testBuilderTryJobForException(self):
 506     git_revision = 'ac4a9f31fe2610bd146857bbd55d7a260003a888'
 507     bot_name = 'linux_perf_bisect_builder'
 508     bisect_job_name = 'testBisectJobname'
 509     patch = None
 510     patch_content = '/dev/null'
 511     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 512     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 513     try_cmd = [
 514         (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
 515         (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)),
 516         (['branch', '-D', new_branch], ('None', 0)),
 517         (['update-index', '--refresh', '-q'], (None, 0)),
 518         (['diff-index', 'HEAD'], (None, 0)),
 519         (['checkout', '-b', new_branch], ('None', 0)),
 520         (['branch', '--set-upstream-to', parent_branch],
 521          ('Setuptream fails', 0)),
 522         (['try',
 523           '-b', bot_name,
 524           '-r', git_revision,
 525           '-n', bisect_job_name,
 526           '--svn_repo=%s' % bisect_perf_regression.SVN_REPO_URL,
 527           '--diff=%s' % patch_content
 528          ], (None, 1)),
 529     ]
 530     self._AssertRunGitExceptions(try_cmd,
 531                                  bisect_perf_regression._BuilderTryjob,
 532                                  git_revision, bot_name, bisect_job_name, patch)
 533
 534   def testBuilderTryJob(self):
 535     git_revision = 'ac4a9f31fe2610bd146857bbd55d7a260003a888'
 536     bot_name = 'linux_perf_bisect_builder'
 537     bisect_job_name = 'testBisectJobname'
 538     patch = None
 539     patch_content = '/dev/null'
 540     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 541     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 542     try_cmd = [
 543         (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
 544         (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)),
 545         (['branch', '-D', new_branch], ('None', 0)),
 546         (['update-index', '--refresh', '-q'], (None, 0)),
 547         (['diff-index', 'HEAD'], (None, 0)),
 548         (['checkout', '-b', new_branch], ('None', 0)),
 549         (['branch', '--set-upstream-to', parent_branch],
 550          ('Setuptream fails', 0)),
 551         (['try',
 552           '-b', bot_name,
 553           '-r', git_revision,
 554           '-n', bisect_job_name,
 555           '--svn_repo=%s' % bisect_perf_regression.SVN_REPO_URL,
 556           '--diff=%s' % patch_content
 557          ], (None, 0)),
 558     ]
 559     self._SetupRunGitMock(try_cmd)
 560     bisect_perf_regression._BuilderTryjob(
 561         git_revision, bot_name, bisect_job_name, patch)
 562
 563
 564 if __name__ == '__main__':
 565   unittest.main()
 566