tools/auto_bisect/bisect_perf_regression_test.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import os
   6 import re
   7 import shutil
   8 import sys
   9 import unittest
  10
  11 SRC = os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir)
  12 sys.path.append(os.path.join(SRC, 'third_party', 'pymock'))
  13
  14 import bisect_perf_regression
  15 import bisect_utils
  16 import fetch_build
  17 import mock
  18 import source_control
  19
  20
  21 # Regression confidence: 0%
  22 CLEAR_NON_REGRESSION = [
  23     # Mean: 30.223 Std. Dev.: 11.383
  24     [[16.886], [16.909], [16.99], [17.723], [17.952], [18.118], [19.028],
  25      [19.552], [21.954], [38.573], [38.839], [38.965], [40.007], [40.572],
  26      [41.491], [42.002], [42.33], [43.109], [43.238]],
  27     # Mean: 34.76 Std. Dev.: 11.516
  28     [[16.426], [17.347], [20.593], [21.177], [22.791], [27.843], [28.383],
  29      [28.46], [29.143], [40.058], [40.303], [40.558], [41.918], [42.44],
  30      [45.223], [46.494], [50.002], [50.625], [50.839]]
  31 ]
  32
  33 # Regression confidence: ~ 90%
  34 ALMOST_REGRESSION = [
  35     # Mean: 30.042 Std. Dev.: 2.002
  36     [[26.146], [28.04], [28.053], [28.074], [28.168], [28.209], [28.471],
  37      [28.652], [28.664], [30.862], [30.973], [31.002], [31.897], [31.929],
  38      [31.99], [32.214], [32.323], [32.452], [32.696]],
  39     # Mean: 33.008 Std. Dev.: 4.265
  40     [[34.963], [30.741], [39.677], [39.512], [34.314], [31.39], [34.361],
  41      [25.2], [30.489], [29.434]]
  42 ]
  43
  44 # Regression confidence: ~ 98%
  45 BARELY_REGRESSION = [
  46     # Mean: 28.828 Std. Dev.: 1.993
  47     [[26.96], [27.605], [27.768], [27.829], [28.006], [28.206], [28.393],
  48      [28.911], [28.933], [30.38], [30.462], [30.808], [31.74], [31.805],
  49      [31.899], [32.077], [32.454], [32.597], [33.155]],
  50     # Mean: 31.156 Std. Dev.: 1.980
  51     [[28.729], [29.112], [29.258], [29.454], [29.789], [30.036], [30.098],
  52      [30.174], [30.534], [32.285], [32.295], [32.552], [32.572], [32.967],
  53      [33.165], [33.403], [33.588], [33.744], [34.147], [35.84]]
  54 ]
  55
  56 # Regression confidence: 99.5%
  57 CLEAR_REGRESSION = [
  58     # Mean: 30.254 Std. Dev.: 2.987
  59     [[26.494], [26.621], [26.701], [26.997], [26.997], [27.05], [27.37],
  60      [27.488], [27.556], [31.846], [32.192], [32.21], [32.586], [32.596],
  61      [32.618], [32.95], [32.979], [33.421], [33.457], [34.97]],
  62     # Mean: 33.190 Std. Dev.: 2.972
  63     [[29.547], [29.713], [29.835], [30.132], [30.132], [30.33], [30.406],
  64      [30.592], [30.72], [34.486], [35.247], [35.253], [35.335], [35.378],
  65      [35.934], [36.233], [36.41], [36.947], [37.982]]
  66 ]
  67
  68 # Regression confidence > 95%, taken from: crbug.com/434318
  69 # Specifically from Builder android_nexus10_perf_bisect Build #1198
  70 MULTIPLE_VALUES = [
  71     [
  72         [18.916, 22.371, 8.527, 5.877, 5.407, 9.476, 8.100, 5.334,
  73          4.507, 4.842, 8.485, 8.308, 27.490, 4.560, 4.804, 23.068, 17.577,
  74          17.346, 26.738, 60.330, 32.307, 5.468, 27.803, 27.373, 17.823,
  75          5.158, 27.439, 5.236, 11.413],
  76         [18.999, 22.642, 8.158, 5.995, 5.495, 9.499, 8.092, 5.324,
  77          4.468, 4.788, 8.248, 7.853, 27.533, 4.410, 4.622, 22.341, 22.313,
  78          17.072, 26.731, 57.513, 33.001, 5.500, 28.297, 27.277, 26.462,
  79          5.009, 27.361, 5.130, 10.955]
  80     ],
  81     [
  82         [18.238, 22.365, 8.555, 5.939, 5.437, 9.463, 7.047, 5.345, 4.517,
  83          4.796, 8.593, 7.901, 27.499, 4.378, 5.040, 4.904, 4.816, 4.828,
  84          4.853, 57.363, 34.184, 5.482, 28.190, 27.290, 26.694, 5.099,
  85          4.905, 5.290, 4.813],
  86         [18.301, 22.522, 8.035, 6.021, 5.565, 9.037, 6.998, 5.321, 4.485,
  87          4.768, 8.397, 7.865, 27.636, 4.640, 5.015, 4.962, 4.933, 4.977,
  88          4.961, 60.648, 34.593, 5.538, 28.454, 27.297, 26.490, 5.099, 5,
  89          5.247, 4.945],
  90         [18.907, 23.368, 8.100, 6.169, 5.621, 9.971, 8.161, 5.331, 4.513,
  91          4.837, 8.255, 7.852, 26.209, 4.388, 5.045, 5.029, 5.032, 4.946,
  92          4.973, 60.334, 33.377, 5.499, 28.275, 27.550, 26.103, 5.108,
  93          4.951, 5.285, 4.910],
  94         [18.715, 23.748, 8.128, 6.148, 5.691, 9.361, 8.106, 5.334, 4.528,
  95          4.965, 8.261, 7.851, 27.282, 4.391, 4.949, 4.981, 4.964, 4.935,
  96          4.933, 60.231, 33.361, 5.489, 28.106, 27.457, 26.648, 5.108,
  97          4.963, 5.272, 4.954]
  98     ]
  99 ]
 100
 101 # Default options for the dry run
 102 DEFAULT_OPTIONS = {
 103     'debug_ignore_build': True,
 104     'debug_ignore_sync': True,
 105     'debug_ignore_perf_test': True,
 106     'debug_ignore_regression_confidence': True,
 107     'command': 'fake_command',
 108     'metric': 'fake/metric',
 109     'good_revision': 280000,
 110     'bad_revision': 280005,
 111 }
 112
 113 # This global is a placeholder for a generator to be defined by the test cases
 114 # that use _MockRunTests.
 115 _MockResultsGenerator = (x for x in [])
 116
 117 def _MakeMockRunTests(bisect_mode_is_return_code=False):
 118   def _MockRunTests(*args, **kwargs):  # pylint: disable=unused-argument
 119     return _FakeTestResult(
 120         _MockResultsGenerator.next(), bisect_mode_is_return_code)
 121
 122   return _MockRunTests
 123
 124
 125 def _FakeTestResult(values, bisect_mode_is_return_code):
 126   mean = 0.0
 127   if bisect_mode_is_return_code:
 128     mean = 0 if (all(v == 0 for v in values)) else 1
 129   result_dict = {'mean': mean, 'std_err': 0.0, 'std_dev': 0.0, 'values': values}
 130   success_code = 0
 131   return (result_dict, success_code)
 132
 133
 134 def _GetBisectPerformanceMetricsInstance(options_dict):
 135   """Returns an instance of the BisectPerformanceMetrics class."""
 136   opts = bisect_perf_regression.BisectOptions.FromDict(options_dict)
 137   return bisect_perf_regression.BisectPerformanceMetrics(opts, os.getcwd())
 138
 139
 140 def _GetExtendedOptions(improvement_dir, fake_first, ignore_confidence=True):
 141   """Returns the a copy of the default options dict plus some options."""
 142   result = dict(DEFAULT_OPTIONS)
 143   result.update({
 144       'improvement_direction': improvement_dir,
 145       'debug_fake_first_test_mean': fake_first,
 146       'debug_ignore_regression_confidence': ignore_confidence})
 147   return result
 148
 149
 150 def _GenericDryRun(options, print_results=False):
 151   """Performs a dry run of the bisector.
 152
 153   Args:
 154     options: Dictionary containing the options for the bisect instance.
 155     print_results: Boolean telling whether to call FormatAndPrintResults.
 156
 157   Returns:
 158     The results dictionary as returned by the bisect Run method.
 159   """
 160   _AbortIfThereAreStagedChanges()
 161   # Disable rmtree to avoid deleting local trees.
 162   old_rmtree = shutil.rmtree
 163   shutil.rmtree = lambda path, on_error: None
 164   # git reset HEAD may be run during the dry run, which removes staged changes.
 165   try:
 166     bisect_instance = _GetBisectPerformanceMetricsInstance(options)
 167     results = bisect_instance.Run(
 168         bisect_instance.opts.command, bisect_instance.opts.bad_revision,
 169         bisect_instance.opts.good_revision, bisect_instance.opts.metric)
 170
 171     if print_results:
 172       bisect_instance.printer.FormatAndPrintResults(results)
 173
 174     return results
 175   finally:
 176     shutil.rmtree = old_rmtree
 177
 178
 179 def _AbortIfThereAreStagedChanges():
 180   """Exits the test prematurely if there are staged changes."""
 181   # The output of "git status --short" will be an empty string if there are
 182   # no staged changes in the current branch. Untracked files are ignored
 183   # because when running the presubmit on the trybot there are sometimes
 184   # untracked changes to the run-perf-test.cfg and bisect.cfg files.
 185   status_output = bisect_utils.CheckRunGit(
 186       ['status', '--short', '--untracked-files=no'])
 187   if status_output:
 188     print 'There are un-committed changes in the current branch.'
 189     print 'Aborting the tests to avoid destroying local changes. Changes:'
 190     print status_output
 191     sys.exit(1)
 192
 193
 194 class BisectPerfRegressionTest(unittest.TestCase):
 195   """Test case for other functions and classes in bisect-perf-regression.py."""
 196
 197   def setUp(self):
 198     self.cwd = os.getcwd()
 199     os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__),
 200                                           os.path.pardir, os.path.pardir)))
 201
 202   def tearDown(self):
 203     os.chdir(self.cwd)
 204
 205   def testBisectOptionsCanPrintHelp(self):
 206     """Tests that the argument parser can be made and can print help."""
 207     bisect_options = bisect_perf_regression.BisectOptions()
 208     parser = bisect_options._CreateCommandLineParser()
 209     parser.format_help()
 210
 211   def testParseDEPSStringManually(self):
 212     """Tests DEPS parsing."""
 213     deps_file_contents = """
 214     vars = {
 215         'ffmpeg_hash':
 216              '@ac4a9f31fe2610bd146857bbd55d7a260003a888',
 217         'webkit_url':
 218              'https://chromium.googlesource.com/chromium/blink.git',
 219         'git_url':
 220              'https://chromium.googlesource.com',
 221         'webkit_rev':
 222              '@e01ac0a267d1017288bc67fa3c366b10469d8a24',
 223         'angle_revision':
 224              '74697cf2064c0a2c0d7e1b1b28db439286766a05'
 225     }"""
 226
 227     # Should only expect SVN/git revisions to come through, and URLs should be
 228     # filtered out.
 229     expected_vars_dict = {
 230         'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888',
 231         'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24',
 232         'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05'
 233     }
 234     # Testing private function.
 235     # pylint: disable=W0212
 236     vars_dict = bisect_perf_regression._ParseRevisionsFromDEPSFileManually(
 237         deps_file_contents)
 238     self.assertEqual(vars_dict, expected_vars_dict)
 239
 240   def _AssertParseResult(self, expected_values, result_string):
 241     """Asserts some values are parsed from a RESULT line."""
 242     results_template = ('RESULT other_chart: other_trace= 123 count\n'
 243                         'RESULT my_chart: my_trace= %(value)s\n')
 244     results = results_template % {'value': result_string}
 245     metric = ['my_chart', 'my_trace']
 246     # Testing private function.
 247     # pylint: disable=W0212
 248     values = bisect_perf_regression._TryParseResultValuesFromOutput(
 249         metric, results)
 250     self.assertEqual(expected_values, values)
 251
 252   def testTryParseResultValuesFromOutput_WithSingleValue(self):
 253     """Tests result pattern <*>RESULT <graph>: <trace>= <value>"""
 254     self._AssertParseResult([66.88], '66.88 kb')
 255     self._AssertParseResult([66.88], '66.88 ')
 256     self._AssertParseResult([-66.88], '-66.88 kb')
 257     self._AssertParseResult([66], '66 kb')
 258     self._AssertParseResult([0.66], '.66 kb')
 259     self._AssertParseResult([], '. kb')
 260     self._AssertParseResult([], 'aaa kb')
 261
 262   def testTryParseResultValuesFromOutput_WithMultiValue(self):
 263     """Tests result pattern <*>RESULT <graph>: <trace>= [<value>,<value>, ..]"""
 264     self._AssertParseResult([66.88], '[66.88] kb')
 265     self._AssertParseResult([66.88, 99.44], '[66.88, 99.44]kb')
 266     self._AssertParseResult([66.88, 99.44], '[ 66.88, 99.44 ]')
 267     self._AssertParseResult([-66.88, 99.44], '[-66.88, 99.44] kb')
 268     self._AssertParseResult([-66, 99], '[-66,99] kb')
 269     self._AssertParseResult([-66, 99], '[-66,99,] kb')
 270     self._AssertParseResult([-66, 0.99], '[-66,.99] kb')
 271     self._AssertParseResult([], '[] kb')
 272     self._AssertParseResult([], '[-66,abc] kb')
 273
 274   def testTryParseResultValuesFromOutputWithMeanStd(self):
 275     """Tests result pattern <*>RESULT <graph>: <trace>= {<mean, std}"""
 276     self._AssertParseResult([33.22], '{33.22, 3.6} kb')
 277     self._AssertParseResult([33.22], '{33.22, 3.6} kb')
 278     self._AssertParseResult([33.22], '{33.22,3.6}kb')
 279     self._AssertParseResult([33.22], '{33.22,3.6} kb')
 280     self._AssertParseResult([33.22], '{ 33.22,3.6 }kb')
 281     self._AssertParseResult([-33.22], '{-33.22,3.6}kb')
 282     self._AssertParseResult([22], '{22,6}kb')
 283     self._AssertParseResult([.22], '{.22,6}kb')
 284     self._AssertParseResult([], '{.22,6, 44}kb')
 285     self._AssertParseResult([], '{}kb')
 286     self._AssertParseResult([], '{XYZ}kb')
 287
 288   # This method doesn't reference self; it fails if an error is thrown.
 289   # pylint: disable=R0201
 290   def testDryRun(self):
 291     """Does a dry run of the bisect script.
 292
 293     This serves as a smoke test to catch errors in the basic execution of the
 294     script.
 295     """
 296     _GenericDryRun(DEFAULT_OPTIONS, True)
 297
 298   def testBisectImprovementDirectionFails(self):
 299     """Dry run of a bisect with an improvement instead of regression."""
 300     # Test result goes from 0 to 100 where higher is better
 301     results = _GenericDryRun(_GetExtendedOptions(1, 100))
 302     self.assertIsNotNone(results.error)
 303     self.assertIn('not a regression', results.error)
 304
 305     # Test result goes from 0 to -100 where lower is better
 306     results = _GenericDryRun(_GetExtendedOptions(-1, -100))
 307     self.assertIsNotNone(results.error)
 308     self.assertIn('not a regression', results.error)
 309
 310   def testBisectImprovementDirectionSucceeds(self):
 311     """Bisects with improvement direction matching regression range."""
 312     # Test result goes from 0 to 100 where lower is better
 313     results = _GenericDryRun(_GetExtendedOptions(-1, 100))
 314     self.assertIsNone(results.error)
 315     # Test result goes from 0 to -100 where higher is better
 316     results = _GenericDryRun(_GetExtendedOptions(1, -100))
 317     self.assertIsNone(results.error)
 318
 319   def _CheckAbortsEarly(self, results):
 320     """Returns True if the bisect job would abort early."""
 321     global _MockResultsGenerator
 322     _MockResultsGenerator = (r for r in results)
 323     bisect_class = bisect_perf_regression.BisectPerformanceMetrics
 324     original_run_tests = bisect_class.RunPerformanceTestAndParseResults
 325     bisect_class.RunPerformanceTestAndParseResults = _MakeMockRunTests()
 326
 327     try:
 328       dry_run_results = _GenericDryRun(_GetExtendedOptions(0, 0, False))
 329     except StopIteration:
 330       # If StopIteration was raised, that means that the next value after
 331       # the first two values was requested, so the job was not aborted.
 332       return False
 333     finally:
 334       bisect_class.RunPerformanceTestAndParseResults = original_run_tests
 335
 336     # If the job was aborted, there should be a warning about it.
 337     self.assertTrue(
 338         any('did not clearly reproduce a regression' in w
 339             for w in dry_run_results.warnings))
 340     return True
 341
 342   def testBisectAbortedOnClearNonRegression(self):
 343     self.assertTrue(self._CheckAbortsEarly(CLEAR_NON_REGRESSION))
 344
 345   def testBisectNotAborted_AlmostRegression(self):
 346     self.assertFalse(self._CheckAbortsEarly(ALMOST_REGRESSION))
 347
 348   def testBisectNotAborted_ClearRegression(self):
 349     self.assertFalse(self._CheckAbortsEarly(CLEAR_REGRESSION))
 350
 351   def testBisectNotAborted_BarelyRegression(self):
 352     self.assertFalse(self._CheckAbortsEarly(BARELY_REGRESSION))
 353
 354   def testBisectNotAborted_MultipleValues(self):
 355     self.assertFalse(self._CheckAbortsEarly(MULTIPLE_VALUES))
 356
 357   def _CheckAbortsEarlyForReturnCode(self, results):
 358     """Returns True if the bisect job would abort early in return code mode."""
 359     global _MockResultsGenerator
 360     _MockResultsGenerator = (r for r in results)
 361     bisect_class = bisect_perf_regression.BisectPerformanceMetrics
 362     original_run_tests = bisect_class.RunPerformanceTestAndParseResults
 363     bisect_class.RunPerformanceTestAndParseResults = _MakeMockRunTests(True)
 364     options = dict(DEFAULT_OPTIONS)
 365     options.update({'bisect_mode': 'return_code'})
 366     try:
 367       dry_run_results = _GenericDryRun(options)
 368     except StopIteration:
 369       # If StopIteration was raised, that means that the next value after
 370       # the first two values was requested, so the job was not aborted.
 371       return False
 372     finally:
 373       bisect_class.RunPerformanceTestAndParseResults = original_run_tests
 374
 375     # If the job was aborted, there should be a warning about it.
 376     if ('known good and known bad revisions returned same' in
 377         dry_run_results.abort_reason):
 378       return True
 379     return False
 380
 381   def testBisectAbortOn_SameReturnCode(self):
 382     self.assertTrue(self._CheckAbortsEarlyForReturnCode([[0,0,0], [0,0,0]]))
 383
 384   def testBisectNotAbortedOn_DifferentReturnCode(self):
 385     self.assertFalse(self._CheckAbortsEarlyForReturnCode([[1,1,1], [0,0,0]]))
 386
 387   def testGetCommitPosition(self):
 388     cp_git_rev = '7017a81991de983e12ab50dfc071c70e06979531'
 389     self.assertEqual(291765, source_control.GetCommitPosition(cp_git_rev))
 390
 391     svn_git_rev = 'e6db23a037cad47299a94b155b95eebd1ee61a58'
 392     self.assertEqual(291467, source_control.GetCommitPosition(svn_git_rev))
 393
 394   def testGetCommitPositionForV8(self):
 395     bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS)
 396     v8_rev = '21d700eedcdd6570eff22ece724b63a5eefe78cb'
 397     depot_path = os.path.join(bisect_instance.src_cwd, 'v8')
 398     self.assertEqual(
 399         23634, source_control.GetCommitPosition(v8_rev, depot_path))
 400
 401   def testGetCommitPositionForWebKit(self):
 402     bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS)
 403     wk_rev = 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61'
 404     depot_path = os.path.join(bisect_instance.src_cwd, 'third_party', 'WebKit')
 405     self.assertEqual(
 406         181660, source_control.GetCommitPosition(wk_rev, depot_path))
 407
 408   def testGetCommitPositionForSkia(self):
 409     bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS)
 410     skia_rev = 'a94d028eCheckAbortsEarly0f2c77f159b3dac95eb90c3b4cf48c61'
 411     depot_path = os.path.join(bisect_instance.src_cwd, 'third_party', 'skia')
 412     # Skia doesn't use commit positions, and GetCommitPosition should
 413     # return None for repos that don't use commit positions.
 414     self.assertIsNone(source_control.GetCommitPosition(skia_rev, depot_path))
 415
 416   def testUpdateDepsContent(self):
 417     bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS)
 418     deps_file = 'DEPS'
 419     # We are intentionally reading DEPS file contents instead of string literal
 420     # with few lines from DEPS because to check if the format we are expecting
 421     # to search is not changed in DEPS content.
 422     # TODO (prasadv): Add a separate test to validate the DEPS contents with the
 423     # format that bisect script expects.
 424     deps_contents = bisect_perf_regression.ReadStringFromFile(deps_file)
 425     deps_key = 'v8_revision'
 426     depot = 'v8'
 427     git_revision = 'a12345789a23456789a123456789a123456789'
 428     updated_content = bisect_instance.UpdateDepsContents(
 429         deps_contents, depot, git_revision, deps_key)
 430     self.assertIsNotNone(updated_content)
 431     ss = re.compile('["\']%s["\']: ["\']%s["\']' % (deps_key, git_revision))
 432     self.assertIsNotNone(re.search(ss, updated_content))
 433
 434   @mock.patch('bisect_utils.RunGClient')
 435   def testSyncToRevisionForChromium(self, mock_RunGClient):
 436     bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS)
 437     mock_RunGClient.return_value = 0
 438     bisect_instance._SyncRevision(
 439         'chromium', 'e6db23a037cad47299a94b155b95eebd1ee61a58', 'gclient')
 440     expected_params = [
 441         'sync',
 442         '--verbose',
 443         '--nohooks',
 444         '--force',
 445         '--delete_unversioned_trees',
 446         '--revision',
 447         'src@e6db23a037cad47299a94b155b95eebd1ee61a58',
 448     ]
 449
 450     mock_RunGClient.assert_called_with(expected_params, cwd=None)
 451
 452   @mock.patch('bisect_utils.RunGit')
 453   def testSyncToRevisionForWebKit(self, mock_RunGit):
 454     bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS)
 455     mock_RunGit.return_value = None, None
 456     bisect_instance._SyncRevision(
 457         'webkit', 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61', None)
 458     expected_params = ['checkout', 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61']
 459     mock_RunGit.assert_called_with(expected_params)
 460
 461   def testTryJobSvnRepo_PerfBuilderType_ReturnsRepoUrl(self):
 462     self.assertEqual(
 463         bisect_perf_regression.PERF_SVN_REPO_URL,
 464         bisect_perf_regression._TryJobSvnRepo(fetch_build.PERF_BUILDER))
 465
 466   def testTryJobSvnRepo_FullBuilderType_ReturnsRepoUrl(self):
 467     self.assertEqual(
 468         bisect_perf_regression.FULL_SVN_REPO_URL,
 469         bisect_perf_regression._TryJobSvnRepo(fetch_build.FULL_BUILDER))
 470
 471   def testTryJobSvnRepo_WithUnknownBuilderType_ThrowsError(self):
 472     with self.assertRaises(NotImplementedError):
 473       bisect_perf_regression._TryJobSvnRepo('foo')
 474
 475   def _CheckIsDownloadable(self, depot, target_platform='chromium',
 476                            builder_type='perf'):
 477     opts = dict(DEFAULT_OPTIONS)
 478     opts.update({'target_platform': target_platform,
 479                  'builder_type': builder_type})
 480     bisect_instance = _GetBisectPerformanceMetricsInstance(opts)
 481     return bisect_instance.IsDownloadable(depot)
 482
 483   def testIsDownloadable_ChromiumDepot_ReturnsTrue(self):
 484     self.assertTrue(self._CheckIsDownloadable(depot='chromium'))
 485
 486   def testIsDownloadable_DEPSDepot_ReturnsTrue(self):
 487     self.assertTrue(self._CheckIsDownloadable(depot='v8'))
 488
 489   def testIsDownloadable_AndroidChromeDepot_ReturnsTrue(self):
 490     self.assertTrue(self._CheckIsDownloadable(
 491         depot='android-chrome', target_platform='android-chrome'))
 492
 493   def testIsDownloadable_AndroidChromeWithDEPSChromium_ReturnsFalse(self):
 494     self.assertFalse(self._CheckIsDownloadable(
 495         depot='chromium', target_platform='android-chrome'))
 496
 497   def testIsDownloadable_AndroidChromeWithDEPSV8_ReturnsFalse(self):
 498     self.assertFalse(self._CheckIsDownloadable(
 499         depot='v8', target_platform='android-chrome'))
 500
 501   def testIsDownloadable_NoBuilderType_ReturnsFalse(self):
 502     self.assertFalse(
 503         self._CheckIsDownloadable(depot='chromium', builder_type=''))
 504
 505
 506 class DepotDirectoryRegistryTest(unittest.TestCase):
 507
 508   def setUp(self):
 509     self.old_chdir = os.chdir
 510     os.chdir = self.mockChdir
 511     self.old_depot_names = bisect_utils.DEPOT_NAMES
 512     bisect_utils.DEPOT_NAMES = ['mock_depot']
 513     self.old_depot_deps_name = bisect_utils.DEPOT_DEPS_NAME
 514     bisect_utils.DEPOT_DEPS_NAME = {'mock_depot': {'src': 'src/foo'}}
 515
 516     self.registry = bisect_perf_regression.DepotDirectoryRegistry('/mock/src')
 517     self.cur_dir = None
 518
 519   def tearDown(self):
 520     os.chdir = self.old_chdir
 521     bisect_utils.DEPOT_NAMES = self.old_depot_names
 522     bisect_utils.DEPOT_DEPS_NAME = self.old_depot_deps_name
 523
 524   def mockChdir(self, new_dir):
 525     self.cur_dir = new_dir
 526
 527   def testReturnsCorrectResultForChrome(self):
 528     self.assertEqual(self.registry.GetDepotDir('chromium'), '/mock/src')
 529
 530   def testUsesDepotSpecToInitializeRegistry(self):
 531     self.assertEqual(self.registry.GetDepotDir('mock_depot'), '/mock/src/foo')
 532
 533   def testChangedTheDirectory(self):
 534     self.registry.ChangeToDepotDir('mock_depot')
 535     self.assertEqual(self.cur_dir, '/mock/src/foo')
 536
 537
 538 # The tests below test private functions (W0212).
 539 # pylint: disable=W0212
 540 class GitTryJobTestCases(unittest.TestCase):
 541
 542   """Test case for bisect try job."""
 543   def setUp(self):
 544     bisect_utils_patcher = mock.patch('bisect_perf_regression.bisect_utils')
 545     self.mock_bisect_utils = bisect_utils_patcher.start()
 546     self.addCleanup(bisect_utils_patcher.stop)
 547
 548   def _SetupRunGitMock(self, git_cmds):
 549     """Setup RunGit mock with expected output for given git command."""
 550     def side_effect(git_cmd_args):
 551       for val in git_cmds:
 552         if set(val[0]) == set(git_cmd_args):
 553           return val[1]
 554     self.mock_bisect_utils.RunGit = mock.Mock(side_effect=side_effect)
 555
 556   def _AssertRunGitExceptions(self, git_cmds, func, *args):
 557     """Setup RunGit mock and tests RunGitException.
 558
 559     Args:
 560       git_cmds: List of tuples with git command and expected output.
 561       func: Callback function to be executed.
 562       args: List of arguments to be passed to the function.
 563     """
 564     self._SetupRunGitMock(git_cmds)
 565     self.assertRaises(bisect_perf_regression.RunGitError,
 566                       func,
 567                       *args)
 568
 569   def testNotGitRepo(self):
 570     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 571     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 572     cmds = [(['rev-parse', '--abbrev-ref', 'HEAD'], (None, 128))]
 573     self._AssertRunGitExceptions(cmds,
 574                                  bisect_perf_regression._PrepareBisectBranch,
 575                                  parent_branch, new_branch)
 576
 577   def testFailedCheckoutMaster(self):
 578     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 579     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 580     cmds = [
 581         (['rev-parse', '--abbrev-ref', 'HEAD'], (new_branch, 0)),
 582         (['checkout', '-f', parent_branch], ('Checkout Failed', 1)),
 583     ]
 584     self._AssertRunGitExceptions(cmds,
 585                                  bisect_perf_regression._PrepareBisectBranch,
 586                                  parent_branch, new_branch)
 587
 588   def testDeleteBisectBranchIfExists(self):
 589     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 590     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 591     cmds = [
 592         (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
 593         (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)),
 594         (['branch', '-D', new_branch], ('Failed to delete branch', 128)),
 595     ]
 596     self._AssertRunGitExceptions(cmds,
 597                                  bisect_perf_regression._PrepareBisectBranch,
 598                                  parent_branch, new_branch)
 599
 600   def testCreatNewBranchFails(self):
 601     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 602     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 603     cmds = [
 604         (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
 605         (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)),
 606         (['branch', '-D', new_branch], ('None', 0)),
 607         (['update-index', '--refresh', '-q'], (None, 0)),
 608         (['diff-index', 'HEAD'], (None, 0)),
 609         (['checkout', '-b', new_branch], ('Failed to create branch', 128)),
 610     ]
 611     self._AssertRunGitExceptions(cmds,
 612                                  bisect_perf_regression._PrepareBisectBranch,
 613                                  parent_branch, new_branch)
 614
 615   def testSetUpstreamToFails(self):
 616     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 617     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 618     cmds = [
 619         (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
 620         (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)),
 621         (['branch', '-D', new_branch], ('None', 0)),
 622         (['update-index', '--refresh', '-q'], (None, 0)),
 623         (['diff-index', 'HEAD'], (None, 0)),
 624         (['checkout', '-b', new_branch], ('None', 0)),
 625         (['branch', '--set-upstream-to', parent_branch],
 626          ('Setuptream fails', 1)),
 627     ]
 628     self._AssertRunGitExceptions(cmds,
 629                                  bisect_perf_regression._PrepareBisectBranch,
 630                                  parent_branch, new_branch)
 631
 632   def testStartBuilderTryJobForException(self):
 633     git_revision = 'ac4a9f31fe2610bd146857bbd55d7a260003a888'
 634     bot_name = 'linux_perf_bisect_builder'
 635     bisect_job_name = 'testBisectJobname'
 636     patch = None
 637     patch_content = '/dev/null'
 638     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 639     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 640     try_cmd = [
 641         (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
 642         (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)),
 643         (['branch', '-D', new_branch], ('None', 0)),
 644         (['update-index', '--refresh', '-q'], (None, 0)),
 645         (['diff-index', 'HEAD'], (None, 0)),
 646         (['checkout', '-b', new_branch], ('None', 0)),
 647         (['branch', '--set-upstream-to', parent_branch],
 648          ('Setuptream fails', 0)),
 649         (['try',
 650           '--bot=%s' % bot_name,
 651           '--revision=%s' % git_revision,
 652           '--name=%s' % bisect_job_name,
 653           '--svn_repo=%s' % bisect_perf_regression.PERF_SVN_REPO_URL,
 654           '--diff=%s' % patch_content],
 655          (None, 1)),
 656     ]
 657     self._AssertRunGitExceptions(
 658         try_cmd, bisect_perf_regression._StartBuilderTryJob,
 659         fetch_build.PERF_BUILDER, git_revision, bot_name, bisect_job_name,
 660         patch)
 661
 662   def testBuilderTryJob(self):
 663     git_revision = 'ac4a9f31fe2610bd146857bbd55d7a260003a888'
 664     bot_name = 'linux_perf_bisect_builder'
 665     bisect_job_name = 'testBisectJobname'
 666     patch = None
 667     patch_content = '/dev/null'
 668     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 669     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 670     try_cmd = [
 671         (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
 672         (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)),
 673         (['branch', '-D', new_branch], ('None', 0)),
 674         (['update-index', '--refresh', '-q'], (None, 0)),
 675         (['diff-index', 'HEAD'], (None, 0)),
 676         (['checkout', '-b', new_branch], ('None', 0)),
 677         (['branch', '--set-upstream-to', parent_branch],
 678          ('Setuptream fails', 0)),
 679         (['try',
 680           '--bot=%s' % bot_name,
 681           '--revision=%s' % git_revision,
 682           '--name=%s' % bisect_job_name,
 683           '--svn_repo=%s' % bisect_perf_regression.PERF_SVN_REPO_URL,
 684           '--diff=%s' % patch_content],
 685          (None, 0)),
 686     ]
 687     self._SetupRunGitMock(try_cmd)
 688     bisect_perf_regression._StartBuilderTryJob(
 689         fetch_build.PERF_BUILDER, git_revision, bot_name, bisect_job_name,
 690         patch)
 691
 692
 693 if __name__ == '__main__':
 694   unittest.main()