tools/auto_bisect/bisect_perf_regression_test.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import os
   6 import re
   7 import shutil
   8 import sys
   9 import unittest
  10
  11 SRC = os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir)
  12 sys.path.append(os.path.join(SRC, 'third_party', 'pymock'))
  13
  14 import bisect_perf_regression
  15 import bisect_utils
  16 import fetch_build
  17 import mock
  18 import source_control
  19
  20
  21 # Regression confidence: 0%
  22 CLEAR_NON_REGRESSION = [
  23     # Mean: 30.223 Std. Dev.: 11.383
  24     [[16.886], [16.909], [16.99], [17.723], [17.952], [18.118], [19.028],
  25      [19.552], [21.954], [38.573], [38.839], [38.965], [40.007], [40.572],
  26      [41.491], [42.002], [42.33], [43.109], [43.238]],
  27     # Mean: 34.76 Std. Dev.: 11.516
  28     [[16.426], [17.347], [20.593], [21.177], [22.791], [27.843], [28.383],
  29      [28.46], [29.143], [40.058], [40.303], [40.558], [41.918], [42.44],
  30      [45.223], [46.494], [50.002], [50.625], [50.839]]
  31 ]
  32
  33 # Regression confidence: ~ 90%
  34 ALMOST_REGRESSION = [
  35     # Mean: 30.042 Std. Dev.: 2.002
  36     [[26.146], [28.04], [28.053], [28.074], [28.168], [28.209], [28.471],
  37      [28.652], [28.664], [30.862], [30.973], [31.002], [31.897], [31.929],
  38      [31.99], [32.214], [32.323], [32.452], [32.696]],
  39     # Mean: 33.008 Std. Dev.: 4.265
  40     [[34.963], [30.741], [39.677], [39.512], [34.314], [31.39], [34.361],
  41      [25.2], [30.489], [29.434]]
  42 ]
  43
  44 # Regression confidence: ~ 98%
  45 BARELY_REGRESSION = [
  46     # Mean: 28.828 Std. Dev.: 1.993
  47     [[26.96], [27.605], [27.768], [27.829], [28.006], [28.206], [28.393],
  48      [28.911], [28.933], [30.38], [30.462], [30.808], [31.74], [31.805],
  49      [31.899], [32.077], [32.454], [32.597], [33.155]],
  50     # Mean: 31.156 Std. Dev.: 1.980
  51     [[28.729], [29.112], [29.258], [29.454], [29.789], [30.036], [30.098],
  52      [30.174], [30.534], [32.285], [32.295], [32.552], [32.572], [32.967],
  53      [33.165], [33.403], [33.588], [33.744], [34.147], [35.84]]
  54 ]
  55
  56 # Regression confidence: 99.5%
  57 CLEAR_REGRESSION = [
  58     # Mean: 30.254 Std. Dev.: 2.987
  59     [[26.494], [26.621], [26.701], [26.997], [26.997], [27.05], [27.37],
  60      [27.488], [27.556], [31.846], [32.192], [32.21], [32.586], [32.596],
  61      [32.618], [32.95], [32.979], [33.421], [33.457], [34.97]],
  62     # Mean: 33.190 Std. Dev.: 2.972
  63     [[29.547], [29.713], [29.835], [30.132], [30.132], [30.33], [30.406],
  64      [30.592], [30.72], [34.486], [35.247], [35.253], [35.335], [35.378],
  65      [35.934], [36.233], [36.41], [36.947], [37.982]]
  66 ]
  67
  68 # Regression confidence > 95%, taken from: crbug.com/434318
  69 # Specifically from Builder android_nexus10_perf_bisect Build #1198
  70 MULTIPLE_VALUES = [
  71     [
  72         [18.916, 22.371, 8.527, 5.877, 5.407, 9.476, 8.100, 5.334,
  73          4.507, 4.842, 8.485, 8.308, 27.490, 4.560, 4.804, 23.068, 17.577,
  74          17.346, 26.738, 60.330, 32.307, 5.468, 27.803, 27.373, 17.823,
  75          5.158, 27.439, 5.236, 11.413],
  76         [18.999, 22.642, 8.158, 5.995, 5.495, 9.499, 8.092, 5.324,
  77          4.468, 4.788, 8.248, 7.853, 27.533, 4.410, 4.622, 22.341, 22.313,
  78          17.072, 26.731, 57.513, 33.001, 5.500, 28.297, 27.277, 26.462,
  79          5.009, 27.361, 5.130, 10.955]
  80     ],
  81     [
  82         [18.238, 22.365, 8.555, 5.939, 5.437, 9.463, 7.047, 5.345, 4.517,
  83          4.796, 8.593, 7.901, 27.499, 4.378, 5.040, 4.904, 4.816, 4.828,
  84          4.853, 57.363, 34.184, 5.482, 28.190, 27.290, 26.694, 5.099,
  85          4.905, 5.290, 4.813],
  86         [18.301, 22.522, 8.035, 6.021, 5.565, 9.037, 6.998, 5.321, 4.485,
  87          4.768, 8.397, 7.865, 27.636, 4.640, 5.015, 4.962, 4.933, 4.977,
  88          4.961, 60.648, 34.593, 5.538, 28.454, 27.297, 26.490, 5.099, 5,
  89          5.247, 4.945],
  90         [18.907, 23.368, 8.100, 6.169, 5.621, 9.971, 8.161, 5.331, 4.513,
  91          4.837, 8.255, 7.852, 26.209, 4.388, 5.045, 5.029, 5.032, 4.946,
  92          4.973, 60.334, 33.377, 5.499, 28.275, 27.550, 26.103, 5.108,
  93          4.951, 5.285, 4.910],
  94         [18.715, 23.748, 8.128, 6.148, 5.691, 9.361, 8.106, 5.334, 4.528,
  95          4.965, 8.261, 7.851, 27.282, 4.391, 4.949, 4.981, 4.964, 4.935,
  96          4.933, 60.231, 33.361, 5.489, 28.106, 27.457, 26.648, 5.108,
  97          4.963, 5.272, 4.954]
  98     ]
  99 ]
 100
 101 # Default options for the dry run
 102 DEFAULT_OPTIONS = {
 103     'debug_ignore_build': True,
 104     'debug_ignore_sync': True,
 105     'debug_ignore_perf_test': True,
 106     'debug_ignore_regression_confidence': True,
 107     'command': 'fake_command',
 108     'metric': 'fake/metric',
 109     'good_revision': 280000,
 110     'bad_revision': 280005,
 111 }
 112
 113 # This global is a placeholder for a generator to be defined by the test cases
 114 # that use _MockRunTests.
 115 _MockResultsGenerator = (x for x in [])
 116
 117
 118 def _MockRunTests(*args, **kwargs):  # pylint: disable=unused-argument
 119   return _FakeTestResult(_MockResultsGenerator.next())
 120
 121
 122 def _FakeTestResult(values):
 123   result_dict = {'mean': 0.0, 'std_err': 0.0, 'std_dev': 0.0, 'values': values}
 124   success_code = 0
 125   return (result_dict, success_code)
 126
 127
 128 def _GetBisectPerformanceMetricsInstance(options_dict):
 129   """Returns an instance of the BisectPerformanceMetrics class."""
 130   opts = bisect_perf_regression.BisectOptions.FromDict(options_dict)
 131   return bisect_perf_regression.BisectPerformanceMetrics(opts, os.getcwd())
 132
 133
 134 def _GetExtendedOptions(improvement_dir, fake_first, ignore_confidence=True):
 135   """Returns the a copy of the default options dict plus some options."""
 136   result = dict(DEFAULT_OPTIONS)
 137   result.update({
 138       'improvement_direction': improvement_dir,
 139       'debug_fake_first_test_mean': fake_first,
 140       'debug_ignore_regression_confidence': ignore_confidence})
 141   return result
 142
 143
 144 def _GenericDryRun(options, print_results=False):
 145   """Performs a dry run of the bisector.
 146
 147   Args:
 148     options: Dictionary containing the options for the bisect instance.
 149     print_results: Boolean telling whether to call FormatAndPrintResults.
 150
 151   Returns:
 152     The results dictionary as returned by the bisect Run method.
 153   """
 154   _AbortIfThereAreStagedChanges()
 155   # Disable rmtree to avoid deleting local trees.
 156   old_rmtree = shutil.rmtree
 157   shutil.rmtree = lambda path, on_error: None
 158   # git reset HEAD may be run during the dry run, which removes staged changes.
 159   try:
 160     bisect_instance = _GetBisectPerformanceMetricsInstance(options)
 161     results = bisect_instance.Run(
 162         bisect_instance.opts.command, bisect_instance.opts.bad_revision,
 163         bisect_instance.opts.good_revision, bisect_instance.opts.metric)
 164
 165     if print_results:
 166       bisect_instance.printer.FormatAndPrintResults(results)
 167
 168     return results
 169   finally:
 170     shutil.rmtree = old_rmtree
 171
 172
 173 def _AbortIfThereAreStagedChanges():
 174   """Exits the test prematurely if there are staged changes."""
 175   # The output of "git status --short" will be an empty string if there are
 176   # no staged changes in the current branch. Untracked files are ignored
 177   # because when running the presubmit on the trybot there are sometimes
 178   # untracked changes to the run-perf-test.cfg and bisect.cfg files.
 179   status_output = bisect_utils.CheckRunGit(
 180       ['status', '--short', '--untracked-files=no'])
 181   if status_output:
 182     print 'There are un-committed changes in the current branch.'
 183     print 'Aborting the tests to avoid destroying local changes. Changes:'
 184     print status_output
 185     sys.exit(1)
 186
 187
 188 class BisectPerfRegressionTest(unittest.TestCase):
 189   """Test case for other functions and classes in bisect-perf-regression.py."""
 190
 191   def setUp(self):
 192     self.cwd = os.getcwd()
 193     os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__),
 194                                           os.path.pardir, os.path.pardir)))
 195
 196   def tearDown(self):
 197     os.chdir(self.cwd)
 198
 199   def testBisectOptionsCanPrintHelp(self):
 200     """Tests that the argument parser can be made and can print help."""
 201     bisect_options = bisect_perf_regression.BisectOptions()
 202     parser = bisect_options._CreateCommandLineParser()
 203     parser.format_help()
 204
 205   def testParseDEPSStringManually(self):
 206     """Tests DEPS parsing."""
 207     deps_file_contents = """
 208     vars = {
 209         'ffmpeg_hash':
 210              '@ac4a9f31fe2610bd146857bbd55d7a260003a888',
 211         'webkit_url':
 212              'https://chromium.googlesource.com/chromium/blink.git',
 213         'git_url':
 214              'https://chromium.googlesource.com',
 215         'webkit_rev':
 216              '@e01ac0a267d1017288bc67fa3c366b10469d8a24',
 217         'angle_revision':
 218              '74697cf2064c0a2c0d7e1b1b28db439286766a05'
 219     }"""
 220
 221     # Should only expect SVN/git revisions to come through, and URLs should be
 222     # filtered out.
 223     expected_vars_dict = {
 224         'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888',
 225         'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24',
 226         'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05'
 227     }
 228     # Testing private function.
 229     # pylint: disable=W0212
 230     vars_dict = bisect_perf_regression._ParseRevisionsFromDEPSFileManually(
 231         deps_file_contents)
 232     self.assertEqual(vars_dict, expected_vars_dict)
 233
 234   def _AssertParseResult(self, expected_values, result_string):
 235     """Asserts some values are parsed from a RESULT line."""
 236     results_template = ('RESULT other_chart: other_trace= 123 count\n'
 237                         'RESULT my_chart: my_trace= %(value)s\n')
 238     results = results_template % {'value': result_string}
 239     metric = ['my_chart', 'my_trace']
 240     # Testing private function.
 241     # pylint: disable=W0212
 242     values = bisect_perf_regression._TryParseResultValuesFromOutput(
 243         metric, results)
 244     self.assertEqual(expected_values, values)
 245
 246   def testTryParseResultValuesFromOutput_WithSingleValue(self):
 247     """Tests result pattern <*>RESULT <graph>: <trace>= <value>"""
 248     self._AssertParseResult([66.88], '66.88 kb')
 249     self._AssertParseResult([66.88], '66.88 ')
 250     self._AssertParseResult([-66.88], '-66.88 kb')
 251     self._AssertParseResult([66], '66 kb')
 252     self._AssertParseResult([0.66], '.66 kb')
 253     self._AssertParseResult([], '. kb')
 254     self._AssertParseResult([], 'aaa kb')
 255
 256   def testTryParseResultValuesFromOutput_WithMultiValue(self):
 257     """Tests result pattern <*>RESULT <graph>: <trace>= [<value>,<value>, ..]"""
 258     self._AssertParseResult([66.88], '[66.88] kb')
 259     self._AssertParseResult([66.88, 99.44], '[66.88, 99.44]kb')
 260     self._AssertParseResult([66.88, 99.44], '[ 66.88, 99.44 ]')
 261     self._AssertParseResult([-66.88, 99.44], '[-66.88, 99.44] kb')
 262     self._AssertParseResult([-66, 99], '[-66,99] kb')
 263     self._AssertParseResult([-66, 99], '[-66,99,] kb')
 264     self._AssertParseResult([-66, 0.99], '[-66,.99] kb')
 265     self._AssertParseResult([], '[] kb')
 266     self._AssertParseResult([], '[-66,abc] kb')
 267
 268   def testTryParseResultValuesFromOutputWithMeanStd(self):
 269     """Tests result pattern <*>RESULT <graph>: <trace>= {<mean, std}"""
 270     self._AssertParseResult([33.22], '{33.22, 3.6} kb')
 271     self._AssertParseResult([33.22], '{33.22, 3.6} kb')
 272     self._AssertParseResult([33.22], '{33.22,3.6}kb')
 273     self._AssertParseResult([33.22], '{33.22,3.6} kb')
 274     self._AssertParseResult([33.22], '{ 33.22,3.6 }kb')
 275     self._AssertParseResult([-33.22], '{-33.22,3.6}kb')
 276     self._AssertParseResult([22], '{22,6}kb')
 277     self._AssertParseResult([.22], '{.22,6}kb')
 278     self._AssertParseResult([], '{.22,6, 44}kb')
 279     self._AssertParseResult([], '{}kb')
 280     self._AssertParseResult([], '{XYZ}kb')
 281
 282   def _AssertCompatibleCommand(
 283       self, expected_command, original_command, revision, target_platform):
 284     """Tests the modification of the command that might be done.
 285
 286     This modification to the command is done in order to get a Telemetry
 287     command that works; before some revisions, the browser name that Telemetry
 288     expects is different in some cases, but we want it to work anyway.
 289
 290     Specifically, only for android:
 291       After r276628, only android-chrome-shell works.
 292       Prior to r274857, only android-chromium-testshell works.
 293       In the range [274857, 276628], both work.
 294     """
 295     bisect_options = bisect_perf_regression.BisectOptions()
 296     bisect_options.output_buildbot_annotations = None
 297     bisect_instance = bisect_perf_regression.BisectPerformanceMetrics(
 298         bisect_options, os.getcwd())
 299     bisect_instance.opts.target_platform = target_platform
 300     git_revision = source_control.ResolveToRevision(
 301         revision, 'chromium', bisect_utils.DEPOT_DEPS_NAME, 100)
 302     depot = 'chromium'
 303     command = bisect_instance.GetCompatibleCommand(
 304         original_command, git_revision, depot)
 305     self.assertEqual(expected_command, command)
 306
 307   def testGetCompatibleCommand_ChangeToTestShell(self):
 308     # For revisions <= r274857, only android-chromium-testshell is used.
 309     self._AssertCompatibleCommand(
 310         'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo',
 311         'tools/perf/run_benchmark -v --browser=android-chrome-shell foo',
 312         274857, 'android')
 313
 314   def testGetCompatibleCommand_ChangeToShell(self):
 315     # For revisions >= r276728, only android-chrome-shell can be used.
 316     self._AssertCompatibleCommand(
 317         'tools/perf/run_benchmark -v --browser=android-chrome-shell foo',
 318         'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo',
 319         276628, 'android')
 320
 321   def testGetCompatibleCommand_NoChange(self):
 322     # For revisions < r276728, android-chromium-testshell can be used.
 323     self._AssertCompatibleCommand(
 324         'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo',
 325         'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo',
 326         274858, 'android')
 327     # For revisions > r274857, android-chrome-shell can be used.
 328     self._AssertCompatibleCommand(
 329         'tools/perf/run_benchmark -v --browser=android-chrome-shell foo',
 330         'tools/perf/run_benchmark -v --browser=android-chrome-shell foo',
 331         274858, 'android')
 332
 333   def testGetCompatibleCommand_NonAndroidPlatform(self):
 334     # In most cases, there's no need to change Telemetry command.
 335     # For revisions >= r276728, only android-chrome-shell can be used.
 336     self._AssertCompatibleCommand(
 337         'tools/perf/run_benchmark -v --browser=release foo',
 338         'tools/perf/run_benchmark -v --browser=release foo',
 339         276628, 'chromium')
 340
 341   # This method doesn't reference self; it fails if an error is thrown.
 342   # pylint: disable=R0201
 343   def testDryRun(self):
 344     """Does a dry run of the bisect script.
 345
 346     This serves as a smoke test to catch errors in the basic execution of the
 347     script.
 348     """
 349     _GenericDryRun(DEFAULT_OPTIONS, True)
 350
 351   def testBisectImprovementDirectionFails(self):
 352     """Dry run of a bisect with an improvement instead of regression."""
 353     # Test result goes from 0 to 100 where higher is better
 354     results = _GenericDryRun(_GetExtendedOptions(1, 100))
 355     self.assertIsNotNone(results.error)
 356     self.assertIn('not a regression', results.error)
 357
 358     # Test result goes from 0 to -100 where lower is better
 359     results = _GenericDryRun(_GetExtendedOptions(-1, -100))
 360     self.assertIsNotNone(results.error)
 361     self.assertIn('not a regression', results.error)
 362
 363   def testBisectImprovementDirectionSucceeds(self):
 364     """Bisects with improvement direction matching regression range."""
 365     # Test result goes from 0 to 100 where lower is better
 366     results = _GenericDryRun(_GetExtendedOptions(-1, 100))
 367     self.assertIsNone(results.error)
 368     # Test result goes from 0 to -100 where higher is better
 369     results = _GenericDryRun(_GetExtendedOptions(1, -100))
 370     self.assertIsNone(results.error)
 371
 372   def _CheckAbortsEarly(self, results):
 373     """Returns True if the bisect job would abort early."""
 374     global _MockResultsGenerator
 375     _MockResultsGenerator = (r for r in results)
 376     bisect_class = bisect_perf_regression.BisectPerformanceMetrics
 377     original_run_tests = bisect_class.RunPerformanceTestAndParseResults
 378     bisect_class.RunPerformanceTestAndParseResults = _MockRunTests
 379
 380     try:
 381       dry_run_results = _GenericDryRun(_GetExtendedOptions(0, 0, False))
 382     except StopIteration:
 383       # If StopIteration was raised, that means that the next value after
 384       # the first two values was requested, so the job was not aborted.
 385       return False
 386     finally:
 387       bisect_class.RunPerformanceTestAndParseResults = original_run_tests
 388
 389     # If the job was aborted, there should be a warning about it.
 390     assert [w for w in dry_run_results.warnings
 391             if 'could not reproduce the regression' in w]
 392     return True
 393
 394   def testBisectAbortedOnClearNonRegression(self):
 395     self.assertTrue(self._CheckAbortsEarly(CLEAR_NON_REGRESSION))
 396
 397   def testBisectNotAborted_AlmostRegression(self):
 398     self.assertFalse(self._CheckAbortsEarly(ALMOST_REGRESSION))
 399
 400   def testBisectNotAborted_ClearRegression(self):
 401     self.assertFalse(self._CheckAbortsEarly(CLEAR_REGRESSION))
 402
 403   def testBisectNotAborted_BarelyRegression(self):
 404     self.assertFalse(self._CheckAbortsEarly(BARELY_REGRESSION))
 405
 406   def testBisectNotAborted_MultipleValues(self):
 407     self.assertFalse(self._CheckAbortsEarly(MULTIPLE_VALUES))
 408
 409   def testGetCommitPosition(self):
 410     cp_git_rev = '7017a81991de983e12ab50dfc071c70e06979531'
 411     self.assertEqual(291765, source_control.GetCommitPosition(cp_git_rev))
 412
 413     svn_git_rev = 'e6db23a037cad47299a94b155b95eebd1ee61a58'
 414     self.assertEqual(291467, source_control.GetCommitPosition(svn_git_rev))
 415
 416   def testGetCommitPositionForV8(self):
 417     bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS)
 418     v8_rev = '21d700eedcdd6570eff22ece724b63a5eefe78cb'
 419     depot_path = os.path.join(bisect_instance.src_cwd, 'v8')
 420     self.assertEqual(
 421         23634, source_control.GetCommitPosition(v8_rev, depot_path))
 422
 423   def testGetCommitPositionForWebKit(self):
 424     bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS)
 425     wk_rev = 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61'
 426     depot_path = os.path.join(bisect_instance.src_cwd, 'third_party', 'WebKit')
 427     self.assertEqual(
 428         181660, source_control.GetCommitPosition(wk_rev, depot_path))
 429
 430   def testGetCommitPositionForSkia(self):
 431     bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS)
 432     skia_rev = 'a94d028eCheckAbortsEarly0f2c77f159b3dac95eb90c3b4cf48c61'
 433     depot_path = os.path.join(bisect_instance.src_cwd, 'third_party', 'skia')
 434     # Skia doesn't use commit positions, and GetCommitPosition should
 435     # return None for repos that don't use commit positions.
 436     self.assertIsNone(source_control.GetCommitPosition(skia_rev, depot_path))
 437
 438   def testUpdateDepsContent(self):
 439     bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS)
 440     deps_file = 'DEPS'
 441     # We are intentionally reading DEPS file contents instead of string literal
 442     # with few lines from DEPS because to check if the format we are expecting
 443     # to search is not changed in DEPS content.
 444     # TODO (prasadv): Add a separate test to validate the DEPS contents with the
 445     # format that bisect script expects.
 446     deps_contents = bisect_perf_regression.ReadStringFromFile(deps_file)
 447     deps_key = 'v8_revision'
 448     depot = 'v8'
 449     git_revision = 'a12345789a23456789a123456789a123456789'
 450     updated_content = bisect_instance.UpdateDepsContents(
 451         deps_contents, depot, git_revision, deps_key)
 452     self.assertIsNotNone(updated_content)
 453     ss = re.compile('["\']%s["\']: ["\']%s["\']' % (deps_key, git_revision))
 454     self.assertIsNotNone(re.search(ss, updated_content))
 455
 456   @mock.patch('bisect_utils.RunGClient')
 457   def testSyncToRevisionForChromium(self, mock_RunGClient):
 458     bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS)
 459     mock_RunGClient.return_value = 0
 460     bisect_instance._SyncRevision(
 461         'chromium', 'e6db23a037cad47299a94b155b95eebd1ee61a58', 'gclient')
 462     expected_params = [
 463         'sync',
 464         '--verbose',
 465         '--nohooks',
 466         '--force',
 467         '--delete_unversioned_trees',
 468         '--revision',
 469         'src@e6db23a037cad47299a94b155b95eebd1ee61a58',
 470     ]
 471
 472     mock_RunGClient.assert_called_with(expected_params, cwd=None)
 473
 474   @mock.patch('bisect_utils.RunGit')
 475   def testSyncToRevisionForWebKit(self, mock_RunGit):
 476     bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS)
 477     mock_RunGit.return_value = None, None
 478     bisect_instance._SyncRevision(
 479         'webkit', 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61', None)
 480     expected_params = ['checkout', 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61']
 481     mock_RunGit.assert_called_with(expected_params)
 482
 483   def testTryJobSvnRepo_PerfBuilderType_ReturnsRepoUrl(self):
 484     self.assertEqual(
 485         bisect_perf_regression.PERF_SVN_REPO_URL,
 486         bisect_perf_regression._TryJobSvnRepo(fetch_build.PERF_BUILDER))
 487
 488   def testTryJobSvnRepo_FullBuilderType_ReturnsRepoUrl(self):
 489     self.assertEqual(
 490         bisect_perf_regression.FULL_SVN_REPO_URL,
 491         bisect_perf_regression._TryJobSvnRepo(fetch_build.FULL_BUILDER))
 492
 493   def testTryJobSvnRepo_WithUnknownBuilderType_ThrowsError(self):
 494     with self.assertRaises(NotImplementedError):
 495       bisect_perf_regression._TryJobSvnRepo('foo')
 496
 497   def _CheckIsDownloadable(self, depot, target_platform='chromium',
 498                            builder_type='perf'):
 499     opts = dict(DEFAULT_OPTIONS)
 500     opts.update({'target_platform': target_platform,
 501                  'builder_type': builder_type})
 502     bisect_instance = _GetBisectPerformanceMetricsInstance(opts)
 503     return bisect_instance.IsDownloadable(depot)
 504
 505   def testIsDownloadable_ChromiumDepot_ReturnsTrue(self):
 506     self.assertTrue(self._CheckIsDownloadable(depot='chromium'))
 507
 508   def testIsDownloadable_DEPSDepot_ReturnsTrue(self):
 509     self.assertTrue(self._CheckIsDownloadable(depot='v8'))
 510
 511   def testIsDownloadable_AndroidChromeDepot_ReturnsTrue(self):
 512     self.assertTrue(self._CheckIsDownloadable(
 513         depot='android-chrome', target_platform='android-chrome'))
 514
 515   def testIsDownloadable_AndroidChromeWithDEPSChromium_ReturnsFalse(self):
 516     self.assertFalse(self._CheckIsDownloadable(
 517         depot='chromium', target_platform='android-chrome'))
 518
 519   def testIsDownloadable_AndroidChromeWithDEPSV8_ReturnsFalse(self):
 520     self.assertFalse(self._CheckIsDownloadable(
 521         depot='v8', target_platform='android-chrome'))
 522
 523   def testIsDownloadable_NoBuilderType_ReturnsFalse(self):
 524     self.assertFalse(
 525         self._CheckIsDownloadable(depot='chromium', builder_type=''))
 526
 527
 528 class DepotDirectoryRegistryTest(unittest.TestCase):
 529
 530   def setUp(self):
 531     self.old_chdir = os.chdir
 532     os.chdir = self.mockChdir
 533     self.old_depot_names = bisect_utils.DEPOT_NAMES
 534     bisect_utils.DEPOT_NAMES = ['mock_depot']
 535     self.old_depot_deps_name = bisect_utils.DEPOT_DEPS_NAME
 536     bisect_utils.DEPOT_DEPS_NAME = {'mock_depot': {'src': 'src/foo'}}
 537
 538     self.registry = bisect_perf_regression.DepotDirectoryRegistry('/mock/src')
 539     self.cur_dir = None
 540
 541   def tearDown(self):
 542     os.chdir = self.old_chdir
 543     bisect_utils.DEPOT_NAMES = self.old_depot_names
 544     bisect_utils.DEPOT_DEPS_NAME = self.old_depot_deps_name
 545
 546   def mockChdir(self, new_dir):
 547     self.cur_dir = new_dir
 548
 549   def testReturnsCorrectResultForChrome(self):
 550     self.assertEqual(self.registry.GetDepotDir('chromium'), '/mock/src')
 551
 552   def testUsesDepotSpecToInitializeRegistry(self):
 553     self.assertEqual(self.registry.GetDepotDir('mock_depot'), '/mock/src/foo')
 554
 555   def testChangedTheDirectory(self):
 556     self.registry.ChangeToDepotDir('mock_depot')
 557     self.assertEqual(self.cur_dir, '/mock/src/foo')
 558
 559
 560 # The tests below test private functions (W0212).
 561 # pylint: disable=W0212
 562 class GitTryJobTestCases(unittest.TestCase):
 563
 564   """Test case for bisect try job."""
 565   def setUp(self):
 566     bisect_utils_patcher = mock.patch('bisect_perf_regression.bisect_utils')
 567     self.mock_bisect_utils = bisect_utils_patcher.start()
 568     self.addCleanup(bisect_utils_patcher.stop)
 569
 570   def _SetupRunGitMock(self, git_cmds):
 571     """Setup RunGit mock with expected output for given git command."""
 572     def side_effect(git_cmd_args):
 573       for val in git_cmds:
 574         if set(val[0]) == set(git_cmd_args):
 575           return val[1]
 576     self.mock_bisect_utils.RunGit = mock.Mock(side_effect=side_effect)
 577
 578   def _AssertRunGitExceptions(self, git_cmds, func, *args):
 579     """Setup RunGit mock and tests RunGitException.
 580
 581     Args:
 582       git_cmds: List of tuples with git command and expected output.
 583       func: Callback function to be executed.
 584       args: List of arguments to be passed to the function.
 585     """
 586     self._SetupRunGitMock(git_cmds)
 587     self.assertRaises(bisect_perf_regression.RunGitError,
 588                       func,
 589                       *args)
 590
 591   def testNotGitRepo(self):
 592     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 593     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 594     cmds = [(['rev-parse', '--abbrev-ref', 'HEAD'], (None, 128))]
 595     self._AssertRunGitExceptions(cmds,
 596                                  bisect_perf_regression._PrepareBisectBranch,
 597                                  parent_branch, new_branch)
 598
 599   def testFailedCheckoutMaster(self):
 600     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 601     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 602     cmds = [
 603         (['rev-parse', '--abbrev-ref', 'HEAD'], (new_branch, 0)),
 604         (['checkout', '-f', parent_branch], ('Checkout Failed', 1)),
 605     ]
 606     self._AssertRunGitExceptions(cmds,
 607                                  bisect_perf_regression._PrepareBisectBranch,
 608                                  parent_branch, new_branch)
 609
 610   def testDeleteBisectBranchIfExists(self):
 611     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 612     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 613     cmds = [
 614         (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
 615         (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)),
 616         (['branch', '-D', new_branch], ('Failed to delete branch', 128)),
 617     ]
 618     self._AssertRunGitExceptions(cmds,
 619                                  bisect_perf_regression._PrepareBisectBranch,
 620                                  parent_branch, new_branch)
 621
 622   def testCreatNewBranchFails(self):
 623     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 624     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 625     cmds = [
 626         (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
 627         (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)),
 628         (['branch', '-D', new_branch], ('None', 0)),
 629         (['update-index', '--refresh', '-q'], (None, 0)),
 630         (['diff-index', 'HEAD'], (None, 0)),
 631         (['checkout', '-b', new_branch], ('Failed to create branch', 128)),
 632     ]
 633     self._AssertRunGitExceptions(cmds,
 634                                  bisect_perf_regression._PrepareBisectBranch,
 635                                  parent_branch, new_branch)
 636
 637   def testSetUpstreamToFails(self):
 638     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 639     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 640     cmds = [
 641         (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
 642         (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)),
 643         (['branch', '-D', new_branch], ('None', 0)),
 644         (['update-index', '--refresh', '-q'], (None, 0)),
 645         (['diff-index', 'HEAD'], (None, 0)),
 646         (['checkout', '-b', new_branch], ('None', 0)),
 647         (['branch', '--set-upstream-to', parent_branch],
 648          ('Setuptream fails', 1)),
 649     ]
 650     self._AssertRunGitExceptions(cmds,
 651                                  bisect_perf_regression._PrepareBisectBranch,
 652                                  parent_branch, new_branch)
 653
 654   def testStartBuilderTryJobForException(self):
 655     git_revision = 'ac4a9f31fe2610bd146857bbd55d7a260003a888'
 656     bot_name = 'linux_perf_bisect_builder'
 657     bisect_job_name = 'testBisectJobname'
 658     patch = None
 659     patch_content = '/dev/null'
 660     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 661     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 662     try_cmd = [
 663         (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
 664         (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)),
 665         (['branch', '-D', new_branch], ('None', 0)),
 666         (['update-index', '--refresh', '-q'], (None, 0)),
 667         (['diff-index', 'HEAD'], (None, 0)),
 668         (['checkout', '-b', new_branch], ('None', 0)),
 669         (['branch', '--set-upstream-to', parent_branch],
 670          ('Setuptream fails', 0)),
 671         (['try',
 672           '--bot=%s' % bot_name,
 673           '--revision=%s' % git_revision,
 674           '--name=%s' % bisect_job_name,
 675           '--svn_repo=%s' % bisect_perf_regression.PERF_SVN_REPO_URL,
 676           '--diff=%s' % patch_content],
 677          (None, 1)),
 678     ]
 679     self._AssertRunGitExceptions(
 680         try_cmd, bisect_perf_regression._StartBuilderTryJob,
 681         fetch_build.PERF_BUILDER, git_revision, bot_name, bisect_job_name,
 682         patch)
 683
 684   def testBuilderTryJob(self):
 685     git_revision = 'ac4a9f31fe2610bd146857bbd55d7a260003a888'
 686     bot_name = 'linux_perf_bisect_builder'
 687     bisect_job_name = 'testBisectJobname'
 688     patch = None
 689     patch_content = '/dev/null'
 690     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 691     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 692     try_cmd = [
 693         (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
 694         (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)),
 695         (['branch', '-D', new_branch], ('None', 0)),
 696         (['update-index', '--refresh', '-q'], (None, 0)),
 697         (['diff-index', 'HEAD'], (None, 0)),
 698         (['checkout', '-b', new_branch], ('None', 0)),
 699         (['branch', '--set-upstream-to', parent_branch],
 700          ('Setuptream fails', 0)),
 701         (['try',
 702           '--bot=%s' % bot_name,
 703           '--revision=%s' % git_revision,
 704           '--name=%s' % bisect_job_name,
 705           '--svn_repo=%s' % bisect_perf_regression.PERF_SVN_REPO_URL,
 706           '--diff=%s' % patch_content],
 707          (None, 0)),
 708     ]
 709     self._SetupRunGitMock(try_cmd)
 710     bisect_perf_regression._StartBuilderTryJob(
 711         fetch_build.PERF_BUILDER, git_revision, bot_name, bisect_job_name,
 712         patch)
 713
 714
 715 if __name__ == '__main__':
 716   unittest.main()