tools/auto_bisect/bisect_results_test.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import os
   6 import unittest
   7
   8 from bisect_results import BisectResults
   9 import source_control
  10
  11
  12 class MockDepotRegistry(object):
  13   def ChangeToDepotDir(self, depot):
  14     pass
  15
  16
  17 class MockRevisionState(object):
  18   def __init__(self, revision, index, depot='chromium', value=None,
  19                perf_time=0, build_time=0, passed='?', external=None):
  20     self.depot = depot
  21     self.revision = revision
  22     self.index = index
  23     self.value = value
  24     self.perf_time = perf_time
  25     self.build_time = build_time
  26     self.passed = passed
  27     self.external = external
  28
  29
  30 class MockBisectState(object):
  31
  32   def __init__(self):
  33     self.mock_revision_states = []
  34
  35     mock_bad_val = {'values': [100, 105, 95]}
  36     for i, rev in enumerate(['a', 'b']):
  37       mock_rev_state = MockRevisionState(rev, i, value=mock_bad_val, passed=0)
  38       self.mock_revision_states.append(mock_rev_state)
  39
  40     mock_good_val = {'values': [1, 2, 3]}
  41     for i, rev in enumerate(['c', 'd', 'e'], start=2):
  42       mock_rev_state = MockRevisionState(rev, i, value=mock_good_val, passed=1)
  43       self.mock_revision_states.append(mock_rev_state)
  44
  45   def GetRevisionStates(self):
  46     return self.mock_revision_states
  47
  48
  49 class MockBisectOptions(object):
  50
  51   def __init__(self):
  52     self.repeat_test_count = 3
  53
  54
  55 class BisectResultsTest(unittest.TestCase):
  56
  57   def setUp(self):
  58     self.mock_bisect_state = MockBisectState()
  59     self.mock_depot_registry = MockDepotRegistry()
  60     self.mock_opts = MockBisectOptions()
  61     self.mock_warnings = []
  62
  63     self.original_getcwd = os.getcwd
  64     self.original_chdir = os.chdir
  65     self.original_query_revision_info = source_control.QueryRevisionInfo
  66
  67     os.getcwd = lambda: '/path'
  68     os.chdir = lambda _: None
  69
  70     revision_infos = {'b': {'test': 'b'}, 'c': {'test': 'c'}}
  71     source_control.QueryRevisionInfo = lambda rev: revision_infos[rev]
  72
  73   def tearDown(self):
  74     os.getcwd = self.original_getcwd
  75     os.chdir = self.original_chdir
  76     source_control.QueryRevisionInfo = self.original_query_revision_info
  77
  78   def _AssertConfidence(self, score, bad_values, good_values):
  79     """Checks whether the given sets of values have a given confidence score.
  80
  81     The score represents our confidence that the two sets of values wouldn't
  82     be as different as they are just by chance; that is, that some real change
  83     occurred between the two sets of values.
  84
  85     Args:
  86       score: Expected confidence score.
  87       bad_values: First list of numbers.
  88       good_values: Second list of numbers.
  89     """
  90     confidence = BisectResults.ConfidenceScore(bad_values, good_values)
  91     self.assertEqual(score, confidence)
  92
  93   def testConfidenceScoreIsZeroOnTooFewLists(self):
  94     self._AssertConfidence(0.0, [], [1, 2])
  95     self._AssertConfidence(0.0, [1, 2], [])
  96     self._AssertConfidence(0.0, [1], [1, 2])
  97     self._AssertConfidence(0.0, [1, 2], [1])
  98
  99   def testConfidenceScore_ZeroConfidence(self):
 100     # The good and bad sets contain the same values, so the confidence that
 101     # they're different should be zero.
 102     self._AssertConfidence(0.0, [4, 5, 7, 6, 8, 7], [8, 7, 6, 7, 5, 4])
 103
 104   def testConfidenceScore_MediumConfidence(self):
 105     self._AssertConfidence(80.0, [0, 1, 1, 1, 2, 2], [1, 1, 1, 3, 3, 4])
 106
 107   def testConfidenceScore_HighConfidence(self):
 108     self._AssertConfidence(95.0, [0, 1, 1, 1, 2, 2], [1, 2, 2, 3, 3, 4])
 109
 110   def testConfidenceScore_VeryHighConfidence(self):
 111     # Confidence is high if the two sets of values have no internal variance.
 112     self._AssertConfidence(99.9, [1, 1, 1, 1], [1.2, 1.2, 1.2, 1.2])
 113     self._AssertConfidence(99.9, [1, 1, 1, 1], [1.01, 1.01, 1.01, 1.01])
 114
 115   def testConfidenceScore_UnbalancedSampleSize(self):
 116     # The second set of numbers only contains one number, so confidence is 0.
 117     self._AssertConfidence(0.0, [1.1, 1.2, 1.1, 1.2, 1.0, 1.3, 1.2], [1.4])
 118
 119   def testConfidenceScore_EmptySample(self):
 120     # Confidence is zero if either or both samples are empty.
 121     self._AssertConfidence(0.0, [], [])
 122     self._AssertConfidence(0.0, [], [1.1, 1.2, 1.1, 1.2, 1.0, 1.3, 1.2, 1.3])
 123     self._AssertConfidence(0.0, [1.1, 1.2, 1.1, 1.2, 1.0, 1.3, 1.2, 1.3], [])
 124
 125   def testConfidenceScore_FunctionalTestResults(self):
 126     self._AssertConfidence(80.0, [1, 1, 0, 1, 1, 1, 0, 1], [0, 0, 1, 0, 1, 0])
 127     self._AssertConfidence(99.9, [1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0])
 128
 129   def testConfidenceScore_RealWorldCases(self):
 130     """This method contains a set of data from actual bisect results.
 131
 132     The confidence scores asserted below were all copied from the actual
 133     results, so the purpose of this test method is mainly to show what the
 134     results for real cases are, and compare when we change the confidence
 135     score function in the future.
 136     """
 137     self._AssertConfidence(80, [133, 130, 132, 132, 130, 129], [129, 129, 125])
 138     self._AssertConfidence(99.5, [668, 667], [498, 498, 499])
 139     self._AssertConfidence(80, [67, 68], [65, 65, 67])
 140     self._AssertConfidence(0, [514], [514])
 141     self._AssertConfidence(90, [616, 613, 607, 615], [617, 619, 619, 617])
 142     self._AssertConfidence(0, [3.5, 5.8, 4.7, 3.5, 3.6], [2.8])
 143     self._AssertConfidence(90, [3, 3, 3], [2, 2, 2, 3])
 144     self._AssertConfidence(0, [1999004, 1999627], [223355])
 145     self._AssertConfidence(90, [1040, 934, 961], [876, 875, 789])
 146     self._AssertConfidence(90, [309, 305, 304], [302, 302, 299, 303, 298])
 147
 148   def testCorrectlyFindsBreakingRange(self):
 149     revision_states = self.mock_bisect_state.mock_revision_states
 150     revision_states[0].passed = 0
 151     revision_states[1].passed = 0
 152     revision_states[2].passed = 1
 153     revision_states[3].passed = 1
 154     revision_states[4].passed = 1
 155
 156     results = BisectResults(self.mock_bisect_state, self.mock_depot_registry,
 157                             self.mock_opts, self.mock_warnings)
 158     self.assertEqual(revision_states[2], results.first_working_revision)
 159     self.assertEqual(revision_states[1], results.last_broken_revision)
 160
 161   def testCorrectlyFindsBreakingRangeNotInOrder(self):
 162     revision_states = self.mock_bisect_state.mock_revision_states
 163     revision_states[0].passed = 0
 164     revision_states[1].passed = 1
 165     revision_states[2].passed = 0
 166     revision_states[3].passed = 1
 167     revision_states[4].passed = 1
 168
 169     results = BisectResults(self.mock_bisect_state, self.mock_depot_registry,
 170                             self.mock_opts, self.mock_warnings)
 171     self.assertEqual(revision_states[1], results.first_working_revision)
 172     self.assertEqual(revision_states[2], results.last_broken_revision)
 173
 174   def testCorrectlyFindsBreakingRangeIncompleteBisect(self):
 175     revision_states = self.mock_bisect_state.mock_revision_states
 176     revision_states[0].passed = 0
 177     revision_states[1].passed = 0
 178     revision_states[2].passed = '?'
 179     revision_states[3].passed = 1
 180     revision_states[4].passed = 1
 181
 182     results = BisectResults(self.mock_bisect_state, self.mock_depot_registry,
 183                             self.mock_opts, self.mock_warnings)
 184     self.assertEqual(revision_states[3], results.first_working_revision)
 185     self.assertEqual(revision_states[1], results.last_broken_revision)
 186
 187   def testFindBreakingRangeAllPassed(self):
 188     revision_states = self.mock_bisect_state.mock_revision_states
 189     revision_states[0].passed = 1
 190     revision_states[1].passed = 1
 191     revision_states[2].passed = 1
 192     revision_states[3].passed = 1
 193     revision_states[4].passed = 1
 194
 195     results = BisectResults(self.mock_bisect_state, self.mock_depot_registry,
 196                             self.mock_opts, self.mock_warnings)
 197     self.assertEqual(revision_states[0], results.first_working_revision)
 198     self.assertIsNone(results.last_broken_revision)
 199
 200   def testFindBreakingRangeNonePassed(self):
 201     revision_states = self.mock_bisect_state.mock_revision_states
 202     revision_states[0].passed = 0
 203     revision_states[1].passed = 0
 204     revision_states[2].passed = 0
 205     revision_states[3].passed = 0
 206     revision_states[4].passed = 0
 207
 208     results = BisectResults(self.mock_bisect_state, self.mock_depot_registry,
 209                             self.mock_opts, self.mock_warnings)
 210     self.assertIsNone(results.first_working_revision)
 211     self.assertEqual(revision_states[4], results.last_broken_revision)
 212
 213   def testCorrectlyComputesRegressionStatistics(self):
 214     revision_states = self.mock_bisect_state.mock_revision_states
 215     revision_states[0].passed = 0
 216     revision_states[0].value = {'values': [1000, 999, 998]}
 217     revision_states[1].passed = 0
 218     revision_states[1].value = {'values': [980, 1000, 999]}
 219     revision_states[2].passed = 1
 220     revision_states[2].value = {'values': [50, 45, 55]}
 221     revision_states[3].passed = 1
 222     revision_states[3].value = {'values': [45, 56, 45]}
 223     revision_states[4].passed = 1
 224     revision_states[4].value = {'values': [51, 41, 58]}
 225
 226     results = BisectResults(self.mock_bisect_state, self.mock_depot_registry,
 227                             self.mock_opts, self.mock_warnings)
 228     self.assertAlmostEqual(99.9, results.confidence)
 229     self.assertAlmostEqual(1909.86547085, results.regression_size)
 230     self.assertAlmostEqual(7.16625904, results.regression_std_err)
 231
 232   def testFindsCulpritRevisions(self):
 233     revision_states = self.mock_bisect_state.mock_revision_states
 234     revision_states[1].depot = 'chromium'
 235     revision_states[2].depot = 'webkit'
 236
 237     results = BisectResults(self.mock_bisect_state, self.mock_depot_registry,
 238                             self.mock_opts, self.mock_warnings)
 239
 240     self.assertEqual(1, len(results.culprit_revisions))
 241     self.assertEqual(('b', {'test': 'b'}, 'chromium'),
 242                      results.culprit_revisions[0])
 243
 244   def testNoResultBasedWarningsForNormalState(self):
 245     results = BisectResults(self.mock_bisect_state, self.mock_depot_registry,
 246                             self.mock_opts, self.mock_warnings)
 247     self.assertEqual(0, len(results.warnings))
 248
 249   def testWarningForMultipleCulpritRevisions(self):
 250     self.mock_bisect_state.mock_revision_states[2].passed = 'Skipped'
 251     results = BisectResults(self.mock_bisect_state, self.mock_depot_registry,
 252                             self.mock_opts, self.mock_warnings)
 253     self.assertEqual(1, len(results.warnings))
 254
 255   def testWarningForTooLowRetryLimit(self):
 256     self.mock_opts.repeat_test_count = 1
 257     results = BisectResults(self.mock_bisect_state, self.mock_depot_registry,
 258                             self.mock_opts, self.mock_warnings)
 259     self.assertEqual(1, len(results.warnings))
 260
 261   def testWarningForTooLowConfidence(self):
 262     revision_states = self.mock_bisect_state.mock_revision_states
 263     revision_states[2].value = {'values': [95, 90, 90]}
 264     revision_states[3].value = {'values': [95, 90, 90]}
 265     revision_states[4].value = {'values': [95, 90, 90]}
 266     results = BisectResults(self.mock_bisect_state, self.mock_depot_registry,
 267                             self.mock_opts, self.mock_warnings)
 268     self.assertGreater(results.confidence, 0)
 269     self.assertEqual(1, len(results.warnings))
 270
 271   def testWarningForZeroConfidence(self):
 272     revision_states = self.mock_bisect_state.mock_revision_states
 273     revision_states[2].value = {'values': [100, 105, 95]}
 274     revision_states[3].value = {'values': [100, 105, 95]}
 275     revision_states[4].value = {'values': [100, 105, 95]}
 276     results = BisectResults(self.mock_bisect_state, self.mock_depot_registry,
 277                             self.mock_opts, self.mock_warnings)
 278     self.assertEqual(0, results.confidence)
 279     self.assertEqual(1, len(results.warnings))
 280
 281
 282 if __name__ == '__main__':
 283   unittest.main()