tools/bisect-perf-regression_test.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import math
   6 import unittest
   7
   8 # Special import necessary because filename contains dash characters.
   9 bisect_perf_module = __import__('bisect-perf-regression')
  10
  11
  12 RESULTS_OUTPUT = """RESULT write_operations: write_operations= 23089 count
  13 RESULT read_bytes_gpu: read_bytes_gpu= 35201 kb
  14 RESULT write_bytes_gpu: write_bytes_gpu= 542 kb
  15 RESULT telemetry_page_measurement_results: num_failed= 0 count
  16 RESULT telemetry_page_measurement_results: num_errored= 0 count
  17 *RESULT Total: Total_ref= %(value)s
  18 """
  19
  20
  21 class BisectPerfRegressionTest(unittest.TestCase):
  22   """Test case for top-level functions in the bisect-perf-regrssion module."""
  23
  24   def setUp(self):
  25     """Sets up the test environment before each test method."""
  26     pass
  27
  28   def tearDown(self):
  29     """Cleans up the test environment after each test method."""
  30     pass
  31
  32   def testParseDEPSStringManually(self):
  33     """Tests DEPS parsing."""
  34     bisect_options = bisect_perf_module.BisectOptions()
  35     bisect_instance = bisect_perf_module.BisectPerformanceMetrics(
  36         None, bisect_options)
  37
  38     deps_file_contents = """
  39 vars = {
  40     'ffmpeg_hash':
  41          '@ac4a9f31fe2610bd146857bbd55d7a260003a888',
  42     'webkit_url':
  43          'https://chromium.googlesource.com/chromium/blink.git',
  44     'git_url':
  45          'https://chromium.googlesource.com',
  46     'webkit_rev':
  47          '@e01ac0a267d1017288bc67fa3c366b10469d8a24',
  48     'angle_revision':
  49          '74697cf2064c0a2c0d7e1b1b28db439286766a05'
  50 }"""
  51
  52     # Should only expect svn/git revisions to come through, and urls to be
  53     # filtered out.
  54     expected_vars_dict = {
  55         'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888',
  56         'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24',
  57         'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05'
  58     }
  59     vars_dict = bisect_instance._ParseRevisionsFromDEPSFileManually(
  60         deps_file_contents)
  61     self.assertEqual(vars_dict, expected_vars_dict)
  62
  63   def testCalculateTruncatedMeanRaisesError(self):
  64     """CalculateTrunctedMean raises an error when passed an empty list."""
  65     with self.assertRaises(TypeError):
  66       bisect_perf_module.CalculateTruncatedMean([], 0)
  67
  68   def testCalculateMeanSingleNum(self):
  69     """Tests the CalculateMean function with a single number."""
  70     self.assertEqual(3.0, bisect_perf_module.CalculateMean([3]))
  71
  72   def testCalculateMeanShortList(self):
  73     """Tests the CalculateMean function with a short list."""
  74     self.assertEqual(0.5, bisect_perf_module.CalculateMean([-3, 0, 1, 4]))
  75
  76   def testCalculateMeanCompareAlternateImplementation(self):
  77     """Tests CalculateMean by comparing against an alternate implementation."""
  78     def AlternateMeanFunction(values):
  79       """Simple arithmetic mean function."""
  80       return sum(values) / float(len(values))
  81     test_values_lists = [[1], [5, 6.5, 1.2, 3], [-3, 0, 1, 4],
  82                          [-3, -1, 0.12, 0.752, 3.33, 8, 16, 32, 439]]
  83     for values in test_values_lists:
  84       self.assertEqual(
  85           AlternateMeanFunction(values),
  86           bisect_perf_module.CalculateMean(values))
  87
  88   def testCalculateConfidence(self):
  89     """Tests the confidence calculation."""
  90     bad_values = [[0, 1], [1, 2]]
  91     good_values = [[6, 7], [7, 8]]
  92     # Closest means are mean(1, 2) and mean(6, 7).
  93     distance = 6.5 - 1.5
  94     # Standard deviation of [n-1, n, n, n+1] is 0.8165.
  95     stddev_sum = 0.8165 + 0.8165
  96     # Expected confidence is an int in the range [0, 100].
  97     expected_confidence = min(100, int(100 * distance / float(stddev_sum)))
  98     self.assertEqual(
  99         expected_confidence,
 100         bisect_perf_module.CalculateConfidence(bad_values, good_values))
 101
 102   def testCalculateConfidence0(self):
 103     """Tests the confidence calculation when it's expected to be 0."""
 104     bad_values = [[0, 1], [1, 2], [4, 5], [0, 2]]
 105     good_values = [[4, 5], [6, 7], [7, 8]]
 106     # Both groups have value lists with means of 4.5, which means distance
 107     # between groups is zero, and thus confidence is zero.
 108     self.assertEqual(
 109         0, bisect_perf_module.CalculateConfidence(bad_values, good_values))
 110
 111   def testCalculateConfidence100(self):
 112     """Tests the confidence calculation when it's expected to be 100."""
 113     bad_values = [[1, 1], [1, 1]]
 114     good_values = [[1.2, 1.2], [1.2, 1.2]]
 115     # Standard deviation in both groups is zero, so confidence is 100.
 116     self.assertEqual(
 117         100, bisect_perf_module.CalculateConfidence(bad_values, good_values))
 118
 119   def testCalculateRelativeChange(self):
 120     """Tests the common cases for calculating relative change."""
 121     # The change is relative to the first value, regardless of which is bigger.
 122     self.assertEqual(0.5, bisect_perf_module.CalculateRelativeChange(1.0, 1.5))
 123     self.assertEqual(0.5, bisect_perf_module.CalculateRelativeChange(2.0, 1.0))
 124
 125   def testCalculateRelativeChangeFromZero(self):
 126     """Tests what happens when relative change from zero is calculated."""
 127     # If the first number is zero, then the result is not a number.
 128     self.assertEqual(0, bisect_perf_module.CalculateRelativeChange(0, 0))
 129     self.assertTrue(
 130         math.isnan(bisect_perf_module.CalculateRelativeChange(0, 1)))
 131     self.assertTrue(
 132         math.isnan(bisect_perf_module.CalculateRelativeChange(0, -1)))
 133
 134   def testCalculateRelativeChangeWithNegatives(self):
 135     """Tests that relative change given is always positive."""
 136     self.assertEqual(3.0, bisect_perf_module.CalculateRelativeChange(-1, 2))
 137     self.assertEqual(3.0, bisect_perf_module.CalculateRelativeChange(1, -2))
 138     self.assertEqual(1.0, bisect_perf_module.CalculateRelativeChange(-1, -2))
 139
 140   def testTryParseResultValuesFromOutputWithSingleValue(self):
 141     """Tests result pattern <*>RESULT <graph>: <trace>= <value>"""
 142     bisect_options = bisect_perf_module.BisectOptions()
 143     bisect_instance = bisect_perf_module.BisectPerformanceMetrics(
 144         None, bisect_options)
 145     metrics = ['Total', 'Total_ref']
 146     self.assertEqual(
 147         [66.88], bisect_instance.TryParseResultValuesFromOutput(
 148             metrics, RESULTS_OUTPUT % {'value': '66.88 kb'}))
 149     self.assertEqual(
 150         [66.88], bisect_instance.TryParseResultValuesFromOutput(
 151             metrics, RESULTS_OUTPUT % {'value': '66.88kb'}))
 152     self.assertEqual(
 153         [66.88], bisect_instance.TryParseResultValuesFromOutput(
 154             metrics, RESULTS_OUTPUT % {'value': ' 66.88 '}))
 155     self.assertEqual(
 156         [-66.88], bisect_instance.TryParseResultValuesFromOutput(
 157             metrics, RESULTS_OUTPUT % {'value': ' -66.88 kb'}))
 158     self.assertEqual(
 159         [66], bisect_instance.TryParseResultValuesFromOutput(
 160             metrics, RESULTS_OUTPUT % {'value': '66 kb'}))
 161     self.assertEqual(
 162         [.66], bisect_instance.TryParseResultValuesFromOutput(
 163             metrics, RESULTS_OUTPUT % {'value': '.66 kb'}))
 164     self.assertEqual(
 165         [], bisect_instance.TryParseResultValuesFromOutput(
 166             metrics, RESULTS_OUTPUT % {'value': '. kb'}))
 167     self.assertEqual(
 168         [], bisect_instance.TryParseResultValuesFromOutput(
 169             metrics, RESULTS_OUTPUT % {'value': 'aaa kb'}))
 170
 171   def testTryParseResultValuesFromOutputWithMulitValue(self):
 172     """Tests result pattern <*>RESULT <graph>: <trace>= [<value>,<value>, ..]"""
 173     bisect_options = bisect_perf_module.BisectOptions()
 174     bisect_instance = bisect_perf_module.BisectPerformanceMetrics(
 175         None, bisect_options)
 176     metrics = ['Total', 'Total_ref']
 177     self.assertEqual(
 178         [66.88], bisect_instance.TryParseResultValuesFromOutput(
 179             metrics, RESULTS_OUTPUT % {'value': '[66.88] kb'}))
 180     self.assertEqual(
 181         [66.88, 99.44], bisect_instance.TryParseResultValuesFromOutput(
 182             metrics, RESULTS_OUTPUT % {'value': '[66.88, 99.44]kb'}))
 183     self.assertEqual(
 184         [66.88, 99.44], bisect_instance.TryParseResultValuesFromOutput(
 185             metrics, RESULTS_OUTPUT % {'value': '[ 66.88, 99.44 ]'}))
 186     self.assertEqual(
 187         [-66.88, 99.44], bisect_instance.TryParseResultValuesFromOutput(
 188             metrics, RESULTS_OUTPUT % {'value': '[-66.88,99.44] kb'}))
 189     self.assertEqual(
 190         [-66, 99], bisect_instance.TryParseResultValuesFromOutput(
 191             metrics, RESULTS_OUTPUT % {'value': '[-66,99] kb'}))
 192     self.assertEqual(
 193         [-66, 99], bisect_instance.TryParseResultValuesFromOutput(
 194             metrics, RESULTS_OUTPUT % {'value': '[-66,99,] kb'}))
 195     self.assertEqual(
 196         [.66, .99], bisect_instance.TryParseResultValuesFromOutput(
 197             metrics, RESULTS_OUTPUT % {'value': '[.66,.99] kb'}))
 198     self.assertEqual(
 199         [], bisect_instance.TryParseResultValuesFromOutput(
 200             metrics, RESULTS_OUTPUT % {'value': '[] kb'}))
 201     self.assertEqual(
 202         [], bisect_instance.TryParseResultValuesFromOutput(
 203             metrics, RESULTS_OUTPUT % {'value': '[-66,abc] kb'}))
 204
 205   def testTryParseResultValuesFromOutputWithMeanStd(self):
 206     """Tests result pattern <*>RESULT <graph>: <trace>= {<mean, std}"""
 207     bisect_options = bisect_perf_module.BisectOptions()
 208     bisect_instance = bisect_perf_module.BisectPerformanceMetrics(
 209         None, bisect_options)
 210     metrics = ['Total', 'Total_ref']
 211     self.assertEqual(
 212         [33.22], bisect_instance.TryParseResultValuesFromOutput(
 213             metrics, RESULTS_OUTPUT % {'value': '{33.22, 3.6} kb'}))
 214     self.assertEqual(
 215         [33.22], bisect_instance.TryParseResultValuesFromOutput(
 216             metrics, RESULTS_OUTPUT % {'value': '{33.22,3.6}kb'}))
 217     self.assertEqual(
 218         [33.22], bisect_instance.TryParseResultValuesFromOutput(
 219             metrics, RESULTS_OUTPUT % {'value': '{33.22,3.6} kb'}))
 220     self.assertEqual(
 221         [33.22], bisect_instance.TryParseResultValuesFromOutput(
 222             metrics, RESULTS_OUTPUT % {'value': '{ 33.22,3.6 }kb'}))
 223     self.assertEqual(
 224         [-33.22], bisect_instance.TryParseResultValuesFromOutput(
 225             metrics, RESULTS_OUTPUT % {'value': '{-33.22,3.6}kb'}))
 226     self.assertEqual(
 227         [22], bisect_instance.TryParseResultValuesFromOutput(
 228             metrics, RESULTS_OUTPUT % {'value': '{22,6}kb'}))
 229     self.assertEqual(
 230         [.22], bisect_instance.TryParseResultValuesFromOutput(
 231             metrics, RESULTS_OUTPUT % {'value': '{.22,6}kb'}))
 232     self.assertEqual(
 233         [], bisect_instance.TryParseResultValuesFromOutput(
 234             metrics, RESULTS_OUTPUT % {'value': '{.22,6, 44}kb'}))
 235     self.assertEqual(
 236         [], bisect_instance.TryParseResultValuesFromOutput(
 237             metrics, RESULTS_OUTPUT % {'value': '{}kb'}))
 238     self.assertEqual(
 239         [], bisect_instance.TryParseResultValuesFromOutput(
 240             metrics, RESULTS_OUTPUT % {'value': '{XYZ}kb'}))
 241
 242   def testGetCompatibleCommand(self):
 243     bisect_options = bisect_perf_module.BisectOptions()
 244     bisect_options.output_buildbot_annotations = None
 245     source_control = bisect_perf_module.DetermineAndCreateSourceControl(
 246         bisect_options)
 247     bisect_instance = bisect_perf_module.BisectPerformanceMetrics(
 248         source_control, bisect_options)
 249     bisect_instance.opts.target_platform = 'android'
 250     depot = 'chromium'
 251     # android-chrome-shell -> android-chromium-testshell
 252     revision = 274857
 253     git_revision = bisect_instance.source_control.ResolveToRevision(
 254         revision, 'chromium', 100)
 255     command = ('tools/perf/run_benchmark -v '
 256                '--browser=android-chrome-shell page_cycler.intl_ja_zh')
 257     expected_command = ('tools/perf/run_benchmark -v --browser='
 258                         'android-chromium-testshell page_cycler.intl_ja_zh')
 259     self.assertEqual(
 260         bisect_instance.GetCompatibleCommand(command, git_revision, depot),
 261         expected_command)
 262
 263     # android-chromium-testshell -> android-chromium-testshell
 264     revision = 274858
 265     git_revision = bisect_instance.source_control.ResolveToRevision(
 266         revision, 'chromium', 100)
 267     command = ('tools/perf/run_benchmark -v '
 268                '--browser=android-chromium-testshell page_cycler.intl_ja_zh')
 269     expected_command = ('tools/perf/run_benchmark -v --browser='
 270                         'android-chromium-testshell page_cycler.intl_ja_zh')
 271     self.assertEqual(
 272         bisect_instance.GetCompatibleCommand(command, git_revision, depot),
 273         expected_command)
 274
 275     # android-chromium-testshell -> android-chrome-shell
 276     revision = 276628
 277     git_revision = bisect_instance.source_control.ResolveToRevision(
 278         revision, 'chromium', 100)
 279     command = ('tools/perf/run_benchmark -v '
 280                '--browser=android-chromium-testshell page_cycler.intl_ja_zh')
 281     expected_command = ('tools/perf/run_benchmark -v --browser='
 282                         'android-chrome-shell page_cycler.intl_ja_zh')
 283     self.assertEqual(
 284         bisect_instance.GetCompatibleCommand(command, git_revision, depot),
 285         expected_command)
 286     # android-chrome-shell -> android-chrome-shell
 287     command = ('tools/perf/run_benchmark -v '
 288                '--browser=android-chrome-shell page_cycler.intl_ja_zh')
 289     expected_command = ('tools/perf/run_benchmark -v --browser='
 290                         'android-chrome-shell page_cycler.intl_ja_zh')
 291     self.assertEqual(
 292         bisect_instance.GetCompatibleCommand(command, git_revision, depot),
 293         expected_command)
 294     # Not android platform
 295     bisect_instance.opts.target_platform = 'chromium'
 296     command = ('tools/perf/run_benchmark -v '
 297                '--browser=release page_cycler.intl_ja_zh')
 298     expected_command = ('tools/perf/run_benchmark -v --browser='
 299                         'release page_cycler.intl_ja_zh')
 300     self.assertEqual(
 301         bisect_instance.GetCompatibleCommand(command, git_revision, depot),
 302         expected_command)
 303
 304 if __name__ == '__main__':
 305   unittest.main()