tools/perf_expectations/update_perf_expectations_unittest.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5 """Unit tests for update_perf_expectations."""
   6 import copy
   7 from StringIO import StringIO
   8 import unittest
   9 import make_expectations as perf_ex_lib
  10 import update_perf_expectations as upe_mod
  11
  12
  13 # A separate .json file contains the list of test cases we'll use.
  14 # The tests used to be defined inline here, but are >80 characters in length.
  15 # Now they are expected to be defined in file ./sample_test_cases.json.
  16 # Create a dictionary of tests using .json file.
  17 all_tests = perf_ex_lib.ConvertJsonIntoDict(
  18     perf_ex_lib.ReadFile('sample_test_cases.json'))
  19 # Get all keys.
  20 all_tests_keys = all_tests.keys()
  21
  22
  23 def VerifyPreparedTests(self, tests_to_update, reva, revb):
  24   # Work with a copy of the set of tests.
  25   all_tests_copy = copy.deepcopy(all_tests)
  26   upe_mod.PrepareTestsForUpdate(tests_to_update, all_tests_copy, reva, revb)
  27   # Make sure reva < revb
  28   if reva > revb:
  29     temp = reva
  30     reva = revb
  31     revb = temp
  32   # Run through all tests and make sure only those that were
  33   # specified to be modified had their 'sha1' value removed.
  34   for test_key in all_tests_keys:
  35     new_test_value = all_tests_copy[test_key]
  36     original_test_value = all_tests[test_key]
  37     if test_key in tests_to_update:
  38       # Make sure there is no "sha1".
  39       self.assertFalse('sha1' in new_test_value)
  40       # Make sure reva and revb values are correctly set.
  41       self.assertEqual(reva, new_test_value['reva'])
  42       self.assertEqual(revb, new_test_value['revb'])
  43     else:
  44       # Make sure there is an "sha1" value
  45       self.assertTrue('sha1' in new_test_value)
  46       # Make sure the sha1, reva and revb values have not changed.
  47       self.assertEqual(original_test_value['sha1'], new_test_value['sha1'])
  48       self.assertEqual(original_test_value['reva'], new_test_value['reva'])
  49       self.assertEqual(original_test_value['revb'], new_test_value['revb'])
  50
  51
  52 class UpdatePerfExpectationsTest(unittest.TestCase):
  53   def testFilterMatch(self):
  54     """Verifies different regular expressions test filter."""
  55     self.maxDiff = None
  56     # Tests to update specified by a single literal string.
  57     tests_to_update = 'win-release/media_tests_av_perf/fps/tulip2.webm'
  58     expected_tests_list = ['win-release/media_tests_av_perf/fps/tulip2.webm']
  59     self.assertEqual(expected_tests_list,
  60                      upe_mod.GetMatchingTests(tests_to_update,
  61                                               all_tests_keys))
  62
  63     # Tests to update specified by a single reg-ex
  64     tests_to_update = 'win-release/media_tests_av_perf/fps.*'
  65     expected_tests_list = ['win-release/media_tests_av_perf/fps/crowd1080.webm',
  66                            'win-release/media_tests_av_perf/fps/crowd2160.webm',
  67                            'win-release/media_tests_av_perf/fps/crowd360.webm',
  68                            'win-release/media_tests_av_perf/fps/crowd480.webm',
  69                            'win-release/media_tests_av_perf/fps/crowd720.webm',
  70                            'win-release/media_tests_av_perf/fps/tulip2.m4a',
  71                            'win-release/media_tests_av_perf/fps/tulip2.mp3',
  72                            'win-release/media_tests_av_perf/fps/tulip2.mp4',
  73                            'win-release/media_tests_av_perf/fps/tulip2.ogg',
  74                            'win-release/media_tests_av_perf/fps/tulip2.ogv',
  75                            'win-release/media_tests_av_perf/fps/tulip2.wav',
  76                            'win-release/media_tests_av_perf/fps/tulip2.webm']
  77     actual_list = upe_mod.GetMatchingTests(tests_to_update,
  78                                            all_tests_keys)
  79     actual_list.sort()
  80     self.assertEqual(expected_tests_list, actual_list)
  81
  82     # Tests to update are specified by a single reg-ex, spanning multiple OSes.
  83     tests_to_update = '.*-release/media_tests_av_perf/fps.*'
  84     expected_tests_list = ['linux-release/media_tests_av_perf/fps/tulip2.m4a',
  85                            'linux-release/media_tests_av_perf/fps/tulip2.mp3',
  86                            'linux-release/media_tests_av_perf/fps/tulip2.mp4',
  87                            'linux-release/media_tests_av_perf/fps/tulip2.ogg',
  88                            'linux-release/media_tests_av_perf/fps/tulip2.ogv',
  89                            'linux-release/media_tests_av_perf/fps/tulip2.wav',
  90                            'win-release/media_tests_av_perf/fps/crowd1080.webm',
  91                            'win-release/media_tests_av_perf/fps/crowd2160.webm',
  92                            'win-release/media_tests_av_perf/fps/crowd360.webm',
  93                            'win-release/media_tests_av_perf/fps/crowd480.webm',
  94                            'win-release/media_tests_av_perf/fps/crowd720.webm',
  95                            'win-release/media_tests_av_perf/fps/tulip2.m4a',
  96                            'win-release/media_tests_av_perf/fps/tulip2.mp3',
  97                            'win-release/media_tests_av_perf/fps/tulip2.mp4',
  98                            'win-release/media_tests_av_perf/fps/tulip2.ogg',
  99                            'win-release/media_tests_av_perf/fps/tulip2.ogv',
 100                            'win-release/media_tests_av_perf/fps/tulip2.wav',
 101                            'win-release/media_tests_av_perf/fps/tulip2.webm']
 102     actual_list = upe_mod.GetMatchingTests(tests_to_update,
 103                                            all_tests_keys)
 104     actual_list.sort()
 105     self.assertEqual(expected_tests_list, actual_list)
 106
 107   def testLinesFromInputFile(self):
 108     """Verifies different string formats specified in input file."""
 109
 110     # Tests to update have been specified by a single literal string in
 111     # an input file.
 112     # Use the StringIO class to mock a file object.
 113     lines_from_file = StringIO(
 114         'win-release/media_tests_av_perf/fps/tulip2.webm')
 115     contents = lines_from_file.read()
 116     expected_tests_list = ['win-release/media_tests_av_perf/fps/tulip2.webm']
 117     actual_list = upe_mod.GetTestsToUpdate(contents, all_tests_keys)
 118     actual_list.sort()
 119     self.assertEqual(expected_tests_list, actual_list)
 120     lines_from_file.close()
 121
 122     # Tests to update specified by a single reg-ex in an input file.
 123     lines_from_file = StringIO('win-release/media_tests_av_perf/fps/tulip2.*\n')
 124     contents = lines_from_file.read()
 125     expected_tests_list = ['win-release/media_tests_av_perf/fps/tulip2.m4a',
 126                            'win-release/media_tests_av_perf/fps/tulip2.mp3',
 127                            'win-release/media_tests_av_perf/fps/tulip2.mp4',
 128                            'win-release/media_tests_av_perf/fps/tulip2.ogg',
 129                            'win-release/media_tests_av_perf/fps/tulip2.ogv',
 130                            'win-release/media_tests_av_perf/fps/tulip2.wav',
 131                            'win-release/media_tests_av_perf/fps/tulip2.webm']
 132     actual_list = upe_mod.GetTestsToUpdate(contents, all_tests_keys)
 133     actual_list.sort()
 134     self.assertEqual(expected_tests_list, actual_list)
 135     lines_from_file.close()
 136
 137     # Tests to update specified by multiple lines in an input file.
 138     lines_from_file = StringIO(
 139         '.*-release/media_tests_av_perf/fps/tulip2.*\n'
 140         'win-release/media_tests_av_perf/dropped_fps/tulip2.*\n'
 141         'linux-release/media_tests_av_perf/audio_latency/latency')
 142     contents = lines_from_file.read()
 143     expected_tests_list = [
 144         'linux-release/media_tests_av_perf/audio_latency/latency',
 145         'linux-release/media_tests_av_perf/fps/tulip2.m4a',
 146         'linux-release/media_tests_av_perf/fps/tulip2.mp3',
 147         'linux-release/media_tests_av_perf/fps/tulip2.mp4',
 148         'linux-release/media_tests_av_perf/fps/tulip2.ogg',
 149         'linux-release/media_tests_av_perf/fps/tulip2.ogv',
 150         'linux-release/media_tests_av_perf/fps/tulip2.wav',
 151         'win-release/media_tests_av_perf/dropped_fps/tulip2.wav',
 152         'win-release/media_tests_av_perf/dropped_fps/tulip2.webm',
 153         'win-release/media_tests_av_perf/fps/tulip2.m4a',
 154         'win-release/media_tests_av_perf/fps/tulip2.mp3',
 155         'win-release/media_tests_av_perf/fps/tulip2.mp4',
 156         'win-release/media_tests_av_perf/fps/tulip2.ogg',
 157         'win-release/media_tests_av_perf/fps/tulip2.ogv',
 158         'win-release/media_tests_av_perf/fps/tulip2.wav',
 159         'win-release/media_tests_av_perf/fps/tulip2.webm']
 160     actual_list = upe_mod.GetTestsToUpdate(contents, all_tests_keys)
 161     actual_list.sort()
 162     self.assertEqual(expected_tests_list, actual_list)
 163     lines_from_file.close()
 164
 165   def testPreparingForUpdate(self):
 166     """Verifies that tests to be modified are changed as expected."""
 167     tests_to_update = [
 168         'linux-release/media_tests_av_perf/audio_latency/latency',
 169         'linux-release/media_tests_av_perf/fps/tulip2.m4a',
 170         'linux-release/media_tests_av_perf/fps/tulip2.mp3',
 171         'linux-release/media_tests_av_perf/fps/tulip2.mp4',
 172         'linux-release/media_tests_av_perf/fps/tulip2.ogg',
 173         'linux-release/media_tests_av_perf/fps/tulip2.ogv',
 174         'linux-release/media_tests_av_perf/fps/tulip2.wav',
 175         'win-release/media_tests_av_perf/dropped_fps/tulip2.wav',
 176         'win-release/media_tests_av_perf/dropped_fps/tulip2.webm',
 177         'win-release/media_tests_av_perf/fps/tulip2.mp3',
 178         'win-release/media_tests_av_perf/fps/tulip2.mp4',
 179         'win-release/media_tests_av_perf/fps/tulip2.ogg',
 180         'win-release/media_tests_av_perf/fps/tulip2.ogv',
 181         'win-release/media_tests_av_perf/fps/tulip2.wav',
 182         'win-release/media_tests_av_perf/fps/tulip2.webm']
 183     # Test regular positive integers.
 184     reva = 12345
 185     revb = 54321
 186     VerifyPreparedTests(self, tests_to_update, reva, revb)
 187     # Test negative values.
 188     reva = -54321
 189     revb = 12345
 190     with self.assertRaises(ValueError):
 191       upe_mod.PrepareTestsForUpdate(tests_to_update, all_tests, reva, revb)
 192     # Test reva greater than revb.
 193     reva = 54321
 194     revb = 12345
 195     upe_mod.PrepareTestsForUpdate(tests_to_update, all_tests, reva, revb)
 196     # Test non-integer values
 197     reva = 'sds'
 198     revb = 12345
 199     with self.assertRaises(ValueError):
 200       upe_mod.PrepareTestsForUpdate(tests_to_update, all_tests, reva, revb)
 201
 202
 203 if __name__ == '__main__':
 204   unittest.main()