tools/perf_expectations/tests/perf_expectations_unittest.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Verify perf_expectations.json can be loaded using simplejson.
   7
   8 perf_expectations.json is a JSON-formatted file.  This script verifies
   9 that simplejson can load it correctly.  It should catch most common
  10 formatting problems.
  11 """
  12
  13 import subprocess
  14 import sys
  15 import os
  16 import unittest
  17 import re
  18
  19 simplejson = None
  20
  21 def OnTestsLoad():
  22   old_path = sys.path
  23   script_path = os.path.dirname(sys.argv[0])
  24   load_path = None
  25   global simplejson
  26
  27   # This test script should be stored in src/tools/perf_expectations/.  That
  28   # directory will most commonly live in 2 locations:
  29   #
  30   #   - a regular Chromium checkout, in which case src/third_party
  31   #     is where to look for simplejson
  32   #
  33   #   - a buildbot checkout, in which case .../pylibs is where
  34   #     to look for simplejson
  35   #
  36   # Locate and install the correct path based on what we can find.
  37   #
  38   for path in ('../../../third_party', '../../../../../pylibs'):
  39     path = os.path.join(script_path, path)
  40     if os.path.exists(path) and os.path.isdir(path):
  41       load_path = os.path.abspath(path)
  42       break
  43
  44   if load_path is None:
  45     msg = "%s expects to live within a Chromium checkout" % sys.argv[0]
  46     raise Exception, "Error locating simplejson load path (%s)" % msg
  47
  48   # Try importing simplejson once.  If this succeeds, we found it and will
  49   # load it again later properly.  Fail if we cannot load it.
  50   sys.path.append(load_path)
  51   try:
  52     import simplejson as Simplejson
  53     simplejson = Simplejson
  54   except ImportError, e:
  55     msg = "%s expects to live within a Chromium checkout" % sys.argv[0]
  56     raise Exception, "Error trying to import simplejson from %s (%s)" % \
  57                      (load_path, msg)
  58   finally:
  59     sys.path = old_path
  60   return True
  61
  62 def LoadJsonFile(filename):
  63   f = open(filename, 'r')
  64   try:
  65     data = simplejson.load(f)
  66   except ValueError, e:
  67     f.seek(0)
  68     print "Error reading %s:\n%s" % (filename,
  69                                      f.read()[:50]+'...')
  70     raise e
  71   f.close()
  72   return data
  73
  74 OnTestsLoad()
  75
  76 CONFIG_JSON = os.path.join(os.path.dirname(sys.argv[0]),
  77                            '../chromium_perf_expectations.cfg')
  78 MAKE_EXPECTATIONS = os.path.join(os.path.dirname(sys.argv[0]),
  79                                  '../make_expectations.py')
  80 PERF_EXPECTATIONS = os.path.join(os.path.dirname(sys.argv[0]),
  81                                  '../perf_expectations.json')
  82
  83
  84 class PerfExpectationsUnittest(unittest.TestCase):
  85   def testPerfExpectations(self):
  86     # Test data is dictionary.
  87     perf_data = LoadJsonFile(PERF_EXPECTATIONS)
  88     if not isinstance(perf_data, dict):
  89       raise Exception('perf expectations is not a dict')
  90
  91     # Test the 'load' key.
  92     if not 'load' in perf_data:
  93       raise Exception("perf expectations is missing a load key")
  94     if not isinstance(perf_data['load'], bool):
  95       raise Exception("perf expectations load key has non-bool value")
  96
  97     # Test all key values are dictionaries.
  98     bad_keys = []
  99     for key in perf_data:
 100       if key == 'load':
 101         continue
 102       if not isinstance(perf_data[key], dict):
 103         bad_keys.append(key)
 104     if len(bad_keys) > 0:
 105       msg = "perf expectations keys have non-dict values"
 106       raise Exception("%s: %s" % (msg, bad_keys))
 107
 108     # Test all key values have delta and var keys.
 109     for key in perf_data:
 110       if key == 'load':
 111         continue
 112
 113       # First check if regress/improve is in the key's data.
 114       if 'regress' in perf_data[key]:
 115         if 'improve' not in perf_data[key]:
 116           bad_keys.append(key)
 117         if (not isinstance(perf_data[key]['regress'], int) and
 118             not isinstance(perf_data[key]['regress'], float)):
 119           bad_keys.append(key)
 120         if (not isinstance(perf_data[key]['improve'], int) and
 121             not isinstance(perf_data[key]['improve'], float)):
 122           bad_keys.append(key)
 123       else:
 124         # Otherwise check if delta/var is in the key's data.
 125         if 'delta' not in perf_data[key] or 'var' not in perf_data[key]:
 126           bad_keys.append(key)
 127         if (not isinstance(perf_data[key]['delta'], int) and
 128             not isinstance(perf_data[key]['delta'], float)):
 129           bad_keys.append(key)
 130         if (not isinstance(perf_data[key]['var'], int) and
 131             not isinstance(perf_data[key]['var'], float)):
 132           bad_keys.append(key)
 133
 134     if len(bad_keys) > 0:
 135       msg = "perf expectations key values missing or invalid delta/var"
 136       raise Exception("%s: %s" % (msg, bad_keys))
 137
 138     # Test all keys have the correct format.
 139     for key in perf_data:
 140       if key == 'load':
 141         continue
 142       # tools/buildbot/scripts/master/log_parser.py should have a matching
 143       # regular expression.
 144       if not re.match(r"^([\w\.-]+)/([\w\.-]+)/([\w\.-]+)/([\w\.-]+)$", key):
 145         bad_keys.append(key)
 146     if len(bad_keys) > 0:
 147       msg = "perf expectations keys in bad format, expected a/b/c/d"
 148       raise Exception("%s: %s" % (msg, bad_keys))
 149
 150   def testNoUpdatesNeeded(self):
 151     p = subprocess.Popen([MAKE_EXPECTATIONS, '-s'], stdout=subprocess.PIPE)
 152     p.wait();
 153     self.assertEqual(p.returncode, 0,
 154         msg='Update expectations first by running ./make_expectations.py')
 155
 156   def testConfigFile(self):
 157     # Test that the config file can be parsed as JSON.
 158     config = LoadJsonFile(CONFIG_JSON)
 159     # Require the following keys.
 160     if 'base_url' not in config:
 161       raise Exception('base_url not specified in config file')
 162     if 'perf_file' not in config:
 163       raise Exception('perf_file not specified in config file')
 164
 165
 166 if __name__ == '__main__':
 167   unittest.main()