chrome/test/mini_installer/test_installer.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """This script tests the installer with test cases specified in the config file.
   6
   7 For each test case, it checks that the machine states after the execution of
   8 each command match the expected machine states. For more details, take a look at
   9 the design documentation at http://goo.gl/Q0rGM6
  10 """
  11
  12 import argparse
  13 import datetime
  14 import inspect
  15 import json
  16 import os
  17 import subprocess
  18 import sys
  19 import time
  20 import unittest
  21 import _winreg
  22
  23 from variable_expander import VariableExpander
  24 import verifier_runner
  25
  26
  27 def LogMessage(message):
  28   """Logs a message to stderr.
  29
  30   Args:
  31     message: The message string to be logged.
  32   """
  33   now = datetime.datetime.now()
  34   frameinfo = inspect.getframeinfo(inspect.currentframe().f_back)
  35   filename = os.path.basename(frameinfo.filename)
  36   line = frameinfo.lineno
  37   sys.stderr.write('[%s:%s(%s)] %s\n' % (now.strftime('%m%d/%H%M%S'),
  38                                          filename, line, message))
  39
  40
  41 class Config:
  42   """Describes the machine states, actions, and test cases.
  43
  44   Attributes:
  45     states: A dictionary where each key is a state name and the associated value
  46         is a property dictionary describing that state.
  47     actions: A dictionary where each key is an action name and the associated
  48         value is the action's command.
  49     tests: An array of test cases.
  50   """
  51   def __init__(self):
  52     self.states = {}
  53     self.actions = {}
  54     self.tests = []
  55
  56
  57 class InstallerTest(unittest.TestCase):
  58   """Tests a test case in the config file."""
  59
  60   def __init__(self, name, test, config, variable_expander, quiet):
  61     """Constructor.
  62
  63     Args:
  64       name: The name of this test.
  65       test: An array of alternating state names and action names, starting and
  66           ending with state names.
  67       config: The Config object.
  68       variable_expander: A VariableExpander object.
  69     """
  70     super(InstallerTest, self).__init__()
  71     self._name = name
  72     self._test = test
  73     self._config = config
  74     self._variable_expander = variable_expander
  75     self._quiet = quiet
  76     self._verifier_runner = verifier_runner.VerifierRunner()
  77     self._clean_on_teardown = True
  78
  79   def __str__(self):
  80     """Returns a string representing the test case.
  81
  82     Returns:
  83       A string created by joining state names and action names together with
  84       ' -> ', for example, 'Test: clean -> install chrome -> chrome_installed'.
  85     """
  86     return '%s: %s\n' % (self._name, ' -> '.join(self._test))
  87
  88   def id(self):
  89     """Returns the name of the test."""
  90     # Overridden from unittest.TestCase so that id() contains the name of the
  91     # test case from the config file in place of the name of this class's test
  92     # function.
  93     return unittest.TestCase.id(self).replace(self._testMethodName, self._name)
  94
  95   def runTest(self):
  96     """Run the test case."""
  97     # |test| is an array of alternating state names and action names, starting
  98     # and ending with state names. Therefore, its length must be odd.
  99     self.assertEqual(1, len(self._test) % 2,
 100                      'The length of test array must be odd')
 101
 102     state = self._test[0]
 103     self._VerifyState(state)
 104
 105     # Starting at index 1, we loop through pairs of (action, state).
 106     for i in range(1, len(self._test), 2):
 107       action = self._test[i]
 108       if not self._quiet:
 109         LogMessage('Beginning action %s' % action)
 110       RunCommand(self._config.actions[action], self._variable_expander)
 111       if not self._quiet:
 112         LogMessage('Finished action %s' % action)
 113
 114       state = self._test[i + 1]
 115       self._VerifyState(state)
 116
 117     # If the test makes it here, it means it was successful, because RunCommand
 118     # and _VerifyState throw an exception on failure.
 119     self._clean_on_teardown = False
 120
 121   def tearDown(self):
 122     """Cleans up the machine if the test case fails."""
 123     if self._clean_on_teardown:
 124       RunCleanCommand(True, self._variable_expander)
 125
 126   def shortDescription(self):
 127     """Overridden from unittest.TestCase.
 128
 129     We return None as the short description to suppress its printing.
 130     The default implementation of this method returns the docstring of the
 131     runTest method, which is not useful since it's the same for every test case.
 132     The description from the __str__ method is informative enough.
 133     """
 134     return None
 135
 136   def _VerifyState(self, state):
 137     """Verifies that the current machine state matches a given state.
 138
 139     Args:
 140       state: A state name.
 141     """
 142     if not self._quiet:
 143       LogMessage('Verifying state %s' % state)
 144     try:
 145       self._verifier_runner.VerifyAll(self._config.states[state],
 146                                       self._variable_expander)
 147     except AssertionError as e:
 148       # If an AssertionError occurs, we intercept it and add the state name
 149       # to the error message so that we know where the test fails.
 150       raise AssertionError("In state '%s', %s" % (state, e))
 151
 152
 153 def RunCommand(command, variable_expander):
 154   """Runs the given command from the current file's directory.
 155
 156   This function throws an Exception if the command returns with non-zero exit
 157   status.
 158
 159   Args:
 160     command: A command to run. It is expanded using Expand.
 161     variable_expander: A VariableExpander object.
 162   """
 163   expanded_command = variable_expander.Expand(command)
 164   script_dir = os.path.dirname(os.path.abspath(__file__))
 165   exit_status = subprocess.call(expanded_command, shell=True, cwd=script_dir)
 166   if exit_status != 0:
 167     raise Exception('Command %s returned non-zero exit status %s' % (
 168         expanded_command, exit_status))
 169
 170
 171 def DeleteGoogleUpdateRegistration(system_level, variable_expander):
 172   """Deletes Chrome's registration with Google Update.
 173
 174   Args:
 175     system_level: True if system-level Chrome is to be deleted.
 176     variable_expander: A VariableExpander object.
 177   """
 178   root = (_winreg.HKEY_LOCAL_MACHINE if system_level
 179           else _winreg.HKEY_CURRENT_USER)
 180   key_name = variable_expander.Expand('$CHROME_UPDATE_REGISTRY_SUBKEY')
 181   try:
 182     key_handle = _winreg.OpenKey(root, key_name, 0,
 183                                  _winreg.KEY_SET_VALUE |
 184                                  _winreg.KEY_WOW64_32KEY)
 185     _winreg.DeleteValue(key_handle, 'pv')
 186   except WindowsError:
 187     # The key isn't present, so there is no value to delete.
 188     pass
 189
 190
 191 def RunCleanCommand(force_clean, variable_expander):
 192   """Puts the machine in the clean state (i.e. Chrome not installed).
 193
 194   Args:
 195     force_clean: A boolean indicating whether to force cleaning existing
 196         installations.
 197     variable_expander: A VariableExpander object.
 198   """
 199   # TODO(sukolsak): Handle Chrome SxS installs.
 200   interactive_option = '--interactive' if not force_clean else ''
 201   for system_level in (False, True):
 202     level_option = '--system-level' if system_level else ''
 203     command = ('python uninstall_chrome.py '
 204                '--chrome-long-name="$CHROME_LONG_NAME" '
 205                '--no-error-if-absent %s %s' %
 206                (level_option, interactive_option))
 207     RunCommand(command, variable_expander)
 208     if force_clean:
 209       DeleteGoogleUpdateRegistration(system_level, variable_expander)
 210
 211
 212 def MergePropertyDictionaries(current_property, new_property):
 213   """Merges the new property dictionary into the current property dictionary.
 214
 215   This is different from general dictionary merging in that, in case there are
 216   keys with the same name, we merge values together in the first level, and we
 217   override earlier values in the second level. For more details, take a look at
 218   http://goo.gl/uE0RoR
 219
 220   Args:
 221     current_property: The property dictionary to be modified.
 222     new_property: The new property dictionary.
 223   """
 224   for key, value in new_property.iteritems():
 225     if key not in current_property:
 226       current_property[key] = value
 227     else:
 228       assert(isinstance(current_property[key], dict) and
 229           isinstance(value, dict))
 230       # This merges two dictionaries together. In case there are keys with
 231       # the same name, the latter will override the former.
 232       current_property[key] = dict(
 233           current_property[key].items() + value.items())
 234
 235
 236 def ParsePropertyFiles(directory, filenames):
 237   """Parses an array of .prop files.
 238
 239   Args:
 240     property_filenames: An array of Property filenames.
 241     directory: The directory where the Config file and all Property files
 242         reside in.
 243
 244   Returns:
 245     A property dictionary created by merging all property dictionaries specified
 246         in the array.
 247   """
 248   current_property = {}
 249   for filename in filenames:
 250     path = os.path.join(directory, filename)
 251     new_property = json.load(open(path))
 252     MergePropertyDictionaries(current_property, new_property)
 253   return current_property
 254
 255
 256 def ParseConfigFile(filename):
 257   """Parses a .config file.
 258
 259   Args:
 260     config_filename: A Config filename.
 261
 262   Returns:
 263     A Config object.
 264   """
 265   with open(filename, 'r') as fp:
 266     config_data = json.load(fp)
 267   directory = os.path.dirname(os.path.abspath(filename))
 268
 269   config = Config()
 270   config.tests = config_data['tests']
 271   for state_name, state_property_filenames in config_data['states']:
 272     config.states[state_name] = ParsePropertyFiles(directory,
 273                                                    state_property_filenames)
 274   for action_name, action_command in config_data['actions']:
 275     config.actions[action_name] = action_command
 276   return config
 277
 278
 279 def main():
 280   parser = argparse.ArgumentParser()
 281   parser.add_argument('--build-dir', default='out',
 282                       help='Path to main build directory (the parent of the '
 283                       'Release or Debug directory)')
 284   parser.add_argument('--target', default='Release',
 285                       help='Build target (Release or Debug)')
 286   parser.add_argument('--force-clean', action='store_true', default=False,
 287                       help='Force cleaning existing installations')
 288   parser.add_argument('-q', '--quiet', action='store_true', default=False,
 289                       help='Reduce test runner output')
 290   parser.add_argument('--write-full-results-to', metavar='FILENAME',
 291                       help='Path to write the list of full results to.')
 292   parser.add_argument('--config', metavar='FILENAME',
 293                       help='Path to test configuration file')
 294   parser.add_argument('test', nargs='*',
 295                       help='Name(s) of tests to run.')
 296   args = parser.parse_args()
 297   if not args.config:
 298     parser.error('missing mandatory --config FILENAME argument')
 299
 300   mini_installer_path = os.path.join(args.build_dir, args.target,
 301                                      'mini_installer.exe')
 302   assert os.path.exists(mini_installer_path), ('Could not find file %s' %
 303                                                mini_installer_path)
 304
 305   suite = unittest.TestSuite()
 306
 307   # Set the env var used by mini_installer.exe to decide to not show UI.
 308   os.environ['MINI_INSTALLER_TEST'] = '1'
 309   config = ParseConfigFile(args.config)
 310
 311   variable_expander = VariableExpander(mini_installer_path)
 312   RunCleanCommand(args.force_clean, variable_expander)
 313   for test in config.tests:
 314     # If tests were specified via |tests|, their names are formatted like so:
 315     test_name = '%s/%s/%s' % (InstallerTest.__module__,
 316                               InstallerTest.__name__,
 317                               test['name'])
 318     if not args.test or test_name in args.test:
 319       suite.addTest(InstallerTest(test['name'], test['traversal'], config,
 320                                   variable_expander, args.quiet))
 321
 322   verbosity = 2 if not args.quiet else 1
 323   result = unittest.TextTestRunner(verbosity=verbosity).run(suite)
 324   if args.write_full_results_to:
 325     with open(args.write_full_results_to, 'w') as fp:
 326       json.dump(_FullResults(suite, result, {}), fp, indent=2)
 327       fp.write('\n')
 328   return 0 if result.wasSuccessful() else 1
 329
 330
 331 # TODO(dpranke): Find a way for this to be shared with the mojo and other tests.
 332 TEST_SEPARATOR = '.'
 333
 334
 335 def _FullResults(suite, result, metadata):
 336   """Convert the unittest results to the Chromium JSON test result format.
 337
 338   This matches run-webkit-tests (the layout tests) and the flakiness dashboard.
 339   """
 340
 341   full_results = {}
 342   full_results['interrupted'] = False
 343   full_results['path_delimiter'] = TEST_SEPARATOR
 344   full_results['version'] = 3
 345   full_results['seconds_since_epoch'] = time.time()
 346   for md in metadata:
 347     key, val = md.split('=', 1)
 348     full_results[key] = val
 349
 350   all_test_names = _AllTestNames(suite)
 351   failed_test_names = _FailedTestNames(result)
 352
 353   full_results['num_failures_by_type'] = {
 354       'FAIL': len(failed_test_names),
 355       'PASS': len(all_test_names) - len(failed_test_names),
 356   }
 357
 358   full_results['tests'] = {}
 359
 360   for test_name in all_test_names:
 361     value = {}
 362     value['expected'] = 'PASS'
 363     if test_name in failed_test_names:
 364       value['actual'] = 'FAIL'
 365       value['is_unexpected'] = True
 366     else:
 367       value['actual'] = 'PASS'
 368     _AddPathToTrie(full_results['tests'], test_name, value)
 369
 370   return full_results
 371
 372
 373 def _AllTestNames(suite):
 374   test_names = []
 375   # _tests is protected  pylint: disable=W0212
 376   for test in suite._tests:
 377     if isinstance(test, unittest.suite.TestSuite):
 378       test_names.extend(_AllTestNames(test))
 379     else:
 380       test_names.append(test.id())
 381   return test_names
 382
 383
 384 def _FailedTestNames(result):
 385   return set(test.id() for test, _ in result.failures + result.errors)
 386
 387
 388 def _AddPathToTrie(trie, path, value):
 389   if TEST_SEPARATOR not in path:
 390     trie[path] = value
 391     return
 392   directory, rest = path.split(TEST_SEPARATOR, 1)
 393   if directory not in trie:
 394     trie[directory] = {}
 395   _AddPathToTrie(trie[directory], rest, value)
 396
 397
 398 if __name__ == '__main__':
 399   sys.exit(main())