chrome/test/mini_installer/test_installer.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """This script tests the installer with test cases specified in the config file.
   6
   7 For each test case, it checks that the machine states after the execution of
   8 each command match the expected machine states. For more details, take a look at
   9 the design documentation at http://goo.gl/Q0rGM6
  10 """
  11
  12 import argparse
  13 import datetime
  14 import inspect
  15 import json
  16 import os
  17 import subprocess
  18 import sys
  19 import time
  20 import unittest
  21 import _winreg
  22
  23 from variable_expander import VariableExpander
  24 import verifier_runner
  25
  26
  27 def LogMessage(message):
  28   """Logs a message to stderr.
  29
  30   Args:
  31     message: The message string to be logged.
  32   """
  33   now = datetime.datetime.now()
  34   frameinfo = inspect.getframeinfo(inspect.currentframe().f_back)
  35   filename = os.path.basename(frameinfo.filename)
  36   line = frameinfo.lineno
  37   sys.stderr.write('[%s:%s(%s)] %s\n' % (now.strftime('%m%d/%H%M%S'),
  38                                          filename, line, message))
  39
  40
  41 class Config:
  42   """Describes the machine states, actions, and test cases.
  43
  44   Attributes:
  45     states: A dictionary where each key is a state name and the associated value
  46         is a property dictionary describing that state.
  47     actions: A dictionary where each key is an action name and the associated
  48         value is the action's command.
  49     tests: An array of test cases.
  50   """
  51   def __init__(self):
  52     self.states = {}
  53     self.actions = {}
  54     self.tests = []
  55
  56
  57 class InstallerTest(unittest.TestCase):
  58   """Tests a test case in the config file."""
  59
  60   def __init__(self, name, test, config, variable_expander, quiet):
  61     """Constructor.
  62
  63     Args:
  64       name: The name of this test.
  65       test: An array of alternating state names and action names, starting and
  66           ending with state names.
  67       config: The Config object.
  68       variable_expander: A VariableExpander object.
  69     """
  70     super(InstallerTest, self).__init__()
  71     self._name = name
  72     self._test = test
  73     self._config = config
  74     self._variable_expander = variable_expander
  75     self._quiet = quiet
  76     self._verifier_runner = verifier_runner.VerifierRunner()
  77     self._clean_on_teardown = True
  78
  79   def __str__(self):
  80     """Returns a string representing the test case.
  81
  82     Returns:
  83       A string created by joining state names and action names together with
  84       ' -> ', for example, 'Test: clean -> install chrome -> chrome_installed'.
  85     """
  86     return '%s: %s\n' % (self._name, ' -> '.join(self._test))
  87
  88   def id(self):
  89     """Returns the name of the test."""
  90     # Overridden from unittest.TestCase so that id() contains the name of the
  91     # test case from the config file in place of the name of this class's test
  92     # function.
  93     return unittest.TestCase.id(self).replace(self._testMethodName, self._name)
  94
  95   def runTest(self):
  96     """Run the test case."""
  97     # |test| is an array of alternating state names and action names, starting
  98     # and ending with state names. Therefore, its length must be odd.
  99     self.assertEqual(1, len(self._test) % 2,
 100                      'The length of test array must be odd')
 101
 102     state = self._test[0]
 103     self._VerifyState(state)
 104
 105     # Starting at index 1, we loop through pairs of (action, state).
 106     for i in range(1, len(self._test), 2):
 107       action = self._test[i]
 108       if not self._quiet:
 109         LogMessage('Beginning action %s' % action)
 110       RunCommand(self._config.actions[action], self._variable_expander)
 111       if not self._quiet:
 112         LogMessage('Finished action %s' % action)
 113
 114       state = self._test[i + 1]
 115       self._VerifyState(state)
 116
 117     # If the test makes it here, it means it was successful, because RunCommand
 118     # and _VerifyState throw an exception on failure.
 119     self._clean_on_teardown = False
 120
 121   def tearDown(self):
 122     """Cleans up the machine if the test case fails."""
 123     if self._clean_on_teardown:
 124       RunCleanCommand(True, self._variable_expander)
 125
 126   def shortDescription(self):
 127     """Overridden from unittest.TestCase.
 128
 129     We return None as the short description to suppress its printing.
 130     The default implementation of this method returns the docstring of the
 131     runTest method, which is not useful since it's the same for every test case.
 132     The description from the __str__ method is informative enough.
 133     """
 134     return None
 135
 136   def _VerifyState(self, state):
 137     """Verifies that the current machine state matches a given state.
 138
 139     Args:
 140       state: A state name.
 141     """
 142     if not self._quiet:
 143       LogMessage('Verifying state %s' % state)
 144     try:
 145       self._verifier_runner.VerifyAll(self._config.states[state],
 146                                       self._variable_expander)
 147     except AssertionError as e:
 148       # If an AssertionError occurs, we intercept it and add the state name
 149       # to the error message so that we know where the test fails.
 150       raise AssertionError("In state '%s', %s" % (state, e))
 151
 152
 153 def RunCommand(command, variable_expander):
 154   """Runs the given command from the current file's directory.
 155
 156   This function throws an Exception if the command returns with non-zero exit
 157   status.
 158
 159   Args:
 160     command: A command to run. It is expanded using Expand.
 161     variable_expander: A VariableExpander object.
 162   """
 163   expanded_command = variable_expander.Expand(command)
 164   script_dir = os.path.dirname(os.path.abspath(__file__))
 165   exit_status = subprocess.call(expanded_command, shell=True, cwd=script_dir)
 166   if exit_status != 0:
 167     raise Exception('Command %s returned non-zero exit status %s' % (
 168         expanded_command, exit_status))
 169
 170
 171 def DeleteGoogleUpdateRegistration(system_level, variable_expander):
 172   """Deletes Chrome's registration with Google Update.
 173
 174   Args:
 175     system_level: True if system-level Chrome is to be deleted.
 176     variable_expander: A VariableExpander object.
 177   """
 178   root = (_winreg.HKEY_LOCAL_MACHINE if system_level
 179           else _winreg.HKEY_CURRENT_USER)
 180   key_name = variable_expander.Expand('$CHROME_UPDATE_REGISTRY_SUBKEY')
 181   try:
 182     key_handle = _winreg.OpenKey(root, key_name, 0,
 183                                  _winreg.KEY_SET_VALUE |
 184                                  _winreg.KEY_WOW64_32KEY)
 185     _winreg.DeleteValue(key_handle, 'pv')
 186   except WindowsError:
 187     # The key isn't present, so there is no value to delete.
 188     pass
 189
 190
 191 def RunCleanCommand(force_clean, variable_expander):
 192   """Puts the machine in the clean state (i.e. Chrome not installed).
 193
 194   Args:
 195     force_clean: A boolean indicating whether to force cleaning existing
 196         installations.
 197     variable_expander: A VariableExpander object.
 198   """
 199   # TODO(sukolsak): Handle Chrome SxS installs.
 200   interactive_option = '--interactive' if not force_clean else ''
 201   for system_level in (False, True):
 202     level_option = '--system-level' if system_level else ''
 203     command = ('python uninstall_chrome.py '
 204                '--chrome-long-name="$CHROME_LONG_NAME" '
 205                '--no-error-if-absent %s %s' %
 206                (level_option, interactive_option))
 207     RunCommand(command, variable_expander)
 208     if force_clean:
 209       DeleteGoogleUpdateRegistration(system_level, variable_expander)
 210
 211
 212 def MergePropertyDictionaries(current_property, new_property):
 213   """Merges the new property dictionary into the current property dictionary.
 214
 215   This is different from general dictionary merging in that, in case there are
 216   keys with the same name, we merge values together in the first level, and we
 217   override earlier values in the second level. For more details, take a look at
 218   http://goo.gl/uE0RoR
 219
 220   Args:
 221     current_property: The property dictionary to be modified.
 222     new_property: The new property dictionary.
 223   """
 224   for key, value in new_property.iteritems():
 225     if key not in current_property:
 226       current_property[key] = value
 227     else:
 228       assert(isinstance(current_property[key], dict) and
 229           isinstance(value, dict))
 230       # This merges two dictionaries together. In case there are keys with
 231       # the same name, the latter will override the former.
 232       current_property[key] = dict(
 233           current_property[key].items() + value.items())
 234
 235
 236 def ParsePropertyFiles(directory, filenames):
 237   """Parses an array of .prop files.
 238
 239   Args:
 240     property_filenames: An array of Property filenames.
 241     directory: The directory where the Config file and all Property files
 242         reside in.
 243
 244   Returns:
 245     A property dictionary created by merging all property dictionaries specified
 246         in the array.
 247   """
 248   current_property = {}
 249   for filename in filenames:
 250     path = os.path.join(directory, filename)
 251     new_property = json.load(open(path))
 252     MergePropertyDictionaries(current_property, new_property)
 253   return current_property
 254
 255
 256 def ParseConfigFile(filename):
 257   """Parses a .config file.
 258
 259   Args:
 260     config_filename: A Config filename.
 261
 262   Returns:
 263     A Config object.
 264   """
 265   with open(filename, 'r') as fp:
 266     config_data = json.load(fp)
 267   directory = os.path.dirname(os.path.abspath(filename))
 268
 269   config = Config()
 270   config.tests = config_data['tests']
 271   for state_name, state_property_filenames in config_data['states']:
 272     config.states[state_name] = ParsePropertyFiles(directory,
 273                                                    state_property_filenames)
 274   for action_name, action_command in config_data['actions']:
 275     config.actions[action_name] = action_command
 276   return config
 277
 278
 279 def IsComponentBuild(mini_installer_path):
 280   """ Invokes the mini_installer asking whether it is a component build.
 281
 282   Args:
 283     mini_installer_path: The path to mini_installer.exe.
 284
 285   Returns:
 286     True if the mini_installer is a component build, False otherwise.
 287   """
 288   query_command = [ mini_installer_path, '--query-component-build' ]
 289   exit_status = subprocess.call(query_command)
 290   return exit_status == 0
 291
 292
 293 def main():
 294   parser = argparse.ArgumentParser()
 295   parser.add_argument('--build-dir', default='out',
 296                       help='Path to main build directory (the parent of the '
 297                       'Release or Debug directory)')
 298   parser.add_argument('--target', default='Release',
 299                       help='Build target (Release or Debug)')
 300   parser.add_argument('--force-clean', action='store_true', default=False,
 301                       help='Force cleaning existing installations')
 302   parser.add_argument('-q', '--quiet', action='store_true', default=False,
 303                       help='Reduce test runner output')
 304   parser.add_argument('--write-full-results-to', metavar='FILENAME',
 305                       help='Path to write the list of full results to.')
 306   parser.add_argument('--config', metavar='FILENAME',
 307                       help='Path to test configuration file')
 308   parser.add_argument('test', nargs='*',
 309                       help='Name(s) of tests to run.')
 310   args = parser.parse_args()
 311   if not args.config:
 312     parser.error('missing mandatory --config FILENAME argument')
 313
 314   mini_installer_path = os.path.join(args.build_dir, args.target,
 315                                      'mini_installer.exe')
 316   assert os.path.exists(mini_installer_path), ('Could not find file %s' %
 317                                                mini_installer_path)
 318
 319   suite = unittest.TestSuite()
 320
 321   # Set the env var used by mini_installer.exe to decide to not show UI.
 322   os.environ['MINI_INSTALLER_TEST'] = '1'
 323   is_component_build = IsComponentBuild(mini_installer_path)
 324   if not is_component_build:
 325     config = ParseConfigFile(args.config)
 326
 327     variable_expander = VariableExpander(mini_installer_path)
 328     RunCleanCommand(args.force_clean, variable_expander)
 329     for test in config.tests:
 330       # If tests were specified via |tests|, their names are formatted like so:
 331       test_name = '%s/%s/%s' % (InstallerTest.__module__,
 332                                 InstallerTest.__name__,
 333                                 test['name'])
 334       if not args.test or test_name in args.test:
 335         suite.addTest(InstallerTest(test['name'], test['traversal'], config,
 336                                     variable_expander, args.quiet))
 337
 338   verbosity = 2 if not args.quiet else 1
 339   result = unittest.TextTestRunner(verbosity=verbosity).run(suite)
 340   if is_component_build:
 341     sys.stderr.write('Component build is currently unsupported by the '
 342                      'mini_installer: http://crbug.com/377839\n')
 343   if args.write_full_results_to:
 344     with open(args.write_full_results_to, 'w') as fp:
 345       json.dump(_FullResults(suite, result, {}), fp, indent=2)
 346       fp.write('\n')
 347   return 0 if result.wasSuccessful() else 1
 348
 349
 350 # TODO(dpranke): Find a way for this to be shared with the mojo and other tests.
 351 TEST_SEPARATOR = '.'
 352
 353
 354 def _FullResults(suite, result, metadata):
 355   """Convert the unittest results to the Chromium JSON test result format.
 356
 357   This matches run-webkit-tests (the layout tests) and the flakiness dashboard.
 358   """
 359
 360   full_results = {}
 361   full_results['interrupted'] = False
 362   full_results['path_delimiter'] = TEST_SEPARATOR
 363   full_results['version'] = 3
 364   full_results['seconds_since_epoch'] = time.time()
 365   for md in metadata:
 366     key, val = md.split('=', 1)
 367     full_results[key] = val
 368
 369   all_test_names = _AllTestNames(suite)
 370   failed_test_names = _FailedTestNames(result)
 371
 372   full_results['num_failures_by_type'] = {
 373       'FAIL': len(failed_test_names),
 374       'PASS': len(all_test_names) - len(failed_test_names),
 375   }
 376
 377   full_results['tests'] = {}
 378
 379   for test_name in all_test_names:
 380     value = {}
 381     value['expected'] = 'PASS'
 382     if test_name in failed_test_names:
 383       value['actual'] = 'FAIL'
 384       value['is_unexpected'] = True
 385     else:
 386       value['actual'] = 'PASS'
 387     _AddPathToTrie(full_results['tests'], test_name, value)
 388
 389   return full_results
 390
 391
 392 def _AllTestNames(suite):
 393   test_names = []
 394   # _tests is protected  pylint: disable=W0212
 395   for test in suite._tests:
 396     if isinstance(test, unittest.suite.TestSuite):
 397       test_names.extend(_AllTestNames(test))
 398     else:
 399       test_names.append(test.id())
 400   return test_names
 401
 402
 403 def _FailedTestNames(result):
 404   return set(test.id() for test, _ in result.failures + result.errors)
 405
 406
 407 def _AddPathToTrie(trie, path, value):
 408   if TEST_SEPARATOR not in path:
 409     trie[path] = value
 410     return
 411   directory, rest = path.split(TEST_SEPARATOR, 1)
 412   if directory not in trie:
 413     trie[directory] = {}
 414   _AddPathToTrie(trie[directory], rest, value)
 415
 416
 417 if __name__ == '__main__':
 418   sys.exit(main())