chrome/test/mini_installer/test_installer.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """This script tests the installer with test cases specified in the config file.
   6
   7 For each test case, it checks that the machine states after the execution of
   8 each command match the expected machine states. For more details, take a look at
   9 the design documentation at http://goo.gl/Q0rGM6
  10 """
  11
  12 import argparse
  13 import datetime
  14 import inspect
  15 import json
  16 import os
  17 import subprocess
  18 import sys
  19 import time
  20 import unittest
  21 import _winreg
  22
  23 from variable_expander import VariableExpander
  24 import verifier_runner
  25
  26
  27 def LogMessage(message):
  28   """Logs a message to stderr.
  29
  30   Args:
  31     message: The message string to be logged.
  32   """
  33   now = datetime.datetime.now()
  34   frameinfo = inspect.getframeinfo(inspect.currentframe().f_back)
  35   filename = os.path.basename(frameinfo.filename)
  36   line = frameinfo.lineno
  37   sys.stderr.write('[%s:%s(%s)] %s\n' % (now.strftime('%m%d/%H%M%S'),
  38                                          filename, line, message))
  39
  40
  41 class Config:
  42   """Describes the machine states, actions, and test cases.
  43
  44   Attributes:
  45     states: A dictionary where each key is a state name and the associated value
  46         is a property dictionary describing that state.
  47     actions: A dictionary where each key is an action name and the associated
  48         value is the action's command.
  49     tests: An array of test cases.
  50   """
  51   def __init__(self):
  52     self.states = {}
  53     self.actions = {}
  54     self.tests = []
  55
  56
  57 class InstallerTest(unittest.TestCase):
  58   """Tests a test case in the config file."""
  59
  60   def __init__(self, name, test, config, variable_expander, quiet):
  61     """Constructor.
  62
  63     Args:
  64       name: The name of this test.
  65       test: An array of alternating state names and action names, starting and
  66           ending with state names.
  67       config: The Config object.
  68       variable_expander: A VariableExpander object.
  69     """
  70     super(InstallerTest, self).__init__()
  71     self._name = name
  72     self._test = test
  73     self._config = config
  74     self._variable_expander = variable_expander
  75     self._quiet = quiet
  76     self._verifier_runner = verifier_runner.VerifierRunner()
  77     self._clean_on_teardown = True
  78
  79   def __str__(self):
  80     """Returns a string representing the test case.
  81
  82     Returns:
  83       A string created by joining state names and action names together with
  84       ' -> ', for example, 'Test: clean -> install chrome -> chrome_installed'.
  85     """
  86     return '%s: %s\n' % (self._name, ' -> '.join(self._test))
  87
  88   def id(self):
  89     """Returns the name of the test."""
  90     # Overridden from unittest.TestCase so that id() contains the name of the
  91     # test case from the config file in place of the name of this class's test
  92     # function.
  93     return unittest.TestCase.id(self).replace(self._testMethodName, self._name)
  94
  95   def runTest(self):
  96     """Run the test case."""
  97     # |test| is an array of alternating state names and action names, starting
  98     # and ending with state names. Therefore, its length must be odd.
  99     self.assertEqual(1, len(self._test) % 2,
 100                      'The length of test array must be odd')
 101
 102     state = self._test[0]
 103     self._VerifyState(state)
 104
 105     # Starting at index 1, we loop through pairs of (action, state).
 106     for i in range(1, len(self._test), 2):
 107       action = self._test[i]
 108       if not self._quiet:
 109         LogMessage('Beginning action %s' % action)
 110       RunCommand(self._config.actions[action], self._variable_expander)
 111       if not self._quiet:
 112         LogMessage('Finished action %s' % action)
 113
 114       state = self._test[i + 1]
 115       self._VerifyState(state)
 116
 117     # If the test makes it here, it means it was successful, because RunCommand
 118     # and _VerifyState throw an exception on failure.
 119     self._clean_on_teardown = False
 120
 121   def tearDown(self):
 122     """Cleans up the machine if the test case fails."""
 123     if self._clean_on_teardown:
 124       RunCleanCommand(True, self._variable_expander)
 125
 126   def shortDescription(self):
 127     """Overridden from unittest.TestCase.
 128
 129     We return None as the short description to suppress its printing.
 130     The default implementation of this method returns the docstring of the
 131     runTest method, which is not useful since it's the same for every test case.
 132     The description from the __str__ method is informative enough.
 133     """
 134     return None
 135
 136   def _VerifyState(self, state):
 137     """Verifies that the current machine state matches a given state.
 138
 139     Args:
 140       state: A state name.
 141     """
 142     if not self._quiet:
 143       LogMessage('Verifying state %s' % state)
 144     try:
 145       self._verifier_runner.VerifyAll(self._config.states[state],
 146                                       self._variable_expander)
 147     except AssertionError as e:
 148       # If an AssertionError occurs, we intercept it and add the state name
 149       # to the error message so that we know where the test fails.
 150       raise AssertionError("In state '%s', %s" % (state, e))
 151
 152
 153 def RunCommand(command, variable_expander):
 154   """Runs the given command from the current file's directory.
 155
 156   This function throws an Exception if the command returns with non-zero exit
 157   status.
 158
 159   Args:
 160     command: A command to run. It is expanded using Expand.
 161     variable_expander: A VariableExpander object.
 162   """
 163   expanded_command = variable_expander.Expand(command)
 164   script_dir = os.path.dirname(os.path.abspath(__file__))
 165   exit_status = subprocess.call(expanded_command, shell=True, cwd=script_dir)
 166   if exit_status != 0:
 167     raise Exception('Command %s returned non-zero exit status %s' % (
 168         expanded_command, exit_status))
 169
 170
 171 def DeleteGoogleUpdateRegistration(system_level, registry_subkey,
 172                                    variable_expander):
 173   """Deletes Chrome's registration with Google Update.
 174
 175   Args:
 176     system_level: True if system-level Chrome is to be deleted.
 177     registry_subkey: The pre-expansion registry subkey for the product.
 178     variable_expander: A VariableExpander object.
 179   """
 180   root = (_winreg.HKEY_LOCAL_MACHINE if system_level
 181           else _winreg.HKEY_CURRENT_USER)
 182   key_name = variable_expander.Expand(registry_subkey)
 183   try:
 184     key_handle = _winreg.OpenKey(root, key_name, 0,
 185                                  _winreg.KEY_SET_VALUE |
 186                                  _winreg.KEY_WOW64_32KEY)
 187     _winreg.DeleteValue(key_handle, 'pv')
 188   except WindowsError:
 189     # The key isn't present, so there is no value to delete.
 190     pass
 191
 192
 193 def RunCleanCommand(force_clean, variable_expander):
 194   """Puts the machine in the clean state (i.e. Chrome not installed).
 195
 196   Args:
 197     force_clean: A boolean indicating whether to force cleaning existing
 198         installations.
 199     variable_expander: A VariableExpander object.
 200   """
 201   # A list of (system_level, product_name, product_switch, registry_subkey)
 202   # tuples for the possible installed products.
 203   data = [
 204     (False, '$CHROME_LONG_NAME', '',
 205      '$CHROME_UPDATE_REGISTRY_SUBKEY'),
 206     (True, '$CHROME_LONG_NAME', '--system-level',
 207      '$CHROME_UPDATE_REGISTRY_SUBKEY'),
 208   ]
 209   if variable_expander.Expand('$SUPPORTS_SXS') == 'True':
 210     data.append((False, '$CHROME_LONG_NAME_SXS', '',
 211                  '$CHROME_UPDATE_REGISTRY_SUBKEY_SXS'))
 212
 213   interactive_option = '--interactive' if not force_clean else ''
 214   for system_level, product_name, product_switch, registry_subkey in data:
 215     command = ('python uninstall_chrome.py '
 216                '--chrome-long-name="%s" '
 217                '--no-error-if-absent %s %s' %
 218                (product_name, product_switch, interactive_option))
 219     RunCommand(command, variable_expander)
 220     if force_clean:
 221       DeleteGoogleUpdateRegistration(system_level, registry_subkey,
 222                                      variable_expander)
 223
 224
 225 def MergePropertyDictionaries(current_property, new_property):
 226   """Merges the new property dictionary into the current property dictionary.
 227
 228   This is different from general dictionary merging in that, in case there are
 229   keys with the same name, we merge values together in the first level, and we
 230   override earlier values in the second level. For more details, take a look at
 231   http://goo.gl/uE0RoR
 232
 233   Args:
 234     current_property: The property dictionary to be modified.
 235     new_property: The new property dictionary.
 236   """
 237   for key, value in new_property.iteritems():
 238     if key not in current_property:
 239       current_property[key] = value
 240     else:
 241       assert(isinstance(current_property[key], dict) and
 242           isinstance(value, dict))
 243       # This merges two dictionaries together. In case there are keys with
 244       # the same name, the latter will override the former.
 245       current_property[key] = dict(
 246           current_property[key].items() + value.items())
 247
 248
 249 def FilterConditionalElem(elem, condition_name, variable_expander):
 250   """Returns True if a conditional element should be processed.
 251
 252   Args:
 253     elem: A dictionary.
 254     condition_name: The name of the condition property in |elem|.
 255     variable_expander: A variable expander used to evaluate conditions.
 256
 257   Returns:
 258     True if |elem| should be processed.
 259   """
 260   if condition_name not in elem:
 261     return True
 262   condition = variable_expander.Expand(elem[condition_name])
 263   return eval(condition, {'__builtins__': {'False': False, 'True': True}})
 264
 265
 266 def ParsePropertyFiles(directory, filenames, variable_expander):
 267   """Parses an array of .prop files.
 268
 269   Args:
 270     directory: The directory where the Config file and all Property files
 271         reside in.
 272     filenames: An array of Property filenames.
 273     variable_expander: A variable expander used to evaluate conditions.
 274
 275   Returns:
 276     A property dictionary created by merging all property dictionaries specified
 277         in the array.
 278   """
 279   current_property = {}
 280   for filename in filenames:
 281     path = os.path.join(directory, filename)
 282     new_property = json.load(open(path))
 283     if not FilterConditionalElem(new_property, 'Condition', variable_expander):
 284       continue
 285     # Remove any Condition from the propery dict before merging since it serves
 286     # no purpose from here on out.
 287     if 'Condition' in new_property:
 288       del new_property['Condition']
 289     MergePropertyDictionaries(current_property, new_property)
 290   return current_property
 291
 292
 293 def ParseConfigFile(filename, variable_expander):
 294   """Parses a .config file.
 295
 296   Args:
 297     config_filename: A Config filename.
 298
 299   Returns:
 300     A Config object.
 301   """
 302   with open(filename, 'r') as fp:
 303     config_data = json.load(fp)
 304   directory = os.path.dirname(os.path.abspath(filename))
 305
 306   config = Config()
 307   config.tests = config_data['tests']
 308   # Drop conditional tests that should not be run in the current configuration.
 309   config.tests = filter(lambda t: FilterConditionalElem(t, 'condition',
 310                                                         variable_expander),
 311                         config.tests)
 312   for state_name, state_property_filenames in config_data['states']:
 313     config.states[state_name] = ParsePropertyFiles(directory,
 314                                                    state_property_filenames,
 315                                                    variable_expander)
 316   for action_name, action_command in config_data['actions']:
 317     config.actions[action_name] = action_command
 318   return config
 319
 320
 321 def main():
 322   parser = argparse.ArgumentParser()
 323   parser.add_argument('--build-dir', default='out',
 324                       help='Path to main build directory (the parent of the '
 325                       'Release or Debug directory)')
 326   parser.add_argument('--target', default='Release',
 327                       help='Build target (Release or Debug)')
 328   parser.add_argument('--force-clean', action='store_true', default=False,
 329                       help='Force cleaning existing installations')
 330   parser.add_argument('-q', '--quiet', action='store_true', default=False,
 331                       help='Reduce test runner output')
 332   parser.add_argument('--write-full-results-to', metavar='FILENAME',
 333                       help='Path to write the list of full results to.')
 334   parser.add_argument('--config', metavar='FILENAME',
 335                       help='Path to test configuration file')
 336   parser.add_argument('test', nargs='*',
 337                       help='Name(s) of tests to run.')
 338   args = parser.parse_args()
 339   if not args.config:
 340     parser.error('missing mandatory --config FILENAME argument')
 341
 342   mini_installer_path = os.path.join(args.build_dir, args.target,
 343                                      'mini_installer.exe')
 344   assert os.path.exists(mini_installer_path), ('Could not find file %s' %
 345                                                mini_installer_path)
 346
 347   suite = unittest.TestSuite()
 348
 349   # Set the env var used by mini_installer.exe to decide to not show UI.
 350   os.environ['MINI_INSTALLER_TEST'] = '1'
 351
 352   variable_expander = VariableExpander(mini_installer_path)
 353   config = ParseConfigFile(args.config, variable_expander)
 354
 355   RunCleanCommand(args.force_clean, variable_expander)
 356   for test in config.tests:
 357     # If tests were specified via |tests|, their names are formatted like so:
 358     test_name = '%s/%s/%s' % (InstallerTest.__module__,
 359                               InstallerTest.__name__,
 360                               test['name'])
 361     if not args.test or test_name in args.test:
 362       suite.addTest(InstallerTest(test['name'], test['traversal'], config,
 363                                   variable_expander, args.quiet))
 364
 365   verbosity = 2 if not args.quiet else 1
 366   result = unittest.TextTestRunner(verbosity=verbosity).run(suite)
 367   if args.write_full_results_to:
 368     with open(args.write_full_results_to, 'w') as fp:
 369       json.dump(_FullResults(suite, result, {}), fp, indent=2)
 370       fp.write('\n')
 371   return 0 if result.wasSuccessful() else 1
 372
 373
 374 # TODO(dpranke): Find a way for this to be shared with the mojo and other tests.
 375 TEST_SEPARATOR = '.'
 376
 377
 378 def _FullResults(suite, result, metadata):
 379   """Convert the unittest results to the Chromium JSON test result format.
 380
 381   This matches run-webkit-tests (the layout tests) and the flakiness dashboard.
 382   """
 383
 384   full_results = {}
 385   full_results['interrupted'] = False
 386   full_results['path_delimiter'] = TEST_SEPARATOR
 387   full_results['version'] = 3
 388   full_results['seconds_since_epoch'] = time.time()
 389   for md in metadata:
 390     key, val = md.split('=', 1)
 391     full_results[key] = val
 392
 393   all_test_names = _AllTestNames(suite)
 394   failed_test_names = _FailedTestNames(result)
 395
 396   full_results['num_failures_by_type'] = {
 397       'FAIL': len(failed_test_names),
 398       'PASS': len(all_test_names) - len(failed_test_names),
 399   }
 400
 401   full_results['tests'] = {}
 402
 403   for test_name in all_test_names:
 404     value = {}
 405     value['expected'] = 'PASS'
 406     if test_name in failed_test_names:
 407       value['actual'] = 'FAIL'
 408       value['is_unexpected'] = True
 409     else:
 410       value['actual'] = 'PASS'
 411     _AddPathToTrie(full_results['tests'], test_name, value)
 412
 413   return full_results
 414
 415
 416 def _AllTestNames(suite):
 417   test_names = []
 418   # _tests is protected  pylint: disable=W0212
 419   for test in suite._tests:
 420     if isinstance(test, unittest.suite.TestSuite):
 421       test_names.extend(_AllTestNames(test))
 422     else:
 423       test_names.append(test.id())
 424   return test_names
 425
 426
 427 def _FailedTestNames(result):
 428   return set(test.id() for test, _ in result.failures + result.errors)
 429
 430
 431 def _AddPathToTrie(trie, path, value):
 432   if TEST_SEPARATOR not in path:
 433     trie[path] = value
 434     return
 435   directory, rest = path.split(TEST_SEPARATOR, 1)
 436   if directory not in trie:
 437     trie[directory] = {}
 438   _AddPathToTrie(trie[directory], rest, value)
 439
 440
 441 if __name__ == '__main__':
 442   sys.exit(main())