Update CrOS OOBE throbber to MD throbber; delete old asset
[chromium-blink-merge.git] / chrome / test / mini_installer / test_installer.py
blob723459b28a9d3e00dd1eaa4d6d91c62a03950127
1 # Copyright 2013 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
5 """This script tests the installer with test cases specified in the config file.
7 For each test case, it checks that the machine states after the execution of
8 each command match the expected machine states. For more details, take a look at
9 the design documentation at http://goo.gl/Q0rGM6
10 """
12 import argparse
13 import datetime
14 import inspect
15 import json
16 import os
17 import subprocess
18 import sys
19 import time
20 import unittest
21 import _winreg
23 from variable_expander import VariableExpander
24 import verifier_runner
27 def LogMessage(message):
28 """Logs a message to stderr.
30 Args:
31 message: The message string to be logged.
32 """
33 now = datetime.datetime.now()
34 frameinfo = inspect.getframeinfo(inspect.currentframe().f_back)
35 filename = os.path.basename(frameinfo.filename)
36 line = frameinfo.lineno
37 sys.stderr.write('[%s:%s(%s)] %s\n' % (now.strftime('%m%d/%H%M%S'),
38 filename, line, message))
41 class Config:
42 """Describes the machine states, actions, and test cases.
44 Attributes:
45 states: A dictionary where each key is a state name and the associated value
46 is a property dictionary describing that state.
47 actions: A dictionary where each key is an action name and the associated
48 value is the action's command.
49 tests: An array of test cases.
50 """
51 def __init__(self):
52 self.states = {}
53 self.actions = {}
54 self.tests = []
57 class InstallerTest(unittest.TestCase):
58 """Tests a test case in the config file."""
60 def __init__(self, name, test, config, variable_expander, quiet):
61 """Constructor.
63 Args:
64 name: The name of this test.
65 test: An array of alternating state names and action names, starting and
66 ending with state names.
67 config: The Config object.
68 variable_expander: A VariableExpander object.
69 """
70 super(InstallerTest, self).__init__()
71 self._name = name
72 self._test = test
73 self._config = config
74 self._variable_expander = variable_expander
75 self._quiet = quiet
76 self._verifier_runner = verifier_runner.VerifierRunner()
77 self._clean_on_teardown = True
79 def __str__(self):
80 """Returns a string representing the test case.
82 Returns:
83 A string created by joining state names and action names together with
84 ' -> ', for example, 'Test: clean -> install chrome -> chrome_installed'.
85 """
86 return '%s: %s\n' % (self._name, ' -> '.join(self._test))
88 def id(self):
89 """Returns the name of the test."""
90 # Overridden from unittest.TestCase so that id() contains the name of the
91 # test case from the config file in place of the name of this class's test
92 # function.
93 return unittest.TestCase.id(self).replace(self._testMethodName, self._name)
95 def runTest(self):
96 """Run the test case."""
97 # |test| is an array of alternating state names and action names, starting
98 # and ending with state names. Therefore, its length must be odd.
99 self.assertEqual(1, len(self._test) % 2,
100 'The length of test array must be odd')
102 state = self._test[0]
103 self._VerifyState(state)
105 # Starting at index 1, we loop through pairs of (action, state).
106 for i in range(1, len(self._test), 2):
107 action = self._test[i]
108 if not self._quiet:
109 LogMessage('Beginning action %s' % action)
110 RunCommand(self._config.actions[action], self._variable_expander)
111 if not self._quiet:
112 LogMessage('Finished action %s' % action)
114 state = self._test[i + 1]
115 self._VerifyState(state)
117 # If the test makes it here, it means it was successful, because RunCommand
118 # and _VerifyState throw an exception on failure.
119 self._clean_on_teardown = False
121 def tearDown(self):
122 """Cleans up the machine if the test case fails."""
123 if self._clean_on_teardown:
124 RunCleanCommand(True, self._variable_expander)
126 def shortDescription(self):
127 """Overridden from unittest.TestCase.
129 We return None as the short description to suppress its printing.
130 The default implementation of this method returns the docstring of the
131 runTest method, which is not useful since it's the same for every test case.
132 The description from the __str__ method is informative enough.
134 return None
136 def _VerifyState(self, state):
137 """Verifies that the current machine state matches a given state.
139 Args:
140 state: A state name.
142 if not self._quiet:
143 LogMessage('Verifying state %s' % state)
144 try:
145 self._verifier_runner.VerifyAll(self._config.states[state],
146 self._variable_expander)
147 except AssertionError as e:
148 # If an AssertionError occurs, we intercept it and add the state name
149 # to the error message so that we know where the test fails.
150 raise AssertionError("In state '%s', %s" % (state, e))
153 def RunCommand(command, variable_expander):
154 """Runs the given command from the current file's directory.
156 This function throws an Exception if the command returns with non-zero exit
157 status.
159 Args:
160 command: A command to run. It is expanded using Expand.
161 variable_expander: A VariableExpander object.
163 expanded_command = variable_expander.Expand(command)
164 script_dir = os.path.dirname(os.path.abspath(__file__))
165 exit_status = subprocess.call(expanded_command, shell=True, cwd=script_dir)
166 if exit_status != 0:
167 raise Exception('Command %s returned non-zero exit status %s' % (
168 expanded_command, exit_status))
171 def DeleteGoogleUpdateRegistration(system_level, registry_subkey,
172 variable_expander):
173 """Deletes Chrome's registration with Google Update.
175 Args:
176 system_level: True if system-level Chrome is to be deleted.
177 registry_subkey: The pre-expansion registry subkey for the product.
178 variable_expander: A VariableExpander object.
180 root = (_winreg.HKEY_LOCAL_MACHINE if system_level
181 else _winreg.HKEY_CURRENT_USER)
182 key_name = variable_expander.Expand(registry_subkey)
183 try:
184 key_handle = _winreg.OpenKey(root, key_name, 0,
185 _winreg.KEY_SET_VALUE |
186 _winreg.KEY_WOW64_32KEY)
187 _winreg.DeleteValue(key_handle, 'pv')
188 except WindowsError:
189 # The key isn't present, so there is no value to delete.
190 pass
193 def RunCleanCommand(force_clean, variable_expander):
194 """Puts the machine in the clean state (i.e. Chrome not installed).
196 Args:
197 force_clean: A boolean indicating whether to force cleaning existing
198 installations.
199 variable_expander: A VariableExpander object.
201 # A list of (system_level, product_name, product_switch, registry_subkey)
202 # tuples for the possible installed products.
203 data = [
204 (False, '$CHROME_LONG_NAME', '',
205 '$CHROME_UPDATE_REGISTRY_SUBKEY'),
206 (True, '$CHROME_LONG_NAME', '--system-level',
207 '$CHROME_UPDATE_REGISTRY_SUBKEY'),
209 if variable_expander.Expand('$SUPPORTS_SXS') == 'True':
210 data.append((False, '$CHROME_LONG_NAME_SXS', '',
211 '$CHROME_UPDATE_REGISTRY_SUBKEY_SXS'))
213 interactive_option = '--interactive' if not force_clean else ''
214 for system_level, product_name, product_switch, registry_subkey in data:
215 command = ('python uninstall_chrome.py '
216 '--chrome-long-name="%s" '
217 '--no-error-if-absent %s %s' %
218 (product_name, product_switch, interactive_option))
219 RunCommand(command, variable_expander)
220 if force_clean:
221 DeleteGoogleUpdateRegistration(system_level, registry_subkey,
222 variable_expander)
225 def MergePropertyDictionaries(current_property, new_property):
226 """Merges the new property dictionary into the current property dictionary.
228 This is different from general dictionary merging in that, in case there are
229 keys with the same name, we merge values together in the first level, and we
230 override earlier values in the second level. For more details, take a look at
231 http://goo.gl/uE0RoR
233 Args:
234 current_property: The property dictionary to be modified.
235 new_property: The new property dictionary.
237 for key, value in new_property.iteritems():
238 if key not in current_property:
239 current_property[key] = value
240 else:
241 assert(isinstance(current_property[key], dict) and
242 isinstance(value, dict))
243 # This merges two dictionaries together. In case there are keys with
244 # the same name, the latter will override the former.
245 current_property[key] = dict(
246 current_property[key].items() + value.items())
249 def FilterConditionalElem(elem, condition_name, variable_expander):
250 """Returns True if a conditional element should be processed.
252 Args:
253 elem: A dictionary.
254 condition_name: The name of the condition property in |elem|.
255 variable_expander: A variable expander used to evaluate conditions.
257 Returns:
258 True if |elem| should be processed.
260 if condition_name not in elem:
261 return True
262 condition = variable_expander.Expand(elem[condition_name])
263 return eval(condition, {'__builtins__': {'False': False, 'True': True}})
266 def ParsePropertyFiles(directory, filenames, variable_expander):
267 """Parses an array of .prop files.
269 Args:
270 directory: The directory where the Config file and all Property files
271 reside in.
272 filenames: An array of Property filenames.
273 variable_expander: A variable expander used to evaluate conditions.
275 Returns:
276 A property dictionary created by merging all property dictionaries specified
277 in the array.
279 current_property = {}
280 for filename in filenames:
281 path = os.path.join(directory, filename)
282 new_property = json.load(open(path))
283 if not FilterConditionalElem(new_property, 'Condition', variable_expander):
284 continue
285 # Remove any Condition from the propery dict before merging since it serves
286 # no purpose from here on out.
287 if 'Condition' in new_property:
288 del new_property['Condition']
289 MergePropertyDictionaries(current_property, new_property)
290 return current_property
293 def ParseConfigFile(filename, variable_expander):
294 """Parses a .config file.
296 Args:
297 config_filename: A Config filename.
299 Returns:
300 A Config object.
302 with open(filename, 'r') as fp:
303 config_data = json.load(fp)
304 directory = os.path.dirname(os.path.abspath(filename))
306 config = Config()
307 config.tests = config_data['tests']
308 # Drop conditional tests that should not be run in the current configuration.
309 config.tests = filter(lambda t: FilterConditionalElem(t, 'condition',
310 variable_expander),
311 config.tests)
312 for state_name, state_property_filenames in config_data['states']:
313 config.states[state_name] = ParsePropertyFiles(directory,
314 state_property_filenames,
315 variable_expander)
316 for action_name, action_command in config_data['actions']:
317 config.actions[action_name] = action_command
318 return config
321 def main():
322 parser = argparse.ArgumentParser()
323 parser.add_argument('--build-dir', default='out',
324 help='Path to main build directory (the parent of the '
325 'Release or Debug directory)')
326 parser.add_argument('--target', default='Release',
327 help='Build target (Release or Debug)')
328 parser.add_argument('--force-clean', action='store_true', default=False,
329 help='Force cleaning existing installations')
330 parser.add_argument('-q', '--quiet', action='store_true', default=False,
331 help='Reduce test runner output')
332 parser.add_argument('--write-full-results-to', metavar='FILENAME',
333 help='Path to write the list of full results to.')
334 parser.add_argument('--config', metavar='FILENAME',
335 help='Path to test configuration file')
336 parser.add_argument('test', nargs='*',
337 help='Name(s) of tests to run.')
338 args = parser.parse_args()
339 if not args.config:
340 parser.error('missing mandatory --config FILENAME argument')
342 mini_installer_path = os.path.join(args.build_dir, args.target,
343 'mini_installer.exe')
344 assert os.path.exists(mini_installer_path), ('Could not find file %s' %
345 mini_installer_path)
347 suite = unittest.TestSuite()
349 # Set the env var used by mini_installer.exe to decide to not show UI.
350 os.environ['MINI_INSTALLER_TEST'] = '1'
352 variable_expander = VariableExpander(mini_installer_path)
353 config = ParseConfigFile(args.config, variable_expander)
355 RunCleanCommand(args.force_clean, variable_expander)
356 for test in config.tests:
357 # If tests were specified via |tests|, their names are formatted like so:
358 test_name = '%s/%s/%s' % (InstallerTest.__module__,
359 InstallerTest.__name__,
360 test['name'])
361 if not args.test or test_name in args.test:
362 suite.addTest(InstallerTest(test['name'], test['traversal'], config,
363 variable_expander, args.quiet))
365 verbosity = 2 if not args.quiet else 1
366 result = unittest.TextTestRunner(verbosity=verbosity).run(suite)
367 if args.write_full_results_to:
368 with open(args.write_full_results_to, 'w') as fp:
369 json.dump(_FullResults(suite, result, {}), fp, indent=2)
370 fp.write('\n')
371 return 0 if result.wasSuccessful() else 1
374 # TODO(dpranke): Find a way for this to be shared with the mojo and other tests.
375 TEST_SEPARATOR = '.'
378 def _FullResults(suite, result, metadata):
379 """Convert the unittest results to the Chromium JSON test result format.
381 This matches run-webkit-tests (the layout tests) and the flakiness dashboard.
384 full_results = {}
385 full_results['interrupted'] = False
386 full_results['path_delimiter'] = TEST_SEPARATOR
387 full_results['version'] = 3
388 full_results['seconds_since_epoch'] = time.time()
389 for md in metadata:
390 key, val = md.split('=', 1)
391 full_results[key] = val
393 all_test_names = _AllTestNames(suite)
394 failed_test_names = _FailedTestNames(result)
396 full_results['num_failures_by_type'] = {
397 'FAIL': len(failed_test_names),
398 'PASS': len(all_test_names) - len(failed_test_names),
401 full_results['tests'] = {}
403 for test_name in all_test_names:
404 value = {}
405 value['expected'] = 'PASS'
406 if test_name in failed_test_names:
407 value['actual'] = 'FAIL'
408 value['is_unexpected'] = True
409 else:
410 value['actual'] = 'PASS'
411 _AddPathToTrie(full_results['tests'], test_name, value)
413 return full_results
416 def _AllTestNames(suite):
417 test_names = []
418 # _tests is protected pylint: disable=W0212
419 for test in suite._tests:
420 if isinstance(test, unittest.suite.TestSuite):
421 test_names.extend(_AllTestNames(test))
422 else:
423 test_names.append(test.id())
424 return test_names
427 def _FailedTestNames(result):
428 return set(test.id() for test, _ in result.failures + result.errors)
431 def _AddPathToTrie(trie, path, value):
432 if TEST_SEPARATOR not in path:
433 trie[path] = value
434 return
435 directory, rest = path.split(TEST_SEPARATOR, 1)
436 if directory not in trie:
437 trie[directory] = {}
438 _AddPathToTrie(trie[directory], rest, value)
441 if __name__ == '__main__':
442 sys.exit(main())