mojo/tools/apptest_runner.py

   1 #!/usr/bin/env python
   2 # Copyright 2014 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 '''A test runner for gtest application tests.'''
   7
   8 import argparse
   9 import json
  10 import logging
  11 import os
  12 import string
  13 import sys
  14 import time
  15
  16 from mopy import gtest
  17 from mopy.config import Config
  18
  19
  20 APPTESTS = os.path.abspath(os.path.join(__file__, '..', 'data', 'apptests'))
  21
  22
  23 def main():
  24   parser = argparse.ArgumentParser(description='An application test runner.')
  25   parser.add_argument('build_dir', type=str, help='The build output directory.')
  26   parser.add_argument('--verbose', default=False, action='store_true',
  27                       help='Print additional logging information.')
  28   parser.add_argument('--repeat-count', default=1, metavar='INT',
  29                       action='store', type=int,
  30                       help='The number of times to repeat the set of tests.')
  31   parser.add_argument('--write-full-results-to', metavar='FILENAME',
  32                       help='The path to write the JSON list of full results.')
  33   parser.add_argument('--test-list-file', metavar='FILENAME', type=file,
  34                       default=APPTESTS, help='The file listing tests to run.')
  35   parser.add_argument('--apptest-filter', default='',
  36                       help='A comma-separated list of mojo:apptests to run.')
  37   args, commandline_args = parser.parse_known_args()
  38
  39   logger = logging.getLogger()
  40   logging.basicConfig(stream=sys.stdout, format='%(levelname)s:%(message)s')
  41   logger.setLevel(logging.DEBUG if args.verbose else logging.WARNING)
  42   logger.debug('Initialized logging: level=%s' % logger.level)
  43
  44   logger.debug('Test list file: %s', args.test_list_file)
  45   config = Config(args.build_dir, is_verbose=args.verbose,
  46                   apk_name='MojoRunnerApptests.apk')
  47   execution_globals = {'config': config}
  48   exec args.test_list_file in execution_globals
  49   test_list = execution_globals['tests']
  50   logger.debug('Test list: %s' % test_list)
  51
  52   shell = None
  53   if config.target_os == Config.OS_ANDROID:
  54     from mopy.android import AndroidShell
  55     shell = AndroidShell(config)
  56     result = shell.InitShell()
  57     if result != 0:
  58       return result
  59
  60   tests = []
  61   failed = []
  62   failed_suites = 0
  63   apptest_filter = [a for a in string.split(args.apptest_filter, ',') if a]
  64   gtest_filter = [a for a in commandline_args if a.startswith('--gtest_filter')]
  65   for _ in range(args.repeat_count):
  66     for test_dict in test_list:
  67       test = test_dict['test']
  68       test_name = test_dict.get('name', test)
  69       test_type = test_dict.get('type', 'gtest')
  70       test_args = test_dict.get('args', []) + commandline_args
  71       if apptest_filter and not set(apptest_filter) & set([test, test_name]):
  72         continue;
  73
  74       print 'Running %s...%s' % (test_name, ('\n' if args.verbose else '')),
  75       sys.stdout.flush()
  76
  77       assert test_type in ('gtest', 'gtest_isolated')
  78       isolate = test_type == 'gtest_isolated'
  79       (ran, fail) = gtest.run_apptest(config, shell, test_args, test, isolate)
  80       # Ignore empty fixture lists when the commandline has a gtest filter flag.
  81       if gtest_filter and not ran and not fail:
  82         print '[ NO TESTS ] ' + (test_name if args.verbose else '')
  83         continue
  84       # Use the apptest name if the whole suite failed or no fixtures were run.
  85       fail = [test_name] if (not ran and (not fail or fail == [test])) else fail
  86       tests.extend(ran)
  87       failed.extend(fail)
  88       result = ran and not fail
  89       print '[  PASSED  ]' if result else '[  FAILED  ]',
  90       print test_name if args.verbose or not result else ''
  91       # Abort when 3 apptest suites, or a tenth of all, have failed.
  92       # base::TestLauncher does this for timeouts and unknown results.
  93       failed_suites += 0 if result else 1
  94       if failed_suites >= max(3, len(test_list) / 10):
  95         print 'Too many failing suites (%d), exiting now.' % failed_suites
  96         failed.append('Test runner aborted for excessive failures.')
  97         break;
  98
  99     if failed:
 100       break;
 101
 102   print '[==========] %d tests ran.' % len(tests)
 103   print '[  PASSED  ] %d tests.' % (len(tests) - len(failed))
 104   if failed:
 105     print '[  FAILED  ] %d tests, listed below:' % len(failed)
 106     for failure in failed:
 107       print '[  FAILED  ] %s' % failure
 108
 109   if args.write_full_results_to:
 110     _WriteJSONResults(tests, failed, args.write_full_results_to)
 111
 112   return 1 if failed else 0
 113
 114
 115 def _WriteJSONResults(tests, failed, write_full_results_to):
 116   '''Write the apptest results in the Chromium JSON test results format.
 117      See <http://www.chromium.org/developers/the-json-test-results-format>
 118      TODO(msw): Use Chromium and TYP testing infrastructure.
 119      TODO(msw): Use GTest Suite.Fixture names, not the apptest names.
 120      Adapted from chrome/test/mini_installer/test_installer.py
 121   '''
 122   results = {
 123     'interrupted': False,
 124     'path_delimiter': '.',
 125     'version': 3,
 126     'seconds_since_epoch': time.time(),
 127     'num_failures_by_type': {
 128       'FAIL': len(failed),
 129       'PASS': len(tests) - len(failed),
 130     },
 131     'tests': {}
 132   }
 133
 134   for test in tests:
 135     value = {
 136       'expected': 'PASS',
 137       'actual': 'FAIL' if test in failed else 'PASS',
 138       'is_unexpected': True if test in failed else False,
 139     }
 140     _AddPathToTrie(results['tests'], test, value)
 141
 142   with open(write_full_results_to, 'w') as fp:
 143     json.dump(results, fp, indent=2)
 144     fp.write('\n')
 145
 146   return results
 147
 148
 149 def _AddPathToTrie(trie, path, value):
 150   if '.' not in path:
 151     trie[path] = value
 152     return
 153   directory, rest = path.split('.', 1)
 154   if directory not in trie:
 155     trie[directory] = {}
 156   _AddPathToTrie(trie[directory], rest, value)
 157
 158
 159 if __name__ == '__main__':
 160   sys.exit(main())