mojo/tools/apptest_runner.py

   1 #!/usr/bin/env python
   2 # Copyright 2014 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 '''A test runner for gtest application tests.'''
   7
   8 import argparse
   9 import json
  10 import logging
  11 import os
  12 import sys
  13 import time
  14
  15 from mopy import gtest
  16 from mopy.config import Config
  17
  18
  19 APPTESTS = os.path.abspath(os.path.join(__file__, '..', 'data', 'apptests'))
  20
  21
  22 def main():
  23   parser = argparse.ArgumentParser(description='An application test runner.')
  24   parser.add_argument('build_dir', type=str, help='The build output directory.')
  25   parser.add_argument('--verbose', default=False, action='store_true',
  26                       help='Print additional logging information.')
  27   parser.add_argument('--repeat-count', default=1, metavar='INT',
  28                       action='store', type=int,
  29                       help='The number of times to repeat the set of tests.')
  30   parser.add_argument('--write-full-results-to', metavar='FILENAME',
  31                       help='The path to write the JSON list of full results.')
  32   parser.add_argument('--test-list-file', metavar='FILENAME', type=file,
  33                       default=APPTESTS, help='The file listing tests to run.')
  34   args = parser.parse_args()
  35
  36   gtest.set_color()
  37   logger = logging.getLogger()
  38   logging.basicConfig(stream=sys.stdout, format='%(levelname)s:%(message)s')
  39   logger.setLevel(logging.DEBUG if args.verbose else logging.WARNING)
  40   logger.debug('Initialized logging: level=%s' % logger.level)
  41
  42   logger.debug('Test list file: %s', args.test_list_file)
  43   config = Config(args.build_dir, is_verbose=args.verbose,
  44                   apk_name='MojoRunnerApptests.apk')
  45   execution_globals = {'config': config}
  46   exec args.test_list_file in execution_globals
  47   test_list = execution_globals['tests']
  48   logger.debug('Test list: %s' % test_list)
  49
  50   shell = None
  51   if config.target_os == Config.OS_ANDROID:
  52     from mopy.android import AndroidShell
  53     shell = AndroidShell(config)
  54     result = shell.InitShell()
  55     if result != 0:
  56       return result
  57
  58   tests = []
  59   failed = []
  60   failed_suites = 0
  61   for _ in range(args.repeat_count):
  62     for test_dict in test_list:
  63       test = test_dict['test']
  64       test_name = test_dict.get('name', test)
  65       test_type = test_dict.get('type', 'gtest')
  66       test_args = test_dict.get('args', [])
  67
  68       print 'Running %s...%s' % (test_name, ('\n' if args.verbose else '')),
  69       sys.stdout.flush()
  70
  71       assert test_type in ('gtest', 'gtest_isolated')
  72       isolate = test_type == 'gtest_isolated'
  73       (test, fail) = gtest.run_apptest(config, shell, test_args, test, isolate)
  74       tests.extend(test)
  75       failed.extend(fail)
  76       result = test and not fail
  77       print '[  PASSED  ]' if result else '[  FAILED  ]',
  78       print test_name if args.verbose or not result else ''
  79       # Abort when 3 apptest suites, or a tenth of all, have failed.
  80       # base::TestLauncher does this for timeouts and unknown results.
  81       failed_suites += 0 if result else 1
  82       if failed_suites >= max(3, len(test_list) / 10):
  83         print 'Too many failing suites (%d), exiting now.' % failed_suites
  84         failed.append('Test runner aborted for excessive failures.')
  85         break;
  86
  87     if failed:
  88       break;
  89
  90   print '[==========] %d tests ran.' % len(tests)
  91   print '[  PASSED  ] %d tests.' % (len(tests) - len(failed))
  92   if failed:
  93     print '[  FAILED  ] %d tests, listed below:' % len(failed)
  94     for failure in failed:
  95       print '[  FAILED  ] %s' % failure
  96
  97   if args.write_full_results_to:
  98     _WriteJSONResults(tests, failed, args.write_full_results_to)
  99
 100   return 1 if failed else 0
 101
 102
 103 def _WriteJSONResults(tests, failed, write_full_results_to):
 104   '''Write the apptest results in the Chromium JSON test results format.
 105      See <http://www.chromium.org/developers/the-json-test-results-format>
 106      TODO(msw): Use Chromium and TYP testing infrastructure.
 107      TODO(msw): Use GTest Suite.Fixture names, not the apptest names.
 108      Adapted from chrome/test/mini_installer/test_installer.py
 109   '''
 110   results = {
 111     'interrupted': False,
 112     'path_delimiter': '.',
 113     'version': 3,
 114     'seconds_since_epoch': time.time(),
 115     'num_failures_by_type': {
 116       'FAIL': len(failed),
 117       'PASS': len(tests) - len(failed),
 118     },
 119     'tests': {}
 120   }
 121
 122   for test in tests:
 123     value = {
 124       'expected': 'PASS',
 125       'actual': 'FAIL' if test in failed else 'PASS',
 126       'is_unexpected': True if test in failed else False,
 127     }
 128     _AddPathToTrie(results['tests'], test, value)
 129
 130   with open(write_full_results_to, 'w') as fp:
 131     json.dump(results, fp, indent=2)
 132     fp.write('\n')
 133
 134   return results
 135
 136
 137 def _AddPathToTrie(trie, path, value):
 138   if '.' not in path:
 139     trie[path] = value
 140     return
 141   directory, rest = path.split('.', 1)
 142   if directory not in trie:
 143     trie[directory] = {}
 144   _AddPathToTrie(trie[directory], rest, value)
 145
 146
 147 if __name__ == '__main__':
 148   sys.exit(main())