mojo/tools/apptest_runner.py

   1 #!/usr/bin/env python
   2 # Copyright 2014 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 '''A test runner for gtest application tests.'''
   7
   8 import argparse
   9 import json
  10 import logging
  11 import os
  12 import sys
  13 import time
  14
  15 from mopy import gtest
  16 from mopy.config import Config
  17
  18
  19 APPTESTS = os.path.abspath(os.path.join(__file__, '..', 'data', 'apptests'))
  20
  21
  22 def main():
  23   parser = argparse.ArgumentParser(description='An application test runner.')
  24   parser.add_argument('build_dir', type=str, help='The build output directory.')
  25   parser.add_argument('--verbose', default=False, action='store_true',
  26                       help='Print additional logging information.')
  27   parser.add_argument('--repeat-count', default=1, metavar='INT',
  28                       action='store', type=int,
  29                       help='The number of times to repeat the set of tests.')
  30   parser.add_argument('--write-full-results-to', metavar='FILENAME',
  31                       help='The path to write the JSON list of full results.')
  32   parser.add_argument('--test-list-file', metavar='FILENAME', type=file,
  33                       default=APPTESTS, help='The file listing tests to run.')
  34   args = parser.parse_args()
  35
  36   logger = logging.getLogger()
  37   logging.basicConfig(stream=sys.stdout, format='%(levelname)s:%(message)s')
  38   logger.setLevel(logging.DEBUG if args.verbose else logging.WARNING)
  39   logger.debug('Initialized logging: level=%s' % logger.level)
  40
  41   logger.debug('Test list file: %s', args.test_list_file)
  42   config = Config(args.build_dir, is_verbose=args.verbose,
  43                   apk_name='MojoRunnerApptests.apk')
  44   execution_globals = {'config': config}
  45   exec args.test_list_file in execution_globals
  46   test_list = execution_globals['tests']
  47   logger.debug('Test list: %s' % test_list)
  48
  49   shell = None
  50   if config.target_os == Config.OS_ANDROID:
  51     from mopy.android import AndroidShell
  52     shell = AndroidShell(config)
  53     result = shell.InitShell()
  54     if result != 0:
  55       return result
  56
  57   tests = []
  58   failed = []
  59   failed_suites = 0
  60   for _ in range(args.repeat_count):
  61     for test_dict in test_list:
  62       test = test_dict['test']
  63       test_name = test_dict.get('name', test)
  64       test_type = test_dict.get('type', 'gtest')
  65       test_args = test_dict.get('args', [])
  66
  67       print 'Running %s...%s' % (test_name, ('\n' if args.verbose else '')),
  68       sys.stdout.flush()
  69
  70       assert test_type in ('gtest', 'gtest_isolated')
  71       isolate = test_type == 'gtest_isolated'
  72       (test, fail) = gtest.run_apptest(config, shell, test_args, test, isolate)
  73       tests.extend(test)
  74       failed.extend(fail)
  75       result = test and not fail
  76       print '[  PASSED  ]' if result else '[  FAILED  ]',
  77       print test_name if args.verbose or not result else ''
  78       # Abort when 3 apptest suites, or a tenth of all, have failed.
  79       # base::TestLauncher does this for timeouts and unknown results.
  80       failed_suites += 0 if result else 1
  81       if failed_suites >= max(3, len(test_list) / 10):
  82         print 'Too many failing suites (%d), exiting now.' % failed_suites
  83         failed.append('Test runner aborted for excessive failures.')
  84         break;
  85
  86     if failed:
  87       break;
  88
  89   print '[==========] %d tests ran.' % len(tests)
  90   print '[  PASSED  ] %d tests.' % (len(tests) - len(failed))
  91   if failed:
  92     print '[  FAILED  ] %d tests, listed below:' % len(failed)
  93     for failure in failed:
  94       print '[  FAILED  ] %s' % failure
  95
  96   if args.write_full_results_to:
  97     _WriteJSONResults(tests, failed, args.write_full_results_to)
  98
  99   return 1 if failed else 0
 100
 101
 102 def _WriteJSONResults(tests, failed, write_full_results_to):
 103   '''Write the apptest results in the Chromium JSON test results format.
 104      See <http://www.chromium.org/developers/the-json-test-results-format>
 105      TODO(msw): Use Chromium and TYP testing infrastructure.
 106      TODO(msw): Use GTest Suite.Fixture names, not the apptest names.
 107      Adapted from chrome/test/mini_installer/test_installer.py
 108   '''
 109   results = {
 110     'interrupted': False,
 111     'path_delimiter': '.',
 112     'version': 3,
 113     'seconds_since_epoch': time.time(),
 114     'num_failures_by_type': {
 115       'FAIL': len(failed),
 116       'PASS': len(tests) - len(failed),
 117     },
 118     'tests': {}
 119   }
 120
 121   for test in tests:
 122     value = {
 123       'expected': 'PASS',
 124       'actual': 'FAIL' if test in failed else 'PASS',
 125       'is_unexpected': True if test in failed else False,
 126     }
 127     _AddPathToTrie(results['tests'], test, value)
 128
 129   with open(write_full_results_to, 'w') as fp:
 130     json.dump(results, fp, indent=2)
 131     fp.write('\n')
 132
 133   return results
 134
 135
 136 def _AddPathToTrie(trie, path, value):
 137   if '.' not in path:
 138     trie[path] = value
 139     return
 140   directory, rest = path.split('.', 1)
 141   if directory not in trie:
 142     trie[directory] = {}
 143   _AddPathToTrie(trie[directory], rest, value)
 144
 145
 146 if __name__ == '__main__':
 147   sys.exit(main())