mojo/tools/apptest_runner.py

   1 #!/usr/bin/env python
   2 # Copyright 2014 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """A test runner for gtest application tests."""
   7
   8 import argparse
   9 import json
  10 import logging
  11 import os
  12 import sys
  13 import time
  14
  15 from mopy import gtest
  16 from mopy.config import Config
  17
  18
  19 def main():
  20   parser = argparse.ArgumentParser(description="An application test runner.")
  21   parser.add_argument("build_dir", type=str, help="The build output directory.")
  22   parser.add_argument("--verbose", default=False, action='store_true',
  23                       help="Print additional logging information.")
  24   parser.add_argument('--repeat-count', default=1, metavar='INT',
  25                       action='store', type=int,
  26                       help="The number of times to repeat the set of tests.")
  27   parser.add_argument('--write-full-results-to', metavar='FILENAME',
  28                       help='The path to write the JSON list of full results.')
  29   parser.add_argument("--test-list-file", metavar='FILENAME', type=file,
  30                       default=os.path.abspath(os.path.join(__file__, os.pardir,
  31                                                            "data", "apptests")),
  32                       help="The file listing apptests to run.")
  33   args = parser.parse_args()
  34
  35   gtest.set_color()
  36   logger = logging.getLogger()
  37   logging.basicConfig(stream=sys.stdout, format="%(levelname)s:%(message)s")
  38   logger.setLevel(logging.DEBUG if args.verbose else logging.WARNING)
  39   logger.debug("Initialized logging: level=%s" % logger.level)
  40
  41   logger.debug("Test list file: %s", args.test_list_file)
  42   config = Config(args.build_dir)
  43   execution_globals = {"config": config}
  44   exec args.test_list_file in execution_globals
  45   test_list = execution_globals["tests"]
  46   logger.debug("Test list: %s" % test_list)
  47
  48   shell = None
  49   if config.target_os == Config.OS_ANDROID:
  50     from mopy.android import AndroidShell
  51     shell = AndroidShell(config)
  52     result = shell.InitShell()
  53     if result != 0:
  54       return result
  55
  56   tests = []
  57   failed = []
  58   for _ in range(args.repeat_count):
  59     for test_dict in test_list:
  60       test = test_dict["test"]
  61       test_name = test_dict.get("name", test)
  62       test_type = test_dict.get("type", "gtest")
  63       test_args = test_dict.get("args", [])
  64
  65       print "Running %s...%s" % (test_name, ("\n" if args.verbose else "")),
  66       sys.stdout.flush()
  67
  68       assert test_type in ("gtest", "gtest_isolated")
  69       isolate = test_type == "gtest_isolated"
  70       (test, fail) = gtest.run_apptest(config, shell, test_args, test, isolate)
  71       tests.extend(test)
  72       failed.extend(fail)
  73       result = test and not fail
  74       print "[  PASSED  ]" if result else "[  FAILED  ]",
  75       print test_name if args.verbose or not result else ""
  76
  77     if failed:
  78       break;
  79
  80   print "[==========] %d tests ran." % len(tests)
  81   print "[  PASSED  ] %d tests." % (len(tests) - len(failed))
  82   if failed:
  83     print "[  FAILED  ] %d tests, listed below:" % len(failed)
  84     for failure in failed:
  85       print "[  FAILED  ] %s" % failure
  86
  87   if args.write_full_results_to:
  88     _WriteJSONResults(tests, failed, args.write_full_results_to)
  89
  90   return 1 if failed else 0
  91
  92
  93 def _WriteJSONResults(tests, failed, write_full_results_to):
  94   """Write the apptest results in the Chromium JSON test results format.
  95      See <http://www.chromium.org/developers/the-json-test-results-format>
  96      TODO(msw): Use Chromium and TYP testing infrastructure.
  97      TODO(msw): Use GTest Suite.Fixture names, not the apptest names.
  98      Adapted from chrome/test/mini_installer/test_installer.py
  99   """
 100   results = {
 101     'interrupted': False,
 102     'path_delimiter': '.',
 103     'version': 3,
 104     'seconds_since_epoch': time.time(),
 105     'num_failures_by_type': {
 106       'FAIL': len(failed),
 107       'PASS': len(tests) - len(failed),
 108     },
 109     'tests': {}
 110   }
 111
 112   for test in tests:
 113     value = {
 114       'expected': 'PASS',
 115       'actual': 'FAIL' if test in failed else 'PASS',
 116       'is_unexpected': True if test in failed else False,
 117     }
 118     _AddPathToTrie(results['tests'], test, value)
 119
 120   with open(write_full_results_to, 'w') as fp:
 121     json.dump(results, fp, indent=2)
 122     fp.write('\n')
 123
 124   return results
 125
 126
 127 def _AddPathToTrie(trie, path, value):
 128   if '.' not in path:
 129     trie[path] = value
 130     return
 131   directory, rest = path.split('.', 1)
 132   if directory not in trie:
 133     trie[directory] = {}
 134   _AddPathToTrie(trie[directory], rest, value)
 135
 136
 137 if __name__ == '__main__':
 138   sys.exit(main())