third_party/WebKit/Tools/Scripts/print-json-test-results

   1 #!/usr/bin/python
   2 import json
   3 import optparse
   4 import os
   5 import sys
   6
   7 from webkitpy.common.host import Host
   8 from webkitpy.layout_tests.port import platform_options, configuration_options
   9
  10
  11 def main(argv):
  12     parser = optparse.OptionParser(usage='%prog [path-to-results.json]')
  13     parser.add_option('--failures', action='store_true',
  14                       help='show failing tests')
  15     parser.add_option('--flakes', action='store_true',
  16                       help='show flaky tests')
  17     parser.add_option('--expected', action='store_true',
  18                       help='include expected results along with unexpected')
  19     parser.add_option('--passes', action='store_true',
  20                       help='show passing tests')
  21     parser.add_option('--ignored-failures-path', action='store',
  22                       help='ignore failures seen in a previous run')
  23     parser.add_options(platform_options())
  24     parser.add_options(configuration_options())
  25     options, args = parser.parse_args(argv)
  26
  27     host = Host()
  28     if args:
  29         if args[0] == '-':
  30             txt = sys.stdin.read()
  31         elif os.path.exists(args[0]):
  32             with open(args[0], 'r') as fp:
  33                 txt = fp.read()
  34         else:
  35             print >> sys.stderr, "file not found: %s" % args[0]
  36             sys.exit(1)
  37     else:
  38         txt = host.filesystem.read_text_file(host.filesystem.join(host.port_factory.get(options=options).results_directory(), 'full_results.json'))
  39
  40     if txt.startswith('ADD_RESULTS(') and txt.endswith(');'):
  41         txt = txt[12:-2]  # ignore optional JSONP wrapper
  42     results = json.loads(txt)
  43
  44     passes, failures, flakes = decode_results(results, options.expected)
  45
  46     tests_to_print = []
  47     if options.passes:
  48         tests_to_print += passes.keys()
  49     if options.failures:
  50         tests_to_print += failures.keys()
  51     if options.flakes:
  52         tests_to_print += flakes.keys()
  53     print "\n".join(sorted(tests_to_print))
  54
  55     if options.ignored_failures_path:
  56         with open(options.ignored_failures_path, 'r') as fp:
  57             txt = fp.read()
  58         if txt.startswith('ADD_RESULTS(') and txt.endswith(');'):
  59             txt = txt[12:-2]  # ignore optional JSONP wrapper
  60         results = json.loads(txt)
  61         _, ignored_failures, _ = decode_results(results, options.expected)
  62         new_failures = set(failures.keys()) - set(ignored_failures.keys())
  63         if new_failures:
  64             print "New failures:"
  65             print "\n".join(sorted(new_failures))
  66             print
  67         if ignored_failures:
  68             print "Ignored failures:"
  69             print "\n".join(sorted(ignored_failures.keys()))
  70         if new_failures:
  71             return 1
  72         return 0
  73
  74
  75 def decode_results(results, include_expected=False):
  76     tests = convert_trie_to_flat_paths(results['tests'])
  77     failures = {}
  78     flakes = {}
  79     passes = {}
  80     for (test, result) in tests.iteritems():
  81         if include_expected or result.get('is_unexpected'):
  82             actual_results = result['actual'].split()
  83             expected_results = result['expected'].split()
  84             if len(actual_results) > 1:
  85                 if actual_results[1] in expected_results:
  86                     flakes[test] = actual_results[0]
  87                 else:
  88                     # We report the first failure type back, even if the second
  89                     # was more severe.
  90                     failures[test] = actual_results[0]
  91             elif actual_results[0] == 'PASS':
  92                 passes[test] = result
  93             else:
  94                 failures[test] = actual_results[0]
  95
  96     return (passes, failures, flakes)
  97
  98
  99 def convert_trie_to_flat_paths(trie, prefix=None):
 100     # Cloned from webkitpy.layout_tests.layout_package.json_results_generator
 101     # so that this code can stand alone.
 102     result = {}
 103     for name, data in trie.iteritems():
 104         if prefix:
 105             name = prefix + "/" + name
 106
 107         if len(data) and not "actual" in data and not "expected" in data:
 108             result.update(convert_trie_to_flat_paths(data, name))
 109         else:
 110             result[name] = data
 111
 112     return result
 113
 114
 115 if __name__ ==  '__main__':
 116     sys.exit(main(sys.argv[1:]))