testing/scripts/common.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import argparse
   6 import contextlib
   7 import json
   8 import os
   9 import subprocess
  10 import sys
  11 import tempfile
  12
  13
  14 SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
  15 SRC_DIR = os.path.abspath(
  16     os.path.join(SCRIPT_DIR, os.path.pardir, os.path.pardir))
  17
  18
  19 # run-webkit-tests returns the number of failures as the return
  20 # code, but caps the return code at 101 to avoid overflow or colliding
  21 # with reserved values from the shell.
  22 MAX_FAILURES_EXIT_STATUS = 101
  23
  24
  25 def run_script(argv, funcs):
  26   def parse_json(path):
  27     with open(path) as f:
  28       return json.load(f)
  29   parser = argparse.ArgumentParser()
  30   # TODO(phajdan.jr): Make build-config-fs required after passing it in recipe.
  31   parser.add_argument('--build-config-fs')
  32   parser.add_argument('--paths', type=parse_json, default={})
  33   parser.add_argument('--properties', type=parse_json, default={})
  34
  35   subparsers = parser.add_subparsers()
  36
  37   run_parser = subparsers.add_parser('run')
  38   run_parser.add_argument(
  39       '--output', type=argparse.FileType('w'), required=True)
  40   run_parser.add_argument('--filter-file', type=argparse.FileType('r'))
  41   run_parser.set_defaults(func=funcs['run'])
  42
  43   run_parser = subparsers.add_parser('compile_targets')
  44   run_parser.add_argument(
  45       '--output', type=argparse.FileType('w'), required=True)
  46   run_parser.set_defaults(func=funcs['compile_targets'])
  47
  48   args = parser.parse_args(argv)
  49   return args.func(args)
  50
  51
  52 def run_command(argv):
  53   print 'Running %r' % argv
  54   rc = subprocess.call(argv)
  55   print 'Command %r returned exit code %d' % (argv, rc)
  56   return rc
  57
  58
  59 def run_runtest(cmd_args, runtest_args):
  60   return run_command([
  61       sys.executable,
  62       os.path.join(cmd_args.paths['build'], 'scripts', 'tools', 'runit.py'),
  63       '--show-path',
  64       sys.executable,
  65       os.path.join(cmd_args.paths['build'], 'scripts', 'slave', 'runtest.py'),
  66       '--target', cmd_args.build_config_fs,
  67       '--xvfb',
  68       '--builder-name', cmd_args.properties['buildername'],
  69       '--slave-name', cmd_args.properties['slavename'],
  70       '--build-number', str(cmd_args.properties['buildnumber']),
  71       '--build-properties', json.dumps(cmd_args.properties),
  72   ] + runtest_args)
  73
  74
  75 @contextlib.contextmanager
  76 def temporary_file():
  77   fd, path = tempfile.mkstemp()
  78   os.close(fd)
  79   try:
  80     yield path
  81   finally:
  82     os.remove(path)
  83
  84
  85 def parse_common_test_results(json_results, test_separator='/'):
  86   def convert_trie_to_flat_paths(trie, prefix=None):
  87     # Also see webkitpy.layout_tests.layout_package.json_results_generator
  88     result = {}
  89     for name, data in trie.iteritems():
  90       if prefix:
  91         name = prefix + test_separator + name
  92       if len(data) and not 'actual' in data and not 'expected' in data:
  93         result.update(convert_trie_to_flat_paths(data, name))
  94       else:
  95         result[name] = data
  96     return result
  97
  98   results = {
  99     'passes': {},
 100     'unexpected_passes': {},
 101     'failures': {},
 102     'unexpected_failures': {},
 103     'flakes': {},
 104     'unexpected_flakes': {},
 105   }
 106
 107   # TODO(dpranke): crbug.com/357866 - we should simplify the handling of
 108   # both the return code and parsing the actual results, below.
 109
 110   passing_statuses = ('PASS', 'SLOW', 'NEEDSREBASELINE',
 111                         'NEEDSMANUALREBASELINE')
 112
 113   for test, result in convert_trie_to_flat_paths(
 114       json_results['tests']).iteritems():
 115     key = 'unexpected_' if result.get('is_unexpected') else ''
 116     data = result['actual']
 117     actual_results = data.split()
 118     last_result = actual_results[-1]
 119     expected_results = result['expected'].split()
 120
 121     if (len(actual_results) > 1 and
 122         (last_result in expected_results or last_result in passing_statuses)):
 123       key += 'flakes'
 124     elif last_result in passing_statuses:
 125       key += 'passes'
 126       # TODO(dpranke): crbug.com/357867 ...  Why are we assigning result
 127       # instead of actual_result here. Do we even need these things to be
 128       # hashes, or just lists?
 129       data = result
 130     else:
 131       key += 'failures'
 132     results[key][test] = data
 133
 134   return results