testing/buildbot/manage.py

   1 #!/usr/bin/env python
   2 # Copyright 2015 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Toolbox to manage all the json files in this directory.
   7
   8 It can reformat them in their canonical format or ensures they are well
   9 formatted.
  10 """
  11
  12 import argparse
  13 import collections
  14 import glob
  15 import json
  16 import os
  17 import subprocess
  18 import sys
  19
  20
  21 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
  22 SRC_DIR = os.path.dirname(os.path.dirname(THIS_DIR))
  23 sys.path.insert(0, os.path.join(SRC_DIR, 'third_party', 'colorama', 'src'))
  24
  25 import colorama
  26
  27
  28 SKIP = {
  29   # These are not 'builders'.
  30   'compile_targets', 'gtest_tests', 'filter_compile_builders',
  31   'non_filter_builders', 'non_filter_tests_builders',
  32
  33   # These are not supported on Swarming yet.
  34   # http://crbug.com/472205
  35   'Chromium Mac 10.10',
  36   # http://crbug.com/441429
  37   'Linux Trusty (32)', 'Linux Trusty (dbg)(32)',
  38
  39   # One off builders. Note that Swarming does support ARM.
  40   'Linux ARM Cross-Compile',
  41   'Site Isolation Linux',
  42   'Site Isolation Win',
  43 }
  44
  45
  46 def upgrade_test(test):
  47   """Converts from old style string to new style dict."""
  48   if isinstance(test, basestring):
  49     return {'test': test}
  50   assert isinstance(test, dict)
  51   return test
  52
  53
  54 def get_isolates():
  55   """Returns the list of all isolate files."""
  56   files = subprocess.check_output(['git', 'ls-files'], cwd=SRC_DIR).splitlines()
  57   return [os.path.basename(f) for f in files if f.endswith('.isolate')]
  58
  59
  60 def main():
  61   colorama.init()
  62   parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__)
  63   group = parser.add_mutually_exclusive_group(required=True)
  64   group.add_argument(
  65       '-c', '--check', action='store_true', help='Only check the files')
  66   group.add_argument(
  67       '--convert', action='store_true',
  68       help='Convert a test to run on Swarming everywhere')
  69   group.add_argument(
  70       '--remaining', action='store_true',
  71       help='Count the number of tests not yet running on Swarming')
  72   group.add_argument(
  73       '-w', '--write', action='store_true', help='Rewrite the files')
  74   parser.add_argument(
  75       'test_name', nargs='?',
  76       help='The test name to print which configs to update; only to be used '
  77            'with --remaining')
  78   args = parser.parse_args()
  79
  80   if args.convert or args.remaining:
  81     isolates = get_isolates()
  82
  83   if args.convert:
  84     if not args.test_name:
  85       parser.error('A test name is required with --convert')
  86     if args.test_name + '.isolate' not in isolates:
  87       parser.error('Create %s.isolate first' % args.test_name)
  88
  89   # Stats when running in --remaining mode;
  90   tests_location = collections.defaultdict(
  91       lambda: {
  92         'count_run_local': 0, 'count_run_on_swarming': 0, 'local_configs': {}
  93       })
  94
  95   result = 0
  96   for filepath in glob.glob(os.path.join(THIS_DIR, '*.json')):
  97     filename = os.path.basename(filepath)
  98     with open(filepath) as f:
  99       content = f.read()
 100     config = json.loads(content)
 101     for builder, data in sorted(config.iteritems()):
 102       if builder in SKIP:
 103         # Oddities.
 104         continue
 105
 106       if not isinstance(data, dict):
 107         print('%s: %s is broken: %s' % (filename, builder, data))
 108         continue
 109
 110       if 'gtest_tests' in data:
 111         config[builder]['gtest_tests'] = sorted(
 112           (upgrade_test(l) for l in data['gtest_tests']),
 113           key=lambda x: x['test'])
 114
 115         if args.remaining:
 116           for test in data['gtest_tests']:
 117             name = test['test']
 118             if test.get('swarming', {}).get('can_use_on_swarming_builders'):
 119               tests_location[name]['count_run_on_swarming'] += 1
 120             else:
 121               tests_location[name]['count_run_local'] += 1
 122               tests_location[name]['local_configs'].setdefault(
 123                   filename, []).append(builder)
 124         elif args.convert:
 125           for test in data['gtest_tests']:
 126             if test['test'] != args.test_name:
 127               continue
 128             test.setdefault('swarming', {})
 129             if not test['swarming'].get('can_use_on_swarming_builders'):
 130               print('- %s: %s' % (filename, builder))
 131               test['swarming']['can_use_on_swarming_builders'] = True
 132
 133     expected = json.dumps(
 134         config, sort_keys=True, indent=2, separators=(',', ': ')) + '\n'
 135     if content != expected:
 136       result = 1
 137       if args.write or args.convert:
 138         with open(filepath, 'wb') as f:
 139           f.write(expected)
 140         if args.write:
 141           print('Updated %s' % filename)
 142       else:
 143         print('%s is not in canonical format' % filename)
 144
 145   if args.remaining:
 146     if args.test_name:
 147       if args.test_name not in tests_location:
 148         print('Unknown test %s' % args.test_name)
 149         return 1
 150       for config, builders in sorted(
 151           tests_location[args.test_name]['local_configs'].iteritems()):
 152         print('%s:' % config)
 153         for builder in sorted(builders):
 154           print('  %s' % builder)
 155     else:
 156       l = max(map(len, tests_location))
 157       print('%-*s%sLocal       %sSwarming  %sMissing isolate' %
 158           (l, 'Test', colorama.Fore.RED, colorama.Fore.GREEN,
 159             colorama.Fore.MAGENTA))
 160       total_local = 0
 161       total_swarming = 0
 162       for name, location in sorted(tests_location.iteritems()):
 163         if not location['count_run_on_swarming']:
 164           c = colorama.Fore.RED
 165         elif location['count_run_local']:
 166           c = colorama.Fore.YELLOW
 167         else:
 168           c = colorama.Fore.GREEN
 169         total_local += location['count_run_local']
 170         total_swarming += location['count_run_on_swarming']
 171         missing_isolate = ''
 172         if name + '.isolate' not in isolates:
 173           missing_isolate = colorama.Fore.MAGENTA + '*'
 174         print('%s%-*s %4d           %4d    %s' %
 175             (c, l, name, location['count_run_local'],
 176               location['count_run_on_swarming'], missing_isolate))
 177
 178       total = total_local + total_swarming
 179       p_local = 100. * total_local / total
 180       p_swarming = 100. * total_swarming / total
 181       print('%s%-*s %4d (%4.1f%%)   %4d (%4.1f%%)' %
 182           (colorama.Fore.WHITE, l, 'Total:', total_local, p_local,
 183             total_swarming, p_swarming))
 184       print('%-*s                %4d' % (l, 'Total executions:', total))
 185   return result
 186
 187
 188 if __name__ == "__main__":
 189   sys.exit(main())