testing/buildbot/manage.py

   1 #!/usr/bin/env python
   2 # Copyright 2015 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Toolbox to manage all the json files in this directory.
   7
   8 It can reformat them in their canonical format or ensures they are well
   9 formatted.
  10 """
  11
  12 import argparse
  13 import collections
  14 import glob
  15 import json
  16 import os
  17 import subprocess
  18 import sys
  19
  20
  21 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
  22 SRC_DIR = os.path.dirname(os.path.dirname(THIS_DIR))
  23 sys.path.insert(0, os.path.join(SRC_DIR, 'third_party', 'colorama', 'src'))
  24
  25 import colorama
  26
  27
  28 SKIP = {
  29   # These are not 'builders'.
  30   'compile_targets', 'gtest_tests', 'filter_compile_builders',
  31   'non_filter_builders', 'non_filter_tests_builders',
  32
  33   # These are not supported on Swarming yet.
  34   # http://crbug.com/472205
  35   'Chromium Mac 10.10',
  36   # http://crbug.com/441429
  37   'Linux Trusty (32)', 'Linux Trusty (dbg)(32)',
  38
  39   # One off builders. Note that Swarming does support ARM.
  40   'Linux ARM Cross-Compile',
  41   'Site Isolation Linux',
  42   'Site Isolation Win',
  43 }
  44
  45
  46 def upgrade_test(test):
  47   """Converts from old style string to new style dict."""
  48   if isinstance(test, basestring):
  49     return {'test': test}
  50   assert isinstance(test, dict)
  51   return test
  52
  53
  54 def get_isolates():
  55   """Returns the list of all isolate files."""
  56   files = subprocess.check_output(['git', 'ls-files'], cwd=SRC_DIR).splitlines()
  57   return [os.path.basename(f) for f in files if f.endswith('.isolate')]
  58
  59
  60 def main():
  61   colorama.init()
  62   parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__)
  63   group = parser.add_mutually_exclusive_group(required=True)
  64   group.add_argument(
  65       '-c', '--check', action='store_true', help='Only check the files')
  66   group.add_argument(
  67       '--convert', action='store_true',
  68       help='Convert a test to run on Swarming everywhere')
  69   group.add_argument(
  70       '--remaining', action='store_true',
  71       help='Count the number of tests not yet running on Swarming')
  72   group.add_argument(
  73       '-w', '--write', action='store_true', help='Rewrite the files')
  74   parser.add_argument(
  75       'test_name', nargs='?',
  76       help='The test name to print which configs to update; only to be used '
  77            'with --remaining')
  78   args = parser.parse_args()
  79
  80   if args.convert or args.remaining:
  81     isolates = get_isolates()
  82
  83   if args.convert:
  84     if not args.test_name:
  85       parser.error('A test name is required with --convert')
  86     if args.test_name + '.isolate' not in isolates:
  87       parser.error('Create %s.isolate first' % args.test_name)
  88
  89   # Stats when running in --remaining mode;
  90   tests_location = collections.defaultdict(
  91       lambda: {
  92         'count_run_local': 0, 'count_run_on_swarming': 0, 'local_configs': {}
  93       })
  94
  95   result = 0
  96   for filepath in glob.glob(os.path.join(THIS_DIR, '*.json')):
  97     filename = os.path.basename(filepath)
  98     with open(filepath) as f:
  99       content = f.read()
 100     try:
 101       config = json.loads(content)
 102     except ValueError as e:
 103       print "Exception raised while checking %s: %s" % (filepath, e)
 104       raise
 105     for builder, data in sorted(config.iteritems()):
 106       if builder in SKIP:
 107         # Oddities.
 108         continue
 109
 110       if not isinstance(data, dict):
 111         print('%s: %s is broken: %s' % (filename, builder, data))
 112         continue
 113
 114       if 'gtest_tests' in data:
 115         config[builder]['gtest_tests'] = sorted(
 116           (upgrade_test(l) for l in data['gtest_tests']),
 117           key=lambda x: x['test'])
 118
 119         if args.remaining:
 120           for test in data['gtest_tests']:
 121             name = test['test']
 122             if test.get('swarming', {}).get('can_use_on_swarming_builders'):
 123               tests_location[name]['count_run_on_swarming'] += 1
 124             else:
 125               tests_location[name]['count_run_local'] += 1
 126               tests_location[name]['local_configs'].setdefault(
 127                   filename, []).append(builder)
 128         elif args.convert:
 129           for test in data['gtest_tests']:
 130             if test['test'] != args.test_name:
 131               continue
 132             test.setdefault('swarming', {})
 133             if not test['swarming'].get('can_use_on_swarming_builders'):
 134               print('- %s: %s' % (filename, builder))
 135               test['swarming']['can_use_on_swarming_builders'] = True
 136
 137     expected = json.dumps(
 138         config, sort_keys=True, indent=2, separators=(',', ': ')) + '\n'
 139     if content != expected:
 140       result = 1
 141       if args.write or args.convert:
 142         with open(filepath, 'wb') as f:
 143           f.write(expected)
 144         if args.write:
 145           print('Updated %s' % filename)
 146       else:
 147         print('%s is not in canonical format' % filename)
 148
 149   if args.remaining:
 150     if args.test_name:
 151       if args.test_name not in tests_location:
 152         print('Unknown test %s' % args.test_name)
 153         return 1
 154       for config, builders in sorted(
 155           tests_location[args.test_name]['local_configs'].iteritems()):
 156         print('%s:' % config)
 157         for builder in sorted(builders):
 158           print('  %s' % builder)
 159     else:
 160       l = max(map(len, tests_location))
 161       print('%-*s%sLocal       %sSwarming  %sMissing isolate' %
 162           (l, 'Test', colorama.Fore.RED, colorama.Fore.GREEN,
 163             colorama.Fore.MAGENTA))
 164       total_local = 0
 165       total_swarming = 0
 166       for name, location in sorted(tests_location.iteritems()):
 167         if not location['count_run_on_swarming']:
 168           c = colorama.Fore.RED
 169         elif location['count_run_local']:
 170           c = colorama.Fore.YELLOW
 171         else:
 172           c = colorama.Fore.GREEN
 173         total_local += location['count_run_local']
 174         total_swarming += location['count_run_on_swarming']
 175         missing_isolate = ''
 176         if name + '.isolate' not in isolates:
 177           missing_isolate = colorama.Fore.MAGENTA + '*'
 178         print('%s%-*s %4d           %4d    %s' %
 179             (c, l, name, location['count_run_local'],
 180               location['count_run_on_swarming'], missing_isolate))
 181
 182       total = total_local + total_swarming
 183       p_local = 100. * total_local / total
 184       p_swarming = 100. * total_swarming / total
 185       print('%s%-*s %4d (%4.1f%%)   %4d (%4.1f%%)' %
 186           (colorama.Fore.WHITE, l, 'Total:', total_local, p_local,
 187             total_swarming, p_swarming))
 188       print('%-*s                %4d' % (l, 'Total executions:', total))
 189   return result
 190
 191
 192 if __name__ == "__main__":
 193   sys.exit(main())