testing/buildbot/manage.py

   1 #!/usr/bin/env python
   2 # Copyright 2015 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Toolbox to manage all the json files in this directory.
   7
   8 It can reformat them in their canonical format or ensures they are well
   9 formatted.
  10 """
  11
  12 import argparse
  13 import ast
  14 import collections
  15 import glob
  16 import json
  17 import os
  18 import subprocess
  19 import sys
  20
  21
  22 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
  23 SRC_DIR = os.path.dirname(os.path.dirname(THIS_DIR))
  24 sys.path.insert(0, os.path.join(SRC_DIR, 'third_party', 'colorama', 'src'))
  25
  26 import colorama
  27
  28
  29 SKIP = {
  30   # These are not 'builders'.
  31   'compile_targets', 'gtest_tests', 'filter_compile_builders',
  32   'non_filter_builders', 'non_filter_tests_builders',
  33
  34   # These are not supported on Swarming yet.
  35   # http://crbug.com/472205
  36   'Chromium Mac 10.10',
  37   # http://crbug.com/441429
  38   'Linux Trusty (32)', 'Linux Trusty (dbg)(32)',
  39
  40   # http://crbug.com/480053
  41   'Linux GN',
  42   'Linux GN (dbg)',
  43
  44   # Unmaintained builders on chromium.fyi
  45   'ClangToTMac',
  46   'ClangToTMacASan',
  47
  48   # This builder is fine, but win8_chromium_ng uses GN and this configuration,
  49   # which breaks everything.
  50   'Win8 Aura',
  51
  52   # One off builders. Note that Swarming does support ARM.
  53   'Linux ARM Cross-Compile',
  54   'Site Isolation Linux',
  55   'Site Isolation Win',
  56 }
  57
  58
  59 # TODO(GYP): These targets have not been ported to GN yet.
  60 SKIP_NINJA_TO_GN_TARGETS = {
  61   'cast_media_unittests',
  62   'cast_shell_browser_test',
  63   'chromevox_tests',
  64   'nacl_helper_nonsfi_unittests',
  65 }
  66
  67
  68 class Error(Exception):
  69   """Processing error."""
  70
  71
  72 def get_isolates():
  73   """Returns the list of all isolate files."""
  74   files = subprocess.check_output(['git', 'ls-files'], cwd=SRC_DIR).splitlines()
  75   return [os.path.basename(f) for f in files if f.endswith('.isolate')]
  76
  77
  78 def process_builder_convert(data, test_name):
  79   """Converts 'test_name' to run on Swarming in 'data'.
  80
  81   Returns True if 'test_name' was found.
  82   """
  83   result = False
  84   for test in data['gtest_tests']:
  85     if test['test'] != test_name:
  86       continue
  87     test.setdefault('swarming', {})
  88     if not test['swarming'].get('can_use_on_swarming_builders'):
  89       test['swarming']['can_use_on_swarming_builders'] = True
  90     result = True
  91   return result
  92
  93
  94 def process_builder_remaining(data, filename, builder, tests_location):
  95   """Calculates tests_location when mode is --remaining."""
  96   for test in data['gtest_tests']:
  97     name = test['test']
  98     if test.get('swarming', {}).get('can_use_on_swarming_builders'):
  99       tests_location[name]['count_run_on_swarming'] += 1
 100     else:
 101       tests_location[name]['count_run_local'] += 1
 102       tests_location[name]['local_configs'].setdefault(
 103           filename, []).append(builder)
 104
 105
 106 def process_file(mode, test_name, tests_location, filepath, ninja_targets,
 107                  ninja_targets_seen):
 108   """Processes a json file describing what tests should be run for each recipe.
 109
 110   The action depends on mode. Updates tests_location.
 111
 112   Return False if the process exit code should be 1.
 113   """
 114   filename = os.path.basename(filepath)
 115   with open(filepath) as f:
 116     content = f.read()
 117   try:
 118     config = json.loads(content)
 119   except ValueError as e:
 120     raise Error('Exception raised while checking %s: %s' % (filepath, e))
 121
 122   for builder, data in sorted(config.iteritems()):
 123     if builder in SKIP:
 124       # Oddities.
 125       continue
 126     if not isinstance(data, dict):
 127       raise Error('%s: %s is broken: %s' % (filename, builder, data))
 128     if 'gtest_tests' not in data:
 129       continue
 130     if not isinstance(data['gtest_tests'], list):
 131       raise Error(
 132           '%s: %s is broken: %s' % (filename, builder, data['gtest_tests']))
 133     if not all(isinstance(g, dict) for g in data['gtest_tests']):
 134       raise Error(
 135           '%s: %s is broken: %s' % (filename, builder, data['gtest_tests']))
 136
 137     for d in data['gtest_tests']:
 138       if (d['test'] not in ninja_targets and
 139           d['test'] not in SKIP_NINJA_TO_GN_TARGETS):
 140         raise Error('%s: %s / %s is not listed in gn_isolate_map.pyl.' %
 141                     (filename, builder, d['test']))
 142       elif d['test'] in ninja_targets:
 143         ninja_targets_seen.add(d['test'])
 144
 145     config[builder]['gtest_tests'] = sorted(
 146         data['gtest_tests'], key=lambda x: x['test'])
 147
 148     # The trick here is that process_builder_remaining() is called before
 149     # process_builder_convert() so tests_location can be used to know how many
 150     # tests were converted.
 151     if mode in ('convert', 'remaining'):
 152       process_builder_remaining(data, filename, builder, tests_location)
 153     if mode == 'convert':
 154       process_builder_convert(data, test_name)
 155
 156   expected = json.dumps(
 157       config, sort_keys=True, indent=2, separators=(',', ': ')) + '\n'
 158   if content != expected:
 159     if mode in ('convert', 'write'):
 160       with open(filepath, 'wb') as f:
 161         f.write(expected)
 162       if mode == 'write':
 163         print('Updated %s' % filename)
 164     else:
 165       print('%s is not in canonical format' % filename)
 166       print('run `testing/buildbot/manage.py -w` to fix')
 167     return mode != 'check'
 168   return True
 169
 170
 171 def print_convert(test_name, tests_location):
 172   """Prints statistics for a test being converted for use in a CL description.
 173   """
 174   data = tests_location[test_name]
 175   print('Convert %s to run exclusively on Swarming' % test_name)
 176   print('')
 177   print('%d configs already ran on Swarming' % data['count_run_on_swarming'])
 178   print('%d used to run locally and were converted:' % data['count_run_local'])
 179   for master, builders in sorted(data['local_configs'].iteritems()):
 180     for builder in builders:
 181       print('- %s: %s' % (master, builder))
 182   print('')
 183   print('Ran:')
 184   print('  ./manage.py --convert %s' % test_name)
 185   print('')
 186   print('R=')
 187   print('BUG=98637')
 188
 189
 190 def print_remaining(test_name, tests_location):
 191   """Prints a visual summary of what tests are yet to be converted to run on
 192   Swarming.
 193   """
 194   if test_name:
 195     if test_name not in tests_location:
 196       raise Error('Unknown test %s' % test_name)
 197     for config, builders in sorted(
 198         tests_location[test_name]['local_configs'].iteritems()):
 199       print('%s:' % config)
 200       for builder in sorted(builders):
 201         print('  %s' % builder)
 202     return
 203
 204   isolates = get_isolates()
 205   l = max(map(len, tests_location))
 206   print('%-*s%sLocal       %sSwarming  %sMissing isolate' %
 207       (l, 'Test', colorama.Fore.RED, colorama.Fore.GREEN,
 208         colorama.Fore.MAGENTA))
 209   total_local = 0
 210   total_swarming = 0
 211   for name, location in sorted(tests_location.iteritems()):
 212     if not location['count_run_on_swarming']:
 213       c = colorama.Fore.RED
 214     elif location['count_run_local']:
 215       c = colorama.Fore.YELLOW
 216     else:
 217       c = colorama.Fore.GREEN
 218     total_local += location['count_run_local']
 219     total_swarming += location['count_run_on_swarming']
 220     missing_isolate = ''
 221     if name + '.isolate' not in isolates:
 222       missing_isolate = colorama.Fore.MAGENTA + '*'
 223     print('%s%-*s %4d           %4d    %s' %
 224         (c, l, name, location['count_run_local'],
 225           location['count_run_on_swarming'], missing_isolate))
 226
 227   total = total_local + total_swarming
 228   p_local = 100. * total_local / total
 229   p_swarming = 100. * total_swarming / total
 230   print('%s%-*s %4d (%4.1f%%)   %4d (%4.1f%%)' %
 231       (colorama.Fore.WHITE, l, 'Total:', total_local, p_local,
 232         total_swarming, p_swarming))
 233   print('%-*s                %4d' % (l, 'Total executions:', total))
 234
 235
 236 def main():
 237   colorama.init()
 238   parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__)
 239   group = parser.add_mutually_exclusive_group(required=True)
 240   group.add_argument(
 241       '-c', '--check', dest='mode', action='store_const', const='check',
 242       default='check', help='Only check the files')
 243   group.add_argument(
 244       '--convert', dest='mode', action='store_const', const='convert',
 245       help='Convert a test to run on Swarming everywhere')
 246   group.add_argument(
 247       '--remaining', dest='mode', action='store_const', const='remaining',
 248       help='Count the number of tests not yet running on Swarming')
 249   group.add_argument(
 250       '-w', '--write', dest='mode', action='store_const', const='write',
 251       help='Rewrite the files')
 252   parser.add_argument(
 253       'test_name', nargs='?',
 254       help='The test name to print which configs to update; only to be used '
 255            'with --remaining')
 256   args = parser.parse_args()
 257
 258   if args.mode == 'convert':
 259     if not args.test_name:
 260       parser.error('A test name is required with --convert')
 261     if args.test_name + '.isolate' not in get_isolates():
 262       parser.error('Create %s.isolate first' % args.test_name)
 263
 264   # Stats when running in --remaining mode;
 265   tests_location = collections.defaultdict(
 266       lambda: {
 267         'count_run_local': 0, 'count_run_on_swarming': 0, 'local_configs': {}
 268       })
 269
 270   with open(os.path.join(THIS_DIR, "gn_isolate_map.pyl")) as fp:
 271     gn_isolate_map = ast.literal_eval(fp.read())
 272     ninja_targets = {k: v['label'] for k, v in gn_isolate_map.items()}
 273
 274   try:
 275     result = 0
 276     ninja_targets_seen = set()
 277     for filepath in glob.glob(os.path.join(THIS_DIR, '*.json')):
 278       if not process_file(args.mode, args.test_name, tests_location, filepath,
 279                           ninja_targets, ninja_targets_seen):
 280         result = 1
 281
 282     extra_targets = set(ninja_targets) - ninja_targets_seen
 283     if extra_targets:
 284       if len(extra_targets) > 1:
 285         extra_targets_str = ', '.join(extra_targets) + ' are'
 286       else:
 287         extra_targets_str = list(extra_targets)[0] + ' is'
 288       raise Error('%s listed in ninja_to_gn.pyl but not in any .json files' %
 289                   extra_targets_str)
 290
 291     if args.mode == 'convert':
 292       print_convert(args.test_name, tests_location)
 293     elif args.mode == 'remaining':
 294       print_remaining(args.test_name, tests_location)
 295     return result
 296   except Error as e:
 297     sys.stderr.write('%s\n' % e)
 298     return 1
 299
 300
 301 if __name__ == "__main__":
 302   sys.exit(main())