testing/buildbot/manage.py

   1 #!/usr/bin/env python
   2 # Copyright 2015 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Toolbox to manage all the json files in this directory.
   7
   8 It can reformat them in their canonical format or ensures they are well
   9 formatted.
  10 """
  11
  12 import argparse
  13 import ast
  14 import collections
  15 import glob
  16 import json
  17 import os
  18 import subprocess
  19 import sys
  20
  21
  22 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
  23 SRC_DIR = os.path.dirname(os.path.dirname(THIS_DIR))
  24 sys.path.insert(0, os.path.join(SRC_DIR, 'third_party', 'colorama', 'src'))
  25
  26 import colorama
  27
  28
  29 SKIP = {
  30   # These are not 'builders'.
  31   'compile_targets', 'gtest_tests', 'filter_compile_builders',
  32   'non_filter_builders', 'non_filter_tests_builders',
  33
  34   # These are not supported on Swarming yet.
  35   # http://crbug.com/472205
  36   'Chromium Mac 10.10',
  37   # http://crbug.com/441429
  38   'Linux Trusty (32)', 'Linux Trusty (dbg)(32)',
  39
  40   # http://crbug.com/480053
  41   'Linux GN',
  42   'Linux GN (dbg)',
  43
  44   # Unmaintained builders on chromium.fyi
  45   'ClangToTMac',
  46   'ClangToTMacASan',
  47
  48   # This builder is fine, but win8_chromium_ng uses GN and this configuration,
  49   # which breaks everything.
  50   'Win8 Aura',
  51
  52   # One off builders. Note that Swarming does support ARM.
  53   'Linux ARM Cross-Compile',
  54   'Site Isolation Linux',
  55   'Site Isolation Win',
  56 }
  57
  58
  59 SKIP_GN_ISOLATE_MAP_TARGETS = {
  60   # TODO(GYP): These targets have not been ported to GN yet.
  61   'cast_media_unittests',
  62   'cast_shell_browser_test',
  63   'chromevox_tests',
  64   'nacl_helper_nonsfi_unittests',
  65
  66   # These targets are run on the bots but not listed in the
  67   # buildbot JSON files.
  68   'angle_end2end_tests',
  69   'content_gl_tests',
  70   'gl_tests',
  71   'gles2_conform_test',
  72   'tab_capture_end2end_tests',
  73   'telemetry_gpu_test',
  74 }
  75
  76
  77 class Error(Exception):
  78   """Processing error."""
  79
  80
  81 def get_isolates():
  82   """Returns the list of all isolate files."""
  83   files = subprocess.check_output(['git', 'ls-files'], cwd=SRC_DIR).splitlines()
  84   return [os.path.basename(f) for f in files if f.endswith('.isolate')]
  85
  86
  87 def process_builder_convert(data, test_name):
  88   """Converts 'test_name' to run on Swarming in 'data'.
  89
  90   Returns True if 'test_name' was found.
  91   """
  92   result = False
  93   for test in data['gtest_tests']:
  94     if test['test'] != test_name:
  95       continue
  96     test.setdefault('swarming', {})
  97     if not test['swarming'].get('can_use_on_swarming_builders'):
  98       test['swarming']['can_use_on_swarming_builders'] = True
  99     result = True
 100   return result
 101
 102
 103 def process_builder_remaining(data, filename, builder, tests_location):
 104   """Calculates tests_location when mode is --remaining."""
 105   for test in data['gtest_tests']:
 106     name = test['test']
 107     if test.get('swarming', {}).get('can_use_on_swarming_builders'):
 108       tests_location[name]['count_run_on_swarming'] += 1
 109     else:
 110       tests_location[name]['count_run_local'] += 1
 111       tests_location[name]['local_configs'].setdefault(
 112           filename, []).append(builder)
 113
 114
 115 def process_file(mode, test_name, tests_location, filepath, ninja_targets,
 116                  ninja_targets_seen):
 117   """Processes a json file describing what tests should be run for each recipe.
 118
 119   The action depends on mode. Updates tests_location.
 120
 121   Return False if the process exit code should be 1.
 122   """
 123   filename = os.path.basename(filepath)
 124   with open(filepath) as f:
 125     content = f.read()
 126   try:
 127     config = json.loads(content)
 128   except ValueError as e:
 129     raise Error('Exception raised while checking %s: %s' % (filepath, e))
 130
 131   for builder, data in sorted(config.iteritems()):
 132     if builder in SKIP:
 133       # Oddities.
 134       continue
 135     if not isinstance(data, dict):
 136       raise Error('%s: %s is broken: %s' % (filename, builder, data))
 137     if 'gtest_tests' not in data:
 138       continue
 139     if not isinstance(data['gtest_tests'], list):
 140       raise Error(
 141           '%s: %s is broken: %s' % (filename, builder, data['gtest_tests']))
 142     if not all(isinstance(g, dict) for g in data['gtest_tests']):
 143       raise Error(
 144           '%s: %s is broken: %s' % (filename, builder, data['gtest_tests']))
 145
 146     for d in data['gtest_tests']:
 147       if (d['test'] not in ninja_targets and
 148           d['test'] not in SKIP_GN_ISOLATE_MAP_TARGETS):
 149         raise Error('%s: %s / %s is not listed in gn_isolate_map.pyl.' %
 150                     (filename, builder, d['test']))
 151       elif d['test'] in ninja_targets:
 152         ninja_targets_seen.add(d['test'])
 153
 154     config[builder]['gtest_tests'] = sorted(
 155         data['gtest_tests'], key=lambda x: x['test'])
 156
 157     # The trick here is that process_builder_remaining() is called before
 158     # process_builder_convert() so tests_location can be used to know how many
 159     # tests were converted.
 160     if mode in ('convert', 'remaining'):
 161       process_builder_remaining(data, filename, builder, tests_location)
 162     if mode == 'convert':
 163       process_builder_convert(data, test_name)
 164
 165   expected = json.dumps(
 166       config, sort_keys=True, indent=2, separators=(',', ': ')) + '\n'
 167   if content != expected:
 168     if mode in ('convert', 'write'):
 169       with open(filepath, 'wb') as f:
 170         f.write(expected)
 171       if mode == 'write':
 172         print('Updated %s' % filename)
 173     else:
 174       print('%s is not in canonical format' % filename)
 175       print('run `testing/buildbot/manage.py -w` to fix')
 176     return mode != 'check'
 177   return True
 178
 179
 180 def print_convert(test_name, tests_location):
 181   """Prints statistics for a test being converted for use in a CL description.
 182   """
 183   data = tests_location[test_name]
 184   print('Convert %s to run exclusively on Swarming' % test_name)
 185   print('')
 186   print('%d configs already ran on Swarming' % data['count_run_on_swarming'])
 187   print('%d used to run locally and were converted:' % data['count_run_local'])
 188   for master, builders in sorted(data['local_configs'].iteritems()):
 189     for builder in builders:
 190       print('- %s: %s' % (master, builder))
 191   print('')
 192   print('Ran:')
 193   print('  ./manage.py --convert %s' % test_name)
 194   print('')
 195   print('R=')
 196   print('BUG=98637')
 197
 198
 199 def print_remaining(test_name, tests_location):
 200   """Prints a visual summary of what tests are yet to be converted to run on
 201   Swarming.
 202   """
 203   if test_name:
 204     if test_name not in tests_location:
 205       raise Error('Unknown test %s' % test_name)
 206     for config, builders in sorted(
 207         tests_location[test_name]['local_configs'].iteritems()):
 208       print('%s:' % config)
 209       for builder in sorted(builders):
 210         print('  %s' % builder)
 211     return
 212
 213   isolates = get_isolates()
 214   l = max(map(len, tests_location))
 215   print('%-*s%sLocal       %sSwarming  %sMissing isolate' %
 216       (l, 'Test', colorama.Fore.RED, colorama.Fore.GREEN,
 217         colorama.Fore.MAGENTA))
 218   total_local = 0
 219   total_swarming = 0
 220   for name, location in sorted(tests_location.iteritems()):
 221     if not location['count_run_on_swarming']:
 222       c = colorama.Fore.RED
 223     elif location['count_run_local']:
 224       c = colorama.Fore.YELLOW
 225     else:
 226       c = colorama.Fore.GREEN
 227     total_local += location['count_run_local']
 228     total_swarming += location['count_run_on_swarming']
 229     missing_isolate = ''
 230     if name + '.isolate' not in isolates:
 231       missing_isolate = colorama.Fore.MAGENTA + '*'
 232     print('%s%-*s %4d           %4d    %s' %
 233         (c, l, name, location['count_run_local'],
 234           location['count_run_on_swarming'], missing_isolate))
 235
 236   total = total_local + total_swarming
 237   p_local = 100. * total_local / total
 238   p_swarming = 100. * total_swarming / total
 239   print('%s%-*s %4d (%4.1f%%)   %4d (%4.1f%%)' %
 240       (colorama.Fore.WHITE, l, 'Total:', total_local, p_local,
 241         total_swarming, p_swarming))
 242   print('%-*s                %4d' % (l, 'Total executions:', total))
 243
 244
 245 def main():
 246   colorama.init()
 247   parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__)
 248   group = parser.add_mutually_exclusive_group(required=True)
 249   group.add_argument(
 250       '-c', '--check', dest='mode', action='store_const', const='check',
 251       default='check', help='Only check the files')
 252   group.add_argument(
 253       '--convert', dest='mode', action='store_const', const='convert',
 254       help='Convert a test to run on Swarming everywhere')
 255   group.add_argument(
 256       '--remaining', dest='mode', action='store_const', const='remaining',
 257       help='Count the number of tests not yet running on Swarming')
 258   group.add_argument(
 259       '-w', '--write', dest='mode', action='store_const', const='write',
 260       help='Rewrite the files')
 261   parser.add_argument(
 262       'test_name', nargs='?',
 263       help='The test name to print which configs to update; only to be used '
 264            'with --remaining')
 265   args = parser.parse_args()
 266
 267   if args.mode == 'convert':
 268     if not args.test_name:
 269       parser.error('A test name is required with --convert')
 270     if args.test_name + '.isolate' not in get_isolates():
 271       parser.error('Create %s.isolate first' % args.test_name)
 272
 273   # Stats when running in --remaining mode;
 274   tests_location = collections.defaultdict(
 275       lambda: {
 276         'count_run_local': 0, 'count_run_on_swarming': 0, 'local_configs': {}
 277       })
 278
 279   with open(os.path.join(THIS_DIR, "gn_isolate_map.pyl")) as fp:
 280     gn_isolate_map = ast.literal_eval(fp.read())
 281     ninja_targets = {k: v['label'] for k, v in gn_isolate_map.items()}
 282
 283   try:
 284     result = 0
 285     ninja_targets_seen = set()
 286     for filepath in glob.glob(os.path.join(THIS_DIR, '*.json')):
 287       if not process_file(args.mode, args.test_name, tests_location, filepath,
 288                           ninja_targets, ninja_targets_seen):
 289         result = 1
 290
 291     extra_targets = (set(ninja_targets) - ninja_targets_seen -
 292                      SKIP_GN_ISOLATE_MAP_TARGETS)
 293     if extra_targets:
 294       if len(extra_targets) > 1:
 295         extra_targets_str = ', '.join(extra_targets) + ' are'
 296       else:
 297         extra_targets_str = list(extra_targets)[0] + ' is'
 298       raise Error('%s listed in gn_isolate_map.pyl but not in any .json '
 299                   'files' % extra_targets_str)
 300
 301     if args.mode == 'convert':
 302       print_convert(args.test_name, tests_location)
 303     elif args.mode == 'remaining':
 304       print_remaining(args.test_name, tests_location)
 305     return result
 306   except Error as e:
 307     sys.stderr.write('%s\n' % e)
 308     return 1
 309
 310
 311 if __name__ == "__main__":
 312   sys.exit(main())