testing/buildbot/manage.py

   1 #!/usr/bin/env python
   2 # Copyright 2015 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Toolbox to manage all the json files in this directory.
   7
   8 It can reformat them in their canonical format or ensures they are well
   9 formatted.
  10 """
  11
  12 import argparse
  13 import ast
  14 import collections
  15 import glob
  16 import json
  17 import os
  18 import subprocess
  19 import sys
  20
  21
  22 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
  23 SRC_DIR = os.path.dirname(os.path.dirname(THIS_DIR))
  24 BLINK_DIR = os.path.join(SRC_DIR, 'third_party', 'WebKit')
  25 sys.path.insert(0, os.path.join(SRC_DIR, 'third_party', 'colorama', 'src'))
  26
  27 import colorama
  28
  29
  30 SKIP = {
  31   # These are not 'builders'.
  32   'compile_targets', 'gtest_tests', 'filter_compile_builders',
  33   'non_filter_builders', 'non_filter_tests_builders',
  34
  35   # These are not supported on Swarming yet.
  36   # http://crbug.com/472205
  37   'Chromium Mac 10.10',
  38   # http://crbug.com/441429
  39   'Linux Trusty (32)', 'Linux Trusty (dbg)(32)',
  40
  41   # http://crbug.com/480053
  42   'Linux GN',
  43   'Linux GN (dbg)',
  44
  45   # Unmaintained builders on chromium.fyi
  46   'ClangToTMac',
  47   'ClangToTMacASan',
  48
  49   # This builder is fine, but win8_chromium_ng uses GN and this configuration,
  50   # which breaks everything.
  51   'Win8 Aura',
  52
  53   # One off builders. Note that Swarming does support ARM.
  54   'Linux ARM Cross-Compile',
  55   'Site Isolation Linux',
  56   'Site Isolation Win',
  57 }
  58
  59
  60 SKIP_GN_ISOLATE_MAP_TARGETS = {
  61   # TODO(GYP): These targets have not been ported to GN yet.
  62   'cast_media_unittests',
  63   'cast_shell_browser_test',
  64   'chromevox_tests',
  65   'nacl_helper_nonsfi_unittests',
  66
  67   # These targets are run on the bots but not listed in the
  68   # buildbot JSON files.
  69   'angle_end2end_tests',
  70   'content_gl_tests',
  71   'gl_tests',
  72   'gles2_conform_test',
  73   'tab_capture_end2end_tests',
  74   'telemetry_gpu_test',
  75 }
  76
  77
  78 class Error(Exception):
  79   """Processing error."""
  80
  81
  82 def get_isolates():
  83   """Returns the list of all isolate files."""
  84
  85   def git_ls_files(cwd):
  86     return subprocess.check_output(['git', 'ls-files'], cwd=cwd).splitlines()
  87
  88   files = git_ls_files(SRC_DIR) + git_ls_files(BLINK_DIR)
  89   return [os.path.basename(f) for f in files if f.endswith('.isolate')]
  90
  91
  92 def process_builder_convert(data, test_name):
  93   """Converts 'test_name' to run on Swarming in 'data'.
  94
  95   Returns True if 'test_name' was found.
  96   """
  97   result = False
  98   for test in data['gtest_tests']:
  99     if test['test'] != test_name:
 100       continue
 101     test.setdefault('swarming', {})
 102     if not test['swarming'].get('can_use_on_swarming_builders'):
 103       test['swarming']['can_use_on_swarming_builders'] = True
 104     result = True
 105   return result
 106
 107
 108 def process_builder_remaining(data, filename, builder, tests_location):
 109   """Calculates tests_location when mode is --remaining."""
 110   for test in data['gtest_tests']:
 111     name = test['test']
 112     if test.get('swarming', {}).get('can_use_on_swarming_builders'):
 113       tests_location[name]['count_run_on_swarming'] += 1
 114     else:
 115       tests_location[name]['count_run_local'] += 1
 116       tests_location[name]['local_configs'].setdefault(
 117           filename, []).append(builder)
 118
 119
 120 def process_file(mode, test_name, tests_location, filepath, ninja_targets,
 121                  ninja_targets_seen):
 122   """Processes a json file describing what tests should be run for each recipe.
 123
 124   The action depends on mode. Updates tests_location.
 125
 126   Return False if the process exit code should be 1.
 127   """
 128   filename = os.path.basename(filepath)
 129   with open(filepath) as f:
 130     content = f.read()
 131   try:
 132     config = json.loads(content)
 133   except ValueError as e:
 134     raise Error('Exception raised while checking %s: %s' % (filepath, e))
 135
 136   for builder, data in sorted(config.iteritems()):
 137     if builder in SKIP:
 138       # Oddities.
 139       continue
 140     if not isinstance(data, dict):
 141       raise Error('%s: %s is broken: %s' % (filename, builder, data))
 142     if 'gtest_tests' not in data:
 143       continue
 144     if not isinstance(data['gtest_tests'], list):
 145       raise Error(
 146           '%s: %s is broken: %s' % (filename, builder, data['gtest_tests']))
 147     if not all(isinstance(g, dict) for g in data['gtest_tests']):
 148       raise Error(
 149           '%s: %s is broken: %s' % (filename, builder, data['gtest_tests']))
 150
 151     for d in data['gtest_tests']:
 152       if (d['test'] not in ninja_targets and
 153           d['test'] not in SKIP_GN_ISOLATE_MAP_TARGETS):
 154         raise Error('%s: %s / %s is not listed in gn_isolate_map.pyl.' %
 155                     (filename, builder, d['test']))
 156       elif d['test'] in ninja_targets:
 157         ninja_targets_seen.add(d['test'])
 158
 159     config[builder]['gtest_tests'] = sorted(
 160         data['gtest_tests'], key=lambda x: x['test'])
 161
 162     # The trick here is that process_builder_remaining() is called before
 163     # process_builder_convert() so tests_location can be used to know how many
 164     # tests were converted.
 165     if mode in ('convert', 'remaining'):
 166       process_builder_remaining(data, filename, builder, tests_location)
 167     if mode == 'convert':
 168       process_builder_convert(data, test_name)
 169
 170   expected = json.dumps(
 171       config, sort_keys=True, indent=2, separators=(',', ': ')) + '\n'
 172   if content != expected:
 173     if mode in ('convert', 'write'):
 174       with open(filepath, 'wb') as f:
 175         f.write(expected)
 176       if mode == 'write':
 177         print('Updated %s' % filename)
 178     else:
 179       print('%s is not in canonical format' % filename)
 180       print('run `testing/buildbot/manage.py -w` to fix')
 181     return mode != 'check'
 182   return True
 183
 184
 185 def print_convert(test_name, tests_location):
 186   """Prints statistics for a test being converted for use in a CL description.
 187   """
 188   data = tests_location[test_name]
 189   print('Convert %s to run exclusively on Swarming' % test_name)
 190   print('')
 191   print('%d configs already ran on Swarming' % data['count_run_on_swarming'])
 192   print('%d used to run locally and were converted:' % data['count_run_local'])
 193   for master, builders in sorted(data['local_configs'].iteritems()):
 194     for builder in builders:
 195       print('- %s: %s' % (master, builder))
 196   print('')
 197   print('Ran:')
 198   print('  ./manage.py --convert %s' % test_name)
 199   print('')
 200   print('R=')
 201   print('BUG=98637')
 202
 203
 204 def print_remaining(test_name, tests_location):
 205   """Prints a visual summary of what tests are yet to be converted to run on
 206   Swarming.
 207   """
 208   if test_name:
 209     if test_name not in tests_location:
 210       raise Error('Unknown test %s' % test_name)
 211     for config, builders in sorted(
 212         tests_location[test_name]['local_configs'].iteritems()):
 213       print('%s:' % config)
 214       for builder in sorted(builders):
 215         print('  %s' % builder)
 216     return
 217
 218   isolates = get_isolates()
 219   l = max(map(len, tests_location))
 220   print('%-*s%sLocal       %sSwarming  %sMissing isolate' %
 221       (l, 'Test', colorama.Fore.RED, colorama.Fore.GREEN,
 222         colorama.Fore.MAGENTA))
 223   total_local = 0
 224   total_swarming = 0
 225   for name, location in sorted(tests_location.iteritems()):
 226     if not location['count_run_on_swarming']:
 227       c = colorama.Fore.RED
 228     elif location['count_run_local']:
 229       c = colorama.Fore.YELLOW
 230     else:
 231       c = colorama.Fore.GREEN
 232     total_local += location['count_run_local']
 233     total_swarming += location['count_run_on_swarming']
 234     missing_isolate = ''
 235     if name + '.isolate' not in isolates:
 236       missing_isolate = colorama.Fore.MAGENTA + '*'
 237     print('%s%-*s %4d           %4d    %s' %
 238         (c, l, name, location['count_run_local'],
 239           location['count_run_on_swarming'], missing_isolate))
 240
 241   total = total_local + total_swarming
 242   p_local = 100. * total_local / total
 243   p_swarming = 100. * total_swarming / total
 244   print('%s%-*s %4d (%4.1f%%)   %4d (%4.1f%%)' %
 245       (colorama.Fore.WHITE, l, 'Total:', total_local, p_local,
 246         total_swarming, p_swarming))
 247   print('%-*s                %4d' % (l, 'Total executions:', total))
 248
 249
 250 def main():
 251   colorama.init()
 252   parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__)
 253   group = parser.add_mutually_exclusive_group(required=True)
 254   group.add_argument(
 255       '-c', '--check', dest='mode', action='store_const', const='check',
 256       default='check', help='Only check the files')
 257   group.add_argument(
 258       '--convert', dest='mode', action='store_const', const='convert',
 259       help='Convert a test to run on Swarming everywhere')
 260   group.add_argument(
 261       '--remaining', dest='mode', action='store_const', const='remaining',
 262       help='Count the number of tests not yet running on Swarming')
 263   group.add_argument(
 264       '-w', '--write', dest='mode', action='store_const', const='write',
 265       help='Rewrite the files')
 266   parser.add_argument(
 267       'test_name', nargs='?',
 268       help='The test name to print which configs to update; only to be used '
 269            'with --remaining')
 270   args = parser.parse_args()
 271
 272   if args.mode == 'convert':
 273     if not args.test_name:
 274       parser.error('A test name is required with --convert')
 275     if args.test_name + '.isolate' not in get_isolates():
 276       parser.error('Create %s.isolate first' % args.test_name)
 277
 278   # Stats when running in --remaining mode;
 279   tests_location = collections.defaultdict(
 280       lambda: {
 281         'count_run_local': 0, 'count_run_on_swarming': 0, 'local_configs': {}
 282       })
 283
 284   with open(os.path.join(THIS_DIR, "gn_isolate_map.pyl")) as fp:
 285     gn_isolate_map = ast.literal_eval(fp.read())
 286     ninja_targets = {k: v['label'] for k, v in gn_isolate_map.items()}
 287
 288   try:
 289     result = 0
 290     ninja_targets_seen = set()
 291     for filepath in glob.glob(os.path.join(THIS_DIR, '*.json')):
 292       if not process_file(args.mode, args.test_name, tests_location, filepath,
 293                           ninja_targets, ninja_targets_seen):
 294         result = 1
 295
 296     extra_targets = (set(ninja_targets) - ninja_targets_seen -
 297                      SKIP_GN_ISOLATE_MAP_TARGETS)
 298     if extra_targets:
 299       if len(extra_targets) > 1:
 300         extra_targets_str = ', '.join(extra_targets) + ' are'
 301       else:
 302         extra_targets_str = list(extra_targets)[0] + ' is'
 303       raise Error('%s listed in gn_isolate_map.pyl but not in any .json '
 304                   'files' % extra_targets_str)
 305
 306     if args.mode == 'convert':
 307       print_convert(args.test_name, tests_location)
 308     elif args.mode == 'remaining':
 309       print_remaining(args.test_name, tests_location)
 310     return result
 311   except Error as e:
 312     sys.stderr.write('%s\n' % e)
 313     return 1
 314
 315
 316 if __name__ == "__main__":
 317   sys.exit(main())