tools/flakiness/find_flakiness.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2011 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Contains two functions that run different test cases and the same test
   7 case in parallel repeatedly to identify flaky tests.
   8 """
   9
  10
  11 import os
  12 import re
  13 import subprocess
  14 import time
  15
  16
  17 # Defaults for FindShardingFlakiness().
  18 FF_DATA_SUFFIX = '_flakies'
  19 FF_SLEEP_INTERVAL = 10.0
  20 FF_NUM_ITERATIONS = 100
  21 FF_SUPERVISOR_ARGS = ['-r3', '--random-seed']
  22
  23 # Defaults for FindUnaryFlakiness().
  24 FF_OUTPUT_SUFFIX = '_purges'
  25 FF_NUM_PROCS = 20
  26 FF_NUM_REPEATS = 10
  27 FF_TIMEOUT = 600
  28
  29
  30 def FindShardingFlakiness(test_path, data_path, supervisor_args):
  31   """Finds flaky test cases by sharding and running a test for the specified
  32   number of times. The data file is read at the beginning of each run to find
  33   the last known counts and is overwritten at the end of each run with the new
  34   counts. There is an optional sleep interval between each run so the script can
  35   be killed without losing the data, useful for overnight (or weekend!) runs.
  36   """
  37
  38   failed_tests = {}
  39   # Read a previously written data file.
  40   if os.path.exists(data_path):
  41     data_file = open(data_path, 'r')
  42     num_runs = int(data_file.readline().split(' ')[0])
  43     num_passes = int(data_file.readline().split(' ')[0])
  44     for line in data_file:
  45       if line:
  46         split_line = line.split(' -> ')
  47         failed_tests[split_line[0]] = int(split_line[1])
  48     data_file.close()
  49   # No data file found.
  50   else:
  51     num_runs = 0
  52     num_passes = 0
  53
  54   log_lines = False
  55   args = ['python', '../sharding_supervisor/sharding_supervisor.py']
  56   args.extend(supervisor_args + [test_path])
  57   proc = subprocess.Popen(args, stderr=subprocess.PIPE)
  58
  59   # Shard the test and collect failures.
  60   while True:
  61     line = proc.stderr.readline()
  62     if not line:
  63       if proc.poll() is not None:
  64         break
  65       continue
  66     print line.rstrip()
  67     if log_lines:
  68       line = line.rstrip()
  69       if line in failed_tests:
  70         failed_tests[line] += 1
  71       else:
  72         failed_tests[line] = 1
  73     elif line.find('FAILED TESTS:') >= 0:
  74       log_lines = True
  75   num_runs += 1
  76   if proc.returncode == 0:
  77     num_passes += 1
  78
  79   # Write the data file and print results.
  80   data_file = open(data_path, 'w')
  81   print '%i runs' % num_runs
  82   data_file.write('%i runs\n' % num_runs)
  83   print '%i passes' % num_passes
  84   data_file.write('%i passes\n' % num_passes)
  85   for (test, count) in failed_tests.iteritems():
  86     print '%s -> %i' % (test, count)
  87     data_file.write('%s -> %i\n' % (test, count))
  88   data_file.close()
  89
  90
  91 def FindUnaryFlakiness(test_path, output_path, num_procs, num_repeats, timeout):
  92   """Runs all the test cases in a given test in parallel with itself, to get at
  93   those that hold on to shared resources. The idea is that if a test uses a
  94   unary resource, then running many instances of this test will purge out some
  95   of them as failures or timeouts.
  96   """
  97
  98   test_name_regex = r'((\w+/)?\w+\.\w+(/\d+)?)'
  99   test_start = re.compile('\[\s+RUN\s+\] ' + test_name_regex)
 100   test_list = []
 101
 102   # Run the test to discover all the test cases.
 103   proc = subprocess.Popen([test_path], stdout=subprocess.PIPE)
 104   while True:
 105     line = proc.stdout.readline()
 106     if not line:
 107       if proc.poll() is not None:
 108         break
 109       continue
 110     print line.rstrip()
 111     results = test_start.search(line)
 112     if results:
 113       test_list.append(results.group(1))
 114
 115   failures = []
 116   index = 0
 117   total = len(test_list)
 118
 119   # Run each test case in parallel with itself.
 120   for test_name in test_list:
 121     num_fails = 0
 122     num_terminated = 0
 123     procs = []
 124     args = [test_path, '--gtest_filter=' + test_name,
 125             '--gtest_repeat=%i' % num_repeats]
 126     while len(procs) < num_procs:
 127       procs.append(subprocess.Popen(args))
 128     seconds = 0
 129     while procs:
 130       for proc in procs:
 131         if proc.poll() is not None:
 132           if proc.returncode != 0:
 133             ++num_fails
 134           procs.remove(proc)
 135       # Timeout exceeded, kill the remaining processes and make a note.
 136       if seconds > timeout:
 137         num_fails += len(procs)
 138         num_terminated = len(procs)
 139         while procs:
 140           procs.pop().terminate()
 141       time.sleep(1.0)
 142       seconds += 1
 143     if num_fails:
 144       line = '%s: %i failed' % (test_name, num_fails)
 145       if num_terminated:
 146         line += ' (%i terminated)' % num_terminated
 147       failures.append(line)
 148     print '%s (%i / %i): %i failed' % (test_name, index, total, num_fails)
 149     index += 1
 150     time.sleep(1.0)
 151
 152   # Print the results and write the data file.
 153   print failures
 154   data_file = open(output_path, 'w')
 155   for line in failures:
 156     data_file.write(line + '\n')
 157   data_file.close()
 158
 159
 160 def main():
 161   if not args:
 162     parser.error('You must specify a path to test!')
 163   if not os.path.exists(args[0]):
 164     parser.error('%s does not exist!' % args[0])
 165
 166   data_path = os.path.basename(args[0]) + FF_DATA_SUFFIX
 167   output_path = os.path.basename(args[0]) + FF_OUTPUT_SUFFIX
 168
 169   for i in range(FF_NUM_ITERATIONS):
 170     FindShardingFlakiness(args[0], data_path, FF_SUPERVISOR_ARGS)
 171     print 'That was just iteration %i of %i.' % (i + 1, FF_NUM_ITERATIONS)
 172     time.sleep(FF_SLEEP_INTERVAL)
 173
 174   FindUnaryFlakiness(
 175       args[0], output_path, FF_NUM_PROCS, FF_NUM_REPEATS, FF_TIMEOUT)
 176
 177
 178 if __name__ == '__main__':
 179   main()