utils/update_test_checks.py

   1 #!/usr/bin/env python2.7
   2
   3 """A script to generate FileCheck statements for 'opt' regression tests.
   4
   5 This script is a utility to update LLVM opt test cases with new
   6 FileCheck patterns. It can either update all of the tests in the file or
   7 a single test function.
   8
   9 Example usage:
  10 $ update_test_checks.py --opt=../bin/opt test/foo.ll
  11
  12 Workflow:
  13 1. Make a compiler patch that requires updating some number of FileCheck lines
  14    in regression test files.
  15 2. Save the patch and revert it from your local work area.
  16 3. Update the RUN-lines in the affected regression tests to look canonical.
  17    Example: "; RUN: opt < %s -instcombine -S | FileCheck %s"
  18 4. Refresh the FileCheck lines for either the entire file or select functions by
  19    running this script.
  20 5. Commit the fresh baseline of checks.
  21 6. Apply your patch from step 1 and rebuild your local binaries.
  22 7. Re-run this script on affected regression tests.
  23 8. Check the diffs to ensure the script has done something reasonable.
  24 9. Submit a patch including the regression test diffs for review.
  25
  26 A common pattern is to have the script insert complete checking of every
  27 instruction. Then, edit it down to only check the relevant instructions.
  28 The script is designed to make adding checks to a test case fast, it is *not*
  29 designed to be authoratitive about what constitutes a good test!
  30 """
  31
  32 import argparse
  33 import itertools
  34 import os         # Used to advertise this file's name ("autogenerated_note").
  35 import string
  36 import subprocess
  37 import sys
  38 import tempfile
  39 import re
  40
  41 from UpdateTestChecks import common
  42
  43 ADVERT = '; NOTE: Assertions have been autogenerated by '
  44
  45 # RegEx: this is where the magic happens.
  46
  47 IR_FUNCTION_RE = re.compile('^\s*define\s+(?:internal\s+)?[^@]*@([\w-]+)\s*\(')
  48
  49
  50
  51
  52
  53 def main():
  54   from argparse import RawTextHelpFormatter
  55   parser = argparse.ArgumentParser(description=__doc__, formatter_class=RawTextHelpFormatter)
  56   parser.add_argument('-v', '--verbose', action='store_true',
  57                       help='Show verbose output')
  58   parser.add_argument('--opt-binary', default='opt',
  59                       help='The opt binary used to generate the test case')
  60   parser.add_argument(
  61       '--function', help='The function in the test file to update')
  62   parser.add_argument('tests', nargs='+')
  63   args = parser.parse_args()
  64
  65   autogenerated_note = (ADVERT + 'utils/' + os.path.basename(__file__))
  66
  67   opt_basename = os.path.basename(args.opt_binary)
  68   if (opt_basename != "opt"):
  69     print >>sys.stderr, 'ERROR: Unexpected opt name: ' + opt_basename
  70     sys.exit(1)
  71
  72   for test in args.tests:
  73     if args.verbose:
  74       print >>sys.stderr, 'Scanning for RUN lines in test file: %s' % (test,)
  75     with open(test) as f:
  76       input_lines = [l.rstrip() for l in f]
  77
  78     raw_lines = [m.group(1)
  79                  for m in [common.RUN_LINE_RE.match(l) for l in input_lines] if m]
  80     run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
  81     for l in raw_lines[1:]:
  82       if run_lines[-1].endswith("\\"):
  83         run_lines[-1] = run_lines[-1].rstrip("\\") + " " + l
  84       else:
  85         run_lines.append(l)
  86
  87     if args.verbose:
  88       print >>sys.stderr, 'Found %d RUN lines:' % (len(run_lines),)
  89       for l in run_lines:
  90         print >>sys.stderr, '  RUN: ' + l
  91
  92     prefix_list = []
  93     for l in run_lines:
  94       (tool_cmd, filecheck_cmd) = tuple([cmd.strip() for cmd in l.split('|', 1)])
  95
  96       if not tool_cmd.startswith(opt_basename + ' '):
  97         print >>sys.stderr, 'WARNING: Skipping non-%s RUN line: %s' % (opt_basename, l)
  98         continue
  99
 100       if not filecheck_cmd.startswith('FileCheck '):
 101         print >>sys.stderr, 'WARNING: Skipping non-FileChecked RUN line: ' + l
 102         continue
 103
 104       tool_cmd_args = tool_cmd[len(opt_basename):].strip()
 105       tool_cmd_args = tool_cmd_args.replace('< %s', '').replace('%s', '').strip()
 106
 107       check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
 108                                for item in m.group(1).split(',')]
 109       if not check_prefixes:
 110         check_prefixes = ['CHECK']
 111
 112       # FIXME: We should use multiple check prefixes to common check lines. For
 113       # now, we just ignore all but the last.
 114       prefix_list.append((check_prefixes, tool_cmd_args))
 115
 116     func_dict = {}
 117     for prefixes, _ in prefix_list:
 118       for prefix in prefixes:
 119         func_dict.update({prefix: dict()})
 120     for prefixes, opt_args in prefix_list:
 121       if args.verbose:
 122         print >>sys.stderr, 'Extracted opt cmd: ' + opt_basename + ' ' + opt_args
 123         print >>sys.stderr, 'Extracted FileCheck prefixes: ' + str(prefixes)
 124
 125       raw_tool_output = common.invoke_tool(args.opt_binary, opt_args, test)
 126       common.build_function_body_dictionary(
 127               common.OPT_FUNCTION_RE, common.scrub_body, [],
 128               raw_tool_output, prefixes, func_dict, args.verbose)
 129
 130     is_in_function = False
 131     is_in_function_start = False
 132     prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
 133     if args.verbose:
 134       print >>sys.stderr, 'Rewriting FileCheck prefixes: %s' % (prefix_set,)
 135     output_lines = []
 136     output_lines.append(autogenerated_note)
 137
 138     for input_line in input_lines:
 139       if is_in_function_start:
 140         if input_line == '':
 141           continue
 142         if input_line.lstrip().startswith(';'):
 143           m = common.CHECK_RE.match(input_line)
 144           if not m or m.group(1) not in prefix_set:
 145             output_lines.append(input_line)
 146             continue
 147
 148         # Print out the various check lines here.
 149         common.add_ir_checks(output_lines, ';', prefix_list, func_dict, func_name)
 150         is_in_function_start = False
 151
 152       if is_in_function:
 153         if common.should_add_line_to_output(input_line, prefix_set):
 154           # This input line of the function body will go as-is into the output.
 155           # Except make leading whitespace uniform: 2 spaces.
 156           input_line = common.SCRUB_LEADING_WHITESPACE_RE.sub(r'  ', input_line)
 157           output_lines.append(input_line)
 158         else:
 159           continue
 160         if input_line.strip() == '}':
 161           is_in_function = False
 162         continue
 163
 164       # Discard any previous script advertising.
 165       if input_line.startswith(ADVERT):
 166         continue
 167
 168       # If it's outside a function, it just gets copied to the output.
 169       output_lines.append(input_line)
 170
 171       m = IR_FUNCTION_RE.match(input_line)
 172       if not m:
 173         continue
 174       func_name = m.group(1)
 175       if args.function is not None and func_name != args.function:
 176         # When filtering on a specific function, skip all others.
 177         continue
 178       is_in_function = is_in_function_start = True
 179
 180     if args.verbose:
 181       print>>sys.stderr, 'Writing %d lines to %s...' % (len(output_lines), test)
 182
 183     with open(test, 'wb') as f:
 184       f.writelines([l + '\n' for l in output_lines])
 185
 186
 187 if __name__ == '__main__':
 188   main()