utils/update_test_checks.py

   1 #!/usr/bin/env python
   2
   3 """A script to generate FileCheck statements for 'opt' regression tests.
   4
   5 This script is a utility to update LLVM opt test cases with new
   6 FileCheck patterns. It can either update all of the tests in the file or
   7 a single test function.
   8
   9 Example usage:
  10 $ update_test_checks.py --opt=../bin/opt test/foo.ll
  11
  12 Workflow:
  13 1. Make a compiler patch that requires updating some number of FileCheck lines
  14    in regression test files.
  15 2. Save the patch and revert it from your local work area.
  16 3. Update the RUN-lines in the affected regression tests to look canonical.
  17    Example: "; RUN: opt < %s -instcombine -S | FileCheck %s"
  18 4. Refresh the FileCheck lines for either the entire file or select functions by
  19    running this script.
  20 5. Commit the fresh baseline of checks.
  21 6. Apply your patch from step 1 and rebuild your local binaries.
  22 7. Re-run this script on affected regression tests.
  23 8. Check the diffs to ensure the script has done something reasonable.
  24 9. Submit a patch including the regression test diffs for review.
  25
  26 A common pattern is to have the script insert complete checking of every
  27 instruction. Then, edit it down to only check the relevant instructions.
  28 The script is designed to make adding checks to a test case fast, it is *not*
  29 designed to be authoratitive about what constitutes a good test!
  30 """
  31
  32 from __future__ import print_function
  33
  34 import argparse
  35 import glob
  36 import itertools
  37 import os         # Used to advertise this file's name ("autogenerated_note").
  38 import string
  39 import subprocess
  40 import sys
  41 import tempfile
  42 import re
  43
  44 from UpdateTestChecks import common
  45
  46 ADVERT = '; NOTE: Assertions have been autogenerated by '
  47
  48 # RegEx: this is where the magic happens.
  49
  50 IR_FUNCTION_RE = re.compile('^\s*define\s+(?:internal\s+)?[^@]*@([\w-]+)\s*\(')
  51
  52
  53
  54
  55
  56 def main():
  57   from argparse import RawTextHelpFormatter
  58   parser = argparse.ArgumentParser(description=__doc__, formatter_class=RawTextHelpFormatter)
  59   parser.add_argument('-v', '--verbose', action='store_true',
  60                       help='Show verbose output')
  61   parser.add_argument('--opt-binary', default='opt',
  62                       help='The opt binary used to generate the test case')
  63   parser.add_argument(
  64       '--function', help='The function in the test file to update')
  65   parser.add_argument('tests', nargs='+')
  66   args = parser.parse_args()
  67
  68   autogenerated_note = (ADVERT + 'utils/' + os.path.basename(__file__))
  69
  70   opt_basename = os.path.basename(args.opt_binary)
  71   if not re.match(r'^opt(-\d+)?$', opt_basename):
  72     print('ERROR: Unexpected opt name: ' + opt_basename, file=sys.stderr)
  73     sys.exit(1)
  74   opt_basename = 'opt'
  75
  76   test_paths = [test for pattern in args.tests for test in glob.glob(pattern)]
  77   for test in test_paths:
  78     if args.verbose:
  79       print('Scanning for RUN lines in test file: %s' % (test,), file=sys.stderr)
  80     with open(test) as f:
  81       input_lines = [l.rstrip() for l in f]
  82
  83     raw_lines = [m.group(1)
  84                  for m in [common.RUN_LINE_RE.match(l) for l in input_lines] if m]
  85     run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
  86     for l in raw_lines[1:]:
  87       if run_lines[-1].endswith("\\"):
  88         run_lines[-1] = run_lines[-1].rstrip("\\") + " " + l
  89       else:
  90         run_lines.append(l)
  91
  92     if args.verbose:
  93       print('Found %d RUN lines:' % (len(run_lines),), file=sys.stderr)
  94       for l in run_lines:
  95         print('  RUN: ' + l, file=sys.stderr)
  96
  97     prefix_list = []
  98     for l in run_lines:
  99       (tool_cmd, filecheck_cmd) = tuple([cmd.strip() for cmd in l.split('|', 1)])
 100
 101       if not tool_cmd.startswith(opt_basename + ' '):
 102         print('WARNING: Skipping non-%s RUN line: %s' % (opt_basename, l), file=sys.stderr)
 103         continue
 104
 105       if not filecheck_cmd.startswith('FileCheck '):
 106         print('WARNING: Skipping non-FileChecked RUN line: ' + l, file=sys.stderr)
 107         continue
 108
 109       tool_cmd_args = tool_cmd[len(opt_basename):].strip()
 110       tool_cmd_args = tool_cmd_args.replace('< %s', '').replace('%s', '').strip()
 111
 112       check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
 113                                for item in m.group(1).split(',')]
 114       if not check_prefixes:
 115         check_prefixes = ['CHECK']
 116
 117       # FIXME: We should use multiple check prefixes to common check lines. For
 118       # now, we just ignore all but the last.
 119       prefix_list.append((check_prefixes, tool_cmd_args))
 120
 121     func_dict = {}
 122     for prefixes, _ in prefix_list:
 123       for prefix in prefixes:
 124         func_dict.update({prefix: dict()})
 125     for prefixes, opt_args in prefix_list:
 126       if args.verbose:
 127         print('Extracted opt cmd: ' + opt_basename + ' ' + opt_args, file=sys.stderr)
 128         print('Extracted FileCheck prefixes: ' + str(prefixes), file=sys.stderr)
 129
 130       raw_tool_output = common.invoke_tool(args.opt_binary, opt_args, test)
 131       common.build_function_body_dictionary(
 132               common.OPT_FUNCTION_RE, common.scrub_body, [],
 133               raw_tool_output, prefixes, func_dict, args.verbose)
 134
 135     is_in_function = False
 136     is_in_function_start = False
 137     prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
 138     if args.verbose:
 139       print('Rewriting FileCheck prefixes: %s' % (prefix_set,), file=sys.stderr)
 140     output_lines = []
 141     output_lines.append(autogenerated_note)
 142
 143     for input_line in input_lines:
 144       if is_in_function_start:
 145         if input_line == '':
 146           continue
 147         if input_line.lstrip().startswith(';'):
 148           m = common.CHECK_RE.match(input_line)
 149           if not m or m.group(1) not in prefix_set:
 150             output_lines.append(input_line)
 151             continue
 152
 153         # Print out the various check lines here.
 154         common.add_ir_checks(output_lines, ';', prefix_list, func_dict, func_name)
 155         is_in_function_start = False
 156
 157       if is_in_function:
 158         if common.should_add_line_to_output(input_line, prefix_set):
 159           # This input line of the function body will go as-is into the output.
 160           # Except make leading whitespace uniform: 2 spaces.
 161           input_line = common.SCRUB_LEADING_WHITESPACE_RE.sub(r'  ', input_line)
 162           output_lines.append(input_line)
 163         else:
 164           continue
 165         if input_line.strip() == '}':
 166           is_in_function = False
 167         continue
 168
 169       # Discard any previous script advertising.
 170       if input_line.startswith(ADVERT):
 171         continue
 172
 173       # If it's outside a function, it just gets copied to the output.
 174       output_lines.append(input_line)
 175
 176       m = IR_FUNCTION_RE.match(input_line)
 177       if not m:
 178         continue
 179       func_name = m.group(1)
 180       if args.function is not None and func_name != args.function:
 181         # When filtering on a specific function, skip all others.
 182         continue
 183       is_in_function = is_in_function_start = True
 184
 185     if args.verbose:
 186       print('Writing %d lines to %s...' % (len(output_lines), test), file=sys.stderr)
 187
 188     with open(test, 'wb') as f:
 189       f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
 190
 191
 192 if __name__ == '__main__':
 193   main()