utils/update_test_checks.py

   1 #!/usr/bin/env python
   2
   3 """A script to generate FileCheck statements for 'opt' regression tests.
   4
   5 This script is a utility to update LLVM opt test cases with new
   6 FileCheck patterns. It can either update all of the tests in the file or
   7 a single test function.
   8
   9 Example usage:
  10 $ update_test_checks.py --opt=../bin/opt test/foo.ll
  11
  12 Workflow:
  13 1. Make a compiler patch that requires updating some number of FileCheck lines
  14    in regression test files.
  15 2. Save the patch and revert it from your local work area.
  16 3. Update the RUN-lines in the affected regression tests to look canonical.
  17    Example: "; RUN: opt < %s -instcombine -S | FileCheck %s"
  18 4. Refresh the FileCheck lines for either the entire file or select functions by
  19    running this script.
  20 5. Commit the fresh baseline of checks.
  21 6. Apply your patch from step 1 and rebuild your local binaries.
  22 7. Re-run this script on affected regression tests.
  23 8. Check the diffs to ensure the script has done something reasonable.
  24 9. Submit a patch including the regression test diffs for review.
  25
  26 A common pattern is to have the script insert complete checking of every
  27 instruction. Then, edit it down to only check the relevant instructions.
  28 The script is designed to make adding checks to a test case fast, it is *not*
  29 designed to be authoratitive about what constitutes a good test!
  30 """
  31
  32 from __future__ import print_function
  33
  34 import argparse
  35 import glob
  36 import itertools
  37 import os         # Used to advertise this file's name ("autogenerated_note").
  38 import string
  39 import subprocess
  40 import sys
  41 import tempfile
  42 import re
  43
  44 from UpdateTestChecks import common
  45
  46 ADVERT = '; NOTE: Assertions have been autogenerated by '
  47
  48 # RegEx: this is where the magic happens.
  49
  50 IR_FUNCTION_RE = re.compile('^\s*define\s+(?:internal\s+)?[^@]*@([\w-]+)\s*\(')
  51
  52
  53
  54
  55
  56 def main():
  57   from argparse import RawTextHelpFormatter
  58   parser = argparse.ArgumentParser(description=__doc__, formatter_class=RawTextHelpFormatter)
  59   parser.add_argument('-v', '--verbose', action='store_true',
  60                       help='Show verbose output')
  61   parser.add_argument('--opt-binary', default='opt',
  62                       help='The opt binary used to generate the test case')
  63   parser.add_argument(
  64       '--function', help='The function in the test file to update')
  65   parser.add_argument('tests', nargs='+')
  66   args = parser.parse_args()
  67
  68   autogenerated_note = (ADVERT + 'utils/' + os.path.basename(__file__))
  69
  70   opt_basename = os.path.basename(args.opt_binary)
  71   if (opt_basename != "opt"):
  72     print('ERROR: Unexpected opt name: ' + opt_basename, file=sys.stderr)
  73     sys.exit(1)
  74
  75   test_paths = [test for pattern in args.tests for test in glob.glob(pattern)]
  76   for test in test_paths:
  77     if args.verbose:
  78       print('Scanning for RUN lines in test file: %s' % (test,), file=sys.stderr)
  79     with open(test) as f:
  80       input_lines = [l.rstrip() for l in f]
  81
  82     raw_lines = [m.group(1)
  83                  for m in [common.RUN_LINE_RE.match(l) for l in input_lines] if m]
  84     run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
  85     for l in raw_lines[1:]:
  86       if run_lines[-1].endswith("\\"):
  87         run_lines[-1] = run_lines[-1].rstrip("\\") + " " + l
  88       else:
  89         run_lines.append(l)
  90
  91     if args.verbose:
  92       print('Found %d RUN lines:' % (len(run_lines),), file=sys.stderr)
  93       for l in run_lines:
  94         print('  RUN: ' + l, file=sys.stderr)
  95
  96     prefix_list = []
  97     for l in run_lines:
  98       (tool_cmd, filecheck_cmd) = tuple([cmd.strip() for cmd in l.split('|', 1)])
  99
 100       if not tool_cmd.startswith(opt_basename + ' '):
 101         print('WARNING: Skipping non-%s RUN line: %s' % (opt_basename, l), file=sys.stderr)
 102         continue
 103
 104       if not filecheck_cmd.startswith('FileCheck '):
 105         print('WARNING: Skipping non-FileChecked RUN line: ' + l, file=sys.stderr)
 106         continue
 107
 108       tool_cmd_args = tool_cmd[len(opt_basename):].strip()
 109       tool_cmd_args = tool_cmd_args.replace('< %s', '').replace('%s', '').strip()
 110
 111       check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
 112                                for item in m.group(1).split(',')]
 113       if not check_prefixes:
 114         check_prefixes = ['CHECK']
 115
 116       # FIXME: We should use multiple check prefixes to common check lines. For
 117       # now, we just ignore all but the last.
 118       prefix_list.append((check_prefixes, tool_cmd_args))
 119
 120     func_dict = {}
 121     for prefixes, _ in prefix_list:
 122       for prefix in prefixes:
 123         func_dict.update({prefix: dict()})
 124     for prefixes, opt_args in prefix_list:
 125       if args.verbose:
 126         print('Extracted opt cmd: ' + opt_basename + ' ' + opt_args, file=sys.stderr)
 127         print('Extracted FileCheck prefixes: ' + str(prefixes), file=sys.stderr)
 128
 129       raw_tool_output = common.invoke_tool(args.opt_binary, opt_args, test)
 130       common.build_function_body_dictionary(
 131               common.OPT_FUNCTION_RE, common.scrub_body, [],
 132               raw_tool_output, prefixes, func_dict, args.verbose)
 133
 134     is_in_function = False
 135     is_in_function_start = False
 136     prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
 137     if args.verbose:
 138       print('Rewriting FileCheck prefixes: %s' % (prefix_set,), file=sys.stderr)
 139     output_lines = []
 140     output_lines.append(autogenerated_note)
 141
 142     for input_line in input_lines:
 143       if is_in_function_start:
 144         if input_line == '':
 145           continue
 146         if input_line.lstrip().startswith(';'):
 147           m = common.CHECK_RE.match(input_line)
 148           if not m or m.group(1) not in prefix_set:
 149             output_lines.append(input_line)
 150             continue
 151
 152         # Print out the various check lines here.
 153         common.add_ir_checks(output_lines, ';', prefix_list, func_dict, func_name)
 154         is_in_function_start = False
 155
 156       if is_in_function:
 157         if common.should_add_line_to_output(input_line, prefix_set):
 158           # This input line of the function body will go as-is into the output.
 159           # Except make leading whitespace uniform: 2 spaces.
 160           input_line = common.SCRUB_LEADING_WHITESPACE_RE.sub(r'  ', input_line)
 161           output_lines.append(input_line)
 162         else:
 163           continue
 164         if input_line.strip() == '}':
 165           is_in_function = False
 166         continue
 167
 168       # Discard any previous script advertising.
 169       if input_line.startswith(ADVERT):
 170         continue
 171
 172       # If it's outside a function, it just gets copied to the output.
 173       output_lines.append(input_line)
 174
 175       m = IR_FUNCTION_RE.match(input_line)
 176       if not m:
 177         continue
 178       func_name = m.group(1)
 179       if args.function is not None and func_name != args.function:
 180         # When filtering on a specific function, skip all others.
 181         continue
 182       is_in_function = is_in_function_start = True
 183
 184     if args.verbose:
 185       print('Writing %d lines to %s...' % (len(output_lines), test), file=sys.stderr)
 186
 187     with open(test, 'wb') as f:
 188       f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
 189
 190
 191 if __name__ == '__main__':
 192   main()