utils/update_test_checks.py

   1 #!/usr/bin/env python
   2
   3 """A script to generate FileCheck statements for 'opt' regression tests.
   4
   5 This script is a utility to update LLVM opt test cases with new
   6 FileCheck patterns. It can either update all of the tests in the file or
   7 a single test function.
   8
   9 Example usage:
  10 $ update_test_checks.py --opt=../bin/opt test/foo.ll
  11
  12 Workflow:
  13 1. Make a compiler patch that requires updating some number of FileCheck lines
  14    in regression test files.
  15 2. Save the patch and revert it from your local work area.
  16 3. Update the RUN-lines in the affected regression tests to look canonical.
  17    Example: "; RUN: opt < %s -instcombine -S | FileCheck %s"
  18 4. Refresh the FileCheck lines for either the entire file or select functions by
  19    running this script.
  20 5. Commit the fresh baseline of checks.
  21 6. Apply your patch from step 1 and rebuild your local binaries.
  22 7. Re-run this script on affected regression tests.
  23 8. Check the diffs to ensure the script has done something reasonable.
  24 9. Submit a patch including the regression test diffs for review.
  25
  26 A common pattern is to have the script insert complete checking of every
  27 instruction. Then, edit it down to only check the relevant instructions.
  28 The script is designed to make adding checks to a test case fast, it is *not*
  29 designed to be authoratitive about what constitutes a good test!
  30 """
  31
  32 from __future__ import print_function
  33
  34 import argparse
  35 import glob
  36 import itertools
  37 import os         # Used to advertise this file's name ("autogenerated_note").
  38 import string
  39 import subprocess
  40 import sys
  41 import tempfile
  42 import re
  43
  44 from UpdateTestChecks import common
  45
  46 ADVERT = '; NOTE: Assertions have been autogenerated by '
  47
  48 # RegEx: this is where the magic happens.
  49
  50 IR_FUNCTION_RE = re.compile('^\s*define\s+(?:internal\s+)?[^@]*@([\w-]+)\s*\(')
  51
  52
  53
  54
  55
  56 def main():
  57   from argparse import RawTextHelpFormatter
  58   parser = argparse.ArgumentParser(description=__doc__, formatter_class=RawTextHelpFormatter)
  59   parser.add_argument('-v', '--verbose', action='store_true',
  60                       help='Show verbose output')
  61   parser.add_argument('--opt-binary', default='opt',
  62                       help='The opt binary used to generate the test case')
  63   parser.add_argument(
  64       '--function', help='The function in the test file to update')
  65   parser.add_argument('tests', nargs='+')
  66   args = parser.parse_args()
  67
  68   autogenerated_note = (ADVERT + 'utils/' + os.path.basename(__file__))
  69
  70   opt_basename = os.path.basename(args.opt_binary)
  71   if not re.match(r'^opt(-\d+)?$', opt_basename):
  72     print('ERROR: Unexpected opt name: ' + opt_basename, file=sys.stderr)
  73     sys.exit(1)
  74   opt_basename = 'opt'
  75
  76   test_paths = []
  77   for test in args.tests:
  78     if not glob.glob(test):
  79       print('WARNING: Test file \'%s\' was not found. Ignoring it.' % (test,), file=sys.stderr)
  80       continue
  81     test_paths.append(test)
  82
  83   for test in test_paths:
  84     if args.verbose:
  85       print('Scanning for RUN lines in test file: %s' % (test,), file=sys.stderr)
  86     with open(test) as f:
  87       input_lines = [l.rstrip() for l in f]
  88
  89     raw_lines = [m.group(1)
  90                  for m in [common.RUN_LINE_RE.match(l) for l in input_lines] if m]
  91     run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
  92     for l in raw_lines[1:]:
  93       if run_lines[-1].endswith("\\"):
  94         run_lines[-1] = run_lines[-1].rstrip("\\") + " " + l
  95       else:
  96         run_lines.append(l)
  97
  98     if args.verbose:
  99       print('Found %d RUN lines:' % (len(run_lines),), file=sys.stderr)
 100       for l in run_lines:
 101         print('  RUN: ' + l, file=sys.stderr)
 102
 103     prefix_list = []
 104     for l in run_lines:
 105       (tool_cmd, filecheck_cmd) = tuple([cmd.strip() for cmd in l.split('|', 1)])
 106
 107       if not tool_cmd.startswith(opt_basename + ' '):
 108         print('WARNING: Skipping non-%s RUN line: %s' % (opt_basename, l), file=sys.stderr)
 109         continue
 110
 111       if not filecheck_cmd.startswith('FileCheck '):
 112         print('WARNING: Skipping non-FileChecked RUN line: ' + l, file=sys.stderr)
 113         continue
 114
 115       tool_cmd_args = tool_cmd[len(opt_basename):].strip()
 116       tool_cmd_args = tool_cmd_args.replace('< %s', '').replace('%s', '').strip()
 117
 118       check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
 119                                for item in m.group(1).split(',')]
 120       if not check_prefixes:
 121         check_prefixes = ['CHECK']
 122
 123       # FIXME: We should use multiple check prefixes to common check lines. For
 124       # now, we just ignore all but the last.
 125       prefix_list.append((check_prefixes, tool_cmd_args))
 126
 127     func_dict = {}
 128     for prefixes, _ in prefix_list:
 129       for prefix in prefixes:
 130         func_dict.update({prefix: dict()})
 131     for prefixes, opt_args in prefix_list:
 132       if args.verbose:
 133         print('Extracted opt cmd: ' + opt_basename + ' ' + opt_args, file=sys.stderr)
 134         print('Extracted FileCheck prefixes: ' + str(prefixes), file=sys.stderr)
 135
 136       raw_tool_output = common.invoke_tool(args.opt_binary, opt_args, test)
 137       common.build_function_body_dictionary(
 138               common.OPT_FUNCTION_RE, common.scrub_body, [],
 139               raw_tool_output, prefixes, func_dict, args.verbose)
 140
 141     is_in_function = False
 142     is_in_function_start = False
 143     prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
 144     if args.verbose:
 145       print('Rewriting FileCheck prefixes: %s' % (prefix_set,), file=sys.stderr)
 146     output_lines = []
 147     output_lines.append(autogenerated_note)
 148
 149     for input_line in input_lines:
 150       if is_in_function_start:
 151         if input_line == '':
 152           continue
 153         if input_line.lstrip().startswith(';'):
 154           m = common.CHECK_RE.match(input_line)
 155           if not m or m.group(1) not in prefix_set:
 156             output_lines.append(input_line)
 157             continue
 158
 159         # Print out the various check lines here.
 160         common.add_ir_checks(output_lines, ';', prefix_list, func_dict, func_name)
 161         is_in_function_start = False
 162
 163       if is_in_function:
 164         if common.should_add_line_to_output(input_line, prefix_set):
 165           # This input line of the function body will go as-is into the output.
 166           # Except make leading whitespace uniform: 2 spaces.
 167           input_line = common.SCRUB_LEADING_WHITESPACE_RE.sub(r'  ', input_line)
 168           output_lines.append(input_line)
 169         else:
 170           continue
 171         if input_line.strip() == '}':
 172           is_in_function = False
 173         continue
 174
 175       # Discard any previous script advertising.
 176       if input_line.startswith(ADVERT):
 177         continue
 178
 179       # If it's outside a function, it just gets copied to the output.
 180       output_lines.append(input_line)
 181
 182       m = IR_FUNCTION_RE.match(input_line)
 183       if not m:
 184         continue
 185       func_name = m.group(1)
 186       if args.function is not None and func_name != args.function:
 187         # When filtering on a specific function, skip all others.
 188         continue
 189       is_in_function = is_in_function_start = True
 190
 191     if args.verbose:
 192       print('Writing %d lines to %s...' % (len(output_lines), test), file=sys.stderr)
 193
 194     with open(test, 'wb') as f:
 195       f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
 196
 197
 198 if __name__ == '__main__':
 199   main()