utils/update_test_checks.py

   1 #!/usr/bin/env python
   2
   3 """A script to generate FileCheck statements for 'opt' regression tests.
   4
   5 This script is a utility to update LLVM opt test cases with new
   6 FileCheck patterns. It can either update all of the tests in the file or
   7 a single test function.
   8
   9 Example usage:
  10 $ update_test_checks.py --opt=../bin/opt test/foo.ll
  11
  12 Workflow:
  13 1. Make a compiler patch that requires updating some number of FileCheck lines
  14    in regression test files.
  15 2. Save the patch and revert it from your local work area.
  16 3. Update the RUN-lines in the affected regression tests to look canonical.
  17    Example: "; RUN: opt < %s -instcombine -S | FileCheck %s"
  18 4. Refresh the FileCheck lines for either the entire file or select functions by
  19    running this script.
  20 5. Commit the fresh baseline of checks.
  21 6. Apply your patch from step 1 and rebuild your local binaries.
  22 7. Re-run this script on affected regression tests.
  23 8. Check the diffs to ensure the script has done something reasonable.
  24 9. Submit a patch including the regression test diffs for review.
  25
  26 A common pattern is to have the script insert complete checking of every
  27 instruction. Then, edit it down to only check the relevant instructions.
  28 The script is designed to make adding checks to a test case fast, it is *not*
  29 designed to be authoratitive about what constitutes a good test!
  30 """
  31
  32 from __future__ import print_function
  33
  34 import argparse
  35 import glob
  36 import itertools
  37 import os         # Used to advertise this file's name ("autogenerated_note").
  38 import string
  39 import subprocess
  40 import sys
  41 import tempfile
  42 import re
  43
  44 from UpdateTestChecks import common
  45
  46 ADVERT = '; NOTE: Assertions have been autogenerated by '
  47
  48 # RegEx: this is where the magic happens.
  49
  50 IR_FUNCTION_RE = re.compile('^\s*define\s+(?:internal\s+)?[^@]*@([\w-]+)\s*\(')
  51
  52
  53
  54
  55
  56 def main():
  57   from argparse import RawTextHelpFormatter
  58   parser = argparse.ArgumentParser(description=__doc__, formatter_class=RawTextHelpFormatter)
  59   parser.add_argument('-v', '--verbose', action='store_true',
  60                       help='Show verbose output')
  61   parser.add_argument('--opt-binary', default='opt',
  62                       help='The opt binary used to generate the test case')
  63   parser.add_argument(
  64       '--function', help='The function in the test file to update')
  65   parser.add_argument('-u', '--update-only', action='store_true',
  66                       help='Only update test if it was already autogened')
  67   parser.add_argument('-p', '--preserve-names', action='store_true',
  68                       help='Do not scrub IR names')
  69   parser.add_argument('tests', nargs='+')
  70   args = parser.parse_args()
  71
  72   script_name = os.path.basename(__file__)
  73   autogenerated_note = (ADVERT + 'utils/' + script_name)
  74
  75   opt_basename = os.path.basename(args.opt_binary)
  76   if not re.match(r'^opt(-\d+)?$', opt_basename):
  77     common.error('Unexpected opt name: ' + opt_basename)
  78     sys.exit(1)
  79   opt_basename = 'opt'
  80
  81   for test in args.tests:
  82     if not glob.glob(test):
  83       common.warn("Test file pattern '%s' was not found. Ignoring it." % (test,))
  84       continue
  85
  86   # On Windows we must expand the patterns ourselves.
  87   test_paths = [test for pattern in args.tests for test in glob.glob(pattern)]
  88   for test in test_paths:
  89     if args.verbose:
  90       print('Scanning for RUN lines in test file: ' + test, file=sys.stderr)
  91     with open(test) as f:
  92       input_lines = [l.rstrip() for l in f]
  93
  94     first_line = input_lines[0] if input_lines else ""
  95     if 'autogenerated' in first_line and script_name not in first_line:
  96       common.warn("Skipping test which wasn't autogenerated by " + script_name, test)
  97       continue
  98
  99     if args.update_only:
 100       if not first_line or 'autogenerated' not in first_line:
 101         common.warn("Skipping test which isn't autogenerated: " + test)
 102         continue
 103
 104     raw_lines = [m.group(1)
 105                  for m in [common.RUN_LINE_RE.match(l) for l in input_lines] if m]
 106     run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
 107     for l in raw_lines[1:]:
 108       if run_lines[-1].endswith('\\'):
 109         run_lines[-1] = run_lines[-1].rstrip('\\') + ' ' + l
 110       else:
 111         run_lines.append(l)
 112
 113     if args.verbose:
 114       print('Found %d RUN lines:' % (len(run_lines),), file=sys.stderr)
 115       for l in run_lines:
 116         print('  RUN: ' + l, file=sys.stderr)
 117
 118     prefix_list = []
 119     for l in run_lines:
 120       if '|' not in l:
 121         common.warn('Skipping unparseable RUN line: ' + l)
 122         continue
 123
 124       (tool_cmd, filecheck_cmd) = tuple([cmd.strip() for cmd in l.split('|', 1)])
 125       common.verify_filecheck_prefixes(filecheck_cmd)
 126       if not tool_cmd.startswith(opt_basename + ' '):
 127         common.warn('Skipping non-%s RUN line: %s' % (opt_basename, l))
 128         continue
 129
 130       if not filecheck_cmd.startswith('FileCheck '):
 131         common.warn('Skipping non-FileChecked RUN line: ' + l)
 132         continue
 133
 134       tool_cmd_args = tool_cmd[len(opt_basename):].strip()
 135       tool_cmd_args = tool_cmd_args.replace('< %s', '').replace('%s', '').strip()
 136
 137       check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
 138                                for item in m.group(1).split(',')]
 139       if not check_prefixes:
 140         check_prefixes = ['CHECK']
 141
 142       # FIXME: We should use multiple check prefixes to common check lines. For
 143       # now, we just ignore all but the last.
 144       prefix_list.append((check_prefixes, tool_cmd_args))
 145
 146     func_dict = {}
 147     for prefixes, _ in prefix_list:
 148       for prefix in prefixes:
 149         func_dict.update({prefix: dict()})
 150     for prefixes, opt_args in prefix_list:
 151       if args.verbose:
 152         print('Extracted opt cmd: ' + opt_basename + ' ' + opt_args, file=sys.stderr)
 153         print('Extracted FileCheck prefixes: ' + str(prefixes), file=sys.stderr)
 154
 155       raw_tool_output = common.invoke_tool(args.opt_binary, opt_args, test)
 156       common.build_function_body_dictionary(
 157               common.OPT_FUNCTION_RE, common.scrub_body, [],
 158               raw_tool_output, prefixes, func_dict, args.verbose)
 159
 160     is_in_function = False
 161     is_in_function_start = False
 162     prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
 163     if args.verbose:
 164       print('Rewriting FileCheck prefixes: %s' % (prefix_set,), file=sys.stderr)
 165     output_lines = []
 166     output_lines.append(autogenerated_note)
 167
 168     for input_line in input_lines:
 169       if is_in_function_start:
 170         if input_line == '':
 171           continue
 172         if input_line.lstrip().startswith(';'):
 173           m = common.CHECK_RE.match(input_line)
 174           if not m or m.group(1) not in prefix_set:
 175             output_lines.append(input_line)
 176             continue
 177
 178         # Print out the various check lines here.
 179         common.add_ir_checks(output_lines, ';', prefix_list, func_dict,
 180                              func_name, args.preserve_names)
 181         is_in_function_start = False
 182
 183       if is_in_function:
 184         if common.should_add_line_to_output(input_line, prefix_set):
 185           # This input line of the function body will go as-is into the output.
 186           # Except make leading whitespace uniform: 2 spaces.
 187           input_line = common.SCRUB_LEADING_WHITESPACE_RE.sub(r'  ', input_line)
 188           output_lines.append(input_line)
 189         else:
 190           continue
 191         if input_line.strip() == '}':
 192           is_in_function = False
 193         continue
 194
 195       # Discard any previous script advertising.
 196       if input_line.startswith(ADVERT):
 197         continue
 198
 199       # If it's outside a function, it just gets copied to the output.
 200       output_lines.append(input_line)
 201
 202       m = IR_FUNCTION_RE.match(input_line)
 203       if not m:
 204         continue
 205       func_name = m.group(1)
 206       if args.function is not None and func_name != args.function:
 207         # When filtering on a specific function, skip all others.
 208         continue
 209       is_in_function = is_in_function_start = True
 210
 211     if args.verbose:
 212       print('Writing %d lines to %s...' % (len(output_lines), test), file=sys.stderr)
 213
 214     with open(test, 'wb') as f:
 215       f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
 216
 217
 218 if __name__ == '__main__':
 219   main()