utils/update_test_checks.py

   1 #!/usr/bin/env python
   2
   3 """A script to generate FileCheck statements for 'opt' regression tests.
   4
   5 This script is a utility to update LLVM opt test cases with new
   6 FileCheck patterns. It can either update all of the tests in the file or
   7 a single test function.
   8
   9 Example usage:
  10 $ update_test_checks.py --opt=../bin/opt test/foo.ll
  11
  12 Workflow:
  13 1. Make a compiler patch that requires updating some number of FileCheck lines
  14    in regression test files.
  15 2. Save the patch and revert it from your local work area.
  16 3. Update the RUN-lines in the affected regression tests to look canonical.
  17    Example: "; RUN: opt < %s -instcombine -S | FileCheck %s"
  18 4. Refresh the FileCheck lines for either the entire file or select functions by
  19    running this script.
  20 5. Commit the fresh baseline of checks.
  21 6. Apply your patch from step 1 and rebuild your local binaries.
  22 7. Re-run this script on affected regression tests.
  23 8. Check the diffs to ensure the script has done something reasonable.
  24 9. Submit a patch including the regression test diffs for review.
  25
  26 A common pattern is to have the script insert complete checking of every
  27 instruction. Then, edit it down to only check the relevant instructions.
  28 The script is designed to make adding checks to a test case fast, it is *not*
  29 designed to be authoratitive about what constitutes a good test!
  30 """
  31
  32 from __future__ import print_function
  33
  34 import argparse
  35 import glob
  36 import itertools
  37 import os         # Used to advertise this file's name ("autogenerated_note").
  38 import string
  39 import subprocess
  40 import sys
  41 import tempfile
  42 import re
  43
  44 from UpdateTestChecks import common
  45
  46 ADVERT = '; NOTE: Assertions have been autogenerated by '
  47
  48 # RegEx: this is where the magic happens.
  49
  50 IR_FUNCTION_RE = re.compile('^\s*define\s+(?:internal\s+)?[^@]*@([\w-]+)\s*\(')
  51
  52
  53
  54
  55
  56 def main():
  57   from argparse import RawTextHelpFormatter
  58   parser = argparse.ArgumentParser(description=__doc__, formatter_class=RawTextHelpFormatter)
  59   parser.add_argument('-v', '--verbose', action='store_true',
  60                       help='Show verbose output')
  61   parser.add_argument('--opt-binary', default='opt',
  62                       help='The opt binary used to generate the test case')
  63   parser.add_argument(
  64       '--function', help='The function in the test file to update')
  65   parser.add_argument('-u', '--update-only', action='store_true',
  66                       help='Only update test if it was already autogened')
  67   parser.add_argument('tests', nargs='+')
  68   args = parser.parse_args()
  69
  70   script_name = os.path.basename(__file__)
  71   autogenerated_note = (ADVERT + 'utils/' + script_name)
  72
  73   opt_basename = os.path.basename(args.opt_binary)
  74   if not re.match(r'^opt(-\d+)?$', opt_basename):
  75     common.error('Unexpected opt name: ' + opt_basename)
  76     sys.exit(1)
  77   opt_basename = 'opt'
  78
  79   test_paths = []
  80   for test in args.tests:
  81     if not glob.glob(test):
  82       common.warn("Test file '%s' was not found. Ignoring it." % (test,))
  83       continue
  84     test_paths.append(test)
  85
  86   for test in test_paths:
  87     if args.verbose:
  88       print('Scanning for RUN lines in test file: ' + test, file=sys.stderr)
  89     with open(test) as f:
  90       input_lines = [l.rstrip() for l in f]
  91
  92     first_line = input_lines[0] if input_lines else ""
  93     if 'autogenerated' in first_line and script_name not in first_line:
  94       common.warn("Skipping test which wasn't autogenerated by " + script_name, test)
  95       continue
  96
  97     if args.update_only:
  98       if not first_line or 'autogenerated' not in first_line:
  99         common.warn("Skipping test which isn't autogenerated: " + test)
 100         continue
 101
 102     raw_lines = [m.group(1)
 103                  for m in [common.RUN_LINE_RE.match(l) for l in input_lines] if m]
 104     run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
 105     for l in raw_lines[1:]:
 106       if run_lines[-1].endswith('\\'):
 107         run_lines[-1] = run_lines[-1].rstrip('\\') + ' ' + l
 108       else:
 109         run_lines.append(l)
 110
 111     if args.verbose:
 112       print('Found %d RUN lines:' % (len(run_lines),), file=sys.stderr)
 113       for l in run_lines:
 114         print('  RUN: ' + l, file=sys.stderr)
 115
 116     prefix_list = []
 117     for l in run_lines:
 118       if '|' not in l:
 119         common.warn('Skipping unparseable RUN line: ' + l)
 120         continue
 121
 122       (tool_cmd, filecheck_cmd) = tuple([cmd.strip() for cmd in l.split('|', 1)])
 123       common.verify_filecheck_prefixes(filecheck_cmd)
 124       if not tool_cmd.startswith(opt_basename + ' '):
 125         common.warn('Skipping non-%s RUN line: %s' % (opt_basename, l))
 126         continue
 127
 128       if not filecheck_cmd.startswith('FileCheck '):
 129         common.warn('Skipping non-FileChecked RUN line: ' + l)
 130         continue
 131
 132       tool_cmd_args = tool_cmd[len(opt_basename):].strip()
 133       tool_cmd_args = tool_cmd_args.replace('< %s', '').replace('%s', '').strip()
 134
 135       check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
 136                                for item in m.group(1).split(',')]
 137       if not check_prefixes:
 138         check_prefixes = ['CHECK']
 139
 140       # FIXME: We should use multiple check prefixes to common check lines. For
 141       # now, we just ignore all but the last.
 142       prefix_list.append((check_prefixes, tool_cmd_args))
 143
 144     func_dict = {}
 145     for prefixes, _ in prefix_list:
 146       for prefix in prefixes:
 147         func_dict.update({prefix: dict()})
 148     for prefixes, opt_args in prefix_list:
 149       if args.verbose:
 150         print('Extracted opt cmd: ' + opt_basename + ' ' + opt_args, file=sys.stderr)
 151         print('Extracted FileCheck prefixes: ' + str(prefixes), file=sys.stderr)
 152
 153       raw_tool_output = common.invoke_tool(args.opt_binary, opt_args, test)
 154       common.build_function_body_dictionary(
 155               common.OPT_FUNCTION_RE, common.scrub_body, [],
 156               raw_tool_output, prefixes, func_dict, args.verbose)
 157
 158     is_in_function = False
 159     is_in_function_start = False
 160     prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
 161     if args.verbose:
 162       print('Rewriting FileCheck prefixes: %s' % (prefix_set,), file=sys.stderr)
 163     output_lines = []
 164     output_lines.append(autogenerated_note)
 165
 166     for input_line in input_lines:
 167       if is_in_function_start:
 168         if input_line == '':
 169           continue
 170         if input_line.lstrip().startswith(';'):
 171           m = common.CHECK_RE.match(input_line)
 172           if not m or m.group(1) not in prefix_set:
 173             output_lines.append(input_line)
 174             continue
 175
 176         # Print out the various check lines here.
 177         common.add_ir_checks(output_lines, ';', prefix_list, func_dict, func_name)
 178         is_in_function_start = False
 179
 180       if is_in_function:
 181         if common.should_add_line_to_output(input_line, prefix_set):
 182           # This input line of the function body will go as-is into the output.
 183           # Except make leading whitespace uniform: 2 spaces.
 184           input_line = common.SCRUB_LEADING_WHITESPACE_RE.sub(r'  ', input_line)
 185           output_lines.append(input_line)
 186         else:
 187           continue
 188         if input_line.strip() == '}':
 189           is_in_function = False
 190         continue
 191
 192       # Discard any previous script advertising.
 193       if input_line.startswith(ADVERT):
 194         continue
 195
 196       # If it's outside a function, it just gets copied to the output.
 197       output_lines.append(input_line)
 198
 199       m = IR_FUNCTION_RE.match(input_line)
 200       if not m:
 201         continue
 202       func_name = m.group(1)
 203       if args.function is not None and func_name != args.function:
 204         # When filtering on a specific function, skip all others.
 205         continue
 206       is_in_function = is_in_function_start = True
 207
 208     if args.verbose:
 209       print('Writing %d lines to %s...' % (len(output_lines), test), file=sys.stderr)
 210
 211     with open(test, 'wb') as f:
 212       f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
 213
 214
 215 if __name__ == '__main__':
 216   main()