llvm/utils/update_test_checks.py

   1 #!/usr/bin/env python3
   2
   3 """A script to generate FileCheck statements for 'opt' regression tests.
   4
   5 This script is a utility to update LLVM opt test cases with new
   6 FileCheck patterns. It can either update all of the tests in the file or
   7 a single test function.
   8
   9 Example usage:
  10 $ update_test_checks.py --opt=../bin/opt test/foo.ll
  11
  12 Workflow:
  13 1. Make a compiler patch that requires updating some number of FileCheck lines
  14    in regression test files.
  15 2. Save the patch and revert it from your local work area.
  16 3. Update the RUN-lines in the affected regression tests to look canonical.
  17    Example: "; RUN: opt < %s -instcombine -S | FileCheck %s"
  18 4. Refresh the FileCheck lines for either the entire file or select functions by
  19    running this script.
  20 5. Commit the fresh baseline of checks.
  21 6. Apply your patch from step 1 and rebuild your local binaries.
  22 7. Re-run this script on affected regression tests.
  23 8. Check the diffs to ensure the script has done something reasonable.
  24 9. Submit a patch including the regression test diffs for review.
  25
  26 A common pattern is to have the script insert complete checking of every
  27 instruction. Then, edit it down to only check the relevant instructions.
  28 The script is designed to make adding checks to a test case fast, it is *not*
  29 designed to be authoratitive about what constitutes a good test!
  30 """
  31
  32 from __future__ import print_function
  33
  34 import argparse
  35 import glob
  36 import itertools
  37 import os         # Used to advertise this file's name ("autogenerated_note").
  38 import string
  39 import subprocess
  40 import sys
  41 import tempfile
  42 import re
  43
  44 from UpdateTestChecks import common
  45
  46 ADVERT = '; NOTE: Assertions have been autogenerated by '
  47
  48 # RegEx: this is where the magic happens.
  49
  50 IR_FUNCTION_RE = re.compile('^\s*define\s+(?:internal\s+)?[^@]*@([\w-]+)\s*\(')
  51
  52
  53
  54
  55
  56 def main():
  57   from argparse import RawTextHelpFormatter
  58   parser = argparse.ArgumentParser(description=__doc__, formatter_class=RawTextHelpFormatter)
  59   parser.add_argument('--opt-binary', default='opt',
  60                       help='The opt binary used to generate the test case')
  61   parser.add_argument(
  62       '--function', help='The function in the test file to update')
  63   parser.add_argument('-p', '--preserve-names', action='store_true',
  64                       help='Do not scrub IR names')
  65   parser.add_argument('--function-signature', action='store_true',
  66                       help='Keep function signature information around for the check line')
  67   parser.add_argument('--scrub-attributes', action='store_true',
  68                       help='Remove attribute annotations (#0) from the end of check line')
  69   parser.add_argument('tests', nargs='+')
  70   args = common.parse_commandline_args(parser)
  71
  72   script_name = os.path.basename(__file__)
  73   autogenerated_note = (ADVERT + 'utils/' + script_name)
  74
  75   opt_basename = os.path.basename(args.opt_binary)
  76   if not re.match(r'^opt(-\d+)?$', opt_basename):
  77     common.error('Unexpected opt name: ' + opt_basename)
  78     sys.exit(1)
  79   opt_basename = 'opt'
  80
  81   for test in args.tests:
  82     if not glob.glob(test):
  83       common.warn("Test file pattern '%s' was not found. Ignoring it." % (test,))
  84       continue
  85
  86   # On Windows we must expand the patterns ourselves.
  87   test_paths = [test for pattern in args.tests for test in glob.glob(pattern)]
  88   for test in test_paths:
  89     with open(test) as f:
  90       input_lines = [l.rstrip() for l in f]
  91
  92     first_line = input_lines[0] if input_lines else ""
  93     if 'autogenerated' in first_line and script_name not in first_line:
  94       common.warn("Skipping test which wasn't autogenerated by " + script_name, test)
  95       continue
  96
  97     if args.update_only:
  98       if not first_line or 'autogenerated' not in first_line:
  99         common.warn("Skipping test which isn't autogenerated: " + test)
 100         continue
 101
 102     run_lines = common.find_run_lines(test, input_lines)
 103
 104     # If requested we scrub trailing attribute annotations, e.g., '#0', together with whitespaces
 105     if args.scrub_attributes:
 106       common.SCRUB_TRAILING_WHITESPACE_TEST_RE = common.SCRUB_TRAILING_WHITESPACE_AND_ATTRIBUTES_RE
 107     else:
 108       common.SCRUB_TRAILING_WHITESPACE_TEST_RE = common.SCRUB_TRAILING_WHITESPACE_RE
 109
 110     prefix_list = []
 111     for l in run_lines:
 112       if '|' not in l:
 113         common.warn('Skipping unparseable RUN line: ' + l)
 114         continue
 115
 116       (tool_cmd, filecheck_cmd) = tuple([cmd.strip() for cmd in l.split('|', 1)])
 117       common.verify_filecheck_prefixes(filecheck_cmd)
 118       if not tool_cmd.startswith(opt_basename + ' '):
 119         common.warn('Skipping non-%s RUN line: %s' % (opt_basename, l))
 120         continue
 121
 122       if not filecheck_cmd.startswith('FileCheck '):
 123         common.warn('Skipping non-FileChecked RUN line: ' + l)
 124         continue
 125
 126       tool_cmd_args = tool_cmd[len(opt_basename):].strip()
 127       tool_cmd_args = tool_cmd_args.replace('< %s', '').replace('%s', '').strip()
 128
 129       check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
 130                                for item in m.group(1).split(',')]
 131       if not check_prefixes:
 132         check_prefixes = ['CHECK']
 133
 134       # FIXME: We should use multiple check prefixes to common check lines. For
 135       # now, we just ignore all but the last.
 136       prefix_list.append((check_prefixes, tool_cmd_args))
 137
 138     func_dict = {}
 139     for prefixes, _ in prefix_list:
 140       for prefix in prefixes:
 141         func_dict.update({prefix: dict()})
 142     for prefixes, opt_args in prefix_list:
 143       common.debug('Extracted opt cmd: ' + opt_basename + ' ' + opt_args)
 144       common.debug('Extracted FileCheck prefixes: ' + str(prefixes))
 145
 146       raw_tool_output = common.invoke_tool(args.opt_binary, opt_args, test)
 147       common.build_function_body_dictionary(
 148               common.OPT_FUNCTION_RE, common.scrub_body, [],
 149               raw_tool_output, prefixes, func_dict, args.verbose,
 150               args.function_signature)
 151
 152     is_in_function = False
 153     is_in_function_start = False
 154     prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
 155     common.debug('Rewriting FileCheck prefixes:', str(prefix_set))
 156     output_lines = []
 157     output_lines.append(autogenerated_note)
 158
 159     for input_line in input_lines:
 160       if is_in_function_start:
 161         if input_line == '':
 162           continue
 163         if input_line.lstrip().startswith(';'):
 164           m = common.CHECK_RE.match(input_line)
 165           if not m or m.group(1) not in prefix_set:
 166             output_lines.append(input_line)
 167             continue
 168
 169         # Print out the various check lines here.
 170         common.add_ir_checks(output_lines, ';', prefix_list, func_dict,
 171                              func_name, args.preserve_names, args.function_signature)
 172         is_in_function_start = False
 173
 174       if is_in_function:
 175         if common.should_add_line_to_output(input_line, prefix_set):
 176           # This input line of the function body will go as-is into the output.
 177           # Except make leading whitespace uniform: 2 spaces.
 178           input_line = common.SCRUB_LEADING_WHITESPACE_RE.sub(r'  ', input_line)
 179           output_lines.append(input_line)
 180         else:
 181           continue
 182         if input_line.strip() == '}':
 183           is_in_function = False
 184         continue
 185
 186       # Discard any previous script advertising.
 187       if input_line.startswith(ADVERT):
 188         continue
 189
 190       # If it's outside a function, it just gets copied to the output.
 191       output_lines.append(input_line)
 192
 193       m = IR_FUNCTION_RE.match(input_line)
 194       if not m:
 195         continue
 196       func_name = m.group(1)
 197       if args.function is not None and func_name != args.function:
 198         # When filtering on a specific function, skip all others.
 199         continue
 200       is_in_function = is_in_function_start = True
 201
 202     common.debug('Writing %d lines to %s...' % (len(output_lines), test))
 203
 204     with open(test, 'wb') as f:
 205       f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
 206
 207
 208 if __name__ == '__main__':
 209   main()