llvm/utils/update_analyze_test_checks.py

   1 #!/usr/bin/env python3
   2
   3 """A script to generate FileCheck statements for 'opt' analysis tests.
   4
   5 This script is a utility to update LLVM opt analysis test cases with new
   6 FileCheck patterns. It can either update all of the tests in the file or
   7 a single test function.
   8
   9 Example usage:
  10 $ update_analyze_test_checks.py --opt=../bin/opt test/foo.ll
  11
  12 Workflow:
  13 1. Make a compiler patch that requires updating some number of FileCheck lines
  14    in regression test files.
  15 2. Save the patch and revert it from your local work area.
  16 3. Update the RUN-lines in the affected regression tests to look canonical.
  17    Example: "; RUN: opt < %s -analyze -cost-model -S | FileCheck %s"
  18 4. Refresh the FileCheck lines for either the entire file or select functions by
  19    running this script.
  20 5. Commit the fresh baseline of checks.
  21 6. Apply your patch from step 1 and rebuild your local binaries.
  22 7. Re-run this script on affected regression tests.
  23 8. Check the diffs to ensure the script has done something reasonable.
  24 9. Submit a patch including the regression test diffs for review.
  25
  26 A common pattern is to have the script insert complete checking of every
  27 instruction. Then, edit it down to only check the relevant instructions.
  28 The script is designed to make adding checks to a test case fast, it is *not*
  29 designed to be authoratitive about what constitutes a good test!
  30 """
  31
  32 from __future__ import print_function
  33
  34 import argparse
  35 import glob
  36 import itertools
  37 import os         # Used to advertise this file's name ("autogenerated_note").
  38 import string
  39 import subprocess
  40 import sys
  41 import tempfile
  42 import re
  43
  44 from UpdateTestChecks import common
  45
  46 ADVERT = '; NOTE: Assertions have been autogenerated by '
  47
  48 def main():
  49   from argparse import RawTextHelpFormatter
  50   parser = argparse.ArgumentParser(description=__doc__, formatter_class=RawTextHelpFormatter)
  51   parser.add_argument('--opt-binary', default='opt',
  52                       help='The opt binary used to generate the test case')
  53   parser.add_argument(
  54       '--function', help='The function in the test file to update')
  55   parser.add_argument('tests', nargs='+')
  56   args = common.parse_commandline_args(parser)
  57
  58   script_name = os.path.basename(__file__)
  59   autogenerated_note = (ADVERT + 'utils/' + script_name)
  60
  61   opt_basename = os.path.basename(args.opt_binary)
  62   if (opt_basename != "opt"):
  63     common.error('Unexpected opt name: ' + opt_basename)
  64     sys.exit(1)
  65
  66   test_paths = [test for pattern in args.tests for test in glob.glob(pattern)]
  67   for test in test_paths:
  68     with open(test) as f:
  69       input_lines = [l.rstrip() for l in f]
  70
  71     first_line = input_lines[0] if input_lines else ""
  72     if 'autogenerated' in first_line and script_name not in first_line:
  73       common.warn("Skipping test which wasn't autogenerated by " + script_name + ": " + test)
  74       continue
  75
  76     if args.update_only:
  77       if not first_line or 'autogenerated' not in first_line:
  78         common.warn("Skipping test which isn't autogenerated: " + test)
  79         continue
  80
  81     run_lines = common.find_run_lines(test, input_lines)
  82     prefix_list = []
  83     for l in run_lines:
  84       if '|' not in l:
  85         common.warn('Skipping unparseable RUN line: ' + l)
  86         continue
  87
  88       (tool_cmd, filecheck_cmd) = tuple([cmd.strip() for cmd in l.split('|', 1)])
  89       common.verify_filecheck_prefixes(filecheck_cmd)
  90
  91       if not tool_cmd.startswith(opt_basename + ' '):
  92         common.warn('WSkipping non-%s RUN line: %s' % (opt_basename, l))
  93         continue
  94
  95       if not filecheck_cmd.startswith('FileCheck '):
  96         common.warn('Skipping non-FileChecked RUN line: ' + l)
  97         continue
  98
  99       tool_cmd_args = tool_cmd[len(opt_basename):].strip()
 100       tool_cmd_args = tool_cmd_args.replace('< %s', '').replace('%s', '').strip()
 101
 102       check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
 103                                for item in m.group(1).split(',')]
 104       if not check_prefixes:
 105         check_prefixes = ['CHECK']
 106
 107       # FIXME: We should use multiple check prefixes to common check lines. For
 108       # now, we just ignore all but the last.
 109       prefix_list.append((check_prefixes, tool_cmd_args))
 110
 111     builder = common.FunctionTestBuilder(
 112       run_list = prefix_list,
 113       flags = type('', (object,), {
 114             'verbose': args.verbose,
 115             'function_signature': False,
 116             'check_attributes': False,
 117             'replace_value_regex': []}),
 118       scrubber_args = [],
 119       path=test)
 120
 121     for prefixes, opt_args in prefix_list:
 122       common.debug('Extracted opt cmd:', opt_basename, opt_args, file=sys.stderr)
 123       common.debug('Extracted FileCheck prefixes:', str(prefixes), file=sys.stderr)
 124
 125       raw_tool_outputs = common.invoke_tool(args.opt_binary, opt_args, test)
 126
 127       # Split analysis outputs by "Printing analysis " declarations.
 128       for raw_tool_output in re.split(r'Printing analysis ', raw_tool_outputs):
 129         builder.process_run_line(common.ANALYZE_FUNCTION_RE, common.scrub_body,
 130                                  raw_tool_output, prefixes, False)
 131
 132     func_dict = builder.finish_and_get_func_dict()
 133     is_in_function = False
 134     is_in_function_start = False
 135     prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
 136     common.debug('Rewriting FileCheck prefixes:', str(prefix_set), file=sys.stderr)
 137     output_lines = []
 138     output_lines.append(autogenerated_note)
 139
 140     for input_line in input_lines:
 141       if is_in_function_start:
 142         if input_line == '':
 143           continue
 144         if input_line.lstrip().startswith(';'):
 145           m = common.CHECK_RE.match(input_line)
 146           if not m or m.group(1) not in prefix_set:
 147             output_lines.append(input_line)
 148             continue
 149
 150         # Print out the various check lines here.
 151         common.add_analyze_checks(output_lines, ';', prefix_list, func_dict, func_name)
 152         is_in_function_start = False
 153
 154       if is_in_function:
 155         if common.should_add_line_to_output(input_line, prefix_set):
 156           # This input line of the function body will go as-is into the output.
 157           # Except make leading whitespace uniform: 2 spaces.
 158           input_line = common.SCRUB_LEADING_WHITESPACE_RE.sub(r'  ', input_line)
 159           output_lines.append(input_line)
 160         else:
 161           continue
 162         if input_line.strip() == '}':
 163           is_in_function = False
 164         continue
 165
 166       # Discard any previous script advertising.
 167       if input_line.startswith(ADVERT):
 168         continue
 169
 170       # If it's outside a function, it just gets copied to the output.
 171       output_lines.append(input_line)
 172
 173       m = common.IR_FUNCTION_RE.match(input_line)
 174       if not m:
 175         continue
 176       func_name = m.group(1)
 177       if args.function is not None and func_name != args.function:
 178         # When filtering on a specific function, skip all others.
 179         continue
 180       is_in_function = is_in_function_start = True
 181
 182     common.debug('Writing %d lines to %s...' % (len(output_lines), test))
 183
 184     with open(test, 'wb') as f:
 185       f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
 186
 187
 188 if __name__ == '__main__':
 189   main()