llvm/utils/update_test_checks.py

   1 #!/usr/bin/env python3
   2
   3 """A script to generate FileCheck statements for 'opt' regression tests.
   4
   5 This script is a utility to update LLVM opt test cases with new
   6 FileCheck patterns. It can either update all of the tests in the file or
   7 a single test function.
   8
   9 Example usage:
  10 $ update_test_checks.py --opt=../bin/opt test/foo.ll
  11
  12 Workflow:
  13 1. Make a compiler patch that requires updating some number of FileCheck lines
  14    in regression test files.
  15 2. Save the patch and revert it from your local work area.
  16 3. Update the RUN-lines in the affected regression tests to look canonical.
  17    Example: "; RUN: opt < %s -instcombine -S | FileCheck %s"
  18 4. Refresh the FileCheck lines for either the entire file or select functions by
  19    running this script.
  20 5. Commit the fresh baseline of checks.
  21 6. Apply your patch from step 1 and rebuild your local binaries.
  22 7. Re-run this script on affected regression tests.
  23 8. Check the diffs to ensure the script has done something reasonable.
  24 9. Submit a patch including the regression test diffs for review.
  25
  26 A common pattern is to have the script insert complete checking of every
  27 instruction. Then, edit it down to only check the relevant instructions.
  28 The script is designed to make adding checks to a test case fast, it is *not*
  29 designed to be authoratitive about what constitutes a good test!
  30 """
  31
  32 from __future__ import print_function
  33
  34 import argparse
  35 import os  # Used to advertise this file's name ("autogenerated_note").
  36 import re
  37 import sys
  38
  39 from UpdateTestChecks import common
  40
  41
  42 def main():
  43   from argparse import RawTextHelpFormatter
  44   parser = argparse.ArgumentParser(description=__doc__, formatter_class=RawTextHelpFormatter)
  45   parser.add_argument('--opt-binary', default='opt',
  46                       help='The opt binary used to generate the test case')
  47   parser.add_argument(
  48       '--function', help='The function in the test file to update')
  49   parser.add_argument('-p', '--preserve-names', action='store_true',
  50                       help='Do not scrub IR names')
  51   parser.add_argument('--function-signature', action='store_true',
  52                       help='Keep function signature information around for the check line')
  53   parser.add_argument('--scrub-attributes', action='store_true',
  54                       help='Remove attribute annotations (#0) from the end of check line')
  55   parser.add_argument('--check-attributes', action='store_true',
  56                       help='Check "Function Attributes" for functions')
  57   parser.add_argument('--check-globals', action='store_true',
  58                       help='Check global entries (global variables, metadata, attribute sets, ...) for functions')
  59   parser.add_argument('tests', nargs='+')
  60   initial_args = common.parse_commandline_args(parser)
  61
  62   script_name = os.path.basename(__file__)
  63   opt_basename = os.path.basename(initial_args.opt_binary)
  64   if not re.match(r'^opt(-\d+)?(\.exe)?$', opt_basename):
  65     common.error('Unexpected opt name: ' + opt_basename)
  66     sys.exit(1)
  67   opt_basename = 'opt'
  68
  69   for ti in common.itertests(initial_args.tests, parser,
  70                              script_name='utils/' + script_name):
  71     # If requested we scrub trailing attribute annotations, e.g., '#0', together with whitespaces
  72     if ti.args.scrub_attributes:
  73       common.SCRUB_TRAILING_WHITESPACE_TEST_RE = common.SCRUB_TRAILING_WHITESPACE_AND_ATTRIBUTES_RE
  74     else:
  75       common.SCRUB_TRAILING_WHITESPACE_TEST_RE = common.SCRUB_TRAILING_WHITESPACE_RE
  76
  77     prefix_list = []
  78     for l in ti.run_lines:
  79       if '|' not in l:
  80         common.warn('Skipping unparseable RUN line: ' + l)
  81         continue
  82
  83       commands = [cmd.strip() for cmd in l.split('|')]
  84       assert len(commands) >= 2
  85       preprocess_cmd = None
  86       if len(commands) > 2:
  87         preprocess_cmd = " | ".join(commands[:-2])
  88       tool_cmd = commands[-2]
  89       filecheck_cmd = commands[-1]
  90       common.verify_filecheck_prefixes(filecheck_cmd)
  91       if not tool_cmd.startswith(opt_basename + ' '):
  92         common.warn('Skipping non-%s RUN line: %s' % (opt_basename, l))
  93         continue
  94
  95       if not filecheck_cmd.startswith('FileCheck '):
  96         common.warn('Skipping non-FileChecked RUN line: ' + l)
  97         continue
  98
  99       tool_cmd_args = tool_cmd[len(opt_basename):].strip()
 100       tool_cmd_args = tool_cmd_args.replace('< %s', '').replace('%s', '').strip()
 101
 102       check_prefixes = [item for m in
 103                         common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
 104                         for item in m.group(1).split(',')]
 105       if not check_prefixes:
 106         check_prefixes = ['CHECK']
 107
 108       # FIXME: We should use multiple check prefixes to common check lines. For
 109       # now, we just ignore all but the last.
 110       prefix_list.append((check_prefixes, tool_cmd_args, preprocess_cmd))
 111
 112     global_vars_seen_dict = {}
 113     builder = common.FunctionTestBuilder(
 114       run_list=prefix_list,
 115       flags=ti.args,
 116       scrubber_args=[],
 117       path=ti.path)
 118
 119     for prefixes, opt_args, preprocess_cmd in prefix_list:
 120       common.debug('Extracted opt cmd: ' + opt_basename + ' ' + opt_args)
 121       common.debug('Extracted FileCheck prefixes: ' + str(prefixes))
 122
 123       raw_tool_output = common.invoke_tool(ti.args.opt_binary, opt_args,
 124                                            ti.path, preprocess_cmd=preprocess_cmd,
 125                                            verbose=ti.args.verbose)
 126       builder.process_run_line(common.OPT_FUNCTION_RE, common.scrub_body,
 127               raw_tool_output, prefixes, False)
 128
 129     func_dict = builder.finish_and_get_func_dict()
 130     is_in_function = False
 131     is_in_function_start = False
 132     has_checked_pre_function_globals = False
 133     prefix_set = set([prefix for prefixes, _, _ in prefix_list for prefix in prefixes])
 134     common.debug('Rewriting FileCheck prefixes:', str(prefix_set))
 135     output_lines = []
 136
 137     include_generated_funcs = common.find_arg_in_test(ti,
 138                                                       lambda args: ti.args.include_generated_funcs,
 139                                                       '--include-generated-funcs',
 140                                                       True)
 141
 142     if include_generated_funcs:
 143       # Generate the appropriate checks for each function.  We need to emit
 144       # these in the order according to the generated output so that CHECK-LABEL
 145       # works properly.  func_order provides that.
 146
 147       # We can't predict where various passes might insert functions so we can't
 148       # be sure the input function order is maintained.  Therefore, first spit
 149       # out all the source lines.
 150       common.dump_input_lines(output_lines, ti, prefix_set, ';')
 151
 152       args = ti.args
 153       if args.check_globals:
 154           common.add_global_checks(builder.global_var_dict(), ';', prefix_list, output_lines, global_vars_seen_dict, args.preserve_names, True)
 155
 156       # Now generate all the checks.
 157       common.add_checks_at_end(output_lines, prefix_list, builder.func_order(),
 158                                ';', lambda my_output_lines, prefixes, func:
 159                                common.add_ir_checks(my_output_lines, ';',
 160                                                     prefixes,
 161                                                     func_dict, func, False,
 162                                                     args.function_signature,
 163                                                     global_vars_seen_dict))
 164     else:
 165       # "Normal" mode.
 166       for input_line_info in ti.iterlines(output_lines):
 167         input_line = input_line_info.line
 168         args = input_line_info.args
 169         if is_in_function_start:
 170           if input_line == '':
 171             continue
 172           if input_line.lstrip().startswith(';'):
 173             m = common.CHECK_RE.match(input_line)
 174             if not m or m.group(1) not in prefix_set:
 175               output_lines.append(input_line)
 176               continue
 177
 178           # Print out the various check lines here.
 179           common.add_ir_checks(output_lines, ';', prefix_list, func_dict,
 180                                func_name, args.preserve_names, args.function_signature,
 181                                global_vars_seen_dict)
 182           is_in_function_start = False
 183
 184         m = common.IR_FUNCTION_RE.match(input_line)
 185         if m and not has_checked_pre_function_globals:
 186             if args.check_globals:
 187                 common.add_global_checks(builder.global_var_dict(), ';', prefix_list, output_lines, global_vars_seen_dict, args.preserve_names, True)
 188             has_checked_pre_function_globals = True
 189
 190         if common.should_add_line_to_output(input_line, prefix_set, not is_in_function):
 191             # This input line of the function body will go as-is into the output.
 192             # Except make leading whitespace uniform: 2 spaces.
 193             input_line = common.SCRUB_LEADING_WHITESPACE_RE.sub(r'  ', input_line)
 194             output_lines.append(input_line)
 195             if input_line.strip() == '}':
 196                  is_in_function = False
 197                  continue
 198
 199         if is_in_function:
 200            continue
 201
 202         m = common.IR_FUNCTION_RE.match(input_line)
 203         if not m:
 204           continue
 205         func_name = m.group(1)
 206         if args.function is not None and func_name != args.function:
 207           # When filtering on a specific function, skip all others.
 208           continue
 209         is_in_function = is_in_function_start = True
 210
 211     if args.check_globals:
 212         common.add_global_checks(builder.global_var_dict(), ';', prefix_list, output_lines, global_vars_seen_dict, args.preserve_names, False)
 213     common.debug('Writing %d lines to %s...' % (len(output_lines), ti.path))
 214
 215     with open(ti.path, 'wb') as f:
 216       f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
 217
 218
 219 if __name__ == '__main__':
 220   main()