llvm/utils/update_test_checks.py

   1 #!/usr/bin/env python3
   2
   3 """A script to generate FileCheck statements for 'opt' regression tests.
   4
   5 This script is a utility to update LLVM opt test cases with new
   6 FileCheck patterns. It can either update all of the tests in the file or
   7 a single test function.
   8
   9 Example usage:
  10
  11 # Default to using `opt` as found in your PATH.
  12 $ update_test_checks.py test/foo.ll
  13
  14 # Override the path lookup.
  15 $ update_test_checks.py --tool-binary=../bin/opt test/foo.ll
  16
  17 # Use a custom tool instead of `opt`.
  18 $ update_test_checks.py --tool=yourtool test/foo.ll
  19
  20 Workflow:
  21 1. Make a compiler patch that requires updating some number of FileCheck lines
  22    in regression test files.
  23 2. Save the patch and revert it from your local work area.
  24 3. Update the RUN-lines in the affected regression tests to look canonical.
  25    Example: "; RUN: opt < %s -instcombine -S | FileCheck %s"
  26 4. Refresh the FileCheck lines for either the entire file or select functions by
  27    running this script.
  28 5. Commit the fresh baseline of checks.
  29 6. Apply your patch from step 1 and rebuild your local binaries.
  30 7. Re-run this script on affected regression tests.
  31 8. Check the diffs to ensure the script has done something reasonable.
  32 9. Submit a patch including the regression test diffs for review.
  33 """
  34
  35 from __future__ import print_function
  36
  37 import argparse
  38 import os  # Used to advertise this file's name ("autogenerated_note").
  39 import re
  40 import sys
  41
  42 from UpdateTestChecks import common
  43
  44
  45 def main():
  46     from argparse import RawTextHelpFormatter
  47
  48     parser = argparse.ArgumentParser(
  49         description=__doc__, formatter_class=RawTextHelpFormatter
  50     )
  51     parser.add_argument(
  52         "--tool",
  53         default="opt",
  54         help='The name of the tool used to generate the test case (defaults to "opt")',
  55     )
  56     parser.add_argument(
  57         "--tool-binary",
  58         "--opt-binary",
  59         help="The tool binary used to generate the test case",
  60     )
  61     parser.add_argument("--function", help="The function in the test file to update")
  62     parser.add_argument(
  63         "-p", "--preserve-names", action="store_true", help="Do not scrub IR names"
  64     )
  65     parser.add_argument(
  66         "--function-signature",
  67         action="store_true",
  68         help="Keep function signature information around for the check line",
  69     )
  70     parser.add_argument(
  71         "--scrub-attributes",
  72         action="store_true",
  73         help="Remove attribute annotations (#0) from the end of check line",
  74     )
  75     parser.add_argument(
  76         "--check-attributes",
  77         action="store_true",
  78         help='Check "Function Attributes" for functions',
  79     )
  80     parser.add_argument(
  81         "--check-globals",
  82         action="store_true",
  83         help="Check global entries (global variables, metadata, attribute sets, ...) for functions",
  84     )
  85     parser.add_argument("tests", nargs="+")
  86     initial_args = common.parse_commandline_args(parser)
  87
  88     script_name = os.path.basename(__file__)
  89
  90     if initial_args.tool_binary:
  91         tool_basename = os.path.basename(initial_args.tool_binary)
  92         if not re.match(r"^%s(-\d+)?(\.exe)?$" % (initial_args.tool), tool_basename):
  93             common.error("Unexpected tool name: " + tool_basename)
  94             sys.exit(1)
  95
  96     for ti in common.itertests(
  97         initial_args.tests, parser, script_name="utils/" + script_name
  98     ):
  99         # If requested we scrub trailing attribute annotations, e.g., '#0', together with whitespaces
 100         if ti.args.scrub_attributes:
 101             common.SCRUB_TRAILING_WHITESPACE_TEST_RE = (
 102                 common.SCRUB_TRAILING_WHITESPACE_AND_ATTRIBUTES_RE
 103             )
 104         else:
 105             common.SCRUB_TRAILING_WHITESPACE_TEST_RE = (
 106                 common.SCRUB_TRAILING_WHITESPACE_RE
 107             )
 108
 109         tool_basename = ti.args.tool
 110
 111         prefix_list = []
 112         for l in ti.run_lines:
 113             if "|" not in l:
 114                 common.warn("Skipping unparsable RUN line: " + l)
 115                 continue
 116
 117             commands = [cmd.strip() for cmd in l.split("|")]
 118             assert len(commands) >= 2
 119             preprocess_cmd = None
 120             if len(commands) > 2:
 121                 preprocess_cmd = " | ".join(commands[:-2])
 122             tool_cmd = commands[-2]
 123             filecheck_cmd = commands[-1]
 124             common.verify_filecheck_prefixes(filecheck_cmd)
 125             if not tool_cmd.startswith(tool_basename + " "):
 126                 common.warn("Skipping non-%s RUN line: %s" % (tool_basename, l))
 127                 continue
 128
 129             if not filecheck_cmd.startswith("FileCheck "):
 130                 common.warn("Skipping non-FileChecked RUN line: " + l)
 131                 continue
 132
 133             tool_cmd_args = tool_cmd[len(tool_basename) :].strip()
 134             tool_cmd_args = tool_cmd_args.replace("< %s", "").replace("%s", "").strip()
 135             check_prefixes = common.get_check_prefixes(filecheck_cmd)
 136
 137             # FIXME: We should use multiple check prefixes to common check lines. For
 138             # now, we just ignore all but the last.
 139             prefix_list.append((check_prefixes, tool_cmd_args, preprocess_cmd))
 140
 141         global_vars_seen_dict = {}
 142         builder = common.FunctionTestBuilder(
 143             run_list=prefix_list, flags=ti.args, scrubber_args=[], path=ti.path
 144         )
 145
 146         tool_binary = ti.args.tool_binary
 147         if not tool_binary:
 148             tool_binary = tool_basename
 149
 150         for prefixes, tool_args, preprocess_cmd in prefix_list:
 151             common.debug("Extracted tool cmd: " + tool_basename + " " + tool_args)
 152             common.debug("Extracted FileCheck prefixes: " + str(prefixes))
 153
 154             raw_tool_output = common.invoke_tool(
 155                 tool_binary,
 156                 tool_args,
 157                 ti.path,
 158                 preprocess_cmd=preprocess_cmd,
 159                 verbose=ti.args.verbose,
 160             )
 161             builder.process_run_line(
 162                 common.OPT_FUNCTION_RE,
 163                 common.scrub_body,
 164                 raw_tool_output,
 165                 prefixes,
 166                 False,
 167             )
 168             builder.processed_prefixes(prefixes)
 169
 170         func_dict = builder.finish_and_get_func_dict()
 171         is_in_function = False
 172         is_in_function_start = False
 173         has_checked_pre_function_globals = False
 174         prefix_set = set(
 175             [prefix for prefixes, _, _ in prefix_list for prefix in prefixes]
 176         )
 177         common.debug("Rewriting FileCheck prefixes:", str(prefix_set))
 178         output_lines = []
 179
 180         include_generated_funcs = common.find_arg_in_test(
 181             ti,
 182             lambda args: ti.args.include_generated_funcs,
 183             "--include-generated-funcs",
 184             True,
 185         )
 186         generated_prefixes = []
 187         if include_generated_funcs:
 188             # Generate the appropriate checks for each function.  We need to emit
 189             # these in the order according to the generated output so that CHECK-LABEL
 190             # works properly.  func_order provides that.
 191
 192             # We can't predict where various passes might insert functions so we can't
 193             # be sure the input function order is maintained.  Therefore, first spit
 194             # out all the source lines.
 195             common.dump_input_lines(output_lines, ti, prefix_set, ";")
 196
 197             args = ti.args
 198             if args.check_globals:
 199                 generated_prefixes.extend(
 200                     common.add_global_checks(
 201                         builder.global_var_dict(),
 202                         ";",
 203                         prefix_list,
 204                         output_lines,
 205                         global_vars_seen_dict,
 206                         args.preserve_names,
 207                         True,
 208                     )
 209                 )
 210
 211             # Now generate all the checks.
 212             generated_prefixes.extend(
 213                 common.add_checks_at_end(
 214                     output_lines,
 215                     prefix_list,
 216                     builder.func_order(),
 217                     ";",
 218                     lambda my_output_lines, prefixes, func: common.add_ir_checks(
 219                         my_output_lines,
 220                         ";",
 221                         prefixes,
 222                         func_dict,
 223                         func,
 224                         False,
 225                         args.function_signature,
 226                         args.version,
 227                         global_vars_seen_dict,
 228                         is_filtered=builder.is_filtered(),
 229                     ),
 230                 )
 231             )
 232         else:
 233             # "Normal" mode.
 234             for input_line_info in ti.iterlines(output_lines):
 235                 input_line = input_line_info.line
 236                 args = input_line_info.args
 237                 if is_in_function_start:
 238                     if input_line == "":
 239                         continue
 240                     if input_line.lstrip().startswith(";"):
 241                         m = common.CHECK_RE.match(input_line)
 242                         if not m or m.group(1) not in prefix_set:
 243                             output_lines.append(input_line)
 244                             continue
 245
 246                     # Print out the various check lines here.
 247                     generated_prefixes.extend(
 248                         common.add_ir_checks(
 249                             output_lines,
 250                             ";",
 251                             prefix_list,
 252                             func_dict,
 253                             func_name,
 254                             args.preserve_names,
 255                             args.function_signature,
 256                             args.version,
 257                             global_vars_seen_dict,
 258                             is_filtered=builder.is_filtered(),
 259                         )
 260                     )
 261                     is_in_function_start = False
 262
 263                 m = common.IR_FUNCTION_RE.match(input_line)
 264                 if m and not has_checked_pre_function_globals:
 265                     if args.check_globals:
 266                         generated_prefixes.extend(
 267                             common.add_global_checks(
 268                                 builder.global_var_dict(),
 269                                 ";",
 270                                 prefix_list,
 271                                 output_lines,
 272                                 global_vars_seen_dict,
 273                                 args.preserve_names,
 274                                 True,
 275                             )
 276                         )
 277                     has_checked_pre_function_globals = True
 278
 279                 if common.should_add_line_to_output(
 280                     input_line, prefix_set, not is_in_function
 281                 ):
 282                     # This input line of the function body will go as-is into the output.
 283                     # Except make leading whitespace uniform: 2 spaces.
 284                     input_line = common.SCRUB_LEADING_WHITESPACE_RE.sub(
 285                         r"  ", input_line
 286                     )
 287                     output_lines.append(input_line)
 288                     if input_line.strip() == "}":
 289                         is_in_function = False
 290                         continue
 291
 292                 if is_in_function:
 293                     continue
 294
 295                 m = common.IR_FUNCTION_RE.match(input_line)
 296                 if not m:
 297                     continue
 298                 func_name = m.group(1)
 299                 if args.function is not None and func_name != args.function:
 300                     # When filtering on a specific function, skip all others.
 301                     continue
 302                 is_in_function = is_in_function_start = True
 303
 304         if args.check_globals:
 305             generated_prefixes.extend(
 306                 common.add_global_checks(
 307                     builder.global_var_dict(),
 308                     ";",
 309                     prefix_list,
 310                     output_lines,
 311                     global_vars_seen_dict,
 312                     args.preserve_names,
 313                     False,
 314                 )
 315             )
 316         if ti.args.gen_unused_prefix_body:
 317             output_lines.extend(
 318                 ti.get_checks_for_unused_prefixes(prefix_list, generated_prefixes)
 319             )
 320         common.debug("Writing %d lines to %s..." % (len(output_lines), ti.path))
 321
 322         with open(ti.path, "wb") as f:
 323             f.writelines(["{}\n".format(l).encode("utf-8") for l in output_lines])
 324
 325
 326 if __name__ == "__main__":
 327     main()