llvm/utils/update_test_checks.py

   1 #!/usr/bin/env python3
   2
   3 """A script to generate FileCheck statements for 'opt' regression tests.
   4
   5 This script is a utility to update LLVM opt test cases with new
   6 FileCheck patterns. It can either update all of the tests in the file or
   7 a single test function.
   8
   9 Example usage:
  10
  11 # Default to using `opt` as found in your PATH.
  12 $ update_test_checks.py test/foo.ll
  13
  14 # Override the path lookup.
  15 $ update_test_checks.py --tool-binary=../bin/opt test/foo.ll
  16
  17 # Use a custom tool instead of `opt`.
  18 $ update_test_checks.py --tool=yourtool test/foo.ll
  19
  20 Workflow:
  21 1. Make a compiler patch that requires updating some number of FileCheck lines
  22    in regression test files.
  23 2. Save the patch and revert it from your local work area.
  24 3. Update the RUN-lines in the affected regression tests to look canonical.
  25    Example: "; RUN: opt < %s -instcombine -S | FileCheck %s"
  26 4. Refresh the FileCheck lines for either the entire file or select functions by
  27    running this script.
  28 5. Commit the fresh baseline of checks.
  29 6. Apply your patch from step 1 and rebuild your local binaries.
  30 7. Re-run this script on affected regression tests.
  31 8. Check the diffs to ensure the script has done something reasonable.
  32 9. Submit a patch including the regression test diffs for review.
  33 """
  34
  35 from __future__ import print_function
  36
  37 import argparse
  38 import os  # Used to advertise this file's name ("autogenerated_note").
  39 import re
  40 import sys
  41
  42 from UpdateTestChecks import common
  43
  44
  45 def main():
  46     from argparse import RawTextHelpFormatter
  47
  48     parser = argparse.ArgumentParser(
  49         description=__doc__, formatter_class=RawTextHelpFormatter
  50     )
  51     parser.add_argument(
  52         "--tool",
  53         default="opt",
  54         help='The name of the tool used to generate the test case (defaults to "opt")',
  55     )
  56     parser.add_argument(
  57         "--tool-binary",
  58         "--opt-binary",
  59         help="The tool binary used to generate the test case",
  60     )
  61     parser.add_argument("--function", help="The function in the test file to update")
  62     parser.add_argument(
  63         "-p", "--preserve-names", action="store_true", help="Do not scrub IR names"
  64     )
  65     parser.add_argument(
  66         "--function-signature",
  67         action="store_true",
  68         help="Keep function signature information around for the check line",
  69     )
  70     parser.add_argument(
  71         "--scrub-attributes",
  72         action="store_true",
  73         help="Remove attribute annotations (#0) from the end of check line",
  74     )
  75     parser.add_argument(
  76         "--check-attributes",
  77         action="store_true",
  78         help='Check "Function Attributes" for functions',
  79     )
  80     parser.add_argument(
  81         "--check-globals",
  82         nargs="?",
  83         const="all",
  84         default="default",
  85         choices=["none", "smart", "all"],
  86         help="Check global entries (global variables, metadata, attribute sets, ...) for functions",
  87     )
  88     parser.add_argument(
  89         "--reset-variable-names",
  90         action="store_true",
  91         help="Reset all variable names to correspond closely to the variable names in IR. "
  92         "This tends to result in larger diffs.",
  93     )
  94     parser.add_argument("tests", nargs="+")
  95     initial_args = common.parse_commandline_args(parser)
  96
  97     script_name = os.path.basename(__file__)
  98
  99     if initial_args.tool_binary:
 100         tool_basename = os.path.basename(initial_args.tool_binary)
 101         if not re.match(r"^%s(-\d+)?(\.exe)?$" % (initial_args.tool), tool_basename):
 102             common.error("Unexpected tool name: " + tool_basename)
 103             sys.exit(1)
 104
 105     for ti in common.itertests(
 106         initial_args.tests, parser, script_name="utils/" + script_name
 107     ):
 108         # If requested we scrub trailing attribute annotations, e.g., '#0', together with whitespaces
 109         if ti.args.scrub_attributes:
 110             common.SCRUB_TRAILING_WHITESPACE_TEST_RE = (
 111                 common.SCRUB_TRAILING_WHITESPACE_AND_ATTRIBUTES_RE
 112             )
 113         else:
 114             common.SCRUB_TRAILING_WHITESPACE_TEST_RE = (
 115                 common.SCRUB_TRAILING_WHITESPACE_RE
 116             )
 117
 118         tool_basename = ti.args.tool
 119
 120         prefix_list = []
 121         for l in ti.run_lines:
 122             if "|" not in l:
 123                 common.warn("Skipping unparsable RUN line: " + l)
 124                 continue
 125
 126             cropped_content = l
 127             if "%if" in l:
 128                 match = re.search(r"%{\s*(.*?)\s*%}", l)
 129                 if match:
 130                     cropped_content = match.group(1)
 131
 132             commands = [cmd.strip() for cmd in cropped_content.split("|")]
 133             assert len(commands) >= 2
 134             preprocess_cmd = None
 135             if len(commands) > 2:
 136                 preprocess_cmd = " | ".join(commands[:-2])
 137             tool_cmd = commands[-2]
 138             filecheck_cmd = commands[-1]
 139             common.verify_filecheck_prefixes(filecheck_cmd)
 140             if not tool_cmd.startswith(tool_basename + " "):
 141                 common.warn("Skipping non-%s RUN line: %s" % (tool_basename, l))
 142                 continue
 143
 144             if not filecheck_cmd.startswith("FileCheck "):
 145                 common.warn("Skipping non-FileChecked RUN line: " + l)
 146                 continue
 147
 148             tool_cmd_args = tool_cmd[len(tool_basename) :].strip()
 149             tool_cmd_args = tool_cmd_args.replace("< %s", "").replace("%s", "").strip()
 150             check_prefixes = common.get_check_prefixes(filecheck_cmd)
 151
 152             # FIXME: We should use multiple check prefixes to common check lines. For
 153             # now, we just ignore all but the last.
 154             prefix_list.append((check_prefixes, tool_cmd_args, preprocess_cmd))
 155
 156         ginfo = common.make_ir_generalizer(ti.args.version)
 157         global_vars_seen_dict = {}
 158         builder = common.FunctionTestBuilder(
 159             run_list=prefix_list,
 160             flags=ti.args,
 161             scrubber_args=[],
 162             path=ti.path,
 163             ginfo=ginfo,
 164         )
 165
 166         tool_binary = ti.args.tool_binary
 167         if not tool_binary:
 168             tool_binary = tool_basename
 169
 170         for prefixes, tool_args, preprocess_cmd in prefix_list:
 171             common.debug("Extracted tool cmd: " + tool_basename + " " + tool_args)
 172             common.debug("Extracted FileCheck prefixes: " + str(prefixes))
 173
 174             raw_tool_output = common.invoke_tool(
 175                 tool_binary,
 176                 tool_args,
 177                 ti.path,
 178                 preprocess_cmd=preprocess_cmd,
 179                 verbose=ti.args.verbose,
 180             )
 181             builder.process_run_line(
 182                 common.OPT_FUNCTION_RE,
 183                 common.scrub_body,
 184                 raw_tool_output,
 185                 prefixes,
 186             )
 187             builder.processed_prefixes(prefixes)
 188
 189         prefix_set = set(
 190             [prefix for prefixes, _, _ in prefix_list for prefix in prefixes]
 191         )
 192
 193         if not ti.args.reset_variable_names:
 194             original_check_lines = common.collect_original_check_lines(ti, prefix_set)
 195         else:
 196             original_check_lines = {}
 197
 198         func_dict = builder.finish_and_get_func_dict()
 199         is_in_function = False
 200         is_in_function_start = False
 201         has_checked_pre_function_globals = False
 202         common.debug("Rewriting FileCheck prefixes:", str(prefix_set))
 203         output_lines = []
 204
 205         include_generated_funcs = common.find_arg_in_test(
 206             ti,
 207             lambda args: ti.args.include_generated_funcs,
 208             "--include-generated-funcs",
 209             True,
 210         )
 211         generated_prefixes = []
 212         if include_generated_funcs:
 213             # Generate the appropriate checks for each function.  We need to emit
 214             # these in the order according to the generated output so that CHECK-LABEL
 215             # works properly.  func_order provides that.
 216
 217             # We can't predict where various passes might insert functions so we can't
 218             # be sure the input function order is maintained.  Therefore, first spit
 219             # out all the source lines.
 220             common.dump_input_lines(output_lines, ti, prefix_set, ";")
 221
 222             args = ti.args
 223             if args.check_globals != 'none':
 224                 generated_prefixes.extend(
 225                     common.add_global_checks(
 226                         builder.global_var_dict(),
 227                         ";",
 228                         prefix_list,
 229                         output_lines,
 230                         ginfo,
 231                         global_vars_seen_dict,
 232                         args.preserve_names,
 233                         True,
 234                         args.check_globals,
 235                     )
 236                 )
 237
 238             # Now generate all the checks.
 239             generated_prefixes.extend(
 240                 common.add_checks_at_end(
 241                     output_lines,
 242                     prefix_list,
 243                     builder.func_order(),
 244                     ";",
 245                     lambda my_output_lines, prefixes, func: common.add_ir_checks(
 246                         my_output_lines,
 247                         ";",
 248                         prefixes,
 249                         func_dict,
 250                         func,
 251                         False,
 252                         args.function_signature,
 253                         ginfo,
 254                         global_vars_seen_dict,
 255                         is_filtered=builder.is_filtered(),
 256                         original_check_lines=original_check_lines.get(func, {}),
 257                     ),
 258                 )
 259             )
 260         else:
 261             # "Normal" mode.
 262             dropped_previous_line = False
 263             for input_line_info in ti.iterlines(output_lines):
 264                 input_line = input_line_info.line
 265                 args = input_line_info.args
 266                 if is_in_function_start:
 267                     if input_line == "":
 268                         continue
 269                     if input_line.lstrip().startswith(";"):
 270                         m = common.CHECK_RE.match(input_line)
 271                         if not m or m.group(1) not in prefix_set:
 272                             output_lines.append(input_line)
 273                             continue
 274
 275                     # Print out the various check lines here.
 276                     generated_prefixes.extend(
 277                         common.add_ir_checks(
 278                             output_lines,
 279                             ";",
 280                             prefix_list,
 281                             func_dict,
 282                             func_name,
 283                             args.preserve_names,
 284                             args.function_signature,
 285                             ginfo,
 286                             global_vars_seen_dict,
 287                             is_filtered=builder.is_filtered(),
 288                             original_check_lines=original_check_lines.get(
 289                                 func_name, {}
 290                             ),
 291                         )
 292                     )
 293                     is_in_function_start = False
 294
 295                 m = common.IR_FUNCTION_RE.match(input_line)
 296                 if m and not has_checked_pre_function_globals:
 297                     if args.check_globals:
 298                         generated_prefixes.extend(
 299                             common.add_global_checks(
 300                                 builder.global_var_dict(),
 301                                 ";",
 302                                 prefix_list,
 303                                 output_lines,
 304                                 ginfo,
 305                                 global_vars_seen_dict,
 306                                 args.preserve_names,
 307                                 True,
 308                                 args.check_globals,
 309                             )
 310                         )
 311                     has_checked_pre_function_globals = True
 312
 313                 if common.should_add_line_to_output(
 314                     input_line,
 315                     prefix_set,
 316                     skip_global_checks=not is_in_function,
 317                     skip_same_checks=dropped_previous_line,
 318                 ):
 319                     # This input line of the function body will go as-is into the output.
 320                     # Except make leading whitespace uniform: 2 spaces.
 321                     input_line = common.SCRUB_LEADING_WHITESPACE_RE.sub(
 322                         r"  ", input_line
 323                     )
 324                     output_lines.append(input_line)
 325                     dropped_previous_line = False
 326                     if input_line.strip() == "}":
 327                         is_in_function = False
 328                         continue
 329                 else:
 330                     # If we are removing a check line, and the next line is CHECK-SAME, it MUST also be removed
 331                     dropped_previous_line = True
 332
 333                 if is_in_function:
 334                     continue
 335
 336                 m = common.IR_FUNCTION_RE.match(input_line)
 337                 if not m:
 338                     continue
 339                 func_name = m.group(1)
 340                 if args.function is not None and func_name != args.function:
 341                     # When filtering on a specific function, skip all others.
 342                     continue
 343                 is_in_function = is_in_function_start = True
 344
 345         if args.check_globals != 'none':
 346             generated_prefixes.extend(
 347                 common.add_global_checks(
 348                     builder.global_var_dict(),
 349                     ";",
 350                     prefix_list,
 351                     output_lines,
 352                     ginfo,
 353                     global_vars_seen_dict,
 354                     args.preserve_names,
 355                     False,
 356                     args.check_globals,
 357                 )
 358             )
 359         if ti.args.gen_unused_prefix_body:
 360             output_lines.extend(
 361                 ti.get_checks_for_unused_prefixes(prefix_list, generated_prefixes)
 362             )
 363         common.debug("Writing %d lines to %s..." % (len(output_lines), ti.path))
 364
 365         with open(ti.path, "wb") as f:
 366             f.writelines(["{}\n".format(l).encode("utf-8") for l in output_lines])
 367
 368
 369 if __name__ == "__main__":
 370     main()