llvm/utils/update_test_checks.py

   1 #!/usr/bin/env python3
   2
   3 """A script to generate FileCheck statements for 'opt' regression tests.
   4
   5 This script is a utility to update LLVM opt test cases with new
   6 FileCheck patterns. It can either update all of the tests in the file or
   7 a single test function.
   8
   9 Example usage:
  10
  11 # Default to using `opt` as found in your PATH.
  12 $ update_test_checks.py test/foo.ll
  13
  14 # Override the path lookup.
  15 $ update_test_checks.py --tool-binary=../bin/opt test/foo.ll
  16
  17 # Use a custom tool instead of `opt`.
  18 $ update_test_checks.py --tool=yourtool test/foo.ll
  19
  20 Workflow:
  21 1. Make a compiler patch that requires updating some number of FileCheck lines
  22    in regression test files.
  23 2. Save the patch and revert it from your local work area.
  24 3. Update the RUN-lines in the affected regression tests to look canonical.
  25    Example: "; RUN: opt < %s -instcombine -S | FileCheck %s"
  26 4. Refresh the FileCheck lines for either the entire file or select functions by
  27    running this script.
  28 5. Commit the fresh baseline of checks.
  29 6. Apply your patch from step 1 and rebuild your local binaries.
  30 7. Re-run this script on affected regression tests.
  31 8. Check the diffs to ensure the script has done something reasonable.
  32 9. Submit a patch including the regression test diffs for review.
  33 """
  34
  35 from __future__ import print_function
  36
  37 import argparse
  38 import os  # Used to advertise this file's name ("autogenerated_note").
  39 import re
  40 import sys
  41
  42 from UpdateTestChecks import common
  43
  44
  45 def main():
  46     from argparse import RawTextHelpFormatter
  47
  48     parser = argparse.ArgumentParser(
  49         description=__doc__, formatter_class=RawTextHelpFormatter
  50     )
  51     parser.add_argument(
  52         "--tool",
  53         default="opt",
  54         help='The name of the tool used to generate the test case (defaults to "opt")',
  55     )
  56     parser.add_argument(
  57         "--tool-binary",
  58         "--opt-binary",
  59         help="The tool binary used to generate the test case",
  60     )
  61     parser.add_argument("--function", help="The function in the test file to update")
  62     parser.add_argument(
  63         "-p", "--preserve-names", action="store_true", help="Do not scrub IR names"
  64     )
  65     parser.add_argument(
  66         "--function-signature",
  67         action="store_true",
  68         help="Keep function signature information around for the check line",
  69     )
  70     parser.add_argument(
  71         "--scrub-attributes",
  72         action="store_true",
  73         help="Remove attribute annotations (#0) from the end of check line",
  74     )
  75     parser.add_argument(
  76         "--check-attributes",
  77         action="store_true",
  78         help='Check "Function Attributes" for functions',
  79     )
  80     parser.add_argument(
  81         "--check-globals",
  82         nargs="?",
  83         const="all",
  84         default="default",
  85         choices=["none", "smart", "all"],
  86         help="Check global entries (global variables, metadata, attribute sets, ...) for functions",
  87     )
  88     parser.add_argument("tests", nargs="+")
  89     initial_args = common.parse_commandline_args(parser)
  90
  91     script_name = os.path.basename(__file__)
  92
  93     if initial_args.tool_binary:
  94         tool_basename = os.path.basename(initial_args.tool_binary)
  95         if not re.match(r"^%s(-\d+)?(\.exe)?$" % (initial_args.tool), tool_basename):
  96             common.error("Unexpected tool name: " + tool_basename)
  97             sys.exit(1)
  98
  99     for ti in common.itertests(
 100         initial_args.tests, parser, script_name="utils/" + script_name
 101     ):
 102         # If requested we scrub trailing attribute annotations, e.g., '#0', together with whitespaces
 103         if ti.args.scrub_attributes:
 104             common.SCRUB_TRAILING_WHITESPACE_TEST_RE = (
 105                 common.SCRUB_TRAILING_WHITESPACE_AND_ATTRIBUTES_RE
 106             )
 107         else:
 108             common.SCRUB_TRAILING_WHITESPACE_TEST_RE = (
 109                 common.SCRUB_TRAILING_WHITESPACE_RE
 110             )
 111
 112         tool_basename = ti.args.tool
 113
 114         prefix_list = []
 115         for l in ti.run_lines:
 116             if "|" not in l:
 117                 common.warn("Skipping unparsable RUN line: " + l)
 118                 continue
 119
 120             commands = [cmd.strip() for cmd in l.split("|")]
 121             assert len(commands) >= 2
 122             preprocess_cmd = None
 123             if len(commands) > 2:
 124                 preprocess_cmd = " | ".join(commands[:-2])
 125             tool_cmd = commands[-2]
 126             filecheck_cmd = commands[-1]
 127             common.verify_filecheck_prefixes(filecheck_cmd)
 128             if not tool_cmd.startswith(tool_basename + " "):
 129                 common.warn("Skipping non-%s RUN line: %s" % (tool_basename, l))
 130                 continue
 131
 132             if not filecheck_cmd.startswith("FileCheck "):
 133                 common.warn("Skipping non-FileChecked RUN line: " + l)
 134                 continue
 135
 136             tool_cmd_args = tool_cmd[len(tool_basename) :].strip()
 137             tool_cmd_args = tool_cmd_args.replace("< %s", "").replace("%s", "").strip()
 138             check_prefixes = common.get_check_prefixes(filecheck_cmd)
 139
 140             # FIXME: We should use multiple check prefixes to common check lines. For
 141             # now, we just ignore all but the last.
 142             prefix_list.append((check_prefixes, tool_cmd_args, preprocess_cmd))
 143
 144         global_vars_seen_dict = {}
 145         builder = common.FunctionTestBuilder(
 146             run_list=prefix_list, flags=ti.args, scrubber_args=[], path=ti.path
 147         )
 148
 149         tool_binary = ti.args.tool_binary
 150         if not tool_binary:
 151             tool_binary = tool_basename
 152
 153         for prefixes, tool_args, preprocess_cmd in prefix_list:
 154             common.debug("Extracted tool cmd: " + tool_basename + " " + tool_args)
 155             common.debug("Extracted FileCheck prefixes: " + str(prefixes))
 156
 157             raw_tool_output = common.invoke_tool(
 158                 tool_binary,
 159                 tool_args,
 160                 ti.path,
 161                 preprocess_cmd=preprocess_cmd,
 162                 verbose=ti.args.verbose,
 163             )
 164             builder.process_run_line(
 165                 common.OPT_FUNCTION_RE,
 166                 common.scrub_body,
 167                 raw_tool_output,
 168                 prefixes,
 169                 False,
 170             )
 171             builder.processed_prefixes(prefixes)
 172
 173         func_dict = builder.finish_and_get_func_dict()
 174         is_in_function = False
 175         is_in_function_start = False
 176         has_checked_pre_function_globals = False
 177         prefix_set = set(
 178             [prefix for prefixes, _, _ in prefix_list for prefix in prefixes]
 179         )
 180         common.debug("Rewriting FileCheck prefixes:", str(prefix_set))
 181         output_lines = []
 182
 183         include_generated_funcs = common.find_arg_in_test(
 184             ti,
 185             lambda args: ti.args.include_generated_funcs,
 186             "--include-generated-funcs",
 187             True,
 188         )
 189         generated_prefixes = []
 190         if include_generated_funcs:
 191             # Generate the appropriate checks for each function.  We need to emit
 192             # these in the order according to the generated output so that CHECK-LABEL
 193             # works properly.  func_order provides that.
 194
 195             # We can't predict where various passes might insert functions so we can't
 196             # be sure the input function order is maintained.  Therefore, first spit
 197             # out all the source lines.
 198             common.dump_input_lines(output_lines, ti, prefix_set, ";")
 199
 200             args = ti.args
 201             if args.check_globals != 'none':
 202                 generated_prefixes.extend(
 203                     common.add_global_checks(
 204                         builder.global_var_dict(),
 205                         ";",
 206                         prefix_list,
 207                         output_lines,
 208                         global_vars_seen_dict,
 209                         args.preserve_names,
 210                         True,
 211                         args.check_globals,
 212                     )
 213                 )
 214
 215             # Now generate all the checks.
 216             generated_prefixes.extend(
 217                 common.add_checks_at_end(
 218                     output_lines,
 219                     prefix_list,
 220                     builder.func_order(),
 221                     ";",
 222                     lambda my_output_lines, prefixes, func: common.add_ir_checks(
 223                         my_output_lines,
 224                         ";",
 225                         prefixes,
 226                         func_dict,
 227                         func,
 228                         False,
 229                         args.function_signature,
 230                         args.version,
 231                         global_vars_seen_dict,
 232                         is_filtered=builder.is_filtered(),
 233                     ),
 234                 )
 235             )
 236         else:
 237             # "Normal" mode.
 238             for input_line_info in ti.iterlines(output_lines):
 239                 input_line = input_line_info.line
 240                 args = input_line_info.args
 241                 if is_in_function_start:
 242                     if input_line == "":
 243                         continue
 244                     if input_line.lstrip().startswith(";"):
 245                         m = common.CHECK_RE.match(input_line)
 246                         if not m or m.group(1) not in prefix_set:
 247                             output_lines.append(input_line)
 248                             continue
 249
 250                     # Print out the various check lines here.
 251                     generated_prefixes.extend(
 252                         common.add_ir_checks(
 253                             output_lines,
 254                             ";",
 255                             prefix_list,
 256                             func_dict,
 257                             func_name,
 258                             args.preserve_names,
 259                             args.function_signature,
 260                             args.version,
 261                             global_vars_seen_dict,
 262                             is_filtered=builder.is_filtered(),
 263                         )
 264                     )
 265                     is_in_function_start = False
 266
 267                 m = common.IR_FUNCTION_RE.match(input_line)
 268                 if m and not has_checked_pre_function_globals:
 269                     if args.check_globals:
 270                         generated_prefixes.extend(
 271                             common.add_global_checks(
 272                                 builder.global_var_dict(),
 273                                 ";",
 274                                 prefix_list,
 275                                 output_lines,
 276                                 global_vars_seen_dict,
 277                                 args.preserve_names,
 278                                 True,
 279                                 args.check_globals,
 280                             )
 281                         )
 282                     has_checked_pre_function_globals = True
 283
 284                 if common.should_add_line_to_output(
 285                     input_line, prefix_set, not is_in_function
 286                 ):
 287                     # This input line of the function body will go as-is into the output.
 288                     # Except make leading whitespace uniform: 2 spaces.
 289                     input_line = common.SCRUB_LEADING_WHITESPACE_RE.sub(
 290                         r"  ", input_line
 291                     )
 292                     output_lines.append(input_line)
 293                     if input_line.strip() == "}":
 294                         is_in_function = False
 295                         continue
 296
 297                 if is_in_function:
 298                     continue
 299
 300                 m = common.IR_FUNCTION_RE.match(input_line)
 301                 if not m:
 302                     continue
 303                 func_name = m.group(1)
 304                 if args.function is not None and func_name != args.function:
 305                     # When filtering on a specific function, skip all others.
 306                     continue
 307                 is_in_function = is_in_function_start = True
 308
 309         if args.check_globals != 'none':
 310             generated_prefixes.extend(
 311                 common.add_global_checks(
 312                     builder.global_var_dict(),
 313                     ";",
 314                     prefix_list,
 315                     output_lines,
 316                     global_vars_seen_dict,
 317                     args.preserve_names,
 318                     False,
 319                     args.check_globals,
 320                 )
 321             )
 322         if ti.args.gen_unused_prefix_body:
 323             output_lines.extend(
 324                 ti.get_checks_for_unused_prefixes(prefix_list, generated_prefixes)
 325             )
 326         common.debug("Writing %d lines to %s..." % (len(output_lines), ti.path))
 327
 328         with open(ti.path, "wb") as f:
 329             f.writelines(["{}\n".format(l).encode("utf-8") for l in output_lines])
 330
 331
 332 if __name__ == "__main__":
 333     main()