llvm/utils/update_analyze_test_checks.py

   1 #!/usr/bin/env python3
   2
   3 """A script to generate FileCheck statements for 'opt' analysis tests.
   4
   5 This script is a utility to update LLVM opt analysis test cases with new
   6 FileCheck patterns. It can either update all of the tests in the file or
   7 a single test function.
   8
   9 Example usage:
  10 $ update_analyze_test_checks.py --opt=../bin/opt test/foo.ll
  11
  12 Workflow:
  13 1. Make a compiler patch that requires updating some number of FileCheck lines
  14    in regression test files.
  15 2. Save the patch and revert it from your local work area.
  16 3. Update the RUN-lines in the affected regression tests to look canonical.
  17    Example: "; RUN: opt < %s -passes='print<cost-model>' -disable-output 2>&1 | FileCheck %s"
  18 4. Refresh the FileCheck lines for either the entire file or select functions by
  19    running this script.
  20 5. Commit the fresh baseline of checks.
  21 6. Apply your patch from step 1 and rebuild your local binaries.
  22 7. Re-run this script on affected regression tests.
  23 8. Check the diffs to ensure the script has done something reasonable.
  24 9. Submit a patch including the regression test diffs for review.
  25
  26 A common pattern is to have the script insert complete checking of every
  27 instruction. Then, edit it down to only check the relevant instructions.
  28 The script is designed to make adding checks to a test case fast, it is *not*
  29 designed to be authoratitive about what constitutes a good test!
  30 """
  31
  32 from __future__ import print_function
  33
  34 import argparse
  35 import os         # Used to advertise this file's name ("autogenerated_note").
  36 import sys
  37 import re
  38
  39 from UpdateTestChecks import common
  40
  41 def main():
  42   from argparse import RawTextHelpFormatter
  43   parser = argparse.ArgumentParser(description=__doc__, formatter_class=RawTextHelpFormatter)
  44   parser.add_argument('--opt-binary', default='opt',
  45                       help='The opt binary used to generate the test case')
  46   parser.add_argument(
  47       '--function', help='The function in the test file to update')
  48   parser.add_argument('tests', nargs='+')
  49   initial_args = common.parse_commandline_args(parser)
  50
  51   script_name = os.path.basename(__file__)
  52
  53   opt_basename = os.path.basename(initial_args.opt_binary)
  54   if (opt_basename != "opt"):
  55     common.error('Unexpected opt name: ' + opt_basename)
  56     sys.exit(1)
  57
  58   for ti in common.itertests(initial_args.tests, parser,
  59                              script_name='utils/' + script_name):
  60     triple_in_ir = None
  61     for l in ti.input_lines:
  62       m = common.TRIPLE_IR_RE.match(l)
  63       if m:
  64         triple_in_ir = m.groups()[0]
  65         break
  66
  67     prefix_list = []
  68     for l in ti.run_lines:
  69       if '|' not in l:
  70         common.warn('Skipping unparsable RUN line: ' + l)
  71         continue
  72
  73       (tool_cmd, filecheck_cmd) = tuple([cmd.strip() for cmd in l.split('|', 1)])
  74       common.verify_filecheck_prefixes(filecheck_cmd)
  75
  76       if not tool_cmd.startswith(opt_basename + ' '):
  77         common.warn('WSkipping non-%s RUN line: %s' % (opt_basename, l))
  78         continue
  79
  80       if not filecheck_cmd.startswith('FileCheck '):
  81         common.warn('Skipping non-FileChecked RUN line: ' + l)
  82         continue
  83
  84       tool_cmd_args = tool_cmd[len(opt_basename):].strip()
  85       tool_cmd_args = tool_cmd_args.replace('< %s', '').replace('%s', '').strip()
  86       check_prefixes = common.get_check_prefixes(filecheck_cmd)
  87
  88       # FIXME: We should use multiple check prefixes to common check lines. For
  89       # now, we just ignore all but the last.
  90       prefix_list.append((check_prefixes, tool_cmd_args))
  91
  92     builder = common.FunctionTestBuilder(
  93       run_list = prefix_list,
  94       flags = type('', (object,), {
  95             'verbose': ti.args.verbose,
  96             'filters': ti.args.filters,
  97             'function_signature': False,
  98             'check_attributes': False,
  99             'replace_value_regex': []}),
 100       scrubber_args = [],
 101       path=ti.path)
 102
 103     for prefixes, opt_args in prefix_list:
 104       common.debug('Extracted opt cmd:', opt_basename, opt_args, file=sys.stderr)
 105       common.debug('Extracted FileCheck prefixes:', str(prefixes), file=sys.stderr)
 106
 107       raw_tool_outputs = common.invoke_tool(ti.args.opt_binary, opt_args, ti.path)
 108
 109       if re.search(r'Printing analysis ', raw_tool_outputs) is not None:
 110         # Split analysis outputs by "Printing analysis " declarations.
 111         for raw_tool_output in re.split(r'Printing analysis ', raw_tool_outputs):
 112           builder.process_run_line(common.ANALYZE_FUNCTION_RE, common.scrub_body,
 113                                   raw_tool_output, prefixes, False)
 114       elif re.search(r'LV: Checking a loop in ', raw_tool_outputs) is not None:
 115         # Split analysis outputs by "Printing analysis " declarations.
 116         for raw_tool_output in re.split(r'LV: Checking a loop in ', raw_tool_outputs):
 117           builder.process_run_line(common.LV_DEBUG_RE, common.scrub_body,
 118                                   raw_tool_output, prefixes, False)
 119       else:
 120         common.warn('Don\'t know how to deal with this output')
 121         continue
 122
 123       builder.processed_prefixes(prefixes)
 124
 125     func_dict = builder.finish_and_get_func_dict()
 126     is_in_function = False
 127     is_in_function_start = False
 128     prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
 129     common.debug('Rewriting FileCheck prefixes:', str(prefix_set), file=sys.stderr)
 130     output_lines = []
 131
 132     generated_prefixes = []
 133     for input_info in ti.iterlines(output_lines):
 134       input_line = input_info.line
 135       args = input_info.args
 136       if is_in_function_start:
 137         if input_line == '':
 138           continue
 139         if input_line.lstrip().startswith(';'):
 140           m = common.CHECK_RE.match(input_line)
 141           if not m or m.group(1) not in prefix_set:
 142             output_lines.append(input_line)
 143             continue
 144
 145         # Print out the various check lines here.
 146         generated_prefixes.extend(
 147             common.add_analyze_checks(
 148                 output_lines,
 149                 ';',
 150                 prefix_list,
 151                 func_dict,
 152                 func_name,
 153                 is_filtered=builder.is_filtered()))
 154         is_in_function_start = False
 155
 156       if is_in_function:
 157         if common.should_add_line_to_output(input_line, prefix_set):
 158           # This input line of the function body will go as-is into the output.
 159           # Except make leading whitespace uniform: 2 spaces.
 160           input_line = common.SCRUB_LEADING_WHITESPACE_RE.sub(r'  ', input_line)
 161           output_lines.append(input_line)
 162         else:
 163           continue
 164         if input_line.strip() == '}':
 165           is_in_function = False
 166         continue
 167
 168       # If it's outside a function, it just gets copied to the output.
 169       output_lines.append(input_line)
 170
 171       m = common.IR_FUNCTION_RE.match(input_line)
 172       if not m:
 173         continue
 174       func_name = m.group(1)
 175       if ti.args.function is not None and func_name != ti.args.function:
 176         # When filtering on a specific function, skip all others.
 177         continue
 178       is_in_function = is_in_function_start = True
 179
 180     if ti.args.gen_unused_prefix_body:
 181       output_lines.extend(
 182           ti.get_checks_for_unused_prefixes(prefix_list, generated_prefixes))
 183
 184     common.debug('Writing %d lines to %s...' % (len(output_lines), ti.path))
 185
 186     with open(ti.path, 'wb') as f:
 187       f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
 188
 189
 190 if __name__ == '__main__':
 191   main()