llvm/utils/rsp_bisect.py

   1 #!/usr/bin/env python3
   2 #===----------------------------------------------------------------------===##
   3 #
   4 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   5 # See https://llvm.org/LICENSE.txt for license information.
   6 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   7 #
   8 #===----------------------------------------------------------------------===##
   9 """Script to bisect over files in an rsp file.
  10
  11 This is mostly used for detecting which file contains a miscompile between two
  12 compiler revisions. It does this by bisecting over an rsp file. Between two
  13 build directories, this script will make the rsp file reference the current
  14 build directory's version of some set of the rsp's object files/libraries, and
  15 reference the other build directory's version of the same files for the
  16 remaining set of object files/libraries.
  17
  18 Build the target in two separate directories with the two compiler revisions,
  19 keeping the rsp file around since ninja by default deletes the rsp file after
  20 building.
  21 $ ninja -d keeprsp mytarget
  22
  23 Create a script to build the target and run an interesting test. Get the
  24 command to build the target via
  25 $ ninja -t commands | grep mytarget
  26 The command to build the target should reference the rsp file.
  27 This script doesn't care if the test script returns 0 or 1 for specifically the
  28 successful or failing test, just that the test script returns a different
  29 return code for success vs failure.
  30 Since the command that `ninja -t commands` is run from the build directory,
  31 usually the test script cd's to the build directory.
  32
  33 $ rsp_bisect.py --test=path/to/test_script --rsp=path/to/build/target.rsp
  34     --other_rel_path=../Other
  35 where --other_rel_path is the relative path from the first build directory to
  36 the other build directory. This is prepended to files in the rsp.
  37
  38
  39 For a full example, if the foo target is suspected to contain a miscompile in
  40 some file, have two different build directories, buildgood/ and buildbad/ and
  41 run
  42 $ ninja -d keeprsp foo
  43 in both so we have two versions of all relevant object files that may contain a
  44 miscompile, one built by a good compiler and one by a bad compiler.
  45
  46 In buildgood/, run
  47 $ ninja -t commands | grep '-o .*foo'
  48 to get the command to link the files together. It may look something like
  49   clang -o foo @foo.rsp
  50
  51 Now create a test script that runs the link step and whatever test reproduces a
  52 miscompile and returns a non-zero exit code when there is a miscompile. For
  53 example
  54 ```
  55   #!/bin/bash
  56   # immediately bail out of script if any command returns a non-zero return code
  57   set -e
  58   clang -o foo @foo.rsp
  59   ./foo
  60 ```
  61
  62 With buildgood/ as the working directory, run
  63 $ path/to/llvm-project/llvm/utils/rsp_bisect.py \
  64     --test=path/to/test_script --rsp=./foo.rsp --other_rel_path=../buildbad/
  65 If rsp_bisect is successful, it will print the first file in the rsp file that
  66 when using the bad build directory's version causes the test script to return a
  67 different return code. foo.rsp.0 and foo.rsp.1 will also be written. foo.rsp.0
  68 will be a copy of foo.rsp with the relevant file using the version in
  69 buildgood/, and foo.rsp.1 will be a copy of foo.rsp with the relevant file
  70 using the version in buildbad/.
  71
  72 """
  73
  74 import argparse
  75 import os
  76 import subprocess
  77 import sys
  78
  79
  80 def is_path(s):
  81   return '/' in s
  82
  83
  84 def run_test(test):
  85   """Runs the test and returns whether it was successful or not."""
  86   return subprocess.run([test], capture_output=True).returncode == 0
  87
  88
  89 def modify_rsp(rsp_entries, other_rel_path, modify_after_num):
  90   """Create a modified rsp file for use in bisection.
  91
  92   Returns a new list from rsp.
  93   For each file in rsp after the first modify_after_num files, prepend
  94   other_rel_path.
  95   """
  96   ret = []
  97   for r in rsp_entries:
  98     if is_path(r):
  99       if modify_after_num == 0:
 100         r = os.path.join(other_rel_path, r)
 101       else:
 102         modify_after_num -= 1
 103     ret.append(r)
 104   assert modify_after_num == 0
 105   return ret
 106
 107
 108 def test_modified_rsp(test, modified_rsp_entries, rsp_path):
 109   """Write the rsp file to disk and run the test."""
 110   with open(rsp_path, 'w') as f:
 111     f.write(' '.join(modified_rsp_entries))
 112   return run_test(test)
 113
 114
 115 def bisect(test, zero_result, rsp_entries, num_files_in_rsp, other_rel_path, rsp_path):
 116   """Bisect over rsp entries.
 117
 118   Args:
 119       zero_result: the test result when modify_after_num is 0.
 120
 121   Returns:
 122       The index of the file in the rsp file where the test result changes.
 123   """
 124   lower = 0
 125   upper = num_files_in_rsp
 126   while lower != upper - 1:
 127     assert lower < upper - 1
 128     mid = int((lower + upper) / 2)
 129     assert lower != mid and mid != upper
 130     print('Trying {} ({}-{})'.format(mid, lower, upper))
 131     result = test_modified_rsp(test, modify_rsp(rsp_entries, other_rel_path, mid),
 132                                rsp_path)
 133     if zero_result == result:
 134       lower = mid
 135     else:
 136       upper = mid
 137   return upper
 138
 139
 140 def main():
 141   parser = argparse.ArgumentParser()
 142   parser.add_argument('--test',
 143                       help='Binary to test if current setup is good or bad',
 144                       required=True)
 145   parser.add_argument('--rsp', help='rsp file', required=True)
 146   parser.add_argument(
 147       '--other-rel-path',
 148       help='Relative path from current build directory to other build ' +
 149       'directory, e.g. from "out/Default" to "out/Other" specify "../Other"',
 150       required=True)
 151   args = parser.parse_args()
 152
 153   with open(args.rsp, 'r') as f:
 154     rsp_entries = f.read()
 155   rsp_entries = rsp_entries.split()
 156   num_files_in_rsp = sum(1 for a in rsp_entries if is_path(a))
 157   if num_files_in_rsp == 0:
 158     print('No files in rsp?')
 159     return 1
 160   print('{} files in rsp'.format(num_files_in_rsp))
 161
 162   try:
 163     print('Initial testing')
 164     test0 = test_modified_rsp(args.test, modify_rsp(rsp_entries, args.other_rel_path,
 165                                                     0), args.rsp)
 166     test_all = test_modified_rsp(
 167         args.test, modify_rsp(rsp_entries, args.other_rel_path, num_files_in_rsp),
 168         args.rsp)
 169
 170     if test0 == test_all:
 171       print('Test returned same exit code for both build directories')
 172       return 1
 173
 174     print('First build directory returned ' + ('0' if test_all else '1'))
 175
 176     result = bisect(args.test, test0, rsp_entries, num_files_in_rsp,
 177                     args.other_rel_path, args.rsp)
 178     print('First file change: {} ({})'.format(
 179         list(filter(is_path, rsp_entries))[result - 1], result))
 180
 181     rsp_out_0 = args.rsp + '.0'
 182     rsp_out_1 = args.rsp + '.1'
 183     with open(rsp_out_0, 'w') as f:
 184       f.write(' '.join(modify_rsp(rsp_entries, args.other_rel_path, result - 1)))
 185     with open(rsp_out_1, 'w') as f:
 186       f.write(' '.join(modify_rsp(rsp_entries, args.other_rel_path, result)))
 187     print('Bisection point rsp files written to {} and {}'.format(
 188         rsp_out_0, rsp_out_1))
 189   finally:
 190     # Always make sure to write the original rsp file contents back so it's
 191     # less of a pain to rerun this script.
 192     with open(args.rsp, 'w') as f:
 193       f.write(' '.join(rsp_entries))
 194
 195
 196 if __name__ == '__main__':
 197   sys.exit(main())