llvm/utils/rsp_bisect.py

   1 #!/usr/bin/env python3
   2 # ===----------------------------------------------------------------------===##
   3 #
   4 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   5 # See https://llvm.org/LICENSE.txt for license information.
   6 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   7 #
   8 # ===----------------------------------------------------------------------===##
   9 """Script to bisect over files in an rsp file.
  10
  11 This is mostly used for detecting which file contains a miscompile between two
  12 compiler revisions. It does this by bisecting over an rsp file. Between two
  13 build directories, this script will make the rsp file reference the current
  14 build directory's version of some set of the rsp's object files/libraries, and
  15 reference the other build directory's version of the same files for the
  16 remaining set of object files/libraries.
  17
  18 Build the target in two separate directories with the two compiler revisions,
  19 keeping the rsp file around since ninja by default deletes the rsp file after
  20 building.
  21 $ ninja -d keeprsp mytarget
  22
  23 Create a script to build the target and run an interesting test. Get the
  24 command to build the target via
  25 $ ninja -t commands | grep mytarget
  26 The command to build the target should reference the rsp file.
  27 This script doesn't care if the test script returns 0 or 1 for specifically the
  28 successful or failing test, just that the test script returns a different
  29 return code for success vs failure.
  30 Since the command that `ninja -t commands` is run from the build directory,
  31 usually the test script cd's to the build directory.
  32
  33 $ rsp_bisect.py --test=path/to/test_script --rsp=path/to/build/target.rsp
  34     --other_rel_path=../Other
  35 where --other_rel_path is the relative path from the first build directory to
  36 the other build directory. This is prepended to files in the rsp.
  37
  38
  39 For a full example, if the foo target is suspected to contain a miscompile in
  40 some file, have two different build directories, buildgood/ and buildbad/ and
  41 run
  42 $ ninja -d keeprsp foo
  43 in both so we have two versions of all relevant object files that may contain a
  44 miscompile, one built by a good compiler and one by a bad compiler.
  45
  46 In buildgood/, run
  47 $ ninja -t commands | grep '-o .*foo'
  48 to get the command to link the files together. It may look something like
  49   clang -o foo @foo.rsp
  50
  51 Now create a test script that runs the link step and whatever test reproduces a
  52 miscompile and returns a non-zero exit code when there is a miscompile. For
  53 example
  54 ```
  55   #!/bin/bash
  56   # immediately bail out of script if any command returns a non-zero return code
  57   set -e
  58   clang -o foo @foo.rsp
  59   ./foo
  60 ```
  61
  62 With buildgood/ as the working directory, run
  63 $ path/to/llvm-project/llvm/utils/rsp_bisect.py \
  64     --test=path/to/test_script --rsp=./foo.rsp --other_rel_path=../buildbad/
  65 If rsp_bisect is successful, it will print the first file in the rsp file that
  66 when using the bad build directory's version causes the test script to return a
  67 different return code. foo.rsp.0 and foo.rsp.1 will also be written. foo.rsp.0
  68 will be a copy of foo.rsp with the relevant file using the version in
  69 buildgood/, and foo.rsp.1 will be a copy of foo.rsp with the relevant file
  70 using the version in buildbad/.
  71
  72 """
  73
  74 import argparse
  75 import os
  76 import subprocess
  77 import sys
  78
  79
  80 def is_path(s):
  81     return "/" in s
  82
  83
  84 def run_test(test):
  85     """Runs the test and returns whether it was successful or not."""
  86     return subprocess.run([test], capture_output=True).returncode == 0
  87
  88
  89 def modify_rsp(rsp_entries, other_rel_path, modify_after_num):
  90     """Create a modified rsp file for use in bisection.
  91
  92     Returns a new list from rsp.
  93     For each file in rsp after the first modify_after_num files, prepend
  94     other_rel_path.
  95     """
  96     ret = []
  97     for r in rsp_entries:
  98         if is_path(r):
  99             if modify_after_num == 0:
 100                 r = os.path.join(other_rel_path, r)
 101             else:
 102                 modify_after_num -= 1
 103         ret.append(r)
 104     assert modify_after_num == 0
 105     return ret
 106
 107
 108 def test_modified_rsp(test, modified_rsp_entries, rsp_path):
 109     """Write the rsp file to disk and run the test."""
 110     with open(rsp_path, "w") as f:
 111         f.write(" ".join(modified_rsp_entries))
 112     return run_test(test)
 113
 114
 115 def bisect(test, zero_result, rsp_entries, num_files_in_rsp, other_rel_path, rsp_path):
 116     """Bisect over rsp entries.
 117
 118     Args:
 119         zero_result: the test result when modify_after_num is 0.
 120
 121     Returns:
 122         The index of the file in the rsp file where the test result changes.
 123     """
 124     lower = 0
 125     upper = num_files_in_rsp
 126     while lower != upper - 1:
 127         assert lower < upper - 1
 128         mid = int((lower + upper) / 2)
 129         assert lower != mid and mid != upper
 130         print("Trying {} ({}-{})".format(mid, lower, upper))
 131         result = test_modified_rsp(
 132             test, modify_rsp(rsp_entries, other_rel_path, mid), rsp_path
 133         )
 134         if zero_result == result:
 135             lower = mid
 136         else:
 137             upper = mid
 138     return upper
 139
 140
 141 def main():
 142     parser = argparse.ArgumentParser()
 143     parser.add_argument(
 144         "--test", help="Binary to test if current setup is good or bad", required=True
 145     )
 146     parser.add_argument("--rsp", help="rsp file", required=True)
 147     parser.add_argument(
 148         "--other-rel-path",
 149         help="Relative path from current build directory to other build "
 150         + 'directory, e.g. from "out/Default" to "out/Other" specify "../Other"',
 151         required=True,
 152     )
 153     args = parser.parse_args()
 154
 155     with open(args.rsp, "r") as f:
 156         rsp_entries = f.read()
 157     rsp_entries = rsp_entries.split()
 158     num_files_in_rsp = sum(1 for a in rsp_entries if is_path(a))
 159     if num_files_in_rsp == 0:
 160         print("No files in rsp?")
 161         return 1
 162     print("{} files in rsp".format(num_files_in_rsp))
 163
 164     try:
 165         print("Initial testing")
 166         test0 = test_modified_rsp(
 167             args.test, modify_rsp(rsp_entries, args.other_rel_path, 0), args.rsp
 168         )
 169         test_all = test_modified_rsp(
 170             args.test,
 171             modify_rsp(rsp_entries, args.other_rel_path, num_files_in_rsp),
 172             args.rsp,
 173         )
 174
 175         if test0 == test_all:
 176             print("Test returned same exit code for both build directories")
 177             return 1
 178
 179         print("First build directory returned " + ("0" if test_all else "1"))
 180
 181         result = bisect(
 182             args.test,
 183             test0,
 184             rsp_entries,
 185             num_files_in_rsp,
 186             args.other_rel_path,
 187             args.rsp,
 188         )
 189         print(
 190             "First file change: {} ({})".format(
 191                 list(filter(is_path, rsp_entries))[result - 1], result
 192             )
 193         )
 194
 195         rsp_out_0 = args.rsp + ".0"
 196         rsp_out_1 = args.rsp + ".1"
 197         with open(rsp_out_0, "w") as f:
 198             f.write(" ".join(modify_rsp(rsp_entries, args.other_rel_path, result - 1)))
 199         with open(rsp_out_1, "w") as f:
 200             f.write(" ".join(modify_rsp(rsp_entries, args.other_rel_path, result)))
 201         print(
 202             "Bisection point rsp files written to {} and {}".format(
 203                 rsp_out_0, rsp_out_1
 204             )
 205         )
 206     finally:
 207         # Always make sure to write the original rsp file contents back so it's
 208         # less of a pain to rerun this script.
 209         with open(args.rsp, "w") as f:
 210             f.write(" ".join(rsp_entries))
 211
 212
 213 if __name__ == "__main__":
 214     sys.exit(main())