llvm-project

1
#!/usr/bin/env python3
2
# ===----------------------------------------------------------------------===##
3
#
4
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5
# See https://llvm.org/LICENSE.txt for license information.
6
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7
#
8
# ===----------------------------------------------------------------------===##
9
"""Script to bisect over files in an rsp file.
10

11
This is mostly used for detecting which file contains a miscompile between two
12
compiler revisions. It does this by bisecting over an rsp file. Between two
13
build directories, this script will make the rsp file reference the current
14
build directory's version of some set of the rsp's object files/libraries, and
15
reference the other build directory's version of the same files for the
16
remaining set of object files/libraries.
17

18
Build the target in two separate directories with the two compiler revisions,
19
keeping the rsp file around since ninja by default deletes the rsp file after
20
building.
21
$ ninja -d keeprsp mytarget
22

23
Create a script to build the target and run an interesting test. Get the
24
command to build the target via
25
$ ninja -t commands | grep mytarget
26
The command to build the target should reference the rsp file.
27
This script doesn't care if the test script returns 0 or 1 for specifically the
28
successful or failing test, just that the test script returns a different
29
return code for success vs failure.
30
Since the command that `ninja -t commands` is run from the build directory,
31
usually the test script cd's to the build directory.
32

33
$ rsp_bisect.py --test=path/to/test_script --rsp=path/to/build/target.rsp
34
    --other_rel_path=../Other
35
where --other_rel_path is the relative path from the first build directory to
36
the other build directory. This is prepended to files in the rsp.
37

38

39
For a full example, if the foo target is suspected to contain a miscompile in
40
some file, have two different build directories, buildgood/ and buildbad/ and
41
run
42
$ ninja -d keeprsp foo
43
in both so we have two versions of all relevant object files that may contain a
44
miscompile, one built by a good compiler and one by a bad compiler.
45

46
In buildgood/, run
47
$ ninja -t commands | grep '-o .*foo'
48
to get the command to link the files together. It may look something like
49
  clang -o foo @foo.rsp
50

51
Now create a test script that runs the link step and whatever test reproduces a
52
miscompile and returns a non-zero exit code when there is a miscompile. For
53
example
54
```
55
  #!/bin/bash
56
  # immediately bail out of script if any command returns a non-zero return code
57
  set -e
58
  clang -o foo @foo.rsp
59
  ./foo
60
```
61

62
With buildgood/ as the working directory, run
63
$ path/to/llvm-project/llvm/utils/rsp_bisect.py \
64
    --test=path/to/test_script --rsp=./foo.rsp --other_rel_path=../buildbad/
65
If rsp_bisect is successful, it will print the first file in the rsp file that
66
when using the bad build directory's version causes the test script to return a
67
different return code. foo.rsp.0 and foo.rsp.1 will also be written. foo.rsp.0
68
will be a copy of foo.rsp with the relevant file using the version in
69
buildgood/, and foo.rsp.1 will be a copy of foo.rsp with the relevant file
70
using the version in buildbad/.
71

72
"""
73

74
import argparse
75
import os
76
import subprocess
77
import sys
78

79

80
def is_path(s):
81
    return "/" in s
82

83

84
def run_test(test):
85
    """Runs the test and returns whether it was successful or not."""
86
    return subprocess.run([test], capture_output=True).returncode == 0
87

88

89
def modify_rsp(rsp_entries, other_rel_path, modify_after_num):
90
    """Create a modified rsp file for use in bisection.
91

92
    Returns a new list from rsp.
93
    For each file in rsp after the first modify_after_num files, prepend
94
    other_rel_path.
95
    """
96
    ret = []
97
    for r in rsp_entries:
98
        if is_path(r):
99
            if modify_after_num == 0:
100
                r = os.path.join(other_rel_path, r)
101
            else:
102
                modify_after_num -= 1
103
        ret.append(r)
104
    assert modify_after_num == 0
105
    return ret
106

107

108
def test_modified_rsp(test, modified_rsp_entries, rsp_path):
109
    """Write the rsp file to disk and run the test."""
110
    with open(rsp_path, "w") as f:
111
        f.write(" ".join(modified_rsp_entries))
112
    return run_test(test)
113

114

115
def bisect(test, zero_result, rsp_entries, num_files_in_rsp, other_rel_path, rsp_path):
116
    """Bisect over rsp entries.
117

118
    Args:
119
        zero_result: the test result when modify_after_num is 0.
120

121
    Returns:
122
        The index of the file in the rsp file where the test result changes.
123
    """
124
    lower = 0
125
    upper = num_files_in_rsp
126
    while lower != upper - 1:
127
        assert lower < upper - 1
128
        mid = int((lower + upper) / 2)
129
        assert lower != mid and mid != upper
130
        print("Trying {} ({}-{})".format(mid, lower, upper))
131
        result = test_modified_rsp(
132
            test, modify_rsp(rsp_entries, other_rel_path, mid), rsp_path
133
        )
134
        if zero_result == result:
135
            lower = mid
136
        else:
137
            upper = mid
138
    return upper
139

140

141
def main():
142
    parser = argparse.ArgumentParser()
143
    parser.add_argument(
144
        "--test", help="Binary to test if current setup is good or bad", required=True
145
    )
146
    parser.add_argument("--rsp", help="rsp file", required=True)
147
    parser.add_argument(
148
        "--other-rel-path",
149
        help="Relative path from current build directory to other build "
150
        + 'directory, e.g. from "out/Default" to "out/Other" specify "../Other"',
151
        required=True,
152
    )
153
    args = parser.parse_args()
154

155
    with open(args.rsp, "r") as f:
156
        rsp_entries = f.read()
157
    rsp_entries = rsp_entries.split()
158
    num_files_in_rsp = sum(1 for a in rsp_entries if is_path(a))
159
    if num_files_in_rsp == 0:
160
        print("No files in rsp?")
161
        return 1
162
    print("{} files in rsp".format(num_files_in_rsp))
163

164
    try:
165
        print("Initial testing")
166
        test0 = test_modified_rsp(
167
            args.test, modify_rsp(rsp_entries, args.other_rel_path, 0), args.rsp
168
        )
169
        test_all = test_modified_rsp(
170
            args.test,
171
            modify_rsp(rsp_entries, args.other_rel_path, num_files_in_rsp),
172
            args.rsp,
173
        )
174

175
        if test0 == test_all:
176
            print("Test returned same exit code for both build directories")
177
            return 1
178

179
        print("First build directory returned " + ("0" if test_all else "1"))
180

181
        result = bisect(
182
            args.test,
183
            test0,
184
            rsp_entries,
185
            num_files_in_rsp,
186
            args.other_rel_path,
187
            args.rsp,
188
        )
189
        print(
190
            "First file change: {} ({})".format(
191
                list(filter(is_path, rsp_entries))[result - 1], result
192
            )
193
        )
194

195
        rsp_out_0 = args.rsp + ".0"
196
        rsp_out_1 = args.rsp + ".1"
197
        with open(rsp_out_0, "w") as f:
198
            f.write(" ".join(modify_rsp(rsp_entries, args.other_rel_path, result - 1)))
199
        with open(rsp_out_1, "w") as f:
200
            f.write(" ".join(modify_rsp(rsp_entries, args.other_rel_path, result)))
201
        print(
202
            "Bisection point rsp files written to {} and {}".format(
203
                rsp_out_0, rsp_out_1
204
            )
205
        )
206
    finally:
207
        # Always make sure to write the original rsp file contents back so it's
208
        # less of a pain to rerun this script.
209
        with open(args.rsp, "w") as f:
210
            f.write(" ".join(rsp_entries))
211

212

213
if __name__ == "__main__":
214
    sys.exit(main())
215