pytorch

1
#!/usr/bin/env python3
2
import argparse
3
import os
4
import subprocess
5
from pathlib import Path
6

7
from common import (
8
    get_testcases,
9
    is_failure,
10
    is_passing_skipped_test,
11
    is_unexpected_success,
12
    key,
13
    open_test_results,
14
)
15
from download_reports import download_reports
16

17

18
"""
19
Usage: update_failures.py /path/to/dynamo_test_failures.py /path/to/test commit_sha
20

21
Best-effort updates the xfail and skip files under test directory
22
by parsing test reports.
23

24
You'll need to provide the commit_sha for the latest commit on a PR
25
from which we will pull CI test results.
26

27
Instructions:
28
- On your PR, add the "keep-going" label to ensure that all the tests are
29
  failing (as opposed to CI stopping on the first failure). You may need to
30
  restart your test jobs by force-pushing to your branch for CI to pick
31
  up the "keep-going" label.
32
- Wait for all the tests to finish running.
33
- Find the full SHA of your commit and run this command.
34

35
This script requires the `gh` cli. You'll need to install it and then
36
authenticate with it via `gh auth login` before using this script.
37
https://docs.github.com/en/github-cli/github-cli/quickstart
38
"""
39

40

41
def patch_file(
42
    filename, test_dir, unexpected_successes, new_xfails, new_skips, unexpected_skips
43
):
44
    failures_directory = os.path.join(test_dir, "dynamo_expected_failures")
45
    skips_directory = os.path.join(test_dir, "dynamo_skips")
46

47
    dynamo_expected_failures = set(os.listdir(failures_directory))
48
    dynamo_skips = set(os.listdir(skips_directory))
49

50
    # These are hand written skips
51
    extra_dynamo_skips = set()
52
    with open(filename) as f:
53
        start = False
54
        for text in f.readlines():
55
            text = text.strip()
56
            if start:
57
                if text == "}":
58
                    break
59
                extra_dynamo_skips.add(text.strip(',"'))
60
            else:
61
                if text == "extra_dynamo_skips = {":
62
                    start = True
63

64
    def format(testcase):
65
        classname = testcase.attrib["classname"]
66
        name = testcase.attrib["name"]
67
        return f"{classname}.{name}"
68

69
    formatted_unexpected_successes = {
70
        f"{format(test)}" for test in unexpected_successes.values()
71
    }
72
    formatted_unexpected_skips = {
73
        f"{format(test)}" for test in unexpected_skips.values()
74
    }
75
    formatted_new_xfails = [f"{format(test)}" for test in new_xfails.values()]
76
    formatted_new_skips = [f"{format(test)}" for test in new_skips.values()]
77

78
    def remove_file(path, name):
79
        file = os.path.join(path, name)
80
        cmd = ["git", "rm", file]
81
        subprocess.run(cmd)
82

83
    def add_file(path, name):
84
        file = os.path.join(path, name)
85
        with open(file, "w") as fp:
86
            pass
87
        cmd = ["git", "add", file]
88
        subprocess.run(cmd)
89

90
    covered_unexpected_successes = set()
91

92
    # dynamo_expected_failures
93
    for test in dynamo_expected_failures:
94
        if test in formatted_unexpected_successes:
95
            covered_unexpected_successes.add(test)
96
            remove_file(failures_directory, test)
97
    for test in formatted_new_xfails:
98
        add_file(failures_directory, test)
99

100
    leftover_unexpected_successes = (
101
        formatted_unexpected_successes - covered_unexpected_successes
102
    )
103
    if len(leftover_unexpected_successes) > 0:
104
        print(
105
            "WARNING: we were unable to remove these "
106
            f"{len(leftover_unexpected_successes)} expectedFailures:"
107
        )
108
        for stuff in leftover_unexpected_successes:
109
            print(stuff)
110

111
    # dynamo_skips
112
    for test in dynamo_skips:
113
        if test in formatted_unexpected_skips:
114
            remove_file(skips_directory, test)
115
    for test in extra_dynamo_skips:
116
        if test in formatted_unexpected_skips:
117
            print(
118
                f"WARNING: {test} in dynamo_test_failures.py needs to be removed manually"
119
            )
120
    for test in formatted_new_skips:
121
        add_file(skips_directory, test)
122

123

124
def get_intersection_and_outside(a_dict, b_dict):
125
    a = set(a_dict.keys())
126
    b = set(b_dict.keys())
127
    intersection = a.intersection(b)
128
    outside = (a.union(b)) - intersection
129

130
    def build_dict(keys):
131
        result = {}
132
        for k in keys:
133
            if k in a_dict:
134
                result[k] = a_dict[k]
135
            else:
136
                result[k] = b_dict[k]
137
        return result
138

139
    return build_dict(intersection), build_dict(outside)
140

141

142
def update(filename, test_dir, py38_dir, py311_dir, also_remove_skips):
143
    def read_test_results(directory):
144
        xmls = open_test_results(directory)
145
        testcases = get_testcases(xmls)
146
        unexpected_successes = {
147
            key(test): test for test in testcases if is_unexpected_success(test)
148
        }
149
        failures = {key(test): test for test in testcases if is_failure(test)}
150
        passing_skipped_tests = {
151
            key(test): test for test in testcases if is_passing_skipped_test(test)
152
        }
153
        return unexpected_successes, failures, passing_skipped_tests
154

155
    (
156
        py38_unexpected_successes,
157
        py38_failures,
158
        py38_passing_skipped_tests,
159
    ) = read_test_results(py38_dir)
160
    (
161
        py311_unexpected_successes,
162
        py311_failures,
163
        py311_passing_skipped_tests,
164
    ) = read_test_results(py311_dir)
165

166
    unexpected_successes = {**py38_unexpected_successes, **py311_unexpected_successes}
167
    _, skips = get_intersection_and_outside(
168
        py38_unexpected_successes, py311_unexpected_successes
169
    )
170
    xfails, more_skips = get_intersection_and_outside(py38_failures, py311_failures)
171
    if also_remove_skips:
172
        unexpected_skips, _ = get_intersection_and_outside(
173
            py38_passing_skipped_tests, py311_passing_skipped_tests
174
        )
175
    else:
176
        unexpected_skips = {}
177
    all_skips = {**skips, **more_skips}
178
    print(
179
        f"Discovered {len(unexpected_successes)} new unexpected successes, "
180
        f"{len(xfails)} new xfails, {len(all_skips)} new skips, {len(unexpected_skips)} new unexpected skips"
181
    )
182
    return patch_file(
183
        filename, test_dir, unexpected_successes, xfails, all_skips, unexpected_skips
184
    )
185

186

187
if __name__ == "__main__":
188
    parser = argparse.ArgumentParser(
189
        prog="update_dynamo_test_failures",
190
        description="Read from logs and update the dynamo_test_failures file",
191
    )
192
    # dynamo_test_failures path
193
    parser.add_argument(
194
        "filename",
195
        nargs="?",
196
        default=str(
197
            Path(__file__).absolute().parent.parent.parent
198
            / "torch/testing/_internal/dynamo_test_failures.py"
199
        ),
200
        help="Optional path to dynamo_test_failures.py",
201
    )
202
    # test path
203
    parser.add_argument(
204
        "test_dir",
205
        nargs="?",
206
        default=str(Path(__file__).absolute().parent.parent.parent / "test"),
207
        help="Optional path to test folder",
208
    )
209
    parser.add_argument(
210
        "commit",
211
        help=(
212
            "The commit sha for the latest commit on a PR from which we will "
213
            "pull CI test results, e.g. 7e5f597aeeba30c390c05f7d316829b3798064a5"
214
        ),
215
    )
216
    parser.add_argument(
217
        "--also-remove-skips",
218
        help="Also attempt to remove skips. WARNING: does not guard against test flakiness",
219
        action="store_true",
220
    )
221
    args = parser.parse_args()
222
    assert Path(args.filename).exists(), args.filename
223
    assert Path(args.test_dir).exists(), args.test_dir
224
    dynamo39, dynamo311 = download_reports(args.commit, ("dynamo39", "dynamo311"))
225
    update(args.filename, args.test_dir, dynamo39, dynamo311, args.also_remove_skips)
226