pytorch

1
"""
2
This script takes a pytest CSV file produced by pytest --csv foo.csv
3
and summarizes it into a more minimal CSV that is good for uploading
4
to Google Sheets.  We have been using this with dynamic shapes to
5
understand how many tests fail when we turn on dynamic shapes.  If
6
you have a test suite with a lot of skips or xfails, if force the
7
tests to run anyway, this can help you understand what the actual
8
errors things are failing with are.
9

10
The resulting csv is written to stdout.  An easy way to get the csv
11
onto your local file system is to send it to GitHub Gist:
12

13
    $ python scripts/analysis/format_test_csv.py foo.csv | gh gist create -
14

15
See also scripts/analysis/run_test_csv.sh
16
"""
17

18
import argparse
19
import csv
20
import subprocess
21
import sys
22

23

24
parser = argparse.ArgumentParser(
25
    description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
26
)
27
parser.add_argument("--log-url", type=str, default="", help="URL of raw logs")
28
parser.add_argument("file", help="pytest CSV file to format")
29
args = parser.parse_args()
30

31
out = csv.writer(sys.stdout, dialect="excel")
32
hash = subprocess.check_output(
33
    "git rev-parse HEAD".split(" "), encoding="utf-8"
34
).rstrip()
35

36
out.writerow([hash, args.log_url, ""])
37

38
with open(args.file) as f:
39
    reader = csv.DictReader(f)
40
    for row in reader:
41
        if row["status"] not in {"failed", "error"}:
42
            continue
43
        msg = row["message"].split("\n")[0]
44
        msg.replace(
45
            " - erroring out! It's likely that this is caused by data-dependent control flow or similar.",
46
            "",
47
        )
48
        msg.replace("\t", " ")
49
        # Feel free to edit this; the idea is to remove prefixes that are
50
        # just gooping up the resulting spreadsheet output
51
        name = row["name"].replace("test_make_fx_symbolic_exhaustive_", "")
52
        out.writerow([name, msg, ""])
53