llvm-project

1
#!/usr/bin/env python3
2
import argparse
3
import csv
4
import re
5
import sys
6
import os
7
from statistics import geometric_mean
8

9
TIMING_LOG_RE = re.compile(r"(.*)/(.*).tmp(.*)")
10

11

12
def main():
13
    parser = argparse.ArgumentParser(
14
        description="BOLT NFC stat parser",
15
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
16
    )
17
    parser.add_argument(
18
        "input", nargs="+", help="timing.log files produced by llvm-bolt-wrapper"
19
    )
20
    parser.add_argument(
21
        "--check_longer_than",
22
        default=2,
23
        type=float,
24
        help="Only warn on tests longer than X seconds for at least one side",
25
    )
26
    parser.add_argument(
27
        "--threshold_single",
28
        default=10,
29
        type=float,
30
        help="Threshold for a single test result swing, abs percent",
31
    ),
32
    parser.add_argument(
33
        "--threshold_agg",
34
        default=5,
35
        type=float,
36
        help="Threshold for geomean test results swing, abs percent",
37
    ),
38
    parser.add_argument("--verbose", "-v", action="store_true")
39
    args = parser.parse_args()
40

41
    def fmt_delta(value, exc_threshold, above_bound=True):
42
        formatted_value = format(value, "+.2%")
43
        if not above_bound:
44
            formatted_value += "?"
45
        elif exc_threshold and sys.stdout.isatty():  # terminal supports colors
46
            return f"\033[1m{formatted_value}\033[0m"
47
        return formatted_value
48

49
    # Ratios for geomean computation
50
    time_ratios = []
51
    mem_ratios = []
52
    # Whether any test exceeds the single test threshold (mem or time)
53
    threshold_single = False
54
    # Whether geomean exceeds aggregate test threshold (mem or time)
55
    threshold_agg = False
56

57
    if args.verbose:
58
        print(f"# Individual test threshold: +-{args.threshold_single}%")
59
        print(f"# Aggregate (geomean) test threshold: +-{args.threshold_agg}%")
60
        print(
61
            f"# Checking time swings for tests with runtime >"
62
            f"{args.check_longer_than}s - otherwise marked as ?"
63
        )
64
        print("Test/binary BOLT_wall_time BOLT_max_rss")
65

66
    for input_file in args.input:
67
        input_dir = os.path.dirname(input_file)
68
        with open(input_file) as timing_file:
69
            timing_reader = csv.reader(timing_file, delimiter=";")
70
            for row in timing_reader:
71
                test_name = row[0]
72
                m = TIMING_LOG_RE.match(row[0])
73
                if m:
74
                    test_name = f"{input_dir}/{m.groups()[1]}/{m.groups()[2]}"
75
                else:
76
                    # Prepend input dir to unparsed test name
77
                    test_name = input_dir + "#" + test_name
78
                time_a, time_b = float(row[1]), float(row[3])
79
                mem_a, mem_b = int(row[2]), int(row[4])
80
                # Check if time is above bound for at least one side
81
                time_above_bound = any(
82
                    [x > args.check_longer_than for x in [time_a, time_b]]
83
                )
84
                # Compute B/A ratios (for % delta and geomean)
85
                time_ratio = time_b / time_a if time_a else float('nan')
86
                mem_ratio = mem_b / mem_a if mem_a else float('nan')
87
                # Keep ratios for geomean
88
                if time_above_bound and time_ratio > 0:  # must be >0 for gmean
89
                    time_ratios += [time_ratio]
90
                mem_ratios += [mem_ratio]
91
                # Deltas: (B/A)-1 = (B-A)/A
92
                time_delta = time_ratio - 1
93
                mem_delta = mem_ratio - 1
94
                # Check individual test results vs single test threshold
95
                time_exc = (
96
                    100.0 * abs(time_delta) > args.threshold_single and time_above_bound
97
                )
98
                mem_exc = 100.0 * abs(mem_delta) > args.threshold_single
99
                if time_exc or mem_exc:
100
                    threshold_single = True
101
                # Print deltas with formatting in verbose mode
102
                if args.verbose or time_exc or mem_exc:
103
                    print(
104
                        test_name,
105
                        fmt_delta(time_delta, time_exc, time_above_bound),
106
                        fmt_delta(mem_delta, mem_exc),
107
                    )
108

109
    time_gmean_delta = geometric_mean(time_ratios) - 1
110
    mem_gmean_delta = geometric_mean(mem_ratios) - 1
111
    time_agg_threshold = 100.0 * abs(time_gmean_delta) > args.threshold_agg
112
    mem_agg_threshold = 100.0 * abs(mem_gmean_delta) > args.threshold_agg
113
    if time_agg_threshold or mem_agg_threshold:
114
        threshold_agg = True
115
    if time_agg_threshold or mem_agg_threshold or args.verbose:
116
        print(
117
            "Geomean",
118
            fmt_delta(time_gmean_delta, time_agg_threshold),
119
            fmt_delta(mem_gmean_delta, mem_agg_threshold),
120
        )
121
    exit(threshold_single or threshold_agg)
122

123

124
if __name__ == "__main__":
125
    main()
126