glusterfs

1
#!/usr/bin/python3
2

3
from __future__ import print_function
4
import blessings
5
import requests
6
from requests.packages.urllib3.exceptions import InsecureRequestWarning
7
import re
8
import argparse
9
from collections import defaultdict
10
from datetime import timedelta, datetime
11
from pystache import render
12

13
# This tool goes though the Gluster regression links and checks for failures
14

15
BASE = 'https://build.gluster.org'
16
TERM = blessings.Terminal()
17
MAX_BUILDS = 1000
18
summary = defaultdict(list)
19
VERBOSE = None
20

21

22
def process_failure(url, node):
23
    text = requests.get(url, verify=False).text
24
    accum = []
25
    for t in text.split('\n'):
26
        if t.find("Result: FAIL") != -1:
27
            for t2 in accum:
28
                if VERBOSE:
29
                    print(t2.encode('utf-8'))
30
                if t2.find("Wstat") != -1:
31
                    test_case = re.search('\./tests/.*\.t', t2)
32
                    if test_case:
33
                        summary[test_case.group()].append((url, node))
34
            accum = []
35
        elif t.find("cur_cores=/") != -1:
36
            summary["core"].append([t.split("/")[1]])
37
            summary["core"].append(url)
38
        else:
39
            accum.append(t)
40

41

42
def print_summary(failed_builds, total_builds, html=False):
43
    # All the templates
44
    count = [
45
            '{{failed}} of {{total}} regressions failed',
46
            '<p><b>{{failed}}</b> of <b>{{total}}</b> regressions failed</p>'
47
    ]
48
    regression_link = [
49
            '\tRegression Link: {{link}}\n'
50
            '\tNode: {{node}}',
51
            '<p>&emsp;Regression Link: {{link}}</p>'
52
            '<p>&emsp;Node: {{node}}</p>'
53
    ]
54
    component = [
55
            '\tComponent: {{comp}}',
56
            '<p>&emsp;Component: {{comp}}</p>'
57
    ]
58
    failure_count = [
59
            ''.join([
60
                TERM.red,
61
                '{{test}} ; Failed {{count}} times',
62
                TERM.normal
63
            ]),
64
            (
65
                '<p><font color="red"><b>{{test}};</b> Failed <b>{{count}}'
66
                '</b> times</font></p>'
67
            )
68
    ]
69

70
    template = 0
71
    if html:
72
        template = 1
73
    print(render(
74
            count[template],
75
            {'failed': failed_builds, 'total': total_builds}
76
    ))
77
    for k, v in summary.items():
78
        if k == 'core':
79
            print(''.join([TERM.red, "Found cores:", TERM.normal]))
80
            for comp, link in zip(v[::2], v[1::2]):
81
                print(render(component[template], {'comp': comp}))
82
                print(render(
83
                        regression_link[template],
84
                        {'link': link[0], 'node': link[1]}
85
                ))
86
        else:
87
            print(render(failure_count[template], {'test': k, 'count': len(v)}))
88
            for link in v:
89
                print(render(
90
                        regression_link[template],
91
                        {'link': link[0], 'node': link[1]}
92
                ))
93

94

95
def get_summary(cut_off_date, reg_link):
96
    '''
97
    Get links to the failed jobs
98
    '''
99
    success_count = 0
100
    failure_count = 0
101
    for page in range(0, MAX_BUILDS, 100):
102
        build_info = requests.get(''.join([
103
                BASE,
104
                reg_link,
105
                'api/json?depth=1&tree=allBuilds'
106
                '[url,result,timestamp,builtOn]',
107
                '{{{0},{1}}}'.format(page, page+100)
108
        ]), verify=False).json()
109
        for build in build_info.get('allBuilds'):
110
            if datetime.fromtimestamp(build['timestamp']/1000) < cut_off_date:
111
                # stop when timestamp older than cut off date
112
                return failure_count, failure_count + success_count
113
            if build['result'] in [None, 'SUCCESS']:
114
                # pass when build is a success or ongoing
115
                success_count += 1
116
                continue
117
            if VERBOSE:
118
                print(''.join([
119
                    TERM.red,
120
                    'FAILURE on {0}'.format(build['url']),
121
                    TERM.normal
122
                ]))
123
            url = ''.join([build['url'], 'consoleText'])
124
            failure_count += 1
125
            process_failure(url, build['builtOn'])
126
    return failure_count, failure_count + success_count
127

128

129
def main(num_days, regression_link, html_report):
130
    cut_off_date = datetime.today() - timedelta(days=num_days)
131
    failure = 0
132
    total = 0
133
    for reg in regression_link:
134
        if reg == 'centos':
135
            reg_link = '/job/centos6-regression/'
136
        elif reg == 'netbsd':
137
            reg_link = '/job/netbsd7-regression/'
138
        else:
139
            reg_link = reg
140
        counts = get_summary(cut_off_date, reg_link)
141
        failure += counts[0]
142
        total += counts[1]
143
    print_summary(failure, total, html_report)
144

145

146
if __name__ == '__main__':
147
    requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
148
    parser = argparse.ArgumentParser()
149
    parser.add_argument("get-summary")
150
    parser.add_argument(
151
            "last_no_of_days",
152
            default=1,
153
            type=int,
154
            help="Regression summary of last number of days"
155
    )
156
    parser.add_argument(
157
            "regression_link",
158
            default="centos",
159
            nargs='+',
160
            help="\"centos\" | \"netbsd\" | any other regression link"
161
    )
162
    parser.add_argument(
163
            "--verbose",
164
            default=False,
165
            action="store_true",
166
            help="Print a detailed report of each test case that is failed"
167
    )
168
    parser.add_argument(
169
            "--html-report",
170
            default=False,
171
            action="store_true",
172
            help="Print a brief report of failed regressions in html format"
173
    )
174
    args = parser.parse_args()
175
    VERBOSE = args.verbose
176
    main(
177
        num_days=args.last_no_of_days,
178
        regression_link=args.regression_link,
179
        html_report=args.html_report
180
    )
181