llvm-project

1
#!/usr/bin/env python3
2
# -*- coding: UTF-8 -*-
3

4
# Polly/LLVM update_check.py
5
# Update lit FileCheck files by replacing the 'CHECK:' lines by the actual output of the 'RUN:' command.
6

7
import argparse
8
import os
9
import subprocess
10
import shlex
11
import re
12

13

14
polly_src_dir = """@POLLY_SOURCE_DIR@"""
15
polly_lib_dir = """@POLLY_LIB_DIR@"""
16
shlibext = """@LLVM_SHLIBEXT@"""
17
llvm_tools_dir = """@LLVM_TOOLS_DIR@"""
18
llvm_polly_link_into_tools = not """@LLVM_POLLY_LINK_INTO_TOOLS@""".lower() in {
19
    "",
20
    "0",
21
    "n",
22
    "no",
23
    "off",
24
    "false",
25
    "notfound",
26
    "llvm_polly_link_into_tools-notfound",
27
}
28

29
runre = re.compile(r"\s*\;\s*RUN\s*\:(?P<tool>.*)")
30
filecheckre = re.compile(r"\s*(?P<tool>.*)\|\s*(?P<filecheck>FileCheck\s[^|]*)")
31
emptyline = re.compile(r"\s*(\;\s*)?")
32
commentline = re.compile(r"\s*(\;.*)?")
33

34

35
def ltrim_emptylines(lines, meta=None):
36
    while len(lines) and emptyline.fullmatch(lines[0]):
37
        del lines[0]
38
        if meta is not None:
39
            del meta[0]
40

41

42
def rtrim_emptylines(lines):
43
    while len(lines) and emptyline.fullmatch(lines[-1]):
44
        del lines[-1]
45

46

47
def trim_emptylines(lines):
48
    ltrim_emptylines(lines)
49
    rtrim_emptylines(lines)
50

51

52
def complete_exename(path, filename):
53
    complpath = os.path.join(path, filename)
54
    if os.path.isfile(complpath):
55
        return complpath
56
    elif os.path.isfile(complpath + ".exe"):
57
        return complpath + ".exe"
58
    return filename
59

60

61
def indention(line):
62
    for i, c in enumerate(line):
63
        if c != " " and c != "\t":
64
            return i
65
    return None
66

67

68
def common_indent(lines):
69
    indentions = (indention(line) for line in lines)
70
    indentions = (indent for indent in indentions if indent is not None)
71
    return min(indentions, default=0)
72

73

74
funcre = re.compile(r"^    Function: \S*$")
75
regionre = re.compile(r"^    Region: \S*$")
76
depthre = re.compile(r"^    Max Loop Depth: .*")
77
paramre = re.compile(r"    [0-9a-z-A-Z_]+\: .*")
78

79

80
def classyfier1(lines):
81
    i = iter(lines)
82
    line = i.__next__()
83
    while True:
84
        if line.startswith(
85
            "Printing analysis 'Polly - Calculate dependences' for region: "
86
        ):
87
            yield {"PrintingDependenceInfo"}
88
        elif line.startswith("remark: "):
89
            yield {"Remark"}
90
        elif funcre.fullmatch(line):
91
            yield {"Function"}
92
        elif regionre.fullmatch(line):
93
            yield {"Region"}
94
        elif depthre.fullmatch(line):
95
            yield {"MaxLoopDepth"}
96
        elif line == "    Invariant Accesses: {":
97
            while True:
98
                yield {"InvariantAccesses"}
99
                if line == "    }":
100
                    break
101
                line = i.__next__()
102
        elif line == "    Context:":
103
            yield {"Context"}
104
            line = i.__next__()
105
            yield {"Context"}
106
        elif line == "    Assumed Context:":
107
            yield {"AssumedContext"}
108
            line = i.__next__()
109
            yield {"AssumedContext"}
110
        elif line == "    Invalid Context:":
111
            yield {"InvalidContext"}
112
            line = i.__next__()
113
            yield {"InvalidContext"}
114
        elif line == "    Boundary Context:":
115
            yield {"BoundaryContext"}
116
            line = i.__next__()
117
            yield {"BoundaryContext"}
118
            line = i.__next__()
119
            while paramre.fullmatch(line):
120
                yield {"Param"}
121
                line = i.__next__()
122
            continue
123
        elif line == "    Arrays {":
124
            while True:
125
                yield {"Arrays"}
126
                if line == "    }":
127
                    break
128
                line = i.__next__()
129
        elif line == "    Arrays (Bounds as pw_affs) {":
130
            while True:
131
                yield {"PwAffArrays"}
132
                if line == "    }":
133
                    break
134
                line = i.__next__()
135
        elif line.startswith("    Alias Groups ("):
136
            while True:
137
                yield {"AliasGroups"}
138
                line = i.__next__()
139
                if not line.startswith("        "):
140
                    break
141
            continue
142
        elif line == "    Statements {":
143
            while True:
144
                yield {"Statements"}
145
                if line == "    }":
146
                    break
147
                line = i.__next__()
148
        elif line == "    RAW dependences:":
149
            yield {"RAWDep", "BasicDep", "Dep", "DepInfo"}
150
            line = i.__next__()
151
            while line.startswith("        "):
152
                yield {"RAWDep", "BasicDep", "Dep", "DepInfo"}
153
                line = i.__next__()
154
            continue
155
        elif line == "    WAR dependences:":
156
            yield {"WARDep", "BasicDep", "Dep", "DepInfo"}
157
            line = i.__next__()
158
            while line.startswith("        "):
159
                yield {"WARDep", "BasicDep", "Dep", "DepInfo"}
160
                line = i.__next__()
161
            continue
162
        elif line == "    WAW dependences:":
163
            yield {"WAWDep", "BasicDep", "Dep", "DepInfo"}
164
            line = i.__next__()
165
            while line.startswith("        "):
166
                yield {"WAWDep", "BasicDep", "Dep", "DepInfo"}
167
                line = i.__next__()
168
            continue
169
        elif line == "    Reduction dependences:":
170
            yield {"RedDep", "Dep", "DepInfo"}
171
            line = i.__next__()
172
            while line.startswith("        "):
173
                yield {"RedDep", "Dep", "DepInfo"}
174
                line = i.__next__()
175
            continue
176
        elif line == "    Transitive closure of reduction dependences:":
177
            yield {"TransitiveClosureDep", "DepInfo"}
178
            line = i.__next__()
179
            while line.startswith("        "):
180
                yield {"TransitiveClosureDep", "DepInfo"}
181
                line = i.__next__()
182
            continue
183
        elif line.startswith("New access function '"):
184
            yield {"NewAccessFunction"}
185
        elif line == "Schedule before flattening {":
186
            while True:
187
                yield {"ScheduleBeforeFlattening"}
188
                if line == "}":
189
                    break
190
                line = i.__next__()
191
        elif line == "Schedule after flattening {":
192
            while True:
193
                yield {"ScheduleAfterFlattening"}
194
                if line == "}":
195
                    break
196
                line = i.__next__()
197
        else:
198
            yield set()
199
        line = i.__next__()
200

201

202
def classyfier2(lines):
203
    i = iter(lines)
204
    line = i.__next__()
205
    while True:
206
        if funcre.fullmatch(line):
207
            while line.startswith("    "):
208
                yield {"FunctionDetail"}
209
                line = i.__next__()
210
            continue
211
        elif line.startswith(
212
            "Printing analysis 'Polly - Generate an AST from the SCoP (isl)' for region: "
213
        ):
214
            yield {"PrintingIslAst"}
215
            line = i.__next__()
216
            while not line.startswith("Printing analysis"):
217
                yield {"AstDetail"}
218
                line = i.__next__()
219
            continue
220
        else:
221
            yield set()
222
        line = i.__next__()
223

224

225
replrepl = {"{{": "{{[{][{]}}", "}}": "{{[}][}]}}", "[[": "{{\[\[}}", "]]": "{{\]\]}}"}
226
replre = re.compile("|".join(re.escape(k) for k in replrepl.keys()))
227

228

229
def main():
230
    parser = argparse.ArgumentParser(description="Update CHECK lines")
231
    parser.add_argument(
232
        "testfile", help="File to update (absolute or relative to --testdir)"
233
    )
234
    parser.add_argument(
235
        "--check-style",
236
        choices=["CHECK", "CHECK-NEXT"],
237
        default="CHECK-NEXT",
238
        help="What kind of checks lines to generate",
239
    )
240
    parser.add_argument(
241
        "--check-position",
242
        choices=["end", "before-content", "autodetect"],
243
        default="autodetect",
244
        help="Where to add the CHECK lines into the file; 'autodetect' searches for the first 'CHECK' line ind inserts it there",
245
    )
246
    parser.add_argument(
247
        "--check-include",
248
        action="append",
249
        default=[],
250
        help="What parts of the output lines to check; use syntax 'CHECK=include' to apply to one CHECK-prefix only (by default, everything)",
251
    )
252
    parser.add_argument(
253
        "--check-label-include",
254
        action="append",
255
        default=[],
256
        help="Use CHECK-LABEL for these includes",
257
    )
258
    parser.add_argument(
259
        "--check-part-newline",
260
        action="store_true",
261
        help="Add empty line between different check parts",
262
    )
263
    parser.add_argument(
264
        "--prefix-only",
265
        action="append",
266
        default=None,
267
        help="Update only these prefixes (default: all)",
268
    )
269
    parser.add_argument("--bindir", help="Location of the opt program")
270
    parser.add_argument("--testdir", help="Root dir for unit tests")
271
    parser.add_argument(
272
        "--inplace", "-i", action="store_true", help="Replace input file"
273
    )
274
    parser.add_argument("--output", "-o", help="Write changed input to this file")
275
    known = parser.parse_args()
276

277
    if not known.inplace and known.output is None:
278
        print("Must specify what to do with output (--output or --inplace)")
279
        exit(1)
280
    if known.inplace and known.output is not None:
281
        print("--inplace and --output are mutually exclusive")
282
        exit(1)
283

284
    outfile = known.output
285

286
    filecheckparser = argparse.ArgumentParser(add_help=False)
287
    filecheckparser.add_argument("-check-prefix", "--check-prefix", default="CHECK")
288

289
    filename = known.testfile
290
    for dir in [".", known.testdir, os.path.join(polly_src_dir, "test"), polly_src_dir]:
291
        if not dir:
292
            continue
293
        testfilename = os.path.join(dir, filename)
294
        if os.path.isfile(testfilename):
295
            filename = testfilename
296
            break
297

298
    if known.inplace:
299
        outfile = filename
300

301
    allchecklines = []
302
    checkprefixes = []
303

304
    with open(filename, "r") as file:
305
        oldlines = [line.rstrip("\r\n") for line in file.readlines()]
306

307
    runlines = []
308
    for line in oldlines:
309
        m = runre.match(line)
310
        if m:
311
            runlines.append(m.group("tool"))
312

313
    continuation = ""
314
    newrunlines = []
315
    for line in runlines:
316
        if line.endswith("\\"):
317
            continuation += line[:-2] + " "
318
        else:
319
            newrunlines.append(continuation + line)
320
            continuation = ""
321
    if continuation:
322
        newrunlines.append(continuation)
323

324
    for line in newrunlines:
325
        m = filecheckre.match(line)
326
        if not m:
327
            continue
328

329
        tool, filecheck = m.group("tool", "filecheck")
330
        filecheck = shlex.split(filecheck)
331
        tool = shlex.split(tool)
332
        if known.bindir is not None:
333
            tool[0] = complete_exename(known.bindir, tool[0])
334
        if os.path.isdir(llvm_tools_dir):
335
            tool[0] = complete_exename(llvm_tools_dir, tool[0])
336
        check_prefix = filecheckparser.parse_known_args(filecheck)[0].check_prefix
337
        if known.prefix_only is not None and not check_prefix in known.prefix_only:
338
            continue
339
        if check_prefix in checkprefixes:
340
            continue
341
        checkprefixes.append(check_prefix)
342

343
        newtool = []
344
        optstderr = None
345
        for toolarg in tool:
346
            toolarg = toolarg.replace("%s", filename)
347
            toolarg = toolarg.replace("%S", os.path.dirname(filename))
348
            if toolarg == "%loadPolly":
349
                if not llvm_polly_link_into_tools:
350
                    newtool += [
351
                        "-load",
352
                        os.path.join(polly_lib_dir, "LLVMPolly" + shlibext),
353
                    ]
354
                newtool.append("-polly-process-unprofitable")
355
                newtool.append("-polly-remarks-minimal")
356
            elif toolarg == "2>&1":
357
                optstderr = subprocess.STDOUT
358
            else:
359
                newtool.append(toolarg)
360
        tool = newtool
361

362
        inpfile = None
363
        i = 1
364
        while i < len(tool):
365
            if tool[i] == "<":
366
                inpfile = tool[i + 1]
367
                del tool[i : i + 2]
368
                continue
369
            i += 1
370
        if inpfile:
371
            with open(inpfile) as inp:
372
                retlines = subprocess.check_output(
373
                    tool, universal_newlines=True, stdin=inp, stderr=optstderr
374
                )
375
        else:
376
            retlines = subprocess.check_output(
377
                tool, universal_newlines=True, stderr=optstderr
378
            )
379
        retlines = [line.replace("\t", "    ") for line in retlines.splitlines()]
380
        check_include = []
381
        for checkme in known.check_include + known.check_label_include:
382
            parts = checkme.split("=")
383
            if len(parts) == 2:
384
                if parts[0] == check_prefix:
385
                    check_include.append(parts[1])
386
            else:
387
                check_include.append(checkme)
388

389
        if check_include:
390
            filtered_retlines = []
391
            classified_retlines = []
392
            lastmatch = None
393
            for line, kind in (
394
                (line, class1.union(class2))
395
                for line, class1, class2 in zip(
396
                    retlines, classyfier1(retlines), classyfier2(retlines)
397
                )
398
            ):
399
                match = kind.intersection(check_include)
400
                if match:
401
                    if lastmatch != match:
402
                        filtered_retlines.append("")
403
                        classified_retlines.append({"Separator"})
404
                    filtered_retlines.append(line)
405
                    classified_retlines.append(kind)
406
                lastmatch = match
407

408
            retlines = filtered_retlines
409
        else:
410
            classified_retlines = (set() for line in retlines)
411

412
        rtrim_emptylines(retlines)
413
        ltrim_emptylines(retlines, classified_retlines)
414
        retlines = [
415
            replre.sub(lambda m: replrepl[m.group(0)], line) for line in retlines
416
        ]
417
        indent = common_indent(retlines)
418
        retlines = [line[indent:] for line in retlines]
419
        checklines = []
420
        previous_was_empty = True
421
        for line, kind in zip(retlines, classified_retlines):
422
            if line:
423
                if known.check_style == "CHECK" and known.check_label_include:
424
                    if not kind.isdisjoint(known.check_label_include):
425
                        checklines.append("; " + check_prefix + "-LABEL: " + line)
426
                    else:
427
                        checklines.append("; " + check_prefix + ":       " + line)
428
                elif known.check_style == "CHECK":
429
                    checklines.append("; " + check_prefix + ": " + line)
430
                elif known.check_label_include and known.check_label_include:
431
                    if not kind.isdisjoint(known.check_label_include):
432
                        checklines.append("; " + check_prefix + "-LABEL: " + line)
433
                    elif previous_was_empty:
434
                        checklines.append("; " + check_prefix + ":       " + line)
435
                    else:
436
                        checklines.append("; " + check_prefix + "-NEXT:  " + line)
437
                else:
438
                    if previous_was_empty:
439
                        checklines.append("; " + check_prefix + ":      " + line)
440
                    else:
441
                        checklines.append("; " + check_prefix + "-NEXT: " + line)
442
                previous_was_empty = False
443
            else:
444
                if not "Separator" in kind or known.check_part_newline:
445
                    checklines.append(";")
446
                previous_was_empty = True
447
        allchecklines.append(checklines)
448

449
    if not checkprefixes:
450
        return
451

452
    checkre = re.compile(
453
        r"^\s*\;\s*("
454
        + "|".join([re.escape(s) for s in checkprefixes])
455
        + ")(\-NEXT|\-DAG|\-NOT|\-LABEL|\-SAME)?\s*\:"
456
    )
457
    firstcheckline = None
458
    firstnoncommentline = None
459
    headerlines = []
460
    newlines = []
461
    uptonowlines = []
462
    emptylines = []
463
    lastwascheck = False
464
    for line in oldlines:
465
        if checkre.match(line):
466
            if firstcheckline is None:
467
                firstcheckline = len(newlines) + len(emptylines)
468
            if not lastwascheck:
469
                uptonowlines += emptylines
470
            emptylines = []
471
            lastwascheck = True
472
        elif emptyline.fullmatch(line):
473
            emptylines.append(line)
474
        else:
475
            newlines += uptonowlines
476
            newlines += emptylines
477
            newlines.append(line)
478
            emptylines = []
479
            uptonowlines = []
480
            lastwascheck = False
481

482
    for i, line in enumerate(newlines):
483
        if not commentline.fullmatch(line):
484
            firstnoncommentline = i
485
            break
486

487
    with open(outfile, "w", newline="") as file:
488

489
        def writelines(lines):
490
            for line in lines:
491
                file.write(line)
492
                file.write("\n")
493

494
        if firstcheckline is not None and known.check_position == "autodetect":
495
            writelines(newlines[:firstcheckline])
496
            writelines(uptonowlines)
497
            for i, checklines in enumerate(allchecklines):
498
                if i != 0:
499
                    file.write("\n")
500
                writelines(checklines)
501
            writelines(newlines[firstcheckline:])
502
            writelines(emptylines)
503
        elif (
504
            firstnoncommentline is not None and known.check_position == "before-content"
505
        ):
506
            headerlines = newlines[:firstnoncommentline]
507
            rtrim_emptylines(headerlines)
508
            contentlines = newlines[firstnoncommentline:]
509
            ltrim_emptylines(contentlines)
510

511
            writelines(headerlines)
512
            for checklines in allchecklines:
513
                file.write("\n")
514
                writelines(checklines)
515
            file.write("\n")
516
            writelines(contentlines)
517
            writelines(uptonowlines)
518
            writelines(emptylines)
519
        else:
520
            writelines(newlines)
521
            rtrim_emptylines(newlines)
522
            for checklines in allchecklines:
523
                file.write("\n\n")
524
                writelines(checklines)
525

526

527
if __name__ == "__main__":
528
    main()
529