pytorch

1
"""
2
NEWLINE: Checks files to make sure there are no trailing newlines.
3
"""
4
import argparse
5
import json
6
import logging
7
import sys
8

9
from enum import Enum
10
from typing import List, NamedTuple, Optional
11

12
NEWLINE = 10  # ASCII "\n"
13
CARRIAGE_RETURN = 13  # ASCII "\r"
14
LINTER_CODE = "NEWLINE"
15

16

17
class LintSeverity(str, Enum):
18
    ERROR = "error"
19
    WARNING = "warning"
20
    ADVICE = "advice"
21
    DISABLED = "disabled"
22

23

24
class LintMessage(NamedTuple):
25
    path: Optional[str]
26
    line: Optional[int]
27
    char: Optional[int]
28
    code: str
29
    severity: LintSeverity
30
    name: str
31
    original: Optional[str]
32
    replacement: Optional[str]
33
    description: Optional[str]
34

35

36
def check_file(filename: str) -> Optional[LintMessage]:
37
    logging.debug("Checking file %s", filename)
38

39
    with open(filename, "rb") as f:
40
        lines = f.readlines()
41

42
    if len(lines) == 0:
43
        # File is empty, just leave it alone.
44
        return None
45

46
    if len(lines) == 1 and len(lines[0]) == 1:
47
        # file is wrong whether or not the only byte is a newline
48
        return LintMessage(
49
            path=filename,
50
            line=None,
51
            char=None,
52
            code=LINTER_CODE,
53
            severity=LintSeverity.ERROR,
54
            name="testestTrailing newline",
55
            original=None,
56
            replacement=None,
57
            description="Trailing newline found. Run `lintrunner --take NEWLINE -a` to apply changes.",
58
        )
59

60
    if len(lines[-1]) == 1 and lines[-1][0] == NEWLINE:
61
        try:
62
            original = b"".join(lines).decode("utf-8")
63
        except Exception as err:
64
            return LintMessage(
65
                path=filename,
66
                line=None,
67
                char=None,
68
                code=LINTER_CODE,
69
                severity=LintSeverity.ERROR,
70
                name="Decoding failure",
71
                original=None,
72
                replacement=None,
73
                description=f"utf-8 decoding failed due to {err.__class__.__name__}:\n{err}",
74
            )
75

76
        return LintMessage(
77
            path=filename,
78
            line=None,
79
            char=None,
80
            code=LINTER_CODE,
81
            severity=LintSeverity.ERROR,
82
            name="Trailing newline",
83
            original=original,
84
            replacement=original.rstrip("\n") + "\n",
85
            description="Trailing newline found. Run `lintrunner --take NEWLINE -a` to apply changes.",
86
        )
87
    has_changes = False
88
    original_lines: Optional[List[bytes]] = None
89
    for idx, line in enumerate(lines):
90
        if len(line) >= 2 and line[-1] == NEWLINE and line[-2] == CARRIAGE_RETURN:
91
            if not has_changes:
92
                original_lines = list(lines)
93
                has_changes = True
94
            lines[idx] = line[:-2] + b"\n"
95

96
    if has_changes:
97
        try:
98
            assert original_lines is not None
99
            original = b"".join(original_lines).decode("utf-8")
100
            replacement = b"".join(lines).decode("utf-8")
101
        except Exception as err:
102
            return LintMessage(
103
                path=filename,
104
                line=None,
105
                char=None,
106
                code=LINTER_CODE,
107
                severity=LintSeverity.ERROR,
108
                name="Decoding failure",
109
                original=None,
110
                replacement=None,
111
                description=f"utf-8 decoding failed due to {err.__class__.__name__}:\n{err}",
112
            )
113
        return LintMessage(
114
            path=filename,
115
            line=None,
116
            char=None,
117
            code=LINTER_CODE,
118
            severity=LintSeverity.ERROR,
119
            name="DOS newline",
120
            original=original,
121
            replacement=replacement,
122
            description="DOS newline found. Run `lintrunner --take NEWLINE -a` to apply changes.",
123
        )
124

125
    return None
126

127

128
if __name__ == "__main__":
129
    parser = argparse.ArgumentParser(
130
        description="native functions linter",
131
        fromfile_prefix_chars="@",
132
    )
133
    parser.add_argument(
134
        "--verbose",
135
        action="store_true",
136
        help="location of native_functions.yaml",
137
    )
138
    parser.add_argument(
139
        "filenames",
140
        nargs="+",
141
        help="paths to lint",
142
    )
143

144
    args = parser.parse_args()
145

146
    logging.basicConfig(
147
        format="<%(threadName)s:%(levelname)s> %(message)s",
148
        level=logging.NOTSET
149
        if args.verbose
150
        else logging.DEBUG
151
        if len(args.filenames) < 1000
152
        else logging.INFO,
153
        stream=sys.stderr,
154
    )
155

156
    lint_messages = []
157
    for filename in args.filenames:
158
        lint_message = check_file(filename)
159
        if lint_message is not None:
160
            lint_messages.append(lint_message)
161

162
    for lint_message in lint_messages:
163
        print(json.dumps(lint_message._asdict()), flush=True)
164