transformers

1
# coding=utf-8
2
# Copyright 2021 The HuggingFace Team. All rights reserved.
3
#
4
# Licensed under the Apache License, Version 2.0 (the "License");
5
# you may not use this file except in compliance with the License.
6
# You may obtain a copy of the License at
7
#
8
#     http://www.apache.org/licenses/LICENSE-2.0
9
#
10
# Unless required by applicable law or agreed to in writing, software
11
# distributed under the License is distributed on an "AS IS" BASIS,
12
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
# See the License for the specific language governing permissions and
14
# limitations under the License.
15
"""
16
Utility that prepares the repository for releases (or patches) by updating all versions in the relevant places. It
17
also performs some post-release cleanup, by updating the links in the main README to respective model doc pages (from
18
main to stable).
19

20
To prepare for a release, use from the root of the repo on the release branch with:
21

22
```bash
23
python release.py
24
```
25

26
or use `make pre-release`.
27

28
To prepare for a patch release, use from the root of the repo on the release branch with:
29

30
```bash
31
python release.py --patch
32
```
33

34
or use `make pre-patch`.
35

36
To do the post-release cleanup, use from the root of the repo on the main branch with:
37

38
```bash
39
python release.py --post_release
40
```
41

42
or use `make post-release`.
43
"""
44
import argparse
45
import os
46
import re
47

48
import packaging.version
49

50

51
# All paths are defined with the intent that this script should be run from the root of the repo.
52
PATH_TO_EXAMPLES = "examples/"
53
# This maps a type of file to the pattern to look for when searching where the version is defined, as well as the
54
# template to follow when replacing it with the new version.
55
REPLACE_PATTERNS = {
56
    "examples": (re.compile(r'^check_min_version\("[^"]+"\)\s*$', re.MULTILINE), 'check_min_version("VERSION")\n'),
57
    "init": (re.compile(r'^__version__\s+=\s+"([^"]+)"\s*$', re.MULTILINE), '__version__ = "VERSION"\n'),
58
    "setup": (re.compile(r'^(\s*)version\s*=\s*"[^"]+",', re.MULTILINE), r'\1version="VERSION",'),
59
}
60
# This maps a type of file to its path in Transformers
61
REPLACE_FILES = {
62
    "init": "src/transformers/__init__.py",
63
    "setup": "setup.py",
64
}
65
README_FILE = "README.md"
66

67

68
def update_version_in_file(fname: str, version: str, file_type: str):
69
    """
70
    Update the version of Transformers in one file.
71

72
    Args:
73
        fname (`str`): The path to the file where we want to update the version.
74
        version (`str`): The new version to set in the file.
75
        file_type (`str`): The type of the file (should be a key in `REPLACE_PATTERNS`).
76
    """
77
    with open(fname, "r", encoding="utf-8", newline="\n") as f:
78
        code = f.read()
79
    re_pattern, replace = REPLACE_PATTERNS[file_type]
80
    replace = replace.replace("VERSION", version)
81
    code = re_pattern.sub(replace, code)
82
    with open(fname, "w", encoding="utf-8", newline="\n") as f:
83
        f.write(code)
84

85

86
def update_version_in_examples(version: str):
87
    """
88
    Update the version in all examples files.
89

90
    Args:
91
        version (`str`): The new version to set in the examples.
92
    """
93
    for folder, directories, fnames in os.walk(PATH_TO_EXAMPLES):
94
        # Removing some of the folders with non-actively maintained examples from the walk
95
        if "research_projects" in directories:
96
            directories.remove("research_projects")
97
        if "legacy" in directories:
98
            directories.remove("legacy")
99
        for fname in fnames:
100
            if fname.endswith(".py"):
101
                update_version_in_file(os.path.join(folder, fname), version, file_type="examples")
102

103

104
def global_version_update(version: str, patch: bool = False):
105
    """
106
    Update the version in all needed files.
107

108
    Args:
109
        version (`str`): The new version to set everywhere.
110
        patch (`bool`, *optional*, defaults to `False`): Whether or not this is a patch release.
111
    """
112
    for pattern, fname in REPLACE_FILES.items():
113
        update_version_in_file(fname, version, pattern)
114
    if not patch:
115
        # We don't update the version in the examples for patch releases.
116
        update_version_in_examples(version)
117

118

119
def clean_main_ref_in_model_list():
120
    """
121
    Replace the links from main doc to stable doc in the model list of the README.
122
    """
123
    # If the introduction or the conclusion of the list change, the prompts may need to be updated.
124
    _start_prompt = "🤗 Transformers currently provides the following architectures"
125
    _end_prompt = "1. Want to contribute a new model?"
126
    with open(README_FILE, "r", encoding="utf-8", newline="\n") as f:
127
        lines = f.readlines()
128

129
    # Find the start of the list.
130
    start_index = 0
131
    while not lines[start_index].startswith(_start_prompt):
132
        start_index += 1
133
    start_index += 1
134

135
    index = start_index
136
    # Update the lines in the model list.
137
    while not lines[index].startswith(_end_prompt):
138
        if lines[index].startswith("1."):
139
            lines[index] = lines[index].replace(
140
                "https://huggingface.co/docs/transformers/main/model_doc",
141
                "https://huggingface.co/docs/transformers/model_doc",
142
            )
143
        index += 1
144

145
    with open(README_FILE, "w", encoding="utf-8", newline="\n") as f:
146
        f.writelines(lines)
147

148

149
def get_version() -> packaging.version.Version:
150
    """
151
    Reads the current version in the main __init__.
152
    """
153
    with open(REPLACE_FILES["init"], "r") as f:
154
        code = f.read()
155
    default_version = REPLACE_PATTERNS["init"][0].search(code).groups()[0]
156
    return packaging.version.parse(default_version)
157

158

159
def pre_release_work(patch: bool = False):
160
    """
161
    Do all the necessary pre-release steps:
162
    - figure out the next minor release version and ask confirmation
163
    - update the version eveywhere
164
    - clean-up the model list in the main README
165

166
    Args:
167
        patch (`bool`, *optional*, defaults to `False`): Whether or not this is a patch release.
168
    """
169
    # First let's get the default version: base version if we are in dev, bump minor otherwise.
170
    default_version = get_version()
171
    if patch and default_version.is_devrelease:
172
        raise ValueError("Can't create a patch version from the dev branch, checkout a released version!")
173
    if default_version.is_devrelease:
174
        default_version = default_version.base_version
175
    elif patch:
176
        default_version = f"{default_version.major}.{default_version.minor}.{default_version.micro + 1}"
177
    else:
178
        default_version = f"{default_version.major}.{default_version.minor + 1}.0"
179

180
    # Now let's ask nicely if we have found the right version.
181
    version = input(f"Which version are you releasing? [{default_version}]")
182
    if len(version) == 0:
183
        version = default_version
184

185
    print(f"Updating version to {version}.")
186
    global_version_update(version, patch=patch)
187
    if not patch:
188
        print("Cleaning main README, don't forget to run `make fix-copies`.")
189
        clean_main_ref_in_model_list()
190

191

192
def post_release_work():
193
    """
194
    Do all the necesarry post-release steps:
195
    - figure out the next dev version and ask confirmation
196
    - update the version eveywhere
197
    - clean-up the model list in the main README
198
    """
199
    # First let's get the current version
200
    current_version = get_version()
201
    dev_version = f"{current_version.major}.{current_version.minor + 1}.0.dev0"
202
    current_version = current_version.base_version
203

204
    # Check with the user we got that right.
205
    version = input(f"Which version are we developing now? [{dev_version}]")
206
    if len(version) == 0:
207
        version = dev_version
208

209
    print(f"Updating version to {version}.")
210
    global_version_update(version)
211
    print("Cleaning main README, don't forget to run `make fix-copies`.")
212
    clean_main_ref_in_model_list()
213

214

215
if __name__ == "__main__":
216
    parser = argparse.ArgumentParser()
217
    parser.add_argument("--post_release", action="store_true", help="Whether this is pre or post release.")
218
    parser.add_argument("--patch", action="store_true", help="Whether or not this is a patch release.")
219
    args = parser.parse_args()
220
    if not args.post_release:
221
        pre_release_work(patch=args.patch)
222
    elif args.patch:
223
        print("Nothing to do after a patch :-)")
224
    else:
225
        post_release_work()
226