utils/check_inits.py - main - githubmirror/transformers

utils/check_inits.py
142 строки5 KB
guarin
Add check_model_inits.py (#47656)
07 авг 2026, 10:35
07 авг 2026, 10:355f121dc
# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Utility that regenerates `src/transformers/models/__init__.py` from the import structure on disk.

The `TYPE_CHECKING` half of that init only exists to give type checkers a static view of what the
`_LazyModule` half exposes at runtime, so both halves are derived from the same `define_import_structure`
call. Hand-writing the `TYPE_CHECKING` half lets new models go missing and removed ones linger; runtime
is unaffected either way, so the `imports` checker cannot catch it.

Usage (from the root of the repo):

Check that the init is up to date (used in `make check-repo`):

```bash
python utils/check_inits.py
```

Regenerate it if needed (used in `make fix-repo`):

```bash
python utils/check_inits.py --fix_and_overwrite
```
"""

import argparse
import difflib
import re
from pathlib import Path

from transformers.utils.import_utils import define_import_structure


CHECKER_CONFIG = {
    "name": "inits",
    "label": "Model init files",
    "cache_globs": ["src/transformers/models/**/*.py"],
    "check_args": [],
    "fix_args": ["--fix_and_overwrite"],
}

REPO_ROOT = Path(__file__).parent.parent
MODELS_PATH = REPO_ROOT / "src" / "transformers" / "models"
MODELS_INIT_PATH = MODELS_PATH / "__init__.py"

AUTO_GENERATED_HEADER = """#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
#           This file was automatically generated from the model directories in `src/transformers/models`.
#           Do NOT edit this file manually as any edits will be overwritten by auto-generation of the file.
#                      A model is picked up once one of its modules contains an `__all__`.
#                   Regenerate the file with: `python utils/check_inits.py --fix_and_overwrite`
#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""

MODELS_INIT_TEMPLATE = """from typing import TYPE_CHECKING

from ..utils import _LazyModule
from ..utils.import_utils import define_import_structure


if TYPE_CHECKING:
{imports}else:
    import sys

    _file = globals()["__file__"]
    sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__)
"""


def natural_sort_key(name: str) -> tuple[str | int, ...]:
    """Sort key matching `ruff`'s import ordering, which compares digit runs numerically."""
    return tuple(int(part) if part.isdigit() else part for part in re.split(r"(\d+)", name))


def get_model_names() -> list[str]:
    """Return the models exposed by `models/__init__.py`, sorted."""
    import_structure = define_import_structure(str(MODELS_INIT_PATH))
    model_names = {module.split(".")[0] for modules in import_structure.values() for module in modules}
    return sorted(model_names, key=natural_sort_key)


def generate_models_init() -> str:
    """Render the full expected content of `models/__init__.py`."""
    imports = "".join(f"    from .{model_name} import *\n" for model_name in get_model_names())
    return AUTO_GENERATED_HEADER + MODELS_INIT_TEMPLATE.format(imports=imports)


def main(overwrite: bool):
    old_content = MODELS_INIT_PATH.read_text(encoding="utf-8")
    new_content = generate_models_init()

    if old_content == new_content:
        return

    if overwrite:
        MODELS_INIT_PATH.write_text(new_content, encoding="utf-8")
        return

    relative_path = MODELS_INIT_PATH.relative_to(REPO_ROOT)
    diff = "".join(
        difflib.unified_diff(
            old_content.splitlines(keepends=True),
            new_content.splitlines(keepends=True),
            fromfile=f"{relative_path} (on disk)",
            tofile=f"{relative_path} (regenerated)",
        )
    )
    raise Exception(
        f"`{relative_path}` is not consistent with the import structure on disk.\n"
        "Run `make fix-repo` or `python utils/check_inits.py --fix_and_overwrite` to fix it.\n\n"
        f"Diff (on disk → regenerated):\n{diff}"
    )


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--fix_and_overwrite", action="store_true", help="Whether to fix inconsistencies.")
    args = parser.parse_args()
    main(overwrite=args.fix_and_overwrite)