/
githubmirror
/
transformers
Обзор
Документация
Войти
/
githubmirror
/
transformers
Код
Запросы
0
Пакеты
0
Релизы
0
Аналитика
Безопасность
backup-before-rebase
setup.py
363 строки
13 KB
Arthur
Clean: revert rebase-noise files to origin/main
18 май 2026, 04:22
18 май 2026, 04:22
a9bebae
Код
Авторство
О чём код?
# Copyright 2021 The HuggingFace Team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Simple check list from AllenNLP repo: https://github.com/allenai/allennlp/blob/main/setup.py To create the package for pypi. 1. Create the release branch named: v<RELEASE>-release, for example v4.19-release. For a patch release checkout the current release branch. If releasing on a special branch, copy the updated README.md on the main branch for the commit you will make for the post-release and run `make fix-repo` on the main branch as well. 2. Run `make pre-release` (or `make pre-patch` for a patch release) and commit these changes with the message: "Release: <VERSION>" and push. 3. Go back to the main branch and run `make post-release` then `make fix-repo`. Commit these changes with the message "v<NEXT_VERSION>.dev.0" and push to main. # If you were just cutting the branch in preparation for a release, you can stop here for now. 4. Wait for the tests on the release branch to be completed and be green (otherwise revert and fix bugs) 5. On the release branch, add a tag in git to mark the release: "git tag v<VERSION> -m 'Adds tag v<VERSION> for pypi' " Push the tag to git: git push --tags origin v<RELEASE>-release 6. Have a core maintainer review and approve the deployment to pypi. """ import re import shutil import sys from pathlib import Path from setuptools import Command, find_packages, setup # Supported Python version range (min, max) SUPPORTED_PYTHON_VERSIONS = (10, 14) # 3.10 to 3.14 PYTHON_MINOR_VERSION = sys.version_info.minor # Remove stale transformers.egg-info directory to avoid https://github.com/pypa/pip/issues/5466 stale_egg_info = Path(__file__).parent / "transformers.egg-info" if stale_egg_info.exists(): print( f"Warning: {stale_egg_info} exists.\n\n" "If you recently updated transformers to 3.0 or later, this is expected,\n" "but it may prevent transformers from installing in editable mode.\n\n" "This directory is automatically generated by Python's packaging tools.\n" "I will remove it now.\n\n" "See https://github.com/pypa/pip/issues/5466 for details.\n" ) shutil.rmtree(stale_egg_info) # IMPORTANT: # 1. all dependencies should be listed here with their version requirements if any # 2. once modified, run: `make fix-repo` to update src/transformers/dependency_versions_table.py _deps = [ "Pillow>=10.0.1,<=15.0", "accelerate>=1.1.0", "av", "beautifulsoup4", "blobfile", "codecarbon>=2.8.1", "datasets>=2.15.0", # We need either this pin or pyarrow<21.0.0 "deepspeed>=0.9.3", "dill<0.3.5", "evaluate>=0.4.6", "faiss-cpu", "fastapi", "filelock", "fugashi>=1.0", "GitPython<3.1.19", "hf-doc-builder", "huggingface-hub>=1.5.0,<2.0", "ipadic>=1.0.0,<2.0", "jinja2>=3.1.0", "jmespath>=1.0.1", "kenlm", "kernels>=0.12.0,<0.13", "librosa", "mistral-common[image]>=1.10.0", "nltk<=3.8.1", "num2words", "numpy>=1.17", "openai>=1.98.0", "opencv-python", "optimum-benchmark>=0.3.0", "optuna", "pandas<2.3.0", # `datasets` requires `pandas` while `pandas==2.3.0` has issues with CircleCI on 2025/06/05 "packaging>=20.0", "parameterized>=0.9", # older version of parameterized cause pytest collection to fail on .expand "peft>=0.18.0", "phonemizer", "protobuf", "psutil", "pyyaml>=5.1", "pydantic>=2", "pytest>=7.2.0,<9.0.0", "pytest-asyncio>=1.2.0", "pytest-random-order", "pytest-rerunfailures<16.0", "pytest-timeout", "pytest-env", "pytest-xdist", "pytest-order", "python>=3.10.0", "regex>=2025.10.22", "rhoknp>=1.1.0,<1.3.1", "rjieba", "rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1", "ruff==0.14.10", # When bumping `transformers-mlinter`, sync repo-local rule overrides from # `utils/rules.toml` back into the released package. "transformers-mlinter==0.1.1", "ty==0.0.20", # `sacrebleu` not used in `transformers`. However, it is needed in several tests, when a test calls # `evaluate.load("sacrebleu")`. This metric is used in the examples that we use to test the `Trainer` with, in the # `Trainer` tests (see references to `run_translation.py`). "sacrebleu>=1.4.12,<2.0.0", "sacremoses", "safetensors>=0.4.3", "sagemaker>=2.31.0", "schedulefree>=1.2.6", "scikit-learn", "scipy", "sentencepiece>=0.1.91,!=0.1.92", "starlette", "sudachipy>=0.6.6", "sudachidict_core>=20220729", "tensorboard", "timeout-decorator", "tomli", "tiktoken", "timm>=1.0.23", "tokenizers>=0.22.0,<=0.23.0", "torch>=2.4", "torchaudio", "torchvision", "pyctcdecode>=0.4.0", "tqdm>=4.27", "typer", "unidic>=1.0.2", "unidic_lite>=1.0.7", "urllib3<2.0.0", "uvicorn", "pytest-rich", "libcst", "rich", "ray[tune]>=2.7.0", "opentelemetry-api", "opentelemetry-exporter-otlp", "opentelemetry-sdk", ] # This is a lookup table with items like: {"tokenizers": "tokenizers==0.9.4", "packaging": "packaging"}, i.e. # some of the values are versioned whereas others aren't. deps = {b: a for a, b in (re.findall(r"^(([^!=<>~ ]+)(?:[!=<>~ ].*)?$)", x)[0] for x in _deps)} def deps_list(*pkgs): return [deps[pkg] for pkg in pkgs] extras = {} extras["torch"] = deps_list("torch", "accelerate") extras["vision"] = deps_list("torchvision", "Pillow") extras["audio"] = deps_list("torchaudio", "librosa", "pyctcdecode", "phonemizer") if PYTHON_MINOR_VERSION < 13: extras["audio"] += deps_list("kenlm") extras["video"] = deps_list("av") extras["timm"] = deps_list("timm") extras["quality"] = deps_list( "datasets", "ruff", "GitPython", "urllib3", "libcst", "rich", "ty", "tomli", "transformers-mlinter" ) extras["docs"] = deps_list("hf-doc-builder") extras["kernels"] = deps_list("kernels") extras["sentencepiece"] = deps_list("sentencepiece", "protobuf") extras["tiktoken"] = deps_list("tiktoken", "blobfile") extras["mistral-common"] = deps_list("mistral-common[image]") extras["chat_template"] = deps_list("jinja2", "jmespath") extras["sklearn"] = deps_list("scikit-learn") extras["accelerate"] = deps_list("accelerate") extras["retrieval"] = deps_list("faiss-cpu", "datasets") extras["sagemaker"] = deps_list("sagemaker") extras["deepspeed"] = deps_list("deepspeed", "accelerate") extras["optuna"] = deps_list("optuna") extras["integrations"] = deps_list("kernels", "optuna", "codecarbon") if PYTHON_MINOR_VERSION < 14: extras["ray"] = deps_list("ray[tune]") extras["integrations"] += extras["ray"] extras["codecarbon"] = deps_list("codecarbon") extras["serving"] = deps_list("openai", "pydantic", "uvicorn", "fastapi", "starlette", "rich") + extras["torch"] extras["num2words"] = deps_list("num2words") extras["benchmark"] = deps_list("optimum-benchmark") extras["ja"] = deps_list("fugashi", "ipadic", "unidic_lite", "unidic", "rhoknp") if PYTHON_MINOR_VERSION < 14: extras["ja"] += deps_list("sudachipy", "sudachidict_core") # OpenTelemetry dependencies for metrics collection in continuous batching extras["open-telemetry"] = deps_list("opentelemetry-api", "opentelemetry-exporter-otlp", "opentelemetry-sdk") extras["testing"] = ( deps_list( "pytest", "pytest-asyncio", "pytest-random-order", "pytest-rich", "pytest-xdist", "pytest-order", "pytest-rerunfailures", "pytest-timeout", "pytest-env", "timeout-decorator", "parameterized", "psutil", "dill", "evaluate", "rouge-score", "nltk", "sacremoses", "rjieba", "beautifulsoup4", "tensorboard", "sacrebleu", # needed in trainer tests, see references to `run_translation.py` "filelock", # filesystem locks, e.g., to prevent parallel downloads ) + extras["docs"] + extras["quality"] + extras["retrieval"] + extras["sentencepiece"] + extras["serving"] ) extras["testing"] += extras["mistral-common"] extras["deepspeed-testing"] = extras["deepspeed"] + extras["testing"] + extras["optuna"] + extras["sentencepiece"] extras["all"] = ( extras["torch"] + extras["vision"] + extras["audio"] + extras["video"] + extras["kernels"] + extras["timm"] + extras["sentencepiece"] + extras["tiktoken"] + extras["chat_template"] + extras["num2words"] ) extras["all"] += extras["mistral-common"] extras["dev"] = extras["all"] + extras["testing"] + extras["ja"] + extras["sklearn"] # Those define the hard dependencies of `transformers` install_requires = [ deps["huggingface-hub"], deps["numpy"], deps["packaging"], # utilities from PyPA to e.g., compare versions deps["pyyaml"], # used for the model cards metadata deps["regex"], # for OpenAI GPT deps["tokenizers"], deps["typer"], # CLI utilities. In practice, already a dependency of huggingface_hub but we use it as well deps["safetensors"], deps["tqdm"], # progress bars in model download and training scripts ] class DepsTableUpdateCommand(Command): """ A custom distutils command that updates the dependency table. usage: python setup.py deps_table_update """ description = "build runtime dependency table" user_options = [ # format: (long option, short option, description). ("dep-table-update", None, "updates src/transformers/dependency_versions_table.py"), ] def initialize_options(self): pass def finalize_options(self): pass def run(self): if SUPPORTED_PYTHON_VERSIONS[0] > PYTHON_MINOR_VERSION: print( f"Table updated only when running 3.{SUPPORTED_PYTHON_VERSIONS[0]}.x, detected version is {sys.version}." ) return entries = "\n".join([f' "{k}": "{v}",' for k, v in deps.items()]) content = [ "# THIS FILE HAS BEEN AUTOGENERATED. To update:", "# 1. modify the `_deps` dict in setup.py", "# 2. run `make fix-repo``", "deps = {", entries, "}", "", ] target = "src/transformers/dependency_versions_table.py" with open(target, "w", encoding="utf-8", newline="\n") as f: f.write("\n".join(content)) if __name__ == "__main__": # Generate python_requires from supported version range min_version, max_version = SUPPORTED_PYTHON_VERSIONS python_requires = f">=3.{min_version}.0" # Generate Python version classifiers dynamically python_classifiers = ["Programming Language :: Python :: 3"] for minor in range(min_version, max_version + 1): python_classifiers.append(f"Programming Language :: Python :: 3.{minor}") setup( name="transformers", version="5.8.0.dev0", # expected format is one of x.y.z.dev0, or x.y.z.rc1 or x.y.z (no to dashes, yes to dots) author="The Hugging Face team (past and future) with the help of all our contributors (https://github.com/huggingface/transformers/graphs/contributors)", author_email="transformers@huggingface.co", description="Transformers: the model-definition framework for state-of-the-art machine learning models in text, vision, audio, and multimodal models, for both inference and training.", long_description=open("README.md", "r", encoding="utf-8").read(), long_description_content_type="text/markdown", keywords="machine-learning nlp python pytorch transformer llm vlm deep-learning inference training model-hub pretrained-models llama gemma qwen", license="Apache 2.0 License", url="https://github.com/huggingface/transformers", package_dir={"": "src"}, packages=find_packages("src"), include_package_data=True, package_data={"": ["**/*.cu", "**/*.cpp", "**/*.cuh", "**/*.h", "**/*.pyx", "py.typed"]}, zip_safe=False, extras_require=extras, entry_points={"console_scripts": ["transformers=transformers.cli.transformers:main"]}, python_requires=python_requires, install_requires=list(install_requires), classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Intended Audience :: Education", "Intended Audience :: Science/Research", "Operating System :: OS Independent", ] + python_classifiers + [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ], cmdclass={"deps_table_update": DepsTableUpdateCommand}, )