alignment-handbook

1
# Copyright 2023 The HuggingFace Team. All rights reserved.
2
#
3
# Licensed under the Apache License, Version 2.0 (the "License");
4
# you may not use this file except in compliance with the License.
5
# You may obtain a copy of the License at
6
#
7
#     http://www.apache.org/licenses/LICENSE-2.0
8
#
9
# Unless required by applicable law or agreed to in writing, software
10
# distributed under the License is distributed on an "AS IS" BASIS,
11
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
# See the License for the specific language governing permissions and
13
# limitations under the License.
14
#
15
# Adapted from huggingface/transformers: https://github.com/huggingface/transformers/blob/21a2d900eceeded7be9edc445b56877b95eda4ca/setup.py
16

17

18
import re
19
import shutil
20
from pathlib import Path
21

22
from setuptools import find_packages, setup
23

24

25
# Remove stale alignment.egg-info directory to avoid https://github.com/pypa/pip/issues/5466
26
stale_egg_info = Path(__file__).parent / "alignment.egg-info"
27
if stale_egg_info.exists():
28
    print(
29
        (
30
            "Warning: {} exists.\n\n"
31
            "If you recently updated alignment, this is expected,\n"
32
            "but it may prevent alignment from installing in editable mode.\n\n"
33
            "This directory is automatically generated by Python's packaging tools.\n"
34
            "I will remove it now.\n\n"
35
            "See https://github.com/pypa/pip/issues/5466 for details.\n"
36
        ).format(stale_egg_info)
37
    )
38
    shutil.rmtree(stale_egg_info)
39

40

41
# IMPORTANT: all dependencies should be listed here with their version requirements, if any.
42
#   * If a dependency is fast-moving (e.g. transformers), pin to the exact version
43
_deps = [
44
    "accelerate==0.27.2",
45
    "bitsandbytes==0.41.2.post2",
46
    "black==23.1.0",
47
    "datasets==2.14.6",
48
    "deepspeed==0.12.2",
49
    "einops>=0.6.1",
50
    "evaluate==0.4.0",
51
    "flake8>=6.0.0",
52
    "hf-doc-builder>=0.4.0",
53
    "hf_transfer>=0.1.4",
54
    "huggingface-hub>=0.19.2,<1.0",
55
    "isort>=5.12.0",
56
    "ninja>=1.11.1",
57
    "numpy>=1.24.2",
58
    "packaging>=23.0",
59
    "parameterized>=0.9.0",
60
    "peft==0.7.1",
61
    "protobuf<=3.20.2",  # Needed to avoid conflicts with `transformers`
62
    "pytest",
63
    "safetensors>=0.3.3",
64
    "sentencepiece>=0.1.99",
65
    "scipy",
66
    "tensorboard",
67
    "torch==2.1.2",
68
    "transformers>=4.38.2",  # Fixes RoPE computation
69
    "trl==0.7.10",
70
    "jinja2>=3.0.0",
71
    "tqdm>=4.64.1",
72
]
73

74
# this is a lookup table with items like:
75
#
76
# tokenizers: "tokenizers==0.9.4"
77
# packaging: "packaging"
78
#
79
# some of the values are versioned whereas others aren't.
80
deps = {b: a for a, b in (re.findall(r"^(([^!=<>~ \[\]]+)(?:\[[^\]]+\])?(?:[!=<>~ ].*)?$)", x)[0] for x in _deps)}
81

82

83
def deps_list(*pkgs):
84
    return [deps[pkg] for pkg in pkgs]
85

86

87
extras = {}
88
extras["tests"] = deps_list("pytest", "parameterized")
89
extras["torch"] = deps_list("torch")
90
extras["quality"] = deps_list("black", "isort", "flake8")
91
extras["docs"] = deps_list("hf-doc-builder")
92
extras["dev"] = extras["docs"] + extras["quality"] + extras["tests"]
93

94
# core dependencies shared across the whole project - keep this to a bare minimum :)
95
install_requires = [
96
    deps["accelerate"],
97
    deps["bitsandbytes"],
98
    deps["einops"],
99
    deps["evaluate"],
100
    deps["datasets"],
101
    deps["deepspeed"],
102
    deps["hf_transfer"],
103
    deps["huggingface-hub"],
104
    deps["jinja2"],
105
    deps["ninja"],
106
    deps["numpy"],
107
    deps["packaging"],  # utilities from PyPA to e.g., compare versions
108
    deps["peft"],
109
    deps["protobuf"],
110
    deps["safetensors"],
111
    deps["sentencepiece"],
112
    deps["scipy"],
113
    deps["tensorboard"],
114
    deps["tqdm"],  # progress bars in model download and training scripts
115
    deps["transformers"],
116
    deps["trl"],
117
]
118

119
setup(
120
    name="alignment-handbook",
121
    version="0.4.0.dev0",  # expected format is one of x.y.z.dev0, or x.y.z.rc1 or x.y.z (no to dashes, yes to dots)
122
    author="The Hugging Face team (past and future)",
123
    author_email="lewis@huggingface.co",
124
    description="The Alignment Handbook",
125
    long_description=open("README.md", "r", encoding="utf-8").read(),
126
    long_description_content_type="text/markdown",
127
    keywords="nlp deep learning rlhf llm",
128
    license="Apache",
129
    url="https://github.com/huggingface/alignment-handbook",
130
    package_dir={"": "src"},
131
    packages=find_packages("src"),
132
    zip_safe=False,
133
    extras_require=extras,
134
    python_requires=">=3.10.9",
135
    install_requires=install_requires,
136
    classifiers=[
137
        "Development Status :: 3 - Alpha",
138
        "Intended Audience :: Developers",
139
        "Intended Audience :: Education",
140
        "Intended Audience :: Science/Research",
141
        "License :: OSI Approved :: Apache Software License",
142
        "Operating System :: OS Independent",
143
        "Programming Language :: Python :: 3",
144
        "Programming Language :: Python :: 3.10",
145
        "Topic :: Scientific/Engineering :: Artificial Intelligence",
146
    ],
147
)
148