paddlenlp

1
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
2
#
3
# Licensed under the Apache License, Version 2.0 (the "License");
4
# you may not use this file except in compliance with the License.
5
# You may obtain a copy of the License at
6
#
7
# http://www.apache.org/licenses/LICENSE-2.0
8
#
9
# Unless required by applicable law or agreed to in writing, software
10
# distributed under the License is distributed on an "AS IS" BASIS,
11
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
# See the License for the specific language governing permissions and
13
# limitations under the License.
14

15
import os
16

17
from setuptools import Distribution, setup
18
from setuptools.command.install import install
19

20

21
class BinaryDistribution(Distribution):
22
    # when build the package, it will add
23
    # platform name such as "cp37-cp37m-linux_x86_64"
24
    def has_ext_modules(self):
25
        return True
26

27

28
class InstallPlatlib(install):
29
    def finalize_options(self):
30
        install.finalize_options(self)
31
        if self.distribution.has_ext_modules():
32
            self.install_lib = self.install_platlib
33

34

35
if os.name != "nt":
36
    package_data = {"fast_tokenizer": ["core_tokenizers.so", "commit.log"]}
37
    package_data["fast_tokenizer.libs"] = []
38
else:
39
    package_data = {"fast_tokenizer": ["core_tokenizers.pyd", "core_tokenizers.lib", "commit.log"]}
40
    # Add icu dll
41
    package_data["fast_tokenizer.libs"] = ["icuuc70.dll", "icudt70.dll"]
42

43

44
def get_version():
45
    f = open(os.path.join("python", "fast_tokenizer", "__init__.py"))
46
    lines = f.readlines()
47
    version = ""
48
    for line in lines:
49
        if line.startswith("__version__"):
50
            version = line.split("=")[1]
51
            version = version.strip().replace('"', "")
52
            break
53
    return version
54

55

56
long_description = "PaddleNLP Fast Tokenizer Library written in C++ "
57
setup(
58
    name="fast-tokenizer-python",
59
    version=get_version(),
60
    author="PaddlePaddle Speech and Language Team",
61
    author_email="paddlesl@baidu.com",
62
    description=long_description,
63
    long_description=long_description,
64
    zip_safe=False,
65
    url="https://github.com/PaddlePaddle/PaddleNLP/fast_tokenizer",
66
    package_dir={"": "python"},
67
    packages=[
68
        "fast_tokenizer",
69
        "fast_tokenizer.tokenizers_impl",
70
        "fast_tokenizer.normalizers",
71
        "fast_tokenizer.pretokenizers",
72
        "fast_tokenizer.models",
73
        "fast_tokenizer.postprocessors",
74
        "fast_tokenizer.libs",
75
        "fast_tokenizer.decoders",
76
    ],
77
    package_data=package_data,
78
    extras_require={"test": ["pytest>=6.0"]},
79
    python_requires=">=3.6",
80
    cmdclass={"install": InstallPlatlib},
81
    license="Apache 2.0",
82
    distclass=BinaryDistribution,
83
    classifiers=[
84
        "Development Status :: 5 - Production/Stable",
85
        "Operating System :: OS Independent",
86
        "Intended Audience :: Developers",
87
        "Intended Audience :: Education",
88
        "Intended Audience :: Science/Research",
89
        "License :: OSI Approved :: Apache Software License",
90
        "Programming Language :: C++",
91
        "Programming Language :: Python :: 3.6",
92
        "Programming Language :: Python :: 3.7",
93
        "Programming Language :: Python :: 3.8",
94
        "Programming Language :: Python :: 3.9",
95
        "Programming Language :: Python :: 3.10",
96
    ],
97
)
98