mindsdb

1
import os
2
import glob
3

4
from setuptools import find_packages, setup
5

6
# A special env var that allows us to disable the installation of the default extras for advanced users / containers / etc
7
MINDSDB_PIP_INSTALL_DEFAULT_EXTRAS = (
8
    True
9
    if os.getenv("MINDSDB_PIP_INSTALL_DEFAULT_EXTRAS", "true").lower() == "true"
10
    else False
11
)
12
DEFAULT_PIP_EXTRAS = [
13
    line.split("#")[0].rstrip()
14
    for line in open("default_handlers.txt").read().splitlines()
15
    if not line.strip().startswith("#")
16
]
17

18

19
class Deps:
20
    pkgs = []
21
    pkgs_exclude = ["tests", "tests.*"]
22
    new_links = []
23
    extras = {}
24

25

26
about = {}
27
with open("mindsdb/__about__.py") as fp:
28
    exec(fp.read(), about)
29

30

31
with open("README.md", "r", encoding="utf8") as fh:
32
    long_description = fh.read()
33

34

35
def expand_requirements_links(requirements: list) -> list:
36
    """Expand requirements that contain links to other requirement files"""
37
    to_add = []
38
    to_remove = []
39

40
    for requirement in requirements:
41
        if requirement.startswith("-r "):
42
            if os.path.exists(requirement.split()[1]):
43
                with open(requirement.split()[1]) as fh:
44
                    to_add += expand_requirements_links(
45
                        [req.strip() for req in fh.read().splitlines()]
46
                    )
47
            to_remove.append(requirement)
48

49
    for req in to_remove:
50
        requirements.remove(req)
51
    for req in to_add:
52
        requirements.append(req)
53

54
    return list(set(requirements))  # Remove duplicates
55

56

57
def define_deps():
58
    """Reads requirements.txt requirements-extra.txt files and preprocess it
59
    to be feed into setuptools.
60

61
    This is the only possible way (we found)
62
    how requirements.txt can be reused in setup.py
63
    using dependencies from private github repositories.
64

65
    Links must be appendend by `-{StringWithAtLeastOneNumber}`
66
    or something like that, so e.g. `-9231` works as well as
67
    `1.1.0`. This is ignored by the setuptools, but has to be there.
68

69
    Warnings:
70
        to make pip respect the links, you have to use
71
        `--process-dependency-links` switch. So e.g.:
72
        `pip install --process-dependency-links {git-url}`
73

74
    Returns:
75
         list of packages, extras and dependency links.
76
    """
77
    with open(os.path.normpath('requirements/requirements.txt')) as req_file:
78
        defaults = [req.strip() for req in req_file.read().splitlines()]
79

80
    links = []
81
    requirements = []
82
    for r in defaults:
83
        if 'git+https' in r:
84
            pkg = r.split('#')[-1]
85
            links.append(r + '-9876543210')
86
            requirements.append(pkg.replace('egg=', ''))
87
        else:
88
            requirements.append(r.strip())
89

90
    extra_requirements = {}
91
    full_requirements = []
92
    for fn in os.listdir(os.path.normpath('./requirements')):
93
        extra = []
94
        if fn.startswith('requirements-') and fn.endswith('.txt'):
95
            extra_name = fn.replace('requirements-', '').replace('.txt', '')
96
            with open(os.path.normpath(f"./requirements/{fn}")) as fp:
97
                extra = [req.strip() for req in fp.read().splitlines()]
98
            extra_requirements[extra_name] = extra
99
            full_requirements += extra
100

101
    extra_requirements['all_extras'] = list(set(full_requirements))
102

103
    full_handlers_requirements = []
104
    handlers_dir_path = os.path.normpath('./mindsdb/integrations/handlers')
105
    for fn in os.listdir(handlers_dir_path):
106
        if os.path.isdir(os.path.join(handlers_dir_path, fn)) and fn.endswith(
107
            "_handler"
108
        ):
109
            extra = []
110
            for req_file_path in glob.glob(
111
                os.path.join(handlers_dir_path, fn, "requirements*.txt")
112
            ):
113
                extra_name = fn.replace("_handler", "")
114
                file_name = os.path.basename(req_file_path)
115
                if file_name != "requirements.txt":
116
                    extra_name += "-" + file_name.replace("requirements_", "").replace(
117
                        ".txt", ""
118
                    )
119

120
                # If requirements.txt in our handler folder, import them as our extra's requirements
121
                if os.path.exists(req_file_path):
122
                    with open(req_file_path) as fp:
123
                        extra = expand_requirements_links(
124
                            [req.strip() for req in fp.read().splitlines()]
125
                        )
126

127
                    extra_requirements[extra_name] = extra
128
                    full_handlers_requirements += extra
129

130
                # Even with no requirements in our handler, list the handler as an extra (with no reqs)
131
                else:
132
                    extra_requirements[extra_name] = []
133

134
                # If this is a default extra and if we want to install defaults (enabled by default)
135
                #   then add it to the default requirements needing to install
136
                if (
137
                    MINDSDB_PIP_INSTALL_DEFAULT_EXTRAS
138
                    and extra_name in DEFAULT_PIP_EXTRAS
139
                    and extra
140
                ):
141
                    requirements += extra
142

143
    extra_requirements['all_handlers_extras'] = list(set(full_handlers_requirements))
144

145
    Deps.pkgs = requirements
146
    Deps.extras = extra_requirements
147
    Deps.new_links = links
148

149
    return Deps
150

151

152
deps = define_deps()
153

154
setup(
155
    name=about['__title__'],
156
    version=about['__version__'],
157
    url=about['__github__'],
158
    download_url=about['__pypi__'],
159
    license=about['__license__'],
160
    author=about['__author__'],
161
    author_email=about['__email__'],
162
    description=about['__description__'],
163
    long_description=long_description,
164
    long_description_content_type="text/markdown",
165
    packages=find_packages(exclude=deps.pkgs_exclude),
166
    install_requires=deps.pkgs,
167
    dependency_links=deps.new_links,
168
    extras_require=deps.extras,
169
    include_package_data=True,
170
    classifiers=[
171
        "Programming Language :: Python :: 3",
172
        "Operating System :: OS Independent",
173
    ],
174
    python_requires=">=3.8,<=3.10",
175
)
176