haystack

1
[build-system]
2
requires = [
3
  "hatchling>=1.8.0",
4
]
5
build-backend = "hatchling.build"
6

7
[project]
8
name = "haystack-ai"
9
dynamic = [
10
  "version",
11
]
12
description = "LLM framework to build customizable, production-ready LLM applications. Connect components (models, vector DBs, file converters) to pipelines or agents that can interact with your data."
13
readme = "README.md"
14
license = "Apache-2.0"
15
requires-python = ">=3.8"
16
authors = [
17
  { name = "deepset.ai", email = "malte.pietsch@deepset.ai" },
18
]
19
keywords = [
20
  "BERT",
21
  "QA",
22
  "Question-Answering",
23
  "Reader",
24
  "Retriever",
25
  "albert",
26
  "language-model",
27
  "mrc",
28
  "roberta",
29
  "search",
30
  "semantic-search",
31
  "squad",
32
  "transfer-learning",
33
  "transformer",
34
]
35
classifiers = [
36
  "Development Status :: 5 - Production/Stable",
37
  "Intended Audience :: Science/Research",
38
  "License :: Freely Distributable",
39
  "License :: OSI Approved :: Apache Software License",
40
  "Operating System :: OS Independent",
41
  "Programming Language :: Python",
42
  "Programming Language :: Python :: 3",
43
  "Programming Language :: Python :: 3.8",
44
  "Programming Language :: Python :: 3.9",
45
  "Programming Language :: Python :: 3.10",
46
  "Topic :: Scientific/Engineering :: Artificial Intelligence",
47
]
48
dependencies = [
49
  "pandas",
50
  "haystack-bm25",
51
  "tqdm",
52
  "tenacity",
53
  "lazy-imports",
54
  "openai>=1.1.0",
55
  "Jinja2",
56
  "posthog",  # telemetry
57
  "pyyaml",
58
  "more-itertools",  # TextDocumentSplitter
59
  "networkx", # Pipeline graphs
60
  "typing_extensions>=4.7", # typing support for Python 3.8
61
  "boilerpy3", # Fulltext extraction from HTML pages
62
]
63

64
[tool.hatch.envs.default]
65
dependencies = [
66
  "pre-commit",
67
  # Type check
68
  "mypy",
69
  # Test
70
  "pytest",
71
  "pytest-cov",
72
  "pytest-custom_exit_code",  # used in the CI
73
  "pytest-asyncio",
74
  "flaky",
75
  "responses",
76
  "tox",
77
  "coverage",
78
  "python-multipart",
79
  "psutil",
80
  # Linting
81
  "pylint",
82
  "ruff",
83
  # Documentation
84
  "toml",
85
  "reno",
86
  # dulwich is a reno dependency, they pin it at >=0.15.0 so pip takes ton of time to resolve the dependency tree.
87
  # We pin it here to avoid taking too much time.
88
  # https://opendev.org/openstack/reno/src/branch/master/requirements.txt#L7
89
  "dulwich>=0.21.0,<1.0.0",
90
  # Version specified following Black stability policy:
91
  # https://black.readthedocs.io/en/stable/the_black_code_style/index.html#stability-policy
92
  "black[jupyter]~=23.0",
93
]
94

95
[tool.hatch.envs.default.scripts]
96
format = "black ."
97
format-check = "black --check ."
98

99
[tool.hatch.envs.test]
100
extra-dependencies = [
101
  "transformers[torch,sentencepiece]==4.37.2",  # ExtractiveReader, TransformersSimilarityRanker, LocalWhisperTranscriber, HFGenerators...
102
  "spacy>=3.7,<3.8",  # NamedEntityExtractor
103
  "spacy-curated-transformers>=0.2,<=0.3",  # NamedEntityExtractor
104
  "en-core-web-trf @ https://github.com/explosion/spacy-models/releases/download/en_core_web_trf-3.7.3/en_core_web_trf-3.7.3-py3-none-any.whl",  # NamedEntityExtractor
105

106
  # Converters
107
  "pypdf",  # PyPDFConverter
108
  "markdown-it-py",  # MarkdownToDocument
109
  "mdit_plain",  # MarkdownToDocument
110
  "tika",  # TikaDocumentConverter
111
  "azure-ai-formrecognizer>=3.2.0b2",  # AzureOCRDocumentConverter
112
  "langdetect",  # TextLanguageRouter and DocumentLanguageClassifier
113
  "sentence-transformers>=2.2.0",  # SentenceTransformersTextEmbedder and SentenceTransformersDocumentEmbedder
114
  "openai-whisper>=20231106",  # LocalWhisperTranscriber
115
  "chroma-haystack",  # pipeline predefined templates
116

117
  # OpenAPI
118
  "jsonref",  # OpenAPIServiceConnector, OpenAPIServiceToFunctions
119
  "openapi3",
120

121
  # Validation
122
  "jsonschema",
123

124
  # Tracing
125
  "opentelemetry-sdk",
126
  "ddtrace",
127

128
  # Structured logging
129
  "structlog",
130
]
131

132
[tool.hatch.envs.test.scripts]
133
e2e = "pytest e2e"
134
unit = 'pytest --cov-report xml:coverage.xml --cov="haystack" -m "not integration" test'
135
integration = 'pytest --maxfail=5 -m "integration" test'
136
integration-mac = 'pytest --maxfail=5 -m "integration" test -k "not tika"'
137
integration-windows = 'pytest --maxfail=5 -m "integration" test -k "not tika"'
138
types = "mypy --install-types --non-interactive --cache-dir=.mypy_cache/ {args:haystack}"
139
lint = [
140
  "ruff {args:haystack}",
141
  "pylint -ry -j 0 {args:haystack}"
142
]
143
lint-fix = [
144
  "black .",
145
  "ruff {args:haystack} --fix",
146
]
147

148
[tool.hatch.envs.readme]
149
detached = true  # To avoid installing the dependencies from the default environment
150
dependencies = [
151
  "haystack-pydoc-tools",
152
]
153

154
[tool.hatch.envs.readme.scripts]
155
sync = "./.github/utils/pydoc-markdown.sh"
156
delete-outdated = "python ./.github/utils/delete_outdated_docs.py {args}"
157

158
[tool.hatch.envs.snippets]
159
extra-dependencies = [
160
  "torch",
161
  "pydantic",
162
]
163

164
[project.urls]
165
"CI: GitHub" = "https://github.com/deepset-ai/haystack/actions"
166
"Docs: RTD" = "https://haystack.deepset.ai/overview/intro"
167
"GitHub: issues" = "https://github.com/deepset-ai/haystack/issues"
168
"GitHub: repo" = "https://github.com/deepset-ai/haystack"
169
Homepage = "https://github.com/deepset-ai/haystack"
170

171
[tool.hatch.version]
172
path = "VERSION.txt"
173
pattern = "(?P<version>.+)"
174

175
[tool.hatch.metadata]
176
allow-direct-references = true
177

178
[tool.hatch.build.targets.sdist]
179
include = [
180
  "/haystack",
181
  "/VERSION.txt",
182
]
183

184
[tool.hatch.build.targets.wheel]
185
packages = [
186
  "haystack",
187
]
188

189
[tool.black]
190
line-length = 120
191
skip_magic_trailing_comma = true  # For compatibility with pydoc>=4.6, check if still needed.
192

193
[tool.codespell]
194
ignore-words-list = "ans,astroid,nd,ned,nin,ue,rouge,ist"
195
quiet-level = 3
196
skip = "test/nodes/*,test/others/*,test/samples/*"
197

198
[tool.pylint.'MESSAGES CONTROL']
199
max-line-length=120
200
disable = [
201

202
  # To keep
203
  "fixme",
204
  "c-extension-no-member",
205

206
  # To review:
207
  "missing-docstring",
208
  "unused-argument",
209
  "no-member",
210
  "line-too-long",
211
  "protected-access",
212
  "too-few-public-methods",
213
  "raise-missing-from",
214
  "invalid-name",
215
  "duplicate-code",
216
  "arguments-differ",
217
  "consider-using-f-string",
218
  "no-else-return",
219
  "attribute-defined-outside-init",
220
  "super-with-arguments",
221
  "redefined-builtin",
222
  "abstract-method",
223
  "unspecified-encoding",
224
  "unidiomatic-typecheck",
225
  "no-name-in-module",
226
  "consider-using-with",
227
  "redefined-outer-name",
228
  "arguments-renamed",
229
  "unnecessary-pass",
230
  "broad-except",
231
  "unnecessary-comprehension",
232
  "subprocess-run-check",
233
  "singleton-comparison",
234
  "consider-iterating-dictionary",
235
  "undefined-loop-variable",
236
  "consider-using-in",
237
  "bare-except",
238
  "unexpected-keyword-arg",
239
  "simplifiable-if-expression",
240
  "use-list-literal",
241
  "broad-exception-raised",
242

243
  # To review later
244
  "cyclic-import",
245
  "import-outside-toplevel",
246
  "deprecated-method",
247
]
248
[tool.pylint.'DESIGN']
249
max-args = 38  # Default is 5
250
max-attributes = 28  # Default is 7
251
max-branches = 34  # Default is 12
252
max-locals = 45  # Default is 15
253
max-module-lines = 2468  # Default is 1000
254
max-nested-blocks = 9  # Default is 5
255
max-statements = 206  # Default is 50
256
[tool.pylint.'SIMILARITIES']
257
min-similarity-lines=6
258

259
[tool.pytest.ini_options]
260
minversion = "6.0"
261
addopts = "--strict-markers"
262
markers = [
263
  "unit: unit tests",
264
  "integration: integration tests",
265

266
  "generator: generator tests",
267
  "summarizer: summarizer tests",
268
  "embedding_dim: uses a document store with non-default embedding dimension (e.g @pytest.mark.embedding_dim(128))",
269

270
  "tika: requires Tika container",
271
  "parsr: requires Parsr container",
272
  "ocr: requires Tesseract",
273

274
  "elasticsearch: requires Elasticsearch container",
275
  "weaviate: requires Weaviate container",
276
  "pinecone: requires Pinecone credentials",
277
  "faiss: uses FAISS",
278
  "opensearch",
279
  "document_store",
280
]
281
log_cli = true
282

283
[tool.mypy]
284
warn_return_any = false
285
warn_unused_configs = true
286
ignore_missing_imports = true
287

288
[tool.ruff]
289
line-length = 1486
290
target-version = "py38"
291

292
[tool.ruff.lint]
293
select = [
294
  "AIR",    # Airflow
295
  "ASYNC",  # flake8-async
296
  "C4",     # flake8-comprehensions
297
  "C90",    # McCabe cyclomatic complexity
298
  "CPY",    # flake8-copyright
299
  "DJ",     # flake8-django
300
  "E501",   # Long lines
301
  "EXE",    # flake8-executable
302
  "F",      # Pyflakes
303
  "FURB",   # refurb
304
  "INT",    # flake8-gettext
305
  "PERF",   # Perflint
306
  "PL",     # Pylint
307
  "Q",      # flake8-quotes
308
  "SIM",    # flake8-simplify
309
  "SLOT",   # flake8-slots
310
  "T10",    # flake8-debugger
311
  "W",      # pycodestyle
312
  "YTT",    # flake8-2020
313
  "I"       # isort
314
  # "E",    # pycodestyle
315
  # "NPY",  # NumPy-specific rules
316
  # "PD",   # pandas-vet
317
  # "PT",   # flake8-pytest-style
318
  # "UP",   # pyupgrade
319
]
320

321
ignore = [
322
  "F401",     # unused-import
323
  "PERF203",	# `try`-`except` within a loop incurs performance overhead
324
  "PERF401",	# Use a list comprehension to create a transformed list
325
  "PLR1714",  # repeated-equality-comparison
326
  "PLR5501",  # collapsible-else-if
327
  "PLW0603",  # global-statement
328
  "PLW1510",  # subprocess-run-without-check
329
  "PLW2901",  # redefined-loop-name
330
  "SIM108",   # if-else-block-instead-of-if-exp
331
  "SIM115",   # open-file-with-context-handler
332
  "SIM118",   # in-dict-keys
333
]
334

335
[tool.ruff.lint.mccabe]
336
max-complexity = 28
337

338
[tool.ruff.lint.per-file-ignores]
339
"examples/basic_qa_pipeline.py" = ["C416"]
340
"haystack/preview/testing/document_store.py" = ["C416", "F821"]
341
"haystack/telemetry.py" = ["F821"]
342

343
[tool.ruff.lint.pylint]
344
allow-magic-value-types = ["float", "int", "str"]
345
max-args = 38  # Default is 5
346
max-branches = 32  # Default is 12
347
max-public-methods = 90  # Default is 20
348
max-returns = 9  # Default is 6
349
max-statements = 105  # Default is 50
350

351
[tool.coverage.run]
352
omit = [
353
    "haystack/testing/*",
354
]
355