DataProcessingFramework

1
[build-system]
2
requires = [
3
    "hatchling >= 1.13.0",
4
    "wheel",
5
    "Cython~=3.0.5"
6
]
7
build-backend = "hatchling.build"
8

9
[project]
10
name = 'DPF'
11
dynamic = [
12
  'version'
13
]
14
readme = 'README.md'
15
requires-python = '>=3.9'
16
dependencies = [
17
  "torch",
18
  "numpy",
19
  "scipy",
20
  "pillow",
21
  "tqdm",
22
  "pandas",
23
  "pandarallel",
24
  "opencv-python==4.8.0.76",
25
  "fsconnectors @ git+https://github.com/ai-forever/fsconnectors.git",
26
  "imageio",
27
  "imageio[pyav]"
28
]
29

30
[project.optional-dependencies]
31
dev = ['pytest', 'mypy', 'ruff', 'isort']
32
filters = [
33
    'torchvision',
34
    'CLIP @ git+https://github.com/openai/CLIP.git',
35
    'CRAFT @ git+https://github.com/boomb0om/CRAFT-text-detection/',
36
    'salesforce-lavis',
37
    'py3langid',
38
    'deep_translator',
39
    'huggingface_hub'
40
]
41
nsfw_detector = ['tensorflow', 'autokeras']
42
llava = [
43
    'transformers>=4.29',
44
    'LLaVA @ git+https://github.com/haotian-liu/LLaVA.git@bb4b22748618fb6cbec35d8d4642fbbd32dec76a'
45
]
46
video_llava = [
47
    'videollava @ git+https://github.com/PKU-YuanGroup/Video-LLaVA',
48
    'decord',
49
    'pytorchvideo @ git+https://github.com/facebookresearch/pytorchvideo.git@28fe037d212663c6a24f373b94cc5d478c8c1a1d'
50
]
51
lita = [
52
    'LITA @ git+https://github.com/NVlabs/LITA',
53
    'gdown'
54
]
55

56
[tool.hatch.version]
57
path = "DPF/__init__.py"
58

59
[tool.hatch.metadata]
60
allow-direct-references = true
61

62
# mypy settings
63
[tool.mypy]
64
strict = true
65
implicit_reexport = true
66
disable_error_code = ["import-untyped"]
67

68
[[tool.mypy.overrides]]
69
module = "DPF.filters.images.ocr_model.*"
70
ignore_errors = true
71

72
[[tool.mypy.overrides]]
73
module = "DPF.filters.videos.raft_core.*"
74
ignore_errors = true
75

76
[[tool.mypy.overrides]]
77
module = "DPF.filters.videos.video_llava_filter"
78
disable_error_code = ["import-not-found"]
79

80
[[tool.mypy.overrides]]
81
module = "DPF.filters.videos.lita_filter"
82
disable_error_code = ["import-not-found", "call-overload", "no-untyped-def"]
83

84
[[tool.mypy.overrides]]
85
module = "DPF.filters.images.llava_captioning_filter"
86
disable_error_code = ["import-not-found"]
87

88
[[tool.mypy.overrides]]
89
module = "DPF.filters.images.ocr_filter"
90
disable_error_code = ["attr-defined", "no-untyped-call"]
91

92
[[tool.mypy.overrides]]
93
module = "DPF.filters.utils.fp16_module"
94
disable_error_code = ["attr-defined", "no-untyped-def", "no-untyped-call", "assignment"]
95

96
# isort setting
97
[tool.isort]
98
profile = "black"
99

100
# ruff settings
101
[tool.ruff]
102
extend-exclude = [
103
    "DPF/filters/videos/raft_core/*",
104
    "DPF/filters/images/ocr_model/*"
105
]
106

107
[tool.ruff.lint]
108
select = [
109
    "E",  # pycodestyle errors
110
    "W",  # pycodestyle warnings
111
    "F",  # pyflakes
112
    "I",  # isort
113
    "C",  # flake8-comprehensions
114
    "B",  # flake8-bugbear
115
    "UP", # pyupgrade
116
]
117
ignore = [
118
    "E501", # line too long, handled by black
119
    "B008", # do not perform function calls in argument defaults
120
    "C901", # too complex
121
    "W191", # indentation contains tabs
122
    "E741", # Ambiguous variable name
123
]
124
ignore-init-module-imports = true
125

126
[tool.ruff.lint.per-file-ignores]
127
"__init__.py" = ["F401"]
128

129
[tool.ruff.lint.pyupgrade]
130
keep-runtime-typing = true
131