allennlp

1
SRC = allennlp
2

3
MD_DOCS_ROOT = docs/
4
MD_DOCS_API_ROOT = $(MD_DOCS_ROOT)api/
5
MD_DOCS_SRC = $(filter-out $(SRC)/__main__.py %/__init__.py $(SRC)/version.py,$(shell find $(SRC) -type f -name '*.py' | grep -v -E 'tests/'))
6
MD_DOCS = $(subst .py,.md,$(subst $(SRC)/,$(MD_DOCS_API_ROOT),$(MD_DOCS_SRC)))
7
MD_DOCS_CMD = python scripts/py2md.py
8
MD_DOCS_CONF = mkdocs.yml
9
MD_DOCS_CONF_SRC = mkdocs-skeleton.yml
10
MD_DOCS_TGT = site/
11
MD_DOCS_EXTRAS = $(addprefix $(MD_DOCS_ROOT),README.md CHANGELOG.md CONTRIBUTING.md)
12

13
TORCH_INSTALL = pip install torch torchvision -c constraints.txt
14
DOCKER_TORCH_VERSION = 1.12.0-cuda11.3-python3.8
15
DOCKER_TEST_TORCH_VERSION = 1.12.0-cuda11.3-python3.8
16

17
DOCKER_TAG = latest
18
DOCKER_IMAGE_NAME = allennlp/allennlp:$(DOCKER_TAG)
19
DOCKER_TEST_IMAGE_NAME = allennlp/test:$(DOCKER_TAG)
20
DOCKER_RUN_CMD = docker run --rm \
21
		-v $$HOME/.allennlp:/root/.allennlp \
22
		-v $$HOME/.cache/huggingface:/root/.cache/huggingface \
23
		-v $$HOME/nltk_data:/root/nltk_data
24

25
# These nltk packages are used by the 'checklist' module. They are downloaded automatically
26
# if not found when `checklist` is imported, but it's good to download the ahead of time
27
# to avoid potential race conditions.
28
NLTK_DOWNLOAD_CMD = python -c 'import nltk; [nltk.download(p) for p in ("wordnet", "wordnet_ic", "sentiwordnet", "omw", "omw-1.4")]'
29

30
ifeq ($(shell uname),Darwin)
31
ifeq ($(shell which gsed),)
32
$(error Please install GNU sed with 'brew install gnu-sed')
33
else
34
SED = gsed
35
endif
36
else
37
SED = sed
38
endif
39

40
.PHONY : version
41
version :
42
	@python -c 'from allennlp.version import VERSION; print(f"AllenNLP v{VERSION}")'
43

44
.PHONY : check-for-cuda
45
check-for-cuda :
46
	@python -c 'import torch; assert torch.cuda.is_available(); print("Cuda is available")'
47

48
#
49
# Testing helpers.
50
#
51

52
.PHONY : flake8
53
flake8 :
54
	flake8 allennlp tests scripts 
55

56
.PHONY : format
57
format :
58
	black --check allennlp tests scripts 
59

60
.PHONY : typecheck
61
typecheck :
62
	mypy allennlp tests scripts --cache-dir=/dev/null
63

64
.PHONY : test
65
test :
66
	pytest --color=yes -v -rf --durations=40 \
67
			--cov-config=.coveragerc \
68
			--cov=$(SRC) \
69
			--cov-report=xml
70

71
.PHONY : test-without-checklist
72
test-without-checklist :
73
	pytest --color=yes -v -rf --durations=40 \
74
			--cov-config=.coveragerc \
75
			--cov=$(SRC) \
76
			--cov-report=xml \
77
			--ignore-glob=*checklist*
78

79
.PHONY : test-checklist
80
test-checklist :
81
	pytest --color=yes -v -rf --durations=40 \
82
			--cov-config=.coveragerc \
83
			--cov=$(SRC) \
84
			--cov-report=xml \
85
			tests/ \
86
			-k checklist
87

88

89
.PHONY : gpu-tests
90
gpu-tests : check-for-cuda
91
	pytest --color=yes -v -rf --durations=20 \
92
			--cov-config=.coveragerc \
93
			--cov=$(SRC) \
94
			--cov-report=xml \
95
			-m gpu
96

97
.PHONY : benchmarks
98
benchmarks :
99
	pytest -c benchmarks/pytest.ini benchmarks/
100

101
#
102
# Setup helpers
103
#
104

105
.PHONY : download-extras
106
download-extras :
107
	$(NLTK_DOWNLOAD_CMD)
108

109
.PHONY : install
110
install :
111
	# Due to a weird thing with pip, we may need egg-info before running `pip install -e`.
112
	# See https://github.com/pypa/pip/issues/4537.
113
	# python setup.py install_egg_info
114
	# Install torch ecosystem first.
115
	$(TORCH_INSTALL)
116
	pip install --upgrade pip
117
	pip install pip-tools
118
	pip-compile requirements.in -o final_requirements.txt --allow-unsafe --rebuild --verbose
119
	pip install -e . -r final_requirements.txt
120
	# These nltk packages are used by the 'checklist' module.
121
	$(NLTK_DOWNLOAD_CMD)
122

123
#
124
# Documention helpers.
125
#
126

127
.PHONY : build-all-api-docs
128
build-all-api-docs :
129
	@$(MD_DOCS_CMD) $(subst /,.,$(subst .py,,$(MD_DOCS_SRC))) -o $(MD_DOCS)
130

131
.PHONY : build-docs
132
build-docs : build-all-api-docs $(MD_DOCS_CONF) $(MD_DOCS) $(MD_DOCS_EXTRAS)
133
	mkdocs build
134

135
.PHONY : serve-docs
136
serve-docs : build-all-api-docs $(MD_DOCS_CONF) $(MD_DOCS) $(MD_DOCS_EXTRAS)
137
	mkdocs serve --dirtyreload
138

139
.PHONY : update-docs
140
update-docs : $(MD_DOCS) $(MD_DOCS_EXTRAS)
141

142
$(MD_DOCS_ROOT)README.md : README.md
143
	cp $< $@
144
	# Alter the relative path of the README image for the docs.
145
	$(SED) -i '1s/docs/./' $@
146
	# Alter external doc links to relative links.
147
	$(SED) -i 's|https://docs.allennlp.org/main/api/|api/|' $@
148

149
$(MD_DOCS_ROOT)%.md : %.md
150
	cp $< $@
151

152
$(MD_DOCS_CONF) : $(MD_DOCS_CONF_SRC) $(MD_DOCS)
153
	python scripts/build_docs_config.py $@ $(MD_DOCS_CONF_SRC) $(MD_DOCS_ROOT) $(MD_DOCS_API_ROOT)
154

155
$(MD_DOCS_API_ROOT)%.md : $(SRC)/%.py scripts/py2md.py
156
	mkdir -p $(shell dirname $@)
157
	$(MD_DOCS_CMD) $(subst /,.,$(subst .py,,$<)) --out $@
158

159
.PHONY : clean
160
clean :
161
	rm -rf $(MD_DOCS_TGT)
162
	rm -rf $(MD_DOCS_API_ROOT)
163
	rm -f $(MD_DOCS_ROOT)*.md
164
	rm -rf .pytest_cache/
165
	rm -rf allennlp.egg-info/
166
	rm -rf dist/
167
	rm -rf build/
168
	find . | grep -E '(\.mypy_cache|__pycache__|\.pyc|\.pyo$$)' | xargs rm -rf
169

170
#
171
# Docker helpers.
172
#
173

174
.PHONY : docker-image
175
docker-image :
176
	docker build \
177
		--pull \
178
		-f Dockerfile \
179
		--build-arg TORCH=$(DOCKER_TORCH_VERSION) \
180
		-t $(DOCKER_IMAGE_NAME) .
181

182
DOCKER_GPUS = --gpus all
183

184
.PHONY : docker-run
185
docker-run :
186
	$(DOCKER_RUN_CMD) $(DOCKER_GPUS) $(DOCKER_IMAGE_NAME) $(ARGS)
187

188
.PHONY : docker-test-image
189
docker-test-image :
190
	docker build \
191
		--pull \
192
		-f Dockerfile.test \
193
		--build-arg TORCH=$(DOCKER_TEST_TORCH_VERSION) \
194
		-t $(DOCKER_TEST_IMAGE_NAME) .
195

196
.PHONY : docker-test-run
197
docker-test-run :
198
	$(DOCKER_RUN_CMD) --shm-size 2G $(DOCKER_GPUS) $(DOCKER_TEST_IMAGE_NAME) $(ARGS)
199