streamlit

1
# Copyright (c) Streamlit Inc. (2018-2022) Snowflake Inc. (2022-2024)
2
#
3
# Licensed under the Apache License, Version 2.0 (the "License");
4
# you may not use this file except in compliance with the License.
5
# You may obtain a copy of the License at
6
#
7
#     http://www.apache.org/licenses/LICENSE-2.0
8
#
9
# Unless required by applicable law or agreed to in writing, software
10
# distributed under the License is distributed on an "AS IS" BASIS,
11
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
# See the License for the specific language governing permissions and
13
# limitations under the License.
14

15
# Make uses /bin/sh by default, but we are using some bash features.  On Ubuntu
16
# /bin/sh is POSIX compliant, ie it's not bash.  So let's be explicit:
17
SHELL=/bin/bash
18

19
INSTALL_DEV_REQS ?= true
20
INSTALL_TEST_REQS ?= true
21
USE_CONSTRAINTS_FILE ?= true
22
PYTHON_VERSION := $(shell python --version | cut -d " " -f 2 | cut -d "." -f 1-2)
23
GITHUB_REPOSITORY ?= streamlit/streamlit
24
CONSTRAINTS_BRANCH ?= constraints-develop
25
CONSTRAINTS_URL ?= https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/${CONSTRAINTS_BRANCH}/constraints-${PYTHON_VERSION}.txt
26

27
# Black magic to get module directories
28
PYTHON_MODULES := $(foreach initpy, $(foreach dir, $(wildcard lib/*), $(wildcard $(dir)/__init__.py)), $(realpath $(dir $(initpy))))
29

30
.PHONY: help
31
help:
32
	@# Magic line used to create self-documenting makefiles.
33
	@# See https://stackoverflow.com/a/35730928
34
	@awk '/^#/{c=substr($$0,3);next}c&&/^[[:alpha:]][[:alnum:]_-]+:/{print substr($$1,1,index($$1,":")),c}1{c=0}' Makefile | column -s: -t
35

36
.PHONY: all
37
# Get dependencies, build frontend, install Streamlit into Python environment.
38
all: init frontend install
39

40
.PHONY: all-devel
41
# Get dependencies and install Streamlit into Python environment -- but do not build the frontend.
42
all-devel: init develop pre-commit-install
43
	@echo ""
44
	@echo "    The frontend has *not* been rebuilt."
45
	@echo "    If you need to make a wheel file or test S3 sharing, run:"
46
	@echo ""
47
	@echo "    make frontend"
48
	@echo ""
49

50
.PHONY: mini-devel
51
# Get minimal dependencies for development and install Streamlit into Python
52
# environment -- but do not build the frontend.
53
mini-devel: mini-init develop pre-commit-install
54

55
.PHONY: build-deps
56
# An even smaller installation than mini-devel. Installs the bare minimum
57
# necessary to build Streamlit (by leaving out some dependencies necessary for
58
# the development process). Does not build the frontend.
59
build-deps: mini-init develop
60

61
.PHONY: init
62
# Install all Python and JS dependencies.
63
init: python-init-all react-init protobuf
64

65
.PHONY: mini-init
66
# Install minimal Python and JS dependencies for development.
67
mini-init: python-init-dev-only react-init protobuf
68

69
.PHONY: frontend
70
# Build frontend into static files.
71
frontend: react-build
72

73
.PHONY: install
74
# Install Streamlit into your Python environment.
75
install:
76
	cd lib ; python setup.py install
77

78
.PHONY: develop
79
# Install Streamlit as links in your Python environment, pointing to local workspace.
80
develop:
81
	INSTALL_DEV_REQS=false INSTALL_TEST_REQS=false make python-init
82

83
.PHONY: python-init-all
84
# Install Streamlit and all (test and dev) requirements
85
python-init-all:
86
	INSTALL_DEV_REQS=true INSTALL_TEST_REQS=true make python-init
87

88
.PHONY: python-init-dev-only
89
# Install Streamlit and dev requirements
90
python-init-dev-only:
91
	INSTALL_DEV_REQS=true INSTALL_TEST_REQS=false make python-init
92

93
.PHONY: python-init-test-only
94
# Install Streamlit and test requirements
95
python-init-test-only: lib/test-requirements.txt
96
	INSTALL_DEV_REQS=false INSTALL_TEST_REQS=true make python-init
97

98
.PHONY: python-init-test-min-deps
99
# Install Streamlit and test requirements, with minimum dependency versions
100
python-init-test-min-deps:
101
	INSTALL_DEV_REQS=false INSTALL_TEST_REQS=true USE_CONSTRAINTS_FILE=true CONSTRAINTS_URL="lib/min-constraints-gen.txt" make python-init
102

103
.PHONY: python-init
104
python-init:
105
	pip_args=("--editable" "lib[snowflake]");\
106
	if [ "${USE_CONSTRAINTS_FILE}" = "true" ] ; then\
107
		pip_args+=(--constraint "${CONSTRAINTS_URL}"); \
108
	fi;\
109
	if [ "${INSTALL_DEV_REQS}" = "true" ] ; then\
110
		pip_args+=("--requirement" "lib/dev-requirements.txt"); \
111
	fi;\
112
	if [ "${INSTALL_TEST_REQS}" = "true" ] ; then\
113
		pip_args+=("--requirement" "lib/test-requirements.txt"); \
114
	fi;\
115
	echo "Running command: pip install $${pip_args[@]}";\
116
	pip install $${pip_args[@]};
117
	if [ "${INSTALL_TEST_REQS}" = "true" ] ; then\
118
		python -m playwright install --with-deps; \
119
	fi;\
120

121
.PHONY: pylint
122
# Verify that our Python files are properly formatted.
123
pylint:
124
	# Does not modify any files. Returns with a non-zero
125
	# status if anything is not properly formatted. (This isn't really
126
	# "linting"; we're not checking anything but code style.)
127
	if command -v "black" > /dev/null; then \
128
		$(BLACK) --diff --check examples/ && \
129
		$(BLACK) --diff --check lib/streamlit/ --exclude=/*_pb2.py$/ && \
130
		$(BLACK) --diff --check lib/tests/ && \
131
		$(BLACK) --diff --check e2e/scripts/ ; \
132
	fi
133

134
.PHONY: pyformat
135
# Fix Python files that are not properly formatted.
136
pyformat:
137
	pre-commit run black --all-files --hook-stage manual
138
	pre-commit run isort --all-files --hook-stage manual
139

140
.PHONY: pytest
141
# Run Python unit tests.
142
pytest:
143
	cd lib; \
144
		PYTHONPATH=. \
145
		pytest -v \
146
			--junitxml=test-reports/pytest/junit.xml \
147
			-l tests/ \
148
			$(PYTHON_MODULES)
149

150
# Run Python integration tests for snowflake.
151
pytest-snowflake:
152
	cd lib; \
153
		PYTHONPATH=. \
154
		pytest -v \
155
			--junitxml=test-reports/pytest/junit.xml \
156
			--require-snowflake \
157
			-l tests/ \
158
			$(PYTHON_MODULES)
159

160
.PHONY: mypy
161
# Run Mypy static type checker.
162
mypy:
163
	./scripts/mypy
164

165
.PHONY: integration-tests
166
# Run all our e2e tests in "bare" mode and check for non-zero exit codes.
167
integration-tests:
168
	python3 scripts/run_bare_integration_tests.py
169

170
.PHONY: cli-smoke-tests
171
# Verify that CLI boots as expected when called with `python -m streamlit`
172
cli-smoke-tests:
173
	python3 scripts/cli_smoke_tests.py
174

175
.PHONY: cli-regression-tests
176
# Verify that CLI boots as expected when called with `python -m streamlit`
177
cli-regression-tests: install
178
	pytest scripts/cli_regression_tests.py
179

180
.PHONY: distribution
181
# Create Python distribution files in dist/.
182
distribution:
183
	# Get rid of the old build and dist folders to make sure that we clean old js and css.
184
	rm -rfv lib/build lib/dist
185
	cd lib ; python3 setup.py bdist_wheel --universal sdist
186

187
.PHONY: package
188
# Build lib and frontend, and then run 'distribution'.
189
package: build-deps frontend distribution
190

191
.PHONY: conda-distribution
192
# Create conda distribution files in lib/conda-recipe/dist.
193
conda-distribution:
194
	rm -rf lib/conda-recipe/dist
195
	mkdir lib/conda-recipe/dist
196
	# This can take upwards of 20 minutes to complete in a fresh conda installation! (Dependency solving is slow.)
197
	# NOTE: Running the following command requires both conda and conda-build to
198
	# be installed.
199
	GIT_HASH=$$(git rev-parse --short HEAD) conda build lib/conda-recipe --output-folder lib/conda-recipe/dist
200

201
.PHONY: conda-package
202
# Build lib and (maybe) frontend assets, and then run 'conda-distribution'
203
conda-package: build-deps
204
	if [ "${SNOWPARK_CONDA_BUILD}" = "1" ] ; then\
205
		echo "Creating Snowpark conda build, so skipping building frontend assets."; \
206
	else \
207
		make frontend; \
208
	fi
209
	make conda-distribution;
210

211
.PHONY: clean
212
# Remove all generated files.
213
clean:
214
	cd lib; rm -rf build dist  .eggs *.egg-info
215
	rm -rf lib/conda-recipe/dist
216
	find . -name '*.pyc' -type f -delete || true
217
	find . -name __pycache__ -type d -delete || true
218
	find . -name .pytest_cache -exec rm -rfv {} \; || true
219
	rm -rf .mypy_cache
220
	rm -f lib/streamlit/proto/*_pb2.py*
221
	rm -rf lib/streamlit/static
222
	rm -f lib/Pipfile.lock
223
	rm -rf frontend/app/build
224
	rm -rf frontend/node_modules
225
	rm -rf frontend/app/node_modules
226
	rm -rf frontend/lib/node_modules
227
	rm -rf frontend/test_results
228
	rm -f frontend/lib/src/proto.js
229
	rm -f frontend/lib/src/proto.d.ts
230
	rm -rf frontend/public/reports
231
	rm -rf frontend/lib/dist
232
	rm -rf ~/.cache/pre-commit
233
	rm -rf e2e_playwright/test-results
234
	find . -name .streamlit -type d -exec rm -rfv {} \; || true
235
	cd lib; rm -rf .coverage .coverage\.*
236

237
MIN_PROTOC_VERSION = 3.20
238
.PHONY: check-protoc
239
# Ensure protoc is installed and is >= MIN_PROTOC_VERSION.
240
check-protoc:
241
	@# We support Python protobuf 4.21, which is incompatible with code generated from
242
	@# protoc < 3.20
243
	@if ! command -v protoc &> /dev/null ; then \
244
		echo "protoc not installed."; \
245
		exit 1; \
246
	fi; \
247
	\
248
	PROTOC_VERSION=$$(protoc --version | cut -d ' ' -f 2); \
249
	\
250
	if [[ $$(echo -e "$$PROTOC_VERSION\n$(MIN_PROTOC_VERSION)" | sort -V | head -n1) != $(MIN_PROTOC_VERSION) ]]; then \
251
	  echo "Error: protoc version $${PROTOC_VERSION} is < $(MIN_PROTOC_VERSION)"; \
252
	  exit 1; \
253
	else \
254
	  echo "protoc version $${PROTOC_VERSION} is >= than $(MIN_PROTOC_VERSION)"; \
255
	fi
256

257
.PHONY: protobuf
258
# Recompile Protobufs for Python and the frontend.
259
protobuf: check-protoc
260
	protoc \
261
		--proto_path=proto \
262
		--python_out=lib \
263
		--mypy_out=lib \
264
		proto/streamlit/proto/*.proto
265

266
	@# JS protobuf generation. The --es6 flag generates a proper es6 module.
267
	cd frontend/ ; ( \
268
		echo "/* eslint-disable */" ; \
269
		echo ; \
270
		yarn --silent pbjs \
271
			../proto/streamlit/proto/*.proto \
272
			--path=proto -t static-module --wrap es6 \
273
	) > ./lib/src/proto.js
274

275
	@# Typescript type declarations for our generated protobufs
276
	cd frontend/ ; ( \
277
		echo "/* eslint-disable */" ; \
278
		echo ; \
279
		yarn --silent pbts ./lib/src/proto.js \
280
	) > ./lib/src/proto.d.ts
281

282
.PHONY: react-init
283
react-init:
284
	cd frontend/ ; yarn install --frozen-lockfile
285

286
.PHONY: react-build
287
react-build:
288
	cd frontend/ ; yarn run build
289
	rsync -av --delete --delete-excluded --exclude=reports \
290
		frontend/app/build/ lib/streamlit/static/
291

292
.PHONY: frontend-fast
293
# Build frontend into static files faster by setting BUILD_AS_FAST_AS_POSSIBLE=true flag, which disables eslint and typechecking.
294
frontend-fast:
295
	cd frontend/ ; yarn run buildFast
296
	rsync -av --delete --delete-excluded --exclude=reports \
297
		frontend/app/build/ lib/streamlit/static/
298

299
.PHONY: frontend-lib
300
# Build the frontend library
301
frontend-lib:
302
	cd frontend/ ; yarn run buildLib;
303

304
.PHONY: frontend-app
305
# Build the frontend app. One must build the frontend lib first before building the app.
306
frontend-app:
307
	cd frontend/ ; yarn run buildApp
308

309
.PHONY: jslint
310
# Lint the JS code
311
jslint:
312
	cd frontend; \
313
		yarn lint;
314

315
.PHONY: tstypecheck
316
# Type check the JS/TS code
317
tstypecheck:
318
	pre-commit run typecheck-lib --all-files --hook-stage manual && pre-commit run typecheck-app --all-files --hook-stage manual
319

320
.PHONY: jsformat
321
# Fix formatting issues in our JavaScript & TypeScript files.
322
jsformat:
323
	pre-commit run prettier --all-files --hook-stage manual
324

325
.PHONY: jstest
326
# Run JS unit tests.
327
jstest:
328
	cd frontend; TESTPATH=$(TESTPATH) yarn run test
329

330
.PHONY: jscoverage
331
# Run JS unit tests and generate a coverage report.
332
jscoverage:
333
	cd frontend; yarn run test --coverage --watchAll=false
334

335
.PHONY: e2etest
336
# Run E2E tests.
337
e2etest:
338
	./scripts/run_e2e_tests.py
339

340
.PHONY: playwright
341
# Run playwright E2E tests.
342
playwright:
343
	cd e2e_playwright; \
344
	rm -rf ./test-results; \
345
	pytest --browser webkit --browser chromium --browser firefox --video retain-on-failure --screenshot only-on-failure --output ./test-results/ -n auto --reruns 1 --reruns-delay 1 --rerun-except "Missing snapshot" --durations=5 -r aR -v
346

347
.PHONY: loc
348
# Count the number of lines of code in the project.
349
loc:
350
	find . -iname '*.py' -or -iname '*.js'  | \
351
		egrep -v "(node_modules)|(_pb2)|(lib\/streamlit\/proto)|(dist\/)" | \
352
		xargs wc
353

354
.PHONY: distribute
355
# Upload the package to PyPI.
356
distribute:
357
	cd lib/dist; \
358
		twine upload $$(ls -t *.whl | head -n 1); \
359
		twine upload $$(ls -t *.tar.gz | head -n 1)
360

361
.PHONY: notices
362
# Rebuild the NOTICES file.
363
notices:
364
	cd frontend; \
365
		yarn licenses generate-disclaimer --silent --production --ignore-platform > ../NOTICES
366

367
	@# When `yarn licenses` is run in a yarn workspace, it misnames the project as
368
	@# "WORKSPACE AGGREGATOR 2B7C80A7 6734 4A68 BB93 8CC72B9A5DEA". We fix that here.
369
	@# There also isn't a portable way to invoke `sed` to edit files in-place, so we have
370
	@# sed create a NOTICES.bak backup file that we immediately delete afterwards.
371
	sed -i'.bak' 's/PORTIONS OF THE .*PRODUCT/PORTIONS OF THE STREAMLIT PRODUCT/' NOTICES
372
	rm -f NOTICES.bak
373

374
	./scripts/append_license.sh frontend/app/src/assets/fonts/Source_Code_Pro/Source-Code-Pro.LICENSE
375
	./scripts/append_license.sh frontend/app/src/assets/fonts/Source_Sans_Pro/Source-Sans-Pro.LICENSE
376
	./scripts/append_license.sh frontend/app/src/assets/fonts/Source_Serif_Pro/Source-Serif-Pro.LICENSE
377
	./scripts/append_license.sh frontend/app/src/assets/img/Material-Icons.LICENSE
378
	./scripts/append_license.sh frontend/app/src/assets/img/Open-Iconic.LICENSE
379
	./scripts/append_license.sh frontend/lib/src/vendor/bokeh/bokeh-LICENSE.txt
380
	./scripts/append_license.sh frontend/lib/src/vendor/twemoji-LICENSE.txt
381
	./scripts/append_license.sh frontend/app/src/vendor/Segment-LICENSE.txt
382
	./scripts/append_license.sh frontend/lib/src/vendor/react-bootstrap-LICENSE.txt
383
	./scripts/append_license.sh lib/streamlit/vendor/ipython/IPython-LICENSE.txt
384

385
.PHONY: headers
386
# Update the license header on all source files.
387
headers:
388
	pre-commit run insert-license --all-files --hook-stage manual
389
	pre-commit run license-headers --all-files --hook-stage manual
390

391
.PHONY: gen-min-dep-constraints
392
# Write the minimum versions of our dependencies to a constraints file.
393
gen-min-dep-constraints:
394
	make develop >/dev/null
395
	python scripts/get_min_versions.py >lib/min-constraints-gen.txt
396

397
.PHONY: pre-commit-install
398
pre-commit-install:
399
	pre-commit install
400

401
.PHONY: ensure-relative-imports
402
# ensure relative imports exist within the lib/dist folder when doing yarn buildLibProd
403
ensure-relative-imports:
404
	./scripts/ensure_relative_imports.sh
405

406
.PHONY frontend-lib-prod:
407
# build the production version for @streamlit/lib
408
frontend-lib-prod:
409
	cd frontend/ ; yarn run buildLibProd;
410

411
.PHONY streamlit-lib-prod:
412
# build the production version for @streamlit/lib
413
# while also doing a make init so it's a single command
414
streamlit-lib-prod:
415
	make mini-init;
416
	make frontend-lib-prod;
417

418