text-generation-inference

1
include Makefile-flash-att
2
include Makefile-flash-att-v2
3
include Makefile-vllm
4
include Makefile-awq
5
include Makefile-eetq
6
include Makefile-selective-scan
7

8
unit-tests:
9
	pytest -s -vv -m "not private" tests
10

11
gen-server:
12
	# Compile protos
13
	pip install grpcio-tools==1.51.1 mypy-protobuf==3.4.0 'types-protobuf>=3.20.4' --no-cache-dir
14
	mkdir text_generation_server/pb || true
15
	python -m grpc_tools.protoc -I../proto --python_out=text_generation_server/pb \
16
		--grpc_python_out=text_generation_server/pb --mypy_out=text_generation_server/pb ../proto/generate.proto
17
	find text_generation_server/pb/ -type f -name "*.py" -print0 -exec sed -i -e 's/^\(import.*pb2\)/from . \1/g' {} \;
18
	touch text_generation_server/pb/__init__.py
19

20
install-megablocks:
21
	pip install git+https://github.com/OlivierDehaene/megablocks@181709df192de9a941fdf3a641cdc65a0462996e
22

23
install: gen-server
24
	pip install pip --upgrade
25
	pip install -r requirements_cuda.txt
26
	pip install -e ".[bnb, accelerate, quantize, peft, outlines]"
27

28
run-dev:
29
	SAFETENSORS_FAST_GPU=1 python -m torch.distributed.run --nproc_per_node=2 text_generation_server/cli.py serve bigscience/bloom-560m --sharded
30

31
export-requirements:
32
	poetry export -o requirements_cuda.txt --extras bnb --without-hashes
33
	poetry export -o requirements_rocm.txt --without-hashes
34