optimum-habana

1
#  Copyright 2022 The HuggingFace Team. All rights reserved.
2
#
3
#  Licensed under the Apache License, Version 2.0 (the "License");
4
#  you may not use this file except in compliance with the License.
5
#  You may obtain a copy of the License at
6
#
7
#      http://www.apache.org/licenses/LICENSE-2.0
8
#
9
#  Unless required by applicable law or agreed to in writing, software
10
#  distributed under the License is distributed on an "AS IS" BASIS,
11
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
#  See the License for the specific language governing permissions and
13
#  limitations under the License.
14
SHELL := /bin/bash
15
CURRENT_DIR = $(shell pwd)
16
DEFAULT_CLONE_URL := https://github.com/huggingface/optimum-habana.git
17
# If CLONE_URL is empty, revert to DEFAULT_CLONE_URL
18
REAL_CLONE_URL = $(if $(CLONE_URL),$(CLONE_URL),$(DEFAULT_CLONE_URL))
19

20

21
.PHONY:	style test
22

23
# Run code quality checks
24
style_check: clean
25
	pip install -U pip ruff
26
	ruff check . setup.py
27
	ruff format --check . setup.py
28

29
style: clean
30
	pip install -U pip ruff
31
	ruff check . setup.py --fix
32
	ruff format . setup.py
33

34
# Run unit and integration tests
35
fast_tests:
36
	python -m pip install .[tests]
37
	python -m pytest tests/test_gaudi_configuration.py tests/test_trainer_distributed.py tests/test_trainer.py tests/test_trainer_seq2seq.py
38

39
# Run unit and integration tests related to Diffusers
40
fast_tests_diffusers:
41
	python -m pip install .[tests]
42
	python -m pytest tests/test_diffusers.py
43

44
# Run single-card non-regression tests
45
slow_tests_1x: test_installs
46
	python -m pytest tests/test_examples.py -v -s -k "single_card"
47

48
# Run multi-card non-regression tests
49
slow_tests_8x: test_installs
50
	python -m pytest tests/test_examples.py -v -s -k "multi_card"
51

52
# Run DeepSpeed non-regression tests
53
slow_tests_deepspeed: test_installs
54
	python -m pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.14.0
55
	python -m pytest tests/test_examples.py -v -s -k "deepspeed"
56

57
slow_tests_diffusers: test_installs
58
	python -m pip install git+https://github.com/huggingface/diffusers.git
59
	python -m pytest tests/test_diffusers.py -v -s -k "test_no_"
60
	python -m pytest tests/test_diffusers.py -v -s -k "test_textual_inversion"
61
	python -m pip install peft==0.7.0
62
	python -m pytest tests/test_diffusers.py -v -s -k "test_train_text_to_image_"
63
	python -m pytest tests/test_diffusers.py -v -s -k "test_train_controlnet"
64

65
# Run text-generation non-regression tests
66
slow_tests_text_generation_example: test_installs
67
	python -m pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.14.0
68
	python -m pytest tests/test_text_generation_example.py tests/test_encoder_decoder.py -v -s --token $(TOKEN)
69

70
slow_tests_fsdp: test_installs
71
	python -m pytest tests/test_fsdp_examples.py -v -s
72

73
# Check if examples are up to date with the Transformers library
74
example_diff_tests: test_installs
75
	python -m pytest tests/test_examples_match_transformers.py
76

77
# Utilities to release to PyPi
78
build_dist_install_tools:
79
	python -m pip install build
80
	python -m pip install twine
81

82
build_dist:
83
	rm -fr build
84
	rm -fr dist
85
	python -m build
86

87
pypi_upload: build_dist
88
	python -m twine upload dist/*
89

90
build_doc_docker_image:
91
	docker build -t doc_maker --build-arg commit_sha=$(COMMIT_SHA_SUBPACKAGE) --build-arg clone_url=$(REAL_CLONE_URL) ./docs
92

93
doc: build_doc_docker_image
94
	@test -n "$(BUILD_DIR)" || (echo "BUILD_DIR is empty." ; exit 1)
95
	@test -n "$(VERSION)" || (echo "VERSION is empty." ; exit 1)
96
	docker run -v $(CURRENT_DIR):/doc_folder --workdir=/doc_folder doc_maker \
97
	doc-builder build optimum.habana /optimum-habana/docs/source/ \
98
		--repo_name optimum-habana \
99
		--build_dir $(BUILD_DIR) \
100
		--version $(VERSION) \
101
		--version_tag_suffix "" \
102
		--html \
103
		--clean
104

105
clean:
106
	find . -name "habana_log.livealloc.log_*" -type f -delete
107
	find . -name .lock -type f -delete
108
	find . -name .graph_dumps -type d -exec rm -r {} +
109
	find . -name save-hpu.pdb -type f -delete
110
	find . -name checkpoints.json -type f -delete
111
	rm -rf regression/
112
	rm -rf tmp_trainer/
113
	rm -rf test/
114
	rm -rf build/
115
	rm -rf dist/
116
	rm -rf optimum_habana.egg-info/
117
	rm -rf hpu_profile/
118

119
test_installs:
120
	python -m pip install .[tests]
121