Dragonfly2

1
# Copyright The Dragonfly Authors.
2
#
3
# Licensed under the Apache License, Version 2.0 (the "License");
4
# you may not use this file except in compliance with the License.
5
# You may obtain a copy of the License at
6
#
7
#      http://www.apache.org/licenses/LICENSE-2.0
8
#
9
# Unless required by applicable law or agreed to in writing, software
10
# distributed under the License is distributed on an "AS IS" BASIS,
11
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
# See the License for the specific language governing permissions and
13
# limitations under the License.
14

15
PROJECT_NAME := "d7y.io/dragonfly/v2"
16
DFGET_NAME := "dfget"
17
DFCACHE_NAME := "dfcache"
18
DFSTORE_NAME := "dfstore"
19
SEMVER := "2.1.0"
20
VERSION_RELEASE := "1"
21
PKG := "$(PROJECT_NAME)"
22
PKG_LIST := $(shell go list ${PKG}/... | grep -v /vendor/ | grep -v '\(/test/\)')
23
GIT_COMMIT := $(shell git rev-parse --verify HEAD --short=7)
24
GIT_COMMIT_LONG := $(shell git rev-parse --verify HEAD)
25
DFGET_ARCHIVE_PREFIX := "$(DFGET_NAME)_$(SEMVER)-$(VERSION_RELEASE)_$(GIT_COMMIT)"
26
DFCACHE_ARCHIVE_PREFIX := "$(DFCACHE_NAME)_$(SEMVER)-$(VERSION_RELEASE)_$(GIT_COMMIT)"
27
DFSTORE_ARCHIVE_PREFIX := "$(DFSTORE_NAME)_$(SEMVER)-$(VERSION_RELEASE)_$(GIT_COMMIT)"
28

29
all: help
30

31
# Prepare required folders for build.
32
build-dirs:
33
	@mkdir -p ./bin
34
.PHONY: build-dirs
35

36
# Build dragonfly.
37
docker-build: docker-build-dfdaemon docker-build-scheduler docker-build-manager docker-build-trainer
38
	@echo "Build image done."
39
.PHONY: docker-build
40

41
# Push dragonfly images.
42
docker-push: docker-push-dfdaemon docker-push-scheduler docker-push-manager docker-build-trainer
43
	@echo "Push image done."
44
.PHONY: docker-push
45

46
# Build dfdaemon image.
47
docker-build-dfdaemon:
48
	@echo "Begin to use docker build dfdaemon image."
49
	./hack/docker-build.sh dfdaemon
50
.PHONY: docker-build-dfdaemon
51

52
# Build scheduler image.
53
docker-build-scheduler:
54
	@echo "Begin to use docker build scheduler image."
55
	./hack/docker-build.sh scheduler
56
.PHONY: docker-build-scheduler
57

58
# Build manager image.
59
docker-build-manager:
60
	@echo "Begin to use docker build manager image."
61
	./hack/docker-build.sh manager
62
.PHONY: docker-build-manager
63

64
# Build trainer image.
65
docker-build-trainer:
66
	@echo "Begin to use docker build trainer image."
67
	./hack/docker-build.sh trainer
68
.PHONY: docker-build-trainer
69

70
# Build testing tools image.
71
docker-build-testing-tools: build-dirs
72
	@echo "Begin to testing tools image."
73
	./test/tools/no-content-length/build.sh
74
.PHONY: docker-build-testing-tools
75

76
# Push dfdaemon image.
77
docker-push-dfdaemon: docker-build-dfdaemon
78
	@echo "Begin to push dfdaemon docker image."
79
	./hack/docker-push.sh dfdaemon
80
.PHONY: docker-push-dfdaemon
81

82
# Push scheduler image.
83
docker-push-scheduler: docker-build-scheduler
84
	@echo "Begin to push scheduler docker image."
85
	./hack/docker-push.sh scheduler
86
.PHONY: docker-push-scheduler
87

88
# Push manager image.
89
docker-push-manager: docker-build-manager
90
	@echo "Begin to push manager docker image."
91
	./hack/docker-push.sh manager
92
.PHONY: docker-push-manager
93

94
# Push trainer image.
95
docker-push-trainer: docker-build-trainer
96
	@echo "Begin to push trainer docker image."
97
	./hack/docker-push.sh trainer
98
.PHONY: docker-push-trainer
99

100
# Build dragonfly.
101
build: build-manager build-scheduler build-trainer build-dfget build-dfcache build-dfstore
102
.PHONY: build
103

104
# Build dfget.
105
build-dfget: build-dirs
106
	@echo "Begin to build dfget."
107
	./hack/build.sh dfget
108
.PHONY: build-dfget
109

110
# Build linux dfget.
111
build-linux-dfget: build-dirs
112
	@echo "Begin to build linux dfget."
113
	GOOS=linux GOARCH=amd64 ./hack/build.sh dfget
114
.PHONY: build-linux-dfget
115

116
# Build dfcache.
117
build-dfcache: build-dirs
118
	@echo "Begin to build dfcache."
119
	./hack/build.sh dfcache
120
.PHONY: build-dfcache
121

122
# Build linux dfcache.
123
build-linux-dfcache: build-dirs
124
	@echo "Begin to build linux dfcache."
125
	GOOS=linux GOARCH=amd64 ./hack/build.sh dfcache
126
.PHONY: build-linux-dfcache
127

128
# Build dfstore.
129
build-dfstore: build-dirs
130
	@echo "Begin to build dfstore."
131
	./hack/build.sh dfstore
132
.PHONY: build-dfstore
133

134
# Build linux dfcache.
135
build-linux-dfstore: build-dirs
136
	@echo "Begin to build linux dfstore."
137
	GOOS=linux GOARCH=amd64 ./hack/build.sh dfstore
138
.PHONY: build-linux-dfstore
139

140
# Build scheduler.
141
build-scheduler: build-dirs
142
	@echo "Begin to build scheduler."
143
	./hack/build.sh scheduler
144
.PHONY: build-scheduler
145

146
# Build manager.
147
build-manager: build-dirs build-manager-console
148
	@echo "Begin to build manager."
149
	make build-manager-server
150
.PHONY: build-manager
151

152
# Build manager server.
153
build-manager-server: build-dirs
154
	@echo "Begin to build manager server."
155
	./hack/build.sh manager
156
.PHONY: build-manager
157

158
# Build manager console.
159
build-manager-console: build-dirs
160
	@echo "Begin to build manager console."
161
	./hack/build.sh manager-console
162
.PHONY: build-manager-console
163

164
# Build trainer.
165
build-trainer: build-dirs
166
	@echo "Begin to build trainer."
167
	./hack/build.sh trainer
168
.PHONY: build-trainer
169

170
# Install dfget.
171
install-dfget:
172
	@echo "Begin to install dfget."
173
	./hack/install.sh install dfget
174
.PHONY: install-dfget
175

176
# Install scheduler.
177
install-scheduler:
178
	@echo "Begin to install scheduler."
179
	./hack/install.sh install scheduler
180
.PHONY: install-scheduler
181

182
# Install manager.
183
install-manager:
184
	@echo "Begin to install manager."
185
	./hack/install.sh install manager
186
.PHONY: install-manager
187

188
# Install trainer.
189
install-trainer:
190
	@echo "Begin to install trainer."
191
	./hack/install.sh install trainer 
192
.PHONY: install-trainer
193

194
# Build rpm dfget.
195
build-rpm-dfget: build-linux-dfget
196
	@echo "Begin to build rpm dfget."
197
	@docker run --rm \
198
	-v "$(PWD)/build:/root/build" \
199
	-v "$(PWD)/build/package/docs:/root/docs" \
200
	-v "$(PWD)/LICENSE:/root/License" \
201
	-v "$(PWD)/CHANGELOG.md:/root/CHANGELOG.md" \
202
	-v "$(PWD)/bin:/root/bin" \
203
	-e "SEMVER=$(SEMVER)" \
204
	-e "VERSION_RELEASE=$(VERSION_RELEASE)" \
205
	goreleaser/nfpm pkg \
206
		--config /root/build/package/nfpm/dfget.yaml \
207
		--target /root/bin/$(DFGET_ARCHIVE_PREFIX)_linux_amd64.rpm
208
	@echo "Build package output: ./bin/$(DFGET_ARCHIVE_PREFIX)_linux_amd64.rpm"
209
.PHONY: build-rpm-dfget
210

211
# Build rpm dfcache.
212
build-rpm-dfcache: build-linux-dfcache build-dfcache-man-page
213
	@echo "Begin to build rpm dfcache."
214
	@docker run --rm \
215
	-v "$(PWD)/build:/root/build" \
216
	-v "$(PWD)/build/package/docs:/root/docs" \
217
	-v "$(PWD)/LICENSE:/root/License" \
218
	-v "$(PWD)/CHANGELOG.md:/root/CHANGELOG.md" \
219
	-v "$(PWD)/bin:/root/bin" \
220
	-e "SEMVER=$(SEMVER)" \
221
	-e "VERSION_RELEASE=$(VERSION_RELEASE)" \
222
	goreleaser/nfpm pkg \
223
		--config /root/build/package/nfpm/dfcache.yaml \
224
		--target /root/bin/$(DFCACHE_ARCHIVE_PREFIX)_linux_amd64.rpm
225
	@echo "Build package output: ./bin/$(DFCACHE_ARCHIVE_PREFIX)_linux_amd64.rpm"
226
.PHONY: build-rpm-dfcache
227

228
# Build rpm dfstore.
229
build-rpm-dfstore: build-linux-dfstore
230
	@echo "Begin to build rpm dfstore."
231
	@docker run --rm \
232
	-v "$(PWD)/build:/root/build" \
233
	-v "$(PWD)/build/package/docs:/root/docs" \
234
	-v "$(PWD)/LICENSE:/root/License" \
235
	-v "$(PWD)/CHANGELOG.md:/root/CHANGELOG.md" \
236
	-v "$(PWD)/bin:/root/bin" \
237
	-e "SEMVER=$(SEMVER)" \
238
	-e "VERSION_RELEASE=$(VERSION_RELEASE)" \
239
	goreleaser/nfpm pkg \
240
		--config /root/build/package/nfpm/dfstore.yaml \
241
		--target /root/bin/$(DFSTORE_ARCHIVE_PREFIX)_linux_amd64.rpm
242
	@echo "Build package output: ./bin/$(DFSTORE_ARCHIVE_PREFIX)_linux_amd64.rpm"
243
.PHONY: build-rpm-dfstore
244

245
# Build deb dfget.
246
build-deb-dfget: build-linux-dfget
247
	@echo "Begin to build deb dfget."
248
	@docker run --rm \
249
	-v "$(PWD)/build:/root/build" \
250
	-v "$(PWD)/build/package/docs:/root/docs" \
251
	-v "$(PWD)/LICENSE:/root/License" \
252
	-v "$(PWD)/CHANGELOG.md:/root/CHANGELOG.md" \
253
	-v "$(PWD)/bin:/root/bin" \
254
	-e "SEMVER=$(SEMVER)" \
255
	-e "VERSION_RELEASE=$(VERSION_RELEASE)" \
256
	goreleaser/nfpm pkg \
257
		--config /root/build/package/nfpm/dfget.yaml \
258
		--target /root/bin/$(DFGET_ARCHIVE_PREFIX)_linux_amd64.deb
259
	@echo "Build package output: ./bin/$(DFGET_ARCHIVE_PREFIX)_linux_amd64.deb"
260
.PHONY: build-deb-dfget
261

262
# Build deb dfcache.
263
build-deb-dfcache: build-linux-dfcache build-dfcache-man-page
264
	@echo "Begin to build deb dfcache."
265
	@docker run --rm \
266
	-v "$(PWD)/build:/root/build" \
267
	-v "$(PWD)/build/package/docs:/root/docs" \
268
	-v "$(PWD)/LICENSE:/root/License" \
269
	-v "$(PWD)/CHANGELOG.md:/root/CHANGELOG.md" \
270
	-v "$(PWD)/bin:/root/bin" \
271
	-e "SEMVER=$(SEMVER)" \
272
	-e "VERSION_RELEASE=$(VERSION_RELEASE)" \
273
	goreleaser/nfpm pkg \
274
		--config /root/build/package/nfpm/dfcache.yaml \
275
		--target /root/bin/$(DFCACHE_ARCHIVE_PREFIX)_linux_amd64.deb
276
	@echo "Build package output: ./bin/$(DFCACHE_ARCHIVE_PREFIX)_linux_amd64.deb"
277
.PHONY: build-deb-dfcache
278

279
# Build deb dfstore
280
build-deb-dfstore: build-linux-dfstore
281
	@echo "Begin to build deb dfstore"
282
	@docker run --rm \
283
	-v "$(PWD)/build:/root/build" \
284
	-v "$(PWD)/build/package/docs:/root/docs" \
285
	-v "$(PWD)/LICENSE:/root/License" \
286
	-v "$(PWD)/CHANGELOG.md:/root/CHANGELOG.md" \
287
	-v "$(PWD)/bin:/root/bin" \
288
	-e "SEMVER=$(SEMVER)" \
289
	-e "VERSION_RELEASE=$(VERSION_RELEASE)" \
290
	goreleaser/nfpm pkg \
291
		--config /root/build/package/nfpm/dfstore.yaml \
292
		--target /root/bin/$(DFSTORE_ARCHIVE_PREFIX)_linux_amd64.deb
293
	@echo "Build package output: ./bin/$(DFSTORE_ARCHIVE_PREFIX)_linux_amd64.deb"
294
.PHONY: build-deb-dfstore
295

296
# Generate man page.
297
build-man-page: build-dfget-man-page build-dfcache-man-page build-dfstore-man-page
298
.PHONY: build-man-page
299

300
# Generate dfget man page.
301
build-dfget-man-page:
302
	@pandoc -s -t man ./build/package/docs/dfget.1.md -o ./build/package/docs/dfget.1
303
.PHONY: build-dfget-man-page
304

305
# Genrate dfcache man pages.
306
build-dfcache-man-page:
307
	@pandoc -s -t man ./build/package/docs/dfcache/dfcache.md -o ./build/package/docs/dfcache/dfcache.1
308
	@pandoc -s -t man ./build/package/docs/dfcache/dfcache_delete.md -o ./build/package/docs/dfcache/dfcache-delete.1
309
	@pandoc -s -t man ./build/package/docs/dfcache/dfcache_doc.md -o ./build/package/docs/dfcache/dfcache-doc.1
310
	@pandoc -s -t man ./build/package/docs/dfcache/dfcache_export.md -o ./build/package/docs/dfcache/dfcache-export.1
311
	@pandoc -s -t man ./build/package/docs/dfcache/dfcache_import.md -o ./build/package/docs/dfcache/dfcache-import.1
312
	@pandoc -s -t man ./build/package/docs/dfcache/dfcache_plugin.md -o ./build/package/docs/dfcache/dfcache-plugin.1
313
	@pandoc -s -t man ./build/package/docs/dfcache/dfcache_stat.md -o ./build/package/docs/dfcache/dfcache-stat.1
314
	@pandoc -s -t man ./build/package/docs/dfcache/dfcache_version.md -o ./build/package/docs/dfcache/dfcache-version.1
315
.PHONY: build-dfcache-man-page
316

317
# Genrate dfstore man pages.
318
build-dfstore-man-page:
319
	@pandoc -s -t man ./build/package/docs/dfstore/dfstore.md -o ./build/package/docs/dfstore/dfstore.1
320
	@pandoc -s -t man ./build/package/docs/dfstore/dfstore_copy.md -o ./build/package/docs/dfstore/dfstore-copy.1
321
	@pandoc -s -t man ./build/package/docs/dfstore/dfstore_remove.md -o ./build/package/docs/dfstore/dfstore-remove.1
322
	@pandoc -s -t man ./build/package/docs/dfstore/dfstore_version.md -o ./build/package/docs/dfstore/dfstore-version.1
323
.PHONY: build-dfstore-man-page
324

325
# Generate e2e sha256sum.
326
build-e2e-sha256sum:
327
	@GOOS=linux GOARCH=amd64 go build -o /tmp/sha256sum-offset test/tools/sha256sum-offset/main.go
328
.PHONY: build-e2e-sha256sum
329

330
# Generate e2e download grpc test binary.
331
build-e2e-download-grpc-test:
332
	@GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /tmp/download-grpc-test test/tools/download-grpc-test/main.go
333
.PHONY: build-e2e-download-grpc-test
334

335
# Run unittests.
336
test:
337
	@go test -v -race -short ${PKG_LIST}
338
.PHONY: test
339

340
# Run tests with coverage.
341
test-coverage:
342
	@go test -v -race -short ${PKG_LIST} -coverprofile cover.out -covermode=atomic
343
	@cat cover.out >> coverage.txt
344
.PHONY: test-coverage
345

346
# Run github actions E2E tests with coverage.
347
actions-e2e-test-coverage:
348
	@ginkgo -v -r --race --fail-fast --cover --trace --show-node-events test/e2e
349
	@cat coverprofile.out >> coverage.txt
350
.PHONY: actions-e2e-test-coverage
351

352
# Install E2E tests environment.
353
install-e2e-test:
354
	@./hack/install-e2e-test.sh
355
.PHONY: install-e2e-test
356

357
# Run E2E tests.
358
e2e-test: install-e2e-test build-e2e-sha256sum build-e2e-download-grpc-test
359
	@ginkgo -v -r --race --fail-fast --cover --trace --show-node-events test/e2e
360
.PHONY: e2e-test
361

362
# Run E2E tests with coverage.
363
e2e-test-coverage: install-e2e-test build-e2e-sha256sum build-e2e-download-grpc-test
364
	@ginkgo -v -r --race --fail-fast --cover --trace --show-node-events test/e2e
365
	@cat coverprofile.out >> coverage.txt
366
.PHONY: e2e-test-coverage
367

368
# Clean E2E tests.
369
clean-e2e-test: 
370
	@kind delete cluster
371
	@echo "cleaning log file."
372
	@rm -rf test/e2e/*.log
373
.PHONY: clean-e2e-test
374

375
# Kind load dragonfly.
376
kind-load: kind-load-scheduler kind-load-dfdaemon kind-load-manager kind-load-trainer kind-load-testing-tools
377
	@echo "Kind load image done."
378
.PHONY: kind-load
379

380
# Run kind load docker scheduler.
381
kind-load-scheduler:
382
	@./hack/kind-load.sh scheduler
383
.PHONY: kind-load-scheduler
384

385
# Run kind load docker dfget.
386
kind-load-dfdaemon:
387
	@./hack/kind-load.sh dfdaemon
388
.PHONY: kind-load-dfget
389

390
# Run kind load docker manager.
391
kind-load-manager:
392
	@./hack/kind-load.sh manager
393
.PHONY: kind-load-manager
394

395
# Run kind load docker trainer.
396
kind-load-trainer:
397
	@./hack/kind-load.sh trainer
398
.PHONY: kind-load-trainer
399

400
# Run kind load docker testing tools.
401
kind-load-testing-tools:
402
	@./hack/kind-load.sh no-content-length
403
.PHONY: kind-load-testing-tools
404

405
# Run code lint.
406
lint: markdownlint
407
	@echo "Begin to golangci-lint."
408
	@golangci-lint run
409
.PHONY: lint
410

411
# Run markdown lint.
412
markdownlint:
413
	@echo "Begin to markdownlint."
414
	@./hack/markdownlint.sh
415
.PHONY: markdownlint
416

417
# Run go generate.
418
generate:
419
	@go generate ${PKG_LIST}
420
.PHONY: generate
421

422
# Generate swagger files.
423
swag:
424
	@swag init --parseDependency --parseInternal -g cmd/manager/main.go -o api/manager
425

426
# Generate changelog.
427
changelog:
428
	@git-chglog -o CHANGELOG.md
429
.PHONY: changelog
430

431
# Clear compiled files.
432
clean:
433
	@go clean
434
	@rm -rf bin .go .cache
435
.PHONY: clean
436

437
help: 
438
	@echo "make build-dirs                     prepare required folders for build"
439
	@echo "make docker-build                   build dragonfly image"
440
	@echo "make docker-push                    push dragonfly image"
441
	@echo "make docker-build-dfdaemon          build dfdaemon image"
442
	@echo "make docker-build-scheduler         build scheduler image"
443
	@echo "make docker-build-manager           build manager image"
444
	@echo "make docker-build-trainer           build trainer image"
445
	@echo "make docker-push-dfdaemon           push dfdaemon image"
446
	@echo "make docker-push-scheduler          push scheduler image"
447
	@echo "make docker-push-manager            push manager image"
448
	@echo "make docker-push-trainer            push trainer image"
449
	@echo "make build                          build dragonfly"
450
	@echo "make build-dfget                    build dfget"
451
	@echo "make build-linux-dfget              build linux dfget"
452
	@echo "make build-dfcache                  build dfcache"
453
	@echo "make build-linux-dfcache            build linux dfcache"
454
	@echo "make build-dfstore                  build dfstore"
455
	@echo "make build-linux-dfstore            build linux dfstore"
456
	@echo "make build-scheduler                build scheduler"
457
	@echo "make build-manager                  build manager"
458
	@echo "make build-manager-server           build manager server"
459
	@echo "make build-manager-console          build manager console"
460
	@echo "make build-trainer                  build trainer"
461
	@echo "make build-e2e-sha256sum            build sha256sum test tool"
462
	@echo "make build-e2e-download-grpc-test   build download grpc test tool"
463
	@echo "make install-dfget                  install dfget"
464
	@echo "make install-scheduler              install scheduler"
465
	@echo "make install-manager                install manager"
466
	@echo "make install-trainer                install trainer"
467
	@echo "make build-rpm-dfget                build rpm dfget"
468
	@echo "make build-rpm-dfcache              build rpm dfcache"
469
	@echo "make build-rpm-dfstore              build rpm dfstore"
470
	@echo "make build-deb-dfget                build deb dfget"
471
	@echo "make build-deb-dfcache              build deb dfcache"
472
	@echo "make build-deb-dfstore              build deb dfstore"
473
	@echo "make build-man-page                 generate man page"
474
	@echo "make build-dfget-man-page           generate dfget man page"
475
	@echo "make build-dfcache-man-page         generate dfcache man page"
476
	@echo "make build-dfstore-man-page         generate dfstore man page"
477
	@echo "make test                           run unit tests"
478
	@echo "make test-coverage                  run tests with coverage"
479
	@echo "make actions-e2e-test-coverage      run github actons E2E tests with coverage"
480
	@echo "make install-e2e-test               install E2E tests environment"
481
	@echo "make e2e-test                       run e2e tests"
482
	@echo "make e2e-test-coverage              run e2e tests with coverage"
483
	@echo "make clean-e2e-test                 clean e2e tests"
484
	@echo "make kind-load                      kind load docker image"
485
	@echo "make kind-load-scheduler            kind load scheduler docker image"
486
	@echo "make kind-load-dfdaemon             kind load dfdaemon docker image"
487
	@echo "make kind-load-manager              kind load manager docker image"
488
	@echo "make kind-load-trainer              kind load trainer docker image"
489
	@echo "make kind-load-testing-tools        kind load testing tools docker image"
490
	@echo "make lint                           run code lint"
491
	@echo "make markdownlint                   run markdown lint"
492
	@echo "make generate                       run go generate"
493
	@echo "make swag                           generate swagger api docs"
494
	@echo "make changelog                      generate CHANGELOG.md"
495
	@echo "make clean                          clean"
496