wal-g

1
MAIN_PG_PATH := main/pg
2
MAIN_MYSQL_PATH := main/mysql
3
MAIN_SQLSERVER_PATH := main/sqlserver
4
MAIN_REDIS_PATH := main/redis
5
MAIN_MONGO_PATH := main/mongo
6
MAIN_FDB_PATH := main/fdb
7
MAIN_GP_PATH := main/gp
8
MAIN_ETCD_PATH := main/etcd
9
DOCKER_COMMON := golang ubuntu ubuntu_20_04 s3
10
CMD_FILES = $(wildcard cmd/**/*.go)
11
PKG_FILES = $(wildcard internal/**/*.go internal/**/**/*.go internal/*.go)
12
TEST_FILES = $(wildcard test/*.go testtools/*.go)
13
PKG := github.com/wal-g/wal-g
14
COVERAGE_FILE := coverage.out
15
TEST := "pg_tests"
16
MYSQL_TEST := "mysql_base_tests"
17
MONGO_MAJOR ?= "4.2"
18
MONGO_VERSION ?= "4.2.8"
19
GOLANGCI_LINT_VERSION ?= "v1.52.2"
20
REDIS_VERSION ?= "5.0.8"
21
TOOLS_MOD_DIR := ./internal/tools
22

23
BUILD_TAGS:=
24

25
ifdef USE_BROTLI
26
	BUILD_TAGS:=$(BUILD_TAGS) brotli
27
endif
28

29
ifdef USE_LIBSODIUM
30
	BUILD_TAGS:=$(BUILD_TAGS) libsodium
31
endif
32

33
ifdef USE_LZO
34
	BUILD_TAGS:=$(BUILD_TAGS) lzo
35
endif
36

37
.PHONY: unittest fmt lint clean install_tools
38

39
test: deps unittest pg_build mysql_build redis_build mongo_build gp_build unlink_brotli pg_integration_test mysql_integration_test redis_integration_test fdb_integration_test gp_integration_test etcd_integration_test
40

41
pg_test: deps pg_build unlink_brotli pg_integration_test
42

43
pg_build: $(CMD_FILES) $(PKG_FILES)
44
	(cd $(MAIN_PG_PATH) && go build -mod vendor -tags "$(BUILD_TAGS)" -o wal-g -ldflags "-s -w -X github.com/wal-g/wal-g/cmd/pg.buildDate=`date -u +%Y.%m.%d_%H:%M:%S` -X github.com/wal-g/wal-g/cmd/pg.gitRevision=`git rev-parse --short HEAD` -X github.com/wal-g/wal-g/cmd/pg.walgVersion=`git tag -l --points-at HEAD`")
45

46
install_and_build_pg: deps pg_build
47

48
pg_build_image:
49
	# There are dependencies between container images.
50
	# Running in one command leads to using outdated images and fails on clean system.
51
	# It can not be fixed with depends_on in compose file. https://github.com/docker/compose/issues/6332
52
	docker-compose build $(DOCKER_COMMON)
53
	docker-compose build pg
54
	docker-compose build pg_build_docker_prefix
55

56
pg_save_image: install_and_build_pg pg_build_image
57
	mkdir -p ${CACHE_FOLDER}
58
	sudo rm -rf ${CACHE_FOLDER}/*
59
	docker save ${IMAGE} | gzip -c > ${CACHE_FILE_DOCKER_PREFIX}
60
	docker save ${IMAGE_UBUNTU_18_04} | gzip -c > ${CACHE_FILE_UBUNTU_18_04}
61
	docker save ${IMAGE_UBUNTU_20_04} | gzip -c > ${CACHE_FILE_UBUNTU_20_04}
62
	docker save ${IMAGE_GOLANG} | gzip -c > ${CACHE_FILE_GOLANG}
63
	ls ${CACHE_FOLDER}
64

65
pg_integration_test: clean_compose
66
	@if [ "x" = "${CACHE_FILE_DOCKER_PREFIX}x" ]; then\
67
		echo "Rebuild";\
68
		make install_and_build_pg;\
69
		make pg_build_image;\
70
	else\
71
		docker load -i ${CACHE_FILE_DOCKER_PREFIX};\
72
	fi
73
	@if echo "$(TEST)" | grep -Fqe "pgbackrest"; then\
74
		docker-compose build pg_pgbackrest;\
75
	fi
76
	@if echo "$(TEST)" | grep -Fqe "pg_ssh_"; then\
77
		docker-compose build ssh;\
78
	fi
79

80
	docker-compose up --exit-code-from $(TEST) $(TEST)
81
	# Run tests with dependencies if we run all tests
82
	@if [ "$(TEST)" = "pg_tests" ]; then\
83
		docker-compose build pg_pgbackrest ssh swift pg_wal_perftest_with_throttling &&\
84
		docker-compose up --exit-code-from pg_ssh_backup_test pg_ssh_backup_test &&\
85
		docker-compose up --exit-code-from pg_storage_swift_test pg_storage_swift_test &&\
86
		docker-compose up --exit-code-from pg_storage_ssh_test pg_storage_ssh_test &&\
87
		docker-compose up --exit-code-from pg_pgbackrest_backup_fetch_test pg_pgbackrest_backup_fetch_test &&\
88
		docker-compose down &&\
89
		sleep 5 &&\
90
		docker-compose up --exit-code-from pg_wal_perftest_with_throttling pg_wal_perftest_with_throttling ;\
91
	fi
92
	make clean_compose
93

94
.PHONY: clean_compose
95
clean_compose:
96
	services=$$(docker-compose ps -a --format '{{.Name}} {{.Service}}' | grep wal-g_ | cut -w -f 2); \
97
		if [ "$$services" ]; then docker-compose down $$services; fi
98

99
all_unittests: deps unittest
100

101
# todo Should we remove this target as a duplicate of pg_integration_test?
102
pg_int_tests_only:
103
	docker-compose build pg_tests
104
	docker-compose up --exit-code-from pg_tests pg_tests
105

106
pg_clean:
107
	(cd $(MAIN_PG_PATH) && go clean)
108
	./cleanup.sh
109

110
pg_install: pg_build
111
	mv $(MAIN_PG_PATH)/wal-g $(GOBIN)/wal-g
112

113
mysql_base: deps mysql_build unlink_brotli
114
mysql_test: deps mysql_build unlink_brotli mysql_integration_test
115

116
mysql_build: $(CMD_FILES) $(PKG_FILES)
117
	(cd $(MAIN_MYSQL_PATH) && go build -mod vendor -tags "$(BUILD_TAGS)" -o wal-g -ldflags "-s -w -X github.com/wal-g/wal-g/cmd/mysql.buildDate=`date -u +%Y.%m.%d_%H:%M:%S` -X github.com/wal-g/wal-g/cmd/mysql.gitRevision=`git rev-parse --short HEAD` -X github.com/wal-g/wal-g/cmd/mysql.walgVersion=`git tag -l --points-at HEAD`")
118

119
sqlserver_build: $(CMD_FILES) $(PKG_FILES)
120
	(cd $(MAIN_SQLSERVER_PATH) && go build -mod vendor -tags "$(BUILD_TAGS)" -o wal-g -ldflags "-s -w -X github.com/wal-g/wal-g/cmd/sqlserver.buildDate=`date -u +%Y.%m.%d_%H:%M:%S` -X github.com/wal-g/wal-g/cmd/sqlserver.gitRevision=`git rev-parse --short HEAD` -X github.com/wal-g/wal-g/cmd/sqlserver.walgVersion=`git tag -l --points-at HEAD`")
121

122
load_docker_common:
123
	@if [ "x" = "${CACHE_FOLDER}x" ]; then\
124
		echo "Rebuild";\
125
		docker-compose build $(DOCKER_COMMON);\
126
	else\
127
		docker load -i ${CACHE_FILE_UBUNTU_18_04};\
128
		docker load -i ${CACHE_FILE_UBUNTU_20_04};\
129
		docker load -i ${CACHE_FILE_GOLANG};\
130
	fi
131

132
mysql_integration_test: deps mysql_build unlink_brotli load_docker_common
133
	./link_brotli.sh
134
	docker-compose build mysql $(MYSQL_TEST)
135
	docker-compose up --force-recreate --exit-code-from $(MYSQL_TEST) $(MYSQL_TEST)
136

137
mysql_clean:
138
	(cd $(MAIN_MYSQL_PATH) && go clean)
139
	./cleanup.sh
140

141
mysql_install: mysql_build
142
	mv $(MAIN_MYSQL_PATH)/wal-g $(GOBIN)/wal-g
143

144
mariadb_test: deps mysql_build unlink_brotli mariadb_integration_test
145

146
mariadb_integration_test: unlink_brotli load_docker_common
147
	./link_brotli.sh
148
	docker-compose build mariadb mariadb_tests
149
	docker-compose up --force-recreate --exit-code-from mariadb_tests mariadb_tests
150

151
mongo_test: deps mongo_build unlink_brotli
152

153
mongo_build: $(CMD_FILES) $(PKG_FILES)
154
	(cd $(MAIN_MONGO_PATH) && go build $(BUILD_ARGS) -mod vendor -tags "$(BUILD_TAGS)" -o wal-g -ldflags "-s -w -X github.com/wal-g/wal-g/cmd/mongo.buildDate=`date -u +%Y.%m.%d_%H:%M:%S` -X github.com/wal-g/wal-g/cmd/mongo.gitRevision=`git rev-parse --short HEAD` -X github.com/wal-g/wal-g/cmd/mongo.walgVersion=`git tag -l --points-at HEAD`")
155

156
mongo_install: mongo_build
157
	mv $(MAIN_MONGO_PATH)/wal-g $(GOBIN)/wal-g
158

159
mongo_features:
160
	set -e
161
	make go_deps
162
	cd tests_func/ && MONGO_MAJOR=$(MONGO_MAJOR) MONGO_VERSION=$(MONGO_VERSION) go test -v -count=1 -timeout 20m  --tf.test=true --tf.debug=true --tf.clean=true --tf.stop=true --tf.database=mongodb
163

164
clean_mongo_features:
165
	set -e
166
	cd tests_func/ && MONGO_MAJOR=$(MONGO_MAJOR) MONGO_VERSION=$(MONGO_VERSION) go test -v -count=1  -timeout 5m --tf.test=false --tf.debug=false --tf.clean=true --tf.stop=true --tf.database=mongodb
167

168
fdb_build: $(CMD_FILES) $(PKG_FILES)
169
	(cd $(MAIN_FDB_PATH) && go build -mod vendor -tags "$(BUILD_TAGS)" -o wal-g -ldflags "-s -w")
170

171
fdb_install: fdb_build
172
	mv $(MAIN_FDB_PATH)/wal-g $(GOBIN)/wal-g
173

174
fdb_integration_test: load_docker_common
175
	docker-compose down -v
176
	docker-compose build fdb_tests
177
	docker-compose up --force-recreate --renew-anon-volumes --exit-code-from fdb_tests fdb_tests
178

179
redis_test: deps redis_build unlink_brotli redis_integration_test
180

181
redis_build: $(CMD_FILES) $(PKG_FILES)
182
	(cd $(MAIN_REDIS_PATH) && go build -mod vendor -tags "$(BUILD_TAGS)" -o wal-g -ldflags "-s -w -X github.com/wal-g/wal-g/cmd/redis.buildDate=`date -u +%Y.%m.%d_%H:%M:%S` -X github.com/wal-g/wal-g/cmd/redis.gitRevision=`git rev-parse --short HEAD` -X github.com/wal-g/wal-g/cmd/redis.walgVersion=`git tag -l --points-at HEAD`")
183

184
redis_integration_test: load_docker_common
185
	docker-compose build redis redis_tests
186
	docker-compose up --exit-code-from redis_tests redis_tests
187

188
redis_clean:
189
	(cd $(MAIN_REDIS_PATH) && go clean)
190
	./cleanup.sh
191

192
redis_install: redis_build
193
	mv $(MAIN_REDIS_PATH)/wal-g $(GOBIN)/wal-g
194

195
redis_features:
196
	set -e
197
	make go_deps
198
	cd tests_func/ && REDIS_VERSION=$(REDIS_VERSION) go test -v -count=1 -timeout 20m  --tf.test=true --tf.debug=false --tf.clean=true --tf.stop=true --tf.database=redis
199

200
clean_redis_features:
201
	set -e
202
	cd tests_func/ && REDIS_VERSION=$(REDIS_VERSION) go test -v -count=1  -timeout 5m --tf.test=false --tf.debug=false --tf.clean=true --tf.stop=true --tf.database=redis
203

204
etcd_test: deps etcd_build unlink_brotli etcd_integration_test
205

206
etcd_build: $(CMD_FILES) $(PKG_FILES)
207
	(cd $(MAIN_ETCD_PATH) && go build -mod vendor -tags "$(BUILD_TAGS)" -o wal-g -ldflags "-s -w -X github.com/wal-g/wal-g/cmd/etcd.buildDate=`date -u +%Y.%m.%d_%H:%M:%S` -X github.com/wal-g/wal-g/cmd/etcd.gitRevision=`git rev-parse --short HEAD` -X github.com/wal-g/wal-g/cmd/etcd.walgVersion=`git tag -l --points-at HEAD`")
208

209
etcd_install: etcd_build
210
	mv $(MAIN_ETCD_PATH)/wal-g $(GOBIN)/wal-g
211

212
etcd_clean:
213
	(cd $(MAIN_ETCD_PATH) && go clean)
214
	./cleanup.sh
215

216
# refactor
217
etcd_integration_test: load_docker_common
218
	docker-compose build etcd etcd_tests
219
	docker-compose up --exit-code-from etcd_tests etcd_tests
220

221
gp_build: $(CMD_FILES) $(PKG_FILES)
222
	(cd $(MAIN_GP_PATH) && go build -mod vendor -tags "$(BUILD_TAGS)" -o wal-g -ldflags "-s -w -X github.com/wal-g/wal-g/cmd/gp.buildDate=`date -u +%Y.%m.%d_%H:%M:%S` -X github.com/wal-g/wal-g/cmd/gp.gitRevision=`git rev-parse --short HEAD` -X github.com/wal-g/wal-g/cmd/gp.walgVersion=`git tag -l --points-at HEAD`")
223

224
gp_clean:
225
	(cd $(MAIN_GP_PATH) && go clean)
226
	./cleanup.sh
227

228
gp_install: gp_build
229
	mv $(MAIN_GP_PATH)/wal-g $(GOBIN)/wal-g
230

231
gp_test: deps gp_build unlink_brotli gp_integration_test
232

233
gp_integration_test: load_docker_common
234
	docker-compose build gp gp_tests
235
	docker-compose up --exit-code-from gp_tests gp_tests
236

237
st_test: deps pg_build unlink_brotli st_integration_test
238

239
st_integration_test: load_docker_common
240
	docker-compose build st_tests
241
	docker-compose up --exit-code-from st_tests st_tests
242

243
unittest:
244
	go list ./... | grep -Ev 'vendor|submodules|tmp' | xargs go vet
245
	go test -mod vendor -v $(TEST_MODIFIER) -tags "$(BUILD_TAGS)" ./internal/...
246
	go test -mod vendor -v $(TEST_MODIFIER) -tags "$(BUILD_TAGS)" ./pkg/...
247
	go test -mod vendor -v $(TEST_MODIFIER) -tags "$(BUILD_TAGS)" ./utility/...
248

249
coverage:
250
	go list ./... | grep -Ev 'vendor|submodules|tmp' | xargs go test -v $(TEST_MODIFIER) -coverprofile=$(COVERAGE_FILE) | grep -v 'no test files'
251
	go tool cover -html=$(COVERAGE_FILE)
252

253
install_tools:
254
	cd $(TOOLS_MOD_DIR) && go install golang.org/x/tools/cmd/goimports
255
	$(warning Please run make docker_lint for lint. It is unreliable to use self-compiled golangci-lint. \
256
		https://golangci-lint.run/usage/install/#install-from-source)
257
	cd $(TOOLS_MOD_DIR) && go install github.com/golangci/golangci-lint/cmd/golangci-lint
258

259
fmt: $(CMD_FILES) $(PKG_FILES) $(TEST_FILES)
260
	gofmt -s -w $(CMD_FILES) $(PKG_FILES) $(TEST_FILES)
261

262
goimports: install_tools $(CMD_FILES) $(PKG_FILES) $(TEST_FILES)
263
	goimports -w $(CMD_FILES) $(PKG_FILES) $(TEST_FILES)
264

265
lint: install_tools
266
	golangci-lint run --allow-parallel-runners ./...
267

268
docker_lint:
269
	docker build -t wal-g/lint --build-arg TAG=$(GOLANGCI_LINT_VERSION) - < docker/lint/Dockerfile
270
	docker run --rm -v `pwd`:/app \
271
		-v wal-g_lint_cache:/cache -e GOLANGCI_LINT_CACHE=/cache/lint \
272
		-e GOCACHE=/cache/go -e GOMODCACHE=/cache/gomod \
273
		wal-g/lint golangci-lint run -v
274

275
deps: go_deps link_external_deps
276

277
go_deps:
278
	git submodule update --init
279
	go mod vendor
280
ifdef USE_LZO
281
	sed -i 's|\(#cgo LDFLAGS:\) .*|\1 -Wl,-Bstatic -llzo2 -Wl,-Bdynamic|' vendor/github.com/cyberdelia/lzo/lzo.go
282
endif
283

284
link_external_deps: link_brotli link_libsodium
285

286
unlink_external_deps: unlink_brotli unlink_libsodium
287

288
install:
289
	@echo "Nothing to be done. Use pg_install/mysql_install/mongo_install/fdb_install/gp_install/etcd_install... instead."
290

291
link_brotli:
292
	@if [ -n "${USE_BROTLI}" ]; then ./link_brotli.sh; fi
293
	@if [ -z "${USE_BROTLI}" ]; then echo "info: USE_BROTLI is not set, skipping 'link_brotli' task"; fi
294

295
link_libsodium:
296
	@if [ ! -z "${USE_LIBSODIUM}" ]; then\
297
		./link_libsodium.sh;\
298
	fi
299

300
unlink_brotli:
301
	rm -rf vendor/github.com/google/brotli/*
302
	if [ -n "${USE_BROTLI}" ] ; then mv tmp/brotli/* vendor/github.com/google/brotli/; fi
303
	rm -rf tmp/brotli
304

305
unlink_libsodium:
306
	rm -rf tmp/libsodium
307

308
build_client:
309
	cd cmd/daemonclient && \
310
	go build -o ../../bin/walg-daemon-client -ldflags "-s -w -X main.buildDate=`date -u +%Y.%m.%d_%H:%M:%S` -X main.gitRevision=`git rev-parse --short HEAD` -X main.version=`git tag -l --points-at HEAD`"
311