kuma

1
//go:build !windows
2
// +build !windows
3

4
package cmd
5

6
import (
7
	"context"
8
	"fmt"
9
	"io"
10
	"os"
11
	"path/filepath"
12
	"strconv"
13
	"strings"
14
	"syscall"
15

16
	envoy_bootstrap_v3 "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3"
17
	. "github.com/onsi/ginkgo/v2"
18
	. "github.com/onsi/gomega"
19

20
	"github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/envoy"
21
	kuma_cmd "github.com/kumahq/kuma/pkg/cmd"
22
	kumadp "github.com/kumahq/kuma/pkg/config/app/kuma-dp"
23
	util_proto "github.com/kumahq/kuma/pkg/util/proto"
24
	"github.com/kumahq/kuma/pkg/xds/bootstrap/types"
25
)
26

27
var _ = Describe("run", func() {
28
	var cancel func()
29
	var ctx context.Context
30
	opts := kuma_cmd.RunCmdOpts{
31
		SetupSignalHandler: func() (context.Context, context.Context) {
32
			return ctx, ctx
33
		},
34
	}
35

36
	var tmpDir string
37

38
	BeforeEach(func() {
39
		ctx, cancel = context.WithCancel(context.Background())
40
		var err error
41
		tmpDir, err = os.MkdirTemp("", "")
42
		Expect(err).ToNot(HaveOccurred())
43
	})
44
	AfterEach(func() {
45
		if tmpDir != "" {
46
			// when
47
			err := os.RemoveAll(tmpDir)
48
			// then
49
			Expect(err).ToNot(HaveOccurred())
50
		}
51
	})
52

53
	var backupEnvVars []string
54

55
	BeforeEach(func() {
56
		backupEnvVars = os.Environ()
57
	})
58
	AfterEach(func() {
59
		os.Clearenv()
60
		for _, envVar := range backupEnvVars {
61
			parts := strings.SplitN(envVar, "=", 2)
62
			Expect(os.Setenv(parts[0], parts[1])).To(Succeed())
63
		}
64
	})
65

66
	type testCase struct {
67
		envVars      map[string]string
68
		args         []string
69
		expectedFile string
70
	}
71
	DescribeTable("should be possible to start dataplane (Envoy) using `kuma-dp run`",
72
		func(givenFunc func() testCase) {
73
			given := givenFunc()
74

75
			// setup
76
			envoyPidFile := filepath.Join(tmpDir, "envoy-mock.pid")
77
			envoyCmdlineFile := filepath.Join(tmpDir, "envoy-mock.cmdline")
78
			corednsPidFile := filepath.Join(tmpDir, "coredns-mock.pid")
79
			corednsCmdlineFile := filepath.Join(tmpDir, "coredns-mock.cmdline")
80

81
			// and
82
			env := given.envVars
83
			env["ENVOY_MOCK_PID_FILE"] = envoyPidFile
84
			env["ENVOY_MOCK_CMDLINE_FILE"] = envoyCmdlineFile
85
			env["COREDNS_MOCK_PID_FILE"] = corednsPidFile
86
			env["COREDNS_MOCK_CMDLINE_FILE"] = corednsCmdlineFile
87
			for key, value := range env {
88
				Expect(os.Setenv(key, value)).To(Succeed())
89
			}
90

91
			// given
92
			rootCtx := DefaultRootContext()
93
			rootCtx.BootstrapGenerator = func(_ context.Context, _ string, cfg kumadp.Config, _ envoy.BootstrapParams) (*envoy_bootstrap_v3.Bootstrap, *types.KumaSidecarConfiguration, error) {
94
				respBytes, err := os.ReadFile(filepath.Join("testdata", "bootstrap-config.golden.yaml"))
95
				Expect(err).ToNot(HaveOccurred())
96
				bootstrap := &envoy_bootstrap_v3.Bootstrap{}
97
				if err := util_proto.FromYAML(respBytes, bootstrap); err != nil {
98
					return nil, nil, err
99
				}
100
				return bootstrap, &types.KumaSidecarConfiguration{}, nil
101
			}
102

103
			reader, writer := io.Pipe()
104
			go func() {
105
				defer GinkgoRecover()
106
				_, err := io.ReadAll(reader)
107
				Expect(err).ToNot(HaveOccurred())
108
			}()
109

110
			cmd := NewRootCmd(opts, rootCtx)
111
			cmd.SetArgs(append([]string{"run"}, given.args...))
112
			cmd.SetOut(writer)
113
			cmd.SetErr(writer)
114

115
			// when
116
			By("starting the Kuma DP")
117
			errCh := make(chan error)
118
			go func() {
119
				defer close(errCh)
120
				errCh <- cmd.Execute()
121
			}()
122

123
			// then
124
			var actualConfigFile string
125
			envoyPid := verifyComponentProcess("Envoy", envoyPidFile, envoyCmdlineFile, func(actualArgs []string) {
126
				Expect(actualArgs[0]).To(Equal("--version"))
127
				Expect(actualArgs[1]).To(Equal("--config-path"))
128
				actualConfigFile = actualArgs[2]
129
				Expect(actualConfigFile).To(BeARegularFile())
130
				if given.expectedFile != "" {
131
					Expect(actualArgs[2]).To(Equal(given.expectedFile))
132
				}
133
			})
134

135
			corednsPid := verifyComponentProcess("coredns", corednsPidFile, corednsCmdlineFile, func(actualArgs []string) {
136
				Expect(actualArgs).To(HaveLen(3))
137
				Expect(actualArgs[0]).To(Equal("-conf"))
138
				Expect(actualArgs[2]).To(Equal("-quiet"))
139
			})
140

141
			// when
142
			By("signaling the dataplane manager to stop")
143
			// we need to close writer, otherwise Cmd#Wait will never finish.
144
			Expect(writer.Close()).To(Succeed())
145
			cancel()
146

147
			// then
148
			err := <-errCh
149
			Expect(err).ToNot(HaveOccurred())
150

151
			By("waiting for dataplane (Envoy) to get stopped")
152
			Eventually(func() bool {
153
				// send sig 0 to check whether Envoy process still exists
154
				err := syscall.Kill(int(envoyPid), syscall.Signal(0))
155
				// we expect Envoy process to get killed by now
156
				return err != nil
157
			}, "5s", "100ms").Should(BeTrue())
158
			By("waiting for dataplane (coredns) to get stopped")
159
			Eventually(func() bool {
160
				// send sig 0 to check whether Envoy process still exists
161
				err := syscall.Kill(int(corednsPid), syscall.Signal(0))
162
				// we expect Envoy process to get killed by now
163
				return err != nil
164
			}, "5s", "100ms").Should(BeTrue())
165

166
			By("verifying that temporary configuration dir gets removed")
167
			if given.expectedFile == "" {
168
				Expect(actualConfigFile).NotTo(BeAnExistingFile())
169
			}
170

171
			By("verifying that explicit configuration dir is not removed")
172
			if given.expectedFile != "" {
173
				Expect(given.expectedFile).To(BeAnExistingFile())
174
			}
175
		},
176
		Entry("can be launched with env vars", func() testCase {
177
			return testCase{
178
				envVars: map[string]string{
179
					"KUMA_CONTROL_PLANE_API_SERVER_URL":  "http://localhost:1234",
180
					"KUMA_DATAPLANE_NAME":                "example",
181
					"KUMA_DATAPLANE_MESH":                "default",
182
					"KUMA_DATAPLANE_RUNTIME_BINARY_PATH": filepath.Join("testdata", "envoy-mock.sleep.sh"),
183
					// Notice: KUMA_DATAPLANE_RUNTIME_CONFIG_DIR is not set in order to let `kuma-dp` to create a temporary directory
184
					"KUMA_DNS_CORE_DNS_BINARY_PATH": filepath.Join("testdata", "coredns-mock.sleep.sh"),
185
				},
186
				args:         []string{},
187
				expectedFile: "",
188
			}
189
		}),
190
		Entry("can be launched with env vars and given config dir", func() testCase {
191
			return testCase{
192
				envVars: map[string]string{
193
					"KUMA_CONTROL_PLANE_API_SERVER_URL":  "http://localhost:1234",
194
					"KUMA_DATAPLANE_NAME":                "example",
195
					"KUMA_DATAPLANE_MESH":                "default",
196
					"KUMA_DATAPLANE_RUNTIME_BINARY_PATH": filepath.Join("testdata", "envoy-mock.sleep.sh"),
197
					"KUMA_DATAPLANE_RUNTIME_CONFIG_DIR":  tmpDir,
198
					"KUMA_DNS_CORE_DNS_BINARY_PATH":      filepath.Join("testdata", "coredns-mock.sleep.sh"),
199
				},
200
				args:         []string{},
201
				expectedFile: filepath.Join(tmpDir, "bootstrap.yaml"),
202
			}
203
		}),
204
		Entry("can be launched with args", func() testCase {
205
			return testCase{
206
				envVars: map[string]string{},
207
				args: []string{
208
					"--cp-address", "http://localhost:1234",
209
					"--name", "example",
210
					"--mesh", "default",
211
					"--binary-path", filepath.Join("testdata", "envoy-mock.sleep.sh"),
212
					// Notice: --config-dir is not set in order to let `kuma-dp` to create a temporary directory
213
					"--dns-coredns-path", filepath.Join("testdata", "coredns-mock.sleep.sh"),
214
				},
215
				expectedFile: "",
216
			}
217
		}),
218
		Entry("can be launched with args and given config dir", func() testCase {
219
			return testCase{
220
				envVars: map[string]string{},
221
				args: []string{
222
					"--cp-address", "http://localhost:1234",
223
					"--name", "example",
224
					"--mesh", "default",
225
					"--binary-path", filepath.Join("testdata", "envoy-mock.sleep.sh"),
226
					"--config-dir", tmpDir,
227
					"--dns-coredns-path", filepath.Join("testdata", "coredns-mock.sleep.sh"),
228
				},
229
				expectedFile: filepath.Join(tmpDir, "bootstrap.yaml"),
230
			}
231
		}),
232
		Entry("can be launched with args and dataplane token", func() testCase {
233
			return testCase{
234
				envVars: map[string]string{},
235
				args: []string{
236
					"--cp-address", "http://localhost:1234",
237
					"--name", "example",
238
					"--mesh", "default",
239
					"--binary-path", filepath.Join("testdata", "envoy-mock.sleep.sh"),
240
					"--dataplane-token-file", filepath.Join("testdata", "token"),
241
					// Notice: --config-dir is not set in order to let `kuma-dp` to create a temporary directory
242
					"--dns-coredns-path", filepath.Join("testdata", "coredns-mock.sleep.sh"),
243
				},
244
				expectedFile: "",
245
			}
246
		}),
247
		Entry("can be launched without Envoy Admin API (env vars)", func() testCase {
248
			return testCase{
249
				envVars: map[string]string{
250
					"KUMA_CONTROL_PLANE_API_SERVER_URL":  "http://localhost:1234",
251
					"KUMA_DATAPLANE_NAME":                "example",
252
					"KUMA_DATAPLANE_MESH":                "default",
253
					"KUMA_DATAPLANE_RUNTIME_BINARY_PATH": filepath.Join("testdata", "envoy-mock.sleep.sh"),
254
					// Notice: KUMA_DATAPLANE_RUNTIME_CONFIG_DIR is not set in order to let `kuma-dp` to create a temporary directory
255
					"KUMA_DNS_CORE_DNS_BINARY_PATH": filepath.Join("testdata", "coredns-mock.sleep.sh"),
256
				},
257
				args:         []string{},
258
				expectedFile: "",
259
			}
260
		}),
261
		Entry("can be launched without Envoy Admin API (command-line args)", func() testCase {
262
			return testCase{
263
				envVars: map[string]string{},
264
				args: []string{
265
					"--cp-address", "http://localhost:1234",
266
					"--name", "example",
267
					"--mesh", "default",
268
					"--binary-path", filepath.Join("testdata", "envoy-mock.sleep.sh"),
269
					// Notice: --config-dir is not set in order to let `kuma-dp` to create a temporary directory
270
					"--dns-coredns-path", filepath.Join("testdata", "coredns-mock.sleep.sh"),
271
				},
272
				expectedFile: "",
273
			}
274
		}),
275
		Entry("can be launched with dataplane template", func() testCase {
276
			return testCase{
277
				envVars: map[string]string{},
278
				args: []string{
279
					"--cp-address", "http://localhost:1234",
280
					"--binary-path", filepath.Join("testdata", "envoy-mock.sleep.sh"),
281
					"--dataplane-token-file", filepath.Join("testdata", "token"),
282
					"--dataplane-file", filepath.Join("testdata", "dataplane_template.yaml"),
283
					"--dataplane-var", "name=example",
284
					"--dataplane-var", "address=127.0.0.1",
285
					"--dns-coredns-path", filepath.Join("testdata", "coredns-mock.sleep.sh"),
286
				},
287
				expectedFile: "",
288
			}
289
		}),
290
		Entry("can be launched with given coredns configuration path", func() testCase {
291
			corefileTemplate := filepath.Join(tmpDir, "Corefile")
292
			_ = os.WriteFile(corefileTemplate, []byte("abcd"), 0o600)
293
			return testCase{
294
				envVars: map[string]string{
295
					"KUMA_CONTROL_PLANE_API_SERVER_URL":      "http://localhost:1234",
296
					"KUMA_DATAPLANE_NAME":                    "example",
297
					"KUMA_DATAPLANE_MESH":                    "default",
298
					"KUMA_DATAPLANE_RUNTIME_BINARY_PATH":     filepath.Join("testdata", "envoy-mock.sleep.sh"),
299
					"KUMA_DATAPLANE_RUNTIME_CONFIG_DIR":      tmpDir,
300
					"KUMA_DNS_CORE_DNS_BINARY_PATH":          filepath.Join("testdata", "coredns-mock.sleep.sh"),
301
					"KUMA_DNS_CORE_DNS_CONFIG_TEMPLATE_PATH": corefileTemplate,
302
				},
303
				args:         []string{},
304
				expectedFile: filepath.Join(tmpDir, "bootstrap.yaml"),
305
			}
306
		}),
307
	)
308

309
	It("should fail when name and mesh is provided with dataplane definition", func() {
310
		// given
311
		cmd := NewRootCmd(opts, DefaultRootContext())
312
		cmd.SetArgs([]string{
313
			"run",
314
			"--cp-address", "http://localhost:1234",
315
			"--binary-path", filepath.Join("testdata", "envoy-mock.sleep.sh"),
316
			"--dataplane-file", filepath.Join("testdata", "dataplane_template.yaml"),
317
			"--dataplane-var", "name=example",
318
			"--dataplane-var", "address=127.0.0.1",
319
			"--name=xyz",
320
			"--mesh=xyz",
321
			"--dns-coredns-path", filepath.Join("testdata", "coredns-mock.sleep.sh"),
322
		})
323

324
		// when
325
		err := cmd.Execute()
326

327
		// then
328
		Expect(err).To(HaveOccurred())
329
		Expect(err.Error()).To(ContainSubstring("--name and --mesh cannot be specified"))
330
	})
331

332
	It("should fail when the proxy type is unknown", func() {
333
		// given
334
		cmd := NewRootCmd(opts, DefaultRootContext())
335
		cmd.SetArgs([]string{
336
			"run",
337
			"--cp-address", "http://localhost:1234",
338
			"--binary-path", filepath.Join("testdata", "envoy-mock.sleep.sh"),
339
			"--dataplane-file", filepath.Join("testdata", "dataplane_template.yaml"),
340
			"--dns-coredns-path", filepath.Join("testdata", "coredns-mock.sleep.sh"),
341
			"--proxy-type", "phoney",
342
		})
343

344
		// when
345
		err := cmd.Execute()
346

347
		// then
348
		Expect(err).To(HaveOccurred())
349
		Expect(err.Error()).To(ContainSubstring("invalid proxy type"))
350
	})
351
}, Ordered)
352

353
func verifyComponentProcess(processDescription, pidfile string, cmdlinefile string, argsVerifier func(expectedArgs []string)) int64 {
354
	var pid int64
355
	By(fmt.Sprintf("waiting for dataplane (%s) to get started", processDescription))
356
	Eventually(func() bool {
357
		data, err := os.ReadFile(pidfile)
358
		if err != nil {
359
			return false
360
		}
361
		pid, err = strconv.ParseInt(strings.TrimSpace(string(data)), 10, 32)
362
		return err == nil
363
	}, "5s", "100ms").Should(BeTrue())
364
	Expect(pid).ToNot(BeZero())
365

366
	By(fmt.Sprintf("verifying the arguments %s was launched with", processDescription))
367
	// when
368
	cmdline, err := os.ReadFile(cmdlinefile)
369

370
	// then
371
	Expect(err).ToNot(HaveOccurred())
372
	// and
373
	if argsVerifier != nil {
374
		actualArgs := strings.FieldsFunc(string(cmdline), func(c rune) bool {
375
			return c == '\n'
376
		})
377
		argsVerifier(actualArgs)
378
	}
379
	return pid
380
}
381