kuma

1
package gateway
2

3
import (
4
	"encoding/base64"
5
	"fmt"
6
	"strings"
7

8
	"github.com/gruntwork-io/terratest/modules/k8s"
9
	. "github.com/onsi/ginkgo/v2"
10
	. "github.com/onsi/gomega"
11

12
	"github.com/kumahq/kuma/pkg/plugins/policies/meshtrafficpermission/api/v1alpha1"
13
	. "github.com/kumahq/kuma/test/framework"
14
	"github.com/kumahq/kuma/test/framework/client"
15
	"github.com/kumahq/kuma/test/framework/deployments/testserver"
16
	"github.com/kumahq/kuma/test/framework/envs/kubernetes"
17
)
18

19
func Mtls() {
20
	meshName := "gateway-mtls"
21
	namespace := "gateway-mtls"
22
	clientNamespace := "gateway-mtls-client"
23

24
	meshGateway := `
25
apiVersion: kuma.io/v1alpha1
26
kind: MeshGateway
27
metadata:
28
  name: mtls-edge-gateway
29
mesh: gateway-mtls
30
spec:
31
  selectors:
32
  - match:
33
      kuma.io/service: mtls-edge-gateway
34
  conf:
35
    listeners:
36
    - port: 8080
37
      protocol: HTTP
38
      hostname: example.kuma.io
39
      tags:
40
        hostname: example.kuma.io
41
    - port: 8081
42
      protocol: TCP
43
      tags:
44
        protocol: tcp
45
    - port: 8082
46
      protocol: TLS
47
      tls:
48
        mode: PASSTHROUGH
49
      hostname: example-passthrough.kuma.io
50
      tags:
51
        name: tls-passthrough
52
    - port: 8083
53
      protocol: TLS
54
      tls:
55
        mode: TERMINATE
56
        certificates:
57
        - secret: kuma-io-certificate-k8s-mtls
58
      tags:
59
        name: tls-terminate
60
`
61

62
	BeforeAll(func() {
63
		httpsSecret := func() string {
64
			cert, key, err := CreateCertsFor("example.kuma.io")
65
			Expect(err).To(Succeed())
66
			secretData := base64.StdEncoding.EncodeToString([]byte(strings.Join([]string{key, cert}, "\n")))
67
			return fmt.Sprintf(`
68
apiVersion: v1
69
kind: Secret
70
metadata:
71
  name: kuma-io-certificate-k8s-mtls
72
  namespace: %s
73
  labels:
74
    kuma.io/mesh: gateway-mtls
75
data:
76
  value: %s
77
type: system.kuma.io/secret
78
`, Config.KumaNamespace, secretData)
79
		}
80
		err := NewClusterSetup().
81
			Install(MTLSMeshKubernetes(meshName)).
82
			Install(NamespaceWithSidecarInjection(namespace)).
83
			Install(Namespace(clientNamespace)).
84
			Install(testserver.Install(
85
				testserver.WithName("demo-client"),
86
				testserver.WithNamespace(clientNamespace),
87
			)).
88
			Install(YamlK8s(httpsSecret())).
89
			Install(YamlK8s(meshGateway)).
90
			Install(MeshTrafficPermissionAllowAllKubernetes(meshName)).
91
			Install(YamlK8s(MkGatewayInstance("mtls-edge-gateway", namespace, meshName))).
92
			Setup(kubernetes.Cluster)
93
		Expect(err).ToNot(HaveOccurred())
94
	})
95

96
	E2EAfterAll(func() {
97
		Expect(kubernetes.Cluster.TriggerDeleteNamespace(namespace)).To(Succeed())
98
		Expect(kubernetes.Cluster.TriggerDeleteNamespace(clientNamespace)).To(Succeed())
99
		Expect(kubernetes.Cluster.DeleteMesh(meshName)).To(Succeed())
100
	})
101

102
	Context("HTTP", func() {
103
		meshGatewayRouteHTTP := `
104
apiVersion: kuma.io/v1alpha1
105
kind: MeshGatewayRoute
106
metadata:
107
  name: mtls-edge-gateway-http
108
mesh: gateway-mtls
109
spec:
110
  selectors:
111
  - match:
112
      kuma.io/service: mtls-edge-gateway
113
      hostname: example.kuma.io
114
  conf:
115
    http:
116
      rules:
117
      - matches:
118
        - path:
119
            match: PREFIX
120
            value: /prefix-trailing/middle/
121
        filters:
122
        - rewrite:
123
            replacePrefixMatch: /middle
124
        backends:
125
        - destination:
126
            kuma.io/service: echo-server_gateway-mtls_svc_80
127
      - matches:
128
        - path:
129
            match: PREFIX
130
            value: /prefix/middle
131
        filters:
132
        - rewrite:
133
            replacePrefixMatch: /middle
134
        backends:
135
        - destination:
136
            kuma.io/service: echo-server_gateway-mtls_svc_80
137
      - matches:
138
        - path:
139
            match: PREFIX
140
            value: /drop-prefix
141
        filters:
142
        - rewrite:
143
            replacePrefixMatch: /
144
        backends:
145
        - destination:
146
            kuma.io/service: echo-server_gateway-mtls_svc_80
147
      - matches:
148
        - path:
149
            match: PREFIX
150
            value: /drop-prefix-trailing/
151
        filters:
152
        - rewrite:
153
            replacePrefixMatch: /
154
        backends:
155
        - destination:
156
            kuma.io/service: echo-server_gateway-mtls_svc_80
157
      - matches:
158
        - path:
159
            match: PREFIX
160
            value: /non-accessible
161
        backends:
162
        - destination:
163
            kuma.io/service: non-accessible-echo-server_gateway-mtls_svc_80
164
      - matches:
165
        - path:
166
            match: PREFIX
167
            value: /
168
        backends:
169
        - destination:
170
            kuma.io/service: echo-server_gateway-mtls_svc_80
171
`
172
		BeforeAll(func() {
173
			err := NewClusterSetup().
174
				Install(testserver.Install(
175
					testserver.WithMesh(meshName),
176
					testserver.WithName("echo-server"),
177
					testserver.WithNamespace(namespace),
178
					testserver.WithEchoArgs("echo", "--instance", "kubernetes"),
179
				)).
180
				Install(testserver.Install(
181
					testserver.WithMesh(meshName),
182
					testserver.WithName("non-accessible-echo-server"),
183
					testserver.WithNamespace(namespace),
184
					testserver.WithEchoArgs("echo", "--instance", "non-accessible-echo-server"),
185
				)).
186
				Install(YamlK8s(meshGatewayRouteHTTP)).
187
				Setup(kubernetes.Cluster)
188
			Expect(err).ToNot(HaveOccurred())
189
		})
190

191
		BeforeEach(func() {
192
			Expect(kubernetes.Cluster.Install(MeshTrafficPermissionAllowAllKubernetes(meshName))).To(Succeed())
193
		})
194

195
		It("should proxy simple HTTP requests", func() {
196
			Eventually(func(g Gomega) {
197
				response, err := client.CollectEchoResponse(
198
					kubernetes.Cluster, "demo-client",
199
					"http://mtls-edge-gateway.gateway-mtls:8080/",
200
					client.WithHeader("host", "example.kuma.io"),
201
					client.FromKubernetesPod(clientNamespace, "demo-client"),
202
				)
203

204
				g.Expect(err).ToNot(HaveOccurred())
205
				g.Expect(response.Instance).To(Equal("kubernetes"))
206
			}, "30s", "1s").Should(Succeed())
207
		})
208

209
		replacePrefix := func(prefix string) func() {
210
			return func() {
211
				Specify("when the prefix is the entire path", func() {
212
					Eventually(func(g Gomega) {
213
						response, err := client.CollectEchoResponse(
214
							kubernetes.Cluster, "demo-client", fmt.Sprintf("http://mtls-edge-gateway.gateway-mtls:8080/%s/middle", prefix),
215
							client.WithHeader("host", "example.kuma.io"),
216
							client.FromKubernetesPod(clientNamespace, "demo-client"),
217
						)
218

219
						g.Expect(err).ToNot(HaveOccurred())
220
						g.Expect(response.Received.Path).To(Equal("/middle"))
221
					}, "30s", "1s").Should(Succeed())
222
				})
223

224
				Specify("when it's a non-trivial prefix", func() {
225
					Eventually(func(g Gomega) {
226
						response, err := client.CollectEchoResponse(
227
							kubernetes.Cluster, "demo-client", fmt.Sprintf("http://mtls-edge-gateway.gateway-mtls:8080/%s/middle/tail", prefix),
228
							client.WithHeader("host", "example.kuma.io"),
229
							client.FromKubernetesPod(clientNamespace, "demo-client"),
230
						)
231

232
						g.Expect(err).ToNot(HaveOccurred())
233
						g.Expect(response.Received.Path).To(Equal("/middle/tail"))
234
					}, "30s", "1s").Should(Succeed())
235
				})
236

237
				Specify("ignoring non-path-separated prefixes", func() {
238
					Eventually(func(g Gomega) {
239
						response, err := client.CollectEchoResponse(
240
							kubernetes.Cluster, "demo-client", fmt.Sprintf("http://mtls-edge-gateway.gateway-mtls:8080/%s/middle_andmore", prefix),
241
							client.WithHeader("host", "example.kuma.io"),
242
							client.FromKubernetesPod(clientNamespace, "demo-client"),
243
						)
244

245
						g.Expect(err).ToNot(HaveOccurred())
246
						g.Expect(response.Received.Path).To(Equal(fmt.Sprintf("/%s/middle_andmore", prefix)))
247
					}, "30s", "1s").Should(Succeed())
248
				})
249
			}
250
		}
251

252
		replacePrefixWithRoot := func(prefix string) func() {
253
			return func() {
254
				Specify("when the prefix is the entire path", func() {
255
					Eventually(func(g Gomega) {
256
						response, err := client.CollectEchoResponse(
257
							kubernetes.Cluster, "demo-client", fmt.Sprintf("http://mtls-edge-gateway.gateway-mtls:8080/%s", prefix),
258
							client.WithHeader("host", "example.kuma.io"),
259
							client.FromKubernetesPod(clientNamespace, "demo-client"),
260
						)
261

262
						g.Expect(err).ToNot(HaveOccurred())
263
						g.Expect(response.Received.Path).To(Equal("/"))
264
					}, "30s", "1s").Should(Succeed())
265
				})
266

267
				Specify("when it's a non-trivial prefix", func() {
268
					Eventually(func(g Gomega) {
269
						response, err := client.CollectEchoResponse(
270
							kubernetes.Cluster, "demo-client", fmt.Sprintf("http://mtls-edge-gateway.gateway-mtls:8080/%s/tail", prefix),
271
							client.WithHeader("host", "example.kuma.io"),
272
							client.FromKubernetesPod(clientNamespace, "demo-client"),
273
						)
274

275
						g.Expect(err).ToNot(HaveOccurred())
276
						g.Expect(response.Received.Path).To(Equal("/tail"))
277
					}, "30s", "1s").Should(Succeed())
278
				})
279

280
				Specify("ignoring non-path-separated prefixes", func() {
281
					Eventually(func(g Gomega) {
282
						response, err := client.CollectEchoResponse(
283
							kubernetes.Cluster, "demo-client", fmt.Sprintf("http://mtls-edge-gateway.gateway-mtls:8080/%s_andmore", prefix),
284
							client.WithHeader("host", "example.kuma.io"),
285
							client.FromKubernetesPod(clientNamespace, "demo-client"),
286
						)
287

288
						g.Expect(err).ToNot(HaveOccurred())
289
						g.Expect(response.Received.Path).To(Equal(fmt.Sprintf("/%s_andmore", prefix)))
290
					}, "30s", "1s").Should(Succeed())
291
				})
292
			}
293
		}
294

295
		Describe("replacing a path prefix", replacePrefix("prefix"))
296
		Describe("replacing a path prefix with trailing prefix", replacePrefix("prefix-trailing"))
297

298
		Describe("replacing a path prefix with /", replacePrefixWithRoot("drop-prefix"))
299
		Describe("replacing a path prefix with /", replacePrefixWithRoot("drop-prefix-trailing"))
300

301
		It("should not access a service for which we don't have traffic permission", func() {
302
			Expect(DeleteMeshResources(kubernetes.Cluster, meshName, v1alpha1.MeshTrafficPermissionResourceTypeDescriptor)).To(Succeed())
303
			tp := `
304
apiVersion: kuma.io/v1alpha1
305
kind: MeshTrafficPermission
306
metadata:
307
  namespace: kuma-system
308
  name: tp-non-accessible-echo-server.kuma-system
309
  labels:
310
    kuma.io/mesh: gateway-mtls
311
spec:
312
  targetRef:
313
    kind: MeshService
314
    name: non-accessible-echo-server_gateway-mtls_svc_80
315
  from:
316
    - targetRef:
317
        kind: MeshService
318
        name: not-mtls-edge-gateway
319
      default:
320
        action: Allow`
321
			Expect(kubernetes.Cluster.Install(YamlK8s(tp))).To(Succeed())
322

323
			Eventually(func(g Gomega) {
324
				status, err := client.CollectFailure(
325
					kubernetes.Cluster, "demo-client", "http://mtls-edge-gateway.gateway-mtls:8080/non-accessible",
326
					client.WithHeader("host", "example.kuma.io"),
327
					client.FromKubernetesPod(clientNamespace, "demo-client"),
328
				)
329

330
				g.Expect(err).ToNot(HaveOccurred())
331
				g.Expect(status.ResponseCode).To(Equal(403))
332
			}, "30s", "1s").Should(Succeed())
333

334
			Expect(DeleteMeshPolicyOrError(
335
				kubernetes.Cluster,
336
				v1alpha1.MeshTrafficPermissionResourceTypeDescriptor,
337
				"tp-non-accessible-echo-server.kuma-system",
338
			)).To(Succeed())
339
		})
340

341
		It("should access a service when we have mesh traffic permission using MeshGateway", func() {
342
			// when no MeshTrafficPermission
343
			Expect(DeleteMeshResources(kubernetes.Cluster, meshName, v1alpha1.MeshTrafficPermissionResourceTypeDescriptor)).To(Succeed())
344

345
			// then cannot reach service
346
			Eventually(func(g Gomega) {
347
				response, err := client.CollectFailure(
348
					kubernetes.Cluster, "demo-client",
349
					"http://mtls-edge-gateway.gateway-mtls:8080/",
350
					client.WithHeader("host", "example.kuma.io"),
351
					client.FromKubernetesPod(clientNamespace, "demo-client"),
352
					client.NoFail(),
353
					client.OutputFormat(`{ "received": { "status": %{response_code} } }`),
354
				)
355

356
				g.Expect(err).ToNot(HaveOccurred())
357
				g.Expect(response.ResponseCode).To(Equal(403))
358
			}, "30s", "1s").Should(Succeed())
359

360
			tp := `
361
apiVersion: kuma.io/v1alpha1
362
kind: MeshTrafficPermission
363
metadata:
364
  namespace: kuma-system
365
  name: access-echo-server.kuma-system
366
  labels:
367
    kuma.io/mesh: gateway-mtls
368
spec:
369
  targetRef:
370
    kind: MeshService
371
    name: echo-server_gateway-mtls_svc_80
372
  from:
373
    - targetRef:
374
        kind: MeshGateway
375
        name: mtls-edge-gateway
376
      default:
377
        action: Allow`
378
			Expect(kubernetes.Cluster.Install(YamlK8s(tp))).To(Succeed())
379

380
			Eventually(func(g Gomega) {
381
				response, err := client.CollectEchoResponse(
382
					kubernetes.Cluster, "demo-client",
383
					"http://mtls-edge-gateway.gateway-mtls:8080/",
384
					client.WithHeader("host", "example.kuma.io"),
385
					client.FromKubernetesPod(clientNamespace, "demo-client"),
386
				)
387

388
				g.Expect(err).ToNot(HaveOccurred())
389
				g.Expect(response.Instance).To(Equal("kubernetes"))
390
			}, "30s", "1s").Should(Succeed())
391

392
			Expect(DeleteMeshPolicyOrError(
393
				kubernetes.Cluster,
394
				v1alpha1.MeshTrafficPermissionResourceTypeDescriptor,
395
				"access-echo-server.kuma-system",
396
			)).To(Succeed())
397
		})
398
	})
399

400
	Context("TCP", func() {
401
		tcpRoute := `
402
apiVersion: kuma.io/v1alpha1
403
kind: MeshGatewayRoute
404
metadata:
405
  name: mtls-gateway-tcp
406
mesh: gateway-mtls
407
spec:
408
  selectors:
409
  - match:
410
      kuma.io/service: mtls-edge-gateway
411
      protocol: tcp
412
  conf:
413
    tcp:
414
      rules:
415
      - backends:
416
        - destination:
417
            kuma.io/service: tcp-server_gateway-mtls_svc_80
418
`
419

420
		BeforeAll(func() {
421
			err := NewClusterSetup().
422
				Install(YamlK8s(tcpRoute)).
423
				Install(testserver.Install(
424
					testserver.WithMesh(meshName),
425
					testserver.WithName("tcp-server"),
426
					testserver.WithNamespace(namespace),
427
					testserver.WithHealthCheckTCPArgs("health-check", "tcp", "--port", "80"),
428
				)).
429
				Install(MeshTrafficPermissionAllowAllKubernetes(meshName)).
430
				Setup(kubernetes.Cluster)
431
			Expect(err).ToNot(HaveOccurred())
432
		})
433

434
		It("should proxy TCP connections", func() {
435
			Eventually(func(g Gomega) {
436
				response, err := client.CollectTCPResponse(kubernetes.Cluster, "demo-client", "telnet://mtls-edge-gateway.gateway-mtls:8081", "request",
437
					client.FromKubernetesPod(clientNamespace, "demo-client"),
438
				)
439

440
				g.Expect(err).ToNot(HaveOccurred())
441
				g.Expect(response).Should(Equal("response"))
442
			}, "30s", "1s").Should(Succeed())
443
		})
444
	})
445

446
	Context("TLS", func() {
447
		tlsServerRoute := `
448
apiVersion: kuma.io/v1alpha1
449
kind: MeshGatewayRoute
450
metadata:
451
  name: mtls-gateway-tls-passthrough
452
mesh: gateway-mtls
453
spec:
454
  selectors:
455
  - match:
456
      kuma.io/service: mtls-edge-gateway
457
      name: tls-passthrough
458
  conf:
459
    tcp:
460
      rules:
461
      - backends:
462
        - destination:
463
            kuma.io/service: tls-server_gateway-mtls_svc_443
464
`
465
		tcpServerRoute := `
466
apiVersion: kuma.io/v1alpha1
467
kind: MeshGatewayRoute
468
metadata:
469
  name: mtls-gateway-tls-terminate
470
mesh: gateway-mtls
471
spec:
472
  selectors:
473
  - match:
474
      kuma.io/service: mtls-edge-gateway
475
      name: tls-terminate
476
  conf:
477
    tcp:
478
      rules:
479
      - backends:
480
        - destination:
481
            kuma.io/service: tcp-server_gateway-mtls_svc_80
482
`
483

484
		BeforeAll(func() {
485
			cert, key, err := CreateCertsFor("example.kuma.io")
486
			Expect(err).To(Succeed())
487

488
			setup := NewClusterSetup().
489
				Install(YamlK8s(tlsServerRoute)).
490
				Install(YamlK8s(tcpServerRoute)).
491
				Install(testserver.Install(
492
					testserver.WithMesh(meshName),
493
					testserver.WithName("tls-server"),
494
					testserver.WithTLS(key, cert),
495
					testserver.WithNamespace(namespace),
496
				)).
497
				Install(testserver.Install(
498
					testserver.WithMesh(meshName),
499
					testserver.WithName("tcp-server"),
500
					testserver.WithNamespace(namespace),
501
					testserver.WithHealthCheckTCPArgs("health-check", "tcp", "--port", "80"),
502
				)).
503
				Install(MeshTrafficPermissionAllowAllKubernetes(meshName))
504
			Expect(setup.Setup(kubernetes.Cluster)).To(Succeed())
505
		})
506

507
		It("should passthrough TLS connections", func() {
508
			Eventually(func(g Gomega) {
509
				clusterIP, err := k8s.RunKubectlAndGetOutputE(
510
					kubernetes.Cluster.GetTesting(),
511
					kubernetes.Cluster.GetKubectlOptions(namespace),
512
					"get", "service", "mtls-edge-gateway", "-ojsonpath={.spec.clusterIP}",
513
				)
514
				g.Expect(err).ToNot(HaveOccurred())
515

516
				response, err := client.CollectEchoResponse(
517
					kubernetes.Cluster, "demo-client",
518
					"https://example-passthrough.kuma.io:8082/",
519
					client.Resolve("example-passthrough.kuma.io:8082", clusterIP),
520
					client.Insecure(),
521
					client.FromKubernetesPod(clientNamespace, "demo-client"),
522
				)
523

524
				g.Expect(err).ToNot(HaveOccurred())
525
				g.Expect(response.Instance).To(HavePrefix("tls-server"))
526
			}, "30s", "1s").Should(Succeed())
527
		})
528

529
		It("should not passthrough TLS connections that don't match SNI", func() {
530
			Consistently(func(g Gomega) {
531
				clusterIP, err := k8s.RunKubectlAndGetOutputE(
532
					kubernetes.Cluster.GetTesting(),
533
					kubernetes.Cluster.GetKubectlOptions(namespace),
534
					"get", "service", "mtls-edge-gateway", "-ojsonpath={.spec.clusterIP}",
535
				)
536
				g.Expect(err).ToNot(HaveOccurred())
537

538
				g.Expect(err).ToNot(HaveOccurred())
539
				status, err := client.CollectFailure(
540
					kubernetes.Cluster, "demo-client",
541
					"https://example-other-hostname.kuma.io:8082/",
542
					client.Resolve("example-other-hostname.kuma.io:8082", clusterIP),
543
					client.Insecure(),
544
					client.FromKubernetesPod(clientNamespace, "demo-client"),
545
				)
546

547
				g.Expect(err).ToNot(HaveOccurred())
548
				g.Expect(status.Exitcode).To(Equal(35))
549
			}, "30s", "1s").Should(Succeed())
550
		})
551

552
		It("should terminate TLS and proxy TCP connections", func() {
553
			Eventually(func(g Gomega) {
554
				response, err := client.CollectTLSResponse(kubernetes.Cluster, "demo-client", "mtls-edge-gateway.gateway-mtls:8083", "request",
555
					client.FromKubernetesPod(clientNamespace, "demo-client"),
556
				)
557

558
				g.Expect(err).ToNot(HaveOccurred())
559
				g.Expect(response).Should(Equal("response"))
560
			}, "30s", "1s").Should(Succeed())
561
		})
562
	})
563
}
564