kuma
705 строк · 21.2 Кб
1package gateway
2
3import (
4"encoding/base64"
5"fmt"
6"strings"
7
8"github.com/gruntwork-io/terratest/modules/k8s"
9. "github.com/onsi/ginkgo/v2"
10. "github.com/onsi/gomega"
11
12. "github.com/kumahq/kuma/test/framework"
13"github.com/kumahq/kuma/test/framework/client"
14"github.com/kumahq/kuma/test/framework/deployments/testserver"
15"github.com/kumahq/kuma/test/framework/envs/kubernetes"
16)
17
18func Gateway() {
19meshName := "simple-gateway"
20namespace := "simple-gateway"
21clientNamespace := "client-simple-gateway"
22meshGateway := `
23apiVersion: kuma.io/v1alpha1
24kind: MeshGateway
25metadata:
26name: simple-gateway
27mesh: simple-gateway
28spec:
29selectors:
30- match:
31kuma.io/service: simple-gateway
32conf:
33listeners:
34- port: 8080
35protocol: HTTP
36hostname: example.kuma.io
37tags:
38hostname: example.kuma.io
39- port: 8081
40protocol: HTTPS
41tls:
42mode: TERMINATE
43certificates:
44- secret: kuma-io-certificate-k8s` +
45// secret names have to be unique because
46// we're removing secrets using owner reference, and we're relying on async namespace deletion,
47// so we could have a situation where the secrets are not yet deleted,
48// and we're trying to create a new mesh with the same secret name which k8s treats as changing the mesh of the secret
49// and results in "cannot change mesh of the Secret. Delete the Secret first and apply it again."
50// https://app.circleci.com/pipelines/github/kumahq/kuma/24848/workflows/f33edb5a-74cb-45ae-b0c2-4b14bf289585/jobs/497239
51`
52hostname: example.kuma.io
53tags:
54hostname: example.kuma.io
55- port: 8081
56protocol: HTTPS
57tls:
58mode: TERMINATE
59certificates:
60- secret: kuma-io-certificate-k8s` +
61// secret names have to be unique because
62// we're removing secrets using owner reference, and we're relying on async namespace deletion,
63// so we could have a situation where the secrets are not yet deleted,
64// and we're trying to create a new mesh with the same secret name which k8s treats as changing the mesh of the secret
65// and results in "cannot change mesh of the Secret. Delete the Secret first and apply it again."
66// https://app.circleci.com/pipelines/github/kumahq/kuma/24848/workflows/f33edb5a-74cb-45ae-b0c2-4b14bf289585/jobs/497239
67`
68hostname: otherexample.kuma.io
69tags:
70hostname: otherexample.kuma.io
71- port: 8082
72protocol: HTTP
73hostname: '*'
74- port: 8083
75protocol: TCP
76`
77httpsSecret := func() string {
78cert, key, err := CreateCertsFor("example.kuma.io")
79Expect(err).To(Succeed())
80secretData := base64.StdEncoding.EncodeToString([]byte(strings.Join([]string{key, cert}, "\n")))
81return fmt.Sprintf(`
82apiVersion: v1
83kind: Secret
84metadata:
85name: kuma-io-certificate-k8s
86namespace: %s
87labels:
88kuma.io/mesh: simple-gateway
89data:
90value: %s
91type: system.kuma.io/secret
92`, Config.KumaNamespace, secretData)
93}
94var clusterIP string
95
96BeforeAll(func() {
97err := NewClusterSetup().
98Install(MTLSMeshKubernetes(meshName)).
99Install(Namespace(clientNamespace)).
100Install(NamespaceWithSidecarInjection(namespace)).
101Install(testserver.Install(
102testserver.WithName("demo-client"),
103testserver.WithNamespace(clientNamespace),
104)).
105Install(testserver.Install(
106testserver.WithName("echo-server"),
107testserver.WithMesh(meshName),
108testserver.WithNamespace(namespace),
109testserver.WithEchoArgs("echo", "--instance", "echo-server"),
110)).
111Install(YamlK8s(httpsSecret())).
112Install(YamlK8s(meshGateway)).
113Install(YamlK8s(MkGatewayInstance("simple-gateway", namespace, meshName))).
114Install(MeshTrafficPermissionAllowAllKubernetes(meshName)).
115Setup(kubernetes.Cluster)
116Expect(err).ToNot(HaveOccurred())
117
118Eventually(func(g Gomega) {
119var err error
120clusterIP, err = k8s.RunKubectlAndGetOutputE(
121kubernetes.Cluster.GetTesting(),
122kubernetes.Cluster.GetKubectlOptions(namespace),
123"get", "service", "simple-gateway", "-ojsonpath={.spec.clusterIP}",
124)
125g.Expect(err).ToNot(HaveOccurred())
126g.Expect(clusterIP).ToNot(BeEmpty())
127}, "30s", "1s").Should(Succeed())
128})
129
130E2EAfterAll(func() {
131Expect(kubernetes.Cluster.TriggerDeleteNamespace(namespace)).To(Succeed())
132Expect(kubernetes.Cluster.TriggerDeleteNamespace(clientNamespace)).To(Succeed())
133Expect(kubernetes.Cluster.DeleteMesh(meshName)).To(Succeed())
134})
135
136meshGatewayRoutes := func(name, path, destination string) []string {
137return []string{
138fmt.Sprintf(`
139apiVersion: kuma.io/v1alpha1
140kind: MeshGatewayRoute
141metadata:
142name: %s
143mesh: simple-gateway
144spec:
145selectors:
146- match:
147kuma.io/service: simple-gateway
148conf:
149http:
150rules:
151- matches:
152- path:
153match: PREFIX
154value: %s
155backends:
156- destination:
157kuma.io/service: %s
158- matches:
159- path:
160match: PREFIX
161value: /rewrite-host%s
162filters:
163- rewrite:
164hostToBackendHostname: true
165backends:
166- destination:
167kuma.io/service: %s
168`, name, path, destination, path, destination),
169fmt.Sprintf(`
170apiVersion: kuma.io/v1alpha1
171kind: MeshGatewayRoute
172metadata:
173name: %s-specific
174mesh: simple-gateway
175spec:
176selectors:
177- match:
178kuma.io/service: simple-gateway
179conf:
180http:
181hostnames:
182- specific.kuma.io
183rules:
184- matches:
185- path:
186match: PREFIX
187value: %s
188filters:
189- requestHeader:
190add:
191- name: x-specific-hostname-header
192value: "true"
193backends:
194- destination:
195kuma.io/service: %s
196- matches:
197- path:
198match: PREFIX
199value: /rewrite-host%s
200filters:
201- rewrite:
202hostToBackendHostname: true
203backends:
204- destination:
205kuma.io/service: %s
206`, name, path, destination, path, destination),
207fmt.Sprintf(`
208apiVersion: kuma.io/v1alpha1
209kind: MeshGatewayRoute
210metadata:
211name: %s-specific-listener
212mesh: simple-gateway
213spec:
214selectors:
215- match:
216kuma.io/service: simple-gateway
217hostname: otherexample.kuma.io
218conf:
219http:
220rules:
221- matches:
222- path:
223match: PREFIX
224value: "%s-specific-listener"
225filters:
226- requestHeader:
227add:
228- name: x-listener-by-hostname-header
229value: "true"
230backends:
231- destination:
232kuma.io/service: %s
233`, name, path, destination),
234}
235}
236
237httpRoute := func(name, path, destination string) []string {
238return []string{
239fmt.Sprintf(`
240apiVersion: kuma.io/v1alpha1
241kind: MeshHTTPRoute
242metadata:
243name: %s
244namespace: %s
245labels:
246kuma.io/mesh: simple-gateway
247spec:
248targetRef:
249kind: MeshGateway
250name: simple-gateway
251to:
252- targetRef:
253kind: Mesh
254rules:
255- matches:
256- path:
257type: PathPrefix
258value: "%s"
259default:
260backendRefs:
261- kind: MeshService
262name: "%s"
263- targetRef:
264kind: Mesh
265hostnames:
266- specific.kuma.io
267rules:
268- matches:
269- path:
270type: PathPrefix
271value: "%s"
272default:
273filters:
274- type: RequestHeaderModifier
275requestHeaderModifier:
276add:
277- name: x-specific-hostname-header
278value: "true"
279backendRefs:
280- kind: MeshService
281name: "%s"
282`, name, Config.KumaNamespace, path, destination, path, destination),
283fmt.Sprintf(`
284apiVersion: kuma.io/v1alpha1
285kind: MeshHTTPRoute
286metadata:
287name: %s
288namespace: %s
289labels:
290kuma.io/mesh: simple-gateway
291spec:
292targetRef:
293kind: MeshGateway
294name: simple-gateway
295tags:
296hostname: otherexample.kuma.io
297to:
298- targetRef:
299kind: Mesh
300rules:
301- matches:
302- path:
303type: PathPrefix
304value: "%s-specific-listener"
305default:
306filters:
307- type: RequestHeaderModifier
308requestHeaderModifier:
309add:
310- name: x-listener-by-hostname-header
311value: "true"
312backendRefs:
313- kind: MeshService
314name: "%s"
315`, name+"-hostname-specific", Config.KumaNamespace, path, destination),
316}
317}
318
319basicRouting := func(name string, routeYAMLs []string) {
320Context(fmt.Sprintf("Mesh service - %s", name), func() {
321BeforeAll(func() {
322Expect(NewClusterSetup().
323Install(YamlK8s(routeYAMLs...)).
324Setup(kubernetes.Cluster),
325).To(Succeed())
326})
327
328E2EAfterAll(func() {
329Expect(NewClusterSetup().
330Install(DeleteYamlK8s(routeYAMLs...)).
331Setup(kubernetes.Cluster),
332).To(Succeed())
333})
334
335It("should proxy to service via HTTP without port in host", func() {
336Eventually(func(g Gomega) {
337response, err := client.CollectEchoResponse(
338kubernetes.Cluster, "demo-client",
339"http://simple-gateway.simple-gateway:8080/",
340client.WithHeader("host", "example.kuma.io"),
341client.FromKubernetesPod(clientNamespace, "demo-client"),
342)
343
344g.Expect(err).ToNot(HaveOccurred())
345g.Expect(response.Instance).To(Equal("echo-server"))
346g.Expect(response.Received.Headers).ToNot(HaveKey("X-Specific-Hostname-Header"))
347}, "1m", "1s").Should(Succeed())
348})
349It("should proxy to service via HTTP with port in host", func() {
350Eventually(func(g Gomega) {
351response, err := client.CollectEchoResponse(
352kubernetes.Cluster, "demo-client",
353"http://simple-gateway.simple-gateway:8080/",
354client.WithHeader("host", "example.kuma.io:8080"),
355client.FromKubernetesPod(clientNamespace, "demo-client"),
356)
357
358g.Expect(err).ToNot(HaveOccurred())
359g.Expect(response.Instance).To(Equal("echo-server"))
360g.Expect(response.Received.Headers).ToNot(HaveKey("X-Specific-Hostname-Header"))
361}, "30s", "1s").Should(Succeed())
362})
363
364It("should proxy to service via HTTPS", func() {
365Eventually(func(g Gomega) {
366response, err := client.CollectEchoResponse(
367kubernetes.Cluster, "demo-client",
368"https://example.kuma.io:8081/",
369client.Resolve("example.kuma.io:8081", clusterIP),
370client.FromKubernetesPod(clientNamespace, "demo-client"),
371client.Insecure(),
372)
373
374g.Expect(err).ToNot(HaveOccurred())
375g.Expect(response.Instance).To(Equal("echo-server"))
376g.Expect(response.Received.Headers).ToNot(HaveKey("X-Specific-Hostname-Header"))
377}, "30s", "1s").Should(Succeed())
378})
379
380It("should automatically set host header from service address", func() {
381Eventually(func(g Gomega) {
382response, err := client.CollectEchoResponse(
383kubernetes.Cluster, "demo-client",
384"http://simple-gateway.simple-gateway:8082/",
385client.WithHeader("host", "example.kuma.io"),
386client.FromKubernetesPod(clientNamespace, "demo-client"),
387)
388
389g.Expect(err).ToNot(HaveOccurred())
390g.Expect(response.Received.Headers["Host"]).To(HaveLen(1))
391g.Expect(response.Received.Headers["Host"]).To(ContainElements("example.kuma.io"))
392g.Expect(response.Received.Headers).ToNot(HaveKey("X-Specific-Hostname-Header"))
393}, "30s", "1s").Should(Succeed())
394})
395
396It("should route based on host name", func() {
397Eventually(func(g Gomega) {
398response, err := client.CollectEchoResponse(
399kubernetes.Cluster, "demo-client",
400"http://simple-gateway.simple-gateway:8082/",
401client.WithHeader("host", "specific.kuma.io"),
402client.FromKubernetesPod(clientNamespace, "demo-client"),
403)
404
405g.Expect(err).ToNot(HaveOccurred())
406g.Expect(response.Received.Headers["Host"]).To(HaveLen(1))
407g.Expect(response.Received.Headers["Host"]).To(ContainElements("specific.kuma.io"))
408g.Expect(response.Received.Headers["X-Specific-Hostname-Header"]).To(ContainElements("true"))
409}, "30s", "1s").Should(Succeed())
410})
411
412It("should match routes by SNI", func() {
413Eventually(func(g Gomega) {
414response, err := client.CollectEchoResponse(
415kubernetes.Cluster, "demo-client",
416"https://otherexample.kuma.io:8081/-specific-listener",
417client.Resolve("otherexample.kuma.io:8081", clusterIP),
418client.FromKubernetesPod(clientNamespace, "demo-client"),
419client.Insecure(),
420)
421
422g.Expect(err).ToNot(HaveOccurred())
423g.Expect(response.Received.Headers["Host"]).To(HaveLen(1))
424g.Expect(response.Received.Headers["Host"]).To(ContainElements("otherexample.kuma.io:8081"))
425g.Expect(response.Received.Headers["X-Listener-By-Hostname-Header"]).To(ContainElements("true"))
426}, "30s", "1s").Should(Succeed())
427})
428
429It("should isolate routes by SNI", func() {
430Eventually(func(g Gomega) {
431response, err := client.CollectEchoResponse(
432kubernetes.Cluster, "demo-client",
433"https://example.kuma.io:8081/-specific-listener",
434client.Resolve("example.kuma.io:8081", clusterIP),
435client.FromKubernetesPod(clientNamespace, "demo-client"),
436client.Insecure(),
437)
438
439g.Expect(err).ToNot(HaveOccurred())
440g.Expect(response.Received.Headers["Host"]).To(HaveLen(1))
441g.Expect(response.Received.Headers["Host"]).To(ContainElements("example.kuma.io:8081"))
442g.Expect(response.Received.Headers["X-Listener-By-Hostname-Header"]).NotTo(ContainElements("true"))
443}, "30s", "1s").Should(Succeed())
444})
445
446It("should check both SNI and Host", func() {
447Consistently(func(g Gomega) {
448status, err := client.CollectFailure(
449kubernetes.Cluster, "demo-client",
450"https://otherexample.kuma.io:8081/-specific-listener",
451client.Resolve("otherexample.kuma.io:8081", clusterIP),
452// Note the header differs from the SNI
453client.WithHeader("host", "example.kuma.io"),
454client.FromKubernetesPod(clientNamespace, "demo-client"),
455client.Insecure(),
456)
457
458g.Expect(err).ToNot(HaveOccurred())
459g.Expect(status.ResponseCode).To(Equal(404))
460}, "30s", "1s").Should(Succeed())
461})
462})
463}
464
465basicRouting("MeshGatewayRoute", meshGatewayRoutes("internal-service", "/", "echo-server_simple-gateway_svc_80"))
466basicRouting("MeshHTTPRoute", httpRoute("internal-service", "/", "echo-server_simple-gateway_svc_80"))
467
468Context("Rate Limit", func() {
469rt := `apiVersion: kuma.io/v1alpha1
470kind: RateLimit
471metadata:
472name: gateway-rate-limit
473mesh: simple-gateway
474spec:
475sources:
476- match:
477kuma.io/service: simple-gateway
478destinations:
479- match:
480kuma.io/service: rt-echo-server_simple-gateway_svc_80
481conf:
482http:
483requests: 5
484interval: 10s`
485routes := meshGatewayRoutes("rt-echo-server", "/rt", "rt-echo-server_simple-gateway_svc_80")
486
487BeforeAll(func() {
488err := NewClusterSetup().
489Install(testserver.Install(
490testserver.WithName("rt-echo-server"),
491testserver.WithMesh(meshName),
492testserver.WithNamespace(namespace),
493testserver.WithEchoArgs("echo", "--instance", "rt-echo-server"),
494)).
495Install(YamlK8s(rt)).
496Install(YamlK8s(routes...)).
497Setup(kubernetes.Cluster)
498Expect(err).ToNot(HaveOccurred())
499})
500
501AfterAll(func() {
502err := NewClusterSetup().
503Install(DeleteYamlK8s(routes...)).
504Setup(kubernetes.Cluster)
505Expect(err).ToNot(HaveOccurred())
506})
507
508It("should rate limit", func() {
509Eventually(func(g Gomega) {
510response, err := client.CollectFailure(
511kubernetes.Cluster, "demo-client",
512"http://simple-gateway.simple-gateway:8080/rt",
513client.WithHeader("host", "example.kuma.io"),
514client.FromKubernetesPod(clientNamespace, "demo-client"),
515client.NoFail(),
516client.OutputFormat(`{ "received": { "status": %{response_code} } }`),
517)
518
519g.Expect(err).ToNot(HaveOccurred())
520g.Expect(response.ResponseCode).To(Equal(429))
521}, "30s", "1s").Should(Succeed())
522})
523})
524
525Context("TCPRoute", func() {
526routes := []string{
527fmt.Sprintf(`
528apiVersion: kuma.io/v1alpha1
529kind: MeshTCPRoute
530metadata:
531name: simple-gateway-tcp-route
532namespace: %s
533labels:
534kuma.io/mesh: simple-gateway
535spec:
536targetRef:
537kind: MeshGateway
538name: simple-gateway
539to:
540- targetRef:
541kind: Mesh
542rules:
543- default:
544backendRefs:
545- kind: MeshService
546name: test-tcp-server_simple-gateway_svc_80
547`, Config.KumaNamespace),
548}
549BeforeAll(func() {
550err := NewClusterSetup().
551Install(testserver.Install(
552testserver.WithName("test-tcp-server"),
553testserver.WithServicePortAppProtocol("tcp"),
554testserver.WithMesh(meshName),
555testserver.WithNamespace(namespace),
556)).
557Install(YamlK8s(routes...)).
558Setup(kubernetes.Cluster)
559Expect(err).ToNot(HaveOccurred())
560})
561AfterAll(func() {
562err := NewClusterSetup().
563Install(DeleteYamlK8s(routes...)).
564Setup(kubernetes.Cluster)
565Expect(err).ToNot(HaveOccurred())
566})
567
568It("should work on TCP service", func() {
569Eventually(func(g Gomega) {
570response, err := client.CollectEchoResponse(
571kubernetes.Cluster, "demo-client",
572"http://simple-gateway.simple-gateway:8083/",
573client.FromKubernetesPod(clientNamespace, "demo-client"),
574)
575g.Expect(err).ToNot(HaveOccurred())
576g.Expect(response.Instance).To(HavePrefix("test-tcp-server"))
577}, "30s", "1s").Should(Succeed())
578})
579})
580
581Context("External Service", func() {
582externalService := `
583apiVersion: kuma.io/v1alpha1
584kind: ExternalService
585mesh: simple-gateway
586metadata:
587name: simple-gateway-external-service
588spec:
589tags:
590kuma.io/service: external-service
591kuma.io/protocol: http
592networking:
593address: es-echo-server.client-simple-gateway.svc.cluster.local:80`
594
595BeforeAll(func() {
596err := NewClusterSetup().
597Install(testserver.Install(
598testserver.WithName("es-echo-server"),
599testserver.WithNamespace(clientNamespace),
600testserver.WithEchoArgs("echo", "--instance", "es-echo-server"),
601)).
602Install(YamlK8s(externalService)).
603Setup(kubernetes.Cluster)
604Expect(err).ToNot(HaveOccurred())
605})
606
607It("should proxy to service via HTTP", func() {
608route := meshGatewayRoutes("es-echo-server", "/external-service", "external-service")
609setup := NewClusterSetup().Install(YamlK8s(route...))
610Expect(setup.Setup(kubernetes.Cluster)).To(Succeed())
611
612Eventually(func(g Gomega) {
613response, err := client.CollectEchoResponse(
614kubernetes.Cluster, "demo-client",
615"http://simple-gateway.simple-gateway:8080/external-service",
616client.WithHeader("host", "example.kuma.io"),
617client.FromKubernetesPod(clientNamespace, "demo-client"),
618)
619
620g.Expect(err).ToNot(HaveOccurred())
621g.Expect(response.Instance).To(Equal("es-echo-server"))
622}, "30s", "1s").Should(Succeed())
623
624Expect(NewClusterSetup().Install(DeleteYamlK8s(route...)).Setup(kubernetes.Cluster)).To(Succeed())
625})
626
627It("should automatically set host header from external service address when rewrite.hostToBackendHostname is set to true", func() {
628route := meshGatewayRoutes("es-echo-server", "/external-service", "external-service")
629setup := NewClusterSetup().Install(YamlK8s(route...))
630Expect(setup.Setup(kubernetes.Cluster)).To(Succeed())
631
632// don't rewrite host to backend hostname
633Eventually(func(g Gomega) {
634response, err := client.CollectEchoResponse(
635kubernetes.Cluster, "demo-client",
636"http://simple-gateway.simple-gateway:8080/external-service",
637client.WithHeader("host", "example.kuma.io"),
638client.FromKubernetesPod(clientNamespace, "demo-client"),
639)
640
641g.Expect(err).ToNot(HaveOccurred())
642g.Expect(response.Received.Headers["Host"]).To(HaveLen(1))
643g.Expect(response.Received.Headers["Host"]).To(ContainElements("example.kuma.io"))
644}, "30s", "1s").Should(Succeed())
645
646// rewrite host to backend hostname
647Eventually(func(g Gomega) {
648response, err := client.CollectEchoResponse(
649kubernetes.Cluster, "demo-client",
650"http://simple-gateway.simple-gateway:8080/rewrite-host/external-service",
651client.WithHeader("host", "example.kuma.io"),
652client.FromKubernetesPod(clientNamespace, "demo-client"),
653)
654
655g.Expect(err).ToNot(HaveOccurred())
656g.Expect(response.Received.Headers["Host"]).To(HaveLen(1))
657g.Expect(response.Received.Headers["Host"]).To(ContainElements("es-echo-server.client-simple-gateway.svc.cluster.local"))
658}, "30s", "1s").Should(Succeed())
659
660Expect(NewClusterSetup().Install(DeleteYamlK8s(route...)).Setup(kubernetes.Cluster)).To(Succeed())
661})
662
663It("should handle external-service excluded by tags", func() {
664route := fmt.Sprintf(`
665apiVersion: kuma.io/v1alpha1
666kind: MeshGatewayRoute
667metadata:
668name: %s
669mesh: simple-gateway
670spec:
671selectors:
672- match:
673kuma.io/service: simple-gateway
674conf:
675http:
676rules:
677- matches:
678- path:
679match: PREFIX
680value: %s
681backends:
682- destination:
683kuma.io/service: %s
684nonexistent: tag
685`, "es-echo-server-broken", "/external-service", "external-service")
686
687setup := NewClusterSetup().Install(YamlK8s(route))
688Expect(setup.Setup(kubernetes.Cluster)).To(Succeed())
689
690Eventually(func(g Gomega) {
691response, err := client.CollectFailure(
692kubernetes.Cluster, "demo-client",
693"http://simple-gateway.simple-gateway:8080/external-service",
694client.WithHeader("host", "example.kuma.io"),
695client.FromKubernetesPod(clientNamespace, "demo-client"),
696)
697
698g.Expect(err).ToNot(HaveOccurred())
699g.Expect(response.ResponseCode).To(Equal(503))
700}, "30s", "1s").Should(Succeed())
701
702Expect(NewClusterSetup().Install(DeleteYamlK8s(route)).Setup(kubernetes.Cluster)).To(Succeed())
703})
704})
705}
706