kuma

1
package observability
2

3
import (
4
	"fmt"
5
	"net"
6

7
	. "github.com/onsi/ginkgo/v2"
8
	. "github.com/onsi/gomega"
9

10
	. "github.com/kumahq/kuma/test/framework"
11
	"github.com/kumahq/kuma/test/framework/client"
12
	"github.com/kumahq/kuma/test/framework/deployments/testserver"
13
	"github.com/kumahq/kuma/test/framework/envs/kubernetes"
14
)
15

16
func MeshAndMetricsAggregate(name string) InstallFunc {
17
	mesh := fmt.Sprintf(`
18
apiVersion: kuma.io/v1alpha1
19
kind: Mesh
20
metadata:
21
  name: %s
22
spec:
23
  metrics:
24
    enabledBackend: prom-1
25
    backends:
26
      - name: prom-1
27
        type: prometheus
28
        conf:
29
          port: 1234
30
          path: /metrics
31
          envoy:
32
            filterRegex: concurrency
33
          skipMTLS: true
34
          aggregate:
35
          - name: path-stats
36
            path: "/path-stats"
37
            port: 80
38
          - name: mesh-default
39
            path: "/mesh-default"
40
            port: 80
41
          - name: service-to-override
42
            path: "/service-to-override"
43
            port: 80
44
          - name: not-working-service
45
            path: "/not-working-service"
46
            port: 81`, name)
47
	return YamlK8s(mesh)
48
}
49

50
func MeshAndMetricsEnabled(name string) InstallFunc {
51
	mesh := fmt.Sprintf(`
52
apiVersion: kuma.io/v1alpha1
53
kind: Mesh
54
metadata:
55
  name: %s
56
spec:
57
  metrics:
58
    enabledBackend: prom-1
59
    backends:
60
      - name: prom-1
61
        type: prometheus
62
        conf:
63
          envoy:
64
            filterRegex: concurrency
65
          port: 1234
66
          path: /metrics
67
          skipMTLS: true`, name)
68
	return YamlK8s(mesh)
69
}
70

71
func MeshAndEnvoyMetricsFilters(name string, usedOnly string) InstallFunc {
72
	mesh := fmt.Sprintf(`
73
apiVersion: kuma.io/v1alpha1
74
kind: Mesh
75
metadata:
76
  name: %s
77
spec:
78
  metrics:
79
    enabledBackend: prom-1
80
    backends:
81
      - name: prom-1
82
        type: prometheus
83
        conf:
84
          port: 5555
85
          path: /metrics/stats
86
          envoy:
87
            filterRegex: http2_act.*
88
            usedOnly: %s
89
          skipMTLS: true
90
          aggregate:
91
          - name: path-stats
92
            path: "/path-stats"
93
            port: 80`, name, usedOnly)
94
	return YamlK8s(mesh)
95
}
96

97
func ApplicationsMetrics() {
98
	const namespace = "applications-metrics"
99
	const mesh = "applications-metrics"
100
	const meshNoAggregate = "applications-metrics-no-aggregeate"
101
	const meshEnvoyFilter = "applications-metrics-envoy-filter"
102

103
	BeforeAll(func() {
104
		err := NewClusterSetup().
105
			Install(MeshAndMetricsAggregate(mesh)).
106
			Install(MeshAndMetricsEnabled(meshNoAggregate)).
107
			Install(MeshAndEnvoyMetricsFilters(meshEnvoyFilter, "false")).
108
			Install(NamespaceWithSidecarInjection(namespace)).
109
			Install(testserver.Install(
110
				testserver.WithNamespace(namespace),
111
				testserver.WithMesh(mesh),
112
				testserver.WithName("test-server"),
113
			)).
114
			Install(testserver.Install(
115
				testserver.WithNamespace(namespace),
116
				testserver.WithMesh(meshEnvoyFilter),
117
				testserver.WithName("test-server-filter"),
118
			)).
119
			Install(testserver.Install(
120
				testserver.WithNamespace(namespace),
121
				testserver.WithMesh(meshNoAggregate),
122
				testserver.WithName("test-server-dp-metrics"),
123
				testserver.WithoutProbes(), // when application binds to localhost you cannot access it
124
				testserver.WithEchoArgs("--ip", "localhost"),
125
				testserver.WithPodAnnotations(map[string]string{
126
					"prometheus.metrics.kuma.io/aggregate-app-path":       "/my-app",
127
					"prometheus.metrics.kuma.io/aggregate-app-port":       "80",
128
					"prometheus.metrics.kuma.io/aggregate-app-address":    "localhost",
129
					"prometheus.metrics.kuma.io/aggregate-other-app-path": "/other-app",
130
					"prometheus.metrics.kuma.io/aggregate-other-app-port": "80",
131
				}))).
132
			Install(testserver.Install(
133
				testserver.WithNamespace(namespace),
134
				testserver.WithMesh(mesh),
135
				testserver.WithName("test-server-override-mesh"),
136
				testserver.WithPodAnnotations(map[string]string{
137
					"prometheus.metrics.kuma.io/aggregate-path-stats-enabled":       "false",
138
					"prometheus.metrics.kuma.io/aggregate-app-path":                 "/my-app",
139
					"prometheus.metrics.kuma.io/aggregate-app-port":                 "80",
140
					"prometheus.metrics.kuma.io/aggregate-service-to-override-path": "/overridden",
141
					"prometheus.metrics.kuma.io/aggregate-service-to-override-port": "80",
142
				}))).
143
			Setup(kubernetes.Cluster)
144
		Expect(err).To(Succeed())
145
	})
146
	E2EAfterAll(func() {
147
		Expect(kubernetes.Cluster.TriggerDeleteNamespace(namespace)).To(Succeed())
148
		Expect(kubernetes.Cluster.DeleteMesh(mesh)).To(Succeed())
149
		Expect(kubernetes.Cluster.DeleteMesh(meshNoAggregate)).To(Succeed())
150
	})
151

152
	It("should scrape metrics defined in mesh and not fail when defined service doesn't exist", func() {
153
		// given
154
		podIp, err := PodIPOfApp(kubernetes.Cluster, "test-server", namespace)
155
		Expect(err).ToNot(HaveOccurred())
156

157
		// when
158
		stdout, _, err := client.CollectResponse(
159
			kubernetes.Cluster, "test-server", "http://"+net.JoinHostPort(podIp, "1234")+"/metrics",
160
			client.FromKubernetesPod(namespace, "test-server"),
161
		)
162

163
		// then
164
		Expect(err).ToNot(HaveOccurred())
165
		Expect(stdout).ToNot(BeNil())
166
		// response returned by test-server
167
		Expect(stdout).To(ContainSubstring("path-stats"))
168
		// metric from envoy
169
		Expect(stdout).To(ContainSubstring("envoy_server_concurrency"))
170
		// response doesn't exist
171
		Expect(stdout).ToNot(ContainSubstring("not-working-service"))
172
	})
173

174
	It("should override mesh configuration with annotation", func() {
175
		// given
176
		podIp, err := PodIPOfApp(kubernetes.Cluster, "test-server-override-mesh", namespace)
177
		Expect(err).ToNot(HaveOccurred())
178

179
		// when
180
		stdout, _, err := client.CollectResponse(
181
			kubernetes.Cluster, "test-server-override-mesh", "http://"+net.JoinHostPort(podIp, "1234")+"/metrics",
182
			client.FromKubernetesPod(namespace, "test-server-override-mesh"),
183
		)
184

185
		// then
186
		Expect(err).ToNot(HaveOccurred())
187
		Expect(stdout).ToNot(BeNil())
188

189
		// response doesn't exist because was disabled
190
		Expect(stdout).ToNot(ContainSubstring("path-stats"))
191
		// path has been overridden
192
		Expect(stdout).ToNot(ContainSubstring("service-to-override"))
193
		// response doesn't exist
194
		Expect(stdout).ToNot(ContainSubstring("not-working-service"))
195

196
		// overridden by pod
197
		Expect(stdout).To(ContainSubstring("overridden"))
198
		// overridden by pod
199
		Expect(stdout).To(ContainSubstring("mesh-default"))
200
		// added in pod
201
		Expect(stdout).To(ContainSubstring("my-app"))
202
		// metric from envoy
203
		Expect(stdout).To(ContainSubstring("envoy_server_concurrency"))
204
	})
205

206
	It("should use only configuration from dataplane", func() {
207
		// given
208
		podIp, err := PodIPOfApp(kubernetes.Cluster, "test-server-dp-metrics", namespace)
209
		Expect(err).ToNot(HaveOccurred())
210

211
		// when
212
		stdout, _, err := client.CollectResponse(
213
			kubernetes.Cluster, "test-server-dp-metrics", "http://"+net.JoinHostPort(podIp, "1234")+"/metrics",
214
			client.FromKubernetesPod(namespace, "test-server-dp-metrics"),
215
		)
216

217
		// then
218
		Expect(err).ToNot(HaveOccurred())
219
		Expect(stdout).ToNot(BeNil())
220

221
		// response doesn't exist because was disabled
222
		Expect(stdout).ToNot(ContainSubstring("path-stats"))
223
		// path has been overridden
224
		Expect(stdout).ToNot(ContainSubstring("service-to-override"))
225

226
		// overridden by pod
227
		Expect(stdout).To(ContainSubstring("my-app"))
228
		// overridden by pod but binding to localhost so it's not available
229
		Expect(stdout).ToNot(ContainSubstring("other-app"))
230
		// metric from envoy
231
		Expect(stdout).To(ContainSubstring("envoy_server_concurrency"))
232
	})
233

234
	It("should return filtered Envoy metrics and react for change of usedOnly parameter", func() {
235
		// given
236
		podIp, err := PodIPOfApp(kubernetes.Cluster, "test-server-filter", namespace)
237
		Expect(err).ToNot(HaveOccurred())
238

239
		// when
240
		stdout, _, err := client.CollectResponse(
241
			kubernetes.Cluster, "test-server-filter", "http://"+net.JoinHostPort(podIp, "5555")+"/metrics/stats",
242
			client.FromKubernetesPod(namespace, "test-server-filter"),
243
		)
244

245
		// then
246
		Expect(err).ToNot(HaveOccurred())
247
		Expect(stdout).ToNot(BeNil())
248

249
		// other application metrics
250
		Expect(stdout).To(ContainSubstring("path-stats"))
251
		// metric from envoy
252
		Expect(stdout).To(ContainSubstring("kuma_envoy_admin"))
253

254
		// when usedOnly is enabled
255
		Expect(MeshAndEnvoyMetricsFilters(meshEnvoyFilter, "true")(kubernetes.Cluster)).To(Succeed())
256

257
		// then
258
		Eventually(func(g Gomega) {
259
			stdout, _, err := client.CollectResponse(
260
				kubernetes.Cluster, "test-server-filter", "http://"+net.JoinHostPort(podIp, "5555")+"/metrics/stats",
261
				client.FromKubernetesPod(namespace, "test-server-filter"),
262
			)
263
			g.Expect(err).ToNot(HaveOccurred())
264
			g.Expect(stdout).To(And(
265
				ContainSubstring("path-stats"),
266
				Not(ContainSubstring("kuma_envoy_admin")),
267
			))
268
		}, "30s", "1s").Should(Succeed())
269
	})
270
}
271