prometheus

1
// Copyright 2018 The Prometheus Authors
2
// Licensed under the Apache License, Version 2.0 (the "License");
3
// you may not use this file except in compliance with the License.
4
// You may obtain a copy of the License at
5
//
6
// http://www.apache.org/licenses/LICENSE-2.0
7
//
8
// Unless required by applicable law or agreed to in writing, software
9
// distributed under the License is distributed on an "AS IS" BASIS,
10
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11
// See the License for the specific language governing permissions and
12
// limitations under the License.
13

14
package discovery
15

16
import (
17
	"context"
18
	"fmt"
19
	"net/url"
20
	"time"
21

22
	"github.com/prometheus/client_golang/prometheus"
23
	"k8s.io/client-go/tools/metrics"
24
	"k8s.io/client-go/util/workqueue"
25
)
26

27
// This file registers metrics used by the Kubernetes Go client (k8s.io/client-go).
28
// Unfortunately, k8s.io/client-go metrics are global.
29
// If we instantiate multiple k8s SD instances, their k8s/client-go metrics will overlap.
30
// To prevent us from displaying misleading metrics, we register k8s.io/client-go metrics
31
// outside of the Kubernetes SD.
32

33
const (
34
	KubernetesMetricsNamespace = "prometheus_sd_kubernetes"
35
	workqueueMetricsNamespace  = KubernetesMetricsNamespace + "_workqueue"
36
)
37

38
var (
39
	clientGoRequestMetrics  = &clientGoRequestMetricAdapter{}
40
	clientGoWorkloadMetrics = &clientGoWorkqueueMetricsProvider{}
41
)
42

43
var (
44
	// Metrics for client-go's HTTP requests.
45
	clientGoRequestResultMetricVec = prometheus.NewCounterVec(
46
		prometheus.CounterOpts{
47
			Namespace: KubernetesMetricsNamespace,
48
			Name:      "http_request_total",
49
			Help:      "Total number of HTTP requests to the Kubernetes API by status code.",
50
		},
51
		[]string{"status_code"},
52
	)
53
	clientGoRequestLatencyMetricVec = prometheus.NewSummaryVec(
54
		prometheus.SummaryOpts{
55
			Namespace:  KubernetesMetricsNamespace,
56
			Name:       "http_request_duration_seconds",
57
			Help:       "Summary of latencies for HTTP requests to the Kubernetes API by endpoint.",
58
			Objectives: map[float64]float64{},
59
		},
60
		[]string{"endpoint"},
61
	)
62

63
	// Definition of metrics for client-go workflow metrics provider.
64
	clientGoWorkqueueDepthMetricVec = prometheus.NewGaugeVec(
65
		prometheus.GaugeOpts{
66
			Namespace: workqueueMetricsNamespace,
67
			Name:      "depth",
68
			Help:      "Current depth of the work queue.",
69
		},
70
		[]string{"queue_name"},
71
	)
72
	clientGoWorkqueueAddsMetricVec = prometheus.NewCounterVec(
73
		prometheus.CounterOpts{
74
			Namespace: workqueueMetricsNamespace,
75
			Name:      "items_total",
76
			Help:      "Total number of items added to the work queue.",
77
		},
78
		[]string{"queue_name"},
79
	)
80
	clientGoWorkqueueLatencyMetricVec = prometheus.NewSummaryVec(
81
		prometheus.SummaryOpts{
82
			Namespace:  workqueueMetricsNamespace,
83
			Name:       "latency_seconds",
84
			Help:       "How long an item stays in the work queue.",
85
			Objectives: map[float64]float64{},
86
		},
87
		[]string{"queue_name"},
88
	)
89
	clientGoWorkqueueUnfinishedWorkSecondsMetricVec = prometheus.NewGaugeVec(
90
		prometheus.GaugeOpts{
91
			Namespace: workqueueMetricsNamespace,
92
			Name:      "unfinished_work_seconds",
93
			Help:      "How long an item has remained unfinished in the work queue.",
94
		},
95
		[]string{"queue_name"},
96
	)
97
	clientGoWorkqueueLongestRunningProcessorMetricVec = prometheus.NewGaugeVec(
98
		prometheus.GaugeOpts{
99
			Namespace: workqueueMetricsNamespace,
100
			Name:      "longest_running_processor_seconds",
101
			Help:      "Duration of the longest running processor in the work queue.",
102
		},
103
		[]string{"queue_name"},
104
	)
105
	clientGoWorkqueueWorkDurationMetricVec = prometheus.NewSummaryVec(
106
		prometheus.SummaryOpts{
107
			Namespace:  workqueueMetricsNamespace,
108
			Name:       "work_duration_seconds",
109
			Help:       "How long processing an item from the work queue takes.",
110
			Objectives: map[float64]float64{},
111
		},
112
		[]string{"queue_name"},
113
	)
114
)
115

116
// Definition of dummy metric used as a placeholder if we don't want to observe some data.
117
type noopMetric struct{}
118

119
func (noopMetric) Inc()            {}
120
func (noopMetric) Dec()            {}
121
func (noopMetric) Observe(float64) {}
122
func (noopMetric) Set(float64)     {}
123

124
// Definition of client-go metrics adapters for HTTP requests observation.
125
type clientGoRequestMetricAdapter struct{}
126

127
// Returns all of the Prometheus metrics derived from k8s.io/client-go.
128
// This may be used tu register and unregister the metrics.
129
func clientGoMetrics() []prometheus.Collector {
130
	return []prometheus.Collector{
131
		clientGoRequestResultMetricVec,
132
		clientGoRequestLatencyMetricVec,
133
		clientGoWorkqueueDepthMetricVec,
134
		clientGoWorkqueueAddsMetricVec,
135
		clientGoWorkqueueLatencyMetricVec,
136
		clientGoWorkqueueUnfinishedWorkSecondsMetricVec,
137
		clientGoWorkqueueLongestRunningProcessorMetricVec,
138
		clientGoWorkqueueWorkDurationMetricVec,
139
	}
140
}
141

142
func RegisterK8sClientMetricsWithPrometheus(registerer prometheus.Registerer) error {
143
	clientGoRequestMetrics.RegisterWithK8sGoClient()
144
	clientGoWorkloadMetrics.RegisterWithK8sGoClient()
145

146
	for _, collector := range clientGoMetrics() {
147
		err := registerer.Register(collector)
148
		if err != nil {
149
			return fmt.Errorf("failed to register Kubernetes Go Client metrics: %w", err)
150
		}
151
	}
152
	return nil
153
}
154

155
func (f *clientGoRequestMetricAdapter) RegisterWithK8sGoClient() {
156
	metrics.Register(
157
		metrics.RegisterOpts{
158
			RequestLatency: f,
159
			RequestResult:  f,
160
		},
161
	)
162
}
163

164
func (clientGoRequestMetricAdapter) Increment(_ context.Context, code, _, _ string) {
165
	clientGoRequestResultMetricVec.WithLabelValues(code).Inc()
166
}
167

168
func (clientGoRequestMetricAdapter) Observe(_ context.Context, _ string, u url.URL, latency time.Duration) {
169
	clientGoRequestLatencyMetricVec.WithLabelValues(u.EscapedPath()).Observe(latency.Seconds())
170
}
171

172
// Definition of client-go workqueue metrics provider definition.
173
type clientGoWorkqueueMetricsProvider struct{}
174

175
func (f *clientGoWorkqueueMetricsProvider) RegisterWithK8sGoClient() {
176
	workqueue.SetProvider(f)
177
}
178

179
func (f *clientGoWorkqueueMetricsProvider) NewDepthMetric(name string) workqueue.GaugeMetric {
180
	return clientGoWorkqueueDepthMetricVec.WithLabelValues(name)
181
}
182

183
func (f *clientGoWorkqueueMetricsProvider) NewAddsMetric(name string) workqueue.CounterMetric {
184
	return clientGoWorkqueueAddsMetricVec.WithLabelValues(name)
185
}
186

187
func (f *clientGoWorkqueueMetricsProvider) NewLatencyMetric(name string) workqueue.HistogramMetric {
188
	return clientGoWorkqueueLatencyMetricVec.WithLabelValues(name)
189
}
190

191
func (f *clientGoWorkqueueMetricsProvider) NewWorkDurationMetric(name string) workqueue.HistogramMetric {
192
	return clientGoWorkqueueWorkDurationMetricVec.WithLabelValues(name)
193
}
194

195
func (f *clientGoWorkqueueMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) workqueue.SettableGaugeMetric {
196
	return clientGoWorkqueueUnfinishedWorkSecondsMetricVec.WithLabelValues(name)
197
}
198

199
func (f *clientGoWorkqueueMetricsProvider) NewLongestRunningProcessorSecondsMetric(name string) workqueue.SettableGaugeMetric {
200
	return clientGoWorkqueueLongestRunningProcessorMetricVec.WithLabelValues(name)
201
}
202

203
func (clientGoWorkqueueMetricsProvider) NewRetriesMetric(string) workqueue.CounterMetric {
204
	// Retries are not used so the metric is omitted.
205
	return noopMetric{}
206
}
207