prometheus

1
// Copyright 2013 The Prometheus Authors
2
// Licensed under the Apache License, Version 2.0 (the "License");
3
// you may not use this file except in compliance with the License.
4
// You may obtain a copy of the License at
5
//
6
// http://www.apache.org/licenses/LICENSE-2.0
7
//
8
// Unless required by applicable law or agreed to in writing, software
9
// distributed under the License is distributed on an "AS IS" BASIS,
10
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11
// See the License for the specific language governing permissions and
12
// limitations under the License.
13

14
package scrape
15

16
import (
17
	"crypto/tls"
18
	"crypto/x509"
19
	"fmt"
20
	"net/http"
21
	"net/http/httptest"
22
	"net/url"
23
	"os"
24
	"strconv"
25
	"strings"
26
	"testing"
27
	"time"
28

29
	config_util "github.com/prometheus/common/config"
30
	"github.com/prometheus/common/model"
31
	"github.com/stretchr/testify/require"
32

33
	"github.com/prometheus/prometheus/config"
34
	"github.com/prometheus/prometheus/discovery/targetgroup"
35
	"github.com/prometheus/prometheus/model/histogram"
36
	"github.com/prometheus/prometheus/model/labels"
37
)
38

39
const (
40
	caCertPath = "testdata/ca.cer"
41
)
42

43
func TestTargetLabels(t *testing.T) {
44
	target := newTestTarget("example.com:80", 0, labels.FromStrings("job", "some_job", "foo", "bar"))
45
	want := labels.FromStrings(model.JobLabel, "some_job", "foo", "bar")
46
	b := labels.NewScratchBuilder(0)
47
	got := target.Labels(&b)
48
	require.Equal(t, want, got)
49
	i := 0
50
	target.LabelsRange(func(l labels.Label) {
51
		switch i {
52
		case 0:
53
			require.Equal(t, labels.Label{Name: "foo", Value: "bar"}, l)
54
		case 1:
55
			require.Equal(t, labels.Label{Name: model.JobLabel, Value: "some_job"}, l)
56
		}
57
		i++
58
	})
59
	require.Equal(t, 2, i)
60
}
61

62
func TestTargetOffset(t *testing.T) {
63
	interval := 10 * time.Second
64
	offsetSeed := uint64(0)
65

66
	offsets := make([]time.Duration, 10000)
67

68
	// Calculate offsets for 10000 different targets.
69
	for i := range offsets {
70
		target := newTestTarget("example.com:80", 0, labels.FromStrings(
71
			"label", strconv.Itoa(i),
72
		))
73
		offsets[i] = target.offset(interval, offsetSeed)
74
	}
75

76
	// Put the offsets into buckets and validate that they are all
77
	// within bounds.
78
	bucketSize := 1 * time.Second
79
	buckets := make([]int, interval/bucketSize)
80

81
	for _, offset := range offsets {
82
		require.InDelta(t, time.Duration(0), offset, float64(interval), "Offset %v out of bounds.", offset)
83

84
		bucket := offset / bucketSize
85
		buckets[bucket]++
86
	}
87

88
	t.Log(buckets)
89

90
	// Calculate whether the number of targets per bucket
91
	// does not differ more than a given tolerance.
92
	avg := len(offsets) / len(buckets)
93
	tolerance := 0.15
94

95
	for _, bucket := range buckets {
96
		diff := bucket - avg
97
		if diff < 0 {
98
			diff = -diff
99
		}
100

101
		require.LessOrEqual(t, float64(diff)/float64(avg), tolerance, "Bucket out of tolerance bounds.")
102
	}
103
}
104

105
func TestTargetURL(t *testing.T) {
106
	params := url.Values{
107
		"abc": []string{"foo", "bar", "baz"},
108
		"xyz": []string{"hoo"},
109
	}
110
	labels := labels.FromMap(map[string]string{
111
		model.AddressLabel:     "example.com:1234",
112
		model.SchemeLabel:      "https",
113
		model.MetricsPathLabel: "/metricz",
114
		"__param_abc":          "overwrite",
115
		"__param_cde":          "huu",
116
	})
117
	target := NewTarget(labels, labels, params)
118

119
	// The reserved labels are concatenated into a full URL. The first value for each
120
	// URL query parameter can be set/modified via labels as well.
121
	expectedParams := url.Values{
122
		"abc": []string{"overwrite", "bar", "baz"},
123
		"cde": []string{"huu"},
124
		"xyz": []string{"hoo"},
125
	}
126
	expectedURL := &url.URL{
127
		Scheme:   "https",
128
		Host:     "example.com:1234",
129
		Path:     "/metricz",
130
		RawQuery: expectedParams.Encode(),
131
	}
132

133
	require.Equal(t, expectedURL, target.URL())
134
}
135

136
func newTestTarget(targetURL string, _ time.Duration, lbls labels.Labels) *Target {
137
	lb := labels.NewBuilder(lbls)
138
	lb.Set(model.SchemeLabel, "http")
139
	lb.Set(model.AddressLabel, strings.TrimPrefix(targetURL, "http://"))
140
	lb.Set(model.MetricsPathLabel, "/metrics")
141

142
	return &Target{labels: lb.Labels()}
143
}
144

145
func TestNewHTTPBearerToken(t *testing.T) {
146
	server := httptest.NewServer(
147
		http.HandlerFunc(
148
			func(w http.ResponseWriter, r *http.Request) {
149
				expected := "Bearer 1234"
150
				received := r.Header.Get("Authorization")
151
				require.Equal(t, expected, received, "Authorization header was not set correctly.")
152
			},
153
		),
154
	)
155
	defer server.Close()
156

157
	cfg := config_util.HTTPClientConfig{
158
		BearerToken: "1234",
159
	}
160
	c, err := config_util.NewClientFromConfig(cfg, "test")
161
	require.NoError(t, err)
162
	_, err = c.Get(server.URL)
163
	require.NoError(t, err)
164
}
165

166
func TestNewHTTPBearerTokenFile(t *testing.T) {
167
	server := httptest.NewServer(
168
		http.HandlerFunc(
169
			func(w http.ResponseWriter, r *http.Request) {
170
				expected := "Bearer 12345"
171
				received := r.Header.Get("Authorization")
172
				require.Equal(t, expected, received, "Authorization header was not set correctly.")
173
			},
174
		),
175
	)
176
	defer server.Close()
177

178
	cfg := config_util.HTTPClientConfig{
179
		BearerTokenFile: "testdata/bearertoken.txt",
180
	}
181
	c, err := config_util.NewClientFromConfig(cfg, "test")
182
	require.NoError(t, err)
183
	_, err = c.Get(server.URL)
184
	require.NoError(t, err)
185
}
186

187
func TestNewHTTPBasicAuth(t *testing.T) {
188
	server := httptest.NewServer(
189
		http.HandlerFunc(
190
			func(w http.ResponseWriter, r *http.Request) {
191
				username, password, ok := r.BasicAuth()
192
				require.True(t, ok, "Basic authorization header was not set correctly.")
193
				require.Equal(t, "user", username)
194
				require.Equal(t, "password123", password)
195
			},
196
		),
197
	)
198
	defer server.Close()
199

200
	cfg := config_util.HTTPClientConfig{
201
		BasicAuth: &config_util.BasicAuth{
202
			Username: "user",
203
			Password: "password123",
204
		},
205
	}
206
	c, err := config_util.NewClientFromConfig(cfg, "test")
207
	require.NoError(t, err)
208
	_, err = c.Get(server.URL)
209
	require.NoError(t, err)
210
}
211

212
func TestNewHTTPCACert(t *testing.T) {
213
	server := httptest.NewUnstartedServer(
214
		http.HandlerFunc(
215
			func(w http.ResponseWriter, r *http.Request) {
216
				w.Header().Set("Content-Type", `text/plain; version=0.0.4`)
217
				w.Write([]byte{})
218
			},
219
		),
220
	)
221
	server.TLS = newTLSConfig("server", t)
222
	server.StartTLS()
223
	defer server.Close()
224

225
	cfg := config_util.HTTPClientConfig{
226
		TLSConfig: config_util.TLSConfig{
227
			CAFile: caCertPath,
228
		},
229
	}
230
	c, err := config_util.NewClientFromConfig(cfg, "test")
231
	require.NoError(t, err)
232
	_, err = c.Get(server.URL)
233
	require.NoError(t, err)
234
}
235

236
func TestNewHTTPClientCert(t *testing.T) {
237
	server := httptest.NewUnstartedServer(
238
		http.HandlerFunc(
239
			func(w http.ResponseWriter, r *http.Request) {
240
				w.Header().Set("Content-Type", `text/plain; version=0.0.4`)
241
				w.Write([]byte{})
242
			},
243
		),
244
	)
245
	tlsConfig := newTLSConfig("server", t)
246
	tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert
247
	tlsConfig.ClientCAs = tlsConfig.RootCAs
248
	server.TLS = tlsConfig
249
	server.StartTLS()
250
	defer server.Close()
251

252
	cfg := config_util.HTTPClientConfig{
253
		TLSConfig: config_util.TLSConfig{
254
			CAFile:   caCertPath,
255
			CertFile: "testdata/client.cer",
256
			KeyFile:  "testdata/client.key",
257
		},
258
	}
259
	c, err := config_util.NewClientFromConfig(cfg, "test")
260
	require.NoError(t, err)
261
	_, err = c.Get(server.URL)
262
	require.NoError(t, err)
263
}
264

265
func TestNewHTTPWithServerName(t *testing.T) {
266
	server := httptest.NewUnstartedServer(
267
		http.HandlerFunc(
268
			func(w http.ResponseWriter, r *http.Request) {
269
				w.Header().Set("Content-Type", `text/plain; version=0.0.4`)
270
				w.Write([]byte{})
271
			},
272
		),
273
	)
274
	server.TLS = newTLSConfig("servername", t)
275
	server.StartTLS()
276
	defer server.Close()
277

278
	cfg := config_util.HTTPClientConfig{
279
		TLSConfig: config_util.TLSConfig{
280
			CAFile:     caCertPath,
281
			ServerName: "prometheus.rocks",
282
		},
283
	}
284
	c, err := config_util.NewClientFromConfig(cfg, "test")
285
	require.NoError(t, err)
286
	_, err = c.Get(server.URL)
287
	require.NoError(t, err)
288
}
289

290
func TestNewHTTPWithBadServerName(t *testing.T) {
291
	server := httptest.NewUnstartedServer(
292
		http.HandlerFunc(
293
			func(w http.ResponseWriter, r *http.Request) {
294
				w.Header().Set("Content-Type", `text/plain; version=0.0.4`)
295
				w.Write([]byte{})
296
			},
297
		),
298
	)
299
	server.TLS = newTLSConfig("servername", t)
300
	server.StartTLS()
301
	defer server.Close()
302

303
	cfg := config_util.HTTPClientConfig{
304
		TLSConfig: config_util.TLSConfig{
305
			CAFile:     caCertPath,
306
			ServerName: "badname",
307
		},
308
	}
309
	c, err := config_util.NewClientFromConfig(cfg, "test")
310
	require.NoError(t, err)
311
	_, err = c.Get(server.URL)
312
	require.Error(t, err)
313
}
314

315
func newTLSConfig(certName string, t *testing.T) *tls.Config {
316
	tlsConfig := &tls.Config{}
317
	caCertPool := x509.NewCertPool()
318
	caCert, err := os.ReadFile(caCertPath)
319
	require.NoError(t, err, "Couldn't read CA cert.")
320
	caCertPool.AppendCertsFromPEM(caCert)
321
	tlsConfig.RootCAs = caCertPool
322
	tlsConfig.ServerName = "127.0.0.1"
323
	certPath := fmt.Sprintf("testdata/%s.cer", certName)
324
	keyPath := fmt.Sprintf("testdata/%s.key", certName)
325
	cert, err := tls.LoadX509KeyPair(certPath, keyPath)
326
	require.NoError(t, err, "Unable to use specified server cert (%s) & key (%v).", certPath, keyPath)
327
	tlsConfig.Certificates = []tls.Certificate{cert}
328
	return tlsConfig
329
}
330

331
func TestNewClientWithBadTLSConfig(t *testing.T) {
332
	cfg := config_util.HTTPClientConfig{
333
		TLSConfig: config_util.TLSConfig{
334
			CAFile:   "testdata/nonexistent_ca.cer",
335
			CertFile: "testdata/nonexistent_client.cer",
336
			KeyFile:  "testdata/nonexistent_client.key",
337
		},
338
	}
339
	_, err := config_util.NewClientFromConfig(cfg, "test")
340
	require.Error(t, err)
341
}
342

343
func TestTargetsFromGroup(t *testing.T) {
344
	expectedError := "instance 0 in group : no address"
345

346
	cfg := config.ScrapeConfig{
347
		ScrapeTimeout:  model.Duration(10 * time.Second),
348
		ScrapeInterval: model.Duration(1 * time.Minute),
349
	}
350
	lb := labels.NewBuilder(labels.EmptyLabels())
351
	targets, failures := TargetsFromGroup(&targetgroup.Group{Targets: []model.LabelSet{{}, {model.AddressLabel: "localhost:9090"}}}, &cfg, false, nil, lb)
352
	require.Len(t, targets, 1)
353
	require.Len(t, failures, 1)
354
	require.EqualError(t, failures[0], expectedError)
355
}
356

357
func BenchmarkTargetsFromGroup(b *testing.B) {
358
	// Simulate Kubernetes service-discovery and use subset of rules from typical Prometheus config.
359
	cfgText := `
360
scrape_configs:
361
  - job_name: job1
362
    scrape_interval: 15s
363
    scrape_timeout: 10s
364
    relabel_configs:
365
    - source_labels: [__meta_kubernetes_pod_container_port_name]
366
      separator: ;
367
      regex: .*-metrics
368
      replacement: $1
369
      action: keep
370
    - source_labels: [__meta_kubernetes_pod_phase]
371
      separator: ;
372
      regex: Succeeded|Failed
373
      replacement: $1
374
      action: drop
375
    - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_pod_label_name]
376
      separator: /
377
      regex: (.*)
378
      target_label: job
379
      replacement: $1
380
      action: replace
381
    - source_labels: [__meta_kubernetes_namespace]
382
      separator: ;
383
      regex: (.*)
384
      target_label: namespace
385
      replacement: $1
386
      action: replace
387
    - source_labels: [__meta_kubernetes_pod_name]
388
      separator: ;
389
      regex: (.*)
390
      target_label: pod
391
      replacement: $1
392
      action: replace
393
    - source_labels: [__meta_kubernetes_pod_container_name]
394
      separator: ;
395
      regex: (.*)
396
      target_label: container
397
      replacement: $1
398
      action: replace
399
    - source_labels: [__meta_kubernetes_pod_name, __meta_kubernetes_pod_container_name,
400
        __meta_kubernetes_pod_container_port_name]
401
      separator: ':'
402
      regex: (.*)
403
      target_label: instance
404
      replacement: $1
405
      action: replace
406
    - separator: ;
407
      regex: (.*)
408
      target_label: cluster
409
      replacement: dev-us-central-0
410
      action: replace
411
`
412
	config := loadConfiguration(b, cfgText)
413
	for _, nTargets := range []int{1, 10, 100} {
414
		b.Run(fmt.Sprintf("%d_targets", nTargets), func(b *testing.B) {
415
			targets := []model.LabelSet{}
416
			for i := 0; i < nTargets; i++ {
417
				labels := model.LabelSet{
418
					model.AddressLabel:                            model.LabelValue(fmt.Sprintf("localhost:%d", i)),
419
					"__meta_kubernetes_namespace":                 "some_namespace",
420
					"__meta_kubernetes_pod_container_name":        "some_container",
421
					"__meta_kubernetes_pod_container_port_name":   "http-metrics",
422
					"__meta_kubernetes_pod_container_port_number": "80",
423
					"__meta_kubernetes_pod_label_name":            "some_name",
424
					"__meta_kubernetes_pod_name":                  "some_pod",
425
					"__meta_kubernetes_pod_phase":                 "Running",
426
				}
427
				// Add some more labels, because Kubernetes SD generates a lot
428
				for i := 0; i < 10; i++ {
429
					labels[model.LabelName(fmt.Sprintf("__meta_kubernetes_pod_label_extra%d", i))] = "a_label_abcdefgh"
430
					labels[model.LabelName(fmt.Sprintf("__meta_kubernetes_pod_labelpresent_extra%d", i))] = "true"
431
				}
432
				targets = append(targets, labels)
433
			}
434
			var tgets []*Target
435
			lb := labels.NewBuilder(labels.EmptyLabels())
436
			group := &targetgroup.Group{Targets: targets}
437
			for i := 0; i < b.N; i++ {
438
				tgets, _ = TargetsFromGroup(group, config.ScrapeConfigs[0], false, tgets, lb)
439
				if len(targets) != nTargets {
440
					b.Fatalf("Expected %d targets, got %d", nTargets, len(targets))
441
				}
442
			}
443
		})
444
	}
445
}
446

447
func TestBucketLimitAppender(t *testing.T) {
448
	example := histogram.Histogram{
449
		Schema:        0,
450
		Count:         21,
451
		Sum:           33,
452
		ZeroThreshold: 0.001,
453
		ZeroCount:     3,
454
		PositiveSpans: []histogram.Span{
455
			{Offset: 0, Length: 3},
456
		},
457
		PositiveBuckets: []int64{3, 0, 0},
458
		NegativeSpans: []histogram.Span{
459
			{Offset: 0, Length: 3},
460
		},
461
		NegativeBuckets: []int64{3, 0, 0},
462
	}
463

464
	bigGap := histogram.Histogram{
465
		Schema:        0,
466
		Count:         21,
467
		Sum:           33,
468
		ZeroThreshold: 0.001,
469
		ZeroCount:     3,
470
		PositiveSpans: []histogram.Span{
471
			{Offset: 1, Length: 1}, // in (1, 2]
472
			{Offset: 2, Length: 1}, // in (8, 16]
473
		},
474
		PositiveBuckets: []int64{1, 0}, // 1, 1
475
	}
476

477
	cases := []struct {
478
		h                 histogram.Histogram
479
		limit             int
480
		expectError       bool
481
		expectBucketCount int
482
		expectSchema      int32
483
	}{
484
		{
485
			h:           example,
486
			limit:       3,
487
			expectError: true,
488
		},
489
		{
490
			h:                 example,
491
			limit:             4,
492
			expectError:       false,
493
			expectBucketCount: 4,
494
			expectSchema:      -1,
495
		},
496
		{
497
			h:                 example,
498
			limit:             10,
499
			expectError:       false,
500
			expectBucketCount: 6,
501
			expectSchema:      0,
502
		},
503
		{
504
			h:                 bigGap,
505
			limit:             1,
506
			expectError:       false,
507
			expectBucketCount: 1,
508
			expectSchema:      -2,
509
		},
510
	}
511

512
	resApp := &collectResultAppender{}
513

514
	for _, c := range cases {
515
		for _, floatHisto := range []bool{true, false} {
516
			t.Run(fmt.Sprintf("floatHistogram=%t", floatHisto), func(t *testing.T) {
517
				app := &bucketLimitAppender{Appender: resApp, limit: c.limit}
518
				ts := int64(10 * time.Minute / time.Millisecond)
519
				lbls := labels.FromStrings("__name__", "sparse_histogram_series")
520
				var err error
521
				if floatHisto {
522
					fh := c.h.Copy().ToFloat(nil)
523
					_, err = app.AppendHistogram(0, lbls, ts, nil, fh)
524
					if c.expectError {
525
						require.Error(t, err)
526
					} else {
527
						require.Equal(t, c.expectSchema, fh.Schema)
528
						require.Equal(t, c.expectBucketCount, len(fh.NegativeBuckets)+len(fh.PositiveBuckets))
529
						require.NoError(t, err)
530
					}
531
				} else {
532
					h := c.h.Copy()
533
					_, err = app.AppendHistogram(0, lbls, ts, h, nil)
534
					if c.expectError {
535
						require.Error(t, err)
536
					} else {
537
						require.Equal(t, c.expectSchema, h.Schema)
538
						require.Equal(t, c.expectBucketCount, len(h.NegativeBuckets)+len(h.PositiveBuckets))
539
						require.NoError(t, err)
540
					}
541
				}
542
				require.NoError(t, app.Commit())
543
			})
544
		}
545
	}
546
}
547

548
func TestMaxSchemaAppender(t *testing.T) {
549
	example := histogram.Histogram{
550
		Schema:        0,
551
		Count:         21,
552
		Sum:           33,
553
		ZeroThreshold: 0.001,
554
		ZeroCount:     3,
555
		PositiveSpans: []histogram.Span{
556
			{Offset: 0, Length: 3},
557
		},
558
		PositiveBuckets: []int64{3, 0, 0},
559
		NegativeSpans: []histogram.Span{
560
			{Offset: 0, Length: 3},
561
		},
562
		NegativeBuckets: []int64{3, 0, 0},
563
	}
564

565
	cases := []struct {
566
		h            histogram.Histogram
567
		maxSchema    int32
568
		expectSchema int32
569
	}{
570
		{
571
			h:            example,
572
			maxSchema:    -1,
573
			expectSchema: -1,
574
		},
575
		{
576
			h:            example,
577
			maxSchema:    0,
578
			expectSchema: 0,
579
		},
580
	}
581

582
	resApp := &collectResultAppender{}
583

584
	for _, c := range cases {
585
		for _, floatHisto := range []bool{true, false} {
586
			t.Run(fmt.Sprintf("floatHistogram=%t", floatHisto), func(t *testing.T) {
587
				app := &maxSchemaAppender{Appender: resApp, maxSchema: c.maxSchema}
588
				ts := int64(10 * time.Minute / time.Millisecond)
589
				lbls := labels.FromStrings("__name__", "sparse_histogram_series")
590
				var err error
591
				if floatHisto {
592
					fh := c.h.Copy().ToFloat(nil)
593
					_, err = app.AppendHistogram(0, lbls, ts, nil, fh)
594
					require.Equal(t, c.expectSchema, fh.Schema)
595
					require.NoError(t, err)
596
				} else {
597
					h := c.h.Copy()
598
					_, err = app.AppendHistogram(0, lbls, ts, h, nil)
599
					require.Equal(t, c.expectSchema, h.Schema)
600
					require.NoError(t, err)
601
				}
602
				require.NoError(t, app.Commit())
603
			})
604
		}
605
	}
606
}
607