prometheus

1
// Copyright 2016 The Prometheus Authors
2
// Licensed under the Apache License, Version 2.0 (the "License");
3
// you may not use this file except in compliance with the License.
4
// You may obtain a copy of the License at
5
//
6
//     http://www.apache.org/licenses/LICENSE-2.0
7
//
8
// Unless required by applicable law or agreed to in writing, software
9
// distributed under the License is distributed on an "AS IS" BASIS,
10
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11
// See the License for the specific language governing permissions and
12
// limitations under the License.
13

14
package web
15

16
import (
17
	"bytes"
18
	"context"
19
	"errors"
20
	"fmt"
21
	"io"
22
	"net/http"
23
	"net/http/httptest"
24
	"sort"
25
	"strconv"
26
	"strings"
27
	"testing"
28
	"time"
29

30
	"github.com/prometheus/common/model"
31
	"github.com/stretchr/testify/require"
32

33
	"github.com/prometheus/prometheus/config"
34
	"github.com/prometheus/prometheus/model/histogram"
35
	"github.com/prometheus/prometheus/model/labels"
36
	"github.com/prometheus/prometheus/model/textparse"
37
	"github.com/prometheus/prometheus/promql"
38
	"github.com/prometheus/prometheus/promql/promqltest"
39
	"github.com/prometheus/prometheus/storage"
40
	"github.com/prometheus/prometheus/tsdb"
41
	"github.com/prometheus/prometheus/util/teststorage"
42
	"github.com/prometheus/prometheus/util/testutil"
43
)
44

45
var scenarios = map[string]struct {
46
	params         string
47
	externalLabels labels.Labels
48
	code           int
49
	body           string
50
}{
51
	"empty": {
52
		params: "",
53
		code:   http.StatusOK,
54
		body:   ``,
55
	},
56
	"match nothing": {
57
		params: "match[]=does_not_match_anything",
58
		code:   http.StatusOK,
59
		body:   ``,
60
	},
61
	"invalid params from the beginning": {
62
		params: "match[]=-not-a-valid-metric-name",
63
		code:   http.StatusBadRequest,
64
		body: `1:1: parse error: unexpected <op:->
65
`,
66
	},
67
	"invalid params somewhere in the middle": {
68
		params: "match[]=not-a-valid-metric-name",
69
		code:   http.StatusBadRequest,
70
		body: `1:4: parse error: unexpected <op:->
71
`,
72
	},
73
	"test_metric1": {
74
		params: "match[]=test_metric1",
75
		code:   http.StatusOK,
76
		body: `# TYPE test_metric1 untyped
77
test_metric1{foo="bar",instance="i"} 10000 6000000
78
test_metric1{foo="boo",instance="i"} 1 6000000
79
`,
80
	},
81
	"test_metric2": {
82
		params: "match[]=test_metric2",
83
		code:   http.StatusOK,
84
		body: `# TYPE test_metric2 untyped
85
test_metric2{foo="boo",instance="i"} 1 6000000
86
`,
87
	},
88
	"test_metric_without_labels": {
89
		params: "match[]=test_metric_without_labels",
90
		code:   http.StatusOK,
91
		body: `# TYPE test_metric_without_labels untyped
92
test_metric_without_labels{instance=""} 1001 6000000
93
`,
94
	},
95
	"test_stale_metric": {
96
		params: "match[]=test_metric_stale",
97
		code:   http.StatusOK,
98
		body:   ``,
99
	},
100
	"test_old_metric": {
101
		params: "match[]=test_metric_old",
102
		code:   http.StatusOK,
103
		body: `# TYPE test_metric_old untyped
104
test_metric_old{instance=""} 981 5880000
105
`,
106
	},
107
	"{foo='boo'}": {
108
		params: "match[]={foo='boo'}",
109
		code:   http.StatusOK,
110
		body: `# TYPE test_metric1 untyped
111
test_metric1{foo="boo",instance="i"} 1 6000000
112
# TYPE test_metric2 untyped
113
test_metric2{foo="boo",instance="i"} 1 6000000
114
`,
115
	},
116
	"two matchers": {
117
		params: "match[]=test_metric1&match[]=test_metric2",
118
		code:   http.StatusOK,
119
		body: `# TYPE test_metric1 untyped
120
test_metric1{foo="bar",instance="i"} 10000 6000000
121
test_metric1{foo="boo",instance="i"} 1 6000000
122
# TYPE test_metric2 untyped
123
test_metric2{foo="boo",instance="i"} 1 6000000
124
`,
125
	},
126
	"two matchers with overlap": {
127
		params: "match[]={__name__=~'test_metric1'}&match[]={foo='bar'}",
128
		code:   http.StatusOK,
129
		body: `# TYPE test_metric1 untyped
130
test_metric1{foo="bar",instance="i"} 10000 6000000
131
test_metric1{foo="boo",instance="i"} 1 6000000
132
`,
133
	},
134
	"everything": {
135
		params: "match[]={__name__=~'.%2b'}", // '%2b' is an URL-encoded '+'.
136
		code:   http.StatusOK,
137
		body: `# TYPE test_metric1 untyped
138
test_metric1{foo="bar",instance="i"} 10000 6000000
139
test_metric1{foo="boo",instance="i"} 1 6000000
140
# TYPE test_metric2 untyped
141
test_metric2{foo="boo",instance="i"} 1 6000000
142
# TYPE test_metric_old untyped
143
test_metric_old{instance=""} 981 5880000
144
# TYPE test_metric_without_labels untyped
145
test_metric_without_labels{instance=""} 1001 6000000
146
`,
147
	},
148
	"empty label value matches everything that doesn't have that label": {
149
		params: "match[]={foo='',__name__=~'.%2b'}",
150
		code:   http.StatusOK,
151
		body: `# TYPE test_metric_old untyped
152
test_metric_old{instance=""} 981 5880000
153
# TYPE test_metric_without_labels untyped
154
test_metric_without_labels{instance=""} 1001 6000000
155
`,
156
	},
157
	"empty label value for a label that doesn't exist at all, matches everything": {
158
		params: "match[]={bar='',__name__=~'.%2b'}",
159
		code:   http.StatusOK,
160
		body: `# TYPE test_metric1 untyped
161
test_metric1{foo="bar",instance="i"} 10000 6000000
162
test_metric1{foo="boo",instance="i"} 1 6000000
163
# TYPE test_metric2 untyped
164
test_metric2{foo="boo",instance="i"} 1 6000000
165
# TYPE test_metric_old untyped
166
test_metric_old{instance=""} 981 5880000
167
# TYPE test_metric_without_labels untyped
168
test_metric_without_labels{instance=""} 1001 6000000
169
`,
170
	},
171
	"external labels are added if not already present": {
172
		params:         "match[]={__name__=~'.%2b'}", // '%2b' is an URL-encoded '+'.
173
		externalLabels: labels.FromStrings("foo", "baz", "zone", "ie"),
174
		code:           http.StatusOK,
175
		body: `# TYPE test_metric1 untyped
176
test_metric1{foo="bar",instance="i",zone="ie"} 10000 6000000
177
test_metric1{foo="boo",instance="i",zone="ie"} 1 6000000
178
# TYPE test_metric2 untyped
179
test_metric2{foo="boo",instance="i",zone="ie"} 1 6000000
180
# TYPE test_metric_old untyped
181
test_metric_old{foo="baz",instance="",zone="ie"} 981 5880000
182
# TYPE test_metric_without_labels untyped
183
test_metric_without_labels{foo="baz",instance="",zone="ie"} 1001 6000000
184
`,
185
	},
186
	"instance is an external label": {
187
		// This makes no sense as a configuration, but we should
188
		// know what it does anyway.
189
		params:         "match[]={__name__=~'.%2b'}", // '%2b' is an URL-encoded '+'.
190
		externalLabels: labels.FromStrings("instance", "baz"),
191
		code:           http.StatusOK,
192
		body: `# TYPE test_metric1 untyped
193
test_metric1{foo="bar",instance="i"} 10000 6000000
194
test_metric1{foo="boo",instance="i"} 1 6000000
195
# TYPE test_metric2 untyped
196
test_metric2{foo="boo",instance="i"} 1 6000000
197
# TYPE test_metric_old untyped
198
test_metric_old{instance="baz"} 981 5880000
199
# TYPE test_metric_without_labels untyped
200
test_metric_without_labels{instance="baz"} 1001 6000000
201
`,
202
	},
203
}
204

205
func TestFederation(t *testing.T) {
206
	storage := promqltest.LoadedStorage(t, `
207
		load 1m
208
			test_metric1{foo="bar",instance="i"}    0+100x100
209
			test_metric1{foo="boo",instance="i"}    1+0x100
210
			test_metric2{foo="boo",instance="i"}    1+0x100
211
			test_metric_without_labels 1+10x100
212
			test_metric_stale                       1+10x99 stale
213
			test_metric_old                         1+10x98
214
	`)
215
	t.Cleanup(func() { storage.Close() })
216

217
	h := &Handler{
218
		localStorage:  &dbAdapter{storage.DB},
219
		lookbackDelta: 5 * time.Minute,
220
		now:           func() model.Time { return 101 * 60 * 1000 }, // 101min after epoch.
221
		config: &config.Config{
222
			GlobalConfig: config.GlobalConfig{},
223
		},
224
	}
225

226
	for name, scenario := range scenarios {
227
		t.Run(name, func(t *testing.T) {
228
			h.config.GlobalConfig.ExternalLabels = scenario.externalLabels
229
			req := httptest.NewRequest(http.MethodGet, "http://example.org/federate?"+scenario.params, nil)
230
			res := httptest.NewRecorder()
231

232
			h.federation(res, req)
233
			require.Equal(t, scenario.code, res.Code)
234
			require.Equal(t, scenario.body, normalizeBody(res.Body))
235
		})
236
	}
237
}
238

239
type notReadyReadStorage struct {
240
	LocalStorage
241
}
242

243
func (notReadyReadStorage) Querier(int64, int64) (storage.Querier, error) {
244
	return nil, fmt.Errorf("wrap: %w", tsdb.ErrNotReady)
245
}
246

247
func (notReadyReadStorage) StartTime() (int64, error) {
248
	return 0, fmt.Errorf("wrap: %w", tsdb.ErrNotReady)
249
}
250

251
func (notReadyReadStorage) Stats(string, int) (*tsdb.Stats, error) {
252
	return nil, fmt.Errorf("wrap: %w", tsdb.ErrNotReady)
253
}
254

255
// Regression test for https://github.com/prometheus/prometheus/issues/7181.
256
func TestFederation_NotReady(t *testing.T) {
257
	for name, scenario := range scenarios {
258
		t.Run(name, func(t *testing.T) {
259
			h := &Handler{
260
				localStorage:  notReadyReadStorage{},
261
				lookbackDelta: 5 * time.Minute,
262
				now:           func() model.Time { return 101 * 60 * 1000 }, // 101min after epoch.
263
				config: &config.Config{
264
					GlobalConfig: config.GlobalConfig{
265
						ExternalLabels: scenario.externalLabels,
266
					},
267
				},
268
			}
269

270
			req := httptest.NewRequest(http.MethodGet, "http://example.org/federate?"+scenario.params, nil)
271
			res := httptest.NewRecorder()
272

273
			h.federation(res, req)
274
			if scenario.code == http.StatusBadRequest {
275
				// Request are expected to be checked before DB readiness.
276
				require.Equal(t, http.StatusBadRequest, res.Code)
277
				return
278
			}
279
			require.Equal(t, http.StatusServiceUnavailable, res.Code)
280
		})
281
	}
282
}
283

284
// normalizeBody sorts the lines within a metric to make it easy to verify the body.
285
// (Federation is not taking care of sorting within a metric family.)
286
func normalizeBody(body *bytes.Buffer) string {
287
	var (
288
		lines    []string
289
		lastHash int
290
	)
291
	for line, err := body.ReadString('\n'); err == nil; line, err = body.ReadString('\n') {
292
		if line[0] == '#' && len(lines) > 0 {
293
			sort.Strings(lines[lastHash+1:])
294
			lastHash = len(lines)
295
		}
296
		lines = append(lines, line)
297
	}
298
	if len(lines) > 0 {
299
		sort.Strings(lines[lastHash+1:])
300
	}
301
	return strings.Join(lines, "")
302
}
303

304
func TestFederationWithNativeHistograms(t *testing.T) {
305
	storage := teststorage.New(t)
306
	t.Cleanup(func() { storage.Close() })
307

308
	var expVec promql.Vector
309

310
	db := storage.DB
311
	hist := &histogram.Histogram{
312
		Count:         12,
313
		ZeroCount:     2,
314
		ZeroThreshold: 0.001,
315
		Sum:           39.4,
316
		Schema:        1,
317
		PositiveSpans: []histogram.Span{
318
			{Offset: 0, Length: 2},
319
			{Offset: 1, Length: 2},
320
		},
321
		PositiveBuckets: []int64{1, 1, -1, 0},
322
		NegativeSpans: []histogram.Span{
323
			{Offset: 0, Length: 2},
324
			{Offset: 1, Length: 2},
325
		},
326
		NegativeBuckets: []int64{1, 1, -1, 0},
327
	}
328
	histWithoutZeroBucket := &histogram.Histogram{
329
		Count:  20,
330
		Sum:    99.23,
331
		Schema: 1,
332
		PositiveSpans: []histogram.Span{
333
			{Offset: 0, Length: 2},
334
			{Offset: 1, Length: 2},
335
		},
336
		PositiveBuckets: []int64{2, 2, -2, 0},
337
		NegativeSpans: []histogram.Span{
338
			{Offset: 0, Length: 2},
339
			{Offset: 1, Length: 2},
340
		},
341
		NegativeBuckets: []int64{2, 2, -2, 0},
342
	}
343
	app := db.Appender(context.Background())
344
	for i := 0; i < 6; i++ {
345
		l := labels.FromStrings("__name__", "test_metric", "foo", strconv.Itoa(i))
346
		expL := labels.FromStrings("__name__", "test_metric", "instance", "", "foo", strconv.Itoa(i))
347
		var err error
348
		switch i {
349
		case 0, 3:
350
			_, err = app.Append(0, l, 100*60*1000, float64(i*100))
351
			expVec = append(expVec, promql.Sample{
352
				T:      100 * 60 * 1000,
353
				F:      float64(i * 100),
354
				Metric: expL,
355
			})
356
		case 4:
357
			_, err = app.AppendHistogram(0, l, 100*60*1000, histWithoutZeroBucket.Copy(), nil)
358
			expVec = append(expVec, promql.Sample{
359
				T:      100 * 60 * 1000,
360
				H:      histWithoutZeroBucket.ToFloat(nil),
361
				Metric: expL,
362
			})
363
		default:
364
			hist.ZeroCount++
365
			hist.Count++
366
			_, err = app.AppendHistogram(0, l, 100*60*1000, hist.Copy(), nil)
367
			expVec = append(expVec, promql.Sample{
368
				T:      100 * 60 * 1000,
369
				H:      hist.ToFloat(nil),
370
				Metric: expL,
371
			})
372
		}
373
		require.NoError(t, err)
374
	}
375
	require.NoError(t, app.Commit())
376

377
	h := &Handler{
378
		localStorage:  &dbAdapter{db},
379
		lookbackDelta: 5 * time.Minute,
380
		now:           func() model.Time { return 101 * 60 * 1000 }, // 101min after epoch.
381
		config: &config.Config{
382
			GlobalConfig: config.GlobalConfig{},
383
		},
384
	}
385

386
	req := httptest.NewRequest(http.MethodGet, "http://example.org/federate?match[]=test_metric", nil)
387
	req.Header.Add("Accept", `application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited,application/openmetrics-text;version=1.0.0;q=0.8,application/openmetrics-text;version=0.0.1;q=0.75,text/plain;version=0.0.4;q=0.5,*/*;q=0.1`)
388
	res := httptest.NewRecorder()
389

390
	h.federation(res, req)
391

392
	require.Equal(t, http.StatusOK, res.Code)
393
	body, err := io.ReadAll(res.Body)
394
	require.NoError(t, err)
395
	p := textparse.NewProtobufParser(body, false, labels.NewSymbolTable())
396
	var actVec promql.Vector
397
	metricFamilies := 0
398
	l := labels.Labels{}
399
	for {
400
		et, err := p.Next()
401
		if err != nil && errors.Is(err, io.EOF) {
402
			break
403
		}
404
		require.NoError(t, err)
405
		if et == textparse.EntryHistogram || et == textparse.EntrySeries {
406
			p.Metric(&l)
407
		}
408
		switch et {
409
		case textparse.EntryHelp:
410
			metricFamilies++
411
		case textparse.EntryHistogram:
412
			_, parsedTimestamp, h, fh := p.Histogram()
413
			require.Nil(t, h)
414
			actVec = append(actVec, promql.Sample{
415
				T:      *parsedTimestamp,
416
				H:      fh,
417
				Metric: l,
418
			})
419
		case textparse.EntrySeries:
420
			_, parsedTimestamp, f := p.Series()
421
			actVec = append(actVec, promql.Sample{
422
				T:      *parsedTimestamp,
423
				F:      f,
424
				Metric: l,
425
			})
426
		}
427
	}
428

429
	// TODO(codesome): Once PromQL is able to set the CounterResetHint on histograms,
430
	// test it with switching histogram types for metric families.
431
	require.Equal(t, 4, metricFamilies)
432
	testutil.RequireEqual(t, expVec, actVec)
433
}
434