prometheus

1
// Copyright 2017 The Prometheus Authors
2
// Licensed under the Apache License, Version 2.0 (the "License");
3
// you may not use this file except in compliance with the License.
4
// You may obtain a copy of the License at
5
//
6
// http://www.apache.org/licenses/LICENSE-2.0
7
//
8
// Unless required by applicable law or agreed to in writing, software
9
// distributed under the License is distributed on an "AS IS" BASIS,
10
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11
// See the License for the specific language governing permissions and
12
// limitations under the License.
13

14
package storage
15

16
import (
17
	"math/rand"
18
	"testing"
19

20
	"github.com/stretchr/testify/require"
21

22
	"github.com/prometheus/prometheus/model/histogram"
23
	"github.com/prometheus/prometheus/tsdb/chunkenc"
24
	"github.com/prometheus/prometheus/tsdb/tsdbutil"
25
)
26

27
func TestSampleRing(t *testing.T) {
28
	cases := []struct {
29
		input []int64
30
		delta int64
31
		size  int
32
	}{
33
		{
34
			input: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
35
			delta: 2,
36
			size:  1,
37
		},
38
		{
39
			input: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
40
			delta: 2,
41
			size:  2,
42
		},
43
		{
44
			input: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
45
			delta: 7,
46
			size:  3,
47
		},
48
		{
49
			input: []int64{1, 2, 3, 4, 5, 16, 17, 18, 19, 20},
50
			delta: 7,
51
			size:  1,
52
		},
53
		{
54
			input: []int64{1, 2, 3, 4, 6},
55
			delta: 4,
56
			size:  4,
57
		},
58
	}
59
	for _, c := range cases {
60
		r := newSampleRing(c.delta, c.size, chunkenc.ValFloat)
61

62
		input := []fSample{}
63
		for _, t := range c.input {
64
			input = append(input, fSample{
65
				t: t,
66
				f: float64(rand.Intn(100)),
67
			})
68
		}
69

70
		for i, s := range input {
71
			r.add(s)
72
			buffered := r.samples()
73

74
			for _, sold := range input[:i] {
75
				found := false
76
				for _, bs := range buffered {
77
					if bs.T() == sold.t && bs.F() == sold.f {
78
						found = true
79
						break
80
					}
81
				}
82

83
				if found {
84
					require.GreaterOrEqual(t, sold.t, s.t-c.delta, "%d: unexpected sample %d in buffer; buffer %v", i, sold.t, buffered)
85
				} else {
86
					require.Less(t, sold.t, s.t-c.delta, "%d: expected sample %d to be in buffer but was not; buffer %v", i, sold.t, buffered)
87
				}
88
			}
89
		}
90
	}
91
}
92

93
func TestSampleRingMixed(t *testing.T) {
94
	h1 := tsdbutil.GenerateTestHistogram(1)
95
	h2 := tsdbutil.GenerateTestHistogram(2)
96

97
	// With ValNone as the preferred type, nothing should be initialized.
98
	r := newSampleRing(10, 2, chunkenc.ValNone)
99
	require.Zero(t, len(r.fBuf))
100
	require.Zero(t, len(r.hBuf))
101
	require.Zero(t, len(r.fhBuf))
102
	require.Zero(t, len(r.iBuf))
103

104
	// But then mixed adds should work as expected.
105
	r.addF(fSample{t: 1, f: 3.14})
106
	r.addH(hSample{t: 2, h: h1})
107

108
	it := r.iterator()
109

110
	require.Equal(t, chunkenc.ValFloat, it.Next())
111
	ts, f := it.At()
112
	require.Equal(t, int64(1), ts)
113
	require.Equal(t, 3.14, f)
114
	require.Equal(t, chunkenc.ValHistogram, it.Next())
115
	var h *histogram.Histogram
116
	ts, h = it.AtHistogram()
117
	require.Equal(t, int64(2), ts)
118
	require.Equal(t, h1, h)
119
	require.Equal(t, chunkenc.ValNone, it.Next())
120

121
	r.reset()
122
	it = r.iterator()
123
	require.Equal(t, chunkenc.ValNone, it.Next())
124

125
	r.addF(fSample{t: 3, f: 4.2})
126
	r.addH(hSample{t: 4, h: h2})
127

128
	it = r.iterator()
129

130
	require.Equal(t, chunkenc.ValFloat, it.Next())
131
	ts, f = it.At()
132
	require.Equal(t, int64(3), ts)
133
	require.Equal(t, 4.2, f)
134
	require.Equal(t, chunkenc.ValHistogram, it.Next())
135
	ts, h = it.AtHistogram()
136
	require.Equal(t, int64(4), ts)
137
	require.Equal(t, h2, h)
138
	require.Equal(t, chunkenc.ValNone, it.Next())
139
}
140

141
func TestSampleRingAtFloatHistogram(t *testing.T) {
142
	fh1 := tsdbutil.GenerateTestFloatHistogram(1)
143
	fh2 := tsdbutil.GenerateTestFloatHistogram(2)
144
	h1 := tsdbutil.GenerateTestHistogram(3)
145
	h2 := tsdbutil.GenerateTestHistogram(4)
146

147
	// With ValNone as the preferred type, nothing should be initialized.
148
	r := newSampleRing(10, 2, chunkenc.ValNone)
149
	require.Zero(t, len(r.fBuf))
150
	require.Zero(t, len(r.hBuf))
151
	require.Zero(t, len(r.fhBuf))
152
	require.Zero(t, len(r.iBuf))
153

154
	var (
155
		h  *histogram.Histogram
156
		fh *histogram.FloatHistogram
157
		ts int64
158
	)
159

160
	it := r.iterator()
161
	require.Equal(t, chunkenc.ValNone, it.Next())
162

163
	r.addFH(fhSample{t: 1, fh: fh1})
164
	r.addFH(fhSample{t: 2, fh: fh2})
165

166
	it = r.iterator()
167

168
	require.Equal(t, chunkenc.ValFloatHistogram, it.Next())
169
	ts, fh = it.AtFloatHistogram(fh)
170
	require.Equal(t, int64(1), ts)
171
	require.Equal(t, fh1, fh)
172
	require.Equal(t, chunkenc.ValFloatHistogram, it.Next())
173
	ts, fh = it.AtFloatHistogram(fh)
174
	require.Equal(t, int64(2), ts)
175
	require.Equal(t, fh2, fh)
176
	require.Equal(t, chunkenc.ValNone, it.Next())
177

178
	r.reset()
179
	it = r.iterator()
180
	require.Equal(t, chunkenc.ValNone, it.Next())
181

182
	r.addH(hSample{t: 3, h: h1})
183
	r.addH(hSample{t: 4, h: h2})
184

185
	it = r.iterator()
186

187
	require.Equal(t, chunkenc.ValHistogram, it.Next())
188
	ts, h = it.AtHistogram()
189
	require.Equal(t, int64(3), ts)
190
	require.Equal(t, h1, h)
191
	ts, fh = it.AtFloatHistogram(fh)
192
	require.Equal(t, int64(3), ts)
193
	require.Equal(t, h1.ToFloat(nil), fh)
194
	require.Equal(t, chunkenc.ValHistogram, it.Next())
195
	ts, h = it.AtHistogram()
196
	require.Equal(t, int64(4), ts)
197
	require.Equal(t, h2, h)
198
	ts, fh = it.AtFloatHistogram(fh)
199
	require.Equal(t, int64(4), ts)
200
	require.Equal(t, h2.ToFloat(nil), fh)
201
	require.Equal(t, chunkenc.ValNone, it.Next())
202
}
203

204
func TestBufferedSeriesIterator(t *testing.T) {
205
	var it *BufferedSeriesIterator
206

207
	bufferEq := func(exp []fSample) {
208
		var b []fSample
209
		bit := it.Buffer()
210
		for bit.Next() == chunkenc.ValFloat {
211
			t, f := bit.At()
212
			b = append(b, fSample{t: t, f: f})
213
		}
214
		require.Equal(t, exp, b, "buffer mismatch")
215
	}
216
	sampleEq := func(ets int64, ev float64) {
217
		ts, v := it.At()
218
		require.Equal(t, ets, ts, "timestamp mismatch")
219
		require.Equal(t, ev, v, "value mismatch")
220
	}
221
	prevSampleEq := func(ets int64, ev float64, eok bool) {
222
		s, ok := it.PeekBack(1)
223
		require.Equal(t, eok, ok, "exist mismatch")
224
		require.Equal(t, ets, s.T(), "timestamp mismatch")
225
		require.Equal(t, ev, s.F(), "value mismatch")
226
	}
227

228
	it = NewBufferIterator(NewListSeriesIterator(samples{
229
		fSample{t: 1, f: 2},
230
		fSample{t: 2, f: 3},
231
		fSample{t: 3, f: 4},
232
		fSample{t: 4, f: 5},
233
		fSample{t: 5, f: 6},
234
		fSample{t: 99, f: 8},
235
		fSample{t: 100, f: 9},
236
		fSample{t: 101, f: 10},
237
	}), 2)
238

239
	require.Equal(t, chunkenc.ValFloat, it.Seek(-123), "seek failed")
240
	sampleEq(1, 2)
241
	prevSampleEq(0, 0, false)
242
	bufferEq(nil)
243

244
	require.Equal(t, chunkenc.ValFloat, it.Next(), "next failed")
245
	sampleEq(2, 3)
246
	prevSampleEq(1, 2, true)
247
	bufferEq([]fSample{{t: 1, f: 2}})
248

249
	require.Equal(t, chunkenc.ValFloat, it.Next(), "next failed")
250
	require.Equal(t, chunkenc.ValFloat, it.Next(), "next failed")
251
	require.Equal(t, chunkenc.ValFloat, it.Next(), "next failed")
252
	sampleEq(5, 6)
253
	prevSampleEq(4, 5, true)
254
	bufferEq([]fSample{{t: 2, f: 3}, {t: 3, f: 4}, {t: 4, f: 5}})
255

256
	require.Equal(t, chunkenc.ValFloat, it.Seek(5), "seek failed")
257
	sampleEq(5, 6)
258
	prevSampleEq(4, 5, true)
259
	bufferEq([]fSample{{t: 2, f: 3}, {t: 3, f: 4}, {t: 4, f: 5}})
260

261
	require.Equal(t, chunkenc.ValFloat, it.Seek(101), "seek failed")
262
	sampleEq(101, 10)
263
	prevSampleEq(100, 9, true)
264
	bufferEq([]fSample{{t: 99, f: 8}, {t: 100, f: 9}})
265

266
	require.Equal(t, chunkenc.ValNone, it.Next(), "next succeeded unexpectedly")
267
	require.Equal(t, chunkenc.ValNone, it.Seek(1024), "seek succeeded unexpectedly")
268
}
269

270
// At() should not be called once Next() returns false.
271
func TestBufferedSeriesIteratorNoBadAt(t *testing.T) {
272
	done := false
273

274
	m := &mockSeriesIterator{
275
		seek: func(int64) chunkenc.ValueType { return chunkenc.ValNone },
276
		at: func() (int64, float64) {
277
			require.False(t, done, "unexpectedly done")
278
			done = true
279
			return 0, 0
280
		},
281
		next: func() chunkenc.ValueType {
282
			if done {
283
				return chunkenc.ValNone
284
			}
285
			return chunkenc.ValFloat
286
		},
287
		err: func() error { return nil },
288
	}
289

290
	it := NewBufferIterator(m, 60)
291
	it.Next()
292
	it.Next()
293
}
294

295
func TestBufferedSeriesIteratorMixedHistograms(t *testing.T) {
296
	histograms := tsdbutil.GenerateTestHistograms(2)
297

298
	it := NewBufferIterator(NewListSeriesIterator(samples{
299
		fhSample{t: 1, fh: histograms[0].ToFloat(nil)},
300
		hSample{t: 2, h: histograms[1]},
301
	}), 2)
302

303
	require.Equal(t, chunkenc.ValNone, it.Seek(3))
304
	require.NoError(t, it.Err())
305

306
	buf := it.Buffer()
307

308
	require.Equal(t, chunkenc.ValFloatHistogram, buf.Next())
309
	_, fh := buf.AtFloatHistogram(nil)
310
	require.Equal(t, histograms[0].ToFloat(nil), fh)
311

312
	require.Equal(t, chunkenc.ValHistogram, buf.Next())
313
	_, fh = buf.AtFloatHistogram(nil)
314
	require.Equal(t, histograms[1].ToFloat(nil), fh)
315
}
316

317
func BenchmarkBufferedSeriesIterator(b *testing.B) {
318
	// Simulate a 5 minute rate.
319
	it := NewBufferIterator(newFakeSeriesIterator(int64(b.N), 30), 5*60)
320

321
	b.SetBytes(16)
322
	b.ReportAllocs()
323
	b.ResetTimer()
324

325
	for it.Next() != chunkenc.ValNone {
326
		// Scan everything.
327
	}
328
	require.NoError(b, it.Err())
329
}
330

331
type mockSeriesIterator struct {
332
	seek func(int64) chunkenc.ValueType
333
	at   func() (int64, float64)
334
	next func() chunkenc.ValueType
335
	err  func() error
336
}
337

338
func (m *mockSeriesIterator) Seek(t int64) chunkenc.ValueType { return m.seek(t) }
339
func (m *mockSeriesIterator) At() (int64, float64)            { return m.at() }
340
func (m *mockSeriesIterator) Next() chunkenc.ValueType        { return m.next() }
341
func (m *mockSeriesIterator) Err() error                      { return m.err() }
342

343
func (m *mockSeriesIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) {
344
	return 0, nil // Not really mocked.
345
}
346

347
func (m *mockSeriesIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
348
	return 0, nil // Not really mocked.
349
}
350

351
func (m *mockSeriesIterator) AtT() int64 {
352
	return 0 // Not really mocked.
353
}
354

355
type fakeSeriesIterator struct {
356
	nsamples int64
357
	step     int64
358
	idx      int64
359
}
360

361
func newFakeSeriesIterator(nsamples, step int64) *fakeSeriesIterator {
362
	return &fakeSeriesIterator{nsamples: nsamples, step: step, idx: -1}
363
}
364

365
func (it *fakeSeriesIterator) At() (int64, float64) {
366
	return it.idx * it.step, 123 // Value doesn't matter.
367
}
368

369
func (it *fakeSeriesIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) {
370
	return it.idx * it.step, &histogram.Histogram{} // Value doesn't matter.
371
}
372

373
func (it *fakeSeriesIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
374
	return it.idx * it.step, &histogram.FloatHistogram{} // Value doesn't matter.
375
}
376

377
func (it *fakeSeriesIterator) AtT() int64 {
378
	return it.idx * it.step
379
}
380

381
func (it *fakeSeriesIterator) Next() chunkenc.ValueType {
382
	it.idx++
383
	if it.idx >= it.nsamples {
384
		return chunkenc.ValNone
385
	}
386
	return chunkenc.ValFloat
387
}
388

389
func (it *fakeSeriesIterator) Seek(t int64) chunkenc.ValueType {
390
	it.idx = t / it.step
391
	if it.idx >= it.nsamples {
392
		return chunkenc.ValNone
393
	}
394
	return chunkenc.ValFloat
395
}
396

397
func (it *fakeSeriesIterator) Err() error { return nil }
398