prometheus/storage/buffer_test.go

287 lines
7.4 KiB
Go
Raw Normal View History

// Copyright 2017 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package storage
import (
"math/rand"
"testing"
"github.com/stretchr/testify/require"
Style cleanup of all the changes in sparsehistogram so far A lot of this code was hacked together, literally during a hackathon. This commit intends not to change the code substantially, but just make the code obey the usual style practices. A (possibly incomplete) list of areas: * Generally address linter warnings. * The `pgk` directory is deprecated as per dev-summit. No new packages should be added to it. I moved the new `pkg/histogram` package to `model` anticipating what's proposed in #9478. * Make the naming of the Sparse Histogram more consistent. Including abbreviations, there were just too many names for it: SparseHistogram, Histogram, Histo, hist, his, shs, h. The idea is to call it "Histogram" in general. Only add "Sparse" if it is needed to avoid confusion with conventional Histograms (which is rare because the TSDB really has no notion of conventional Histograms). Use abbreviations only in local scope, and then really abbreviate (not just removing three out of seven letters like in "Histo"). This is in the spirit of https://github.com/golang/go/wiki/CodeReviewComments#variable-names * Several other minor name changes. * A lot of formatting of doc comments. For one, following https://github.com/golang/go/wiki/CodeReviewComments#comment-sentences , but also layout question, anticipating how things will look like when rendered by `godoc` (even where `godoc` doesn't render them right now because they are for unexported types or not a doc comment at all but just a normal code comment - consistency is queen!). * Re-enabled `TestQueryLog` and `TestEndopints` (they pass now, leaving them disabled was presumably an oversight). * Bucket iterator for histogram.Histogram is now created with a method. * HistogramChunk.iterator now allows iterator recycling. (I think @dieterbe only commented it out because he was confused by the question in the comment.) * HistogramAppender.Append panics now because we decided to treat staleness marker differently. Signed-off-by: beorn7 <beorn@grafana.com>
2021-10-09 06:57:07 -07:00
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/tsdb/chunkenc"
"github.com/prometheus/prometheus/tsdb/tsdbutil"
)
func TestSampleRing(t *testing.T) {
cases := []struct {
input []int64
delta int64
size int
}{
{
input: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
delta: 2,
size: 1,
},
{
input: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
delta: 2,
size: 2,
},
{
input: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
delta: 7,
size: 3,
},
{
input: []int64{1, 2, 3, 4, 5, 16, 17, 18, 19, 20},
delta: 7,
size: 1,
},
{
input: []int64{1, 2, 3, 4, 6},
delta: 4,
size: 4,
},
}
for _, c := range cases {
r := newSampleRing(c.delta, c.size, chunkenc.ValFloat)
input := []fSample{}
for _, t := range c.input {
input = append(input, fSample{
t: t,
f: float64(rand.Intn(100)),
})
}
for i, s := range input {
r.add(s)
buffered := r.samples()
for _, sold := range input[:i] {
found := false
for _, bs := range buffered {
if bs.T() == sold.t && bs.F() == sold.f {
found = true
break
}
}
if found {
require.GreaterOrEqual(t, sold.t, s.t-c.delta, "%d: unexpected sample %d in buffer; buffer %v", i, sold.t, buffered)
} else {
require.Less(t, sold.t, s.t-c.delta, "%d: expected sample %d to be in buffer but was not; buffer %v", i, sold.t, buffered)
}
}
}
}
}
func TestBufferedSeriesIterator(t *testing.T) {
var it *BufferedSeriesIterator
bufferEq := func(exp []fSample) {
var b []fSample
bit := it.Buffer()
for bit.Next() == chunkenc.ValFloat {
t, f := bit.At()
b = append(b, fSample{t: t, f: f})
}
require.Equal(t, exp, b, "buffer mismatch")
}
sampleEq := func(ets int64, ev float64) {
ts, v := it.At()
require.Equal(t, ets, ts, "timestamp mismatch")
require.Equal(t, ev, v, "value mismatch")
}
prevSampleEq := func(ets int64, ev float64, eok bool) {
s, ok := it.PeekBack(1)
require.Equal(t, eok, ok, "exist mismatch")
require.Equal(t, ets, s.T(), "timestamp mismatch")
require.Equal(t, ev, s.F(), "value mismatch")
}
it = NewBufferIterator(NewListSeriesIterator(samples{
fSample{t: 1, f: 2},
fSample{t: 2, f: 3},
fSample{t: 3, f: 4},
fSample{t: 4, f: 5},
fSample{t: 5, f: 6},
fSample{t: 99, f: 8},
fSample{t: 100, f: 9},
fSample{t: 101, f: 10},
}), 2)
require.Equal(t, chunkenc.ValFloat, it.Seek(-123), "seek failed")
sampleEq(1, 2)
prevSampleEq(0, 0, false)
bufferEq(nil)
require.Equal(t, chunkenc.ValFloat, it.Next(), "next failed")
sampleEq(2, 3)
prevSampleEq(1, 2, true)
bufferEq([]fSample{{t: 1, f: 2}})
require.Equal(t, chunkenc.ValFloat, it.Next(), "next failed")
require.Equal(t, chunkenc.ValFloat, it.Next(), "next failed")
require.Equal(t, chunkenc.ValFloat, it.Next(), "next failed")
sampleEq(5, 6)
prevSampleEq(4, 5, true)
bufferEq([]fSample{{t: 2, f: 3}, {t: 3, f: 4}, {t: 4, f: 5}})
require.Equal(t, chunkenc.ValFloat, it.Seek(5), "seek failed")
sampleEq(5, 6)
prevSampleEq(4, 5, true)
bufferEq([]fSample{{t: 2, f: 3}, {t: 3, f: 4}, {t: 4, f: 5}})
require.Equal(t, chunkenc.ValFloat, it.Seek(101), "seek failed")
sampleEq(101, 10)
prevSampleEq(100, 9, true)
bufferEq([]fSample{{t: 99, f: 8}, {t: 100, f: 9}})
require.Equal(t, chunkenc.ValNone, it.Next(), "next succeeded unexpectedly")
require.Equal(t, chunkenc.ValNone, it.Seek(1024), "seek succeeded unexpectedly")
}
2016-12-30 01:45:56 -08:00
// At() should not be called once Next() returns false.
func TestBufferedSeriesIteratorNoBadAt(t *testing.T) {
done := false
m := &mockSeriesIterator{
seek: func(int64) chunkenc.ValueType { return chunkenc.ValNone },
at: func() (int64, float64) {
require.False(t, done, "unexpectedly done")
done = true
return 0, 0
},
next: func() chunkenc.ValueType {
if done {
return chunkenc.ValNone
}
return chunkenc.ValFloat
},
err: func() error { return nil },
}
it := NewBufferIterator(m, 60)
it.Next()
it.Next()
}
func TestBufferedSeriesIteratorMixedHistograms(t *testing.T) {
histograms := tsdbutil.GenerateTestHistograms(2)
it := NewBufferIterator(NewListSeriesIterator(samples{
fhSample{t: 1, fh: histograms[0].ToFloat()},
hSample{t: 2, h: histograms[1]},
}), 2)
require.Equal(t, chunkenc.ValNone, it.Seek(3))
require.NoError(t, it.Err())
buf := it.Buffer()
require.Equal(t, chunkenc.ValFloatHistogram, buf.Next())
_, fh := buf.AtFloatHistogram()
require.Equal(t, histograms[0].ToFloat(), fh)
require.Equal(t, chunkenc.ValHistogram, buf.Next())
_, fh = buf.AtFloatHistogram()
require.Equal(t, histograms[1].ToFloat(), fh)
}
func BenchmarkBufferedSeriesIterator(b *testing.B) {
// Simulate a 5 minute rate.
it := NewBufferIterator(newFakeSeriesIterator(int64(b.N), 30), 5*60)
b.SetBytes(16)
b.ReportAllocs()
b.ResetTimer()
for it.Next() != chunkenc.ValNone { // nolint:revive
// Scan everything.
}
require.NoError(b, it.Err())
}
2016-12-30 01:45:56 -08:00
type mockSeriesIterator struct {
seek func(int64) chunkenc.ValueType
at func() (int64, float64)
next func() chunkenc.ValueType
err func() error
2016-12-30 01:45:56 -08:00
}
func (m *mockSeriesIterator) Seek(t int64) chunkenc.ValueType { return m.seek(t) }
func (m *mockSeriesIterator) At() (int64, float64) { return m.at() }
func (m *mockSeriesIterator) Next() chunkenc.ValueType { return m.next() }
func (m *mockSeriesIterator) Err() error { return m.err() }
func (m *mockSeriesIterator) AtHistogram() (int64, *histogram.Histogram) {
return 0, nil // Not really mocked.
}
func (m *mockSeriesIterator) AtFloatHistogram() (int64, *histogram.FloatHistogram) {
return 0, nil // Not really mocked.
}
func (m *mockSeriesIterator) AtT() int64 {
return 0 // Not really mocked.
}
2016-12-30 01:45:56 -08:00
type fakeSeriesIterator struct {
nsamples int64
step int64
idx int64
}
func newFakeSeriesIterator(nsamples, step int64) *fakeSeriesIterator {
return &fakeSeriesIterator{nsamples: nsamples, step: step, idx: -1}
}
func (it *fakeSeriesIterator) At() (int64, float64) {
return it.idx * it.step, 123 // Value doesn't matter.
}
func (it *fakeSeriesIterator) AtHistogram() (int64, *histogram.Histogram) {
return it.idx * it.step, &histogram.Histogram{} // Value doesn't matter.
}
func (it *fakeSeriesIterator) AtFloatHistogram() (int64, *histogram.FloatHistogram) {
return it.idx * it.step, &histogram.FloatHistogram{} // Value doesn't matter.
}
func (it *fakeSeriesIterator) AtT() int64 {
return it.idx * it.step
}
func (it *fakeSeriesIterator) Next() chunkenc.ValueType {
it.idx++
if it.idx >= it.nsamples {
return chunkenc.ValNone
}
return chunkenc.ValFloat
}
func (it *fakeSeriesIterator) Seek(t int64) chunkenc.ValueType {
it.idx = t / it.step
if it.idx >= it.nsamples {
return chunkenc.ValNone
}
return chunkenc.ValFloat
}
storage: Added Chunks{Queryable/Querier/SeriesSet/Series/Iteratable. Added generic Merge{SeriesSet/Querier} implementation. (#7005) * storage: Added Chunks{Queryable/Querier/SeriesSet/Series/Iteratable. Added generic Merge{SeriesSet/Querier} implementation. ## Rationales: In many places (e.g. chunk Remote read, Thanos Receive fetching chunk from TSDB), we operate on encoded chunks not samples. This means that we unnecessary decode/encode, wasting CPU, time and memory. This PR adds chunk iterator interfaces and makes the merge code to be reused between both seriesSets I will make the use of it in following PR inside tsdb itself. For now fanout implements it and mergers. All merges now also allows passing series mergers. This opens doors for custom deduplications other than TSDB vertical ones (e.g. offline one we have in Thanos). ## Changes * Added Chunk versions of all iterating methods. It all starts in Querier/ChunkQuerier. The plan is that Storage will implement both chunked and samples. * Added Seek to chunks.Iterator interface for iterating over chunks. * NewMergeChunkQuerier was added; Both this and NewMergeQuerier are now using generigMergeQuerier to share the code. Generic code was added. * Improved tests. * Added some TODO for further simplifications in next PRs. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Addressed Brian's comments. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Moved s/Labeled/SeriesLabels as per Krasi suggestion. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Addressed Krasi's comments. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Second iteration of Krasi comments. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Another round of comments. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
2020-03-24 13:15:47 -07:00
func (it *fakeSeriesIterator) Err() error { return nil }