diff --git a/storage/buffer.go b/storage/buffer.go index 38f559103..b1b5f8148 100644 --- a/storage/buffer.go +++ b/storage/buffer.go @@ -19,7 +19,7 @@ import ( "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/tsdb/chunkenc" - "github.com/prometheus/prometheus/tsdb/tsdbutil" + "github.com/prometheus/prometheus/tsdb/chunks" ) // BufferedSeriesIterator wraps an iterator with a look-back buffer. @@ -69,7 +69,7 @@ func (b *BufferedSeriesIterator) ReduceDelta(delta int64) bool { // PeekBack returns the nth previous element of the iterator. If there is none buffered, // ok is false. -func (b *BufferedSeriesIterator) PeekBack(n int) (sample tsdbutil.Sample, ok bool) { +func (b *BufferedSeriesIterator) PeekBack(n int) (sample chunks.Sample, ok bool) { return b.buf.nthLast(n) } @@ -247,7 +247,7 @@ type sampleRing struct { // allowed to be populated!) This avoids the overhead of the interface // wrapper for the happy (and by far most common) case of homogenous // samples. - iBuf []tsdbutil.Sample + iBuf []chunks.Sample fBuf []fSample hBuf []hSample fhBuf []fhSample @@ -289,7 +289,7 @@ func newSampleRing(delta int64, size int, typ chunkenc.ValueType) *sampleRing { case chunkenc.ValFloatHistogram: r.fhBuf = make([]fhSample, size) default: - r.iBuf = make([]tsdbutil.Sample, size) + r.iBuf = make([]chunks.Sample, size) } return r } @@ -383,7 +383,7 @@ func (it *sampleRingIterator) AtT() int64 { return it.t } -func (r *sampleRing) at(i int) tsdbutil.Sample { +func (r *sampleRing) at(i int) chunks.Sample { j := (r.f + i) % len(r.iBuf) return r.iBuf[j] } @@ -408,7 +408,7 @@ func (r *sampleRing) atFH(i int) fhSample { // implementation. If you know you are dealing with one of the implementations // from this package (fSample, hSample, fhSample), call one of the specialized // methods addF, addH, or addFH for better performance. -func (r *sampleRing) add(s tsdbutil.Sample) { +func (r *sampleRing) add(s chunks.Sample) { if r.bufInUse == noBuf { // First sample. switch s := s.(type) { @@ -519,7 +519,7 @@ func (r *sampleRing) addFH(s fhSample) { } } -// genericAdd is a generic implementation of adding a tsdbutil.Sample +// genericAdd is a generic implementation of adding a chunks.Sample // implementation to a buffer of a sample ring. However, the Go compiler // currently (go1.20) decides to not expand the code during compile time, but // creates dynamic code to handle the different types. That has a significant @@ -529,7 +529,7 @@ func (r *sampleRing) addFH(s fhSample) { // Therefore, genericAdd has been manually implemented for all the types // (addSample, addF, addH, addFH) below. // -// func genericAdd[T tsdbutil.Sample](s T, buf []T, r *sampleRing) []T { +// func genericAdd[T chunks.Sample](s T, buf []T, r *sampleRing) []T { // l := len(buf) // // Grow the ring buffer if it fits no more elements. // if l == 0 { @@ -568,15 +568,15 @@ func (r *sampleRing) addFH(s fhSample) { // } // addSample is a handcoded specialization of genericAdd (see above). -func addSample(s tsdbutil.Sample, buf []tsdbutil.Sample, r *sampleRing) []tsdbutil.Sample { +func addSample(s chunks.Sample, buf []chunks.Sample, r *sampleRing) []chunks.Sample { l := len(buf) // Grow the ring buffer if it fits no more elements. if l == 0 { - buf = make([]tsdbutil.Sample, 16) + buf = make([]chunks.Sample, 16) l = 16 } if l == r.l { - newBuf := make([]tsdbutil.Sample, 2*l) + newBuf := make([]chunks.Sample, 2*l) copy(newBuf[l+r.f:], buf[r.f:]) copy(newBuf, buf[:r.f]) @@ -748,7 +748,7 @@ func (r *sampleRing) reduceDelta(delta int64) bool { return true } -func genericReduceDelta[T tsdbutil.Sample](buf []T, r *sampleRing) { +func genericReduceDelta[T chunks.Sample](buf []T, r *sampleRing) { // Free head of the buffer of samples that just fell out of the range. l := len(buf) tmin := buf[r.i].T() - r.delta @@ -762,7 +762,7 @@ func genericReduceDelta[T tsdbutil.Sample](buf []T, r *sampleRing) { } // nthLast returns the nth most recent element added to the ring. -func (r *sampleRing) nthLast(n int) (tsdbutil.Sample, bool) { +func (r *sampleRing) nthLast(n int) (chunks.Sample, bool) { if n > r.l { return fSample{}, false } @@ -779,8 +779,8 @@ func (r *sampleRing) nthLast(n int) (tsdbutil.Sample, bool) { } } -func (r *sampleRing) samples() []tsdbutil.Sample { - res := make([]tsdbutil.Sample, r.l) +func (r *sampleRing) samples() []chunks.Sample { + res := make([]chunks.Sample, r.l) k := r.f + r.l var j int diff --git a/storage/merge_test.go b/storage/merge_test.go index b0544c2d8..e0b938be7 100644 --- a/storage/merge_test.go +++ b/storage/merge_test.go @@ -25,6 +25,7 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/prometheus/prometheus/tsdb/chunks" "github.com/prometheus/prometheus/tsdb/tsdbutil" ) @@ -62,116 +63,116 @@ func TestMergeQuerierWithChainMerger(t *testing.T) { { name: "one querier, two series", querierSeries: [][]Series{{ - NewListSeries(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}), - NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}}), + NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}), + NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}}), }}, expected: NewMockSeriesSet( - NewListSeries(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}), - NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}}), + NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}), + NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}}), ), }, { name: "two queriers, one different series each", querierSeries: [][]Series{{ - NewListSeries(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}), + NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}), }, { - NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}}), + NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}}), }}, expected: NewMockSeriesSet( - NewListSeries(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}), - NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}}), + NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}), + NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}}), ), }, { name: "two time unsorted queriers, two series each", querierSeries: [][]Series{{ - NewListSeries(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{5, 5}, fSample{6, 6}}), - NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}}), + NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{5, 5}, fSample{6, 6}}), + NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}}), }, { - NewListSeries(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}), - NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{3, 3}, fSample{4, 4}}), + NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}), + NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{3, 3}, fSample{4, 4}}), }}, expected: NewMockSeriesSet( NewListSeries( labels.FromStrings("bar", "baz"), - []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}, fSample{6, 6}}, + []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}, fSample{6, 6}}, ), NewListSeries( labels.FromStrings("foo", "bar"), - []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{4, 4}}, + []chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{4, 4}}, ), ), }, { name: "five queriers, only two queriers have two time unsorted series each", querierSeries: [][]Series{{}, {}, { - NewListSeries(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{5, 5}, fSample{6, 6}}), - NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}}), + NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{5, 5}, fSample{6, 6}}), + NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}}), }, { - NewListSeries(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}), - NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{3, 3}, fSample{4, 4}}), + NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}), + NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{3, 3}, fSample{4, 4}}), }, {}}, expected: NewMockSeriesSet( NewListSeries( labels.FromStrings("bar", "baz"), - []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}, fSample{6, 6}}, + []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}, fSample{6, 6}}, ), NewListSeries( labels.FromStrings("foo", "bar"), - []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{4, 4}}, + []chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{4, 4}}, ), ), }, { name: "two queriers, only two queriers have two time unsorted series each, with 3 noop and one nil querier together", querierSeries: [][]Series{{}, {}, { - NewListSeries(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{5, 5}, fSample{6, 6}}), - NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}}), + NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{5, 5}, fSample{6, 6}}), + NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}}), }, { - NewListSeries(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}), - NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{3, 3}, fSample{4, 4}}), + NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}), + NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{3, 3}, fSample{4, 4}}), }, {}}, extraQueriers: []Querier{NoopQuerier(), NoopQuerier(), nil, NoopQuerier()}, expected: NewMockSeriesSet( NewListSeries( labels.FromStrings("bar", "baz"), - []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}, fSample{6, 6}}, + []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}, fSample{6, 6}}, ), NewListSeries( labels.FromStrings("foo", "bar"), - []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{4, 4}}, + []chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{4, 4}}, ), ), }, { name: "two queriers, with two series, one is overlapping", querierSeries: [][]Series{{}, {}, { - NewListSeries(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{2, 21}, fSample{3, 31}, fSample{5, 5}, fSample{6, 6}}), - NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}}), + NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{2, 21}, fSample{3, 31}, fSample{5, 5}, fSample{6, 6}}), + NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}}), }, { - NewListSeries(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 22}, fSample{3, 32}}), - NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{3, 3}, fSample{4, 4}}), + NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 22}, fSample{3, 32}}), + NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{3, 3}, fSample{4, 4}}), }, {}}, expected: NewMockSeriesSet( NewListSeries( labels.FromStrings("bar", "baz"), - []tsdbutil.Sample{fSample{1, 1}, fSample{2, 21}, fSample{3, 31}, fSample{5, 5}, fSample{6, 6}}, + []chunks.Sample{fSample{1, 1}, fSample{2, 21}, fSample{3, 31}, fSample{5, 5}, fSample{6, 6}}, ), NewListSeries( labels.FromStrings("foo", "bar"), - []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{4, 4}}, + []chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{4, 4}}, ), ), }, { name: "two queries, one with NaN samples series", querierSeries: [][]Series{{ - NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{0, math.NaN()}}), + NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, math.NaN()}}), }, { - NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{1, 1}}), + NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{1, 1}}), }}, expected: NewMockSeriesSet( - NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{0, math.NaN()}, fSample{1, 1}}), + NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, math.NaN()}, fSample{1, 1}}), ), }, } { @@ -245,108 +246,108 @@ func TestMergeChunkQuerierWithNoVerticalChunkSeriesMerger(t *testing.T) { { name: "one querier, two series", chkQuerierSeries: [][]ChunkSeries{{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}}, []tsdbutil.Sample{fSample{3, 3}}), - NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}}, []tsdbutil.Sample{fSample{2, 2}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}}), + NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}}, []chunks.Sample{fSample{2, 2}}), }}, expected: NewMockChunkSeriesSet( - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}}, []tsdbutil.Sample{fSample{3, 3}}), - NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}}, []tsdbutil.Sample{fSample{2, 2}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}}), + NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}}, []chunks.Sample{fSample{2, 2}}), ), }, { name: "two secondaries, one different series each", chkQuerierSeries: [][]ChunkSeries{{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}}, []tsdbutil.Sample{fSample{3, 3}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}}), }, { - NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}}, []tsdbutil.Sample{fSample{2, 2}}), + NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}}, []chunks.Sample{fSample{2, 2}}), }}, expected: NewMockChunkSeriesSet( - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}}, []tsdbutil.Sample{fSample{3, 3}}), - NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}}, []tsdbutil.Sample{fSample{2, 2}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}}), + NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}}, []chunks.Sample{fSample{2, 2}}), ), }, { name: "two secondaries, two not in time order series each", chkQuerierSeries: [][]ChunkSeries{{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{5, 5}}, []tsdbutil.Sample{fSample{6, 6}}), - NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}}, []tsdbutil.Sample{fSample{2, 2}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{5, 5}}, []chunks.Sample{fSample{6, 6}}), + NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}}, []chunks.Sample{fSample{2, 2}}), }, { - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}}, []tsdbutil.Sample{fSample{3, 3}}), - NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{3, 3}}, []tsdbutil.Sample{fSample{4, 4}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}}), + NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{3, 3}}, []chunks.Sample{fSample{4, 4}}), }}, expected: NewMockChunkSeriesSet( NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), - []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}}, - []tsdbutil.Sample{fSample{3, 3}}, - []tsdbutil.Sample{fSample{5, 5}}, - []tsdbutil.Sample{fSample{6, 6}}, + []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, + []chunks.Sample{fSample{3, 3}}, + []chunks.Sample{fSample{5, 5}}, + []chunks.Sample{fSample{6, 6}}, ), NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), - []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}}, - []tsdbutil.Sample{fSample{2, 2}}, - []tsdbutil.Sample{fSample{3, 3}}, - []tsdbutil.Sample{fSample{4, 4}}, + []chunks.Sample{fSample{0, 0}, fSample{1, 1}}, + []chunks.Sample{fSample{2, 2}}, + []chunks.Sample{fSample{3, 3}}, + []chunks.Sample{fSample{4, 4}}, ), ), }, { name: "five secondaries, only two have two not in time order series each", chkQuerierSeries: [][]ChunkSeries{{}, {}, { - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{5, 5}}, []tsdbutil.Sample{fSample{6, 6}}), - NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}}, []tsdbutil.Sample{fSample{2, 2}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{5, 5}}, []chunks.Sample{fSample{6, 6}}), + NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}}, []chunks.Sample{fSample{2, 2}}), }, { - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}}, []tsdbutil.Sample{fSample{3, 3}}), - NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{3, 3}}, []tsdbutil.Sample{fSample{4, 4}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}}), + NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{3, 3}}, []chunks.Sample{fSample{4, 4}}), }, {}}, expected: NewMockChunkSeriesSet( NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), - []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}}, - []tsdbutil.Sample{fSample{3, 3}}, - []tsdbutil.Sample{fSample{5, 5}}, - []tsdbutil.Sample{fSample{6, 6}}, + []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, + []chunks.Sample{fSample{3, 3}}, + []chunks.Sample{fSample{5, 5}}, + []chunks.Sample{fSample{6, 6}}, ), NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), - []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}}, - []tsdbutil.Sample{fSample{2, 2}}, - []tsdbutil.Sample{fSample{3, 3}}, - []tsdbutil.Sample{fSample{4, 4}}, + []chunks.Sample{fSample{0, 0}, fSample{1, 1}}, + []chunks.Sample{fSample{2, 2}}, + []chunks.Sample{fSample{3, 3}}, + []chunks.Sample{fSample{4, 4}}, ), ), }, { name: "two secondaries, with two not in time order series each, with 3 noop queries and one nil together", chkQuerierSeries: [][]ChunkSeries{{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{5, 5}}, []tsdbutil.Sample{fSample{6, 6}}), - NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}}, []tsdbutil.Sample{fSample{2, 2}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{5, 5}}, []chunks.Sample{fSample{6, 6}}), + NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}}, []chunks.Sample{fSample{2, 2}}), }, { - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}}, []tsdbutil.Sample{fSample{3, 3}}), - NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{3, 3}}, []tsdbutil.Sample{fSample{4, 4}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}}), + NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{3, 3}}, []chunks.Sample{fSample{4, 4}}), }}, extraQueriers: []ChunkQuerier{NoopChunkedQuerier(), NoopChunkedQuerier(), nil, NoopChunkedQuerier()}, expected: NewMockChunkSeriesSet( NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), - []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}}, - []tsdbutil.Sample{fSample{3, 3}}, - []tsdbutil.Sample{fSample{5, 5}}, - []tsdbutil.Sample{fSample{6, 6}}, + []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, + []chunks.Sample{fSample{3, 3}}, + []chunks.Sample{fSample{5, 5}}, + []chunks.Sample{fSample{6, 6}}, ), NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), - []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}}, - []tsdbutil.Sample{fSample{2, 2}}, - []tsdbutil.Sample{fSample{3, 3}}, - []tsdbutil.Sample{fSample{4, 4}}, + []chunks.Sample{fSample{0, 0}, fSample{1, 1}}, + []chunks.Sample{fSample{2, 2}}, + []chunks.Sample{fSample{3, 3}}, + []chunks.Sample{fSample{4, 4}}, ), ), }, { name: "two queries, one with NaN samples series", chkQuerierSeries: [][]ChunkSeries{{ - NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{0, math.NaN()}}), + NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, math.NaN()}}), }, { - NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{1, 1}}), + NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{1, 1}}), }}, expected: NewMockChunkSeriesSet( - NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{fSample{0, math.NaN()}}, []tsdbutil.Sample{fSample{1, 1}}), + NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, math.NaN()}}, []chunks.Sample{fSample{1, 1}}), ), }, } { @@ -408,9 +409,9 @@ func TestCompactingChunkSeriesMerger(t *testing.T) { { name: "single series", input: []ChunkSeries{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}}, []tsdbutil.Sample{fSample{3, 3}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}}), }, - expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}}, []tsdbutil.Sample{fSample{3, 3}}), + expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}}), }, { name: "two empty series", @@ -423,142 +424,142 @@ func TestCompactingChunkSeriesMerger(t *testing.T) { { name: "two non overlapping", input: []ChunkSeries{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}}, []tsdbutil.Sample{fSample{3, 3}, fSample{5, 5}}), - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{7, 7}, fSample{9, 9}}, []tsdbutil.Sample{fSample{10, 10}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}, fSample{5, 5}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{7, 7}, fSample{9, 9}}, []chunks.Sample{fSample{10, 10}}), }, - expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}}, []tsdbutil.Sample{fSample{3, 3}, fSample{5, 5}}, []tsdbutil.Sample{fSample{7, 7}, fSample{9, 9}}, []tsdbutil.Sample{fSample{10, 10}}), + expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}, fSample{5, 5}}, []chunks.Sample{fSample{7, 7}, fSample{9, 9}}, []chunks.Sample{fSample{10, 10}}), }, { name: "two overlapping", input: []ChunkSeries{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}}, []tsdbutil.Sample{fSample{3, 3}, fSample{8, 8}}), - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{7, 7}, fSample{9, 9}}, []tsdbutil.Sample{fSample{10, 10}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}, fSample{8, 8}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{7, 7}, fSample{9, 9}}, []chunks.Sample{fSample{10, 10}}), }, - expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}}, []tsdbutil.Sample{fSample{3, 3}, fSample{7, 7}, fSample{8, 8}, fSample{9, 9}}, []tsdbutil.Sample{fSample{10, 10}}), + expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}, fSample{7, 7}, fSample{8, 8}, fSample{9, 9}}, []chunks.Sample{fSample{10, 10}}), }, { name: "two duplicated", input: []ChunkSeries{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}), - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}), }, - expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}), + expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}), }, { name: "three overlapping", input: []ChunkSeries{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}), - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{2, 2}, fSample{3, 3}, fSample{6, 6}}), - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{0, 0}, fSample{4, 4}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{2, 2}, fSample{3, 3}, fSample{6, 6}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 0}, fSample{4, 4}}), }, - expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{4, 4}, fSample{5, 5}, fSample{6, 6}}), + expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{4, 4}, fSample{5, 5}, fSample{6, 6}}), }, { name: "three in chained overlap", input: []ChunkSeries{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}), - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{4, 4}, fSample{6, 66}}), - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{6, 6}, fSample{10, 10}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{4, 4}, fSample{6, 66}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{6, 6}, fSample{10, 10}}), }, - expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{4, 4}, fSample{5, 5}, fSample{6, 66}, fSample{10, 10}}), + expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{4, 4}, fSample{5, 5}, fSample{6, 66}, fSample{10, 10}}), }, { name: "three in chained overlap complex", input: []ChunkSeries{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{0, 0}, fSample{5, 5}}, []tsdbutil.Sample{fSample{10, 10}, fSample{15, 15}}), - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{2, 2}, fSample{20, 20}}, []tsdbutil.Sample{fSample{25, 25}, fSample{30, 30}}), - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{18, 18}, fSample{26, 26}}, []tsdbutil.Sample{fSample{31, 31}, fSample{35, 35}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 0}, fSample{5, 5}}, []chunks.Sample{fSample{10, 10}, fSample{15, 15}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{2, 2}, fSample{20, 20}}, []chunks.Sample{fSample{25, 25}, fSample{30, 30}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{18, 18}, fSample{26, 26}}, []chunks.Sample{fSample{31, 31}, fSample{35, 35}}), }, expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), - []tsdbutil.Sample{fSample{0, 0}, fSample{2, 2}, fSample{5, 5}, fSample{10, 10}, fSample{15, 15}, fSample{18, 18}, fSample{20, 20}, fSample{25, 25}, fSample{26, 26}, fSample{30, 30}}, - []tsdbutil.Sample{fSample{31, 31}, fSample{35, 35}}, + []chunks.Sample{fSample{0, 0}, fSample{2, 2}, fSample{5, 5}, fSample{10, 10}, fSample{15, 15}, fSample{18, 18}, fSample{20, 20}, fSample{25, 25}, fSample{26, 26}, fSample{30, 30}}, + []chunks.Sample{fSample{31, 31}, fSample{35, 35}}, ), }, { name: "110 overlapping", input: []ChunkSeries{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), tsdbutil.GenerateSamples(0, 110)), // [0 - 110) - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), tsdbutil.GenerateSamples(60, 50)), // [60 - 110) + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), chunks.GenerateSamples(0, 110)), // [0 - 110) + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), chunks.GenerateSamples(60, 50)), // [60 - 110) }, expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), - tsdbutil.GenerateSamples(0, 110), + chunks.GenerateSamples(0, 110), ), }, { name: "150 overlapping samples, split chunk", input: []ChunkSeries{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), tsdbutil.GenerateSamples(0, 90)), // [0 - 90) - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), tsdbutil.GenerateSamples(60, 90)), // [90 - 150) + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), chunks.GenerateSamples(0, 90)), // [0 - 90) + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), chunks.GenerateSamples(60, 90)), // [90 - 150) }, expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), - tsdbutil.GenerateSamples(0, 120), - tsdbutil.GenerateSamples(120, 30), + chunks.GenerateSamples(0, 120), + chunks.GenerateSamples(120, 30), ), }, { name: "histogram chunks overlapping", input: []ChunkSeries{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{histogramSample(0), histogramSample(5)}, []tsdbutil.Sample{histogramSample(10), histogramSample(15)}), - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{histogramSample(2), histogramSample(20)}, []tsdbutil.Sample{histogramSample(25), histogramSample(30)}), - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{histogramSample(18), histogramSample(26)}, []tsdbutil.Sample{histogramSample(31), histogramSample(35)}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{histogramSample(0), histogramSample(5)}, []chunks.Sample{histogramSample(10), histogramSample(15)}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{histogramSample(2), histogramSample(20)}, []chunks.Sample{histogramSample(25), histogramSample(30)}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{histogramSample(18), histogramSample(26)}, []chunks.Sample{histogramSample(31), histogramSample(35)}), }, expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), - []tsdbutil.Sample{histogramSample(0), histogramSample(2), histogramSample(5), histogramSample(10), histogramSample(15), histogramSample(18), histogramSample(20), histogramSample(25), histogramSample(26), histogramSample(30)}, - []tsdbutil.Sample{histogramSample(31), histogramSample(35)}, + []chunks.Sample{histogramSample(0), histogramSample(2), histogramSample(5), histogramSample(10), histogramSample(15), histogramSample(18), histogramSample(20), histogramSample(25), histogramSample(26), histogramSample(30)}, + []chunks.Sample{histogramSample(31), histogramSample(35)}, ), }, { name: "histogram chunks overlapping with float chunks", input: []ChunkSeries{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{histogramSample(0), histogramSample(5)}, []tsdbutil.Sample{histogramSample(10), histogramSample(15)}), - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{12, 12}}, []tsdbutil.Sample{fSample{14, 14}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{histogramSample(0), histogramSample(5)}, []chunks.Sample{histogramSample(10), histogramSample(15)}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{12, 12}}, []chunks.Sample{fSample{14, 14}}), }, expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), - []tsdbutil.Sample{histogramSample(0)}, - []tsdbutil.Sample{fSample{1, 1}}, - []tsdbutil.Sample{histogramSample(5), histogramSample(10)}, - []tsdbutil.Sample{fSample{12, 12}, fSample{14, 14}}, - []tsdbutil.Sample{histogramSample(15)}, + []chunks.Sample{histogramSample(0)}, + []chunks.Sample{fSample{1, 1}}, + []chunks.Sample{histogramSample(5), histogramSample(10)}, + []chunks.Sample{fSample{12, 12}, fSample{14, 14}}, + []chunks.Sample{histogramSample(15)}, ), }, { name: "float histogram chunks overlapping", input: []ChunkSeries{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{floatHistogramSample(0), floatHistogramSample(5)}, []tsdbutil.Sample{floatHistogramSample(10), floatHistogramSample(15)}), - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{floatHistogramSample(2), floatHistogramSample(20)}, []tsdbutil.Sample{floatHistogramSample(25), floatHistogramSample(30)}), - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{floatHistogramSample(18), floatHistogramSample(26)}, []tsdbutil.Sample{floatHistogramSample(31), floatHistogramSample(35)}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{floatHistogramSample(0), floatHistogramSample(5)}, []chunks.Sample{floatHistogramSample(10), floatHistogramSample(15)}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{floatHistogramSample(2), floatHistogramSample(20)}, []chunks.Sample{floatHistogramSample(25), floatHistogramSample(30)}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{floatHistogramSample(18), floatHistogramSample(26)}, []chunks.Sample{floatHistogramSample(31), floatHistogramSample(35)}), }, expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), - []tsdbutil.Sample{floatHistogramSample(0), floatHistogramSample(2), floatHistogramSample(5), floatHistogramSample(10), floatHistogramSample(15), floatHistogramSample(18), floatHistogramSample(20), floatHistogramSample(25), floatHistogramSample(26), floatHistogramSample(30)}, - []tsdbutil.Sample{floatHistogramSample(31), floatHistogramSample(35)}, + []chunks.Sample{floatHistogramSample(0), floatHistogramSample(2), floatHistogramSample(5), floatHistogramSample(10), floatHistogramSample(15), floatHistogramSample(18), floatHistogramSample(20), floatHistogramSample(25), floatHistogramSample(26), floatHistogramSample(30)}, + []chunks.Sample{floatHistogramSample(31), floatHistogramSample(35)}, ), }, { name: "float histogram chunks overlapping with float chunks", input: []ChunkSeries{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{floatHistogramSample(0), floatHistogramSample(5)}, []tsdbutil.Sample{floatHistogramSample(10), floatHistogramSample(15)}), - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{12, 12}}, []tsdbutil.Sample{fSample{14, 14}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{floatHistogramSample(0), floatHistogramSample(5)}, []chunks.Sample{floatHistogramSample(10), floatHistogramSample(15)}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{12, 12}}, []chunks.Sample{fSample{14, 14}}), }, expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), - []tsdbutil.Sample{floatHistogramSample(0)}, - []tsdbutil.Sample{fSample{1, 1}}, - []tsdbutil.Sample{floatHistogramSample(5), floatHistogramSample(10)}, - []tsdbutil.Sample{fSample{12, 12}, fSample{14, 14}}, - []tsdbutil.Sample{floatHistogramSample(15)}, + []chunks.Sample{floatHistogramSample(0)}, + []chunks.Sample{fSample{1, 1}}, + []chunks.Sample{floatHistogramSample(5), floatHistogramSample(10)}, + []chunks.Sample{fSample{12, 12}, fSample{14, 14}}, + []chunks.Sample{floatHistogramSample(15)}, ), }, { name: "float histogram chunks overlapping with histogram chunks", input: []ChunkSeries{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{floatHistogramSample(0), floatHistogramSample(5)}, []tsdbutil.Sample{floatHistogramSample(10), floatHistogramSample(15)}), - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{histogramSample(1), histogramSample(12)}, []tsdbutil.Sample{histogramSample(14)}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{floatHistogramSample(0), floatHistogramSample(5)}, []chunks.Sample{floatHistogramSample(10), floatHistogramSample(15)}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{histogramSample(1), histogramSample(12)}, []chunks.Sample{histogramSample(14)}), }, expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), - []tsdbutil.Sample{floatHistogramSample(0)}, - []tsdbutil.Sample{histogramSample(1)}, - []tsdbutil.Sample{floatHistogramSample(5), floatHistogramSample(10)}, - []tsdbutil.Sample{histogramSample(12), histogramSample(14)}, - []tsdbutil.Sample{floatHistogramSample(15)}, + []chunks.Sample{floatHistogramSample(0)}, + []chunks.Sample{histogramSample(1)}, + []chunks.Sample{floatHistogramSample(5), floatHistogramSample(10)}, + []chunks.Sample{histogramSample(12), histogramSample(14)}, + []chunks.Sample{floatHistogramSample(15)}, ), }, } { @@ -592,9 +593,9 @@ func TestConcatenatingChunkSeriesMerger(t *testing.T) { { name: "single series", input: []ChunkSeries{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}}, []tsdbutil.Sample{fSample{3, 3}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}}), }, - expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}}, []tsdbutil.Sample{fSample{3, 3}}), + expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}}), }, { name: "two empty series", @@ -607,92 +608,92 @@ func TestConcatenatingChunkSeriesMerger(t *testing.T) { { name: "two non overlapping", input: []ChunkSeries{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}}, []tsdbutil.Sample{fSample{3, 3}, fSample{5, 5}}), - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{7, 7}, fSample{9, 9}}, []tsdbutil.Sample{fSample{10, 10}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}, fSample{5, 5}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{7, 7}, fSample{9, 9}}, []chunks.Sample{fSample{10, 10}}), }, - expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}}, []tsdbutil.Sample{fSample{3, 3}, fSample{5, 5}}, []tsdbutil.Sample{fSample{7, 7}, fSample{9, 9}}, []tsdbutil.Sample{fSample{10, 10}}), + expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}, fSample{5, 5}}, []chunks.Sample{fSample{7, 7}, fSample{9, 9}}, []chunks.Sample{fSample{10, 10}}), }, { name: "two overlapping", input: []ChunkSeries{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}}, []tsdbutil.Sample{fSample{3, 3}, fSample{8, 8}}), - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{7, 7}, fSample{9, 9}}, []tsdbutil.Sample{fSample{10, 10}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}, fSample{8, 8}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{7, 7}, fSample{9, 9}}, []chunks.Sample{fSample{10, 10}}), }, expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), - []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}}, []tsdbutil.Sample{fSample{3, 3}, fSample{8, 8}}, - []tsdbutil.Sample{fSample{7, 7}, fSample{9, 9}}, []tsdbutil.Sample{fSample{10, 10}}, + []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}, fSample{8, 8}}, + []chunks.Sample{fSample{7, 7}, fSample{9, 9}}, []chunks.Sample{fSample{10, 10}}, ), }, { name: "two duplicated", input: []ChunkSeries{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}), - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}), }, expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), - []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}, - []tsdbutil.Sample{fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}, + []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}, + []chunks.Sample{fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}, ), }, { name: "three overlapping", input: []ChunkSeries{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}), - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{2, 2}, fSample{3, 3}, fSample{6, 6}}), - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{0, 0}, fSample{4, 4}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{2, 2}, fSample{3, 3}, fSample{6, 6}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 0}, fSample{4, 4}}), }, expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), - []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}, - []tsdbutil.Sample{fSample{2, 2}, fSample{3, 3}, fSample{6, 6}}, - []tsdbutil.Sample{fSample{0, 0}, fSample{4, 4}}, + []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}, + []chunks.Sample{fSample{2, 2}, fSample{3, 3}, fSample{6, 6}}, + []chunks.Sample{fSample{0, 0}, fSample{4, 4}}, ), }, { name: "three in chained overlap", input: []ChunkSeries{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}), - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{4, 4}, fSample{6, 66}}), - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{6, 6}, fSample{10, 10}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{4, 4}, fSample{6, 66}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{6, 6}, fSample{10, 10}}), }, expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), - []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}, - []tsdbutil.Sample{fSample{4, 4}, fSample{6, 66}}, - []tsdbutil.Sample{fSample{6, 6}, fSample{10, 10}}, + []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}, + []chunks.Sample{fSample{4, 4}, fSample{6, 66}}, + []chunks.Sample{fSample{6, 6}, fSample{10, 10}}, ), }, { name: "three in chained overlap complex", input: []ChunkSeries{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{0, 0}, fSample{5, 5}}, []tsdbutil.Sample{fSample{10, 10}, fSample{15, 15}}), - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{2, 2}, fSample{20, 20}}, []tsdbutil.Sample{fSample{25, 25}, fSample{30, 30}}), - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{fSample{18, 18}, fSample{26, 26}}, []tsdbutil.Sample{fSample{31, 31}, fSample{35, 35}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 0}, fSample{5, 5}}, []chunks.Sample{fSample{10, 10}, fSample{15, 15}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{2, 2}, fSample{20, 20}}, []chunks.Sample{fSample{25, 25}, fSample{30, 30}}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{18, 18}, fSample{26, 26}}, []chunks.Sample{fSample{31, 31}, fSample{35, 35}}), }, expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), - []tsdbutil.Sample{fSample{0, 0}, fSample{5, 5}}, []tsdbutil.Sample{fSample{10, 10}, fSample{15, 15}}, - []tsdbutil.Sample{fSample{2, 2}, fSample{20, 20}}, []tsdbutil.Sample{fSample{25, 25}, fSample{30, 30}}, - []tsdbutil.Sample{fSample{18, 18}, fSample{26, 26}}, []tsdbutil.Sample{fSample{31, 31}, fSample{35, 35}}, + []chunks.Sample{fSample{0, 0}, fSample{5, 5}}, []chunks.Sample{fSample{10, 10}, fSample{15, 15}}, + []chunks.Sample{fSample{2, 2}, fSample{20, 20}}, []chunks.Sample{fSample{25, 25}, fSample{30, 30}}, + []chunks.Sample{fSample{18, 18}, fSample{26, 26}}, []chunks.Sample{fSample{31, 31}, fSample{35, 35}}, ), }, { name: "110 overlapping", input: []ChunkSeries{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), tsdbutil.GenerateSamples(0, 110)), // [0 - 110) - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), tsdbutil.GenerateSamples(60, 50)), // [60 - 110) + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), chunks.GenerateSamples(0, 110)), // [0 - 110) + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), chunks.GenerateSamples(60, 50)), // [60 - 110) }, expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), - tsdbutil.GenerateSamples(0, 110), - tsdbutil.GenerateSamples(60, 50), + chunks.GenerateSamples(0, 110), + chunks.GenerateSamples(60, 50), ), }, { name: "150 overlapping samples, simply concatenated and no splits", input: []ChunkSeries{ - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), tsdbutil.GenerateSamples(0, 90)), // [0 - 90) - NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), tsdbutil.GenerateSamples(60, 90)), // [90 - 150) + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), chunks.GenerateSamples(0, 90)), // [0 - 90) + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), chunks.GenerateSamples(60, 90)), // [90 - 150) }, expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), - tsdbutil.GenerateSamples(0, 90), - tsdbutil.GenerateSamples(60, 90), + chunks.GenerateSamples(0, 90), + chunks.GenerateSamples(60, 90), ), }, } { @@ -803,20 +804,20 @@ func (m *mockChunkSeriesSet) Warnings() Warnings { return nil } func TestChainSampleIterator(t *testing.T) { for _, tc := range []struct { input []chunkenc.Iterator - expected []tsdbutil.Sample + expected []chunks.Sample }{ { input: []chunkenc.Iterator{ NewListSeriesIterator(samples{fSample{0, 0}, fSample{1, 1}}), }, - expected: []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}}, + expected: []chunks.Sample{fSample{0, 0}, fSample{1, 1}}, }, { input: []chunkenc.Iterator{ NewListSeriesIterator(samples{fSample{0, 0}, fSample{1, 1}}), NewListSeriesIterator(samples{fSample{2, 2}, fSample{3, 3}}), }, - expected: []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}, + expected: []chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}, }, { input: []chunkenc.Iterator{ @@ -824,7 +825,7 @@ func TestChainSampleIterator(t *testing.T) { NewListSeriesIterator(samples{fSample{1, 1}, fSample{4, 4}}), NewListSeriesIterator(samples{fSample{2, 2}, fSample{5, 5}}), }, - expected: []tsdbutil.Sample{ + expected: []chunks.Sample{ fSample{0, 0}, fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{4, 4}, fSample{5, 5}, }, }, @@ -838,7 +839,7 @@ func TestChainSampleIterator(t *testing.T) { NewListSeriesIterator(samples{}), NewListSeriesIterator(samples{}), }, - expected: []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}, + expected: []chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}, }, } { merged := ChainSampleIteratorFromIterators(nil, tc.input) @@ -852,14 +853,14 @@ func TestChainSampleIteratorSeek(t *testing.T) { for _, tc := range []struct { input []chunkenc.Iterator seek int64 - expected []tsdbutil.Sample + expected []chunks.Sample }{ { input: []chunkenc.Iterator{ NewListSeriesIterator(samples{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}}), }, seek: 1, - expected: []tsdbutil.Sample{fSample{1, 1}, fSample{2, 2}}, + expected: []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, }, { input: []chunkenc.Iterator{ @@ -867,7 +868,7 @@ func TestChainSampleIteratorSeek(t *testing.T) { NewListSeriesIterator(samples{fSample{2, 2}, fSample{3, 3}}), }, seek: 2, - expected: []tsdbutil.Sample{fSample{2, 2}, fSample{3, 3}}, + expected: []chunks.Sample{fSample{2, 2}, fSample{3, 3}}, }, { input: []chunkenc.Iterator{ @@ -876,7 +877,7 @@ func TestChainSampleIteratorSeek(t *testing.T) { NewListSeriesIterator(samples{fSample{2, 2}, fSample{5, 5}}), }, seek: 2, - expected: []tsdbutil.Sample{fSample{2, 2}, fSample{3, 3}, fSample{4, 4}, fSample{5, 5}}, + expected: []chunks.Sample{fSample{2, 2}, fSample{3, 3}, fSample{4, 4}, fSample{5, 5}}, }, { input: []chunkenc.Iterator{ @@ -884,11 +885,11 @@ func TestChainSampleIteratorSeek(t *testing.T) { NewListSeriesIterator(samples{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}}), }, seek: 0, - expected: []tsdbutil.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}, + expected: []chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}, }, } { merged := ChainSampleIteratorFromIterators(nil, tc.input) - actual := []tsdbutil.Sample{} + actual := []chunks.Sample{} if merged.Seek(tc.seek) == chunkenc.ValFloat { t, f := merged.At() actual = append(actual, fSample{t, f}) @@ -904,7 +905,7 @@ func makeSeries(numSeries, numSamples int) []Series { series := []Series{} for j := 0; j < numSeries; j++ { labels := labels.FromStrings("foo", fmt.Sprintf("bar%d", j)) - samples := []tsdbutil.Sample{} + samples := []chunks.Sample{} for k := 0; k < numSamples; k++ { samples = append(samples, fSample{t: int64(k), f: float64(k)}) } diff --git a/storage/remote/read_handler_test.go b/storage/remote/read_handler_test.go index 7e8618615..fdb9f04dd 100644 --- a/storage/remote/read_handler_test.go +++ b/storage/remote/read_handler_test.go @@ -196,10 +196,10 @@ func BenchmarkStreamReadEndpoint(b *testing.B) { } func TestStreamReadEndpoint(t *testing.T) { - // First with 120 samples. We expect 1 frame with 1 chunk. - // Second with 121 samples, We expect 1 frame with 2 chunks. - // Third with 241 samples. We expect 1 frame with 2 chunks, and 1 frame with 1 chunk for the same series due to bytes limit. - // Fourth with 120 histogram samples. We expect 1 frame with 1 chunk. + // First with 120 float samples. We expect 1 frame with 1 chunk. + // Second with 121 float samples, We expect 1 frame with 2 chunks. + // Third with 241 float samples. We expect 1 frame with 2 chunks, and 1 frame with 1 chunk for the same series due to bytes limit. + // Fourth with 25 histogram samples. We expect 1 frame with 1 chunk. store := promql.LoadedStorage(t, ` load 1m test_metric1{foo="bar1",baz="qux"} 0+100x119 @@ -208,7 +208,7 @@ func TestStreamReadEndpoint(t *testing.T) { `) defer store.Close() - addNativeHistogramsToTestSuite(t, store, 120) + addNativeHistogramsToTestSuite(t, store, 25) api := NewReadHandler(nil, nil, store, func() config.Config { return config.Config{ @@ -420,8 +420,8 @@ func TestStreamReadEndpoint(t *testing.T) { Chunks: []prompb.Chunk{ { Type: prompb.Chunk_FLOAT_HISTOGRAM, - MaxTimeMs: 7140000, - Data: []byte("\000x\000\377?PbM\322\361\251\374\214\244\224e$\242@(\000\000\000\000\000\000@\000\000\000\000\000\000\000@2ffffff?\360\000\000\000\000\000\000@\000\000\000\000\000\000\000?\360\000\000\000\000\000\000?\360\000\000\000\000\000\000?\360\000\000\000\000\000\000@\000\000\000\000\000\000\000?\360\000\000\000\000\000\000?\360\000\000\000\000\000\000\370\352`\326/v\003\322\037\302_\377\330\017\t\177\377\t\177\377\t\177\377`<%\377\374%\377\364\275a{4\237\377\377\377\377\377\375\200\365\205\354\007\260\036\300z\302\366\003\330\r\244\217\275\240\365\353\237\377\377\377\377\377\375a{A\353\013\326\027\254/h=az\302\320\276\300\270\254\314\314\314\314\314\333A\354\013\332\017h=\240\366\005\355\007\264\032\t\231\22333333;\002\347`^\300\275\201s\260/`Z8\324'\352\037\274\352\023\346gP\2372\334$\356\007\273)\377\377\377\377\377\377P\237\270\036\241?P\237\250O\334\017P\237\250Om\027\373B\366\347\265UUUUUw\003\332\027\270\036\340{\201\355\013\334\017p4\231\235\231\231\231\231\231\236\320\271\332\027\264/h\\\355\013\332\026\3307{\003\3732\344\314\314\314\314\314\347`~fv\007\346Q1\335\225\252\252\252\252\252\253\260?\035\201\375\201\375\201\370\354\017\354\017\245\347\267<\377\377\377\377\377\377\031\3061\234b\324O\306:\365\357\377\377\377\377\377\374\3439\3163\232\005\353\023\340\254\314\314\314\314\314\307X\237\030\307X\237\030\240\316\360p\311\231\231\231\231\231\265\211\373\301\353\023\365\211\372\304\375\340\365\211\372\304\364\017\334\027\007\252\252\252\252\252\253x=\301{\301\357\007\274\036\340\275\340\367\203C\231\216\177\377\377\377\377\377\270.w\005\356\013\334\027;\202\367\005\240^\320\374\026\252\252\252\252\252\251\332\037\231\235\241\371\224\031\214-\231\231\231\231\231\235\241\370\355\017\355\017\355\017\307h\177h}\035\347<\231\231\231\231\231\2323\214c8\305\002c\005\252\252\252\252\252\252q\234\347\031\315\006\366\t\360\317\377\377\377\377\377\343\260O\214c\260O\214P&\030=UUUUU[\004\370v\t\373\004\375\202|;\004\375\202z\037\307\034\231\231\231\231\231\232\030\341\206\030\341\205\002a\202\26333333\0341\307\0341\306\203s\302\352\252\252\252\252\252\206xa\206xa@\230tq\376*\252\252\252\252\253<3\317<3\316\237q\300\026\252\252\252\252\252\260\307\0140\307\014-\342\346\030\007\263333328c\2168c\215/H\377\340\t333338t\217\376\030a\322?\370an\033\236\000\340;UUUUU]#\377\340\017H\377\364\217\377H\377\370\003\322?\375#\377G\357\013\200'\377\377\377\377\377\370\003\336\027\300\037\000|\001\357\013\340\017\2004\314\300.\252\252\252\252\252\273\302\347x^\360\275\341s\274/xZ/p~\007\254\314\314\314\314\314\235\301\371\231\334\037\231m\037\314`\t33333=\301\370\356\017\356\017\356\017\307p\177p}\027\234\003\352\252\252\252\252\253\031\3061\234b\204\306\000\237\377\377\377\377\377\234g9\306sN\366\211\360\035\252\252\252\252\252\254v\211\361\214v\211\361\212\023\014\002\331\231\231\231\231\231\264O\207h\237\264O\332'\303\264O\332'\243\361\300\022fffffp\307\0140\307\014(L0}\252\252\252\252\252\254p\307\034p\307\033a\037s\300\023\377\377\377\377\377\360\317\0140\317\014(&\030\007\325UUUUVxg\236xg\235\013\307\000I\231\231\231\231\231\303\0340\303\0340\243\230`:\314\314\314\314\314\310\341\2168\341\2164\027\260_\300.\252\252\252\252\252\260\354\027\360\303\016\301\177\014(f\014\001?\377\377\377\377\377\260_\301\330/\366\013\375\202\376\016\301\177\260_\240\370p=\252\252\252\252\252\254\030p`\301\207\006\ny\203\000I\231\231\231\231\231\303\203\016\03480\341\240\270\360\017fffffd\030\360`\301\217\006\n\031\203\000Z\252\252\252\252\252\307\203\036|\3703\347\264\031\3770\340\007\252\252\252\252\252\2500\340\301\203\016\014\027\021&\014\001\237\377\377\377\377\377\016\0148p\340\303\206\340.\343\300\013UUUUUPc\301\203\006<\030)0`\033fffffh\360c\307\217\006\367\t\360\002L\314\314\314\314\314w\t\361\214w\t\361\212\t\206\000\332\252\252\252\252\252\334'\303\270O\334'\356\023\341\334'\356\023\320\374p\002\177\377\377\377\377\3741\303\0141\303\n\t\206\001\272\252\252\252\252\252\216\030\343\216\030\343Gs\300\032\314\314\314\314\314\320\317\0140\317\014(&\030\001&fffffxg\236xg\235\013\307\001\375UUUUUC\0340\303\0340\247\230`\004\377\377\377\377\377\370\341\2168\341\2164\027\264_\300\033UUUUUP\355\027\360\303\016\321\177\014(f\014\003l\314\314\314\314\315\264_\301\332/\366\213\375\242\376\016\321\177\264_\240\370p\002L\314\314\314\314\314\030p`\301\207\006\n9\203\000mUUUUUC\203\016\03480\341\240\270\360\002\177\377\377\377\377\374\030\360`\301\217\006\n\031\203\007\375UUUUUG\203\036\201\360p\001?\377\377\377\377\374\014\034\014\014\014\034\014\013Y\177\366\006\000]UUUUU\203\201\203\203\203\201\203\203@.\036\000\35333333\201\207\201\201\201\207\201\201@f\006\000$\314\314\314\314\314\207\201\207\207\207\201\207\207@\336\016\000}UUUUU\201\203\201\201\201\203\201\201@&\006\000'\377\377\377\377\377\203\201\203\203\203\201\203\203@n>\001\355UUUUU\201\217\201\201\201\217\201\201@&\006\000[33333\217\201\217\217\217\201\217\217"), + MaxTimeMs: 1440000, + Data: []byte("\x00\x19\x00\xff?PbM\xd2\xf1\xa9\xfc\x8c\xa4\x94e$\xa2@(\x00\x00\x00\x00\x00\x00@\x00\x00\x00\x00\x00\x00\x00@2ffffff?\xf0\x00\x00\x00\x00\x00\x00@\x00\x00\x00\x00\x00\x00\x00?\xf0\x00\x00\x00\x00\x00\x00?\xf0\x00\x00\x00\x00\x00\x00?\xf0\x00\x00\x00\x00\x00\x00@\x00\x00\x00\x00\x00\x00\x00?\xf0\x00\x00\x00\x00\x00\x00?\xf0\x00\x00\x00\x00\x00\x00\xf8\xea`\xd6/v\x03\xd2\x1f\xc2_\xff\xd8\x0f\t\x7f\xff\t\x7f\xff\t\x7f\xff`<%\xff\xfc%\xff\xf4\xbda{4\x9f\xff\xff\xff\xff\xff\xfd\x80\xf5\x85\xec\a\xb0\x1e\xc0z\xc2\xf6\x03\xd8\r\xa4\x8f\xbd\xa0\xf5\xeb\x9f\xff\xff\xff\xff\xff\xfda{A\xeb\v\xd6\x17\xac/h=az\xc2о\xc0\xb8\xac\xcc\xcc\xcc\xcc\xcc\xdbA\xec\v\xda\x0fh=\xa0\xf6\x05\xed\a\xb4\x1a\t\x99\x9333333;\x02\xe7`^\xc0\xbd\x81s\xb0/`Z8\xd4'\xea\x1f\xbc\xea\x13\xe6gP\x9f2\xdc$\xee\a\xbb)\xff\xff\xff\xff\xff\xffP\x9f\xb8\x1e\xa1?P\x9f\xa8O\xdc\x0fP\x9f\xa8Om\x17\xfbB\xf6\xe7\xb5UUUUUw\x03\xda\x17\xb8\x1e\xe0{\x81\xed\v\xdc\x0fp4\x99\x9d\x99\x99\x99\x99\x99\x9eй\xda\x17\xb4/h\\\xed\v\xda\x16\xd87{\x03\xfb2\xe4\xcc\xcc\xcc\xcc\xcc\xe7`~fv\a\xe6Q1ݕ\xaa\xaa\xaa\xaa\xaa\xab\xb0?\x1d\x81\xfd\x81\xfd\x81\xf8\xec\x0f\xec\x0f\xa5\xe7\xb7<\xff\xff\xff\xff\xff\xff\x19\xc61\x9cb\xd4O\xc6:\xf5\xef\xff\xff\xff\xff\xff\xfc\xe39\xce3\x9a\x05\xeb\x13\xe0\xac\xcc\xcc\xcc\xcc\xcc\xc7X\x9f\x18\xc7X\x9f\x18\xa0\xce\xf0pə\x99\x99\x99\x99\xb5\x89\xfb\xc1\xeb\x13\xf5\x89\xfa\xc4\xfd\xe0\xf5\x89\xfa\xc4\xf4\x0f\xdc\x17\a\xaa\xaa\xaa\xaa\xaa\xabx=\xc1{\xc1\xef\a\xbc\x1e\xe0\xbd\xe0\xf7\x83C\x99\x8e\x7f\xff\xff\xff\xff\xff\xb8.w\x05\xee\v\xdc\x17;\x82\xf7\x05\xa0^\xd0\xfc\x16\xaa\xaa\xaa\xaa\xaa\xa9\xda\x1f\x99\x9d\xa1\xf9\x94\x19\x8c-\x99\x99\x99\x99\x99\x9d\xa1\xf8\xed\x0f\xed\x0f\xed\x0f\xc7h\x7fh}\x1d\xe7<\x99\x99\x99\x99\x99\x9a3\x8cc8\xc5\x02c\x05\xaa\xaa\xaa\xaa\xaa\xaaq\x9c\xe7\x19\xcd\x06\xf6\t\xf0\xcf\xff\xff\xff\xff\xff\xe3\xb0O\x8cc\xb0O\x8cP&\x18=UUUUU[\x04\xf8v\t\xfb\x04\xfd\x82|;\x04\xfd\x82z\x1f\xc7\x1c\x99\x99\x99\x99\x99\x9a\x18\xe1\x86\x18\xe1\x84"), }, }, }, diff --git a/storage/series.go b/storage/series.go index e88964d88..b111505aa 100644 --- a/storage/series.go +++ b/storage/series.go @@ -22,7 +22,6 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/chunks" - "github.com/prometheus/prometheus/tsdb/tsdbutil" ) type SeriesEntry struct { @@ -42,7 +41,7 @@ func (s *ChunkSeriesEntry) Labels() labels.Labels { return func (s *ChunkSeriesEntry) Iterator(it chunks.Iterator) chunks.Iterator { return s.ChunkIteratorFn(it) } // NewListSeries returns series entry with iterator that allows to iterate over provided samples. -func NewListSeries(lset labels.Labels, s []tsdbutil.Sample) *SeriesEntry { +func NewListSeries(lset labels.Labels, s []chunks.Sample) *SeriesEntry { samplesS := Samples(samples(s)) return &SeriesEntry{ Lset: lset, @@ -59,10 +58,10 @@ func NewListSeries(lset labels.Labels, s []tsdbutil.Sample) *SeriesEntry { // NewListChunkSeriesFromSamples returns chunk series entry that allows to iterate over provided samples. // NOTE: It uses inefficient chunks encoding implementation, not caring about chunk size. // Use only for testing. -func NewListChunkSeriesFromSamples(lset labels.Labels, samples ...[]tsdbutil.Sample) *ChunkSeriesEntry { +func NewListChunkSeriesFromSamples(lset labels.Labels, samples ...[]chunks.Sample) *ChunkSeriesEntry { chksFromSamples := make([]chunks.Meta, 0, len(samples)) for _, s := range samples { - cfs, err := tsdbutil.ChunkFromSamples(s) + cfs, err := chunks.ChunkFromSamples(s) if err != nil { return &ChunkSeriesEntry{ Lset: lset, @@ -98,14 +97,14 @@ type listSeriesIterator struct { idx int } -type samples []tsdbutil.Sample +type samples []chunks.Sample -func (s samples) Get(i int) tsdbutil.Sample { return s[i] } -func (s samples) Len() int { return len(s) } +func (s samples) Get(i int) chunks.Sample { return s[i] } +func (s samples) Len() int { return len(s) } -// Samples interface allows to work on arrays of types that are compatible with tsdbutil.Sample. +// Samples interface allows to work on arrays of types that are compatible with chunks.Sample. type Samples interface { - Get(i int) tsdbutil.Sample + Get(i int) chunks.Sample Len() int } @@ -412,9 +411,9 @@ func (e errChunksIterator) Err() error { return e.err } // ExpandSamples iterates over all samples in the iterator, buffering all in slice. // Optionally it takes samples constructor, useful when you want to compare sample slices with different // sample implementations. if nil, sample type from this package will be used. -func ExpandSamples(iter chunkenc.Iterator, newSampleFn func(t int64, f float64, h *histogram.Histogram, fh *histogram.FloatHistogram) tsdbutil.Sample) ([]tsdbutil.Sample, error) { +func ExpandSamples(iter chunkenc.Iterator, newSampleFn func(t int64, f float64, h *histogram.Histogram, fh *histogram.FloatHistogram) chunks.Sample) ([]chunks.Sample, error) { if newSampleFn == nil { - newSampleFn = func(t int64, f float64, h *histogram.Histogram, fh *histogram.FloatHistogram) tsdbutil.Sample { + newSampleFn = func(t int64, f float64, h *histogram.Histogram, fh *histogram.FloatHistogram) chunks.Sample { switch { case h != nil: return hSample{t, h} @@ -426,7 +425,7 @@ func ExpandSamples(iter chunkenc.Iterator, newSampleFn func(t int64, f float64, } } - var result []tsdbutil.Sample + var result []chunks.Sample for { switch iter.Next() { case chunkenc.ValNone: diff --git a/storage/series_test.go b/storage/series_test.go index 5c74fae09..669ade3a1 100644 --- a/storage/series_test.go +++ b/storage/series_test.go @@ -25,7 +25,6 @@ import ( "github.com/prometheus/prometheus/model/value" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/chunks" - "github.com/prometheus/prometheus/tsdb/tsdbutil" ) func TestListSeriesIterator(t *testing.T) { @@ -78,11 +77,11 @@ func TestListSeriesIterator(t *testing.T) { func TestChunkSeriesSetToSeriesSet(t *testing.T) { series := []struct { lbs labels.Labels - samples []tsdbutil.Sample + samples []chunks.Sample }{ { lbs: labels.FromStrings("__name__", "up", "instance", "localhost:8080"), - samples: []tsdbutil.Sample{ + samples: []chunks.Sample{ fSample{t: 1, f: 1}, fSample{t: 2, f: 2}, fSample{t: 3, f: 3}, @@ -90,7 +89,7 @@ func TestChunkSeriesSetToSeriesSet(t *testing.T) { }, }, { lbs: labels.FromStrings("__name__", "up", "instance", "localhost:8081"), - samples: []tsdbutil.Sample{ + samples: []chunks.Sample{ fSample{t: 1, f: 2}, fSample{t: 2, f: 3}, fSample{t: 3, f: 4}, @@ -126,7 +125,7 @@ func TestChunkSeriesSetToSeriesSet(t *testing.T) { } type histogramTest struct { - samples []tsdbutil.Sample + samples []chunks.Sample expectedCounterResetHeaders []chunkenc.CounterResetHeader } @@ -270,34 +269,34 @@ func TestHistogramSeriesToChunks(t *testing.T) { tests := map[string]histogramTest{ "single histogram to single chunk": { - samples: []tsdbutil.Sample{ + samples: []chunks.Sample{ hSample{t: 1, h: h1}, }, expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.UnknownCounterReset}, }, "two histograms encoded to a single chunk": { - samples: []tsdbutil.Sample{ + samples: []chunks.Sample{ hSample{t: 1, h: h1}, hSample{t: 2, h: h2}, }, expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.UnknownCounterReset}, }, "two histograms encoded to two chunks": { - samples: []tsdbutil.Sample{ + samples: []chunks.Sample{ hSample{t: 1, h: h2}, hSample{t: 2, h: h1}, }, expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.UnknownCounterReset, chunkenc.CounterReset}, }, "histogram and stale sample encoded to two chunks": { - samples: []tsdbutil.Sample{ + samples: []chunks.Sample{ hSample{t: 1, h: staleHistogram}, hSample{t: 2, h: h1}, }, expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.UnknownCounterReset, chunkenc.UnknownCounterReset}, }, "histogram and reduction in bucket encoded to two chunks": { - samples: []tsdbutil.Sample{ + samples: []chunks.Sample{ hSample{t: 1, h: h1}, hSample{t: 2, h: h2down}, }, @@ -305,34 +304,34 @@ func TestHistogramSeriesToChunks(t *testing.T) { }, // Float histograms. "single float histogram to single chunk": { - samples: []tsdbutil.Sample{ + samples: []chunks.Sample{ fhSample{t: 1, fh: fh1}, }, expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.UnknownCounterReset}, }, "two float histograms encoded to a single chunk": { - samples: []tsdbutil.Sample{ + samples: []chunks.Sample{ fhSample{t: 1, fh: fh1}, fhSample{t: 2, fh: fh2}, }, expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.UnknownCounterReset}, }, "two float histograms encoded to two chunks": { - samples: []tsdbutil.Sample{ + samples: []chunks.Sample{ fhSample{t: 1, fh: fh2}, fhSample{t: 2, fh: fh1}, }, expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.UnknownCounterReset, chunkenc.CounterReset}, }, "float histogram and stale sample encoded to two chunks": { - samples: []tsdbutil.Sample{ + samples: []chunks.Sample{ fhSample{t: 1, fh: staleFloatHistogram}, fhSample{t: 2, fh: fh1}, }, expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.UnknownCounterReset, chunkenc.UnknownCounterReset}, }, "float histogram and reduction in bucket encoded to two chunks": { - samples: []tsdbutil.Sample{ + samples: []chunks.Sample{ fhSample{t: 1, fh: fh1}, fhSample{t: 2, fh: fh2down}, }, @@ -340,61 +339,61 @@ func TestHistogramSeriesToChunks(t *testing.T) { }, // Mixed. "histogram and float histogram encoded to two chunks": { - samples: []tsdbutil.Sample{ + samples: []chunks.Sample{ hSample{t: 1, h: h1}, fhSample{t: 2, fh: fh2}, }, expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.UnknownCounterReset, chunkenc.UnknownCounterReset}, }, "float histogram and histogram encoded to two chunks": { - samples: []tsdbutil.Sample{ + samples: []chunks.Sample{ fhSample{t: 1, fh: fh1}, hSample{t: 2, h: h2}, }, expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.UnknownCounterReset, chunkenc.UnknownCounterReset}, }, "histogram and stale float histogram encoded to two chunks": { - samples: []tsdbutil.Sample{ + samples: []chunks.Sample{ hSample{t: 1, h: h1}, fhSample{t: 2, fh: staleFloatHistogram}, }, expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.UnknownCounterReset, chunkenc.UnknownCounterReset}, }, "single gauge histogram encoded to one chunk": { - samples: []tsdbutil.Sample{ + samples: []chunks.Sample{ hSample{t: 1, h: gh1}, }, expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.GaugeType}, }, "two gauge histograms encoded to one chunk when counter increases": { - samples: []tsdbutil.Sample{ + samples: []chunks.Sample{ hSample{t: 1, h: gh1}, hSample{t: 2, h: gh2}, }, expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.GaugeType}, }, "two gauge histograms encoded to one chunk when counter decreases": { - samples: []tsdbutil.Sample{ + samples: []chunks.Sample{ hSample{t: 1, h: gh2}, hSample{t: 2, h: gh1}, }, expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.GaugeType}, }, "single gauge float histogram encoded to one chunk": { - samples: []tsdbutil.Sample{ + samples: []chunks.Sample{ fhSample{t: 1, fh: gfh1}, }, expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.GaugeType}, }, "two float gauge histograms encoded to one chunk when counter increases": { - samples: []tsdbutil.Sample{ + samples: []chunks.Sample{ fhSample{t: 1, fh: gfh1}, fhSample{t: 2, fh: gfh2}, }, expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.GaugeType}, }, "two float gauge histograms encoded to one chunk when counter decreases": { - samples: []tsdbutil.Sample{ + samples: []chunks.Sample{ fhSample{t: 1, fh: gfh2}, fhSample{t: 2, fh: gfh1}, }, @@ -411,7 +410,7 @@ func TestHistogramSeriesToChunks(t *testing.T) { func testHistogramsSeriesToChunks(t *testing.T, test histogramTest) { lbs := labels.FromStrings("__name__", "up", "instance", "localhost:8080") - copiedSamples := []tsdbutil.Sample{} + copiedSamples := []chunks.Sample{} for _, s := range test.samples { switch cs := s.(type) { case hSample: @@ -470,7 +469,7 @@ func testHistogramsSeriesToChunks(t *testing.T, test histogramTest) { } } -func expandHistogramSamples(chunks []chunks.Meta) (result []tsdbutil.Sample) { +func expandHistogramSamples(chunks []chunks.Meta) (result []chunks.Sample) { if len(chunks) == 0 { return } diff --git a/tsdb/agent/db_test.go b/tsdb/agent/db_test.go index 5c71d548f..7eda6110c 100644 --- a/tsdb/agent/db_test.go +++ b/tsdb/agent/db_test.go @@ -33,6 +33,7 @@ import ( "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/storage/remote" "github.com/prometheus/prometheus/tsdb" + "github.com/prometheus/prometheus/tsdb/chunks" "github.com/prometheus/prometheus/tsdb/record" "github.com/prometheus/prometheus/tsdb/tsdbutil" "github.com/prometheus/prometheus/tsdb/wlog" @@ -132,7 +133,7 @@ func TestCommit(t *testing.T) { lset := labels.New(l...) for i := 0; i < numDatapoints; i++ { - sample := tsdbutil.GenerateSamples(0, 1) + sample := chunks.GenerateSamples(0, 1) ref, err := app.Append(0, lset, sample[0].T(), sample[0].F()) require.NoError(t, err) @@ -247,7 +248,7 @@ func TestRollback(t *testing.T) { lset := labels.New(l...) for i := 0; i < numDatapoints; i++ { - sample := tsdbutil.GenerateSamples(0, 1) + sample := chunks.GenerateSamples(0, 1) _, err := app.Append(0, lset, sample[0].T(), sample[0].F()) require.NoError(t, err) } diff --git a/tsdb/block_test.go b/tsdb/block_test.go index e0d928aca..d65d76d8f 100644 --- a/tsdb/block_test.go +++ b/tsdb/block_test.go @@ -36,7 +36,6 @@ import ( "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/chunks" "github.com/prometheus/prometheus/tsdb/fileutil" - "github.com/prometheus/prometheus/tsdb/tsdbutil" "github.com/prometheus/prometheus/tsdb/wlog" ) @@ -175,7 +174,7 @@ func TestCorruptedChunk(t *testing.T) { t.Run(tc.name, func(t *testing.T) { tmpdir := t.TempDir() - series := storage.NewListSeries(labels.FromStrings("a", "b"), []tsdbutil.Sample{sample{1, 1, nil, nil}}) + series := storage.NewListSeries(labels.FromStrings("a", "b"), []chunks.Sample{sample{1, 1, nil, nil}}) blockDir := createBlock(t, tmpdir, []storage.Series{series}) files, err := sequenceFiles(chunkDir(blockDir)) require.NoError(t, err) @@ -218,7 +217,7 @@ func TestLabelValuesWithMatchers(t *testing.T) { seriesEntries = append(seriesEntries, storage.NewListSeries(labels.FromStrings( "tens", fmt.Sprintf("value%d", i/10), "unique", fmt.Sprintf("value%d", i), - ), []tsdbutil.Sample{sample{100, 0, nil, nil}})) + ), []chunks.Sample{sample{100, 0, nil, nil}})) } blockDir := createBlock(t, tmpdir, seriesEntries) @@ -353,12 +352,12 @@ func TestReadIndexFormatV1(t *testing.T) { q, err := NewBlockQuerier(block, 0, 1000) require.NoError(t, err) require.Equal(t, query(t, q, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")), - map[string][]tsdbutil.Sample{`{foo="bar"}`: {sample{t: 1, f: 2}}}) + map[string][]chunks.Sample{`{foo="bar"}`: {sample{t: 1, f: 2}}}) q, err = NewBlockQuerier(block, 0, 1000) require.NoError(t, err) require.Equal(t, query(t, q, labels.MustNewMatcher(labels.MatchNotRegexp, "foo", "^.?$")), - map[string][]tsdbutil.Sample{ + map[string][]chunks.Sample{ `{foo="bar"}`: {sample{t: 1, f: 2}}, `{foo="baz"}`: {sample{t: 3, f: 4}}, }) @@ -376,7 +375,7 @@ func BenchmarkLabelValuesWithMatchers(b *testing.B) { "a_unique", fmt.Sprintf("value%d", i), "b_tens", fmt.Sprintf("value%d", i/(metricCount/10)), "c_ninety", fmt.Sprintf("value%d", i/(metricCount/10)/9), // "0" for the first 90%, then "1" - ), []tsdbutil.Sample{sample{100, 0, nil, nil}})) + ), []chunks.Sample{sample{100, 0, nil, nil}})) } blockDir := createBlock(b, tmpdir, seriesEntries) @@ -412,13 +411,13 @@ func TestLabelNamesWithMatchers(t *testing.T) { for i := 0; i < 100; i++ { seriesEntries = append(seriesEntries, storage.NewListSeries(labels.FromStrings( "unique", fmt.Sprintf("value%d", i), - ), []tsdbutil.Sample{sample{100, 0, nil, nil}})) + ), []chunks.Sample{sample{100, 0, nil, nil}})) if i%10 == 0 { seriesEntries = append(seriesEntries, storage.NewListSeries(labels.FromStrings( "tens", fmt.Sprintf("value%d", i/10), "unique", fmt.Sprintf("value%d", i), - ), []tsdbutil.Sample{sample{100, 0, nil, nil}})) + ), []chunks.Sample{sample{100, 0, nil, nil}})) } if i%20 == 0 { @@ -426,7 +425,7 @@ func TestLabelNamesWithMatchers(t *testing.T) { "tens", fmt.Sprintf("value%d", i/10), "twenties", fmt.Sprintf("value%d", i/20), "unique", fmt.Sprintf("value%d", i), - ), []tsdbutil.Sample{sample{100, 0, nil, nil}})) + ), []chunks.Sample{sample{100, 0, nil, nil}})) } } @@ -552,7 +551,7 @@ func createHeadWithOOOSamples(tb testing.TB, w *wlog.WL, series []storage.Series require.NoError(tb, err) oooSampleLabels := make([]labels.Labels, 0, len(series)) - oooSamples := make([]tsdbutil.SampleSlice, 0, len(series)) + oooSamples := make([]chunks.SampleSlice, 0, len(series)) var it chunkenc.Iterator totalSamples := 0 @@ -561,7 +560,7 @@ func createHeadWithOOOSamples(tb testing.TB, w *wlog.WL, series []storage.Series ref := storage.SeriesRef(0) it = s.Iterator(it) lset := s.Labels() - os := tsdbutil.SampleSlice{} + os := chunks.SampleSlice{} count := 0 for it.Next() == chunkenc.ValFloat { totalSamples++ @@ -612,14 +611,14 @@ const ( // genSeries generates series of float64 samples with a given number of labels and values. func genSeries(totalSeries, labelCount int, mint, maxt int64) []storage.Series { - return genSeriesFromSampleGenerator(totalSeries, labelCount, mint, maxt, 1, func(ts int64) tsdbutil.Sample { + return genSeriesFromSampleGenerator(totalSeries, labelCount, mint, maxt, 1, func(ts int64) chunks.Sample { return sample{t: ts, f: rand.Float64()} }) } // genHistogramSeries generates series of histogram samples with a given number of labels and values. func genHistogramSeries(totalSeries, labelCount int, mint, maxt, step int64, floatHistogram bool) []storage.Series { - return genSeriesFromSampleGenerator(totalSeries, labelCount, mint, maxt, step, func(ts int64) tsdbutil.Sample { + return genSeriesFromSampleGenerator(totalSeries, labelCount, mint, maxt, step, func(ts int64) chunks.Sample { h := &histogram.Histogram{ Count: 7 + uint64(ts*5), ZeroCount: 2 + uint64(ts), @@ -653,7 +652,7 @@ func genHistogramSeries(totalSeries, labelCount int, mint, maxt, step int64, flo func genHistogramAndFloatSeries(totalSeries, labelCount int, mint, maxt, step int64, floatHistogram bool) []storage.Series { floatSample := false count := 0 - return genSeriesFromSampleGenerator(totalSeries, labelCount, mint, maxt, step, func(ts int64) tsdbutil.Sample { + return genSeriesFromSampleGenerator(totalSeries, labelCount, mint, maxt, step, func(ts int64) chunks.Sample { count++ var s sample if floatSample { @@ -694,7 +693,7 @@ func genHistogramAndFloatSeries(totalSeries, labelCount int, mint, maxt, step in }) } -func genSeriesFromSampleGenerator(totalSeries, labelCount int, mint, maxt, step int64, generator func(ts int64) tsdbutil.Sample) []storage.Series { +func genSeriesFromSampleGenerator(totalSeries, labelCount int, mint, maxt, step int64, generator func(ts int64) chunks.Sample) []storage.Series { if totalSeries == 0 || labelCount == 0 { return nil } @@ -707,7 +706,7 @@ func genSeriesFromSampleGenerator(totalSeries, labelCount int, mint, maxt, step for j := 1; len(lbls) < labelCount; j++ { lbls[defaultLabelName+strconv.Itoa(j)] = defaultLabelValue + strconv.Itoa(j) } - samples := make([]tsdbutil.Sample, 0, (maxt-mint)/step+1) + samples := make([]chunks.Sample, 0, (maxt-mint)/step+1) for t := mint; t < maxt; t += step { samples = append(samples, generator(t)) } @@ -727,7 +726,7 @@ func populateSeries(lbls []map[string]string, mint, maxt int64) []storage.Series if len(lbl) == 0 { continue } - samples := make([]tsdbutil.Sample, 0, maxt-mint+1) + samples := make([]chunks.Sample, 0, maxt-mint+1) for t := mint; t <= maxt; t++ { samples = append(samples, sample{t: t, f: rand.Float64()}) } diff --git a/tsdb/blockwriter_test.go b/tsdb/blockwriter_test.go index 84ea8d51b..d8240b53c 100644 --- a/tsdb/blockwriter_test.go +++ b/tsdb/blockwriter_test.go @@ -23,7 +23,7 @@ import ( "github.com/stretchr/testify/require" "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/tsdb/tsdbutil" + "github.com/prometheus/prometheus/tsdb/chunks" ) func TestBlockWriter(t *testing.T) { @@ -52,9 +52,9 @@ func TestBlockWriter(t *testing.T) { q, err := NewBlockQuerier(b, math.MinInt64, math.MaxInt64) require.NoError(t, err) series := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "", ".*")) - sample1 := []tsdbutil.Sample{sample{t: ts1, f: v1}} - sample2 := []tsdbutil.Sample{sample{t: ts2, f: v2}} - expectedSeries := map[string][]tsdbutil.Sample{"{a=\"b\"}": sample1, "{c=\"d\"}": sample2} + sample1 := []chunks.Sample{sample{t: ts1, f: v1}} + sample2 := []chunks.Sample{sample{t: ts2, f: v2}} + expectedSeries := map[string][]chunks.Sample{"{a=\"b\"}": sample1, "{c=\"d\"}": sample2} require.Equal(t, expectedSeries, series) require.NoError(t, w.Close()) diff --git a/tsdb/chunkenc/chunk.go b/tsdb/chunkenc/chunk.go index d6a6f4614..e7ff5b165 100644 --- a/tsdb/chunkenc/chunk.go +++ b/tsdb/chunkenc/chunk.go @@ -52,6 +52,20 @@ func IsValidEncoding(e Encoding) bool { return e == EncXOR || e == EncHistogram || e == EncFloatHistogram } +const ( + // MaxBytesPerXORChunk is the maximum size an XOR chunk can be. + MaxBytesPerXORChunk = 1024 + // TargetBytesPerHistogramChunk sets a size target for each histogram chunk. + TargetBytesPerHistogramChunk = 1024 + // MinSamplesPerHistogramChunk sets a minimum sample count for histogram chunks. This is desirable because a single + // histogram sample can be larger than TargetBytesPerHistogramChunk but we want to avoid too-small sample count + // chunks so we can achieve some measure of compression advantage even while dealing with really large histograms. + // Note that this minimum sample count is not enforced across chunk range boundaries (for example, if the chunk + // range is 100 and the first sample in the chunk range is 99, the next sample will be included in a new chunk + // resulting in the old chunk containing only a single sample). + MinSamplesPerHistogramChunk = 10 +) + // Chunk holds a sequence of sample pairs that can be iterated over and appended to. type Chunk interface { // Bytes returns the underlying byte slice of the chunk. diff --git a/tsdb/chunks/chunks.go b/tsdb/chunks/chunks.go index 9817fe47a..88fc5924b 100644 --- a/tsdb/chunks/chunks.go +++ b/tsdb/chunks/chunks.go @@ -140,6 +140,73 @@ type Meta struct { OOOLastMinTime, OOOLastMaxTime int64 } +// ChunkFromSamples requires all samples to have the same type. +func ChunkFromSamples(s []Sample) (Meta, error) { + return ChunkFromSamplesGeneric(SampleSlice(s)) +} + +// ChunkFromSamplesGeneric requires all samples to have the same type. +func ChunkFromSamplesGeneric(s Samples) (Meta, error) { + emptyChunk := Meta{Chunk: chunkenc.NewXORChunk()} + mint, maxt := int64(0), int64(0) + + if s.Len() > 0 { + mint, maxt = s.Get(0).T(), s.Get(s.Len()-1).T() + } + + if s.Len() == 0 { + return emptyChunk, nil + } + + sampleType := s.Get(0).Type() + c, err := chunkenc.NewEmptyChunk(sampleType.ChunkEncoding()) + if err != nil { + return Meta{}, err + } + + ca, _ := c.Appender() + var newChunk chunkenc.Chunk + + for i := 0; i < s.Len(); i++ { + switch sampleType { + case chunkenc.ValFloat: + ca.Append(s.Get(i).T(), s.Get(i).F()) + case chunkenc.ValHistogram: + newChunk, _, ca, err = ca.AppendHistogram(nil, s.Get(i).T(), s.Get(i).H(), false) + if err != nil { + return emptyChunk, err + } + if newChunk != nil { + return emptyChunk, fmt.Errorf("did not expect to start a second chunk") + } + case chunkenc.ValFloatHistogram: + newChunk, _, ca, err = ca.AppendFloatHistogram(nil, s.Get(i).T(), s.Get(i).FH(), false) + if err != nil { + return emptyChunk, err + } + if newChunk != nil { + return emptyChunk, fmt.Errorf("did not expect to start a second chunk") + } + default: + panic(fmt.Sprintf("unknown sample type %s", sampleType.String())) + } + } + return Meta{ + MinTime: mint, + MaxTime: maxt, + Chunk: c, + }, nil +} + +// PopulatedChunk creates a chunk populated with samples every second starting at minTime +func PopulatedChunk(numSamples int, minTime int64) (Meta, error) { + samples := make([]Sample, numSamples) + for i := 0; i < numSamples; i++ { + samples[i] = sample{t: minTime + int64(i*1000), f: 1.0} + } + return ChunkFromSamples(samples) +} + // Iterator iterates over the chunks of a single time series. type Iterator interface { // At returns the current meta. diff --git a/tsdb/chunks/samples.go b/tsdb/chunks/samples.go new file mode 100644 index 000000000..638660c70 --- /dev/null +++ b/tsdb/chunks/samples.go @@ -0,0 +1,89 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package chunks + +import ( + "github.com/prometheus/prometheus/model/histogram" + "github.com/prometheus/prometheus/tsdb/chunkenc" +) + +type Samples interface { + Get(i int) Sample + Len() int +} + +type Sample interface { + T() int64 + F() float64 + H() *histogram.Histogram + FH() *histogram.FloatHistogram + Type() chunkenc.ValueType +} + +type SampleSlice []Sample + +func (s SampleSlice) Get(i int) Sample { return s[i] } +func (s SampleSlice) Len() int { return len(s) } + +type sample struct { + t int64 + f float64 + h *histogram.Histogram + fh *histogram.FloatHistogram +} + +func (s sample) T() int64 { + return s.t +} + +func (s sample) F() float64 { + return s.f +} + +func (s sample) H() *histogram.Histogram { + return s.h +} + +func (s sample) FH() *histogram.FloatHistogram { + return s.fh +} + +func (s sample) Type() chunkenc.ValueType { + switch { + case s.h != nil: + return chunkenc.ValHistogram + case s.fh != nil: + return chunkenc.ValFloatHistogram + default: + return chunkenc.ValFloat + } +} + +// GenerateSamples starting at start and counting up numSamples. +func GenerateSamples(start, numSamples int) []Sample { + return generateSamples(start, numSamples, func(i int) Sample { + return sample{ + t: int64(i), + f: float64(i), + } + }) +} + +func generateSamples(start, numSamples int, gen func(int) Sample) []Sample { + samples := make([]Sample, 0, numSamples) + for i := start; i < start+numSamples; i++ { + samples = append(samples, gen(i)) + } + return samples +} diff --git a/tsdb/compact_test.go b/tsdb/compact_test.go index 512a6ecfb..2ef25d91a 100644 --- a/tsdb/compact_test.go +++ b/tsdb/compact_test.go @@ -38,7 +38,6 @@ import ( "github.com/prometheus/prometheus/tsdb/chunks" "github.com/prometheus/prometheus/tsdb/fileutil" "github.com/prometheus/prometheus/tsdb/tombstones" - "github.com/prometheus/prometheus/tsdb/tsdbutil" "github.com/prometheus/prometheus/tsdb/wlog" ) @@ -1316,7 +1315,7 @@ func TestHeadCompactionWithHistograms(t *testing.T) { minute := func(m int) int64 { return int64(m) * time.Minute.Milliseconds() } ctx := context.Background() appendHistogram := func( - lbls labels.Labels, from, to int, h *histogram.Histogram, exp *[]tsdbutil.Sample, + lbls labels.Labels, from, to int, h *histogram.Histogram, exp *[]chunks.Sample, ) { t.Helper() app := head.Appender(ctx) @@ -1345,7 +1344,7 @@ func TestHeadCompactionWithHistograms(t *testing.T) { } require.NoError(t, app.Commit()) } - appendFloat := func(lbls labels.Labels, from, to int, exp *[]tsdbutil.Sample) { + appendFloat := func(lbls labels.Labels, from, to int, exp *[]chunks.Sample) { t.Helper() app := head.Appender(ctx) for tsMinute := from; tsMinute <= to; tsMinute++ { @@ -1361,7 +1360,7 @@ func TestHeadCompactionWithHistograms(t *testing.T) { series2 = labels.FromStrings("foo", "bar2") series3 = labels.FromStrings("foo", "bar3") series4 = labels.FromStrings("foo", "bar4") - exp1, exp2, exp3, exp4 []tsdbutil.Sample + exp1, exp2, exp3, exp4 []chunks.Sample ) h := &histogram.Histogram{ Count: 15, @@ -1419,7 +1418,7 @@ func TestHeadCompactionWithHistograms(t *testing.T) { require.NoError(t, err) actHists := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.*")) - require.Equal(t, map[string][]tsdbutil.Sample{ + require.Equal(t, map[string][]chunks.Sample{ series1.String(): exp1, series2.String(): exp2, series3.String(): exp3, diff --git a/tsdb/db_test.go b/tsdb/db_test.go index e8b6d7c3b..ffe8b7cc0 100644 --- a/tsdb/db_test.go +++ b/tsdb/db_test.go @@ -87,18 +87,18 @@ func openTestDB(t testing.TB, opts *Options, rngs []int64) (db *DB) { } // query runs a matcher query against the querier and fully expands its data. -func query(t testing.TB, q storage.Querier, matchers ...*labels.Matcher) map[string][]tsdbutil.Sample { +func query(t testing.TB, q storage.Querier, matchers ...*labels.Matcher) map[string][]chunks.Sample { ss := q.Select(false, nil, matchers...) defer func() { require.NoError(t, q.Close()) }() var it chunkenc.Iterator - result := map[string][]tsdbutil.Sample{} + result := map[string][]chunks.Sample{} for ss.Next() { series := ss.At() - samples := []tsdbutil.Sample{} + samples := []chunks.Sample{} it = series.Iterator(it) for typ := it.Next(); typ != chunkenc.ValNone; typ = it.Next() { switch typ { @@ -131,12 +131,12 @@ func query(t testing.TB, q storage.Querier, matchers ...*labels.Matcher) map[str } // queryAndExpandChunks runs a matcher query against the querier and fully expands its data into samples. -func queryAndExpandChunks(t testing.TB, q storage.ChunkQuerier, matchers ...*labels.Matcher) map[string][][]tsdbutil.Sample { +func queryAndExpandChunks(t testing.TB, q storage.ChunkQuerier, matchers ...*labels.Matcher) map[string][][]chunks.Sample { s := queryChunks(t, q, matchers...) - res := make(map[string][][]tsdbutil.Sample) + res := make(map[string][][]chunks.Sample) for k, v := range s { - var samples [][]tsdbutil.Sample + var samples [][]chunks.Sample for _, chk := range v { sam, err := storage.ExpandSamples(chk.Chunk.Iterator(nil), nil) require.NoError(t, err) @@ -222,7 +222,7 @@ func TestDataAvailableOnlyAfterCommit(t *testing.T) { querier, err := db.Querier(context.TODO(), 0, 1) require.NoError(t, err) seriesSet := query(t, querier, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) - require.Equal(t, map[string][]tsdbutil.Sample{}, seriesSet) + require.Equal(t, map[string][]chunks.Sample{}, seriesSet) err = app.Commit() require.NoError(t, err) @@ -233,7 +233,7 @@ func TestDataAvailableOnlyAfterCommit(t *testing.T) { seriesSet = query(t, querier, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) - require.Equal(t, map[string][]tsdbutil.Sample{`{foo="bar"}`: {sample{t: 0, f: 0}}}, seriesSet) + require.Equal(t, map[string][]chunks.Sample{`{foo="bar"}`: {sample{t: 0, f: 0}}}, seriesSet) } // TestNoPanicAfterWALCorruption ensures that querying the db after a WAL corruption doesn't cause a panic. @@ -243,7 +243,7 @@ func TestNoPanicAfterWALCorruption(t *testing.T) { // Append until the first mmaped head chunk. // This is to ensure that all samples can be read from the mmaped chunks when the WAL is corrupted. - var expSamples []tsdbutil.Sample + var expSamples []chunks.Sample var maxt int64 ctx := context.Background() { @@ -289,7 +289,7 @@ func TestNoPanicAfterWALCorruption(t *testing.T) { require.NoError(t, err) seriesSet := query(t, querier, labels.MustNewMatcher(labels.MatchEqual, "", "")) // The last sample should be missing as it was after the WAL segment corruption. - require.Equal(t, map[string][]tsdbutil.Sample{`{foo="bar"}`: expSamples[0 : len(expSamples)-1]}, seriesSet) + require.Equal(t, map[string][]chunks.Sample{`{foo="bar"}`: expSamples[0 : len(expSamples)-1]}, seriesSet) } } @@ -312,7 +312,7 @@ func TestDataNotAvailableAfterRollback(t *testing.T) { seriesSet := query(t, querier, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) - require.Equal(t, map[string][]tsdbutil.Sample{}, seriesSet) + require.Equal(t, map[string][]chunks.Sample{}, seriesSet) } func TestDBAppenderAddRef(t *testing.T) { @@ -362,7 +362,7 @@ func TestDBAppenderAddRef(t *testing.T) { res := query(t, q, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) - require.Equal(t, map[string][]tsdbutil.Sample{ + require.Equal(t, map[string][]chunks.Sample{ labels.FromStrings("a", "b").String(): { sample{t: 123, f: 0}, sample{t: 124, f: 1}, @@ -455,7 +455,7 @@ Outer: res := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) - expSamples := make([]tsdbutil.Sample, 0, len(c.remaint)) + expSamples := make([]chunks.Sample, 0, len(c.remaint)) for _, ts := range c.remaint { expSamples = append(expSamples, sample{ts, smpls[ts], nil, nil}) } @@ -615,7 +615,7 @@ func TestSkippingInvalidValuesInSameTxn(t *testing.T) { ssMap := query(t, q, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) - require.Equal(t, map[string][]tsdbutil.Sample{ + require.Equal(t, map[string][]chunks.Sample{ labels.New(labels.Label{Name: "a", Value: "b"}).String(): {sample{0, 1, nil, nil}}, }, ssMap) @@ -632,7 +632,7 @@ func TestSkippingInvalidValuesInSameTxn(t *testing.T) { ssMap = query(t, q, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) - require.Equal(t, map[string][]tsdbutil.Sample{ + require.Equal(t, map[string][]chunks.Sample{ labels.New(labels.Label{Name: "a", Value: "b"}).String(): {sample{0, 1, nil, nil}, sample{10, 3, nil, nil}}, }, ssMap) } @@ -783,7 +783,7 @@ Outer: res := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) - expSamples := make([]tsdbutil.Sample, 0, len(c.remaint)) + expSamples := make([]chunks.Sample, 0, len(c.remaint)) for _, ts := range c.remaint { expSamples = append(expSamples, sample{ts, smpls[ts], nil, nil}) } @@ -869,9 +869,9 @@ func TestDB_e2e(t *testing.T) { }, } - seriesMap := map[string][]tsdbutil.Sample{} + seriesMap := map[string][]chunks.Sample{} for _, l := range lbls { - seriesMap[labels.New(l...).String()] = []tsdbutil.Sample{} + seriesMap[labels.New(l...).String()] = []chunks.Sample{} } db := openTestDB(t, nil, nil) @@ -884,7 +884,7 @@ func TestDB_e2e(t *testing.T) { for _, l := range lbls { lset := labels.New(l...) - series := []tsdbutil.Sample{} + series := []chunks.Sample{} ts := rand.Int63n(300) for i := 0; i < numDatapoints; i++ { @@ -942,7 +942,7 @@ func TestDB_e2e(t *testing.T) { mint := rand.Int63n(300) maxt := mint + rand.Int63n(timeInterval*int64(numDatapoints)) - expected := map[string][]tsdbutil.Sample{} + expected := map[string][]chunks.Sample{} // Build the mockSeriesSet. for _, m := range matched { @@ -956,7 +956,7 @@ func TestDB_e2e(t *testing.T) { require.NoError(t, err) ss := q.Select(false, nil, qry.ms...) - result := map[string][]tsdbutil.Sample{} + result := map[string][]chunks.Sample{} for ss.Next() { x := ss.At() @@ -1220,7 +1220,7 @@ func TestTombstoneClean(t *testing.T) { res := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) - expSamples := make([]tsdbutil.Sample, 0, len(c.remaint)) + expSamples := make([]chunks.Sample, 0, len(c.remaint)) for _, ts := range c.remaint { expSamples = append(expSamples, sample{ts, smpls[ts], nil, nil}) } @@ -2386,8 +2386,8 @@ func TestDBReadOnly(t *testing.T) { logger = log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr)) expBlocks []*Block expBlock *Block - expSeries map[string][]tsdbutil.Sample - expChunks map[string][][]tsdbutil.Sample + expSeries map[string][]chunks.Sample + expChunks map[string][][]chunks.Sample expDBHash []byte matchAll = labels.MustNewMatcher(labels.MatchEqual, "", "") err error @@ -2714,8 +2714,8 @@ func TestDBQueryDoesntSeeAppendsAfterCreation(t *testing.T) { require.Equal(t, map[string][]sample{`{foo="bar"}`: {{t: 0, f: 0}}}, seriesSet) } -func assureChunkFromSamples(t *testing.T, samples []tsdbutil.Sample) chunks.Meta { - chks, err := tsdbutil.ChunkFromSamples(samples) +func assureChunkFromSamples(t *testing.T, samples []chunks.Sample) chunks.Meta { + chks, err := chunks.ChunkFromSamples(samples) require.NoError(t, err) return chks } @@ -2723,11 +2723,11 @@ func assureChunkFromSamples(t *testing.T, samples []tsdbutil.Sample) chunks.Meta // TestChunkWriter_ReadAfterWrite ensures that chunk segment are cut at the set segment size and // that the resulted segments includes the expected chunks data. func TestChunkWriter_ReadAfterWrite(t *testing.T) { - chk1 := assureChunkFromSamples(t, []tsdbutil.Sample{sample{1, 1, nil, nil}}) - chk2 := assureChunkFromSamples(t, []tsdbutil.Sample{sample{1, 2, nil, nil}}) - chk3 := assureChunkFromSamples(t, []tsdbutil.Sample{sample{1, 3, nil, nil}}) - chk4 := assureChunkFromSamples(t, []tsdbutil.Sample{sample{1, 4, nil, nil}}) - chk5 := assureChunkFromSamples(t, []tsdbutil.Sample{sample{1, 5, nil, nil}}) + chk1 := assureChunkFromSamples(t, []chunks.Sample{sample{1, 1, nil, nil}}) + chk2 := assureChunkFromSamples(t, []chunks.Sample{sample{1, 2, nil, nil}}) + chk3 := assureChunkFromSamples(t, []chunks.Sample{sample{1, 3, nil, nil}}) + chk4 := assureChunkFromSamples(t, []chunks.Sample{sample{1, 4, nil, nil}}) + chk5 := assureChunkFromSamples(t, []chunks.Sample{sample{1, 5, nil, nil}}) chunkSize := len(chk1.Chunk.Bytes()) + chunks.MaxChunkLengthFieldSize + chunks.ChunkEncodingSize + crc32.Size tests := []struct { @@ -2927,11 +2927,11 @@ func TestRangeForTimestamp(t *testing.T) { // Regression test for https://github.com/prometheus/prometheus/pull/6514. func TestChunkReader_ConcurrentReads(t *testing.T) { chks := []chunks.Meta{ - assureChunkFromSamples(t, []tsdbutil.Sample{sample{1, 1, nil, nil}}), - assureChunkFromSamples(t, []tsdbutil.Sample{sample{1, 2, nil, nil}}), - assureChunkFromSamples(t, []tsdbutil.Sample{sample{1, 3, nil, nil}}), - assureChunkFromSamples(t, []tsdbutil.Sample{sample{1, 4, nil, nil}}), - assureChunkFromSamples(t, []tsdbutil.Sample{sample{1, 5, nil, nil}}), + assureChunkFromSamples(t, []chunks.Sample{sample{1, 1, nil, nil}}), + assureChunkFromSamples(t, []chunks.Sample{sample{1, 2, nil, nil}}), + assureChunkFromSamples(t, []chunks.Sample{sample{1, 3, nil, nil}}), + assureChunkFromSamples(t, []chunks.Sample{sample{1, 4, nil, nil}}), + assureChunkFromSamples(t, []chunks.Sample{sample{1, 5, nil, nil}}), } tempDir := t.TempDir() @@ -4156,7 +4156,7 @@ func TestOOOCompaction(t *testing.T) { addSample(90, 310) verifyDBSamples := func() { - var series1Samples, series2Samples []tsdbutil.Sample + var series1Samples, series2Samples []chunks.Sample for _, r := range [][2]int64{{90, 119}, {120, 239}, {240, 350}} { fromMins, toMins := r[0], r[1] for min := fromMins; min <= toMins; min++ { @@ -4165,7 +4165,7 @@ func TestOOOCompaction(t *testing.T) { series2Samples = append(series2Samples, sample{ts, float64(2 * ts), nil, nil}) } } - expRes := map[string][]tsdbutil.Sample{ + expRes := map[string][]chunks.Sample{ series1.String(): series1Samples, series2.String(): series2Samples, } @@ -4225,14 +4225,14 @@ func TestOOOCompaction(t *testing.T) { checkEmptyOOOChunk(series2) verifySamples := func(block *Block, fromMins, toMins int64) { - series1Samples := make([]tsdbutil.Sample, 0, toMins-fromMins+1) - series2Samples := make([]tsdbutil.Sample, 0, toMins-fromMins+1) + series1Samples := make([]chunks.Sample, 0, toMins-fromMins+1) + series2Samples := make([]chunks.Sample, 0, toMins-fromMins+1) for min := fromMins; min <= toMins; min++ { ts := min * time.Minute.Milliseconds() series1Samples = append(series1Samples, sample{ts, float64(ts), nil, nil}) series2Samples = append(series2Samples, sample{ts, float64(2 * ts), nil, nil}) } - expRes := map[string][]tsdbutil.Sample{ + expRes := map[string][]chunks.Sample{ series1.String(): series1Samples, series2.String(): series2Samples, } @@ -4356,14 +4356,14 @@ func TestOOOCompactionWithNormalCompaction(t *testing.T) { } verifySamples := func(block *Block, fromMins, toMins int64) { - series1Samples := make([]tsdbutil.Sample, 0, toMins-fromMins+1) - series2Samples := make([]tsdbutil.Sample, 0, toMins-fromMins+1) + series1Samples := make([]chunks.Sample, 0, toMins-fromMins+1) + series2Samples := make([]chunks.Sample, 0, toMins-fromMins+1) for min := fromMins; min <= toMins; min++ { ts := min * time.Minute.Milliseconds() series1Samples = append(series1Samples, sample{ts, float64(ts), nil, nil}) series2Samples = append(series2Samples, sample{ts, float64(2 * ts), nil, nil}) } - expRes := map[string][]tsdbutil.Sample{ + expRes := map[string][]chunks.Sample{ series1.String(): series1Samples, series2.String(): series2Samples, } @@ -4456,14 +4456,14 @@ func TestOOOCompactionWithDisabledWriteLog(t *testing.T) { } verifySamples := func(block *Block, fromMins, toMins int64) { - series1Samples := make([]tsdbutil.Sample, 0, toMins-fromMins+1) - series2Samples := make([]tsdbutil.Sample, 0, toMins-fromMins+1) + series1Samples := make([]chunks.Sample, 0, toMins-fromMins+1) + series2Samples := make([]chunks.Sample, 0, toMins-fromMins+1) for min := fromMins; min <= toMins; min++ { ts := min * time.Minute.Milliseconds() series1Samples = append(series1Samples, sample{ts, float64(ts), nil, nil}) series2Samples = append(series2Samples, sample{ts, float64(2 * ts), nil, nil}) } - expRes := map[string][]tsdbutil.Sample{ + expRes := map[string][]chunks.Sample{ series1.String(): series1Samples, series2.String(): series2Samples, } @@ -4549,14 +4549,14 @@ func TestOOOQueryAfterRestartWithSnapshotAndRemovedWBL(t *testing.T) { } verifySamples := func(fromMins, toMins int64) { - series1Samples := make([]tsdbutil.Sample, 0, toMins-fromMins+1) - series2Samples := make([]tsdbutil.Sample, 0, toMins-fromMins+1) + series1Samples := make([]chunks.Sample, 0, toMins-fromMins+1) + series2Samples := make([]chunks.Sample, 0, toMins-fromMins+1) for min := fromMins; min <= toMins; min++ { ts := min * time.Minute.Milliseconds() series1Samples = append(series1Samples, sample{ts, float64(ts), nil, nil}) series2Samples = append(series2Samples, sample{ts, float64(2 * ts), nil, nil}) } - expRes := map[string][]tsdbutil.Sample{ + expRes := map[string][]chunks.Sample{ series1.String(): series1Samples, series2.String(): series2Samples, } @@ -4598,7 +4598,7 @@ func Test_Querier_OOOQuery(t *testing.T) { series1 := labels.FromStrings("foo", "bar1") minutes := func(m int64) int64 { return m * time.Minute.Milliseconds() } - addSample := func(db *DB, fromMins, toMins, queryMinT, queryMaxT int64, expSamples []tsdbutil.Sample) ([]tsdbutil.Sample, int) { + addSample := func(db *DB, fromMins, toMins, queryMinT, queryMaxT int64, expSamples []chunks.Sample) ([]chunks.Sample, int) { app := db.Appender(context.Background()) totalAppended := 0 for min := fromMins; min <= toMins; min += time.Minute.Milliseconds() { @@ -4649,7 +4649,7 @@ func Test_Querier_OOOQuery(t *testing.T) { require.NoError(t, db.Close()) }() - var expSamples []tsdbutil.Sample + var expSamples []chunks.Sample // Add in-order samples. expSamples, _ = addSample(db, tc.inOrderMinT, tc.inOrderMaxT, tc.queryMinT, tc.queryMaxT, expSamples) @@ -4683,7 +4683,7 @@ func Test_ChunkQuerier_OOOQuery(t *testing.T) { series1 := labels.FromStrings("foo", "bar1") minutes := func(m int64) int64 { return m * time.Minute.Milliseconds() } - addSample := func(db *DB, fromMins, toMins, queryMinT, queryMaxT int64, expSamples []tsdbutil.Sample) ([]tsdbutil.Sample, int) { + addSample := func(db *DB, fromMins, toMins, queryMinT, queryMaxT int64, expSamples []chunks.Sample) ([]chunks.Sample, int) { app := db.Appender(context.Background()) totalAppended := 0 for min := fromMins; min <= toMins; min += time.Minute.Milliseconds() { @@ -4734,7 +4734,7 @@ func Test_ChunkQuerier_OOOQuery(t *testing.T) { require.NoError(t, db.Close()) }() - var expSamples []tsdbutil.Sample + var expSamples []chunks.Sample // Add in-order samples. expSamples, _ = addSample(db, tc.inOrderMinT, tc.inOrderMaxT, tc.queryMinT, tc.queryMaxT, expSamples) @@ -4754,7 +4754,7 @@ func Test_ChunkQuerier_OOOQuery(t *testing.T) { require.NotNil(t, chks[series1.String()]) require.Equal(t, 1, len(chks)) require.Equal(t, float64(oooSamples), prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamplesAppended), "number of ooo appended samples mismatch") - var gotSamples []tsdbutil.Sample + var gotSamples []chunks.Sample for _, chunk := range chks[series1.String()] { it := chunk.Chunk.Iterator(nil) for it.Next() == chunkenc.ValFloat { @@ -4782,7 +4782,7 @@ func TestOOOAppendAndQuery(t *testing.T) { s2 := labels.FromStrings("foo", "bar2") minutes := func(m int64) int64 { return m * time.Minute.Milliseconds() } - appendedSamples := make(map[string][]tsdbutil.Sample) + appendedSamples := make(map[string][]chunks.Sample) totalSamples := 0 addSample := func(lbls labels.Labels, fromMins, toMins int64, faceError bool) { app := db.Appender(context.Background()) @@ -4819,7 +4819,7 @@ func TestOOOAppendAndQuery(t *testing.T) { appendedSamples[k] = v } - expSamples := make(map[string][]tsdbutil.Sample) + expSamples := make(map[string][]chunks.Sample) for k, samples := range appendedSamples { for _, s := range samples { if s.T() < from { @@ -4903,7 +4903,7 @@ func TestOOODisabled(t *testing.T) { s1 := labels.FromStrings("foo", "bar1") minutes := func(m int64) int64 { return m * time.Minute.Milliseconds() } - expSamples := make(map[string][]tsdbutil.Sample) + expSamples := make(map[string][]chunks.Sample) totalSamples := 0 failedSamples := 0 addSample := func(lbls labels.Labels, fromMins, toMins int64, faceError bool) { @@ -4971,7 +4971,7 @@ func TestWBLAndMmapReplay(t *testing.T) { s1 := labels.FromStrings("foo", "bar1") minutes := func(m int64) int64 { return m * time.Minute.Milliseconds() } - expSamples := make(map[string][]tsdbutil.Sample) + expSamples := make(map[string][]chunks.Sample) totalSamples := 0 addSample := func(lbls labels.Labels, fromMins, toMins int64) { app := db.Appender(context.Background()) @@ -4987,7 +4987,7 @@ func TestWBLAndMmapReplay(t *testing.T) { require.NoError(t, app.Commit()) } - testQuery := func(exp map[string][]tsdbutil.Sample) { + testQuery := func(exp map[string][]chunks.Sample) { querier, err := db.Querier(context.TODO(), math.MinInt64, math.MaxInt64) require.NoError(t, err) @@ -5017,7 +5017,7 @@ func TestWBLAndMmapReplay(t *testing.T) { ms, created, err := db.head.getOrCreate(s1.Hash(), s1) require.False(t, created) require.NoError(t, err) - var s1MmapSamples []tsdbutil.Sample + var s1MmapSamples []chunks.Sample for _, mc := range ms.ooo.oooMmappedChunks { chk, err := db.head.chunkDiskMapper.Chunk(mc.ref) require.NoError(t, err) @@ -5076,7 +5076,7 @@ func TestWBLAndMmapReplay(t *testing.T) { require.Equal(t, oooMint, db.head.MinOOOTime()) require.Equal(t, oooMaxt, db.head.MaxOOOTime()) inOrderSample := expSamples[s1.String()][len(expSamples[s1.String()])-1] - testQuery(map[string][]tsdbutil.Sample{ + testQuery(map[string][]chunks.Sample{ s1.String(): append(s1MmapSamples, inOrderSample), }) require.NoError(t, db.Close()) @@ -5247,12 +5247,12 @@ func TestOOOCompactionFailure(t *testing.T) { require.Equal(t, int64(0), f.Size()) verifySamples := func(block *Block, fromMins, toMins int64) { - series1Samples := make([]tsdbutil.Sample, 0, toMins-fromMins+1) + series1Samples := make([]chunks.Sample, 0, toMins-fromMins+1) for min := fromMins; min <= toMins; min++ { ts := min * time.Minute.Milliseconds() series1Samples = append(series1Samples, sample{ts, float64(ts), nil, nil}) } - expRes := map[string][]tsdbutil.Sample{ + expRes := map[string][]chunks.Sample{ series1.String(): series1Samples, } @@ -5295,7 +5295,7 @@ func TestWBLCorruption(t *testing.T) { }) series1 := labels.FromStrings("foo", "bar1") - var allSamples, expAfterRestart []tsdbutil.Sample + var allSamples, expAfterRestart []chunks.Sample addSamples := func(fromMins, toMins int64, afterRestart bool) { app := db.Appender(context.Background()) for min := fromMins; min <= toMins; min++ { @@ -5367,12 +5367,12 @@ func TestWBLCorruption(t *testing.T) { require.NoError(t, err) require.Greater(t, f2.Size(), int64(100)) - verifySamples := func(expSamples []tsdbutil.Sample) { + verifySamples := func(expSamples []chunks.Sample) { sort.Slice(expSamples, func(i, j int) bool { return expSamples[i].T() < expSamples[j].T() }) - expRes := map[string][]tsdbutil.Sample{ + expRes := map[string][]chunks.Sample{ series1.String(): expSamples, } @@ -5441,7 +5441,7 @@ func TestOOOMmapCorruption(t *testing.T) { }) series1 := labels.FromStrings("foo", "bar1") - var allSamples, expInMmapChunks []tsdbutil.Sample + var allSamples, expInMmapChunks []chunks.Sample addSamples := func(fromMins, toMins int64, inMmapAfterCorruption bool) { app := db.Appender(context.Background()) for min := fromMins; min <= toMins; min++ { @@ -5475,12 +5475,12 @@ func TestOOOMmapCorruption(t *testing.T) { db.head.chunkDiskMapper.CutNewFile() addSamples(260, 290, false) - verifySamples := func(expSamples []tsdbutil.Sample) { + verifySamples := func(expSamples []chunks.Sample) { sort.Slice(expSamples, func(i, j int) bool { return expSamples[i].T() < expSamples[j].T() }) - expRes := map[string][]tsdbutil.Sample{ + expRes := map[string][]chunks.Sample{ series1.String(): expSamples, } @@ -5577,7 +5577,7 @@ func TestOutOfOrderRuntimeConfig(t *testing.T) { } series1 := labels.FromStrings("foo", "bar1") - addSamples := func(t *testing.T, db *DB, fromMins, toMins int64, success bool, allSamples []tsdbutil.Sample) []tsdbutil.Sample { + addSamples := func(t *testing.T, db *DB, fromMins, toMins int64, success bool, allSamples []chunks.Sample) []chunks.Sample { app := db.Appender(context.Background()) for min := fromMins; min <= toMins; min++ { ts := min * time.Minute.Milliseconds() @@ -5593,12 +5593,12 @@ func TestOutOfOrderRuntimeConfig(t *testing.T) { return allSamples } - verifySamples := func(t *testing.T, db *DB, expSamples []tsdbutil.Sample) { + verifySamples := func(t *testing.T, db *DB, expSamples []chunks.Sample) { sort.Slice(expSamples, func(i, j int) bool { return expSamples[i].T() < expSamples[j].T() }) - expRes := map[string][]tsdbutil.Sample{ + expRes := map[string][]chunks.Sample{ series1.String(): expSamples, } @@ -5626,7 +5626,7 @@ func TestOutOfOrderRuntimeConfig(t *testing.T) { } t.Run("increase time window", func(t *testing.T) { - var allSamples []tsdbutil.Sample + var allSamples []chunks.Sample db := getDB(30 * time.Minute.Milliseconds()) // In-order. @@ -5656,7 +5656,7 @@ func TestOutOfOrderRuntimeConfig(t *testing.T) { }) t.Run("decrease time window and increase again", func(t *testing.T) { - var allSamples []tsdbutil.Sample + var allSamples []chunks.Sample db := getDB(60 * time.Minute.Milliseconds()) // In-order. @@ -5695,7 +5695,7 @@ func TestOutOfOrderRuntimeConfig(t *testing.T) { }) t.Run("disabled to enabled", func(t *testing.T) { - var allSamples []tsdbutil.Sample + var allSamples []chunks.Sample db := getDB(0) // In-order. @@ -5724,7 +5724,7 @@ func TestOutOfOrderRuntimeConfig(t *testing.T) { }) t.Run("enabled to disabled", func(t *testing.T) { - var allSamples []tsdbutil.Sample + var allSamples []chunks.Sample db := getDB(60 * time.Minute.Milliseconds()) // In-order. @@ -5754,7 +5754,7 @@ func TestOutOfOrderRuntimeConfig(t *testing.T) { }) t.Run("disabled to disabled", func(t *testing.T) { - var allSamples []tsdbutil.Sample + var allSamples []chunks.Sample db := getDB(0) // In-order. @@ -5795,13 +5795,13 @@ func TestNoGapAfterRestartWithOOO(t *testing.T) { } verifySamples := func(t *testing.T, db *DB, fromMins, toMins int64) { - var expSamples []tsdbutil.Sample + var expSamples []chunks.Sample for min := fromMins; min <= toMins; min++ { ts := min * time.Minute.Milliseconds() expSamples = append(expSamples, sample{t: ts, f: float64(ts)}) } - expRes := map[string][]tsdbutil.Sample{ + expRes := map[string][]chunks.Sample{ series1.String(): expSamples, } @@ -5898,7 +5898,7 @@ func TestWblReplayAfterOOODisableAndRestart(t *testing.T) { }) series1 := labels.FromStrings("foo", "bar1") - var allSamples []tsdbutil.Sample + var allSamples []chunks.Sample addSamples := func(fromMins, toMins int64) { app := db.Appender(context.Background()) for min := fromMins; min <= toMins; min++ { @@ -5915,12 +5915,12 @@ func TestWblReplayAfterOOODisableAndRestart(t *testing.T) { // OOO samples. addSamples(250, 260) - verifySamples := func(expSamples []tsdbutil.Sample) { + verifySamples := func(expSamples []chunks.Sample) { sort.Slice(expSamples, func(i, j int) bool { return expSamples[i].T() < expSamples[j].T() }) - expRes := map[string][]tsdbutil.Sample{ + expRes := map[string][]chunks.Sample{ series1.String(): expSamples, } @@ -5957,7 +5957,7 @@ func TestPanicOnApplyConfig(t *testing.T) { }) series1 := labels.FromStrings("foo", "bar1") - var allSamples []tsdbutil.Sample + var allSamples []chunks.Sample addSamples := func(fromMins, toMins int64) { app := db.Appender(context.Background()) for min := fromMins; min <= toMins; min++ { @@ -6005,7 +6005,7 @@ func TestDiskFillingUpAfterDisablingOOO(t *testing.T) { }) series1 := labels.FromStrings("foo", "bar1") - var allSamples []tsdbutil.Sample + var allSamples []chunks.Sample addSamples := func(fromMins, toMins int64) { app := db.Appender(context.Background()) for min := fromMins; min <= toMins; min++ { @@ -6096,7 +6096,7 @@ func testHistogramAppendAndQueryHelper(t *testing.T, floatHistogram bool) { ctx := context.Background() appendHistogram := func( lbls labels.Labels, tsMinute int, h *histogram.Histogram, - exp *[]tsdbutil.Sample, expCRH histogram.CounterResetHint, + exp *[]chunks.Sample, expCRH histogram.CounterResetHint, ) { t.Helper() var err error @@ -6115,7 +6115,7 @@ func testHistogramAppendAndQueryHelper(t *testing.T, floatHistogram bool) { require.NoError(t, err) require.NoError(t, app.Commit()) } - appendFloat := func(lbls labels.Labels, tsMinute int, val float64, exp *[]tsdbutil.Sample) { + appendFloat := func(lbls labels.Labels, tsMinute int, val float64, exp *[]chunks.Sample) { t.Helper() app := db.Appender(ctx) _, err := app.Append(0, lbls, minute(tsMinute), val) @@ -6124,7 +6124,7 @@ func testHistogramAppendAndQueryHelper(t *testing.T, floatHistogram bool) { *exp = append(*exp, sample{t: minute(tsMinute), f: val}) } - testQuery := func(name, value string, exp map[string][]tsdbutil.Sample) { + testQuery := func(name, value string, exp map[string][]chunks.Sample) { t.Helper() q, err := db.Querier(ctx, math.MinInt64, math.MaxInt64) require.NoError(t, err) @@ -6155,7 +6155,7 @@ func testHistogramAppendAndQueryHelper(t *testing.T, floatHistogram bool) { series2 = labels.FromStrings("foo", "bar2") series3 = labels.FromStrings("foo", "bar3") series4 = labels.FromStrings("foo", "bar4") - exp1, exp2, exp3, exp4 []tsdbutil.Sample + exp1, exp2, exp3, exp4 []chunks.Sample ) // TODO(codesome): test everything for negative buckets as well. @@ -6163,23 +6163,23 @@ func testHistogramAppendAndQueryHelper(t *testing.T, floatHistogram bool) { h := baseH.Copy() // This is shared across all sub tests. appendHistogram(series1, 100, h, &exp1, histogram.UnknownCounterReset) - testQuery("foo", "bar1", map[string][]tsdbutil.Sample{series1.String(): exp1}) + testQuery("foo", "bar1", map[string][]chunks.Sample{series1.String(): exp1}) h.PositiveBuckets[0]++ h.NegativeBuckets[0] += 2 h.Count += 10 appendHistogram(series1, 101, h, &exp1, histogram.NotCounterReset) - testQuery("foo", "bar1", map[string][]tsdbutil.Sample{series1.String(): exp1}) + testQuery("foo", "bar1", map[string][]chunks.Sample{series1.String(): exp1}) t.Run("changing schema", func(t *testing.T) { h.Schema = 2 appendHistogram(series1, 102, h, &exp1, histogram.UnknownCounterReset) - testQuery("foo", "bar1", map[string][]tsdbutil.Sample{series1.String(): exp1}) + testQuery("foo", "bar1", map[string][]chunks.Sample{series1.String(): exp1}) // Schema back to old. h.Schema = 1 appendHistogram(series1, 103, h, &exp1, histogram.UnknownCounterReset) - testQuery("foo", "bar1", map[string][]tsdbutil.Sample{series1.String(): exp1}) + testQuery("foo", "bar1", map[string][]chunks.Sample{series1.String(): exp1}) }) t.Run("new buckets incoming", func(t *testing.T) { @@ -6208,7 +6208,7 @@ func testHistogramAppendAndQueryHelper(t *testing.T, floatHistogram bool) { h.PositiveBuckets = append(h.PositiveBuckets, 1) h.Count += 3 appendHistogram(series1, 104, h, &exp1, histogram.NotCounterReset) - testQuery("foo", "bar1", map[string][]tsdbutil.Sample{series1.String(): exp1}) + testQuery("foo", "bar1", map[string][]chunks.Sample{series1.String(): exp1}) // Because of the previous two histograms being on the active chunk, // and the next append is only adding a new bucket, the active chunk @@ -6246,7 +6246,7 @@ func testHistogramAppendAndQueryHelper(t *testing.T, floatHistogram bool) { // {2, 1, -1, 0, 1} -> {2, 1, 0, -1, 0, 1} h.PositiveBuckets = append(h.PositiveBuckets[:2], append([]int64{0}, h.PositiveBuckets[2:]...)...) appendHistogram(series1, 105, h, &exp1, histogram.NotCounterReset) - testQuery("foo", "bar1", map[string][]tsdbutil.Sample{series1.String(): exp1}) + testQuery("foo", "bar1", map[string][]chunks.Sample{series1.String(): exp1}) // We add 4 more histograms to clear out the buffer and see the re-encoded histograms. appendHistogram(series1, 106, h, &exp1, histogram.NotCounterReset) @@ -6279,14 +6279,14 @@ func testHistogramAppendAndQueryHelper(t *testing.T, floatHistogram bool) { exp1[l-6] = sample{t: exp1[l-6].T(), h: h6} } - testQuery("foo", "bar1", map[string][]tsdbutil.Sample{series1.String(): exp1}) + testQuery("foo", "bar1", map[string][]chunks.Sample{series1.String(): exp1}) }) t.Run("buckets disappearing", func(t *testing.T) { h.PositiveSpans[1].Length-- h.PositiveBuckets = h.PositiveBuckets[:len(h.PositiveBuckets)-1] appendHistogram(series1, 110, h, &exp1, histogram.CounterReset) - testQuery("foo", "bar1", map[string][]tsdbutil.Sample{series1.String(): exp1}) + testQuery("foo", "bar1", map[string][]chunks.Sample{series1.String(): exp1}) }) }) @@ -6294,22 +6294,22 @@ func testHistogramAppendAndQueryHelper(t *testing.T, floatHistogram bool) { appendFloat(series2, 100, 100, &exp2) appendFloat(series2, 101, 101, &exp2) appendFloat(series2, 102, 102, &exp2) - testQuery("foo", "bar2", map[string][]tsdbutil.Sample{series2.String(): exp2}) + testQuery("foo", "bar2", map[string][]chunks.Sample{series2.String(): exp2}) h := baseH.Copy() appendHistogram(series2, 103, h, &exp2, histogram.UnknownCounterReset) appendHistogram(series2, 104, h, &exp2, histogram.NotCounterReset) appendHistogram(series2, 105, h, &exp2, histogram.NotCounterReset) - testQuery("foo", "bar2", map[string][]tsdbutil.Sample{series2.String(): exp2}) + testQuery("foo", "bar2", map[string][]chunks.Sample{series2.String(): exp2}) // Switching between float and histograms again. appendFloat(series2, 106, 106, &exp2) appendFloat(series2, 107, 107, &exp2) - testQuery("foo", "bar2", map[string][]tsdbutil.Sample{series2.String(): exp2}) + testQuery("foo", "bar2", map[string][]chunks.Sample{series2.String(): exp2}) appendHistogram(series2, 108, h, &exp2, histogram.UnknownCounterReset) appendHistogram(series2, 109, h, &exp2, histogram.NotCounterReset) - testQuery("foo", "bar2", map[string][]tsdbutil.Sample{series2.String(): exp2}) + testQuery("foo", "bar2", map[string][]chunks.Sample{series2.String(): exp2}) }) t.Run("series starting with histogram and then getting float", func(t *testing.T) { @@ -6317,21 +6317,21 @@ func testHistogramAppendAndQueryHelper(t *testing.T, floatHistogram bool) { appendHistogram(series3, 101, h, &exp3, histogram.UnknownCounterReset) appendHistogram(series3, 102, h, &exp3, histogram.NotCounterReset) appendHistogram(series3, 103, h, &exp3, histogram.NotCounterReset) - testQuery("foo", "bar3", map[string][]tsdbutil.Sample{series3.String(): exp3}) + testQuery("foo", "bar3", map[string][]chunks.Sample{series3.String(): exp3}) appendFloat(series3, 104, 100, &exp3) appendFloat(series3, 105, 101, &exp3) appendFloat(series3, 106, 102, &exp3) - testQuery("foo", "bar3", map[string][]tsdbutil.Sample{series3.String(): exp3}) + testQuery("foo", "bar3", map[string][]chunks.Sample{series3.String(): exp3}) // Switching between histogram and float again. appendHistogram(series3, 107, h, &exp3, histogram.UnknownCounterReset) appendHistogram(series3, 108, h, &exp3, histogram.NotCounterReset) - testQuery("foo", "bar3", map[string][]tsdbutil.Sample{series3.String(): exp3}) + testQuery("foo", "bar3", map[string][]chunks.Sample{series3.String(): exp3}) appendFloat(series3, 109, 106, &exp3) appendFloat(series3, 110, 107, &exp3) - testQuery("foo", "bar3", map[string][]tsdbutil.Sample{series3.String(): exp3}) + testQuery("foo", "bar3", map[string][]chunks.Sample{series3.String(): exp3}) }) t.Run("query mix of histogram and float series", func(t *testing.T) { @@ -6340,7 +6340,7 @@ func testHistogramAppendAndQueryHelper(t *testing.T, floatHistogram bool) { appendFloat(series4, 101, 101, &exp4) appendFloat(series4, 102, 102, &exp4) - testQuery("foo", "bar.*", map[string][]tsdbutil.Sample{ + testQuery("foo", "bar.*", map[string][]chunks.Sample{ series1.String(): exp1, series2.String(): exp2, series3.String(): exp3, @@ -6365,7 +6365,7 @@ func TestQueryHistogramFromBlocksWithCompaction(t *testing.T) { ctx := context.Background() var it chunkenc.Iterator - exp := make(map[string][]tsdbutil.Sample) + exp := make(map[string][]chunks.Sample) for _, series := range blockSeries { createBlock(t, db.Dir(), series) @@ -6448,7 +6448,7 @@ func TestQueryHistogramFromBlocksWithCompaction(t *testing.T) { t.Run("serial blocks with either histograms or floats in a block and not both", func(t *testing.T) { testBlockQuerying(t, genHistogramSeries(10, 5, minute(0), minute(119), minute(1), floatHistogram), - genSeriesFromSampleGenerator(10, 5, minute(120), minute(239), minute(1), func(ts int64) tsdbutil.Sample { + genSeriesFromSampleGenerator(10, 5, minute(120), minute(239), minute(1), func(ts int64) chunks.Sample { return sample{t: ts, f: rand.Float64()} }), genHistogramSeries(10, 5, minute(240), minute(359), minute(1), floatHistogram), @@ -6460,7 +6460,7 @@ func TestQueryHistogramFromBlocksWithCompaction(t *testing.T) { genHistogramAndFloatSeries(10, 5, minute(0), minute(60), minute(1), floatHistogram), genHistogramSeries(10, 5, minute(61), minute(120), minute(1), floatHistogram), genHistogramAndFloatSeries(10, 5, minute(121), minute(180), minute(1), floatHistogram), - genSeriesFromSampleGenerator(10, 5, minute(181), minute(240), minute(1), func(ts int64) tsdbutil.Sample { + genSeriesFromSampleGenerator(10, 5, minute(181), minute(240), minute(1), func(ts int64) chunks.Sample { return sample{t: ts, f: rand.Float64()} }), ) @@ -6477,7 +6477,7 @@ func TestQueryHistogramFromBlocksWithCompaction(t *testing.T) { t.Run("overlapping blocks with only histograms and only float in a series", func(t *testing.T) { testBlockQuerying(t, genHistogramSeries(10, 5, minute(0), minute(120), minute(3), floatHistogram), - genSeriesFromSampleGenerator(10, 5, minute(1), minute(120), minute(3), func(ts int64) tsdbutil.Sample { + genSeriesFromSampleGenerator(10, 5, minute(1), minute(120), minute(3), func(ts int64) chunks.Sample { return sample{t: ts, f: rand.Float64()} }), genHistogramSeries(10, 5, minute(2), minute(120), minute(3), floatHistogram), @@ -6489,7 +6489,7 @@ func TestQueryHistogramFromBlocksWithCompaction(t *testing.T) { genHistogramAndFloatSeries(10, 5, minute(0), minute(60), minute(3), floatHistogram), genHistogramSeries(10, 5, minute(46), minute(100), minute(3), floatHistogram), genHistogramAndFloatSeries(10, 5, minute(89), minute(140), minute(3), floatHistogram), - genSeriesFromSampleGenerator(10, 5, minute(126), minute(200), minute(3), func(ts int64) tsdbutil.Sample { + genSeriesFromSampleGenerator(10, 5, minute(126), minute(200), minute(3), func(ts int64) chunks.Sample { return sample{t: ts, f: rand.Float64()} }), ) @@ -6546,7 +6546,7 @@ func TestNativeHistogramFlag(t *testing.T) { q, err := db.Querier(context.Background(), math.MinInt, math.MaxInt64) require.NoError(t, err) act := query(t, q, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) - require.Equal(t, map[string][]tsdbutil.Sample{ + require.Equal(t, map[string][]chunks.Sample{ l.String(): {sample{t: 200, h: h}, sample{t: 205, fh: h.ToFloat()}}, }, act) } @@ -6559,7 +6559,7 @@ func TestNativeHistogramFlag(t *testing.T) { // actual series contains a counter reset hint "UnknownCounterReset". // "GaugeType" hints are still strictly checked, and any "UnknownCounterReset" // in an expected series has to be matched precisely by the actual series. -func compareSeries(t require.TestingT, expected, actual map[string][]tsdbutil.Sample) { +func compareSeries(t require.TestingT, expected, actual map[string][]chunks.Sample) { if len(expected) != len(actual) { // The reason for the difference is not the counter reset hints // (alone), so let's use the pretty diffing by the require diff --git a/tsdb/head.go b/tsdb/head.go index 34a289a98..cfda3f644 100644 --- a/tsdb/head.go +++ b/tsdb/head.go @@ -42,7 +42,6 @@ import ( "github.com/prometheus/prometheus/tsdb/index" "github.com/prometheus/prometheus/tsdb/record" "github.com/prometheus/prometheus/tsdb/tombstones" - "github.com/prometheus/prometheus/tsdb/tsdbutil" "github.com/prometheus/prometheus/tsdb/wlog" "github.com/prometheus/prometheus/util/zeropool" ) @@ -1918,7 +1917,7 @@ type sample struct { fh *histogram.FloatHistogram } -func newSample(t int64, v float64, h *histogram.Histogram, fh *histogram.FloatHistogram) tsdbutil.Sample { +func newSample(t int64, v float64, h *histogram.Histogram, fh *histogram.FloatHistogram) chunks.Sample { return sample{t, v, h, fh} } @@ -1967,7 +1966,8 @@ type memSeries struct { mmMaxTime int64 // Max time of any mmapped chunk, only used during WAL replay. - nextAt int64 // Timestamp at which to cut the next chunk. + nextAt int64 // Timestamp at which to cut the next chunk. + histogramChunkHasComputedEndTime bool // True if nextAt has been predicted for the current histograms chunk; false otherwise. // We keep the last value here (in addition to appending it to the chunk) so we can check for duplicates. lastValue float64 diff --git a/tsdb/head_append.go b/tsdb/head_append.go index 8c548fcd9..901694375 100644 --- a/tsdb/head_append.go +++ b/tsdb/head_append.go @@ -1160,7 +1160,7 @@ func (s *memSeries) appendHistogram(t int64, h *histogram.Histogram, appendID ui // Ignoring ok is ok, since we don't want to compare to the wrong previous appender anyway. prevApp, _ := s.app.(*chunkenc.HistogramAppender) - c, sampleInOrder, chunkCreated := s.appendPreprocessor(t, chunkenc.EncHistogram, o) + c, sampleInOrder, chunkCreated := s.histogramsAppendPreprocessor(t, chunkenc.EncHistogram, o) if !sampleInOrder { return sampleInOrder, chunkCreated } @@ -1217,7 +1217,7 @@ func (s *memSeries) appendFloatHistogram(t int64, fh *histogram.FloatHistogram, // Ignoring ok is ok, since we don't want to compare to the wrong previous appender anyway. prevApp, _ := s.app.(*chunkenc.FloatHistogramAppender) - c, sampleInOrder, chunkCreated := s.appendPreprocessor(t, chunkenc.EncFloatHistogram, o) + c, sampleInOrder, chunkCreated := s.histogramsAppendPreprocessor(t, chunkenc.EncFloatHistogram, o) if !sampleInOrder { return sampleInOrder, chunkCreated } @@ -1262,10 +1262,16 @@ func (s *memSeries) appendFloatHistogram(t int64, fh *histogram.FloatHistogram, return true, true } -// appendPreprocessor takes care of cutting new chunks and m-mapping old chunks. +// appendPreprocessor takes care of cutting new XOR chunks and m-mapping old ones. XOR chunks are cut based on the +// number of samples they contain with a soft cap in bytes. // It is unsafe to call this concurrently with s.iterator(...) without holding the series lock. // This should be called only when appending data. func (s *memSeries) appendPreprocessor(t int64, e chunkenc.Encoding, o chunkOpts) (c *memChunk, sampleInOrder, chunkCreated bool) { + // We target chunkenc.MaxBytesPerXORChunk as a hard for the size of an XOR chunk. We must determine whether to cut + // a new head chunk without knowing the size of the next sample, however, so we assume the next sample will be a + // maximally-sized sample (19 bytes). + const maxBytesPerXORChunk = chunkenc.MaxBytesPerXORChunk - 19 + c = s.headChunks if c == nil { @@ -1276,6 +1282,9 @@ func (s *memSeries) appendPreprocessor(t int64, e chunkenc.Encoding, o chunkOpts // There is no head chunk in this series yet, create the first chunk for the sample. c = s.cutNewHeadChunk(t, e, o.chunkRange) chunkCreated = true + } else if len(c.chunk.Bytes()) > maxBytesPerXORChunk { + c = s.cutNewHeadChunk(t, e, o.chunkRange) + chunkCreated = true } // Out of order sample. @@ -1304,7 +1313,7 @@ func (s *memSeries) appendPreprocessor(t int64, e chunkenc.Encoding, o chunkOpts // the remaining chunks in the current chunk range. // At latest it must happen at the timestamp set when the chunk was cut. if numSamples == o.samplesPerChunk/4 { - s.nextAt = computeChunkEndTime(c.minTime, c.maxTime, s.nextAt) + s.nextAt = computeChunkEndTime(c.minTime, c.maxTime, s.nextAt, 4) } // If numSamples > samplesPerChunk*2 then our previous prediction was invalid, // most likely because samples rate has changed and now they are arriving more frequently. @@ -1319,17 +1328,95 @@ func (s *memSeries) appendPreprocessor(t int64, e chunkenc.Encoding, o chunkOpts return c, true, chunkCreated } +// histogramsAppendPreprocessor takes care of cutting new histogram chunks and m-mapping old ones. Histogram chunks are +// cut based on their size in bytes. +// It is unsafe to call this concurrently with s.iterator(...) without holding the series lock. +// This should be called only when appending data. +func (s *memSeries) histogramsAppendPreprocessor(t int64, e chunkenc.Encoding, o chunkOpts) (c *memChunk, sampleInOrder, chunkCreated bool) { + c = s.headChunks + + if c == nil { + if len(s.mmappedChunks) > 0 && s.mmappedChunks[len(s.mmappedChunks)-1].maxTime >= t { + // Out of order sample. Sample timestamp is already in the mmapped chunks, so ignore it. + return c, false, false + } + // There is no head chunk in this series yet, create the first chunk for the sample. + c = s.cutNewHeadChunk(t, e, o.chunkRange) + chunkCreated = true + } + + // Out of order sample. + if c.maxTime >= t { + return c, false, chunkCreated + } + + if c.chunk.Encoding() != e { + // The chunk encoding expected by this append is different than the head chunk's + // encoding. So we cut a new chunk with the expected encoding. + c = s.cutNewHeadChunk(t, e, o.chunkRange) + chunkCreated = true + } + + numSamples := c.chunk.NumSamples() + targetBytes := chunkenc.TargetBytesPerHistogramChunk + numBytes := len(c.chunk.Bytes()) + + if numSamples == 0 { + // It could be the new chunk created after reading the chunk snapshot, + // hence we fix the minTime of the chunk here. + c.minTime = t + s.nextAt = rangeForTimestamp(c.minTime, o.chunkRange) + } + + // Below, we will enforce chunkenc.MinSamplesPerHistogramChunk. There are, however, two cases that supersede it: + // - The current chunk range is ending before chunkenc.MinSamplesPerHistogramChunk will be satisfied. + // - s.nextAt was set while loading a chunk snapshot with the intent that a new chunk be cut on the next append. + var nextChunkRangeStart int64 + if s.histogramChunkHasComputedEndTime { + nextChunkRangeStart = rangeForTimestamp(c.minTime, o.chunkRange) + } else { + // If we haven't yet computed an end time yet, s.nextAt is either set to + // rangeForTimestamp(c.minTime, o.chunkRange) or was set while loading a chunk snapshot. Either way, we want to + // skip enforcing chunkenc.MinSamplesPerHistogramChunk. + nextChunkRangeStart = s.nextAt + } + + // If we reach 25% of a chunk's desired maximum size, predict an end time + // for this chunk that will try to make samples equally distributed within + // the remaining chunks in the current chunk range. + // At the latest it must happen at the timestamp set when the chunk was cut. + if !s.histogramChunkHasComputedEndTime && numBytes >= targetBytes/4 { + ratioToFull := float64(targetBytes) / float64(numBytes) + s.nextAt = computeChunkEndTime(c.minTime, c.maxTime, s.nextAt, ratioToFull) + s.histogramChunkHasComputedEndTime = true + } + // If numBytes > targetBytes*2 then our previous prediction was invalid. This could happen if the sample rate has + // increased or if the bucket/span count has increased. + // Note that next chunk will have its nextAt recalculated for the new rate. + if (t >= s.nextAt || numBytes >= targetBytes*2) && (numSamples >= chunkenc.MinSamplesPerHistogramChunk || t >= nextChunkRangeStart) { + c = s.cutNewHeadChunk(t, e, o.chunkRange) + chunkCreated = true + } + + // The new chunk will also need a new computed end time. + if chunkCreated { + s.histogramChunkHasComputedEndTime = false + } + + return c, true, chunkCreated +} + // computeChunkEndTime estimates the end timestamp based the beginning of a // chunk, its current timestamp and the upper bound up to which we insert data. -// It assumes that the time range is 1/4 full. +// It assumes that the time range is 1/ratioToFull full. // Assuming that the samples will keep arriving at the same rate, it will make the // remaining n chunks within this chunk range (before max) equally sized. -func computeChunkEndTime(start, cur, max int64) int64 { - n := (max - start) / ((cur - start + 1) * 4) +func computeChunkEndTime(start, cur, max int64, ratioToFull float64) int64 { + n := float64(max-start) / (float64(cur-start+1) * ratioToFull) if n <= 1 { return max } - return start + (max-start)/n + return int64(float64(start) + float64(max-start)/math.Floor(n)) } func (s *memSeries) cutNewHeadChunk(mint int64, e chunkenc.Encoding, chunkRange int64) *memChunk { diff --git a/tsdb/head_test.go b/tsdb/head_test.go index c7fea3f9a..a58afba87 100644 --- a/tsdb/head_test.go +++ b/tsdb/head_test.go @@ -416,7 +416,7 @@ func TestHead_HighConcurrencyReadAndWrite(t *testing.T) { } // queryHead is a helper to query the head for a given time range and labelset. - queryHead := func(mint, maxt uint64, label labels.Label) (map[string][]tsdbutil.Sample, error) { + queryHead := func(mint, maxt uint64, label labels.Label) (map[string][]chunks.Sample, error) { q, err := NewBlockQuerier(head, int64(mint), int64(maxt)) if err != nil { return nil, err @@ -662,7 +662,7 @@ func TestHead_WALMultiRef(t *testing.T) { series := query(t, q, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) // The samples before the new ref should be discarded since Head truncation // happens only after compacting the Head. - require.Equal(t, map[string][]tsdbutil.Sample{`{foo="bar"}`: { + require.Equal(t, map[string][]chunks.Sample{`{foo="bar"}`: { sample{1700, 3, nil, nil}, sample{2000, 4, nil, nil}, }}, series) @@ -1143,8 +1143,8 @@ func TestHeadDeleteSimple(t *testing.T) { actSeriesSet := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, lblDefault.Name, lblDefault.Value)) require.NoError(t, q.Close()) expSeriesSet := newMockSeriesSet([]storage.Series{ - storage.NewListSeries(lblsDefault, func() []tsdbutil.Sample { - ss := make([]tsdbutil.Sample, 0, len(c.smplsExp)) + storage.NewListSeries(lblsDefault, func() []chunks.Sample { + ss := make([]chunks.Sample, 0, len(c.smplsExp)) for _, s := range c.smplsExp { ss = append(ss, s) } @@ -1223,7 +1223,7 @@ func TestDeleteUntilCurMax(t *testing.T) { it = exps.Iterator(nil) resSamples, err := storage.ExpandSamples(it, newSample) require.NoError(t, err) - require.Equal(t, []tsdbutil.Sample{sample{11, 1, nil, nil}}, resSamples) + require.Equal(t, []chunks.Sample{sample{11, 1, nil, nil}}, resSamples) for res.Next() { } require.NoError(t, res.Err()) @@ -1321,9 +1321,9 @@ func TestDelete_e2e(t *testing.T) { {Name: "job", Value: "prom-k8s"}, }, } - seriesMap := map[string][]tsdbutil.Sample{} + seriesMap := map[string][]chunks.Sample{} for _, l := range lbls { - seriesMap[labels.New(l...).String()] = []tsdbutil.Sample{} + seriesMap[labels.New(l...).String()] = []chunks.Sample{} } hb, _ := newTestHead(t, 100000, wlog.CompressionNone, false) @@ -1334,7 +1334,7 @@ func TestDelete_e2e(t *testing.T) { app := hb.Appender(context.Background()) for _, l := range lbls { ls := labels.New(l...) - series := []tsdbutil.Sample{} + series := []chunks.Sample{} ts := rand.Int63n(300) for i := 0; i < numDatapoints; i++ { v := rand.Float64() @@ -1433,7 +1433,7 @@ func TestDelete_e2e(t *testing.T) { } } -func boundedSamples(full []tsdbutil.Sample, mint, maxt int64) []tsdbutil.Sample { +func boundedSamples(full []chunks.Sample, mint, maxt int64) []chunks.Sample { for len(full) > 0 { if full[0].T() >= mint { break @@ -1450,8 +1450,8 @@ func boundedSamples(full []tsdbutil.Sample, mint, maxt int64) []tsdbutil.Sample return full } -func deletedSamples(full []tsdbutil.Sample, dranges tombstones.Intervals) []tsdbutil.Sample { - ds := make([]tsdbutil.Sample, 0, len(full)) +func deletedSamples(full []chunks.Sample, dranges tombstones.Intervals) []chunks.Sample { + ds := make([]chunks.Sample, 0, len(full)) Outer: for _, s := range full { for _, r := range dranges { @@ -1466,44 +1466,58 @@ Outer: } func TestComputeChunkEndTime(t *testing.T) { - cases := []struct { + cases := map[string]struct { start, cur, max int64 + ratioToFull float64 res int64 }{ - { - start: 0, - cur: 250, - max: 1000, - res: 1000, + "exactly 1/4 full, even increment": { + start: 0, + cur: 250, + max: 1000, + ratioToFull: 4, + res: 1000, }, - { - start: 100, - cur: 200, - max: 1000, - res: 550, + "exactly 1/4 full, uneven increment": { + start: 100, + cur: 200, + max: 1000, + ratioToFull: 4, + res: 550, + }, + "decimal ratio to full": { + start: 5000, + cur: 5110, + max: 10000, + ratioToFull: 4.2, + res: 5500, }, // Case where we fit floored 0 chunks. Must catch division by 0 // and default to maximum time. - { - start: 0, - cur: 500, - max: 1000, - res: 1000, + "fit floored 0 chunks": { + start: 0, + cur: 500, + max: 1000, + ratioToFull: 4, + res: 1000, }, // Catch division by zero for cur == start. Strictly not a possible case. - { - start: 100, - cur: 100, - max: 1000, - res: 104, + "cur == start": { + start: 100, + cur: 100, + max: 1000, + ratioToFull: 4, + res: 104, }, } - for _, c := range cases { - got := computeChunkEndTime(c.start, c.cur, c.max) - if got != c.res { - t.Errorf("expected %d for (start: %d, cur: %d, max: %d), got %d", c.res, c.start, c.cur, c.max, got) - } + for testName, tc := range cases { + t.Run(testName, func(t *testing.T) { + got := computeChunkEndTime(tc.start, tc.cur, tc.max, tc.ratioToFull) + if got != tc.res { + t.Errorf("expected %d for (start: %d, cur: %d, max: %d, ratioToFull: %f), got %d", tc.res, tc.start, tc.cur, tc.max, tc.ratioToFull, got) + } + }) } } @@ -2967,7 +2981,7 @@ func TestDataMissingOnQueryDuringCompaction(t *testing.T) { ) // Appends samples to span over 1.5 block ranges. - expSamples := make([]tsdbutil.Sample, 0) + expSamples := make([]chunks.Sample, 0) // 7 chunks with 15s scrape interval. for i := int64(0); i <= 120*7; i++ { ts := i * DefaultBlockDuration / (4 * 120) @@ -2997,7 +3011,7 @@ func TestDataMissingOnQueryDuringCompaction(t *testing.T) { // Querying the querier that was got before compaction. series := query(t, q, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) - require.Equal(t, map[string][]tsdbutil.Sample{`{a="b"}`: expSamples}, series) + require.Equal(t, map[string][]chunks.Sample{`{a="b"}`: expSamples}, series) wg.Wait() } @@ -3117,7 +3131,7 @@ func TestAppendHistogram(t *testing.T) { ingestTs := int64(0) app := head.Appender(context.Background()) - expHistograms := make([]tsdbutil.Sample, 0, 2*numHistograms) + expHistograms := make([]chunks.Sample, 0, 2*numHistograms) // Counter integer histograms. for _, h := range tsdbutil.GenerateTestHistograms(numHistograms) { @@ -3143,7 +3157,7 @@ func TestAppendHistogram(t *testing.T) { } } - expFloatHistograms := make([]tsdbutil.Sample, 0, 2*numHistograms) + expFloatHistograms := make([]chunks.Sample, 0, 2*numHistograms) // Counter float histograms. for _, fh := range tsdbutil.GenerateTestFloatHistograms(numHistograms) { @@ -3184,8 +3198,8 @@ func TestAppendHistogram(t *testing.T) { require.False(t, ss.Next()) it := s.Iterator(nil) - actHistograms := make([]tsdbutil.Sample, 0, len(expHistograms)) - actFloatHistograms := make([]tsdbutil.Sample, 0, len(expFloatHistograms)) + actHistograms := make([]chunks.Sample, 0, len(expHistograms)) + actFloatHistograms := make([]chunks.Sample, 0, len(expFloatHistograms)) for typ := it.Next(); typ != chunkenc.ValNone; typ = it.Next() { switch typ { case chunkenc.ValHistogram: @@ -3199,13 +3213,13 @@ func TestAppendHistogram(t *testing.T) { compareSeries( t, - map[string][]tsdbutil.Sample{"dummy": expHistograms}, - map[string][]tsdbutil.Sample{"dummy": actHistograms}, + map[string][]chunks.Sample{"dummy": expHistograms}, + map[string][]chunks.Sample{"dummy": actHistograms}, ) compareSeries( t, - map[string][]tsdbutil.Sample{"dummy": expFloatHistograms}, - map[string][]tsdbutil.Sample{"dummy": actFloatHistograms}, + map[string][]chunks.Sample{"dummy": expFloatHistograms}, + map[string][]chunks.Sample{"dummy": actFloatHistograms}, ) }) } @@ -3222,7 +3236,7 @@ func TestHistogramInWALAndMmapChunk(t *testing.T) { s1 := labels.FromStrings("a", "b1") k1 := s1.String() numHistograms := 300 - exp := map[string][]tsdbutil.Sample{} + exp := map[string][]chunks.Sample{} ts := int64(0) var app storage.Appender for _, gauge := range []bool{true, false} { @@ -3273,10 +3287,10 @@ func TestHistogramInWALAndMmapChunk(t *testing.T) { head.mmapHeadChunks() } - // There should be 11 mmap chunks in s1. + // There should be 20 mmap chunks in s1. ms := head.series.getByHash(s1.Hash(), s1) - require.Len(t, ms.mmappedChunks, 11) - expMmapChunks := make([]*mmappedChunk, 0, 11) + require.Len(t, ms.mmappedChunks, 25) + expMmapChunks := make([]*mmappedChunk, 0, 20) for _, mmap := range ms.mmappedChunks { require.Greater(t, mmap.numSamples, uint16(0)) cpy := *mmap @@ -3408,9 +3422,9 @@ func TestChunkSnapshot(t *testing.T) { } numSeries := 10 - expSeries := make(map[string][]tsdbutil.Sample) - expHist := make(map[string][]tsdbutil.Sample) - expFloatHist := make(map[string][]tsdbutil.Sample) + expSeries := make(map[string][]chunks.Sample) + expHist := make(map[string][]chunks.Sample) + expFloatHist := make(map[string][]chunks.Sample) expTombstones := make(map[storage.SeriesRef]tombstones.Intervals) expExemplars := make([]ex, 0) histograms := tsdbutil.GenerateTestGaugeHistograms(481) @@ -3959,7 +3973,7 @@ func testHistogramStaleSampleHelper(t *testing.T, floatHistogram bool) { } func TestHistogramCounterResetHeader(t *testing.T) { - for _, floatHisto := range []bool{true, false} { + for _, floatHisto := range []bool{true} { // FIXME t.Run(fmt.Sprintf("floatHistogram=%t", floatHisto), func(t *testing.T) { l := labels.FromStrings("a", "b") head, _ := newTestHead(t, 1000, wlog.CompressionNone, false) @@ -4026,10 +4040,16 @@ func TestHistogramCounterResetHeader(t *testing.T) { appendHistogram(h) checkExpCounterResetHeader(chunkenc.CounterReset) - // Add 2 non-counter reset histogram chunks. - for i := 0; i < 250; i++ { + // Add 2 non-counter reset histogram chunks (each chunk targets 1024 bytes which contains ~500 int histogram + // samples or ~1000 float histogram samples). + numAppend := 2000 + if floatHisto { + numAppend = 1000 + } + for i := 0; i < numAppend; i++ { appendHistogram(h) } + checkExpCounterResetHeader(chunkenc.NotCounterReset, chunkenc.NotCounterReset) // Changing schema will cut a new chunk with unknown counter reset. @@ -4054,8 +4074,8 @@ func TestHistogramCounterResetHeader(t *testing.T) { appendHistogram(h) checkExpCounterResetHeader(chunkenc.CounterReset) - // Add 2 non-counter reset histograms. Just to have some non-counter reset chunks in between. - for i := 0; i < 250; i++ { + // Add 2 non-counter reset histogram chunks. Just to have some non-counter reset chunks in between. + for i := 0; i < 2000; i++ { appendHistogram(h) } checkExpCounterResetHeader(chunkenc.NotCounterReset, chunkenc.NotCounterReset) @@ -4088,7 +4108,7 @@ func TestAppendingDifferentEncodingToSameSeries(t *testing.T) { floatHists := tsdbutil.GenerateTestFloatHistograms(10) lbls := labels.FromStrings("a", "b") - var expResult []tsdbutil.Sample + var expResult []chunks.Sample checkExpChunks := func(count int) { ms, created, err := db.Head().getOrCreate(lbls.Hash(), lbls) require.NoError(t, err) @@ -4098,59 +4118,59 @@ func TestAppendingDifferentEncodingToSameSeries(t *testing.T) { } appends := []struct { - samples []tsdbutil.Sample + samples []chunks.Sample expChunks int err error // If this is empty, samples above will be taken instead of this. - addToExp []tsdbutil.Sample + addToExp []chunks.Sample }{ // Histograms that end up in the expected samples are copied here so that we // can independently set the CounterResetHint later. { - samples: []tsdbutil.Sample{sample{t: 100, h: hists[0].Copy()}}, + samples: []chunks.Sample{sample{t: 100, h: hists[0].Copy()}}, expChunks: 1, }, { - samples: []tsdbutil.Sample{sample{t: 200, f: 2}}, + samples: []chunks.Sample{sample{t: 200, f: 2}}, expChunks: 2, }, { - samples: []tsdbutil.Sample{sample{t: 210, fh: floatHists[0].Copy()}}, + samples: []chunks.Sample{sample{t: 210, fh: floatHists[0].Copy()}}, expChunks: 3, }, { - samples: []tsdbutil.Sample{sample{t: 220, h: hists[1].Copy()}}, + samples: []chunks.Sample{sample{t: 220, h: hists[1].Copy()}}, expChunks: 4, }, { - samples: []tsdbutil.Sample{sample{t: 230, fh: floatHists[3].Copy()}}, + samples: []chunks.Sample{sample{t: 230, fh: floatHists[3].Copy()}}, expChunks: 5, }, { - samples: []tsdbutil.Sample{sample{t: 100, h: hists[2].Copy()}}, + samples: []chunks.Sample{sample{t: 100, h: hists[2].Copy()}}, err: storage.ErrOutOfOrderSample, }, { - samples: []tsdbutil.Sample{sample{t: 300, h: hists[3].Copy()}}, + samples: []chunks.Sample{sample{t: 300, h: hists[3].Copy()}}, expChunks: 6, }, { - samples: []tsdbutil.Sample{sample{t: 100, f: 2}}, + samples: []chunks.Sample{sample{t: 100, f: 2}}, err: storage.ErrOutOfOrderSample, }, { - samples: []tsdbutil.Sample{sample{t: 100, fh: floatHists[4].Copy()}}, + samples: []chunks.Sample{sample{t: 100, fh: floatHists[4].Copy()}}, err: storage.ErrOutOfOrderSample, }, { // Combination of histograms and float64 in the same commit. The behaviour is undefined, but we want to also // verify how TSDB would behave. Here the histogram is appended at the end, hence will be considered as out of order. - samples: []tsdbutil.Sample{ + samples: []chunks.Sample{ sample{t: 400, f: 4}, sample{t: 500, h: hists[5]}, // This won't be committed. sample{t: 600, f: 6}, }, - addToExp: []tsdbutil.Sample{ + addToExp: []chunks.Sample{ sample{t: 400, f: 4}, sample{t: 600, f: 6}, }, @@ -4158,12 +4178,12 @@ func TestAppendingDifferentEncodingToSameSeries(t *testing.T) { }, { // Here the histogram is appended at the end, hence the first histogram is out of order. - samples: []tsdbutil.Sample{ + samples: []chunks.Sample{ sample{t: 700, h: hists[7]}, // Out of order w.r.t. the next float64 sample that is appended first. sample{t: 800, f: 8}, sample{t: 900, h: hists[9]}, }, - addToExp: []tsdbutil.Sample{ + addToExp: []chunks.Sample{ sample{t: 800, f: 8}, sample{t: 900, h: hists[9].Copy()}, }, @@ -4171,11 +4191,11 @@ func TestAppendingDifferentEncodingToSameSeries(t *testing.T) { }, { // Float histogram is appended at the end. - samples: []tsdbutil.Sample{ + samples: []chunks.Sample{ sample{t: 1000, fh: floatHists[7]}, // Out of order w.r.t. the next histogram. sample{t: 1100, h: hists[9]}, }, - addToExp: []tsdbutil.Sample{ + addToExp: []chunks.Sample{ sample{t: 1100, h: hists[9].Copy()}, }, expChunks: 8, @@ -4220,7 +4240,7 @@ func TestAppendingDifferentEncodingToSameSeries(t *testing.T) { require.NoError(t, err) series := query(t, q, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) - require.Equal(t, map[string][]tsdbutil.Sample{lbls.String(): expResult}, series) + require.Equal(t, map[string][]chunks.Sample{lbls.String(): expResult}, series) } // Tests https://github.com/prometheus/prometheus/issues/9725. @@ -4654,7 +4674,7 @@ func TestReplayAfterMmapReplayError(t *testing.T) { itvl := int64(15 * time.Second / time.Millisecond) lastTs := int64(0) lbls := labels.FromStrings("__name__", "testing", "foo", "bar") - var expSamples []tsdbutil.Sample + var expSamples []chunks.Sample addSamples := func(numSamples int) { app := h.Appender(context.Background()) var ref storage.SeriesRef @@ -4703,7 +4723,7 @@ func TestReplayAfterMmapReplayError(t *testing.T) { q, err := NewBlockQuerier(h, 0, lastTs) require.NoError(t, err) res := query(t, q, labels.MustNewMatcher(labels.MatchEqual, "__name__", "testing")) - require.Equal(t, map[string][]tsdbutil.Sample{lbls.String(): expSamples}, res) + require.Equal(t, map[string][]chunks.Sample{lbls.String(): expSamples}, res) require.NoError(t, h.Close()) } @@ -4818,23 +4838,22 @@ func TestHistogramValidation(t *testing.T) { } func BenchmarkHistogramValidation(b *testing.B) { - histograms := generateBigTestHistograms(b.N) + histograms := generateBigTestHistograms(b.N, 500) b.ResetTimer() for _, h := range histograms { require.NoError(b, ValidateHistogram(h)) } } -func generateBigTestHistograms(n int) []*histogram.Histogram { - const numBuckets = 500 +func generateBigTestHistograms(numHistograms, numBuckets int) []*histogram.Histogram { numSpans := numBuckets / 10 bucketsPerSide := numBuckets / 2 spanLength := uint32(bucketsPerSide / numSpans) - // Given all bucket deltas are 1, sum n + 1. + // Given all bucket deltas are 1, sum numHistograms + 1. observationCount := numBuckets / 2 * (1 + numBuckets) var histograms []*histogram.Histogram - for i := 0; i < n; i++ { + for i := 0; i < numHistograms; i++ { h := &histogram.Histogram{ Count: uint64(i + observationCount), ZeroCount: uint64(i), @@ -4848,7 +4867,7 @@ func generateBigTestHistograms(n int) []*histogram.Histogram { } for j := 0; j < numSpans; j++ { - s := histogram.Span{Offset: 1 + int32(i), Length: spanLength} + s := histogram.Span{Offset: 1, Length: spanLength} h.NegativeSpans[j] = s h.PositiveSpans[j] = s } @@ -5197,3 +5216,186 @@ func TestSnapshotAheadOfWALError(t *testing.T) { require.NoError(t, head.Close()) } + +func BenchmarkCuttingHeadHistogramChunks(b *testing.B) { + const ( + numSamples = 50000 + numBuckets = 100 + ) + samples := generateBigTestHistograms(numSamples, numBuckets) + + h, _ := newTestHead(b, DefaultBlockDuration, wlog.CompressionNone, false) + defer func() { + require.NoError(b, h.Close()) + }() + + a := h.Appender(context.Background()) + ts := time.Now().UnixMilli() + lbls := labels.FromStrings("foo", "bar") + + b.ResetTimer() + + for _, s := range samples { + _, err := a.AppendHistogram(0, lbls, ts, s, nil) + require.NoError(b, err) + } +} + +func TestCuttingNewHeadChunks(t *testing.T) { + testCases := map[string]struct { + numTotalSamples int + timestampJitter bool + floatValFunc func(i int) float64 + histValFunc func(i int) *histogram.Histogram + expectedChks []struct { + numSamples int + numBytes int + } + }{ + "float samples": { + numTotalSamples: 180, + floatValFunc: func(i int) float64 { + return 1. + }, + expectedChks: []struct { + numSamples int + numBytes int + }{ + {numSamples: 120, numBytes: 46}, + {numSamples: 60, numBytes: 32}, + }, + }, + "large float samples": { + // Normally 120 samples would fit into a single chunk but these chunks violate the 1005 byte soft cap. + numTotalSamples: 120, + timestampJitter: true, + floatValFunc: func(i int) float64 { + // Flipping between these two make each sample val take at least 64 bits. + vals := []float64{math.MaxFloat64, 0x00} + return vals[i%len(vals)] + }, + expectedChks: []struct { + numSamples int + numBytes int + }{ + {99, 1008}, + {21, 219}, + }, + }, + "small histograms": { + numTotalSamples: 240, + histValFunc: func() func(i int) *histogram.Histogram { + hists := generateBigTestHistograms(240, 10) + return func(i int) *histogram.Histogram { + return hists[i] + } + }(), + expectedChks: []struct { + numSamples int + numBytes int + }{ + {120, 1087}, + {120, 1039}, + }, + }, + "large histograms": { + numTotalSamples: 240, + histValFunc: func() func(i int) *histogram.Histogram { + hists := generateBigTestHistograms(240, 100) + return func(i int) *histogram.Histogram { + return hists[i] + } + }(), + expectedChks: []struct { + numSamples int + numBytes int + }{ + {30, 696}, + {30, 700}, + {30, 708}, + {30, 693}, + {30, 691}, + {30, 692}, + {30, 695}, + {30, 694}, + }, + }, + "really large histograms": { + // Really large histograms; each chunk can only contain a single histogram but we have a 10 sample minimum + // per chunk. + numTotalSamples: 11, + histValFunc: func() func(i int) *histogram.Histogram { + hists := generateBigTestHistograms(11, 100000) + return func(i int) *histogram.Histogram { + return hists[i] + } + }(), + expectedChks: []struct { + numSamples int + numBytes int + }{ + {10, 200103}, + {1, 87540}, + }, + }, + } + for testName, tc := range testCases { + t.Run(testName, func(t *testing.T) { + h, _ := newTestHead(t, DefaultBlockDuration, wlog.CompressionNone, false) + defer func() { + require.NoError(t, h.Close()) + }() + + a := h.Appender(context.Background()) + + ts := int64(10000) + lbls := labels.FromStrings("foo", "bar") + jitter := []int64{0, 1} // A bit of jitter to prevent dod=0. + + for i := 0; i < tc.numTotalSamples; i++ { + if tc.floatValFunc != nil { + _, err := a.Append(0, lbls, ts, tc.floatValFunc(i)) + require.NoError(t, err) + } else if tc.histValFunc != nil { + _, err := a.AppendHistogram(0, lbls, ts, tc.histValFunc(i), nil) + require.NoError(t, err) + } + + ts += int64(60 * time.Second / time.Millisecond) + if tc.timestampJitter { + ts += jitter[i%len(jitter)] + } + } + + require.NoError(t, a.Commit()) + + idxReader, err := h.Index() + require.NoError(t, err) + + chkReader, err := h.Chunks() + require.NoError(t, err) + + p, err := idxReader.Postings("foo", "bar") + require.NoError(t, err) + + var lblBuilder labels.ScratchBuilder + + for p.Next() { + sRef := p.At() + + chkMetas := make([]chunks.Meta, len(tc.expectedChks)) + require.NoError(t, idxReader.Series(sRef, &lblBuilder, &chkMetas)) + + require.Len(t, chkMetas, len(tc.expectedChks)) + + for i, expected := range tc.expectedChks { + chk, err := chkReader.Chunk(chkMetas[i]) + require.NoError(t, err) + + require.Equal(t, expected.numSamples, chk.NumSamples()) + require.Len(t, chk.Bytes(), expected.numBytes) + } + } + }) + } +} diff --git a/tsdb/ooo_head_read_test.go b/tsdb/ooo_head_read_test.go index 6c0038f89..013b59aa5 100644 --- a/tsdb/ooo_head_read_test.go +++ b/tsdb/ooo_head_read_test.go @@ -28,7 +28,6 @@ import ( "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/chunks" - "github.com/prometheus/prometheus/tsdb/tsdbutil" "github.com/prometheus/prometheus/tsdb/wlog" ) @@ -496,16 +495,16 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) { queryMinT int64 queryMaxT int64 firstInOrderSampleAt int64 - inputSamples tsdbutil.SampleSlice + inputSamples chunks.SampleSlice expChunkError bool - expChunksSamples []tsdbutil.SampleSlice + expChunksSamples []chunks.SampleSlice }{ { name: "Getting the head when there are no overlapping chunks returns just the samples in the head", queryMinT: minutes(0), queryMaxT: minutes(100), firstInOrderSampleAt: minutes(120), - inputSamples: tsdbutil.SampleSlice{ + inputSamples: chunks.SampleSlice{ sample{t: minutes(30), f: float64(0)}, sample{t: minutes(40), f: float64(0)}, }, @@ -514,7 +513,7 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) { // Query Interval [------------------------------------------------------------------------------------------] // Chunk 0: Current Head [--------] (With 2 samples) // Output Graphically [--------] (With 2 samples) - expChunksSamples: []tsdbutil.SampleSlice{ + expChunksSamples: []chunks.SampleSlice{ { sample{t: minutes(30), f: float64(0)}, sample{t: minutes(40), f: float64(0)}, @@ -526,7 +525,7 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) { queryMinT: minutes(0), queryMaxT: minutes(100), firstInOrderSampleAt: minutes(120), - inputSamples: tsdbutil.SampleSlice{ + inputSamples: chunks.SampleSlice{ // opts.OOOCapMax is 5 so these will be mmapped to the first mmapped chunk sample{t: minutes(41), f: float64(0)}, sample{t: minutes(42), f: float64(0)}, @@ -544,7 +543,7 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) { // Chunk 0 [---] (With 5 samples) // Chunk 1: Current Head [-----------------] (With 2 samples) // Output Graphically [-----------------] (With 7 samples) - expChunksSamples: []tsdbutil.SampleSlice{ + expChunksSamples: []chunks.SampleSlice{ { sample{t: minutes(30), f: float64(1)}, sample{t: minutes(41), f: float64(0)}, @@ -561,7 +560,7 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) { queryMinT: minutes(0), queryMaxT: minutes(100), firstInOrderSampleAt: minutes(120), - inputSamples: tsdbutil.SampleSlice{ + inputSamples: chunks.SampleSlice{ // Chunk 0 sample{t: minutes(10), f: float64(0)}, sample{t: minutes(12), f: float64(0)}, @@ -592,7 +591,7 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) { // Chunk 2 [--------] // Chunk 3: Current Head [--------] // Output Graphically [----------------][-----------------] - expChunksSamples: []tsdbutil.SampleSlice{ + expChunksSamples: []chunks.SampleSlice{ { sample{t: minutes(10), f: float64(0)}, sample{t: minutes(12), f: float64(0)}, @@ -619,7 +618,7 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) { queryMinT: minutes(0), queryMaxT: minutes(100), firstInOrderSampleAt: minutes(120), - inputSamples: tsdbutil.SampleSlice{ + inputSamples: chunks.SampleSlice{ // Chunk 0 sample{t: minutes(40), f: float64(0)}, sample{t: minutes(42), f: float64(0)}, @@ -650,7 +649,7 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) { // Chunk 2 [-------] // Chunk 3: Current Head [--------] // Output Graphically [----------------][-----------------] - expChunksSamples: []tsdbutil.SampleSlice{ + expChunksSamples: []chunks.SampleSlice{ { sample{t: minutes(10), f: float64(3)}, sample{t: minutes(20), f: float64(2)}, @@ -677,7 +676,7 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) { queryMinT: minutes(0), queryMaxT: minutes(100), firstInOrderSampleAt: minutes(120), - inputSamples: tsdbutil.SampleSlice{ + inputSamples: chunks.SampleSlice{ // Chunk 0 sample{t: minutes(10), f: float64(0)}, sample{t: minutes(12), f: float64(0)}, @@ -708,7 +707,7 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) { // Chunk 2 [-------] // Chunk 3: Current Head [-------] // Output Graphically [-------][-------][-------][--------] - expChunksSamples: []tsdbutil.SampleSlice{ + expChunksSamples: []chunks.SampleSlice{ { sample{t: minutes(10), f: float64(0)}, sample{t: minutes(12), f: float64(0)}, @@ -741,7 +740,7 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) { queryMinT: minutes(0), queryMaxT: minutes(100), firstInOrderSampleAt: minutes(120), - inputSamples: tsdbutil.SampleSlice{ + inputSamples: chunks.SampleSlice{ // Chunk 0 sample{t: minutes(10), f: float64(0)}, sample{t: minutes(15), f: float64(0)}, @@ -765,7 +764,7 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) { // Chunk 1 [--------------------] // Chunk 2 Current Head [--------------] // Output Graphically [-----------------------------------] - expChunksSamples: []tsdbutil.SampleSlice{ + expChunksSamples: []chunks.SampleSlice{ { sample{t: minutes(10), f: float64(0)}, sample{t: minutes(15), f: float64(0)}, @@ -784,7 +783,7 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) { queryMinT: minutes(12), queryMaxT: minutes(33), firstInOrderSampleAt: minutes(120), - inputSamples: tsdbutil.SampleSlice{ + inputSamples: chunks.SampleSlice{ // Chunk 0 sample{t: minutes(10), f: float64(0)}, sample{t: minutes(15), f: float64(0)}, @@ -808,7 +807,7 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) { // Chunk 1 [--------------------] // Chunk 2 Current Head [--------------] // Output Graphically [-----------------------------------] - expChunksSamples: []tsdbutil.SampleSlice{ + expChunksSamples: []chunks.SampleSlice{ { sample{t: minutes(10), f: float64(0)}, sample{t: minutes(15), f: float64(0)}, @@ -853,7 +852,7 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) { c, err := cr.Chunk(chks[i]) require.NoError(t, err) - var resultSamples tsdbutil.SampleSlice + var resultSamples chunks.SampleSlice it := c.Iterator(nil) for it.Next() == chunkenc.ValFloat { t, v := it.At() @@ -892,17 +891,17 @@ func TestOOOHeadChunkReader_Chunk_ConsistentQueryResponseDespiteOfHeadExpanding( queryMinT int64 queryMaxT int64 firstInOrderSampleAt int64 - initialSamples tsdbutil.SampleSlice - samplesAfterSeriesCall tsdbutil.SampleSlice + initialSamples chunks.SampleSlice + samplesAfterSeriesCall chunks.SampleSlice expChunkError bool - expChunksSamples []tsdbutil.SampleSlice + expChunksSamples []chunks.SampleSlice }{ { name: "Current head gets old, new and in between sample after Series call, they all should be omitted from the result", queryMinT: minutes(0), queryMaxT: minutes(100), firstInOrderSampleAt: minutes(120), - initialSamples: tsdbutil.SampleSlice{ + initialSamples: chunks.SampleSlice{ // Chunk 0 sample{t: minutes(20), f: float64(0)}, sample{t: minutes(22), f: float64(0)}, @@ -913,7 +912,7 @@ func TestOOOHeadChunkReader_Chunk_ConsistentQueryResponseDespiteOfHeadExpanding( sample{t: minutes(25), f: float64(1)}, sample{t: minutes(35), f: float64(1)}, }, - samplesAfterSeriesCall: tsdbutil.SampleSlice{ + samplesAfterSeriesCall: chunks.SampleSlice{ sample{t: minutes(10), f: float64(1)}, sample{t: minutes(32), f: float64(1)}, sample{t: minutes(50), f: float64(1)}, @@ -926,7 +925,7 @@ func TestOOOHeadChunkReader_Chunk_ConsistentQueryResponseDespiteOfHeadExpanding( // New samples added after Series() // Chunk 1: Current Head [-----------------------------------] (5 samples) // Output Graphically [------------] (With 8 samples, samples newer than lastmint or older than lastmaxt are omitted but the ones in between are kept) - expChunksSamples: []tsdbutil.SampleSlice{ + expChunksSamples: []chunks.SampleSlice{ { sample{t: minutes(20), f: float64(0)}, sample{t: minutes(22), f: float64(0)}, @@ -944,7 +943,7 @@ func TestOOOHeadChunkReader_Chunk_ConsistentQueryResponseDespiteOfHeadExpanding( queryMinT: minutes(0), queryMaxT: minutes(100), firstInOrderSampleAt: minutes(120), - initialSamples: tsdbutil.SampleSlice{ + initialSamples: chunks.SampleSlice{ // Chunk 0 sample{t: minutes(20), f: float64(0)}, sample{t: minutes(22), f: float64(0)}, @@ -955,7 +954,7 @@ func TestOOOHeadChunkReader_Chunk_ConsistentQueryResponseDespiteOfHeadExpanding( sample{t: minutes(25), f: float64(1)}, sample{t: minutes(35), f: float64(1)}, }, - samplesAfterSeriesCall: tsdbutil.SampleSlice{ + samplesAfterSeriesCall: chunks.SampleSlice{ sample{t: minutes(10), f: float64(1)}, sample{t: minutes(32), f: float64(1)}, sample{t: minutes(50), f: float64(1)}, @@ -972,7 +971,7 @@ func TestOOOHeadChunkReader_Chunk_ConsistentQueryResponseDespiteOfHeadExpanding( // Chunk 1 (mmapped) [-------------------------] (5 samples) // Chunk 2: Current Head [-----------] (2 samples) // Output Graphically [------------] (8 samples) It has 5 from Chunk 0 and 3 from Chunk 1 - expChunksSamples: []tsdbutil.SampleSlice{ + expChunksSamples: []chunks.SampleSlice{ { sample{t: minutes(20), f: float64(0)}, sample{t: minutes(22), f: float64(0)}, @@ -1024,7 +1023,7 @@ func TestOOOHeadChunkReader_Chunk_ConsistentQueryResponseDespiteOfHeadExpanding( c, err := cr.Chunk(chks[i]) require.NoError(t, err) - var resultSamples tsdbutil.SampleSlice + var resultSamples chunks.SampleSlice it := c.Iterator(nil) for it.Next() == chunkenc.ValFloat { ts, v := it.At() diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go index a63041b53..24005ab5f 100644 --- a/tsdb/querier_test.go +++ b/tsdb/querier_test.go @@ -239,7 +239,7 @@ func testBlockQuerier(t *testing.T, c blockQuerierTestCase, ir IndexReader, cr C require.Equal(t, errExp, errRes) require.Equal(t, len(chksExp), len(chksRes)) - var exp, act [][]tsdbutil.Sample + var exp, act [][]chunks.Sample for i := range chksExp { samples, err := storage.ExpandSamples(chksExp[i].Chunk.Iterator(nil), nil) require.NoError(t, err) @@ -291,24 +291,24 @@ func TestBlockQuerier(t *testing.T) { ms: []*labels.Matcher{labels.MustNewMatcher(labels.MatchRegexp, "a", ".*")}, exp: newMockSeriesSet([]storage.Series{ storage.NewListSeries(labels.FromStrings("a", "a"), - []tsdbutil.Sample{sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 4, nil, nil}, sample{5, 2, nil, nil}, sample{6, 3, nil, nil}, sample{7, 4, nil, nil}}, + []chunks.Sample{sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 4, nil, nil}, sample{5, 2, nil, nil}, sample{6, 3, nil, nil}, sample{7, 4, nil, nil}}, ), storage.NewListSeries(labels.FromStrings("a", "a", "b", "b"), - []tsdbutil.Sample{sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}, sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, + []chunks.Sample{sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}, sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, ), storage.NewListSeries(labels.FromStrings("b", "b"), - []tsdbutil.Sample{sample{1, 3, nil, nil}, sample{2, 2, nil, nil}, sample{3, 6, nil, nil}, sample{5, 1, nil, nil}, sample{6, 7, nil, nil}, sample{7, 2, nil, nil}}, + []chunks.Sample{sample{1, 3, nil, nil}, sample{2, 2, nil, nil}, sample{3, 6, nil, nil}, sample{5, 1, nil, nil}, sample{6, 7, nil, nil}, sample{7, 2, nil, nil}}, ), }), expChks: newMockChunkSeriesSet([]storage.ChunkSeries{ storage.NewListChunkSeriesFromSamples(labels.FromStrings("a", "a"), - []tsdbutil.Sample{sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 4, nil, nil}}, []tsdbutil.Sample{sample{5, 2, nil, nil}, sample{6, 3, nil, nil}, sample{7, 4, nil, nil}}, + []chunks.Sample{sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 4, nil, nil}}, []chunks.Sample{sample{5, 2, nil, nil}, sample{6, 3, nil, nil}, sample{7, 4, nil, nil}}, ), storage.NewListChunkSeriesFromSamples(labels.FromStrings("a", "a", "b", "b"), - []tsdbutil.Sample{sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}}, []tsdbutil.Sample{sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, + []chunks.Sample{sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}}, []chunks.Sample{sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, ), storage.NewListChunkSeriesFromSamples(labels.FromStrings("b", "b"), - []tsdbutil.Sample{sample{1, 3, nil, nil}, sample{2, 2, nil, nil}, sample{3, 6, nil, nil}}, []tsdbutil.Sample{sample{5, 1, nil, nil}, sample{6, 7, nil, nil}, sample{7, 2, nil, nil}}, + []chunks.Sample{sample{1, 3, nil, nil}, sample{2, 2, nil, nil}, sample{3, 6, nil, nil}}, []chunks.Sample{sample{5, 1, nil, nil}, sample{6, 7, nil, nil}, sample{7, 2, nil, nil}}, ), }), }, @@ -318,18 +318,18 @@ func TestBlockQuerier(t *testing.T) { ms: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "a", "a")}, exp: newMockSeriesSet([]storage.Series{ storage.NewListSeries(labels.FromStrings("a", "a"), - []tsdbutil.Sample{sample{2, 3, nil, nil}, sample{3, 4, nil, nil}, sample{5, 2, nil, nil}, sample{6, 3, nil, nil}}, + []chunks.Sample{sample{2, 3, nil, nil}, sample{3, 4, nil, nil}, sample{5, 2, nil, nil}, sample{6, 3, nil, nil}}, ), storage.NewListSeries(labels.FromStrings("a", "a", "b", "b"), - []tsdbutil.Sample{sample{2, 2, nil, nil}, sample{3, 3, nil, nil}, sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, + []chunks.Sample{sample{2, 2, nil, nil}, sample{3, 3, nil, nil}, sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, ), }), expChks: newMockChunkSeriesSet([]storage.ChunkSeries{ storage.NewListChunkSeriesFromSamples(labels.FromStrings("a", "a"), - []tsdbutil.Sample{sample{2, 3, nil, nil}, sample{3, 4, nil, nil}}, []tsdbutil.Sample{sample{5, 2, nil, nil}, sample{6, 3, nil, nil}}, + []chunks.Sample{sample{2, 3, nil, nil}, sample{3, 4, nil, nil}}, []chunks.Sample{sample{5, 2, nil, nil}, sample{6, 3, nil, nil}}, ), storage.NewListChunkSeriesFromSamples(labels.FromStrings("a", "a", "b", "b"), - []tsdbutil.Sample{sample{2, 2, nil, nil}, sample{3, 3, nil, nil}}, []tsdbutil.Sample{sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, + []chunks.Sample{sample{2, 2, nil, nil}, sample{3, 3, nil, nil}}, []chunks.Sample{sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, ), }), }, @@ -342,20 +342,20 @@ func TestBlockQuerier(t *testing.T) { ms: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "a", "a")}, exp: newMockSeriesSet([]storage.Series{ storage.NewListSeries(labels.FromStrings("a", "a"), - []tsdbutil.Sample{sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 4, nil, nil}, sample{5, 2, nil, nil}, sample{6, 3, nil, nil}, sample{7, 4, nil, nil}}, + []chunks.Sample{sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 4, nil, nil}, sample{5, 2, nil, nil}, sample{6, 3, nil, nil}, sample{7, 4, nil, nil}}, ), storage.NewListSeries(labels.FromStrings("a", "a", "b", "b"), - []tsdbutil.Sample{sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}, sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, + []chunks.Sample{sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}, sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, ), }), expChks: newMockChunkSeriesSet([]storage.ChunkSeries{ storage.NewListChunkSeriesFromSamples(labels.FromStrings("a", "a"), - []tsdbutil.Sample{sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 4, nil, nil}}, - []tsdbutil.Sample{sample{5, 2, nil, nil}, sample{6, 3, nil, nil}, sample{7, 4, nil, nil}}, + []chunks.Sample{sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 4, nil, nil}}, + []chunks.Sample{sample{5, 2, nil, nil}, sample{6, 3, nil, nil}, sample{7, 4, nil, nil}}, ), storage.NewListChunkSeriesFromSamples(labels.FromStrings("a", "a", "b", "b"), - []tsdbutil.Sample{sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}}, - []tsdbutil.Sample{sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, + []chunks.Sample{sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}}, + []chunks.Sample{sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, ), }), }, @@ -368,18 +368,18 @@ func TestBlockQuerier(t *testing.T) { ms: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "a", "a")}, exp: newMockSeriesSet([]storage.Series{ storage.NewListSeries(labels.FromStrings("a", "a"), - []tsdbutil.Sample{sample{5, 2, nil, nil}, sample{6, 3, nil, nil}, sample{7, 4, nil, nil}}, + []chunks.Sample{sample{5, 2, nil, nil}, sample{6, 3, nil, nil}, sample{7, 4, nil, nil}}, ), storage.NewListSeries(labels.FromStrings("a", "a", "b", "b"), - []tsdbutil.Sample{sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, + []chunks.Sample{sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, ), }), expChks: newMockChunkSeriesSet([]storage.ChunkSeries{ storage.NewListChunkSeriesFromSamples(labels.FromStrings("a", "a"), - []tsdbutil.Sample{sample{5, 2, nil, nil}, sample{6, 3, nil, nil}, sample{7, 4, nil, nil}}, + []chunks.Sample{sample{5, 2, nil, nil}, sample{6, 3, nil, nil}, sample{7, 4, nil, nil}}, ), storage.NewListChunkSeriesFromSamples(labels.FromStrings("a", "a", "b", "b"), - []tsdbutil.Sample{sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, + []chunks.Sample{sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, ), }), }, @@ -427,24 +427,24 @@ func TestBlockQuerier_AgainstHeadWithOpenChunks(t *testing.T) { ms: []*labels.Matcher{labels.MustNewMatcher(labels.MatchRegexp, "a", ".*")}, exp: newMockSeriesSet([]storage.Series{ storage.NewListSeries(labels.FromStrings("a", "a"), - []tsdbutil.Sample{sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 4, nil, nil}, sample{5, 2, nil, nil}, sample{6, 3, nil, nil}, sample{7, 4, nil, nil}}, + []chunks.Sample{sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 4, nil, nil}, sample{5, 2, nil, nil}, sample{6, 3, nil, nil}, sample{7, 4, nil, nil}}, ), storage.NewListSeries(labels.FromStrings("a", "a", "b", "b"), - []tsdbutil.Sample{sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}, sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, + []chunks.Sample{sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}, sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, ), storage.NewListSeries(labels.FromStrings("b", "b"), - []tsdbutil.Sample{sample{1, 3, nil, nil}, sample{2, 2, nil, nil}, sample{3, 6, nil, nil}, sample{5, 1, nil, nil}, sample{6, 7, nil, nil}, sample{7, 2, nil, nil}}, + []chunks.Sample{sample{1, 3, nil, nil}, sample{2, 2, nil, nil}, sample{3, 6, nil, nil}, sample{5, 1, nil, nil}, sample{6, 7, nil, nil}, sample{7, 2, nil, nil}}, ), }), expChks: newMockChunkSeriesSet([]storage.ChunkSeries{ storage.NewListChunkSeriesFromSamples(labels.FromStrings("a", "a"), - []tsdbutil.Sample{sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 4, nil, nil}, sample{5, 2, nil, nil}, sample{6, 3, nil, nil}, sample{7, 4, nil, nil}}, + []chunks.Sample{sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 4, nil, nil}, sample{5, 2, nil, nil}, sample{6, 3, nil, nil}, sample{7, 4, nil, nil}}, ), storage.NewListChunkSeriesFromSamples(labels.FromStrings("a", "a", "b", "b"), - []tsdbutil.Sample{sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}, sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, + []chunks.Sample{sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}, sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, ), storage.NewListChunkSeriesFromSamples(labels.FromStrings("b", "b"), - []tsdbutil.Sample{sample{1, 3, nil, nil}, sample{2, 2, nil, nil}, sample{3, 6, nil, nil}, sample{5, 1, nil, nil}, sample{6, 7, nil, nil}, sample{7, 2, nil, nil}}, + []chunks.Sample{sample{1, 3, nil, nil}, sample{2, 2, nil, nil}, sample{3, 6, nil, nil}, sample{5, 1, nil, nil}, sample{6, 7, nil, nil}, sample{7, 2, nil, nil}}, ), }), }, @@ -454,18 +454,18 @@ func TestBlockQuerier_AgainstHeadWithOpenChunks(t *testing.T) { ms: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "a", "a")}, exp: newMockSeriesSet([]storage.Series{ storage.NewListSeries(labels.FromStrings("a", "a"), - []tsdbutil.Sample{sample{2, 3, nil, nil}, sample{3, 4, nil, nil}, sample{5, 2, nil, nil}, sample{6, 3, nil, nil}}, + []chunks.Sample{sample{2, 3, nil, nil}, sample{3, 4, nil, nil}, sample{5, 2, nil, nil}, sample{6, 3, nil, nil}}, ), storage.NewListSeries(labels.FromStrings("a", "a", "b", "b"), - []tsdbutil.Sample{sample{2, 2, nil, nil}, sample{3, 3, nil, nil}, sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, + []chunks.Sample{sample{2, 2, nil, nil}, sample{3, 3, nil, nil}, sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, ), }), expChks: newMockChunkSeriesSet([]storage.ChunkSeries{ storage.NewListChunkSeriesFromSamples(labels.FromStrings("a", "a"), - []tsdbutil.Sample{sample{2, 3, nil, nil}, sample{3, 4, nil, nil}, sample{5, 2, nil, nil}, sample{6, 3, nil, nil}}, + []chunks.Sample{sample{2, 3, nil, nil}, sample{3, 4, nil, nil}, sample{5, 2, nil, nil}, sample{6, 3, nil, nil}}, ), storage.NewListChunkSeriesFromSamples(labels.FromStrings("a", "a", "b", "b"), - []tsdbutil.Sample{sample{2, 2, nil, nil}, sample{3, 3, nil, nil}, sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, + []chunks.Sample{sample{2, 2, nil, nil}, sample{3, 3, nil, nil}, sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, ), }), }, @@ -509,18 +509,18 @@ func TestBlockQuerier_TrimmingDoesNotModifyOriginalTombstoneIntervals(t *testing ms: []*labels.Matcher{labels.MustNewMatcher(labels.MatchRegexp, "a", "a")}, exp: newMockSeriesSet([]storage.Series{ storage.NewListSeries(labels.FromStrings("a", "a"), - []tsdbutil.Sample{sample{3, 4, nil, nil}, sample{5, 2, nil, nil}, sample{6, 3, nil, nil}}, + []chunks.Sample{sample{3, 4, nil, nil}, sample{5, 2, nil, nil}, sample{6, 3, nil, nil}}, ), storage.NewListSeries(labels.FromStrings("a", "a", "b", "b"), - []tsdbutil.Sample{sample{3, 3, nil, nil}, sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, + []chunks.Sample{sample{3, 3, nil, nil}, sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, ), }), expChks: newMockChunkSeriesSet([]storage.ChunkSeries{ storage.NewListChunkSeriesFromSamples(labels.FromStrings("a", "a"), - []tsdbutil.Sample{sample{3, 4, nil, nil}}, []tsdbutil.Sample{sample{5, 2, nil, nil}, sample{6, 3, nil, nil}}, + []chunks.Sample{sample{3, 4, nil, nil}}, []chunks.Sample{sample{5, 2, nil, nil}, sample{6, 3, nil, nil}}, ), storage.NewListChunkSeriesFromSamples(labels.FromStrings("a", "a", "b", "b"), - []tsdbutil.Sample{sample{3, 3, nil, nil}}, []tsdbutil.Sample{sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, + []chunks.Sample{sample{3, 3, nil, nil}}, []chunks.Sample{sample{5, 3, nil, nil}, sample{6, 6, nil, nil}}, ), }), } @@ -608,24 +608,24 @@ func TestBlockQuerierDelete(t *testing.T) { ms: []*labels.Matcher{labels.MustNewMatcher(labels.MatchRegexp, "a", ".*")}, exp: newMockSeriesSet([]storage.Series{ storage.NewListSeries(labels.FromStrings("a", "a"), - []tsdbutil.Sample{sample{5, 2, nil, nil}, sample{6, 3, nil, nil}, sample{7, 4, nil, nil}}, + []chunks.Sample{sample{5, 2, nil, nil}, sample{6, 3, nil, nil}, sample{7, 4, nil, nil}}, ), storage.NewListSeries(labels.FromStrings("a", "a", "b", "b"), - []tsdbutil.Sample{sample{5, 3, nil, nil}}, + []chunks.Sample{sample{5, 3, nil, nil}}, ), storage.NewListSeries(labels.FromStrings("b", "b"), - []tsdbutil.Sample{sample{1, 3, nil, nil}, sample{2, 2, nil, nil}, sample{3, 6, nil, nil}, sample{5, 1, nil, nil}}, + []chunks.Sample{sample{1, 3, nil, nil}, sample{2, 2, nil, nil}, sample{3, 6, nil, nil}, sample{5, 1, nil, nil}}, ), }), expChks: newMockChunkSeriesSet([]storage.ChunkSeries{ storage.NewListChunkSeriesFromSamples(labels.FromStrings("a", "a"), - []tsdbutil.Sample{sample{5, 2, nil, nil}, sample{6, 3, nil, nil}, sample{7, 4, nil, nil}}, + []chunks.Sample{sample{5, 2, nil, nil}, sample{6, 3, nil, nil}, sample{7, 4, nil, nil}}, ), storage.NewListChunkSeriesFromSamples(labels.FromStrings("a", "a", "b", "b"), - []tsdbutil.Sample{sample{5, 3, nil, nil}}, + []chunks.Sample{sample{5, 3, nil, nil}}, ), storage.NewListChunkSeriesFromSamples(labels.FromStrings("b", "b"), - []tsdbutil.Sample{sample{1, 3, nil, nil}, sample{2, 2, nil, nil}, sample{3, 6, nil, nil}}, []tsdbutil.Sample{sample{5, 1, nil, nil}}, + []chunks.Sample{sample{1, 3, nil, nil}, sample{2, 2, nil, nil}, sample{3, 6, nil, nil}}, []chunks.Sample{sample{5, 1, nil, nil}}, ), }), }, @@ -635,18 +635,18 @@ func TestBlockQuerierDelete(t *testing.T) { ms: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "a", "a")}, exp: newMockSeriesSet([]storage.Series{ storage.NewListSeries(labels.FromStrings("a", "a"), - []tsdbutil.Sample{sample{5, 2, nil, nil}, sample{6, 3, nil, nil}}, + []chunks.Sample{sample{5, 2, nil, nil}, sample{6, 3, nil, nil}}, ), storage.NewListSeries(labels.FromStrings("a", "a", "b", "b"), - []tsdbutil.Sample{sample{5, 3, nil, nil}}, + []chunks.Sample{sample{5, 3, nil, nil}}, ), }), expChks: newMockChunkSeriesSet([]storage.ChunkSeries{ storage.NewListChunkSeriesFromSamples(labels.FromStrings("a", "a"), - []tsdbutil.Sample{sample{5, 2, nil, nil}, sample{6, 3, nil, nil}}, + []chunks.Sample{sample{5, 2, nil, nil}, sample{6, 3, nil, nil}}, ), storage.NewListChunkSeriesFromSamples(labels.FromStrings("a", "a", "b", "b"), - []tsdbutil.Sample{sample{5, 3, nil, nil}}, + []chunks.Sample{sample{5, 3, nil, nil}}, ), }), }, @@ -663,14 +663,14 @@ type fakeChunksReader struct { chks map[chunks.ChunkRef]chunkenc.Chunk } -func createFakeReaderAndNotPopulatedChunks(s ...[]tsdbutil.Sample) (*fakeChunksReader, []chunks.Meta) { +func createFakeReaderAndNotPopulatedChunks(s ...[]chunks.Sample) (*fakeChunksReader, []chunks.Meta) { f := &fakeChunksReader{ chks: map[chunks.ChunkRef]chunkenc.Chunk{}, } chks := make([]chunks.Meta, 0, len(s)) for ref, samples := range s { - chk, _ := tsdbutil.ChunkFromSamples(samples) + chk, _ := chunks.ChunkFromSamples(samples) f.chks[chunks.ChunkRef(ref)] = chk.Chunk chks = append(chks, chunks.Meta{ @@ -693,9 +693,9 @@ func (r *fakeChunksReader) Chunk(meta chunks.Meta) (chunkenc.Chunk, error) { func TestPopulateWithTombSeriesIterators(t *testing.T) { cases := []struct { name string - chks [][]tsdbutil.Sample + chks [][]chunks.Sample - expected []tsdbutil.Sample + expected []chunks.Sample expectedChks []chunks.Meta intervals tombstones.Intervals @@ -706,79 +706,79 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { }{ { name: "no chunk", - chks: [][]tsdbutil.Sample{}, + chks: [][]chunks.Sample{}, }, { name: "one empty chunk", // This should never happen. - chks: [][]tsdbutil.Sample{{}}, + chks: [][]chunks.Sample{{}}, expectedChks: []chunks.Meta{ - assureChunkFromSamples(t, []tsdbutil.Sample{}), + assureChunkFromSamples(t, []chunks.Sample{}), }, }, { name: "three empty chunks", // This should never happen. - chks: [][]tsdbutil.Sample{{}, {}, {}}, + chks: [][]chunks.Sample{{}, {}, {}}, expectedChks: []chunks.Meta{ - assureChunkFromSamples(t, []tsdbutil.Sample{}), - assureChunkFromSamples(t, []tsdbutil.Sample{}), - assureChunkFromSamples(t, []tsdbutil.Sample{}), + assureChunkFromSamples(t, []chunks.Sample{}), + assureChunkFromSamples(t, []chunks.Sample{}), + assureChunkFromSamples(t, []chunks.Sample{}), }, }, { name: "one chunk", - chks: [][]tsdbutil.Sample{ + chks: [][]chunks.Sample{ {sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}}, }, - expected: []tsdbutil.Sample{ + expected: []chunks.Sample{ sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}, }, expectedChks: []chunks.Meta{ - assureChunkFromSamples(t, []tsdbutil.Sample{ + assureChunkFromSamples(t, []chunks.Sample{ sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}, }), }, }, { name: "two full chunks", - chks: [][]tsdbutil.Sample{ + chks: [][]chunks.Sample{ {sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}}, {sample{7, 89, nil, nil}, sample{9, 8, nil, nil}}, }, - expected: []tsdbutil.Sample{ + expected: []chunks.Sample{ sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}, sample{7, 89, nil, nil}, sample{9, 8, nil, nil}, }, expectedChks: []chunks.Meta{ - assureChunkFromSamples(t, []tsdbutil.Sample{ + assureChunkFromSamples(t, []chunks.Sample{ sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}, }), - assureChunkFromSamples(t, []tsdbutil.Sample{ + assureChunkFromSamples(t, []chunks.Sample{ sample{7, 89, nil, nil}, sample{9, 8, nil, nil}, }), }, }, { name: "three full chunks", - chks: [][]tsdbutil.Sample{ + chks: [][]chunks.Sample{ {sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}}, {sample{7, 89, nil, nil}, sample{9, 8, nil, nil}}, {sample{10, 22, nil, nil}, sample{203, 3493, nil, nil}}, }, - expected: []tsdbutil.Sample{ + expected: []chunks.Sample{ sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}, sample{7, 89, nil, nil}, sample{9, 8, nil, nil}, sample{10, 22, nil, nil}, sample{203, 3493, nil, nil}, }, expectedChks: []chunks.Meta{ - assureChunkFromSamples(t, []tsdbutil.Sample{ + assureChunkFromSamples(t, []chunks.Sample{ sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}, }), - assureChunkFromSamples(t, []tsdbutil.Sample{ + assureChunkFromSamples(t, []chunks.Sample{ sample{7, 89, nil, nil}, sample{9, 8, nil, nil}, }), - assureChunkFromSamples(t, []tsdbutil.Sample{ + assureChunkFromSamples(t, []chunks.Sample{ sample{10, 22, nil, nil}, sample{203, 3493, nil, nil}, }), }, @@ -786,14 +786,14 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { // Seek cases. { name: "three empty chunks and seek", // This should never happen. - chks: [][]tsdbutil.Sample{{}, {}, {}}, + chks: [][]chunks.Sample{{}, {}, {}}, seek: 1, seekSuccess: false, }, { name: "two chunks and seek beyond chunks", - chks: [][]tsdbutil.Sample{ + chks: [][]chunks.Sample{ {sample{1, 2, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}}, {sample{7, 89, nil, nil}, sample{9, 8, nil, nil}}, }, @@ -803,92 +803,92 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { }, { name: "two chunks and seek on middle of first chunk", - chks: [][]tsdbutil.Sample{ + chks: [][]chunks.Sample{ {sample{1, 2, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}}, {sample{7, 89, nil, nil}, sample{9, 8, nil, nil}}, }, seek: 2, seekSuccess: true, - expected: []tsdbutil.Sample{ + expected: []chunks.Sample{ sample{3, 5, nil, nil}, sample{6, 1, nil, nil}, sample{7, 89, nil, nil}, sample{9, 8, nil, nil}, }, }, { name: "two chunks and seek before first chunk", - chks: [][]tsdbutil.Sample{ + chks: [][]chunks.Sample{ {sample{1, 2, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}}, {sample{7, 89, nil, nil}, sample{9, 8, nil, nil}}, }, seek: -32, seekSuccess: true, - expected: []tsdbutil.Sample{ + expected: []chunks.Sample{ sample{1, 2, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}, sample{7, 89, nil, nil}, sample{9, 8, nil, nil}, }, }, // Deletion / Trim cases. { name: "no chunk with deletion interval", - chks: [][]tsdbutil.Sample{}, + chks: [][]chunks.Sample{}, intervals: tombstones.Intervals{{Mint: 20, Maxt: 21}}, }, { name: "two chunks with trimmed first and last samples from edge chunks", - chks: [][]tsdbutil.Sample{ + chks: [][]chunks.Sample{ {sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}}, {sample{7, 89, nil, nil}, sample{9, 8, nil, nil}}, }, intervals: tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 9, Maxt: math.MaxInt64}), - expected: []tsdbutil.Sample{ + expected: []chunks.Sample{ sample{3, 5, nil, nil}, sample{6, 1, nil, nil}, sample{7, 89, nil, nil}, }, expectedChks: []chunks.Meta{ - assureChunkFromSamples(t, []tsdbutil.Sample{ + assureChunkFromSamples(t, []chunks.Sample{ sample{3, 5, nil, nil}, sample{6, 1, nil, nil}, }), - assureChunkFromSamples(t, []tsdbutil.Sample{ + assureChunkFromSamples(t, []chunks.Sample{ sample{7, 89, nil, nil}, }), }, }, { name: "two chunks with trimmed middle sample of first chunk", - chks: [][]tsdbutil.Sample{ + chks: [][]chunks.Sample{ {sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}}, {sample{7, 89, nil, nil}, sample{9, 8, nil, nil}}, }, intervals: tombstones.Intervals{{Mint: 2, Maxt: 3}}, - expected: []tsdbutil.Sample{ + expected: []chunks.Sample{ sample{1, 2, nil, nil}, sample{6, 1, nil, nil}, sample{7, 89, nil, nil}, sample{9, 8, nil, nil}, }, expectedChks: []chunks.Meta{ - assureChunkFromSamples(t, []tsdbutil.Sample{ + assureChunkFromSamples(t, []chunks.Sample{ sample{1, 2, nil, nil}, sample{6, 1, nil, nil}, }), - assureChunkFromSamples(t, []tsdbutil.Sample{ + assureChunkFromSamples(t, []chunks.Sample{ sample{7, 89, nil, nil}, sample{9, 8, nil, nil}, }), }, }, { name: "two chunks with deletion across two chunks", - chks: [][]tsdbutil.Sample{ + chks: [][]chunks.Sample{ {sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}}, {sample{7, 89, nil, nil}, sample{9, 8, nil, nil}}, }, intervals: tombstones.Intervals{{Mint: 6, Maxt: 7}}, - expected: []tsdbutil.Sample{ + expected: []chunks.Sample{ sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, sample{9, 8, nil, nil}, }, expectedChks: []chunks.Meta{ - assureChunkFromSamples(t, []tsdbutil.Sample{ + assureChunkFromSamples(t, []chunks.Sample{ sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, }), - assureChunkFromSamples(t, []tsdbutil.Sample{ + assureChunkFromSamples(t, []chunks.Sample{ sample{9, 8, nil, nil}, }), }, @@ -896,7 +896,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { // Deletion with seek. { name: "two chunks with trimmed first and last samples from edge chunks, seek from middle of first chunk", - chks: [][]tsdbutil.Sample{ + chks: [][]chunks.Sample{ {sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}}, {sample{7, 89, nil, nil}, sample{9, 8, nil, nil}}, }, @@ -904,13 +904,13 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { seek: 3, seekSuccess: true, - expected: []tsdbutil.Sample{ + expected: []chunks.Sample{ sample{3, 5, nil, nil}, sample{6, 1, nil, nil}, sample{7, 89, nil, nil}, }, }, { name: "one histogram chunk", - chks: [][]tsdbutil.Sample{ + chks: [][]chunks.Sample{ { sample{1, 0, tsdbutil.GenerateTestHistogram(1), nil}, sample{2, 0, tsdbutil.GenerateTestHistogram(2), nil}, @@ -918,14 +918,14 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{6, 0, tsdbutil.GenerateTestHistogram(6), nil}, }, }, - expected: []tsdbutil.Sample{ + expected: []chunks.Sample{ sample{1, 0, tsdbutil.GenerateTestHistogram(1), nil}, sample{2, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(2)), nil}, sample{3, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(3)), nil}, sample{6, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(6)), nil}, }, expectedChks: []chunks.Meta{ - assureChunkFromSamples(t, []tsdbutil.Sample{ + assureChunkFromSamples(t, []chunks.Sample{ sample{1, 0, tsdbutil.GenerateTestHistogram(1), nil}, sample{2, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(2)), nil}, sample{3, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(3)), nil}, @@ -935,7 +935,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { }, { name: "one histogram chunk intersect with deletion interval", - chks: [][]tsdbutil.Sample{ + chks: [][]chunks.Sample{ { sample{1, 0, tsdbutil.GenerateTestHistogram(1), nil}, sample{2, 0, tsdbutil.GenerateTestHistogram(2), nil}, @@ -944,13 +944,13 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { }, }, intervals: tombstones.Intervals{{Mint: 5, Maxt: 20}}, - expected: []tsdbutil.Sample{ + expected: []chunks.Sample{ sample{1, 0, tsdbutil.GenerateTestHistogram(1), nil}, sample{2, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(2)), nil}, sample{3, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(3)), nil}, }, expectedChks: []chunks.Meta{ - assureChunkFromSamples(t, []tsdbutil.Sample{ + assureChunkFromSamples(t, []chunks.Sample{ sample{1, 0, tsdbutil.GenerateTestHistogram(1), nil}, sample{2, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(2)), nil}, sample{3, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(3)), nil}, @@ -959,7 +959,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { }, { name: "one float histogram chunk", - chks: [][]tsdbutil.Sample{ + chks: [][]chunks.Sample{ { sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(1)}, sample{2, 0, nil, tsdbutil.GenerateTestFloatHistogram(2)}, @@ -967,14 +967,14 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{6, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, }, }, - expected: []tsdbutil.Sample{ + expected: []chunks.Sample{ sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(1)}, sample{2, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(2))}, sample{3, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(3))}, sample{6, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(6))}, }, expectedChks: []chunks.Meta{ - assureChunkFromSamples(t, []tsdbutil.Sample{ + assureChunkFromSamples(t, []chunks.Sample{ sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(1)}, sample{2, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(2))}, sample{3, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(3))}, @@ -984,7 +984,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { }, { name: "one float histogram chunk intersect with deletion interval", - chks: [][]tsdbutil.Sample{ + chks: [][]chunks.Sample{ { sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(1)}, sample{2, 0, nil, tsdbutil.GenerateTestFloatHistogram(2)}, @@ -993,13 +993,13 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { }, }, intervals: tombstones.Intervals{{Mint: 5, Maxt: 20}}, - expected: []tsdbutil.Sample{ + expected: []chunks.Sample{ sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(1)}, sample{2, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(2))}, sample{3, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(3))}, }, expectedChks: []chunks.Meta{ - assureChunkFromSamples(t, []tsdbutil.Sample{ + assureChunkFromSamples(t, []chunks.Sample{ sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(1)}, sample{2, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(2))}, sample{3, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(3))}, @@ -1008,7 +1008,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { }, { name: "one gauge histogram chunk", - chks: [][]tsdbutil.Sample{ + chks: [][]chunks.Sample{ { sample{1, 0, tsdbutil.GenerateTestGaugeHistogram(1), nil}, sample{2, 0, tsdbutil.GenerateTestGaugeHistogram(2), nil}, @@ -1016,14 +1016,14 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{6, 0, tsdbutil.GenerateTestGaugeHistogram(6), nil}, }, }, - expected: []tsdbutil.Sample{ + expected: []chunks.Sample{ sample{1, 0, tsdbutil.GenerateTestGaugeHistogram(1), nil}, sample{2, 0, tsdbutil.GenerateTestGaugeHistogram(2), nil}, sample{3, 0, tsdbutil.GenerateTestGaugeHistogram(3), nil}, sample{6, 0, tsdbutil.GenerateTestGaugeHistogram(6), nil}, }, expectedChks: []chunks.Meta{ - assureChunkFromSamples(t, []tsdbutil.Sample{ + assureChunkFromSamples(t, []chunks.Sample{ sample{1, 0, tsdbutil.GenerateTestGaugeHistogram(1), nil}, sample{2, 0, tsdbutil.GenerateTestGaugeHistogram(2), nil}, sample{3, 0, tsdbutil.GenerateTestGaugeHistogram(3), nil}, @@ -1033,7 +1033,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { }, { name: "one gauge histogram chunk intersect with deletion interval", - chks: [][]tsdbutil.Sample{ + chks: [][]chunks.Sample{ { sample{1, 0, tsdbutil.GenerateTestGaugeHistogram(1), nil}, sample{2, 0, tsdbutil.GenerateTestGaugeHistogram(2), nil}, @@ -1042,13 +1042,13 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { }, }, intervals: tombstones.Intervals{{Mint: 5, Maxt: 20}}, - expected: []tsdbutil.Sample{ + expected: []chunks.Sample{ sample{1, 0, tsdbutil.GenerateTestGaugeHistogram(1), nil}, sample{2, 0, tsdbutil.GenerateTestGaugeHistogram(2), nil}, sample{3, 0, tsdbutil.GenerateTestGaugeHistogram(3), nil}, }, expectedChks: []chunks.Meta{ - assureChunkFromSamples(t, []tsdbutil.Sample{ + assureChunkFromSamples(t, []chunks.Sample{ sample{1, 0, tsdbutil.GenerateTestGaugeHistogram(1), nil}, sample{2, 0, tsdbutil.GenerateTestGaugeHistogram(2), nil}, sample{3, 0, tsdbutil.GenerateTestGaugeHistogram(3), nil}, @@ -1057,7 +1057,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { }, { name: "one gauge float histogram", - chks: [][]tsdbutil.Sample{ + chks: [][]chunks.Sample{ { sample{1, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(1)}, sample{2, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(2)}, @@ -1065,14 +1065,14 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{6, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(6)}, }, }, - expected: []tsdbutil.Sample{ + expected: []chunks.Sample{ sample{1, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(1)}, sample{2, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(2)}, sample{3, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3)}, sample{6, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(6)}, }, expectedChks: []chunks.Meta{ - assureChunkFromSamples(t, []tsdbutil.Sample{ + assureChunkFromSamples(t, []chunks.Sample{ sample{1, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(1)}, sample{2, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(2)}, sample{3, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3)}, @@ -1082,7 +1082,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { }, { name: "one gauge float histogram chunk intersect with deletion interval", - chks: [][]tsdbutil.Sample{ + chks: [][]chunks.Sample{ { sample{1, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(1)}, sample{2, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(2)}, @@ -1091,13 +1091,13 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { }, }, intervals: tombstones.Intervals{{Mint: 5, Maxt: 20}}, - expected: []tsdbutil.Sample{ + expected: []chunks.Sample{ sample{1, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(1)}, sample{2, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(2)}, sample{3, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3)}, }, expectedChks: []chunks.Meta{ - assureChunkFromSamples(t, []tsdbutil.Sample{ + assureChunkFromSamples(t, []chunks.Sample{ sample{1, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(1)}, sample{2, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(2)}, sample{3, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3)}, @@ -1112,7 +1112,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { it := &populateWithDelSeriesIterator{} it.reset(ulid.ULID{}, f, chkMetas, tc.intervals) - var r []tsdbutil.Sample + var r []chunks.Sample if tc.seek != 0 { require.Equal(t, tc.seekSuccess, it.Seek(tc.seek) == chunkenc.ValFloat) require.Equal(t, tc.seekSuccess, it.Seek(tc.seek) == chunkenc.ValFloat) // Next one should be noop. @@ -1158,9 +1158,9 @@ func rmChunkRefs(chks []chunks.Meta) { // Regression for: https://github.com/prometheus/tsdb/pull/97 func TestPopulateWithDelSeriesIterator_DoubleSeek(t *testing.T) { f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []tsdbutil.Sample{}, - []tsdbutil.Sample{sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}}, - []tsdbutil.Sample{sample{4, 4, nil, nil}, sample{5, 5, nil, nil}}, + []chunks.Sample{}, + []chunks.Sample{sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}}, + []chunks.Sample{sample{4, 4, nil, nil}, sample{5, 5, nil, nil}}, ) it := &populateWithDelSeriesIterator{} @@ -1177,9 +1177,9 @@ func TestPopulateWithDelSeriesIterator_DoubleSeek(t *testing.T) { // skipped to the end when seeking a value in the current chunk. func TestPopulateWithDelSeriesIterator_SeekInCurrentChunk(t *testing.T) { f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []tsdbutil.Sample{}, - []tsdbutil.Sample{sample{1, 2, nil, nil}, sample{3, 4, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, - []tsdbutil.Sample{}, + []chunks.Sample{}, + []chunks.Sample{sample{1, 2, nil, nil}, sample{3, 4, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, + []chunks.Sample{}, ) it := &populateWithDelSeriesIterator{} @@ -1197,7 +1197,7 @@ func TestPopulateWithDelSeriesIterator_SeekInCurrentChunk(t *testing.T) { func TestPopulateWithDelSeriesIterator_SeekWithMinTime(t *testing.T) { f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []tsdbutil.Sample{sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{6, 8, nil, nil}}, + []chunks.Sample{sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{6, 8, nil, nil}}, ) it := &populateWithDelSeriesIterator{} @@ -1210,7 +1210,7 @@ func TestPopulateWithDelSeriesIterator_SeekWithMinTime(t *testing.T) { // Seek gets called and advances beyond the max time, which was just accepted as a valid sample. func TestPopulateWithDelSeriesIterator_NextWithMinTime(t *testing.T) { f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []tsdbutil.Sample{sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, + []chunks.Sample{sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, ) it := &populateWithDelSeriesIterator{} diff --git a/tsdb/tsdbutil/chunks.go b/tsdb/tsdbutil/chunks.go deleted file mode 100644 index c9d796f9b..000000000 --- a/tsdb/tsdbutil/chunks.go +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright 2018 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tsdbutil - -import ( - "fmt" - - "github.com/prometheus/prometheus/model/histogram" - "github.com/prometheus/prometheus/tsdb/chunkenc" - "github.com/prometheus/prometheus/tsdb/chunks" -) - -type Samples interface { - Get(i int) Sample - Len() int -} - -type Sample interface { - T() int64 - F() float64 - H() *histogram.Histogram - FH() *histogram.FloatHistogram - Type() chunkenc.ValueType -} - -type SampleSlice []Sample - -func (s SampleSlice) Get(i int) Sample { return s[i] } -func (s SampleSlice) Len() int { return len(s) } - -// ChunkFromSamples requires all samples to have the same type. -func ChunkFromSamples(s []Sample) (chunks.Meta, error) { - return ChunkFromSamplesGeneric(SampleSlice(s)) -} - -// ChunkFromSamplesGeneric requires all samples to have the same type. -func ChunkFromSamplesGeneric(s Samples) (chunks.Meta, error) { - emptyChunk := chunks.Meta{Chunk: chunkenc.NewXORChunk()} - mint, maxt := int64(0), int64(0) - - if s.Len() > 0 { - mint, maxt = s.Get(0).T(), s.Get(s.Len()-1).T() - } - - if s.Len() == 0 { - return emptyChunk, nil - } - - sampleType := s.Get(0).Type() - c, err := chunkenc.NewEmptyChunk(sampleType.ChunkEncoding()) - if err != nil { - return chunks.Meta{}, err - } - - ca, _ := c.Appender() - var newChunk chunkenc.Chunk - - for i := 0; i < s.Len(); i++ { - switch sampleType { - case chunkenc.ValFloat: - ca.Append(s.Get(i).T(), s.Get(i).F()) - case chunkenc.ValHistogram: - newChunk, _, ca, err = ca.AppendHistogram(nil, s.Get(i).T(), s.Get(i).H(), false) - if err != nil { - return emptyChunk, err - } - if newChunk != nil { - return emptyChunk, fmt.Errorf("did not expect to start a second chunk") - } - case chunkenc.ValFloatHistogram: - newChunk, _, ca, err = ca.AppendFloatHistogram(nil, s.Get(i).T(), s.Get(i).FH(), false) - if err != nil { - return emptyChunk, err - } - if newChunk != nil { - return emptyChunk, fmt.Errorf("did not expect to start a second chunk") - } - default: - panic(fmt.Sprintf("unknown sample type %s", sampleType.String())) - } - } - return chunks.Meta{ - MinTime: mint, - MaxTime: maxt, - Chunk: c, - }, nil -} - -type sample struct { - t int64 - f float64 - h *histogram.Histogram - fh *histogram.FloatHistogram -} - -func (s sample) T() int64 { - return s.t -} - -func (s sample) F() float64 { - return s.f -} - -func (s sample) H() *histogram.Histogram { - return s.h -} - -func (s sample) FH() *histogram.FloatHistogram { - return s.fh -} - -func (s sample) Type() chunkenc.ValueType { - switch { - case s.h != nil: - return chunkenc.ValHistogram - case s.fh != nil: - return chunkenc.ValFloatHistogram - default: - return chunkenc.ValFloat - } -} - -// PopulatedChunk creates a chunk populated with samples every second starting at minTime -func PopulatedChunk(numSamples int, minTime int64) (chunks.Meta, error) { - samples := make([]Sample, numSamples) - for i := 0; i < numSamples; i++ { - samples[i] = sample{t: minTime + int64(i*1000), f: 1.0} - } - return ChunkFromSamples(samples) -} - -// GenerateSamples starting at start and counting up numSamples. -func GenerateSamples(start, numSamples int) []Sample { - return generateSamples(start, numSamples, func(i int) Sample { - return sample{ - t: int64(i), - f: float64(i), - } - }) -} - -func generateSamples(start, numSamples int, gen func(int) Sample) []Sample { - samples := make([]Sample, 0, numSamples) - for i := start; i < start+numSamples; i++ { - samples = append(samples, gen(i)) - } - return samples -} diff --git a/tsdb/tsdbutil/histogram.go b/tsdb/tsdbutil/histogram.go index 8270da686..0327815c4 100644 --- a/tsdb/tsdbutil/histogram.go +++ b/tsdb/tsdbutil/histogram.go @@ -14,7 +14,7 @@ package tsdbutil import ( - "math/rand" + "math" "github.com/prometheus/prometheus/model/histogram" ) @@ -53,7 +53,8 @@ func GenerateTestHistogram(i int) *histogram.Histogram { func GenerateTestGaugeHistograms(n int) (r []*histogram.Histogram) { for x := 0; x < n; x++ { - r = append(r, GenerateTestGaugeHistogram(rand.Intn(n))) + i := int(math.Sin(float64(x))*100) + 100 + r = append(r, GenerateTestGaugeHistogram(i)) } return r } @@ -98,7 +99,8 @@ func GenerateTestFloatHistogram(i int) *histogram.FloatHistogram { func GenerateTestGaugeFloatHistograms(n int) (r []*histogram.FloatHistogram) { for x := 0; x < n; x++ { - r = append(r, GenerateTestGaugeFloatHistogram(rand.Intn(n))) + i := int(math.Sin(float64(x))*100) + 100 + r = append(r, GenerateTestGaugeFloatHistogram(i)) } return r }