From 46be85f2dc054920bcf0fa7718de099436ee8491 Mon Sep 17 00:00:00 2001 From: Jeanette Tan Date: Wed, 1 Nov 2023 19:53:41 +0800 Subject: [PATCH 1/8] Make TestPopulateWithDelSeriesIterator tests cover histogram types and check MinTime Signed-off-by: Jeanette Tan --- tsdb/querier_test.go | 237 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 195 insertions(+), 42 deletions(-) diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go index fc6c688010..096e0ff729 100644 --- a/tsdb/querier_test.go +++ b/tsdb/querier_test.go @@ -1160,65 +1160,218 @@ func rmChunkRefs(chks []chunks.Meta) { // Regression for: https://github.com/prometheus/tsdb/pull/97 func TestPopulateWithDelSeriesIterator_DoubleSeek(t *testing.T) { - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{}, - []chunks.Sample{sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}}, - []chunks.Sample{sample{4, 4, nil, nil}, sample{5, 5, nil, nil}}, - ) - - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, chunkenc.ValFloat, it.Seek(1)) - require.Equal(t, chunkenc.ValFloat, it.Seek(2)) - require.Equal(t, chunkenc.ValFloat, it.Seek(2)) - ts, v := it.At() - require.Equal(t, int64(2), ts) - require.Equal(t, float64(2), v) + t.Run("float", func(t *testing.T) { + valType := chunkenc.ValFloat + f, chkMetas := createFakeReaderAndNotPopulatedChunks( + []chunks.Sample{}, + []chunks.Sample{sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}}, + []chunks.Sample{sample{4, 4, nil, nil}, sample{5, 5, nil, nil}}, + ) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, valType, it.Seek(1)) + require.Equal(t, valType, it.Seek(2)) + require.Equal(t, valType, it.Seek(2)) + ts, v := it.At() + require.Equal(t, int64(2), ts) + require.Equal(t, float64(2), v) + require.Equal(t, int64(0), chkMetas[0].MinTime) + require.Equal(t, int64(1), chkMetas[1].MinTime) + require.Equal(t, int64(4), chkMetas[2].MinTime) + }) + t.Run("histogram", func(t *testing.T) { + valType := chunkenc.ValHistogram + f, chkMetas := createFakeReaderAndNotPopulatedChunks( + []chunks.Sample{}, + []chunks.Sample{sample{1, 0, tsdbutil.GenerateTestHistogram(1), nil}, sample{2, 0, tsdbutil.GenerateTestHistogram(2), nil}, sample{3, 0, tsdbutil.GenerateTestHistogram(3), nil}}, + []chunks.Sample{sample{4, 0, tsdbutil.GenerateTestHistogram(4), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(5), nil}}, + ) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, valType, it.Seek(1)) + require.Equal(t, valType, it.Seek(2)) + require.Equal(t, valType, it.Seek(2)) + ts, h := it.AtHistogram() + require.Equal(t, int64(2), ts) + h.CounterResetHint = histogram.UnknownCounterReset + require.Equal(t, tsdbutil.GenerateTestHistogram(2), h) + require.Equal(t, int64(0), chkMetas[0].MinTime) + require.Equal(t, int64(1), chkMetas[1].MinTime) + require.Equal(t, int64(4), chkMetas[2].MinTime) + }) + t.Run("float histogram", func(t *testing.T) { + valType := chunkenc.ValFloatHistogram + f, chkMetas := createFakeReaderAndNotPopulatedChunks( + []chunks.Sample{}, + []chunks.Sample{sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(1)}, sample{2, 0, nil, tsdbutil.GenerateTestFloatHistogram(2)}, sample{3, 0, nil, tsdbutil.GenerateTestFloatHistogram(3)}}, + []chunks.Sample{sample{4, 0, nil, tsdbutil.GenerateTestFloatHistogram(4)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(5)}}, + ) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, valType, it.Seek(1)) + require.Equal(t, valType, it.Seek(2)) + require.Equal(t, valType, it.Seek(2)) + ts, h := it.AtFloatHistogram() + require.Equal(t, int64(2), ts) + h.CounterResetHint = histogram.UnknownCounterReset + require.Equal(t, tsdbutil.GenerateTestFloatHistogram(2), h) + require.Equal(t, int64(0), chkMetas[0].MinTime) + require.Equal(t, int64(1), chkMetas[1].MinTime) + require.Equal(t, int64(4), chkMetas[2].MinTime) + }) } // Regression when seeked chunks were still found via binary search and we always // skipped to the end when seeking a value in the current chunk. func TestPopulateWithDelSeriesIterator_SeekInCurrentChunk(t *testing.T) { - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{}, - []chunks.Sample{sample{1, 2, nil, nil}, sample{3, 4, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, - []chunks.Sample{}, - ) + t.Run("float", func(t *testing.T) { + valType := chunkenc.ValFloat + f, chkMetas := createFakeReaderAndNotPopulatedChunks( + []chunks.Sample{}, + []chunks.Sample{sample{1, 2, nil, nil}, sample{3, 4, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, + []chunks.Sample{}, + ) - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, chunkenc.ValFloat, it.Next()) - ts, v := it.At() - require.Equal(t, int64(1), ts) - require.Equal(t, float64(2), v) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, valType, it.Next()) + ts, v := it.At() + require.Equal(t, int64(1), ts) + require.Equal(t, float64(2), v) - require.Equal(t, chunkenc.ValFloat, it.Seek(4)) - ts, v = it.At() - require.Equal(t, int64(5), ts) - require.Equal(t, float64(6), v) + require.Equal(t, valType, it.Seek(4)) + ts, v = it.At() + require.Equal(t, int64(5), ts) + require.Equal(t, float64(6), v) + + require.Equal(t, int64(0), chkMetas[0].MinTime) + require.Equal(t, int64(1), chkMetas[1].MinTime) + require.Equal(t, int64(0), chkMetas[2].MinTime) + }) + t.Run("histogram", func(t *testing.T) { + valType := chunkenc.ValHistogram + f, chkMetas := createFakeReaderAndNotPopulatedChunks( + []chunks.Sample{}, + []chunks.Sample{sample{1, 0, tsdbutil.GenerateTestHistogram(2), nil}, sample{3, 0, tsdbutil.GenerateTestHistogram(4), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{7, 0, tsdbutil.GenerateTestHistogram(8), nil}}, + []chunks.Sample{}, + ) + + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, valType, it.Next()) + ts, h := it.AtHistogram() + require.Equal(t, int64(1), ts) + require.Equal(t, tsdbutil.GenerateTestHistogram(2), h) + + require.Equal(t, valType, it.Seek(4)) + ts, h = it.AtHistogram() + require.Equal(t, int64(5), ts) + h.CounterResetHint = histogram.UnknownCounterReset + require.Equal(t, tsdbutil.GenerateTestHistogram(6), h) + + require.Equal(t, int64(0), chkMetas[0].MinTime) + require.Equal(t, int64(1), chkMetas[1].MinTime) + require.Equal(t, int64(0), chkMetas[2].MinTime) + }) + t.Run("float histogram", func(t *testing.T) { + valType := chunkenc.ValFloatHistogram + f, chkMetas := createFakeReaderAndNotPopulatedChunks( + []chunks.Sample{}, + []chunks.Sample{sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(2)}, sample{3, 0, nil, tsdbutil.GenerateTestFloatHistogram(4)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{7, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, + []chunks.Sample{}, + ) + + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, valType, it.Next()) + ts, h := it.AtFloatHistogram() + require.Equal(t, int64(1), ts) + require.Equal(t, tsdbutil.GenerateTestFloatHistogram(2), h) + + require.Equal(t, valType, it.Seek(4)) + ts, h = it.AtFloatHistogram() + require.Equal(t, int64(5), ts) + h.CounterResetHint = histogram.UnknownCounterReset + require.Equal(t, tsdbutil.GenerateTestFloatHistogram(6), h) + + require.Equal(t, int64(0), chkMetas[0].MinTime) + require.Equal(t, int64(1), chkMetas[1].MinTime) + require.Equal(t, int64(0), chkMetas[2].MinTime) + }) } func TestPopulateWithDelSeriesIterator_SeekWithMinTime(t *testing.T) { - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{6, 8, nil, nil}}, - ) + t.Run("float", func(t *testing.T) { + valType := chunkenc.ValFloat + f, chkMetas := createFakeReaderAndNotPopulatedChunks( + []chunks.Sample{sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{6, 8, nil, nil}}, + ) - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, chunkenc.ValNone, it.Seek(7)) - require.Equal(t, chunkenc.ValFloat, it.Seek(3)) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, chunkenc.ValNone, it.Seek(7)) + require.Equal(t, valType, it.Seek(3)) + require.Equal(t, int64(1), chkMetas[0].MinTime) + }) + t.Run("histogram", func(t *testing.T) { + valType := chunkenc.ValHistogram + f, chkMetas := createFakeReaderAndNotPopulatedChunks( + []chunks.Sample{sample{1, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{6, 0, tsdbutil.GenerateTestHistogram(8), nil}}, + ) + + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, chunkenc.ValNone, it.Seek(7)) + require.Equal(t, valType, it.Seek(3)) + require.Equal(t, int64(1), chkMetas[0].MinTime) + }) + t.Run("float histogram", func(t *testing.T) { + valType := chunkenc.ValFloatHistogram + f, chkMetas := createFakeReaderAndNotPopulatedChunks( + []chunks.Sample{sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{6, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, + ) + + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, chunkenc.ValNone, it.Seek(7)) + require.Equal(t, valType, it.Seek(3)) + require.Equal(t, int64(1), chkMetas[0].MinTime) + }) } // Regression when calling Next() with a time bounded to fit within two samples. // Seek gets called and advances beyond the max time, which was just accepted as a valid sample. func TestPopulateWithDelSeriesIterator_NextWithMinTime(t *testing.T) { - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, - ) + t.Run("float", func(t *testing.T) { + f, chkMetas := createFakeReaderAndNotPopulatedChunks( + []chunks.Sample{sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, + ) - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 4, Maxt: math.MaxInt64})) - require.Equal(t, chunkenc.ValNone, it.Next()) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 4, Maxt: math.MaxInt64})) + require.Equal(t, chunkenc.ValNone, it.Next()) + require.Equal(t, int64(1), chkMetas[0].MinTime) + }) + t.Run("histogram", func(t *testing.T) { + f, chkMetas := createFakeReaderAndNotPopulatedChunks( + []chunks.Sample{sample{1, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{7, 0, tsdbutil.GenerateTestHistogram(8), nil}}, + ) + + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 4, Maxt: math.MaxInt64})) + require.Equal(t, chunkenc.ValNone, it.Next()) + require.Equal(t, int64(1), chkMetas[0].MinTime) + }) + t.Run("float histogram", func(t *testing.T) { + f, chkMetas := createFakeReaderAndNotPopulatedChunks( + []chunks.Sample{sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{7, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, + ) + + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 4, Maxt: math.MaxInt64})) + require.Equal(t, chunkenc.ValNone, it.Next()) + require.Equal(t, int64(1), chkMetas[0].MinTime) + }) } // Test the cost of merging series sets for different number of merged sets and their size. From 04aabdd7cc86fedba3dd2e614fa5289b746bda28 Mon Sep 17 00:00:00 2001 From: Jeanette Tan Date: Wed, 1 Nov 2023 19:53:41 +0800 Subject: [PATCH 2/8] Refactor TestPopulateWithDelSeriesIterator unit tests to reuse more code Signed-off-by: Jeanette Tan --- tsdb/querier_test.go | 385 +++++++++++++++++++++---------------------- 1 file changed, 189 insertions(+), 196 deletions(-) diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go index 096e0ff729..1c50724d7c 100644 --- a/tsdb/querier_test.go +++ b/tsdb/querier_test.go @@ -1158,220 +1158,213 @@ func rmChunkRefs(chks []chunks.Meta) { } } +func checkCurrVal(t *testing.T, valType chunkenc.ValueType, it *populateWithDelSeriesIterator, expectedTs, expectedValue int) { + switch valType { + case chunkenc.ValFloat: + ts, v := it.At() + require.Equal(t, int64(expectedTs), ts) + require.Equal(t, float64(expectedValue), v) + case chunkenc.ValHistogram: + ts, h := it.AtHistogram() + require.Equal(t, int64(expectedTs), ts) + h.CounterResetHint = histogram.UnknownCounterReset + require.Equal(t, tsdbutil.GenerateTestHistogram(expectedValue), h) + case chunkenc.ValFloatHistogram: + ts, h := it.AtFloatHistogram() + require.Equal(t, int64(expectedTs), ts) + h.CounterResetHint = histogram.UnknownCounterReset + require.Equal(t, tsdbutil.GenerateTestFloatHistogram(expectedValue), h) + default: + panic("unexpected value type") + } +} + // Regression for: https://github.com/prometheus/tsdb/pull/97 func TestPopulateWithDelSeriesIterator_DoubleSeek(t *testing.T) { - t.Run("float", func(t *testing.T) { - valType := chunkenc.ValFloat - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{}, - []chunks.Sample{sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}}, - []chunks.Sample{sample{4, 4, nil, nil}, sample{5, 5, nil, nil}}, - ) - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, valType, it.Seek(1)) - require.Equal(t, valType, it.Seek(2)) - require.Equal(t, valType, it.Seek(2)) - ts, v := it.At() - require.Equal(t, int64(2), ts) - require.Equal(t, float64(2), v) - require.Equal(t, int64(0), chkMetas[0].MinTime) - require.Equal(t, int64(1), chkMetas[1].MinTime) - require.Equal(t, int64(4), chkMetas[2].MinTime) - }) - t.Run("histogram", func(t *testing.T) { - valType := chunkenc.ValHistogram - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{}, - []chunks.Sample{sample{1, 0, tsdbutil.GenerateTestHistogram(1), nil}, sample{2, 0, tsdbutil.GenerateTestHistogram(2), nil}, sample{3, 0, tsdbutil.GenerateTestHistogram(3), nil}}, - []chunks.Sample{sample{4, 0, tsdbutil.GenerateTestHistogram(4), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(5), nil}}, - ) - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, valType, it.Seek(1)) - require.Equal(t, valType, it.Seek(2)) - require.Equal(t, valType, it.Seek(2)) - ts, h := it.AtHistogram() - require.Equal(t, int64(2), ts) - h.CounterResetHint = histogram.UnknownCounterReset - require.Equal(t, tsdbutil.GenerateTestHistogram(2), h) - require.Equal(t, int64(0), chkMetas[0].MinTime) - require.Equal(t, int64(1), chkMetas[1].MinTime) - require.Equal(t, int64(4), chkMetas[2].MinTime) - }) - t.Run("float histogram", func(t *testing.T) { - valType := chunkenc.ValFloatHistogram - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{}, - []chunks.Sample{sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(1)}, sample{2, 0, nil, tsdbutil.GenerateTestFloatHistogram(2)}, sample{3, 0, nil, tsdbutil.GenerateTestFloatHistogram(3)}}, - []chunks.Sample{sample{4, 0, nil, tsdbutil.GenerateTestFloatHistogram(4)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(5)}}, - ) - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, valType, it.Seek(1)) - require.Equal(t, valType, it.Seek(2)) - require.Equal(t, valType, it.Seek(2)) - ts, h := it.AtFloatHistogram() - require.Equal(t, int64(2), ts) - h.CounterResetHint = histogram.UnknownCounterReset - require.Equal(t, tsdbutil.GenerateTestFloatHistogram(2), h) - require.Equal(t, int64(0), chkMetas[0].MinTime) - require.Equal(t, int64(1), chkMetas[1].MinTime) - require.Equal(t, int64(4), chkMetas[2].MinTime) - }) + cases := []struct { + name string + valType chunkenc.ValueType + chks [][]chunks.Sample + }{ + { + name: "float", + valType: chunkenc.ValFloat, + chks: [][]chunks.Sample{ + {}, + {sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}}, + {sample{4, 4, nil, nil}, sample{5, 5, nil, nil}}, + }, + }, + { + name: "histogram", + valType: chunkenc.ValHistogram, + chks: [][]chunks.Sample{ + {}, + {sample{1, 0, tsdbutil.GenerateTestHistogram(1), nil}, sample{2, 0, tsdbutil.GenerateTestHistogram(2), nil}, sample{3, 0, tsdbutil.GenerateTestHistogram(3), nil}}, + {sample{4, 0, tsdbutil.GenerateTestHistogram(4), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(5), nil}}, + }, + }, + { + name: "float histogram", + valType: chunkenc.ValFloatHistogram, + chks: [][]chunks.Sample{ + {}, + {sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(1)}, sample{2, 0, nil, tsdbutil.GenerateTestFloatHistogram(2)}, sample{3, 0, nil, tsdbutil.GenerateTestFloatHistogram(3)}}, + {sample{4, 0, nil, tsdbutil.GenerateTestFloatHistogram(4)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(5)}}, + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f, chkMetas := createFakeReaderAndNotPopulatedChunks(tc.chks...) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, tc.valType, it.Seek(1)) + require.Equal(t, tc.valType, it.Seek(2)) + require.Equal(t, tc.valType, it.Seek(2)) + checkCurrVal(t, tc.valType, it, 2, 2) + require.Equal(t, int64(0), chkMetas[0].MinTime) + require.Equal(t, int64(1), chkMetas[1].MinTime) + require.Equal(t, int64(4), chkMetas[2].MinTime) + }) + } } // Regression when seeked chunks were still found via binary search and we always // skipped to the end when seeking a value in the current chunk. func TestPopulateWithDelSeriesIterator_SeekInCurrentChunk(t *testing.T) { - t.Run("float", func(t *testing.T) { - valType := chunkenc.ValFloat - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{}, - []chunks.Sample{sample{1, 2, nil, nil}, sample{3, 4, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, - []chunks.Sample{}, - ) + cases := []struct { + name string + valType chunkenc.ValueType + chks [][]chunks.Sample + }{ + { + name: "float", + valType: chunkenc.ValFloat, + chks: [][]chunks.Sample{ + {}, + {sample{1, 2, nil, nil}, sample{3, 4, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, + {}, + }, + }, + { + name: "histogram", + valType: chunkenc.ValHistogram, + chks: [][]chunks.Sample{ + {}, + {sample{1, 0, tsdbutil.GenerateTestHistogram(2), nil}, sample{3, 0, tsdbutil.GenerateTestHistogram(4), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{7, 0, tsdbutil.GenerateTestHistogram(8), nil}}, + {}, + }, + }, + { + name: "float histogram", + valType: chunkenc.ValFloatHistogram, + chks: [][]chunks.Sample{ + {}, + {sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(2)}, sample{3, 0, nil, tsdbutil.GenerateTestFloatHistogram(4)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{7, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, + {}, + }, + }, + } - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, valType, it.Next()) - ts, v := it.At() - require.Equal(t, int64(1), ts) - require.Equal(t, float64(2), v) - - require.Equal(t, valType, it.Seek(4)) - ts, v = it.At() - require.Equal(t, int64(5), ts) - require.Equal(t, float64(6), v) - - require.Equal(t, int64(0), chkMetas[0].MinTime) - require.Equal(t, int64(1), chkMetas[1].MinTime) - require.Equal(t, int64(0), chkMetas[2].MinTime) - }) - t.Run("histogram", func(t *testing.T) { - valType := chunkenc.ValHistogram - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{}, - []chunks.Sample{sample{1, 0, tsdbutil.GenerateTestHistogram(2), nil}, sample{3, 0, tsdbutil.GenerateTestHistogram(4), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{7, 0, tsdbutil.GenerateTestHistogram(8), nil}}, - []chunks.Sample{}, - ) - - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, valType, it.Next()) - ts, h := it.AtHistogram() - require.Equal(t, int64(1), ts) - require.Equal(t, tsdbutil.GenerateTestHistogram(2), h) - - require.Equal(t, valType, it.Seek(4)) - ts, h = it.AtHistogram() - require.Equal(t, int64(5), ts) - h.CounterResetHint = histogram.UnknownCounterReset - require.Equal(t, tsdbutil.GenerateTestHistogram(6), h) - - require.Equal(t, int64(0), chkMetas[0].MinTime) - require.Equal(t, int64(1), chkMetas[1].MinTime) - require.Equal(t, int64(0), chkMetas[2].MinTime) - }) - t.Run("float histogram", func(t *testing.T) { - valType := chunkenc.ValFloatHistogram - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{}, - []chunks.Sample{sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(2)}, sample{3, 0, nil, tsdbutil.GenerateTestFloatHistogram(4)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{7, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, - []chunks.Sample{}, - ) - - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, valType, it.Next()) - ts, h := it.AtFloatHistogram() - require.Equal(t, int64(1), ts) - require.Equal(t, tsdbutil.GenerateTestFloatHistogram(2), h) - - require.Equal(t, valType, it.Seek(4)) - ts, h = it.AtFloatHistogram() - require.Equal(t, int64(5), ts) - h.CounterResetHint = histogram.UnknownCounterReset - require.Equal(t, tsdbutil.GenerateTestFloatHistogram(6), h) - - require.Equal(t, int64(0), chkMetas[0].MinTime) - require.Equal(t, int64(1), chkMetas[1].MinTime) - require.Equal(t, int64(0), chkMetas[2].MinTime) - }) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f, chkMetas := createFakeReaderAndNotPopulatedChunks(tc.chks...) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, tc.valType, it.Next()) + checkCurrVal(t, tc.valType, it, 1, 2) + require.Equal(t, tc.valType, it.Seek(4)) + checkCurrVal(t, tc.valType, it, 5, 6) + require.Equal(t, int64(0), chkMetas[0].MinTime) + require.Equal(t, int64(1), chkMetas[1].MinTime) + require.Equal(t, int64(0), chkMetas[2].MinTime) + }) + } } func TestPopulateWithDelSeriesIterator_SeekWithMinTime(t *testing.T) { - t.Run("float", func(t *testing.T) { - valType := chunkenc.ValFloat - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{6, 8, nil, nil}}, - ) + cases := []struct { + name string + valType chunkenc.ValueType + chks [][]chunks.Sample + }{ + { + name: "float", + valType: chunkenc.ValFloat, + chks: [][]chunks.Sample{ + {sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{6, 8, nil, nil}}, + }, + }, + { + name: "histogram", + valType: chunkenc.ValHistogram, + chks: [][]chunks.Sample{ + {sample{1, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{6, 0, tsdbutil.GenerateTestHistogram(8), nil}}, + }, + }, + { + name: "float histogram", + valType: chunkenc.ValFloatHistogram, + chks: [][]chunks.Sample{ + {sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{6, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, + }, + }, + } - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, chunkenc.ValNone, it.Seek(7)) - require.Equal(t, valType, it.Seek(3)) - require.Equal(t, int64(1), chkMetas[0].MinTime) - }) - t.Run("histogram", func(t *testing.T) { - valType := chunkenc.ValHistogram - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{sample{1, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{6, 0, tsdbutil.GenerateTestHistogram(8), nil}}, - ) - - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, chunkenc.ValNone, it.Seek(7)) - require.Equal(t, valType, it.Seek(3)) - require.Equal(t, int64(1), chkMetas[0].MinTime) - }) - t.Run("float histogram", func(t *testing.T) { - valType := chunkenc.ValFloatHistogram - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{6, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, - ) - - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, chunkenc.ValNone, it.Seek(7)) - require.Equal(t, valType, it.Seek(3)) - require.Equal(t, int64(1), chkMetas[0].MinTime) - }) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f, chkMetas := createFakeReaderAndNotPopulatedChunks(tc.chks...) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, chunkenc.ValNone, it.Seek(7)) + require.Equal(t, tc.valType, it.Seek(3)) + require.Equal(t, int64(1), chkMetas[0].MinTime) + }) + } } // Regression when calling Next() with a time bounded to fit within two samples. // Seek gets called and advances beyond the max time, which was just accepted as a valid sample. func TestPopulateWithDelSeriesIterator_NextWithMinTime(t *testing.T) { - t.Run("float", func(t *testing.T) { - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, - ) + cases := []struct { + name string + valType chunkenc.ValueType + chks [][]chunks.Sample + }{ + { + name: "float", + valType: chunkenc.ValFloat, + chks: [][]chunks.Sample{ + {sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, + }, + }, + { + name: "histogram", + valType: chunkenc.ValHistogram, + chks: [][]chunks.Sample{ + {sample{1, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{7, 0, tsdbutil.GenerateTestHistogram(8), nil}}, + }, + }, + { + name: "float histogram", + valType: chunkenc.ValFloatHistogram, + chks: [][]chunks.Sample{ + {sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{7, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, + }, + }, + } - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 4, Maxt: math.MaxInt64})) - require.Equal(t, chunkenc.ValNone, it.Next()) - require.Equal(t, int64(1), chkMetas[0].MinTime) - }) - t.Run("histogram", func(t *testing.T) { - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{sample{1, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{7, 0, tsdbutil.GenerateTestHistogram(8), nil}}, - ) - - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 4, Maxt: math.MaxInt64})) - require.Equal(t, chunkenc.ValNone, it.Next()) - require.Equal(t, int64(1), chkMetas[0].MinTime) - }) - t.Run("float histogram", func(t *testing.T) { - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{7, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, - ) - - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 4, Maxt: math.MaxInt64})) - require.Equal(t, chunkenc.ValNone, it.Next()) - require.Equal(t, int64(1), chkMetas[0].MinTime) - }) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f, chkMetas := createFakeReaderAndNotPopulatedChunks(tc.chks...) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 4, Maxt: math.MaxInt64})) + require.Equal(t, chunkenc.ValNone, it.Next()) + require.Equal(t, int64(1), chkMetas[0].MinTime) + }) + } } // Test the cost of merging series sets for different number of merged sets and their size. From 7a4a1127b70d91ee3805238c61957ada1874b13a Mon Sep 17 00:00:00 2001 From: Jeanette Tan Date: Wed, 1 Nov 2023 19:53:41 +0800 Subject: [PATCH 3/8] Expand TestPopulateWithTombSeriesIterators to test min max times of chunks, including mixed chunks Signed-off-by: Jeanette Tan --- tsdb/querier_test.go | 124 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 122 insertions(+), 2 deletions(-) diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go index 1c50724d7c..5ad5003f1f 100644 --- a/tsdb/querier_test.go +++ b/tsdb/querier_test.go @@ -694,12 +694,16 @@ func (r *fakeChunksReader) Chunk(meta chunks.Meta) (chunkenc.Chunk, error) { } func TestPopulateWithTombSeriesIterators(t *testing.T) { + type minMaxTimes struct { + minTime, maxTime int64 + } cases := []struct { name string chks [][]chunks.Sample - expected []chunks.Sample - expectedChks []chunks.Meta + expected []chunks.Sample + expectedChks []chunks.Meta + expectedMinMaxTimes []minMaxTimes intervals tombstones.Intervals @@ -718,6 +722,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { expectedChks: []chunks.Meta{ assureChunkFromSamples(t, []chunks.Sample{}), }, + expectedMinMaxTimes: []minMaxTimes{{0, 0}}, }, { name: "three empty chunks", // This should never happen. @@ -728,6 +733,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { assureChunkFromSamples(t, []chunks.Sample{}), assureChunkFromSamples(t, []chunks.Sample{}), }, + expectedMinMaxTimes: []minMaxTimes{{0, 0}, {0, 0}, {0, 0}}, }, { name: "one chunk", @@ -743,6 +749,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}}, }, { name: "two full chunks", @@ -762,6 +769,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{7, 89, nil, nil}, sample{9, 8, nil, nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}, {7, 9}}, }, { name: "three full chunks", @@ -785,6 +793,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{10, 22, nil, nil}, sample{203, 3493, nil, nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}, {7, 9}, {10, 203}}, }, // Seek cases. { @@ -855,6 +864,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{7, 89, nil, nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{3, 6}, {7, 7}}, }, { name: "two chunks with trimmed middle sample of first chunk", @@ -875,6 +885,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{7, 89, nil, nil}, sample{9, 8, nil, nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}, {7, 9}}, }, { name: "two chunks with deletion across two chunks", @@ -895,6 +906,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{9, 8, nil, nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 3}, {9, 9}}, }, // Deletion with seek. { @@ -935,6 +947,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{6, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(6)), nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}}, }, { name: "one histogram chunk intersect with deletion interval", @@ -959,6 +972,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{3, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(3)), nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 3}}, }, { name: "one float histogram chunk", @@ -984,6 +998,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{6, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(6))}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}}, }, { name: "one float histogram chunk intersect with deletion interval", @@ -1008,6 +1023,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{3, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(3))}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 3}}, }, { name: "one gauge histogram chunk", @@ -1033,6 +1049,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{6, 0, tsdbutil.GenerateTestGaugeHistogram(6), nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}}, }, { name: "one gauge histogram chunk intersect with deletion interval", @@ -1057,6 +1074,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{3, 0, tsdbutil.GenerateTestGaugeHistogram(3), nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 3}}, }, { name: "one gauge float histogram", @@ -1082,6 +1100,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{6, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(6)}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}}, }, { name: "one gauge float histogram chunk intersect with deletion interval", @@ -1106,6 +1125,102 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{3, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3)}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 3}}, + }, + { + name: "three full mixed chunks", + chks: [][]chunks.Sample{ + {sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}}, + { + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, + sample{9, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, + }, + { + sample{10, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, + sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }, + }, + + expected: []chunks.Sample{ + sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}, sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, sample{9, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, sample{10, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }, + expectedChks: []chunks.Meta{ + assureChunkFromSamples(t, []chunks.Sample{ + sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}, + }), + assureChunkFromSamples(t, []chunks.Sample{ + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, + sample{9, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, + }), + assureChunkFromSamples(t, []chunks.Sample{ + sample{10, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, + sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }), + }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}, {7, 9}, {10, 203}}, + }, + { + name: "three full mixed chunks in different order", + chks: [][]chunks.Sample{ + { + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, + sample{9, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, + }, + {sample{11, 2, nil, nil}, sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, sample{16, 1, nil, nil}}, + { + sample{100, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, + sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }, + }, + + expected: []chunks.Sample{ + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, sample{9, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, sample{11, 2, nil, nil}, sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, sample{16, 1, nil, nil}, sample{100, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }, + expectedChks: []chunks.Meta{ + assureChunkFromSamples(t, []chunks.Sample{ + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, + sample{9, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, + }), + assureChunkFromSamples(t, []chunks.Sample{ + sample{11, 2, nil, nil}, sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, sample{16, 1, nil, nil}, + }), + assureChunkFromSamples(t, []chunks.Sample{ + sample{100, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, + sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }), + }, + expectedMinMaxTimes: []minMaxTimes{{7, 9}, {11, 16}, {100, 203}}, + }, + { + name: "three full mixed chunks in different order intersect with deletion interval", + chks: [][]chunks.Sample{ + { + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, + sample{9, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, + }, + {sample{11, 2, nil, nil}, sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, sample{16, 1, nil, nil}}, + { + sample{100, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, + sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }, + }, + intervals: tombstones.Intervals{{Mint: 8, Maxt: 11}, {Mint: 15, Maxt: 150}}, + + expected: []chunks.Sample{ + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }, + expectedChks: []chunks.Meta{ + assureChunkFromSamples(t, []chunks.Sample{ + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, + }), + assureChunkFromSamples(t, []chunks.Sample{ + sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, + }), + assureChunkFromSamples(t, []chunks.Sample{ + sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }), + }, + expectedMinMaxTimes: []minMaxTimes{{7, 7}, {12, 13}, {203, 203}}, }, } for _, tc := range cases { @@ -1147,6 +1262,11 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { rmChunkRefs(expandedResult) rmChunkRefs(tc.expectedChks) require.Equal(t, tc.expectedChks, expandedResult) + + for i, meta := range expandedResult { + require.Equal(t, tc.expectedMinMaxTimes[i].minTime, meta.MinTime) + require.Equal(t, tc.expectedMinMaxTimes[i].maxTime, meta.MaxTime) + } }) }) } From 2f7060bd5afa40d50e7251cacd08a69bb8c4684f Mon Sep 17 00:00:00 2001 From: Jeanette Tan Date: Wed, 1 Nov 2023 20:04:23 +0800 Subject: [PATCH 4/8] Expand TestPopulateWithTombSeriesIterators to test earlier deletion intervals for histogram chunks as well as time-overlapping chunks Signed-off-by: Jeanette Tan --- tsdb/querier_test.go | 132 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 128 insertions(+), 4 deletions(-) diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go index 5ad5003f1f..720d5c6997 100644 --- a/tsdb/querier_test.go +++ b/tsdb/querier_test.go @@ -950,7 +950,30 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { expectedMinMaxTimes: []minMaxTimes{{1, 6}}, }, { - name: "one histogram chunk intersect with deletion interval", + name: "one histogram chunk intersect with earlier deletion interval", + chks: [][]chunks.Sample{ + { + sample{1, 0, tsdbutil.GenerateTestHistogram(1), nil}, + sample{2, 0, tsdbutil.GenerateTestHistogram(2), nil}, + sample{3, 0, tsdbutil.GenerateTestHistogram(3), nil}, + sample{6, 0, tsdbutil.GenerateTestHistogram(6), nil}, + }, + }, + intervals: tombstones.Intervals{{Mint: 1, Maxt: 2}}, + expected: []chunks.Sample{ + sample{3, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(3)), nil}, + sample{6, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(6)), nil}, + }, + expectedChks: []chunks.Meta{ + assureChunkFromSamples(t, []chunks.Sample{ + sample{3, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(3)), nil}, + sample{6, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(6)), nil}, + }), + }, + expectedMinMaxTimes: []minMaxTimes{{3, 6}}, + }, + { + name: "one histogram chunk intersect with later deletion interval", chks: [][]chunks.Sample{ { sample{1, 0, tsdbutil.GenerateTestHistogram(1), nil}, @@ -1001,7 +1024,30 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { expectedMinMaxTimes: []minMaxTimes{{1, 6}}, }, { - name: "one float histogram chunk intersect with deletion interval", + name: "one float histogram chunk intersect with earlier deletion interval", + chks: [][]chunks.Sample{ + { + sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(1)}, + sample{2, 0, nil, tsdbutil.GenerateTestFloatHistogram(2)}, + sample{3, 0, nil, tsdbutil.GenerateTestFloatHistogram(3)}, + sample{6, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, + }, + }, + intervals: tombstones.Intervals{{Mint: 1, Maxt: 2}}, + expected: []chunks.Sample{ + sample{3, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(3))}, + sample{6, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(6))}, + }, + expectedChks: []chunks.Meta{ + assureChunkFromSamples(t, []chunks.Sample{ + sample{3, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(3))}, + sample{6, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(6))}, + }), + }, + expectedMinMaxTimes: []minMaxTimes{{3, 6}}, + }, + { + name: "one float histogram chunk intersect with later deletion interval", chks: [][]chunks.Sample{ { sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(1)}, @@ -1052,7 +1098,30 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { expectedMinMaxTimes: []minMaxTimes{{1, 6}}, }, { - name: "one gauge histogram chunk intersect with deletion interval", + name: "one gauge histogram chunk intersect with earlier deletion interval", + chks: [][]chunks.Sample{ + { + sample{1, 0, tsdbutil.GenerateTestGaugeHistogram(1), nil}, + sample{2, 0, tsdbutil.GenerateTestGaugeHistogram(2), nil}, + sample{3, 0, tsdbutil.GenerateTestGaugeHistogram(3), nil}, + sample{6, 0, tsdbutil.GenerateTestGaugeHistogram(6), nil}, + }, + }, + intervals: tombstones.Intervals{{Mint: 1, Maxt: 2}}, + expected: []chunks.Sample{ + sample{3, 0, tsdbutil.GenerateTestGaugeHistogram(3), nil}, + sample{6, 0, tsdbutil.GenerateTestGaugeHistogram(6), nil}, + }, + expectedChks: []chunks.Meta{ + assureChunkFromSamples(t, []chunks.Sample{ + sample{3, 0, tsdbutil.GenerateTestGaugeHistogram(3), nil}, + sample{6, 0, tsdbutil.GenerateTestGaugeHistogram(6), nil}, + }), + }, + expectedMinMaxTimes: []minMaxTimes{{3, 6}}, + }, + { + name: "one gauge histogram chunk intersect with later deletion interval", chks: [][]chunks.Sample{ { sample{1, 0, tsdbutil.GenerateTestGaugeHistogram(1), nil}, @@ -1103,7 +1172,30 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { expectedMinMaxTimes: []minMaxTimes{{1, 6}}, }, { - name: "one gauge float histogram chunk intersect with deletion interval", + name: "one gauge float histogram chunk intersect with earlier deletion interval", + chks: [][]chunks.Sample{ + { + sample{1, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(1)}, + sample{2, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(2)}, + sample{3, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3)}, + sample{6, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(6)}, + }, + }, + intervals: tombstones.Intervals{{Mint: 1, Maxt: 2}}, + expected: []chunks.Sample{ + sample{3, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3)}, + sample{6, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(6)}, + }, + expectedChks: []chunks.Meta{ + assureChunkFromSamples(t, []chunks.Sample{ + sample{3, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3)}, + sample{6, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(6)}, + }), + }, + expectedMinMaxTimes: []minMaxTimes{{3, 6}}, + }, + { + name: "one gauge float histogram chunk intersect with later deletion interval", chks: [][]chunks.Sample{ { sample{1, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(1)}, @@ -1222,6 +1314,38 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { }, expectedMinMaxTimes: []minMaxTimes{{7, 7}, {12, 13}, {203, 203}}, }, + { + name: "three full mixed chunks overlapping", + chks: [][]chunks.Sample{ + { + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, + sample{12, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, + }, + {sample{11, 2, nil, nil}, sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, sample{16, 1, nil, nil}}, + { + sample{10, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, + sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }, + }, + + expected: []chunks.Sample{ + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, sample{12, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, sample{11, 2, nil, nil}, sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, sample{16, 1, nil, nil}, sample{10, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }, + expectedChks: []chunks.Meta{ + assureChunkFromSamples(t, []chunks.Sample{ + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, + sample{12, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, + }), + assureChunkFromSamples(t, []chunks.Sample{ + sample{11, 2, nil, nil}, sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, sample{16, 1, nil, nil}, + }), + assureChunkFromSamples(t, []chunks.Sample{ + sample{10, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, + sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }), + }, + expectedMinMaxTimes: []minMaxTimes{{7, 12}, {11, 16}, {10, 203}}, + }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { From 27abf09e7fe80237d4925874de4d2e524460e71a Mon Sep 17 00:00:00 2001 From: Jeanette Tan Date: Wed, 1 Nov 2023 19:53:41 +0800 Subject: [PATCH 5/8] Fix missing MinTime in histogram chunks Signed-off-by: Jeanette Tan --- tsdb/querier.go | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/tsdb/querier.go b/tsdb/querier.go index 96406f1bdf..9a01e8c49b 100644 --- a/tsdb/querier.go +++ b/tsdb/querier.go @@ -855,13 +855,18 @@ func (p *populateWithDelChunkSeriesIterator) Next() bool { if app, err = newChunk.Appender(); err != nil { break } - - for vt := valueType; vt != chunkenc.ValNone; vt = p.currDelIter.Next() { + var h *histogram.Histogram + t, h = p.currDelIter.AtHistogram() + p.curr.MinTime = t + _, _, app, err = app.AppendHistogram(nil, t, h, true) + if err != nil { + break + } + for vt := p.currDelIter.Next(); vt != chunkenc.ValNone; vt = p.currDelIter.Next() { if vt != chunkenc.ValHistogram { err = fmt.Errorf("found value type %v in histogram chunk", vt) break } - var h *histogram.Histogram t, h = p.currDelIter.AtHistogram() _, _, app, err = app.AppendHistogram(nil, t, h, true) if err != nil { @@ -890,13 +895,18 @@ func (p *populateWithDelChunkSeriesIterator) Next() bool { if app, err = newChunk.Appender(); err != nil { break } - - for vt := valueType; vt != chunkenc.ValNone; vt = p.currDelIter.Next() { + var h *histogram.FloatHistogram + t, h = p.currDelIter.AtFloatHistogram() + p.curr.MinTime = t + _, _, app, err = app.AppendFloatHistogram(nil, t, h, true) + if err != nil { + break + } + for vt := p.currDelIter.Next(); vt != chunkenc.ValNone; vt = p.currDelIter.Next() { if vt != chunkenc.ValFloatHistogram { err = fmt.Errorf("found value type %v in histogram chunk", vt) break } - var h *histogram.FloatHistogram t, h = p.currDelIter.AtFloatHistogram() _, _, app, err = app.AppendFloatHistogram(nil, t, h, true) if err != nil { From 4296ecbd14dd2639eb5768ab66e4c61ea86c7b33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gy=C3=B6rgy=20Krajcsovits?= Date: Wed, 1 Nov 2023 15:52:04 +0100 Subject: [PATCH 6/8] tsdb/compact_test.go: test mixed typed series with PopulateBlock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add testcase and update test so that it can test native histograms as well. Signed-off-by: György Krajcsovits --- tsdb/compact_test.go | 43 ++++++++++++++++++++++++++++++++++++++++--- tsdb/querier_test.go | 33 ++++++++++++++++++++++++++++----- 2 files changed, 68 insertions(+), 8 deletions(-) diff --git a/tsdb/compact_test.go b/tsdb/compact_test.go index 098be8bfa0..2d98b30502 100644 --- a/tsdb/compact_test.go +++ b/tsdb/compact_test.go @@ -38,6 +38,7 @@ import ( "github.com/prometheus/prometheus/tsdb/chunks" "github.com/prometheus/prometheus/tsdb/fileutil" "github.com/prometheus/prometheus/tsdb/tombstones" + "github.com/prometheus/prometheus/tsdb/tsdbutil" "github.com/prometheus/prometheus/tsdb/wlog" ) @@ -933,6 +934,31 @@ func TestCompaction_populateBlock(t *testing.T) { }, }, }, + { + title: "Populate from mixed type series and expect sample inside the interval only", // regression test for populateWithDelChunkSeriesIterator failing to set minTime on chunks + compactMinTime: 1, + compactMaxTime: 11, + inputSeriesSamples: [][]seriesSamples{ + { + { + lset: map[string]string{"a": "1"}, + chunks: [][]sample{ + {{t: 0, h: tsdbutil.GenerateTestHistogram(0)}, {t: 1, h: tsdbutil.GenerateTestHistogram(1)}}, + {{t: 10, f: 1}, {t: 11, f: 2}}, + }, + }, + }, + }, + expSeriesSamples: []seriesSamples{ + { + lset: map[string]string{"a": "1"}, + chunks: [][]sample{ + {{t: 1, h: tsdbutil.GenerateTestHistogram(1)}}, + {{t: 10, f: 1}}, + }, + }, + }, + }, } { t.Run(tc.title, func(t *testing.T) { blocks := make([]BlockReader, 0, len(tc.inputSeriesSamples)) @@ -974,12 +1000,23 @@ func TestCompaction_populateBlock(t *testing.T) { firstTs int64 = math.MaxInt64 s sample ) - for iter.Next() == chunkenc.ValFloat { - s.t, s.f = iter.At() + for vt := iter.Next(); vt != chunkenc.ValNone; vt = iter.Next() { + switch vt { + case chunkenc.ValFloat: + s.t, s.f = iter.At() + samples = append(samples, s) + case chunkenc.ValHistogram: + s.t, s.h = iter.AtHistogram() + samples = append(samples, s) + case chunkenc.ValFloatHistogram: + s.t, s.fh = iter.AtFloatHistogram() + samples = append(samples, s) + default: + require.Fail(t, "unexpected value type") + } if firstTs == math.MaxInt64 { firstTs = s.t } - samples = append(samples, s) } // Check if chunk has correct min, max times. diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go index 720d5c6997..17a99c6e38 100644 --- a/tsdb/querier_test.go +++ b/tsdb/querier_test.go @@ -133,12 +133,35 @@ func createIdxChkReaders(t *testing.T, tc []seriesSamples) (IndexReader, ChunkRe Ref: chunkRef, }) - chunk := chunkenc.NewXORChunk() - app, _ := chunk.Appender() - for _, smpl := range chk { - app.Append(smpl.t, smpl.f) + switch { + case chk[0].fh != nil: + chunk := chunkenc.NewFloatHistogramChunk() + app, _ := chunk.Appender() + for _, smpl := range chk { + require.NotNil(t, smpl.fh, "chunk can only contain one type of sample") + _, _, _, err := app.AppendFloatHistogram(nil, smpl.t, smpl.fh, true) + require.NoError(t, err, "chunk should be appendable") + } + chkReader[chunkRef] = chunk + case chk[0].h != nil: + chunk := chunkenc.NewHistogramChunk() + app, _ := chunk.Appender() + for _, smpl := range chk { + require.NotNil(t, smpl.h, "chunk can only contain one type of sample") + _, _, _, err := app.AppendHistogram(nil, smpl.t, smpl.h, true) + require.NoError(t, err, "chunk should be appendable") + } + chkReader[chunkRef] = chunk + default: + chunk := chunkenc.NewXORChunk() + app, _ := chunk.Appender() + for _, smpl := range chk { + require.Nil(t, smpl.h, "chunk can only contain one type of sample") + require.Nil(t, smpl.fh, "chunk can only contain one type of sample") + app.Append(smpl.t, smpl.f) + } + chkReader[chunkRef] = chunk } - chkReader[chunkRef] = chunk chunkRef++ } ls := labels.FromMap(s.lset) From 3ccaaa40ba088b74e69a471e95fce9958bd291a5 Mon Sep 17 00:00:00 2001 From: Jeanette Tan Date: Thu, 2 Nov 2023 13:37:07 +0800 Subject: [PATCH 7/8] Fix according to code review Signed-off-by: Jeanette Tan --- tsdb/compact_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tsdb/compact_test.go b/tsdb/compact_test.go index 2d98b30502..cf8c2439ac 100644 --- a/tsdb/compact_test.go +++ b/tsdb/compact_test.go @@ -935,7 +935,8 @@ func TestCompaction_populateBlock(t *testing.T) { }, }, { - title: "Populate from mixed type series and expect sample inside the interval only", // regression test for populateWithDelChunkSeriesIterator failing to set minTime on chunks + // Regression test for populateWithDelChunkSeriesIterator failing to set minTime on chunks. + title: "Populate from mixed type series and expect sample inside the interval only.", compactMinTime: 1, compactMaxTime: 11, inputSeriesSamples: [][]seriesSamples{ From 52eb303031450f06e4d2da6c3f08681c0a5b8bb3 Mon Sep 17 00:00:00 2001 From: Jeanette Tan Date: Thu, 2 Nov 2023 21:23:05 +0800 Subject: [PATCH 8/8] Refactor assigning MinTime in histogram chunks Signed-off-by: Jeanette Tan --- tsdb/querier.go | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/tsdb/querier.go b/tsdb/querier.go index 9a01e8c49b..a832c0d1ba 100644 --- a/tsdb/querier.go +++ b/tsdb/querier.go @@ -840,6 +840,7 @@ func (p *populateWithDelChunkSeriesIterator) Next() bool { } return false } + p.curr.MinTime = p.currDelIter.AtT() // Re-encode the chunk if iterator is provider. This means that it has // some samples to be deleted or chunk is opened. @@ -855,18 +856,12 @@ func (p *populateWithDelChunkSeriesIterator) Next() bool { if app, err = newChunk.Appender(); err != nil { break } - var h *histogram.Histogram - t, h = p.currDelIter.AtHistogram() - p.curr.MinTime = t - _, _, app, err = app.AppendHistogram(nil, t, h, true) - if err != nil { - break - } - for vt := p.currDelIter.Next(); vt != chunkenc.ValNone; vt = p.currDelIter.Next() { + for vt := valueType; vt != chunkenc.ValNone; vt = p.currDelIter.Next() { if vt != chunkenc.ValHistogram { err = fmt.Errorf("found value type %v in histogram chunk", vt) break } + var h *histogram.Histogram t, h = p.currDelIter.AtHistogram() _, _, app, err = app.AppendHistogram(nil, t, h, true) if err != nil { @@ -878,15 +873,12 @@ func (p *populateWithDelChunkSeriesIterator) Next() bool { if app, err = newChunk.Appender(); err != nil { break } - var v float64 - t, v = p.currDelIter.At() - p.curr.MinTime = t - app.Append(t, v) - for vt := p.currDelIter.Next(); vt != chunkenc.ValNone; vt = p.currDelIter.Next() { + for vt := valueType; vt != chunkenc.ValNone; vt = p.currDelIter.Next() { if vt != chunkenc.ValFloat { err = fmt.Errorf("found value type %v in float chunk", vt) break } + var v float64 t, v = p.currDelIter.At() app.Append(t, v) } @@ -895,18 +887,12 @@ func (p *populateWithDelChunkSeriesIterator) Next() bool { if app, err = newChunk.Appender(); err != nil { break } - var h *histogram.FloatHistogram - t, h = p.currDelIter.AtFloatHistogram() - p.curr.MinTime = t - _, _, app, err = app.AppendFloatHistogram(nil, t, h, true) - if err != nil { - break - } - for vt := p.currDelIter.Next(); vt != chunkenc.ValNone; vt = p.currDelIter.Next() { + for vt := valueType; vt != chunkenc.ValNone; vt = p.currDelIter.Next() { if vt != chunkenc.ValFloatHistogram { err = fmt.Errorf("found value type %v in histogram chunk", vt) break } + var h *histogram.FloatHistogram t, h = p.currDelIter.AtFloatHistogram() _, _, app, err = app.AppendFloatHistogram(nil, t, h, true) if err != nil {