diff --git a/CHANGELOG.md b/CHANGELOG.md index bc14091cde..35f9f77682 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 2.48.0-rc.2 / 2023-11-02 + +* [ENHANCEMENT] Scraping: Add configuration option for tracking staleness of scraped timestamps. #13060 +* [BUGFIX] Storage: Fix crash caused by incorrect mixed samples handling. #13055 +* [BUGFIX] TSDB: Fix compactor failures by adding min time to histogram chunks. #13062 + ## 2.48.0-rc.1 / 2023-10-24 * [BUGFIX] PromQL: Reduce inefficiency introduced by warnings/annotations and temporarily remove possible non-counter warnings. #13012 diff --git a/VERSION b/VERSION index f2bfe03dba..10429c7ad0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.48.0-rc.1 +2.48.0-rc.2 diff --git a/tsdb/compact_test.go b/tsdb/compact_test.go index 098be8bfa0..cf8c2439ac 100644 --- a/tsdb/compact_test.go +++ b/tsdb/compact_test.go @@ -38,6 +38,7 @@ import ( "github.com/prometheus/prometheus/tsdb/chunks" "github.com/prometheus/prometheus/tsdb/fileutil" "github.com/prometheus/prometheus/tsdb/tombstones" + "github.com/prometheus/prometheus/tsdb/tsdbutil" "github.com/prometheus/prometheus/tsdb/wlog" ) @@ -933,6 +934,32 @@ func TestCompaction_populateBlock(t *testing.T) { }, }, }, + { + // Regression test for populateWithDelChunkSeriesIterator failing to set minTime on chunks. + title: "Populate from mixed type series and expect sample inside the interval only.", + compactMinTime: 1, + compactMaxTime: 11, + inputSeriesSamples: [][]seriesSamples{ + { + { + lset: map[string]string{"a": "1"}, + chunks: [][]sample{ + {{t: 0, h: tsdbutil.GenerateTestHistogram(0)}, {t: 1, h: tsdbutil.GenerateTestHistogram(1)}}, + {{t: 10, f: 1}, {t: 11, f: 2}}, + }, + }, + }, + }, + expSeriesSamples: []seriesSamples{ + { + lset: map[string]string{"a": "1"}, + chunks: [][]sample{ + {{t: 1, h: tsdbutil.GenerateTestHistogram(1)}}, + {{t: 10, f: 1}}, + }, + }, + }, + }, } { t.Run(tc.title, func(t *testing.T) { blocks := make([]BlockReader, 0, len(tc.inputSeriesSamples)) @@ -974,12 +1001,23 @@ func TestCompaction_populateBlock(t *testing.T) { firstTs int64 = math.MaxInt64 s sample ) - for iter.Next() == chunkenc.ValFloat { - s.t, s.f = iter.At() + for vt := iter.Next(); vt != chunkenc.ValNone; vt = iter.Next() { + switch vt { + case chunkenc.ValFloat: + s.t, s.f = iter.At() + samples = append(samples, s) + case chunkenc.ValHistogram: + s.t, s.h = iter.AtHistogram() + samples = append(samples, s) + case chunkenc.ValFloatHistogram: + s.t, s.fh = iter.AtFloatHistogram() + samples = append(samples, s) + default: + require.Fail(t, "unexpected value type") + } if firstTs == math.MaxInt64 { firstTs = s.t } - samples = append(samples, s) } // Check if chunk has correct min, max times. diff --git a/tsdb/querier.go b/tsdb/querier.go index 96406f1bdf..a832c0d1ba 100644 --- a/tsdb/querier.go +++ b/tsdb/querier.go @@ -840,6 +840,7 @@ func (p *populateWithDelChunkSeriesIterator) Next() bool { } return false } + p.curr.MinTime = p.currDelIter.AtT() // Re-encode the chunk if iterator is provider. This means that it has // some samples to be deleted or chunk is opened. @@ -855,7 +856,6 @@ func (p *populateWithDelChunkSeriesIterator) Next() bool { if app, err = newChunk.Appender(); err != nil { break } - for vt := valueType; vt != chunkenc.ValNone; vt = p.currDelIter.Next() { if vt != chunkenc.ValHistogram { err = fmt.Errorf("found value type %v in histogram chunk", vt) @@ -873,15 +873,12 @@ func (p *populateWithDelChunkSeriesIterator) Next() bool { if app, err = newChunk.Appender(); err != nil { break } - var v float64 - t, v = p.currDelIter.At() - p.curr.MinTime = t - app.Append(t, v) - for vt := p.currDelIter.Next(); vt != chunkenc.ValNone; vt = p.currDelIter.Next() { + for vt := valueType; vt != chunkenc.ValNone; vt = p.currDelIter.Next() { if vt != chunkenc.ValFloat { err = fmt.Errorf("found value type %v in float chunk", vt) break } + var v float64 t, v = p.currDelIter.At() app.Append(t, v) } @@ -890,7 +887,6 @@ func (p *populateWithDelChunkSeriesIterator) Next() bool { if app, err = newChunk.Appender(); err != nil { break } - for vt := valueType; vt != chunkenc.ValNone; vt = p.currDelIter.Next() { if vt != chunkenc.ValFloatHistogram { err = fmt.Errorf("found value type %v in histogram chunk", vt) diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go index 78eddf76ca..9360fb9a94 100644 --- a/tsdb/querier_test.go +++ b/tsdb/querier_test.go @@ -132,12 +132,35 @@ func createIdxChkReaders(t *testing.T, tc []seriesSamples) (IndexReader, ChunkRe Ref: chunkRef, }) - chunk := chunkenc.NewXORChunk() - app, _ := chunk.Appender() - for _, smpl := range chk { - app.Append(smpl.t, smpl.f) + switch { + case chk[0].fh != nil: + chunk := chunkenc.NewFloatHistogramChunk() + app, _ := chunk.Appender() + for _, smpl := range chk { + require.NotNil(t, smpl.fh, "chunk can only contain one type of sample") + _, _, _, err := app.AppendFloatHistogram(nil, smpl.t, smpl.fh, true) + require.NoError(t, err, "chunk should be appendable") + } + chkReader[chunkRef] = chunk + case chk[0].h != nil: + chunk := chunkenc.NewHistogramChunk() + app, _ := chunk.Appender() + for _, smpl := range chk { + require.NotNil(t, smpl.h, "chunk can only contain one type of sample") + _, _, _, err := app.AppendHistogram(nil, smpl.t, smpl.h, true) + require.NoError(t, err, "chunk should be appendable") + } + chkReader[chunkRef] = chunk + default: + chunk := chunkenc.NewXORChunk() + app, _ := chunk.Appender() + for _, smpl := range chk { + require.Nil(t, smpl.h, "chunk can only contain one type of sample") + require.Nil(t, smpl.fh, "chunk can only contain one type of sample") + app.Append(smpl.t, smpl.f) + } + chkReader[chunkRef] = chunk } - chkReader[chunkRef] = chunk chunkRef++ } ls := labels.FromMap(s.lset) @@ -693,12 +716,16 @@ func (r *fakeChunksReader) Chunk(meta chunks.Meta) (chunkenc.Chunk, error) { } func TestPopulateWithTombSeriesIterators(t *testing.T) { + type minMaxTimes struct { + minTime, maxTime int64 + } cases := []struct { name string chks [][]chunks.Sample - expected []chunks.Sample - expectedChks []chunks.Meta + expected []chunks.Sample + expectedChks []chunks.Meta + expectedMinMaxTimes []minMaxTimes intervals tombstones.Intervals @@ -717,6 +744,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { expectedChks: []chunks.Meta{ assureChunkFromSamples(t, []chunks.Sample{}), }, + expectedMinMaxTimes: []minMaxTimes{{0, 0}}, }, { name: "three empty chunks", // This should never happen. @@ -727,6 +755,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { assureChunkFromSamples(t, []chunks.Sample{}), assureChunkFromSamples(t, []chunks.Sample{}), }, + expectedMinMaxTimes: []minMaxTimes{{0, 0}, {0, 0}, {0, 0}}, }, { name: "one chunk", @@ -742,6 +771,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}}, }, { name: "two full chunks", @@ -761,6 +791,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{7, 89, nil, nil}, sample{9, 8, nil, nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}, {7, 9}}, }, { name: "three full chunks", @@ -784,6 +815,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{10, 22, nil, nil}, sample{203, 3493, nil, nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}, {7, 9}, {10, 203}}, }, // Seek cases. { @@ -854,6 +886,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{7, 89, nil, nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{3, 6}, {7, 7}}, }, { name: "two chunks with trimmed middle sample of first chunk", @@ -874,6 +907,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{7, 89, nil, nil}, sample{9, 8, nil, nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}, {7, 9}}, }, { name: "two chunks with deletion across two chunks", @@ -894,6 +928,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{9, 8, nil, nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 3}, {9, 9}}, }, // Deletion with seek. { @@ -934,9 +969,33 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{6, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(6)), nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}}, }, { - name: "one histogram chunk intersect with deletion interval", + name: "one histogram chunk intersect with earlier deletion interval", + chks: [][]chunks.Sample{ + { + sample{1, 0, tsdbutil.GenerateTestHistogram(1), nil}, + sample{2, 0, tsdbutil.GenerateTestHistogram(2), nil}, + sample{3, 0, tsdbutil.GenerateTestHistogram(3), nil}, + sample{6, 0, tsdbutil.GenerateTestHistogram(6), nil}, + }, + }, + intervals: tombstones.Intervals{{Mint: 1, Maxt: 2}}, + expected: []chunks.Sample{ + sample{3, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(3)), nil}, + sample{6, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(6)), nil}, + }, + expectedChks: []chunks.Meta{ + assureChunkFromSamples(t, []chunks.Sample{ + sample{3, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(3)), nil}, + sample{6, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(6)), nil}, + }), + }, + expectedMinMaxTimes: []minMaxTimes{{3, 6}}, + }, + { + name: "one histogram chunk intersect with later deletion interval", chks: [][]chunks.Sample{ { sample{1, 0, tsdbutil.GenerateTestHistogram(1), nil}, @@ -958,6 +1017,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{3, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(3)), nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 3}}, }, { name: "one float histogram chunk", @@ -983,9 +1043,33 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{6, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(6))}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}}, }, { - name: "one float histogram chunk intersect with deletion interval", + name: "one float histogram chunk intersect with earlier deletion interval", + chks: [][]chunks.Sample{ + { + sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(1)}, + sample{2, 0, nil, tsdbutil.GenerateTestFloatHistogram(2)}, + sample{3, 0, nil, tsdbutil.GenerateTestFloatHistogram(3)}, + sample{6, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, + }, + }, + intervals: tombstones.Intervals{{Mint: 1, Maxt: 2}}, + expected: []chunks.Sample{ + sample{3, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(3))}, + sample{6, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(6))}, + }, + expectedChks: []chunks.Meta{ + assureChunkFromSamples(t, []chunks.Sample{ + sample{3, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(3))}, + sample{6, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(6))}, + }), + }, + expectedMinMaxTimes: []minMaxTimes{{3, 6}}, + }, + { + name: "one float histogram chunk intersect with later deletion interval", chks: [][]chunks.Sample{ { sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(1)}, @@ -1007,6 +1091,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{3, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(3))}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 3}}, }, { name: "one gauge histogram chunk", @@ -1032,9 +1117,33 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{6, 0, tsdbutil.GenerateTestGaugeHistogram(6), nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}}, }, { - name: "one gauge histogram chunk intersect with deletion interval", + name: "one gauge histogram chunk intersect with earlier deletion interval", + chks: [][]chunks.Sample{ + { + sample{1, 0, tsdbutil.GenerateTestGaugeHistogram(1), nil}, + sample{2, 0, tsdbutil.GenerateTestGaugeHistogram(2), nil}, + sample{3, 0, tsdbutil.GenerateTestGaugeHistogram(3), nil}, + sample{6, 0, tsdbutil.GenerateTestGaugeHistogram(6), nil}, + }, + }, + intervals: tombstones.Intervals{{Mint: 1, Maxt: 2}}, + expected: []chunks.Sample{ + sample{3, 0, tsdbutil.GenerateTestGaugeHistogram(3), nil}, + sample{6, 0, tsdbutil.GenerateTestGaugeHistogram(6), nil}, + }, + expectedChks: []chunks.Meta{ + assureChunkFromSamples(t, []chunks.Sample{ + sample{3, 0, tsdbutil.GenerateTestGaugeHistogram(3), nil}, + sample{6, 0, tsdbutil.GenerateTestGaugeHistogram(6), nil}, + }), + }, + expectedMinMaxTimes: []minMaxTimes{{3, 6}}, + }, + { + name: "one gauge histogram chunk intersect with later deletion interval", chks: [][]chunks.Sample{ { sample{1, 0, tsdbutil.GenerateTestGaugeHistogram(1), nil}, @@ -1056,6 +1165,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{3, 0, tsdbutil.GenerateTestGaugeHistogram(3), nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 3}}, }, { name: "one gauge float histogram", @@ -1081,9 +1191,33 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{6, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(6)}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}}, }, { - name: "one gauge float histogram chunk intersect with deletion interval", + name: "one gauge float histogram chunk intersect with earlier deletion interval", + chks: [][]chunks.Sample{ + { + sample{1, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(1)}, + sample{2, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(2)}, + sample{3, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3)}, + sample{6, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(6)}, + }, + }, + intervals: tombstones.Intervals{{Mint: 1, Maxt: 2}}, + expected: []chunks.Sample{ + sample{3, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3)}, + sample{6, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(6)}, + }, + expectedChks: []chunks.Meta{ + assureChunkFromSamples(t, []chunks.Sample{ + sample{3, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3)}, + sample{6, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(6)}, + }), + }, + expectedMinMaxTimes: []minMaxTimes{{3, 6}}, + }, + { + name: "one gauge float histogram chunk intersect with later deletion interval", chks: [][]chunks.Sample{ { sample{1, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(1)}, @@ -1105,6 +1239,134 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{3, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3)}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 3}}, + }, + { + name: "three full mixed chunks", + chks: [][]chunks.Sample{ + {sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}}, + { + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, + sample{9, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, + }, + { + sample{10, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, + sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }, + }, + + expected: []chunks.Sample{ + sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}, sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, sample{9, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, sample{10, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }, + expectedChks: []chunks.Meta{ + assureChunkFromSamples(t, []chunks.Sample{ + sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}, + }), + assureChunkFromSamples(t, []chunks.Sample{ + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, + sample{9, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, + }), + assureChunkFromSamples(t, []chunks.Sample{ + sample{10, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, + sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }), + }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}, {7, 9}, {10, 203}}, + }, + { + name: "three full mixed chunks in different order", + chks: [][]chunks.Sample{ + { + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, + sample{9, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, + }, + {sample{11, 2, nil, nil}, sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, sample{16, 1, nil, nil}}, + { + sample{100, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, + sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }, + }, + + expected: []chunks.Sample{ + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, sample{9, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, sample{11, 2, nil, nil}, sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, sample{16, 1, nil, nil}, sample{100, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }, + expectedChks: []chunks.Meta{ + assureChunkFromSamples(t, []chunks.Sample{ + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, + sample{9, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, + }), + assureChunkFromSamples(t, []chunks.Sample{ + sample{11, 2, nil, nil}, sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, sample{16, 1, nil, nil}, + }), + assureChunkFromSamples(t, []chunks.Sample{ + sample{100, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, + sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }), + }, + expectedMinMaxTimes: []minMaxTimes{{7, 9}, {11, 16}, {100, 203}}, + }, + { + name: "three full mixed chunks in different order intersect with deletion interval", + chks: [][]chunks.Sample{ + { + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, + sample{9, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, + }, + {sample{11, 2, nil, nil}, sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, sample{16, 1, nil, nil}}, + { + sample{100, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, + sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }, + }, + intervals: tombstones.Intervals{{Mint: 8, Maxt: 11}, {Mint: 15, Maxt: 150}}, + + expected: []chunks.Sample{ + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }, + expectedChks: []chunks.Meta{ + assureChunkFromSamples(t, []chunks.Sample{ + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, + }), + assureChunkFromSamples(t, []chunks.Sample{ + sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, + }), + assureChunkFromSamples(t, []chunks.Sample{ + sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }), + }, + expectedMinMaxTimes: []minMaxTimes{{7, 7}, {12, 13}, {203, 203}}, + }, + { + name: "three full mixed chunks overlapping", + chks: [][]chunks.Sample{ + { + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, + sample{12, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, + }, + {sample{11, 2, nil, nil}, sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, sample{16, 1, nil, nil}}, + { + sample{10, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, + sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }, + }, + + expected: []chunks.Sample{ + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, sample{12, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, sample{11, 2, nil, nil}, sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, sample{16, 1, nil, nil}, sample{10, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }, + expectedChks: []chunks.Meta{ + assureChunkFromSamples(t, []chunks.Sample{ + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, + sample{12, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, + }), + assureChunkFromSamples(t, []chunks.Sample{ + sample{11, 2, nil, nil}, sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, sample{16, 1, nil, nil}, + }), + assureChunkFromSamples(t, []chunks.Sample{ + sample{10, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, + sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }), + }, + expectedMinMaxTimes: []minMaxTimes{{7, 12}, {11, 16}, {10, 203}}, }, } for _, tc := range cases { @@ -1146,6 +1408,11 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { rmChunkRefs(expandedResult) rmChunkRefs(tc.expectedChks) require.Equal(t, tc.expectedChks, expandedResult) + + for i, meta := range expandedResult { + require.Equal(t, tc.expectedMinMaxTimes[i].minTime, meta.MinTime) + require.Equal(t, tc.expectedMinMaxTimes[i].maxTime, meta.MaxTime) + } }) }) } @@ -1157,67 +1424,213 @@ func rmChunkRefs(chks []chunks.Meta) { } } +func checkCurrVal(t *testing.T, valType chunkenc.ValueType, it *populateWithDelSeriesIterator, expectedTs, expectedValue int) { + switch valType { + case chunkenc.ValFloat: + ts, v := it.At() + require.Equal(t, int64(expectedTs), ts) + require.Equal(t, float64(expectedValue), v) + case chunkenc.ValHistogram: + ts, h := it.AtHistogram() + require.Equal(t, int64(expectedTs), ts) + h.CounterResetHint = histogram.UnknownCounterReset + require.Equal(t, tsdbutil.GenerateTestHistogram(expectedValue), h) + case chunkenc.ValFloatHistogram: + ts, h := it.AtFloatHistogram() + require.Equal(t, int64(expectedTs), ts) + h.CounterResetHint = histogram.UnknownCounterReset + require.Equal(t, tsdbutil.GenerateTestFloatHistogram(expectedValue), h) + default: + panic("unexpected value type") + } +} + // Regression for: https://github.com/prometheus/tsdb/pull/97 func TestPopulateWithDelSeriesIterator_DoubleSeek(t *testing.T) { - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{}, - []chunks.Sample{sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}}, - []chunks.Sample{sample{4, 4, nil, nil}, sample{5, 5, nil, nil}}, - ) + cases := []struct { + name string + valType chunkenc.ValueType + chks [][]chunks.Sample + }{ + { + name: "float", + valType: chunkenc.ValFloat, + chks: [][]chunks.Sample{ + {}, + {sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}}, + {sample{4, 4, nil, nil}, sample{5, 5, nil, nil}}, + }, + }, + { + name: "histogram", + valType: chunkenc.ValHistogram, + chks: [][]chunks.Sample{ + {}, + {sample{1, 0, tsdbutil.GenerateTestHistogram(1), nil}, sample{2, 0, tsdbutil.GenerateTestHistogram(2), nil}, sample{3, 0, tsdbutil.GenerateTestHistogram(3), nil}}, + {sample{4, 0, tsdbutil.GenerateTestHistogram(4), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(5), nil}}, + }, + }, + { + name: "float histogram", + valType: chunkenc.ValFloatHistogram, + chks: [][]chunks.Sample{ + {}, + {sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(1)}, sample{2, 0, nil, tsdbutil.GenerateTestFloatHistogram(2)}, sample{3, 0, nil, tsdbutil.GenerateTestFloatHistogram(3)}}, + {sample{4, 0, nil, tsdbutil.GenerateTestFloatHistogram(4)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(5)}}, + }, + }, + } - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, chunkenc.ValFloat, it.Seek(1)) - require.Equal(t, chunkenc.ValFloat, it.Seek(2)) - require.Equal(t, chunkenc.ValFloat, it.Seek(2)) - ts, v := it.At() - require.Equal(t, int64(2), ts) - require.Equal(t, float64(2), v) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f, chkMetas := createFakeReaderAndNotPopulatedChunks(tc.chks...) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, tc.valType, it.Seek(1)) + require.Equal(t, tc.valType, it.Seek(2)) + require.Equal(t, tc.valType, it.Seek(2)) + checkCurrVal(t, tc.valType, it, 2, 2) + require.Equal(t, int64(0), chkMetas[0].MinTime) + require.Equal(t, int64(1), chkMetas[1].MinTime) + require.Equal(t, int64(4), chkMetas[2].MinTime) + }) + } } // Regression when seeked chunks were still found via binary search and we always // skipped to the end when seeking a value in the current chunk. func TestPopulateWithDelSeriesIterator_SeekInCurrentChunk(t *testing.T) { - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{}, - []chunks.Sample{sample{1, 2, nil, nil}, sample{3, 4, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, - []chunks.Sample{}, - ) + cases := []struct { + name string + valType chunkenc.ValueType + chks [][]chunks.Sample + }{ + { + name: "float", + valType: chunkenc.ValFloat, + chks: [][]chunks.Sample{ + {}, + {sample{1, 2, nil, nil}, sample{3, 4, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, + {}, + }, + }, + { + name: "histogram", + valType: chunkenc.ValHistogram, + chks: [][]chunks.Sample{ + {}, + {sample{1, 0, tsdbutil.GenerateTestHistogram(2), nil}, sample{3, 0, tsdbutil.GenerateTestHistogram(4), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{7, 0, tsdbutil.GenerateTestHistogram(8), nil}}, + {}, + }, + }, + { + name: "float histogram", + valType: chunkenc.ValFloatHistogram, + chks: [][]chunks.Sample{ + {}, + {sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(2)}, sample{3, 0, nil, tsdbutil.GenerateTestFloatHistogram(4)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{7, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, + {}, + }, + }, + } - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, chunkenc.ValFloat, it.Next()) - ts, v := it.At() - require.Equal(t, int64(1), ts) - require.Equal(t, float64(2), v) - - require.Equal(t, chunkenc.ValFloat, it.Seek(4)) - ts, v = it.At() - require.Equal(t, int64(5), ts) - require.Equal(t, float64(6), v) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f, chkMetas := createFakeReaderAndNotPopulatedChunks(tc.chks...) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, tc.valType, it.Next()) + checkCurrVal(t, tc.valType, it, 1, 2) + require.Equal(t, tc.valType, it.Seek(4)) + checkCurrVal(t, tc.valType, it, 5, 6) + require.Equal(t, int64(0), chkMetas[0].MinTime) + require.Equal(t, int64(1), chkMetas[1].MinTime) + require.Equal(t, int64(0), chkMetas[2].MinTime) + }) + } } func TestPopulateWithDelSeriesIterator_SeekWithMinTime(t *testing.T) { - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{6, 8, nil, nil}}, - ) + cases := []struct { + name string + valType chunkenc.ValueType + chks [][]chunks.Sample + }{ + { + name: "float", + valType: chunkenc.ValFloat, + chks: [][]chunks.Sample{ + {sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{6, 8, nil, nil}}, + }, + }, + { + name: "histogram", + valType: chunkenc.ValHistogram, + chks: [][]chunks.Sample{ + {sample{1, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{6, 0, tsdbutil.GenerateTestHistogram(8), nil}}, + }, + }, + { + name: "float histogram", + valType: chunkenc.ValFloatHistogram, + chks: [][]chunks.Sample{ + {sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{6, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, + }, + }, + } - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, chunkenc.ValNone, it.Seek(7)) - require.Equal(t, chunkenc.ValFloat, it.Seek(3)) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f, chkMetas := createFakeReaderAndNotPopulatedChunks(tc.chks...) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, chunkenc.ValNone, it.Seek(7)) + require.Equal(t, tc.valType, it.Seek(3)) + require.Equal(t, int64(1), chkMetas[0].MinTime) + }) + } } // Regression when calling Next() with a time bounded to fit within two samples. // Seek gets called and advances beyond the max time, which was just accepted as a valid sample. func TestPopulateWithDelSeriesIterator_NextWithMinTime(t *testing.T) { - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, - ) + cases := []struct { + name string + valType chunkenc.ValueType + chks [][]chunks.Sample + }{ + { + name: "float", + valType: chunkenc.ValFloat, + chks: [][]chunks.Sample{ + {sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, + }, + }, + { + name: "histogram", + valType: chunkenc.ValHistogram, + chks: [][]chunks.Sample{ + {sample{1, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{7, 0, tsdbutil.GenerateTestHistogram(8), nil}}, + }, + }, + { + name: "float histogram", + valType: chunkenc.ValFloatHistogram, + chks: [][]chunks.Sample{ + {sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{7, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, + }, + }, + } - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 4, Maxt: math.MaxInt64})) - require.Equal(t, chunkenc.ValNone, it.Next()) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f, chkMetas := createFakeReaderAndNotPopulatedChunks(tc.chks...) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 4, Maxt: math.MaxInt64})) + require.Equal(t, chunkenc.ValNone, it.Next()) + require.Equal(t, int64(1), chkMetas[0].MinTime) + }) + } } // Test the cost of merging series sets for different number of merged sets and their size. diff --git a/web/ui/module/codemirror-promql/package.json b/web/ui/module/codemirror-promql/package.json index 0e62d640ab..408891028c 100644 --- a/web/ui/module/codemirror-promql/package.json +++ b/web/ui/module/codemirror-promql/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/codemirror-promql", - "version": "0.48.0-rc.1", + "version": "0.48.0-rc.2", "description": "a CodeMirror mode for the PromQL language", "types": "dist/esm/index.d.ts", "module": "dist/esm/index.js", @@ -29,7 +29,7 @@ }, "homepage": "https://github.com/prometheus/prometheus/blob/main/web/ui/module/codemirror-promql/README.md", "dependencies": { - "@prometheus-io/lezer-promql": "0.48.0-rc.1", + "@prometheus-io/lezer-promql": "0.48.0-rc.2", "lru-cache": "^7.18.3" }, "devDependencies": { diff --git a/web/ui/module/lezer-promql/package.json b/web/ui/module/lezer-promql/package.json index d38b3288de..9f4a19d007 100644 --- a/web/ui/module/lezer-promql/package.json +++ b/web/ui/module/lezer-promql/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/lezer-promql", - "version": "0.48.0-rc.1", + "version": "0.48.0-rc.2", "description": "lezer-based PromQL grammar", "main": "dist/index.cjs", "type": "module", diff --git a/web/ui/package-lock.json b/web/ui/package-lock.json index f951aba10d..43cda8b139 100644 --- a/web/ui/package-lock.json +++ b/web/ui/package-lock.json @@ -1,12 +1,12 @@ { "name": "prometheus-io", - "version": "0.48.0-rc.1", + "version": "0.48.0-rc.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "prometheus-io", - "version": "0.48.0-rc.1", + "version": "0.48.0-rc.2", "workspaces": [ "react-app", "module/*" @@ -30,10 +30,10 @@ }, "module/codemirror-promql": { "name": "@prometheus-io/codemirror-promql", - "version": "0.48.0-rc.1", + "version": "0.48.0-rc.2", "license": "Apache-2.0", "dependencies": { - "@prometheus-io/lezer-promql": "0.48.0-rc.1", + "@prometheus-io/lezer-promql": "0.48.0-rc.2", "lru-cache": "^7.18.3" }, "devDependencies": { @@ -70,7 +70,7 @@ }, "module/lezer-promql": { "name": "@prometheus-io/lezer-promql", - "version": "0.48.0-rc.1", + "version": "0.48.0-rc.2", "license": "Apache-2.0", "devDependencies": { "@lezer/generator": "^1.2.3", @@ -20764,7 +20764,7 @@ }, "react-app": { "name": "@prometheus-io/app", - "version": "0.48.0-rc.1", + "version": "0.48.0-rc.2", "dependencies": { "@codemirror/autocomplete": "^6.7.1", "@codemirror/commands": "^6.2.4", @@ -20782,7 +20782,7 @@ "@lezer/lr": "^1.3.6", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "0.48.0-rc.1", + "@prometheus-io/codemirror-promql": "0.48.0-rc.2", "bootstrap": "^4.6.2", "css.escape": "^1.5.1", "downshift": "^7.6.0", @@ -23422,7 +23422,7 @@ "@lezer/lr": "^1.3.6", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "0.48.0-rc.1", + "@prometheus-io/codemirror-promql": "0.48.0-rc.2", "@testing-library/react-hooks": "^7.0.2", "@types/enzyme": "^3.10.13", "@types/flot": "0.0.32", @@ -23486,7 +23486,7 @@ "@lezer/common": "^1.0.3", "@lezer/highlight": "^1.1.6", "@lezer/lr": "^1.3.6", - "@prometheus-io/lezer-promql": "0.48.0-rc.1", + "@prometheus-io/lezer-promql": "0.48.0-rc.2", "isomorphic-fetch": "^3.0.0", "lru-cache": "^7.18.3", "nock": "^13.3.1" diff --git a/web/ui/package.json b/web/ui/package.json index 76b5ad2a89..9b4dfbc628 100644 --- a/web/ui/package.json +++ b/web/ui/package.json @@ -28,5 +28,5 @@ "ts-jest": "^29.1.0", "typescript": "^4.9.5" }, - "version": "0.48.0-rc.1" + "version": "0.48.0-rc.2" } diff --git a/web/ui/react-app/package.json b/web/ui/react-app/package.json index 3d1de883a5..de0d0e3150 100644 --- a/web/ui/react-app/package.json +++ b/web/ui/react-app/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/app", - "version": "0.48.0-rc.1", + "version": "0.48.0-rc.2", "private": true, "dependencies": { "@codemirror/autocomplete": "^6.7.1", @@ -19,7 +19,7 @@ "@lezer/lr": "^1.3.6", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "0.48.0-rc.1", + "@prometheus-io/codemirror-promql": "0.48.0-rc.2", "bootstrap": "^4.6.2", "css.escape": "^1.5.1", "downshift": "^7.6.0",