From af1d9e01c7e46949c69b863211b2649ea7f448a3 Mon Sep 17 00:00:00 2001 From: Justin Lei <97976793+leizor@users.noreply.github.com> Date: Fri, 10 Feb 2023 03:39:33 -0800 Subject: [PATCH] Refactor tsdbutil for tests/native histograms (#11948) * Add float histograms to ChunkFromSamplesGeneric Signed-off-by: Justin Lei * Add Generate*Samples functions to tsdbutil Signed-off-by: Justin Lei * PR responses Signed-off-by: Justin Lei --------- Signed-off-by: Justin Lei --- promql/engine_test.go | 8 +-- rules/manager_test.go | 4 +- storage/merge_test.go | 58 ++++++++++++++----- tsdb/agent/db_test.go | 30 +++++----- tsdb/head.go | 91 ------------------------------ tsdb/head_test.go | 46 ++++++++-------- tsdb/tsdbutil/chunks.go | 16 ++++-- tsdb/tsdbutil/histogram.go | 110 +++++++++++++++++++++++++++++++++++++ 8 files changed, 211 insertions(+), 152 deletions(-) create mode 100644 tsdb/tsdbutil/histogram.go diff --git a/promql/engine_test.go b/promql/engine_test.go index a7aaedd69..e2e209849 100644 --- a/promql/engine_test.go +++ b/promql/engine_test.go @@ -25,7 +25,7 @@ import ( "github.com/go-kit/log" - "github.com/prometheus/prometheus/util/stats" + "github.com/prometheus/prometheus/tsdb/tsdbutil" "github.com/stretchr/testify/require" "go.uber.org/goleak" @@ -35,7 +35,7 @@ import ( "github.com/prometheus/prometheus/model/timestamp" "github.com/prometheus/prometheus/promql/parser" "github.com/prometheus/prometheus/storage" - "github.com/prometheus/prometheus/tsdb" + "github.com/prometheus/prometheus/util/stats" ) func TestMain(m *testing.M) { @@ -3136,7 +3136,7 @@ func TestNativeHistogramRate(t *testing.T) { lbls := labels.FromStrings("__name__", seriesName) app := test.Storage().Appender(context.TODO()) - for i, h := range tsdb.GenerateTestHistograms(100) { + for i, h := range tsdbutil.GenerateTestHistograms(100) { _, err := app.AppendHistogram(0, lbls, int64(i)*int64(15*time.Second/time.Millisecond), h, nil) require.NoError(t, err) } @@ -3181,7 +3181,7 @@ func TestNativeFloatHistogramRate(t *testing.T) { lbls := labels.FromStrings("__name__", seriesName) app := test.Storage().Appender(context.TODO()) - for i, fh := range tsdb.GenerateTestFloatHistograms(100) { + for i, fh := range tsdbutil.GenerateTestFloatHistograms(100) { _, err := app.AppendHistogram(0, lbls, int64(i)*int64(15*time.Second/time.Millisecond), nil, fh) require.NoError(t, err) } diff --git a/rules/manager_test.go b/rules/manager_test.go index 6f0dd0dda..d287c25ce 100644 --- a/rules/manager_test.go +++ b/rules/manager_test.go @@ -38,8 +38,8 @@ import ( "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/promql/parser" "github.com/prometheus/prometheus/storage" - "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/prometheus/prometheus/tsdb/tsdbutil" "github.com/prometheus/prometheus/util/teststorage" ) @@ -1348,7 +1348,7 @@ func TestNativeHistogramsInRecordingRules(t *testing.T) { // Add some histograms. db := suite.TSDB() - hists := tsdb.GenerateTestHistograms(5) + hists := tsdbutil.GenerateTestHistograms(5) ts := time.Now() app := db.Appender(context.Background()) for i, h := range hists { diff --git a/storage/merge_test.go b/storage/merge_test.go index ad68684c0..6abf8723d 100644 --- a/storage/merge_test.go +++ b/storage/merge_test.go @@ -23,7 +23,6 @@ import ( "github.com/stretchr/testify/require" - "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/tsdbutil" @@ -387,18 +386,11 @@ func TestCompactingChunkSeriesMerger(t *testing.T) { // histogramSample returns a histogram that is unique to the ts. histogramSample := func(ts int64) sample { - idx := ts + 1 - return sample{t: ts, h: &histogram.Histogram{ - Schema: 2, - ZeroThreshold: 0.001, - ZeroCount: 2 * uint64(idx), - Count: 5 * uint64(idx), - Sum: 12.34 * float64(idx), - PositiveSpans: []histogram.Span{{Offset: 1, Length: 2}, {Offset: 2, Length: 1}}, - NegativeSpans: []histogram.Span{{Offset: 2, Length: 1}, {Offset: 1, Length: 2}}, - PositiveBuckets: []int64{1 * idx, -1 * idx, 3 * idx}, - NegativeBuckets: []int64{1 * idx, 2 * idx, 3 * idx}, - }} + return sample{t: ts, h: tsdbutil.GenerateTestHistogram(int(ts + 1))} + } + + floatHistogramSample := func(ts int64) sample { + return sample{t: ts, fh: tsdbutil.GenerateTestFloatHistogram(int(ts + 1))} } for _, tc := range []struct { @@ -529,6 +521,46 @@ func TestCompactingChunkSeriesMerger(t *testing.T) { []tsdbutil.Sample{histogramSample(15)}, ), }, + { + name: "float histogram chunks overlapping", + input: []ChunkSeries{ + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{floatHistogramSample(0), floatHistogramSample(5)}, []tsdbutil.Sample{floatHistogramSample(10), floatHistogramSample(15)}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{floatHistogramSample(2), floatHistogramSample(20)}, []tsdbutil.Sample{floatHistogramSample(25), floatHistogramSample(30)}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{floatHistogramSample(18), floatHistogramSample(26)}, []tsdbutil.Sample{floatHistogramSample(31), floatHistogramSample(35)}), + }, + expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), + []tsdbutil.Sample{floatHistogramSample(0), floatHistogramSample(2), floatHistogramSample(5), floatHistogramSample(10), floatHistogramSample(15), floatHistogramSample(18), floatHistogramSample(20), floatHistogramSample(25), floatHistogramSample(26), floatHistogramSample(30)}, + []tsdbutil.Sample{floatHistogramSample(31), floatHistogramSample(35)}, + ), + }, + { + name: "float histogram chunks overlapping with float chunks", + input: []ChunkSeries{ + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{floatHistogramSample(0), floatHistogramSample(5)}, []tsdbutil.Sample{floatHistogramSample(10), floatHistogramSample(15)}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{sample{1, 1, nil, nil}, sample{12, 12, nil, nil}}, []tsdbutil.Sample{sample{14, 14, nil, nil}}), + }, + expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), + []tsdbutil.Sample{floatHistogramSample(0)}, + []tsdbutil.Sample{sample{1, 1, nil, nil}}, + []tsdbutil.Sample{floatHistogramSample(5), floatHistogramSample(10)}, + []tsdbutil.Sample{sample{12, 12, nil, nil}, sample{14, 14, nil, nil}}, + []tsdbutil.Sample{floatHistogramSample(15)}, + ), + }, + { + name: "float histogram chunks overlapping with histogram chunks", + input: []ChunkSeries{ + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{floatHistogramSample(0), floatHistogramSample(5)}, []tsdbutil.Sample{floatHistogramSample(10), floatHistogramSample(15)}), + NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{histogramSample(1), histogramSample(12)}, []tsdbutil.Sample{histogramSample(14)}), + }, + expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), + []tsdbutil.Sample{floatHistogramSample(0)}, + []tsdbutil.Sample{histogramSample(1)}, + []tsdbutil.Sample{floatHistogramSample(5), floatHistogramSample(10)}, + []tsdbutil.Sample{histogramSample(12), histogramSample(14)}, + []tsdbutil.Sample{floatHistogramSample(15)}, + ), + }, } { t.Run(tc.name, func(t *testing.T) { merged := m(tc.input...) diff --git a/tsdb/agent/db_test.go b/tsdb/agent/db_test.go index 0bd3b8249..9b27aaa0b 100644 --- a/tsdb/agent/db_test.go +++ b/tsdb/agent/db_test.go @@ -54,10 +54,10 @@ func TestDB_InvalidSeries(t *testing.T) { }) t.Run("Histograms", func(t *testing.T) { - _, err := app.AppendHistogram(0, labels.Labels{}, 0, tsdb.GenerateTestHistograms(1)[0], nil) + _, err := app.AppendHistogram(0, labels.Labels{}, 0, tsdbutil.GenerateTestHistograms(1)[0], nil) require.ErrorIs(t, err, tsdb.ErrInvalidSample, "should reject empty labels") - _, err = app.AppendHistogram(0, labels.FromStrings("a", "1", "a", "2"), 0, tsdb.GenerateTestHistograms(1)[0], nil) + _, err = app.AppendHistogram(0, labels.FromStrings("a", "1", "a", "2"), 0, tsdbutil.GenerateTestHistograms(1)[0], nil) require.ErrorIs(t, err, tsdb.ErrInvalidSample, "should reject duplicate labels") }) @@ -151,7 +151,7 @@ func TestCommit(t *testing.T) { for _, l := range lbls { lset := labels.New(l...) - histograms := tsdb.GenerateTestHistograms(numHistograms) + histograms := tsdbutil.GenerateTestHistograms(numHistograms) for i := 0; i < numHistograms; i++ { _, err := app.AppendHistogram(0, lset, int64(i), histograms[i], nil) @@ -163,7 +163,7 @@ func TestCommit(t *testing.T) { for _, l := range lbls { lset := labels.New(l...) - floatHistograms := tsdb.GenerateTestFloatHistograms(numHistograms) + floatHistograms := tsdbutil.GenerateTestFloatHistograms(numHistograms) for i := 0; i < numHistograms; i++ { _, err := app.AppendHistogram(0, lset, int64(i), nil, floatHistograms[i]) @@ -257,7 +257,7 @@ func TestRollback(t *testing.T) { for _, l := range lbls { lset := labels.New(l...) - histograms := tsdb.GenerateTestHistograms(numHistograms) + histograms := tsdbutil.GenerateTestHistograms(numHistograms) for i := 0; i < numHistograms; i++ { _, err := app.AppendHistogram(0, lset, int64(i), histograms[i], nil) @@ -269,7 +269,7 @@ func TestRollback(t *testing.T) { for _, l := range lbls { lset := labels.New(l...) - floatHistograms := tsdb.GenerateTestFloatHistograms(numHistograms) + floatHistograms := tsdbutil.GenerateTestFloatHistograms(numHistograms) for i := 0; i < numHistograms; i++ { _, err := app.AppendHistogram(0, lset, int64(i), nil, floatHistograms[i]) @@ -374,7 +374,7 @@ func TestFullTruncateWAL(t *testing.T) { for _, l := range lbls { lset := labels.New(l...) - histograms := tsdb.GenerateTestHistograms(numHistograms) + histograms := tsdbutil.GenerateTestHistograms(numHistograms) for i := 0; i < numHistograms; i++ { _, err := app.AppendHistogram(0, lset, int64(lastTs), histograms[i], nil) @@ -387,7 +387,7 @@ func TestFullTruncateWAL(t *testing.T) { for _, l := range lbls { lset := labels.New(l...) - floatHistograms := tsdb.GenerateTestFloatHistograms(numHistograms) + floatHistograms := tsdbutil.GenerateTestFloatHistograms(numHistograms) for i := 0; i < numHistograms; i++ { _, err := app.AppendHistogram(0, lset, int64(lastTs), nil, floatHistograms[i]) @@ -436,7 +436,7 @@ func TestPartialTruncateWAL(t *testing.T) { for _, l := range lbls { lset := labels.New(l...) - histograms := tsdb.GenerateTestHistograms(numDatapoints) + histograms := tsdbutil.GenerateTestHistograms(numDatapoints) for i := 0; i < numDatapoints; i++ { _, err := app.AppendHistogram(0, lset, lastTs, histograms[i], nil) @@ -449,7 +449,7 @@ func TestPartialTruncateWAL(t *testing.T) { for _, l := range lbls { lset := labels.New(l...) - floatHistograms := tsdb.GenerateTestFloatHistograms(numDatapoints) + floatHistograms := tsdbutil.GenerateTestFloatHistograms(numDatapoints) for i := 0; i < numDatapoints; i++ { _, err := app.AppendHistogram(0, lset, lastTs, nil, floatHistograms[i]) @@ -475,7 +475,7 @@ func TestPartialTruncateWAL(t *testing.T) { for _, l := range lbls { lset := labels.New(l...) - histograms := tsdb.GenerateTestHistograms(numDatapoints) + histograms := tsdbutil.GenerateTestHistograms(numDatapoints) for i := 0; i < numDatapoints; i++ { _, err := app.AppendHistogram(0, lset, lastTs, histograms[i], nil) @@ -488,7 +488,7 @@ func TestPartialTruncateWAL(t *testing.T) { for _, l := range lbls { lset := labels.New(l...) - floatHistograms := tsdb.GenerateTestFloatHistograms(numDatapoints) + floatHistograms := tsdbutil.GenerateTestFloatHistograms(numDatapoints) for i := 0; i < numDatapoints; i++ { _, err := app.AppendHistogram(0, lset, lastTs, nil, floatHistograms[i]) @@ -529,7 +529,7 @@ func TestWALReplay(t *testing.T) { for _, l := range lbls { lset := labels.New(l...) - histograms := tsdb.GenerateTestHistograms(numHistograms) + histograms := tsdbutil.GenerateTestHistograms(numHistograms) for i := 0; i < numHistograms; i++ { _, err := app.AppendHistogram(0, lset, lastTs, histograms[i], nil) @@ -541,7 +541,7 @@ func TestWALReplay(t *testing.T) { for _, l := range lbls { lset := labels.New(l...) - floatHistograms := tsdb.GenerateTestFloatHistograms(numHistograms) + floatHistograms := tsdbutil.GenerateTestFloatHistograms(numHistograms) for i := 0; i < numHistograms; i++ { _, err := app.AppendHistogram(0, lset, lastTs, nil, floatHistograms[i]) @@ -622,7 +622,7 @@ func Test_ExistingWAL_NextRef(t *testing.T) { } histogramCount := 10 - histograms := tsdb.GenerateTestHistograms(histogramCount) + histograms := tsdbutil.GenerateTestHistograms(histogramCount) // Append series for i := 0; i < histogramCount; i++ { lset := labels.FromStrings(model.MetricNameLabel, fmt.Sprintf("histogram_%d", i)) diff --git a/tsdb/head.go b/tsdb/head.go index 728b3c9d3..833aa8c5f 100644 --- a/tsdb/head.go +++ b/tsdb/head.go @@ -17,7 +17,6 @@ import ( "fmt" "io" "math" - "math/rand" "path/filepath" "sync" "time" @@ -2037,93 +2036,3 @@ func (h *Head) updateWALReplayStatusRead(current int) { h.stats.WALReplayStatus.Current = current } - -func GenerateTestHistograms(n int) (r []*histogram.Histogram) { - for i := 0; i < n; i++ { - h := GenerateTestHistogram(i) - if i > 0 { - h.CounterResetHint = histogram.NotCounterReset - } - r = append(r, h) - } - return r -} - -// Generates a test histogram, it is up to the user to set any known counter reset hint. -func GenerateTestHistogram(i int) *histogram.Histogram { - return &histogram.Histogram{ - Count: 10 + uint64(i*8), - ZeroCount: 2 + uint64(i), - ZeroThreshold: 0.001, - Sum: 18.4 * float64(i+1), - Schema: 1, - PositiveSpans: []histogram.Span{ - {Offset: 0, Length: 2}, - {Offset: 1, Length: 2}, - }, - PositiveBuckets: []int64{int64(i + 1), 1, -1, 0}, - NegativeSpans: []histogram.Span{ - {Offset: 0, Length: 2}, - {Offset: 1, Length: 2}, - }, - NegativeBuckets: []int64{int64(i + 1), 1, -1, 0}, - } -} - -func GenerateTestGaugeHistograms(n int) (r []*histogram.Histogram) { - for x := 0; x < n; x++ { - r = append(r, GenerateTestGaugeHistogram(rand.Intn(n))) - } - return r -} - -func GenerateTestGaugeHistogram(i int) *histogram.Histogram { - h := GenerateTestHistogram(i) - h.CounterResetHint = histogram.GaugeType - return h -} - -func GenerateTestFloatHistograms(n int) (r []*histogram.FloatHistogram) { - for i := 0; i < n; i++ { - h := GenerateTestFloatHistogram(i) - if i > 0 { - h.CounterResetHint = histogram.NotCounterReset - } - r = append(r, h) - } - return r -} - -// Generates a test float histogram, it is up to the user to set any known counter reset hint. -func GenerateTestFloatHistogram(i int) *histogram.FloatHistogram { - return &histogram.FloatHistogram{ - Count: 10 + float64(i*8), - ZeroCount: 2 + float64(i), - ZeroThreshold: 0.001, - Sum: 18.4 * float64(i+1), - Schema: 1, - PositiveSpans: []histogram.Span{ - {Offset: 0, Length: 2}, - {Offset: 1, Length: 2}, - }, - PositiveBuckets: []float64{float64(i + 1), float64(i + 2), float64(i + 1), float64(i + 1)}, - NegativeSpans: []histogram.Span{ - {Offset: 0, Length: 2}, - {Offset: 1, Length: 2}, - }, - NegativeBuckets: []float64{float64(i + 1), float64(i + 2), float64(i + 1), float64(i + 1)}, - } -} - -func GenerateTestGaugeFloatHistograms(n int) (r []*histogram.FloatHistogram) { - for x := 0; x < n; x++ { - r = append(r, GenerateTestGaugeFloatHistogram(rand.Intn(n))) - } - return r -} - -func GenerateTestGaugeFloatHistogram(i int) *histogram.FloatHistogram { - h := GenerateTestFloatHistogram(i) - h.CounterResetHint = histogram.GaugeType - return h -} diff --git a/tsdb/head_test.go b/tsdb/head_test.go index 1130bbe19..08842e4ce 100644 --- a/tsdb/head_test.go +++ b/tsdb/head_test.go @@ -1343,7 +1343,7 @@ func TestMemSeries_appendHistogram(t *testing.T) { s := newMemSeries(labels.Labels{}, 1, defaultIsolationDisabled) - histograms := GenerateTestHistograms(4) + histograms := tsdbutil.GenerateTestHistograms(4) histogramWithOneMoreBucket := histograms[3].Copy() histogramWithOneMoreBucket.Count++ histogramWithOneMoreBucket.Sum += 1.23 @@ -2836,7 +2836,7 @@ func TestAppendHistogram(t *testing.T) { expHistograms := make([]tsdbutil.Sample, 0, 2*numHistograms) // Counter integer histograms. - for _, h := range GenerateTestHistograms(numHistograms) { + for _, h := range tsdbutil.GenerateTestHistograms(numHistograms) { _, err := app.AppendHistogram(0, l, ingestTs, h, nil) require.NoError(t, err) expHistograms = append(expHistograms, sample{t: ingestTs, h: h}) @@ -2848,7 +2848,7 @@ func TestAppendHistogram(t *testing.T) { } // Gauge integer histograms. - for _, h := range GenerateTestGaugeHistograms(numHistograms) { + for _, h := range tsdbutil.GenerateTestGaugeHistograms(numHistograms) { _, err := app.AppendHistogram(0, l, ingestTs, h, nil) require.NoError(t, err) expHistograms = append(expHistograms, sample{t: ingestTs, h: h}) @@ -2862,7 +2862,7 @@ func TestAppendHistogram(t *testing.T) { expFloatHistograms := make([]tsdbutil.Sample, 0, 2*numHistograms) // Counter float histograms. - for _, fh := range GenerateTestFloatHistograms(numHistograms) { + for _, fh := range tsdbutil.GenerateTestFloatHistograms(numHistograms) { _, err := app.AppendHistogram(0, l, ingestTs, nil, fh) require.NoError(t, err) expFloatHistograms = append(expFloatHistograms, sample{t: ingestTs, fh: fh}) @@ -2874,7 +2874,7 @@ func TestAppendHistogram(t *testing.T) { } // Gauge float histograms. - for _, fh := range GenerateTestGaugeFloatHistograms(numHistograms) { + for _, fh := range tsdbutil.GenerateTestGaugeFloatHistograms(numHistograms) { _, err := app.AppendHistogram(0, l, ingestTs, nil, fh) require.NoError(t, err) expFloatHistograms = append(expFloatHistograms, sample{t: ingestTs, fh: fh}) @@ -2944,9 +2944,9 @@ func TestHistogramInWALAndMmapChunk(t *testing.T) { app = head.Appender(context.Background()) var hists []*histogram.Histogram if gauge { - hists = GenerateTestGaugeHistograms(numHistograms) + hists = tsdbutil.GenerateTestGaugeHistograms(numHistograms) } else { - hists = GenerateTestHistograms(numHistograms) + hists = tsdbutil.GenerateTestHistograms(numHistograms) } for _, h := range hists { h.Count = h.Count * 2 @@ -2967,9 +2967,9 @@ func TestHistogramInWALAndMmapChunk(t *testing.T) { app = head.Appender(context.Background()) var hists []*histogram.FloatHistogram if gauge { - hists = GenerateTestGaugeFloatHistograms(numHistograms) + hists = tsdbutil.GenerateTestGaugeFloatHistograms(numHistograms) } else { - hists = GenerateTestFloatHistograms(numHistograms) + hists = tsdbutil.GenerateTestFloatHistograms(numHistograms) } for _, h := range hists { h.Count = h.Count * 2 @@ -3007,9 +3007,9 @@ func TestHistogramInWALAndMmapChunk(t *testing.T) { app = head.Appender(context.Background()) var hists []*histogram.Histogram if gauge { - hists = GenerateTestGaugeHistograms(100) + hists = tsdbutil.GenerateTestGaugeHistograms(100) } else { - hists = GenerateTestHistograms(100) + hists = tsdbutil.GenerateTestHistograms(100) } for _, h := range hists { ts++ @@ -3044,9 +3044,9 @@ func TestHistogramInWALAndMmapChunk(t *testing.T) { app = head.Appender(context.Background()) var hists []*histogram.FloatHistogram if gauge { - hists = GenerateTestGaugeFloatHistograms(100) + hists = tsdbutil.GenerateTestGaugeFloatHistograms(100) } else { - hists = GenerateTestFloatHistograms(100) + hists = tsdbutil.GenerateTestFloatHistograms(100) } for _, h := range hists { ts++ @@ -3422,14 +3422,14 @@ func TestHistogramMetrics(t *testing.T) { for x := 0; x < 5; x++ { expHSeries++ l := labels.FromStrings("a", fmt.Sprintf("b%d", x)) - for i, h := range GenerateTestHistograms(numHistograms) { + for i, h := range tsdbutil.GenerateTestHistograms(numHistograms) { app := head.Appender(context.Background()) _, err := app.AppendHistogram(0, l, int64(i), h, nil) require.NoError(t, err) require.NoError(t, app.Commit()) expHSamples++ } - for i, fh := range GenerateTestFloatHistograms(numHistograms) { + for i, fh := range tsdbutil.GenerateTestFloatHistograms(numHistograms) { app := head.Appender(context.Background()) _, err := app.AppendHistogram(0, l, int64(numHistograms+i), nil, fh) require.NoError(t, err) @@ -3544,7 +3544,7 @@ func testHistogramStaleSampleHelper(t *testing.T, floatHistogram bool) { // Adding stale in the same appender. app := head.Appender(context.Background()) - for _, h := range GenerateTestHistograms(numHistograms) { + for _, h := range tsdbutil.GenerateTestHistograms(numHistograms) { var err error if floatHistogram { _, err = app.AppendHistogram(0, l, 100*int64(len(expHistograms)), nil, h.ToFloat()) @@ -3573,7 +3573,7 @@ func testHistogramStaleSampleHelper(t *testing.T, floatHistogram bool) { // Adding stale in different appender and continuing series after a stale sample. app = head.Appender(context.Background()) - for _, h := range GenerateTestHistograms(2 * numHistograms)[numHistograms:] { + for _, h := range tsdbutil.GenerateTestHistograms(2 * numHistograms)[numHistograms:] { var err error if floatHistogram { _, err = app.AppendHistogram(0, l, 100*int64(len(expHistograms)), nil, h.ToFloat()) @@ -3652,7 +3652,7 @@ func TestHistogramCounterResetHeader(t *testing.T) { } } - h := GenerateTestHistograms(1)[0] + h := tsdbutil.GenerateTestHistograms(1)[0] h.PositiveBuckets = []int64{100, 1, 1, 1} h.NegativeBuckets = []int64{100, 1, 1, 1} h.Count = 1000 @@ -3729,8 +3729,8 @@ func TestAppendingDifferentEncodingToSameSeries(t *testing.T) { }) db.DisableCompactions() - hists := GenerateTestHistograms(10) - floatHists := GenerateTestFloatHistograms(10) + hists := tsdbutil.GenerateTestHistograms(10) + floatHists := tsdbutil.GenerateTestFloatHistograms(10) lbls := labels.FromStrings("a", "b") var expResult []tsdbutil.Sample @@ -4358,7 +4358,7 @@ func TestHistogramValidation(t *testing.T) { errMsgFloat string // To be considered for float histogram only if it is non-empty. }{ "valid histogram": { - h: GenerateTestHistograms(1)[0], + h: tsdbutil.GenerateTestHistograms(1)[0], }, "rejects histogram who has too few negative buckets": { h: &histogram.Histogram{ @@ -4641,7 +4641,7 @@ func TestGaugeHistogramWALAndChunkHeader(t *testing.T) { require.NoError(t, app.Commit()) } - hists := GenerateTestGaugeHistograms(5) + hists := tsdbutil.GenerateTestGaugeHistograms(5) hists[0].CounterResetHint = histogram.UnknownCounterReset appendHistogram(hists[0]) appendHistogram(hists[1]) @@ -4716,7 +4716,7 @@ func TestGaugeFloatHistogramWALAndChunkHeader(t *testing.T) { require.NoError(t, app.Commit()) } - hists := GenerateTestGaugeFloatHistograms(5) + hists := tsdbutil.GenerateTestGaugeFloatHistograms(5) hists[0].CounterResetHint = histogram.UnknownCounterReset appendHistogram(hists[0]) appendHistogram(hists[1]) diff --git a/tsdb/tsdbutil/chunks.go b/tsdb/tsdbutil/chunks.go index 87cc345dd..f9981ffe1 100644 --- a/tsdb/tsdbutil/chunks.go +++ b/tsdb/tsdbutil/chunks.go @@ -72,6 +72,8 @@ func ChunkFromSamplesGeneric(s Samples) chunks.Meta { ca.Append(s.Get(i).T(), s.Get(i).V()) case chunkenc.ValHistogram: ca.AppendHistogram(s.Get(i).T(), s.Get(i).H()) + case chunkenc.ValFloatHistogram: + ca.AppendFloatHistogram(s.Get(i).T(), s.Get(i).FH()) default: panic(fmt.Sprintf("unknown sample type %s", sampleType.String())) } @@ -128,12 +130,18 @@ func PopulatedChunk(numSamples int, minTime int64) chunks.Meta { // GenerateSamples starting at start and counting up numSamples. func GenerateSamples(start, numSamples int) []Sample { - samples := make([]Sample, 0, numSamples) - for i := start; i < start+numSamples; i++ { - samples = append(samples, sample{ + return generateSamples(start, numSamples, func(i int) Sample { + return sample{ t: int64(i), v: float64(i), - }) + } + }) +} + +func generateSamples(start, numSamples int, gen func(int) Sample) []Sample { + samples := make([]Sample, 0, numSamples) + for i := start; i < start+numSamples; i++ { + samples = append(samples, gen(i)) } return samples } diff --git a/tsdb/tsdbutil/histogram.go b/tsdb/tsdbutil/histogram.go new file mode 100644 index 000000000..3c276c841 --- /dev/null +++ b/tsdb/tsdbutil/histogram.go @@ -0,0 +1,110 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tsdbutil + +import ( + "math/rand" + + "github.com/prometheus/prometheus/model/histogram" +) + +func GenerateTestHistograms(n int) (r []*histogram.Histogram) { + for i := 0; i < n; i++ { + h := GenerateTestHistogram(i) + if i > 0 { + h.CounterResetHint = histogram.NotCounterReset + } + r = append(r, h) + } + return r +} + +// GenerateTestHistogram but it is up to the user to set any known counter reset hint. +func GenerateTestHistogram(i int) *histogram.Histogram { + return &histogram.Histogram{ + Count: 10 + uint64(i*8), + ZeroCount: 2 + uint64(i), + ZeroThreshold: 0.001, + Sum: 18.4 * float64(i+1), + Schema: 1, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 2}, + {Offset: 1, Length: 2}, + }, + PositiveBuckets: []int64{int64(i + 1), 1, -1, 0}, + NegativeSpans: []histogram.Span{ + {Offset: 0, Length: 2}, + {Offset: 1, Length: 2}, + }, + NegativeBuckets: []int64{int64(i + 1), 1, -1, 0}, + } +} + +func GenerateTestGaugeHistograms(n int) (r []*histogram.Histogram) { + for x := 0; x < n; x++ { + r = append(r, GenerateTestGaugeHistogram(rand.Intn(n))) + } + return r +} + +func GenerateTestGaugeHistogram(i int) *histogram.Histogram { + h := GenerateTestHistogram(i) + h.CounterResetHint = histogram.GaugeType + return h +} + +func GenerateTestFloatHistograms(n int) (r []*histogram.FloatHistogram) { + for i := 0; i < n; i++ { + h := GenerateTestFloatHistogram(i) + if i > 0 { + h.CounterResetHint = histogram.NotCounterReset + } + r = append(r, h) + } + return r +} + +// GenerateTestFloatHistogram but it is up to the user to set any known counter reset hint. +func GenerateTestFloatHistogram(i int) *histogram.FloatHistogram { + return &histogram.FloatHistogram{ + Count: 10 + float64(i*8), + ZeroCount: 2 + float64(i), + ZeroThreshold: 0.001, + Sum: 18.4 * float64(i+1), + Schema: 1, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 2}, + {Offset: 1, Length: 2}, + }, + PositiveBuckets: []float64{float64(i + 1), float64(i + 2), float64(i + 1), float64(i + 1)}, + NegativeSpans: []histogram.Span{ + {Offset: 0, Length: 2}, + {Offset: 1, Length: 2}, + }, + NegativeBuckets: []float64{float64(i + 1), float64(i + 2), float64(i + 1), float64(i + 1)}, + } +} + +func GenerateTestGaugeFloatHistograms(n int) (r []*histogram.FloatHistogram) { + for x := 0; x < n; x++ { + r = append(r, GenerateTestGaugeFloatHistogram(rand.Intn(n))) + } + return r +} + +func GenerateTestGaugeFloatHistogram(i int) *histogram.FloatHistogram { + h := GenerateTestFloatHistogram(i) + h.CounterResetHint = histogram.GaugeType + return h +}