From c8dce11939281051f044d19289053ff242c831a0 Mon Sep 17 00:00:00 2001 From: amanycodes Date: Fri, 31 Jan 2025 20:07:26 +0530 Subject: [PATCH 1/3] replaced-tsdb-append-error-metrics Signed-off-by: amanycodes --- tsdb/block_test.go | 6 +-- tsdb/db_test.go | 2 +- tsdb/head.go | 121 +++++++++++++++++++++++++------------------- tsdb/head_append.go | 51 ++++++++++++------- tsdb/head_test.go | 22 ++++---- tsdb/testutil.go | 4 +- 6 files changed, 120 insertions(+), 86 deletions(-) diff --git a/tsdb/block_test.go b/tsdb/block_test.go index 3e25cff3da..7cc0735b56 100644 --- a/tsdb/block_test.go +++ b/tsdb/block_test.go @@ -738,7 +738,7 @@ func createHeadWithOOOSamples(tb testing.TB, w *wlog.WL, series []storage.Series require.NoError(tb, app.Commit()) oooSamplesAppended := 0 - require.Equal(tb, float64(0), prom_testutil.ToFloat64(head.metrics.outOfOrderSamplesAppended)) + require.Equal(tb, float64(0), prom_testutil.ToFloat64(head.metrics.successulSamplesAppended.WithLabelValues(oooAppends))) app = head.Appender(context.Background()) for i, lset := range oooSampleLabels { @@ -751,11 +751,11 @@ func createHeadWithOOOSamples(tb testing.TB, w *wlog.WL, series []storage.Series } require.NoError(tb, app.Commit()) - actOOOAppended := prom_testutil.ToFloat64(head.metrics.outOfOrderSamplesAppended) + actOOOAppended := prom_testutil.ToFloat64(head.metrics.successulSamplesAppended.WithLabelValues(oooAppends)) require.GreaterOrEqual(tb, actOOOAppended, float64(oooSamplesAppended-len(series))) require.LessOrEqual(tb, actOOOAppended, float64(oooSamplesAppended)) - require.Equal(tb, float64(totalSamples), prom_testutil.ToFloat64(head.metrics.samplesAppended)) + require.Equal(tb, float64(totalSamples), prom_testutil.ToFloat64(head.metrics.successulSamplesAppended.WithLabelValues(successfulAppends))) return head } diff --git a/tsdb/db_test.go b/tsdb/db_test.go index 7dc60a7304..244300bd86 100644 --- a/tsdb/db_test.go +++ b/tsdb/db_test.go @@ -6897,7 +6897,7 @@ func testOOODisabled(t *testing.T, scenario sampleTypeScenario) { requireEqualSeries(t, expSamples, seriesSet, true) requireEqualOOOSamples(t, 0, db) require.Equal(t, float64(failedSamples), - prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType))+prom_testutil.ToFloat64(db.head.metrics.outOfBoundSamples.WithLabelValues(scenario.sampleType)), + prom_testutil.ToFloat64(db.head.metrics.sampleAppendFailures.WithLabelValues(outOfOrder, scenario.sampleType))+prom_testutil.ToFloat64(db.head.metrics.sampleAppendFailures.WithLabelValues(outOfBounds, scenario.sampleType)), "number of ooo/oob samples mismatch") // Verifying that no OOO artifacts were generated. diff --git a/tsdb/head.go b/tsdb/head.go index 4fbb4b5710..b5fa540c7e 100644 --- a/tsdb/head.go +++ b/tsdb/head.go @@ -342,38 +342,45 @@ func (h *Head) resetInMemoryState() error { } type headMetrics struct { - activeAppenders prometheus.Gauge - series prometheus.GaugeFunc - seriesCreated prometheus.Counter - seriesRemoved prometheus.Counter - seriesNotFound prometheus.Counter - chunks prometheus.Gauge - chunksCreated prometheus.Counter - chunksRemoved prometheus.Counter - gcDuration prometheus.Summary - samplesAppended *prometheus.CounterVec - outOfOrderSamplesAppended *prometheus.CounterVec - outOfBoundSamples *prometheus.CounterVec - outOfOrderSamples *prometheus.CounterVec - tooOldSamples *prometheus.CounterVec - walTruncateDuration prometheus.Summary - walCorruptionsTotal prometheus.Counter - dataTotalReplayDuration prometheus.Gauge - headTruncateFail prometheus.Counter - headTruncateTotal prometheus.Counter - checkpointDeleteFail prometheus.Counter - checkpointDeleteTotal prometheus.Counter - checkpointCreationFail prometheus.Counter - checkpointCreationTotal prometheus.Counter - mmapChunkCorruptionTotal prometheus.Counter - snapshotReplayErrorTotal prometheus.Counter // Will be either 0 or 1. - oooHistogram prometheus.Histogram - mmapChunksTotal prometheus.Counter + activeAppenders prometheus.Gauge + series prometheus.GaugeFunc + seriesCreated prometheus.Counter + seriesRemoved prometheus.Counter + seriesNotFound prometheus.Counter + chunks prometheus.Gauge + chunksCreated prometheus.Counter + chunksRemoved prometheus.Counter + gcDuration prometheus.Summary + successulSamplesAppended *prometheus.CounterVec + // samplesAppended *prometheus.CounterVec + // outOfOrderSamplesAppended *prometheus.CounterVec + // outOfBoundSamples *prometheus.CounterVec + // outOfOrderSamples *prometheus.CounterVec + // tooOldSamples *prometheus.CounterVec + walTruncateDuration prometheus.Summary + walCorruptionsTotal prometheus.Counter + dataTotalReplayDuration prometheus.Gauge + headTruncateFail prometheus.Counter + headTruncateTotal prometheus.Counter + checkpointDeleteFail prometheus.Counter + checkpointDeleteTotal prometheus.Counter + checkpointCreationFail prometheus.Counter + checkpointCreationTotal prometheus.Counter + mmapChunkCorruptionTotal prometheus.Counter + snapshotReplayErrorTotal prometheus.Counter // Will be either 0 or 1. + oooHistogram prometheus.Histogram + mmapChunksTotal prometheus.Counter + sampleAppendFailures *prometheus.CounterVec } const ( sampleMetricTypeFloat = "float" sampleMetricTypeHistogram = "histogram" + outOfBounds = "out_of_bounds" + outOfOrder = "out_of_order" + tooOld = "too_old" + successfulAppends = "successful_appends" + oooAppends = "ooo_appends" ) func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics { @@ -428,26 +435,34 @@ func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics { Name: "prometheus_tsdb_data_replay_duration_seconds", Help: "Time taken to replay the data on disk.", }), - samplesAppended: prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "prometheus_tsdb_head_samples_appended_total", - Help: "Total number of appended samples.", - }, []string{"type"}), - outOfOrderSamplesAppended: prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "prometheus_tsdb_head_out_of_order_samples_appended_total", - Help: "Total number of appended out of order samples.", - }, []string{"type"}), - outOfBoundSamples: prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "prometheus_tsdb_out_of_bound_samples_total", - Help: "Total number of out of bound samples ingestion failed attempts with out of order support disabled.", - }, []string{"type"}), - outOfOrderSamples: prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "prometheus_tsdb_out_of_order_samples_total", - Help: "Total number of out of order samples ingestion failed attempts due to out of order being disabled.", - }, []string{"type"}), - tooOldSamples: prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "prometheus_tsdb_too_old_samples_total", - Help: "Total number of out of order samples ingestion failed attempts with out of support enabled, but sample outside of time window.", - }, []string{"type"}), + successulSamplesAppended: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "prometheus_tsdb_head_sucessful_samples_appended_total", + Help: "Total number of successful appended samples including out of order samples.", + }, []string{"reason", "type"}), + sampleAppendFailures: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "prometheus_tsdb_head_samples_append_failures_total", + Help: "Total number of sample append failures with different reasons.", + }, []string{"reason", "type"}), + // samplesAppended: prometheus.NewCounterVec(prometheus.CounterOpts{ + // Name: "prometheus_tsdb_head_samples_appended_total", + // Help: "Total number of appended samples.", + // }, []string{"type"}), + // outOfOrderSamplesAppended: prometheus.NewCounterVec(prometheus.CounterOpts{ + // Name: "prometheus_tsdb_head_out_of_order_samples_appended_total", + // Help: "Total number of appended out of order samples.", + // }, []string{"type"}), + // outOfBoundSamples: prometheus.NewCounterVec(prometheus.CounterOpts{ + // Name: "prometheus_tsdb_out_of_bound_samples_total", + // Help: "Total number of out of bound samples ingestion failed attempts with out of order support disabled.", + // }, []string{"type"}), + // outOfOrderSamples: prometheus.NewCounterVec(prometheus.CounterOpts{ + // Name: "prometheus_tsdb_out_of_order_samples_total", + // Help: "Total number of out of order samples ingestion failed attempts due to out of order being disabled.", + // }, []string{"type"}), + // tooOldSamples: prometheus.NewCounterVec(prometheus.CounterOpts{ + // Name: "prometheus_tsdb_too_old_samples_total", + // Help: "Total number of out of order samples ingestion failed attempts with out of support enabled, but sample outside of time window.", + // }, []string{"type"}), headTruncateFail: prometheus.NewCounter(prometheus.CounterOpts{ Name: "prometheus_tsdb_head_truncations_failed_total", Help: "Total number of head truncations that failed.", @@ -516,11 +531,13 @@ func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics { m.walTruncateDuration, m.walCorruptionsTotal, m.dataTotalReplayDuration, - m.samplesAppended, - m.outOfOrderSamplesAppended, - m.outOfBoundSamples, - m.outOfOrderSamples, - m.tooOldSamples, + // m.samplesAppended, + m.successulSamplesAppended, + // m.outOfOrderSamplesAppended, + m.sampleAppendFailures, + // m.outOfBoundSamples, + // m.outOfOrderSamples, + // m.tooOldSamples, m.headTruncateFail, m.headTruncateTotal, m.checkpointDeleteFail, diff --git a/tsdb/head_append.go b/tsdb/head_append.go index c94c42bc53..3d7f911389 100644 --- a/tsdb/head_append.go +++ b/tsdb/head_append.go @@ -343,7 +343,8 @@ func (a *headAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64 // Fail fast if OOO is disabled and the sample is out of bounds. // Otherwise a full check will be done later to decide if the sample is in-order or out-of-order. if a.oooTimeWindow == 0 && t < a.minValidTime { - a.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeFloat).Inc() + a.head.metrics.sampleAppendFailures.WithLabelValues(outOfBounds, sampleMetricTypeFloat).Inc() + // a.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeFloat).Inc() return 0, storage.ErrOutOfBounds } @@ -377,7 +378,8 @@ func (a *headAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64 isOOO, delta, err := s.appendable(t, v, a.headMaxt, a.minValidTime, a.oooTimeWindow) if err == nil { if isOOO && a.hints != nil && a.hints.DiscardOutOfOrder { - a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat).Inc() + a.head.metrics.sampleAppendFailures.WithLabelValues(outOfOrder, sampleMetricTypeFloat).Inc() + // a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat).Inc() return 0, storage.ErrOutOfOrderSample } s.pendingCommit = true @@ -388,9 +390,11 @@ func (a *headAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64 if err != nil { switch { case errors.Is(err, storage.ErrOutOfOrderSample): - a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat).Inc() + a.head.metrics.sampleAppendFailures.WithLabelValues(outOfOrder, sampleMetricTypeFloat).Inc() + // a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat).Inc() case errors.Is(err, storage.ErrTooOldSample): - a.head.metrics.tooOldSamples.WithLabelValues(sampleMetricTypeFloat).Inc() + a.head.metrics.sampleAppendFailures.WithLabelValues(tooOld, sampleMetricTypeFloat).Inc() + // a.head.metrics.tooOldSamples.WithLabelValues(sampleMetricTypeFloat).Inc() } return 0, err } @@ -655,7 +659,8 @@ func (a *headAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels // Fail fast if OOO is disabled and the sample is out of bounds. // Otherwise a full check will be done later to decide if the sample is in-order or out-of-order. if (a.oooTimeWindow == 0 || !a.head.opts.EnableOOONativeHistograms.Load()) && t < a.minValidTime { - a.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeHistogram).Inc() + a.head.metrics.sampleAppendFailures.WithLabelValues(outOfBounds, sampleMetricTypeHistogram).Inc() + // a.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeHistogram).Inc() return 0, storage.ErrOutOfBounds } @@ -707,9 +712,11 @@ func (a *headAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels case errors.Is(err, storage.ErrOutOfOrderSample): fallthrough case errors.Is(err, storage.ErrOOONativeHistogramsDisabled): - a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeHistogram).Inc() + a.head.metrics.sampleAppendFailures.WithLabelValues(outOfOrder, sampleMetricTypeHistogram).Inc() + // a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeHistogram).Inc() case errors.Is(err, storage.ErrTooOldSample): - a.head.metrics.tooOldSamples.WithLabelValues(sampleMetricTypeHistogram).Inc() + a.head.metrics.sampleAppendFailures.WithLabelValues(tooOld, sampleMetricTypeHistogram).Inc() + // a.head.metrics.tooOldSamples.WithLabelValues(sampleMetricTypeHistogram).Inc() } return 0, err } @@ -744,9 +751,11 @@ func (a *headAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels case errors.Is(err, storage.ErrOutOfOrderSample): fallthrough case errors.Is(err, storage.ErrOOONativeHistogramsDisabled): - a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeHistogram).Inc() + a.head.metrics.sampleAppendFailures.WithLabelValues(outOfOrder, sampleMetricTypeHistogram).Inc() + // a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeHistogram).Inc() case errors.Is(err, storage.ErrTooOldSample): - a.head.metrics.tooOldSamples.WithLabelValues(sampleMetricTypeHistogram).Inc() + // a.head.metrics.tooOldSamples.WithLabelValues(sampleMetricTypeHistogram).Inc() + a.head.metrics.sampleAppendFailures.WithLabelValues(tooOld, sampleMetricTypeHistogram).Inc() } return 0, err } @@ -1491,14 +1500,22 @@ func (a *headAppender) Commit() (err error) { a.commitFloatHistograms(acc) a.commitMetadata() - a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat).Add(float64(acc.floatOOORejected)) - a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeHistogram).Add(float64(acc.histoOOORejected)) - a.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeFloat).Add(float64(acc.floatOOBRejected)) - a.head.metrics.tooOldSamples.WithLabelValues(sampleMetricTypeFloat).Add(float64(acc.floatTooOldRejected)) - a.head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeFloat).Add(float64(acc.floatsAppended)) - a.head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeHistogram).Add(float64(acc.histogramsAppended)) - a.head.metrics.outOfOrderSamplesAppended.WithLabelValues(sampleMetricTypeFloat).Add(float64(acc.oooFloatsAccepted)) - a.head.metrics.outOfOrderSamplesAppended.WithLabelValues(sampleMetricTypeHistogram).Add(float64(acc.oooHistogramAccepted)) + // a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat).Add(float64(acc.floatOOORejected)) + // a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeHistogram).Add(float64(acc.histoOOORejected)) + a.head.metrics.sampleAppendFailures.WithLabelValues(outOfOrder, sampleMetricTypeHistogram).Add(float64(acc.histoOOORejected)) + a.head.metrics.sampleAppendFailures.WithLabelValues(outOfOrder, sampleMetricTypeFloat).Add(float64(acc.floatOOORejected)) + // a.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeFloat).Add(float64(acc.floatOOBRejected)) + a.head.metrics.sampleAppendFailures.WithLabelValues(outOfBounds, sampleMetricTypeFloat).Add(float64(acc.floatOOBRejected)) + // a.head.metrics.tooOldSamples.WithLabelValues(sampleMetricTypeFloat).Add(float64(acc.floatTooOldRejected)) + a.head.metrics.sampleAppendFailures.WithLabelValues(tooOld, sampleMetricTypeFloat).Add(float64(acc.floatTooOldRejected)) + a.head.metrics.successulSamplesAppended.WithLabelValues(successfulAppends, sampleMetricTypeFloat).Add(float64(acc.floatsAppended)) + a.head.metrics.successulSamplesAppended.WithLabelValues(successfulAppends, sampleMetricTypeHistogram).Add(float64(acc.floatsAppended)) + // a.head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeFloat).Add(float64(acc.floatsAppended)) + // a.head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeHistogram).Add(float64(acc.histogramsAppended)) + a.head.metrics.successulSamplesAppended.WithLabelValues(oooAppends, sampleMetricTypeFloat).Add(float64(acc.oooFloatsAccepted)) + a.head.metrics.successulSamplesAppended.WithLabelValues(oooAppends, sampleMetricTypeHistogram).Add(float64(acc.oooHistogramAccepted)) + // a.head.metrics.outOfOrderSamplesAppended.WithLabelValues(sampleMetricTypeFloat).Add(float64(acc.oooFloatsAccepted)) + // a.head.metrics.outOfOrderSamplesAppended.WithLabelValues(sampleMetricTypeHistogram).Add(float64(acc.oooHistogramAccepted)) a.head.updateMinMaxTime(acc.inOrderMint, acc.inOrderMaxt) a.head.updateMinOOOMaxOOOTime(acc.oooMinT, acc.oooMaxT) diff --git a/tsdb/head_test.go b/tsdb/head_test.go index 33b54a756f..6ec9117350 100644 --- a/tsdb/head_test.go +++ b/tsdb/head_test.go @@ -2768,19 +2768,19 @@ func testOutOfOrderSamplesMetric(t *testing.T, scenario sampleTypeScenario, opti require.NoError(t, app.Commit()) // Test out of order metric. - require.Equal(t, 0.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType))) + require.Equal(t, 0.0, prom_testutil.ToFloat64(db.head.metrics.sampleAppendFailures.WithLabelValues(outOfOrder, scenario.sampleType))) app = db.Appender(ctx) _, err = appendSample(app, 2) require.Equal(t, expectOutOfOrderError, err) - require.Equal(t, 1.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType))) + require.Equal(t, 1.0, prom_testutil.ToFloat64(db.head.metrics.sampleAppendFailures.WithLabelValues(outOfOrder, scenario.sampleType))) _, err = appendSample(app, 3) require.Equal(t, expectOutOfOrderError, err) - require.Equal(t, 2.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType))) + require.Equal(t, 2.0, prom_testutil.ToFloat64(db.head.metrics.sampleAppendFailures.WithLabelValues(outOfOrder, scenario.sampleType))) _, err = appendSample(app, 4) require.Equal(t, expectOutOfOrderError, err) - require.Equal(t, 3.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType))) + require.Equal(t, 3.0, prom_testutil.ToFloat64(db.head.metrics.sampleAppendFailures.WithLabelValues(outOfOrder, scenario.sampleType))) require.NoError(t, app.Commit()) // Compact Head to test out of bound metric. @@ -2796,11 +2796,11 @@ func testOutOfOrderSamplesMetric(t *testing.T, scenario sampleTypeScenario, opti app = db.Appender(ctx) _, err = appendSample(app, db.head.minValidTime.Load()-2) require.Equal(t, storage.ErrOutOfBounds, err) - require.Equal(t, 1.0, prom_testutil.ToFloat64(db.head.metrics.outOfBoundSamples.WithLabelValues(scenario.sampleType))) + require.Equal(t, 1.0, prom_testutil.ToFloat64(db.head.metrics.sampleAppendFailures.WithLabelValues(outOfBounds, scenario.sampleType))) _, err = appendSample(app, db.head.minValidTime.Load()-1) require.Equal(t, storage.ErrOutOfBounds, err) - require.Equal(t, 2.0, prom_testutil.ToFloat64(db.head.metrics.outOfBoundSamples.WithLabelValues(scenario.sampleType))) + require.Equal(t, 2.0, prom_testutil.ToFloat64(db.head.metrics.sampleAppendFailures.WithLabelValues(outOfBounds, scenario.sampleType))) require.NoError(t, app.Commit()) // Some more valid samples for out of order. @@ -2815,15 +2815,15 @@ func testOutOfOrderSamplesMetric(t *testing.T, scenario sampleTypeScenario, opti app = db.Appender(ctx) _, err = appendSample(app, db.head.minValidTime.Load()+DefaultBlockDuration+2) require.Equal(t, expectOutOfOrderError, err) - require.Equal(t, 4.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType))) + require.Equal(t, 4.0, prom_testutil.ToFloat64(db.head.metrics.sampleAppendFailures.WithLabelValues(outOfOrder, scenario.sampleType))) _, err = appendSample(app, db.head.minValidTime.Load()+DefaultBlockDuration+3) require.Equal(t, expectOutOfOrderError, err) - require.Equal(t, 5.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType))) + require.Equal(t, 5.0, prom_testutil.ToFloat64(db.head.metrics.sampleAppendFailures.WithLabelValues(outOfOrder, scenario.sampleType))) _, err = appendSample(app, db.head.minValidTime.Load()+DefaultBlockDuration+4) require.Equal(t, expectOutOfOrderError, err) - require.Equal(t, 6.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType))) + require.Equal(t, 6.0, prom_testutil.ToFloat64(db.head.metrics.sampleAppendFailures.WithLabelValues(outOfOrder, scenario.sampleType))) require.NoError(t, app.Commit()) } @@ -4379,7 +4379,7 @@ func TestHistogramMetrics(t *testing.T) { } } - require.Equal(t, float64(expHSamples), prom_testutil.ToFloat64(head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeHistogram))) + require.Equal(t, float64(expHSamples), prom_testutil.ToFloat64(head.metrics.successulSamplesAppended.WithLabelValues(successfulAppends, sampleMetricTypeHistogram))) require.NoError(t, head.Close()) w, err := wlog.NewSize(nil, nil, head.wal.Dir(), 32768, wlog.CompressionNone) @@ -4388,7 +4388,7 @@ func TestHistogramMetrics(t *testing.T) { require.NoError(t, err) require.NoError(t, head.Init(0)) - require.Equal(t, float64(0), prom_testutil.ToFloat64(head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeHistogram))) // Counter reset. + require.Equal(t, float64(0), prom_testutil.ToFloat64(head.metrics.successulSamplesAppended.WithLabelValues(successfulAppends, sampleMetricTypeHistogram))) // Counter reset. } func TestHistogramStaleSample(t *testing.T) { diff --git a/tsdb/testutil.go b/tsdb/testutil.go index e957b0307b..a8fe6b1391 100644 --- a/tsdb/testutil.go +++ b/tsdb/testutil.go @@ -149,8 +149,8 @@ func requireEqualSeries(t *testing.T, expected, actual map[string][]chunks.Sampl func requireEqualOOOSamples(t *testing.T, expectedSamples int, db *DB) { require.Equal(t, float64(expectedSamples), - prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamplesAppended.WithLabelValues(sampleMetricTypeFloat))+ - prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamplesAppended.WithLabelValues(sampleMetricTypeHistogram)), + prom_testutil.ToFloat64(db.head.metrics.successulSamplesAppended.WithLabelValues(oooAppends, sampleMetricTypeFloat))+ + prom_testutil.ToFloat64(db.head.metrics.successulSamplesAppended.WithLabelValues(oooAppends, sampleMetricTypeHistogram)), "number of ooo appended samples mismatch") } From 011ad4f3d363661c58176331ab41f5a553570053 Mon Sep 17 00:00:00 2001 From: amanycodes Date: Fri, 31 Jan 2025 23:34:01 +0530 Subject: [PATCH 2/3] fixed CI Erros Signed-off-by: amanycodes --- tsdb/block_test.go | 6 +++--- tsdb/head.go | 26 +++++++++++++------------- tsdb/head_append.go | 8 ++++---- tsdb/head_test.go | 4 ++-- tsdb/testutil.go | 4 ++-- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/tsdb/block_test.go b/tsdb/block_test.go index 7cc0735b56..a4b68e6805 100644 --- a/tsdb/block_test.go +++ b/tsdb/block_test.go @@ -738,7 +738,7 @@ func createHeadWithOOOSamples(tb testing.TB, w *wlog.WL, series []storage.Series require.NoError(tb, app.Commit()) oooSamplesAppended := 0 - require.Equal(tb, float64(0), prom_testutil.ToFloat64(head.metrics.successulSamplesAppended.WithLabelValues(oooAppends))) + require.Equal(tb, float64(0), prom_testutil.ToFloat64(head.metrics.successfulSamplesAppended.WithLabelValues(oooAppends))) app = head.Appender(context.Background()) for i, lset := range oooSampleLabels { @@ -751,11 +751,11 @@ func createHeadWithOOOSamples(tb testing.TB, w *wlog.WL, series []storage.Series } require.NoError(tb, app.Commit()) - actOOOAppended := prom_testutil.ToFloat64(head.metrics.successulSamplesAppended.WithLabelValues(oooAppends)) + actOOOAppended := prom_testutil.ToFloat64(head.metrics.successfulSamplesAppended.WithLabelValues(oooAppends)) require.GreaterOrEqual(tb, actOOOAppended, float64(oooSamplesAppended-len(series))) require.LessOrEqual(tb, actOOOAppended, float64(oooSamplesAppended)) - require.Equal(tb, float64(totalSamples), prom_testutil.ToFloat64(head.metrics.successulSamplesAppended.WithLabelValues(successfulAppends))) + require.Equal(tb, float64(totalSamples), prom_testutil.ToFloat64(head.metrics.successfulSamplesAppended.WithLabelValues(successfulAppends))) return head } diff --git a/tsdb/head.go b/tsdb/head.go index b5fa540c7e..bf6bd69826 100644 --- a/tsdb/head.go +++ b/tsdb/head.go @@ -342,16 +342,16 @@ func (h *Head) resetInMemoryState() error { } type headMetrics struct { - activeAppenders prometheus.Gauge - series prometheus.GaugeFunc - seriesCreated prometheus.Counter - seriesRemoved prometheus.Counter - seriesNotFound prometheus.Counter - chunks prometheus.Gauge - chunksCreated prometheus.Counter - chunksRemoved prometheus.Counter - gcDuration prometheus.Summary - successulSamplesAppended *prometheus.CounterVec + activeAppenders prometheus.Gauge + series prometheus.GaugeFunc + seriesCreated prometheus.Counter + seriesRemoved prometheus.Counter + seriesNotFound prometheus.Counter + chunks prometheus.Gauge + chunksCreated prometheus.Counter + chunksRemoved prometheus.Counter + gcDuration prometheus.Summary + successfulSamplesAppended *prometheus.CounterVec // samplesAppended *prometheus.CounterVec // outOfOrderSamplesAppended *prometheus.CounterVec // outOfBoundSamples *prometheus.CounterVec @@ -435,8 +435,8 @@ func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics { Name: "prometheus_tsdb_data_replay_duration_seconds", Help: "Time taken to replay the data on disk.", }), - successulSamplesAppended: prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "prometheus_tsdb_head_sucessful_samples_appended_total", + successfulSamplesAppended: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "prometheus_tsdb_head_successful_samples_appended_total", Help: "Total number of successful appended samples including out of order samples.", }, []string{"reason", "type"}), sampleAppendFailures: prometheus.NewCounterVec(prometheus.CounterOpts{ @@ -532,7 +532,7 @@ func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics { m.walCorruptionsTotal, m.dataTotalReplayDuration, // m.samplesAppended, - m.successulSamplesAppended, + m.successfulSamplesAppended, // m.outOfOrderSamplesAppended, m.sampleAppendFailures, // m.outOfBoundSamples, diff --git a/tsdb/head_append.go b/tsdb/head_append.go index 3d7f911389..c170c757a4 100644 --- a/tsdb/head_append.go +++ b/tsdb/head_append.go @@ -1508,12 +1508,12 @@ func (a *headAppender) Commit() (err error) { a.head.metrics.sampleAppendFailures.WithLabelValues(outOfBounds, sampleMetricTypeFloat).Add(float64(acc.floatOOBRejected)) // a.head.metrics.tooOldSamples.WithLabelValues(sampleMetricTypeFloat).Add(float64(acc.floatTooOldRejected)) a.head.metrics.sampleAppendFailures.WithLabelValues(tooOld, sampleMetricTypeFloat).Add(float64(acc.floatTooOldRejected)) - a.head.metrics.successulSamplesAppended.WithLabelValues(successfulAppends, sampleMetricTypeFloat).Add(float64(acc.floatsAppended)) - a.head.metrics.successulSamplesAppended.WithLabelValues(successfulAppends, sampleMetricTypeHistogram).Add(float64(acc.floatsAppended)) + a.head.metrics.successfulSamplesAppended.WithLabelValues(successfulAppends, sampleMetricTypeFloat).Add(float64(acc.floatsAppended)) + a.head.metrics.successfulSamplesAppended.WithLabelValues(successfulAppends, sampleMetricTypeHistogram).Add(float64(acc.histogramsAppended)) // a.head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeFloat).Add(float64(acc.floatsAppended)) // a.head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeHistogram).Add(float64(acc.histogramsAppended)) - a.head.metrics.successulSamplesAppended.WithLabelValues(oooAppends, sampleMetricTypeFloat).Add(float64(acc.oooFloatsAccepted)) - a.head.metrics.successulSamplesAppended.WithLabelValues(oooAppends, sampleMetricTypeHistogram).Add(float64(acc.oooHistogramAccepted)) + a.head.metrics.successfulSamplesAppended.WithLabelValues(oooAppends, sampleMetricTypeFloat).Add(float64(acc.oooFloatsAccepted)) + a.head.metrics.successfulSamplesAppended.WithLabelValues(oooAppends, sampleMetricTypeHistogram).Add(float64(acc.oooHistogramAccepted)) // a.head.metrics.outOfOrderSamplesAppended.WithLabelValues(sampleMetricTypeFloat).Add(float64(acc.oooFloatsAccepted)) // a.head.metrics.outOfOrderSamplesAppended.WithLabelValues(sampleMetricTypeHistogram).Add(float64(acc.oooHistogramAccepted)) a.head.updateMinMaxTime(acc.inOrderMint, acc.inOrderMaxt) diff --git a/tsdb/head_test.go b/tsdb/head_test.go index 6ec9117350..5c0883ddfa 100644 --- a/tsdb/head_test.go +++ b/tsdb/head_test.go @@ -4379,7 +4379,7 @@ func TestHistogramMetrics(t *testing.T) { } } - require.Equal(t, float64(expHSamples), prom_testutil.ToFloat64(head.metrics.successulSamplesAppended.WithLabelValues(successfulAppends, sampleMetricTypeHistogram))) + require.Equal(t, float64(expHSamples), prom_testutil.ToFloat64(head.metrics.successfulSamplesAppended.WithLabelValues(successfulAppends, sampleMetricTypeHistogram))) require.NoError(t, head.Close()) w, err := wlog.NewSize(nil, nil, head.wal.Dir(), 32768, wlog.CompressionNone) @@ -4388,7 +4388,7 @@ func TestHistogramMetrics(t *testing.T) { require.NoError(t, err) require.NoError(t, head.Init(0)) - require.Equal(t, float64(0), prom_testutil.ToFloat64(head.metrics.successulSamplesAppended.WithLabelValues(successfulAppends, sampleMetricTypeHistogram))) // Counter reset. + require.Equal(t, float64(0), prom_testutil.ToFloat64(head.metrics.successfulSamplesAppended.WithLabelValues(successfulAppends, sampleMetricTypeHistogram))) // Counter reset. } func TestHistogramStaleSample(t *testing.T) { diff --git a/tsdb/testutil.go b/tsdb/testutil.go index a8fe6b1391..03c21c9f2b 100644 --- a/tsdb/testutil.go +++ b/tsdb/testutil.go @@ -149,8 +149,8 @@ func requireEqualSeries(t *testing.T, expected, actual map[string][]chunks.Sampl func requireEqualOOOSamples(t *testing.T, expectedSamples int, db *DB) { require.Equal(t, float64(expectedSamples), - prom_testutil.ToFloat64(db.head.metrics.successulSamplesAppended.WithLabelValues(oooAppends, sampleMetricTypeFloat))+ - prom_testutil.ToFloat64(db.head.metrics.successulSamplesAppended.WithLabelValues(oooAppends, sampleMetricTypeHistogram)), + prom_testutil.ToFloat64(db.head.metrics.successfulSamplesAppended.WithLabelValues(oooAppends, sampleMetricTypeFloat))+ + prom_testutil.ToFloat64(db.head.metrics.successfulSamplesAppended.WithLabelValues(oooAppends, sampleMetricTypeHistogram)), "number of ooo appended samples mismatch") } From 2dcab871cc7ee216427839fbdb083eeba6dff38e Mon Sep 17 00:00:00 2001 From: amanycodes Date: Mon, 3 Feb 2025 18:57:02 +0530 Subject: [PATCH 3/3] replaced success metric and removed comments Signed-off-by: amanycodes --- tsdb/block_test.go | 6 ++-- tsdb/head.go | 80 +++++++++++++++------------------------------ tsdb/head_append.go | 25 +++----------- tsdb/head_test.go | 4 +-- tsdb/testutil.go | 4 +-- 5 files changed, 38 insertions(+), 81 deletions(-) diff --git a/tsdb/block_test.go b/tsdb/block_test.go index a4b68e6805..3e25cff3da 100644 --- a/tsdb/block_test.go +++ b/tsdb/block_test.go @@ -738,7 +738,7 @@ func createHeadWithOOOSamples(tb testing.TB, w *wlog.WL, series []storage.Series require.NoError(tb, app.Commit()) oooSamplesAppended := 0 - require.Equal(tb, float64(0), prom_testutil.ToFloat64(head.metrics.successfulSamplesAppended.WithLabelValues(oooAppends))) + require.Equal(tb, float64(0), prom_testutil.ToFloat64(head.metrics.outOfOrderSamplesAppended)) app = head.Appender(context.Background()) for i, lset := range oooSampleLabels { @@ -751,11 +751,11 @@ func createHeadWithOOOSamples(tb testing.TB, w *wlog.WL, series []storage.Series } require.NoError(tb, app.Commit()) - actOOOAppended := prom_testutil.ToFloat64(head.metrics.successfulSamplesAppended.WithLabelValues(oooAppends)) + actOOOAppended := prom_testutil.ToFloat64(head.metrics.outOfOrderSamplesAppended) require.GreaterOrEqual(tb, actOOOAppended, float64(oooSamplesAppended-len(series))) require.LessOrEqual(tb, actOOOAppended, float64(oooSamplesAppended)) - require.Equal(tb, float64(totalSamples), prom_testutil.ToFloat64(head.metrics.successfulSamplesAppended.WithLabelValues(successfulAppends))) + require.Equal(tb, float64(totalSamples), prom_testutil.ToFloat64(head.metrics.samplesAppended)) return head } diff --git a/tsdb/head.go b/tsdb/head.go index bf6bd69826..0b9a5e62f8 100644 --- a/tsdb/head.go +++ b/tsdb/head.go @@ -351,26 +351,22 @@ type headMetrics struct { chunksCreated prometheus.Counter chunksRemoved prometheus.Counter gcDuration prometheus.Summary - successfulSamplesAppended *prometheus.CounterVec - // samplesAppended *prometheus.CounterVec - // outOfOrderSamplesAppended *prometheus.CounterVec - // outOfBoundSamples *prometheus.CounterVec - // outOfOrderSamples *prometheus.CounterVec - // tooOldSamples *prometheus.CounterVec - walTruncateDuration prometheus.Summary - walCorruptionsTotal prometheus.Counter - dataTotalReplayDuration prometheus.Gauge - headTruncateFail prometheus.Counter - headTruncateTotal prometheus.Counter - checkpointDeleteFail prometheus.Counter - checkpointDeleteTotal prometheus.Counter - checkpointCreationFail prometheus.Counter - checkpointCreationTotal prometheus.Counter - mmapChunkCorruptionTotal prometheus.Counter - snapshotReplayErrorTotal prometheus.Counter // Will be either 0 or 1. - oooHistogram prometheus.Histogram - mmapChunksTotal prometheus.Counter - sampleAppendFailures *prometheus.CounterVec + samplesAppended *prometheus.CounterVec + outOfOrderSamplesAppended *prometheus.CounterVec + walTruncateDuration prometheus.Summary + walCorruptionsTotal prometheus.Counter + dataTotalReplayDuration prometheus.Gauge + headTruncateFail prometheus.Counter + headTruncateTotal prometheus.Counter + checkpointDeleteFail prometheus.Counter + checkpointDeleteTotal prometheus.Counter + checkpointCreationFail prometheus.Counter + checkpointCreationTotal prometheus.Counter + mmapChunkCorruptionTotal prometheus.Counter + snapshotReplayErrorTotal prometheus.Counter // Will be either 0 or 1. + oooHistogram prometheus.Histogram + mmapChunksTotal prometheus.Counter + sampleAppendFailures *prometheus.CounterVec } const ( @@ -379,8 +375,6 @@ const ( outOfBounds = "out_of_bounds" outOfOrder = "out_of_order" tooOld = "too_old" - successfulAppends = "successful_appends" - oooAppends = "ooo_appends" ) func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics { @@ -435,34 +429,18 @@ func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics { Name: "prometheus_tsdb_data_replay_duration_seconds", Help: "Time taken to replay the data on disk.", }), - successfulSamplesAppended: prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "prometheus_tsdb_head_successful_samples_appended_total", - Help: "Total number of successful appended samples including out of order samples.", - }, []string{"reason", "type"}), sampleAppendFailures: prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "prometheus_tsdb_head_samples_append_failures_total", + Name: "prometheus_tsdb_head_append_failures_total", Help: "Total number of sample append failures with different reasons.", }, []string{"reason", "type"}), - // samplesAppended: prometheus.NewCounterVec(prometheus.CounterOpts{ - // Name: "prometheus_tsdb_head_samples_appended_total", - // Help: "Total number of appended samples.", - // }, []string{"type"}), - // outOfOrderSamplesAppended: prometheus.NewCounterVec(prometheus.CounterOpts{ - // Name: "prometheus_tsdb_head_out_of_order_samples_appended_total", - // Help: "Total number of appended out of order samples.", - // }, []string{"type"}), - // outOfBoundSamples: prometheus.NewCounterVec(prometheus.CounterOpts{ - // Name: "prometheus_tsdb_out_of_bound_samples_total", - // Help: "Total number of out of bound samples ingestion failed attempts with out of order support disabled.", - // }, []string{"type"}), - // outOfOrderSamples: prometheus.NewCounterVec(prometheus.CounterOpts{ - // Name: "prometheus_tsdb_out_of_order_samples_total", - // Help: "Total number of out of order samples ingestion failed attempts due to out of order being disabled.", - // }, []string{"type"}), - // tooOldSamples: prometheus.NewCounterVec(prometheus.CounterOpts{ - // Name: "prometheus_tsdb_too_old_samples_total", - // Help: "Total number of out of order samples ingestion failed attempts with out of support enabled, but sample outside of time window.", - // }, []string{"type"}), + samplesAppended: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "prometheus_tsdb_head_samples_appended_total", + Help: "Total number of appended samples.", + }, []string{"type"}), + outOfOrderSamplesAppended: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "prometheus_tsdb_head_out_of_order_samples_appended_total", + Help: "Total number of appended out of order samples.", + }, []string{"type"}), headTruncateFail: prometheus.NewCounter(prometheus.CounterOpts{ Name: "prometheus_tsdb_head_truncations_failed_total", Help: "Total number of head truncations that failed.", @@ -531,13 +509,9 @@ func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics { m.walTruncateDuration, m.walCorruptionsTotal, m.dataTotalReplayDuration, - // m.samplesAppended, - m.successfulSamplesAppended, - // m.outOfOrderSamplesAppended, + m.samplesAppended, + m.outOfOrderSamplesAppended, m.sampleAppendFailures, - // m.outOfBoundSamples, - // m.outOfOrderSamples, - // m.tooOldSamples, m.headTruncateFail, m.headTruncateTotal, m.checkpointDeleteFail, diff --git a/tsdb/head_append.go b/tsdb/head_append.go index c170c757a4..7bd19829af 100644 --- a/tsdb/head_append.go +++ b/tsdb/head_append.go @@ -344,7 +344,6 @@ func (a *headAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64 // Otherwise a full check will be done later to decide if the sample is in-order or out-of-order. if a.oooTimeWindow == 0 && t < a.minValidTime { a.head.metrics.sampleAppendFailures.WithLabelValues(outOfBounds, sampleMetricTypeFloat).Inc() - // a.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeFloat).Inc() return 0, storage.ErrOutOfBounds } @@ -379,7 +378,6 @@ func (a *headAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64 if err == nil { if isOOO && a.hints != nil && a.hints.DiscardOutOfOrder { a.head.metrics.sampleAppendFailures.WithLabelValues(outOfOrder, sampleMetricTypeFloat).Inc() - // a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat).Inc() return 0, storage.ErrOutOfOrderSample } s.pendingCommit = true @@ -391,10 +389,8 @@ func (a *headAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64 switch { case errors.Is(err, storage.ErrOutOfOrderSample): a.head.metrics.sampleAppendFailures.WithLabelValues(outOfOrder, sampleMetricTypeFloat).Inc() - // a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat).Inc() case errors.Is(err, storage.ErrTooOldSample): a.head.metrics.sampleAppendFailures.WithLabelValues(tooOld, sampleMetricTypeFloat).Inc() - // a.head.metrics.tooOldSamples.WithLabelValues(sampleMetricTypeFloat).Inc() } return 0, err } @@ -660,7 +656,6 @@ func (a *headAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels // Otherwise a full check will be done later to decide if the sample is in-order or out-of-order. if (a.oooTimeWindow == 0 || !a.head.opts.EnableOOONativeHistograms.Load()) && t < a.minValidTime { a.head.metrics.sampleAppendFailures.WithLabelValues(outOfBounds, sampleMetricTypeHistogram).Inc() - // a.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeHistogram).Inc() return 0, storage.ErrOutOfBounds } @@ -713,10 +708,8 @@ func (a *headAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels fallthrough case errors.Is(err, storage.ErrOOONativeHistogramsDisabled): a.head.metrics.sampleAppendFailures.WithLabelValues(outOfOrder, sampleMetricTypeHistogram).Inc() - // a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeHistogram).Inc() case errors.Is(err, storage.ErrTooOldSample): a.head.metrics.sampleAppendFailures.WithLabelValues(tooOld, sampleMetricTypeHistogram).Inc() - // a.head.metrics.tooOldSamples.WithLabelValues(sampleMetricTypeHistogram).Inc() } return 0, err } @@ -752,9 +745,7 @@ func (a *headAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels fallthrough case errors.Is(err, storage.ErrOOONativeHistogramsDisabled): a.head.metrics.sampleAppendFailures.WithLabelValues(outOfOrder, sampleMetricTypeHistogram).Inc() - // a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeHistogram).Inc() case errors.Is(err, storage.ErrTooOldSample): - // a.head.metrics.tooOldSamples.WithLabelValues(sampleMetricTypeHistogram).Inc() a.head.metrics.sampleAppendFailures.WithLabelValues(tooOld, sampleMetricTypeHistogram).Inc() } return 0, err @@ -1500,22 +1491,14 @@ func (a *headAppender) Commit() (err error) { a.commitFloatHistograms(acc) a.commitMetadata() - // a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat).Add(float64(acc.floatOOORejected)) - // a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeHistogram).Add(float64(acc.histoOOORejected)) a.head.metrics.sampleAppendFailures.WithLabelValues(outOfOrder, sampleMetricTypeHistogram).Add(float64(acc.histoOOORejected)) a.head.metrics.sampleAppendFailures.WithLabelValues(outOfOrder, sampleMetricTypeFloat).Add(float64(acc.floatOOORejected)) - // a.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeFloat).Add(float64(acc.floatOOBRejected)) a.head.metrics.sampleAppendFailures.WithLabelValues(outOfBounds, sampleMetricTypeFloat).Add(float64(acc.floatOOBRejected)) - // a.head.metrics.tooOldSamples.WithLabelValues(sampleMetricTypeFloat).Add(float64(acc.floatTooOldRejected)) a.head.metrics.sampleAppendFailures.WithLabelValues(tooOld, sampleMetricTypeFloat).Add(float64(acc.floatTooOldRejected)) - a.head.metrics.successfulSamplesAppended.WithLabelValues(successfulAppends, sampleMetricTypeFloat).Add(float64(acc.floatsAppended)) - a.head.metrics.successfulSamplesAppended.WithLabelValues(successfulAppends, sampleMetricTypeHistogram).Add(float64(acc.histogramsAppended)) - // a.head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeFloat).Add(float64(acc.floatsAppended)) - // a.head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeHistogram).Add(float64(acc.histogramsAppended)) - a.head.metrics.successfulSamplesAppended.WithLabelValues(oooAppends, sampleMetricTypeFloat).Add(float64(acc.oooFloatsAccepted)) - a.head.metrics.successfulSamplesAppended.WithLabelValues(oooAppends, sampleMetricTypeHistogram).Add(float64(acc.oooHistogramAccepted)) - // a.head.metrics.outOfOrderSamplesAppended.WithLabelValues(sampleMetricTypeFloat).Add(float64(acc.oooFloatsAccepted)) - // a.head.metrics.outOfOrderSamplesAppended.WithLabelValues(sampleMetricTypeHistogram).Add(float64(acc.oooHistogramAccepted)) + a.head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeFloat).Add(float64(acc.floatsAppended)) + a.head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeHistogram).Add(float64(acc.histogramsAppended)) + a.head.metrics.outOfOrderSamplesAppended.WithLabelValues(sampleMetricTypeFloat).Add(float64(acc.oooFloatsAccepted)) + a.head.metrics.outOfOrderSamplesAppended.WithLabelValues(sampleMetricTypeHistogram).Add(float64(acc.oooHistogramAccepted)) a.head.updateMinMaxTime(acc.inOrderMint, acc.inOrderMaxt) a.head.updateMinOOOMaxOOOTime(acc.oooMinT, acc.oooMaxT) diff --git a/tsdb/head_test.go b/tsdb/head_test.go index 5c0883ddfa..1fe7f5b2bd 100644 --- a/tsdb/head_test.go +++ b/tsdb/head_test.go @@ -4379,7 +4379,7 @@ func TestHistogramMetrics(t *testing.T) { } } - require.Equal(t, float64(expHSamples), prom_testutil.ToFloat64(head.metrics.successfulSamplesAppended.WithLabelValues(successfulAppends, sampleMetricTypeHistogram))) + require.Equal(t, float64(expHSamples), prom_testutil.ToFloat64(head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeHistogram))) require.NoError(t, head.Close()) w, err := wlog.NewSize(nil, nil, head.wal.Dir(), 32768, wlog.CompressionNone) @@ -4388,7 +4388,7 @@ func TestHistogramMetrics(t *testing.T) { require.NoError(t, err) require.NoError(t, head.Init(0)) - require.Equal(t, float64(0), prom_testutil.ToFloat64(head.metrics.successfulSamplesAppended.WithLabelValues(successfulAppends, sampleMetricTypeHistogram))) // Counter reset. + require.Equal(t, float64(0), prom_testutil.ToFloat64(head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeHistogram))) } func TestHistogramStaleSample(t *testing.T) { diff --git a/tsdb/testutil.go b/tsdb/testutil.go index 03c21c9f2b..e957b0307b 100644 --- a/tsdb/testutil.go +++ b/tsdb/testutil.go @@ -149,8 +149,8 @@ func requireEqualSeries(t *testing.T, expected, actual map[string][]chunks.Sampl func requireEqualOOOSamples(t *testing.T, expectedSamples int, db *DB) { require.Equal(t, float64(expectedSamples), - prom_testutil.ToFloat64(db.head.metrics.successfulSamplesAppended.WithLabelValues(oooAppends, sampleMetricTypeFloat))+ - prom_testutil.ToFloat64(db.head.metrics.successfulSamplesAppended.WithLabelValues(oooAppends, sampleMetricTypeHistogram)), + prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamplesAppended.WithLabelValues(sampleMetricTypeFloat))+ + prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamplesAppended.WithLabelValues(sampleMetricTypeHistogram)), "number of ooo appended samples mismatch") }