From fb6a45f06bbe474096162d33c6ac8afcbbb71f45 Mon Sep 17 00:00:00 2001 From: Arve Knudsen Date: Wed, 17 Jan 2024 18:28:06 +0100 Subject: [PATCH 1/2] tsdb/wlog: Only treat unknown record types as failure Signed-off-by: Arve Knudsen --- CHANGELOG.md | 1 + tsdb/wlog/watcher.go | 10 ++++------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1dfcc5c33..e17124abe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ * [ENHANCEMENT] OTLP: Improve remote write format translation performance by using label set hashes for metric identifiers instead of string based ones. #14006 #13991 * [BUGFIX] OTLP: Don't generate target_info unless at least one identifying label is defined. #13991 * [BUGFIX] OTLP: Don't generate target_info unless there are metrics. #13991 +* [BUGFIX] tsdb/wlog.Watcher.readSegmentForGC: Only count unknown record types against record_decode_failures_total metric. #14042 ## 2.52.0-rc.1 / 2024-05-03 diff --git a/tsdb/wlog/watcher.go b/tsdb/wlog/watcher.go index 8ebd9249a..fd4f5f20f 100644 --- a/tsdb/wlog/watcher.go +++ b/tsdb/wlog/watcher.go @@ -685,14 +685,12 @@ func (w *Watcher) readSegmentForGC(r *LiveReader, segmentNum int, _ bool) error } w.writer.UpdateSeriesSegment(series, segmentNum) - // Ignore these; we're only interested in series. - case record.Samples: - case record.Exemplars: - case record.Tombstones: - - default: + case record.Unknown: // Could be corruption, or reading from a WAL from a newer Prometheus. w.recordDecodeFailsMetric.Inc() + + default: + // We're only interested in series. } } if err := r.Err(); err != nil { From 694f717dc44849592b439fce6ffa5fbcdf7957a6 Mon Sep 17 00:00:00 2001 From: Arve Knudsen Date: Tue, 28 May 2024 15:23:50 +0200 Subject: [PATCH 2/2] Watcher.readSegment: Only consider unknown rec types failures Signed-off-by: Arve Knudsen --- tsdb/wlog/watcher.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tsdb/wlog/watcher.go b/tsdb/wlog/watcher.go index fd4f5f20f..5a73acdd4 100644 --- a/tsdb/wlog/watcher.go +++ b/tsdb/wlog/watcher.go @@ -625,6 +625,7 @@ func (w *Watcher) readSegment(r *LiveReader, segmentNum int, tail bool) error { w.writer.AppendHistograms(histogramsToSend) histogramsToSend = histogramsToSend[:0] } + case record.FloatHistogramSamples: // Skip if experimental "histograms over remote write" is not enabled. if !w.sendHistograms { @@ -652,11 +653,13 @@ func (w *Watcher) readSegment(r *LiveReader, segmentNum int, tail bool) error { w.writer.AppendFloatHistograms(floatHistogramsToSend) floatHistogramsToSend = floatHistogramsToSend[:0] } - case record.Tombstones: - default: + case record.Unknown: // Could be corruption, or reading from a WAL from a newer Prometheus. w.recordDecodeFailsMetric.Inc() + + default: + // We're not interested in other types of records. } } if err := r.Err(); err != nil {