From b30db03f35651888e34ac101a06e25d27d15b476 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar <15064823+codesome@users.noreply.github.com> Date: Fri, 1 Oct 2021 17:00:22 +0530 Subject: [PATCH] Cut v2.30.2 (#9426) * Don't error on overlapping m-mapped chunks during WAL replay (#9381) Signed-off-by: Ganesh Vernekar * Reduce log level during WAL replay on overlapping m-map chunks (#9425) Signed-off-by: Ganesh Vernekar * Cut v2.30.2 Signed-off-by: Ganesh Vernekar --- CHANGELOG.md | 4 ++++ VERSION | 2 +- tsdb/head_wal.go | 21 ++++++++++++++++----- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d65c6bd99a..a226f51882 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 2.30.2 / 2021-10-01 + +* [BUGFIX] TSDB: Don't error on overlapping m-mapped chunks during WAL replay. #9381 + ## 2.30.1 / 2021-09-28 * [ENHANCEMENT] Remote Write: Redact remote write URL when used for metric label. #9383 diff --git a/VERSION b/VERSION index bcec02eeb9..0958964f67 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.30.1 +2.30.2 diff --git a/tsdb/head_wal.go b/tsdb/head_wal.go index 6a647dd852..ba98c2e467 100644 --- a/tsdb/head_wal.go +++ b/tsdb/head_wal.go @@ -46,6 +46,8 @@ func (h *Head) loadWAL(r *wal.Reader, multiRef map[uint64]uint64, mmappedChunks // for error reporting. var unknownRefs atomic.Uint64 var unknownExemplarRefs atomic.Uint64 + // Track number of series records that had overlapping m-map chunks. + var mmapOverlappingChunks uint64 // Start workers that each process samples for a partition of the series ID space. // They are connected through a ring of channels which ensures that all sample batches @@ -241,8 +243,6 @@ Outer: } // Checking if the new m-mapped chunks overlap with the already existing ones. - // This should never happen, but we have a check anyway to detect any - // edge cases that we might have missed. if len(mSeries.mmappedChunks) > 0 && len(mmc) > 0 { if overlapsClosedInterval( mSeries.mmappedChunks[0].minTime, @@ -250,9 +250,17 @@ Outer: mmc[0].minTime, mmc[len(mmc)-1].maxTime, ) { - // The m-map chunks for the new series ref overlaps with old m-map chunks. - seriesCreationErr = errors.Errorf("overlapping m-mapped chunks for series %s", mSeries.lset.String()) - break Outer + mmapOverlappingChunks++ + level.Debug(h.logger).Log( + "msg", "M-mapped chunks overlap on a duplicate series record", + "series", mSeries.lset.String(), + "oldref", mSeries.ref, + "oldmint", mSeries.mmappedChunks[0].minTime, + "oldmaxt", mSeries.mmappedChunks[len(mSeries.mmappedChunks)-1].maxTime, + "newref", walSeries.Ref, + "newmint", mmc[0].minTime, + "newmaxt", mmc[len(mmc)-1].maxTime, + ) } } @@ -351,6 +359,9 @@ Outer: if unknownRefs.Load() > 0 || unknownExemplarRefs.Load() > 0 { level.Warn(h.logger).Log("msg", "Unknown series references", "samples", unknownRefs.Load(), "exemplars", unknownExemplarRefs.Load()) } + if mmapOverlappingChunks > 0 { + level.Info(h.logger).Log("msg", "Overlapping m-map chunks on duplicate series records", "count", mmapOverlappingChunks) + } return nil }