From d65f037defb0dd70eef65dbfdb7d420cb01e4576 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Mierzwa?= Date: Fri, 17 Jun 2022 06:51:43 +0100 Subject: [PATCH] Don't increment prometheus_tsdb_compactions_failed_total when context is canceled (#10772) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When restarting Prometheus I sometimes see: caller=db.go:832 level=error component=tsdb msg="compaction failed" err="compact head: persist head block: 2 errors: populate block: context canceled; context canceled" And prometheus_tsdb_compactions_failed_total metric gets incremented. This makes it more difficult to write alerts based on prometheus_tsdb_compactions_failed_total metric since any restart can trigger it. Signed-off-by: Łukasz Mierzwa --- tsdb/db.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tsdb/db.go b/tsdb/db.go index ab0822d73..0678a6ea9 100644 --- a/tsdb/db.go +++ b/tsdb/db.go @@ -888,7 +888,9 @@ func (db *DB) Compact() (returnErr error) { db.cmtx.Lock() defer db.cmtx.Unlock() defer func() { - if returnErr != nil { + if returnErr != nil && !errors.Is(returnErr, context.Canceled) { + // If we got an error because context was canceled then we're most likely + // shutting down TSDB and we don't need to report this on metrics db.metrics.compactionsFailed.Inc() } }()