mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2024-12-26 06:04:05 -08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

queue_manager: add histogram info to error logs (#14326)

Browse source 
Signed-off-by: Piotr Gwizdala <17101802+thampiotr@users.noreply.github.com>
This commit is contained in:
Piotr 2024-06-21 00:45:13 +01:00 committed by GitHub
parent dbd29df5df
commit d78253319d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 11 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
storage/remote/queue_manager.go
View file

@ -16,6 +16,7 @@ package remote
import (
"context"
"errors"
"fmt"
"math"
"strconv"
"sync"
@ -1224,12 +1225,16 @@ func (s *shards) stop() {
// Force an unclean shutdown.
s.hardShutdown()
<-s.done
if dropped := s.samplesDroppedOnHardShutdown.Load(); dropped > 0 {
level.Error(s.qm.logger).Log("msg", "Failed to flush all samples on shutdown", "count", dropped)
// Log error for any dropped samples, exemplars, or histograms.
logDroppedError := func(t string, counter atomic.Uint32) {
if dropped := counter.Load(); dropped > 0 {
level.Error(s.qm.logger).Log("msg", fmt.Sprintf("Failed to flush all %s on shutdown", t), "count", dropped)
}
if dropped := s.exemplarsDroppedOnHardShutdown.Load(); dropped > 0 {
level.Error(s.qm.logger).Log("msg", "Failed to flush all exemplars on shutdown", "count", dropped)
}
logDroppedError("samples", s.samplesDroppedOnHardShutdown)
logDroppedError("exemplars", s.exemplarsDroppedOnHardShutdown)
logDroppedError("histograms", s.histogramsDroppedOnHardShutdown)
}
// enqueue data (sample or exemplar). If the shard is full, shutting down, or
@ -1537,7 +1542,7 @@ func (s *shards) sendSamples(ctx context.Context, samples []prompb.TimeSeries, s
begin := time.Now()
err := s.sendSamplesWithBackoff(ctx, samples, sampleCount, exemplarCount, histogramCount, pBuf, buf)
if err != nil {
level.Error(s.qm.logger).Log("msg", "non-recoverable error", "count", sampleCount, "exemplarCount", exemplarCount, "err", err)
level.Error(s.qm.logger).Log("msg", "non-recoverable error", "count", sampleCount, "exemplarCount", exemplarCount, "histogramCount", histogramCount, "err", err)
s.qm.metrics.failedSamplesTotal.Add(float64(sampleCount))
s.qm.metrics.failedExemplarsTotal.Add(float64(exemplarCount))
s.qm.metrics.failedHistogramsTotal.Add(float64(histogramCount))