diff --git a/retrieval/scrape.go b/retrieval/scrape.go index d04d07313..7029dfbc5 100644 --- a/retrieval/scrape.go +++ b/retrieval/scrape.go @@ -609,6 +609,8 @@ func (sl *scrapeLoop) append(b []byte, ts time.Time) (total, added int, err erro p = textparse.New(b) defTime = timestamp.FromTime(ts) samplesScraped = map[string]labels.Labels{} + numOutOfOrder = 0 + numDuplicates = 0 ) loop: @@ -632,10 +634,12 @@ loop: case errSeriesDropped: continue case storage.ErrOutOfOrderSample: - log.With("timeseries", string(met)).Warn("Out of order sample") + log.With("timeseries", string(met)).Debug("Out of order sample") + numOutOfOrder += 1 continue case storage.ErrDuplicateSampleForTimestamp: - log.With("timeseries", string(met)).Warn("Duplicate sample for timestamp") + numDuplicates += 1 + log.With("timeseries", string(met)).Debug("Duplicate sample for timestamp") continue default: break loop @@ -654,11 +658,13 @@ loop: continue case storage.ErrOutOfOrderSample: err = nil - log.With("timeseries", string(met)).Warn("Out of order sample") + log.With("timeseries", string(met)).Debug("Out of order sample") + numOutOfOrder += 1 continue case storage.ErrDuplicateSampleForTimestamp: err = nil - log.With("timeseries", string(met)).Warn("Duplicate sample for timestamp") + numDuplicates += 1 + log.With("timeseries", string(met)).Debug("Duplicate sample for timestamp") continue default: break loop @@ -678,6 +684,12 @@ loop: if err == nil { err = p.Err() } + if numOutOfOrder > 0 { + log.With("numDropped", numOutOfOrder).Warn("Error on ingesting out-of-order samples") + } + if numDuplicates > 0 { + log.With("numDropped", numDuplicates).Warn("Error on ingesting samples with different value but same timestamp") + } if err == nil { for metric, lset := range sl.samplesInPreviousScrape { if _, ok := samplesScraped[metric]; !ok { @@ -689,8 +701,8 @@ loop: err = nil continue case storage.ErrOutOfOrderSample, storage.ErrDuplicateSampleForTimestamp: - // Do not log here, as this is expected if a target goes away and comes back - // again with a new scrape loop. + // Do not count these in logging, as this is expected if a target + // goes away and comes back again with a new scrape loop. err = nil continue default: