diff --git a/retrieval/target.go b/retrieval/target.go index f45287dd2..bf9adb0c6 100644 --- a/retrieval/target.go +++ b/retrieval/target.go @@ -453,6 +453,7 @@ func (t *Target) scrape(appender storage.SampleAppender) (err error) { var ( samples model.Vector numOutOfOrder int + logger = log.With("target", t.InstanceIdentifier()) ) for { if err = sdec.Decode(&samples); err != nil { @@ -464,14 +465,14 @@ func (t *Target) scrape(appender storage.SampleAppender) (err error) { if err == local.ErrOutOfOrderSample { numOutOfOrder++ } else { - log.Warnf("Error inserting sample %v: %s", s, err) + logger.With("sample", s).Warnf("Error inserting sample: %s", err) } } } } if numOutOfOrder > 0 { - log.Warnf("Error on ingesting %d out-of-order samples") + logger.With("numDropped", numOutOfOrder).Warn("Error on ingesting out-of-order samples") } if err == io.EOF { diff --git a/storage/local/storage.go b/storage/local/storage.go index e7a4c292a..62453d89d 100644 --- a/storage/local/storage.go +++ b/storage/local/storage.go @@ -591,6 +591,7 @@ func (s *memorySeriesStorage) Append(sample *model.Sample) error { series := s.getOrCreateSeries(fp, sample.Metric) if sample.Timestamp <= series.lastTime { + s.fpLocker.Unlock(fp) // Don't log and track equal timestamps, as they are a common occurrence // when using client-side timestamps (e.g. Pushgateway or federation). // It would be even better to also compare the sample values here, but @@ -599,7 +600,6 @@ func (s *memorySeriesStorage) Append(sample *model.Sample) error { s.outOfOrderSamplesCount.Inc() return ErrOutOfOrderSample } - s.fpLocker.Unlock(fp) return nil } completedChunksCount := series.add(&model.SamplePair{ diff --git a/storage/storage.go b/storage/storage.go index 86730d643..5acae673e 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -25,8 +25,7 @@ type SampleAppender interface { // of the sample after Append has returned. Remote storage // implementation will simply drop samples if they cannot keep up with // sending samples. Local storage implementations will only drop metrics - // upon unrecoverable errors. Reporting any errors is done via metrics - // and logs and not the concern of the caller. + // upon unrecoverable errors. Append(*model.Sample) error // NeedsThrottling returns true if the underlying storage wishes to not // receive any more samples. Append will still work but might lead to