mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2025-01-12 06:17:27 -08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Handle scrapes with OutOfBounds metrics better

Browse source 
fixes #2894

Signed-off-by: Goutham Veeramachaneni <goutham@boomerangcommerce.com>
This commit is contained in:
Goutham Veeramachaneni 2017-07-04 11:24:13 +02:00
parent 426125298e
commit 3069bd3996
2 changed files with 19 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
retrieval/scrape.go
View file

@ -722,11 +722,12 @@ func (s samples) Less(i, j int) bool {
func (sl *scrapeLoop) append(b []byte, ts time.Time) (total, added int, err error) { func (sl *scrapeLoop) append(b []byte, ts time.Time) (total, added int, err error) {
var ( var (
app = sl.appender() app = sl.appender()
p = textparse.New(b) p = textparse.New(b)
defTime = timestamp.FromTime(ts) defTime = timestamp.FromTime(ts)
numOutOfOrder = 0 numOutOfOrder = 0
numDuplicates = 0 numDuplicates = 0
numOutOfBounds = 0
) )
var sampleLimitErr error var sampleLimitErr error
@ -761,6 +762,10 @@ loop:
numDuplicates++ numDuplicates++
sl.l.With("timeseries", string(met)).Debug("Duplicate sample for timestamp") sl.l.With("timeseries", string(met)).Debug("Duplicate sample for timestamp")
continue continue
case storage.ErrOutOfBounds:
numOutOfBounds++
sl.l.With("timeseries", string(met)).Debug("Out of bounds metric")
continue
case errSampleLimit: case errSampleLimit:
// Keep on parsing output if we hit the limit, so we report the correct // Keep on parsing output if we hit the limit, so we report the correct
// total number of samples scraped. // total number of samples scraped.
@ -804,6 +809,10 @@ loop:
numDuplicates++ numDuplicates++
sl.l.With("timeseries", string(met)).Debug("Duplicate sample for timestamp") sl.l.With("timeseries", string(met)).Debug("Duplicate sample for timestamp")
continue continue
case storage.ErrOutOfBounds:
numOutOfBounds++
sl.l.With("timeseries", string(met)).Debug("Out of bounds metric")
continue
case errSampleLimit: case errSampleLimit:
sampleLimitErr = err sampleLimitErr = err
added++ added++
@ -832,6 +841,9 @@ loop:
if numDuplicates > 0 { if numDuplicates > 0 {
sl.l.With("numDropped", numDuplicates).Warn("Error on ingesting samples with different value but same timestamp") sl.l.With("numDropped", numDuplicates).Warn("Error on ingesting samples with different value but same timestamp")
} }
if numOutOfBounds > 0 {
sl.l.With("numOutOfBounds", numOutOfBounds).Warn("Error on ingesting samples that are too old")
}
if err == nil { if err == nil {
sl.cache.forEachStale(func(lset labels.Labels) bool { sl.cache.forEachStale(func(lset labels.Labels) bool {
// Series no longer exposed, mark it stale. // Series no longer exposed, mark it stale.

2
storage/interface.go
View file

@ -19,10 +19,12 @@ import (
"github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/labels"
) )
// The errors exposed.
var ( var (
ErrNotFound = errors.New("not found") ErrNotFound = errors.New("not found")
ErrOutOfOrderSample = errors.New("out of order sample") ErrOutOfOrderSample = errors.New("out of order sample")
ErrDuplicateSampleForTimestamp = errors.New("duplicate sample for timestamp") ErrDuplicateSampleForTimestamp = errors.New("duplicate sample for timestamp")
ErrOutOfBounds = errors.New("out of bounds")
) )
// Storage ingests and manages samples, along with various indexes. All methods // Storage ingests and manages samples, along with various indexes. All methods