Additional targetScrapeSample metrics (#3018)

This commit is contained in:
Edward Marshall 2017-08-02 13:10:18 +01:00 committed by Brian Brazil
parent 2ed3a9bd62
commit c490725ac9

View file

@ -86,6 +86,24 @@ var (
Help: "Total number of scrapes that hit the sample limit and were rejected.", Help: "Total number of scrapes that hit the sample limit and were rejected.",
}, },
) )
targetScrapeSampleDuplicate = prometheus.NewCounter(
prometheus.CounterOpts{
Name: "prometheus_target_scrapes_sample_duplicate_timestamp_total",
Help: "Total number of samples rejected due to duplicate timestamps but different values",
},
)
targetScrapeSampleOutOfOrder = prometheus.NewCounter(
prometheus.CounterOpts{
Name: "prometheus_target_scrapes_sample_out_of_order_total",
Help: "Total number of samples rejected due to not being out of the expected order",
},
)
targetScrapeSampleOutOfBounds = prometheus.NewCounter(
prometheus.CounterOpts{
Name: "prometheus_target_scrapes_sample_out_of_bounds_total",
Help: "Total number of samples rejected due to timestamp falling outside of the time bounds",
},
)
) )
func init() { func init() {
@ -94,6 +112,9 @@ func init() {
prometheus.MustRegister(targetSyncIntervalLength) prometheus.MustRegister(targetSyncIntervalLength)
prometheus.MustRegister(targetScrapePoolSyncsCounter) prometheus.MustRegister(targetScrapePoolSyncsCounter)
prometheus.MustRegister(targetScrapeSampleLimit) prometheus.MustRegister(targetScrapeSampleLimit)
prometheus.MustRegister(targetScrapeSampleDuplicate)
prometheus.MustRegister(targetScrapeSampleOutOfOrder)
prometheus.MustRegister(targetScrapeSampleOutOfBounds)
} }
// scrapePool manages scrapes for sets of targets. // scrapePool manages scrapes for sets of targets.
@ -764,16 +785,19 @@ loop:
err = nil err = nil
continue continue
case storage.ErrOutOfOrderSample: case storage.ErrOutOfOrderSample:
sl.l.With("timeseries", string(met)).Debug("Out of order sample")
numOutOfOrder++ numOutOfOrder++
sl.l.With("timeseries", string(met)).Debug("Out of order sample")
targetScrapeSampleOutOfOrder.Inc()
continue continue
case storage.ErrDuplicateSampleForTimestamp: case storage.ErrDuplicateSampleForTimestamp:
numDuplicates++ numDuplicates++
sl.l.With("timeseries", string(met)).Debug("Duplicate sample for timestamp") sl.l.With("timeseries", string(met)).Debug("Duplicate sample for timestamp")
targetScrapeSampleDuplicate.Inc()
continue continue
case storage.ErrOutOfBounds: case storage.ErrOutOfBounds:
numOutOfBounds++ numOutOfBounds++
sl.l.With("timeseries", string(met)).Debug("Out of bounds metric") sl.l.With("timeseries", string(met)).Debug("Out of bounds metric")
targetScrapeSampleOutOfBounds.Inc()
continue continue
case errSampleLimit: case errSampleLimit:
// Keep on parsing output if we hit the limit, so we report the correct // Keep on parsing output if we hit the limit, so we report the correct
@ -810,18 +834,21 @@ loop:
continue continue
case storage.ErrOutOfOrderSample: case storage.ErrOutOfOrderSample:
err = nil err = nil
sl.l.With("timeseries", string(met)).Debug("Out of order sample")
numOutOfOrder++ numOutOfOrder++
sl.l.With("timeseries", string(met)).Debug("Out of order sample")
targetScrapeSampleOutOfOrder.Inc()
continue continue
case storage.ErrDuplicateSampleForTimestamp: case storage.ErrDuplicateSampleForTimestamp:
err = nil err = nil
numDuplicates++ numDuplicates++
sl.l.With("timeseries", string(met)).Debug("Duplicate sample for timestamp") sl.l.With("timeseries", string(met)).Debug("Duplicate sample for timestamp")
targetScrapeSampleDuplicate.Inc()
continue continue
case storage.ErrOutOfBounds: case storage.ErrOutOfBounds:
err = nil err = nil
numOutOfBounds++ numOutOfBounds++
sl.l.With("timeseries", string(met)).Debug("Out of bounds metric") sl.l.With("timeseries", string(met)).Debug("Out of bounds metric")
targetScrapeSampleOutOfBounds.Inc()
continue continue
case errSampleLimit: case errSampleLimit:
sampleLimitErr = err sampleLimitErr = err