Handle scrapes with OutOfBounds metrics better

fixes #2894

Signed-off-by: Goutham Veeramachaneni <goutham@boomerangcommerce.com>

retrieval/scrape.go

@@ -722,11 +722,12 @@ func (s samples) Less(i, j int) bool {
 
 func (sl *scrapeLoop) append(b []byte, ts time.Time) (total, added int, err error) {
 	var (
-		app           = sl.appender()
-		p             = textparse.New(b)
-		defTime       = timestamp.FromTime(ts)
-		numOutOfOrder = 0
-		numDuplicates = 0
+		app            = sl.appender()
+		p              = textparse.New(b)
+		defTime        = timestamp.FromTime(ts)
+		numOutOfOrder  = 0
+		numDuplicates  = 0
+		numOutOfBounds = 0
 	)
 	var sampleLimitErr error
 
@@ -761,6 +762,10 @@ loop:
 				numDuplicates++
 				sl.l.With("timeseries", string(met)).Debug("Duplicate sample for timestamp")
 				continue
+			case storage.ErrOutOfBounds:
+				numOutOfBounds++
+				sl.l.With("timeseries", string(met)).Debug("Out of bounds metric")
+				continue
 			case errSampleLimit:
 				// Keep on parsing output if we hit the limit, so we report the correct
 				// total number of samples scraped.
@@ -804,6 +809,10 @@ loop:
 				numDuplicates++
 				sl.l.With("timeseries", string(met)).Debug("Duplicate sample for timestamp")
 				continue
+			case storage.ErrOutOfBounds:
+				numOutOfBounds++
+				sl.l.With("timeseries", string(met)).Debug("Out of bounds metric")
+				continue
 			case errSampleLimit:
 				sampleLimitErr = err
 				added++
@@ -832,6 +841,9 @@ loop:
 	if numDuplicates > 0 {
 		sl.l.With("numDropped", numDuplicates).Warn("Error on ingesting samples with different value but same timestamp")
 	}
+	if numOutOfBounds > 0 {
+		sl.l.With("numOutOfBounds", numOutOfBounds).Warn("Error on ingesting samples that are too old")
+	}
 	if err == nil {
 		sl.cache.forEachStale(func(lset labels.Labels) bool {
 			// Series no longer exposed, mark it stale.

storage/interface.go

@@ -19,10 +19,12 @@ import (
 	"github.com/prometheus/prometheus/pkg/labels"
 )
 
+// The errors exposed.
 var (
 	ErrNotFound                    = errors.New("not found")
 	ErrOutOfOrderSample            = errors.New("out of order sample")
 	ErrDuplicateSampleForTimestamp = errors.New("duplicate sample for timestamp")
+	ErrOutOfBounds                 = errors.New("out of bounds")
 )
 
 // Storage ingests and manages samples, along with various indexes. All methods