Don't fail scrape if one sample violates ordering.

In Prometheus 1.x one sample that is out of order
or that has a duplicate timestamp is discarded, and
the rest of the scrape ingestion continues on.
This will now also be true for 2.0.
This commit is contained in:
Brian Brazil 2017-05-03 17:20:07 +01:00
parent fd5c5a50a3
commit 3c45400130
2 changed files with 46 additions and 2 deletions

View file

@ -567,7 +567,7 @@ loop:
samplesScraped[sl.lsetCache[ref].str] = sl.lsetCache[ref].lset samplesScraped[sl.lsetCache[ref].str] = sl.lsetCache[ref].lset
case storage.ErrNotFound: case storage.ErrNotFound:
ok = false ok = false
case errSeriesDropped: case errSeriesDropped, storage.ErrOutOfOrderSample, storage.ErrDuplicateSampleForTimestamp:
continue continue
default: default:
break loop break loop
@ -581,7 +581,7 @@ loop:
// TODO(fabxc): also add a dropped-cache? // TODO(fabxc): also add a dropped-cache?
switch err { switch err {
case nil: case nil:
case errSeriesDropped: case errSeriesDropped, storage.ErrOutOfOrderSample, storage.ErrDuplicateSampleForTimestamp:
continue continue
default: default:
break loop break loop

View file

@ -653,6 +653,50 @@ func TestScrapeLoopAppendNoStalenessIfTimestamp(t *testing.T) {
} }
type errorAppender struct {
collectResultAppender
}
func (app *errorAppender) Add(lset labels.Labels, t int64, v float64) (uint64, error) {
if lset.Get(model.MetricNameLabel) == "out_of_order" {
return 0, storage.ErrOutOfOrderSample
} else if lset.Get(model.MetricNameLabel) == "amend" {
return 0, storage.ErrDuplicateSampleForTimestamp
}
return app.collectResultAppender.Add(lset, t, v)
}
func (app *errorAppender) AddFast(ref uint64, t int64, v float64) error {
return app.collectResultAppender.AddFast(ref, t, v)
}
func TestScrapeLoopAppendGracefullyIfAmendOrOutOfOrder(t *testing.T) {
app := &errorAppender{}
sl := &scrapeLoop{
appender: func() storage.Appender { return app },
reportAppender: func() storage.Appender { return nopAppender{} },
refCache: map[string]uint64{},
lsetCache: map[uint64]lsetCacheEntry{},
}
now := time.Unix(1, 0)
_, _, err := sl.append([]byte("out_of_order 1\namend 1\nnormal 1\n"), now)
if err != nil {
t.Fatalf("Unexpected append error: %s", err)
}
want := []sample{
{
metric: labels.FromStrings(model.MetricNameLabel, "normal"),
t: timestamp.FromTime(now),
v: 1,
},
}
if !reflect.DeepEqual(want, app.result) {
t.Fatalf("Appended samples not as expected. Wanted: %+v Got: %+v", want, app.result)
}
}
func TestTargetScraperScrapeOK(t *testing.T) { func TestTargetScraperScrapeOK(t *testing.T) {
const ( const (
configTimeout = 1500 * time.Millisecond configTimeout = 1500 * time.Millisecond