From fd5c5a50a32084953940cb3161b4a5ecd811e5d6 Mon Sep 17 00:00:00 2001
From: Brian Brazil <brian.brazil@robustperception.io>
Date: Wed, 3 May 2017 16:51:45 +0100
Subject: [PATCH] Add stale markers on parse error.

If we fail to parse the result of a scrape,
we should treat that as a failed scrape and
add stale markers.
---
 retrieval/scrape.go      |  5 ++++
 retrieval/scrape_test.go | 52 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)

diff --git a/retrieval/scrape.go b/retrieval/scrape.go
index 49063eb25..50400297e 100644
--- a/retrieval/scrape.go
+++ b/retrieval/scrape.go
@@ -497,6 +497,11 @@ func (sl *scrapeLoop) run(interval, timeout time.Duration, errc chan<- error) {
 		// we still call sl.append to trigger stale markers.
 		if total, added, err = sl.append(b, start); err != nil {
 			log.With("err", err).Error("append failed")
+			// The append failed, probably due to a parse error.
+			// Call sl.append again with an empty scrape to trigger stale markers.
+			if _, _, err = sl.append([]byte{}, start); err != nil {
+				log.With("err", err).Error("failure append failed")
+			}
 		}
 
 		sl.report(start, time.Since(start), total, added, err)
diff --git a/retrieval/scrape_test.go b/retrieval/scrape_test.go
index e7d9faca0..4378b9f7b 100644
--- a/retrieval/scrape_test.go
+++ b/retrieval/scrape_test.go
@@ -486,6 +486,58 @@ func TestScrapeLoopRunCreatesStaleMarkersOnFailedScrape(t *testing.T) {
 	}
 }
 
+func TestScrapeLoopRunCreatesStaleMarkersOnParseFailure(t *testing.T) {
+	appender := &collectResultAppender{}
+	var (
+		signal = make(chan struct{})
+
+		scraper    = &testScraper{}
+		app        = func() storage.Appender { return appender }
+		reportApp  = func() storage.Appender { return &nopAppender{} }
+		numScrapes = 0
+	)
+	defer close(signal)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	sl := newScrapeLoop(ctx, scraper, app, reportApp)
+
+	// Succeed once, several failures, then stop.
+	scraper.scrapeFunc = func(ctx context.Context, w io.Writer) error {
+		numScrapes += 1
+		if numScrapes == 1 {
+			w.Write([]byte("metric_a 42\n"))
+			return nil
+		} else if numScrapes == 2 {
+			w.Write([]byte("7&-\n"))
+			return nil
+		} else if numScrapes == 3 {
+			cancel()
+		}
+		return fmt.Errorf("Scrape failed.")
+	}
+
+	go func() {
+		sl.run(10*time.Millisecond, time.Hour, nil)
+		signal <- struct{}{}
+	}()
+
+	select {
+	case <-signal:
+	case <-time.After(5 * time.Second):
+		t.Fatalf("Scrape wasn't stopped.")
+	}
+
+	if len(appender.result) != 2 {
+		t.Fatalf("Appended samples not as expected. Wanted: %d samples Got: %d", 2, len(appender.result))
+	}
+	if appender.result[0].v != 42.0 {
+		t.Fatalf("Appended first sample not as expected. Wanted: %f Got: %f", appender.result[0], 42)
+	}
+	if !value.IsStaleNaN(appender.result[1].v) {
+		t.Fatalf("Appended second sample not as expected. Wanted: stale NaN Got: %x", math.Float64bits(appender.result[1].v))
+	}
+}
+
 func TestScrapeLoopAppend(t *testing.T) {
 	app := &collectResultAppender{}
 	sl := &scrapeLoop{