From 45e5775f9b6a3ec58823edc6fe83c5f7f54e6962 Mon Sep 17 00:00:00 2001
From: beorn7 <beorn@soundcloud.com>
Date: Thu, 19 May 2016 16:22:49 +0200
Subject: [PATCH] Add missing logging of out-of-order samples

So far, out-of-order samples during rule evaluation were not logged,
and neither scrape health samples. The latter are unlikely to cause
any errors. That's why I'm logging them always now. (It's alway highly
irregular should it happen.) For rules, I have used the same plumbing
as for samples, just with a different wording in the message to mark
them as a result of rule evaluation.
---
 retrieval/scrape.go |  8 ++++++--
 rules/manager.go    | 24 +++++++++++++++++++++++-
 2 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/retrieval/scrape.go b/retrieval/scrape.go
index e34482dd98..3df04b616b 100644
--- a/retrieval/scrape.go
+++ b/retrieval/scrape.go
@@ -496,6 +496,10 @@ func (sl *scrapeLoop) report(start time.Time, duration time.Duration, err error)
 		Value:     model.SampleValue(float64(duration) / float64(time.Second)),
 	}
 
-	sl.reportAppender.Append(healthSample)
-	sl.reportAppender.Append(durationSample)
+	if err := sl.reportAppender.Append(healthSample); err != nil {
+		log.With("sample", healthSample).With("error", err).Warn("Scrape health sample discarded")
+	}
+	if err := sl.reportAppender.Append(durationSample); err != nil {
+		log.With("sample", durationSample).With("error", err).Warn("Scrape duration sample discarded")
+	}
 }
diff --git a/rules/manager.go b/rules/manager.go
index d1736dd6fa..449e874eae 100644
--- a/rules/manager.go
+++ b/rules/manager.go
@@ -31,6 +31,7 @@ import (
 	"github.com/prometheus/prometheus/notifier"
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/storage"
+	"github.com/prometheus/prometheus/storage/local"
 	"github.com/prometheus/prometheus/template"
 	"github.com/prometheus/prometheus/util/strutil"
 )
@@ -270,8 +271,29 @@ func (g *Group) eval() {
 			if ar, ok := rule.(*AlertingRule); ok {
 				g.sendAlerts(ar, now)
 			}
+			var (
+				numOutOfOrder = 0
+				numDuplicates = 0
+			)
 			for _, s := range vector {
-				g.opts.SampleAppender.Append(s)
+				if err := g.opts.SampleAppender.Append(s); err != nil {
+					switch err {
+					case local.ErrOutOfOrderSample:
+						numOutOfOrder++
+						log.With("sample", s).With("error", err).Debug("Rule evaluation result discarded")
+					case local.ErrDuplicateSampleForTimestamp:
+						numDuplicates++
+						log.With("sample", s).With("error", err).Debug("Rule evaluation result discarded")
+					default:
+						log.With("sample", s).With("error", err).Warn("Rule evaluation result discarded")
+					}
+				}
+			}
+			if numOutOfOrder > 0 {
+				log.With("numDropped", numOutOfOrder).Warn("Error on ingesting out-of-order result from rule evaluation")
+			}
+			if numDuplicates > 0 {
+				log.With("numDropped", numDuplicates).Warn("Error on ingesting results from rule evaluation with different value but same timestamp")
 			}
 		}(rule)
 	}