Merge pull request #440 from prometheus/report-rule-failures

Report rule eval failures, remove extraction.Result type

main.go

@@ -23,7 +23,6 @@ import (
 	"time"
 
 	"github.com/golang/glog"
-	"github.com/prometheus/client_golang/extraction"
 
 	clientmodel "github.com/prometheus/client_golang/model"
 	registry "github.com/prometheus/client_golang/prometheus"
@@ -81,7 +80,7 @@ var (
 )
 
 type prometheus struct {
-	unwrittenSamples chan *extraction.Result
+	unwrittenSamples chan clientmodel.Samples
 
 	ruleManager         manager.RuleManager
 	targetManager       retrieval.TargetManager
@@ -102,7 +101,7 @@ func NewPrometheus() *prometheus {
 		glog.Fatalf("Error loading configuration from %s: %v", *configFile, err)
 	}
 
-	unwrittenSamples := make(chan *extraction.Result, *samplesQueueCapacity)
+	unwrittenSamples := make(chan clientmodel.Samples, *samplesQueueCapacity)
 
 	ingester := &retrieval.MergeLabelsIngester{
 		Labels:          conf.GlobalLabels(),
@@ -214,11 +213,11 @@ func (p *prometheus) Serve() {
 		}
 	}()
 
-	for block := range p.unwrittenSamples {
-		if block.Err == nil && len(block.Samples) > 0 {
-			p.storage.AppendSamples(block.Samples)
+	for samples := range p.unwrittenSamples {
+		if len(samples) > 0 {
+			p.storage.AppendSamples(samples)
 			if p.remoteTSDBQueue != nil {
-				p.remoteTSDBQueue.Queue(block.Samples)
+				p.remoteTSDBQueue.Queue(samples)
 			}
 		}
 	}

retrieval/helpers_test.go

@@ -14,11 +14,11 @@
 package retrieval
 
 import (
-	"github.com/prometheus/client_golang/extraction"
+	clientmodel "github.com/prometheus/client_golang/model"
 )
 
 type nopIngester struct{}
 
-func (i nopIngester) Ingest(*extraction.Result) error {
+func (i nopIngester) Ingest(clientmodel.Samples) error {
 	return nil
 }

retrieval/ingester.go

@@ -31,19 +31,19 @@ type MergeLabelsIngester struct {
 
 // Ingest ingests the provided extraction result by merging in i.Labels and then
 // handing it over to i.Ingester.
-func (i *MergeLabelsIngester) Ingest(r *extraction.Result) error {
-	for _, s := range r.Samples {
+func (i *MergeLabelsIngester) Ingest(samples clientmodel.Samples) error {
+	for _, s := range samples {
 		s.Metric.MergeFromLabelSet(i.Labels, i.CollisionPrefix)
 	}
 
-	return i.Ingester.Ingest(r)
+	return i.Ingester.Ingest(samples)
 }
 
 // ChannelIngester feeds results into a channel without modifying them.
-type ChannelIngester chan<- *extraction.Result
+type ChannelIngester chan<- clientmodel.Samples
 
 // Ingest ingests the provided extraction result by sending it to i.
-func (i ChannelIngester) Ingest(r *extraction.Result) error {
-	i <- r
+func (i ChannelIngester) Ingest(s clientmodel.Samples) error {
+	i <- s
 	return nil
 }

retrieval/target.go

@@ -205,10 +205,7 @@ func (t *target) recordScrapeHealth(ingester extraction.Ingester, timestamp clie
 		Value:     clientmodel.SampleValue(float64(scrapeDuration) / float64(time.Second)),
 	}
 
-	ingester.Ingest(&extraction.Result{
-		Err:     nil,
-		Samples: clientmodel.Samples{healthSample, durationSample},
-	})
+	ingester.Ingest(clientmodel.Samples{healthSample, durationSample})
 }
 
 // RunScraper implements Target.

retrieval/target_test.go

@@ -22,17 +22,15 @@ import (
 
 	clientmodel "github.com/prometheus/client_golang/model"
 
-	"github.com/prometheus/client_golang/extraction"
-
 	"github.com/prometheus/prometheus/utility"
 )
 
 type collectResultIngester struct {
-	result *extraction.Result
+	result clientmodel.Samples
 }
 
-func (i *collectResultIngester) Ingest(r *extraction.Result) error {
-	i.result = r
+func (i *collectResultIngester) Ingest(s clientmodel.Samples) error {
+	i.result = s
 	return nil
 }
 
@@ -57,15 +55,15 @@ func TestTargetRecordScrapeHealth(t *testing.T) {
 
 	now := clientmodel.Now()
 	ingester := &collectResultIngester{}
-	testTarget.recordScrapeHealth(ingester, now, true, 2 * time.Second)
+	testTarget.recordScrapeHealth(ingester, now, true, 2*time.Second)
 
 	result := ingester.result
 
-	if len(result.Samples) != 2 {
-		t.Fatalf("Expected two samples, got %d", len(result.Samples))
+	if len(result) != 2 {
+		t.Fatalf("Expected two samples, got %d", len(result))
 	}
 
-	actual := result.Samples[0]
+	actual := result[0]
 	expected := &clientmodel.Sample{
 		Metric: clientmodel.Metric{
 			clientmodel.MetricNameLabel: scrapeHealthMetricName,
@@ -76,15 +74,11 @@ func TestTargetRecordScrapeHealth(t *testing.T) {
 		Value:     1,
 	}
 
-	if result.Err != nil {
-		t.Fatalf("Got unexpected error: %v", result.Err)
-	}
-
 	if !actual.Equal(expected) {
 		t.Fatalf("Expected and actual samples not equal. Expected: %v, actual: %v", expected, actual)
 	}
 
-	actual = result.Samples[1]
+	actual = result[1]
 	expected = &clientmodel.Sample{
 		Metric: clientmodel.Metric{
 			clientmodel.MetricNameLabel: scrapeDurationMetricName,
@@ -95,10 +89,6 @@ func TestTargetRecordScrapeHealth(t *testing.T) {
 		Value:     2.0,
 	}
 
-	if result.Err != nil {
-		t.Fatalf("Got unexpected error: %v", result.Err)
-	}
-
 	if !actual.Equal(expected) {
 		t.Fatalf("Expected and actual samples not equal. Expected: %v, actual: %v", expected, actual)
 	}

rules/manager/manager.go

@@ -19,7 +19,6 @@ import (
 	"time"
 
 	"github.com/golang/glog"
-	"github.com/prometheus/client_golang/extraction"
 	"github.com/prometheus/client_golang/prometheus"
 
 	clientmodel "github.com/prometheus/client_golang/model"
@@ -49,6 +48,13 @@ var (
 		},
 		[]string{ruleTypeLabel},
 	)
+	evalFailures = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: namespace,
+			Name:      "rule_evaluation_failures_total",
+			Help:      "The total number of rule evaluation failures.",
+		},
+	)
 	iterationDuration = prometheus.NewSummary(prometheus.SummaryOpts{
 		Namespace:  namespace,
 		Name:       "evaluator_duration_milliseconds",
@@ -59,6 +65,7 @@ var (
 
 func init() {
 	prometheus.MustRegister(iterationDuration)
+	prometheus.MustRegister(evalFailures)
 	prometheus.MustRegister(evalDuration)
 }
 
@@ -87,7 +94,7 @@ type ruleManager struct {
 	interval time.Duration
 	storage  local.Storage
 
-	results             chan<- *extraction.Result
+	results             chan<- clientmodel.Samples
 	notificationHandler *notification.NotificationHandler
 
 	prometheusURL string
@@ -99,7 +106,7 @@ type RuleManagerOptions struct {
 	Storage            local.Storage
 
 	NotificationHandler *notification.NotificationHandler
-	Results             chan<- *extraction.Result
+	Results             chan<- clientmodel.Samples
 
 	PrometheusURL string
 }
@@ -202,7 +209,7 @@ func (m *ruleManager) queueAlertNotifications(rule *rules.AlertingRule, timestam
 	m.notificationHandler.SubmitReqs(notifications)
 }
 
-func (m *ruleManager) runIteration(results chan<- *extraction.Result) {
+func (m *ruleManager) runIteration(results chan<- clientmodel.Samples) {
 	now := clientmodel.Now()
 	wg := sync.WaitGroup{}
 
@@ -229,9 +236,12 @@ func (m *ruleManager) runIteration(results chan<- *extraction.Result) {
 					Timestamp: s.Timestamp,
 				}
 			}
-			m.results <- &extraction.Result{
-				Samples: samples,
-				Err:     err,
+
+			if err != nil {
+				evalFailures.Inc()
+				glog.Warningf("Error while evaluating rule %q: %s", rule, err)
+			} else {
+				m.results <- samples
 			}
 
 			switch r := rule.(type) {