From e329cbf673e0427d9755972f0062ad02dfc17f17 Mon Sep 17 00:00:00 2001 From: Wei Guo Date: Tue, 27 Nov 2018 14:38:13 +0800 Subject: [PATCH] Add metric prometheus_rule_group_last_evaluation for recording and alerting (#4852) * add metric prometheus_rule_group_last_evaluation for recording and alerting Signed-off-by: Wei Guo * fix issues from comments Signed-off-by: Wei Guo --- rules/manager.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/rules/manager.go b/rules/manager.go index df6e5af1e..d654b6d57 100644 --- a/rules/manager.go +++ b/rules/manager.go @@ -90,6 +90,12 @@ var ( Name: "rule_group_iterations_total", Help: "The total number of scheduled rule group evaluations, whether executed or missed.", }) + lastEvaluation = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "rule_group_last_evaluation_timestamp_seconds"), + "The timestamp of the last rule group evaluation in seconds.", + []string{"rule_group"}, + nil, + ) lastDuration = prometheus.NewDesc( prometheus.BuildFQName(namespace, "", "rule_group_last_duration_seconds"), "The duration of the last rule group evaluation.", @@ -818,6 +824,7 @@ func (m *Manager) AlertingRules() []*AlertingRule { // Describe implements prometheus.Collector. func (m *Manager) Describe(ch chan<- *prometheus.Desc) { + ch <- lastEvaluation ch <- lastDuration ch <- groupInterval } @@ -825,10 +832,20 @@ func (m *Manager) Describe(ch chan<- *prometheus.Desc) { // Collect implements prometheus.Collector. func (m *Manager) Collect(ch chan<- prometheus.Metric) { for _, g := range m.RuleGroups() { + lastEvaluationTime := g.GetEvaluationTimestamp() + lastEvaluationTimestamp := math.Inf(-1) + if !lastEvaluationTime.IsZero() { + lastEvaluationTimestamp = float64(lastEvaluationTime.UnixNano()) / 1e9 + } + key := groupKey(g.file, g.name) + ch <- prometheus.MustNewConstMetric(lastEvaluation, + prometheus.GaugeValue, + lastEvaluationTimestamp, + key) ch <- prometheus.MustNewConstMetric(lastDuration, prometheus.GaugeValue, g.GetEvaluationDuration().Seconds(), - groupKey(g.file, g.name)) + key) } for _, g := range m.RuleGroups() { ch <- prometheus.MustNewConstMetric(groupInterval,