Add metric prometheus_rule_group_last_evaluation for recording and alerting (#4852)

* add metric prometheus_rule_group_last_evaluation for recording and alerting

Signed-off-by: Wei Guo <me@imkira.com>

* fix issues from comments

Signed-off-by: Wei Guo <me@imkira.com>
This commit is contained in:
Wei Guo 2018-11-27 14:38:13 +08:00 committed by Julius Volz
parent d2f0f54d68
commit e329cbf673

View file

@ -90,6 +90,12 @@ var (
Name: "rule_group_iterations_total", Name: "rule_group_iterations_total",
Help: "The total number of scheduled rule group evaluations, whether executed or missed.", Help: "The total number of scheduled rule group evaluations, whether executed or missed.",
}) })
lastEvaluation = prometheus.NewDesc(
prometheus.BuildFQName(namespace, "", "rule_group_last_evaluation_timestamp_seconds"),
"The timestamp of the last rule group evaluation in seconds.",
[]string{"rule_group"},
nil,
)
lastDuration = prometheus.NewDesc( lastDuration = prometheus.NewDesc(
prometheus.BuildFQName(namespace, "", "rule_group_last_duration_seconds"), prometheus.BuildFQName(namespace, "", "rule_group_last_duration_seconds"),
"The duration of the last rule group evaluation.", "The duration of the last rule group evaluation.",
@ -818,6 +824,7 @@ func (m *Manager) AlertingRules() []*AlertingRule {
// Describe implements prometheus.Collector. // Describe implements prometheus.Collector.
func (m *Manager) Describe(ch chan<- *prometheus.Desc) { func (m *Manager) Describe(ch chan<- *prometheus.Desc) {
ch <- lastEvaluation
ch <- lastDuration ch <- lastDuration
ch <- groupInterval ch <- groupInterval
} }
@ -825,10 +832,20 @@ func (m *Manager) Describe(ch chan<- *prometheus.Desc) {
// Collect implements prometheus.Collector. // Collect implements prometheus.Collector.
func (m *Manager) Collect(ch chan<- prometheus.Metric) { func (m *Manager) Collect(ch chan<- prometheus.Metric) {
for _, g := range m.RuleGroups() { for _, g := range m.RuleGroups() {
lastEvaluationTime := g.GetEvaluationTimestamp()
lastEvaluationTimestamp := math.Inf(-1)
if !lastEvaluationTime.IsZero() {
lastEvaluationTimestamp = float64(lastEvaluationTime.UnixNano()) / 1e9
}
key := groupKey(g.file, g.name)
ch <- prometheus.MustNewConstMetric(lastEvaluation,
prometheus.GaugeValue,
lastEvaluationTimestamp,
key)
ch <- prometheus.MustNewConstMetric(lastDuration, ch <- prometheus.MustNewConstMetric(lastDuration,
prometheus.GaugeValue, prometheus.GaugeValue,
g.GetEvaluationDuration().Seconds(), g.GetEvaluationDuration().Seconds(),
groupKey(g.file, g.name)) key)
} }
for _, g := range m.RuleGroups() { for _, g := range m.RuleGroups() {
ch <- prometheus.MustNewConstMetric(groupInterval, ch <- prometheus.MustNewConstMetric(groupInterval,