mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2024-11-09 23:24:05 -08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

add evalDuration histogram and ruleCount counter for rules

Browse source 
Change-Id: I3508fe72526348d96b8158828388c3ac8d7c3fa9
This commit is contained in:
Stuart Nelson 2013-12-10 10:54:35 -05:00
parent 6947ee9bc9
commit b75adfebad
2 changed files with 33 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
rules/manager.go
View file

@ -14,6 +14,7 @@
package rules
import (
"fmt"
"sync"
"time"
@ -89,7 +90,7 @@ func (m *ruleManager) Run() {
case <-ticker.C:
start := time.Now()
m.runIteration(m.results)
evalDurations.Add(map[string]string{intervalKey: m.interval.String()}, float64(time.Since(start)/time.Millisecond))
iterationDuration.Add(map[string]string{intervalLabel: m.interval.String()}, float64(time.Since(start)/time.Millisecond))
case <-m.done:
glog.Info("RuleManager exiting...")
break
@ -146,7 +147,11 @@ func (m *ruleManager) runIteration(results chan<- *extraction.Result) {
// BUG(julius): Look at fixing thundering herd.
go func(rule Rule) {
defer wg.Done()
start := time.Now()
vector, err := rule.Eval(now, m.storage)
duration := time.Since(start)
samples := make(clientmodel.Samples, len(vector))
copy(samples, vector)
m.results <- &extraction.Result{
@ -154,8 +159,14 @@ func (m *ruleManager) runIteration(results chan<- *extraction.Result) {
Err: err,
}
if alertingRule, ok := rule.(*AlertingRule); ok {
m.queueAlertNotifications(alertingRule)
switch r := rule.(type) {
case *AlertingRule:
m.queueAlertNotifications(r)
recordOutcome(alertingRuleType, duration)
case *RecordingRule:
recordOutcome(recordingRuleType, duration)
default:
panic(fmt.Sprintf("Unknown rule type: %T", rule))
}
}(rule)
}

24
rules/telemetry.go
View file

@ -14,21 +14,35 @@
package rules
import (
"time"
"github.com/prometheus/client_golang/prometheus"
)
const (
intervalKey = "interval"
intervalLabel = "interval"
ruleTypeLabel = "rule_type"
alertingRuleType = "alerting"
recordingRuleType = "recording"
)
var (
evalDurations = prometheus.NewHistogram(&prometheus.HistogramSpecification{
evalDuration = prometheus.NewDefaultHistogram()
evalCount = prometheus.NewCounter()
iterationDuration = prometheus.NewHistogram(&prometheus.HistogramSpecification{
Starts: prometheus.LogarithmicSizedBucketsFor(0, 10000),
BucketBuilder: prometheus.AccumulatingBucketBuilder(prometheus.EvictAndReplaceWith(10, prometheus.AverageReducer), 100),
ReportablePercentiles: []float64{0.01, 0.05, 0.5, 0.90, 0.99}})
evalDuration = prometheus.NewCounter()
)
func init() {
prometheus.Register("prometheus_evaluator_duration_ms", "The duration for each evaluation pool to execute.", prometheus.NilLabels, evalDurations)
func recordOutcome(ruleType string, duration time.Duration) {
millisecondDuration := float64(duration / time.Millisecond)
evalCount.Increment(map[string]string{ruleTypeLabel: ruleType})
evalDuration.Add(map[string]string{ruleTypeLabel: ruleType}, millisecondDuration)
}
func init() {
prometheus.Register("prometheus_evaluator_duration_ms", "The duration for each evaluation pool to execute.", prometheus.NilLabels, iterationDuration)
prometheus.Register("prometheus_rule_evaluation_duration_ms", "The duration for a rule to execute.", prometheus.NilLabels, evalDuration)
prometheus.Register("prometheus_rule_evaluation_count", "The number of rules evaluated.", prometheus.NilLabels, evalCount)
}