From e4fabe135abb99b7ae085e937776611bd07de2ad Mon Sep 17 00:00:00 2001 From: Fabian Reinartz Date: Mon, 14 Dec 2015 12:00:44 +0100 Subject: [PATCH 1/6] Set StartsAt to time of first firing state --- rules/manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules/manager.go b/rules/manager.go index d226f6b69..0a801c257 100644 --- a/rules/manager.go +++ b/rules/manager.go @@ -227,7 +227,7 @@ func (m *Manager) sendAlertNotifications(rule *AlertingRule, timestamp model.Tim } alerts = append(alerts, &model.Alert{ - StartsAt: aa.ActiveSince.Time(), + StartsAt: aa.ActiveSince.Time().Add(rule.holdDuration), Labels: labels, Annotations: annotations, GeneratorURL: m.externalURL.String() + strutil.GraphLinkForExpression(rule.vector.String()), From 52e5224f5a4904da2dcd2c0c1658ec2de2d93893 Mon Sep 17 00:00:00 2001 From: Fabian Reinartz Date: Mon, 14 Dec 2015 17:40:40 +0100 Subject: [PATCH 2/6] Refactor rules/ package --- cmd/prometheus/main.go | 2 +- notification/notification.go | 3 + rules/alerting.go | 214 +++++++++-------- rules/manager.go | 451 ++++++++++++++++++++--------------- rules/manager_test.go | 45 +--- rules/recording.go | 4 +- web/ui/bindata.go | 2 +- 7 files changed, 375 insertions(+), 346 deletions(-) diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index deb5e1030..c3a1c1475 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -155,7 +155,7 @@ func Main() int { prometheus.MustRegister(configSuccess) prometheus.MustRegister(configSuccessTime) - go ruleManager.Run() + // go ruleManager.Run() defer ruleManager.Stop() go notificationHandler.Run() diff --git a/notification/notification.go b/notification/notification.go index b446c4fe7..00c5f7f2a 100644 --- a/notification/notification.go +++ b/notification/notification.go @@ -168,6 +168,9 @@ func (n *Handler) Run() { alerts := n.nextBatch() + if len(alerts) == 0 { + continue + } if n.opts.AlertmanagerURL == "" { log.Warn("No AlertManager configured, not dispatching %d alerts", len(alerts)) n.dropped.Add(float64(len(alerts))) diff --git a/rules/alerting.go b/rules/alerting.go index 12a21ed76..785cb216f 100644 --- a/rules/alerting.go +++ b/rules/alerting.go @@ -46,52 +46,21 @@ func (s AlertState) String() string { return "pending" case StateFiring: return "firing" - default: - panic("undefined") } + panic(fmt.Errorf("unknown alert state: %v", s)) } const ( - // StateInactive is the state of an alert that is either firing nor pending. StateInactive AlertState = iota - // StatePending is the state of an alert that has been active for less than - // the configured threshold duration. StatePending - // StateFiring is the state of an alert that has been active for longer than - // the configured threshold duration. StateFiring ) -// Alert is used to track active (pending/firing) alerts over time. -type Alert struct { - // The name of the alert. - Name string - // The vector element labelset triggering this alert. - Labels model.LabelSet - // The state of the alert (Pending or Firing). - State AlertState - // The time when the alert first transitioned into Pending state. - ActiveSince model.Time - // The value of the alert expression for this vector element. - Value model.SampleValue -} - -// sample returns a Sample suitable for recording the alert. -func (a Alert) sample(timestamp model.Time, value model.SampleValue) *model.Sample { - recordedMetric := make(model.Metric, len(a.Labels)+3) - for label, value := range a.Labels { - recordedMetric[label] = value - } - - recordedMetric[model.MetricNameLabel] = alertMetricName - recordedMetric[alertNameLabel] = model.LabelValue(a.Name) - recordedMetric[alertStateLabel] = model.LabelValue(a.State.String()) - - return &model.Sample{ - Metric: recordedMetric, - Value: value, - Timestamp: timestamp, - } +type alertInstance struct { + metric model.Metric + value model.SampleValue + state AlertState + activeSince model.Time } // An AlertingRule generates alerts from its vector expression. @@ -109,10 +78,10 @@ type AlertingRule struct { annotations model.LabelSet // Protects the below. - mutex sync.Mutex + mtx sync.Mutex // A map of alerts which are currently active (Pending or Firing), keyed by // the fingerprint of the labelset they correspond to. - activeAlerts map[model.Fingerprint]*Alert + active map[model.Fingerprint]*alertInstance } // NewAlertingRule constructs a new AlertingRule. @@ -123,8 +92,7 @@ func NewAlertingRule(name string, vec promql.Expr, hold time.Duration, lbls, ann holdDuration: hold, labels: lbls, annotations: anns, - - activeAlerts: map[model.Fingerprint]*Alert{}, + active: map[model.Fingerprint]*alertInstance{}, } } @@ -133,65 +101,127 @@ func (rule *AlertingRule) Name() string { return rule.name } +func (r *AlertingRule) sample(ai *alertInstance, ts model.Time, set bool) *model.Sample { + // Build alert labels in order they can be overwritten. + metric := model.Metric(r.labels.Clone()) + + for ln, lv := range ai.metric { + metric[ln] = lv + } + + metric[model.MetricNameLabel] = alertMetricName + metric[model.AlertNameLabel] = model.LabelValue(r.name) + metric[alertStateLabel] = model.LabelValue(ai.state.String()) + + s := &model.Sample{ + Metric: metric, + Timestamp: ts, + Value: 0, + } + if set { + s.Value = 1 + } + return s +} + // eval evaluates the rule expression and then creates pending alerts and fires // or removes previously pending alerts accordingly. -func (rule *AlertingRule) eval(timestamp model.Time, engine *promql.Engine) (model.Vector, error) { - query, err := engine.NewInstantQuery(rule.vector.String(), timestamp) +func (r *AlertingRule) eval(ts model.Time, engine *promql.Engine) (model.Vector, error) { + query, err := engine.NewInstantQuery(r.vector.String(), ts) if err != nil { return nil, err } - exprResult, err := query.Exec().Vector() + res, err := query.Exec().Vector() if err != nil { return nil, err } - rule.mutex.Lock() - defer rule.mutex.Unlock() + r.mtx.Lock() + defer r.mtx.Unlock() // Create pending alerts for any new vector elements in the alert expression // or update the expression value for existing elements. resultFPs := map[model.Fingerprint]struct{}{} - for _, sample := range exprResult { - fp := sample.Metric.Fingerprint() + + for _, smpl := range res { + fp := smpl.Metric.Fingerprint() resultFPs[fp] = struct{}{} - if alert, ok := rule.activeAlerts[fp]; !ok { - labels := model.LabelSet(sample.Metric.Clone()) - labels = labels.Merge(rule.labels) - if _, ok := labels[model.MetricNameLabel]; ok { - delete(labels, model.MetricNameLabel) - } - rule.activeAlerts[fp] = &Alert{ - Name: rule.name, - Labels: labels, - State: StatePending, - ActiveSince: timestamp, - Value: sample.Value, - } - } else { - alert.Value = sample.Value - } - } - - var vector model.Vector - - // Check if any pending alerts should be removed or fire now. Write out alert timeseries. - for fp, activeAlert := range rule.activeAlerts { - if _, ok := resultFPs[fp]; !ok { - vector = append(vector, activeAlert.sample(timestamp, 0)) - delete(rule.activeAlerts, fp) + if ai, ok := r.active[fp]; ok { + ai.value = smpl.Value continue } - if activeAlert.State == StatePending && timestamp.Sub(activeAlert.ActiveSince) >= rule.holdDuration { - vector = append(vector, activeAlert.sample(timestamp, 0)) - activeAlert.State = StateFiring - } + delete(smpl.Metric, model.MetricNameLabel) - vector = append(vector, activeAlert.sample(timestamp, 1)) + r.active[fp] = &alertInstance{ + metric: smpl.Metric, + activeSince: ts, + state: StatePending, + value: smpl.Value, + } } - return vector, nil + var vec model.Vector + // Check if any pending alerts should be removed or fire now. Write out alert timeseries. + for fp, ai := range r.active { + if _, ok := resultFPs[fp]; !ok { + delete(r.active, fp) + vec = append(vec, r.sample(ai, ts, false)) + continue + } + + if ai.state != StateFiring && ts.Sub(ai.activeSince) >= r.holdDuration { + vec = append(vec, r.sample(ai, ts, false)) + ai.state = StateFiring + } + + vec = append(vec, r.sample(ai, ts, true)) + } + + return vec, nil +} + +// Alert is the user-level representation of a single instance of an alerting rule. +type Alert struct { + State AlertState + Labels model.LabelSet + ActiveSince model.Time + Value model.SampleValue +} + +func (r *AlertingRule) State() AlertState { + r.mtx.Lock() + defer r.mtx.Unlock() + + maxState := StateInactive + for _, ai := range r.active { + if ai.state > maxState { + maxState = ai.state + } + } + return maxState +} + +// ActiveAlerts returns a slice of active alerts. +func (r *AlertingRule) ActiveAlerts() []*Alert { + r.mtx.Lock() + defer r.mtx.Unlock() + + alerts := make([]*Alert, 0, len(r.active)) + for _, ai := range r.active { + labels := r.labels.Clone() + for ln, lv := range ai.metric { + labels[ln] = lv + } + alerts = append(alerts, &Alert{ + State: ai.state, + Labels: labels, + ActiveSince: ai.activeSince, + Value: ai.value, + }) + } + return alerts } func (rule *AlertingRule) String() string { @@ -230,29 +260,3 @@ func (rule *AlertingRule) HTMLSnippet(pathPrefix string) template.HTML { } return template.HTML(s) } - -// State returns the "maximum" state: firing > pending > inactive. -func (rule *AlertingRule) State() AlertState { - rule.mutex.Lock() - defer rule.mutex.Unlock() - - maxState := StateInactive - for _, activeAlert := range rule.activeAlerts { - if activeAlert.State > maxState { - maxState = activeAlert.State - } - } - return maxState -} - -// ActiveAlerts returns a slice of active alerts. -func (rule *AlertingRule) ActiveAlerts() []Alert { - rule.mutex.Lock() - defer rule.mutex.Unlock() - - alerts := make([]Alert, 0, len(rule.activeAlerts)) - for _, alert := range rule.activeAlerts { - alerts = append(alerts, *alert) - } - return alerts -} diff --git a/rules/manager.go b/rules/manager.go index 0a801c257..2630ce860 100644 --- a/rules/manager.go +++ b/rules/manager.go @@ -39,9 +39,7 @@ import ( const ( namespace = "prometheus" - ruleTypeLabel = "rule_type" - ruleTypeAlerting = "alerting" - ruleTypeRecording = "recording" + ruleTypeLabel = "rule_type" ) var ( @@ -74,12 +72,18 @@ func init() { prometheus.MustRegister(evalDuration) } +type ruleType string + +const ( + ruleTypeAlert = "alerting" + ruleTypeRecording = "recording" +) + // A Rule encapsulates a vector expression which is evaluated at a specified // interval and acted upon (currently either recorded or used for alerting). type Rule interface { - // Name returns the name of the rule. Name() string - // Eval evaluates the rule, including any associated recording or alerting actions. + // eval evaluates the rule, including any associated recording or alerting actions. eval(model.Time, *promql.Engine) (model.Vector, error) // String returns a human-readable string representation of the rule. String() string @@ -88,306 +92,365 @@ type Rule interface { HTMLSnippet(pathPrefix string) html_template.HTML } -// The Manager manages recording and alerting rules. -type Manager struct { - // Protects the rules list. - sync.Mutex - rules []Rule +type Group struct { + name string + interval time.Duration + rules []Rule + opts *ManagerOptions - done chan bool - - interval time.Duration - queryEngine *promql.Engine - - sampleAppender storage.SampleAppender - notificationHandler *notification.Handler - - externalURL *url.URL + done chan struct{} + terminated chan struct{} } -// ManagerOptions bundles options for the Manager. -type ManagerOptions struct { - EvaluationInterval time.Duration - QueryEngine *promql.Engine - - NotificationHandler *notification.Handler - SampleAppender storage.SampleAppender - - ExternalURL *url.URL -} - -// NewManager returns an implementation of Manager, ready to be started -// by calling the Run method. -func NewManager(o *ManagerOptions) *Manager { - manager := &Manager{ - rules: []Rule{}, - done: make(chan bool), - - interval: o.EvaluationInterval, - sampleAppender: o.SampleAppender, - queryEngine: o.QueryEngine, - notificationHandler: o.NotificationHandler, - externalURL: o.ExternalURL, +func newGroup(name string, opts *ManagerOptions) *Group { + return &Group{ + name: name, + opts: opts, + done: make(chan struct{}), + terminated: make(chan struct{}), } - return manager } -// Run the rule manager's periodic rule evaluation. -func (m *Manager) Run() { - defer log.Info("Rule manager stopped.") +func (g *Group) run() { + defer close(g.terminated) - m.Lock() - lastInterval := m.interval - m.Unlock() + // Wait an initial amount to have consistently slotted intervals. + time.Sleep(g.offset()) - ticker := time.NewTicker(lastInterval) - defer ticker.Stop() + iter := func() { + start := time.Now() + g.eval() + + iterationDuration.Observe(float64(time.Since(start) / time.Millisecond)) + } + iter() + + tick := time.NewTicker(g.interval) + defer tick.Stop() for { - // The outer select clause makes sure that m.done is looked at - // first. Otherwise, if m.runIteration takes longer than - // m.interval, there is only a 50% chance that m.done will be - // looked at before the next m.runIteration call happens. select { - case <-m.done: + case <-g.done: return default: select { - case <-ticker.C: - start := time.Now() - m.runIteration() - iterationDuration.Observe(float64(time.Since(start) / time.Millisecond)) - - m.Lock() - if lastInterval != m.interval { - ticker.Stop() - ticker = time.NewTicker(m.interval) - lastInterval = m.interval - } - m.Unlock() - case <-m.done: + case <-g.done: return + case <-tick.C: + iter() } } } } -// Stop the rule manager's rule evaluation cycles. -func (m *Manager) Stop() { - log.Info("Stopping rule manager...") - m.done <- true +func (g *Group) stop() { + close(g.done) + <-g.terminated } -func (m *Manager) sendAlertNotifications(rule *AlertingRule, timestamp model.Time) { - activeAlerts := rule.ActiveAlerts() - if len(activeAlerts) == 0 { - return +func (g *Group) fingerprint() model.Fingerprint { + l := model.LabelSet{"name": model.LabelValue(g.name)} + return l.Fingerprint() +} + +func (g *Group) offset() time.Duration { + now := time.Now().UnixNano() + + var ( + base = now - (now % int64(g.interval)) + offset = uint64(g.fingerprint()) % uint64(g.interval) + next = base + int64(offset) + ) + + if next < now { + next += int64(g.interval) } + return time.Duration(next - now) +} - alerts := make(model.Alerts, 0, len(activeAlerts)) - - for _, aa := range activeAlerts { - if aa.State != StateFiring { - // BUG: In the future, make AlertManager support pending alerts? +func (g *Group) copyState(from *Group) { + for _, fromRule := range from.rules { + far, ok := fromRule.(*AlertingRule) + if !ok { continue } - - // Provide the alert information to the template. - l := map[string]string{} - for k, v := range aa.Labels { - l[string(k)] = string(v) - } - tmplData := struct { - Labels map[string]string - Value float64 - }{ - Labels: l, - Value: float64(aa.Value), - } - // Inject some convenience variables that are easier to remember for users - // who are not used to Go's templating system. - defs := "{{$labels := .Labels}}{{$value := .Value}}" - - expand := func(text model.LabelValue) model.LabelValue { - tmpl := template.NewTemplateExpander(defs+string(text), "__alert_"+rule.Name(), tmplData, timestamp, m.queryEngine, m.externalURL.Path) - result, err := tmpl.Expand() - if err != nil { - result = err.Error() - log.Warnf("Error expanding alert template %v with data '%v': %v", rule.Name(), tmplData, err) + for _, rule := range g.rules { + ar, ok := rule.(*AlertingRule) + if !ok { + continue + } + if far.Name() == ar.Name() { + ar.active = far.active } - return model.LabelValue(result) } - - labels := aa.Labels.Clone() - labels[model.AlertNameLabel] = model.LabelValue(rule.Name()) - - annotations := rule.annotations.Clone() - for an, av := range rule.annotations { - annotations[an] = expand(av) - } - - alerts = append(alerts, &model.Alert{ - StartsAt: aa.ActiveSince.Time().Add(rule.holdDuration), - Labels: labels, - Annotations: annotations, - GeneratorURL: m.externalURL.String() + strutil.GraphLinkForExpression(rule.vector.String()), - }) } - m.notificationHandler.Send(alerts...) } -func (m *Manager) runIteration() { - now := model.Now() - wg := sync.WaitGroup{} +func (g *Group) eval() { + var ( + now = model.Now() + wg sync.WaitGroup + ) - m.Lock() - rulesSnapshot := make([]Rule, len(m.rules)) - copy(rulesSnapshot, m.rules) - m.Unlock() - - for _, rule := range rulesSnapshot { + for _, rule := range g.rules { wg.Add(1) // BUG(julius): Look at fixing thundering herd. go func(rule Rule) { defer wg.Done() start := time.Now() - vector, err := rule.eval(now, m.queryEngine) - duration := time.Since(start) + vector, err := rule.eval(now, g.opts.QueryEngine) if err != nil { evalFailures.Inc() log.Warnf("Error while evaluating rule %q: %s", rule, err) - return } + var rtyp ruleType switch r := rule.(type) { case *AlertingRule: - m.sendAlertNotifications(r, now) - evalDuration.WithLabelValues(ruleTypeAlerting).Observe( - float64(duration / time.Millisecond), - ) + rtyp = ruleTypeRecording + g.sendAlerts(r, now) + case *RecordingRule: - evalDuration.WithLabelValues(ruleTypeRecording).Observe( - float64(duration / time.Millisecond), - ) + rtyp = ruleTypeAlert + default: panic(fmt.Errorf("unknown rule type: %T", rule)) } + evalDuration.WithLabelValues(string(rtyp)).Observe( + float64(time.Since(start) / time.Millisecond), + ) + for _, s := range vector { - m.sampleAppender.Append(s) + g.opts.SampleAppender.Append(s) } }(rule) } wg.Wait() } -// transferAlertState makes a copy of the state of alerting rules and returns a function -// that restores them in the current state. -func (m *Manager) transferAlertState() func() { - - alertingRules := map[string]*AlertingRule{} - for _, r := range m.rules { - if ar, ok := r.(*AlertingRule); ok { - alertingRules[ar.name] = ar +func (g *Group) sendAlerts(rule *AlertingRule, timestamp model.Time) error { + var alerts model.Alerts + for _, alert := range rule.ActiveAlerts() { + // Only send actually firing alerts. + if alert.State != StateFiring { + continue } + + // Provide the alert information to the template. + l := make(map[string]string, len(alert.Labels)) + for k, v := range alert.Labels { + l[string(k)] = string(v) + } + + tmplData := struct { + Labels map[string]string + Value float64 + }{ + Labels: l, + Value: float64(alert.Value), + } + // Inject some convenience variables that are easier to remember for users + // who are not used to Go's templating system. + defs := "{{$labels := .Labels}}{{$value := .Value}}" + + expand := func(text model.LabelValue) model.LabelValue { + tmpl := template.NewTemplateExpander( + defs+string(text), + "__alert_"+rule.Name(), + tmplData, + timestamp, + g.opts.QueryEngine, + g.opts.ExternalURL.Path, + ) + result, err := tmpl.Expand() + if err != nil { + result = fmt.Sprintf("", err) + log.Warnf("Error expanding alert template %v with data '%v': %s", rule.Name(), tmplData, err) + } + return model.LabelValue(result) + } + + labels := make(model.LabelSet, len(alert.Labels)+1) + for ln, lv := range alert.Labels { + labels[ln] = expand(lv) + } + labels[model.AlertNameLabel] = model.LabelValue(rule.Name()) + + annotations := make(model.LabelSet, len(rule.annotations)) + for an, av := range rule.annotations { + annotations[an] = expand(av) + } + + alerts = append(alerts, &model.Alert{ + StartsAt: alert.ActiveSince.Time().Add(rule.holdDuration), + Labels: labels, + Annotations: annotations, + GeneratorURL: g.opts.ExternalURL.String() + strutil.GraphLinkForExpression(rule.vector.String()), + }) } - return func() { - // Restore alerting rule state. - for _, r := range m.rules { - ar, ok := r.(*AlertingRule) - if !ok { - continue - } - if old, ok := alertingRules[ar.name]; ok { - ar.activeAlerts = old.activeAlerts - } - } + if len(alerts) > 0 { + g.opts.NotificationHandler.Send(alerts...) } + + return nil +} + +// The Manager manages recording and alerting rules. +type Manager struct { + opts *ManagerOptions + groups map[string]*Group + mtx sync.RWMutex +} + +// ManagerOptions bundles options for the Manager. +type ManagerOptions struct { + ExternalURL *url.URL + QueryEngine *promql.Engine + NotificationHandler *notification.Handler + SampleAppender storage.SampleAppender +} + +// NewManager returns an implementation of Manager, ready to be started +// by calling the Run method. +func NewManager(o *ManagerOptions) *Manager { + manager := &Manager{ + groups: map[string]*Group{}, + opts: o, + } + return manager +} + +// Stop the rule manager's rule evaluation cycles. +func (m *Manager) Stop() { + log.Info("Stopping rule manager...") + + for _, eg := range m.groups { + eg.stop() + } + + log.Info("Rule manager stopped.") } // ApplyConfig updates the rule manager's state as the config requires. If // loading the new rules failed the old rule set is restored. Returns true on success. func (m *Manager) ApplyConfig(conf *config.Config) bool { - m.Lock() - defer m.Unlock() - - defer m.transferAlertState()() - - success := true - m.interval = time.Duration(conf.GlobalConfig.EvaluationInterval) - - rulesSnapshot := make([]Rule, len(m.rules)) - copy(rulesSnapshot, m.rules) - m.rules = m.rules[:0] + m.mtx.Lock() + defer m.mtx.Unlock() + // Get all rule files and load the groups they define. var files []string for _, pat := range conf.RuleFiles { fs, err := filepath.Glob(pat) if err != nil { // The only error can be a bad pattern. log.Errorf("Error retrieving rule files for %s: %s", pat, err) - success = false + return false } files = append(files, fs...) } - if err := m.loadRuleFiles(files...); err != nil { - // If loading the new rules failed, restore the old rule set. - m.rules = rulesSnapshot + + groups, err := m.loadGroups(files...) + if err != nil { log.Errorf("Error loading rules, previous rule set restored: %s", err) - success = false + return false } - return success + var wg sync.WaitGroup + + for _, newg := range groups { + // To be replaced with a configurable per-group interval. + newg.interval = time.Duration(conf.GlobalConfig.EvaluationInterval) + + wg.Add(1) + + // If there is an old group with the same identifier, stop it and wait for + // it to finish the current iteration. Then copy its into the new group. + oldg, ok := m.groups[newg.name] + delete(m.groups, newg.name) + + go func(newg *Group) { + if ok { + oldg.stop() + newg.copyState(oldg) + } + go newg.run() + wg.Done() + }(newg) + } + + // Stop remaining old groups. + for _, oldg := range m.groups { + oldg.stop() + } + + wg.Wait() + m.groups = groups + + return true } -// loadRuleFiles loads alerting and recording rules from the given files. -func (m *Manager) loadRuleFiles(filenames ...string) error { +func (m *Manager) loadGroups(filenames ...string) (map[string]*Group, error) { + groups := map[string]*Group{} + + // Currently there is no group syntax implemented. Thus all rules + // are read into a single default group. + g := newGroup("default", m.opts) + groups[g.name] = g + for _, fn := range filenames { content, err := ioutil.ReadFile(fn) if err != nil { - return err + return nil, err } stmts, err := promql.ParseStmts(string(content)) if err != nil { - return fmt.Errorf("error parsing %s: %s", fn, err) + return nil, fmt.Errorf("error parsing %s: %s", fn, err) } for _, stmt := range stmts { + var rule Rule + switch r := stmt.(type) { case *promql.AlertStmt: - rule := NewAlertingRule(r.Name, r.Expr, r.Duration, r.Labels, r.Annotations) - m.rules = append(m.rules, rule) + rule = NewAlertingRule(r.Name, r.Expr, r.Duration, r.Labels, r.Annotations) + case *promql.RecordStmt: - rule := NewRecordingRule(r.Name, r.Expr, r.Labels) - m.rules = append(m.rules, rule) + rule = NewRecordingRule(r.Name, r.Expr, r.Labels) + default: panic("retrieval.Manager.LoadRuleFiles: unknown statement type") } + g.rules = append(g.rules, rule) } } - return nil + + return groups, nil } // Rules returns the list of the manager's rules. func (m *Manager) Rules() []Rule { - m.Lock() - defer m.Unlock() + m.mtx.RLock() + defer m.mtx.RUnlock() + + var rules []Rule + for _, g := range m.groups { + rules = append(rules, g.rules...) + } - rules := make([]Rule, len(m.rules)) - copy(rules, m.rules) return rules } // AlertingRules returns the list of the manager's alerting rules. func (m *Manager) AlertingRules() []*AlertingRule { - m.Lock() - defer m.Unlock() + m.mtx.RLock() + defer m.mtx.RUnlock() alerts := []*AlertingRule{} - for _, rule := range m.rules { + for _, rule := range m.Rules() { if alertingRule, ok := rule.(*AlertingRule); ok { alerts = append(alerts, alertingRule) } diff --git a/rules/manager_test.go b/rules/manager_test.go index 621986877..ffec7974b 100644 --- a/rules/manager_test.go +++ b/rules/manager_test.go @@ -15,7 +15,7 @@ package rules import ( "fmt" - "reflect" + // "reflect" "strings" "testing" "time" @@ -138,46 +138,3 @@ func annotateWithTime(lines []string, timestamp model.Time) []string { } return annotatedLines } - -func TestTransferAlertState(t *testing.T) { - m := NewManager(&ManagerOptions{}) - - alert := &Alert{ - Name: "testalert", - State: StateFiring, - } - - arule := AlertingRule{ - name: "test", - activeAlerts: map[model.Fingerprint]*Alert{}, - } - aruleCopy := arule - - m.rules = append(m.rules, &arule) - - // Set an alert. - arule.activeAlerts[0] = alert - - // Save state and get the restore function. - restore := m.transferAlertState() - - // Remove arule from the rule list and add an unrelated rule and the - // stateless copy of arule. - m.rules = []Rule{ - &AlertingRule{ - name: "test_other", - activeAlerts: map[model.Fingerprint]*Alert{}, - }, - &aruleCopy, - } - - // Apply the restore function. - restore() - - if ar := m.rules[0].(*AlertingRule); len(ar.activeAlerts) != 0 { - t.Fatalf("unexpected alert for unrelated alerting rule") - } - if ar := m.rules[1].(*AlertingRule); !reflect.DeepEqual(ar.activeAlerts[0], alert) { - t.Fatalf("alert state was not restored") - } -} diff --git a/rules/recording.go b/rules/recording.go index 7fcfc32b0..070b43820 100644 --- a/rules/recording.go +++ b/rules/recording.go @@ -40,7 +40,9 @@ func NewRecordingRule(name string, vector promql.Expr, labels model.LabelSet) *R } // Name returns the rule name. -func (rule RecordingRule) Name() string { return rule.name } +func (rule RecordingRule) Name() string { + return rule.name +} // eval evaluates the rule and then overrides the metric names and labels accordingly. func (rule RecordingRule) eval(timestamp model.Time, engine *promql.Engine) (model.Vector, error) { diff --git a/web/ui/bindata.go b/web/ui/bindata.go index 6ab2b970f..1acfe71d2 100644 --- a/web/ui/bindata.go +++ b/web/ui/bindata.go @@ -134,7 +134,7 @@ func webUiTemplatesAlertsHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "web/ui/templates/alerts.html", size: 1707, mode: os.FileMode(420), modTime: time.Unix(1450269200, 0)} + info := bindataFileInfo{name: "web/ui/templates/alerts.html", size: 1707, mode: os.FileMode(420), modTime: time.Unix(1450348618, 0)} a := &asset{bytes: bytes, info: info} return a, nil } From 62075aa037495e8b14b580ec26ca567e64080566 Mon Sep 17 00:00:00 2001 From: Fabian Reinartz Date: Tue, 15 Dec 2015 12:11:38 +0100 Subject: [PATCH 3/6] Reduce noisy no-alertmanager warning --- cmd/prometheus/main.go | 1 - notification/notification.go | 6 +++++- rules/manager_test.go | 1 - 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index c3a1c1475..00aa3f87c 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -155,7 +155,6 @@ func Main() int { prometheus.MustRegister(configSuccess) prometheus.MustRegister(configSuccessTime) - // go ruleManager.Run() defer ruleManager.Stop() go notificationHandler.Run() diff --git a/notification/notification.go b/notification/notification.go index 00c5f7f2a..500c43eaf 100644 --- a/notification/notification.go +++ b/notification/notification.go @@ -159,6 +159,11 @@ func (n *Handler) nextBatch() []*model.Alert { // Run dispatches notifications continuously. func (n *Handler) Run() { + // Just warn one in the beginning to prevent nosiy logs. + if n.opts.AlertmanagerURL == "" { + log.Warnf("No AlertManager configured, not dispatching any alerts") + } + for { select { case <-n.ctx.Done(): @@ -172,7 +177,6 @@ func (n *Handler) Run() { continue } if n.opts.AlertmanagerURL == "" { - log.Warn("No AlertManager configured, not dispatching %d alerts", len(alerts)) n.dropped.Add(float64(len(alerts))) continue } diff --git a/rules/manager_test.go b/rules/manager_test.go index ffec7974b..463388e15 100644 --- a/rules/manager_test.go +++ b/rules/manager_test.go @@ -15,7 +15,6 @@ package rules import ( "fmt" - // "reflect" "strings" "testing" "time" From f69e668fc468dfa2feb13603237810b8d748d87e Mon Sep 17 00:00:00 2001 From: Fabian Reinartz Date: Tue, 15 Dec 2015 13:15:07 +0100 Subject: [PATCH 4/6] Improve rules/ instrumentation This commit adds a counter for the total number of rule evaluations and standardizes the units to seconds. --- rules/manager.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/rules/manager.go b/rules/manager.go index 2630ce860..352e17837 100644 --- a/rules/manager.go +++ b/rules/manager.go @@ -46,7 +46,7 @@ var ( evalDuration = prometheus.NewSummaryVec( prometheus.SummaryOpts{ Namespace: namespace, - Name: "rule_evaluation_duration_milliseconds", + Name: "rule_evaluation_duration_seconds", Help: "The duration for a rule to execute.", }, []string{ruleTypeLabel}, @@ -58,9 +58,16 @@ var ( Help: "The total number of rule evaluation failures.", }, ) + evalTotal = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "rule_evaluations_total", + Help: "The total number of rule evaluations.", + }, + ) iterationDuration = prometheus.NewSummary(prometheus.SummaryOpts{ Namespace: namespace, - Name: "evaluator_duration_milliseconds", + Name: "evaluator_duration_seconds", Help: "The duration for all evaluations to execute.", Objectives: map[float64]float64{0.01: 0.001, 0.05: 0.005, 0.5: 0.05, 0.90: 0.01, 0.99: 0.001}, }) @@ -121,7 +128,7 @@ func (g *Group) run() { start := time.Now() g.eval() - iterationDuration.Observe(float64(time.Since(start) / time.Millisecond)) + iterationDuration.Observe(float64(time.Since(start)) / float64(time.Millisecond)) } iter() @@ -199,6 +206,7 @@ func (g *Group) eval() { defer wg.Done() start := time.Now() + evalTotal.Inc() vector, err := rule.eval(now, g.opts.QueryEngine) if err != nil { @@ -220,7 +228,7 @@ func (g *Group) eval() { } evalDuration.WithLabelValues(string(rtyp)).Observe( - float64(time.Since(start) / time.Millisecond), + float64(time.Since(start)) / float64(time.Second), ) for _, s := range vector { From bf6abac8f4051f7407fffc3b043dad702b1ef429 Mon Sep 17 00:00:00 2001 From: Fabian Reinartz Date: Tue, 15 Dec 2015 19:46:03 +0100 Subject: [PATCH 5/6] Send resolved notifications --- rules/alerting.go | 120 +++++++++++++++++++++-------------- rules/manager.go | 16 +++-- web/ui/bindata.go | 4 +- web/ui/templates/alerts.html | 2 +- 4 files changed, 85 insertions(+), 57 deletions(-) diff --git a/rules/alerting.go b/rules/alerting.go index 785cb216f..861fdd99b 100644 --- a/rules/alerting.go +++ b/rules/alerting.go @@ -38,6 +38,17 @@ const ( // AlertState denotes the state of an active alert. type AlertState int +const ( + // StateInactive is the state of an alert that is either firing nor pending. + StateInactive AlertState = iota + // StatePending is the state of an alert that has been active for less than + // the configured threshold duration. + StatePending + // StateFiring is the state of an alert that has been active for longer than + // the configured threshold duration. + StateFiring +) + func (s AlertState) String() string { switch s { case StateInactive: @@ -50,17 +61,13 @@ func (s AlertState) String() string { panic(fmt.Errorf("unknown alert state: %v", s)) } -const ( - StateInactive AlertState = iota - StatePending - StateFiring -) - -type alertInstance struct { - metric model.Metric - value model.SampleValue - state AlertState - activeSince model.Time +// Alert is the user-level representation of a single instance of an alerting rule. +type Alert struct { + State AlertState + Labels model.LabelSet + Value model.SampleValue + ActiveAt model.Time + ResolvedAt model.Time } // An AlertingRule generates alerts from its vector expression. @@ -81,7 +88,7 @@ type AlertingRule struct { mtx sync.Mutex // A map of alerts which are currently active (Pending or Firing), keyed by // the fingerprint of the labelset they correspond to. - active map[model.Fingerprint]*alertInstance + active map[model.Fingerprint]*Alert } // NewAlertingRule constructs a new AlertingRule. @@ -92,7 +99,7 @@ func NewAlertingRule(name string, vec promql.Expr, hold time.Duration, lbls, ann holdDuration: hold, labels: lbls, annotations: anns, - active: map[model.Fingerprint]*alertInstance{}, + active: map[model.Fingerprint]*Alert{}, } } @@ -101,17 +108,17 @@ func (rule *AlertingRule) Name() string { return rule.name } -func (r *AlertingRule) sample(ai *alertInstance, ts model.Time, set bool) *model.Sample { +func (r *AlertingRule) sample(alert *Alert, ts model.Time, set bool) *model.Sample { // Build alert labels in order they can be overwritten. metric := model.Metric(r.labels.Clone()) - for ln, lv := range ai.metric { + for ln, lv := range alert.Labels { metric[ln] = lv } metric[model.MetricNameLabel] = alertMetricName metric[model.AlertNameLabel] = model.LabelValue(r.name) - metric[alertStateLabel] = model.LabelValue(ai.state.String()) + metric[alertStateLabel] = model.LabelValue(alert.State.String()) s := &model.Sample{ Metric: metric, @@ -124,6 +131,10 @@ func (r *AlertingRule) sample(ai *alertInstance, ts model.Time, set bool) *model return s } +// resolvedRetention is the duration for which a resolved alert instance +// is kept in memory state and consequentally repeatedly sent to the AlertManager. +const resolvedRetention = 15 * time.Minute + // eval evaluates the rule expression and then creates pending alerts and fires // or removes previously pending alerts accordingly. func (r *AlertingRule) eval(ts model.Time, engine *promql.Engine) (model.Vector, error) { @@ -147,57 +158,59 @@ func (r *AlertingRule) eval(ts model.Time, engine *promql.Engine) (model.Vector, fp := smpl.Metric.Fingerprint() resultFPs[fp] = struct{}{} - if ai, ok := r.active[fp]; ok { - ai.value = smpl.Value + if alert, ok := r.active[fp]; ok { + alert.Value = smpl.Value continue } delete(smpl.Metric, model.MetricNameLabel) - r.active[fp] = &alertInstance{ - metric: smpl.Metric, - activeSince: ts, - state: StatePending, - value: smpl.Value, + r.active[fp] = &Alert{ + Labels: model.LabelSet(smpl.Metric), + ActiveAt: ts, + State: StatePending, + Value: smpl.Value, } } var vec model.Vector // Check if any pending alerts should be removed or fire now. Write out alert timeseries. - for fp, ai := range r.active { + for fp, a := range r.active { if _, ok := resultFPs[fp]; !ok { - delete(r.active, fp) - vec = append(vec, r.sample(ai, ts, false)) + if a.State != StateInactive { + vec = append(vec, r.sample(a, ts, false)) + } + // If the alert was previously firing, keep it aroud for a given + // retention time so it is reported as resolved to the AlertManager. + if a.State == StatePending || (a.ResolvedAt != 0 && ts.Sub(a.ResolvedAt) > resolvedRetention) { + delete(r.active, fp) + } + if a.State != StateInactive { + a.State = StateInactive + a.ResolvedAt = ts + } continue } - if ai.state != StateFiring && ts.Sub(ai.activeSince) >= r.holdDuration { - vec = append(vec, r.sample(ai, ts, false)) - ai.state = StateFiring + if a.State == StatePending && ts.Sub(a.ActiveAt) >= r.holdDuration { + vec = append(vec, r.sample(a, ts, false)) + a.State = StateFiring } - vec = append(vec, r.sample(ai, ts, true)) + vec = append(vec, r.sample(a, ts, true)) } return vec, nil } -// Alert is the user-level representation of a single instance of an alerting rule. -type Alert struct { - State AlertState - Labels model.LabelSet - ActiveSince model.Time - Value model.SampleValue -} - func (r *AlertingRule) State() AlertState { r.mtx.Lock() defer r.mtx.Unlock() maxState := StateInactive - for _, ai := range r.active { - if ai.state > maxState { - maxState = ai.state + for _, a := range r.active { + if a.State > maxState { + maxState = a.State } } return maxState @@ -205,21 +218,30 @@ func (r *AlertingRule) State() AlertState { // ActiveAlerts returns a slice of active alerts. func (r *AlertingRule) ActiveAlerts() []*Alert { + var res []*Alert + for _, a := range r.recentAlerts() { + if a.ResolvedAt == 0 { + res = append(res, a) + } + } + return res +} + +func (r *AlertingRule) recentAlerts() []*Alert { r.mtx.Lock() defer r.mtx.Unlock() alerts := make([]*Alert, 0, len(r.active)) - for _, ai := range r.active { + + for _, a := range r.active { labels := r.labels.Clone() - for ln, lv := range ai.metric { + for ln, lv := range a.Labels { labels[ln] = lv } - alerts = append(alerts, &Alert{ - State: ai.state, - Labels: labels, - ActiveSince: ai.activeSince, - Value: ai.value, - }) + anew := *a + anew.Labels = labels + + alerts = append(alerts, &anew) } return alerts } diff --git a/rules/manager.go b/rules/manager.go index 352e17837..1ff9eefe6 100644 --- a/rules/manager.go +++ b/rules/manager.go @@ -241,9 +241,10 @@ func (g *Group) eval() { func (g *Group) sendAlerts(rule *AlertingRule, timestamp model.Time) error { var alerts model.Alerts - for _, alert := range rule.ActiveAlerts() { + + for _, alert := range rule.recentAlerts() { // Only send actually firing alerts. - if alert.State != StateFiring { + if alert.State == StatePending { continue } @@ -292,12 +293,17 @@ func (g *Group) sendAlerts(rule *AlertingRule, timestamp model.Time) error { annotations[an] = expand(av) } - alerts = append(alerts, &model.Alert{ - StartsAt: alert.ActiveSince.Time().Add(rule.holdDuration), + a := &model.Alert{ + StartsAt: alert.ActiveAt.Add(rule.holdDuration).Time(), Labels: labels, Annotations: annotations, GeneratorURL: g.opts.ExternalURL.String() + strutil.GraphLinkForExpression(rule.vector.String()), - }) + } + if alert.ResolvedAt != 0 { + a.EndsAt = alert.ResolvedAt.Time() + } + + alerts = append(alerts, a) } if len(alerts) > 0 { diff --git a/web/ui/bindata.go b/web/ui/bindata.go index 1acfe71d2..92ccc2bd5 100644 --- a/web/ui/bindata.go +++ b/web/ui/bindata.go @@ -119,7 +119,7 @@ func webUiTemplates_baseHtml() (*asset, error) { return a, nil } -var _webUiTemplatesAlertsHtml = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\x8c\x54\x4d\x8f\x9b\x30\x10\xbd\xef\xaf\xb0\xd0\x1e\x5a\xa9\x80\xd4\x63\x45\x90\x56\x7b\xe9\x61\x5b\x55\x9b\x74\xaf\x91\xb1\x27\x8b\xb7\x8e\x41\xb6\x93\x4d\xe4\xf2\xdf\x3b\xb6\x21\x4b\x08\x48\xbd\x24\x78\x66\xde\xf3\xf3\x7c\x39\xc7\x61\x27\x14\x90\xa4\x06\xca\x93\xae\xbb\x23\xa4\x90\x42\xfd\x21\xf6\xdc\xc2\x2a\xb1\x70\xb2\x39\x33\x26\x21\x1a\xe4\x2a\x31\xf6\x2c\xc1\xd4\x00\x36\x21\xb5\x86\xdd\x2a\x71\x8e\xb4\xd4\xd6\xbf\xf0\x20\x4e\xa4\xeb\x72\x63\xa9\x15\xcc\x63\x72\x2a\x41\x5b\x93\x79\x78\xe9\x79\x0d\xd3\xa2\xb5\xc4\x68\xb6\x8c\x7b\xbb\xc0\xde\x10\x55\xe4\x11\x53\xde\x39\x07\x8a\xa3\xbc\xbb\x0f\xc5\xac\x51\x16\x94\xf5\xa2\x0b\x2e\x8e\x84\x49\x6a\xcc\x2a\x98\x29\x06\xe8\x74\x27\x0f\x82\xc7\xab\xeb\xaf\xe5\x43\xa0\x2d\x72\xfc\xf4\x16\x4b\x2b\x09\x03\x26\x1e\xc2\x6f\x5a\x35\x9a\x83\x06\xde\x1f\x59\x23\x25\x6d\x0d\x44\x22\x0f\xac\x1a\x7e\x8e\xdf\xce\xdd\x07\xb1\x6b\xd4\x0e\x9b\xe6\xb9\x79\x7f\xf4\x7c\xe4\xdb\x8a\x64\x0f\x33\x8e\x90\x5e\x0f\xd3\x54\xbd\x42\x1f\x23\xd4\xeb\xf3\x01\xb3\xda\x3b\x23\x2b\xb3\xe2\x08\x51\x71\x64\x1b\x19\x2e\x81\x85\xd5\xc3\x03\x9c\x13\x8a\xc3\x89\xcc\xeb\xc9\x82\xa1\xeb\x48\xf0\x6e\x7d\xa9\x41\xf7\xef\x89\x44\xbc\x2c\xc4\xc0\x25\x30\x83\x29\xab\xe1\xa8\xf1\x9f\x37\xef\xca\xd7\x41\x94\xa4\xa8\x4a\xe7\xb2\x9f\x74\x8f\x4c\x45\x5e\x95\xe4\x93\x73\x12\x14\xb9\x52\xeb\x2f\x09\xc7\xcf\x45\x8e\xac\x83\xd2\xdc\xea\xf2\x56\x75\x94\xc3\x01\xeb\x25\xcd\x44\xcf\xe5\x80\x47\xac\xee\xf8\x8c\x96\x56\x43\x59\xb0\x86\x83\x97\xf4\x7d\xf3\xe3\x69\xad\x44\xdb\x82\x1d\x35\x95\x17\x19\x22\x8a\xdc\x47\x8f\xf9\xf2\x09\x21\x66\x6f\x37\x7d\xc6\x38\xfe\x7f\x7b\xa5\x6e\x8e\xa0\x2f\x7d\x83\x05\x51\xd8\x37\x7d\xd2\x41\xc2\x1e\xbb\xd5\x6c\x83\x3b\x99\xbc\xe7\x23\x27\x13\x8f\xf7\xd5\xe5\x13\xad\x40\x62\xef\xe2\xe7\x8c\x37\x54\x77\xc9\x19\x3b\x87\xac\x85\x62\x8b\x31\x2f\x54\x1e\x16\x9d\x6b\x81\x45\x9e\xc3\x8e\x8b\x3a\xe4\x31\x36\xf6\x72\x2a\xc3\x53\x6f\x6f\xe1\x53\xd3\x88\x4b\xfa\xb7\x7f\x21\xf7\x47\x2f\x32\x0c\x43\xcc\xc6\x84\xb7\xa7\x32\x2d\x55\x43\x2a\x03\x92\x84\xdf\xb4\xd5\x62\x4f\xf5\x39\xc1\x7e\x89\x8c\x5d\xe7\xa7\x26\xb2\x76\x5d\x82\x9b\x06\x91\x73\x32\xe2\xde\x99\x5c\x93\xdf\x4a\x0e\x43\x34\xbe\x3e\xd4\x3d\x56\x3f\xc5\x75\x17\x87\x90\xfc\x25\xe3\x11\x8d\xf3\x89\x43\xe3\xd7\x1f\x6c\x71\x88\x05\xa3\xb6\xc1\x26\xc2\xc5\x9b\x1e\xb0\xa5\x35\xa3\x06\xbc\xec\x61\x88\x7b\xa5\x4b\x12\x30\x30\x96\x3c\x54\x3c\xdb\x88\x3d\x64\xbf\x37\x8f\x1e\xb7\x08\x78\x89\x49\xb8\x8d\x98\x2b\xf1\x34\x1f\x18\xe3\x3b\xfa\x7a\x9e\xae\x83\xe6\x57\xc1\x38\x0a\xad\xc3\x5a\xbd\xf0\xf5\x63\x3a\x84\xfd\x0b\x00\x00\xff\xff\xdb\x8e\x0c\x63\xab\x06\x00\x00") +var _webUiTemplatesAlertsHtml = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\x8c\x54\x4d\x8f\x9b\x30\x10\xbd\xef\xaf\xb0\xd0\x1e\x5a\xa9\x80\xd4\x63\x45\x90\x56\x7b\xe9\x61\x5b\x55\x9b\x74\xaf\x91\xb1\x27\x8b\xb7\x8e\x41\xb6\x93\x4d\xe4\xf2\xdf\x3b\xb6\x21\x4b\x08\x48\xbd\x24\x78\xe6\xcd\xf3\xf3\x7c\x39\xc7\x61\x27\x14\x90\xa4\x06\xca\x93\xae\xbb\x23\xa4\x90\x42\xfd\x21\xf6\xdc\xc2\x2a\xb1\x70\xb2\x39\x33\x26\x21\x1a\xe4\x2a\x31\xf6\x2c\xc1\xd4\x00\x36\x21\xb5\x86\xdd\x2a\x71\x8e\xb4\xd4\xd6\xbf\xf0\x20\x4e\xa4\xeb\x72\x63\xa9\x15\xcc\xc7\xe4\x54\x82\xb6\x26\xf3\xe1\xa5\xe7\x35\x4c\x8b\xd6\x12\xa3\xd9\x72\xdc\xdb\x25\xec\x0d\xa3\x8a\x3c\xc6\x94\x77\xce\x81\xe2\x28\xef\xee\x43\x31\x6b\x94\x05\x65\xbd\xe8\x82\x8b\x23\x61\x92\x1a\xb3\x0a\x66\x8a\x00\x9d\xee\xe4\x41\xf0\x78\x75\xfd\xb5\x7c\x08\xb4\x45\x8e\x9f\xde\x62\x69\x25\x61\x88\x89\x87\xf0\x9b\x56\x8d\xe6\xa0\x81\xf7\x47\xd6\x48\x49\x5b\x03\x91\xc8\x07\x56\x0d\x3f\xc7\x6f\xe7\xee\x83\xd8\x35\x6a\x87\x4d\xf3\xdc\xbc\x3f\x7a\x3e\xf2\x6d\x45\xb2\x87\x19\x47\x48\xaf\x0f\xd3\x54\xbd\x42\x8f\x11\xea\xf5\xf9\x80\x59\xed\x9d\x91\x95\x59\x71\x84\xa8\x38\xb2\x8d\x0c\x17\x60\x61\xf5\xf0\x00\xe7\x84\xe2\x70\x22\xf3\x7a\xb2\x60\xe8\x3a\x12\xbc\x5b\x5f\x6a\xd0\xfd\x7b\x22\x11\x2f\x0b\x31\x70\x09\xcc\x60\xca\x6a\x38\x6a\xfc\xe7\xcd\xbb\xf2\x75\x10\x25\x29\xaa\xd2\xb9\xec\x27\xdd\x23\x53\x91\x57\x25\xf9\xe4\x9c\x04\x45\xae\xd4\xfa\x4b\xc2\xf1\x73\x91\x23\xeb\xa0\x34\xb7\xba\xbc\x55\x1d\xe5\x70\xc0\x7a\x49\x33\xd1\x73\x39\xe0\x11\xab\x3b\x3e\xa3\xa5\xd5\x50\x16\xac\xe1\xe0\x25\x7d\xdf\xfc\x78\x5a\x2b\xd1\xb6\x60\x47\x4d\xe5\x45\x06\x44\x91\x7b\xf4\x98\x2f\x9f\x10\x62\xf6\x76\xd3\x67\x8c\xf1\xff\xdb\x2b\x75\x73\x04\x7d\xe9\x1b\x2c\x88\xc2\xbe\xe9\x93\x0e\x12\xf6\xd8\xad\x66\x1b\xdc\xc9\xe4\x3d\x1f\x39\x99\x78\xbc\xaf\x2e\x9f\x68\x05\x12\x7b\x17\x3f\x67\xbc\xa1\xba\x4b\xce\xd8\x39\x64\x2d\x14\x5b\xc4\xbc\x50\x79\x58\x74\xae\x05\x16\x79\x2e\x76\x5c\xd4\x21\x8f\xb1\xb1\x97\x53\x19\x9e\x7a\x7b\x0b\x9f\x9a\x46\x5c\xd2\xbf\xfd\x0b\xb9\x3f\x7a\x91\x61\x18\x62\x36\x26\xbc\x3d\x95\x69\xa9\x1a\x52\x19\x22\x49\xf8\x4d\x5b\x2d\xf6\x54\x9f\x13\xec\x97\xc8\xd8\x75\x7e\x6a\x22\x6b\xd7\x25\xb8\x69\x30\x72\x4e\x46\xdc\x3b\x93\x6b\xf2\x5b\xc9\x61\x88\xc6\xd7\x87\xba\xc7\xea\xa7\xb8\xee\xe2\x10\x92\xbf\x64\x3c\xa2\x71\x3e\x71\x68\xfc\xfa\x83\x2d\x0e\xb1\x60\xd4\x36\xd8\x44\xb8\x78\xd3\x03\xb6\xb4\x66\xd4\x80\x97\x3d\x0c\x71\xaf\x74\x49\x02\x02\xfb\x65\x61\xb3\x8d\xd8\x43\xf6\x7b\xf3\xe8\x83\x16\xd1\x2f\x31\x03\xb7\x88\xb9\xfa\x4e\x93\x81\x18\xdf\xce\xd7\xc3\x74\x0d\x9a\xdf\x03\x63\x14\x5a\x87\x9d\x7a\xe1\xeb\x67\x74\x80\xfd\x0b\x00\x00\xff\xff\xb3\xf6\xaa\xca\xa8\x06\x00\x00") func webUiTemplatesAlertsHtmlBytes() ([]byte, error) { return bindataRead( @@ -134,7 +134,7 @@ func webUiTemplatesAlertsHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "web/ui/templates/alerts.html", size: 1707, mode: os.FileMode(420), modTime: time.Unix(1450348618, 0)} + info := bindataFileInfo{name: "web/ui/templates/alerts.html", size: 1704, mode: os.FileMode(420), modTime: time.Unix(1450348695, 0)} a := &asset{bytes: bytes, info: info} return a, nil } diff --git a/web/ui/templates/alerts.html b/web/ui/templates/alerts.html index 58a00be34..4bdaec5f1 100644 --- a/web/ui/templates/alerts.html +++ b/web/ui/templates/alerts.html @@ -36,7 +36,7 @@ {{end}} {{.State}} - {{.ActiveSince.Time.UTC}} + {{.ActiveAt.Time.UTC}} {{.Value}} {{end}} From 0cf3c6a9efe454d37861db2fbf990a03f8b6cf27 Mon Sep 17 00:00:00 2001 From: Fabian Reinartz Date: Thu, 17 Dec 2015 11:46:10 +0100 Subject: [PATCH 6/6] Add comments, rename a method --- notification/notification.go | 2 +- rules/alerting.go | 23 ++++++++++++++--------- rules/manager.go | 12 +++++++++++- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/notification/notification.go b/notification/notification.go index 500c43eaf..acf3b49d7 100644 --- a/notification/notification.go +++ b/notification/notification.go @@ -159,7 +159,7 @@ func (n *Handler) nextBatch() []*model.Alert { // Run dispatches notifications continuously. func (n *Handler) Run() { - // Just warn one in the beginning to prevent nosiy logs. + // Just warn once in the beginning to prevent noisy logs. if n.opts.AlertmanagerURL == "" { log.Warnf("No AlertManager configured, not dispatching any alerts") } diff --git a/rules/alerting.go b/rules/alerting.go index 861fdd99b..cb89b8b1a 100644 --- a/rules/alerting.go +++ b/rules/alerting.go @@ -63,11 +63,13 @@ func (s AlertState) String() string { // Alert is the user-level representation of a single instance of an alerting rule. type Alert struct { - State AlertState - Labels model.LabelSet - Value model.SampleValue - ActiveAt model.Time - ResolvedAt model.Time + State AlertState + Labels model.LabelSet + // The value at the last evaluation of the alerting expression. + Value model.SampleValue + // The interval during which the condition of this alert held true. + // ResolvedAt will be 0 to indicate a still active alert. + ActiveAt, ResolvedAt model.Time } // An AlertingRule generates alerts from its vector expression. @@ -109,7 +111,6 @@ func (rule *AlertingRule) Name() string { } func (r *AlertingRule) sample(alert *Alert, ts model.Time, set bool) *model.Sample { - // Build alert labels in order they can be overwritten. metric := model.Metric(r.labels.Clone()) for ln, lv := range alert.Labels { @@ -180,7 +181,7 @@ func (r *AlertingRule) eval(ts model.Time, engine *promql.Engine) (model.Vector, if a.State != StateInactive { vec = append(vec, r.sample(a, ts, false)) } - // If the alert was previously firing, keep it aroud for a given + // If the alert was previously firing, keep it around for a given // retention time so it is reported as resolved to the AlertManager. if a.State == StatePending || (a.ResolvedAt != 0 && ts.Sub(a.ResolvedAt) > resolvedRetention) { delete(r.active, fp) @@ -203,6 +204,8 @@ func (r *AlertingRule) eval(ts model.Time, engine *promql.Engine) (model.Vector, return vec, nil } +// State returns the maximum state of alert instances for this rule. +// StateFiring > StatePending > StateInactive func (r *AlertingRule) State() AlertState { r.mtx.Lock() defer r.mtx.Unlock() @@ -219,7 +222,7 @@ func (r *AlertingRule) State() AlertState { // ActiveAlerts returns a slice of active alerts. func (r *AlertingRule) ActiveAlerts() []*Alert { var res []*Alert - for _, a := range r.recentAlerts() { + for _, a := range r.currentAlerts() { if a.ResolvedAt == 0 { res = append(res, a) } @@ -227,7 +230,9 @@ func (r *AlertingRule) ActiveAlerts() []*Alert { return res } -func (r *AlertingRule) recentAlerts() []*Alert { +// currentAlerts returns all instances of alerts for this rule. This may include +// inactive alerts that were previously firing. +func (r *AlertingRule) currentAlerts() []*Alert { r.mtx.Lock() defer r.mtx.Unlock() diff --git a/rules/manager.go b/rules/manager.go index 1ff9eefe6..3592e76fb 100644 --- a/rules/manager.go +++ b/rules/manager.go @@ -99,6 +99,7 @@ type Rule interface { HTMLSnippet(pathPrefix string) html_template.HTML } +// Group is a set of rules that have a logical relation. type Group struct { name string interval time.Duration @@ -160,6 +161,7 @@ func (g *Group) fingerprint() model.Fingerprint { return l.Fingerprint() } +// offset returns until the next consistently slotted evaluation interval. func (g *Group) offset() time.Duration { now := time.Now().UnixNano() @@ -175,6 +177,7 @@ func (g *Group) offset() time.Duration { return time.Duration(next - now) } +// copyState copies the alerting rule state from the given group. func (g *Group) copyState(from *Group) { for _, fromRule := range from.rules { far, ok := fromRule.(*AlertingRule) @@ -193,6 +196,9 @@ func (g *Group) copyState(from *Group) { } } +// eval runs a single evaluation cycle in which all rules are evaluated in parallel. +// In the future a single group will be evaluated sequentially to properly handle +// rule dependency. func (g *Group) eval() { var ( now = model.Now() @@ -239,10 +245,11 @@ func (g *Group) eval() { wg.Wait() } +// sendAlerts sends alert notifications for the given rule. func (g *Group) sendAlerts(rule *AlertingRule, timestamp model.Time) error { var alerts model.Alerts - for _, alert := range rule.recentAlerts() { + for _, alert := range rule.currentAlerts() { // Only send actually firing alerts. if alert.State == StatePending { continue @@ -407,6 +414,9 @@ func (m *Manager) ApplyConfig(conf *config.Config) bool { return true } +// loadGroups reads groups from a list of files. +// As there's currently no group syntax a single group named "default" containing +// all rules will be returned. func (m *Manager) loadGroups(filenames ...string) (map[string]*Group, error) { groups := map[string]*Group{}