Refactor rules/ package

This commit is contained in:
Fabian Reinartz 2015-12-14 17:40:40 +01:00
parent e4fabe135a
commit 52e5224f5a
7 changed files with 375 additions and 346 deletions

View file

@ -155,7 +155,7 @@ func Main() int {
prometheus.MustRegister(configSuccess) prometheus.MustRegister(configSuccess)
prometheus.MustRegister(configSuccessTime) prometheus.MustRegister(configSuccessTime)
go ruleManager.Run() // go ruleManager.Run()
defer ruleManager.Stop() defer ruleManager.Stop()
go notificationHandler.Run() go notificationHandler.Run()

View file

@ -168,6 +168,9 @@ func (n *Handler) Run() {
alerts := n.nextBatch() alerts := n.nextBatch()
if len(alerts) == 0 {
continue
}
if n.opts.AlertmanagerURL == "" { if n.opts.AlertmanagerURL == "" {
log.Warn("No AlertManager configured, not dispatching %d alerts", len(alerts)) log.Warn("No AlertManager configured, not dispatching %d alerts", len(alerts))
n.dropped.Add(float64(len(alerts))) n.dropped.Add(float64(len(alerts)))

View file

@ -46,52 +46,21 @@ func (s AlertState) String() string {
return "pending" return "pending"
case StateFiring: case StateFiring:
return "firing" return "firing"
default:
panic("undefined")
} }
panic(fmt.Errorf("unknown alert state: %v", s))
} }
const ( const (
// StateInactive is the state of an alert that is either firing nor pending.
StateInactive AlertState = iota StateInactive AlertState = iota
// StatePending is the state of an alert that has been active for less than
// the configured threshold duration.
StatePending StatePending
// StateFiring is the state of an alert that has been active for longer than
// the configured threshold duration.
StateFiring StateFiring
) )
// Alert is used to track active (pending/firing) alerts over time. type alertInstance struct {
type Alert struct { metric model.Metric
// The name of the alert. value model.SampleValue
Name string state AlertState
// The vector element labelset triggering this alert. activeSince model.Time
Labels model.LabelSet
// The state of the alert (Pending or Firing).
State AlertState
// The time when the alert first transitioned into Pending state.
ActiveSince model.Time
// The value of the alert expression for this vector element.
Value model.SampleValue
}
// sample returns a Sample suitable for recording the alert.
func (a Alert) sample(timestamp model.Time, value model.SampleValue) *model.Sample {
recordedMetric := make(model.Metric, len(a.Labels)+3)
for label, value := range a.Labels {
recordedMetric[label] = value
}
recordedMetric[model.MetricNameLabel] = alertMetricName
recordedMetric[alertNameLabel] = model.LabelValue(a.Name)
recordedMetric[alertStateLabel] = model.LabelValue(a.State.String())
return &model.Sample{
Metric: recordedMetric,
Value: value,
Timestamp: timestamp,
}
} }
// An AlertingRule generates alerts from its vector expression. // An AlertingRule generates alerts from its vector expression.
@ -109,10 +78,10 @@ type AlertingRule struct {
annotations model.LabelSet annotations model.LabelSet
// Protects the below. // Protects the below.
mutex sync.Mutex mtx sync.Mutex
// A map of alerts which are currently active (Pending or Firing), keyed by // A map of alerts which are currently active (Pending or Firing), keyed by
// the fingerprint of the labelset they correspond to. // the fingerprint of the labelset they correspond to.
activeAlerts map[model.Fingerprint]*Alert active map[model.Fingerprint]*alertInstance
} }
// NewAlertingRule constructs a new AlertingRule. // NewAlertingRule constructs a new AlertingRule.
@ -123,8 +92,7 @@ func NewAlertingRule(name string, vec promql.Expr, hold time.Duration, lbls, ann
holdDuration: hold, holdDuration: hold,
labels: lbls, labels: lbls,
annotations: anns, annotations: anns,
active: map[model.Fingerprint]*alertInstance{},
activeAlerts: map[model.Fingerprint]*Alert{},
} }
} }
@ -133,65 +101,127 @@ func (rule *AlertingRule) Name() string {
return rule.name return rule.name
} }
func (r *AlertingRule) sample(ai *alertInstance, ts model.Time, set bool) *model.Sample {
// Build alert labels in order they can be overwritten.
metric := model.Metric(r.labels.Clone())
for ln, lv := range ai.metric {
metric[ln] = lv
}
metric[model.MetricNameLabel] = alertMetricName
metric[model.AlertNameLabel] = model.LabelValue(r.name)
metric[alertStateLabel] = model.LabelValue(ai.state.String())
s := &model.Sample{
Metric: metric,
Timestamp: ts,
Value: 0,
}
if set {
s.Value = 1
}
return s
}
// eval evaluates the rule expression and then creates pending alerts and fires // eval evaluates the rule expression and then creates pending alerts and fires
// or removes previously pending alerts accordingly. // or removes previously pending alerts accordingly.
func (rule *AlertingRule) eval(timestamp model.Time, engine *promql.Engine) (model.Vector, error) { func (r *AlertingRule) eval(ts model.Time, engine *promql.Engine) (model.Vector, error) {
query, err := engine.NewInstantQuery(rule.vector.String(), timestamp) query, err := engine.NewInstantQuery(r.vector.String(), ts)
if err != nil { if err != nil {
return nil, err return nil, err
} }
exprResult, err := query.Exec().Vector() res, err := query.Exec().Vector()
if err != nil { if err != nil {
return nil, err return nil, err
} }
rule.mutex.Lock() r.mtx.Lock()
defer rule.mutex.Unlock() defer r.mtx.Unlock()
// Create pending alerts for any new vector elements in the alert expression // Create pending alerts for any new vector elements in the alert expression
// or update the expression value for existing elements. // or update the expression value for existing elements.
resultFPs := map[model.Fingerprint]struct{}{} resultFPs := map[model.Fingerprint]struct{}{}
for _, sample := range exprResult {
fp := sample.Metric.Fingerprint() for _, smpl := range res {
fp := smpl.Metric.Fingerprint()
resultFPs[fp] = struct{}{} resultFPs[fp] = struct{}{}
if alert, ok := rule.activeAlerts[fp]; !ok { if ai, ok := r.active[fp]; ok {
labels := model.LabelSet(sample.Metric.Clone()) ai.value = smpl.Value
labels = labels.Merge(rule.labels)
if _, ok := labels[model.MetricNameLabel]; ok {
delete(labels, model.MetricNameLabel)
}
rule.activeAlerts[fp] = &Alert{
Name: rule.name,
Labels: labels,
State: StatePending,
ActiveSince: timestamp,
Value: sample.Value,
}
} else {
alert.Value = sample.Value
}
}
var vector model.Vector
// Check if any pending alerts should be removed or fire now. Write out alert timeseries.
for fp, activeAlert := range rule.activeAlerts {
if _, ok := resultFPs[fp]; !ok {
vector = append(vector, activeAlert.sample(timestamp, 0))
delete(rule.activeAlerts, fp)
continue continue
} }
if activeAlert.State == StatePending && timestamp.Sub(activeAlert.ActiveSince) >= rule.holdDuration { delete(smpl.Metric, model.MetricNameLabel)
vector = append(vector, activeAlert.sample(timestamp, 0))
activeAlert.State = StateFiring
}
vector = append(vector, activeAlert.sample(timestamp, 1)) r.active[fp] = &alertInstance{
metric: smpl.Metric,
activeSince: ts,
state: StatePending,
value: smpl.Value,
}
} }
return vector, nil var vec model.Vector
// Check if any pending alerts should be removed or fire now. Write out alert timeseries.
for fp, ai := range r.active {
if _, ok := resultFPs[fp]; !ok {
delete(r.active, fp)
vec = append(vec, r.sample(ai, ts, false))
continue
}
if ai.state != StateFiring && ts.Sub(ai.activeSince) >= r.holdDuration {
vec = append(vec, r.sample(ai, ts, false))
ai.state = StateFiring
}
vec = append(vec, r.sample(ai, ts, true))
}
return vec, nil
}
// Alert is the user-level representation of a single instance of an alerting rule.
type Alert struct {
State AlertState
Labels model.LabelSet
ActiveSince model.Time
Value model.SampleValue
}
func (r *AlertingRule) State() AlertState {
r.mtx.Lock()
defer r.mtx.Unlock()
maxState := StateInactive
for _, ai := range r.active {
if ai.state > maxState {
maxState = ai.state
}
}
return maxState
}
// ActiveAlerts returns a slice of active alerts.
func (r *AlertingRule) ActiveAlerts() []*Alert {
r.mtx.Lock()
defer r.mtx.Unlock()
alerts := make([]*Alert, 0, len(r.active))
for _, ai := range r.active {
labels := r.labels.Clone()
for ln, lv := range ai.metric {
labels[ln] = lv
}
alerts = append(alerts, &Alert{
State: ai.state,
Labels: labels,
ActiveSince: ai.activeSince,
Value: ai.value,
})
}
return alerts
} }
func (rule *AlertingRule) String() string { func (rule *AlertingRule) String() string {
@ -230,29 +260,3 @@ func (rule *AlertingRule) HTMLSnippet(pathPrefix string) template.HTML {
} }
return template.HTML(s) return template.HTML(s)
} }
// State returns the "maximum" state: firing > pending > inactive.
func (rule *AlertingRule) State() AlertState {
rule.mutex.Lock()
defer rule.mutex.Unlock()
maxState := StateInactive
for _, activeAlert := range rule.activeAlerts {
if activeAlert.State > maxState {
maxState = activeAlert.State
}
}
return maxState
}
// ActiveAlerts returns a slice of active alerts.
func (rule *AlertingRule) ActiveAlerts() []Alert {
rule.mutex.Lock()
defer rule.mutex.Unlock()
alerts := make([]Alert, 0, len(rule.activeAlerts))
for _, alert := range rule.activeAlerts {
alerts = append(alerts, *alert)
}
return alerts
}

View file

@ -39,9 +39,7 @@ import (
const ( const (
namespace = "prometheus" namespace = "prometheus"
ruleTypeLabel = "rule_type" ruleTypeLabel = "rule_type"
ruleTypeAlerting = "alerting"
ruleTypeRecording = "recording"
) )
var ( var (
@ -74,12 +72,18 @@ func init() {
prometheus.MustRegister(evalDuration) prometheus.MustRegister(evalDuration)
} }
type ruleType string
const (
ruleTypeAlert = "alerting"
ruleTypeRecording = "recording"
)
// A Rule encapsulates a vector expression which is evaluated at a specified // A Rule encapsulates a vector expression which is evaluated at a specified
// interval and acted upon (currently either recorded or used for alerting). // interval and acted upon (currently either recorded or used for alerting).
type Rule interface { type Rule interface {
// Name returns the name of the rule.
Name() string Name() string
// Eval evaluates the rule, including any associated recording or alerting actions. // eval evaluates the rule, including any associated recording or alerting actions.
eval(model.Time, *promql.Engine) (model.Vector, error) eval(model.Time, *promql.Engine) (model.Vector, error)
// String returns a human-readable string representation of the rule. // String returns a human-readable string representation of the rule.
String() string String() string
@ -88,306 +92,365 @@ type Rule interface {
HTMLSnippet(pathPrefix string) html_template.HTML HTMLSnippet(pathPrefix string) html_template.HTML
} }
// The Manager manages recording and alerting rules. type Group struct {
type Manager struct { name string
// Protects the rules list. interval time.Duration
sync.Mutex rules []Rule
rules []Rule opts *ManagerOptions
done chan bool done chan struct{}
terminated chan struct{}
interval time.Duration
queryEngine *promql.Engine
sampleAppender storage.SampleAppender
notificationHandler *notification.Handler
externalURL *url.URL
} }
// ManagerOptions bundles options for the Manager. func newGroup(name string, opts *ManagerOptions) *Group {
type ManagerOptions struct { return &Group{
EvaluationInterval time.Duration name: name,
QueryEngine *promql.Engine opts: opts,
done: make(chan struct{}),
NotificationHandler *notification.Handler terminated: make(chan struct{}),
SampleAppender storage.SampleAppender
ExternalURL *url.URL
}
// NewManager returns an implementation of Manager, ready to be started
// by calling the Run method.
func NewManager(o *ManagerOptions) *Manager {
manager := &Manager{
rules: []Rule{},
done: make(chan bool),
interval: o.EvaluationInterval,
sampleAppender: o.SampleAppender,
queryEngine: o.QueryEngine,
notificationHandler: o.NotificationHandler,
externalURL: o.ExternalURL,
} }
return manager
} }
// Run the rule manager's periodic rule evaluation. func (g *Group) run() {
func (m *Manager) Run() { defer close(g.terminated)
defer log.Info("Rule manager stopped.")
m.Lock() // Wait an initial amount to have consistently slotted intervals.
lastInterval := m.interval time.Sleep(g.offset())
m.Unlock()
ticker := time.NewTicker(lastInterval) iter := func() {
defer ticker.Stop() start := time.Now()
g.eval()
iterationDuration.Observe(float64(time.Since(start) / time.Millisecond))
}
iter()
tick := time.NewTicker(g.interval)
defer tick.Stop()
for { for {
// The outer select clause makes sure that m.done is looked at
// first. Otherwise, if m.runIteration takes longer than
// m.interval, there is only a 50% chance that m.done will be
// looked at before the next m.runIteration call happens.
select { select {
case <-m.done: case <-g.done:
return return
default: default:
select { select {
case <-ticker.C: case <-g.done:
start := time.Now()
m.runIteration()
iterationDuration.Observe(float64(time.Since(start) / time.Millisecond))
m.Lock()
if lastInterval != m.interval {
ticker.Stop()
ticker = time.NewTicker(m.interval)
lastInterval = m.interval
}
m.Unlock()
case <-m.done:
return return
case <-tick.C:
iter()
} }
} }
} }
} }
// Stop the rule manager's rule evaluation cycles. func (g *Group) stop() {
func (m *Manager) Stop() { close(g.done)
log.Info("Stopping rule manager...") <-g.terminated
m.done <- true
} }
func (m *Manager) sendAlertNotifications(rule *AlertingRule, timestamp model.Time) { func (g *Group) fingerprint() model.Fingerprint {
activeAlerts := rule.ActiveAlerts() l := model.LabelSet{"name": model.LabelValue(g.name)}
if len(activeAlerts) == 0 { return l.Fingerprint()
return }
func (g *Group) offset() time.Duration {
now := time.Now().UnixNano()
var (
base = now - (now % int64(g.interval))
offset = uint64(g.fingerprint()) % uint64(g.interval)
next = base + int64(offset)
)
if next < now {
next += int64(g.interval)
} }
return time.Duration(next - now)
}
alerts := make(model.Alerts, 0, len(activeAlerts)) func (g *Group) copyState(from *Group) {
for _, fromRule := range from.rules {
for _, aa := range activeAlerts { far, ok := fromRule.(*AlertingRule)
if aa.State != StateFiring { if !ok {
// BUG: In the future, make AlertManager support pending alerts?
continue continue
} }
for _, rule := range g.rules {
// Provide the alert information to the template. ar, ok := rule.(*AlertingRule)
l := map[string]string{} if !ok {
for k, v := range aa.Labels { continue
l[string(k)] = string(v) }
} if far.Name() == ar.Name() {
tmplData := struct { ar.active = far.active
Labels map[string]string
Value float64
}{
Labels: l,
Value: float64(aa.Value),
}
// Inject some convenience variables that are easier to remember for users
// who are not used to Go's templating system.
defs := "{{$labels := .Labels}}{{$value := .Value}}"
expand := func(text model.LabelValue) model.LabelValue {
tmpl := template.NewTemplateExpander(defs+string(text), "__alert_"+rule.Name(), tmplData, timestamp, m.queryEngine, m.externalURL.Path)
result, err := tmpl.Expand()
if err != nil {
result = err.Error()
log.Warnf("Error expanding alert template %v with data '%v': %v", rule.Name(), tmplData, err)
} }
return model.LabelValue(result)
} }
labels := aa.Labels.Clone()
labels[model.AlertNameLabel] = model.LabelValue(rule.Name())
annotations := rule.annotations.Clone()
for an, av := range rule.annotations {
annotations[an] = expand(av)
}
alerts = append(alerts, &model.Alert{
StartsAt: aa.ActiveSince.Time().Add(rule.holdDuration),
Labels: labels,
Annotations: annotations,
GeneratorURL: m.externalURL.String() + strutil.GraphLinkForExpression(rule.vector.String()),
})
} }
m.notificationHandler.Send(alerts...)
} }
func (m *Manager) runIteration() { func (g *Group) eval() {
now := model.Now() var (
wg := sync.WaitGroup{} now = model.Now()
wg sync.WaitGroup
)
m.Lock() for _, rule := range g.rules {
rulesSnapshot := make([]Rule, len(m.rules))
copy(rulesSnapshot, m.rules)
m.Unlock()
for _, rule := range rulesSnapshot {
wg.Add(1) wg.Add(1)
// BUG(julius): Look at fixing thundering herd. // BUG(julius): Look at fixing thundering herd.
go func(rule Rule) { go func(rule Rule) {
defer wg.Done() defer wg.Done()
start := time.Now() start := time.Now()
vector, err := rule.eval(now, m.queryEngine)
duration := time.Since(start)
vector, err := rule.eval(now, g.opts.QueryEngine)
if err != nil { if err != nil {
evalFailures.Inc() evalFailures.Inc()
log.Warnf("Error while evaluating rule %q: %s", rule, err) log.Warnf("Error while evaluating rule %q: %s", rule, err)
return
} }
var rtyp ruleType
switch r := rule.(type) { switch r := rule.(type) {
case *AlertingRule: case *AlertingRule:
m.sendAlertNotifications(r, now) rtyp = ruleTypeRecording
evalDuration.WithLabelValues(ruleTypeAlerting).Observe( g.sendAlerts(r, now)
float64(duration / time.Millisecond),
)
case *RecordingRule: case *RecordingRule:
evalDuration.WithLabelValues(ruleTypeRecording).Observe( rtyp = ruleTypeAlert
float64(duration / time.Millisecond),
)
default: default:
panic(fmt.Errorf("unknown rule type: %T", rule)) panic(fmt.Errorf("unknown rule type: %T", rule))
} }
evalDuration.WithLabelValues(string(rtyp)).Observe(
float64(time.Since(start) / time.Millisecond),
)
for _, s := range vector { for _, s := range vector {
m.sampleAppender.Append(s) g.opts.SampleAppender.Append(s)
} }
}(rule) }(rule)
} }
wg.Wait() wg.Wait()
} }
// transferAlertState makes a copy of the state of alerting rules and returns a function func (g *Group) sendAlerts(rule *AlertingRule, timestamp model.Time) error {
// that restores them in the current state. var alerts model.Alerts
func (m *Manager) transferAlertState() func() { for _, alert := range rule.ActiveAlerts() {
// Only send actually firing alerts.
alertingRules := map[string]*AlertingRule{} if alert.State != StateFiring {
for _, r := range m.rules { continue
if ar, ok := r.(*AlertingRule); ok {
alertingRules[ar.name] = ar
} }
// Provide the alert information to the template.
l := make(map[string]string, len(alert.Labels))
for k, v := range alert.Labels {
l[string(k)] = string(v)
}
tmplData := struct {
Labels map[string]string
Value float64
}{
Labels: l,
Value: float64(alert.Value),
}
// Inject some convenience variables that are easier to remember for users
// who are not used to Go's templating system.
defs := "{{$labels := .Labels}}{{$value := .Value}}"
expand := func(text model.LabelValue) model.LabelValue {
tmpl := template.NewTemplateExpander(
defs+string(text),
"__alert_"+rule.Name(),
tmplData,
timestamp,
g.opts.QueryEngine,
g.opts.ExternalURL.Path,
)
result, err := tmpl.Expand()
if err != nil {
result = fmt.Sprintf("<error expanding template: %s>", err)
log.Warnf("Error expanding alert template %v with data '%v': %s", rule.Name(), tmplData, err)
}
return model.LabelValue(result)
}
labels := make(model.LabelSet, len(alert.Labels)+1)
for ln, lv := range alert.Labels {
labels[ln] = expand(lv)
}
labels[model.AlertNameLabel] = model.LabelValue(rule.Name())
annotations := make(model.LabelSet, len(rule.annotations))
for an, av := range rule.annotations {
annotations[an] = expand(av)
}
alerts = append(alerts, &model.Alert{
StartsAt: alert.ActiveSince.Time().Add(rule.holdDuration),
Labels: labels,
Annotations: annotations,
GeneratorURL: g.opts.ExternalURL.String() + strutil.GraphLinkForExpression(rule.vector.String()),
})
} }
return func() { if len(alerts) > 0 {
// Restore alerting rule state. g.opts.NotificationHandler.Send(alerts...)
for _, r := range m.rules {
ar, ok := r.(*AlertingRule)
if !ok {
continue
}
if old, ok := alertingRules[ar.name]; ok {
ar.activeAlerts = old.activeAlerts
}
}
} }
return nil
}
// The Manager manages recording and alerting rules.
type Manager struct {
opts *ManagerOptions
groups map[string]*Group
mtx sync.RWMutex
}
// ManagerOptions bundles options for the Manager.
type ManagerOptions struct {
ExternalURL *url.URL
QueryEngine *promql.Engine
NotificationHandler *notification.Handler
SampleAppender storage.SampleAppender
}
// NewManager returns an implementation of Manager, ready to be started
// by calling the Run method.
func NewManager(o *ManagerOptions) *Manager {
manager := &Manager{
groups: map[string]*Group{},
opts: o,
}
return manager
}
// Stop the rule manager's rule evaluation cycles.
func (m *Manager) Stop() {
log.Info("Stopping rule manager...")
for _, eg := range m.groups {
eg.stop()
}
log.Info("Rule manager stopped.")
} }
// ApplyConfig updates the rule manager's state as the config requires. If // ApplyConfig updates the rule manager's state as the config requires. If
// loading the new rules failed the old rule set is restored. Returns true on success. // loading the new rules failed the old rule set is restored. Returns true on success.
func (m *Manager) ApplyConfig(conf *config.Config) bool { func (m *Manager) ApplyConfig(conf *config.Config) bool {
m.Lock() m.mtx.Lock()
defer m.Unlock() defer m.mtx.Unlock()
defer m.transferAlertState()()
success := true
m.interval = time.Duration(conf.GlobalConfig.EvaluationInterval)
rulesSnapshot := make([]Rule, len(m.rules))
copy(rulesSnapshot, m.rules)
m.rules = m.rules[:0]
// Get all rule files and load the groups they define.
var files []string var files []string
for _, pat := range conf.RuleFiles { for _, pat := range conf.RuleFiles {
fs, err := filepath.Glob(pat) fs, err := filepath.Glob(pat)
if err != nil { if err != nil {
// The only error can be a bad pattern. // The only error can be a bad pattern.
log.Errorf("Error retrieving rule files for %s: %s", pat, err) log.Errorf("Error retrieving rule files for %s: %s", pat, err)
success = false return false
} }
files = append(files, fs...) files = append(files, fs...)
} }
if err := m.loadRuleFiles(files...); err != nil {
// If loading the new rules failed, restore the old rule set. groups, err := m.loadGroups(files...)
m.rules = rulesSnapshot if err != nil {
log.Errorf("Error loading rules, previous rule set restored: %s", err) log.Errorf("Error loading rules, previous rule set restored: %s", err)
success = false return false
} }
return success var wg sync.WaitGroup
for _, newg := range groups {
// To be replaced with a configurable per-group interval.
newg.interval = time.Duration(conf.GlobalConfig.EvaluationInterval)
wg.Add(1)
// If there is an old group with the same identifier, stop it and wait for
// it to finish the current iteration. Then copy its into the new group.
oldg, ok := m.groups[newg.name]
delete(m.groups, newg.name)
go func(newg *Group) {
if ok {
oldg.stop()
newg.copyState(oldg)
}
go newg.run()
wg.Done()
}(newg)
}
// Stop remaining old groups.
for _, oldg := range m.groups {
oldg.stop()
}
wg.Wait()
m.groups = groups
return true
} }
// loadRuleFiles loads alerting and recording rules from the given files. func (m *Manager) loadGroups(filenames ...string) (map[string]*Group, error) {
func (m *Manager) loadRuleFiles(filenames ...string) error { groups := map[string]*Group{}
// Currently there is no group syntax implemented. Thus all rules
// are read into a single default group.
g := newGroup("default", m.opts)
groups[g.name] = g
for _, fn := range filenames { for _, fn := range filenames {
content, err := ioutil.ReadFile(fn) content, err := ioutil.ReadFile(fn)
if err != nil { if err != nil {
return err return nil, err
} }
stmts, err := promql.ParseStmts(string(content)) stmts, err := promql.ParseStmts(string(content))
if err != nil { if err != nil {
return fmt.Errorf("error parsing %s: %s", fn, err) return nil, fmt.Errorf("error parsing %s: %s", fn, err)
} }
for _, stmt := range stmts { for _, stmt := range stmts {
var rule Rule
switch r := stmt.(type) { switch r := stmt.(type) {
case *promql.AlertStmt: case *promql.AlertStmt:
rule := NewAlertingRule(r.Name, r.Expr, r.Duration, r.Labels, r.Annotations) rule = NewAlertingRule(r.Name, r.Expr, r.Duration, r.Labels, r.Annotations)
m.rules = append(m.rules, rule)
case *promql.RecordStmt: case *promql.RecordStmt:
rule := NewRecordingRule(r.Name, r.Expr, r.Labels) rule = NewRecordingRule(r.Name, r.Expr, r.Labels)
m.rules = append(m.rules, rule)
default: default:
panic("retrieval.Manager.LoadRuleFiles: unknown statement type") panic("retrieval.Manager.LoadRuleFiles: unknown statement type")
} }
g.rules = append(g.rules, rule)
} }
} }
return nil
return groups, nil
} }
// Rules returns the list of the manager's rules. // Rules returns the list of the manager's rules.
func (m *Manager) Rules() []Rule { func (m *Manager) Rules() []Rule {
m.Lock() m.mtx.RLock()
defer m.Unlock() defer m.mtx.RUnlock()
var rules []Rule
for _, g := range m.groups {
rules = append(rules, g.rules...)
}
rules := make([]Rule, len(m.rules))
copy(rules, m.rules)
return rules return rules
} }
// AlertingRules returns the list of the manager's alerting rules. // AlertingRules returns the list of the manager's alerting rules.
func (m *Manager) AlertingRules() []*AlertingRule { func (m *Manager) AlertingRules() []*AlertingRule {
m.Lock() m.mtx.RLock()
defer m.Unlock() defer m.mtx.RUnlock()
alerts := []*AlertingRule{} alerts := []*AlertingRule{}
for _, rule := range m.rules { for _, rule := range m.Rules() {
if alertingRule, ok := rule.(*AlertingRule); ok { if alertingRule, ok := rule.(*AlertingRule); ok {
alerts = append(alerts, alertingRule) alerts = append(alerts, alertingRule)
} }

View file

@ -15,7 +15,7 @@ package rules
import ( import (
"fmt" "fmt"
"reflect" // "reflect"
"strings" "strings"
"testing" "testing"
"time" "time"
@ -138,46 +138,3 @@ func annotateWithTime(lines []string, timestamp model.Time) []string {
} }
return annotatedLines return annotatedLines
} }
func TestTransferAlertState(t *testing.T) {
m := NewManager(&ManagerOptions{})
alert := &Alert{
Name: "testalert",
State: StateFiring,
}
arule := AlertingRule{
name: "test",
activeAlerts: map[model.Fingerprint]*Alert{},
}
aruleCopy := arule
m.rules = append(m.rules, &arule)
// Set an alert.
arule.activeAlerts[0] = alert
// Save state and get the restore function.
restore := m.transferAlertState()
// Remove arule from the rule list and add an unrelated rule and the
// stateless copy of arule.
m.rules = []Rule{
&AlertingRule{
name: "test_other",
activeAlerts: map[model.Fingerprint]*Alert{},
},
&aruleCopy,
}
// Apply the restore function.
restore()
if ar := m.rules[0].(*AlertingRule); len(ar.activeAlerts) != 0 {
t.Fatalf("unexpected alert for unrelated alerting rule")
}
if ar := m.rules[1].(*AlertingRule); !reflect.DeepEqual(ar.activeAlerts[0], alert) {
t.Fatalf("alert state was not restored")
}
}

View file

@ -40,7 +40,9 @@ func NewRecordingRule(name string, vector promql.Expr, labels model.LabelSet) *R
} }
// Name returns the rule name. // Name returns the rule name.
func (rule RecordingRule) Name() string { return rule.name } func (rule RecordingRule) Name() string {
return rule.name
}
// eval evaluates the rule and then overrides the metric names and labels accordingly. // eval evaluates the rule and then overrides the metric names and labels accordingly.
func (rule RecordingRule) eval(timestamp model.Time, engine *promql.Engine) (model.Vector, error) { func (rule RecordingRule) eval(timestamp model.Time, engine *promql.Engine) (model.Vector, error) {

View file

@ -134,7 +134,7 @@ func webUiTemplatesAlertsHtml() (*asset, error) {
return nil, err return nil, err
} }
info := bindataFileInfo{name: "web/ui/templates/alerts.html", size: 1707, mode: os.FileMode(420), modTime: time.Unix(1450269200, 0)} info := bindataFileInfo{name: "web/ui/templates/alerts.html", size: 1707, mode: os.FileMode(420), modTime: time.Unix(1450348618, 0)}
a := &asset{bytes: bytes, info: info} a := &asset{bytes: bytes, info: info}
return a, nil return a, nil
} }