Use annotation based alerts in rules/

This commit breaks the previously used alert format.
This commit is contained in:
Fabian Reinartz 2015-12-11 17:12:34 +01:00
parent af3a6661ed
commit 7c90db22ed
3 changed files with 21 additions and 36 deletions

View file

@ -105,12 +105,8 @@ type AlertingRule struct {
holdDuration time.Duration holdDuration time.Duration
// Extra labels to attach to the resulting alert sample vectors. // Extra labels to attach to the resulting alert sample vectors.
labels model.LabelSet labels model.LabelSet
// Short alert summary, suitable for email subjects. // Non-identifying key/value pairs.
summary string annotations model.LabelSet
// More detailed alert description.
description string
// A reference to a runbook for the alert.
runbook string
// Protects the below. // Protects the below.
mutex sync.Mutex mutex sync.Mutex
@ -120,23 +116,13 @@ type AlertingRule struct {
} }
// NewAlertingRule constructs a new AlertingRule. // NewAlertingRule constructs a new AlertingRule.
func NewAlertingRule( func NewAlertingRule(name string, vec promql.Expr, hold time.Duration, lbls, anns model.LabelSet) *AlertingRule {
name string,
vector promql.Expr,
holdDuration time.Duration,
labels model.LabelSet,
summary string,
description string,
runbook string,
) *AlertingRule {
return &AlertingRule{ return &AlertingRule{
name: name, name: name,
vector: vector, vector: vec,
holdDuration: holdDuration, holdDuration: hold,
labels: labels, labels: lbls,
summary: summary, annotations: anns,
description: description,
runbook: runbook,
activeAlerts: map[model.Fingerprint]*Alert{}, activeAlerts: map[model.Fingerprint]*Alert{},
} }
@ -217,9 +203,9 @@ func (rule *AlertingRule) String() string {
if len(rule.labels) > 0 { if len(rule.labels) > 0 {
s += fmt.Sprintf("\n\tWITH %s", rule.labels) s += fmt.Sprintf("\n\tWITH %s", rule.labels)
} }
s += fmt.Sprintf("\n\tSUMMARY %q", rule.summary) if len(rule.annotations) > 0 {
s += fmt.Sprintf("\n\tDESCRIPTION %q", rule.description) s += fmt.Sprintf("\n\tANNOTATIONS %s", rule.annotations)
s += fmt.Sprintf("\n\tRUNBOOK %q", rule.runbook) }
return s return s
} }
@ -239,9 +225,9 @@ func (rule *AlertingRule) HTMLSnippet(pathPrefix string) template.HTML {
if len(rule.labels) > 0 { if len(rule.labels) > 0 {
s += fmt.Sprintf("\n WITH %s", rule.labels) s += fmt.Sprintf("\n WITH %s", rule.labels)
} }
s += fmt.Sprintf("\n SUMMARY %q", rule.summary) if len(rule.annotations) > 0 {
s += fmt.Sprintf("\n DESCRIPTION %q", rule.description) s += fmt.Sprintf("\n ANNOTATIONS %s", rule.annotations)
s += fmt.Sprintf("\n RUNBOOK %q", rule.runbook) }
return template.HTML(s) return template.HTML(s)
} }

View file

@ -208,23 +208,22 @@ func (m *Manager) sendAlertNotifications(rule *AlertingRule, timestamp model.Tim
// who are not used to Go's templating system. // who are not used to Go's templating system.
defs := "{{$labels := .Labels}}{{$value := .Value}}" defs := "{{$labels := .Labels}}{{$value := .Value}}"
expand := func(text string) string { expand := func(text model.LabelValue) model.LabelValue {
tmpl := template.NewTemplateExpander(defs+text, "__alert_"+rule.Name(), tmplData, timestamp, m.queryEngine, m.externalURL.Path) tmpl := template.NewTemplateExpander(defs+string(text), "__alert_"+rule.Name(), tmplData, timestamp, m.queryEngine, m.externalURL.Path)
result, err := tmpl.Expand() result, err := tmpl.Expand()
if err != nil { if err != nil {
result = err.Error() result = err.Error()
log.Warnf("Error expanding alert template %v with data '%v': %v", rule.Name(), tmplData, err) log.Warnf("Error expanding alert template %v with data '%v': %v", rule.Name(), tmplData, err)
} }
return result return model.LabelValue(result)
} }
labels := aa.Labels.Clone() labels := aa.Labels.Clone()
labels[model.AlertNameLabel] = model.LabelValue(rule.Name()) labels[model.AlertNameLabel] = model.LabelValue(rule.Name())
annotations := model.LabelSet{ annotations := rule.annotations.Clone()
"summary": model.LabelValue(expand(rule.summary)), for an, av := range rule.annotations {
"description": model.LabelValue(expand(rule.description)), annotations[an] = expand(av)
"runbook": model.LabelValue(expand(rule.runbook)),
} }
alerts = append(alerts, &model.Alert{ alerts = append(alerts, &model.Alert{
@ -359,7 +358,7 @@ func (m *Manager) loadRuleFiles(filenames ...string) error {
for _, stmt := range stmts { for _, stmt := range stmts {
switch r := stmt.(type) { switch r := stmt.(type) {
case *promql.AlertStmt: case *promql.AlertStmt:
rule := NewAlertingRule(r.Name, r.Expr, r.Duration, r.Labels, r.Summary, r.Description, r.Runbook) rule := NewAlertingRule(r.Name, r.Expr, r.Duration, r.Labels, r.Annotations)
m.rules = append(m.rules, rule) m.rules = append(m.rules, rule)
case *promql.RecordStmt: case *promql.RecordStmt:
rule := NewRecordingRule(r.Name, r.Expr, r.Labels) rule := NewRecordingRule(r.Name, r.Expr, r.Labels)

View file

@ -56,7 +56,7 @@ func TestAlertingRule(t *testing.T) {
expr, expr,
time.Minute, time.Minute,
model.LabelSet{"severity": "critical"}, model.LabelSet{"severity": "critical"},
"summary", "description", "runbook", model.LabelSet{},
) )
var tests = []struct { var tests = []struct {