Merge pull request #1263 from prometheus/notify

Annotations for alerting rules

promql/ast.go

@@ -59,9 +59,7 @@ type AlertStmt struct {
 	Expr        Expr
 	Duration    time.Duration
 	Labels      model.LabelSet
-	Summary     string
+	Annotations model.LabelSet
-	Description string
-	Runbook     string
 }
 
 // EvalStmt holds an expression and information on the range it should

promql/lex.go

@@ -154,9 +154,7 @@ const (
 	itemIf
 	itemFor
 	itemWith
-	itemSummary
+	itemAnnotations
-	itemRunbook
-	itemDescription
 	itemKeepCommon
 	itemOffset
 	itemBy
@@ -186,9 +184,7 @@ var key = map[string]itemType{
 	"if":            itemIf,
 	"for":           itemFor,
 	"with":          itemWith,
-	"summary":       itemSummary,
+	"annotations":   itemAnnotations,
-	"runbook":       itemRunbook,
-	"description":   itemDescription,
 	"offset":        itemOffset,
 	"by":            itemBy,
 	"keeping_extra": itemKeepCommon,

promql/lex_test.go

@@ -241,14 +241,8 @@ var tests = []struct {
 		input:    "with",
 		expected: []item{{itemWith, 0, "with"}},
 	}, {
-		input:    "description",
+		input:    "annotations",
-		expected: []item{{itemDescription, 0, "description"}},
+		expected: []item{{itemAnnotations, 0, "annotations"}},
-	}, {
-		input:    "summary",
-		expected: []item{{itemSummary, 0, "summary"}},
-	}, {
-		input:    "runbook",
-		expected: []item{{itemRunbook, 0, "runbook"}},
 	}, {
 		input:    "offset",
 		expected: []item{{itemOffset, 0, "offset"}},

promql/parse.go

@@ -357,9 +357,9 @@ func (p *parser) stmt() Statement {
 
 // alertStmt parses an alert rule.
 //
-//		ALERT name IF expr [FOR duration] [WITH label_set]
+//		ALERT name IF expr [FOR duration]
-//			SUMMARY "summary"
+//			[WITH label_set]
-//			DESCRIPTION "description"
+//			[ANNOTATIONS label_set]
 //
 func (p *parser) alertStmt() *AlertStmt {
 	const ctx = "alert statement"
@@ -389,44 +389,10 @@ func (p *parser) alertStmt() *AlertStmt {
 		lset = p.labelSet()
 	}
 
-	var (
+	annotations := model.LabelSet{}
-		hasSum, hasDesc, hasRunbook bool
+	if p.peek().typ == itemAnnotations {
-		sum, desc, runbook          string
+		p.expect(itemAnnotations, ctx)
-	)
+		annotations = p.labelSet()
-Loop:
-	for {
-		switch p.next().typ {
-		case itemSummary:
-			if hasSum {
-				p.errorf("summary must not be defined twice")
-			}
-			hasSum = true
-			sum = p.unquoteString(p.expect(itemString, ctx).val)
-
-		case itemDescription:
-			if hasDesc {
-				p.errorf("description must not be defined twice")
-			}
-			hasDesc = true
-			desc = p.unquoteString(p.expect(itemString, ctx).val)
-
-		case itemRunbook:
-			if hasRunbook {
-				p.errorf("runbook must not be defined twice")
-			}
-			hasRunbook = true
-			runbook = p.unquoteString(p.expect(itemString, ctx).val)
-
-		default:
-			p.backup()
-			break Loop
-		}
-	}
-	if sum == "" {
-		p.errorf("alert summary missing")
-	}
-	if desc == "" {
-		p.errorf("alert description missing")
 	}
 
 	return &AlertStmt{
@@ -434,9 +400,7 @@ Loop:
 		Expr:        expr,
 		Duration:    duration,
 		Labels:      lset,
-		Summary:     sum,
+		Annotations: annotations,
-		Description: desc,
-		Runbook:     runbook,
 	}
 }
 
@@ -874,11 +838,20 @@ func (p *parser) labelMatchers(operators ...itemType) metric.LabelMatchers {
 
 		matchers = append(matchers, m)
 
+		if p.peek().typ == itemIdentifier {
+			p.errorf("missing comma before next identifier %q", p.peek().val)
+		}
+
 		// Terminate list if last matcher.
 		if p.peek().typ != itemComma {
 			break
 		}
 		p.next()
+
+		// Allow comma after each item in a multi-line listing.
+		if p.peek().typ == itemRightBrace {
+			break
+		}
 	}
 
 	p.expect(itemRightBrace, ctx)

promql/parse_test.go

@@ -1153,15 +1153,19 @@ var testStatement = []struct {
 			    service = "testservice"
 			    # ... more fields here ...
 			  }
-			  SUMMARY "Global request rate low"
+			  ANNOTATIONS {
-			  DESCRIPTION "The global request rate is low"
+			    summary     = "Global request rate low",
+			    description = "The global request rate is low"
+			  }
 
 			foo = bar{label1="value1"}
 
 			ALERT BazAlert IF foo > 10
-			  DESCRIPTION "BazAlert"
+			  ANNOTATIONS {
-			  RUNBOOK     "http://my.url"
+			    description = "BazAlert",
-			  SUMMARY     "Baz"
+			    runbook     = "http://my.url",
+			    summary     = "Baz",
+			  }
 		`,
 		expected: Statements{
 			&RecordStmt{
@@ -1198,8 +1202,10 @@ var testStatement = []struct {
 				}},
 				Labels:   model.LabelSet{"service": "testservice"},
 				Duration: 5 * time.Minute,
-				Summary:     "Global request rate low",
+				Annotations: model.LabelSet{
-				Description: "The global request rate is low",
+					"summary":     "Global request rate low",
+					"description": "The global request rate is low",
+				},
 			},
 			&RecordStmt{
 				Name: "foo",
@@ -1225,9 +1231,11 @@ var testStatement = []struct {
 					RHS: &NumberLiteral{10},
 				},
 				Labels: model.LabelSet{},
-				Summary:     "Baz",
+				Annotations: model.LabelSet{
-				Description: "BazAlert",
+					"summary":     "Baz",
-				Runbook:     "http://my.url",
+					"description": "BazAlert",
+					"runbook":     "http://my.url",
+				},
 			},
 		},
 	}, {
@@ -1248,8 +1256,10 @@ var testStatement = []struct {
 		},
 	}, {
 		input: `ALERT SomeName IF some_metric > 1 
-			SUMMARY "Global request rate low"
+			ANNOTATIONS {
-			DESCRIPTION "The global request rate is low"
+			  summary = "Global request rate low",
+			  description = "The global request rate is low"
+			}
 		`,
 		expected: Statements{
 			&AlertStmt{
@@ -1265,8 +1275,10 @@ var testStatement = []struct {
 					RHS: &NumberLiteral{1},
 				},
 				Labels: model.LabelSet{},
-				Summary:     "Global request rate low",
+				Annotations: model.LabelSet{
-				Description: "The global request rate is low",
+					"summary":     "Global request rate low",
+					"description": "The global request rate is low",
+				},
 			},
 		},
 	}, {
@@ -1276,8 +1288,10 @@ var testStatement = []struct {
 			    service = "testservice"
 			    # ... more fields here ... 
 			  }
-			  SUMMARY "Global request rate low"
+			  ANNOTATIONS {
-			  DESCRIPTION "The global request rate is low"
+			    summary = "Global request rate low"
+			    description = "The global request rate is low"
+			  }
 	  	`,
 		fail: true,
 	}, {
@@ -1323,16 +1337,6 @@ var testStatement = []struct {
 			DESCRIPTION "The global request rate is low"
 		`,
 		fail: true,
-	}, {
-		input: `ALERT SomeName IF some_metric > 1 WITH {} 
-			SUMMARY "Global request rate low"
-		`,
-		fail: true,
-	}, {
-		input: `ALERT SomeName IF some_metric > 1 
-			DESCRIPTION "The global request rate is low"
-		`,
-		fail: true,
 	},
 	// Fuzzing regression tests.
 	{

promql/printer.go

@@ -109,8 +109,9 @@ func (node *AlertStmt) String() string {
 	if len(node.Labels) > 0 {
 		s += fmt.Sprintf("\n\tWITH %s", node.Labels)
 	}
-	s += fmt.Sprintf("\n\tSUMMARY %q", node.Summary)
+	if len(node.Annotations) > 0 {
-	s += fmt.Sprintf("\n\tDESCRIPTION %q", node.Description)
+		s += fmt.Sprintf("\n\tANNOTATIONS %s", node.Labels)
+	}
 	return s
 }
 

rules/alerting.go

@@ -105,12 +105,8 @@ type AlertingRule struct {
 	holdDuration time.Duration
 	// Extra labels to attach to the resulting alert sample vectors.
 	labels model.LabelSet
-	// Short alert summary, suitable for email subjects.
+	// Non-identifying key/value pairs.
-	summary string
+	annotations model.LabelSet
-	// More detailed alert description.
-	description string
-	// A reference to a runbook for the alert.
-	runbook string
 
 	// Protects the below.
 	mutex sync.Mutex
@@ -120,23 +116,13 @@ type AlertingRule struct {
 }
 
 // NewAlertingRule constructs a new AlertingRule.
-func NewAlertingRule(
+func NewAlertingRule(name string, vec promql.Expr, hold time.Duration, lbls, anns model.LabelSet) *AlertingRule {
-	name string,
-	vector promql.Expr,
-	holdDuration time.Duration,
-	labels model.LabelSet,
-	summary string,
-	description string,
-	runbook string,
-) *AlertingRule {
 	return &AlertingRule{
 		name:         name,
-		vector:       vector,
+		vector:       vec,
-		holdDuration: holdDuration,
+		holdDuration: hold,
-		labels:       labels,
+		labels:       lbls,
-		summary:      summary,
+		annotations:  anns,
-		description:  description,
-		runbook:      runbook,
 
 		activeAlerts: map[model.Fingerprint]*Alert{},
 	}
@@ -217,9 +203,9 @@ func (rule *AlertingRule) String() string {
 	if len(rule.labels) > 0 {
 		s += fmt.Sprintf("\n\tWITH %s", rule.labels)
 	}
-	s += fmt.Sprintf("\n\tSUMMARY %q", rule.summary)
+	if len(rule.annotations) > 0 {
-	s += fmt.Sprintf("\n\tDESCRIPTION %q", rule.description)
+		s += fmt.Sprintf("\n\tANNOTATIONS %s", rule.annotations)
-	s += fmt.Sprintf("\n\tRUNBOOK %q", rule.runbook)
+	}
 	return s
 }
 
@@ -239,9 +225,9 @@ func (rule *AlertingRule) HTMLSnippet(pathPrefix string) template.HTML {
 	if len(rule.labels) > 0 {
 		s += fmt.Sprintf("\n  WITH %s", rule.labels)
 	}
-	s += fmt.Sprintf("\n  SUMMARY %q", rule.summary)
+	if len(rule.annotations) > 0 {
-	s += fmt.Sprintf("\n  DESCRIPTION %q", rule.description)
+		s += fmt.Sprintf("\n  ANNOTATIONS %s", rule.annotations)
-	s += fmt.Sprintf("\n  RUNBOOK %q", rule.runbook)
+	}
 	return template.HTML(s)
 }
 

rules/manager.go

@@ -208,23 +208,22 @@ func (m *Manager) sendAlertNotifications(rule *AlertingRule, timestamp model.Tim
 		// who are not used to Go's templating system.
 		defs := "{{$labels := .Labels}}{{$value := .Value}}"
 
-		expand := func(text string) string {
+		expand := func(text model.LabelValue) model.LabelValue {
-			tmpl := template.NewTemplateExpander(defs+text, "__alert_"+rule.Name(), tmplData, timestamp, m.queryEngine, m.externalURL.Path)
+			tmpl := template.NewTemplateExpander(defs+string(text), "__alert_"+rule.Name(), tmplData, timestamp, m.queryEngine, m.externalURL.Path)
 			result, err := tmpl.Expand()
 			if err != nil {
 				result = err.Error()
 				log.Warnf("Error expanding alert template %v with data '%v': %v", rule.Name(), tmplData, err)
 			}
-			return result
+			return model.LabelValue(result)
 		}
 
 		labels := aa.Labels.Clone()
 		labels[model.AlertNameLabel] = model.LabelValue(rule.Name())
 
-		annotations := model.LabelSet{
+		annotations := rule.annotations.Clone()
-			"summary":     model.LabelValue(expand(rule.summary)),
+		for an, av := range rule.annotations {
-			"description": model.LabelValue(expand(rule.description)),
+			annotations[an] = expand(av)
-			"runbook":     model.LabelValue(expand(rule.runbook)),
 		}
 
 		alerts = append(alerts, &model.Alert{
@@ -359,7 +358,7 @@ func (m *Manager) loadRuleFiles(filenames ...string) error {
 		for _, stmt := range stmts {
 			switch r := stmt.(type) {
 			case *promql.AlertStmt:
-				rule := NewAlertingRule(r.Name, r.Expr, r.Duration, r.Labels, r.Summary, r.Description, r.Runbook)
+				rule := NewAlertingRule(r.Name, r.Expr, r.Duration, r.Labels, r.Annotations)
 				m.rules = append(m.rules, rule)
 			case *promql.RecordStmt:
 				rule := NewRecordingRule(r.Name, r.Expr, r.Labels)

rules/manager_test.go

@@ -56,7 +56,7 @@ func TestAlertingRule(t *testing.T) {
 		expr,
 		time.Minute,
 		model.LabelSet{"severity": "critical"},
-		"summary", "description", "runbook",
+		model.LabelSet{},
 	)
 
 	var tests = []struct {