Merge pull request #8878 from LeviHarrison/external-address-alerts

Access external address in alerts

cmd/prometheus/main.go

@@ -527,6 +527,9 @@ func main() {
 		conntrack.DialWithTracing(),
 	)
 
+	// This is passed to ruleManager.Update().
+	var externalURL = cfg.web.ExternalURL.String()
+
 	reloaders := []reloader{
 		{
 			name:     "remote_storage",
@@ -592,6 +595,7 @@ func main() {
 					time.Duration(cfg.GlobalConfig.EvaluationInterval),
 					files,
 					cfg.GlobalConfig.ExternalLabels,
+					externalURL,
 				)
 			},
 		},

cmd/promtool/rules.go

@@ -68,7 +68,7 @@ func newRuleImporter(logger log.Logger, config ruleImporterConfig, apiClient que
 
 // loadGroups parses groups from a list of recording rule files.
 func (importer *ruleImporter) loadGroups(ctx context.Context, filenames []string) (errs []error) {
-	groups, errs := importer.ruleManager.LoadGroups(importer.config.evalInterval, labels.Labels{}, filenames...)
+	groups, errs := importer.ruleManager.LoadGroups(importer.config.evalInterval, labels.Labels{}, "", filenames...)
 	if errs != nil {
 		return errs
 	}

cmd/promtool/unittest.go

@@ -146,6 +146,7 @@ type testGroup struct {
 	AlertRuleTests  []alertTestCase  `yaml:"alert_rule_test,omitempty"`
 	PromqlExprTests []promqlTestCase `yaml:"promql_expr_test,omitempty"`
 	ExternalLabels  labels.Labels    `yaml:"external_labels,omitempty"`
+	ExternalURL     string           `yaml:"external_url,omitempty"`
 	TestGroupName   string           `yaml:"name,omitempty"`
 }
 
@@ -168,7 +169,7 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
 		Logger:     log.NewNopLogger(),
 	}
 	m := rules.NewManager(opts)
-	groupsMap, ers := m.LoadGroups(time.Duration(tg.Interval), tg.ExternalLabels, ruleFiles...)
+	groupsMap, ers := m.LoadGroups(time.Duration(tg.Interval), tg.ExternalLabels, tg.ExternalURL, ruleFiles...)
 	if ers != nil {
 		return ers
 	}

docs/configuration/template_reference.md

@@ -90,9 +90,9 @@ parameterize templates, and have a few other differences.
 
 ### Alert field templates
 
-`.Value`, `.Labels`, and `.ExternalLabels` contain the alert value, the alert
+`.Value`, `.Labels`, `.ExternalLabels`, and `.ExternalURL` contain the alert value, the alert
-labels, and the globally configured external labels, respectively. They are
+labels, the globally configured external labels, and the external URL (configured with `--web.external-url`) respectively. They are
-also exposed as the `$value`, `$labels`, and `$externalLabels` variables for
+also exposed as the `$value`, `$labels`, `$externalLabels`, and `$externalURL` variables for
 convenience.
 
 ### Console templates

docs/configuration/unit_testing_rules.md

@@ -59,6 +59,10 @@ promql_expr_test:
 # External labels accessible to the alert template.
 external_labels:
   [ <labelname>: <string> ... ]
+
+# External URL accessible to the alert template.
+# Usually set using --web.external-url.
+  [ external_url: <string> ]
 ```
 
 ### `<series>`

pkg/rulefmt/rulefmt.go

@@ -223,10 +223,11 @@ func testTemplateParsing(rl *RuleNode) (errs []error) {
 	}
 
 	// Trying to parse templates.
-	tmplData := template.AlertTemplateData(map[string]string{}, map[string]string{}, 0)
+	tmplData := template.AlertTemplateData(map[string]string{}, map[string]string{}, "", 0)
 	defs := []string{
 		"{{$labels := .Labels}}",
 		"{{$externalLabels := .ExternalLabels}}",
+		"{{$externalURL := .ExternalURL}}",
 		"{{$value := .Value}}",
 	}
 	parseTest := func(text string) error {

rules/alerting.go

@@ -121,6 +121,8 @@ type AlertingRule struct {
 	annotations labels.Labels
 	// External labels from the global config.
 	externalLabels map[string]string
+	// The external URL from the --web.external-url flag.
+	externalURL string
 	// true if old state has been restored. We start persisting samples for ALERT_FOR_STATE
 	// only after the restoration.
 	restored bool
@@ -144,7 +146,7 @@ type AlertingRule struct {
 // NewAlertingRule constructs a new AlertingRule.
 func NewAlertingRule(
 	name string, vec parser.Expr, hold time.Duration,
-	labels, annotations, externalLabels labels.Labels,
+	labels, annotations, externalLabels labels.Labels, externalURL string,
 	restored bool, logger log.Logger,
 ) *AlertingRule {
 	el := make(map[string]string, len(externalLabels))
@@ -159,6 +161,7 @@ func NewAlertingRule(
 		labels:         labels,
 		annotations:    annotations,
 		externalLabels: el,
+		externalURL:    externalURL,
 		health:         HealthUnknown,
 		active:         map[uint64]*Alert{},
 		logger:         logger,
@@ -318,12 +321,13 @@ func (r *AlertingRule) Eval(ctx context.Context, ts time.Time, query QueryFunc,
 			l[lbl.Name] = lbl.Value
 		}
 
-		tmplData := template.AlertTemplateData(l, r.externalLabels, smpl.V)
+		tmplData := template.AlertTemplateData(l, r.externalLabels, r.externalURL, smpl.V)
 		// Inject some convenience variables that are easier to remember for users
 		// who are not used to Go's templating system.
 		defs := []string{
 			"{{$labels := .Labels}}",
 			"{{$externalLabels := .ExternalLabels}}",
+			"{{$externalURL := .ExternalURL}}",
 			"{{$value := .Value}}",
 		}
 

rules/alerting_test.go

@@ -32,7 +32,7 @@ import (
 func TestAlertingRuleHTMLSnippet(t *testing.T) {
 	expr, err := parser.ParseExpr(`foo{html="<b>BOLD<b>"}`)
 	require.NoError(t, err)
-	rule := NewAlertingRule("testrule", expr, 0, labels.FromStrings("html", "<b>BOLD</b>"), labels.FromStrings("html", "<b>BOLD</b>"), nil, false, nil)
+	rule := NewAlertingRule("testrule", expr, 0, labels.FromStrings("html", "<b>BOLD</b>"), labels.FromStrings("html", "<b>BOLD</b>"), nil, "", false, nil)
 
 	const want = template.HTML(`alert: <a href="/test/prefix/graph?g0.expr=ALERTS%7Balertname%3D%22testrule%22%7D&g0.tab=1">testrule</a>
 expr: <a href="/test/prefix/graph?g0.expr=foo%7Bhtml%3D%22%3Cb%3EBOLD%3Cb%3E%22%7D&g0.tab=1">foo{html=&#34;&lt;b&gt;BOLD&lt;b&gt;&#34;}</a>
@@ -79,7 +79,7 @@ func TestAlertingRuleState(t *testing.T) {
 	}
 
 	for i, test := range tests {
-		rule := NewAlertingRule(test.name, nil, 0, nil, nil, nil, true, nil)
+		rule := NewAlertingRule(test.name, nil, 0, nil, nil, nil, "", true, nil)
 		rule.active = test.active
 		got := rule.State()
 		require.Equal(t, test.want, got, "test case %d unexpected AlertState, want:%d got:%d", i, test.want, got)
@@ -107,7 +107,7 @@ func TestAlertingRuleLabelsUpdate(t *testing.T) {
 		// If an alert is going back and forth between two label values it will never fire.
 		// Instead, you should write two alerts with constant labels.
 		labels.FromStrings("severity", "{{ if lt $value 80.0 }}critical{{ else }}warning{{ end }}"),
-		nil, nil, true, nil,
+		nil, nil, "", true, nil,
 	)
 
 	results := []promql.Vector{
@@ -208,6 +208,7 @@ func TestAlertingRuleExternalLabelsInTemplate(t *testing.T) {
 		labels.FromStrings("templated_label", "There are {{ len $externalLabels }} external Labels, of which foo is {{ $externalLabels.foo }}."),
 		nil,
 		nil,
+		"",
 		true, log.NewNopLogger(),
 	)
 	ruleWithExternalLabels := NewAlertingRule(
@@ -217,6 +218,7 @@ func TestAlertingRuleExternalLabelsInTemplate(t *testing.T) {
 		labels.FromStrings("templated_label", "There are {{ len $externalLabels }} external Labels, of which foo is {{ $externalLabels.foo }}."),
 		nil,
 		labels.FromStrings("foo", "bar", "dings", "bums"),
+		"",
 		true, log.NewNopLogger(),
 	)
 	result := promql.Vector{
@@ -280,6 +282,100 @@ func TestAlertingRuleExternalLabelsInTemplate(t *testing.T) {
 	require.Equal(t, result, filteredRes)
 }
 
+func TestAlertingRuleExternalURLInTemplate(t *testing.T) {
+	suite, err := promql.NewTest(t, `
+		load 1m
+			http_requests{job="app-server", instance="0"}	75 85 70 70
+	`)
+	require.NoError(t, err)
+	defer suite.Close()
+
+	require.NoError(t, suite.Run())
+
+	expr, err := parser.ParseExpr(`http_requests < 100`)
+	require.NoError(t, err)
+
+	ruleWithoutExternalURL := NewAlertingRule(
+		"ExternalURLDoesNotExist",
+		expr,
+		time.Minute,
+		labels.FromStrings("templated_label", "The external URL is {{ $externalURL }}."),
+		nil,
+		nil,
+		"",
+		true, log.NewNopLogger(),
+	)
+	ruleWithExternalURL := NewAlertingRule(
+		"ExternalURLExists",
+		expr,
+		time.Minute,
+		labels.FromStrings("templated_label", "The external URL is {{ $externalURL }}."),
+		nil,
+		nil,
+		"http://localhost:1234",
+		true, log.NewNopLogger(),
+	)
+	result := promql.Vector{
+		{
+			Metric: labels.FromStrings(
+				"__name__", "ALERTS",
+				"alertname", "ExternalURLDoesNotExist",
+				"alertstate", "pending",
+				"instance", "0",
+				"job", "app-server",
+				"templated_label", "The external URL is .",
+			),
+			Point: promql.Point{V: 1},
+		},
+		{
+			Metric: labels.FromStrings(
+				"__name__", "ALERTS",
+				"alertname", "ExternalURLExists",
+				"alertstate", "pending",
+				"instance", "0",
+				"job", "app-server",
+				"templated_label", "The external URL is http://localhost:1234.",
+			),
+			Point: promql.Point{V: 1},
+		},
+	}
+
+	evalTime := time.Unix(0, 0)
+	result[0].Point.T = timestamp.FromTime(evalTime)
+	result[1].Point.T = timestamp.FromTime(evalTime)
+
+	var filteredRes promql.Vector // After removing 'ALERTS_FOR_STATE' samples.
+	res, err := ruleWithoutExternalURL.Eval(
+		suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil,
+	)
+	require.NoError(t, err)
+	for _, smpl := range res {
+		smplName := smpl.Metric.Get("__name__")
+		if smplName == "ALERTS" {
+			filteredRes = append(filteredRes, smpl)
+		} else {
+			// If not 'ALERTS', it has to be 'ALERTS_FOR_STATE'.
+			require.Equal(t, "ALERTS_FOR_STATE", smplName)
+		}
+	}
+
+	res, err = ruleWithExternalURL.Eval(
+		suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil,
+	)
+	require.NoError(t, err)
+	for _, smpl := range res {
+		smplName := smpl.Metric.Get("__name__")
+		if smplName == "ALERTS" {
+			filteredRes = append(filteredRes, smpl)
+		} else {
+			// If not 'ALERTS', it has to be 'ALERTS_FOR_STATE'.
+			require.Equal(t, "ALERTS_FOR_STATE", smplName)
+		}
+	}
+
+	require.Equal(t, result, filteredRes)
+}
+
 func TestAlertingRuleEmptyLabelFromTemplate(t *testing.T) {
 	suite, err := promql.NewTest(t, `
 		load 1m
@@ -300,6 +396,7 @@ func TestAlertingRuleEmptyLabelFromTemplate(t *testing.T) {
 		labels.FromStrings("empty_label", ""),
 		nil,
 		nil,
+		"",
 		true, log.NewNopLogger(),
 	)
 	result := promql.Vector{
@@ -360,6 +457,7 @@ func TestAlertingRuleDuplicate(t *testing.T) {
 		labels.FromStrings("test", "test"),
 		nil,
 		nil,
+		"",
 		true, log.NewNopLogger(),
 	)
 	_, err := rule.Eval(ctx, now, EngineQueryFunc(engine, storage), nil)

rules/manager.go

@@ -946,11 +946,11 @@ func (m *Manager) Stop() {
 
 // Update the rule manager's state as the config requires. If
 // loading the new rules failed the old rule set is restored.
-func (m *Manager) Update(interval time.Duration, files []string, externalLabels labels.Labels) error {
+func (m *Manager) Update(interval time.Duration, files []string, externalLabels labels.Labels, externalURL string) error {
 	m.mtx.Lock()
 	defer m.mtx.Unlock()
 
-	groups, errs := m.LoadGroups(interval, externalLabels, files...)
+	groups, errs := m.LoadGroups(interval, externalLabels, externalURL, files...)
 	if errs != nil {
 		for _, e := range errs {
 			level.Error(m.logger).Log("msg", "loading groups failed", "err", e)
@@ -1034,7 +1034,7 @@ func (FileLoader) Parse(query string) (parser.Expr, error) { return parser.Parse
 
 // LoadGroups reads groups from a list of files.
 func (m *Manager) LoadGroups(
-	interval time.Duration, externalLabels labels.Labels, filenames ...string,
+	interval time.Duration, externalLabels labels.Labels, externalURL string, filenames ...string,
 ) (map[string]*Group, []error) {
 	groups := make(map[string]*Group)
 
@@ -1067,6 +1067,7 @@ func (m *Manager) LoadGroups(
 						labels.FromMap(r.Labels),
 						labels.FromMap(r.Annotations),
 						externalLabels,
+						externalURL,
 						m.restored,
 						log.With(m.logger, "alert", r.Alert),
 					))

rules/manager_test.go

@@ -64,7 +64,7 @@ func TestAlertingRule(t *testing.T) {
 		expr,
 		time.Minute,
 		labels.FromStrings("severity", "{{\"c\"}}ritical"),
-		nil, nil, true, nil,
+		nil, nil, "", true, nil,
 	)
 	result := promql.Vector{
 		{
@@ -205,7 +205,7 @@ func TestForStateAddSamples(t *testing.T) {
 		expr,
 		time.Minute,
 		labels.FromStrings("severity", "{{\"c\"}}ritical"),
-		nil, nil, true, nil,
+		nil, nil, "", true, nil,
 	)
 	result := promql.Vector{
 		{
@@ -379,7 +379,7 @@ func TestForStateRestore(t *testing.T) {
 		expr,
 		alertForDuration,
 		labels.FromStrings("severity", "critical"),
-		nil, nil, true, nil,
+		nil, nil, "", true, nil,
 	)
 
 	group := NewGroup(GroupOptions{
@@ -445,7 +445,7 @@ func TestForStateRestore(t *testing.T) {
 			expr,
 			alertForDuration,
 			labels.FromStrings("severity", "critical"),
-			nil, nil, false, nil,
+			nil, nil, "", false, nil,
 		)
 		newGroup := NewGroup(GroupOptions{
 			Name:          "default",
@@ -609,13 +609,13 @@ func readSeriesSet(ss storage.SeriesSet) (map[string][]promql.Point, error) {
 func TestCopyState(t *testing.T) {
 	oldGroup := &Group{
 		rules: []Rule{
-			NewAlertingRule("alert", nil, 0, nil, nil, nil, true, nil),
+			NewAlertingRule("alert", nil, 0, nil, nil, nil, "", true, nil),
 			NewRecordingRule("rule1", nil, nil),
 			NewRecordingRule("rule2", nil, nil),
 			NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v1"}}),
 			NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v2"}}),
 			NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v3"}}),
-			NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, nil, true, nil),
+			NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, nil, "", true, nil),
 		},
 		seriesInPreviousEval: []map[string]labels.Labels{
 			{},
@@ -634,10 +634,10 @@ func TestCopyState(t *testing.T) {
 			NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v0"}}),
 			NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v1"}}),
 			NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v2"}}),
-			NewAlertingRule("alert", nil, 0, nil, nil, nil, true, nil),
+			NewAlertingRule("alert", nil, 0, nil, nil, nil, "", true, nil),
 			NewRecordingRule("rule1", nil, nil),
-			NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v0"}}, nil, nil, true, nil),
+			NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v0"}}, nil, nil, "", true, nil),
-			NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, nil, true, nil),
+			NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, nil, "", true, nil),
 			NewRecordingRule("rule4", nil, nil),
 		},
 		seriesInPreviousEval: make([]map[string]labels.Labels, 8),
@@ -725,7 +725,7 @@ func TestUpdate(t *testing.T) {
 	ruleManager.start()
 	defer ruleManager.Stop()
 
-	err := ruleManager.Update(10*time.Second, files, nil)
+	err := ruleManager.Update(10*time.Second, files, nil, "")
 	require.NoError(t, err)
 	require.Greater(t, len(ruleManager.groups), 0, "expected non-empty rule groups")
 	ogs := map[string]*Group{}
@@ -736,7 +736,7 @@ func TestUpdate(t *testing.T) {
 		ogs[h] = g
 	}
 
-	err = ruleManager.Update(10*time.Second, files, nil)
+	err = ruleManager.Update(10*time.Second, files, nil, "")
 	require.NoError(t, err)
 	for h, g := range ruleManager.groups {
 		for _, actual := range g.seriesInPreviousEval {
@@ -755,7 +755,7 @@ func TestUpdate(t *testing.T) {
 	defer os.Remove(tmpFile.Name())
 	defer tmpFile.Close()
 
-	err = ruleManager.Update(10*time.Second, []string{tmpFile.Name()}, nil)
+	err = ruleManager.Update(10*time.Second, []string{tmpFile.Name()}, nil, "")
 	require.NoError(t, err)
 
 	for h, g := range ruleManager.groups {
@@ -826,7 +826,7 @@ func reloadAndValidate(rgs *rulefmt.RuleGroups, t *testing.T, tmpFile *os.File,
 	tmpFile.Seek(0, 0)
 	_, err = tmpFile.Write(bs)
 	require.NoError(t, err)
-	err = ruleManager.Update(10*time.Second, []string{tmpFile.Name()}, nil)
+	err = ruleManager.Update(10*time.Second, []string{tmpFile.Name()}, nil, "")
 	require.NoError(t, err)
 	for h, g := range ruleManager.groups {
 		if ogs[h] == g {
@@ -862,7 +862,7 @@ func TestNotify(t *testing.T) {
 
 	expr, err := parser.ParseExpr("a > 1")
 	require.NoError(t, err)
-	rule := NewAlertingRule("aTooHigh", expr, 0, labels.Labels{}, labels.Labels{}, nil, true, log.NewNopLogger())
+	rule := NewAlertingRule("aTooHigh", expr, 0, labels.Labels{}, labels.Labels{}, nil, "", true, log.NewNopLogger())
 	group := NewGroup(GroupOptions{
 		Name:          "alert",
 		Interval:      time.Second,
@@ -971,7 +971,7 @@ func TestMetricsUpdate(t *testing.T) {
 	}
 
 	for i, c := range cases {
-		err := ruleManager.Update(time.Second, c.files, nil)
+		err := ruleManager.Update(time.Second, c.files, nil, "")
 		require.NoError(t, err)
 		time.Sleep(2 * time.Second)
 		require.Equal(t, c.metrics, countMetrics(), "test %d: invalid count of metrics", i)
@@ -1045,7 +1045,7 @@ func TestGroupStalenessOnRemoval(t *testing.T) {
 
 	var totalStaleNaN int
 	for i, c := range cases {
-		err := ruleManager.Update(time.Second, c.files, nil)
+		err := ruleManager.Update(time.Second, c.files, nil, "")
 		require.NoError(t, err)
 		time.Sleep(3 * time.Second)
 		totalStaleNaN += c.staleNaN
@@ -1087,11 +1087,11 @@ func TestMetricsStalenessOnManagerShutdown(t *testing.T) {
 		}
 	}()
 
-	err := ruleManager.Update(2*time.Second, files, nil)
+	err := ruleManager.Update(2*time.Second, files, nil, "")
 	time.Sleep(4 * time.Second)
 	require.NoError(t, err)
 	start := time.Now()
-	err = ruleManager.Update(3*time.Second, files[:0], nil)
+	err = ruleManager.Update(3*time.Second, files[:0], nil, "")
 	require.NoError(t, err)
 	ruleManager.Stop()
 	stopped = true
@@ -1134,7 +1134,7 @@ func TestGroupHasAlertingRules(t *testing.T) {
 			group: &Group{
 				name: "HasAlertingRule",
 				rules: []Rule{
-					NewAlertingRule("alert", nil, 0, nil, nil, nil, true, nil),
+					NewAlertingRule("alert", nil, 0, nil, nil, nil, "", true, nil),
 					NewRecordingRule("record", nil, nil),
 				},
 			},

template/template.go

@@ -295,14 +295,16 @@ func NewTemplateExpander(
 }
 
 // AlertTemplateData returns the interface to be used in expanding the template.
-func AlertTemplateData(labels map[string]string, externalLabels map[string]string, value float64) interface{} {
+func AlertTemplateData(labels map[string]string, externalLabels map[string]string, externalURL string, value float64) interface{} {
 	return struct {
 		Labels         map[string]string
 		ExternalLabels map[string]string
+		ExternalURL    string
 		Value          float64
 	}{
 		Labels:         labels,
 		ExternalLabels: externalLabels,
+		ExternalURL:    externalURL,
 		Value:          value,
 	}
 }

web/api/v1/api_test.go

@@ -211,6 +211,7 @@ func (m rulesRetrieverMock) AlertingRules() []*rules.AlertingRule {
 		labels.Labels{},
 		labels.Labels{},
 		labels.Labels{},
+		"",
 		true,
 		log.NewNopLogger(),
 	)
@@ -221,6 +222,7 @@ func (m rulesRetrieverMock) AlertingRules() []*rules.AlertingRule {
 		labels.Labels{},
 		labels.Labels{},
 		labels.Labels{},
+		"",
 		true,
 		log.NewNopLogger(),
 	)