mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2024-12-25 05:34:05 -08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Merge pull request #8878 from LeviHarrison/external-address-alerts

Browse source 
Access external address in alerts
This commit is contained in:
Julien Pivotto 2021-06-04 09:14:25 +02:00 committed by GitHub
parent 609ba54b8f 7b18018012
commit 6cbed8c569
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 151 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
cmd/prometheus/main.go
View file

@ -527,6 +527,9 @@ func main() {
conntrack.DialWithTracing(),
)
// This is passed to ruleManager.Update().
var externalURL = cfg.web.ExternalURL.String()
reloaders := []reloader{
{
name: "remote_storage",
@ -592,6 +595,7 @@ func main() {
time.Duration(cfg.GlobalConfig.EvaluationInterval),
files,
cfg.GlobalConfig.ExternalLabels,
externalURL,
)
},
},

2
cmd/promtool/rules.go
View file

@ -68,7 +68,7 @@ func newRuleImporter(logger log.Logger, config ruleImporterConfig, apiClient que
// loadGroups parses groups from a list of recording rule files.
func (importer *ruleImporter) loadGroups(ctx context.Context, filenames []string) (errs []error) {
groups, errs := importer.ruleManager.LoadGroups(importer.config.evalInterval, labels.Labels{}, filenames...)
groups, errs := importer.ruleManager.LoadGroups(importer.config.evalInterval, labels.Labels{}, "", filenames...)
if errs != nil {
return errs
}

3
cmd/promtool/unittest.go
View file

@ -146,6 +146,7 @@ type testGroup struct {
AlertRuleTests []alertTestCase `yaml:"alert_rule_test,omitempty"`
PromqlExprTests []promqlTestCase `yaml:"promql_expr_test,omitempty"`
ExternalLabels labels.Labels `yaml:"external_labels,omitempty"`
ExternalURL string `yaml:"external_url,omitempty"`
TestGroupName string `yaml:"name,omitempty"`
}
@ -168,7 +169,7 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
Logger: log.NewNopLogger(),
}
m := rules.NewManager(opts)
groupsMap, ers := m.LoadGroups(time.Duration(tg.Interval), tg.ExternalLabels, ruleFiles...)
groupsMap, ers := m.LoadGroups(time.Duration(tg.Interval), tg.ExternalLabels, tg.ExternalURL, ruleFiles...)
if ers != nil {
return ers
}

6
docs/configuration/template_reference.md
View file

@ -90,9 +90,9 @@ parameterize templates, and have a few other differences.
### Alert field templates
`.Value`, `.Labels`, and `.ExternalLabels` contain the alert value, the alert
labels, and the globally configured external labels, respectively. They are
also exposed as the `$value`, `$labels`, and `$externalLabels` variables for
`.Value`, `.Labels`, `.ExternalLabels`, and `.ExternalURL` contain the alert value, the alert
labels, the globally configured external labels, and the external URL (configured with `--web.external-url`) respectively. They are
also exposed as the `$value`, `$labels`, `$externalLabels`, and `$externalURL` variables for
convenience.
### Console templates

4
docs/configuration/unit_testing_rules.md
View file

@ -59,6 +59,10 @@ promql_expr_test:
# External labels accessible to the alert template.
external_labels:
[ <labelname>: <string> ... ]
# External URL accessible to the alert template.
# Usually set using --web.external-url.
[ external_url: <string> ]
```
### `<series>`

3
pkg/rulefmt/rulefmt.go
View file

@ -223,10 +223,11 @@ func testTemplateParsing(rl *RuleNode) (errs []error) {
}
// Trying to parse templates.
tmplData := template.AlertTemplateData(map[string]string{}, map[string]string{}, 0)
tmplData := template.AlertTemplateData(map[string]string{}, map[string]string{}, "", 0)
defs := []string{
"{{$labels := .Labels}}",
"{{$externalLabels := .ExternalLabels}}",
"{{$externalURL := .ExternalURL}}",
"{{$value := .Value}}",
}
parseTest := func(text string) error {

8
rules/alerting.go
View file

@ -121,6 +121,8 @@ type AlertingRule struct {
annotations labels.Labels
// External labels from the global config.
externalLabels map[string]string
// The external URL from the --web.external-url flag.
externalURL string
// true if old state has been restored. We start persisting samples for ALERT_FOR_STATE
// only after the restoration.
restored bool
@ -144,7 +146,7 @@ type AlertingRule struct {
// NewAlertingRule constructs a new AlertingRule.
func NewAlertingRule(
name string, vec parser.Expr, hold time.Duration,
labels, annotations, externalLabels labels.Labels,
labels, annotations, externalLabels labels.Labels, externalURL string,
restored bool, logger log.Logger,
) *AlertingRule {
el := make(map[string]string, len(externalLabels))
@ -159,6 +161,7 @@ func NewAlertingRule(
labels: labels,
annotations: annotations,
externalLabels: el,
externalURL: externalURL,
health: HealthUnknown,
active: map[uint64]*Alert{},
logger: logger,
@ -318,12 +321,13 @@ func (r *AlertingRule) Eval(ctx context.Context, ts time.Time, query QueryFunc,
l[lbl.Name] = lbl.Value
}
tmplData := template.AlertTemplateData(l, r.externalLabels, smpl.V)
tmplData := template.AlertTemplateData(l, r.externalLabels, r.externalURL, smpl.V)
// Inject some convenience variables that are easier to remember for users
// who are not used to Go's templating system.
defs := []string{
"{{$labels := .Labels}}",
"{{$externalLabels := .ExternalLabels}}",
"{{$externalURL := .ExternalURL}}",
"{{$value := .Value}}",
}

104
rules/alerting_test.go
View file

@ -32,7 +32,7 @@ import (
func TestAlertingRuleHTMLSnippet(t *testing.T) {
expr, err := parser.ParseExpr(`foo{html="<b>BOLD<b>"}`)
require.NoError(t, err)
rule := NewAlertingRule("testrule", expr, 0, labels.FromStrings("html", "<b>BOLD</b>"), labels.FromStrings("html", "<b>BOLD</b>"), nil, false, nil)
rule := NewAlertingRule("testrule", expr, 0, labels.FromStrings("html", "<b>BOLD</b>"), labels.FromStrings("html", "<b>BOLD</b>"), nil, "", false, nil)
const want = template.HTML(`alert: <a href="/test/prefix/graph?g0.expr=ALERTS%7Balertname%3D%22testrule%22%7D&g0.tab=1">testrule</a>
expr: <a href="/test/prefix/graph?g0.expr=foo%7Bhtml%3D%22%3Cb%3EBOLD%3Cb%3E%22%7D&g0.tab=1">foo{html=&#34;&lt;b&gt;BOLD&lt;b&gt;&#34;}</a>
@ -79,7 +79,7 @@ func TestAlertingRuleState(t *testing.T) {
}
for i, test := range tests {
rule := NewAlertingRule(test.name, nil, 0, nil, nil, nil, true, nil)
rule := NewAlertingRule(test.name, nil, 0, nil, nil, nil, "", true, nil)
rule.active = test.active
got := rule.State()
require.Equal(t, test.want, got, "test case %d unexpected AlertState, want:%d got:%d", i, test.want, got)
@ -107,7 +107,7 @@ func TestAlertingRuleLabelsUpdate(t *testing.T) {
// If an alert is going back and forth between two label values it will never fire.
// Instead, you should write two alerts with constant labels.
labels.FromStrings("severity", "{{ if lt $value 80.0 }}critical{{ else }}warning{{ end }}"),
nil, nil, true, nil,
nil, nil, "", true, nil,
)
results := []promql.Vector{
@ -208,6 +208,7 @@ func TestAlertingRuleExternalLabelsInTemplate(t *testing.T) {
labels.FromStrings("templated_label", "There are {{ len $externalLabels }} external Labels, of which foo is {{ $externalLabels.foo }}."),
nil,
nil,
"",
true, log.NewNopLogger(),
)
ruleWithExternalLabels := NewAlertingRule(
@ -217,6 +218,7 @@ func TestAlertingRuleExternalLabelsInTemplate(t *testing.T) {
labels.FromStrings("templated_label", "There are {{ len $externalLabels }} external Labels, of which foo is {{ $externalLabels.foo }}."),
nil,
labels.FromStrings("foo", "bar", "dings", "bums"),
"",
true, log.NewNopLogger(),
)
result := promql.Vector{
@ -280,6 +282,100 @@ func TestAlertingRuleExternalLabelsInTemplate(t *testing.T) {
require.Equal(t, result, filteredRes)
}
func TestAlertingRuleExternalURLInTemplate(t *testing.T) {
suite, err := promql.NewTest(t, `
load 1m
http_requests{job="app-server", instance="0"} 75 85 70 70
`)
require.NoError(t, err)
defer suite.Close()
require.NoError(t, suite.Run())
expr, err := parser.ParseExpr(`http_requests < 100`)
require.NoError(t, err)
ruleWithoutExternalURL := NewAlertingRule(
"ExternalURLDoesNotExist",
expr,
time.Minute,
labels.FromStrings("templated_label", "The external URL is {{ $externalURL }}."),
nil,
nil,
"",
true, log.NewNopLogger(),
)
ruleWithExternalURL := NewAlertingRule(
"ExternalURLExists",
expr,
time.Minute,
labels.FromStrings("templated_label", "The external URL is {{ $externalURL }}."),
nil,
nil,
"http://localhost:1234",
true, log.NewNopLogger(),
)
result := promql.Vector{
{
Metric: labels.FromStrings(
"__name__", "ALERTS",
"alertname", "ExternalURLDoesNotExist",
"alertstate", "pending",
"instance", "0",
"job", "app-server",
"templated_label", "The external URL is .",
),
Point: promql.Point{V: 1},
},
{
Metric: labels.FromStrings(
"__name__", "ALERTS",
"alertname", "ExternalURLExists",
"alertstate", "pending",
"instance", "0",
"job", "app-server",
"templated_label", "The external URL is http://localhost:1234.",
),
Point: promql.Point{V: 1},
},
}
evalTime := time.Unix(0, 0)
result[0].Point.T = timestamp.FromTime(evalTime)
result[1].Point.T = timestamp.FromTime(evalTime)
var filteredRes promql.Vector // After removing 'ALERTS_FOR_STATE' samples.
res, err := ruleWithoutExternalURL.Eval(
suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil,
)
require.NoError(t, err)
for _, smpl := range res {
smplName := smpl.Metric.Get("__name__")
if smplName == "ALERTS" {
filteredRes = append(filteredRes, smpl)
} else {
// If not 'ALERTS', it has to be 'ALERTS_FOR_STATE'.
require.Equal(t, "ALERTS_FOR_STATE", smplName)
}
}
res, err = ruleWithExternalURL.Eval(
suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil,
)
require.NoError(t, err)
for _, smpl := range res {
smplName := smpl.Metric.Get("__name__")
if smplName == "ALERTS" {
filteredRes = append(filteredRes, smpl)
} else {
// If not 'ALERTS', it has to be 'ALERTS_FOR_STATE'.
require.Equal(t, "ALERTS_FOR_STATE", smplName)
}
}
require.Equal(t, result, filteredRes)
}
func TestAlertingRuleEmptyLabelFromTemplate(t *testing.T) {
suite, err := promql.NewTest(t, `
load 1m
@ -300,6 +396,7 @@ func TestAlertingRuleEmptyLabelFromTemplate(t *testing.T) {
labels.FromStrings("empty_label", ""),
nil,
nil,
"",
true, log.NewNopLogger(),
)
result := promql.Vector{
@ -360,6 +457,7 @@ func TestAlertingRuleDuplicate(t *testing.T) {
labels.FromStrings("test", "test"),
nil,
nil,
"",
true, log.NewNopLogger(),
)
_, err := rule.Eval(ctx, now, EngineQueryFunc(engine, storage), nil)

7
rules/manager.go
View file

@ -946,11 +946,11 @@ func (m *Manager) Stop() {
// Update the rule manager's state as the config requires. If
// loading the new rules failed the old rule set is restored.
func (m *Manager) Update(interval time.Duration, files []string, externalLabels labels.Labels) error {
func (m *Manager) Update(interval time.Duration, files []string, externalLabels labels.Labels, externalURL string) error {
m.mtx.Lock()
defer m.mtx.Unlock()
groups, errs := m.LoadGroups(interval, externalLabels, files...)
groups, errs := m.LoadGroups(interval, externalLabels, externalURL, files...)
if errs != nil {
for _, e := range errs {
level.Error(m.logger).Log("msg", "loading groups failed", "err", e)
@ -1034,7 +1034,7 @@ func (FileLoader) Parse(query string) (parser.Expr, error) { return parser.Parse
// LoadGroups reads groups from a list of files.
func (m *Manager) LoadGroups(
interval time.Duration, externalLabels labels.Labels, filenames ...string,
interval time.Duration, externalLabels labels.Labels, externalURL string, filenames ...string,
) (map[string]*Group, []error) {
groups := make(map[string]*Group)
@ -1067,6 +1067,7 @@ func (m *Manager) LoadGroups(
labels.FromMap(r.Labels),
labels.FromMap(r.Annotations),
externalLabels,
externalURL,
m.restored,
log.With(m.logger, "alert", r.Alert),
))

38
rules/manager_test.go
View file

@ -64,7 +64,7 @@ func TestAlertingRule(t *testing.T) {
expr,
time.Minute,
labels.FromStrings("severity", "{{\"c\"}}ritical"),
nil, nil, true, nil,
nil, nil, "", true, nil,
)
result := promql.Vector{
{
@ -205,7 +205,7 @@ func TestForStateAddSamples(t *testing.T) {
expr,
time.Minute,
labels.FromStrings("severity", "{{\"c\"}}ritical"),
nil, nil, true, nil,
nil, nil, "", true, nil,
)
result := promql.Vector{
{
@ -379,7 +379,7 @@ func TestForStateRestore(t *testing.T) {
expr,
alertForDuration,
labels.FromStrings("severity", "critical"),
nil, nil, true, nil,
nil, nil, "", true, nil,
)
group := NewGroup(GroupOptions{
@ -445,7 +445,7 @@ func TestForStateRestore(t *testing.T) {
expr,
alertForDuration,
labels.FromStrings("severity", "critical"),
nil, nil, false, nil,
nil, nil, "", false, nil,
)
newGroup := NewGroup(GroupOptions{
Name: "default",
@ -609,13 +609,13 @@ func readSeriesSet(ss storage.SeriesSet) (map[string][]promql.Point, error) {
func TestCopyState(t *testing.T) {
oldGroup := &Group{
rules: []Rule{
NewAlertingRule("alert", nil, 0, nil, nil, nil, true, nil),
NewAlertingRule("alert", nil, 0, nil, nil, nil, "", true, nil),
NewRecordingRule("rule1", nil, nil),
NewRecordingRule("rule2", nil, nil),
NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v1"}}),
NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v2"}}),
NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v3"}}),
NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, nil, true, nil),
NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, nil, "", true, nil),
},
seriesInPreviousEval: []map[string]labels.Labels{
{},
@ -634,10 +634,10 @@ func TestCopyState(t *testing.T) {
NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v0"}}),
NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v1"}}),
NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v2"}}),
NewAlertingRule("alert", nil, 0, nil, nil, nil, true, nil),
NewAlertingRule("alert", nil, 0, nil, nil, nil, "", true, nil),
NewRecordingRule("rule1", nil, nil),
NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v0"}}, nil, nil, true, nil),
NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, nil, true, nil),
NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v0"}}, nil, nil, "", true, nil),
NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, nil, "", true, nil),
NewRecordingRule("rule4", nil, nil),
},
seriesInPreviousEval: make([]map[string]labels.Labels, 8),
@ -725,7 +725,7 @@ func TestUpdate(t *testing.T) {
ruleManager.start()
defer ruleManager.Stop()
err := ruleManager.Update(10*time.Second, files, nil)
err := ruleManager.Update(10*time.Second, files, nil, "")
require.NoError(t, err)
require.Greater(t, len(ruleManager.groups), 0, "expected non-empty rule groups")
ogs := map[string]*Group{}
@ -736,7 +736,7 @@ func TestUpdate(t *testing.T) {
ogs[h] = g
}
err = ruleManager.Update(10*time.Second, files, nil)
err = ruleManager.Update(10*time.Second, files, nil, "")
require.NoError(t, err)
for h, g := range ruleManager.groups {
for _, actual := range g.seriesInPreviousEval {
@ -755,7 +755,7 @@ func TestUpdate(t *testing.T) {
defer os.Remove(tmpFile.Name())
defer tmpFile.Close()
err = ruleManager.Update(10*time.Second, []string{tmpFile.Name()}, nil)
err = ruleManager.Update(10*time.Second, []string{tmpFile.Name()}, nil, "")
require.NoError(t, err)
for h, g := range ruleManager.groups {
@ -826,7 +826,7 @@ func reloadAndValidate(rgs *rulefmt.RuleGroups, t *testing.T, tmpFile *os.File,
tmpFile.Seek(0, 0)
_, err = tmpFile.Write(bs)
require.NoError(t, err)
err = ruleManager.Update(10*time.Second, []string{tmpFile.Name()}, nil)
err = ruleManager.Update(10*time.Second, []string{tmpFile.Name()}, nil, "")
require.NoError(t, err)
for h, g := range ruleManager.groups {
if ogs[h] == g {
@ -862,7 +862,7 @@ func TestNotify(t *testing.T) {
expr, err := parser.ParseExpr("a > 1")
require.NoError(t, err)
rule := NewAlertingRule("aTooHigh", expr, 0, labels.Labels{}, labels.Labels{}, nil, true, log.NewNopLogger())
rule := NewAlertingRule("aTooHigh", expr, 0, labels.Labels{}, labels.Labels{}, nil, "", true, log.NewNopLogger())
group := NewGroup(GroupOptions{
Name: "alert",
Interval: time.Second,
@ -971,7 +971,7 @@ func TestMetricsUpdate(t *testing.T) {
}
for i, c := range cases {
err := ruleManager.Update(time.Second, c.files, nil)
err := ruleManager.Update(time.Second, c.files, nil, "")
require.NoError(t, err)
time.Sleep(2 * time.Second)
require.Equal(t, c.metrics, countMetrics(), "test %d: invalid count of metrics", i)
@ -1045,7 +1045,7 @@ func TestGroupStalenessOnRemoval(t *testing.T) {
var totalStaleNaN int
for i, c := range cases {
err := ruleManager.Update(time.Second, c.files, nil)
err := ruleManager.Update(time.Second, c.files, nil, "")
require.NoError(t, err)
time.Sleep(3 * time.Second)
totalStaleNaN += c.staleNaN
@ -1087,11 +1087,11 @@ func TestMetricsStalenessOnManagerShutdown(t *testing.T) {
}
}()
err := ruleManager.Update(2*time.Second, files, nil)
err := ruleManager.Update(2*time.Second, files, nil, "")
time.Sleep(4 * time.Second)
require.NoError(t, err)
start := time.Now()
err = ruleManager.Update(3*time.Second, files[:0], nil)
err = ruleManager.Update(3*time.Second, files[:0], nil, "")
require.NoError(t, err)
ruleManager.Stop()
stopped = true
@ -1134,7 +1134,7 @@ func TestGroupHasAlertingRules(t *testing.T) {
group: &Group{
name: "HasAlertingRule",
rules: []Rule{
NewAlertingRule("alert", nil, 0, nil, nil, nil, true, nil),
NewAlertingRule("alert", nil, 0, nil, nil, nil, "", true, nil),
NewRecordingRule("record", nil, nil),
},
},

4
template/template.go
View file

@ -295,14 +295,16 @@ func NewTemplateExpander(
}
// AlertTemplateData returns the interface to be used in expanding the template.
func AlertTemplateData(labels map[string]string, externalLabels map[string]string, value float64) interface{} {
func AlertTemplateData(labels map[string]string, externalLabels map[string]string, externalURL string, value float64) interface{} {
return struct {
Labels map[string]string
ExternalLabels map[string]string
ExternalURL string
Value float64
}{
Labels: labels,
ExternalLabels: externalLabels,
ExternalURL: externalURL,
Value: value,
}
}

2
web/api/v1/api_test.go
View file

@ -211,6 +211,7 @@ func (m rulesRetrieverMock) AlertingRules() []*rules.AlertingRule {
labels.Labels{},
labels.Labels{},
labels.Labels{},
"",
true,
log.NewNopLogger(),
)
@ -221,6 +222,7 @@ func (m rulesRetrieverMock) AlertingRules() []*rules.AlertingRule {
labels.Labels{},
labels.Labels{},
labels.Labels{},
"",
true,
log.NewNopLogger(),
)