mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2024-12-23 12:44:05 -08:00
Code Issues Actions 6 Packages Projects Releases Wiki Activity

Existing alert labels will update based on templates (#4500)

Browse source 
Signed-off-by: Chris Marchbanks <csmarchbanks@gmail.com>
This commit is contained in:
Chris Marchbanks 2018-08-15 01:52:08 -06:00 committed by Brian Brazil
parent cd54add5b8
commit 11155c7028
2 changed files with 108 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
rules/alerting.go
View file

@ -318,7 +318,8 @@ func (r *AlertingRule) Eval(ctx context.Context, ts time.Time, query QueryFunc,
annotations = append(annotations, labels.Label{Name: a.Name, Value: expand(a.Value)})
}
h := smpl.Metric.Hash()
lbs := lb.Labels()
h := lbs.Hash()
resultFPs[h] = struct{}{}
// Check whether we already have alerting state for the identifying label set.
@ -330,7 +331,7 @@ func (r *AlertingRule) Eval(ctx context.Context, ts time.Time, query QueryFunc,
}
r.active[h] = &Alert{
Labels: lb.Labels(),
Labels: lbs,
Annotations: annotations,
ActiveAt: ts,
State: StatePending,

105
rules/alerting_test.go
View file

@ -15,8 +15,10 @@ package rules
import (
"testing"
"time"
"github.com/prometheus/prometheus/pkg/labels"
"github.com/prometheus/prometheus/pkg/timestamp"
"github.com/prometheus/prometheus/promql"
"github.com/prometheus/prometheus/util/testutil"
)
@ -37,3 +39,106 @@ annotations:
got := rule.HTMLSnippet("/test/prefix")
testutil.Assert(t, want == got, "incorrect HTML snippet; want:\n\n|%v|\n\ngot:\n\n|%v|", want, got)
}
func TestAlertingRuleLabelsUpdate(t *testing.T) {
suite, err := promql.NewTest(t, `
load 1m
http_requests{job="app-server", instance="0"} 75 85 70 70
`)
testutil.Ok(t, err)
defer suite.Close()
err = suite.Run()
testutil.Ok(t, err)
expr, err := promql.ParseExpr(`http_requests < 100`)
testutil.Ok(t, err)
rule := NewAlertingRule(
"HTTPRequestRateLow",
expr,
time.Minute,
// Basing alerting rule labels off of a value that can change is a very bad idea.
// If an alert is going back and forth between two label values it will never fire.
// Instead, you should write two alerts with constant labels.
labels.FromStrings("severity", "{{ if lt $value 80.0 }}critical{{ else }}warning{{ end }}"),
nil, true, nil,
)
results := []promql.Vector{
promql.Vector{
{
Metric: labels.FromStrings(
"__name__", "ALERTS",
"alertname", "HTTPRequestRateLow",
"alertstate", "pending",
"instance", "0",
"job", "app-server",
"severity", "critical",
),
Point: promql.Point{V: 1},
},
},
promql.Vector{
{
Metric: labels.FromStrings(
"__name__", "ALERTS",
"alertname", "HTTPRequestRateLow",
"alertstate", "pending",
"instance", "0",
"job", "app-server",
"severity", "warning",
),
Point: promql.Point{V: 1},
},
},
promql.Vector{
{
Metric: labels.FromStrings(
"__name__", "ALERTS",
"alertname", "HTTPRequestRateLow",
"alertstate", "pending",
"instance", "0",
"job", "app-server",
"severity", "critical",
),
Point: promql.Point{V: 1},
},
},
promql.Vector{
{
Metric: labels.FromStrings(
"__name__", "ALERTS",
"alertname", "HTTPRequestRateLow",
"alertstate", "firing",
"instance", "0",
"job", "app-server",
"severity", "critical",
),
Point: promql.Point{V: 1},
},
},
}
baseTime := time.Unix(0, 0)
for i, result := range results {
t.Logf("case %d", i)
evalTime := baseTime.Add(time.Duration(i) * time.Minute)
result[0].Point.T = timestamp.FromTime(evalTime)
res, err := rule.Eval(suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil)
testutil.Ok(t, err)
var filteredRes promql.Vector // After removing 'ALERTS_FOR_STATE' samples.
for _, smpl := range res {
smplName := smpl.Metric.Get("__name__")
if smplName == "ALERTS" {
filteredRes = append(filteredRes, smpl)
} else {
// If not 'ALERTS', it has to be 'ALERTS_FOR_STATE'.
testutil.Equals(t, smplName, "ALERTS_FOR_STATE")
}
}
testutil.Equals(t, result, filteredRes)
}
}