mirror of
https://github.com/prometheus/prometheus.git
synced 2024-12-24 05:04:05 -08:00
Fix notifier relabel changing the labels of active alerts (#11427)
Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>
This commit is contained in:
parent
6d7f26c46f
commit
46b26c4f09
|
@ -72,7 +72,6 @@ import (
|
|||
"github.com/prometheus/prometheus/tsdb/agent"
|
||||
"github.com/prometheus/prometheus/util/logging"
|
||||
prom_runtime "github.com/prometheus/prometheus/util/runtime"
|
||||
"github.com/prometheus/prometheus/util/strutil"
|
||||
"github.com/prometheus/prometheus/web"
|
||||
)
|
||||
|
||||
|
@ -619,7 +618,7 @@ func main() {
|
|||
Appendable: fanoutStorage,
|
||||
Queryable: localStorage,
|
||||
QueryFunc: rules.EngineQueryFunc(queryEngine, fanoutStorage),
|
||||
NotifyFunc: sendAlerts(notifierManager, cfg.web.ExternalURL.String()),
|
||||
NotifyFunc: rules.SendAlerts(notifierManager, cfg.web.ExternalURL.String()),
|
||||
Context: ctxRule,
|
||||
ExternalURL: cfg.web.ExternalURL,
|
||||
Registerer: prometheus.DefaultRegisterer,
|
||||
|
@ -1270,36 +1269,6 @@ func computeExternalURL(u, listenAddr string) (*url.URL, error) {
|
|||
return eu, nil
|
||||
}
|
||||
|
||||
type sender interface {
|
||||
Send(alerts ...*notifier.Alert)
|
||||
}
|
||||
|
||||
// sendAlerts implements the rules.NotifyFunc for a Notifier.
|
||||
func sendAlerts(s sender, externalURL string) rules.NotifyFunc {
|
||||
return func(ctx context.Context, expr string, alerts ...*rules.Alert) {
|
||||
var res []*notifier.Alert
|
||||
|
||||
for _, alert := range alerts {
|
||||
a := ¬ifier.Alert{
|
||||
StartsAt: alert.FiredAt,
|
||||
Labels: alert.Labels,
|
||||
Annotations: alert.Annotations,
|
||||
GeneratorURL: externalURL + strutil.TableLinkForExpression(expr),
|
||||
}
|
||||
if !alert.ResolvedAt.IsZero() {
|
||||
a.EndsAt = alert.ResolvedAt
|
||||
} else {
|
||||
a.EndsAt = alert.ValidUntil
|
||||
}
|
||||
res = append(res, a)
|
||||
}
|
||||
|
||||
if len(alerts) > 0 {
|
||||
s.Send(res...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// readyStorage implements the Storage interface while allowing to set the actual
|
||||
// storage at a later point in time.
|
||||
type readyStorage struct {
|
||||
|
|
|
@ -198,7 +198,7 @@ func TestSendAlerts(t *testing.T) {
|
|||
}
|
||||
require.Equal(t, tc.exp, alerts)
|
||||
})
|
||||
sendAlerts(senderFunc, "http://localhost:9090")(context.TODO(), "up", tc.in...)
|
||||
rules.SendAlerts(senderFunc, "http://localhost:9090")(context.TODO(), "up", tc.in...)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
@ -507,6 +507,8 @@ func (r *AlertingRule) sendAlerts(ctx context.Context, ts time.Time, resendDelay
|
|||
}
|
||||
alert.ValidUntil = ts.Add(4 * delta)
|
||||
anew := *alert
|
||||
// The notifier re-uses the labels slice, hence make a copy.
|
||||
anew.Labels = alert.Labels.Copy()
|
||||
alerts = append(alerts, &anew)
|
||||
}
|
||||
})
|
||||
|
|
|
@ -20,10 +20,13 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/go-kit/log"
|
||||
"github.com/prometheus/common/model"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
"github.com/prometheus/prometheus/model/relabel"
|
||||
"github.com/prometheus/prometheus/model/timestamp"
|
||||
"github.com/prometheus/prometheus/notifier"
|
||||
"github.com/prometheus/prometheus/promql"
|
||||
"github.com/prometheus/prometheus/promql/parser"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
|
@ -659,3 +662,53 @@ func TestQueryForStateSeries(t *testing.T) {
|
|||
testFunc(tst)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSendAlertsDontAffectActiveAlerts tests a fix for https://github.com/prometheus/prometheus/issues/11424.
|
||||
func TestSendAlertsDontAffectActiveAlerts(t *testing.T) {
|
||||
rule := NewAlertingRule(
|
||||
"TestRule",
|
||||
nil,
|
||||
time.Minute,
|
||||
labels.FromStrings("severity", "critical"),
|
||||
labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil,
|
||||
)
|
||||
|
||||
// Set an active alert.
|
||||
lbls := labels.FromStrings("a1", "1")
|
||||
h := lbls.Hash()
|
||||
al := &Alert{State: StateFiring, Labels: lbls, ActiveAt: time.Now()}
|
||||
rule.active[h] = al
|
||||
|
||||
expr, err := parser.ParseExpr("foo")
|
||||
require.NoError(t, err)
|
||||
rule.vector = expr
|
||||
|
||||
// The relabel rule reproduced the bug here.
|
||||
opts := notifier.Options{
|
||||
QueueCapacity: 1,
|
||||
RelabelConfigs: []*relabel.Config{
|
||||
{
|
||||
SourceLabels: model.LabelNames{"a1"},
|
||||
Regex: relabel.MustNewRegexp("(.+)"),
|
||||
TargetLabel: "a1",
|
||||
Replacement: "bug",
|
||||
Action: "replace",
|
||||
},
|
||||
},
|
||||
}
|
||||
nm := notifier.NewManager(&opts, log.NewNopLogger())
|
||||
|
||||
f := SendAlerts(nm, "")
|
||||
notifyFunc := func(ctx context.Context, expr string, alerts ...*Alert) {
|
||||
require.Len(t, alerts, 1)
|
||||
require.Equal(t, al, alerts[0])
|
||||
f(ctx, expr, alerts...)
|
||||
}
|
||||
|
||||
rule.sendAlerts(context.Background(), time.Now(), 0, 0, notifyFunc)
|
||||
nm.Stop()
|
||||
|
||||
// The relabel rule changes a1=1 to a1=bug.
|
||||
// But the labels with the AlertingRule should not be changed.
|
||||
require.Equal(t, labels.FromStrings("a1", "1"), rule.active[h].Labels)
|
||||
}
|
||||
|
|
|
@ -35,9 +35,11 @@ import (
|
|||
"github.com/prometheus/prometheus/model/rulefmt"
|
||||
"github.com/prometheus/prometheus/model/timestamp"
|
||||
"github.com/prometheus/prometheus/model/value"
|
||||
"github.com/prometheus/prometheus/notifier"
|
||||
"github.com/prometheus/prometheus/promql"
|
||||
"github.com/prometheus/prometheus/promql/parser"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
"github.com/prometheus/prometheus/util/strutil"
|
||||
)
|
||||
|
||||
// RuleHealth describes the health state of a rule.
|
||||
|
@ -1168,3 +1170,33 @@ func (m *Manager) AlertingRules() []*AlertingRule {
|
|||
|
||||
return alerts
|
||||
}
|
||||
|
||||
type Sender interface {
|
||||
Send(alerts ...*notifier.Alert)
|
||||
}
|
||||
|
||||
// SendAlerts implements the rules.NotifyFunc for a Notifier.
|
||||
func SendAlerts(s Sender, externalURL string) NotifyFunc {
|
||||
return func(ctx context.Context, expr string, alerts ...*Alert) {
|
||||
var res []*notifier.Alert
|
||||
|
||||
for _, alert := range alerts {
|
||||
a := ¬ifier.Alert{
|
||||
StartsAt: alert.FiredAt,
|
||||
Labels: alert.Labels,
|
||||
Annotations: alert.Annotations,
|
||||
GeneratorURL: externalURL + strutil.TableLinkForExpression(expr),
|
||||
}
|
||||
if !alert.ResolvedAt.IsZero() {
|
||||
a.EndsAt = alert.ResolvedAt
|
||||
} else {
|
||||
a.EndsAt = alert.ValidUntil
|
||||
}
|
||||
res = append(res, a)
|
||||
}
|
||||
|
||||
if len(alerts) > 0 {
|
||||
s.Send(res...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue