Give a bit more slack for alertmanager send failures. (#7228)

Fixes #5277

Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>
This commit is contained in:
Brian Brazil 2020-05-09 05:37:46 +01:00 committed by GitHub
parent 9e265aba10
commit 5368066b58
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23

View file

@ -485,12 +485,12 @@ func (r *AlertingRule) sendAlerts(ctx context.Context, ts time.Time, resendDelay
r.ForEachActiveAlert(func(alert *Alert) {
if alert.needsSending(ts, resendDelay) {
alert.LastSentAt = ts
// Allow for a couple Eval or Alertmanager send failures
// Allow for two Eval or Alertmanager send failures.
delta := resendDelay
if interval > resendDelay {
delta = interval
}
alert.ValidUntil = ts.Add(3 * delta)
alert.ValidUntil = ts.Add(4 * delta)
anew := *alert
alerts = append(alerts, &anew)
}