mirror of
https://github.com/prometheus/prometheus.git
synced 2024-12-24 05:04:05 -08:00
Send resolved notifications
This commit is contained in:
Fabian Reinartz
parent
f69e668fc4
commit
bf6abac8f4
|
|
@ -38,6 +38,17 @@ const (
|
|||
// AlertState denotes the state of an active alert.
|
||||
type AlertState int
|
||||
|
||||
const (
|
||||
// StateInactive is the state of an alert that is either firing nor pending.
|
||||
StateInactive AlertState = iota
|
||||
// StatePending is the state of an alert that has been active for less than
|
||||
// the configured threshold duration.
|
||||
StatePending
|
||||
// StateFiring is the state of an alert that has been active for longer than
|
||||
// the configured threshold duration.
|
||||
StateFiring
|
||||
)
|
||||
|
||||
func (s AlertState) String() string {
|
||||
switch s {
|
||||
case StateInactive:
|
||||
|
|
@ -50,17 +61,13 @@ func (s AlertState) String() string {
|
|||
panic(fmt.Errorf("unknown alert state: %v", s))
|
||||
}
|
||||
|
||||
const (
|
||||
StateInactive AlertState = iota
|
||||
StatePending
|
||||
StateFiring
|
||||
)
|
||||
|
||||
type alertInstance struct {
|
||||
metric model.Metric
|
||||
value model.SampleValue
|
||||
state AlertState
|
||||
activeSince model.Time
|
||||
// Alert is the user-level representation of a single instance of an alerting rule.
|
||||
type Alert struct {
|
||||
State AlertState
|
||||
Labels model.LabelSet
|
||||
Value model.SampleValue
|
||||
ActiveAt model.Time
|
||||
ResolvedAt model.Time
|
||||
}
|
||||
|
||||
// An AlertingRule generates alerts from its vector expression.
|
||||
|
|
@ -81,7 +88,7 @@ type AlertingRule struct {
|
|||
mtx sync.Mutex
|
||||
// A map of alerts which are currently active (Pending or Firing), keyed by
|
||||
// the fingerprint of the labelset they correspond to.
|
||||
active map[model.Fingerprint]*alertInstance
|
||||
active map[model.Fingerprint]*Alert
|
||||
}
|
||||
|
||||
// NewAlertingRule constructs a new AlertingRule.
|
||||
|
|
@ -92,7 +99,7 @@ func NewAlertingRule(name string, vec promql.Expr, hold time.Duration, lbls, ann
|
|||
holdDuration: hold,
|
||||
labels: lbls,
|
||||
annotations: anns,
|
||||
active: map[model.Fingerprint]*alertInstance{},
|
||||
active: map[model.Fingerprint]*Alert{},
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -101,17 +108,17 @@ func (rule *AlertingRule) Name() string {
|
|||
return rule.name
|
||||
}
|
||||
|
||||
func (r *AlertingRule) sample(ai *alertInstance, ts model.Time, set bool) *model.Sample {
|
||||
func (r *AlertingRule) sample(alert *Alert, ts model.Time, set bool) *model.Sample {
|
||||
// Build alert labels in order they can be overwritten.
|
||||
metric := model.Metric(r.labels.Clone())
|
||||
|
||||
for ln, lv := range ai.metric {
|
||||
for ln, lv := range alert.Labels {
|
||||
metric[ln] = lv
|
||||
}
|
||||
|
||||
metric[model.MetricNameLabel] = alertMetricName
|
||||
metric[model.AlertNameLabel] = model.LabelValue(r.name)
|
||||
metric[alertStateLabel] = model.LabelValue(ai.state.String())
|
||||
metric[alertStateLabel] = model.LabelValue(alert.State.String())
|
||||
|
||||
s := &model.Sample{
|
||||
Metric: metric,
|
||||
|
|
@ -124,6 +131,10 @@ func (r *AlertingRule) sample(ai *alertInstance, ts model.Time, set bool) *model
|
|||
return s
|
||||
}
|
||||
|
||||
// resolvedRetention is the duration for which a resolved alert instance
|
||||
// is kept in memory state and consequentally repeatedly sent to the AlertManager.
|
||||
const resolvedRetention = 15 * time.Minute
|
||||
|
||||
// eval evaluates the rule expression and then creates pending alerts and fires
|
||||
// or removes previously pending alerts accordingly.
|
||||
func (r *AlertingRule) eval(ts model.Time, engine *promql.Engine) (model.Vector, error) {
|
||||
|
|
@ -147,57 +158,59 @@ func (r *AlertingRule) eval(ts model.Time, engine *promql.Engine) (model.Vector,
|
|||
fp := smpl.Metric.Fingerprint()
|
||||
resultFPs[fp] = struct{}{}
|
||||
|
||||
if ai, ok := r.active[fp]; ok {
|
||||
ai.value = smpl.Value
|
||||
if alert, ok := r.active[fp]; ok {
|
||||
alert.Value = smpl.Value
|
||||
continue
|
||||
}
|
||||
|
||||
delete(smpl.Metric, model.MetricNameLabel)
|
||||
|
||||
r.active[fp] = &alertInstance{
|
||||
metric: smpl.Metric,
|
||||
activeSince: ts,
|
||||
state: StatePending,
|
||||
value: smpl.Value,
|
||||
r.active[fp] = &Alert{
|
||||
Labels: model.LabelSet(smpl.Metric),
|
||||
ActiveAt: ts,
|
||||
State: StatePending,
|
||||
Value: smpl.Value,
|
||||
}
|
||||
}
|
||||
|
||||
var vec model.Vector
|
||||
// Check if any pending alerts should be removed or fire now. Write out alert timeseries.
|
||||
for fp, ai := range r.active {
|
||||
for fp, a := range r.active {
|
||||
if _, ok := resultFPs[fp]; !ok {
|
||||
delete(r.active, fp)
|
||||
vec = append(vec, r.sample(ai, ts, false))
|
||||
if a.State != StateInactive {
|
||||
vec = append(vec, r.sample(a, ts, false))
|
||||
}
|
||||
// If the alert was previously firing, keep it aroud for a given
|
||||
// retention time so it is reported as resolved to the AlertManager.
|
||||
if a.State == StatePending || (a.ResolvedAt != 0 && ts.Sub(a.ResolvedAt) > resolvedRetention) {
|
||||
delete(r.active, fp)
|
||||
}
|
||||
if a.State != StateInactive {
|
||||
a.State = StateInactive
|
||||
a.ResolvedAt = ts
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if ai.state != StateFiring && ts.Sub(ai.activeSince) >= r.holdDuration {
|
||||
vec = append(vec, r.sample(ai, ts, false))
|
||||
ai.state = StateFiring
|
||||
if a.State == StatePending && ts.Sub(a.ActiveAt) >= r.holdDuration {
|
||||
vec = append(vec, r.sample(a, ts, false))
|
||||
a.State = StateFiring
|
||||
}
|
||||
|
||||
vec = append(vec, r.sample(ai, ts, true))
|
||||
vec = append(vec, r.sample(a, ts, true))
|
||||
}
|
||||
|
||||
return vec, nil
|
||||
}
|
||||
|
||||
// Alert is the user-level representation of a single instance of an alerting rule.
|
||||
type Alert struct {
|
||||
State AlertState
|
||||
Labels model.LabelSet
|
||||
ActiveSince model.Time
|
||||
Value model.SampleValue
|
||||
}
|
||||
|
||||
func (r *AlertingRule) State() AlertState {
|
||||
r.mtx.Lock()
|
||||
defer r.mtx.Unlock()
|
||||
|
||||
maxState := StateInactive
|
||||
for _, ai := range r.active {
|
||||
if ai.state > maxState {
|
||||
maxState = ai.state
|
||||
for _, a := range r.active {
|
||||
if a.State > maxState {
|
||||
maxState = a.State
|
||||
}
|
||||
}
|
||||
return maxState
|
||||
|
|
@ -205,21 +218,30 @@ func (r *AlertingRule) State() AlertState {
|
|||
|
||||
// ActiveAlerts returns a slice of active alerts.
|
||||
func (r *AlertingRule) ActiveAlerts() []*Alert {
|
||||
var res []*Alert
|
||||
for _, a := range r.recentAlerts() {
|
||||
if a.ResolvedAt == 0 {
|
||||
res = append(res, a)
|
||||
}
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func (r *AlertingRule) recentAlerts() []*Alert {
|
||||
r.mtx.Lock()
|
||||
defer r.mtx.Unlock()
|
||||
|
||||
alerts := make([]*Alert, 0, len(r.active))
|
||||
for _, ai := range r.active {
|
||||
|
||||
for _, a := range r.active {
|
||||
labels := r.labels.Clone()
|
||||
for ln, lv := range ai.metric {
|
||||
for ln, lv := range a.Labels {
|
||||
labels[ln] = lv
|
||||
}
|
||||
alerts = append(alerts, &Alert{
|
||||
State: ai.state,
|
||||
Labels: labels,
|
||||
ActiveSince: ai.activeSince,
|
||||
Value: ai.value,
|
||||
})
|
||||
anew := *a
|
||||
anew.Labels = labels
|
||||
|
||||
alerts = append(alerts, &anew)
|
||||
}
|
||||
return alerts
|
||||
}
|
||||
|
|
|
|||
|
|
@ -241,9 +241,10 @@ func (g *Group) eval() {
|
|||
|
||||
func (g *Group) sendAlerts(rule *AlertingRule, timestamp model.Time) error {
|
||||
var alerts model.Alerts
|
||||
for _, alert := range rule.ActiveAlerts() {
|
||||
|
||||
for _, alert := range rule.recentAlerts() {
|
||||
// Only send actually firing alerts.
|
||||
if alert.State != StateFiring {
|
||||
if alert.State == StatePending {
|
||||
continue
|
||||
}
|
||||
|
||||
|
|
@ -292,12 +293,17 @@ func (g *Group) sendAlerts(rule *AlertingRule, timestamp model.Time) error {
|
|||
annotations[an] = expand(av)
|
||||
}
|
||||
|
||||
alerts = append(alerts, &model.Alert{
|
||||
StartsAt: alert.ActiveSince.Time().Add(rule.holdDuration),
|
||||
a := &model.Alert{
|
||||
StartsAt: alert.ActiveAt.Add(rule.holdDuration).Time(),
|
||||
Labels: labels,
|
||||
Annotations: annotations,
|
||||
GeneratorURL: g.opts.ExternalURL.String() + strutil.GraphLinkForExpression(rule.vector.String()),
|
||||
})
|
||||
}
|
||||
if alert.ResolvedAt != 0 {
|
||||
a.EndsAt = alert.ResolvedAt.Time()
|
||||
}
|
||||
|
||||
alerts = append(alerts, a)
|
||||
}
|
||||
|
||||
if len(alerts) > 0 {
|
||||
|
|
|
|||
|
|
@ -119,7 +119,7 @@ func webUiTemplates_baseHtml() (*asset, error) {
|
|||
return a, nil
|
||||
}
|
||||
|
||||
var _webUiTemplatesAlertsHtml = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\x8c\x54\x4d\x8f\x9b\x30\x10\xbd\xef\xaf\xb0\xd0\x1e\x5a\xa9\x80\xd4\x63\x45\x90\x56\x7b\xe9\x61\x5b\x55\x9b\x74\xaf\x91\xb1\x27\x8b\xb7\x8e\x41\xb6\x93\x4d\xe4\xf2\xdf\x3b\xb6\x21\x4b\x08\x48\xbd\x24\x78\x66\xde\xf3\xf3\x7c\x39\xc7\x61\x27\x14\x90\xa4\x06\xca\x93\xae\xbb\x23\xa4\x90\x42\xfd\x21\xf6\xdc\xc2\x2a\xb1\x70\xb2\x39\x33\x26\x21\x1a\xe4\x2a\x31\xf6\x2c\xc1\xd4\x00\x36\x21\xb5\x86\xdd\x2a\x71\x8e\xb4\xd4\xd6\xbf\xf0\x20\x4e\xa4\xeb\x72\x63\xa9\x15\xcc\x63\x72\x2a\x41\x5b\x93\x79\x78\xe9\x79\x0d\xd3\xa2\xb5\xc4\x68\xb6\x8c\x7b\xbb\xc0\xde\x10\x55\xe4\x11\x53\xde\x39\x07\x8a\xa3\xbc\xbb\x0f\xc5\xac\x51\x16\x94\xf5\xa2\x0b\x2e\x8e\x84\x49\x6a\xcc\x2a\x98\x29\x06\xe8\x74\x27\x0f\x82\xc7\xab\xeb\xaf\xe5\x43\xa0\x2d\x72\xfc\xf4\x16\x4b\x2b\x09\x03\x26\x1e\xc2\x6f\x5a\x35\x9a\x83\x06\xde\x1f\x59\x23\x25\x6d\x0d\x44\x22\x0f\xac\x1a\x7e\x8e\xdf\xce\xdd\x07\xb1\x6b\xd4\x0e\x9b\xe6\xb9\x79\x7f\xf4\x7c\xe4\xdb\x8a\x64\x0f\x33\x8e\x90\x5e\x0f\xd3\x54\xbd\x42\x1f\x23\xd4\xeb\xf3\x01\xb3\xda\x3b\x23\x2b\xb3\xe2\x08\x51\x71\x64\x1b\x19\x2e\x81\x85\xd5\xc3\x03\x9c\x13\x8a\xc3\x89\xcc\xeb\xc9\x82\xa1\xeb\x48\xf0\x6e\x7d\xa9\x41\xf7\xef\x89\x44\xbc\x2c\xc4\xc0\x25\x30\x83\x29\xab\xe1\xa8\xf1\x9f\x37\xef\xca\xd7\x41\x94\xa4\xa8\x4a\xe7\xb2\x9f\x74\x8f\x4c\x45\x5e\x95\xe4\x93\x73\x12\x14\xb9\x52\xeb\x2f\x09\xc7\xcf\x45\x8e\xac\x83\xd2\xdc\xea\xf2\x56\x75\x94\xc3\x01\xeb\x25\xcd\x44\xcf\xe5\x80\x47\xac\xee\xf8\x8c\x96\x56\x43\x59\xb0\x86\x83\x97\xf4\x7d\xf3\xe3\x69\xad\x44\xdb\x82\x1d\x35\x95\x17\x19\x22\x8a\xdc\x47\x8f\xf9\xf2\x09\x21\x66\x6f\x37\x7d\xc6\x38\xfe\x7f\x7b\xa5\x6e\x8e\xa0\x2f\x7d\x83\x05\x51\xd8\x37\x7d\xd2\x41\xc2\x1e\xbb\xd5\x6c\x83\x3b\x99\xbc\xe7\x23\x27\x13\x8f\xf7\xd5\xe5\x13\xad\x40\x62\xef\xe2\xe7\x8c\x37\x54\x77\xc9\x19\x3b\x87\xac\x85\x62\x8b\x31\x2f\x54\x1e\x16\x9d\x6b\x81\x45\x9e\xc3\x8e\x8b\x3a\xe4\x31\x36\xf6\x72\x2a\xc3\x53\x6f\x6f\xe1\x53\xd3\x88\x4b\xfa\xb7\x7f\x21\xf7\x47\x2f\x32\x0c\x43\xcc\xc6\x84\xb7\xa7\x32\x2d\x55\x43\x2a\x03\x92\x84\xdf\xb4\xd5\x62\x4f\xf5\x39\xc1\x7e\x89\x8c\x5d\xe7\xa7\x26\xb2\x76\x5d\x82\x9b\x06\x91\x73\x32\xe2\xde\x99\x5c\x93\xdf\x4a\x0e\x43\x34\xbe\x3e\xd4\x3d\x56\x3f\xc5\x75\x17\x87\x90\xfc\x25\xe3\x11\x8d\xf3\x89\x43\xe3\xd7\x1f\x6c\x71\x88\x05\xa3\xb6\xc1\x26\xc2\xc5\x9b\x1e\xb0\xa5\x35\xa3\x06\xbc\xec\x61\x88\x7b\xa5\x4b\x12\x30\x30\x96\x3c\x54\x3c\xdb\x88\x3d\x64\xbf\x37\x8f\x1e\xb7\x08\x78\x89\x49\xb8\x8d\x98\x2b\xf1\x34\x1f\x18\xe3\x3b\xfa\x7a\x9e\xae\x83\xe6\x57\xc1\x38\x0a\xad\xc3\x5a\xbd\xf0\xf5\x63\x3a\x84\xfd\x0b\x00\x00\xff\xff\xdb\x8e\x0c\x63\xab\x06\x00\x00")
|
||||
var _webUiTemplatesAlertsHtml = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\x8c\x54\x4d\x8f\x9b\x30\x10\xbd\xef\xaf\xb0\xd0\x1e\x5a\xa9\x80\xd4\x63\x45\x90\x56\x7b\xe9\x61\x5b\x55\x9b\x74\xaf\x91\xb1\x27\x8b\xb7\x8e\x41\xb6\x93\x4d\xe4\xf2\xdf\x3b\xb6\x21\x4b\x08\x48\xbd\x24\x78\xe6\xcd\xf3\xf3\x7c\x39\xc7\x61\x27\x14\x90\xa4\x06\xca\x93\xae\xbb\x23\xa4\x90\x42\xfd\x21\xf6\xdc\xc2\x2a\xb1\x70\xb2\x39\x33\x26\x21\x1a\xe4\x2a\x31\xf6\x2c\xc1\xd4\x00\x36\x21\xb5\x86\xdd\x2a\x71\x8e\xb4\xd4\xd6\xbf\xf0\x20\x4e\xa4\xeb\x72\x63\xa9\x15\xcc\xc7\xe4\x54\x82\xb6\x26\xf3\xe1\xa5\xe7\x35\x4c\x8b\xd6\x12\xa3\xd9\x72\xdc\xdb\x25\xec\x0d\xa3\x8a\x3c\xc6\x94\x77\xce\x81\xe2\x28\xef\xee\x43\x31\x6b\x94\x05\x65\xbd\xe8\x82\x8b\x23\x61\x92\x1a\xb3\x0a\x66\x8a\x00\x9d\xee\xe4\x41\xf0\x78\x75\xfd\xb5\x7c\x08\xb4\x45\x8e\x9f\xde\x62\x69\x25\x61\x88\x89\x87\xf0\x9b\x56\x8d\xe6\xa0\x81\xf7\x47\xd6\x48\x49\x5b\x03\x91\xc8\x07\x56\x0d\x3f\xc7\x6f\xe7\xee\x83\xd8\x35\x6a\x87\x4d\xf3\xdc\xbc\x3f\x7a\x3e\xf2\x6d\x45\xb2\x87\x19\x47\x48\xaf\x0f\xd3\x54\xbd\x42\x8f\x11\xea\xf5\xf9\x80\x59\xed\x9d\x91\x95\x59\x71\x84\xa8\x38\xb2\x8d\x0c\x17\x60\x61\xf5\xf0\x00\xe7\x84\xe2\x70\x22\xf3\x7a\xb2\x60\xe8\x3a\x12\xbc\x5b\x5f\x6a\xd0\xfd\x7b\x22\x11\x2f\x0b\x31\x70\x09\xcc\x60\xca\x6a\x38\x6a\xfc\xe7\xcd\xbb\xf2\x75\x10\x25\x29\xaa\xd2\xb9\xec\x27\xdd\x23\x53\x91\x57\x25\xf9\xe4\x9c\x04\x45\xae\xd4\xfa\x4b\xc2\xf1\x73\x91\x23\xeb\xa0\x34\xb7\xba\xbc\x55\x1d\xe5\x70\xc0\x7a\x49\x33\xd1\x73\x39\xe0\x11\xab\x3b\x3e\xa3\xa5\xd5\x50\x16\xac\xe1\xe0\x25\x7d\xdf\xfc\x78\x5a\x2b\xd1\xb6\x60\x47\x4d\xe5\x45\x06\x44\x91\x7b\xf4\x98\x2f\x9f\x10\x62\xf6\x76\xd3\x67\x8c\xf1\xff\xdb\x2b\x75\x73\x04\x7d\xe9\x1b\x2c\x88\xc2\xbe\xe9\x93\x0e\x12\xf6\xd8\xad\x66\x1b\xdc\xc9\xe4\x3d\x1f\x39\x99\x78\xbc\xaf\x2e\x9f\x68\x05\x12\x7b\x17\x3f\x67\xbc\xa1\xba\x4b\xce\xd8\x39\x64\x2d\x14\x5b\xc4\xbc\x50\x79\x58\x74\xae\x05\x16\x79\x2e\x76\x5c\xd4\x21\x8f\xb1\xb1\x97\x53\x19\x9e\x7a\x7b\x0b\x9f\x9a\x46\x5c\xd2\xbf\xfd\x0b\xb9\x3f\x7a\x91\x61\x18\x62\x36\x26\xbc\x3d\x95\x69\xa9\x1a\x52\x19\x22\x49\xf8\x4d\x5b\x2d\xf6\x54\x9f\x13\xec\x97\xc8\xd8\x75\x7e\x6a\x22\x6b\xd7\x25\xb8\x69\x30\x72\x4e\x46\xdc\x3b\x93\x6b\xf2\x5b\xc9\x61\x88\xc6\xd7\x87\xba\xc7\xea\xa7\xb8\xee\xe2\x10\x92\xbf\x64\x3c\xa2\x71\x3e\x71\x68\xfc\xfa\x83\x2d\x0e\xb1\x60\xd4\x36\xd8\x44\xb8\x78\xd3\x03\xb6\xb4\x66\xd4\x80\x97\x3d\x0c\x71\xaf\x74\x49\x02\x02\xfb\x65\x61\xb3\x8d\xd8\x43\xf6\x7b\xf3\xe8\x83\x16\xd1\x2f\x31\x03\xb7\x88\xb9\xfa\x4e\x93\x81\x18\xdf\xce\xd7\xc3\x74\x0d\x9a\xdf\x03\x63\x14\x5a\x87\x9d\x7a\xe1\xeb\x67\x74\x80\xfd\x0b\x00\x00\xff\xff\xb3\xf6\xaa\xca\xa8\x06\x00\x00")
|
||||
|
||||
func webUiTemplatesAlertsHtmlBytes() ([]byte, error) {
|
||||
return bindataRead(
|
||||
|
|
@ -134,7 +134,7 @@ func webUiTemplatesAlertsHtml() (*asset, error) {
|
|||
return nil, err
|
||||
}
|
||||
|
||||
info := bindataFileInfo{name: "web/ui/templates/alerts.html", size: 1707, mode: os.FileMode(420), modTime: time.Unix(1450348618, 0)}
|
||||
info := bindataFileInfo{name: "web/ui/templates/alerts.html", size: 1704, mode: os.FileMode(420), modTime: time.Unix(1450348695, 0)}
|
||||
a := &asset{bytes: bytes, info: info}
|
||||
return a, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@
|
|||
{{end}}
|
||||
</td>
|
||||
<td><span class="alert alert-{{ .State | alertStateToClass }} state_indicator text-uppercase">{{.State}}</span></td>
|
||||
<td>{{.ActiveSince.Time.UTC}}</td>
|
||||
<td>{{.ActiveAt.Time.UTC}}</td>
|
||||
<td>{{.Value}}</td>
|
||||
</tr>
|
||||
{{end}}
|
||||
|
|
|
|||
Loading…
Reference in a new issue