mirror of
https://github.com/prometheus/prometheus.git
synced 2024-12-25 05:34:05 -08:00
*: avoid missed Alertmanager targets (#6455)
This change makes sure that nearly-identical Alertmanager configurations aren't merged together. The config's identifier was the MD5 hash of the configuration serialized to JSON but because `relabel.Regexp` has no public field and doesn't implement the JSON.Marshaler interface, it was always serialized to "{}". In practice, the identifier can be based on the index of the configuration in the list. Signed-off-by: Simon Pasquier <spasquie@redhat.com>
This commit is contained in:
parent
48d25e6fe7
commit
cccd542891
|
@ -16,8 +16,6 @@ package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/md5"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
@ -435,13 +433,8 @@ func main() {
|
||||||
notifierManager.ApplyConfig,
|
notifierManager.ApplyConfig,
|
||||||
func(cfg *config.Config) error {
|
func(cfg *config.Config) error {
|
||||||
c := make(map[string]sd_config.ServiceDiscoveryConfig)
|
c := make(map[string]sd_config.ServiceDiscoveryConfig)
|
||||||
for _, v := range cfg.AlertingConfig.AlertmanagerConfigs {
|
for k, v := range cfg.AlertingConfig.AlertmanagerConfigs.ToMap() {
|
||||||
// AlertmanagerConfigs doesn't hold an unique identifier so we use the config hash as the identifier.
|
c[k] = v.ServiceDiscoveryConfig
|
||||||
b, err := json.Marshal(v)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
c[fmt.Sprintf("%x", md5.Sum(b))] = v.ServiceDiscoveryConfig
|
|
||||||
}
|
}
|
||||||
return discoveryManagerNotify.ApplyConfig(c)
|
return discoveryManagerNotify.ApplyConfig(c)
|
||||||
},
|
},
|
||||||
|
|
|
@ -432,8 +432,8 @@ func (c *ScrapeConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||||
|
|
||||||
// AlertingConfig configures alerting and alertmanager related configs.
|
// AlertingConfig configures alerting and alertmanager related configs.
|
||||||
type AlertingConfig struct {
|
type AlertingConfig struct {
|
||||||
AlertRelabelConfigs []*relabel.Config `yaml:"alert_relabel_configs,omitempty"`
|
AlertRelabelConfigs []*relabel.Config `yaml:"alert_relabel_configs,omitempty"`
|
||||||
AlertmanagerConfigs []*AlertmanagerConfig `yaml:"alertmanagers,omitempty"`
|
AlertmanagerConfigs AlertmanagerConfigs `yaml:"alertmanagers,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// UnmarshalYAML implements the yaml.Unmarshaler interface.
|
// UnmarshalYAML implements the yaml.Unmarshaler interface.
|
||||||
|
@ -454,6 +454,18 @@ func (c *AlertingConfig) UnmarshalYAML(unmarshal func(interface{}) error) error
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AlertmanagerConfigs is a slice of *AlertmanagerConfig.
|
||||||
|
type AlertmanagerConfigs []*AlertmanagerConfig
|
||||||
|
|
||||||
|
// ToMap converts a slice of *AlertmanagerConfig to a map.
|
||||||
|
func (a AlertmanagerConfigs) ToMap() map[string]*AlertmanagerConfig {
|
||||||
|
ret := make(map[string]*AlertmanagerConfig)
|
||||||
|
for i := range a {
|
||||||
|
ret[fmt.Sprintf("config-%d", i)] = a[i]
|
||||||
|
}
|
||||||
|
return ret
|
||||||
|
}
|
||||||
|
|
||||||
// AlertmanagerAPIVersion represents a version of the
|
// AlertmanagerAPIVersion represents a version of the
|
||||||
// github.com/prometheus/alertmanager/api, e.g. 'v1' or 'v2'.
|
// github.com/prometheus/alertmanager/api, e.g. 'v1' or 'v2'.
|
||||||
type AlertmanagerAPIVersion string
|
type AlertmanagerAPIVersion string
|
||||||
|
|
|
@ -16,7 +16,6 @@ package notifier
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/md5"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
@ -262,20 +261,13 @@ func (n *Manager) ApplyConfig(conf *config.Config) error {
|
||||||
|
|
||||||
amSets := make(map[string]*alertmanagerSet)
|
amSets := make(map[string]*alertmanagerSet)
|
||||||
|
|
||||||
for _, cfg := range conf.AlertingConfig.AlertmanagerConfigs {
|
for k, cfg := range conf.AlertingConfig.AlertmanagerConfigs.ToMap() {
|
||||||
ams, err := newAlertmanagerSet(cfg, n.logger)
|
ams, err := newAlertmanagerSet(cfg, n.logger, n.metrics)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
ams.metrics = n.metrics
|
amSets[k] = ams
|
||||||
|
|
||||||
// The config hash is used for the map lookup identifier.
|
|
||||||
b, err := json.Marshal(cfg)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
amSets[fmt.Sprintf("%x", md5.Sum(b))] = ams
|
|
||||||
}
|
}
|
||||||
|
|
||||||
n.alertmanagers = amSets
|
n.alertmanagers = amSets
|
||||||
|
@ -638,15 +630,16 @@ type alertmanagerSet struct {
|
||||||
logger log.Logger
|
logger log.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
func newAlertmanagerSet(cfg *config.AlertmanagerConfig, logger log.Logger) (*alertmanagerSet, error) {
|
func newAlertmanagerSet(cfg *config.AlertmanagerConfig, logger log.Logger, metrics *alertMetrics) (*alertmanagerSet, error) {
|
||||||
client, err := config_util.NewClientFromConfig(cfg.HTTPClientConfig, "alertmanager", false)
|
client, err := config_util.NewClientFromConfig(cfg.HTTPClientConfig, "alertmanager", false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
s := &alertmanagerSet{
|
s := &alertmanagerSet{
|
||||||
client: client,
|
client: client,
|
||||||
cfg: cfg,
|
cfg: cfg,
|
||||||
logger: logger,
|
logger: logger,
|
||||||
|
metrics: metrics,
|
||||||
}
|
}
|
||||||
return s, nil
|
return s, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -16,7 +16,6 @@ package notifier
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/md5"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
|
@ -467,6 +466,7 @@ alerting:
|
||||||
if err := yaml.UnmarshalStrict([]byte(s), cfg); err != nil {
|
if err := yaml.UnmarshalStrict([]byte(s), cfg); err != nil {
|
||||||
t.Fatalf("Unable to load YAML config: %s", err)
|
t.Fatalf("Unable to load YAML config: %s", err)
|
||||||
}
|
}
|
||||||
|
testutil.Equals(t, 1, len(cfg.AlertingConfig.AlertmanagerConfigs))
|
||||||
|
|
||||||
if err := n.ApplyConfig(cfg); err != nil {
|
if err := n.ApplyConfig(cfg); err != nil {
|
||||||
t.Fatalf("Error Applying the config:%v", err)
|
t.Fatalf("Error Applying the config:%v", err)
|
||||||
|
@ -474,18 +474,16 @@ alerting:
|
||||||
|
|
||||||
tgs := make(map[string][]*targetgroup.Group)
|
tgs := make(map[string][]*targetgroup.Group)
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
|
for k := range cfg.AlertingConfig.AlertmanagerConfigs.ToMap() {
|
||||||
b, err := json.Marshal(cfg.AlertingConfig.AlertmanagerConfigs[0])
|
tgs[k] = []*targetgroup.Group{
|
||||||
if err != nil {
|
tt.in,
|
||||||
t.Fatalf("Error creating config hash:%v", err)
|
}
|
||||||
}
|
break
|
||||||
tgs[fmt.Sprintf("%x", md5.Sum(b))] = []*targetgroup.Group{
|
|
||||||
tt.in,
|
|
||||||
}
|
}
|
||||||
n.reload(tgs)
|
n.reload(tgs)
|
||||||
res := n.Alertmanagers()[0].String()
|
res := n.Alertmanagers()[0].String()
|
||||||
|
|
||||||
testutil.Equals(t, res, tt.out)
|
testutil.Equals(t, tt.out, res)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -522,6 +520,7 @@ alerting:
|
||||||
if err := yaml.UnmarshalStrict([]byte(s), cfg); err != nil {
|
if err := yaml.UnmarshalStrict([]byte(s), cfg); err != nil {
|
||||||
t.Fatalf("Unable to load YAML config: %s", err)
|
t.Fatalf("Unable to load YAML config: %s", err)
|
||||||
}
|
}
|
||||||
|
testutil.Equals(t, 1, len(cfg.AlertingConfig.AlertmanagerConfigs))
|
||||||
|
|
||||||
if err := n.ApplyConfig(cfg); err != nil {
|
if err := n.ApplyConfig(cfg); err != nil {
|
||||||
t.Fatalf("Error Applying the config:%v", err)
|
t.Fatalf("Error Applying the config:%v", err)
|
||||||
|
@ -529,20 +528,18 @@ alerting:
|
||||||
|
|
||||||
tgs := make(map[string][]*targetgroup.Group)
|
tgs := make(map[string][]*targetgroup.Group)
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
|
for k := range cfg.AlertingConfig.AlertmanagerConfigs.ToMap() {
|
||||||
|
tgs[k] = []*targetgroup.Group{
|
||||||
|
tt.in,
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
b, err := json.Marshal(cfg.AlertingConfig.AlertmanagerConfigs[0])
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Error creating config hash:%v", err)
|
|
||||||
}
|
|
||||||
tgs[fmt.Sprintf("%x", md5.Sum(b))] = []*targetgroup.Group{
|
|
||||||
tt.in,
|
|
||||||
}
|
|
||||||
n.reload(tgs)
|
n.reload(tgs)
|
||||||
res := n.DroppedAlertmanagers()[0].String()
|
res := n.DroppedAlertmanagers()[0].String()
|
||||||
|
|
||||||
testutil.Equals(t, res, tt.out)
|
testutil.Equals(t, res, tt.out)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func makeInputTargetGroup() *targetgroup.Group {
|
func makeInputTargetGroup() *targetgroup.Group {
|
||||||
|
|
Loading…
Reference in a new issue