From 679d225c8df5fcac2791c288e8b3d15169bbeca3 Mon Sep 17 00:00:00 2001 From: Frederic Branczyk Date: Tue, 9 Aug 2016 10:08:15 +0200 Subject: [PATCH 1/3] allow relabeling of alerts in case of dropping don't even enqueue them --- config/config.go | 7 +++--- notifier/notifier.go | 19 ++++++++++++++- notifier/notifier_test.go | 50 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 71 insertions(+), 5 deletions(-) diff --git a/config/config.go b/config/config.go index 2fd88fc68..942c5322b 100644 --- a/config/config.go +++ b/config/config.go @@ -176,9 +176,10 @@ func (u URL) MarshalYAML() (interface{}, error) { // Config is the top-level configuration for Prometheus's config files. type Config struct { - GlobalConfig GlobalConfig `yaml:"global"` - RuleFiles []string `yaml:"rule_files,omitempty"` - ScrapeConfigs []*ScrapeConfig `yaml:"scrape_configs,omitempty"` + GlobalConfig GlobalConfig `yaml:"global"` + AlertRelabelConfigs []*RelabelConfig `yaml:"alert_relabel_configs,omitempty"` + RuleFiles []string `yaml:"rule_files,omitempty"` + ScrapeConfigs []*ScrapeConfig `yaml:"scrape_configs,omitempty"` // Catches all undefined fields and must be empty after parsing. XXX map[string]interface{} `yaml:",inline"` diff --git a/notifier/notifier.go b/notifier/notifier.go index 4ab8e13e7..8e5b46e3b 100644 --- a/notifier/notifier.go +++ b/notifier/notifier.go @@ -30,6 +30,7 @@ import ( "golang.org/x/net/context/ctxhttp" "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/retrieval" ) const ( @@ -69,9 +70,10 @@ type Options struct { QueueCapacity int Timeout time.Duration ExternalLabels model.LabelSet + RelabelConfigs []*config.RelabelConfig } -// New constructs a neww Notifier. +// New constructs a new Notifier. func New(o *Options) *Notifier { ctx, cancel := context.WithCancel(context.Background()) @@ -136,6 +138,7 @@ func (n *Notifier) ApplyConfig(conf *config.Config) error { defer n.mtx.Unlock() n.opts.ExternalLabels = conf.GlobalConfig.ExternalLabels + n.opts.RelabelConfigs = conf.AlertRelabelConfigs return nil } @@ -208,6 +211,8 @@ func (n *Notifier) Send(alerts ...*model.Alert) { n.mtx.Lock() defer n.mtx.Unlock() + alerts = n.relabelAlerts(alerts) + // Queue capacity should be significantly larger than a single alert // batch could be. if d := len(alerts) - n.opts.QueueCapacity; d > 0 { @@ -231,6 +236,18 @@ func (n *Notifier) Send(alerts ...*model.Alert) { n.setMore() } +func (n *Notifier) relabelAlerts(alerts []*model.Alert) []*model.Alert { + var relabeledAlerts []*model.Alert + for _, alert := range alerts { + labels, _ := retrieval.Relabel(alert.Labels, n.opts.RelabelConfigs...) + if labels != nil { + alert.Labels = labels + relabeledAlerts = append(relabeledAlerts, alert) + } + } + return relabeledAlerts +} + // setMore signals that the alert queue has items. func (n *Notifier) setMore() { // If we cannot send on the channel, it means the signal already exists diff --git a/notifier/notifier_test.go b/notifier/notifier_test.go index f8ed049a4..af87c4085 100644 --- a/notifier/notifier_test.go +++ b/notifier/notifier_test.go @@ -23,6 +23,8 @@ import ( "time" "github.com/prometheus/common/model" + + "github.com/prometheus/prometheus/config" ) func TestPostURL(t *testing.T) { @@ -185,7 +187,53 @@ func TestHandlerSendAll(t *testing.T) { } } -func TestHandlerFull(t *testing.T) { +func TestHandlerRelabel(t *testing.T) { + h := New(&Options{ + QueueCapacity: 3 * maxBatchSize, + RelabelConfigs: []*config.RelabelConfig{ + &config.RelabelConfig{ + SourceLabels: model.LabelNames{"alertname"}, + Action: "drop", + Regex: config.MustNewRegexp("drop"), + }, + &config.RelabelConfig{ + SourceLabels: model.LabelNames{"alertname"}, + TargetLabel: "alertname", + Action: "replace", + Regex: config.MustNewRegexp("rename"), + Replacement: "renamed", + }, + }, + }) + + // This alert should be dropped due to the configuration + h.Send(&model.Alert{ + Labels: model.LabelSet{ + "alertname": "drop", + }, + }) + + // This alert should be replaced due to the configuration + h.Send(&model.Alert{ + Labels: model.LabelSet{ + "alertname": "rename", + }, + }) + + expected := []*model.Alert{ + &model.Alert{ + Labels: model.LabelSet{ + "alertname": "renamed", + }, + }, + } + + if !alertsEqual(expected, h.queue) { + t.Errorf("Expected alerts %v, got %v", expected, h.queue) + } +} + +func TestHandlerQueueing(t *testing.T) { var ( unblock = make(chan struct{}) called = make(chan struct{}) From 7714b9c781d8e5ae869c1fe3679f93f71cf4a0c1 Mon Sep 17 00:00:00 2001 From: Frederic Branczyk Date: Tue, 9 Aug 2016 12:36:23 +0200 Subject: [PATCH 2/3] move relabeling functionality to its own package also remove the returned error as it was always nil --- notifier/notifier.go | 4 ++-- {retrieval => relabel}/relabel.go | 24 ++++++++++-------------- {retrieval => relabel}/relabel_test.go | 7 ++----- retrieval/target.go | 7 +++---- retrieval/targetmanager.go | 9 ++++----- 5 files changed, 21 insertions(+), 30 deletions(-) rename {retrieval => relabel}/relabel.go (84%) rename {retrieval => relabel}/relabel_test.go (97%) diff --git a/notifier/notifier.go b/notifier/notifier.go index 8e5b46e3b..75360aa47 100644 --- a/notifier/notifier.go +++ b/notifier/notifier.go @@ -30,7 +30,7 @@ import ( "golang.org/x/net/context/ctxhttp" "github.com/prometheus/prometheus/config" - "github.com/prometheus/prometheus/retrieval" + "github.com/prometheus/prometheus/relabel" ) const ( @@ -239,7 +239,7 @@ func (n *Notifier) Send(alerts ...*model.Alert) { func (n *Notifier) relabelAlerts(alerts []*model.Alert) []*model.Alert { var relabeledAlerts []*model.Alert for _, alert := range alerts { - labels, _ := retrieval.Relabel(alert.Labels, n.opts.RelabelConfigs...) + labels := relabel.Process(alert.Labels, n.opts.RelabelConfigs...) if labels != nil { alert.Labels = labels relabeledAlerts = append(relabeledAlerts, alert) diff --git a/retrieval/relabel.go b/relabel/relabel.go similarity index 84% rename from retrieval/relabel.go rename to relabel/relabel.go index 8a360fb7b..0e5d616ca 100644 --- a/retrieval/relabel.go +++ b/relabel/relabel.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package retrieval +package relabel import ( "crypto/md5" @@ -23,27 +23,23 @@ import ( "github.com/prometheus/prometheus/config" ) -// Relabel returns a relabeled copy of the given label set. The relabel configurations +// Process returns a relabeled copy of the given label set. The relabel configurations // are applied in order of input. // If a label set is dropped, nil is returned. -func Relabel(labels model.LabelSet, cfgs ...*config.RelabelConfig) (model.LabelSet, error) { +func Process(labels model.LabelSet, cfgs ...*config.RelabelConfig) model.LabelSet { out := model.LabelSet{} for ln, lv := range labels { out[ln] = lv } - var err error for _, cfg := range cfgs { - if out, err = relabel(out, cfg); err != nil { - return nil, err - } - if out == nil { - return nil, nil + if out = relabel(out, cfg); out == nil { + return nil } } - return out, nil + return out } -func relabel(labels model.LabelSet, cfg *config.RelabelConfig) (model.LabelSet, error) { +func relabel(labels model.LabelSet, cfg *config.RelabelConfig) model.LabelSet { values := make([]string, 0, len(cfg.SourceLabels)) for _, ln := range cfg.SourceLabels { values = append(values, string(labels[ln])) @@ -53,11 +49,11 @@ func relabel(labels model.LabelSet, cfg *config.RelabelConfig) (model.LabelSet, switch cfg.Action { case config.RelabelDrop: if cfg.Regex.MatchString(val) { - return nil, nil + return nil } case config.RelabelKeep: if !cfg.Regex.MatchString(val) { - return nil, nil + return nil } case config.RelabelReplace: indexes := cfg.Regex.FindStringSubmatchIndex(val) @@ -90,7 +86,7 @@ func relabel(labels model.LabelSet, cfg *config.RelabelConfig) (model.LabelSet, default: panic(fmt.Errorf("retrieval.relabel: unknown relabel action type %q", cfg.Action)) } - return labels, nil + return labels } // sum64 sums the md5 hash to an uint64. diff --git a/retrieval/relabel_test.go b/relabel/relabel_test.go similarity index 97% rename from retrieval/relabel_test.go rename to relabel/relabel_test.go index ad86c644d..6773ffeb4 100644 --- a/retrieval/relabel_test.go +++ b/relabel/relabel_test.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package retrieval +package relabel import ( "reflect" @@ -280,10 +280,7 @@ func TestRelabel(t *testing.T) { } for i, test := range tests { - res, err := Relabel(test.input, test.relabel...) - if err != nil { - t.Errorf("Test %d: error relabeling: %s", i+1, err) - } + res := Process(test.input, test.relabel...) if !reflect.DeepEqual(res, test.output) { t.Errorf("Test %d: relabel output mismatch: expected %#v, got %#v", i+1, test.output, res) diff --git a/retrieval/target.go b/retrieval/target.go index 7ce3390d3..0c34fb81f 100644 --- a/retrieval/target.go +++ b/retrieval/target.go @@ -26,6 +26,7 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/relabel" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/util/httputil" ) @@ -275,10 +276,8 @@ type relabelAppender struct { } func (app relabelAppender) Append(s *model.Sample) error { - labels, err := Relabel(model.LabelSet(s.Metric), app.relabelings...) - if err != nil { - return fmt.Errorf("metric relabeling error %s: %s", s.Metric, err) - } + labels := relabel.Process(model.LabelSet(s.Metric), app.relabelings...) + // Check if the timeseries was dropped. if labels == nil { return nil diff --git a/retrieval/targetmanager.go b/retrieval/targetmanager.go index 846bbe5f6..56d847d1d 100644 --- a/retrieval/targetmanager.go +++ b/retrieval/targetmanager.go @@ -25,6 +25,7 @@ import ( "golang.org/x/net/context" "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/relabel" "github.com/prometheus/prometheus/retrieval/discovery" "github.com/prometheus/prometheus/storage" ) @@ -448,10 +449,8 @@ func targetsFromGroup(tg *config.TargetGroup, cfg *config.ScrapeConfig) ([]*Targ preRelabelLabels := labels - labels, err := Relabel(labels, cfg.RelabelConfigs...) - if err != nil { - return nil, fmt.Errorf("error while relabeling instance %d in target group %s: %s", i, tg, err) - } + labels := relabel.Process(labels, cfg.RelabelConfigs...) + // Check if the target was dropped. if labels == nil { continue @@ -469,7 +468,7 @@ func targetsFromGroup(tg *config.TargetGroup, cfg *config.ScrapeConfig) ([]*Targ } labels[model.AddressLabel] = model.LabelValue(addr) } - if err = config.CheckTargetAddress(labels[model.AddressLabel]); err != nil { + if err := config.CheckTargetAddress(labels[model.AddressLabel]); err != nil { return nil, err } From b655aa002fba7beaf2da30a038e0e6c5c41d4e03 Mon Sep 17 00:00:00 2001 From: Frederic Branczyk Date: Tue, 9 Aug 2016 13:09:36 +0200 Subject: [PATCH 3/3] introduce top level alerting config node --- config/config.go | 13 +++++++++---- notifier/notifier.go | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/config/config.go b/config/config.go index 942c5322b..6a4042b92 100644 --- a/config/config.go +++ b/config/config.go @@ -176,10 +176,10 @@ func (u URL) MarshalYAML() (interface{}, error) { // Config is the top-level configuration for Prometheus's config files. type Config struct { - GlobalConfig GlobalConfig `yaml:"global"` - AlertRelabelConfigs []*RelabelConfig `yaml:"alert_relabel_configs,omitempty"` - RuleFiles []string `yaml:"rule_files,omitempty"` - ScrapeConfigs []*ScrapeConfig `yaml:"scrape_configs,omitempty"` + GlobalConfig GlobalConfig `yaml:"global"` + AlertingConfig AlertingConfig `yaml:"alerting,omitempty"` + RuleFiles []string `yaml:"rule_files,omitempty"` + ScrapeConfigs []*ScrapeConfig `yaml:"scrape_configs,omitempty"` // Catches all undefined fields and must be empty after parsing. XXX map[string]interface{} `yaml:",inline"` @@ -293,6 +293,11 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { return nil } +// AlertingConfig configures alerting and alertmanager related configs +type AlertingConfig struct { + AlertRelabelConfigs []*RelabelConfig `yaml:"alert_relabel_configs,omitempty"` +} + // GlobalConfig configures values that are used across other configuration // objects. type GlobalConfig struct { diff --git a/notifier/notifier.go b/notifier/notifier.go index 75360aa47..2e36b62b7 100644 --- a/notifier/notifier.go +++ b/notifier/notifier.go @@ -138,7 +138,7 @@ func (n *Notifier) ApplyConfig(conf *config.Config) error { defer n.mtx.Unlock() n.opts.ExternalLabels = conf.GlobalConfig.ExternalLabels - n.opts.RelabelConfigs = conf.AlertRelabelConfigs + n.opts.RelabelConfigs = conf.AlertingConfig.AlertRelabelConfigs return nil }