Merge pull request #1878 from brancz/relabel-alerts

allow relabeling of alerts

config/config.go

@@ -177,6 +177,7 @@ func (u URL) MarshalYAML() (interface{}, error) {
 // Config is the top-level configuration for Prometheus's config files.
 type Config struct {
 	GlobalConfig   GlobalConfig    `yaml:"global"`
+	AlertingConfig AlertingConfig  `yaml:"alerting,omitempty"`
 	RuleFiles      []string        `yaml:"rule_files,omitempty"`
 	ScrapeConfigs  []*ScrapeConfig `yaml:"scrape_configs,omitempty"`
 
@@ -292,6 +293,11 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	return nil
 }
 
+// AlertingConfig configures alerting and alertmanager related configs
+type AlertingConfig struct {
+	AlertRelabelConfigs []*RelabelConfig `yaml:"alert_relabel_configs,omitempty"`
+}
+
 // GlobalConfig configures values that are used across other configuration
 // objects.
 type GlobalConfig struct {

notifier/notifier.go

@@ -30,6 +30,7 @@ import (
 	"golang.org/x/net/context/ctxhttp"
 
 	"github.com/prometheus/prometheus/config"
+	"github.com/prometheus/prometheus/relabel"
 )
 
 const (
@@ -69,9 +70,10 @@ type Options struct {
 	QueueCapacity    int
 	Timeout          time.Duration
 	ExternalLabels   model.LabelSet
+	RelabelConfigs   []*config.RelabelConfig
 }
 
-// New constructs a neww Notifier.
+// New constructs a new Notifier.
 func New(o *Options) *Notifier {
 	ctx, cancel := context.WithCancel(context.Background())
 
@@ -136,6 +138,7 @@ func (n *Notifier) ApplyConfig(conf *config.Config) error {
 	defer n.mtx.Unlock()
 
 	n.opts.ExternalLabels = conf.GlobalConfig.ExternalLabels
+	n.opts.RelabelConfigs = conf.AlertingConfig.AlertRelabelConfigs
 	return nil
 }
 
@@ -208,6 +211,8 @@ func (n *Notifier) Send(alerts ...*model.Alert) {
 	n.mtx.Lock()
 	defer n.mtx.Unlock()
 
+	alerts = n.relabelAlerts(alerts)
+
 	// Queue capacity should be significantly larger than a single alert
 	// batch could be.
 	if d := len(alerts) - n.opts.QueueCapacity; d > 0 {
@@ -231,6 +236,18 @@ func (n *Notifier) Send(alerts ...*model.Alert) {
 	n.setMore()
 }
 
+func (n *Notifier) relabelAlerts(alerts []*model.Alert) []*model.Alert {
+	var relabeledAlerts []*model.Alert
+	for _, alert := range alerts {
+		labels := relabel.Process(alert.Labels, n.opts.RelabelConfigs...)
+		if labels != nil {
+			alert.Labels = labels
+			relabeledAlerts = append(relabeledAlerts, alert)
+		}
+	}
+	return relabeledAlerts
+}
+
 // setMore signals that the alert queue has items.
 func (n *Notifier) setMore() {
 	// If we cannot send on the channel, it means the signal already exists

notifier/notifier_test.go

@@ -23,6 +23,8 @@ import (
 	"time"
 
 	"github.com/prometheus/common/model"
+
+	"github.com/prometheus/prometheus/config"
 )
 
 func TestPostURL(t *testing.T) {
@@ -185,7 +187,53 @@ func TestHandlerSendAll(t *testing.T) {
 	}
 }
 
-func TestHandlerFull(t *testing.T) {
+func TestHandlerRelabel(t *testing.T) {
+	h := New(&Options{
+		QueueCapacity: 3 * maxBatchSize,
+		RelabelConfigs: []*config.RelabelConfig{
+			&config.RelabelConfig{
+				SourceLabels: model.LabelNames{"alertname"},
+				Action:       "drop",
+				Regex:        config.MustNewRegexp("drop"),
+			},
+			&config.RelabelConfig{
+				SourceLabels: model.LabelNames{"alertname"},
+				TargetLabel:  "alertname",
+				Action:       "replace",
+				Regex:        config.MustNewRegexp("rename"),
+				Replacement:  "renamed",
+			},
+		},
+	})
+
+	// This alert should be dropped due to the configuration
+	h.Send(&model.Alert{
+		Labels: model.LabelSet{
+			"alertname": "drop",
+		},
+	})
+
+	// This alert should be replaced due to the configuration
+	h.Send(&model.Alert{
+		Labels: model.LabelSet{
+			"alertname": "rename",
+		},
+	})
+
+	expected := []*model.Alert{
+		&model.Alert{
+			Labels: model.LabelSet{
+				"alertname": "renamed",
+			},
+		},
+	}
+
+	if !alertsEqual(expected, h.queue) {
+		t.Errorf("Expected alerts %v, got %v", expected, h.queue)
+	}
+}
+
+func TestHandlerQueueing(t *testing.T) {
 	var (
 		unblock  = make(chan struct{})
 		called   = make(chan struct{})

relabel/relabel.go

@@ -11,7 +11,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package retrieval
+package relabel
 
 import (
 	"crypto/md5"
@@ -23,27 +23,23 @@ import (
 	"github.com/prometheus/prometheus/config"
 )
 
-// Relabel returns a relabeled copy of the given label set. The relabel configurations
+// Process returns a relabeled copy of the given label set. The relabel configurations
 // are applied in order of input.
 // If a label set is dropped, nil is returned.
-func Relabel(labels model.LabelSet, cfgs ...*config.RelabelConfig) (model.LabelSet, error) {
+func Process(labels model.LabelSet, cfgs ...*config.RelabelConfig) model.LabelSet {
 	out := model.LabelSet{}
 	for ln, lv := range labels {
 		out[ln] = lv
 	}
-	var err error
 	for _, cfg := range cfgs {
-		if out, err = relabel(out, cfg); err != nil {
+		if out = relabel(out, cfg); out == nil {
-			return nil, err
+			return nil
-		}
-		if out == nil {
-			return nil, nil
 		}
 	}
-	return out, nil
+	return out
 }
 
-func relabel(labels model.LabelSet, cfg *config.RelabelConfig) (model.LabelSet, error) {
+func relabel(labels model.LabelSet, cfg *config.RelabelConfig) model.LabelSet {
 	values := make([]string, 0, len(cfg.SourceLabels))
 	for _, ln := range cfg.SourceLabels {
 		values = append(values, string(labels[ln]))
@@ -53,11 +49,11 @@ func relabel(labels model.LabelSet, cfg *config.RelabelConfig) (model.LabelSet,
 	switch cfg.Action {
 	case config.RelabelDrop:
 		if cfg.Regex.MatchString(val) {
-			return nil, nil
+			return nil
 		}
 	case config.RelabelKeep:
 		if !cfg.Regex.MatchString(val) {
-			return nil, nil
+			return nil
 		}
 	case config.RelabelReplace:
 		indexes := cfg.Regex.FindStringSubmatchIndex(val)
@@ -90,7 +86,7 @@ func relabel(labels model.LabelSet, cfg *config.RelabelConfig) (model.LabelSet,
 	default:
 		panic(fmt.Errorf("retrieval.relabel: unknown relabel action type %q", cfg.Action))
 	}
-	return labels, nil
+	return labels
 }
 
 // sum64 sums the md5 hash to an uint64.

relabel/relabel_test.go

@@ -11,7 +11,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package retrieval
+package relabel
 
 import (
 	"reflect"
@@ -280,10 +280,7 @@ func TestRelabel(t *testing.T) {
 	}
 
 	for i, test := range tests {
-		res, err := Relabel(test.input, test.relabel...)
+		res := Process(test.input, test.relabel...)
-		if err != nil {
-			t.Errorf("Test %d: error relabeling: %s", i+1, err)
-		}
 
 		if !reflect.DeepEqual(res, test.output) {
 			t.Errorf("Test %d: relabel output mismatch: expected %#v, got %#v", i+1, test.output, res)

retrieval/target.go

@@ -26,6 +26,7 @@ import (
 	"github.com/prometheus/common/model"
 
 	"github.com/prometheus/prometheus/config"
+	"github.com/prometheus/prometheus/relabel"
 	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/util/httputil"
 )
@@ -275,10 +276,8 @@ type relabelAppender struct {
 }
 
 func (app relabelAppender) Append(s *model.Sample) error {
-	labels, err := Relabel(model.LabelSet(s.Metric), app.relabelings...)
+	labels := relabel.Process(model.LabelSet(s.Metric), app.relabelings...)
-	if err != nil {
+
-		return fmt.Errorf("metric relabeling error %s: %s", s.Metric, err)
-	}
 	// Check if the timeseries was dropped.
 	if labels == nil {
 		return nil

retrieval/targetmanager.go

@@ -25,6 +25,7 @@ import (
 	"golang.org/x/net/context"
 
 	"github.com/prometheus/prometheus/config"
+	"github.com/prometheus/prometheus/relabel"
 	"github.com/prometheus/prometheus/retrieval/discovery"
 	"github.com/prometheus/prometheus/storage"
 )
@@ -448,10 +449,8 @@ func targetsFromGroup(tg *config.TargetGroup, cfg *config.ScrapeConfig) ([]*Targ
 
 		preRelabelLabels := labels
 
-		labels, err := Relabel(labels, cfg.RelabelConfigs...)
+		labels := relabel.Process(labels, cfg.RelabelConfigs...)
-		if err != nil {
+
-			return nil, fmt.Errorf("error while relabeling instance %d in target group %s: %s", i, tg, err)
-		}
 		// Check if the target was dropped.
 		if labels == nil {
 			continue
@@ -469,7 +468,7 @@ func targetsFromGroup(tg *config.TargetGroup, cfg *config.ScrapeConfig) ([]*Targ
 			}
 			labels[model.AddressLabel] = model.LabelValue(addr)
 		}
-		if err = config.CheckTargetAddress(labels[model.AddressLabel]); err != nil {
+		if err := config.CheckTargetAddress(labels[model.AddressLabel]); err != nil {
 			return nil, err
 		}