Add relabelling to target management.

This commit adds a relabelling stage on the set of base
labels from which a target is created. It allows to drop
targets and rewrite any regular or internal label.
This commit is contained in:
Fabian Reinartz 2015-04-29 00:08:58 +02:00
parent 0b619b46d6
commit 945c49a2dd
9 changed files with 483 additions and 25 deletions

View file

@ -14,6 +14,7 @@
package config
import (
"errors"
"fmt"
"regexp"
"strings"
@ -190,6 +191,11 @@ func (c *ScrapeConfig) Validate() error {
return fmt.Errorf("invalid DNS config: %s", err)
}
}
for _, rlcfg := range c.RelabelConfigs() {
if err := rlcfg.Validate(); err != nil {
return fmt.Errorf("invalid relabelling config: %s", err)
}
}
return nil
}
@ -203,6 +209,15 @@ func (c *ScrapeConfig) DNSConfigs() []*DNSConfig {
return dnscfgs
}
// RelabelConfigs returns the relabel configs of the scrape config.
func (c *ScrapeConfig) RelabelConfigs() []*RelabelConfig {
var rlcfgs []*RelabelConfig
for _, rc := range c.GetRelabelConfig() {
rlcfgs = append(rlcfgs, &RelabelConfig{*rc})
}
return rlcfgs
}
// DNSConfig encapsulates the protobuf configuration object for DNS based
// service discovery.
type DNSConfig struct {
@ -222,6 +237,17 @@ func (c *DNSConfig) RefreshInterval() time.Duration {
return stringToDuration(c.GetRefreshInterval())
}
type RelabelConfig struct {
pb.RelabelConfig
}
func (c *RelabelConfig) Validate() error {
if len(c.GetSourceLabel()) == 0 {
return errors.New("at least one source label is required")
}
return nil
}
// TargetGroup is derived from a protobuf TargetGroup and attaches a source to it
// that identifies the origin of the group.
type TargetGroup struct {

View file

@ -58,9 +58,32 @@ message DNSConfig {
optional string refresh_interval = 2 [default = "30s"];
}
// The configuration for relabeling of target label sets.
message RelabelConfig {
// A list of labels from which values are taken and concatenated
// with the configured separator in order.
repeated string source_label = 1;
// Regex against which the concatenation is matched.
required string regex = 2;
// The label to which the resulting string is written in a replacement.
optional string target_label = 3;
// Replacement is the regex replacement pattern to be used.
optional string replacement = 4;
// Separator is the string between concatenated values from the source labels.
optional string separator = 5 [default = ";"];
// Action is the action to be performed for the relabeling.
enum Action {
REPLACE = 0; // Performs a regex replacement.
KEEP = 1; // Drops targets for which the input does not match the regex.
DROP = 2; // Drops targets for which the input does match the regex.
}
optional Action action = 6 [default = REPLACE];
}
// The configuration for a Prometheus job to scrape.
//
// The next field no. is 10.
// The next field no. is 11.
message ScrapeConfig {
// The job name. Must adhere to the regex "[a-zA-Z_][a-zA-Z0-9_-]*".
required string job_name = 1;
@ -75,6 +98,8 @@ message ScrapeConfig {
repeated DNSConfig dns_config = 9;
// List of labeled target groups for this job.
repeated TargetGroup target_group = 5;
// List of relabel configurations.
repeated RelabelConfig relabel_config = 10;
// The HTTP resource path on which to fetch metrics from targets.
optional string metrics_path = 6 [default = "/metrics"];
// The URL scheme with which to fetch metrics from targets.

View file

@ -14,6 +14,7 @@ It has these top-level messages:
GlobalConfig
TargetGroup
DNSConfig
RelabelConfig
ScrapeConfig
PrometheusConfig
*/
@ -26,6 +27,43 @@ import math "math"
var _ = proto.Marshal
var _ = math.Inf
// Action is the action to be performed for the relabeling.
type RelabelConfig_Action int32
const (
RelabelConfig_REPLACE RelabelConfig_Action = 0
RelabelConfig_KEEP RelabelConfig_Action = 1
RelabelConfig_DROP RelabelConfig_Action = 2
)
var RelabelConfig_Action_name = map[int32]string{
0: "REPLACE",
1: "KEEP",
2: "DROP",
}
var RelabelConfig_Action_value = map[string]int32{
"REPLACE": 0,
"KEEP": 1,
"DROP": 2,
}
func (x RelabelConfig_Action) Enum() *RelabelConfig_Action {
p := new(RelabelConfig_Action)
*p = x
return p
}
func (x RelabelConfig_Action) String() string {
return proto.EnumName(RelabelConfig_Action_name, int32(x))
}
func (x *RelabelConfig_Action) UnmarshalJSON(data []byte) error {
value, err := proto.UnmarshalJSONEnum(RelabelConfig_Action_value, data, "RelabelConfig_Action")
if err != nil {
return err
}
*x = RelabelConfig_Action(value)
return nil
}
// A label/value pair suitable for attaching to timeseries.
type LabelPair struct {
// The name of the label. Must adhere to the regex "[a-zA-Z_][a-zA-Z0-9_]*".
@ -149,7 +187,7 @@ func (m *TargetGroup) GetLabels() *LabelPairs {
// The configuration for DNS based service discovery.
type DNSConfig struct {
// The list of DNS-SD service names pointing to SRV records
// The list of DNS-SD service names pointing to SRV records
// containing endpoint information.
Name []string `protobuf:"bytes,1,rep,name=name" json:"name,omitempty"`
// Discovery refresh period when using DNS-SD to discover targets. Must be a
@ -178,9 +216,75 @@ func (m *DNSConfig) GetRefreshInterval() string {
return Default_DNSConfig_RefreshInterval
}
// The configuration for relabeling of target label sets.
type RelabelConfig struct {
// A list of labels from which values are taken and concatenated
// with the configured separator in order.
SourceLabel []string `protobuf:"bytes,1,rep,name=source_label" json:"source_label,omitempty"`
// Regex against which the concatenation is matched.
Regex *string `protobuf:"bytes,2,req,name=regex" json:"regex,omitempty"`
// The label to which the resulting string is written in a replacement.
TargetLabel *string `protobuf:"bytes,3,opt,name=target_label" json:"target_label,omitempty"`
// Replacement is the regex replacement pattern to be used.
Replacement *string `protobuf:"bytes,4,opt,name=replacement" json:"replacement,omitempty"`
// Separator is the string between concatenated values from the source labels.
Separator *string `protobuf:"bytes,5,opt,name=separator,def=;" json:"separator,omitempty"`
Action *RelabelConfig_Action `protobuf:"varint,6,opt,name=action,enum=io.prometheus.RelabelConfig_Action,def=0" json:"action,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *RelabelConfig) Reset() { *m = RelabelConfig{} }
func (m *RelabelConfig) String() string { return proto.CompactTextString(m) }
func (*RelabelConfig) ProtoMessage() {}
const Default_RelabelConfig_Separator string = ";"
const Default_RelabelConfig_Action RelabelConfig_Action = RelabelConfig_REPLACE
func (m *RelabelConfig) GetSourceLabel() []string {
if m != nil {
return m.SourceLabel
}
return nil
}
func (m *RelabelConfig) GetRegex() string {
if m != nil && m.Regex != nil {
return *m.Regex
}
return ""
}
func (m *RelabelConfig) GetTargetLabel() string {
if m != nil && m.TargetLabel != nil {
return *m.TargetLabel
}
return ""
}
func (m *RelabelConfig) GetReplacement() string {
if m != nil && m.Replacement != nil {
return *m.Replacement
}
return ""
}
func (m *RelabelConfig) GetSeparator() string {
if m != nil && m.Separator != nil {
return *m.Separator
}
return Default_RelabelConfig_Separator
}
func (m *RelabelConfig) GetAction() RelabelConfig_Action {
if m != nil && m.Action != nil {
return *m.Action
}
return Default_RelabelConfig_Action
}
// The configuration for a Prometheus job to scrape.
//
// The next field no. is 10.
// The next field no. is 11.
type ScrapeConfig struct {
// The job name. Must adhere to the regex "[a-zA-Z_][a-zA-Z0-9_-]*".
JobName *string `protobuf:"bytes,1,req,name=job_name" json:"job_name,omitempty"`
@ -195,6 +299,8 @@ type ScrapeConfig struct {
DnsConfig []*DNSConfig `protobuf:"bytes,9,rep,name=dns_config" json:"dns_config,omitempty"`
// List of labeled target groups for this job.
TargetGroup []*TargetGroup `protobuf:"bytes,5,rep,name=target_group" json:"target_group,omitempty"`
// List of relabel configurations.
RelabelConfig []*RelabelConfig `protobuf:"bytes,10,rep,name=relabel_config" json:"relabel_config,omitempty"`
// The HTTP resource path on which to fetch metrics from targets.
MetricsPath *string `protobuf:"bytes,6,opt,name=metrics_path,def=/metrics" json:"metrics_path,omitempty"`
// The URL scheme with which to fetch metrics from targets.
@ -245,6 +351,13 @@ func (m *ScrapeConfig) GetTargetGroup() []*TargetGroup {
return nil
}
func (m *ScrapeConfig) GetRelabelConfig() []*RelabelConfig {
if m != nil {
return m.RelabelConfig
}
return nil
}
func (m *ScrapeConfig) GetMetricsPath() string {
if m != nil && m.MetricsPath != nil {
return *m.MetricsPath
@ -289,4 +402,5 @@ func (m *PrometheusConfig) GetScrapeConfig() []*ScrapeConfig {
}
func init() {
proto.RegisterEnum("io.prometheus.RelabelConfig_Action", RelabelConfig_Action_name, RelabelConfig_Action_value)
}

View file

@ -32,6 +32,7 @@ const (
resolvConf = "/etc/resolv.conf"
dnsSourcePrefix = "dns"
DNSNameLabel = clientmodel.MetaLabelPrefix + "dns_srv_name"
// Constants for instrumentation.
namespace = "prometheus"
@ -148,6 +149,7 @@ func (dd *DNSDiscovery) refresh(name string, ch chan<- *config.TargetGroup) erro
target := clientmodel.LabelValue(fmt.Sprintf("%s:%d", addr.Target, addr.Port))
tg.Targets = append(tg.Targets, clientmodel.LabelSet{
clientmodel.AddressLabel: target,
DNSNameLabel: clientmodel.LabelValue(name),
})
}

70
retrieval/relabel.go Normal file
View file

@ -0,0 +1,70 @@
package retrieval
import (
"regexp"
"strings"
clientmodel "github.com/prometheus/client_golang/model"
"github.com/prometheus/prometheus/config"
pb "github.com/prometheus/prometheus/config/generated"
)
// Relabel returns a relabeled copy of the given label set. The relabel configurations
// are applied in order of input.
// If a label set is dropped, nil is returned.
func Relabel(labels clientmodel.LabelSet, cfgs ...*config.RelabelConfig) (clientmodel.LabelSet, error) {
out := clientmodel.LabelSet{}
for ln, lv := range labels {
out[ln] = lv
}
var err error
for _, cfg := range cfgs {
if out, err = relabel(out, cfg); err != nil {
return nil, err
}
if out == nil {
return nil, nil
}
}
return out, nil
}
func relabel(labels clientmodel.LabelSet, cfg *config.RelabelConfig) (clientmodel.LabelSet, error) {
pat, err := regexp.Compile(cfg.GetRegex())
if err != nil {
return nil, err
}
values := make([]string, 0, len(cfg.GetSourceLabel()))
for _, name := range cfg.GetSourceLabel() {
values = append(values, string(labels[clientmodel.LabelName(name)]))
}
val := strings.Join(values, cfg.GetSeparator())
switch cfg.GetAction() {
case pb.RelabelConfig_DROP:
if pat.MatchString(val) {
return nil, nil
}
case pb.RelabelConfig_KEEP:
if !pat.MatchString(val) {
return nil, nil
}
case pb.RelabelConfig_REPLACE:
// If there is no match no replacement must take place.
if !pat.MatchString(val) {
break
}
res := pat.ReplaceAllString(val, cfg.GetReplacement())
ln := clientmodel.LabelName(cfg.GetTargetLabel())
if res == "" {
delete(labels, ln)
} else {
labels[ln] = clientmodel.LabelValue(res)
}
default:
panic("retrieval.relabel: unknown relabel action type")
}
return labels, nil
}

172
retrieval/relabel_test.go Normal file
View file

@ -0,0 +1,172 @@
package retrieval
import (
"reflect"
"testing"
"github.com/golang/protobuf/proto"
clientmodel "github.com/prometheus/client_golang/model"
"github.com/prometheus/prometheus/config"
pb "github.com/prometheus/prometheus/config/generated"
)
func TestRelabel(t *testing.T) {
tests := []struct {
input clientmodel.LabelSet
relabel []pb.RelabelConfig
output clientmodel.LabelSet
}{
{
input: clientmodel.LabelSet{
"a": "foo",
"b": "bar",
"c": "baz",
},
relabel: []pb.RelabelConfig{
{
SourceLabel: []string{"a"},
Regex: proto.String("f(.*)"),
TargetLabel: proto.String("d"),
Separator: proto.String(";"),
Replacement: proto.String("ch${1}-ch${1}"),
Action: pb.RelabelConfig_REPLACE.Enum(),
},
},
output: clientmodel.LabelSet{
"a": "foo",
"b": "bar",
"c": "baz",
"d": "choo-choo",
},
},
{
input: clientmodel.LabelSet{
"a": "foo",
"b": "bar",
"c": "baz",
},
relabel: []pb.RelabelConfig{
{
SourceLabel: []string{"a", "b"},
Regex: proto.String("^f(.*);(.*)r$"),
TargetLabel: proto.String("a"),
Separator: proto.String(";"),
Replacement: proto.String("b${1}${2}m"), // boobam
},
{
SourceLabel: []string{"c", "a"},
Regex: proto.String("(b).*b(.*)ba(.*)"),
TargetLabel: proto.String("d"),
Separator: proto.String(";"),
Replacement: proto.String("$1$2$2$3"),
Action: pb.RelabelConfig_REPLACE.Enum(),
},
},
output: clientmodel.LabelSet{
"a": "boobam",
"b": "bar",
"c": "baz",
"d": "boooom",
},
},
{
input: clientmodel.LabelSet{
"a": "foo",
},
relabel: []pb.RelabelConfig{
{
SourceLabel: []string{"a"},
Regex: proto.String("o$"),
Action: pb.RelabelConfig_DROP.Enum(),
}, {
SourceLabel: []string{"a"},
Regex: proto.String("f(.*)"),
TargetLabel: proto.String("d"),
Separator: proto.String(";"),
Replacement: proto.String("ch$1-ch$1"),
Action: pb.RelabelConfig_REPLACE.Enum(),
},
},
output: nil,
},
{
input: clientmodel.LabelSet{
"a": "foo",
},
relabel: []pb.RelabelConfig{
{
SourceLabel: []string{"a"},
Regex: proto.String("no-match"),
Action: pb.RelabelConfig_DROP.Enum(),
},
},
output: clientmodel.LabelSet{
"a": "foo",
},
},
{
input: clientmodel.LabelSet{
"a": "foo",
},
relabel: []pb.RelabelConfig{
{
SourceLabel: []string{"a"},
Regex: proto.String("no-match"),
Action: pb.RelabelConfig_KEEP.Enum(),
},
},
output: nil,
},
{
input: clientmodel.LabelSet{
"a": "foo",
},
relabel: []pb.RelabelConfig{
{
SourceLabel: []string{"a"},
Regex: proto.String("^f"),
Action: pb.RelabelConfig_KEEP.Enum(),
},
},
output: clientmodel.LabelSet{
"a": "foo",
},
},
{
// No replacement must be applied if there is no match.
input: clientmodel.LabelSet{
"a": "boo",
},
relabel: []pb.RelabelConfig{
{
SourceLabel: []string{"a"},
Regex: proto.String("^f"),
Action: pb.RelabelConfig_REPLACE.Enum(),
TargetLabel: proto.String("b"),
Replacement: proto.String("bar"),
},
},
output: clientmodel.LabelSet{
"a": "boo",
},
},
}
for i, test := range tests {
var relabel []*config.RelabelConfig
for _, rl := range test.relabel {
proto.SetDefaults(&rl)
relabel = append(relabel, &config.RelabelConfig{rl})
}
res, err := Relabel(test.input, relabel...)
if err != nil {
t.Errorf("Test %d: error relabeling: %s", i+1, err)
}
if !reflect.DeepEqual(res, test.output) {
t.Errorf("Test %d: relabel output mismatch: expected %#v, got %#v", i+1, test.output, res)
}
}
}

View file

@ -172,10 +172,10 @@ type target struct {
}
// NewTarget creates a reasonably configured target for querying.
func NewTarget(address string, cfg *config.ScrapeConfig, baseLabels clientmodel.LabelSet) Target {
func NewTarget(cfg *config.ScrapeConfig, baseLabels clientmodel.LabelSet) Target {
t := &target{
url: &url.URL{
Host: address,
Host: string(baseLabels[clientmodel.AddressLabel]),
},
scraperStopping: make(chan struct{}),
scraperStopped: make(chan struct{}),
@ -197,16 +197,16 @@ func (t *target) Update(cfg *config.ScrapeConfig, baseLabels clientmodel.LabelSe
t.deadline = cfg.ScrapeTimeout()
t.httpClient = utility.NewDeadlineClient(cfg.ScrapeTimeout())
t.baseLabels = clientmodel.LabelSet{
clientmodel.InstanceLabel: clientmodel.LabelValue(t.InstanceIdentifier()),
}
t.baseLabels = clientmodel.LabelSet{}
// All remaining internal labels will not be part of the label set.
for name, val := range baseLabels {
if !strings.HasPrefix(string(name), clientmodel.ReservedLabelPrefix) {
t.baseLabels[name] = val
}
}
if _, ok := t.baseLabels[clientmodel.InstanceLabel]; !ok {
t.baseLabels[clientmodel.InstanceLabel] = clientmodel.LabelValue(t.InstanceIdentifier())
}
}
func (t *target) String() string {

View file

@ -81,19 +81,19 @@ func (tm *TargetManager) Run() {
sources := map[string]struct{}{}
for scfg, provs := range tm.providers {
for _, p := range provs {
for _, prov := range provs {
ch := make(chan *config.TargetGroup)
go tm.handleTargetUpdates(scfg, ch)
for _, src := range p.Sources() {
for _, src := range prov.Sources() {
src = fullSource(scfg, src)
sources[src] = struct{}{}
}
// Run the target provider after cleanup of the stale targets is done.
defer func(c chan *config.TargetGroup) {
defer func(p TargetProvider, c chan *config.TargetGroup) {
go p.Run(c)
}(ch)
}(prov, ch)
}
}
@ -326,9 +326,17 @@ func (tm *TargetManager) targetsFromGroup(tg *config.TargetGroup, cfg *config.Sc
}
}
address, ok := labels[clientmodel.AddressLabel]
if !ok {
return nil, fmt.Errorf("Instance %d in target group %s has no address", i, tg)
if _, ok := labels[clientmodel.AddressLabel]; !ok {
return nil, fmt.Errorf("instance %d in target group %s has no address", i, tg)
}
labels, err := Relabel(labels, cfg.RelabelConfigs()...)
if err != nil {
return nil, fmt.Errorf("error while relabelling instance %d in target group %s: %s", i, tg, err)
}
// Check if the target was dropped.
if labels == nil {
continue
}
for ln := range labels {
@ -338,8 +346,8 @@ func (tm *TargetManager) targetsFromGroup(tg *config.TargetGroup, cfg *config.Sc
delete(labels, ln)
}
}
targets = append(targets, NewTarget(string(address), cfg, labels))
tr := NewTarget(cfg, labels)
targets = append(targets, tr)
}
return targets, nil

View file

@ -164,10 +164,45 @@ func TestTargetManagerConfigUpdate(t *testing.T) {
JobName: proto.String("test_job2"),
ScrapeInterval: proto.String("1m"),
TargetGroup: []*pb.TargetGroup{
{Target: []string{"example.org:8080", "example.com:8081"}},
{
Target: []string{"example.org:8080", "example.com:8081"},
Labels: &pb.LabelPairs{Label: []*pb.LabelPair{
{Name: proto.String("foo"), Value: proto.String("bar")},
{Name: proto.String("boom"), Value: proto.String("box")},
}},
},
{Target: []string{"test.com:1234"}},
{
Target: []string{"test.com:1235"},
Labels: &pb.LabelPairs{Label: []*pb.LabelPair{
{Name: proto.String("instance"), Value: proto.String("fixed")},
}},
},
},
RelabelConfig: []*pb.RelabelConfig{
{
SourceLabel: []string{string(clientmodel.AddressLabel)},
Regex: proto.String(`^test\.(.*?):(.*)`),
Replacement: proto.String("foo.${1}:${2}"),
TargetLabel: proto.String(string(clientmodel.AddressLabel)),
}, {
// Add a new label for example.* targets.
SourceLabel: []string{string(clientmodel.AddressLabel), "boom", "foo"},
Regex: proto.String("^example.*?-b([a-z-]+)r$"),
TargetLabel: proto.String("new"),
Replacement: proto.String("$1"),
Separator: proto.String("-"),
}, {
// Drop an existing label.
SourceLabel: []string{"boom"},
Regex: proto.String(".*"),
TargetLabel: proto.String("boom"),
Replacement: proto.String(""),
},
},
}
proto.SetDefaults(testJob1)
proto.SetDefaults(testJob2)
sequence := []struct {
scrapeConfigs []*pb.ScrapeConfig
@ -197,11 +232,14 @@ func TestTargetManagerConfigUpdate(t *testing.T) {
{clientmodel.JobLabel: "test_job1", clientmodel.InstanceLabel: "example.com:80"},
},
"test_job2:static:0": {
{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "example.org:8080"},
{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "example.com:8081"},
{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "example.org:8080", "foo": "bar", "new": "ox-ba"},
{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "example.com:8081", "foo": "bar", "new": "ox-ba"},
},
"test_job2:static:1": {
{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "test.com:1234"},
{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "foo.com:1234"},
},
"test_job2:static:2": {
{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "fixed"},
},
},
}, {
@ -211,11 +249,14 @@ func TestTargetManagerConfigUpdate(t *testing.T) {
scrapeConfigs: []*pb.ScrapeConfig{testJob2},
expected: map[string][]clientmodel.LabelSet{
"test_job2:static:0": {
{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "example.org:8080"},
{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "example.com:8081"},
{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "example.org:8080", "foo": "bar", "new": "ox-ba"},
{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "example.com:8081", "foo": "bar", "new": "ox-ba"},
},
"test_job2:static:1": {
{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "test.com:1234"},
{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "foo.com:1234"},
},
"test_job2:static:2": {
{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "fixed"},
},
},
},