diff --git a/config/config.go b/config/config.go index 844bd79e2..94f45fb0f 100644 --- a/config/config.go +++ b/config/config.go @@ -14,6 +14,7 @@ package config import ( + "errors" "fmt" "regexp" "strings" @@ -190,6 +191,11 @@ func (c *ScrapeConfig) Validate() error { return fmt.Errorf("invalid DNS config: %s", err) } } + for _, rlcfg := range c.RelabelConfigs() { + if err := rlcfg.Validate(); err != nil { + return fmt.Errorf("invalid relabelling config: %s", err) + } + } return nil } @@ -203,6 +209,15 @@ func (c *ScrapeConfig) DNSConfigs() []*DNSConfig { return dnscfgs } +// RelabelConfigs returns the relabel configs of the scrape config. +func (c *ScrapeConfig) RelabelConfigs() []*RelabelConfig { + var rlcfgs []*RelabelConfig + for _, rc := range c.GetRelabelConfig() { + rlcfgs = append(rlcfgs, &RelabelConfig{*rc}) + } + return rlcfgs +} + // DNSConfig encapsulates the protobuf configuration object for DNS based // service discovery. type DNSConfig struct { @@ -222,6 +237,17 @@ func (c *DNSConfig) RefreshInterval() time.Duration { return stringToDuration(c.GetRefreshInterval()) } +type RelabelConfig struct { + pb.RelabelConfig +} + +func (c *RelabelConfig) Validate() error { + if len(c.GetSourceLabel()) == 0 { + return errors.New("at least one source label is required") + } + return nil +} + // TargetGroup is derived from a protobuf TargetGroup and attaches a source to it // that identifies the origin of the group. type TargetGroup struct { diff --git a/config/config.proto b/config/config.proto index 3de910005..c0d534449 100644 --- a/config/config.proto +++ b/config/config.proto @@ -58,9 +58,32 @@ message DNSConfig { optional string refresh_interval = 2 [default = "30s"]; } +// The configuration for relabeling of target label sets. +message RelabelConfig { + // A list of labels from which values are taken and concatenated + // with the configured separator in order. + repeated string source_label = 1; + // Regex against which the concatenation is matched. + required string regex = 2; + // The label to which the resulting string is written in a replacement. + optional string target_label = 3; + // Replacement is the regex replacement pattern to be used. + optional string replacement = 4; + // Separator is the string between concatenated values from the source labels. + optional string separator = 5 [default = ";"]; + + // Action is the action to be performed for the relabeling. + enum Action { + REPLACE = 0; // Performs a regex replacement. + KEEP = 1; // Drops targets for which the input does not match the regex. + DROP = 2; // Drops targets for which the input does match the regex. + } + optional Action action = 6 [default = REPLACE]; +} + // The configuration for a Prometheus job to scrape. // -// The next field no. is 10. +// The next field no. is 11. message ScrapeConfig { // The job name. Must adhere to the regex "[a-zA-Z_][a-zA-Z0-9_-]*". required string job_name = 1; @@ -75,6 +98,8 @@ message ScrapeConfig { repeated DNSConfig dns_config = 9; // List of labeled target groups for this job. repeated TargetGroup target_group = 5; + // List of relabel configurations. + repeated RelabelConfig relabel_config = 10; // The HTTP resource path on which to fetch metrics from targets. optional string metrics_path = 6 [default = "/metrics"]; // The URL scheme with which to fetch metrics from targets. diff --git a/config/generated/config.pb.go b/config/generated/config.pb.go index c44247ab5..6898a9af8 100644 --- a/config/generated/config.pb.go +++ b/config/generated/config.pb.go @@ -14,6 +14,7 @@ It has these top-level messages: GlobalConfig TargetGroup DNSConfig + RelabelConfig ScrapeConfig PrometheusConfig */ @@ -26,6 +27,43 @@ import math "math" var _ = proto.Marshal var _ = math.Inf +// Action is the action to be performed for the relabeling. +type RelabelConfig_Action int32 + +const ( + RelabelConfig_REPLACE RelabelConfig_Action = 0 + RelabelConfig_KEEP RelabelConfig_Action = 1 + RelabelConfig_DROP RelabelConfig_Action = 2 +) + +var RelabelConfig_Action_name = map[int32]string{ + 0: "REPLACE", + 1: "KEEP", + 2: "DROP", +} +var RelabelConfig_Action_value = map[string]int32{ + "REPLACE": 0, + "KEEP": 1, + "DROP": 2, +} + +func (x RelabelConfig_Action) Enum() *RelabelConfig_Action { + p := new(RelabelConfig_Action) + *p = x + return p +} +func (x RelabelConfig_Action) String() string { + return proto.EnumName(RelabelConfig_Action_name, int32(x)) +} +func (x *RelabelConfig_Action) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(RelabelConfig_Action_value, data, "RelabelConfig_Action") + if err != nil { + return err + } + *x = RelabelConfig_Action(value) + return nil +} + // A label/value pair suitable for attaching to timeseries. type LabelPair struct { // The name of the label. Must adhere to the regex "[a-zA-Z_][a-zA-Z0-9_]*". @@ -149,7 +187,7 @@ func (m *TargetGroup) GetLabels() *LabelPairs { // The configuration for DNS based service discovery. type DNSConfig struct { - // The list of DNS-SD service names pointing to SRV records + // The list of DNS-SD service names pointing to SRV records // containing endpoint information. Name []string `protobuf:"bytes,1,rep,name=name" json:"name,omitempty"` // Discovery refresh period when using DNS-SD to discover targets. Must be a @@ -178,9 +216,75 @@ func (m *DNSConfig) GetRefreshInterval() string { return Default_DNSConfig_RefreshInterval } +// The configuration for relabeling of target label sets. +type RelabelConfig struct { + // A list of labels from which values are taken and concatenated + // with the configured separator in order. + SourceLabel []string `protobuf:"bytes,1,rep,name=source_label" json:"source_label,omitempty"` + // Regex against which the concatenation is matched. + Regex *string `protobuf:"bytes,2,req,name=regex" json:"regex,omitempty"` + // The label to which the resulting string is written in a replacement. + TargetLabel *string `protobuf:"bytes,3,opt,name=target_label" json:"target_label,omitempty"` + // Replacement is the regex replacement pattern to be used. + Replacement *string `protobuf:"bytes,4,opt,name=replacement" json:"replacement,omitempty"` + // Separator is the string between concatenated values from the source labels. + Separator *string `protobuf:"bytes,5,opt,name=separator,def=;" json:"separator,omitempty"` + Action *RelabelConfig_Action `protobuf:"varint,6,opt,name=action,enum=io.prometheus.RelabelConfig_Action,def=0" json:"action,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *RelabelConfig) Reset() { *m = RelabelConfig{} } +func (m *RelabelConfig) String() string { return proto.CompactTextString(m) } +func (*RelabelConfig) ProtoMessage() {} + +const Default_RelabelConfig_Separator string = ";" +const Default_RelabelConfig_Action RelabelConfig_Action = RelabelConfig_REPLACE + +func (m *RelabelConfig) GetSourceLabel() []string { + if m != nil { + return m.SourceLabel + } + return nil +} + +func (m *RelabelConfig) GetRegex() string { + if m != nil && m.Regex != nil { + return *m.Regex + } + return "" +} + +func (m *RelabelConfig) GetTargetLabel() string { + if m != nil && m.TargetLabel != nil { + return *m.TargetLabel + } + return "" +} + +func (m *RelabelConfig) GetReplacement() string { + if m != nil && m.Replacement != nil { + return *m.Replacement + } + return "" +} + +func (m *RelabelConfig) GetSeparator() string { + if m != nil && m.Separator != nil { + return *m.Separator + } + return Default_RelabelConfig_Separator +} + +func (m *RelabelConfig) GetAction() RelabelConfig_Action { + if m != nil && m.Action != nil { + return *m.Action + } + return Default_RelabelConfig_Action +} + // The configuration for a Prometheus job to scrape. // -// The next field no. is 10. +// The next field no. is 11. type ScrapeConfig struct { // The job name. Must adhere to the regex "[a-zA-Z_][a-zA-Z0-9_-]*". JobName *string `protobuf:"bytes,1,req,name=job_name" json:"job_name,omitempty"` @@ -195,6 +299,8 @@ type ScrapeConfig struct { DnsConfig []*DNSConfig `protobuf:"bytes,9,rep,name=dns_config" json:"dns_config,omitempty"` // List of labeled target groups for this job. TargetGroup []*TargetGroup `protobuf:"bytes,5,rep,name=target_group" json:"target_group,omitempty"` + // List of relabel configurations. + RelabelConfig []*RelabelConfig `protobuf:"bytes,10,rep,name=relabel_config" json:"relabel_config,omitempty"` // The HTTP resource path on which to fetch metrics from targets. MetricsPath *string `protobuf:"bytes,6,opt,name=metrics_path,def=/metrics" json:"metrics_path,omitempty"` // The URL scheme with which to fetch metrics from targets. @@ -245,6 +351,13 @@ func (m *ScrapeConfig) GetTargetGroup() []*TargetGroup { return nil } +func (m *ScrapeConfig) GetRelabelConfig() []*RelabelConfig { + if m != nil { + return m.RelabelConfig + } + return nil +} + func (m *ScrapeConfig) GetMetricsPath() string { if m != nil && m.MetricsPath != nil { return *m.MetricsPath @@ -289,4 +402,5 @@ func (m *PrometheusConfig) GetScrapeConfig() []*ScrapeConfig { } func init() { + proto.RegisterEnum("io.prometheus.RelabelConfig_Action", RelabelConfig_Action_name, RelabelConfig_Action_value) } diff --git a/retrieval/discovery/dns.go b/retrieval/discovery/dns.go index 70cf3a738..017ea476a 100644 --- a/retrieval/discovery/dns.go +++ b/retrieval/discovery/dns.go @@ -32,6 +32,7 @@ const ( resolvConf = "/etc/resolv.conf" dnsSourcePrefix = "dns" + DNSNameLabel = clientmodel.MetaLabelPrefix + "dns_srv_name" // Constants for instrumentation. namespace = "prometheus" @@ -148,6 +149,7 @@ func (dd *DNSDiscovery) refresh(name string, ch chan<- *config.TargetGroup) erro target := clientmodel.LabelValue(fmt.Sprintf("%s:%d", addr.Target, addr.Port)) tg.Targets = append(tg.Targets, clientmodel.LabelSet{ clientmodel.AddressLabel: target, + DNSNameLabel: clientmodel.LabelValue(name), }) } diff --git a/retrieval/relabel.go b/retrieval/relabel.go new file mode 100644 index 000000000..e0ddb232d --- /dev/null +++ b/retrieval/relabel.go @@ -0,0 +1,70 @@ +package retrieval + +import ( + "regexp" + "strings" + + clientmodel "github.com/prometheus/client_golang/model" + + "github.com/prometheus/prometheus/config" + pb "github.com/prometheus/prometheus/config/generated" +) + +// Relabel returns a relabeled copy of the given label set. The relabel configurations +// are applied in order of input. +// If a label set is dropped, nil is returned. +func Relabel(labels clientmodel.LabelSet, cfgs ...*config.RelabelConfig) (clientmodel.LabelSet, error) { + out := clientmodel.LabelSet{} + for ln, lv := range labels { + out[ln] = lv + } + var err error + for _, cfg := range cfgs { + if out, err = relabel(out, cfg); err != nil { + return nil, err + } + if out == nil { + return nil, nil + } + } + return out, nil +} + +func relabel(labels clientmodel.LabelSet, cfg *config.RelabelConfig) (clientmodel.LabelSet, error) { + pat, err := regexp.Compile(cfg.GetRegex()) + if err != nil { + return nil, err + } + + values := make([]string, 0, len(cfg.GetSourceLabel())) + for _, name := range cfg.GetSourceLabel() { + values = append(values, string(labels[clientmodel.LabelName(name)])) + } + val := strings.Join(values, cfg.GetSeparator()) + + switch cfg.GetAction() { + case pb.RelabelConfig_DROP: + if pat.MatchString(val) { + return nil, nil + } + case pb.RelabelConfig_KEEP: + if !pat.MatchString(val) { + return nil, nil + } + case pb.RelabelConfig_REPLACE: + // If there is no match no replacement must take place. + if !pat.MatchString(val) { + break + } + res := pat.ReplaceAllString(val, cfg.GetReplacement()) + ln := clientmodel.LabelName(cfg.GetTargetLabel()) + if res == "" { + delete(labels, ln) + } else { + labels[ln] = clientmodel.LabelValue(res) + } + default: + panic("retrieval.relabel: unknown relabel action type") + } + return labels, nil +} diff --git a/retrieval/relabel_test.go b/retrieval/relabel_test.go new file mode 100644 index 000000000..0bdfe4315 --- /dev/null +++ b/retrieval/relabel_test.go @@ -0,0 +1,172 @@ +package retrieval + +import ( + "reflect" + "testing" + + "github.com/golang/protobuf/proto" + + clientmodel "github.com/prometheus/client_golang/model" + + "github.com/prometheus/prometheus/config" + pb "github.com/prometheus/prometheus/config/generated" +) + +func TestRelabel(t *testing.T) { + tests := []struct { + input clientmodel.LabelSet + relabel []pb.RelabelConfig + output clientmodel.LabelSet + }{ + { + input: clientmodel.LabelSet{ + "a": "foo", + "b": "bar", + "c": "baz", + }, + relabel: []pb.RelabelConfig{ + { + SourceLabel: []string{"a"}, + Regex: proto.String("f(.*)"), + TargetLabel: proto.String("d"), + Separator: proto.String(";"), + Replacement: proto.String("ch${1}-ch${1}"), + Action: pb.RelabelConfig_REPLACE.Enum(), + }, + }, + output: clientmodel.LabelSet{ + "a": "foo", + "b": "bar", + "c": "baz", + "d": "choo-choo", + }, + }, + { + input: clientmodel.LabelSet{ + "a": "foo", + "b": "bar", + "c": "baz", + }, + relabel: []pb.RelabelConfig{ + { + SourceLabel: []string{"a", "b"}, + Regex: proto.String("^f(.*);(.*)r$"), + TargetLabel: proto.String("a"), + Separator: proto.String(";"), + Replacement: proto.String("b${1}${2}m"), // boobam + }, + { + SourceLabel: []string{"c", "a"}, + Regex: proto.String("(b).*b(.*)ba(.*)"), + TargetLabel: proto.String("d"), + Separator: proto.String(";"), + Replacement: proto.String("$1$2$2$3"), + Action: pb.RelabelConfig_REPLACE.Enum(), + }, + }, + output: clientmodel.LabelSet{ + "a": "boobam", + "b": "bar", + "c": "baz", + "d": "boooom", + }, + }, + { + input: clientmodel.LabelSet{ + "a": "foo", + }, + relabel: []pb.RelabelConfig{ + { + SourceLabel: []string{"a"}, + Regex: proto.String("o$"), + Action: pb.RelabelConfig_DROP.Enum(), + }, { + SourceLabel: []string{"a"}, + Regex: proto.String("f(.*)"), + TargetLabel: proto.String("d"), + Separator: proto.String(";"), + Replacement: proto.String("ch$1-ch$1"), + Action: pb.RelabelConfig_REPLACE.Enum(), + }, + }, + output: nil, + }, + { + input: clientmodel.LabelSet{ + "a": "foo", + }, + relabel: []pb.RelabelConfig{ + { + SourceLabel: []string{"a"}, + Regex: proto.String("no-match"), + Action: pb.RelabelConfig_DROP.Enum(), + }, + }, + output: clientmodel.LabelSet{ + "a": "foo", + }, + }, + { + input: clientmodel.LabelSet{ + "a": "foo", + }, + relabel: []pb.RelabelConfig{ + { + SourceLabel: []string{"a"}, + Regex: proto.String("no-match"), + Action: pb.RelabelConfig_KEEP.Enum(), + }, + }, + output: nil, + }, + { + input: clientmodel.LabelSet{ + "a": "foo", + }, + relabel: []pb.RelabelConfig{ + { + SourceLabel: []string{"a"}, + Regex: proto.String("^f"), + Action: pb.RelabelConfig_KEEP.Enum(), + }, + }, + output: clientmodel.LabelSet{ + "a": "foo", + }, + }, + { + // No replacement must be applied if there is no match. + input: clientmodel.LabelSet{ + "a": "boo", + }, + relabel: []pb.RelabelConfig{ + { + SourceLabel: []string{"a"}, + Regex: proto.String("^f"), + Action: pb.RelabelConfig_REPLACE.Enum(), + TargetLabel: proto.String("b"), + Replacement: proto.String("bar"), + }, + }, + output: clientmodel.LabelSet{ + "a": "boo", + }, + }, + } + + for i, test := range tests { + var relabel []*config.RelabelConfig + for _, rl := range test.relabel { + proto.SetDefaults(&rl) + relabel = append(relabel, &config.RelabelConfig{rl}) + } + res, err := Relabel(test.input, relabel...) + if err != nil { + t.Errorf("Test %d: error relabeling: %s", i+1, err) + } + + if !reflect.DeepEqual(res, test.output) { + t.Errorf("Test %d: relabel output mismatch: expected %#v, got %#v", i+1, test.output, res) + } + } +} diff --git a/retrieval/target.go b/retrieval/target.go index d09dbeecb..cdd3acdca 100644 --- a/retrieval/target.go +++ b/retrieval/target.go @@ -172,10 +172,10 @@ type target struct { } // NewTarget creates a reasonably configured target for querying. -func NewTarget(address string, cfg *config.ScrapeConfig, baseLabels clientmodel.LabelSet) Target { +func NewTarget(cfg *config.ScrapeConfig, baseLabels clientmodel.LabelSet) Target { t := &target{ url: &url.URL{ - Host: address, + Host: string(baseLabels[clientmodel.AddressLabel]), }, scraperStopping: make(chan struct{}), scraperStopped: make(chan struct{}), @@ -197,16 +197,16 @@ func (t *target) Update(cfg *config.ScrapeConfig, baseLabels clientmodel.LabelSe t.deadline = cfg.ScrapeTimeout() t.httpClient = utility.NewDeadlineClient(cfg.ScrapeTimeout()) - t.baseLabels = clientmodel.LabelSet{ - clientmodel.InstanceLabel: clientmodel.LabelValue(t.InstanceIdentifier()), - } - + t.baseLabels = clientmodel.LabelSet{} // All remaining internal labels will not be part of the label set. for name, val := range baseLabels { if !strings.HasPrefix(string(name), clientmodel.ReservedLabelPrefix) { t.baseLabels[name] = val } } + if _, ok := t.baseLabels[clientmodel.InstanceLabel]; !ok { + t.baseLabels[clientmodel.InstanceLabel] = clientmodel.LabelValue(t.InstanceIdentifier()) + } } func (t *target) String() string { diff --git a/retrieval/targetmanager.go b/retrieval/targetmanager.go index 2427dc2cf..66d29bd23 100644 --- a/retrieval/targetmanager.go +++ b/retrieval/targetmanager.go @@ -81,19 +81,19 @@ func (tm *TargetManager) Run() { sources := map[string]struct{}{} for scfg, provs := range tm.providers { - for _, p := range provs { + for _, prov := range provs { ch := make(chan *config.TargetGroup) go tm.handleTargetUpdates(scfg, ch) - for _, src := range p.Sources() { + for _, src := range prov.Sources() { src = fullSource(scfg, src) sources[src] = struct{}{} } // Run the target provider after cleanup of the stale targets is done. - defer func(c chan *config.TargetGroup) { + defer func(p TargetProvider, c chan *config.TargetGroup) { go p.Run(c) - }(ch) + }(prov, ch) } } @@ -326,9 +326,17 @@ func (tm *TargetManager) targetsFromGroup(tg *config.TargetGroup, cfg *config.Sc } } - address, ok := labels[clientmodel.AddressLabel] - if !ok { - return nil, fmt.Errorf("Instance %d in target group %s has no address", i, tg) + if _, ok := labels[clientmodel.AddressLabel]; !ok { + return nil, fmt.Errorf("instance %d in target group %s has no address", i, tg) + } + + labels, err := Relabel(labels, cfg.RelabelConfigs()...) + if err != nil { + return nil, fmt.Errorf("error while relabelling instance %d in target group %s: %s", i, tg, err) + } + // Check if the target was dropped. + if labels == nil { + continue } for ln := range labels { @@ -338,8 +346,8 @@ func (tm *TargetManager) targetsFromGroup(tg *config.TargetGroup, cfg *config.Sc delete(labels, ln) } } - targets = append(targets, NewTarget(string(address), cfg, labels)) - + tr := NewTarget(cfg, labels) + targets = append(targets, tr) } return targets, nil diff --git a/retrieval/targetmanager_test.go b/retrieval/targetmanager_test.go index 1d8315ca2..6b539d1a7 100644 --- a/retrieval/targetmanager_test.go +++ b/retrieval/targetmanager_test.go @@ -164,10 +164,45 @@ func TestTargetManagerConfigUpdate(t *testing.T) { JobName: proto.String("test_job2"), ScrapeInterval: proto.String("1m"), TargetGroup: []*pb.TargetGroup{ - {Target: []string{"example.org:8080", "example.com:8081"}}, + { + Target: []string{"example.org:8080", "example.com:8081"}, + Labels: &pb.LabelPairs{Label: []*pb.LabelPair{ + {Name: proto.String("foo"), Value: proto.String("bar")}, + {Name: proto.String("boom"), Value: proto.String("box")}, + }}, + }, {Target: []string{"test.com:1234"}}, + { + Target: []string{"test.com:1235"}, + Labels: &pb.LabelPairs{Label: []*pb.LabelPair{ + {Name: proto.String("instance"), Value: proto.String("fixed")}, + }}, + }, + }, + RelabelConfig: []*pb.RelabelConfig{ + { + SourceLabel: []string{string(clientmodel.AddressLabel)}, + Regex: proto.String(`^test\.(.*?):(.*)`), + Replacement: proto.String("foo.${1}:${2}"), + TargetLabel: proto.String(string(clientmodel.AddressLabel)), + }, { + // Add a new label for example.* targets. + SourceLabel: []string{string(clientmodel.AddressLabel), "boom", "foo"}, + Regex: proto.String("^example.*?-b([a-z-]+)r$"), + TargetLabel: proto.String("new"), + Replacement: proto.String("$1"), + Separator: proto.String("-"), + }, { + // Drop an existing label. + SourceLabel: []string{"boom"}, + Regex: proto.String(".*"), + TargetLabel: proto.String("boom"), + Replacement: proto.String(""), + }, }, } + proto.SetDefaults(testJob1) + proto.SetDefaults(testJob2) sequence := []struct { scrapeConfigs []*pb.ScrapeConfig @@ -197,11 +232,14 @@ func TestTargetManagerConfigUpdate(t *testing.T) { {clientmodel.JobLabel: "test_job1", clientmodel.InstanceLabel: "example.com:80"}, }, "test_job2:static:0": { - {clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "example.org:8080"}, - {clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "example.com:8081"}, + {clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "example.org:8080", "foo": "bar", "new": "ox-ba"}, + {clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "example.com:8081", "foo": "bar", "new": "ox-ba"}, }, "test_job2:static:1": { - {clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "test.com:1234"}, + {clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "foo.com:1234"}, + }, + "test_job2:static:2": { + {clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "fixed"}, }, }, }, { @@ -211,11 +249,14 @@ func TestTargetManagerConfigUpdate(t *testing.T) { scrapeConfigs: []*pb.ScrapeConfig{testJob2}, expected: map[string][]clientmodel.LabelSet{ "test_job2:static:0": { - {clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "example.org:8080"}, - {clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "example.com:8081"}, + {clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "example.org:8080", "foo": "bar", "new": "ox-ba"}, + {clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "example.com:8081", "foo": "bar", "new": "ox-ba"}, }, "test_job2:static:1": { - {clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "test.com:1234"}, + {clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "foo.com:1234"}, + }, + "test_job2:static:2": { + {clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "fixed"}, }, }, },