From ec94df49d45baa8e680d81e732294eb04fff00de Mon Sep 17 00:00:00 2001 From: Shubheksha Jalan Date: Sat, 30 Dec 2017 01:31:34 +0530 Subject: [PATCH] Refactor SD configuration to remove `config` dependency (#3629) * refactor: move targetGroup struct and CheckOverflow() to their own package * refactor: move auth and security related structs to a utility package, fix import error in utility package * refactor: Azure SD, remove SD struct from config * refactor: DNS SD, remove SD struct from config into dns package * refactor: ec2 SD, move SD struct from config into the ec2 package * refactor: file SD, move SD struct from config to file discovery package * refactor: gce, move SD struct from config to gce discovery package * refactor: move HTTPClientConfig and URL into util/config, fix import error in httputil * refactor: consul, move SD struct from config into consul discovery package * refactor: marathon, move SD struct from config into marathon discovery package * refactor: triton, move SD struct from config to triton discovery package, fix test * refactor: zookeeper, move SD structs from config to zookeeper discovery package * refactor: openstack, remove SD struct from config, move into openstack discovery package * refactor: kubernetes, move SD struct from config into kubernetes discovery package * refactor: notifier, use targetgroup package instead of config * refactor: tests for file, marathon, triton SD - use targetgroup package instead of config.TargetGroup * refactor: retrieval, use targetgroup package instead of config.TargetGroup * refactor: storage, use config util package * refactor: discovery manager, use targetgroup package instead of config.TargetGroup * refactor: use HTTPClient and TLS config from configUtil instead of config * refactor: tests, use targetgroup package instead of config.TargetGroup * refactor: fix tagetgroup.Group pointers that were removed by mistake * refactor: openstack, kubernetes: drop prefixes * refactor: remove import aliases forced due to vscode bug * refactor: move main SD struct out of config into discovery/config * refactor: rename configUtil to config_util * refactor: rename yamlUtil to yaml_config * refactor: kubernetes, remove prefixes * refactor: move the TargetGroup package to discovery/ * refactor: fix order of imports --- cmd/promtool/main.go | 7 +- config/config.go | 854 +------------------------ config/config_test.go | 105 +-- discovery/azure/azure.go | 49 +- discovery/config/config.go | 73 +++ discovery/consul/consul.go | 57 +- discovery/consul/consul_test.go | 6 +- discovery/dns/dns.go | 58 +- discovery/ec2/ec2.go | 61 +- discovery/file/file.go | 57 +- discovery/file/file_test.go | 6 +- discovery/gce/gce.go | 65 +- discovery/kubernetes/endpoints.go | 19 +- discovery/kubernetes/endpoints_test.go | 12 +- discovery/kubernetes/ingress.go | 16 +- discovery/kubernetes/ingress_test.go | 6 +- discovery/kubernetes/kubernetes.go | 103 ++- discovery/kubernetes/node.go | 16 +- discovery/kubernetes/node_test.go | 29 +- discovery/kubernetes/pod.go | 16 +- discovery/kubernetes/pod_test.go | 18 +- discovery/kubernetes/service.go | 16 +- discovery/kubernetes/service_test.go | 18 +- discovery/manager.go | 42 +- discovery/manager_test.go | 77 +-- discovery/marathon/marathon.go | 67 +- discovery/marathon/marathon_test.go | 35 +- discovery/openstack/hypervisor.go | 13 +- discovery/openstack/hypervisor_test.go | 3 +- discovery/openstack/instance.go | 12 +- discovery/openstack/instance_test.go | 3 +- discovery/openstack/openstack.go | 81 ++- discovery/targetgroup/targetgroup.go | 91 +++ discovery/triton/triton.go | 60 +- discovery/triton/triton_test.go | 9 +- discovery/zookeeper/zookeeper.go | 99 ++- notifier/notifier.go | 5 +- notifier/notifier_test.go | 10 +- retrieval/helpers_test.go | 6 +- retrieval/manager.go | 5 +- retrieval/scrape.go | 3 +- retrieval/target.go | 3 +- retrieval/target_test.go | 30 +- storage/remote/client.go | 8 +- storage/remote/client_test.go | 4 +- util/config/config.go | 138 ++++ util/httputil/client.go | 8 +- util/httputil/client_test.go | 48 +- util/yaml/yaml.go | 31 + 49 files changed, 1347 insertions(+), 1211 deletions(-) create mode 100644 discovery/config/config.go create mode 100644 discovery/targetgroup/targetgroup.go create mode 100644 util/config/config.go create mode 100644 util/yaml/yaml.go diff --git a/cmd/promtool/main.go b/cmd/promtool/main.go index 0b72470cd..86fa69075 100644 --- a/cmd/promtool/main.go +++ b/cmd/promtool/main.go @@ -20,14 +20,15 @@ import ( "path/filepath" "strings" - kingpin "gopkg.in/alecthomas/kingpin.v2" - yaml "gopkg.in/yaml.v2" + "gopkg.in/alecthomas/kingpin.v2" + "gopkg.in/yaml.v2" "github.com/prometheus/common/model" "github.com/prometheus/common/version" "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/pkg/rulefmt" "github.com/prometheus/prometheus/promql" + config_util "github.com/prometheus/prometheus/util/config" "github.com/prometheus/prometheus/util/promlint" ) @@ -172,7 +173,7 @@ func checkConfig(filename string) ([]string, error) { return ruleFiles, nil } -func checkTLSConfig(tlsConfig config.TLSConfig) error { +func checkTLSConfig(tlsConfig config_util.TLSConfig) error { if err := checkFileExists(tlsConfig.CertFile); err != nil { return fmt.Errorf("error checking client cert file %q: %s", tlsConfig.CertFile, err) } diff --git a/config/config.go b/config/config.go index 62feb595e..cab3d5b74 100644 --- a/config/config.go +++ b/config/config.go @@ -14,7 +14,6 @@ package config import ( - "encoding/json" "fmt" "io/ioutil" "net/url" @@ -23,14 +22,14 @@ import ( "strings" "time" - "github.com/aws/aws-sdk-go/aws/ec2metadata" - "github.com/aws/aws-sdk-go/aws/session" "github.com/prometheus/common/model" + sd_config "github.com/prometheus/prometheus/discovery/config" + config_util "github.com/prometheus/prometheus/util/config" + yaml_util "github.com/prometheus/prometheus/util/yaml" "gopkg.in/yaml.v2" ) var ( - patFileSDName = regexp.MustCompile(`^[^*]*(\*[^/]*)?\.(json|yml|yaml|JSON|YML|YAML)$`) patRulePath = regexp.MustCompile(`^[^*]*(\*[^/]*)?$`) relabelTarget = regexp.MustCompile(`^(?:(?:[a-zA-Z_]|\$(?:\{\w+\}|\w+))+\w*)+$`) ) @@ -102,74 +101,6 @@ var ( Replacement: "$1", } - // DefaultDNSSDConfig is the default DNS SD configuration. - DefaultDNSSDConfig = DNSSDConfig{ - RefreshInterval: model.Duration(30 * time.Second), - Type: "SRV", - } - - // DefaultFileSDConfig is the default file SD configuration. - DefaultFileSDConfig = FileSDConfig{ - RefreshInterval: model.Duration(5 * time.Minute), - } - - // DefaultConsulSDConfig is the default Consul SD configuration. - DefaultConsulSDConfig = ConsulSDConfig{ - TagSeparator: ",", - Scheme: "http", - } - - // DefaultServersetSDConfig is the default Serverset SD configuration. - DefaultServersetSDConfig = ServersetSDConfig{ - Timeout: model.Duration(10 * time.Second), - } - - // DefaultNerveSDConfig is the default Nerve SD configuration. - DefaultNerveSDConfig = NerveSDConfig{ - Timeout: model.Duration(10 * time.Second), - } - - // DefaultMarathonSDConfig is the default Marathon SD configuration. - DefaultMarathonSDConfig = MarathonSDConfig{ - Timeout: model.Duration(30 * time.Second), - RefreshInterval: model.Duration(30 * time.Second), - } - - // DefaultKubernetesSDConfig is the default Kubernetes SD configuration - DefaultKubernetesSDConfig = KubernetesSDConfig{} - - // DefaultGCESDConfig is the default EC2 SD configuration. - DefaultGCESDConfig = GCESDConfig{ - Port: 80, - TagSeparator: ",", - RefreshInterval: model.Duration(60 * time.Second), - } - - // DefaultEC2SDConfig is the default EC2 SD configuration. - DefaultEC2SDConfig = EC2SDConfig{ - Port: 80, - RefreshInterval: model.Duration(60 * time.Second), - } - - // DefaultOpenstackSDConfig is the default OpenStack SD configuration. - DefaultOpenstackSDConfig = OpenstackSDConfig{ - Port: 80, - RefreshInterval: model.Duration(60 * time.Second), - } - - // DefaultAzureSDConfig is the default Azure SD configuration. - DefaultAzureSDConfig = AzureSDConfig{ - Port: 80, - RefreshInterval: model.Duration(5 * time.Minute), - } - - // DefaultTritonSDConfig is the default Triton SD configuration. - DefaultTritonSDConfig = TritonSDConfig{ - Port: 9163, - RefreshInterval: model.Duration(60 * time.Second), - Version: 1, - } - // DefaultRemoteWriteConfig is the default remote write configuration. DefaultRemoteWriteConfig = RemoteWriteConfig{ RemoteTimeout: model.Duration(30 * time.Second), @@ -200,34 +131,6 @@ var ( } ) -// URL is a custom URL type that allows validation at configuration load time. -type URL struct { - *url.URL -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface for URLs. -func (u *URL) UnmarshalYAML(unmarshal func(interface{}) error) error { - var s string - if err := unmarshal(&s); err != nil { - return err - } - - urlp, err := url.Parse(s) - if err != nil { - return err - } - u.URL = urlp - return nil -} - -// MarshalYAML implements the yaml.Marshaler interface for URLs. -func (u URL) MarshalYAML() (interface{}, error) { - if u.URL != nil { - return u.String(), nil - } - return nil, nil -} - // Config is the top-level configuration for Prometheus's config files. type Config struct { GlobalConfig GlobalConfig `yaml:"global"` @@ -245,23 +148,6 @@ type Config struct { original string } -// Secret special type for storing secrets. -type Secret string - -// UnmarshalYAML implements the yaml.Unmarshaler interface for Secrets. -func (s *Secret) UnmarshalYAML(unmarshal func(interface{}) error) error { - type plain Secret - return unmarshal((*plain)(s)) -} - -// MarshalYAML implements the yaml.Marshaler interface for Secrets. -func (s Secret) MarshalYAML() (interface{}, error) { - if s != "" { - return "", nil - } - return nil, nil -} - // resolveFilepaths joins all relative paths in a configuration // with a given base directory. func resolveFilepaths(baseDir string, cfg *Config) { @@ -276,13 +162,13 @@ func resolveFilepaths(baseDir string, cfg *Config) { cfg.RuleFiles[i] = join(rf) } - clientPaths := func(scfg *HTTPClientConfig) { + clientPaths := func(scfg *config_util.HTTPClientConfig) { scfg.BearerTokenFile = join(scfg.BearerTokenFile) scfg.TLSConfig.CAFile = join(scfg.TLSConfig.CAFile) scfg.TLSConfig.CertFile = join(scfg.TLSConfig.CertFile) scfg.TLSConfig.KeyFile = join(scfg.TLSConfig.KeyFile) } - sdPaths := func(cfg *ServiceDiscoveryConfig) { + sdPaths := func(cfg *sd_config.ServiceDiscoveryConfig) { for _, kcfg := range cfg.KubernetesSDConfigs { kcfg.BearerTokenFile = join(kcfg.BearerTokenFile) kcfg.TLSConfig.CAFile = join(kcfg.TLSConfig.CAFile) @@ -317,17 +203,6 @@ func resolveFilepaths(baseDir string, cfg *Config) { } } -func checkOverflow(m map[string]interface{}, ctx string) error { - if len(m) > 0 { - var keys []string - for k := range m { - keys = append(keys, k) - } - return fmt.Errorf("unknown fields in %s: %s", ctx, strings.Join(keys, ", ")) - } - return nil -} - func (c Config) String() string { b, err := yaml.Marshal(c) if err != nil { @@ -346,7 +221,7 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { if err := unmarshal((*plain)(c)); err != nil { return err } - if err := checkOverflow(c.XXX, "config"); err != nil { + if err := yaml_util.CheckOverflow(c.XXX, "config"); err != nil { return err } // If a global block was open but empty the default global config is overwritten. @@ -412,7 +287,7 @@ func (c *GlobalConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { if err := unmarshal((*plain)(gc)); err != nil { return err } - if err := checkOverflow(gc.XXX, "global config"); err != nil { + if err := yaml_util.CheckOverflow(gc.XXX, "global config"); err != nil { return err } // First set the correct scrape interval, then check that the timeout @@ -445,101 +320,6 @@ func (c *GlobalConfig) isZero() bool { c.EvaluationInterval == 0 } -// TLSConfig configures the options for TLS connections. -type TLSConfig struct { - // The CA cert to use for the targets. - CAFile string `yaml:"ca_file,omitempty"` - // The client cert file for the targets. - CertFile string `yaml:"cert_file,omitempty"` - // The client key file for the targets. - KeyFile string `yaml:"key_file,omitempty"` - // Used to verify the hostname for the targets. - ServerName string `yaml:"server_name,omitempty"` - // Disable target certificate validation. - InsecureSkipVerify bool `yaml:"insecure_skip_verify"` - - // Catches all undefined fields and must be empty after parsing. - XXX map[string]interface{} `yaml:",inline"` -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *TLSConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { - type plain TLSConfig - if err := unmarshal((*plain)(c)); err != nil { - return err - } - return checkOverflow(c.XXX, "TLS config") -} - -// ServiceDiscoveryConfig configures lists of different service discovery mechanisms. -type ServiceDiscoveryConfig struct { - // List of labeled target groups for this job. - StaticConfigs []*TargetGroup `yaml:"static_configs,omitempty"` - // List of DNS service discovery configurations. - DNSSDConfigs []*DNSSDConfig `yaml:"dns_sd_configs,omitempty"` - // List of file service discovery configurations. - FileSDConfigs []*FileSDConfig `yaml:"file_sd_configs,omitempty"` - // List of Consul service discovery configurations. - ConsulSDConfigs []*ConsulSDConfig `yaml:"consul_sd_configs,omitempty"` - // List of Serverset service discovery configurations. - ServersetSDConfigs []*ServersetSDConfig `yaml:"serverset_sd_configs,omitempty"` - // NerveSDConfigs is a list of Nerve service discovery configurations. - NerveSDConfigs []*NerveSDConfig `yaml:"nerve_sd_configs,omitempty"` - // MarathonSDConfigs is a list of Marathon service discovery configurations. - MarathonSDConfigs []*MarathonSDConfig `yaml:"marathon_sd_configs,omitempty"` - // List of Kubernetes service discovery configurations. - KubernetesSDConfigs []*KubernetesSDConfig `yaml:"kubernetes_sd_configs,omitempty"` - // List of GCE service discovery configurations. - GCESDConfigs []*GCESDConfig `yaml:"gce_sd_configs,omitempty"` - // List of EC2 service discovery configurations. - EC2SDConfigs []*EC2SDConfig `yaml:"ec2_sd_configs,omitempty"` - // List of OpenStack service discovery configurations. - OpenstackSDConfigs []*OpenstackSDConfig `yaml:"openstack_sd_configs,omitempty"` - // List of Azure service discovery configurations. - AzureSDConfigs []*AzureSDConfig `yaml:"azure_sd_configs,omitempty"` - // List of Triton service discovery configurations. - TritonSDConfigs []*TritonSDConfig `yaml:"triton_sd_configs,omitempty"` - - // Catches all undefined fields and must be empty after parsing. - XXX map[string]interface{} `yaml:",inline"` -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *ServiceDiscoveryConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { - type plain ServiceDiscoveryConfig - if err := unmarshal((*plain)(c)); err != nil { - return err - } - return checkOverflow(c.XXX, "service discovery config") -} - -// HTTPClientConfig configures an HTTP client. -type HTTPClientConfig struct { - // The HTTP basic authentication credentials for the targets. - BasicAuth *BasicAuth `yaml:"basic_auth,omitempty"` - // The bearer token for the targets. - BearerToken Secret `yaml:"bearer_token,omitempty"` - // The bearer token file for the targets. - BearerTokenFile string `yaml:"bearer_token_file,omitempty"` - // HTTP proxy server to use to connect to the targets. - ProxyURL URL `yaml:"proxy_url,omitempty"` - // TLSConfig to use to connect to the targets. - TLSConfig TLSConfig `yaml:"tls_config,omitempty"` - - // Catches all undefined fields and must be empty after parsing. - XXX map[string]interface{} `yaml:",inline"` -} - -func (c *HTTPClientConfig) validate() error { - if len(c.BearerToken) > 0 && len(c.BearerTokenFile) > 0 { - return fmt.Errorf("at most one of bearer_token & bearer_token_file must be configured") - } - if c.BasicAuth != nil && (len(c.BearerToken) > 0 || len(c.BearerTokenFile) > 0) { - return fmt.Errorf("at most one of basic_auth, bearer_token & bearer_token_file must be configured") - } - return nil -} - // ScrapeConfig configures a scraping unit for Prometheus. type ScrapeConfig struct { // The job name to which the job label is set by default. @@ -562,8 +342,8 @@ type ScrapeConfig struct { // We cannot do proper Go type embedding below as the parser will then parse // values arbitrarily into the overflow maps of further-down types. - ServiceDiscoveryConfig ServiceDiscoveryConfig `yaml:",inline"` - HTTPClientConfig HTTPClientConfig `yaml:",inline"` + ServiceDiscoveryConfig sd_config.ServiceDiscoveryConfig `yaml:",inline"` + HTTPClientConfig config_util.HTTPClientConfig `yaml:",inline"` // List of target relabel configurations. RelabelConfigs []*RelabelConfig `yaml:"relabel_configs,omitempty"` @@ -582,7 +362,7 @@ func (c *ScrapeConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { if err != nil { return err } - if err = checkOverflow(c.XXX, "scrape_config"); err != nil { + if err = yaml_util.CheckOverflow(c.XXX, "scrape_config"); err != nil { return err } if len(c.JobName) == 0 { @@ -592,7 +372,7 @@ func (c *ScrapeConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { // The UnmarshalYAML method of HTTPClientConfig is not being called because it's not a pointer. // We cannot make it a pointer as the parser panics for inlined pointer structs. // Thus we just do its validation here. - if err = c.HTTPClientConfig.validate(); err != nil { + if err = c.HTTPClientConfig.Validate(); err != nil { return err } @@ -627,7 +407,7 @@ func (c *AlertingConfig) UnmarshalYAML(unmarshal func(interface{}) error) error if err := unmarshal((*plain)(c)); err != nil { return err } - return checkOverflow(c.XXX, "alerting config") + return yaml_util.CheckOverflow(c.XXX, "alerting config") } // AlertmanagerConfig configures how Alertmanagers can be discovered and communicated with. @@ -635,8 +415,8 @@ type AlertmanagerConfig struct { // We cannot do proper Go type embedding below as the parser will then parse // values arbitrarily into the overflow maps of further-down types. - ServiceDiscoveryConfig ServiceDiscoveryConfig `yaml:",inline"` - HTTPClientConfig HTTPClientConfig `yaml:",inline"` + ServiceDiscoveryConfig sd_config.ServiceDiscoveryConfig `yaml:",inline"` + HTTPClientConfig config_util.HTTPClientConfig `yaml:",inline"` // The URL scheme to use when talking to Alertmanagers. Scheme string `yaml:"scheme,omitempty"` @@ -659,14 +439,14 @@ func (c *AlertmanagerConfig) UnmarshalYAML(unmarshal func(interface{}) error) er if err := unmarshal((*plain)(c)); err != nil { return err } - if err := checkOverflow(c.XXX, "alertmanager config"); err != nil { + if err := yaml_util.CheckOverflow(c.XXX, "alertmanager config"); err != nil { return err } // The UnmarshalYAML method of HTTPClientConfig is not being called because it's not a pointer. // We cannot make it a pointer as the parser panics for inlined pointer structs. // Thus we just do its validation here. - if err := c.HTTPClientConfig.validate(); err != nil { + if err := c.HTTPClientConfig.Validate(); err != nil { return err } @@ -692,140 +472,15 @@ func CheckTargetAddress(address model.LabelValue) error { return nil } -// BasicAuth contains basic HTTP authentication credentials. -type BasicAuth struct { - Username string `yaml:"username"` - Password Secret `yaml:"password"` - - // Catches all undefined fields and must be empty after parsing. - XXX map[string]interface{} `yaml:",inline"` -} - // ClientCert contains client cert credentials. type ClientCert struct { - Cert string `yaml:"cert"` - Key Secret `yaml:"key"` + Cert string `yaml:"cert"` + Key config_util.Secret `yaml:"key"` // Catches all undefined fields and must be empty after parsing. XXX map[string]interface{} `yaml:",inline"` } -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (a *BasicAuth) UnmarshalYAML(unmarshal func(interface{}) error) error { - type plain BasicAuth - err := unmarshal((*plain)(a)) - if err != nil { - return err - } - return checkOverflow(a.XXX, "basic_auth") -} - -// TargetGroup is a set of targets with a common label set(production , test, staging etc.). -type TargetGroup struct { - // Targets is a list of targets identified by a label set. Each target is - // uniquely identifiable in the group by its address label. - Targets []model.LabelSet - // Labels is a set of labels that is common across all targets in the group. - Labels model.LabelSet - - // Source is an identifier that describes a group of targets. - Source string -} - -func (tg TargetGroup) String() string { - return tg.Source -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (tg *TargetGroup) UnmarshalYAML(unmarshal func(interface{}) error) error { - g := struct { - Targets []string `yaml:"targets"` - Labels model.LabelSet `yaml:"labels"` - XXX map[string]interface{} `yaml:",inline"` - }{} - if err := unmarshal(&g); err != nil { - return err - } - tg.Targets = make([]model.LabelSet, 0, len(g.Targets)) - for _, t := range g.Targets { - tg.Targets = append(tg.Targets, model.LabelSet{ - model.AddressLabel: model.LabelValue(t), - }) - } - tg.Labels = g.Labels - return checkOverflow(g.XXX, "static_config") -} - -// MarshalYAML implements the yaml.Marshaler interface. -func (tg TargetGroup) MarshalYAML() (interface{}, error) { - g := &struct { - Targets []string `yaml:"targets"` - Labels model.LabelSet `yaml:"labels,omitempty"` - }{ - Targets: make([]string, 0, len(tg.Targets)), - Labels: tg.Labels, - } - for _, t := range tg.Targets { - g.Targets = append(g.Targets, string(t[model.AddressLabel])) - } - return g, nil -} - -// UnmarshalJSON implements the json.Unmarshaler interface. -func (tg *TargetGroup) UnmarshalJSON(b []byte) error { - g := struct { - Targets []string `json:"targets"` - Labels model.LabelSet `json:"labels"` - }{} - if err := json.Unmarshal(b, &g); err != nil { - return err - } - tg.Targets = make([]model.LabelSet, 0, len(g.Targets)) - for _, t := range g.Targets { - tg.Targets = append(tg.Targets, model.LabelSet{ - model.AddressLabel: model.LabelValue(t), - }) - } - tg.Labels = g.Labels - return nil -} - -// DNSSDConfig is the configuration for DNS based service discovery. -type DNSSDConfig struct { - Names []string `yaml:"names"` - RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` - Type string `yaml:"type"` - Port int `yaml:"port"` // Ignored for SRV records - // Catches all undefined fields and must be empty after parsing. - XXX map[string]interface{} `yaml:",inline"` -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *DNSSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { - *c = DefaultDNSSDConfig - type plain DNSSDConfig - err := unmarshal((*plain)(c)) - if err != nil { - return err - } - if err := checkOverflow(c.XXX, "dns_sd_config"); err != nil { - return err - } - if len(c.Names) == 0 { - return fmt.Errorf("DNS-SD config must contain at least one SRV record name") - } - switch strings.ToUpper(c.Type) { - case "SRV": - case "A", "AAAA": - if c.Port == 0 { - return fmt.Errorf("a port is required in DNS-SD configs for all record types except SRV") - } - default: - return fmt.Errorf("invalid DNS-SD records type %s", c.Type) - } - return nil -} - // FileSDConfig is the configuration for file based discovery. type FileSDConfig struct { Files []string `yaml:"files"` @@ -835,455 +490,6 @@ type FileSDConfig struct { XXX map[string]interface{} `yaml:",inline"` } -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *FileSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { - *c = DefaultFileSDConfig - type plain FileSDConfig - err := unmarshal((*plain)(c)) - if err != nil { - return err - } - if err := checkOverflow(c.XXX, "file_sd_config"); err != nil { - return err - } - if len(c.Files) == 0 { - return fmt.Errorf("file service discovery config must contain at least one path name") - } - for _, name := range c.Files { - if !patFileSDName.MatchString(name) { - return fmt.Errorf("path name %q is not valid for file discovery", name) - } - } - return nil -} - -// ConsulSDConfig is the configuration for Consul service discovery. -type ConsulSDConfig struct { - Server string `yaml:"server"` - Token Secret `yaml:"token,omitempty"` - Datacenter string `yaml:"datacenter,omitempty"` - TagSeparator string `yaml:"tag_separator,omitempty"` - Scheme string `yaml:"scheme,omitempty"` - Username string `yaml:"username,omitempty"` - Password Secret `yaml:"password,omitempty"` - // The list of services for which targets are discovered. - // Defaults to all services if empty. - Services []string `yaml:"services"` - - TLSConfig TLSConfig `yaml:"tls_config,omitempty"` - // Catches all undefined fields and must be empty after parsing. - XXX map[string]interface{} `yaml:",inline"` -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *ConsulSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { - *c = DefaultConsulSDConfig - type plain ConsulSDConfig - err := unmarshal((*plain)(c)) - if err != nil { - return err - } - if err := checkOverflow(c.XXX, "consul_sd_config"); err != nil { - return err - } - if strings.TrimSpace(c.Server) == "" { - return fmt.Errorf("Consul SD configuration requires a server address") - } - return nil -} - -// ServersetSDConfig is the configuration for Twitter serversets in Zookeeper based discovery. -type ServersetSDConfig struct { - Servers []string `yaml:"servers"` - Paths []string `yaml:"paths"` - Timeout model.Duration `yaml:"timeout,omitempty"` - - // Catches all undefined fields and must be empty after parsing. - XXX map[string]interface{} `yaml:",inline"` -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *ServersetSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { - *c = DefaultServersetSDConfig - type plain ServersetSDConfig - err := unmarshal((*plain)(c)) - if err != nil { - return err - } - if err := checkOverflow(c.XXX, "serverset_sd_config"); err != nil { - return err - } - if len(c.Servers) == 0 { - return fmt.Errorf("serverset SD config must contain at least one Zookeeper server") - } - if len(c.Paths) == 0 { - return fmt.Errorf("serverset SD config must contain at least one path") - } - for _, path := range c.Paths { - if !strings.HasPrefix(path, "/") { - return fmt.Errorf("serverset SD config paths must begin with '/': %s", path) - } - } - return nil -} - -// NerveSDConfig is the configuration for AirBnB's Nerve in Zookeeper based discovery. -type NerveSDConfig struct { - Servers []string `yaml:"servers"` - Paths []string `yaml:"paths"` - Timeout model.Duration `yaml:"timeout,omitempty"` - - // Catches all undefined fields and must be empty after parsing. - XXX map[string]interface{} `yaml:",inline"` -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *NerveSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { - *c = DefaultNerveSDConfig - type plain NerveSDConfig - err := unmarshal((*plain)(c)) - if err != nil { - return err - } - if err := checkOverflow(c.XXX, "nerve_sd_config"); err != nil { - return err - } - if len(c.Servers) == 0 { - return fmt.Errorf("nerve SD config must contain at least one Zookeeper server") - } - if len(c.Paths) == 0 { - return fmt.Errorf("nerve SD config must contain at least one path") - } - for _, path := range c.Paths { - if !strings.HasPrefix(path, "/") { - return fmt.Errorf("nerve SD config paths must begin with '/': %s", path) - } - } - return nil -} - -// MarathonSDConfig is the configuration for services running on Marathon. -type MarathonSDConfig struct { - Servers []string `yaml:"servers,omitempty"` - Timeout model.Duration `yaml:"timeout,omitempty"` - RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` - TLSConfig TLSConfig `yaml:"tls_config,omitempty"` - BearerToken Secret `yaml:"bearer_token,omitempty"` - BearerTokenFile string `yaml:"bearer_token_file,omitempty"` - - // Catches all undefined fields and must be empty after parsing. - XXX map[string]interface{} `yaml:",inline"` -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *MarathonSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { - *c = DefaultMarathonSDConfig - type plain MarathonSDConfig - err := unmarshal((*plain)(c)) - if err != nil { - return err - } - if err := checkOverflow(c.XXX, "marathon_sd_config"); err != nil { - return err - } - if len(c.Servers) == 0 { - return fmt.Errorf("Marathon SD config must contain at least one Marathon server") - } - if len(c.BearerToken) > 0 && len(c.BearerTokenFile) > 0 { - return fmt.Errorf("at most one of bearer_token & bearer_token_file must be configured") - } - - return nil -} - -// KubernetesRole is role of the service in Kubernetes. -type KubernetesRole string - -// The valid options for KubernetesRole. -const ( - KubernetesRoleNode KubernetesRole = "node" - KubernetesRolePod KubernetesRole = "pod" - KubernetesRoleService KubernetesRole = "service" - KubernetesRoleEndpoint KubernetesRole = "endpoints" - KubernetesRoleIngress KubernetesRole = "ingress" -) - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *KubernetesRole) UnmarshalYAML(unmarshal func(interface{}) error) error { - if err := unmarshal((*string)(c)); err != nil { - return err - } - switch *c { - case KubernetesRoleNode, KubernetesRolePod, KubernetesRoleService, KubernetesRoleEndpoint, KubernetesRoleIngress: - return nil - default: - return fmt.Errorf("Unknown Kubernetes SD role %q", *c) - } -} - -// KubernetesSDConfig is the configuration for Kubernetes service discovery. -type KubernetesSDConfig struct { - APIServer URL `yaml:"api_server"` - Role KubernetesRole `yaml:"role"` - BasicAuth *BasicAuth `yaml:"basic_auth,omitempty"` - BearerToken Secret `yaml:"bearer_token,omitempty"` - BearerTokenFile string `yaml:"bearer_token_file,omitempty"` - TLSConfig TLSConfig `yaml:"tls_config,omitempty"` - NamespaceDiscovery KubernetesNamespaceDiscovery `yaml:"namespaces"` - - // Catches all undefined fields and must be empty after parsing. - XXX map[string]interface{} `yaml:",inline"` -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *KubernetesSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { - *c = KubernetesSDConfig{} - type plain KubernetesSDConfig - err := unmarshal((*plain)(c)) - if err != nil { - return err - } - if err := checkOverflow(c.XXX, "kubernetes_sd_config"); err != nil { - return err - } - if c.Role == "" { - return fmt.Errorf("role missing (one of: pod, service, endpoints, node)") - } - if len(c.BearerToken) > 0 && len(c.BearerTokenFile) > 0 { - return fmt.Errorf("at most one of bearer_token & bearer_token_file must be configured") - } - if c.BasicAuth != nil && (len(c.BearerToken) > 0 || len(c.BearerTokenFile) > 0) { - return fmt.Errorf("at most one of basic_auth, bearer_token & bearer_token_file must be configured") - } - if c.APIServer.URL == nil && - (c.BasicAuth != nil || c.BearerToken != "" || c.BearerTokenFile != "" || - c.TLSConfig.CAFile != "" || c.TLSConfig.CertFile != "" || c.TLSConfig.KeyFile != "") { - return fmt.Errorf("to use custom authentication please provide the 'api_server' URL explicitly") - } - return nil -} - -// KubernetesNamespaceDiscovery is the configuration for discovering -// Kubernetes namespaces. -type KubernetesNamespaceDiscovery struct { - Names []string `yaml:"names"` - // Catches all undefined fields and must be empty after parsing. - XXX map[string]interface{} `yaml:",inline"` -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *KubernetesNamespaceDiscovery) UnmarshalYAML(unmarshal func(interface{}) error) error { - *c = KubernetesNamespaceDiscovery{} - type plain KubernetesNamespaceDiscovery - err := unmarshal((*plain)(c)) - if err != nil { - return err - } - return checkOverflow(c.XXX, "namespaces") -} - -// GCESDConfig is the configuration for GCE based service discovery. -type GCESDConfig struct { - // Project: The Google Cloud Project ID - Project string `yaml:"project"` - - // Zone: The zone of the scrape targets. - // If you need to configure multiple zones use multiple gce_sd_configs - Zone string `yaml:"zone"` - - // Filter: Can be used optionally to filter the instance list by other criteria. - // Syntax of this filter string is described here in the filter query parameter section: - // https://cloud.google.com/compute/docs/reference/latest/instances/list - Filter string `yaml:"filter,omitempty"` - - RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` - Port int `yaml:"port"` - TagSeparator string `yaml:"tag_separator,omitempty"` - - // Catches all undefined fields and must be empty after parsing. - XXX map[string]interface{} `yaml:",inline"` -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *GCESDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { - *c = DefaultGCESDConfig - type plain GCESDConfig - err := unmarshal((*plain)(c)) - if err != nil { - return err - } - if err := checkOverflow(c.XXX, "gce_sd_config"); err != nil { - return err - } - if c.Project == "" { - return fmt.Errorf("GCE SD configuration requires a project") - } - if c.Zone == "" { - return fmt.Errorf("GCE SD configuration requires a zone") - } - return nil -} - -// EC2SDConfig is the configuration for EC2 based service discovery. -type EC2SDConfig struct { - Region string `yaml:"region"` - AccessKey string `yaml:"access_key,omitempty"` - SecretKey Secret `yaml:"secret_key,omitempty"` - Profile string `yaml:"profile,omitempty"` - RoleARN string `yaml:"role_arn,omitempty"` - RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` - Port int `yaml:"port"` - - // Catches all undefined fields and must be empty after parsing. - XXX map[string]interface{} `yaml:",inline"` -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *EC2SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { - *c = DefaultEC2SDConfig - type plain EC2SDConfig - err := unmarshal((*plain)(c)) - if err != nil { - return err - } - if err := checkOverflow(c.XXX, "ec2_sd_config"); err != nil { - return err - } - if c.Region == "" { - sess, err := session.NewSession() - if err != nil { - return err - } - metadata := ec2metadata.New(sess) - region, err := metadata.Region() - if err != nil { - return fmt.Errorf("EC2 SD configuration requires a region") - } - c.Region = region - } - return nil -} - -// OpenstackSDConfig is the configuration for OpenStack based service discovery. -type OpenstackSDConfig struct { - IdentityEndpoint string `yaml:"identity_endpoint"` - Username string `yaml:"username"` - UserID string `yaml:"userid"` - Password Secret `yaml:"password"` - ProjectName string `yaml:"project_name"` - ProjectID string `yaml:"project_id"` - DomainName string `yaml:"domain_name"` - DomainID string `yaml:"domain_id"` - Role OpenStackRole `yaml:"role"` - Region string `yaml:"region"` - RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` - Port int `yaml:"port"` - - // Catches all undefined fields and must be empty after parsing. - XXX map[string]interface{} `yaml:",inline"` -} - -// OpenStackRole is role of the target in OpenStack. -type OpenStackRole string - -// The valid options for OpenStackRole. -const ( - // OpenStack document reference - // https://docs.openstack.org/nova/pike/admin/arch.html#hypervisors - OpenStackRoleHypervisor OpenStackRole = "hypervisor" - // OpenStack document reference - // https://docs.openstack.org/horizon/pike/user/launch-instances.html - OpenStackRoleInstance OpenStackRole = "instance" -) - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *OpenStackRole) UnmarshalYAML(unmarshal func(interface{}) error) error { - if err := unmarshal((*string)(c)); err != nil { - return err - } - switch *c { - case OpenStackRoleHypervisor, OpenStackRoleInstance: - return nil - default: - return fmt.Errorf("Unknown OpenStack SD role %q", *c) - } -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *OpenstackSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { - *c = DefaultOpenstackSDConfig - type plain OpenstackSDConfig - err := unmarshal((*plain)(c)) - if err != nil { - return err - } - if c.Role == "" { - return fmt.Errorf("role missing (one of: instance, hypervisor)") - } - return checkOverflow(c.XXX, "openstack_sd_config") -} - -// AzureSDConfig is the configuration for Azure based service discovery. -type AzureSDConfig struct { - Port int `yaml:"port"` - SubscriptionID string `yaml:"subscription_id"` - TenantID string `yaml:"tenant_id,omitempty"` - ClientID string `yaml:"client_id,omitempty"` - ClientSecret Secret `yaml:"client_secret,omitempty"` - RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` - - // Catches all undefined fields and must be empty after parsing. - XXX map[string]interface{} `yaml:",inline"` -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *AzureSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { - *c = DefaultAzureSDConfig - type plain AzureSDConfig - err := unmarshal((*plain)(c)) - if err != nil { - return err - } - - return checkOverflow(c.XXX, "azure_sd_config") -} - -// TritonSDConfig is the configuration for Triton based service discovery. -type TritonSDConfig struct { - Account string `yaml:"account"` - DNSSuffix string `yaml:"dns_suffix"` - Endpoint string `yaml:"endpoint"` - Port int `yaml:"port"` - RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` - TLSConfig TLSConfig `yaml:"tls_config,omitempty"` - Version int `yaml:"version"` - // Catches all undefined fields and must be empty after parsing. - XXX map[string]interface{} `yaml:",inline"` -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *TritonSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { - *c = DefaultTritonSDConfig - type plain TritonSDConfig - err := unmarshal((*plain)(c)) - if err != nil { - return err - } - if c.Account == "" { - return fmt.Errorf("Triton SD configuration requires an account") - } - if c.DNSSuffix == "" { - return fmt.Errorf("Triton SD configuration requires a dns_suffix") - } - if c.Endpoint == "" { - return fmt.Errorf("Triton SD configuration requires an endpoint") - } - if c.RefreshInterval <= 0 { - return fmt.Errorf("Triton SD configuration requires RefreshInterval to be a positive integer") - } - return checkOverflow(c.XXX, "triton_sd_config") -} - // RelabelAction is the action to be performed on relabeling. type RelabelAction string @@ -1348,7 +554,7 @@ func (c *RelabelConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { if err := unmarshal((*plain)(c)); err != nil { return err } - if err := checkOverflow(c.XXX, "relabel_config"); err != nil { + if err := yaml_util.CheckOverflow(c.XXX, "relabel_config"); err != nil { return err } if c.Regex.Regexp == nil { @@ -1429,14 +635,14 @@ func (re Regexp) MarshalYAML() (interface{}, error) { // RemoteWriteConfig is the configuration for writing to remote storage. type RemoteWriteConfig struct { - URL *URL `yaml:"url"` + URL *config_util.URL `yaml:"url"` RemoteTimeout model.Duration `yaml:"remote_timeout,omitempty"` WriteRelabelConfigs []*RelabelConfig `yaml:"write_relabel_configs,omitempty"` // We cannot do proper Go type embedding below as the parser will then parse // values arbitrarily into the overflow maps of further-down types. - HTTPClientConfig HTTPClientConfig `yaml:",inline"` - QueueConfig QueueConfig `yaml:"queue_config,omitempty"` + HTTPClientConfig config_util.HTTPClientConfig `yaml:",inline"` + QueueConfig QueueConfig `yaml:"queue_config,omitempty"` // Catches all undefined fields and must be empty after parsing. XXX map[string]interface{} `yaml:",inline"` @@ -1456,11 +662,11 @@ func (c *RemoteWriteConfig) UnmarshalYAML(unmarshal func(interface{}) error) err // The UnmarshalYAML method of HTTPClientConfig is not being called because it's not a pointer. // We cannot make it a pointer as the parser panics for inlined pointer structs. // Thus we just do its validation here. - if err := c.HTTPClientConfig.validate(); err != nil { + if err := c.HTTPClientConfig.Validate(); err != nil { return err } - return checkOverflow(c.XXX, "remote_write") + return yaml_util.CheckOverflow(c.XXX, "remote_write") } // QueueConfig is the configuration for the queue used to write to remote @@ -1488,12 +694,12 @@ type QueueConfig struct { // RemoteReadConfig is the configuration for reading from remote storage. type RemoteReadConfig struct { - URL *URL `yaml:"url"` - RemoteTimeout model.Duration `yaml:"remote_timeout,omitempty"` - ReadRecent bool `yaml:"read_recent,omitempty"` + URL *config_util.URL `yaml:"url"` + RemoteTimeout model.Duration `yaml:"remote_timeout,omitempty"` + ReadRecent bool `yaml:"read_recent,omitempty"` // We cannot do proper Go type embedding below as the parser will then parse // values arbitrarily into the overflow maps of further-down types. - HTTPClientConfig HTTPClientConfig `yaml:",inline"` + HTTPClientConfig config_util.HTTPClientConfig `yaml:",inline"` // RequiredMatchers is an optional list of equality matchers which have to // be present in a selector to query the remote read endpoint. @@ -1517,9 +723,9 @@ func (c *RemoteReadConfig) UnmarshalYAML(unmarshal func(interface{}) error) erro // The UnmarshalYAML method of HTTPClientConfig is not being called because it's not a pointer. // We cannot make it a pointer as the parser panics for inlined pointer structs. // Thus we just do its validation here. - if err := c.HTTPClientConfig.validate(); err != nil { + if err := c.HTTPClientConfig.Validate(); err != nil { return err } - return checkOverflow(c.XXX, "remote_read") + return yaml_util.CheckOverflow(c.XXX, "remote_read") } diff --git a/config/config_test.go b/config/config_test.go index 2d7807569..795e7cb45 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -23,17 +23,30 @@ import ( "testing" "time" + "github.com/prometheus/prometheus/discovery/azure" + "github.com/prometheus/prometheus/discovery/consul" + "github.com/prometheus/prometheus/discovery/dns" + "github.com/prometheus/prometheus/discovery/ec2" + "github.com/prometheus/prometheus/discovery/file" + "github.com/prometheus/prometheus/discovery/kubernetes" + "github.com/prometheus/prometheus/discovery/marathon" + "github.com/prometheus/prometheus/discovery/targetgroup" + "github.com/prometheus/prometheus/discovery/triton" + "github.com/prometheus/prometheus/discovery/zookeeper" + "github.com/prometheus/common/model" + sd_config "github.com/prometheus/prometheus/discovery/config" + config_util "github.com/prometheus/prometheus/util/config" "github.com/prometheus/prometheus/util/testutil" "gopkg.in/yaml.v2" ) -func mustParseURL(u string) *URL { +func mustParseURL(u string) *config_util.URL { parsed, err := url.Parse(u) if err != nil { panic(err) } - return &URL{URL: parsed} + return &config_util.URL{URL: parsed} } var expectedConf = &Config{ @@ -100,12 +113,12 @@ var expectedConf = &Config{ MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, - HTTPClientConfig: HTTPClientConfig{ + HTTPClientConfig: config_util.HTTPClientConfig{ BearerTokenFile: filepath.FromSlash("testdata/valid_token_file"), }, - ServiceDiscoveryConfig: ServiceDiscoveryConfig{ - StaticConfigs: []*TargetGroup{ + ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{ + StaticConfigs: []*targetgroup.Group{ { Targets: []model.LabelSet{ {model.AddressLabel: "localhost:9090"}, @@ -118,7 +131,7 @@ var expectedConf = &Config{ }, }, - FileSDConfigs: []*FileSDConfig{ + FileSDConfigs: []*file.SDConfig{ { Files: []string{"testdata/foo/*.slow.json", "testdata/foo/*.slow.yml", "testdata/single/file.yml"}, RefreshInterval: model.Duration(10 * time.Minute), @@ -168,8 +181,8 @@ var expectedConf = &Config{ ScrapeTimeout: model.Duration(5 * time.Second), SampleLimit: 1000, - HTTPClientConfig: HTTPClientConfig{ - BasicAuth: &BasicAuth{ + HTTPClientConfig: config_util.HTTPClientConfig{ + BasicAuth: &config_util.BasicAuth{ Username: "admin_name", Password: "multiline\nmysecret\ntest", }, @@ -177,8 +190,8 @@ var expectedConf = &Config{ MetricsPath: "/my_path", Scheme: "https", - ServiceDiscoveryConfig: ServiceDiscoveryConfig{ - DNSSDConfigs: []*DNSSDConfig{ + ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{ + DNSSDConfigs: []*dns.SDConfig{ { Names: []string{ "first.dns.address.domain.com", @@ -259,15 +272,15 @@ var expectedConf = &Config{ MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, - ServiceDiscoveryConfig: ServiceDiscoveryConfig{ - ConsulSDConfigs: []*ConsulSDConfig{ + ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{ + ConsulSDConfigs: []*consul.SDConfig{ { Server: "localhost:1234", Token: "mysecret", Services: []string{"nginx", "cache", "mysql"}, - TagSeparator: DefaultConsulSDConfig.TagSeparator, + TagSeparator: consul.DefaultSDConfig.TagSeparator, Scheme: "https", - TLSConfig: TLSConfig{ + TLSConfig: config_util.TLSConfig{ CertFile: filepath.FromSlash("testdata/valid_cert_file"), KeyFile: filepath.FromSlash("testdata/valid_key_file"), CAFile: filepath.FromSlash("testdata/valid_ca_file"), @@ -297,8 +310,8 @@ var expectedConf = &Config{ MetricsPath: "/metrics", Scheme: "http", - HTTPClientConfig: HTTPClientConfig{ - TLSConfig: TLSConfig{ + HTTPClientConfig: config_util.HTTPClientConfig{ + TLSConfig: config_util.TLSConfig{ CertFile: filepath.FromSlash("testdata/valid_cert_file"), KeyFile: filepath.FromSlash("testdata/valid_key_file"), }, @@ -315,16 +328,16 @@ var expectedConf = &Config{ MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, - ServiceDiscoveryConfig: ServiceDiscoveryConfig{ - KubernetesSDConfigs: []*KubernetesSDConfig{ + ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{ + KubernetesSDConfigs: []*kubernetes.SDConfig{ { APIServer: kubernetesSDHostURL(), - Role: KubernetesRoleEndpoint, - BasicAuth: &BasicAuth{ + Role: kubernetes.RoleEndpoint, + BasicAuth: &config_util.BasicAuth{ Username: "myusername", Password: "mysecret", }, - NamespaceDiscovery: KubernetesNamespaceDiscovery{}, + NamespaceDiscovery: kubernetes.NamespaceDiscovery{}, }, }, }, @@ -338,12 +351,12 @@ var expectedConf = &Config{ MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, - ServiceDiscoveryConfig: ServiceDiscoveryConfig{ - KubernetesSDConfigs: []*KubernetesSDConfig{ + ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{ + KubernetesSDConfigs: []*kubernetes.SDConfig{ { APIServer: kubernetesSDHostURL(), - Role: KubernetesRoleEndpoint, - NamespaceDiscovery: KubernetesNamespaceDiscovery{ + Role: kubernetes.RoleEndpoint, + NamespaceDiscovery: kubernetes.NamespaceDiscovery{ Names: []string{ "default", }, @@ -361,15 +374,15 @@ var expectedConf = &Config{ MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, - ServiceDiscoveryConfig: ServiceDiscoveryConfig{ - MarathonSDConfigs: []*MarathonSDConfig{ + ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{ + MarathonSDConfigs: []*marathon.SDConfig{ { Servers: []string{ "https://marathon.example.com:443", }, Timeout: model.Duration(30 * time.Second), RefreshInterval: model.Duration(30 * time.Second), - TLSConfig: TLSConfig{ + TLSConfig: config_util.TLSConfig{ CertFile: filepath.FromSlash("testdata/valid_cert_file"), KeyFile: filepath.FromSlash("testdata/valid_key_file"), }, @@ -386,8 +399,8 @@ var expectedConf = &Config{ MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, - ServiceDiscoveryConfig: ServiceDiscoveryConfig{ - EC2SDConfigs: []*EC2SDConfig{ + ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{ + EC2SDConfigs: []*ec2.SDConfig{ { Region: "us-east-1", AccessKey: "access", @@ -408,8 +421,8 @@ var expectedConf = &Config{ MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, - ServiceDiscoveryConfig: ServiceDiscoveryConfig{ - AzureSDConfigs: []*AzureSDConfig{ + ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{ + AzureSDConfigs: []*azure.SDConfig{ { SubscriptionID: "11AAAA11-A11A-111A-A111-1111A1111A11", TenantID: "BBBB222B-B2B2-2B22-B222-2BB2222BB2B2", @@ -430,8 +443,8 @@ var expectedConf = &Config{ MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, - ServiceDiscoveryConfig: ServiceDiscoveryConfig{ - NerveSDConfigs: []*NerveSDConfig{ + ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{ + NerveSDConfigs: []*zookeeper.NerveSDConfig{ { Servers: []string{"localhost"}, Paths: []string{"/monitoring"}, @@ -449,8 +462,8 @@ var expectedConf = &Config{ MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, - ServiceDiscoveryConfig: ServiceDiscoveryConfig{ - StaticConfigs: []*TargetGroup{ + ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{ + StaticConfigs: []*targetgroup.Group{ { Targets: []model.LabelSet{ {model.AddressLabel: "localhost:9090"}, @@ -468,8 +481,8 @@ var expectedConf = &Config{ MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, - ServiceDiscoveryConfig: ServiceDiscoveryConfig{ - StaticConfigs: []*TargetGroup{ + ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{ + StaticConfigs: []*targetgroup.Group{ { Targets: []model.LabelSet{ {model.AddressLabel: "localhost:9090"}, @@ -487,8 +500,8 @@ var expectedConf = &Config{ MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, - ServiceDiscoveryConfig: ServiceDiscoveryConfig{ - TritonSDConfigs: []*TritonSDConfig{ + ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{ + TritonSDConfigs: []*triton.SDConfig{ { Account: "testAccount", @@ -497,7 +510,7 @@ var expectedConf = &Config{ Port: 9163, RefreshInterval: model.Duration(60 * time.Second), Version: 1, - TLSConfig: TLSConfig{ + TLSConfig: config_util.TLSConfig{ CertFile: "testdata/valid_cert_file", KeyFile: "testdata/valid_key_file", }, @@ -511,8 +524,8 @@ var expectedConf = &Config{ { Scheme: "https", Timeout: 10 * time.Second, - ServiceDiscoveryConfig: ServiceDiscoveryConfig{ - StaticConfigs: []*TargetGroup{ + ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{ + StaticConfigs: []*targetgroup.Group{ { Targets: []model.LabelSet{ {model.AddressLabel: "1.2.3.4:9093"}, @@ -682,7 +695,7 @@ func TestBadConfigs(t *testing.T) { func TestBadStaticConfigs(t *testing.T) { content, err := ioutil.ReadFile("testdata/static_config.bad.json") testutil.Ok(t, err) - var tg TargetGroup + var tg targetgroup.Group err = json.Unmarshal(content, &tg) testutil.NotOk(t, err, "") } @@ -729,7 +742,7 @@ func TestTargetLabelValidity(t *testing.T) { } } -func kubernetesSDHostURL() URL { +func kubernetesSDHostURL() config_util.URL { tURL, _ := url.Parse("https://localhost:1234") - return URL{URL: tURL} + return config_util.URL{URL: tURL} } diff --git a/discovery/azure/azure.go b/discovery/azure/azure.go index 991a542d1..782b224ea 100644 --- a/discovery/azure/azure.go +++ b/discovery/azure/azure.go @@ -29,8 +29,10 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" + config_util "github.com/prometheus/prometheus/util/config" "github.com/prometheus/prometheus/util/strutil" + yaml_util "github.com/prometheus/prometheus/util/yaml" ) const ( @@ -54,8 +56,39 @@ var ( Name: "prometheus_sd_azure_refresh_duration_seconds", Help: "The duration of a Azure-SD refresh in seconds.", }) + + // DefaultSDConfig is the default Azure SD configuration. + DefaultSDConfig = SDConfig{ + Port: 80, + RefreshInterval: model.Duration(5 * time.Minute), + } ) +// SDConfig is the configuration for Azure based service discovery. +type SDConfig struct { + Port int `yaml:"port"` + SubscriptionID string `yaml:"subscription_id"` + TenantID string `yaml:"tenant_id,omitempty"` + ClientID string `yaml:"client_id,omitempty"` + ClientSecret config_util.Secret `yaml:"client_secret,omitempty"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + + // Catches all undefined fields and must be empty after parsing. + XXX map[string]interface{} `yaml:",inline"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = DefaultSDConfig + type plain SDConfig + err := unmarshal((*plain)(c)) + if err != nil { + return err + } + + return yaml_util.CheckOverflow(c.XXX, "azure_sd_config") +} + func init() { prometheus.MustRegister(azureSDRefreshDuration) prometheus.MustRegister(azureSDRefreshFailuresCount) @@ -64,14 +97,14 @@ func init() { // Discovery periodically performs Azure-SD requests. It implements // the TargetProvider interface. type Discovery struct { - cfg *config.AzureSDConfig + cfg *SDConfig interval time.Duration port int logger log.Logger } // NewDiscovery returns a new AzureDiscovery which periodically refreshes its targets. -func NewDiscovery(cfg *config.AzureSDConfig, logger log.Logger) *Discovery { +func NewDiscovery(cfg *SDConfig, logger log.Logger) *Discovery { if logger == nil { logger = log.NewNopLogger() } @@ -84,7 +117,7 @@ func NewDiscovery(cfg *config.AzureSDConfig, logger log.Logger) *Discovery { } // Run implements the TargetProvider interface. -func (d *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { +func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { ticker := time.NewTicker(d.interval) defer ticker.Stop() @@ -101,7 +134,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { } else { select { case <-ctx.Done(): - case ch <- []*config.TargetGroup{tg}: + case ch <- []*targetgroup.Group{tg}: } } @@ -120,7 +153,7 @@ type azureClient struct { } // createAzureClient is a helper function for creating an Azure compute client to ARM. -func createAzureClient(cfg config.AzureSDConfig) (azureClient, error) { +func createAzureClient(cfg SDConfig) (azureClient, error) { var c azureClient oauthConfig, err := azure.PublicCloud.OAuthConfigForTenant(cfg.TenantID) if err != nil { @@ -162,7 +195,7 @@ func newAzureResourceFromID(id string, logger log.Logger) (azureResource, error) }, nil } -func (d *Discovery) refresh() (tg *config.TargetGroup, err error) { +func (d *Discovery) refresh() (tg *targetgroup.Group, err error) { defer level.Debug(d.logger).Log("msg", "Azure discovery completed") t0 := time.Now() @@ -172,7 +205,7 @@ func (d *Discovery) refresh() (tg *config.TargetGroup, err error) { azureSDRefreshFailuresCount.Inc() } }() - tg = &config.TargetGroup{} + tg = &targetgroup.Group{} client, err := createAzureClient(*d.cfg) if err != nil { return tg, fmt.Errorf("could not create Azure client: %s", err) diff --git a/discovery/config/config.go b/discovery/config/config.go new file mode 100644 index 000000000..079404846 --- /dev/null +++ b/discovery/config/config.go @@ -0,0 +1,73 @@ +// Copyright 2016 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "github.com/prometheus/prometheus/discovery/azure" + "github.com/prometheus/prometheus/discovery/consul" + "github.com/prometheus/prometheus/discovery/dns" + "github.com/prometheus/prometheus/discovery/ec2" + "github.com/prometheus/prometheus/discovery/file" + "github.com/prometheus/prometheus/discovery/gce" + "github.com/prometheus/prometheus/discovery/kubernetes" + "github.com/prometheus/prometheus/discovery/marathon" + "github.com/prometheus/prometheus/discovery/openstack" + "github.com/prometheus/prometheus/discovery/targetgroup" + "github.com/prometheus/prometheus/discovery/triton" + "github.com/prometheus/prometheus/discovery/zookeeper" + + yaml_util "github.com/prometheus/prometheus/util/yaml" +) + +// ServiceDiscoveryConfig configures lists of different service discovery mechanisms. +type ServiceDiscoveryConfig struct { + // List of labeled target groups for this job. + StaticConfigs []*targetgroup.Group `yaml:"static_configs,omitempty"` + // List of DNS service discovery configurations. + DNSSDConfigs []*dns.SDConfig `yaml:"dns_sd_configs,omitempty"` + // List of file service discovery configurations. + FileSDConfigs []*file.SDConfig `yaml:"file_sd_configs,omitempty"` + // List of Consul service discovery configurations. + ConsulSDConfigs []*consul.SDConfig `yaml:"consul_sd_configs,omitempty"` + // List of Serverset service discovery configurations. + ServersetSDConfigs []*zookeeper.ServersetSDConfig `yaml:"serverset_sd_configs,omitempty"` + // NerveSDConfigs is a list of Nerve service discovery configurations. + NerveSDConfigs []*zookeeper.NerveSDConfig `yaml:"nerve_sd_configs,omitempty"` + // MarathonSDConfigs is a list of Marathon service discovery configurations. + MarathonSDConfigs []*marathon.SDConfig `yaml:"marathon_sd_configs,omitempty"` + // List of Kubernetes service discovery configurations. + KubernetesSDConfigs []*kubernetes.SDConfig `yaml:"kubernetes_sd_configs,omitempty"` + // List of GCE service discovery configurations. + GCESDConfigs []*gce.SDConfig `yaml:"gce_sd_configs,omitempty"` + // List of EC2 service discovery configurations. + EC2SDConfigs []*ec2.SDConfig `yaml:"ec2_sd_configs,omitempty"` + // List of OpenStack service discovery configurations. + OpenstackSDConfigs []*openstack.SDConfig `yaml:"openstack_sd_configs,omitempty"` + // List of Azure service discovery configurations. + AzureSDConfigs []*azure.SDConfig `yaml:"azure_sd_configs,omitempty"` + // List of Triton service discovery configurations. + TritonSDConfigs []*triton.SDConfig `yaml:"triton_sd_configs,omitempty"` + + // Catches all undefined fields and must be empty after parsing. + XXX map[string]interface{} `yaml:",inline"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *ServiceDiscoveryConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + type plain ServiceDiscoveryConfig + if err := unmarshal((*plain)(c)); err != nil { + return err + } + return yaml_util.CheckOverflow(c.XXX, "service discovery config") +} diff --git a/discovery/consul/consul.go b/discovery/consul/consul.go index ebea7b8a2..66600dfb5 100644 --- a/discovery/consul/consul.go +++ b/discovery/consul/consul.go @@ -28,9 +28,11 @@ import ( "github.com/mwitkow/go-conntrack" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" + config_util "github.com/prometheus/prometheus/util/config" "github.com/prometheus/prometheus/util/httputil" "github.com/prometheus/prometheus/util/strutil" + yaml_util "github.com/prometheus/prometheus/util/yaml" ) const ( @@ -75,8 +77,49 @@ var ( }, []string{"endpoint", "call"}, ) + + // DefaultSDConfig is the default Consul SD configuration. + DefaultSDConfig = SDConfig{ + TagSeparator: ",", + Scheme: "http", + } ) +// SDConfig is the configuration for Consul service discovery. +type SDConfig struct { + Server string `yaml:"server"` + Token config_util.Secret `yaml:"token,omitempty"` + Datacenter string `yaml:"datacenter,omitempty"` + TagSeparator string `yaml:"tag_separator,omitempty"` + Scheme string `yaml:"scheme,omitempty"` + Username string `yaml:"username,omitempty"` + Password config_util.Secret `yaml:"password,omitempty"` + // The list of services for which targets are discovered. + // Defaults to all services if empty. + Services []string `yaml:"services"` + + TLSConfig config_util.TLSConfig `yaml:"tls_config,omitempty"` + // Catches all undefined fields and must be empty after parsing. + XXX map[string]interface{} `yaml:",inline"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = DefaultSDConfig + type plain SDConfig + err := unmarshal((*plain)(c)) + if err != nil { + return err + } + if err := yaml_util.CheckOverflow(c.XXX, "consul_sd_config"); err != nil { + return err + } + if strings.TrimSpace(c.Server) == "" { + return fmt.Errorf("Consul SD configuration requires a server address") + } + return nil +} + func init() { prometheus.MustRegister(rpcFailuresCount) prometheus.MustRegister(rpcDuration) @@ -98,7 +141,7 @@ type Discovery struct { } // NewDiscovery returns a new Discovery for the given config. -func NewDiscovery(conf *config.ConsulSDConfig, logger log.Logger) (*Discovery, error) { +func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) { if logger == nil { logger = log.NewNopLogger() } @@ -160,7 +203,7 @@ func (d *Discovery) shouldWatch(name string) bool { } // Run implements the TargetProvider interface. -func (d *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { +func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { // Watched services and their cancelation functions. services := map[string]func(){} @@ -243,7 +286,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { select { case <-ctx.Done(): return - case ch <- []*config.TargetGroup{{Source: name}}: + case ch <- []*targetgroup.Group{{Source: name}}: } } } @@ -259,7 +302,7 @@ type consulService struct { logger log.Logger } -func (srv *consulService) watch(ctx context.Context, ch chan<- []*config.TargetGroup) { +func (srv *consulService) watch(ctx context.Context, ch chan<- []*targetgroup.Group) { catalog := srv.client.Catalog() lastIndex := uint64(0) @@ -291,7 +334,7 @@ func (srv *consulService) watch(ctx context.Context, ch chan<- []*config.TargetG } lastIndex = meta.LastIndex - tgroup := config.TargetGroup{ + tgroup := targetgroup.Group{ Source: srv.name, Labels: srv.labels, Targets: make([]model.LabelSet, 0, len(nodes)), @@ -339,7 +382,7 @@ func (srv *consulService) watch(ctx context.Context, ch chan<- []*config.TargetG select { case <-ctx.Done(): return - case ch <- []*config.TargetGroup{&tgroup}: + case ch <- []*targetgroup.Group{&tgroup}: } } } diff --git a/discovery/consul/consul_test.go b/discovery/consul/consul_test.go index cb55f13aa..039967504 100644 --- a/discovery/consul/consul_test.go +++ b/discovery/consul/consul_test.go @@ -15,12 +15,10 @@ package consul import ( "testing" - - "github.com/prometheus/prometheus/config" ) func TestConfiguredService(t *testing.T) { - conf := &config.ConsulSDConfig{ + conf := &SDConfig{ Services: []string{"configuredServiceName"}} consulDiscovery, err := NewDiscovery(conf, nil) @@ -36,7 +34,7 @@ func TestConfiguredService(t *testing.T) { } func TestNonConfiguredService(t *testing.T) { - conf := &config.ConsulSDConfig{} + conf := &SDConfig{} consulDiscovery, err := NewDiscovery(conf, nil) if err != nil { diff --git a/discovery/dns/dns.go b/discovery/dns/dns.go index db8c1b19a..f4b8676bb 100644 --- a/discovery/dns/dns.go +++ b/discovery/dns/dns.go @@ -26,8 +26,8 @@ import ( "github.com/miekg/dns" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" - - "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" + yaml_util "github.com/prometheus/prometheus/util/yaml" ) const ( @@ -52,8 +52,50 @@ var ( Name: "sd_dns_lookup_failures_total", Help: "The number of DNS-SD lookup failures.", }) + + // DefaultSDConfig is the default DNS SD configuration. + DefaultSDConfig = SDConfig{ + RefreshInterval: model.Duration(30 * time.Second), + Type: "SRV", + } ) +// SDConfig is the configuration for DNS based service discovery. +type SDConfig struct { + Names []string `yaml:"names"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + Type string `yaml:"type"` + Port int `yaml:"port"` // Ignored for SRV records + // Catches all undefined fields and must be empty after parsing. + XXX map[string]interface{} `yaml:",inline"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = DefaultSDConfig + type plain SDConfig + err := unmarshal((*plain)(c)) + if err != nil { + return err + } + if err := yaml_util.CheckOverflow(c.XXX, "dns_sd_config"); err != nil { + return err + } + if len(c.Names) == 0 { + return fmt.Errorf("DNS-SD config must contain at least one SRV record name") + } + switch strings.ToUpper(c.Type) { + case "SRV": + case "A", "AAAA": + if c.Port == 0 { + return fmt.Errorf("a port is required in DNS-SD configs for all record types except SRV") + } + default: + return fmt.Errorf("invalid DNS-SD records type %s", c.Type) + } + return nil +} + func init() { prometheus.MustRegister(dnsSDLookupFailuresCount) prometheus.MustRegister(dnsSDLookupsCount) @@ -71,7 +113,7 @@ type Discovery struct { } // NewDiscovery returns a new Discovery which periodically refreshes its targets. -func NewDiscovery(conf *config.DNSSDConfig, logger log.Logger) *Discovery { +func NewDiscovery(conf SDConfig, logger log.Logger) *Discovery { if logger == nil { logger = log.NewNopLogger() } @@ -95,7 +137,7 @@ func NewDiscovery(conf *config.DNSSDConfig, logger log.Logger) *Discovery { } // Run implements the TargetProvider interface. -func (d *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { +func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { ticker := time.NewTicker(d.interval) defer ticker.Stop() @@ -112,7 +154,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { } } -func (d *Discovery) refreshAll(ctx context.Context, ch chan<- []*config.TargetGroup) { +func (d *Discovery) refreshAll(ctx context.Context, ch chan<- []*targetgroup.Group) { var wg sync.WaitGroup wg.Add(len(d.names)) @@ -128,7 +170,7 @@ func (d *Discovery) refreshAll(ctx context.Context, ch chan<- []*config.TargetGr wg.Wait() } -func (d *Discovery) refresh(ctx context.Context, name string, ch chan<- []*config.TargetGroup) error { +func (d *Discovery) refresh(ctx context.Context, name string, ch chan<- []*targetgroup.Group) error { response, err := lookupWithSearchPath(name, d.qtype, d.logger) dnsSDLookupsCount.Inc() if err != nil { @@ -136,7 +178,7 @@ func (d *Discovery) refresh(ctx context.Context, name string, ch chan<- []*confi return err } - tg := &config.TargetGroup{} + tg := &targetgroup.Group{} hostPort := func(a string, p int) model.LabelValue { return model.LabelValue(net.JoinHostPort(a, fmt.Sprintf("%d", p))) } @@ -167,7 +209,7 @@ func (d *Discovery) refresh(ctx context.Context, name string, ch chan<- []*confi select { case <-ctx.Done(): return ctx.Err() - case ch <- []*config.TargetGroup{tg}: + case ch <- []*targetgroup.Group{tg}: } return nil diff --git a/discovery/ec2/ec2.go b/discovery/ec2/ec2.go index 6e538e710..81875ba59 100644 --- a/discovery/ec2/ec2.go +++ b/discovery/ec2/ec2.go @@ -32,8 +32,10 @@ import ( "github.com/prometheus/common/model" "github.com/aws/aws-sdk-go/service/ec2" - "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" + config_util "github.com/prometheus/prometheus/util/config" "github.com/prometheus/prometheus/util/strutil" + yaml_util "github.com/prometheus/prometheus/util/yaml" ) const ( @@ -62,8 +64,53 @@ var ( Name: "prometheus_sd_ec2_refresh_duration_seconds", Help: "The duration of a EC2-SD refresh in seconds.", }) + // DefaultSDConfig is the default EC2 SD configuration. + DefaultSDConfig = SDConfig{ + Port: 80, + RefreshInterval: model.Duration(60 * time.Second), + } ) +// SDConfig is the configuration for EC2 based service discovery. +type SDConfig struct { + Region string `yaml:"region"` + AccessKey string `yaml:"access_key,omitempty"` + SecretKey config_util.Secret `yaml:"secret_key,omitempty"` + Profile string `yaml:"profile,omitempty"` + RoleARN string `yaml:"role_arn,omitempty"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + Port int `yaml:"port"` + + // Catches all undefined fields and must be empty after parsing. + XXX map[string]interface{} `yaml:",inline"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = DefaultSDConfig + type plain SDConfig + err := unmarshal((*plain)(c)) + if err != nil { + return err + } + if err := yaml_util.CheckOverflow(c.XXX, "ec2_sd_config"); err != nil { + return err + } + if c.Region == "" { + sess, err := session.NewSession() + if err != nil { + return err + } + metadata := ec2metadata.New(sess) + region, err := metadata.Region() + if err != nil { + return fmt.Errorf("EC2 SD configuration requires a region") + } + c.Region = region + } + return nil +} + func init() { prometheus.MustRegister(ec2SDRefreshFailuresCount) prometheus.MustRegister(ec2SDRefreshDuration) @@ -81,7 +128,7 @@ type Discovery struct { } // NewDiscovery returns a new EC2Discovery which periodically refreshes its targets. -func NewDiscovery(conf *config.EC2SDConfig, logger log.Logger) *Discovery { +func NewDiscovery(conf *SDConfig, logger log.Logger) *Discovery { creds := credentials.NewStaticCredentials(conf.AccessKey, string(conf.SecretKey), "") if conf.AccessKey == "" && conf.SecretKey == "" { creds = nil @@ -103,7 +150,7 @@ func NewDiscovery(conf *config.EC2SDConfig, logger log.Logger) *Discovery { } // Run implements the TargetProvider interface. -func (d *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { +func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { ticker := time.NewTicker(d.interval) defer ticker.Stop() @@ -113,7 +160,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { level.Error(d.logger).Log("msg", "Refresh failed", "err", err) } else { select { - case ch <- []*config.TargetGroup{tg}: + case ch <- []*targetgroup.Group{tg}: case <-ctx.Done(): return } @@ -129,7 +176,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { } select { - case ch <- []*config.TargetGroup{tg}: + case ch <- []*targetgroup.Group{tg}: case <-ctx.Done(): return } @@ -149,7 +196,7 @@ func (d *Discovery) ec2MetadataAvailable(sess *session.Session) (isAvailable boo return isAvailable } -func (d *Discovery) refresh() (tg *config.TargetGroup, err error) { +func (d *Discovery) refresh() (tg *targetgroup.Group, err error) { t0 := time.Now() defer func() { ec2SDRefreshDuration.Observe(time.Since(t0).Seconds()) @@ -178,7 +225,7 @@ func (d *Discovery) refresh() (tg *config.TargetGroup, err error) { ec2s = ec2.New(sess) } } - tg = &config.TargetGroup{ + tg = &targetgroup.Group{ Source: *d.aws.Region, } if err = ec2s.DescribeInstancesPages(nil, func(p *ec2.DescribeInstancesOutput, lastPage bool) bool { diff --git a/discovery/file/file.go b/discovery/file/file.go index adb71127c..f17d55191 100644 --- a/discovery/file/file.go +++ b/discovery/file/file.go @@ -21,6 +21,7 @@ import ( "io/ioutil" "os" "path/filepath" + "regexp" "strings" "sync" "time" @@ -29,12 +30,52 @@ import ( "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/discovery/targetgroup" + yaml_util "github.com/prometheus/prometheus/util/yaml" "gopkg.in/fsnotify.v1" "gopkg.in/yaml.v2" - - "github.com/prometheus/prometheus/config" ) +var ( + patFileSDName = regexp.MustCompile(`^[^*]*(\*[^/]*)?\.(json|yml|yaml|JSON|YML|YAML)$`) + + // DefaultSDConfig is the default file SD configuration. + DefaultSDConfig = SDConfig{ + RefreshInterval: model.Duration(5 * time.Minute), + } +) + +// SDConfig is the configuration for file based discovery. +type SDConfig struct { + Files []string `yaml:"files"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + + // Catches all undefined fields and must be empty after parsing. + XXX map[string]interface{} `yaml:",inline"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = DefaultSDConfig + type plain SDConfig + err := unmarshal((*plain)(c)) + if err != nil { + return err + } + if err := yaml_util.CheckOverflow(c.XXX, "file_sd_config"); err != nil { + return err + } + if len(c.Files) == 0 { + return fmt.Errorf("file service discovery config must contain at least one path name") + } + for _, name := range c.Files { + if !patFileSDName.MatchString(name) { + return fmt.Errorf("path name %q is not valid for file discovery", name) + } + } + return nil +} + const fileSDFilepathLabel = model.MetaLabelPrefix + "filepath" // TimestampCollector is a Custom Collector for Timestamps of the files. @@ -133,7 +174,7 @@ type Discovery struct { } // NewDiscovery returns a new file discovery for the given paths. -func NewDiscovery(conf *config.FileSDConfig, logger log.Logger) *Discovery { +func NewDiscovery(conf *SDConfig, logger log.Logger) *Discovery { if logger == nil { logger = log.NewNopLogger() } @@ -181,7 +222,7 @@ func (d *Discovery) watchFiles() { } // Run implements the TargetProvider interface. -func (d *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { +func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { watcher, err := fsnotify.NewWatcher() if err != nil { level.Error(d.logger).Log("msg", "Error adding file watcher", "err", err) @@ -271,7 +312,7 @@ func (d *Discovery) stop() { // refresh reads all files matching the discovery's patterns and sends the respective // updated target groups through the channel. -func (d *Discovery) refresh(ctx context.Context, ch chan<- []*config.TargetGroup) { +func (d *Discovery) refresh(ctx context.Context, ch chan<- []*targetgroup.Group) { t0 := time.Now() defer func() { fileSDScanDuration.Observe(time.Since(t0).Seconds()) @@ -303,7 +344,7 @@ func (d *Discovery) refresh(ctx context.Context, ch chan<- []*config.TargetGroup d.deleteTimestamp(f) for i := m; i < n; i++ { select { - case ch <- []*config.TargetGroup{{Source: fileSource(f, i)}}: + case ch <- []*targetgroup.Group{{Source: fileSource(f, i)}}: case <-ctx.Done(): return } @@ -317,7 +358,7 @@ func (d *Discovery) refresh(ctx context.Context, ch chan<- []*config.TargetGroup // readFile reads a JSON or YAML list of targets groups from the file, depending on its // file extension. It returns full configuration target groups. -func (d *Discovery) readFile(filename string) ([]*config.TargetGroup, error) { +func (d *Discovery) readFile(filename string) ([]*targetgroup.Group, error) { fd, err := os.Open(filename) if err != nil { return nil, err @@ -334,7 +375,7 @@ func (d *Discovery) readFile(filename string) ([]*config.TargetGroup, error) { return nil, err } - var targetGroups []*config.TargetGroup + var targetGroups []*targetgroup.Group switch ext := filepath.Ext(filename); strings.ToLower(ext) { case ".json": diff --git a/discovery/file/file_test.go b/discovery/file/file_test.go index 5116ab9bc..16d797745 100644 --- a/discovery/file/file_test.go +++ b/discovery/file/file_test.go @@ -23,7 +23,7 @@ import ( "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" ) const testDir = "fixtures" @@ -42,13 +42,13 @@ func TestFileSD(t *testing.T) { func testFileSD(t *testing.T, prefix, ext string, expect bool) { // As interval refreshing is more of a fallback, we only want to test // whether file watches work as expected. - var conf config.FileSDConfig + var conf SDConfig conf.Files = []string{filepath.Join(testDir, "_*"+ext)} conf.RefreshInterval = model.Duration(1 * time.Hour) var ( fsd = NewDiscovery(&conf, nil) - ch = make(chan []*config.TargetGroup) + ch = make(chan []*targetgroup.Group) ctx, cancel = context.WithCancel(context.Background()) ) go fsd.Run(ctx, ch) diff --git a/discovery/gce/gce.go b/discovery/gce/gce.go index 8172886cf..52b556356 100644 --- a/discovery/gce/gce.go +++ b/discovery/gce/gce.go @@ -26,10 +26,11 @@ import ( "github.com/prometheus/common/model" "golang.org/x/oauth2" "golang.org/x/oauth2/google" - "google.golang.org/api/compute/v1" + compute "google.golang.org/api/compute/v1" - "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/util/strutil" + yaml_util "github.com/prometheus/prometheus/util/yaml" ) const ( @@ -57,8 +58,56 @@ var ( Name: "prometheus_sd_gce_refresh_duration", Help: "The duration of a GCE-SD refresh in seconds.", }) + // DefaultSDConfig is the default GCE SD configuration. + DefaultSDConfig = SDConfig{ + Port: 80, + TagSeparator: ",", + RefreshInterval: model.Duration(60 * time.Second), + } ) +// SDConfig is the configuration for GCE based service discovery. +type SDConfig struct { + // Project: The Google Cloud Project ID + Project string `yaml:"project"` + + // Zone: The zone of the scrape targets. + // If you need to configure multiple zones use multiple gce_sd_configs + Zone string `yaml:"zone"` + + // Filter: Can be used optionally to filter the instance list by other criteria. + // Syntax of this filter string is described here in the filter query parameter section: + // https://cloud.google.com/compute/docs/reference/latest/instances/list + Filter string `yaml:"filter,omitempty"` + + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + Port int `yaml:"port"` + TagSeparator string `yaml:"tag_separator,omitempty"` + + // Catches all undefined fields and must be empty after parsing. + XXX map[string]interface{} `yaml:",inline"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = DefaultSDConfig + type plain SDConfig + err := unmarshal((*plain)(c)) + if err != nil { + return err + } + if err := yaml_util.CheckOverflow(c.XXX, "gce_sd_config"); err != nil { + return err + } + if c.Project == "" { + return fmt.Errorf("GCE SD configuration requires a project") + } + if c.Zone == "" { + return fmt.Errorf("GCE SD configuration requires a zone") + } + return nil +} + func init() { prometheus.MustRegister(gceSDRefreshFailuresCount) prometheus.MustRegister(gceSDRefreshDuration) @@ -80,7 +129,7 @@ type Discovery struct { } // NewDiscovery returns a new Discovery which periodically refreshes its targets. -func NewDiscovery(conf *config.GCESDConfig, logger log.Logger) (*Discovery, error) { +func NewDiscovery(conf SDConfig, logger log.Logger) (*Discovery, error) { if logger == nil { logger = log.NewNopLogger() } @@ -107,14 +156,14 @@ func NewDiscovery(conf *config.GCESDConfig, logger log.Logger) (*Discovery, erro } // Run implements the TargetProvider interface. -func (d *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { +func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { // Get an initial set right away. tg, err := d.refresh() if err != nil { level.Error(d.logger).Log("msg", "Refresh failed", "err", err) } else { select { - case ch <- []*config.TargetGroup{tg}: + case ch <- []*targetgroup.Group{tg}: case <-ctx.Done(): } } @@ -131,7 +180,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { continue } select { - case ch <- []*config.TargetGroup{tg}: + case ch <- []*targetgroup.Group{tg}: case <-ctx.Done(): } case <-ctx.Done(): @@ -140,7 +189,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { } } -func (d *Discovery) refresh() (tg *config.TargetGroup, err error) { +func (d *Discovery) refresh() (tg *targetgroup.Group, err error) { t0 := time.Now() defer func() { gceSDRefreshDuration.Observe(time.Since(t0).Seconds()) @@ -149,7 +198,7 @@ func (d *Discovery) refresh() (tg *config.TargetGroup, err error) { } }() - tg = &config.TargetGroup{ + tg = &targetgroup.Group{ Source: fmt.Sprintf("GCE_%s_%s", d.project, d.zone), } diff --git a/discovery/kubernetes/endpoints.go b/discovery/kubernetes/endpoints.go index 2fcbb971c..62ccc9f5b 100644 --- a/discovery/kubernetes/endpoints.go +++ b/discovery/kubernetes/endpoints.go @@ -22,10 +22,9 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/discovery/targetgroup" apiv1 "k8s.io/client-go/pkg/api/v1" "k8s.io/client-go/tools/cache" - - "github.com/prometheus/prometheus/config" ) // Endpoints discovers new endpoint targets. @@ -60,9 +59,9 @@ func NewEndpoints(l log.Logger, svc, eps, pod cache.SharedInformer) *Endpoints { } // Run implements the retrieval.TargetProvider interface. -func (e *Endpoints) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { +func (e *Endpoints) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { // Send full initial set of endpoint targets. - var initial []*config.TargetGroup + var initial []*targetgroup.Group for _, o := range e.endpointsStore.List() { tg := e.buildEndpoints(o.(*apiv1.Endpoints)) @@ -74,14 +73,14 @@ func (e *Endpoints) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { case ch <- initial: } // Send target groups for pod updates. - send := func(tg *config.TargetGroup) { + send := func(tg *targetgroup.Group) { if tg == nil { return } level.Debug(e.logger).Log("msg", "endpoints update", "tg", fmt.Sprintf("%#v", tg)) select { case <-ctx.Done(): - case ch <- []*config.TargetGroup{tg}: + case ch <- []*targetgroup.Group{tg}: } } @@ -114,7 +113,7 @@ func (e *Endpoints) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { level.Error(e.logger).Log("msg", "converting to Endpoints object failed", "err", err) return } - send(&config.TargetGroup{Source: endpointsSource(eps)}) + send(&targetgroup.Group{Source: endpointsSource(eps)}) }, }) @@ -185,12 +184,12 @@ const ( endpointPortProtocolLabel = metaLabelPrefix + "endpoint_port_protocol" ) -func (e *Endpoints) buildEndpoints(eps *apiv1.Endpoints) *config.TargetGroup { +func (e *Endpoints) buildEndpoints(eps *apiv1.Endpoints) *targetgroup.Group { if len(eps.Subsets) == 0 { return nil } - tg := &config.TargetGroup{ + tg := &targetgroup.Group{ Source: endpointsSource(eps), } tg.Labels = model.LabelSet{ @@ -319,7 +318,7 @@ func (e *Endpoints) resolvePodRef(ref *apiv1.ObjectReference) *apiv1.Pod { return obj.(*apiv1.Pod) } -func (e *Endpoints) addServiceLabels(ns, name string, tg *config.TargetGroup) { +func (e *Endpoints) addServiceLabels(ns, name string, tg *targetgroup.Group) { svc := &apiv1.Service{} svc.Namespace = ns svc.Name = name diff --git a/discovery/kubernetes/endpoints_test.go b/discovery/kubernetes/endpoints_test.go index c00514226..41f95b73e 100644 --- a/discovery/kubernetes/endpoints_test.go +++ b/discovery/kubernetes/endpoints_test.go @@ -17,7 +17,7 @@ import ( "testing" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/pkg/api/v1" @@ -89,7 +89,7 @@ func TestEndpointsDiscoveryInitial(t *testing.T) { k8sDiscoveryTest{ discovery: n, - expectedInitial: []*config.TargetGroup{ + expectedInitial: []*targetgroup.Group{ { Targets: []model.LabelSet{ { @@ -195,7 +195,7 @@ func TestEndpointsDiscoveryAdd(t *testing.T) { ) }() }, - expectedRes: []*config.TargetGroup{ + expectedRes: []*targetgroup.Group{ { Targets: []model.LabelSet{ { @@ -245,7 +245,7 @@ func TestEndpointsDiscoveryDelete(t *testing.T) { k8sDiscoveryTest{ discovery: n, afterStart: func() { go func() { eps.Delete(makeEndpoints()) }() }, - expectedRes: []*config.TargetGroup{ + expectedRes: []*targetgroup.Group{ { Source: "endpoints/default/testendpoints", }, @@ -260,7 +260,7 @@ func TestEndpointsDiscoveryDeleteUnknownCacheState(t *testing.T) { k8sDiscoveryTest{ discovery: n, afterStart: func() { go func() { eps.Delete(cache.DeletedFinalStateUnknown{Obj: makeEndpoints()}) }() }, - expectedRes: []*config.TargetGroup{ + expectedRes: []*targetgroup.Group{ { Source: "endpoints/default/testendpoints", }, @@ -314,7 +314,7 @@ func TestEndpointsDiscoveryUpdate(t *testing.T) { }) }() }, - expectedRes: []*config.TargetGroup{ + expectedRes: []*targetgroup.Group{ { Targets: []model.LabelSet{ { diff --git a/discovery/kubernetes/ingress.go b/discovery/kubernetes/ingress.go index 4b5fae151..58998ccdc 100644 --- a/discovery/kubernetes/ingress.go +++ b/discovery/kubernetes/ingress.go @@ -20,7 +20,7 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/util/strutil" "k8s.io/client-go/pkg/apis/extensions/v1beta1" "k8s.io/client-go/tools/cache" @@ -39,9 +39,9 @@ func NewIngress(l log.Logger, inf cache.SharedInformer) *Ingress { } // Run implements the TargetProvider interface. -func (s *Ingress) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { +func (s *Ingress) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { // Send full initial set of pod targets. - var initial []*config.TargetGroup + var initial []*targetgroup.Group for _, o := range s.store.List() { tg := s.buildIngress(o.(*v1beta1.Ingress)) initial = append(initial, tg) @@ -53,10 +53,10 @@ func (s *Ingress) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { } // Send target groups for ingress updates. - send := func(tg *config.TargetGroup) { + send := func(tg *targetgroup.Group) { select { case <-ctx.Done(): - case ch <- []*config.TargetGroup{tg}: + case ch <- []*targetgroup.Group{tg}: } } s.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ @@ -78,7 +78,7 @@ func (s *Ingress) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { level.Error(s.logger).Log("msg", "converting to Ingress object failed", "err", err.Error()) return } - send(&config.TargetGroup{Source: ingressSource(ingress)}) + send(&targetgroup.Group{Source: ingressSource(ingress)}) }, UpdateFunc: func(_, o interface{}) { eventCount.WithLabelValues("ingress", "update").Inc() @@ -158,8 +158,8 @@ func pathsFromIngressRule(rv *v1beta1.IngressRuleValue) []string { return paths } -func (s *Ingress) buildIngress(ingress *v1beta1.Ingress) *config.TargetGroup { - tg := &config.TargetGroup{ +func (s *Ingress) buildIngress(ingress *v1beta1.Ingress) *targetgroup.Group { + tg := &targetgroup.Group{ Source: ingressSource(ingress), } tg.Labels = ingressLabels(ingress) diff --git a/discovery/kubernetes/ingress_test.go b/discovery/kubernetes/ingress_test.go index 6c8bf7f14..1b250c8d3 100644 --- a/discovery/kubernetes/ingress_test.go +++ b/discovery/kubernetes/ingress_test.go @@ -17,7 +17,7 @@ import ( "testing" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/pkg/apis/extensions/v1beta1" ) @@ -77,12 +77,12 @@ func makeIngress(tls []v1beta1.IngressTLS) *v1beta1.Ingress { } } -func expectedTargetGroups(tls bool) []*config.TargetGroup { +func expectedTargetGroups(tls bool) []*targetgroup.Group { scheme := "http" if tls { scheme = "https" } - return []*config.TargetGroup{ + return []*targetgroup.Group{ { Targets: []model.LabelSet{ { diff --git a/discovery/kubernetes/kubernetes.go b/discovery/kubernetes/kubernetes.go index 934f7f24b..1e92e6444 100644 --- a/discovery/kubernetes/kubernetes.go +++ b/discovery/kubernetes/kubernetes.go @@ -15,6 +15,7 @@ package kubernetes import ( "context" + "fmt" "io/ioutil" "sync" "time" @@ -23,14 +24,16 @@ import ( "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/discovery/targetgroup" + config_util "github.com/prometheus/prometheus/util/config" + yaml_util "github.com/prometheus/prometheus/util/yaml" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/pkg/api" apiv1 "k8s.io/client-go/pkg/api/v1" extensionsv1beta1 "k8s.io/client-go/pkg/apis/extensions/v1beta1" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" - - "github.com/prometheus/prometheus/config" ) const ( @@ -48,8 +51,96 @@ var ( }, []string{"role", "event"}, ) + // DefaultSDConfig is the default Kubernetes SD configuration + DefaultSDConfig = SDConfig{} ) +// Role is role of the service in Kubernetes. +type Role string + +// The valid options for Role. +const ( + RoleNode Role = "node" + RolePod Role = "pod" + RoleService Role = "service" + RoleEndpoint Role = "endpoints" + RoleIngress Role = "ingress" +) + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *Role) UnmarshalYAML(unmarshal func(interface{}) error) error { + if err := unmarshal((*string)(c)); err != nil { + return err + } + switch *c { + case RoleNode, RolePod, RoleService, RoleEndpoint, RoleIngress: + return nil + default: + return fmt.Errorf("Unknown Kubernetes SD role %q", *c) + } +} + +// SDConfig is the configuration for Kubernetes service discovery. +type SDConfig struct { + APIServer config_util.URL `yaml:"api_server"` + Role Role `yaml:"role"` + BasicAuth *config_util.BasicAuth `yaml:"basic_auth,omitempty"` + BearerToken config_util.Secret `yaml:"bearer_token,omitempty"` + BearerTokenFile string `yaml:"bearer_token_file,omitempty"` + TLSConfig config_util.TLSConfig `yaml:"tls_config,omitempty"` + NamespaceDiscovery NamespaceDiscovery `yaml:"namespaces"` + + // Catches all undefined fields and must be empty after parsing. + XXX map[string]interface{} `yaml:",inline"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = SDConfig{} + type plain SDConfig + err := unmarshal((*plain)(c)) + if err != nil { + return err + } + if err := yaml_util.CheckOverflow(c.XXX, "kubernetes_sd_config"); err != nil { + return err + } + if c.Role == "" { + return fmt.Errorf("role missing (one of: pod, service, endpoints, node)") + } + if len(c.BearerToken) > 0 && len(c.BearerTokenFile) > 0 { + return fmt.Errorf("at most one of bearer_token & bearer_token_file must be configured") + } + if c.BasicAuth != nil && (len(c.BearerToken) > 0 || len(c.BearerTokenFile) > 0) { + return fmt.Errorf("at most one of basic_auth, bearer_token & bearer_token_file must be configured") + } + if c.APIServer.URL == nil && + (c.BasicAuth != nil || c.BearerToken != "" || c.BearerTokenFile != "" || + c.TLSConfig.CAFile != "" || c.TLSConfig.CertFile != "" || c.TLSConfig.KeyFile != "") { + return fmt.Errorf("to use custom authentication please provide the 'api_server' URL explicitly") + } + return nil +} + +// NamespaceDiscovery is the configuration for discovering +// Kubernetes namespaces. +type NamespaceDiscovery struct { + Names []string `yaml:"names"` + // Catches all undefined fields and must be empty after parsing. + XXX map[string]interface{} `yaml:",inline"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *NamespaceDiscovery) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = NamespaceDiscovery{} + type plain NamespaceDiscovery + err := unmarshal((*plain)(c)) + if err != nil { + return err + } + return yaml_util.CheckOverflow(c.XXX, "namespaces") +} + func init() { prometheus.MustRegister(eventCount) @@ -65,9 +156,9 @@ func init() { // targets from Kubernetes. type Discovery struct { client kubernetes.Interface - role config.KubernetesRole + role Role logger log.Logger - namespaceDiscovery *config.KubernetesNamespaceDiscovery + namespaceDiscovery *NamespaceDiscovery } func (d *Discovery) getNamespaces() []string { @@ -79,7 +170,7 @@ func (d *Discovery) getNamespaces() []string { } // New creates a new Kubernetes discovery for the given role. -func New(l log.Logger, conf *config.KubernetesSDConfig) (*Discovery, error) { +func New(l log.Logger, conf *SDConfig) (*Discovery, error) { if l == nil { l = log.NewNopLogger() } @@ -154,7 +245,7 @@ func New(l log.Logger, conf *config.KubernetesSDConfig) (*Discovery, error) { const resyncPeriod = 10 * time.Minute // Run implements the TargetProvider interface. -func (d *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { +func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { rclient := d.client.Core().RESTClient() reclient := d.client.Extensions().RESTClient() diff --git a/discovery/kubernetes/node.go b/discovery/kubernetes/node.go index 7e8603446..337a2f234 100644 --- a/discovery/kubernetes/node.go +++ b/discovery/kubernetes/node.go @@ -22,7 +22,7 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/util/strutil" "k8s.io/client-go/pkg/api" apiv1 "k8s.io/client-go/pkg/api/v1" @@ -45,9 +45,9 @@ func NewNode(l log.Logger, inf cache.SharedInformer) *Node { } // Run implements the TargetProvider interface. -func (n *Node) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { +func (n *Node) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { // Send full initial set of pod targets. - var initial []*config.TargetGroup + var initial []*targetgroup.Group for _, o := range n.store.List() { tg := n.buildNode(o.(*apiv1.Node)) initial = append(initial, tg) @@ -59,10 +59,10 @@ func (n *Node) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { } // Send target groups for service updates. - send := func(tg *config.TargetGroup) { + send := func(tg *targetgroup.Group) { select { case <-ctx.Done(): - case ch <- []*config.TargetGroup{tg}: + case ch <- []*targetgroup.Group{tg}: } } n.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ @@ -84,7 +84,7 @@ func (n *Node) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { level.Error(n.logger).Log("msg", "converting to Node object failed", "err", err) return } - send(&config.TargetGroup{Source: nodeSource(node)}) + send(&targetgroup.Group{Source: nodeSource(node)}) }, UpdateFunc: func(_, o interface{}) { eventCount.WithLabelValues("node", "update").Inc() @@ -147,8 +147,8 @@ func nodeLabels(n *apiv1.Node) model.LabelSet { return ls } -func (n *Node) buildNode(node *apiv1.Node) *config.TargetGroup { - tg := &config.TargetGroup{ +func (n *Node) buildNode(node *apiv1.Node) *targetgroup.Group { + tg := &targetgroup.Group{ Source: nodeSource(node), } tg.Labels = nodeLabels(node) diff --git a/discovery/kubernetes/node_test.go b/discovery/kubernetes/node_test.go index 0d2ed3d17..489d05b6a 100644 --- a/discovery/kubernetes/node_test.go +++ b/discovery/kubernetes/node_test.go @@ -22,12 +22,11 @@ import ( "time" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/pkg/api/v1" "k8s.io/client-go/tools/cache" - - "github.com/prometheus/prometheus/config" ) type fakeInformer struct { @@ -105,18 +104,18 @@ func (i *fakeInformer) Update(obj interface{}) { } type targetProvider interface { - Run(ctx context.Context, up chan<- []*config.TargetGroup) + Run(ctx context.Context, up chan<- []*targetgroup.Group) } type k8sDiscoveryTest struct { discovery targetProvider afterStart func() - expectedInitial []*config.TargetGroup - expectedRes []*config.TargetGroup + expectedInitial []*targetgroup.Group + expectedRes []*targetgroup.Group } func (d k8sDiscoveryTest) Run(t *testing.T) { - ch := make(chan []*config.TargetGroup) + ch := make(chan []*targetgroup.Group) ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*10) defer cancel() go func() { @@ -136,7 +135,7 @@ func (d k8sDiscoveryTest) Run(t *testing.T) { } } -func requireTargetGroups(t *testing.T, expected, res []*config.TargetGroup) { +func requireTargetGroups(t *testing.T, expected, res []*targetgroup.Group) { b1, err := json.Marshal(expected) if err != nil { panic(err) @@ -200,7 +199,7 @@ func TestNodeDiscoveryInitial(t *testing.T) { k8sDiscoveryTest{ discovery: n, - expectedInitial: []*config.TargetGroup{ + expectedInitial: []*targetgroup.Group{ { Targets: []model.LabelSet{ { @@ -226,7 +225,7 @@ func TestNodeDiscoveryAdd(t *testing.T) { k8sDiscoveryTest{ discovery: n, afterStart: func() { go func() { i.Add(makeEnumeratedNode(1)) }() }, - expectedRes: []*config.TargetGroup{ + expectedRes: []*targetgroup.Group{ { Targets: []model.LabelSet{ { @@ -251,7 +250,7 @@ func TestNodeDiscoveryDelete(t *testing.T) { k8sDiscoveryTest{ discovery: n, afterStart: func() { go func() { i.Delete(makeEnumeratedNode(0)) }() }, - expectedInitial: []*config.TargetGroup{ + expectedInitial: []*targetgroup.Group{ { Targets: []model.LabelSet{ { @@ -266,7 +265,7 @@ func TestNodeDiscoveryDelete(t *testing.T) { Source: "node/test0", }, }, - expectedRes: []*config.TargetGroup{ + expectedRes: []*targetgroup.Group{ { Source: "node/test0", }, @@ -281,7 +280,7 @@ func TestNodeDiscoveryDeleteUnknownCacheState(t *testing.T) { k8sDiscoveryTest{ discovery: n, afterStart: func() { go func() { i.Delete(cache.DeletedFinalStateUnknown{Obj: makeEnumeratedNode(0)}) }() }, - expectedInitial: []*config.TargetGroup{ + expectedInitial: []*targetgroup.Group{ { Targets: []model.LabelSet{ { @@ -296,7 +295,7 @@ func TestNodeDiscoveryDeleteUnknownCacheState(t *testing.T) { Source: "node/test0", }, }, - expectedRes: []*config.TargetGroup{ + expectedRes: []*targetgroup.Group{ { Source: "node/test0", }, @@ -322,7 +321,7 @@ func TestNodeDiscoveryUpdate(t *testing.T) { ) }() }, - expectedInitial: []*config.TargetGroup{ + expectedInitial: []*targetgroup.Group{ { Targets: []model.LabelSet{ { @@ -337,7 +336,7 @@ func TestNodeDiscoveryUpdate(t *testing.T) { Source: "node/test0", }, }, - expectedRes: []*config.TargetGroup{ + expectedRes: []*targetgroup.Group{ { Targets: []model.LabelSet{ { diff --git a/discovery/kubernetes/pod.go b/discovery/kubernetes/pod.go index 770e8e6e2..f2d2706f3 100644 --- a/discovery/kubernetes/pod.go +++ b/discovery/kubernetes/pod.go @@ -27,7 +27,7 @@ import ( apiv1 "k8s.io/client-go/pkg/api/v1" "k8s.io/client-go/tools/cache" - "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/util/strutil" ) @@ -51,9 +51,9 @@ func NewPod(l log.Logger, pods cache.SharedInformer) *Pod { } // Run implements the TargetProvider interface. -func (p *Pod) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { +func (p *Pod) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { // Send full initial set of pod targets. - var initial []*config.TargetGroup + var initial []*targetgroup.Group for _, o := range p.store.List() { tg := p.buildPod(o.(*apiv1.Pod)) initial = append(initial, tg) @@ -67,11 +67,11 @@ func (p *Pod) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { } // Send target groups for pod updates. - send := func(tg *config.TargetGroup) { + send := func(tg *targetgroup.Group) { level.Debug(p.logger).Log("msg", "pod update", "tg", fmt.Sprintf("%#v", tg)) select { case <-ctx.Done(): - case ch <- []*config.TargetGroup{tg}: + case ch <- []*targetgroup.Group{tg}: } } p.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ @@ -93,7 +93,7 @@ func (p *Pod) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { level.Error(p.logger).Log("msg", "converting to Pod object failed", "err", err) return } - send(&config.TargetGroup{Source: podSource(pod)}) + send(&targetgroup.Group{Source: podSource(pod)}) }, UpdateFunc: func(_, o interface{}) { eventCount.WithLabelValues("pod", "update").Inc() @@ -166,13 +166,13 @@ func podLabels(pod *apiv1.Pod) model.LabelSet { return ls } -func (p *Pod) buildPod(pod *apiv1.Pod) *config.TargetGroup { +func (p *Pod) buildPod(pod *apiv1.Pod) *targetgroup.Group { // During startup the pod may not have an IP yet. This does not even allow // for an up metric, so we skip the target. if len(pod.Status.PodIP) == 0 { return nil } - tg := &config.TargetGroup{ + tg := &targetgroup.Group{ Source: podSource(pod), } tg.Labels = podLabels(pod) diff --git a/discovery/kubernetes/pod_test.go b/discovery/kubernetes/pod_test.go index 7ea09266e..84c5edf51 100644 --- a/discovery/kubernetes/pod_test.go +++ b/discovery/kubernetes/pod_test.go @@ -17,7 +17,7 @@ import ( "testing" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/pkg/api/v1" @@ -123,7 +123,7 @@ func TestPodDiscoveryInitial(t *testing.T) { k8sDiscoveryTest{ discovery: n, - expectedInitial: []*config.TargetGroup{ + expectedInitial: []*targetgroup.Group{ { Targets: []model.LabelSet{ { @@ -168,7 +168,7 @@ func TestPodDiscoveryAdd(t *testing.T) { k8sDiscoveryTest{ discovery: n, afterStart: func() { go func() { i.Add(makePod()) }() }, - expectedRes: []*config.TargetGroup{ + expectedRes: []*targetgroup.Group{ { Targets: []model.LabelSet{ { @@ -201,7 +201,7 @@ func TestPodDiscoveryDelete(t *testing.T) { k8sDiscoveryTest{ discovery: n, afterStart: func() { go func() { i.Delete(makePod()) }() }, - expectedInitial: []*config.TargetGroup{ + expectedInitial: []*targetgroup.Group{ { Targets: []model.LabelSet{ { @@ -224,7 +224,7 @@ func TestPodDiscoveryDelete(t *testing.T) { Source: "pod/default/testpod", }, }, - expectedRes: []*config.TargetGroup{ + expectedRes: []*targetgroup.Group{ { Source: "pod/default/testpod", }, @@ -239,7 +239,7 @@ func TestPodDiscoveryDeleteUnknownCacheState(t *testing.T) { k8sDiscoveryTest{ discovery: n, afterStart: func() { go func() { i.Delete(cache.DeletedFinalStateUnknown{Obj: makePod()}) }() }, - expectedInitial: []*config.TargetGroup{ + expectedInitial: []*targetgroup.Group{ { Targets: []model.LabelSet{ { @@ -262,7 +262,7 @@ func TestPodDiscoveryDeleteUnknownCacheState(t *testing.T) { Source: "pod/default/testpod", }, }, - expectedRes: []*config.TargetGroup{ + expectedRes: []*targetgroup.Group{ { Source: "pod/default/testpod", }, @@ -302,7 +302,7 @@ func TestPodDiscoveryUpdate(t *testing.T) { k8sDiscoveryTest{ discovery: n, afterStart: func() { go func() { i.Update(makePod()) }() }, - expectedInitial: []*config.TargetGroup{ + expectedInitial: []*targetgroup.Group{ { Targets: []model.LabelSet{ { @@ -325,7 +325,7 @@ func TestPodDiscoveryUpdate(t *testing.T) { Source: "pod/default/testpod", }, }, - expectedRes: []*config.TargetGroup{ + expectedRes: []*targetgroup.Group{ { Targets: []model.LabelSet{ { diff --git a/discovery/kubernetes/service.go b/discovery/kubernetes/service.go index 29902555a..1c6c97465 100644 --- a/discovery/kubernetes/service.go +++ b/discovery/kubernetes/service.go @@ -25,7 +25,7 @@ import ( apiv1 "k8s.io/client-go/pkg/api/v1" "k8s.io/client-go/tools/cache" - "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/util/strutil" ) @@ -45,9 +45,9 @@ func NewService(l log.Logger, inf cache.SharedInformer) *Service { } // Run implements the TargetProvider interface. -func (s *Service) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { +func (s *Service) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { // Send full initial set of pod targets. - var initial []*config.TargetGroup + var initial []*targetgroup.Group for _, o := range s.store.List() { tg := s.buildService(o.(*apiv1.Service)) initial = append(initial, tg) @@ -59,10 +59,10 @@ func (s *Service) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { } // Send target groups for service updates. - send := func(tg *config.TargetGroup) { + send := func(tg *targetgroup.Group) { select { case <-ctx.Done(): - case ch <- []*config.TargetGroup{tg}: + case ch <- []*targetgroup.Group{tg}: } } s.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ @@ -84,7 +84,7 @@ func (s *Service) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { level.Error(s.logger).Log("msg", "converting to Service object failed", "err", err) return } - send(&config.TargetGroup{Source: serviceSource(svc)}) + send(&targetgroup.Group{Source: serviceSource(svc)}) }, UpdateFunc: func(_, o interface{}) { eventCount.WithLabelValues("service", "update").Inc() @@ -148,8 +148,8 @@ func serviceLabels(svc *apiv1.Service) model.LabelSet { return ls } -func (s *Service) buildService(svc *apiv1.Service) *config.TargetGroup { - tg := &config.TargetGroup{ +func (s *Service) buildService(svc *apiv1.Service) *targetgroup.Group { + tg := &targetgroup.Group{ Source: serviceSource(svc), } tg.Labels = serviceLabels(svc) diff --git a/discovery/kubernetes/service_test.go b/discovery/kubernetes/service_test.go index 5dd6ad197..f979dba8a 100644 --- a/discovery/kubernetes/service_test.go +++ b/discovery/kubernetes/service_test.go @@ -18,7 +18,7 @@ import ( "testing" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/pkg/api/v1" "k8s.io/client-go/tools/cache" @@ -90,7 +90,7 @@ func TestServiceDiscoveryInitial(t *testing.T) { k8sDiscoveryTest{ discovery: n, - expectedInitial: []*config.TargetGroup{ + expectedInitial: []*targetgroup.Group{ { Targets: []model.LabelSet{ { @@ -122,7 +122,7 @@ func TestServiceDiscoveryAdd(t *testing.T) { k8sDiscoveryTest{ discovery: n, afterStart: func() { go func() { i.Add(makeService()) }() }, - expectedRes: []*config.TargetGroup{ + expectedRes: []*targetgroup.Group{ { Targets: []model.LabelSet{ { @@ -148,7 +148,7 @@ func TestServiceDiscoveryDelete(t *testing.T) { k8sDiscoveryTest{ discovery: n, afterStart: func() { go func() { i.Delete(makeService()) }() }, - expectedInitial: []*config.TargetGroup{ + expectedInitial: []*targetgroup.Group{ { Targets: []model.LabelSet{ { @@ -164,7 +164,7 @@ func TestServiceDiscoveryDelete(t *testing.T) { Source: "svc/default/testservice", }, }, - expectedRes: []*config.TargetGroup{ + expectedRes: []*targetgroup.Group{ { Source: "svc/default/testservice", }, @@ -179,7 +179,7 @@ func TestServiceDiscoveryDeleteUnknownCacheState(t *testing.T) { k8sDiscoveryTest{ discovery: n, afterStart: func() { go func() { i.Delete(cache.DeletedFinalStateUnknown{Obj: makeService()}) }() }, - expectedInitial: []*config.TargetGroup{ + expectedInitial: []*targetgroup.Group{ { Targets: []model.LabelSet{ { @@ -195,7 +195,7 @@ func TestServiceDiscoveryDeleteUnknownCacheState(t *testing.T) { Source: "svc/default/testservice", }, }, - expectedRes: []*config.TargetGroup{ + expectedRes: []*targetgroup.Group{ { Source: "svc/default/testservice", }, @@ -210,7 +210,7 @@ func TestServiceDiscoveryUpdate(t *testing.T) { k8sDiscoveryTest{ discovery: n, afterStart: func() { go func() { i.Update(makeMultiPortService()) }() }, - expectedInitial: []*config.TargetGroup{ + expectedInitial: []*targetgroup.Group{ { Targets: []model.LabelSet{ { @@ -226,7 +226,7 @@ func TestServiceDiscoveryUpdate(t *testing.T) { Source: "svc/default/testservice", }, }, - expectedRes: []*config.TargetGroup{ + expectedRes: []*targetgroup.Group{ { Targets: []model.LabelSet{ { diff --git a/discovery/manager.go b/discovery/manager.go index a76676a57..0a3d107a3 100644 --- a/discovery/manager.go +++ b/discovery/manager.go @@ -22,6 +22,8 @@ import ( "github.com/go-kit/kit/log/level" "github.com/prometheus/prometheus/config" + sd_config "github.com/prometheus/prometheus/discovery/config" + "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/discovery/azure" "github.com/prometheus/prometheus/discovery/consul" @@ -49,7 +51,7 @@ type Discoverer interface { // updated target groups. // Must returns if the context gets canceled. It should not close the update // channel on returning. - Run(ctx context.Context, up chan<- []*config.TargetGroup) + Run(ctx context.Context, up chan<- []*targetgroup.Group) } type poolKey struct { @@ -70,8 +72,8 @@ func NewManager(logger log.Logger) *Manager { return &Manager{ logger: logger, actionCh: make(chan func(context.Context)), - syncCh: make(chan map[string][]*config.TargetGroup), - targets: make(map[poolKey][]*config.TargetGroup), + syncCh: make(chan map[string][]*targetgroup.Group), + targets: make(map[poolKey][]*targetgroup.Group), discoverCancel: []context.CancelFunc{}, } } @@ -81,9 +83,9 @@ type Manager struct { logger log.Logger actionCh chan func(context.Context) discoverCancel []context.CancelFunc - targets map[poolKey][]*config.TargetGroup + targets map[poolKey][]*targetgroup.Group // The sync channels sends the updates in map[targetSetName] where targetSetName is the job value from the scrape config. - syncCh chan map[string][]*config.TargetGroup + syncCh chan map[string][]*targetgroup.Group } // Run starts the background processing @@ -100,7 +102,7 @@ func (m *Manager) Run(ctx context.Context) error { } // SyncCh returns a read only channel used by all Discoverers to send target updates. -func (m *Manager) SyncCh() <-chan map[string][]*config.TargetGroup { +func (m *Manager) SyncCh() <-chan map[string][]*targetgroup.Group { return m.syncCh } @@ -122,7 +124,7 @@ func (m *Manager) ApplyConfig(cfg *config.Config) error { func (m *Manager) startProvider(ctx context.Context, poolKey poolKey, worker Discoverer) { ctx, cancel := context.WithCancel(ctx) - updates := make(chan []*config.TargetGroup) + updates := make(chan []*targetgroup.Group) m.discoverCancel = append(m.discoverCancel, cancel) @@ -130,7 +132,7 @@ func (m *Manager) startProvider(ctx context.Context, poolKey poolKey, worker Dis go m.runProvider(ctx, poolKey, updates) } -func (m *Manager) runProvider(ctx context.Context, poolKey poolKey, updates chan []*config.TargetGroup) { +func (m *Manager) runProvider(ctx context.Context, poolKey poolKey, updates chan []*targetgroup.Group) { for { select { case <-ctx.Done(): @@ -151,11 +153,11 @@ func (m *Manager) cancelDiscoverers() { for _, c := range m.discoverCancel { c() } - m.targets = make(map[poolKey][]*config.TargetGroup) + m.targets = make(map[poolKey][]*targetgroup.Group) m.discoverCancel = nil } -func (m *Manager) addGroup(poolKey poolKey, tg []*config.TargetGroup) { +func (m *Manager) addGroup(poolKey poolKey, tg []*targetgroup.Group) { done := make(chan struct{}) m.actionCh <- func(ctx context.Context) { @@ -168,8 +170,8 @@ func (m *Manager) addGroup(poolKey poolKey, tg []*config.TargetGroup) { <-done } -func (m *Manager) allGroups(pk poolKey) map[string][]*config.TargetGroup { - tSets := make(chan map[string][]*config.TargetGroup) +func (m *Manager) allGroups(pk poolKey) map[string][]*targetgroup.Group { + tSets := make(chan map[string][]*targetgroup.Group) m.actionCh <- func(ctx context.Context) { @@ -180,7 +182,7 @@ func (m *Manager) allGroups(pk poolKey) map[string][]*config.TargetGroup { } sort.Sort(byProvider(pKeys)) - tSetsAll := map[string][]*config.TargetGroup{} + tSetsAll := map[string][]*targetgroup.Group{} for _, pk := range pKeys { for _, tg := range m.targets[pk] { if tg.Source != "" { // Don't add empty targets. @@ -194,7 +196,7 @@ func (m *Manager) allGroups(pk poolKey) map[string][]*config.TargetGroup { } -func (m *Manager) providersFromConfig(cfg config.ServiceDiscoveryConfig) map[string]Discoverer { +func (m *Manager) providersFromConfig(cfg sd_config.ServiceDiscoveryConfig) map[string]Discoverer { providers := map[string]Discoverer{} app := func(mech string, i int, tp Discoverer) { @@ -202,7 +204,7 @@ func (m *Manager) providersFromConfig(cfg config.ServiceDiscoveryConfig) map[str } for i, c := range cfg.DNSSDConfigs { - app("dns", i, dns.NewDiscovery(c, log.With(m.logger, "discovery", "dns"))) + app("dns", i, dns.NewDiscovery(*c, log.With(m.logger, "discovery", "dns"))) } for i, c := range cfg.FileSDConfigs { app("file", i, file.NewDiscovery(c, log.With(m.logger, "discovery", "file"))) @@ -216,7 +218,7 @@ func (m *Manager) providersFromConfig(cfg config.ServiceDiscoveryConfig) map[str app("consul", i, k) } for i, c := range cfg.MarathonSDConfigs { - t, err := marathon.NewDiscovery(c, log.With(m.logger, "discovery", "marathon")) + t, err := marathon.NewDiscovery(*c, log.With(m.logger, "discovery", "marathon")) if err != nil { level.Error(m.logger).Log("msg", "Cannot create Marathon discovery", "err", err) continue @@ -250,7 +252,7 @@ func (m *Manager) providersFromConfig(cfg config.ServiceDiscoveryConfig) map[str } for i, c := range cfg.GCESDConfigs { - gced, err := gce.NewDiscovery(c, log.With(m.logger, "discovery", "gce")) + gced, err := gce.NewDiscovery(*c, log.With(m.logger, "discovery", "gce")) if err != nil { level.Error(m.logger).Log("msg", "Cannot initialize GCE discovery", "err", err) continue @@ -277,12 +279,12 @@ func (m *Manager) providersFromConfig(cfg config.ServiceDiscoveryConfig) map[str // StaticProvider holds a list of target groups that never change. type StaticProvider struct { - TargetGroups []*config.TargetGroup + TargetGroups []*targetgroup.Group } // NewStaticProvider returns a StaticProvider configured with the given // target groups. -func NewStaticProvider(groups []*config.TargetGroup) *StaticProvider { +func NewStaticProvider(groups []*targetgroup.Group) *StaticProvider { for i, tg := range groups { tg.Source = fmt.Sprintf("%d", i) } @@ -290,7 +292,7 @@ func NewStaticProvider(groups []*config.TargetGroup) *StaticProvider { } // Run implements the Worker interface. -func (sd *StaticProvider) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { +func (sd *StaticProvider) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { // We still have to consider that the consumer exits right away in which case // the context will be canceled. select { diff --git a/discovery/manager_test.go b/discovery/manager_test.go index 748d19f18..2e99f097c 100644 --- a/discovery/manager_test.go +++ b/discovery/manager_test.go @@ -23,7 +23,8 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/prometheus/config" - yaml "gopkg.in/yaml.v2" + "github.com/prometheus/prometheus/discovery/targetgroup" + "gopkg.in/yaml.v2" ) // TestDiscoveryManagerSyncCalls checks that the target updates are received in the expected order. @@ -35,7 +36,7 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { testCases := []struct { title string updates map[string][]update - expectedTargets [][]*config.TargetGroup + expectedTargets [][]*targetgroup.Group }{ { title: "Single TP no updates", @@ -58,12 +59,12 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { updates: map[string][]update{ "tp1": { { - targetGroups: []config.TargetGroup{}, + targetGroups: []targetgroup.Group{}, interval: 5, }, }, }, - expectedTargets: [][]*config.TargetGroup{ + expectedTargets: [][]*targetgroup.Group{ {}, }, }, @@ -72,24 +73,24 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { updates: map[string][]update{ "tp1": { { - targetGroups: []config.TargetGroup{}, + targetGroups: []targetgroup.Group{}, interval: 5, }, }, "tp2": { { - targetGroups: []config.TargetGroup{}, + targetGroups: []targetgroup.Group{}, interval: 200, }, }, "tp3": { { - targetGroups: []config.TargetGroup{}, + targetGroups: []targetgroup.Group{}, interval: 100, }, }, }, - expectedTargets: [][]*config.TargetGroup{ + expectedTargets: [][]*targetgroup.Group{ {}, {}, {}, @@ -100,7 +101,7 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { updates: map[string][]update{ "tp1": { { - targetGroups: []config.TargetGroup{ + targetGroups: []targetgroup.Group{ { Source: "initial1", Targets: []model.LabelSet{{"__instance__": "1"}}, @@ -112,7 +113,7 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { }, }, }, - expectedTargets: [][]*config.TargetGroup{ + expectedTargets: [][]*targetgroup.Group{ { { Source: "initial1", @@ -130,7 +131,7 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { updates: map[string][]update{ "tp1": { { - targetGroups: []config.TargetGroup{ + targetGroups: []targetgroup.Group{ { Source: "tp1-initial1", Targets: []model.LabelSet{{"__instance__": "1"}}, @@ -144,7 +145,7 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { }, "tp2": { { - targetGroups: []config.TargetGroup{ + targetGroups: []targetgroup.Group{ { Source: "tp2-initial1", Targets: []model.LabelSet{{"__instance__": "3"}}, @@ -154,7 +155,7 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { }, }, }, - expectedTargets: [][]*config.TargetGroup{ + expectedTargets: [][]*targetgroup.Group{ { { Source: "tp1-initial1", @@ -185,7 +186,7 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { updates: map[string][]update{ "tp1": { { - targetGroups: []config.TargetGroup{ + targetGroups: []targetgroup.Group{ { Source: "initial1", Targets: []model.LabelSet{{"__instance__": "1"}}, @@ -198,12 +199,12 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { interval: 0, }, { - targetGroups: []config.TargetGroup{}, + targetGroups: []targetgroup.Group{}, interval: 10, }, }, }, - expectedTargets: [][]*config.TargetGroup{ + expectedTargets: [][]*targetgroup.Group{ { { Source: "initial1", @@ -222,7 +223,7 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { updates: map[string][]update{ "tp1": { { - targetGroups: []config.TargetGroup{ + targetGroups: []targetgroup.Group{ { Source: "initial1", Targets: []model.LabelSet{{"__instance__": "1"}}, @@ -235,7 +236,7 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { interval: 0, }, { - targetGroups: []config.TargetGroup{ + targetGroups: []targetgroup.Group{ { Source: "update1", Targets: []model.LabelSet{{"__instance__": "3"}}, @@ -249,7 +250,7 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { }, }, }, - expectedTargets: [][]*config.TargetGroup{ + expectedTargets: [][]*targetgroup.Group{ { { Source: "initial1", @@ -277,7 +278,7 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { updates: map[string][]update{ "tp1": { { - targetGroups: []config.TargetGroup{ + targetGroups: []targetgroup.Group{ { Source: "tp1-initial1", Targets: []model.LabelSet{{"__instance__": "1"}}, @@ -290,7 +291,7 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { interval: 10, }, { - targetGroups: []config.TargetGroup{ + targetGroups: []targetgroup.Group{ { Source: "tp1-update1", Targets: []model.LabelSet{{"__instance__": "3"}}, @@ -305,7 +306,7 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { }, "tp2": { { - targetGroups: []config.TargetGroup{ + targetGroups: []targetgroup.Group{ { Source: "tp2-initial1", Targets: []model.LabelSet{{"__instance__": "5"}}, @@ -318,7 +319,7 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { interval: 100, }, { - targetGroups: []config.TargetGroup{ + targetGroups: []targetgroup.Group{ { Source: "tp2-update1", Targets: []model.LabelSet{{"__instance__": "7"}}, @@ -332,7 +333,7 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { }, }, }, - expectedTargets: [][]*config.TargetGroup{ + expectedTargets: [][]*targetgroup.Group{ { { Source: "tp1-initial1", @@ -404,7 +405,7 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { updates: map[string][]update{ "tp1": { { - targetGroups: []config.TargetGroup{ + targetGroups: []targetgroup.Group{ { Source: "tp1-initial1", Targets: []model.LabelSet{{"__instance__": "1"}}, @@ -417,7 +418,7 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { interval: 10, }, { - targetGroups: []config.TargetGroup{ + targetGroups: []targetgroup.Group{ { Source: "tp1-update1", Targets: []model.LabelSet{{"__instance__": "3"}}, @@ -432,7 +433,7 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { }, "tp2": { { - targetGroups: []config.TargetGroup{ + targetGroups: []targetgroup.Group{ { Source: "tp2-initial1", Targets: []model.LabelSet{{"__instance__": "5"}}, @@ -445,7 +446,7 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { interval: 200, }, { - targetGroups: []config.TargetGroup{ + targetGroups: []targetgroup.Group{ { Source: "tp2-update1", Targets: []model.LabelSet{{"__instance__": "7"}}, @@ -459,7 +460,7 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { }, }, }, - expectedTargets: [][]*config.TargetGroup{ + expectedTargets: [][]*targetgroup.Group{ { { Source: "tp1-initial1", @@ -524,7 +525,7 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { updates: map[string][]update{ "tp1": { { - targetGroups: []config.TargetGroup{ + targetGroups: []targetgroup.Group{ { Source: "initial1", Targets: []model.LabelSet{{"__instance__": "1"}}, @@ -537,11 +538,11 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { interval: 30, }, { - targetGroups: []config.TargetGroup{}, + targetGroups: []targetgroup.Group{}, interval: 10, }, { - targetGroups: []config.TargetGroup{ + targetGroups: []targetgroup.Group{ { Source: "update1", Targets: []model.LabelSet{{"__instance__": "3"}}, @@ -555,7 +556,7 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { }, }, }, - expectedTargets: [][]*config.TargetGroup{ + expectedTargets: [][]*targetgroup.Group{ { { Source: "initial1", @@ -627,7 +628,7 @@ func TestDiscoveryManagerSyncCalls(t *testing.T) { } func TestTargetSetRecreatesTargetGroupsEveryRun(t *testing.T) { - verifyPresence := func(tSets map[poolKey][]*config.TargetGroup, poolKey poolKey, label string, present bool) { + verifyPresence := func(tSets map[poolKey][]*targetgroup.Group, poolKey poolKey, label string, present bool) { if _, ok := tSets[poolKey]; !ok { t.Fatalf("'%s' should be present in Pool keys: %v", poolKey, tSets) return @@ -694,13 +695,13 @@ scrape_configs: } type update struct { - targetGroups []config.TargetGroup + targetGroups []targetgroup.Group interval time.Duration } type mockdiscoveryProvider struct { updates []update - up chan<- []*config.TargetGroup + up chan<- []*targetgroup.Group } func newMockDiscoveryProvider(updates []update) mockdiscoveryProvider { @@ -711,7 +712,7 @@ func newMockDiscoveryProvider(updates []update) mockdiscoveryProvider { return tp } -func (tp mockdiscoveryProvider) Run(ctx context.Context, up chan<- []*config.TargetGroup) { +func (tp mockdiscoveryProvider) Run(ctx context.Context, up chan<- []*targetgroup.Group) { tp.up = up tp.sendUpdates() } @@ -721,7 +722,7 @@ func (tp mockdiscoveryProvider) sendUpdates() { time.Sleep(update.interval * time.Millisecond) - tgs := make([]*config.TargetGroup, len(update.targetGroups)) + tgs := make([]*targetgroup.Group, len(update.targetGroups)) for i := range update.targetGroups { tgs[i] = &update.targetGroups[i] } diff --git a/discovery/marathon/marathon.go b/discovery/marathon/marathon.go index 7a897cfe1..5dff4f98c 100644 --- a/discovery/marathon/marathon.go +++ b/discovery/marathon/marathon.go @@ -27,12 +27,14 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" - "github.com/mwitkow/go-conntrack" + conntrack "github.com/mwitkow/go-conntrack" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" + config_util "github.com/prometheus/prometheus/util/config" "github.com/prometheus/prometheus/util/httputil" "github.com/prometheus/prometheus/util/strutil" + yaml_util "github.com/prometheus/prometheus/util/yaml" ) const ( @@ -73,8 +75,47 @@ var ( Name: "sd_marathon_refresh_duration_seconds", Help: "The duration of a Marathon-SD refresh in seconds.", }) + // DefaultSDConfig is the default Marathon SD configuration. + DefaultSDConfig = SDConfig{ + Timeout: model.Duration(30 * time.Second), + RefreshInterval: model.Duration(30 * time.Second), + } ) +// SDConfig is the configuration for services running on Marathon. +type SDConfig struct { + Servers []string `yaml:"servers,omitempty"` + Timeout model.Duration `yaml:"timeout,omitempty"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + TLSConfig config_util.TLSConfig `yaml:"tls_config,omitempty"` + BearerToken config_util.Secret `yaml:"bearer_token,omitempty"` + BearerTokenFile string `yaml:"bearer_token_file,omitempty"` + + // Catches all undefined fields and must be empty after parsing. + XXX map[string]interface{} `yaml:",inline"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = DefaultSDConfig + type plain SDConfig + err := unmarshal((*plain)(c)) + if err != nil { + return err + } + if err := yaml_util.CheckOverflow(c.XXX, "marathon_sd_config"); err != nil { + return err + } + if len(c.Servers) == 0 { + return fmt.Errorf("Marathon SD config must contain at least one Marathon server") + } + if len(c.BearerToken) > 0 && len(c.BearerTokenFile) > 0 { + return fmt.Errorf("at most one of bearer_token & bearer_token_file must be configured") + } + + return nil +} + func init() { prometheus.MustRegister(refreshFailuresCount) prometheus.MustRegister(refreshDuration) @@ -87,14 +128,14 @@ type Discovery struct { client *http.Client servers []string refreshInterval time.Duration - lastRefresh map[string]*config.TargetGroup + lastRefresh map[string]*targetgroup.Group appsClient AppListClient token string logger log.Logger } // NewDiscovery returns a new Marathon Discovery. -func NewDiscovery(conf *config.MarathonSDConfig, logger log.Logger) (*Discovery, error) { +func NewDiscovery(conf SDConfig, logger log.Logger) (*Discovery, error) { if logger == nil { logger = log.NewNopLogger() } @@ -135,7 +176,7 @@ func NewDiscovery(conf *config.MarathonSDConfig, logger log.Logger) (*Discovery, } // Run implements the TargetProvider interface. -func (d *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { +func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { for { select { case <-ctx.Done(): @@ -149,7 +190,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { } } -func (d *Discovery) updateServices(ctx context.Context, ch chan<- []*config.TargetGroup) (err error) { +func (d *Discovery) updateServices(ctx context.Context, ch chan<- []*targetgroup.Group) (err error) { t0 := time.Now() defer func() { refreshDuration.Observe(time.Since(t0).Seconds()) @@ -163,7 +204,7 @@ func (d *Discovery) updateServices(ctx context.Context, ch chan<- []*config.Targ return err } - all := make([]*config.TargetGroup, 0, len(targetMap)) + all := make([]*targetgroup.Group, 0, len(targetMap)) for _, tg := range targetMap { all = append(all, tg) } @@ -181,7 +222,7 @@ func (d *Discovery) updateServices(ctx context.Context, ch chan<- []*config.Targ select { case <-ctx.Done(): return ctx.Err() - case ch <- []*config.TargetGroup{{Source: source}}: + case ch <- []*targetgroup.Group{{Source: source}}: level.Debug(d.logger).Log("msg", "Removing group", "source", source) } } @@ -191,7 +232,7 @@ func (d *Discovery) updateServices(ctx context.Context, ch chan<- []*config.Targ return nil } -func (d *Discovery) fetchTargetGroups() (map[string]*config.TargetGroup, error) { +func (d *Discovery) fetchTargetGroups() (map[string]*targetgroup.Group, error) { url := RandomAppsURL(d.servers) apps, err := d.appsClient(d.client, url, d.token) if err != nil { @@ -294,8 +335,8 @@ func RandomAppsURL(servers []string) string { } // AppsToTargetGroups takes an array of Marathon apps and converts them into target groups. -func AppsToTargetGroups(apps *AppList) map[string]*config.TargetGroup { - tgroups := map[string]*config.TargetGroup{} +func AppsToTargetGroups(apps *AppList) map[string]*targetgroup.Group { + tgroups := map[string]*targetgroup.Group{} for _, a := range apps.Apps { group := createTargetGroup(&a) tgroups[group.Source] = group @@ -303,13 +344,13 @@ func AppsToTargetGroups(apps *AppList) map[string]*config.TargetGroup { return tgroups } -func createTargetGroup(app *App) *config.TargetGroup { +func createTargetGroup(app *App) *targetgroup.Group { var ( targets = targetsForApp(app) appName = model.LabelValue(app.ID) image = model.LabelValue(app.Container.Docker.Image) ) - tg := &config.TargetGroup{ + tg := &targetgroup.Group{ Targets: targets, Labels: model.LabelSet{ appLabel: appName, diff --git a/discovery/marathon/marathon_test.go b/discovery/marathon/marathon_test.go index 5351bdb56..cabd222d6 100644 --- a/discovery/marathon/marathon_test.go +++ b/discovery/marathon/marathon_test.go @@ -21,18 +21,17 @@ import ( "time" "github.com/prometheus/common/model" - - "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" ) var ( marathonValidLabel = map[string]string{"prometheus": "yes"} testServers = []string{"http://localhost:8080"} - conf = config.MarathonSDConfig{Servers: testServers} + conf = SDConfig{Servers: testServers} ) -func testUpdateServices(client AppListClient, ch chan []*config.TargetGroup) error { - md, err := NewDiscovery(&conf, nil) +func testUpdateServices(client AppListClient, ch chan []*targetgroup.Group) error { + md, err := NewDiscovery(conf, nil) if err != nil { return err } @@ -43,7 +42,7 @@ func testUpdateServices(client AppListClient, ch chan []*config.TargetGroup) err func TestMarathonSDHandleError(t *testing.T) { var ( errTesting = errors.New("testing failure") - ch = make(chan []*config.TargetGroup, 1) + ch = make(chan []*targetgroup.Group, 1) client = func(client *http.Client, url, token string) (*AppList, error) { return nil, errTesting } ) if err := testUpdateServices(client, ch); err != errTesting { @@ -58,7 +57,7 @@ func TestMarathonSDHandleError(t *testing.T) { func TestMarathonSDEmptyList(t *testing.T) { var ( - ch = make(chan []*config.TargetGroup, 1) + ch = make(chan []*targetgroup.Group, 1) client = func(client *http.Client, url, token string) (*AppList, error) { return &AppList{}, nil } ) if err := testUpdateServices(client, ch); err != nil { @@ -105,7 +104,7 @@ func marathonTestAppList(labels map[string]string, runningTasks int) *AppList { func TestMarathonSDSendGroup(t *testing.T) { var ( - ch = make(chan []*config.TargetGroup, 1) + ch = make(chan []*targetgroup.Group, 1) client = func(client *http.Client, url, token string) (*AppList, error) { return marathonTestAppList(marathonValidLabel, 1), nil } @@ -139,8 +138,8 @@ func TestMarathonSDSendGroup(t *testing.T) { } func TestMarathonSDRemoveApp(t *testing.T) { - var ch = make(chan []*config.TargetGroup, 1) - md, err := NewDiscovery(&conf, nil) + var ch = make(chan []*targetgroup.Group, 1) + md, err := NewDiscovery(conf, nil) if err != nil { t.Fatalf("%s", err) } @@ -172,11 +171,11 @@ func TestMarathonSDRemoveApp(t *testing.T) { func TestMarathonSDRunAndStop(t *testing.T) { var ( refreshInterval = model.Duration(time.Millisecond * 10) - conf = config.MarathonSDConfig{Servers: testServers, RefreshInterval: refreshInterval} - ch = make(chan []*config.TargetGroup) + conf = SDConfig{Servers: testServers, RefreshInterval: refreshInterval} + ch = make(chan []*targetgroup.Group) doneCh = make(chan error) ) - md, err := NewDiscovery(&conf, nil) + md, err := NewDiscovery(conf, nil) if err != nil { t.Fatalf("%s", err) } @@ -237,7 +236,7 @@ func marathonTestAppListWithMutiplePorts(labels map[string]string, runningTasks func TestMarathonSDSendGroupWithMutiplePort(t *testing.T) { var ( - ch = make(chan []*config.TargetGroup, 1) + ch = make(chan []*targetgroup.Group, 1) client = func(client *http.Client, url, token string) (*AppList, error) { return marathonTestAppListWithMutiplePorts(marathonValidLabel, 1), nil } @@ -304,7 +303,7 @@ func marathonTestZeroTaskPortAppList(labels map[string]string, runningTasks int) func TestMarathonZeroTaskPorts(t *testing.T) { var ( - ch = make(chan []*config.TargetGroup, 1) + ch = make(chan []*targetgroup.Group, 1) client = func(client *http.Client, url, token string) (*AppList, error) { return marathonTestZeroTaskPortAppList(marathonValidLabel, 1), nil } @@ -357,7 +356,7 @@ func marathonTestAppListWithoutPortMappings(labels map[string]string, runningTas func TestMarathonSDSendGroupWithoutPortMappings(t *testing.T) { var ( - ch = make(chan []*config.TargetGroup, 1) + ch = make(chan []*targetgroup.Group, 1) client = func(client *http.Client, url, token string) (*AppList, error) { return marathonTestAppListWithoutPortMappings(marathonValidLabel, 1), nil } @@ -430,7 +429,7 @@ func marathonTestAppListWithoutPortDefinitions(labels map[string]string, running func TestMarathonSDSendGroupWithoutPortDefinitions(t *testing.T) { var ( - ch = make(chan []*config.TargetGroup, 1) + ch = make(chan []*targetgroup.Group, 1) client = func(client *http.Client, url, token string) (*AppList, error) { return marathonTestAppListWithoutPortDefinitions(marathonValidLabel, 1), nil } @@ -505,7 +504,7 @@ func marathonTestAppListWithContainerPortMappings(labels map[string]string, runn func TestMarathonSDSendGroupWithContainerPortMappings(t *testing.T) { var ( - ch = make(chan []*config.TargetGroup, 1) + ch = make(chan []*targetgroup.Group, 1) client = func(client *http.Client, url, token string) (*AppList, error) { return marathonTestAppListWithContainerPortMappings(marathonValidLabel, 1), nil } diff --git a/discovery/openstack/hypervisor.go b/discovery/openstack/hypervisor.go index 80a23466b..c18e2d213 100644 --- a/discovery/openstack/hypervisor.go +++ b/discovery/openstack/hypervisor.go @@ -26,8 +26,7 @@ import ( "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/hypervisors" "github.com/gophercloud/gophercloud/pagination" "github.com/prometheus/common/model" - - "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" ) const ( @@ -55,14 +54,14 @@ func NewHypervisorDiscovery(opts *gophercloud.AuthOptions, } // Run implements the TargetProvider interface. -func (h *HypervisorDiscovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { +func (h *HypervisorDiscovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { // Get an initial set right away. tg, err := h.refresh() if err != nil { level.Error(h.logger).Log("msg", "Unable refresh target groups", "err", err.Error()) } else { select { - case ch <- []*config.TargetGroup{tg}: + case ch <- []*targetgroup.Group{tg}: case <-ctx.Done(): return } @@ -81,7 +80,7 @@ func (h *HypervisorDiscovery) Run(ctx context.Context, ch chan<- []*config.Targe } select { - case ch <- []*config.TargetGroup{tg}: + case ch <- []*targetgroup.Group{tg}: case <-ctx.Done(): return } @@ -91,7 +90,7 @@ func (h *HypervisorDiscovery) Run(ctx context.Context, ch chan<- []*config.Targe } } -func (h *HypervisorDiscovery) refresh() (*config.TargetGroup, error) { +func (h *HypervisorDiscovery) refresh() (*targetgroup.Group, error) { var err error t0 := time.Now() defer func() { @@ -112,7 +111,7 @@ func (h *HypervisorDiscovery) refresh() (*config.TargetGroup, error) { return nil, fmt.Errorf("could not create OpenStack compute session: %s", err) } - tg := &config.TargetGroup{ + tg := &targetgroup.Group{ Source: fmt.Sprintf("OS_" + h.region), } // OpenStack API reference diff --git a/discovery/openstack/hypervisor_test.go b/discovery/openstack/hypervisor_test.go index 1cbf5cfde..8bcadcea6 100644 --- a/discovery/openstack/hypervisor_test.go +++ b/discovery/openstack/hypervisor_test.go @@ -21,7 +21,6 @@ import ( "github.com/stretchr/testify/suite" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/config" ) type OpenstackSDHypervisorTestSuite struct { @@ -48,7 +47,7 @@ func TestOpenstackSDHypervisorSuite(t *testing.T) { } func (s *OpenstackSDHypervisorTestSuite) openstackAuthSuccess() (Discovery, error) { - conf := config.OpenstackSDConfig{ + conf := SDConfig{ IdentityEndpoint: s.Mock.Endpoint(), Password: "test", Username: "test", diff --git a/discovery/openstack/instance.go b/discovery/openstack/instance.go index 66082a0d3..bed934458 100644 --- a/discovery/openstack/instance.go +++ b/discovery/openstack/instance.go @@ -28,7 +28,7 @@ import ( "github.com/gophercloud/gophercloud/pagination" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/util/strutil" ) @@ -63,14 +63,14 @@ func NewInstanceDiscovery(opts *gophercloud.AuthOptions, } // Run implements the TargetProvider interface. -func (i *InstanceDiscovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { +func (i *InstanceDiscovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { // Get an initial set right away. tg, err := i.refresh() if err != nil { level.Error(i.logger).Log("msg", "Unable to refresh target groups", "err", err.Error()) } else { select { - case ch <- []*config.TargetGroup{tg}: + case ch <- []*targetgroup.Group{tg}: case <-ctx.Done(): return } @@ -89,7 +89,7 @@ func (i *InstanceDiscovery) Run(ctx context.Context, ch chan<- []*config.TargetG } select { - case ch <- []*config.TargetGroup{tg}: + case ch <- []*targetgroup.Group{tg}: case <-ctx.Done(): return } @@ -99,7 +99,7 @@ func (i *InstanceDiscovery) Run(ctx context.Context, ch chan<- []*config.TargetG } } -func (i *InstanceDiscovery) refresh() (*config.TargetGroup, error) { +func (i *InstanceDiscovery) refresh() (*targetgroup.Group, error) { var err error t0 := time.Now() defer func() { @@ -145,7 +145,7 @@ func (i *InstanceDiscovery) refresh() (*config.TargetGroup, error) { // https://developer.openstack.org/api-ref/compute/#list-servers opts := servers.ListOpts{} pager := servers.List(client, opts) - tg := &config.TargetGroup{ + tg := &targetgroup.Group{ Source: fmt.Sprintf("OS_" + i.region), } err = pager.EachPage(func(page pagination.Page) (bool, error) { diff --git a/discovery/openstack/instance_test.go b/discovery/openstack/instance_test.go index d320bd09b..49727a205 100644 --- a/discovery/openstack/instance_test.go +++ b/discovery/openstack/instance_test.go @@ -21,7 +21,6 @@ import ( "github.com/stretchr/testify/suite" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/config" ) type OpenstackSDInstanceTestSuite struct { @@ -49,7 +48,7 @@ func TestOpenstackSDInstanceSuite(t *testing.T) { } func (s *OpenstackSDInstanceTestSuite) openstackAuthSuccess() (Discovery, error) { - conf := config.OpenstackSDConfig{ + conf := SDConfig{ IdentityEndpoint: s.Mock.Endpoint(), Password: "test", Username: "test", diff --git a/discovery/openstack/openstack.go b/discovery/openstack/openstack.go index 2e67f8977..b1bafd85d 100644 --- a/discovery/openstack/openstack.go +++ b/discovery/openstack/openstack.go @@ -16,14 +16,17 @@ package openstack import ( "context" "errors" + "fmt" "time" "github.com/go-kit/kit/log" "github.com/gophercloud/gophercloud" "github.com/gophercloud/gophercloud/openstack" "github.com/prometheus/client_golang/prometheus" - - "github.com/prometheus/prometheus/config" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/discovery/targetgroup" + config_util "github.com/prometheus/prometheus/util/config" + yaml_util "github.com/prometheus/prometheus/util/yaml" ) var ( @@ -37,8 +40,72 @@ var ( Name: "prometheus_sd_openstack_refresh_duration_seconds", Help: "The duration of an OpenStack-SD refresh in seconds.", }) + // DefaultSDConfig is the default OpenStack SD configuration. + DefaultSDConfig = SDConfig{ + Port: 80, + RefreshInterval: model.Duration(60 * time.Second), + } ) +// SDConfig is the configuration for OpenStack based service discovery. +type SDConfig struct { + IdentityEndpoint string `yaml:"identity_endpoint"` + Username string `yaml:"username"` + UserID string `yaml:"userid"` + Password config_util.Secret `yaml:"password"` + ProjectName string `yaml:"project_name"` + ProjectID string `yaml:"project_id"` + DomainName string `yaml:"domain_name"` + DomainID string `yaml:"domain_id"` + Role Role `yaml:"role"` + Region string `yaml:"region"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + Port int `yaml:"port"` + + // Catches all undefined fields and must be empty after parsing. + XXX map[string]interface{} `yaml:",inline"` +} + +// OpenStackRole is role of the target in OpenStack. +type Role string + +// The valid options for OpenStackRole. +const ( + // OpenStack document reference + // https://docs.openstack.org/nova/pike/admin/arch.html#hypervisors + OpenStackRoleHypervisor Role = "hypervisor" + // OpenStack document reference + // https://docs.openstack.org/horizon/pike/user/launch-instances.html + OpenStackRoleInstance Role = "instance" +) + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *Role) UnmarshalYAML(unmarshal func(interface{}) error) error { + if err := unmarshal((*string)(c)); err != nil { + return err + } + switch *c { + case OpenStackRoleHypervisor, OpenStackRoleInstance: + return nil + default: + return fmt.Errorf("Unknown OpenStack SD role %q", *c) + } +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = DefaultSDConfig + type plain SDConfig + err := unmarshal((*plain)(c)) + if err != nil { + return err + } + if c.Role == "" { + return fmt.Errorf("role missing (one of: instance, hypervisor)") + } + return yaml_util.CheckOverflow(c.XXX, "openstack_sd_config") +} + func init() { prometheus.MustRegister(refreshFailuresCount) prometheus.MustRegister(refreshDuration) @@ -47,12 +114,12 @@ func init() { // Discovery periodically performs OpenStack-SD requests. It implements // the TargetProvider interface. type Discovery interface { - Run(ctx context.Context, ch chan<- []*config.TargetGroup) - refresh() (tg *config.TargetGroup, err error) + Run(ctx context.Context, ch chan<- []*targetgroup.Group) + refresh() (tg *targetgroup.Group, err error) } // NewDiscovery returns a new OpenStackDiscovery which periodically refreshes its targets. -func NewDiscovery(conf *config.OpenstackSDConfig, l log.Logger) (Discovery, error) { +func NewDiscovery(conf *SDConfig, l log.Logger) (Discovery, error) { var opts gophercloud.AuthOptions if conf.IdentityEndpoint == "" { var err error @@ -73,11 +140,11 @@ func NewDiscovery(conf *config.OpenstackSDConfig, l log.Logger) (Discovery, erro } } switch conf.Role { - case config.OpenStackRoleHypervisor: + case OpenStackRoleHypervisor: hypervisor := NewHypervisorDiscovery(&opts, time.Duration(conf.RefreshInterval), conf.Port, conf.Region, l) return hypervisor, nil - case config.OpenStackRoleInstance: + case OpenStackRoleInstance: instance := NewInstanceDiscovery(&opts, time.Duration(conf.RefreshInterval), conf.Port, conf.Region, l) return instance, nil diff --git a/discovery/targetgroup/targetgroup.go b/discovery/targetgroup/targetgroup.go new file mode 100644 index 000000000..4bafcef9e --- /dev/null +++ b/discovery/targetgroup/targetgroup.go @@ -0,0 +1,91 @@ +// Copyright 2013 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package targetgroup + +import ( + "encoding/json" + + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/util/yaml" +) + +// Group is a set of targets with a common label set(production , test, staging etc.). +type Group struct { + // Targets is a list of targets identified by a label set. Each target is + // uniquely identifiable in the group by its address label. + Targets []model.LabelSet + // Labels is a set of labels that is common across all targets in the group. + Labels model.LabelSet + + // Source is an identifier that describes a group of targets. + Source string +} + +func (tg Group) String() string { + return tg.Source +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (tg *Group) UnmarshalYAML(unmarshal func(interface{}) error) error { + g := struct { + Targets []string `yaml:"targets"` + Labels model.LabelSet `yaml:"labels"` + XXX map[string]interface{} `yaml:",inline"` + }{} + if err := unmarshal(&g); err != nil { + return err + } + tg.Targets = make([]model.LabelSet, 0, len(g.Targets)) + for _, t := range g.Targets { + tg.Targets = append(tg.Targets, model.LabelSet{ + model.AddressLabel: model.LabelValue(t), + }) + } + tg.Labels = g.Labels + return yaml.CheckOverflow(g.XXX, "static_config") +} + +// MarshalYAML implements the yaml.Marshaler interface. +func (tg Group) MarshalYAML() (interface{}, error) { + g := &struct { + Targets []string `yaml:"targets"` + Labels model.LabelSet `yaml:"labels,omitempty"` + }{ + Targets: make([]string, 0, len(tg.Targets)), + Labels: tg.Labels, + } + for _, t := range tg.Targets { + g.Targets = append(g.Targets, string(t[model.AddressLabel])) + } + return g, nil +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (tg *Group) UnmarshalJSON(b []byte) error { + g := struct { + Targets []string `json:"targets"` + Labels model.LabelSet `json:"labels"` + }{} + if err := json.Unmarshal(b, &g); err != nil { + return err + } + tg.Targets = make([]model.LabelSet, 0, len(g.Targets)) + for _, t := range g.Targets { + tg.Targets = append(tg.Targets, model.LabelSet{ + model.AddressLabel: model.LabelValue(t), + }) + } + tg.Labels = g.Labels + return nil +} diff --git a/discovery/triton/triton.go b/discovery/triton/triton.go index 75bc4f48a..b6fde88ba 100644 --- a/discovery/triton/triton.go +++ b/discovery/triton/triton.go @@ -27,8 +27,10 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" + config_util "github.com/prometheus/prometheus/util/config" "github.com/prometheus/prometheus/util/httputil" + yaml_util "github.com/prometheus/prometheus/util/yaml" ) const ( @@ -52,8 +54,50 @@ var ( Name: "prometheus_sd_triton_refresh_duration_seconds", Help: "The duration of a Triton-SD refresh in seconds.", }) + // DefaultSDConfig is the default Triton SD configuration. + DefaultSDConfig = SDConfig{ + Port: 9163, + RefreshInterval: model.Duration(60 * time.Second), + Version: 1, + } ) +// SDConfig is the configuration for Triton based service discovery. +type SDConfig struct { + Account string `yaml:"account"` + DNSSuffix string `yaml:"dns_suffix"` + Endpoint string `yaml:"endpoint"` + Port int `yaml:"port"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + TLSConfig config_util.TLSConfig `yaml:"tls_config,omitempty"` + Version int `yaml:"version"` + // Catches all undefined fields and must be empty after parsing. + XXX map[string]interface{} `yaml:",inline"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = DefaultSDConfig + type plain SDConfig + err := unmarshal((*plain)(c)) + if err != nil { + return err + } + if c.Account == "" { + return fmt.Errorf("Triton SD configuration requires an account") + } + if c.DNSSuffix == "" { + return fmt.Errorf("Triton SD configuration requires a dns_suffix") + } + if c.Endpoint == "" { + return fmt.Errorf("Triton SD configuration requires an endpoint") + } + if c.RefreshInterval <= 0 { + return fmt.Errorf("Triton SD configuration requires RefreshInterval to be a positive integer") + } + return yaml_util.CheckOverflow(c.XXX, "triton_sd_config") +} + func init() { prometheus.MustRegister(refreshFailuresCount) prometheus.MustRegister(refreshDuration) @@ -76,11 +120,11 @@ type Discovery struct { client *http.Client interval time.Duration logger log.Logger - sdConfig *config.TritonSDConfig + sdConfig *SDConfig } // New returns a new Discovery which periodically refreshes its targets. -func New(logger log.Logger, conf *config.TritonSDConfig) (*Discovery, error) { +func New(logger log.Logger, conf *SDConfig) (*Discovery, error) { if logger == nil { logger = log.NewNopLogger() } @@ -108,7 +152,7 @@ func New(logger log.Logger, conf *config.TritonSDConfig) (*Discovery, error) { } // Run implements the TargetProvider interface. -func (d *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { +func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { defer close(ch) ticker := time.NewTicker(d.interval) @@ -119,7 +163,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { if err != nil { level.Error(d.logger).Log("msg", "Refreshing targets failed", "err", err) } else { - ch <- []*config.TargetGroup{tg} + ch <- []*targetgroup.Group{tg} } for { @@ -129,7 +173,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { if err != nil { level.Error(d.logger).Log("msg", "Refreshing targets failed", "err", err) } else { - ch <- []*config.TargetGroup{tg} + ch <- []*targetgroup.Group{tg} } case <-ctx.Done(): return @@ -137,7 +181,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { } } -func (d *Discovery) refresh() (tg *config.TargetGroup, err error) { +func (d *Discovery) refresh() (tg *targetgroup.Group, err error) { t0 := time.Now() defer func() { refreshDuration.Observe(time.Since(t0).Seconds()) @@ -147,7 +191,7 @@ func (d *Discovery) refresh() (tg *config.TargetGroup, err error) { }() var endpoint = fmt.Sprintf("https://%s:%d/v%d/discover", d.sdConfig.Endpoint, d.sdConfig.Port, d.sdConfig.Version) - tg = &config.TargetGroup{ + tg = &targetgroup.Group{ Source: endpoint, } diff --git a/discovery/triton/triton_test.go b/discovery/triton/triton_test.go index 86c59fee8..faccfe5e3 100644 --- a/discovery/triton/triton_test.go +++ b/discovery/triton/triton_test.go @@ -27,11 +27,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" + "github.com/prometheus/prometheus/util/config" ) var ( - conf = config.TritonSDConfig{ + conf = SDConfig{ Account: "testAccount", DNSSuffix: "triton.example.com", Endpoint: "127.0.0.1", @@ -40,7 +41,7 @@ var ( RefreshInterval: 1, TLSConfig: config.TLSConfig{InsecureSkipVerify: true}, } - badconf = config.TritonSDConfig{ + badconf = SDConfig{ Account: "badTestAccount", DNSSuffix: "bad.triton.example.com", Endpoint: "127.0.0.1", @@ -78,7 +79,7 @@ func TestTritonSDNewBadConfig(t *testing.T) { func TestTritonSDRun(t *testing.T) { var ( td, err = New(nil, &conf) - ch = make(chan []*config.TargetGroup) + ch = make(chan []*targetgroup.Group) ctx, cancel = context.WithCancel(context.Background()) ) diff --git a/discovery/zookeeper/zookeeper.go b/discovery/zookeeper/zookeeper.go index 66c5ca228..b76d07b55 100644 --- a/discovery/zookeeper/zookeeper.go +++ b/discovery/zookeeper/zookeeper.go @@ -19,23 +19,106 @@ import ( "fmt" "net" "strconv" + "strings" "time" "github.com/go-kit/kit/log" "github.com/prometheus/common/model" "github.com/samuel/go-zookeeper/zk" - "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/util/strutil" "github.com/prometheus/prometheus/util/treecache" + yaml_util "github.com/prometheus/prometheus/util/yaml" ) +var ( + // DefaultServersetSDConfig is the default Serverset SD configuration. + DefaultServersetSDConfig = ServersetSDConfig{ + Timeout: model.Duration(10 * time.Second), + } + // DefaultNerveSDConfig is the default Nerve SD configuration. + DefaultNerveSDConfig = NerveSDConfig{ + Timeout: model.Duration(10 * time.Second), + } +) + +// ServersetSDConfig is the configuration for Twitter serversets in Zookeeper based discovery. +type ServersetSDConfig struct { + Servers []string `yaml:"servers"` + Paths []string `yaml:"paths"` + Timeout model.Duration `yaml:"timeout,omitempty"` + + // Catches all undefined fields and must be empty after parsing. + XXX map[string]interface{} `yaml:",inline"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *ServersetSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = DefaultServersetSDConfig + type plain ServersetSDConfig + err := unmarshal((*plain)(c)) + if err != nil { + return err + } + if err := yaml_util.CheckOverflow(c.XXX, "serverset_sd_config"); err != nil { + return err + } + if len(c.Servers) == 0 { + return fmt.Errorf("serverset SD config must contain at least one Zookeeper server") + } + if len(c.Paths) == 0 { + return fmt.Errorf("serverset SD config must contain at least one path") + } + for _, path := range c.Paths { + if !strings.HasPrefix(path, "/") { + return fmt.Errorf("serverset SD config paths must begin with '/': %s", path) + } + } + return nil +} + +// NerveSDConfig is the configuration for AirBnB's Nerve in Zookeeper based discovery. +type NerveSDConfig struct { + Servers []string `yaml:"servers"` + Paths []string `yaml:"paths"` + Timeout model.Duration `yaml:"timeout,omitempty"` + + // Catches all undefined fields and must be empty after parsing. + XXX map[string]interface{} `yaml:",inline"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *NerveSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = DefaultNerveSDConfig + type plain NerveSDConfig + err := unmarshal((*plain)(c)) + if err != nil { + return err + } + if err := yaml_util.CheckOverflow(c.XXX, "nerve_sd_config"); err != nil { + return err + } + if len(c.Servers) == 0 { + return fmt.Errorf("nerve SD config must contain at least one Zookeeper server") + } + if len(c.Paths) == 0 { + return fmt.Errorf("nerve SD config must contain at least one path") + } + for _, path := range c.Paths { + if !strings.HasPrefix(path, "/") { + return fmt.Errorf("nerve SD config paths must begin with '/': %s", path) + } + } + return nil +} + // Discovery implements the TargetProvider interface for discovering // targets from Zookeeper. type Discovery struct { conn *zk.Conn - sources map[string]*config.TargetGroup + sources map[string]*targetgroup.Group updates chan treecache.ZookeeperTreeCacheEvent treeCaches []*treecache.ZookeeperTreeCache @@ -45,12 +128,12 @@ type Discovery struct { } // NewNerveDiscovery returns a new Discovery for the given Nerve config. -func NewNerveDiscovery(conf *config.NerveSDConfig, logger log.Logger) *Discovery { +func NewNerveDiscovery(conf *NerveSDConfig, logger log.Logger) *Discovery { return NewDiscovery(conf.Servers, time.Duration(conf.Timeout), conf.Paths, logger, parseNerveMember) } // NewServersetDiscovery returns a new Discovery for the given serverset config. -func NewServersetDiscovery(conf *config.ServersetSDConfig, logger log.Logger) *Discovery { +func NewServersetDiscovery(conf *ServersetSDConfig, logger log.Logger) *Discovery { return NewDiscovery(conf.Servers, time.Duration(conf.Timeout), conf.Paths, logger, parseServersetMember) } @@ -76,7 +159,7 @@ func NewDiscovery( sd := &Discovery{ conn: conn, updates: updates, - sources: map[string]*config.TargetGroup{}, + sources: map[string]*targetgroup.Group{}, parse: pf, logger: logger, } @@ -87,7 +170,7 @@ func NewDiscovery( } // Run implements the TargetProvider interface. -func (d *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { +func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { defer func() { for _, tc := range d.treeCaches { tc.Stop() @@ -103,7 +186,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { case <-ctx.Done(): return case event := <-d.updates: - tg := &config.TargetGroup{ + tg := &targetgroup.Group{ Source: event.Path, } if event.Data != nil { @@ -120,7 +203,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { select { case <-ctx.Done(): return - case ch <- []*config.TargetGroup{tg}: + case ch <- []*targetgroup.Group{tg}: } } } diff --git a/notifier/notifier.go b/notifier/notifier.go index d2c14b765..dfd858f43 100644 --- a/notifier/notifier.go +++ b/notifier/notifier.go @@ -35,6 +35,7 @@ import ( "golang.org/x/net/context/ctxhttp" "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/relabel" "github.com/prometheus/prometheus/util/httputil" @@ -516,7 +517,7 @@ func newAlertmanagerSet(cfg *config.AlertmanagerConfig, logger log.Logger) (*ale // Sync extracts a deduplicated set of Alertmanager endpoints from a list // of target groups definitions. -func (s *alertmanagerSet) Sync(tgs []*config.TargetGroup) { +func (s *alertmanagerSet) Sync(tgs []*targetgroup.Group) { all := []alertmanager{} for _, tg := range tgs { @@ -555,7 +556,7 @@ func postPath(pre string) string { // alertmanagersFromGroup extracts a list of alertmanagers from a target group and an associcated // AlertmanagerConfig. -func alertmanagerFromGroup(tg *config.TargetGroup, cfg *config.AlertmanagerConfig) ([]alertmanager, error) { +func alertmanagerFromGroup(tg *targetgroup.Group, cfg *config.AlertmanagerConfig) ([]alertmanager, error) { var res []alertmanager for _, tlset := range tg.Targets { diff --git a/notifier/notifier_test.go b/notifier/notifier_test.go index a20452189..bff958f3b 100644 --- a/notifier/notifier_test.go +++ b/notifier/notifier_test.go @@ -29,7 +29,9 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/pkg/labels" + config_util "github.com/prometheus/prometheus/util/config" "github.com/prometheus/prometheus/util/httputil" ) @@ -165,8 +167,8 @@ func TestHandlerSendAll(t *testing.T) { h := New(&Options{}, nil) - authClient, _ := httputil.NewClientFromConfig(config.HTTPClientConfig{ - BasicAuth: &config.BasicAuth{ + authClient, _ := httputil.NewClientFromConfig(config_util.HTTPClientConfig{ + BasicAuth: &config_util.BasicAuth{ Username: "prometheus", Password: "testing_password", }, @@ -440,8 +442,8 @@ func TestLabelSetNotReused(t *testing.T) { } } -func makeInputTargetGroup() *config.TargetGroup { - return &config.TargetGroup{ +func makeInputTargetGroup() *targetgroup.Group { + return &targetgroup.Group{ Targets: []model.LabelSet{ { model.AddressLabel: model.LabelValue("1.1.1.1:9090"), diff --git a/retrieval/helpers_test.go b/retrieval/helpers_test.go index 85e63cc4e..6d3d92ed6 100644 --- a/retrieval/helpers_test.go +++ b/retrieval/helpers_test.go @@ -14,7 +14,7 @@ package retrieval import ( - "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage" ) @@ -74,10 +74,10 @@ func (a *collectResultAppender) Rollback() error { return nil } // of TargetGroups through the update channel. type fakeTargetProvider struct { sources []string - update chan *config.TargetGroup + update chan *targetgroup.Group } -func (tp *fakeTargetProvider) Run(ch chan<- config.TargetGroup, done <-chan struct{}) { +func (tp *fakeTargetProvider) Run(ch chan<- targetgroup.Group, done <-chan struct{}) { defer close(ch) for { select { diff --git a/retrieval/manager.go b/retrieval/manager.go index 7d9de9445..adf936022 100644 --- a/retrieval/manager.go +++ b/retrieval/manager.go @@ -20,6 +20,7 @@ import ( "github.com/go-kit/kit/log/level" "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/storage" ) @@ -53,7 +54,7 @@ type ScrapeManager struct { } // Run starts background processing to handle target updates and reload the scraping loops. -func (m *ScrapeManager) Run(tsets <-chan map[string][]*config.TargetGroup) error { +func (m *ScrapeManager) Run(tsets <-chan map[string][]*targetgroup.Group) error { level.Info(m.logger).Log("msg", "Starting scrape manager...") for { @@ -126,7 +127,7 @@ func (m *ScrapeManager) Targets() []*Target { return <-targets } -func (m *ScrapeManager) reload(t map[string][]*config.TargetGroup) error { +func (m *ScrapeManager) reload(t map[string][]*targetgroup.Group) error { for tsetName, tgroup := range t { scrapeConfig, ok := m.scrapeConfigs[tsetName] if !ok { diff --git a/retrieval/scrape.go b/retrieval/scrape.go index d07663e9e..8d1173cd5 100644 --- a/retrieval/scrape.go +++ b/retrieval/scrape.go @@ -34,6 +34,7 @@ import ( "golang.org/x/net/context/ctxhttp" "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/pool" "github.com/prometheus/prometheus/pkg/relabel" @@ -245,7 +246,7 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) { // Sync converts target groups into actual scrape targets and synchronizes // the currently running scraper with the resulting set. -func (sp *scrapePool) Sync(tgs []*config.TargetGroup) { +func (sp *scrapePool) Sync(tgs []*targetgroup.Group) { start := time.Now() var all []*Target diff --git a/retrieval/target.go b/retrieval/target.go index 5a23f578b..900ddb632 100644 --- a/retrieval/target.go +++ b/retrieval/target.go @@ -26,6 +26,7 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/relabel" "github.com/prometheus/prometheus/pkg/value" @@ -341,7 +342,7 @@ func populateLabels(lset labels.Labels, cfg *config.ScrapeConfig) (res, orig lab } // targetsFromGroup builds targets based on the given TargetGroup and config. -func targetsFromGroup(tg *config.TargetGroup, cfg *config.ScrapeConfig) ([]*Target, error) { +func targetsFromGroup(tg *targetgroup.Group, cfg *config.ScrapeConfig) ([]*Target, error) { targets := make([]*Target, 0, len(tg.Targets)) for i, tlset := range tg.Targets { diff --git a/retrieval/target_test.go b/retrieval/target_test.go index 7e53fe1ed..eebacfb1c 100644 --- a/retrieval/target_test.go +++ b/retrieval/target_test.go @@ -28,8 +28,8 @@ import ( "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/pkg/labels" + config_util "github.com/prometheus/prometheus/util/config" "github.com/prometheus/prometheus/util/httputil" ) @@ -148,7 +148,7 @@ func TestNewHTTPBearerToken(t *testing.T) { ) defer server.Close() - cfg := config.HTTPClientConfig{ + cfg := config_util.HTTPClientConfig{ BearerToken: "1234", } c, err := httputil.NewClientFromConfig(cfg, "test") @@ -175,7 +175,7 @@ func TestNewHTTPBearerTokenFile(t *testing.T) { ) defer server.Close() - cfg := config.HTTPClientConfig{ + cfg := config_util.HTTPClientConfig{ BearerTokenFile: "testdata/bearertoken.txt", } c, err := httputil.NewClientFromConfig(cfg, "test") @@ -201,8 +201,8 @@ func TestNewHTTPBasicAuth(t *testing.T) { ) defer server.Close() - cfg := config.HTTPClientConfig{ - BasicAuth: &config.BasicAuth{ + cfg := config_util.HTTPClientConfig{ + BasicAuth: &config_util.BasicAuth{ Username: "user", Password: "password123", }, @@ -230,8 +230,8 @@ func TestNewHTTPCACert(t *testing.T) { server.StartTLS() defer server.Close() - cfg := config.HTTPClientConfig{ - TLSConfig: config.TLSConfig{ + cfg := config_util.HTTPClientConfig{ + TLSConfig: config_util.TLSConfig{ CAFile: caCertPath, }, } @@ -262,8 +262,8 @@ func TestNewHTTPClientCert(t *testing.T) { server.StartTLS() defer server.Close() - cfg := config.HTTPClientConfig{ - TLSConfig: config.TLSConfig{ + cfg := config_util.HTTPClientConfig{ + TLSConfig: config_util.TLSConfig{ CAFile: caCertPath, CertFile: "testdata/client.cer", KeyFile: "testdata/client.key", @@ -292,8 +292,8 @@ func TestNewHTTPWithServerName(t *testing.T) { server.StartTLS() defer server.Close() - cfg := config.HTTPClientConfig{ - TLSConfig: config.TLSConfig{ + cfg := config_util.HTTPClientConfig{ + TLSConfig: config_util.TLSConfig{ CAFile: caCertPath, ServerName: "prometheus.rocks", }, @@ -321,8 +321,8 @@ func TestNewHTTPWithBadServerName(t *testing.T) { server.StartTLS() defer server.Close() - cfg := config.HTTPClientConfig{ - TLSConfig: config.TLSConfig{ + cfg := config_util.HTTPClientConfig{ + TLSConfig: config_util.TLSConfig{ CAFile: caCertPath, ServerName: "badname", }, @@ -359,8 +359,8 @@ func newTLSConfig(certName string, t *testing.T) *tls.Config { } func TestNewClientWithBadTLSConfig(t *testing.T) { - cfg := config.HTTPClientConfig{ - TLSConfig: config.TLSConfig{ + cfg := config_util.HTTPClientConfig{ + TLSConfig: config_util.TLSConfig{ CAFile: "testdata/nonexistent_ca.cer", CertFile: "testdata/nonexistent_client.cer", KeyFile: "testdata/nonexistent_client.key", diff --git a/storage/remote/client.go b/storage/remote/client.go index 8da4d8452..ed5591ffe 100644 --- a/storage/remote/client.go +++ b/storage/remote/client.go @@ -28,8 +28,8 @@ import ( "github.com/prometheus/common/model" "golang.org/x/net/context/ctxhttp" - "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/prompb" + config_util "github.com/prometheus/prometheus/util/config" "github.com/prometheus/prometheus/util/httputil" ) @@ -38,16 +38,16 @@ const maxErrMsgLen = 256 // Client allows reading and writing from/to a remote HTTP endpoint. type Client struct { index int // Used to differentiate clients in metrics. - url *config.URL + url *config_util.URL client *http.Client timeout time.Duration } // ClientConfig configures a Client. type ClientConfig struct { - URL *config.URL + URL *config_util.URL Timeout model.Duration - HTTPClientConfig config.HTTPClientConfig + HTTPClientConfig config_util.HTTPClientConfig } // NewClient creates a new Client. diff --git a/storage/remote/client_test.go b/storage/remote/client_test.go index 2b5893d6d..f04548996 100644 --- a/storage/remote/client_test.go +++ b/storage/remote/client_test.go @@ -24,8 +24,8 @@ import ( "time" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/prompb" + config_util "github.com/prometheus/prometheus/util/config" ) var longErrMessage = strings.Repeat("error message", maxErrMsgLen) @@ -66,7 +66,7 @@ func TestStoreHTTPErrorHandling(t *testing.T) { } c, err := NewClient(0, &ClientConfig{ - URL: &config.URL{URL: serverURL}, + URL: &config_util.URL{URL: serverURL}, Timeout: model.Duration(time.Second), }) if err != nil { diff --git a/util/config/config.go b/util/config/config.go new file mode 100644 index 000000000..df81dd411 --- /dev/null +++ b/util/config/config.go @@ -0,0 +1,138 @@ +// Copyright 2013 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + "net/url" + + yaml_util "github.com/prometheus/prometheus/util/yaml" +) + +// Secret special type for storing secrets. +type Secret string + +// UnmarshalYAML implements the yaml.Unmarshaler interface for Secrets. +func (s *Secret) UnmarshalYAML(unmarshal func(interface{}) error) error { + type plain Secret + return unmarshal((*plain)(s)) +} + +// MarshalYAML implements the yaml.Marshaler interface for Secrets. +func (s Secret) MarshalYAML() (interface{}, error) { + if s != "" { + return "", nil + } + return nil, nil +} + +// TLSConfig configures the options for TLS connections. +type TLSConfig struct { + // The CA cert to use for the targets. + CAFile string `yaml:"ca_file,omitempty"` + // The client cert file for the targets. + CertFile string `yaml:"cert_file,omitempty"` + // The client key file for the targets. + KeyFile string `yaml:"key_file,omitempty"` + // Used to verify the hostname for the targets. + ServerName string `yaml:"server_name,omitempty"` + // Disable target certificate validation. + InsecureSkipVerify bool `yaml:"insecure_skip_verify"` + + // Catches all undefined fields and must be empty after parsing. + XXX map[string]interface{} `yaml:",inline"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *TLSConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + type plain TLSConfig + if err := unmarshal((*plain)(c)); err != nil { + return err + } + return yaml_util.CheckOverflow(c.XXX, "TLS config") +} + +// BasicAuth contains basic HTTP authentication credentials. +type BasicAuth struct { + Username string `yaml:"username"` + Password Secret `yaml:"password"` + + // Catches all undefined fields and must be empty after parsing. + XXX map[string]interface{} `yaml:",inline"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (a *BasicAuth) UnmarshalYAML(unmarshal func(interface{}) error) error { + type plain BasicAuth + err := unmarshal((*plain)(a)) + if err != nil { + return err + } + return yaml_util.CheckOverflow(a.XXX, "basic_auth") +} + +// URL is a custom URL type that allows validation at configuration load time. +type URL struct { + *url.URL +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface for URLs. +func (u *URL) UnmarshalYAML(unmarshal func(interface{}) error) error { + var s string + if err := unmarshal(&s); err != nil { + return err + } + + urlp, err := url.Parse(s) + if err != nil { + return err + } + u.URL = urlp + return nil +} + +// MarshalYAML implements the yaml.Marshaler interface for URLs. +func (u URL) MarshalYAML() (interface{}, error) { + if u.URL != nil { + return u.String(), nil + } + return nil, nil +} + +// HTTPClientConfig configures an HTTP client. +type HTTPClientConfig struct { + // The HTTP basic authentication credentials for the targets. + BasicAuth *BasicAuth `yaml:"basic_auth,omitempty"` + // The bearer token for the targets. + BearerToken Secret `yaml:"bearer_token,omitempty"` + // The bearer token file for the targets. + BearerTokenFile string `yaml:"bearer_token_file,omitempty"` + // HTTP proxy server to use to connect to the targets. + ProxyURL URL `yaml:"proxy_url,omitempty"` + // TLSConfig to use to connect to the targets. + TLSConfig TLSConfig `yaml:"tls_config,omitempty"` + + // Catches all undefined fields and must be empty after parsing. + XXX map[string]interface{} `yaml:",inline"` +} + +func (c *HTTPClientConfig) Validate() error { + if len(c.BearerToken) > 0 && len(c.BearerTokenFile) > 0 { + return fmt.Errorf("at most one of bearer_token & bearer_token_file must be configured") + } + if c.BasicAuth != nil && (len(c.BearerToken) > 0 || len(c.BearerTokenFile) > 0) { + return fmt.Errorf("at most one of basic_auth, bearer_token & bearer_token_file must be configured") + } + return nil +} diff --git a/util/httputil/client.go b/util/httputil/client.go index 99469c37b..9cbddb875 100644 --- a/util/httputil/client.go +++ b/util/httputil/client.go @@ -23,7 +23,7 @@ import ( "time" "github.com/mwitkow/go-conntrack" - "github.com/prometheus/prometheus/config" + config_util "github.com/prometheus/prometheus/util/config" ) // NewClient returns a http.Client using the specified http.RoundTripper. @@ -33,7 +33,7 @@ func newClient(rt http.RoundTripper) *http.Client { // NewClientFromConfig returns a new HTTP client configured for the // given config.HTTPClientConfig. The name is used as go-conntrack metric label. -func NewClientFromConfig(cfg config.HTTPClientConfig, name string) (*http.Client, error) { +func NewClientFromConfig(cfg config_util.HTTPClientConfig, name string) (*http.Client, error) { tlsConfig, err := NewTLSConfig(cfg.TLSConfig) if err != nil { return nil, err @@ -134,8 +134,8 @@ func cloneRequest(r *http.Request) *http.Request { return r2 } -// NewTLSConfig creates a new tls.Config from the given config.TLSConfig. -func NewTLSConfig(cfg config.TLSConfig) (*tls.Config, error) { +// NewTLSConfig creates a new tls.Config from the given config_util.TLSConfig. +func NewTLSConfig(cfg config_util.TLSConfig) (*tls.Config, error) { tlsConfig := &tls.Config{InsecureSkipVerify: cfg.InsecureSkipVerify} // If a CA cert is provided then let's read it in so we can validate the diff --git a/util/httputil/client_test.go b/util/httputil/client_test.go index 617b02602..a537b9da0 100644 --- a/util/httputil/client_test.go +++ b/util/httputil/client_test.go @@ -24,7 +24,7 @@ import ( "strings" "testing" - "github.com/prometheus/prometheus/config" + config_util "github.com/prometheus/prometheus/util/config" "github.com/prometheus/prometheus/util/testutil" ) @@ -77,12 +77,12 @@ func newTestServer(handler func(w http.ResponseWriter, r *http.Request)) (*httpt func TestNewClientFromConfig(t *testing.T) { var newClientValidConfig = []struct { - clientConfig config.HTTPClientConfig + clientConfig config_util.HTTPClientConfig handler func(w http.ResponseWriter, r *http.Request) }{ { - clientConfig: config.HTTPClientConfig{ - TLSConfig: config.TLSConfig{ + clientConfig: config_util.HTTPClientConfig{ + TLSConfig: config_util.TLSConfig{ CAFile: "", CertFile: BarneyCertificatePath, KeyFile: BarneyKeyNoPassPath, @@ -93,8 +93,8 @@ func TestNewClientFromConfig(t *testing.T) { fmt.Fprint(w, ExpectedMessage) }, }, { - clientConfig: config.HTTPClientConfig{ - TLSConfig: config.TLSConfig{ + clientConfig: config_util.HTTPClientConfig{ + TLSConfig: config_util.TLSConfig{ CAFile: TLSCAChainPath, CertFile: BarneyCertificatePath, KeyFile: BarneyKeyNoPassPath, @@ -105,9 +105,9 @@ func TestNewClientFromConfig(t *testing.T) { fmt.Fprint(w, ExpectedMessage) }, }, { - clientConfig: config.HTTPClientConfig{ + clientConfig: config_util.HTTPClientConfig{ BearerToken: BearerToken, - TLSConfig: config.TLSConfig{ + TLSConfig: config_util.TLSConfig{ CAFile: TLSCAChainPath, CertFile: BarneyCertificatePath, KeyFile: BarneyKeyNoPassPath, @@ -124,9 +124,9 @@ func TestNewClientFromConfig(t *testing.T) { } }, }, { - clientConfig: config.HTTPClientConfig{ + clientConfig: config_util.HTTPClientConfig{ BearerTokenFile: BearerTokenFile, - TLSConfig: config.TLSConfig{ + TLSConfig: config_util.TLSConfig{ CAFile: TLSCAChainPath, CertFile: BarneyCertificatePath, KeyFile: BarneyKeyNoPassPath, @@ -143,12 +143,12 @@ func TestNewClientFromConfig(t *testing.T) { } }, }, { - clientConfig: config.HTTPClientConfig{ - BasicAuth: &config.BasicAuth{ + clientConfig: config_util.HTTPClientConfig{ + BasicAuth: &config_util.BasicAuth{ Username: ExpectedUsername, Password: ExpectedPassword, }, - TLSConfig: config.TLSConfig{ + TLSConfig: config_util.TLSConfig{ CAFile: TLSCAChainPath, CertFile: BarneyCertificatePath, KeyFile: BarneyKeyNoPassPath, @@ -205,12 +205,12 @@ func TestNewClientFromConfig(t *testing.T) { func TestNewClientFromInvalidConfig(t *testing.T) { var newClientInvalidConfig = []struct { - clientConfig config.HTTPClientConfig + clientConfig config_util.HTTPClientConfig errorMsg string }{ { - clientConfig: config.HTTPClientConfig{ - TLSConfig: config.TLSConfig{ + clientConfig: config_util.HTTPClientConfig{ + TLSConfig: config_util.TLSConfig{ CAFile: MissingCA, CertFile: "", KeyFile: "", @@ -219,9 +219,9 @@ func TestNewClientFromInvalidConfig(t *testing.T) { }, errorMsg: fmt.Sprintf("unable to use specified CA cert %s:", MissingCA), }, { - clientConfig: config.HTTPClientConfig{ + clientConfig: config_util.HTTPClientConfig{ BearerTokenFile: MissingBearerTokenFile, - TLSConfig: config.TLSConfig{ + TLSConfig: config_util.TLSConfig{ CAFile: TLSCAChainPath, CertFile: BarneyCertificatePath, KeyFile: BarneyKeyNoPassPath, @@ -307,7 +307,7 @@ func TestBasicAuthRoundTripper(t *testing.T) { } func TestTLSConfig(t *testing.T) { - configTLSConfig := config.TLSConfig{ + configTLSConfig := config_util.TLSConfig{ CAFile: TLSCAChainPath, CertFile: BarneyCertificatePath, KeyFile: BarneyKeyNoPassPath, @@ -346,7 +346,7 @@ func TestTLSConfig(t *testing.T) { } func TestTLSConfigEmpty(t *testing.T) { - configTLSConfig := config.TLSConfig{ + configTLSConfig := config_util.TLSConfig{ CAFile: "", CertFile: "", KeyFile: "", @@ -369,11 +369,11 @@ func TestTLSConfigEmpty(t *testing.T) { func TestTLSConfigInvalidCA(t *testing.T) { var invalidTLSConfig = []struct { - configTLSConfig config.TLSConfig + configTLSConfig config_util.TLSConfig errorMessage string }{ { - configTLSConfig: config.TLSConfig{ + configTLSConfig: config_util.TLSConfig{ CAFile: MissingCA, CertFile: "", KeyFile: "", @@ -381,7 +381,7 @@ func TestTLSConfigInvalidCA(t *testing.T) { InsecureSkipVerify: false}, errorMessage: fmt.Sprintf("unable to use specified CA cert %s:", MissingCA), }, { - configTLSConfig: config.TLSConfig{ + configTLSConfig: config_util.TLSConfig{ CAFile: "", CertFile: MissingCert, KeyFile: BarneyKeyNoPassPath, @@ -389,7 +389,7 @@ func TestTLSConfigInvalidCA(t *testing.T) { InsecureSkipVerify: false}, errorMessage: fmt.Sprintf("unable to use specified client cert (%s) & key (%s):", MissingCert, BarneyKeyNoPassPath), }, { - configTLSConfig: config.TLSConfig{ + configTLSConfig: config_util.TLSConfig{ CAFile: "", CertFile: BarneyCertificatePath, KeyFile: MissingKey, diff --git a/util/yaml/yaml.go b/util/yaml/yaml.go new file mode 100644 index 000000000..734efa1df --- /dev/null +++ b/util/yaml/yaml.go @@ -0,0 +1,31 @@ +// Copyright 2013 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package yaml + +import ( + "fmt" + "strings" +) + +// CheckOverflow checks for unknown config params and raises an error if found +func CheckOverflow(m map[string]interface{}, ctx string) error { + if len(m) > 0 { + var keys []string + for k := range m { + keys = append(keys, k) + } + return fmt.Errorf("unknown fields in %s: %s", ctx, strings.Join(keys, ", ")) + } + return nil +}