From 78411d5e8b803b590ba10bb563caab52292a11a2 Mon Sep 17 00:00:00 2001 From: Paulin Todev Date: Tue, 23 Jan 2024 15:53:55 +0000 Subject: [PATCH] SD Managers taking over responsibility for registration of debug metrics (#13375) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SD Managers take over responsibility for SD metrics registration --------- Signed-off-by: Paulin Todev Signed-off-by: Björn Rabenstein Co-authored-by: Björn Rabenstein --- cmd/prometheus/main.go | 15 ++- cmd/promtool/sd.go | 17 +++- discovery/aws/ec2.go | 28 ++++-- discovery/aws/lightsail.go | 29 ++++-- discovery/aws/metrics_ec2.go | 32 ++++++ discovery/aws/metrics_lightsail.go | 32 ++++++ discovery/azure/azure.go | 67 +++++++------ discovery/azure/metrics.go | 64 ++++++++++++ discovery/consul/consul.go | 89 ++++++----------- discovery/consul/consul_test.go | 37 +++++-- discovery/consul/metrics.go | 73 ++++++++++++++ discovery/digitalocean/digitalocean.go | 26 +++-- discovery/digitalocean/digitalocean_test.go | 12 ++- discovery/digitalocean/metrics.go | 32 ++++++ discovery/discoverer_metrics_noop.go | 28 ++++++ discovery/discovery.go | 47 +++++++-- discovery/dns/dns.go | 53 +++++----- discovery/dns/dns_test.go | 11 ++- discovery/dns/metrics.go | 66 +++++++++++++ discovery/eureka/eureka.go | 27 +++-- discovery/eureka/eureka_test.go | 13 ++- discovery/eureka/metrics.go | 32 ++++++ discovery/file/file.go | 66 +++++-------- discovery/file/file_test.go | 24 +++-- discovery/file/metrics.go | 76 ++++++++++++++ discovery/gce/gce.go | 26 +++-- discovery/gce/metrics.go | 32 ++++++ discovery/hetzner/hetzner.go | 26 +++-- discovery/hetzner/metrics.go | 32 ++++++ discovery/http/http.go | 45 +++++---- discovery/http/http_test.go | 44 +++++++-- discovery/http/metrics.go | 57 +++++++++++ discovery/ionos/ionos.go | 30 ++++-- discovery/ionos/metrics.go | 32 ++++++ discovery/kubernetes/kubernetes.go | 65 ++++-------- discovery/kubernetes/kubernetes_test.go | 25 +++-- discovery/kubernetes/metrics.go | 75 ++++++++++++++ discovery/legacymanager/manager.go | 12 ++- discovery/legacymanager/manager_test.go | 58 +++++++++-- discovery/linode/linode.go | 37 ++++--- discovery/linode/linode_test.go | 12 ++- discovery/linode/metrics.go | 57 +++++++++++ discovery/manager.go | 26 ++++- discovery/manager_test.go | 98 ++++++++++++++++--- discovery/marathon/marathon.go | 26 +++-- discovery/marathon/marathon_test.go | 26 ++++- discovery/marathon/metrics.go | 32 ++++++ discovery/metrics.go | 2 +- discovery/metrics_refresh.go | 75 ++++++++++++++ discovery/moby/docker.go | 26 +++-- discovery/moby/docker_test.go | 11 ++- discovery/moby/dockerswarm.go | 26 +++-- discovery/moby/metrics_docker.go | 32 ++++++ discovery/moby/metrics_dockerswarm.go | 32 ++++++ discovery/moby/nodes_test.go | 11 ++- discovery/moby/services_test.go | 20 +++- discovery/moby/tasks_test.go | 11 ++- discovery/nomad/metrics.go | 57 +++++++++++ discovery/nomad/nomad.go | 37 ++++--- discovery/nomad/nomad_test.go | 22 ++++- discovery/openstack/metrics.go | 32 ++++++ discovery/openstack/openstack.go | 26 +++-- discovery/ovhcloud/metrics.go | 32 ++++++ discovery/ovhcloud/ovhcloud.go | 26 +++-- discovery/ovhcloud/ovhcloud_test.go | 12 ++- discovery/puppetdb/metrics.go | 32 ++++++ discovery/puppetdb/puppetdb.go | 26 +++-- discovery/puppetdb/puppetdb_test.go | 47 ++++++++- discovery/refresh/refresh.go | 56 +++-------- discovery/refresh/refresh_test.go | 16 ++- discovery/registry.go | 23 +++++ discovery/scaleway/metrics.go | 32 ++++++ discovery/scaleway/scaleway.go | 26 +++-- discovery/triton/metrics.go | 32 ++++++ discovery/triton/triton.go | 26 +++-- discovery/triton/triton_test.go | 43 ++++++-- discovery/uyuni/metrics.go | 32 ++++++ discovery/uyuni/uyuni.go | 26 +++-- discovery/uyuni/uyuni_test.go | 22 ++++- discovery/vultr/metrics.go | 32 ++++++ discovery/vultr/vultr.go | 26 +++-- discovery/vultr/vultr_test.go | 12 ++- discovery/xds/kuma.go | 44 +++------ discovery/xds/kuma_test.go | 16 ++- discovery/xds/metrics.go | 73 ++++++++++++++ discovery/xds/xds.go | 21 +--- discovery/xds/xds_test.go | 91 ++++++++++------- discovery/zookeeper/zookeeper.go | 11 +++ .../examples/custom-sd/adapter-usage/main.go | 13 ++- .../examples/custom-sd/adapter/adapter.go | 4 +- .../custom-sd/adapter/adapter_test.go | 10 +- 91 files changed, 2522 insertions(+), 629 deletions(-) create mode 100644 discovery/aws/metrics_ec2.go create mode 100644 discovery/aws/metrics_lightsail.go create mode 100644 discovery/azure/metrics.go create mode 100644 discovery/consul/metrics.go create mode 100644 discovery/digitalocean/metrics.go create mode 100644 discovery/discoverer_metrics_noop.go create mode 100644 discovery/dns/metrics.go create mode 100644 discovery/eureka/metrics.go create mode 100644 discovery/file/metrics.go create mode 100644 discovery/gce/metrics.go create mode 100644 discovery/hetzner/metrics.go create mode 100644 discovery/http/metrics.go create mode 100644 discovery/ionos/metrics.go create mode 100644 discovery/kubernetes/metrics.go create mode 100644 discovery/linode/metrics.go create mode 100644 discovery/marathon/metrics.go create mode 100644 discovery/metrics_refresh.go create mode 100644 discovery/moby/metrics_docker.go create mode 100644 discovery/moby/metrics_dockerswarm.go create mode 100644 discovery/nomad/metrics.go create mode 100644 discovery/openstack/metrics.go create mode 100644 discovery/ovhcloud/metrics.go create mode 100644 discovery/puppetdb/metrics.go create mode 100644 discovery/scaleway/metrics.go create mode 100644 discovery/triton/metrics.go create mode 100644 discovery/uyuni/metrics.go create mode 100644 discovery/vultr/metrics.go create mode 100644 discovery/xds/metrics.go diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index d73652fba..f7244646e 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -644,9 +644,16 @@ func main() { level.Error(logger).Log("msg", "failed to register Kubernetes client metrics", "err", err) os.Exit(1) } + + sdMetrics, err := discovery.CreateAndRegisterSDMetrics(prometheus.DefaultRegisterer) + if err != nil { + level.Error(logger).Log("msg", "failed to register service discovery metrics", "err", err) + os.Exit(1) + } + if cfg.enableNewSDManager { { - discMgr := discovery.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, discovery.Name("scrape")) + discMgr := discovery.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, sdMetrics, discovery.Name("scrape")) if discMgr == nil { level.Error(logger).Log("msg", "failed to create a discovery manager scrape") os.Exit(1) @@ -655,7 +662,7 @@ func main() { } { - discMgr := discovery.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, discovery.Name("notify")) + discMgr := discovery.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, sdMetrics, discovery.Name("notify")) if discMgr == nil { level.Error(logger).Log("msg", "failed to create a discovery manager notify") os.Exit(1) @@ -664,7 +671,7 @@ func main() { } } else { { - discMgr := legacymanager.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, legacymanager.Name("scrape")) + discMgr := legacymanager.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, sdMetrics, legacymanager.Name("scrape")) if discMgr == nil { level.Error(logger).Log("msg", "failed to create a discovery manager scrape") os.Exit(1) @@ -673,7 +680,7 @@ func main() { } { - discMgr := legacymanager.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, legacymanager.Name("notify")) + discMgr := legacymanager.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, sdMetrics, legacymanager.Name("notify")) if discMgr == nil { level.Error(logger).Log("msg", "failed to create a discovery manager notify") os.Exit(1) diff --git a/cmd/promtool/sd.go b/cmd/promtool/sd.go index 155152e1a..4892743fc 100644 --- a/cmd/promtool/sd.go +++ b/cmd/promtool/sd.go @@ -78,12 +78,25 @@ func CheckSD(sdConfigFiles, sdJobName string, sdTimeout time.Duration, noDefault defer cancel() for _, cfg := range scrapeConfig.ServiceDiscoveryConfigs { - d, err := cfg.NewDiscoverer(discovery.DiscovererOptions{Logger: logger, Registerer: registerer}) + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) + err := metrics.Register() + if err != nil { + fmt.Fprintln(os.Stderr, "Could not register service discovery metrics", err) + return failureExitCode + } + + d, err := cfg.NewDiscoverer(discovery.DiscovererOptions{Logger: logger, Metrics: metrics}) if err != nil { fmt.Fprintln(os.Stderr, "Could not create new discoverer", err) return failureExitCode } - go d.Run(ctx, targetGroupChan) + go func() { + d.Run(ctx, targetGroupChan) + metrics.Unregister() + refreshMetrics.Unregister() + }() } var targetGroups []*targetgroup.Group diff --git a/discovery/aws/ec2.go b/discovery/aws/ec2.go index 40e6e7cb7..aa79fd9c6 100644 --- a/discovery/aws/ec2.go +++ b/discovery/aws/ec2.go @@ -97,12 +97,19 @@ type EC2SDConfig struct { HTTPClientConfig config.HTTPClientConfig `yaml:",inline"` } +// NewDiscovererMetrics implements discovery.Config. +func (*EC2SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &ec2Metrics{ + refreshMetrics: rmi, + } +} + // Name returns the name of the EC2 Config. func (*EC2SDConfig) Name() string { return "ec2" } // NewDiscoverer returns a Discoverer for the EC2 Config. func (c *EC2SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewEC2Discovery(c, opts.Logger, opts.Registerer), nil + return NewEC2Discovery(c, opts.Logger, opts.Metrics) } // UnmarshalYAML implements the yaml.Unmarshaler interface for the EC2 Config. @@ -148,7 +155,12 @@ type EC2Discovery struct { } // NewEC2Discovery returns a new EC2Discovery which periodically refreshes its targets. -func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger, reg prometheus.Registerer) *EC2Discovery { +func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*EC2Discovery, error) { + m, ok := metrics.(*ec2Metrics) + if !ok { + return nil, fmt.Errorf("invalid discovery metrics type") + } + if logger == nil { logger = log.NewNopLogger() } @@ -158,14 +170,14 @@ func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger, reg prometheus.Regist } d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, - Mech: "ec2", - Interval: time.Duration(d.cfg.RefreshInterval), - RefreshF: d.refresh, - Registry: reg, + Logger: logger, + Mech: "ec2", + Interval: time.Duration(d.cfg.RefreshInterval), + RefreshF: d.refresh, + MetricsInstantiator: m.refreshMetrics, }, ) - return d + return d, nil } func (d *EC2Discovery) ec2Client(context.Context) (*ec2.EC2, error) { diff --git a/discovery/aws/lightsail.go b/discovery/aws/lightsail.go index 5382ea015..86b138be5 100644 --- a/discovery/aws/lightsail.go +++ b/discovery/aws/lightsail.go @@ -80,12 +80,19 @@ type LightsailSDConfig struct { HTTPClientConfig config.HTTPClientConfig `yaml:",inline"` } +// NewDiscovererMetrics implements discovery.Config. +func (*LightsailSDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &lightsailMetrics{ + refreshMetrics: rmi, + } +} + // Name returns the name of the Lightsail Config. func (*LightsailSDConfig) Name() string { return "lightsail" } // NewDiscoverer returns a Discoverer for the Lightsail Config. func (c *LightsailSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewLightsailDiscovery(c, opts.Logger, opts.Registerer), nil + return NewLightsailDiscovery(c, opts.Logger, opts.Metrics) } // UnmarshalYAML implements the yaml.Unmarshaler interface for the Lightsail Config. @@ -122,23 +129,29 @@ type LightsailDiscovery struct { } // NewLightsailDiscovery returns a new LightsailDiscovery which periodically refreshes its targets. -func NewLightsailDiscovery(conf *LightsailSDConfig, logger log.Logger, reg prometheus.Registerer) *LightsailDiscovery { +func NewLightsailDiscovery(conf *LightsailSDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*LightsailDiscovery, error) { + m, ok := metrics.(*lightsailMetrics) + if !ok { + return nil, fmt.Errorf("invalid discovery metrics type") + } + if logger == nil { logger = log.NewNopLogger() } + d := &LightsailDiscovery{ cfg: conf, } d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, - Mech: "lightsail", - Interval: time.Duration(d.cfg.RefreshInterval), - RefreshF: d.refresh, - Registry: reg, + Logger: logger, + Mech: "lightsail", + Interval: time.Duration(d.cfg.RefreshInterval), + RefreshF: d.refresh, + MetricsInstantiator: m.refreshMetrics, }, ) - return d + return d, nil } func (d *LightsailDiscovery) lightsailClient() (*lightsail.Lightsail, error) { diff --git a/discovery/aws/metrics_ec2.go b/discovery/aws/metrics_ec2.go new file mode 100644 index 000000000..73c061c3d --- /dev/null +++ b/discovery/aws/metrics_ec2.go @@ -0,0 +1,32 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aws + +import ( + "github.com/prometheus/prometheus/discovery" +) + +type ec2Metrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator +} + +var _ discovery.DiscovererMetrics = (*ec2Metrics)(nil) + +// Register implements discovery.DiscovererMetrics. +func (m *ec2Metrics) Register() error { + return nil +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *ec2Metrics) Unregister() {} diff --git a/discovery/aws/metrics_lightsail.go b/discovery/aws/metrics_lightsail.go new file mode 100644 index 000000000..69593e701 --- /dev/null +++ b/discovery/aws/metrics_lightsail.go @@ -0,0 +1,32 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aws + +import ( + "github.com/prometheus/prometheus/discovery" +) + +type lightsailMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator +} + +var _ discovery.DiscovererMetrics = (*lightsailMetrics)(nil) + +// Register implements discovery.DiscovererMetrics. +func (m *lightsailMetrics) Register() error { + return nil +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *lightsailMetrics) Unregister() {} diff --git a/discovery/azure/azure.go b/discovery/azure/azure.go index 64f02ec7a..e5e6109b2 100644 --- a/discovery/azure/azure.go +++ b/discovery/azure/azure.go @@ -120,12 +120,17 @@ type SDConfig struct { HTTPClientConfig config_util.HTTPClientConfig `yaml:",inline"` } +// NewDiscovererMetrics implements discovery.Config. +func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return newDiscovererMetrics(reg, rmi) +} + // Name returns the name of the Config. func (*SDConfig) Name() string { return "azure" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger, opts.Registerer) + return NewDiscovery(c, opts.Logger, opts.Metrics) } func validateAuthParam(param, name string) error { @@ -168,45 +173,39 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { type Discovery struct { *refresh.Discovery - logger log.Logger - cfg *SDConfig - port int - cache *cache.Cache[string, *armnetwork.Interface] - failuresCount prometheus.Counter - cacheHitCount prometheus.Counter + logger log.Logger + cfg *SDConfig + port int + cache *cache.Cache[string, *armnetwork.Interface] + metrics *azureMetrics } // NewDiscovery returns a new AzureDiscovery which periodically refreshes its targets. -func NewDiscovery(cfg *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) { +func NewDiscovery(cfg *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { + m, ok := metrics.(*azureMetrics) + if !ok { + return nil, fmt.Errorf("invalid discovery metrics type") + } + if logger == nil { logger = log.NewNopLogger() } l := cache.New(cache.AsLRU[string, *armnetwork.Interface](lru.WithCapacity(5000))) d := &Discovery{ - cfg: cfg, - port: cfg.Port, - logger: logger, - cache: l, - failuresCount: prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "prometheus_sd_azure_failures_total", - Help: "Number of Azure service discovery refresh failures.", - }), - cacheHitCount: prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "prometheus_sd_azure_cache_hit_total", - Help: "Number of cache hit during refresh.", - }), + cfg: cfg, + port: cfg.Port, + logger: logger, + cache: l, + metrics: m, } d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, - Mech: "azure", - Interval: time.Duration(cfg.RefreshInterval), - RefreshF: d.refresh, - Registry: reg, - Metrics: []prometheus.Collector{d.failuresCount, d.cacheHitCount}, + Logger: logger, + Mech: "azure", + Interval: time.Duration(cfg.RefreshInterval), + RefreshF: d.refresh, + MetricsInstantiator: m.refreshMetrics, }, ) @@ -333,14 +332,14 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { client, err := createAzureClient(*d.cfg) if err != nil { - d.failuresCount.Inc() + d.metrics.failuresCount.Inc() return nil, fmt.Errorf("could not create Azure client: %w", err) } client.logger = d.logger machines, err := client.getVMs(ctx, d.cfg.ResourceGroup) if err != nil { - d.failuresCount.Inc() + d.metrics.failuresCount.Inc() return nil, fmt.Errorf("could not get virtual machines: %w", err) } @@ -349,14 +348,14 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { // Load the vms managed by scale sets. scaleSets, err := client.getScaleSets(ctx, d.cfg.ResourceGroup) if err != nil { - d.failuresCount.Inc() + d.metrics.failuresCount.Inc() return nil, fmt.Errorf("could not get virtual machine scale sets: %w", err) } for _, scaleSet := range scaleSets { scaleSetVms, err := client.getScaleSetVMs(ctx, scaleSet) if err != nil { - d.failuresCount.Inc() + d.metrics.failuresCount.Inc() return nil, fmt.Errorf("could not get virtual machine scale set vms: %w", err) } machines = append(machines, scaleSetVms...) @@ -407,7 +406,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { var networkInterface *armnetwork.Interface if v, ok := d.getFromCache(nicID); ok { networkInterface = v - d.cacheHitCount.Add(1) + d.metrics.cacheHitCount.Add(1) } else { if vm.ScaleSet == "" { networkInterface, err = client.getVMNetworkInterfaceByID(ctx, nicID) @@ -480,7 +479,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { var tg targetgroup.Group for tgt := range ch { if tgt.err != nil { - d.failuresCount.Inc() + d.metrics.failuresCount.Inc() return nil, fmt.Errorf("unable to complete Azure service discovery: %w", tgt.err) } if tgt.labelSet != nil { diff --git a/discovery/azure/metrics.go b/discovery/azure/metrics.go new file mode 100644 index 000000000..3e3dbdbfb --- /dev/null +++ b/discovery/azure/metrics.go @@ -0,0 +1,64 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azure + +import ( + "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/prometheus/discovery" +) + +var _ discovery.DiscovererMetrics = (*azureMetrics)(nil) + +type azureMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator + + failuresCount prometheus.Counter + cacheHitCount prometheus.Counter + + metricRegisterer discovery.MetricRegisterer +} + +func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + m := &azureMetrics{ + refreshMetrics: rmi, + failuresCount: prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "prometheus_sd_azure_failures_total", + Help: "Number of Azure service discovery refresh failures.", + }), + cacheHitCount: prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "prometheus_sd_azure_cache_hit_total", + Help: "Number of cache hit during refresh.", + }), + } + + m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{ + m.failuresCount, + m.cacheHitCount, + }) + + return m +} + +// Register implements discovery.DiscovererMetrics. +func (m *azureMetrics) Register() error { + return m.metricRegisterer.RegisterMetrics() +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *azureMetrics) Unregister() { + m.metricRegisterer.UnregisterMetrics() +} diff --git a/discovery/consul/consul.go b/discovery/consul/consul.go index 50f171a78..40eed7697 100644 --- a/discovery/consul/consul.go +++ b/discovery/consul/consul.go @@ -119,12 +119,17 @@ type SDConfig struct { HTTPClientConfig config.HTTPClientConfig `yaml:",inline"` } +// NewDiscovererMetrics implements discovery.Config. +func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return newDiscovererMetrics(reg, rmi) +} + // Name returns the name of the Config. func (*SDConfig) Name() string { return "consul" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger, opts.Registerer) + return NewDiscovery(c, opts.Logger, opts.Metrics) } // SetDirectory joins any relative file paths with dir. @@ -161,27 +166,28 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { // Discovery retrieves target information from a Consul server // and updates them via watches. type Discovery struct { - client *consul.Client - clientDatacenter string - clientNamespace string - clientPartition string - tagSeparator string - watchedServices []string // Set of services which will be discovered. - watchedTags []string // Tags used to filter instances of a service. - watchedNodeMeta map[string]string - allowStale bool - refreshInterval time.Duration - finalizer func() - logger log.Logger - rpcFailuresCount prometheus.Counter - rpcDuration *prometheus.SummaryVec - servicesRPCDuration prometheus.Observer - serviceRPCDuration prometheus.Observer - metricRegisterer discovery.MetricRegisterer + client *consul.Client + clientDatacenter string + clientNamespace string + clientPartition string + tagSeparator string + watchedServices []string // Set of services which will be discovered. + watchedTags []string // Tags used to filter instances of a service. + watchedNodeMeta map[string]string + allowStale bool + refreshInterval time.Duration + finalizer func() + logger log.Logger + metrics *consulMetrics } // NewDiscovery returns a new Discovery for the given config. -func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) { +func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { + m, ok := metrics.(*consulMetrics) + if !ok { + return nil, fmt.Errorf("invalid discovery metrics type") + } + if logger == nil { logger = log.NewNopLogger() } @@ -219,35 +225,9 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) clientPartition: conf.Partition, finalizer: wrapper.CloseIdleConnections, logger: logger, - rpcFailuresCount: prometheus.NewCounter( - prometheus.CounterOpts{ - Namespace: namespace, - Name: "sd_consul_rpc_failures_total", - Help: "The number of Consul RPC call failures.", - }), - rpcDuration: prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Namespace: namespace, - Name: "sd_consul_rpc_duration_seconds", - Help: "The duration of a Consul RPC call in seconds.", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, - }, - []string{"endpoint", "call"}, - ), + metrics: m, } - cd.metricRegisterer = discovery.NewMetricRegisterer( - reg, - []prometheus.Collector{ - cd.rpcFailuresCount, - cd.rpcDuration, - }, - ) - - // Initialize metric vectors. - cd.servicesRPCDuration = cd.rpcDuration.WithLabelValues("catalog", "services") - cd.serviceRPCDuration = cd.rpcDuration.WithLabelValues("catalog", "service") - return cd, nil } @@ -303,7 +283,7 @@ func (d *Discovery) getDatacenter() error { info, err := d.client.Agent().Self() if err != nil { level.Error(d.logger).Log("msg", "Error retrieving datacenter name", "err", err) - d.rpcFailuresCount.Inc() + d.metrics.rpcFailuresCount.Inc() return err } @@ -344,13 +324,6 @@ func (d *Discovery) initialize(ctx context.Context) { // Run implements the Discoverer interface. func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { - err := d.metricRegisterer.RegisterMetrics() - if err != nil { - level.Error(d.logger).Log("msg", "Unable to register metrics", "err", err.Error()) - return - } - defer d.metricRegisterer.UnregisterMetrics() - if d.finalizer != nil { defer d.finalizer() } @@ -399,7 +372,7 @@ func (d *Discovery) watchServices(ctx context.Context, ch chan<- []*targetgroup. t0 := time.Now() srvs, meta, err := catalog.Services(opts.WithContext(ctx)) elapsed := time.Since(t0) - d.servicesRPCDuration.Observe(elapsed.Seconds()) + d.metrics.servicesRPCDuration.Observe(elapsed.Seconds()) // Check the context before in order to exit early. select { @@ -410,7 +383,7 @@ func (d *Discovery) watchServices(ctx context.Context, ch chan<- []*targetgroup. if err != nil { level.Error(d.logger).Log("msg", "Error refreshing service list", "err", err) - d.rpcFailuresCount.Inc() + d.metrics.rpcFailuresCount.Inc() time.Sleep(retryInterval) return } @@ -490,8 +463,8 @@ func (d *Discovery) watchService(ctx context.Context, ch chan<- []*targetgroup.G }, tagSeparator: d.tagSeparator, logger: d.logger, - rpcFailuresCount: d.rpcFailuresCount, - serviceRPCDuration: d.serviceRPCDuration, + rpcFailuresCount: d.metrics.rpcFailuresCount, + serviceRPCDuration: d.metrics.serviceRPCDuration, } go func() { diff --git a/discovery/consul/consul_test.go b/discovery/consul/consul_test.go index 97cb8fbc9..040040ae9 100644 --- a/discovery/consul/consul_test.go +++ b/discovery/consul/consul_test.go @@ -29,6 +29,7 @@ import ( "go.uber.org/goleak" "gopkg.in/yaml.v2" + "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/targetgroup" ) @@ -36,11 +37,25 @@ func TestMain(m *testing.M) { goleak.VerifyTestMain(m) } +// TODO: Add ability to unregister metrics? +func NewTestMetrics(t *testing.T, conf discovery.Config, reg prometheus.Registerer) discovery.DiscovererMetrics { + refreshMetrics := discovery.NewRefreshMetrics(reg) + require.NoError(t, refreshMetrics.Register()) + + metrics := conf.NewDiscovererMetrics(prometheus.NewRegistry(), refreshMetrics) + require.NoError(t, metrics.Register()) + + return metrics +} + func TestConfiguredService(t *testing.T) { conf := &SDConfig{ Services: []string{"configuredServiceName"}, } - consulDiscovery, err := NewDiscovery(conf, nil, prometheus.NewRegistry()) + + metrics := NewTestMetrics(t, conf, prometheus.NewRegistry()) + + consulDiscovery, err := NewDiscovery(conf, nil, metrics) if err != nil { t.Errorf("Unexpected error when initializing discovery %v", err) } @@ -57,7 +72,10 @@ func TestConfiguredServiceWithTag(t *testing.T) { Services: []string{"configuredServiceName"}, ServiceTags: []string{"http"}, } - consulDiscovery, err := NewDiscovery(conf, nil, prometheus.NewRegistry()) + + metrics := NewTestMetrics(t, conf, prometheus.NewRegistry()) + + consulDiscovery, err := NewDiscovery(conf, nil, metrics) if err != nil { t.Errorf("Unexpected error when initializing discovery %v", err) } @@ -152,7 +170,9 @@ func TestConfiguredServiceWithTags(t *testing.T) { } for _, tc := range cases { - consulDiscovery, err := NewDiscovery(tc.conf, nil, prometheus.NewRegistry()) + metrics := NewTestMetrics(t, tc.conf, prometheus.NewRegistry()) + + consulDiscovery, err := NewDiscovery(tc.conf, nil, metrics) if err != nil { t.Errorf("Unexpected error when initializing discovery %v", err) } @@ -160,13 +180,15 @@ func TestConfiguredServiceWithTags(t *testing.T) { if ret != tc.shouldWatch { t.Errorf("Expected should watch? %t, got %t. Watched service and tags: %s %+v, input was %s %+v", tc.shouldWatch, ret, tc.conf.Services, tc.conf.ServiceTags, tc.serviceName, tc.serviceTags) } - } } func TestNonConfiguredService(t *testing.T) { conf := &SDConfig{} - consulDiscovery, err := NewDiscovery(conf, nil, prometheus.NewRegistry()) + + metrics := NewTestMetrics(t, conf, prometheus.NewRegistry()) + + consulDiscovery, err := NewDiscovery(conf, nil, metrics) if err != nil { t.Errorf("Unexpected error when initializing discovery %v", err) } @@ -263,7 +285,10 @@ func newServer(t *testing.T) (*httptest.Server, *SDConfig) { func newDiscovery(t *testing.T, config *SDConfig) *Discovery { logger := log.NewNopLogger() - d, err := NewDiscovery(config, logger, prometheus.NewRegistry()) + + metrics := NewTestMetrics(t, config, prometheus.NewRegistry()) + + d, err := NewDiscovery(config, logger, metrics) require.NoError(t, err) return d } diff --git a/discovery/consul/metrics.go b/discovery/consul/metrics.go new file mode 100644 index 000000000..8266e7cc6 --- /dev/null +++ b/discovery/consul/metrics.go @@ -0,0 +1,73 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package consul + +import ( + "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/prometheus/discovery" +) + +var _ discovery.DiscovererMetrics = (*consulMetrics)(nil) + +type consulMetrics struct { + rpcFailuresCount prometheus.Counter + rpcDuration *prometheus.SummaryVec + + servicesRPCDuration prometheus.Observer + serviceRPCDuration prometheus.Observer + + metricRegisterer discovery.MetricRegisterer +} + +func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + m := &consulMetrics{ + rpcFailuresCount: prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "sd_consul_rpc_failures_total", + Help: "The number of Consul RPC call failures.", + }), + rpcDuration: prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: namespace, + Name: "sd_consul_rpc_duration_seconds", + Help: "The duration of a Consul RPC call in seconds.", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, + []string{"endpoint", "call"}, + ), + } + + m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{ + m.rpcFailuresCount, + m.rpcDuration, + }) + + // Initialize metric vectors. + m.servicesRPCDuration = m.rpcDuration.WithLabelValues("catalog", "services") + m.serviceRPCDuration = m.rpcDuration.WithLabelValues("catalog", "service") + + return m +} + +// Register implements discovery.DiscovererMetrics. +func (m *consulMetrics) Register() error { + return m.metricRegisterer.RegisterMetrics() +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *consulMetrics) Unregister() { + m.metricRegisterer.UnregisterMetrics() +} diff --git a/discovery/digitalocean/digitalocean.go b/discovery/digitalocean/digitalocean.go index 970258de0..18380b729 100644 --- a/discovery/digitalocean/digitalocean.go +++ b/discovery/digitalocean/digitalocean.go @@ -63,6 +63,13 @@ func init() { discovery.RegisterConfig(&SDConfig{}) } +// NewDiscovererMetrics implements discovery.Config. +func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &digitaloceanMetrics{ + refreshMetrics: rmi, + } +} + // SDConfig is the configuration for DigitalOcean based service discovery. type SDConfig struct { HTTPClientConfig config.HTTPClientConfig `yaml:",inline"` @@ -76,7 +83,7 @@ func (*SDConfig) Name() string { return "digitalocean" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger, opts.Registerer) + return NewDiscovery(c, opts.Logger, opts.Metrics) } // SetDirectory joins any relative file paths with dir. @@ -104,7 +111,12 @@ type Discovery struct { } // NewDiscovery returns a new Discovery which periodically refreshes its targets. -func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) { +func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { + m, ok := metrics.(*digitaloceanMetrics) + if !ok { + return nil, fmt.Errorf("invalid discovery metrics type") + } + d := &Discovery{ port: conf.Port, } @@ -127,11 +139,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, - Mech: "digitalocean", - Interval: time.Duration(conf.RefreshInterval), - RefreshF: d.refresh, - Registry: reg, + Logger: logger, + Mech: "digitalocean", + Interval: time.Duration(conf.RefreshInterval), + RefreshF: d.refresh, + MetricsInstantiator: m.refreshMetrics, }, ) return d, nil diff --git a/discovery/digitalocean/digitalocean_test.go b/discovery/digitalocean/digitalocean_test.go index a959b312c..841b5ef97 100644 --- a/discovery/digitalocean/digitalocean_test.go +++ b/discovery/digitalocean/digitalocean_test.go @@ -23,6 +23,8 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" + + "github.com/prometheus/prometheus/discovery" ) type DigitalOceanSDTestSuite struct { @@ -47,7 +49,15 @@ func TestDigitalOceanSDRefresh(t *testing.T) { cfg := DefaultSDConfig cfg.HTTPClientConfig.BearerToken = tokenID - d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry()) + + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + defer metrics.Unregister() + defer refreshMetrics.Unregister() + + d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics) require.NoError(t, err) endpoint, err := url.Parse(sdmock.Mock.Endpoint()) require.NoError(t, err) diff --git a/discovery/digitalocean/metrics.go b/discovery/digitalocean/metrics.go new file mode 100644 index 000000000..91cfd9bf3 --- /dev/null +++ b/discovery/digitalocean/metrics.go @@ -0,0 +1,32 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package digitalocean + +import ( + "github.com/prometheus/prometheus/discovery" +) + +var _ discovery.DiscovererMetrics = (*digitaloceanMetrics)(nil) + +type digitaloceanMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator +} + +// Register implements discovery.DiscovererMetrics. +func (m *digitaloceanMetrics) Register() error { + return nil +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *digitaloceanMetrics) Unregister() {} diff --git a/discovery/discoverer_metrics_noop.go b/discovery/discoverer_metrics_noop.go new file mode 100644 index 000000000..638317ace --- /dev/null +++ b/discovery/discoverer_metrics_noop.go @@ -0,0 +1,28 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package discovery + +// Create a dummy metrics struct, because this SD doesn't have any metrics. +type NoopDiscovererMetrics struct{} + +var _ DiscovererMetrics = (*NoopDiscovererMetrics)(nil) + +// Register implements discovery.DiscovererMetrics. +func (*NoopDiscovererMetrics) Register() error { + return nil +} + +// Unregister implements discovery.DiscovererMetrics. +func (*NoopDiscovererMetrics) Unregister() { +} diff --git a/discovery/discovery.go b/discovery/discovery.go index acc4c1efe..a5826f817 100644 --- a/discovery/discovery.go +++ b/discovery/discovery.go @@ -39,24 +39,47 @@ type Discoverer interface { Run(ctx context.Context, up chan<- []*targetgroup.Group) } +// Internal metrics of service discovery mechanisms. +type DiscovererMetrics interface { + Register() error + Unregister() +} + // DiscovererOptions provides options for a Discoverer. type DiscovererOptions struct { Logger log.Logger - // A registerer for the Discoverer's metrics. - // Some Discoverers may ignore this registerer and use the global one instead. - // For now this will work, because the Prometheus `main` function uses the global registry. - // However, in the future the Prometheus `main` function will be updated to not use the global registry. - // Hence, if a discoverer wants its metrics to be visible via the Prometheus executable's - // `/metrics` endpoint, it should use this explicit registerer. - // TODO(ptodev): Update this comment once the Prometheus `main` function does not use the global registry. - Registerer prometheus.Registerer + Metrics DiscovererMetrics // Extra HTTP client options to expose to Discoverers. This field may be // ignored; Discoverer implementations must opt-in to reading it. HTTPClientOptions []config.HTTPClientOption } +// Metrics used by the "refresh" package. +// We define them here in the "discovery" package in order to avoid a cyclic dependency between +// "discovery" and "refresh". +type RefreshMetrics struct { + Failures prometheus.Counter + Duration prometheus.Observer +} + +// Instantiate the metrics used by the "refresh" package. +type RefreshMetricsInstantiator interface { + Instantiate(mech string) *RefreshMetrics +} + +// An interface for registering, unregistering, and instantiating metrics for the "refresh" package. +// Refresh metrics are registered and unregistered outside of the service discovery mechanism. +// This is so that the same metrics can be reused across different service discovery mechanisms. +// To manage refresh metrics inside the SD mechanism, we'd need to use const labels which are +// specific to that SD. However, doing so would also expose too many unused metrics on +// the Prometheus /metrics endpoint. +type RefreshMetricsManager interface { + DiscovererMetrics + RefreshMetricsInstantiator +} + // A Config provides the configuration and constructor for a Discoverer. type Config interface { // Name returns the name of the discovery mechanism. @@ -65,6 +88,9 @@ type Config interface { // NewDiscoverer returns a Discoverer for the Config // with the given DiscovererOptions. NewDiscoverer(DiscovererOptions) (Discoverer, error) + + // NewDiscovererMetrics returns the metrics used by the service discovery. + NewDiscovererMetrics(prometheus.Registerer, RefreshMetricsInstantiator) DiscovererMetrics } // Configs is a slice of Config values that uses custom YAML marshaling and unmarshaling @@ -119,6 +145,11 @@ func (c StaticConfig) NewDiscoverer(DiscovererOptions) (Discoverer, error) { return staticDiscoverer(c), nil } +// No metrics are needed for this service discovery mechanism. +func (c StaticConfig) NewDiscovererMetrics(prometheus.Registerer, RefreshMetricsInstantiator) DiscovererMetrics { + return &NoopDiscovererMetrics{} +} + type staticDiscoverer []*targetgroup.Group func (c staticDiscoverer) Run(ctx context.Context, up chan<- []*targetgroup.Group) { diff --git a/discovery/dns/dns.go b/discovery/dns/dns.go index 9b6bd6741..cf56a2ad0 100644 --- a/discovery/dns/dns.go +++ b/discovery/dns/dns.go @@ -67,12 +67,17 @@ type SDConfig struct { Port int `yaml:"port"` // Ignored for SRV records } +// NewDiscovererMetrics implements discovery.Config. +func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return newDiscovererMetrics(reg, rmi) +} + // Name returns the name of the Config. func (*SDConfig) Name() string { return "dns" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(*c, opts.Logger, opts.Registerer) + return NewDiscovery(*c, opts.Logger, opts.Metrics) } // UnmarshalYAML implements the yaml.Unmarshaler interface. @@ -102,18 +107,22 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { // the Discoverer interface. type Discovery struct { *refresh.Discovery - names []string - port int - qtype uint16 - logger log.Logger - dnsSDLookupsCount prometheus.Counter - dnsSDLookupFailuresCount prometheus.Counter + names []string + port int + qtype uint16 + logger log.Logger + metrics *dnsMetrics lookupFn func(name string, qtype uint16, logger log.Logger) (*dns.Msg, error) } // NewDiscovery returns a new Discovery which periodically refreshes its targets. -func NewDiscovery(conf SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) { +func NewDiscovery(conf SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { + m, ok := metrics.(*dnsMetrics) + if !ok { + return nil, fmt.Errorf("invalid discovery metrics type") + } + if logger == nil { logger = log.NewNopLogger() } @@ -137,28 +146,16 @@ func NewDiscovery(conf SDConfig, logger log.Logger, reg prometheus.Registerer) ( port: conf.Port, logger: logger, lookupFn: lookupWithSearchPath, - dnsSDLookupsCount: prometheus.NewCounter( - prometheus.CounterOpts{ - Namespace: namespace, - Name: "sd_dns_lookups_total", - Help: "The number of DNS-SD lookups.", - }), - dnsSDLookupFailuresCount: prometheus.NewCounter( - prometheus.CounterOpts{ - Namespace: namespace, - Name: "sd_dns_lookup_failures_total", - Help: "The number of DNS-SD lookup failures.", - }), + metrics: m, } d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, - Mech: "dns", - Interval: time.Duration(conf.RefreshInterval), - RefreshF: d.refresh, - Registry: prometheus.NewRegistry(), - Metrics: []prometheus.Collector{d.dnsSDLookupsCount, d.dnsSDLookupFailuresCount}, + Logger: logger, + Mech: "dns", + Interval: time.Duration(conf.RefreshInterval), + RefreshF: d.refresh, + MetricsInstantiator: m.refreshMetrics, }, ) @@ -195,9 +192,9 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { func (d *Discovery) refreshOne(ctx context.Context, name string, ch chan<- *targetgroup.Group) error { response, err := d.lookupFn(name, d.qtype, d.logger) - d.dnsSDLookupsCount.Inc() + d.metrics.dnsSDLookupsCount.Inc() if err != nil { - d.dnsSDLookupFailuresCount.Inc() + d.metrics.dnsSDLookupFailuresCount.Inc() return err } diff --git a/discovery/dns/dns_test.go b/discovery/dns/dns_test.go index b8dd2efaa..33a976827 100644 --- a/discovery/dns/dns_test.go +++ b/discovery/dns/dns_test.go @@ -28,6 +28,7 @@ import ( "go.uber.org/goleak" "gopkg.in/yaml.v2" + "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/targetgroup" ) @@ -253,13 +254,21 @@ func TestDNS(t *testing.T) { tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() - sd, err := NewDiscovery(tc.config, nil, prometheus.NewRegistry()) + + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := tc.config.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + + sd, err := NewDiscovery(tc.config, nil, metrics) require.NoError(t, err) sd.lookupFn = tc.lookup tgs, err := sd.refresh(context.Background()) require.NoError(t, err) require.Equal(t, tc.expected, tgs) + + metrics.Unregister() }) } } diff --git a/discovery/dns/metrics.go b/discovery/dns/metrics.go new file mode 100644 index 000000000..27c96b53e --- /dev/null +++ b/discovery/dns/metrics.go @@ -0,0 +1,66 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dns + +import ( + "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/prometheus/discovery" +) + +var _ discovery.DiscovererMetrics = (*dnsMetrics)(nil) + +type dnsMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator + + dnsSDLookupsCount prometheus.Counter + dnsSDLookupFailuresCount prometheus.Counter + + metricRegisterer discovery.MetricRegisterer +} + +func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + m := &dnsMetrics{ + refreshMetrics: rmi, + dnsSDLookupsCount: prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "sd_dns_lookups_total", + Help: "The number of DNS-SD lookups.", + }), + dnsSDLookupFailuresCount: prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "sd_dns_lookup_failures_total", + Help: "The number of DNS-SD lookup failures.", + }), + } + + m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{ + m.dnsSDLookupsCount, + m.dnsSDLookupFailuresCount, + }) + + return m +} + +// Register implements discovery.DiscovererMetrics. +func (m *dnsMetrics) Register() error { + return m.metricRegisterer.RegisterMetrics() +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *dnsMetrics) Unregister() { + m.metricRegisterer.UnregisterMetrics() +} diff --git a/discovery/eureka/eureka.go b/discovery/eureka/eureka.go index d3e4084e5..deed89f6f 100644 --- a/discovery/eureka/eureka.go +++ b/discovery/eureka/eureka.go @@ -16,6 +16,7 @@ package eureka import ( "context" "errors" + "fmt" "net" "net/http" "net/url" @@ -76,12 +77,19 @@ type SDConfig struct { RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` } +// NewDiscovererMetrics implements discovery.Config. +func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &eurekaMetrics{ + refreshMetrics: rmi, + } +} + // Name returns the name of the Config. func (*SDConfig) Name() string { return "eureka" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger, opts.Registerer) + return NewDiscovery(c, opts.Logger, opts.Metrics) } // SetDirectory joins any relative file paths with dir. @@ -118,7 +126,12 @@ type Discovery struct { } // NewDiscovery creates a new Eureka discovery for the given role. -func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) { +func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { + m, ok := metrics.(*eurekaMetrics) + if !ok { + return nil, fmt.Errorf("invalid discovery metrics type") + } + rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "eureka_sd") if err != nil { return nil, err @@ -130,11 +143,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) } d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, - Mech: "eureka", - Interval: time.Duration(conf.RefreshInterval), - RefreshF: d.refresh, - Registry: reg, + Logger: logger, + Mech: "eureka", + Interval: time.Duration(conf.RefreshInterval), + RefreshF: d.refresh, + MetricsInstantiator: m.refreshMetrics, }, ) return d, nil diff --git a/discovery/eureka/eureka_test.go b/discovery/eureka/eureka_test.go index 1fe3c710e..b499410bf 100644 --- a/discovery/eureka/eureka_test.go +++ b/discovery/eureka/eureka_test.go @@ -24,6 +24,7 @@ import ( "github.com/prometheus/common/model" "github.com/stretchr/testify/require" + "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/targetgroup" ) @@ -36,7 +37,17 @@ func testUpdateServices(respHandler http.HandlerFunc) ([]*targetgroup.Group, err Server: ts.URL, } - md, err := NewDiscovery(&conf, nil, prometheus.NewRegistry()) + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := conf.NewDiscovererMetrics(reg, refreshMetrics) + err := metrics.Register() + if err != nil { + return nil, err + } + defer metrics.Unregister() + defer refreshMetrics.Unregister() + + md, err := NewDiscovery(&conf, nil, metrics) if err != nil { return nil, err } diff --git a/discovery/eureka/metrics.go b/discovery/eureka/metrics.go new file mode 100644 index 000000000..6d6463ccd --- /dev/null +++ b/discovery/eureka/metrics.go @@ -0,0 +1,32 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package eureka + +import ( + "github.com/prometheus/prometheus/discovery" +) + +var _ discovery.DiscovererMetrics = (*eurekaMetrics)(nil) + +type eurekaMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator +} + +// Register implements discovery.DiscovererMetrics. +func (m *eurekaMetrics) Register() error { + return nil +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *eurekaMetrics) Unregister() {} diff --git a/discovery/file/file.go b/discovery/file/file.go index ef6ed1f5e..e7e9d0870 100644 --- a/discovery/file/file.go +++ b/discovery/file/file.go @@ -57,12 +57,17 @@ type SDConfig struct { RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` } +// NewDiscovererMetrics implements discovery.Config. +func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return newDiscovererMetrics(reg, rmi) +} + // Name returns the name of the Config. func (*SDConfig) Name() string { return "file" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger, opts.Registerer) + return NewDiscovery(c, opts.Logger, opts.Metrics) } // SetDirectory joins any relative file paths with dir. @@ -94,6 +99,9 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { const fileSDFilepathLabel = model.MetaLabelPrefix + "filepath" // TimestampCollector is a Custom Collector for Timestamps of the files. +// TODO(ptodev): Now that each file SD has its own TimestampCollector +// inside discovery/file/metrics.go, we can refactor this collector +// (or get rid of it) as each TimestampCollector instance will only use one discoverer. type TimestampCollector struct { Description *prometheus.Desc discoverers map[*Discovery]struct{} @@ -169,16 +177,16 @@ type Discovery struct { lastRefresh map[string]int logger log.Logger - fileSDReadErrorsCount prometheus.Counter - fileSDScanDuration prometheus.Summary - fileWatcherErrorsCount prometheus.Counter - fileSDTimeStamp *TimestampCollector - - metricRegisterer discovery.MetricRegisterer + metrics *fileMetrics } // NewDiscovery returns a new file discovery for the given paths. -func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) { +func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { + fm, ok := metrics.(*fileMetrics) + if !ok { + return nil, fmt.Errorf("invalid discovery metrics type") + } + if logger == nil { logger = log.NewNopLogger() } @@ -188,33 +196,10 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) interval: time.Duration(conf.RefreshInterval), timestamps: make(map[string]float64), logger: logger, - fileSDReadErrorsCount: prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "prometheus_sd_file_read_errors_total", - Help: "The number of File-SD read errors.", - }), - fileSDScanDuration: prometheus.NewSummary( - prometheus.SummaryOpts{ - Name: "prometheus_sd_file_scan_duration_seconds", - Help: "The duration of the File-SD scan in seconds.", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, - }), - fileWatcherErrorsCount: prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "prometheus_sd_file_watcher_errors_total", - Help: "The number of File-SD errors caused by filesystem watch failures.", - }), - fileSDTimeStamp: NewTimestampCollector(), + metrics: fm, } - disc.fileSDTimeStamp.addDiscoverer(disc) - - disc.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{ - disc.fileSDReadErrorsCount, - disc.fileSDScanDuration, - disc.fileWatcherErrorsCount, - disc.fileSDTimeStamp, - }) + fm.init(disc) return disc, nil } @@ -253,17 +238,10 @@ func (d *Discovery) watchFiles() { // Run implements the Discoverer interface. func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { - err := d.metricRegisterer.RegisterMetrics() - if err != nil { - level.Error(d.logger).Log("msg", "Unable to register metrics", "err", err.Error()) - return - } - defer d.metricRegisterer.UnregisterMetrics() - watcher, err := fsnotify.NewWatcher() if err != nil { level.Error(d.logger).Log("msg", "Error adding file watcher", "err", err) - d.fileWatcherErrorsCount.Inc() + d.metrics.fileWatcherErrorsCount.Inc() return } d.watcher = watcher @@ -327,7 +305,7 @@ func (d *Discovery) stop() { done := make(chan struct{}) defer close(done) - d.fileSDTimeStamp.removeDiscoverer(d) + d.metrics.fileSDTimeStamp.removeDiscoverer(d) // Closing the watcher will deadlock unless all events and errors are drained. go func() { @@ -353,13 +331,13 @@ func (d *Discovery) stop() { func (d *Discovery) refresh(ctx context.Context, ch chan<- []*targetgroup.Group) { t0 := time.Now() defer func() { - d.fileSDScanDuration.Observe(time.Since(t0).Seconds()) + d.metrics.fileSDScanDuration.Observe(time.Since(t0).Seconds()) }() ref := map[string]int{} for _, p := range d.listFiles() { tgroups, err := d.readFile(p) if err != nil { - d.fileSDReadErrorsCount.Inc() + d.metrics.fileSDReadErrorsCount.Inc() level.Error(d.logger).Log("msg", "Error reading file", "path", p, "err", err) // Prevent deletion down below. diff --git a/discovery/file/file_test.go b/discovery/file/file_test.go index 731611832..8a5057089 100644 --- a/discovery/file/file_test.go +++ b/discovery/file/file_test.go @@ -29,6 +29,7 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/goleak" + "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/targetgroup" ) @@ -144,19 +145,28 @@ func (t *testRunner) run(files ...string) { ctx, cancel := context.WithCancel(context.Background()) t.cancelSD = cancel go func() { + conf := &SDConfig{ + Files: files, + // Setting a high refresh interval to make sure that the tests only + // rely on file watches. + RefreshInterval: model.Duration(1 * time.Hour), + } + + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := conf.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + d, err := NewDiscovery( - &SDConfig{ - Files: files, - // Setting a high refresh interval to make sure that the tests only - // rely on file watches. - RefreshInterval: model.Duration(1 * time.Hour), - }, + conf, nil, - prometheus.NewRegistry(), + metrics, ) require.NoError(t, err) d.Run(ctx, t.ch) + + metrics.Unregister() }() } diff --git a/discovery/file/metrics.go b/discovery/file/metrics.go new file mode 100644 index 000000000..c01501e4e --- /dev/null +++ b/discovery/file/metrics.go @@ -0,0 +1,76 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package file + +import ( + "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/prometheus/discovery" +) + +var _ discovery.DiscovererMetrics = (*fileMetrics)(nil) + +type fileMetrics struct { + fileSDReadErrorsCount prometheus.Counter + fileSDScanDuration prometheus.Summary + fileWatcherErrorsCount prometheus.Counter + fileSDTimeStamp *TimestampCollector + + metricRegisterer discovery.MetricRegisterer +} + +func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + fm := &fileMetrics{ + fileSDReadErrorsCount: prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "prometheus_sd_file_read_errors_total", + Help: "The number of File-SD read errors.", + }), + fileSDScanDuration: prometheus.NewSummary( + prometheus.SummaryOpts{ + Name: "prometheus_sd_file_scan_duration_seconds", + Help: "The duration of the File-SD scan in seconds.", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }), + fileWatcherErrorsCount: prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "prometheus_sd_file_watcher_errors_total", + Help: "The number of File-SD errors caused by filesystem watch failures.", + }), + fileSDTimeStamp: NewTimestampCollector(), + } + + fm.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{ + fm.fileSDReadErrorsCount, + fm.fileSDScanDuration, + fm.fileWatcherErrorsCount, + fm.fileSDTimeStamp, + }) + + return fm +} + +// Register implements discovery.DiscovererMetrics. +func (fm *fileMetrics) Register() error { + return fm.metricRegisterer.RegisterMetrics() +} + +// Unregister implements discovery.DiscovererMetrics. +func (fm *fileMetrics) Unregister() { + fm.metricRegisterer.UnregisterMetrics() +} + +func (fm *fileMetrics) init(disc *Discovery) { + fm.fileSDTimeStamp.addDiscoverer(disc) +} diff --git a/discovery/gce/gce.go b/discovery/gce/gce.go index 21a95ee39..15f32dd24 100644 --- a/discovery/gce/gce.go +++ b/discovery/gce/gce.go @@ -82,12 +82,19 @@ type SDConfig struct { TagSeparator string `yaml:"tag_separator,omitempty"` } +// NewDiscovererMetrics implements discovery.Config. +func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &gceMetrics{ + refreshMetrics: rmi, + } +} + // Name returns the name of the Config. func (*SDConfig) Name() string { return "gce" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(*c, opts.Logger, opts.Registerer) + return NewDiscovery(*c, opts.Logger, opts.Metrics) } // UnmarshalYAML implements the yaml.Unmarshaler interface. @@ -122,7 +129,12 @@ type Discovery struct { } // NewDiscovery returns a new Discovery which periodically refreshes its targets. -func NewDiscovery(conf SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) { +func NewDiscovery(conf SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { + m, ok := metrics.(*gceMetrics) + if !ok { + return nil, fmt.Errorf("invalid discovery metrics type") + } + d := &Discovery{ project: conf.Project, zone: conf.Zone, @@ -143,11 +155,11 @@ func NewDiscovery(conf SDConfig, logger log.Logger, reg prometheus.Registerer) ( d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, - Mech: "gce", - Interval: time.Duration(conf.RefreshInterval), - RefreshF: d.refresh, - Registry: reg, + Logger: logger, + Mech: "gce", + Interval: time.Duration(conf.RefreshInterval), + RefreshF: d.refresh, + MetricsInstantiator: m.refreshMetrics, }, ) return d, nil diff --git a/discovery/gce/metrics.go b/discovery/gce/metrics.go new file mode 100644 index 000000000..f53ea5a8c --- /dev/null +++ b/discovery/gce/metrics.go @@ -0,0 +1,32 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gce + +import ( + "github.com/prometheus/prometheus/discovery" +) + +var _ discovery.DiscovererMetrics = (*gceMetrics)(nil) + +type gceMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator +} + +// Register implements discovery.DiscovererMetrics. +func (m *gceMetrics) Register() error { + return nil +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *gceMetrics) Unregister() {} diff --git a/discovery/hetzner/hetzner.go b/discovery/hetzner/hetzner.go index 9d3e6aa65..69c823d38 100644 --- a/discovery/hetzner/hetzner.go +++ b/discovery/hetzner/hetzner.go @@ -63,12 +63,19 @@ type SDConfig struct { robotEndpoint string // For tests only. } +// NewDiscovererMetrics implements discovery.Config. +func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &hetznerMetrics{ + refreshMetrics: rmi, + } +} + // Name returns the name of the Config. func (*SDConfig) Name() string { return "hetzner" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger, opts.Registerer) + return NewDiscovery(c, opts.Logger, opts.Metrics) } type refresher interface { @@ -128,7 +135,12 @@ type Discovery struct { } // NewDiscovery returns a new Discovery which periodically refreshes its targets. -func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*refresh.Discovery, error) { +func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*refresh.Discovery, error) { + m, ok := metrics.(*hetznerMetrics) + if !ok { + return nil, fmt.Errorf("invalid discovery metrics type") + } + r, err := newRefresher(conf, logger) if err != nil { return nil, err @@ -136,11 +148,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) return refresh.NewDiscovery( refresh.Options{ - Logger: logger, - Mech: "hetzner", - Interval: time.Duration(conf.RefreshInterval), - RefreshF: r.refresh, - Registry: reg, + Logger: logger, + Mech: "hetzner", + Interval: time.Duration(conf.RefreshInterval), + RefreshF: r.refresh, + MetricsInstantiator: m.refreshMetrics, }, ), nil } diff --git a/discovery/hetzner/metrics.go b/discovery/hetzner/metrics.go new file mode 100644 index 000000000..27361ee75 --- /dev/null +++ b/discovery/hetzner/metrics.go @@ -0,0 +1,32 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hetzner + +import ( + "github.com/prometheus/prometheus/discovery" +) + +var _ discovery.DiscovererMetrics = (*hetznerMetrics)(nil) + +type hetznerMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator +} + +// Register implements discovery.DiscovererMetrics. +func (m *hetznerMetrics) Register() error { + return nil +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *hetznerMetrics) Unregister() {} diff --git a/discovery/http/http.go b/discovery/http/http.go index c12fdb26d..8dd21ec9e 100644 --- a/discovery/http/http.go +++ b/discovery/http/http.go @@ -58,12 +58,17 @@ type SDConfig struct { URL string `yaml:"url"` } +// NewDiscovererMetrics implements discovery.Config. +func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return newDiscovererMetrics(reg, rmi) +} + // Name returns the name of the Config. func (*SDConfig) Name() string { return "http" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger, opts.HTTPClientOptions, opts.Registerer) + return NewDiscovery(c, opts.Logger, opts.HTTPClientOptions, opts.Metrics) } // SetDirectory joins any relative file paths with dir. @@ -105,11 +110,16 @@ type Discovery struct { client *http.Client refreshInterval time.Duration tgLastLength int - failuresCount prometheus.Counter + metrics *httpMetrics } // NewDiscovery returns a new HTTP discovery for the given config. -func NewDiscovery(conf *SDConfig, logger log.Logger, clientOpts []config.HTTPClientOption, reg prometheus.Registerer) (*Discovery, error) { +func NewDiscovery(conf *SDConfig, logger log.Logger, clientOpts []config.HTTPClientOption, metrics discovery.DiscovererMetrics) (*Discovery, error) { + m, ok := metrics.(*httpMetrics) + if !ok { + return nil, fmt.Errorf("invalid discovery metrics type") + } + if logger == nil { logger = log.NewNopLogger() } @@ -124,21 +134,16 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, clientOpts []config.HTTPCli url: conf.URL, client: client, refreshInterval: time.Duration(conf.RefreshInterval), // Stored to be sent as headers. - failuresCount: prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "prometheus_sd_http_failures_total", - Help: "Number of HTTP service discovery refresh failures.", - }), + metrics: m, } d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, - Mech: "http", - Interval: time.Duration(conf.RefreshInterval), - RefreshF: d.Refresh, - Registry: reg, - Metrics: []prometheus.Collector{d.failuresCount}, + Logger: logger, + Mech: "http", + Interval: time.Duration(conf.RefreshInterval), + RefreshF: d.Refresh, + MetricsInstantiator: m.refreshMetrics, }, ) return d, nil @@ -155,7 +160,7 @@ func (d *Discovery) Refresh(ctx context.Context) ([]*targetgroup.Group, error) { resp, err := d.client.Do(req.WithContext(ctx)) if err != nil { - d.failuresCount.Inc() + d.metrics.failuresCount.Inc() return nil, err } defer func() { @@ -164,31 +169,31 @@ func (d *Discovery) Refresh(ctx context.Context) ([]*targetgroup.Group, error) { }() if resp.StatusCode != http.StatusOK { - d.failuresCount.Inc() + d.metrics.failuresCount.Inc() return nil, fmt.Errorf("server returned HTTP status %s", resp.Status) } if !matchContentType.MatchString(strings.TrimSpace(resp.Header.Get("Content-Type"))) { - d.failuresCount.Inc() + d.metrics.failuresCount.Inc() return nil, fmt.Errorf("unsupported content type %q", resp.Header.Get("Content-Type")) } b, err := io.ReadAll(resp.Body) if err != nil { - d.failuresCount.Inc() + d.metrics.failuresCount.Inc() return nil, err } var targetGroups []*targetgroup.Group if err := json.Unmarshal(b, &targetGroups); err != nil { - d.failuresCount.Inc() + d.metrics.failuresCount.Inc() return nil, err } for i, tg := range targetGroups { if tg == nil { - d.failuresCount.Inc() + d.metrics.failuresCount.Inc() err = errors.New("nil target group item found") return nil, err } diff --git a/discovery/http/http_test.go b/discovery/http/http_test.go index 164719e90..0cafe035d 100644 --- a/discovery/http/http_test.go +++ b/discovery/http/http_test.go @@ -28,6 +28,7 @@ import ( "github.com/prometheus/common/model" "github.com/stretchr/testify/require" + "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/targetgroup" ) @@ -41,7 +42,14 @@ func TestHTTPValidRefresh(t *testing.T) { RefreshInterval: model.Duration(30 * time.Second), } - d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, prometheus.NewRegistry()) + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + defer refreshMetrics.Unregister() + metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + defer metrics.Unregister() + + d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, metrics) require.NoError(t, err) ctx := context.Background() @@ -63,7 +71,7 @@ func TestHTTPValidRefresh(t *testing.T) { }, } require.Equal(t, expectedTargets, tgs) - require.Equal(t, 0.0, getFailureCount(d.failuresCount)) + require.Equal(t, 0.0, getFailureCount(d.metrics.failuresCount)) } func TestHTTPInvalidCode(t *testing.T) { @@ -79,13 +87,20 @@ func TestHTTPInvalidCode(t *testing.T) { RefreshInterval: model.Duration(30 * time.Second), } - d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, prometheus.NewRegistry()) + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + defer refreshMetrics.Unregister() + metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + defer metrics.Unregister() + + d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, metrics) require.NoError(t, err) ctx := context.Background() _, err = d.Refresh(ctx) require.EqualError(t, err, "server returned HTTP status 400 Bad Request") - require.Equal(t, 1.0, getFailureCount(d.failuresCount)) + require.Equal(t, 1.0, getFailureCount(d.metrics.failuresCount)) } func TestHTTPInvalidFormat(t *testing.T) { @@ -101,13 +116,20 @@ func TestHTTPInvalidFormat(t *testing.T) { RefreshInterval: model.Duration(30 * time.Second), } - d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, prometheus.NewRegistry()) + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + defer refreshMetrics.Unregister() + metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + defer metrics.Unregister() + + d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, metrics) require.NoError(t, err) ctx := context.Background() _, err = d.Refresh(ctx) require.EqualError(t, err, `unsupported content type "text/plain; charset=utf-8"`) - require.Equal(t, 1.0, getFailureCount(d.failuresCount)) + require.Equal(t, 1.0, getFailureCount(d.metrics.failuresCount)) } func getFailureCount(failuresCount prometheus.Counter) float64 { @@ -412,7 +434,15 @@ func TestSourceDisappeared(t *testing.T) { URL: ts.URL, RefreshInterval: model.Duration(1 * time.Second), } - d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, prometheus.NewRegistry()) + + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + defer refreshMetrics.Unregister() + metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + defer metrics.Unregister() + + d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, metrics) require.NoError(t, err) for _, test := range cases { ctx := context.Background() diff --git a/discovery/http/metrics.go b/discovery/http/metrics.go new file mode 100644 index 000000000..b1f8b8443 --- /dev/null +++ b/discovery/http/metrics.go @@ -0,0 +1,57 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/prometheus/discovery" +) + +var _ discovery.DiscovererMetrics = (*httpMetrics)(nil) + +type httpMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator + + failuresCount prometheus.Counter + + metricRegisterer discovery.MetricRegisterer +} + +func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + m := &httpMetrics{ + refreshMetrics: rmi, + failuresCount: prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "prometheus_sd_http_failures_total", + Help: "Number of HTTP service discovery refresh failures.", + }), + } + + m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{ + m.failuresCount, + }) + + return m +} + +// Register implements discovery.DiscovererMetrics. +func (m *httpMetrics) Register() error { + return m.metricRegisterer.RegisterMetrics() +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *httpMetrics) Unregister() { + m.metricRegisterer.UnregisterMetrics() +} diff --git a/discovery/ionos/ionos.go b/discovery/ionos/ionos.go index 36623745a..c8b4f7f8e 100644 --- a/discovery/ionos/ionos.go +++ b/discovery/ionos/ionos.go @@ -15,16 +15,16 @@ package ionos import ( "errors" + "fmt" "time" "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/refresh" - - "github.com/prometheus/client_golang/prometheus" ) const ( @@ -43,7 +43,12 @@ func init() { type Discovery struct{} // NewDiscovery returns a new refresh.Discovery for IONOS Cloud. -func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*refresh.Discovery, error) { +func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*refresh.Discovery, error) { + m, ok := metrics.(*ionosMetrics) + if !ok { + return nil, fmt.Errorf("invalid discovery metrics type") + } + if conf.ionosEndpoint == "" { conf.ionosEndpoint = "https://api.ionos.com" } @@ -55,11 +60,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) return refresh.NewDiscovery( refresh.Options{ - Logger: logger, - Mech: "ionos", - Interval: time.Duration(conf.RefreshInterval), - RefreshF: d.refresh, - Registry: reg, + Logger: logger, + Mech: "ionos", + Interval: time.Duration(conf.RefreshInterval), + RefreshF: d.refresh, + MetricsInstantiator: m.refreshMetrics, }, ), nil } @@ -84,6 +89,13 @@ type SDConfig struct { ionosEndpoint string // For tests only. } +// NewDiscovererMetrics implements discovery.Config. +func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &ionosMetrics{ + refreshMetrics: rmi, + } +} + // Name returns the name of the IONOS Cloud service discovery. func (c SDConfig) Name() string { return "ionos" @@ -91,7 +103,7 @@ func (c SDConfig) Name() string { // NewDiscoverer returns a new discovery.Discoverer for IONOS Cloud. func (c SDConfig) NewDiscoverer(options discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(&c, options.Logger, options.Registerer) + return NewDiscovery(&c, options.Logger, options.Metrics) } // UnmarshalYAML implements the yaml.Unmarshaler interface. diff --git a/discovery/ionos/metrics.go b/discovery/ionos/metrics.go new file mode 100644 index 000000000..88e465acf --- /dev/null +++ b/discovery/ionos/metrics.go @@ -0,0 +1,32 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ionos + +import ( + "github.com/prometheus/prometheus/discovery" +) + +var _ discovery.DiscovererMetrics = (*ionosMetrics)(nil) + +type ionosMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator +} + +// Register implements discovery.DiscovererMetrics. +func (m *ionosMetrics) Register() error { + return nil +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *ionosMetrics) Unregister() {} diff --git a/discovery/kubernetes/kubernetes.go b/discovery/kubernetes/kubernetes.go index 5c5f3dfb6..362e05326 100644 --- a/discovery/kubernetes/kubernetes.go +++ b/discovery/kubernetes/kubernetes.go @@ -123,12 +123,17 @@ type SDConfig struct { AttachMetadata AttachMetadataConfig `yaml:"attach_metadata,omitempty"` } +// NewDiscovererMetrics implements discovery.Config. +func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return newDiscovererMetrics(reg, rmi) +} + // Name returns the name of the Config. func (*SDConfig) Name() string { return "kubernetes" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return New(opts.Logger, opts.Registerer, c) + return New(opts.Logger, opts.Metrics, c) } // SetDirectory joins any relative file paths with dir. @@ -265,8 +270,7 @@ type Discovery struct { selectors roleSelector ownNamespace string attachMetadata AttachMetadataConfig - eventCount *prometheus.CounterVec - metricRegisterer discovery.MetricRegisterer + metrics *kubernetesMetrics } func (d *Discovery) getNamespaces() []string { @@ -285,7 +289,12 @@ func (d *Discovery) getNamespaces() []string { } // New creates a new Kubernetes discovery for the given role. -func New(l log.Logger, reg prometheus.Registerer, conf *SDConfig) (*Discovery, error) { +func New(l log.Logger, metrics discovery.DiscovererMetrics, conf *SDConfig) (*Discovery, error) { + m, ok := metrics.(*kubernetesMetrics) + if !ok { + return nil, fmt.Errorf("invalid discovery metrics type") + } + if l == nil { l = log.NewNopLogger() } @@ -348,34 +357,7 @@ func New(l log.Logger, reg prometheus.Registerer, conf *SDConfig) (*Discovery, e selectors: mapSelector(conf.Selectors), ownNamespace: ownNamespace, attachMetadata: conf.AttachMetadata, - eventCount: prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: discovery.KubernetesMetricsNamespace, - Name: "events_total", - Help: "The number of Kubernetes events handled.", - }, - []string{"role", "event"}, - ), - } - - d.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{d.eventCount}) - - // Initialize metric vectors. - for _, role := range []string{ - RoleEndpointSlice.String(), - RoleEndpoint.String(), - RoleNode.String(), - RolePod.String(), - RoleService.String(), - RoleIngress.String(), - } { - for _, evt := range []string{ - MetricLabelRoleAdd, - MetricLabelRoleDelete, - MetricLabelRoleUpdate, - } { - d.eventCount.WithLabelValues(role, evt) - } + metrics: m, } return d, nil @@ -415,13 +397,6 @@ const resyncDisabled = 0 func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { d.Lock() - err := d.metricRegisterer.RegisterMetrics() - if err != nil { - level.Error(d.logger).Log("msg", "Unable to register metrics", "err", err.Error()) - return - } - defer d.metricRegisterer.UnregisterMetrics() - namespaces := d.getNamespaces() switch d.role { @@ -513,7 +488,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { cache.NewSharedInformer(slw, &apiv1.Service{}, resyncDisabled), cache.NewSharedInformer(plw, &apiv1.Pod{}, resyncDisabled), nodeInf, - d.eventCount, + d.metrics.eventCount, ) d.discoverers = append(d.discoverers, eps) go eps.endpointSliceInf.Run(ctx.Done()) @@ -573,7 +548,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { cache.NewSharedInformer(slw, &apiv1.Service{}, resyncDisabled), cache.NewSharedInformer(plw, &apiv1.Pod{}, resyncDisabled), nodeInf, - d.eventCount, + d.metrics.eventCount, ) d.discoverers = append(d.discoverers, eps) go eps.endpointsInf.Run(ctx.Done()) @@ -605,7 +580,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { log.With(d.logger, "role", "pod"), d.newPodsByNodeInformer(plw), nodeInformer, - d.eventCount, + d.metrics.eventCount, ) d.discoverers = append(d.discoverers, pod) go pod.podInf.Run(ctx.Done()) @@ -628,7 +603,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { svc := NewService( log.With(d.logger, "role", "service"), cache.NewSharedInformer(slw, &apiv1.Service{}, resyncDisabled), - d.eventCount, + d.metrics.eventCount, ) d.discoverers = append(d.discoverers, svc) go svc.informer.Run(ctx.Done()) @@ -686,14 +661,14 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { ingress := NewIngress( log.With(d.logger, "role", "ingress"), informer, - d.eventCount, + d.metrics.eventCount, ) d.discoverers = append(d.discoverers, ingress) go ingress.informer.Run(ctx.Done()) } case RoleNode: nodeInformer := d.newNodeInformer(ctx) - node := NewNode(log.With(d.logger, "role", "node"), nodeInformer, d.eventCount) + node := NewNode(log.With(d.logger, "role", "node"), nodeInformer, d.metrics.eventCount) d.discoverers = append(d.discoverers, node) go node.informer.Run(ctx.Done()) default: diff --git a/discovery/kubernetes/kubernetes_test.go b/discovery/kubernetes/kubernetes_test.go index 71c937e94..4071ebc34 100644 --- a/discovery/kubernetes/kubernetes_test.go +++ b/discovery/kubernetes/kubernetes_test.go @@ -51,24 +51,29 @@ func makeDiscoveryWithVersion(role Role, nsDiscovery NamespaceDiscovery, k8sVer fakeDiscovery, _ := clientset.Discovery().(*fakediscovery.FakeDiscovery) fakeDiscovery.FakedServerVersion = &version.Info{GitVersion: k8sVer} + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := newDiscovererMetrics(reg, refreshMetrics) + err := metrics.Register() + if err != nil { + panic(err) + } + // TODO(ptodev): Unregister the metrics at the end of the test. + + kubeMetrics, ok := metrics.(*kubernetesMetrics) + if !ok { + panic("invalid discovery metrics type") + } + d := &Discovery{ client: clientset, logger: log.NewNopLogger(), role: role, namespaceDiscovery: &nsDiscovery, ownNamespace: "own-ns", - eventCount: prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: discovery.KubernetesMetricsNamespace, - Name: "events_total", - Help: "The number of Kubernetes events handled.", - }, - []string{"role", "event"}, - ), + metrics: kubeMetrics, } - d.metricRegisterer = discovery.NewMetricRegisterer(prometheus.NewRegistry(), []prometheus.Collector{d.eventCount}) - return d, clientset } diff --git a/discovery/kubernetes/metrics.go b/discovery/kubernetes/metrics.go new file mode 100644 index 000000000..7d384fb96 --- /dev/null +++ b/discovery/kubernetes/metrics.go @@ -0,0 +1,75 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubernetes + +import ( + "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/prometheus/discovery" +) + +var _ discovery.DiscovererMetrics = (*kubernetesMetrics)(nil) + +type kubernetesMetrics struct { + eventCount *prometheus.CounterVec + + metricRegisterer discovery.MetricRegisterer +} + +func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + m := &kubernetesMetrics{ + eventCount: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: discovery.KubernetesMetricsNamespace, + Name: "events_total", + Help: "The number of Kubernetes events handled.", + }, + []string{"role", "event"}, + ), + } + + m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{ + m.eventCount, + }) + + // Initialize metric vectors. + for _, role := range []string{ + RoleEndpointSlice.String(), + RoleEndpoint.String(), + RoleNode.String(), + RolePod.String(), + RoleService.String(), + RoleIngress.String(), + } { + for _, evt := range []string{ + MetricLabelRoleAdd, + MetricLabelRoleDelete, + MetricLabelRoleUpdate, + } { + m.eventCount.WithLabelValues(role, evt) + } + } + + return m +} + +// Register implements discovery.DiscovererMetrics. +func (m *kubernetesMetrics) Register() error { + return m.metricRegisterer.RegisterMetrics() +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *kubernetesMetrics) Unregister() { + m.metricRegisterer.UnregisterMetrics() +} diff --git a/discovery/legacymanager/manager.go b/discovery/legacymanager/manager.go index 9c80f305a..6fc61485d 100644 --- a/discovery/legacymanager/manager.go +++ b/discovery/legacymanager/manager.go @@ -42,7 +42,7 @@ type provider struct { } // NewManager is the Discovery Manager constructor. -func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Registerer, options ...func(*Manager)) *Manager { +func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Registerer, sdMetrics map[string]discovery.DiscovererMetrics, options ...func(*Manager)) *Manager { if logger == nil { logger = log.NewNopLogger() } @@ -55,6 +55,7 @@ func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Re updatert: 5 * time.Second, triggerSend: make(chan struct{}, 1), registerer: registerer, + sdMetrics: sdMetrics, } for _, option := range options { option(mgr) @@ -62,7 +63,7 @@ func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Re // Register the metrics. // We have to do this after setting all options, so that the name of the Manager is set. - if metrics, err := discovery.NewMetrics(registerer, mgr.name); err == nil { + if metrics, err := discovery.NewManagerMetrics(registerer, mgr.name); err == nil { mgr.metrics = metrics } else { level.Error(logger).Log("msg", "Failed to create discovery manager metrics", "manager", mgr.name, "err", err) @@ -108,7 +109,8 @@ type Manager struct { // A registerer for all service discovery metrics. registerer prometheus.Registerer - metrics *discovery.Metrics + metrics *discovery.Metrics + sdMetrics map[string]discovery.DiscovererMetrics } // Run starts the background processing. @@ -283,8 +285,8 @@ func (m *Manager) registerProviders(cfgs discovery.Configs, setName string) int } typ := cfg.Name() d, err := cfg.NewDiscoverer(discovery.DiscovererOptions{ - Logger: log.With(m.logger, "discovery", typ, "config", setName), - Registerer: m.registerer, + Logger: log.With(m.logger, "discovery", typ, "config", setName), + Metrics: m.sdMetrics[typ], }) if err != nil { level.Error(m.logger).Log("msg", "Cannot create service discovery", "err", err, "type", typ, "config", setName) diff --git a/discovery/legacymanager/manager_test.go b/discovery/legacymanager/manager_test.go index 7a2e8feea..6fbecabc2 100644 --- a/discovery/legacymanager/manager_test.go +++ b/discovery/legacymanager/manager_test.go @@ -36,6 +36,13 @@ func TestMain(m *testing.M) { testutil.TolerantVerifyLeak(m) } +func newTestMetrics(t *testing.T, reg prometheus.Registerer) (*discovery.RefreshMetricsManager, map[string]discovery.DiscovererMetrics) { + refreshMetrics := discovery.NewRefreshMetrics(reg) + sdMetrics, err := discovery.RegisterSDMetrics(reg, refreshMetrics) + require.NoError(t, err) + return &refreshMetrics, sdMetrics +} + // TestTargetUpdatesOrder checks that the target updates are received in the expected order. func TestTargetUpdatesOrder(t *testing.T) { // The order by which the updates are send is determined by the interval passed to the mock discovery adapter @@ -665,7 +672,10 @@ func TestTargetUpdatesOrder(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry()) + reg := prometheus.NewRegistry() + _, sdMetrics := newTestMetrics(t, reg) + + discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond @@ -748,7 +758,11 @@ func verifyPresence(t *testing.T, tSets map[poolKey]map[string]*targetgroup.Grou func TestTargetSetRecreatesTargetGroupsEveryRun(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry()) + + reg := prometheus.NewRegistry() + _, sdMetrics := newTestMetrics(t, reg) + + discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() @@ -777,7 +791,11 @@ func TestTargetSetRecreatesTargetGroupsEveryRun(t *testing.T) { func TestDiscovererConfigs(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry()) + + reg := prometheus.NewRegistry() + _, sdMetrics := newTestMetrics(t, reg) + + discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() @@ -802,7 +820,11 @@ func TestDiscovererConfigs(t *testing.T) { func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry()) + + reg := prometheus.NewRegistry() + _, sdMetrics := newTestMetrics(t, reg) + + discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() @@ -842,7 +864,11 @@ func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) { func TestIdenticalConfigurationsAreCoalesced(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - discoveryManager := NewManager(ctx, nil, prometheus.NewRegistry()) + + reg := prometheus.NewRegistry() + _, sdMetrics := newTestMetrics(t, reg) + + discoveryManager := NewManager(ctx, nil, reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() @@ -874,7 +900,11 @@ func TestApplyConfigDoesNotModifyStaticTargets(t *testing.T) { } ctx, cancel := context.WithCancel(context.Background()) defer cancel() - discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry()) + + reg := prometheus.NewRegistry() + _, sdMetrics := newTestMetrics(t, reg) + + discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() @@ -897,10 +927,19 @@ func (e errorConfig) NewDiscoverer(discovery.DiscovererOptions) (discovery.Disco return nil, e.err } +// NewDiscovererMetrics implements discovery.Config. +func (errorConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &discovery.NoopDiscovererMetrics{} +} + func TestGaugeFailedConfigs(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry()) + + reg := prometheus.NewRegistry() + _, sdMetrics := newTestMetrics(t, reg) + + discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() @@ -1057,7 +1096,10 @@ func TestCoordinationWithReceiver(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - mgr := NewManager(ctx, nil, prometheus.NewRegistry()) + reg := prometheus.NewRegistry() + _, sdMetrics := newTestMetrics(t, reg) + + mgr := NewManager(ctx, nil, reg, sdMetrics) require.NotNil(t, mgr) mgr.updatert = updateDelay go mgr.Run() diff --git a/discovery/linode/linode.go b/discovery/linode/linode.go index 38a5cdad4..94f0a63bb 100644 --- a/discovery/linode/linode.go +++ b/discovery/linode/linode.go @@ -87,12 +87,17 @@ type SDConfig struct { TagSeparator string `yaml:"tag_separator,omitempty"` } +// NewDiscovererMetrics implements discovery.Config. +func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return newDiscovererMetrics(reg, rmi) +} + // Name returns the name of the Config. func (*SDConfig) Name() string { return "linode" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger, opts.Registerer) + return NewDiscovery(c, opts.Logger, opts.Metrics) } // SetDirectory joins any relative file paths with dir. @@ -122,22 +127,23 @@ type Discovery struct { pollCount int lastResults []*targetgroup.Group eventPollingEnabled bool - failuresCount prometheus.Counter + metrics *linodeMetrics } // NewDiscovery returns a new Discovery which periodically refreshes its targets. -func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) { +func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { + m, ok := metrics.(*linodeMetrics) + if !ok { + return nil, fmt.Errorf("invalid discovery metrics type") + } + d := &Discovery{ port: conf.Port, tagSeparator: conf.TagSeparator, pollCount: 0, lastRefreshTimestamp: time.Now().UTC(), eventPollingEnabled: true, - failuresCount: prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "prometheus_sd_linode_failures_total", - Help: "Number of Linode service discovery refresh failures.", - }), + metrics: m, } rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "linode_sd") @@ -156,12 +162,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, - Mech: "linode", - Interval: time.Duration(conf.RefreshInterval), - RefreshF: d.refresh, - Registry: reg, - Metrics: []prometheus.Collector{d.failuresCount}, + Logger: logger, + Mech: "linode", + Interval: time.Duration(conf.RefreshInterval), + RefreshF: d.refresh, + MetricsInstantiator: m.refreshMetrics, }, ) return d, nil @@ -223,14 +228,14 @@ func (d *Discovery) refreshData(ctx context.Context) ([]*targetgroup.Group, erro // Gather all linode instances. instances, err := d.client.ListInstances(ctx, &linodego.ListOptions{PageSize: 500}) if err != nil { - d.failuresCount.Inc() + d.metrics.failuresCount.Inc() return nil, err } // Gather detailed IP address info for all IPs on all linode instances. detailedIPs, err := d.client.ListIPAddresses(ctx, &linodego.ListOptions{PageSize: 500}) if err != nil { - d.failuresCount.Inc() + d.metrics.failuresCount.Inc() return nil, err } diff --git a/discovery/linode/linode_test.go b/discovery/linode/linode_test.go index 536b12090..a6a16d82a 100644 --- a/discovery/linode/linode_test.go +++ b/discovery/linode/linode_test.go @@ -24,6 +24,8 @@ import ( "github.com/prometheus/common/config" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" + + "github.com/prometheus/prometheus/discovery" ) type LinodeSDTestSuite struct { @@ -53,7 +55,15 @@ func TestLinodeSDRefresh(t *testing.T) { Credentials: tokenID, Type: "Bearer", } - d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry()) + + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + defer metrics.Unregister() + defer refreshMetrics.Unregister() + + d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics) require.NoError(t, err) endpoint, err := url.Parse(sdmock.Mock.Endpoint()) require.NoError(t, err) diff --git a/discovery/linode/metrics.go b/discovery/linode/metrics.go new file mode 100644 index 000000000..8f8138922 --- /dev/null +++ b/discovery/linode/metrics.go @@ -0,0 +1,57 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linode + +import ( + "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/prometheus/discovery" +) + +var _ discovery.DiscovererMetrics = (*linodeMetrics)(nil) + +type linodeMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator + + failuresCount prometheus.Counter + + metricRegisterer discovery.MetricRegisterer +} + +func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + m := &linodeMetrics{ + refreshMetrics: rmi, + failuresCount: prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "prometheus_sd_linode_failures_total", + Help: "Number of Linode service discovery refresh failures.", + }), + } + + m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{ + m.failuresCount, + }) + + return m +} + +// Register implements discovery.DiscovererMetrics. +func (m *linodeMetrics) Register() error { + return m.metricRegisterer.RegisterMetrics() +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *linodeMetrics) Unregister() { + m.metricRegisterer.UnregisterMetrics() +} diff --git a/discovery/manager.go b/discovery/manager.go index 67e326c41..e3a263557 100644 --- a/discovery/manager.go +++ b/discovery/manager.go @@ -64,8 +64,24 @@ func (p *Provider) Config() interface{} { return p.config } +// Registers the metrics needed for SD mechanisms. +// Does not register the metrics for the Discovery Manager. +// TODO(ptodev): Add ability to unregister the metrics? +func CreateAndRegisterSDMetrics(reg prometheus.Registerer) (map[string]DiscovererMetrics, error) { + // Some SD mechanisms use the "refresh" package, which has its own metrics. + refreshSdMetrics := NewRefreshMetrics(reg) + + // Register the metrics specific for each SD mechanism, and the ones for the refresh package. + sdMetrics, err := RegisterSDMetrics(reg, refreshSdMetrics) + if err != nil { + return nil, fmt.Errorf("failed to register service discovery metrics: %w", err) + } + + return sdMetrics, nil +} + // NewManager is the Discovery Manager constructor. -func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Registerer, options ...func(*Manager)) *Manager { +func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Registerer, sdMetrics map[string]DiscovererMetrics, options ...func(*Manager)) *Manager { if logger == nil { logger = log.NewNopLogger() } @@ -77,6 +93,7 @@ func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Re updatert: 5 * time.Second, triggerSend: make(chan struct{}, 1), registerer: registerer, + sdMetrics: sdMetrics, } for _, option := range options { option(mgr) @@ -84,7 +101,7 @@ func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Re // Register the metrics. // We have to do this after setting all options, so that the name of the Manager is set. - if metrics, err := NewMetrics(registerer, mgr.name); err == nil { + if metrics, err := NewManagerMetrics(registerer, mgr.name); err == nil { mgr.metrics = metrics } else { level.Error(logger).Log("msg", "Failed to create discovery manager metrics", "manager", mgr.name, "err", err) @@ -143,7 +160,8 @@ type Manager struct { // A registerer for all service discovery metrics. registerer prometheus.Registerer - metrics *Metrics + metrics *Metrics + sdMetrics map[string]DiscovererMetrics } // Providers returns the currently configured SD providers. @@ -402,7 +420,7 @@ func (m *Manager) registerProviders(cfgs Configs, setName string) int { d, err := cfg.NewDiscoverer(DiscovererOptions{ Logger: log.With(m.logger, "discovery", typ, "config", setName), HTTPClientOptions: m.httpOpts, - Registerer: m.registerer, + Metrics: m.sdMetrics[typ], }) if err != nil { level.Error(m.logger).Log("msg", "Cannot create service discovery", "err", err, "type", typ, "config", setName) diff --git a/discovery/manager_test.go b/discovery/manager_test.go index f22de75a4..2f6e03aa5 100644 --- a/discovery/manager_test.go +++ b/discovery/manager_test.go @@ -36,6 +36,13 @@ func TestMain(m *testing.M) { testutil.TolerantVerifyLeak(m) } +func NewTestMetrics(t *testing.T, reg prometheus.Registerer) (*RefreshMetricsManager, map[string]DiscovererMetrics) { + refreshMetrics := NewRefreshMetrics(reg) + sdMetrics, err := RegisterSDMetrics(reg, refreshMetrics) + require.NoError(t, err) + return &refreshMetrics, sdMetrics +} + // TestTargetUpdatesOrder checks that the target updates are received in the expected order. func TestTargetUpdatesOrder(t *testing.T) { // The order by which the updates are send is determined by the interval passed to the mock discovery adapter @@ -665,7 +672,10 @@ func TestTargetUpdatesOrder(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry()) + reg := prometheus.NewRegistry() + _, sdMetrics := NewTestMetrics(t, reg) + + discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond @@ -780,7 +790,11 @@ func pk(provider, setName string, n int) poolKey { func TestTargetSetTargetGroupsPresentOnConfigReload(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry()) + + reg := prometheus.NewRegistry() + _, sdMetrics := NewTestMetrics(t, reg) + + discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() @@ -813,7 +827,11 @@ func TestTargetSetTargetGroupsPresentOnConfigReload(t *testing.T) { func TestTargetSetTargetGroupsPresentOnConfigRename(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry()) + + reg := prometheus.NewRegistry() + _, sdMetrics := NewTestMetrics(t, reg) + + discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() @@ -849,7 +867,11 @@ func TestTargetSetTargetGroupsPresentOnConfigRename(t *testing.T) { func TestTargetSetTargetGroupsPresentOnConfigDuplicateAndDeleteOriginal(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry()) + + reg := prometheus.NewRegistry() + _, sdMetrics := NewTestMetrics(t, reg) + + discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() @@ -888,7 +910,11 @@ func TestTargetSetTargetGroupsPresentOnConfigDuplicateAndDeleteOriginal(t *testi func TestTargetSetTargetGroupsPresentOnConfigChange(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry()) + + reg := prometheus.NewRegistry() + _, sdMetrics := NewTestMetrics(t, reg) + + discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() @@ -950,7 +976,11 @@ func TestTargetSetTargetGroupsPresentOnConfigChange(t *testing.T) { func TestTargetSetRecreatesTargetGroupsOnConfigChange(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry()) + + reg := prometheus.NewRegistry() + _, sdMetrics := NewTestMetrics(t, reg) + + discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() @@ -990,7 +1020,11 @@ func TestTargetSetRecreatesTargetGroupsOnConfigChange(t *testing.T) { func TestDiscovererConfigs(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry()) + + reg := prometheus.NewRegistry() + _, sdMetrics := NewTestMetrics(t, reg) + + discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() @@ -1023,7 +1057,11 @@ func TestDiscovererConfigs(t *testing.T) { func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry()) + + reg := prometheus.NewRegistry() + _, sdMetrics := NewTestMetrics(t, reg) + + discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() @@ -1071,7 +1109,11 @@ func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) { func TestIdenticalConfigurationsAreCoalesced(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - discoveryManager := NewManager(ctx, nil, prometheus.NewRegistry()) + + reg := prometheus.NewRegistry() + _, sdMetrics := NewTestMetrics(t, reg) + + discoveryManager := NewManager(ctx, nil, reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() @@ -1108,7 +1150,11 @@ func TestApplyConfigDoesNotModifyStaticTargets(t *testing.T) { } ctx, cancel := context.WithCancel(context.Background()) defer cancel() - discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry()) + + reg := prometheus.NewRegistry() + _, sdMetrics := NewTestMetrics(t, reg) + + discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() @@ -1129,11 +1175,21 @@ type errorConfig struct{ err error } func (e errorConfig) Name() string { return "error" } func (e errorConfig) NewDiscoverer(DiscovererOptions) (Discoverer, error) { return nil, e.err } +// NewDiscovererMetrics implements discovery.Config. +func (errorConfig) NewDiscovererMetrics(prometheus.Registerer, RefreshMetricsInstantiator) DiscovererMetrics { + return &NoopDiscovererMetrics{} +} + type lockStaticConfig struct { mu *sync.Mutex config StaticConfig } +// NewDiscovererMetrics implements discovery.Config. +func (lockStaticConfig) NewDiscovererMetrics(prometheus.Registerer, RefreshMetricsInstantiator) DiscovererMetrics { + return &NoopDiscovererMetrics{} +} + func (s lockStaticConfig) Name() string { return "lockstatic" } func (s lockStaticConfig) NewDiscoverer(DiscovererOptions) (Discoverer, error) { return (lockStaticDiscoverer)(s), nil @@ -1155,7 +1211,11 @@ func (s lockStaticDiscoverer) Run(ctx context.Context, up chan<- []*targetgroup. func TestGaugeFailedConfigs(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry()) + + reg := prometheus.NewRegistry() + _, sdMetrics := NewTestMetrics(t, reg) + + discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() @@ -1312,7 +1372,10 @@ func TestCoordinationWithReceiver(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - mgr := NewManager(ctx, nil, prometheus.NewRegistry()) + reg := prometheus.NewRegistry() + _, sdMetrics := NewTestMetrics(t, reg) + + mgr := NewManager(ctx, nil, reg, sdMetrics) require.NotNil(t, mgr) mgr.updatert = updateDelay go mgr.Run() @@ -1408,7 +1471,11 @@ func (o onceProvider) Run(_ context.Context, ch chan<- []*targetgroup.Group) { func TestTargetSetTargetGroupsUpdateDuringApplyConfig(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry()) + + reg := prometheus.NewRegistry() + _, sdMetrics := NewTestMetrics(t, reg) + + discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() @@ -1470,6 +1537,11 @@ func newTestDiscoverer() *testDiscoverer { } } +// NewDiscovererMetrics implements discovery.Config. +func (*testDiscoverer) NewDiscovererMetrics(prometheus.Registerer, RefreshMetricsInstantiator) DiscovererMetrics { + return &NoopDiscovererMetrics{} +} + // Name implements Config. func (t *testDiscoverer) Name() string { return "test" diff --git a/discovery/marathon/marathon.go b/discovery/marathon/marathon.go index a6a6252fd..ecad108e4 100644 --- a/discovery/marathon/marathon.go +++ b/discovery/marathon/marathon.go @@ -79,12 +79,19 @@ type SDConfig struct { HTTPClientConfig config.HTTPClientConfig `yaml:",inline"` } +// NewDiscovererMetrics implements discovery.Config. +func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &marathonMetrics{ + refreshMetrics: rmi, + } +} + // Name returns the name of the Config. func (*SDConfig) Name() string { return "marathon" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(*c, opts.Logger, opts.Registerer) + return NewDiscovery(*c, opts.Logger, opts.Metrics) } // SetDirectory joins any relative file paths with dir. @@ -133,7 +140,12 @@ type Discovery struct { } // NewDiscovery returns a new Marathon Discovery. -func NewDiscovery(conf SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) { +func NewDiscovery(conf SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { + m, ok := metrics.(*marathonMetrics) + if !ok { + return nil, fmt.Errorf("invalid discovery metrics type") + } + rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "marathon_sd") if err != nil { return nil, err @@ -156,11 +168,11 @@ func NewDiscovery(conf SDConfig, logger log.Logger, reg prometheus.Registerer) ( } d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, - Mech: "marathon", - Interval: time.Duration(conf.RefreshInterval), - RefreshF: d.refresh, - Registry: reg, + Logger: logger, + Mech: "marathon", + Interval: time.Duration(conf.RefreshInterval), + RefreshF: d.refresh, + MetricsInstantiator: m.refreshMetrics, }, ) return d, nil diff --git a/discovery/marathon/marathon_test.go b/discovery/marathon/marathon_test.go index a1ddce930..c78cc1e3c 100644 --- a/discovery/marathon/marathon_test.go +++ b/discovery/marathon/marathon_test.go @@ -23,7 +23,9 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" + "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/targetgroup" ) @@ -37,7 +39,19 @@ func testConfig() SDConfig { } func testUpdateServices(client appListClient) ([]*targetgroup.Group, error) { - md, err := NewDiscovery(testConfig(), nil, prometheus.NewRegistry()) + cfg := testConfig() + + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) + err := metrics.Register() + if err != nil { + return nil, err + } + defer metrics.Unregister() + defer refreshMetrics.Unregister() + + md, err := NewDiscovery(cfg, nil, metrics) if err != nil { return nil, err } @@ -130,7 +144,15 @@ func TestMarathonSDSendGroup(t *testing.T) { } func TestMarathonSDRemoveApp(t *testing.T) { - md, err := NewDiscovery(testConfig(), nil, prometheus.NewRegistry()) + cfg := testConfig() + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + defer metrics.Unregister() + defer refreshMetrics.Unregister() + + md, err := NewDiscovery(cfg, nil, metrics) if err != nil { t.Fatalf("%s", err) } diff --git a/discovery/marathon/metrics.go b/discovery/marathon/metrics.go new file mode 100644 index 000000000..790c16747 --- /dev/null +++ b/discovery/marathon/metrics.go @@ -0,0 +1,32 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package marathon + +import ( + "github.com/prometheus/prometheus/discovery" +) + +var _ discovery.DiscovererMetrics = (*marathonMetrics)(nil) + +type marathonMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator +} + +// Register implements discovery.DiscovererMetrics. +func (m *marathonMetrics) Register() error { + return nil +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *marathonMetrics) Unregister() {} diff --git a/discovery/metrics.go b/discovery/metrics.go index 6a6060395..a77b86d27 100644 --- a/discovery/metrics.go +++ b/discovery/metrics.go @@ -38,7 +38,7 @@ type Metrics struct { SentUpdates prometheus.Counter } -func NewMetrics(registerer prometheus.Registerer, sdManagerName string) (*Metrics, error) { +func NewManagerMetrics(registerer prometheus.Registerer, sdManagerName string) (*Metrics, error) { m := &Metrics{} m.FailedConfigs = prometheus.NewGauge( diff --git a/discovery/metrics_refresh.go b/discovery/metrics_refresh.go new file mode 100644 index 000000000..d621165ce --- /dev/null +++ b/discovery/metrics_refresh.go @@ -0,0 +1,75 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package discovery + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +// Metric vectors for the "refresh" package. +// We define them here in the "discovery" package in order to avoid a cyclic dependency between +// "discovery" and "refresh". +type RefreshMetricsVecs struct { + failuresVec *prometheus.CounterVec + durationVec *prometheus.SummaryVec + + metricRegisterer MetricRegisterer +} + +var _ RefreshMetricsManager = (*RefreshMetricsVecs)(nil) + +func NewRefreshMetrics(reg prometheus.Registerer) RefreshMetricsManager { + m := &RefreshMetricsVecs{ + failuresVec: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "prometheus_sd_refresh_failures_total", + Help: "Number of refresh failures for the given SD mechanism.", + }, + []string{"mechanism"}), + durationVec: prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Name: "prometheus_sd_refresh_duration_seconds", + Help: "The duration of a refresh in seconds for the given SD mechanism.", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, + []string{"mechanism"}), + } + + // The reason we register metric vectors instead of metrics is so that + // the metrics are not visible until they are recorded. + m.metricRegisterer = NewMetricRegisterer(reg, []prometheus.Collector{ + m.failuresVec, + m.durationVec, + }) + + return m +} + +// Instantiate returns metrics out of metric vectors. +func (m *RefreshMetricsVecs) Instantiate(mech string) *RefreshMetrics { + return &RefreshMetrics{ + Failures: m.failuresVec.WithLabelValues(mech), + Duration: m.durationVec.WithLabelValues(mech), + } +} + +// Register implements discovery.DiscovererMetrics. +func (m *RefreshMetricsVecs) Register() error { + return m.metricRegisterer.RegisterMetrics() +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *RefreshMetricsVecs) Unregister() { + m.metricRegisterer.UnregisterMetrics() +} diff --git a/discovery/moby/docker.go b/discovery/moby/docker.go index a13bb8704..57fa62bb5 100644 --- a/discovery/moby/docker.go +++ b/discovery/moby/docker.go @@ -76,12 +76,19 @@ type DockerSDConfig struct { RefreshInterval model.Duration `yaml:"refresh_interval"` } +// NewDiscovererMetrics implements discovery.Config. +func (*DockerSDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &dockerMetrics{ + refreshMetrics: rmi, + } +} + // Name returns the name of the Config. func (*DockerSDConfig) Name() string { return "docker" } // NewDiscoverer returns a Discoverer for the Config. func (c *DockerSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDockerDiscovery(c, opts.Logger, opts.Registerer) + return NewDockerDiscovery(c, opts.Logger, opts.Metrics) } // SetDirectory joins any relative file paths with dir. @@ -115,8 +122,11 @@ type DockerDiscovery struct { } // NewDockerDiscovery returns a new DockerDiscovery which periodically refreshes its targets. -func NewDockerDiscovery(conf *DockerSDConfig, logger log.Logger, reg prometheus.Registerer) (*DockerDiscovery, error) { - var err error +func NewDockerDiscovery(conf *DockerSDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*DockerDiscovery, error) { + m, ok := metrics.(*dockerMetrics) + if !ok { + return nil, fmt.Errorf("invalid discovery metrics type") + } d := &DockerDiscovery{ port: conf.Port, @@ -167,11 +177,11 @@ func NewDockerDiscovery(conf *DockerSDConfig, logger log.Logger, reg prometheus. d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, - Mech: "docker", - Interval: time.Duration(conf.RefreshInterval), - RefreshF: d.refresh, - Registry: reg, + Logger: logger, + Mech: "docker", + Interval: time.Duration(conf.RefreshInterval), + RefreshF: d.refresh, + MetricsInstantiator: m.refreshMetrics, }, ) return d, nil diff --git a/discovery/moby/docker_test.go b/discovery/moby/docker_test.go index 1a87ad2a1..fec56d3e5 100644 --- a/discovery/moby/docker_test.go +++ b/discovery/moby/docker_test.go @@ -23,6 +23,8 @@ import ( "github.com/prometheus/common/model" "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" + + "github.com/prometheus/prometheus/discovery" ) func TestDockerSDRefresh(t *testing.T) { @@ -38,7 +40,14 @@ host: %s var cfg DockerSDConfig require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg)) - d, err := NewDockerDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry()) + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + defer metrics.Unregister() + defer refreshMetrics.Unregister() + + d, err := NewDockerDiscovery(&cfg, log.NewNopLogger(), metrics) require.NoError(t, err) ctx := context.Background() diff --git a/discovery/moby/dockerswarm.go b/discovery/moby/dockerswarm.go index bd87fea5a..b0147467d 100644 --- a/discovery/moby/dockerswarm.go +++ b/discovery/moby/dockerswarm.go @@ -70,12 +70,19 @@ type Filter struct { Values []string `yaml:"values"` } +// NewDiscovererMetrics implements discovery.Config. +func (*DockerSwarmSDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &dockerswarmMetrics{ + refreshMetrics: rmi, + } +} + // Name returns the name of the Config. func (*DockerSwarmSDConfig) Name() string { return "dockerswarm" } // NewDiscoverer returns a Discoverer for the Config. func (c *DockerSwarmSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger, opts.Registerer) + return NewDiscovery(c, opts.Logger, opts.Metrics) } // SetDirectory joins any relative file paths with dir. @@ -118,8 +125,11 @@ type Discovery struct { } // NewDiscovery returns a new Discovery which periodically refreshes its targets. -func NewDiscovery(conf *DockerSwarmSDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) { - var err error +func NewDiscovery(conf *DockerSwarmSDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { + m, ok := metrics.(*dockerswarmMetrics) + if !ok { + return nil, fmt.Errorf("invalid discovery metrics type") + } d := &Discovery{ port: conf.Port, @@ -170,11 +180,11 @@ func NewDiscovery(conf *DockerSwarmSDConfig, logger log.Logger, reg prometheus.R d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, - Mech: "dockerswarm", - Interval: time.Duration(conf.RefreshInterval), - RefreshF: d.refresh, - Registry: reg, + Logger: logger, + Mech: "dockerswarm", + Interval: time.Duration(conf.RefreshInterval), + RefreshF: d.refresh, + MetricsInstantiator: m.refreshMetrics, }, ) return d, nil diff --git a/discovery/moby/metrics_docker.go b/discovery/moby/metrics_docker.go new file mode 100644 index 000000000..457d520a0 --- /dev/null +++ b/discovery/moby/metrics_docker.go @@ -0,0 +1,32 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package moby + +import ( + "github.com/prometheus/prometheus/discovery" +) + +var _ discovery.DiscovererMetrics = (*dockerMetrics)(nil) + +type dockerMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator +} + +// Register implements discovery.DiscovererMetrics. +func (m *dockerMetrics) Register() error { + return nil +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *dockerMetrics) Unregister() {} diff --git a/discovery/moby/metrics_dockerswarm.go b/discovery/moby/metrics_dockerswarm.go new file mode 100644 index 000000000..227ba6c2b --- /dev/null +++ b/discovery/moby/metrics_dockerswarm.go @@ -0,0 +1,32 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package moby + +import ( + "github.com/prometheus/prometheus/discovery" +) + +var _ discovery.DiscovererMetrics = (*dockerswarmMetrics)(nil) + +type dockerswarmMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator +} + +// Register implements discovery.DiscovererMetrics. +func (m *dockerswarmMetrics) Register() error { + return nil +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *dockerswarmMetrics) Unregister() {} diff --git a/discovery/moby/nodes_test.go b/discovery/moby/nodes_test.go index 512ff7049..4ad1088d1 100644 --- a/discovery/moby/nodes_test.go +++ b/discovery/moby/nodes_test.go @@ -23,6 +23,8 @@ import ( "github.com/prometheus/common/model" "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" + + "github.com/prometheus/prometheus/discovery" ) func TestDockerSwarmNodesSDRefresh(t *testing.T) { @@ -39,7 +41,14 @@ host: %s var cfg DockerSwarmSDConfig require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg)) - d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry()) + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + defer metrics.Unregister() + defer refreshMetrics.Unregister() + + d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics) require.NoError(t, err) ctx := context.Background() diff --git a/discovery/moby/services_test.go b/discovery/moby/services_test.go index 816586dd7..47ca69e33 100644 --- a/discovery/moby/services_test.go +++ b/discovery/moby/services_test.go @@ -23,6 +23,8 @@ import ( "github.com/prometheus/common/model" "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" + + "github.com/prometheus/prometheus/discovery" ) func TestDockerSwarmSDServicesRefresh(t *testing.T) { @@ -39,7 +41,14 @@ host: %s var cfg DockerSwarmSDConfig require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg)) - d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry()) + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + defer metrics.Unregister() + defer refreshMetrics.Unregister() + + d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics) require.NoError(t, err) ctx := context.Background() @@ -333,7 +342,14 @@ filters: var cfg DockerSwarmSDConfig require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg)) - d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry()) + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + defer metrics.Unregister() + defer refreshMetrics.Unregister() + + d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics) require.NoError(t, err) ctx := context.Background() diff --git a/discovery/moby/tasks_test.go b/discovery/moby/tasks_test.go index 764fda343..ef71bc02f 100644 --- a/discovery/moby/tasks_test.go +++ b/discovery/moby/tasks_test.go @@ -23,6 +23,8 @@ import ( "github.com/prometheus/common/model" "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" + + "github.com/prometheus/prometheus/discovery" ) func TestDockerSwarmTasksSDRefresh(t *testing.T) { @@ -39,7 +41,14 @@ host: %s var cfg DockerSwarmSDConfig require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg)) - d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry()) + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + defer metrics.Unregister() + defer refreshMetrics.Unregister() + + d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics) require.NoError(t, err) ctx := context.Background() diff --git a/discovery/nomad/metrics.go b/discovery/nomad/metrics.go new file mode 100644 index 000000000..9707153d9 --- /dev/null +++ b/discovery/nomad/metrics.go @@ -0,0 +1,57 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nomad + +import ( + "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/prometheus/discovery" +) + +var _ discovery.DiscovererMetrics = (*nomadMetrics)(nil) + +type nomadMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator + + failuresCount prometheus.Counter + + metricRegisterer discovery.MetricRegisterer +} + +func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + m := &nomadMetrics{ + refreshMetrics: rmi, + failuresCount: prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "prometheus_sd_nomad_failures_total", + Help: "Number of nomad service discovery refresh failures.", + }), + } + + m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{ + m.failuresCount, + }) + + return m +} + +// Register implements discovery.DiscovererMetrics. +func (m *nomadMetrics) Register() error { + return m.metricRegisterer.RegisterMetrics() +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *nomadMetrics) Unregister() { + m.metricRegisterer.UnregisterMetrics() +} diff --git a/discovery/nomad/nomad.go b/discovery/nomad/nomad.go index 3fdcf714e..d9c48120a 100644 --- a/discovery/nomad/nomad.go +++ b/discovery/nomad/nomad.go @@ -74,12 +74,17 @@ type SDConfig struct { TagSeparator string `yaml:"tag_separator,omitempty"` } +// NewDiscovererMetrics implements discovery.Config. +func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return newDiscovererMetrics(reg, rmi) +} + // Name returns the name of the Config. func (*SDConfig) Name() string { return "nomad" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger, opts.Registerer) + return NewDiscovery(c, opts.Logger, opts.Metrics) } // SetDirectory joins any relative file paths with dir. @@ -112,11 +117,16 @@ type Discovery struct { region string server string tagSeparator string - failuresCount prometheus.Counter + metrics *nomadMetrics } // NewDiscovery returns a new Discovery which periodically refreshes its targets. -func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) { +func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { + m, ok := metrics.(*nomadMetrics) + if !ok { + return nil, fmt.Errorf("invalid discovery metrics type") + } + d := &Discovery{ allowStale: conf.AllowStale, namespace: conf.Namespace, @@ -124,11 +134,7 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) region: conf.Region, server: conf.Server, tagSeparator: conf.TagSeparator, - failuresCount: prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "prometheus_sd_nomad_failures_total", - Help: "Number of nomad service discovery refresh failures.", - }), + metrics: m, } HTTPClient, err := config.NewClientFromConfig(conf.HTTPClientConfig, "nomad_sd") @@ -151,12 +157,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, - Mech: "nomad", - Interval: time.Duration(conf.RefreshInterval), - RefreshF: d.refresh, - Registry: reg, - Metrics: []prometheus.Collector{d.failuresCount}, + Logger: logger, + Mech: "nomad", + Interval: time.Duration(conf.RefreshInterval), + RefreshF: d.refresh, + MetricsInstantiator: m.refreshMetrics, }, ) return d, nil @@ -168,7 +173,7 @@ func (d *Discovery) refresh(context.Context) ([]*targetgroup.Group, error) { } stubs, _, err := d.client.Services().List(opts) if err != nil { - d.failuresCount.Inc() + d.metrics.failuresCount.Inc() return nil, err } @@ -180,7 +185,7 @@ func (d *Discovery) refresh(context.Context) ([]*targetgroup.Group, error) { for _, service := range stub.Services { instances, _, err := d.client.Services().Get(service.ServiceName, opts) if err != nil { - d.failuresCount.Inc() + d.metrics.failuresCount.Inc() return nil, fmt.Errorf("failed to fetch services: %w", err) } diff --git a/discovery/nomad/nomad_test.go b/discovery/nomad/nomad_test.go index ca67a877e..357d4a8e9 100644 --- a/discovery/nomad/nomad_test.go +++ b/discovery/nomad/nomad_test.go @@ -25,6 +25,8 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" + + "github.com/prometheus/prometheus/discovery" ) type NomadSDTestSuite struct { @@ -128,8 +130,16 @@ func TestConfiguredService(t *testing.T) { conf := &SDConfig{ Server: "http://localhost:4646", } - _, err := NewDiscovery(conf, nil, prometheus.NewRegistry()) + + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := conf.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + + _, err := NewDiscovery(conf, nil, metrics) require.NoError(t, err) + + metrics.Unregister() } func TestNomadSDRefresh(t *testing.T) { @@ -142,7 +152,15 @@ func TestNomadSDRefresh(t *testing.T) { cfg := DefaultSDConfig cfg.Server = endpoint.String() - d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry()) + + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + defer metrics.Unregister() + defer refreshMetrics.Unregister() + + d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics) require.NoError(t, err) tgs, err := d.refresh(context.Background()) diff --git a/discovery/openstack/metrics.go b/discovery/openstack/metrics.go new file mode 100644 index 000000000..a64c1a673 --- /dev/null +++ b/discovery/openstack/metrics.go @@ -0,0 +1,32 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openstack + +import ( + "github.com/prometheus/prometheus/discovery" +) + +type openstackMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator +} + +var _ discovery.DiscovererMetrics = (*openstackMetrics)(nil) + +// Register implements discovery.DiscovererMetrics. +func (m *openstackMetrics) Register() error { + return nil +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *openstackMetrics) Unregister() {} diff --git a/discovery/openstack/openstack.go b/discovery/openstack/openstack.go index 9544a7c0f..c98f78788 100644 --- a/discovery/openstack/openstack.go +++ b/discovery/openstack/openstack.go @@ -66,12 +66,19 @@ type SDConfig struct { Availability string `yaml:"availability,omitempty"` } +// NewDiscovererMetrics implements discovery.Config. +func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &openstackMetrics{ + refreshMetrics: rmi, + } +} + // Name returns the name of the Config. func (*SDConfig) Name() string { return "openstack" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger, opts.Registerer) + return NewDiscovery(c, opts.Logger, opts.Metrics) } // SetDirectory joins any relative file paths with dir. @@ -135,18 +142,23 @@ type refresher interface { } // NewDiscovery returns a new OpenStack Discoverer which periodically refreshes its targets. -func NewDiscovery(conf *SDConfig, l log.Logger, reg prometheus.Registerer) (*refresh.Discovery, error) { +func NewDiscovery(conf *SDConfig, l log.Logger, metrics discovery.DiscovererMetrics) (*refresh.Discovery, error) { + m, ok := metrics.(*openstackMetrics) + if !ok { + return nil, fmt.Errorf("invalid discovery metrics type") + } + r, err := newRefresher(conf, l) if err != nil { return nil, err } return refresh.NewDiscovery( refresh.Options{ - Logger: l, - Mech: "openstack", - Interval: time.Duration(conf.RefreshInterval), - RefreshF: r.refresh, - Registry: reg, + Logger: l, + Mech: "openstack", + Interval: time.Duration(conf.RefreshInterval), + RefreshF: r.refresh, + MetricsInstantiator: m.refreshMetrics, }, ), nil } diff --git a/discovery/ovhcloud/metrics.go b/discovery/ovhcloud/metrics.go new file mode 100644 index 000000000..1c5170946 --- /dev/null +++ b/discovery/ovhcloud/metrics.go @@ -0,0 +1,32 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ovhcloud + +import ( + "github.com/prometheus/prometheus/discovery" +) + +var _ discovery.DiscovererMetrics = (*ovhcloudMetrics)(nil) + +type ovhcloudMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator +} + +// Register implements discovery.DiscovererMetrics. +func (m *ovhcloudMetrics) Register() error { + return nil +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *ovhcloudMetrics) Unregister() {} diff --git a/discovery/ovhcloud/ovhcloud.go b/discovery/ovhcloud/ovhcloud.go index eca284a85..988b4482f 100644 --- a/discovery/ovhcloud/ovhcloud.go +++ b/discovery/ovhcloud/ovhcloud.go @@ -53,6 +53,13 @@ type SDConfig struct { Service string `yaml:"service"` } +// NewDiscovererMetrics implements discovery.Config. +func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &ovhcloudMetrics{ + refreshMetrics: rmi, + } +} + // Name implements the Discoverer interface. func (c SDConfig) Name() string { return "ovhcloud" @@ -94,7 +101,7 @@ func createClient(config *SDConfig) (*ovh.Client, error) { // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(options discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, options.Logger, options.Registerer) + return NewDiscovery(c, options.Logger, options.Metrics) } func init() { @@ -141,7 +148,12 @@ func newRefresher(conf *SDConfig, logger log.Logger) (refresher, error) { } // NewDiscovery returns a new OVHcloud Discoverer which periodically refreshes its targets. -func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*refresh.Discovery, error) { +func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*refresh.Discovery, error) { + m, ok := metrics.(*ovhcloudMetrics) + if !ok { + return nil, fmt.Errorf("invalid discovery metrics type") + } + r, err := newRefresher(conf, logger) if err != nil { return nil, err @@ -149,11 +161,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) return refresh.NewDiscovery( refresh.Options{ - Logger: logger, - Mech: "ovhcloud", - Interval: time.Duration(conf.RefreshInterval), - RefreshF: r.refresh, - Registry: reg, + Logger: logger, + Mech: "ovhcloud", + Interval: time.Duration(conf.RefreshInterval), + RefreshF: r.refresh, + MetricsInstantiator: m.refreshMetrics, }, ), nil } diff --git a/discovery/ovhcloud/ovhcloud_test.go b/discovery/ovhcloud/ovhcloud_test.go index 9bd9ea954..53ec9b445 100644 --- a/discovery/ovhcloud/ovhcloud_test.go +++ b/discovery/ovhcloud/ovhcloud_test.go @@ -122,9 +122,17 @@ func TestParseIPs(t *testing.T) { func TestDiscoverer(t *testing.T) { conf, _ := getMockConf("vps") logger := testutil.NewLogger(t) + + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := conf.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + defer metrics.Unregister() + defer refreshMetrics.Unregister() + _, err := conf.NewDiscoverer(discovery.DiscovererOptions{ - Logger: logger, - Registerer: prometheus.NewRegistry(), + Logger: logger, + Metrics: metrics, }) require.NoError(t, err) diff --git a/discovery/puppetdb/metrics.go b/discovery/puppetdb/metrics.go new file mode 100644 index 000000000..2f5faa57c --- /dev/null +++ b/discovery/puppetdb/metrics.go @@ -0,0 +1,32 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package puppetdb + +import ( + "github.com/prometheus/prometheus/discovery" +) + +var _ discovery.DiscovererMetrics = (*puppetdbMetrics)(nil) + +type puppetdbMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator +} + +// Register implements discovery.DiscovererMetrics. +func (m *puppetdbMetrics) Register() error { + return nil +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *puppetdbMetrics) Unregister() {} diff --git a/discovery/puppetdb/puppetdb.go b/discovery/puppetdb/puppetdb.go index 616f2c61e..4355a7ade 100644 --- a/discovery/puppetdb/puppetdb.go +++ b/discovery/puppetdb/puppetdb.go @@ -79,12 +79,19 @@ type SDConfig struct { Port int `yaml:"port"` } +// NewDiscovererMetrics implements discovery.Config. +func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &puppetdbMetrics{ + refreshMetrics: rmi, + } +} + // Name returns the name of the Config. func (*SDConfig) Name() string { return "puppetdb" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger, opts.Registerer) + return NewDiscovery(c, opts.Logger, opts.Metrics) } // SetDirectory joins any relative file paths with dir. @@ -131,7 +138,12 @@ type Discovery struct { } // NewDiscovery returns a new PuppetDB discovery for the given config. -func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) { +func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { + m, ok := metrics.(*puppetdbMetrics) + if !ok { + return nil, fmt.Errorf("invalid discovery metrics type") + } + if logger == nil { logger = log.NewNopLogger() } @@ -158,11 +170,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, - Mech: "http", - Interval: time.Duration(conf.RefreshInterval), - RefreshF: d.refresh, - Registry: reg, + Logger: logger, + Mech: "http", + Interval: time.Duration(conf.RefreshInterval), + RefreshF: d.refresh, + MetricsInstantiator: m.refreshMetrics, }, ) return d, nil diff --git a/discovery/puppetdb/puppetdb_test.go b/discovery/puppetdb/puppetdb_test.go index edd9b9d04..bf9c7b215 100644 --- a/discovery/puppetdb/puppetdb_test.go +++ b/discovery/puppetdb/puppetdb_test.go @@ -28,6 +28,7 @@ import ( "github.com/prometheus/common/model" "github.com/stretchr/testify/require" + "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/targetgroup" ) @@ -63,9 +64,17 @@ func TestPuppetSlashInURL(t *testing.T) { Port: 80, RefreshInterval: model.Duration(30 * time.Second), } - d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry()) + + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + + d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics) require.NoError(t, err) require.Equal(t, apiURL, d.url) + + metrics.Unregister() } } @@ -80,7 +89,12 @@ func TestPuppetDBRefresh(t *testing.T) { RefreshInterval: model.Duration(30 * time.Second), } - d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry()) + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + + d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics) require.NoError(t, err) ctx := context.Background() @@ -107,6 +121,8 @@ func TestPuppetDBRefresh(t *testing.T) { }, } require.Equal(t, expectedTargets, tgs) + + metrics.Unregister() } func TestPuppetDBRefreshWithParameters(t *testing.T) { @@ -121,7 +137,12 @@ func TestPuppetDBRefreshWithParameters(t *testing.T) { RefreshInterval: model.Duration(30 * time.Second), } - d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry()) + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + + d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics) require.NoError(t, err) ctx := context.Background() @@ -158,6 +179,8 @@ func TestPuppetDBRefreshWithParameters(t *testing.T) { }, } require.Equal(t, expectedTargets, tgs) + + metrics.Unregister() } func TestPuppetDBInvalidCode(t *testing.T) { @@ -173,12 +196,19 @@ func TestPuppetDBInvalidCode(t *testing.T) { RefreshInterval: model.Duration(30 * time.Second), } - d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry()) + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + + d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics) require.NoError(t, err) ctx := context.Background() _, err = d.refresh(ctx) require.EqualError(t, err, "server returned HTTP status 400 Bad Request") + + metrics.Unregister() } func TestPuppetDBInvalidFormat(t *testing.T) { @@ -194,10 +224,17 @@ func TestPuppetDBInvalidFormat(t *testing.T) { RefreshInterval: model.Duration(30 * time.Second), } - d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry()) + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + + d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics) require.NoError(t, err) ctx := context.Background() _, err = d.refresh(ctx) require.EqualError(t, err, "unsupported content type text/plain; charset=utf-8") + + metrics.Unregister() } diff --git a/discovery/refresh/refresh.go b/discovery/refresh/refresh.go index 0b0e5a921..f037a90cf 100644 --- a/discovery/refresh/refresh.go +++ b/discovery/refresh/refresh.go @@ -20,19 +20,17 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/targetgroup" ) type Options struct { - Logger log.Logger - Mech string - Interval time.Duration - RefreshF func(ctx context.Context) ([]*targetgroup.Group, error) - Registry prometheus.Registerer - Metrics []prometheus.Collector + Logger log.Logger + Mech string + Interval time.Duration + RefreshF func(ctx context.Context) ([]*targetgroup.Group, error) + MetricsInstantiator discovery.RefreshMetricsInstantiator } // Discovery implements the Discoverer interface. @@ -40,15 +38,13 @@ type Discovery struct { logger log.Logger interval time.Duration refreshf func(ctx context.Context) ([]*targetgroup.Group, error) - - failures prometheus.Counter - duration prometheus.Summary - - metricRegisterer discovery.MetricRegisterer + metrics *discovery.RefreshMetrics } // NewDiscovery returns a Discoverer function that calls a refresh() function at every interval. func NewDiscovery(opts Options) *Discovery { + m := opts.MetricsInstantiator.Instantiate(opts.Mech) + var logger log.Logger if opts.Logger == nil { logger = log.NewNopLogger() @@ -60,44 +56,14 @@ func NewDiscovery(opts Options) *Discovery { logger: logger, interval: opts.Interval, refreshf: opts.RefreshF, - failures: prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "prometheus_sd_refresh_failures_total", - Help: "Number of refresh failures for the given SD mechanism.", - ConstLabels: prometheus.Labels{ - "mechanism": opts.Mech, - }, - }), - duration: prometheus.NewSummary( - prometheus.SummaryOpts{ - Name: "prometheus_sd_refresh_duration_seconds", - Help: "The duration of a refresh in seconds for the given SD mechanism.", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, - ConstLabels: prometheus.Labels{ - "mechanism": opts.Mech, - }, - }), + metrics: m, } - metrics := []prometheus.Collector{d.failures, d.duration} - if opts.Metrics != nil { - metrics = append(metrics, opts.Metrics...) - } - - d.metricRegisterer = discovery.NewMetricRegisterer(opts.Registry, metrics) - return &d } // Run implements the Discoverer interface. func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { - err := d.metricRegisterer.RegisterMetrics() - if err != nil { - level.Error(d.logger).Log("msg", "Unable to register metrics", "err", err.Error()) - return - } - defer d.metricRegisterer.UnregisterMetrics() - // Get an initial set right away. tgs, err := d.refresh(ctx) if err != nil { @@ -140,12 +106,12 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { now := time.Now() defer func() { - d.duration.Observe(time.Since(now).Seconds()) + d.metrics.Duration.Observe(time.Since(now).Seconds()) }() tgs, err := d.refreshf(ctx) if err != nil { - d.failures.Inc() + d.metrics.Failures.Inc() } return tgs, err } diff --git a/discovery/refresh/refresh_test.go b/discovery/refresh/refresh_test.go index 12e7ab3be..407f0a7fa 100644 --- a/discovery/refresh/refresh_test.go +++ b/discovery/refresh/refresh_test.go @@ -24,6 +24,7 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/goleak" + "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/targetgroup" ) @@ -66,13 +67,18 @@ func TestRefresh(t *testing.T) { return nil, fmt.Errorf("some error") } interval := time.Millisecond + + metrics := discovery.NewRefreshMetrics(prometheus.NewRegistry()) + require.NoError(t, metrics.Register()) + defer metrics.Unregister() + d := NewDiscovery( Options{ - Logger: nil, - Mech: "test", - Interval: interval, - RefreshF: refresh, - Registry: prometheus.NewRegistry(), + Logger: nil, + Mech: "test", + Interval: interval, + RefreshF: refresh, + MetricsInstantiator: metrics, }, ) diff --git a/discovery/registry.go b/discovery/registry.go index 13168a07a..1f491d4ca 100644 --- a/discovery/registry.go +++ b/discovery/registry.go @@ -24,6 +24,8 @@ import ( "gopkg.in/yaml.v2" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/discovery/targetgroup" ) @@ -258,3 +260,24 @@ func replaceYAMLTypeError(err error, oldTyp, newTyp reflect.Type) error { } return err } + +// RegisterSDMetrics registers the metrics used by service discovery mechanisms. +// RegisterSDMetrics should be called only once during the lifetime of the Prometheus process. +// There is no need for the Prometheus process to unregister the metrics. +func RegisterSDMetrics(registerer prometheus.Registerer, rmm RefreshMetricsManager) (map[string]DiscovererMetrics, error) { + err := rmm.Register() + if err != nil { + return nil, fmt.Errorf("failed to create service discovery refresh metrics") + } + + metrics := make(map[string]DiscovererMetrics) + for _, conf := range configNames { + currentSdMetrics := conf.NewDiscovererMetrics(registerer, rmm) + err = currentSdMetrics.Register() + if err != nil { + return nil, fmt.Errorf("failed to create service discovery metrics") + } + metrics[conf.Name()] = currentSdMetrics + } + return metrics, nil +} diff --git a/discovery/scaleway/metrics.go b/discovery/scaleway/metrics.go new file mode 100644 index 000000000..a8277d0fb --- /dev/null +++ b/discovery/scaleway/metrics.go @@ -0,0 +1,32 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scaleway + +import ( + "github.com/prometheus/prometheus/discovery" +) + +var _ discovery.DiscovererMetrics = (*scalewayMetrics)(nil) + +type scalewayMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator +} + +// Register implements discovery.DiscovererMetrics. +func (m *scalewayMetrics) Register() error { + return nil +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *scalewayMetrics) Unregister() {} diff --git a/discovery/scaleway/scaleway.go b/discovery/scaleway/scaleway.go index 86527b34e..f8e1a83f5 100644 --- a/discovery/scaleway/scaleway.go +++ b/discovery/scaleway/scaleway.go @@ -104,6 +104,13 @@ type SDConfig struct { Role role `yaml:"role"` } +// NewDiscovererMetrics implements discovery.Config. +func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &scalewayMetrics{ + refreshMetrics: rmi, + } +} + func (c SDConfig) Name() string { return "scaleway" } @@ -161,7 +168,7 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { } func (c SDConfig) NewDiscoverer(options discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(&c, options.Logger, options.Registerer) + return NewDiscovery(&c, options.Logger, options.Metrics) } // SetDirectory joins any relative file paths with dir. @@ -178,7 +185,12 @@ func init() { // the Discoverer interface. type Discovery struct{} -func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*refresh.Discovery, error) { +func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*refresh.Discovery, error) { + m, ok := metrics.(*scalewayMetrics) + if !ok { + return nil, fmt.Errorf("invalid discovery metrics type") + } + r, err := newRefresher(conf) if err != nil { return nil, err @@ -186,11 +198,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) return refresh.NewDiscovery( refresh.Options{ - Logger: logger, - Mech: "scaleway", - Interval: time.Duration(conf.RefreshInterval), - RefreshF: r.refresh, - Registry: reg, + Logger: logger, + Mech: "scaleway", + Interval: time.Duration(conf.RefreshInterval), + RefreshF: r.refresh, + MetricsInstantiator: m.refreshMetrics, }, ), nil } diff --git a/discovery/triton/metrics.go b/discovery/triton/metrics.go new file mode 100644 index 000000000..3e34a840a --- /dev/null +++ b/discovery/triton/metrics.go @@ -0,0 +1,32 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package triton + +import ( + "github.com/prometheus/prometheus/discovery" +) + +var _ discovery.DiscovererMetrics = (*tritonMetrics)(nil) + +type tritonMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator +} + +// Register implements discovery.DiscovererMetrics. +func (m *tritonMetrics) Register() error { + return nil +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *tritonMetrics) Unregister() {} diff --git a/discovery/triton/triton.go b/discovery/triton/triton.go index 4839827ad..e56b7951b 100644 --- a/discovery/triton/triton.go +++ b/discovery/triton/triton.go @@ -70,12 +70,19 @@ type SDConfig struct { Version int `yaml:"version"` } +// NewDiscovererMetrics implements discovery.Config. +func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &tritonMetrics{ + refreshMetrics: rmi, + } +} + // Name returns the name of the Config. func (*SDConfig) Name() string { return "triton" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return New(opts.Logger, c, opts.Registerer) + return New(opts.Logger, c, opts.Metrics) } // SetDirectory joins any relative file paths with dir. @@ -139,7 +146,12 @@ type Discovery struct { } // New returns a new Discovery which periodically refreshes its targets. -func New(logger log.Logger, conf *SDConfig, reg prometheus.Registerer) (*Discovery, error) { +func New(logger log.Logger, conf *SDConfig, metrics discovery.DiscovererMetrics) (*Discovery, error) { + m, ok := metrics.(*tritonMetrics) + if !ok { + return nil, fmt.Errorf("invalid discovery metrics type") + } + tls, err := config.NewTLSConfig(&conf.TLSConfig) if err != nil { return nil, err @@ -161,11 +173,11 @@ func New(logger log.Logger, conf *SDConfig, reg prometheus.Registerer) (*Discove } d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, - Mech: "triton", - Interval: time.Duration(conf.RefreshInterval), - RefreshF: d.refresh, - Registry: reg, + Logger: logger, + Mech: "triton", + Interval: time.Duration(conf.RefreshInterval), + RefreshF: d.refresh, + MetricsInstantiator: m.refreshMetrics, }, ) return d, nil diff --git a/discovery/triton/triton_test.go b/discovery/triton/triton_test.go index fa51a2e47..e37693e6b 100644 --- a/discovery/triton/triton_test.go +++ b/discovery/triton/triton_test.go @@ -28,6 +28,8 @@ import ( "github.com/prometheus/common/config" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" + + "github.com/prometheus/prometheus/discovery" ) var ( @@ -79,12 +81,26 @@ var ( } ) -func newTritonDiscovery(c SDConfig) (*Discovery, error) { - return New(nil, &c, prometheus.NewRegistry()) +func newTritonDiscovery(c SDConfig) (*Discovery, discovery.DiscovererMetrics, error) { + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + // TODO(ptodev): Add the ability to unregister refresh metrics. + metrics := c.NewDiscovererMetrics(reg, refreshMetrics) + err := metrics.Register() + if err != nil { + return nil, nil, err + } + + d, err := New(nil, &c, metrics) + if err != nil { + return nil, nil, err + } + + return d, metrics, nil } func TestTritonSDNew(t *testing.T) { - td, err := newTritonDiscovery(conf) + td, m, err := newTritonDiscovery(conf) require.NoError(t, err) require.NotNil(t, td) require.NotNil(t, td.client) @@ -94,16 +110,17 @@ func TestTritonSDNew(t *testing.T) { require.Equal(t, conf.DNSSuffix, td.sdConfig.DNSSuffix) require.Equal(t, conf.Endpoint, td.sdConfig.Endpoint) require.Equal(t, conf.Port, td.sdConfig.Port) + m.Unregister() } func TestTritonSDNewBadConfig(t *testing.T) { - td, err := newTritonDiscovery(badconf) + td, _, err := newTritonDiscovery(badconf) require.Error(t, err) require.Nil(t, td) } func TestTritonSDNewGroupsConfig(t *testing.T) { - td, err := newTritonDiscovery(groupsconf) + td, m, err := newTritonDiscovery(groupsconf) require.NoError(t, err) require.NotNil(t, td) require.NotNil(t, td.client) @@ -114,10 +131,11 @@ func TestTritonSDNewGroupsConfig(t *testing.T) { require.Equal(t, groupsconf.Endpoint, td.sdConfig.Endpoint) require.Equal(t, groupsconf.Groups, td.sdConfig.Groups) require.Equal(t, groupsconf.Port, td.sdConfig.Port) + m.Unregister() } func TestTritonSDNewCNConfig(t *testing.T) { - td, err := newTritonDiscovery(cnconf) + td, m, err := newTritonDiscovery(cnconf) require.NoError(t, err) require.NotNil(t, td) require.NotNil(t, td.client) @@ -128,6 +146,7 @@ func TestTritonSDNewCNConfig(t *testing.T) { require.Equal(t, cnconf.DNSSuffix, td.sdConfig.DNSSuffix) require.Equal(t, cnconf.Endpoint, td.sdConfig.Endpoint) require.Equal(t, cnconf.Port, td.sdConfig.Port) + m.Unregister() } func TestTritonSDRefreshNoTargets(t *testing.T) { @@ -160,21 +179,23 @@ func TestTritonSDRefreshMultipleTargets(t *testing.T) { } func TestTritonSDRefreshNoServer(t *testing.T) { - td, _ := newTritonDiscovery(conf) + td, m, _ := newTritonDiscovery(conf) _, err := td.refresh(context.Background()) require.Error(t, err) require.True(t, strings.Contains(err.Error(), "an error occurred when requesting targets from the discovery endpoint")) + m.Unregister() } func TestTritonSDRefreshCancelled(t *testing.T) { - td, _ := newTritonDiscovery(conf) + td, m, _ := newTritonDiscovery(conf) ctx, cancel := context.WithCancel(context.Background()) cancel() _, err := td.refresh(ctx) require.Error(t, err) require.True(t, strings.Contains(err.Error(), context.Canceled.Error())) + m.Unregister() } func TestTritonSDRefreshCNsUUIDOnly(t *testing.T) { @@ -211,8 +232,8 @@ func TestTritonSDRefreshCNsWithHostname(t *testing.T) { func testTritonSDRefresh(t *testing.T, c SDConfig, dstr string) []model.LabelSet { var ( - td, _ = newTritonDiscovery(c) - s = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + td, m, _ = newTritonDiscovery(c) + s = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, dstr) })) ) @@ -240,5 +261,7 @@ func testTritonSDRefresh(t *testing.T, c SDConfig, dstr string) []model.LabelSet tg := tgs[0] require.NotNil(t, tg) + m.Unregister() + return tg.Targets } diff --git a/discovery/uyuni/metrics.go b/discovery/uyuni/metrics.go new file mode 100644 index 000000000..6793b5920 --- /dev/null +++ b/discovery/uyuni/metrics.go @@ -0,0 +1,32 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package uyuni + +import ( + "github.com/prometheus/prometheus/discovery" +) + +var _ discovery.DiscovererMetrics = (*uyuniMetrics)(nil) + +type uyuniMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator +} + +// Register implements discovery.DiscovererMetrics. +func (m *uyuniMetrics) Register() error { + return nil +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *uyuniMetrics) Unregister() {} diff --git a/discovery/uyuni/uyuni.go b/discovery/uyuni/uyuni.go index 744f3f96c..e885ef2e8 100644 --- a/discovery/uyuni/uyuni.go +++ b/discovery/uyuni/uyuni.go @@ -111,12 +111,19 @@ type Discovery struct { logger log.Logger } +// NewDiscovererMetrics implements discovery.Config. +func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &uyuniMetrics{ + refreshMetrics: rmi, + } +} + // Name returns the name of the Config. func (*SDConfig) Name() string { return "uyuni" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger, opts.Registerer) + return NewDiscovery(c, opts.Logger, opts.Metrics) } // SetDirectory joins any relative file paths with dir. @@ -204,7 +211,12 @@ func getEndpointInfoForSystems( } // NewDiscovery returns a uyuni discovery for the given configuration. -func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) { +func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { + m, ok := metrics.(*uyuniMetrics) + if !ok { + return nil, fmt.Errorf("invalid discovery metrics type") + } + apiURL, err := url.Parse(conf.Server) if err != nil { return nil, err @@ -229,11 +241,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, - Mech: "uyuni", - Interval: time.Duration(conf.RefreshInterval), - RefreshF: d.refresh, - Registry: reg, + Logger: logger, + Mech: "uyuni", + Interval: time.Duration(conf.RefreshInterval), + RefreshF: d.refresh, + MetricsInstantiator: m.refreshMetrics, }, ) return d, nil diff --git a/discovery/uyuni/uyuni_test.go b/discovery/uyuni/uyuni_test.go index fd03c88f1..09be23e2b 100644 --- a/discovery/uyuni/uyuni_test.go +++ b/discovery/uyuni/uyuni_test.go @@ -25,6 +25,7 @@ import ( "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/targetgroup" ) @@ -37,7 +38,17 @@ func testUpdateServices(respHandler http.HandlerFunc) ([]*targetgroup.Group, err Server: ts.URL, } - md, err := NewDiscovery(&conf, nil, prometheus.NewRegistry()) + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := conf.NewDiscovererMetrics(reg, refreshMetrics) + err := metrics.Register() + if err != nil { + return nil, err + } + defer metrics.Unregister() + defer refreshMetrics.Unregister() + + md, err := NewDiscovery(&conf, nil, metrics) if err != nil { return nil, err } @@ -110,7 +121,14 @@ func TestUyuniSDSkipLogin(t *testing.T) { Server: ts.URL, } - md, err := NewDiscovery(&conf, nil, prometheus.NewRegistry()) + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := conf.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + defer metrics.Unregister() + defer refreshMetrics.Unregister() + + md, err := NewDiscovery(&conf, nil, metrics) if err != nil { t.Error(err) } diff --git a/discovery/vultr/metrics.go b/discovery/vultr/metrics.go new file mode 100644 index 000000000..193436aad --- /dev/null +++ b/discovery/vultr/metrics.go @@ -0,0 +1,32 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vultr + +import ( + "github.com/prometheus/prometheus/discovery" +) + +var _ discovery.DiscovererMetrics = (*vultrMetrics)(nil) + +type vultrMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator +} + +// Register implements discovery.DiscovererMetrics. +func (m *vultrMetrics) Register() error { + return nil +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *vultrMetrics) Unregister() {} diff --git a/discovery/vultr/vultr.go b/discovery/vultr/vultr.go index 129800048..aaa9c64e4 100644 --- a/discovery/vultr/vultr.go +++ b/discovery/vultr/vultr.go @@ -74,12 +74,19 @@ type SDConfig struct { Port int `yaml:"port"` } +// NewDiscovererMetrics implements discovery.Config. +func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &vultrMetrics{ + refreshMetrics: rmi, + } +} + // Name returns the name of the Config. func (*SDConfig) Name() string { return "vultr" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger, opts.Registerer) + return NewDiscovery(c, opts.Logger, opts.Metrics) } // SetDirectory joins any relative file paths with dir. @@ -107,7 +114,12 @@ type Discovery struct { } // NewDiscovery returns a new Discovery which periodically refreshes its targets. -func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) { +func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { + m, ok := metrics.(*vultrMetrics) + if !ok { + return nil, fmt.Errorf("invalid discovery metrics type") + } + d := &Discovery{ port: conf.Port, } @@ -130,11 +142,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, - Mech: "vultr", - Interval: time.Duration(conf.RefreshInterval), - RefreshF: d.refresh, - Registry: reg, + Logger: logger, + Mech: "vultr", + Interval: time.Duration(conf.RefreshInterval), + RefreshF: d.refresh, + MetricsInstantiator: m.refreshMetrics, }, ) return d, nil diff --git a/discovery/vultr/vultr_test.go b/discovery/vultr/vultr_test.go index c50b11d2d..2f12a3552 100644 --- a/discovery/vultr/vultr_test.go +++ b/discovery/vultr/vultr_test.go @@ -23,6 +23,8 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" + + "github.com/prometheus/prometheus/discovery" ) type VultrSDTestSuite struct { @@ -47,7 +49,15 @@ func TestVultrSDRefresh(t *testing.T) { cfg := DefaultSDConfig cfg.HTTPClientConfig.BearerToken = APIKey - d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry()) + + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + defer metrics.Unregister() + defer refreshMetrics.Unregister() + + d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics) require.NoError(t, err) endpoint, err := url.Parse(sdMock.Mock.Endpoint()) require.NoError(t, err) diff --git a/discovery/xds/kuma.go b/discovery/xds/kuma.go index c74bc552c..d1d540aaf 100644 --- a/discovery/xds/kuma.go +++ b/discovery/xds/kuma.go @@ -58,6 +58,11 @@ const ( type KumaSDConfig = SDConfig +// NewDiscovererMetrics implements discovery.Config. +func (*KumaSDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return newDiscovererMetrics(reg, rmi) +} + // UnmarshalYAML implements the yaml.Unmarshaler interface. func (c *KumaSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { *c = DefaultKumaSDConfig @@ -97,7 +102,7 @@ func (c *KumaSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discover logger = log.NewNopLogger() } - return NewKumaHTTPDiscovery(c, logger, opts.Registerer) + return NewKumaHTTPDiscovery(c, logger, opts.Metrics) } func convertKumaV1MonitoringAssignment(assignment *MonitoringAssignment) []model.LabelSet { @@ -153,7 +158,12 @@ func kumaMadsV1ResourceParser(resources []*anypb.Any, typeURL string) ([]model.L return targets, nil } -func NewKumaHTTPDiscovery(conf *KumaSDConfig, logger log.Logger, reg prometheus.Registerer) (discovery.Discoverer, error) { +func NewKumaHTTPDiscovery(conf *KumaSDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (discovery.Discoverer, error) { + m, ok := metrics.(*xdsMetrics) + if !ok { + return nil, fmt.Errorf("invalid discovery metrics type") + } + // Default to "prometheus" if hostname is unavailable. clientID := conf.ClientID if clientID == "" { @@ -189,36 +199,8 @@ func NewKumaHTTPDiscovery(conf *KumaSDConfig, logger log.Logger, reg prometheus. refreshInterval: time.Duration(conf.RefreshInterval), source: "kuma", parseResources: kumaMadsV1ResourceParser, - fetchFailuresCount: prometheus.NewCounter( - prometheus.CounterOpts{ - Namespace: namespace, - Name: "sd_kuma_fetch_failures_total", - Help: "The number of Kuma MADS fetch call failures.", - }), - fetchSkipUpdateCount: prometheus.NewCounter( - prometheus.CounterOpts{ - Namespace: namespace, - Name: "sd_kuma_fetch_skipped_updates_total", - Help: "The number of Kuma MADS fetch calls that result in no updates to the targets.", - }), - fetchDuration: prometheus.NewSummary( - prometheus.SummaryOpts{ - Namespace: namespace, - Name: "sd_kuma_fetch_duration_seconds", - Help: "The duration of a Kuma MADS fetch call.", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, - }, - ), + metrics: m, } - d.metricRegisterer = discovery.NewMetricRegisterer( - reg, - []prometheus.Collector{ - d.fetchFailuresCount, - d.fetchSkipUpdateCount, - d.fetchDuration, - }, - ) - return d, nil } diff --git a/discovery/xds/kuma_test.go b/discovery/xds/kuma_test.go index 5e2417c96..cfb9cbac5 100644 --- a/discovery/xds/kuma_test.go +++ b/discovery/xds/kuma_test.go @@ -28,6 +28,7 @@ import ( "google.golang.org/protobuf/types/known/anypb" "gopkg.in/yaml.v2" + "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/targetgroup" ) @@ -108,7 +109,16 @@ func getKumaMadsV1DiscoveryResponse(resources ...*MonitoringAssignment) (*v3.Dis } func newKumaTestHTTPDiscovery(c KumaSDConfig) (*fetchDiscovery, error) { - kd, err := NewKumaHTTPDiscovery(&c, nopLogger, prometheus.NewRegistry()) + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + // TODO(ptodev): Add the ability to unregister refresh metrics. + metrics := c.NewDiscovererMetrics(reg, refreshMetrics) + err := metrics.Register() + if err != nil { + return nil, err + } + + kd, err := NewKumaHTTPDiscovery(&c, nopLogger, metrics) if err != nil { return nil, err } @@ -207,6 +217,8 @@ func TestNewKumaHTTPDiscovery(t *testing.T) { require.Equal(t, KumaMadsV1ResourceTypeURL, resClient.ResourceTypeURL()) require.Equal(t, kumaConf.ClientID, resClient.ID()) require.Equal(t, KumaMadsV1ResourceType, resClient.config.ResourceType) + + kd.metrics.Unregister() } func TestKumaHTTPDiscoveryRefresh(t *testing.T) { @@ -301,4 +313,6 @@ tls_config: case <-ch: require.Fail(t, "no update expected") } + + kd.metrics.Unregister() } diff --git a/discovery/xds/metrics.go b/discovery/xds/metrics.go new file mode 100644 index 000000000..597d51656 --- /dev/null +++ b/discovery/xds/metrics.go @@ -0,0 +1,73 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package xds + +import ( + "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/prometheus/discovery" +) + +var _ discovery.DiscovererMetrics = (*xdsMetrics)(nil) + +type xdsMetrics struct { + fetchDuration prometheus.Summary + fetchSkipUpdateCount prometheus.Counter + fetchFailuresCount prometheus.Counter + + metricRegisterer discovery.MetricRegisterer +} + +func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + m := &xdsMetrics{ + fetchFailuresCount: prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "sd_kuma_fetch_failures_total", + Help: "The number of Kuma MADS fetch call failures.", + }), + fetchSkipUpdateCount: prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "sd_kuma_fetch_skipped_updates_total", + Help: "The number of Kuma MADS fetch calls that result in no updates to the targets.", + }), + fetchDuration: prometheus.NewSummary( + prometheus.SummaryOpts{ + Namespace: namespace, + Name: "sd_kuma_fetch_duration_seconds", + Help: "The duration of a Kuma MADS fetch call.", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, + ), + } + + m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{ + m.fetchFailuresCount, + m.fetchSkipUpdateCount, + m.fetchDuration, + }) + + return m +} + +// Register implements discovery.DiscovererMetrics. +func (m *xdsMetrics) Register() error { + return m.metricRegisterer.RegisterMetrics() +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *xdsMetrics) Unregister() { + m.metricRegisterer.UnregisterMetrics() +} diff --git a/discovery/xds/xds.go b/discovery/xds/xds.go index 8b6cb7e65..8191d6be1 100644 --- a/discovery/xds/xds.go +++ b/discovery/xds/xds.go @@ -20,7 +20,6 @@ import ( v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" "github.com/go-kit/log" "github.com/go-kit/log/level" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" "github.com/prometheus/common/model" "google.golang.org/protobuf/encoding/protojson" @@ -107,20 +106,10 @@ type fetchDiscovery struct { parseResources resourceParser logger log.Logger - fetchDuration prometheus.Summary - fetchSkipUpdateCount prometheus.Counter - fetchFailuresCount prometheus.Counter - - metricRegisterer discovery.MetricRegisterer + metrics *xdsMetrics } func (d *fetchDiscovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { - err := d.metricRegisterer.RegisterMetrics() - if err != nil { - level.Error(d.logger).Log("msg", "Unable to register metrics", "err", err.Error()) - return - } - defer d.metricRegisterer.UnregisterMetrics() defer d.client.Close() ticker := time.NewTicker(d.refreshInterval) @@ -141,7 +130,7 @@ func (d *fetchDiscovery) poll(ctx context.Context, ch chan<- []*targetgroup.Grou t0 := time.Now() response, err := d.client.Fetch(ctx) elapsed := time.Since(t0) - d.fetchDuration.Observe(elapsed.Seconds()) + d.metrics.fetchDuration.Observe(elapsed.Seconds()) // Check the context before in order to exit early. select { @@ -152,20 +141,20 @@ func (d *fetchDiscovery) poll(ctx context.Context, ch chan<- []*targetgroup.Grou if err != nil { level.Error(d.logger).Log("msg", "error parsing resources", "err", err) - d.fetchFailuresCount.Inc() + d.metrics.fetchFailuresCount.Inc() return } if response == nil { // No update needed. - d.fetchSkipUpdateCount.Inc() + d.metrics.fetchSkipUpdateCount.Inc() return } parsedTargets, err := d.parseResources(response.Resources, response.TypeUrl) if err != nil { level.Error(d.logger).Log("msg", "error parsing resources", "err", err) - d.fetchFailuresCount.Inc() + d.metrics.fetchFailuresCount.Inc() return } diff --git a/discovery/xds/xds_test.go b/discovery/xds/xds_test.go index f57fff996..7cce021c5 100644 --- a/discovery/xds/xds_test.go +++ b/discovery/xds/xds_test.go @@ -29,24 +29,15 @@ import ( "go.uber.org/goleak" "google.golang.org/protobuf/types/known/anypb" + "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/targetgroup" ) -var ( - sdConf = SDConfig{ - Server: "http://127.0.0.1", - RefreshInterval: model.Duration(10 * time.Second), - ClientID: "test-id", - } - - testFetchFailuresCount = prometheus.NewCounter( - prometheus.CounterOpts{}) - testFetchSkipUpdateCount = prometheus.NewCounter( - prometheus.CounterOpts{}) - testFetchDuration = prometheus.NewSummary( - prometheus.SummaryOpts{}, - ) -) +var sdConf = SDConfig{ + Server: "http://127.0.0.1", + RefreshInterval: model.Duration(10 * time.Second), + ClientID: "test-id", +} func TestMain(m *testing.M) { goleak.VerifyTestMain(m) @@ -133,12 +124,22 @@ func TestPollingRefreshSkipUpdate(t *testing.T) { return nil, nil }, } + + cfg := &SDConfig{} + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + + xdsMetrics, ok := metrics.(*xdsMetrics) + if !ok { + require.Fail(t, "invalid discovery metrics type") + } + pd := &fetchDiscovery{ - client: rc, - logger: nopLogger, - fetchDuration: testFetchDuration, - fetchFailuresCount: testFetchFailuresCount, - fetchSkipUpdateCount: testFetchSkipUpdateCount, + client: rc, + logger: nopLogger, + metrics: xdsMetrics, } ctx, cancel := context.WithCancel(context.Background()) @@ -155,6 +156,9 @@ func TestPollingRefreshSkipUpdate(t *testing.T) { case <-ch: require.Fail(t, "no update expected") } + + metrics.Unregister() + refreshMetrics.Unregister() } func TestPollingRefreshAttachesGroupMetadata(t *testing.T) { @@ -167,13 +171,18 @@ func TestPollingRefreshAttachesGroupMetadata(t *testing.T) { return &v3.DiscoveryResponse{}, nil }, } + + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := newDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + xdsMetrics, ok := metrics.(*xdsMetrics) + require.True(t, ok) + pd := &fetchDiscovery{ - source: source, - client: rc, - logger: nopLogger, - fetchDuration: testFetchDuration, - fetchFailuresCount: testFetchFailuresCount, - fetchSkipUpdateCount: testFetchSkipUpdateCount, + source: source, + client: rc, + logger: nopLogger, parseResources: constantResourceParser([]model.LabelSet{ { "__meta_custom_xds_label": "a-value", @@ -186,6 +195,7 @@ func TestPollingRefreshAttachesGroupMetadata(t *testing.T) { "instance": "prometheus-02", }, }, nil), + metrics: xdsMetrics, } ch := make(chan []*targetgroup.Group, 1) pd.poll(context.Background(), ch) @@ -202,6 +212,9 @@ func TestPollingRefreshAttachesGroupMetadata(t *testing.T) { target2 := group.Targets[1] require.Contains(t, target2, model.LabelName("__meta_custom_xds_label")) require.Equal(t, model.LabelValue("a-value"), target2["__meta_custom_xds_label"]) + + metrics.Unregister() + refreshMetrics.Unregister() } func TestPollingDisappearingTargets(t *testing.T) { @@ -243,14 +256,23 @@ func TestPollingDisappearingTargets(t *testing.T) { }, nil } + cfg := &SDConfig{} + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + + xdsMetrics, ok := metrics.(*xdsMetrics) + if !ok { + require.Fail(t, "invalid discovery metrics type") + } + pd := &fetchDiscovery{ - source: source, - client: rc, - logger: nopLogger, - fetchDuration: testFetchDuration, - fetchFailuresCount: testFetchFailuresCount, - fetchSkipUpdateCount: testFetchSkipUpdateCount, - parseResources: parser, + source: source, + client: rc, + logger: nopLogger, + parseResources: parser, + metrics: xdsMetrics, } ch := make(chan []*targetgroup.Group, 1) @@ -271,4 +293,7 @@ func TestPollingDisappearingTargets(t *testing.T) { require.Equal(t, source, groups[0].Source) require.Len(t, groups[0].Targets, 1) + + metrics.Unregister() + refreshMetrics.Unregister() } diff --git a/discovery/zookeeper/zookeeper.go b/discovery/zookeeper/zookeeper.go index 308d63a5f..7d8761527 100644 --- a/discovery/zookeeper/zookeeper.go +++ b/discovery/zookeeper/zookeeper.go @@ -25,6 +25,7 @@ import ( "github.com/go-kit/log" "github.com/go-zookeeper/zk" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/discovery" @@ -56,6 +57,11 @@ type ServersetSDConfig struct { Timeout model.Duration `yaml:"timeout,omitempty"` } +// NewDiscovererMetrics implements discovery.Config. +func (*ServersetSDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &discovery.NoopDiscovererMetrics{} +} + // Name returns the name of the Config. func (*ServersetSDConfig) Name() string { return "serverset" } @@ -93,6 +99,11 @@ type NerveSDConfig struct { Timeout model.Duration `yaml:"timeout,omitempty"` } +// NewDiscovererMetrics implements discovery.Config. +func (*NerveSDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &discovery.NoopDiscovererMetrics{} +} + // Name returns the name of the Config. func (*NerveSDConfig) Name() string { return "nerve" } diff --git a/documentation/examples/custom-sd/adapter-usage/main.go b/documentation/examples/custom-sd/adapter-usage/main.go index f4bba7394..bfbca7b70 100644 --- a/documentation/examples/custom-sd/adapter-usage/main.go +++ b/documentation/examples/custom-sd/adapter-usage/main.go @@ -28,8 +28,10 @@ import ( "github.com/alecthomas/kingpin/v2" "github.com/go-kit/log" "github.com/go-kit/log/level" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + prom_discovery "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/documentation/examples/custom-sd/adapter" "github.com/prometheus/prometheus/util/strutil" @@ -273,7 +275,16 @@ func main() { level.Error(logger).Log("msg", "failed to create discovery metrics", "err", err) os.Exit(1) } - sdAdapter := adapter.NewAdapter(ctx, *outputFile, "exampleSD", disc, logger) + + reg := prometheus.NewRegistry() + refreshMetrics := prom_discovery.NewRefreshMetrics(reg) + metrics, err := prom_discovery.RegisterSDMetrics(reg, refreshMetrics) + if err != nil { + level.Error(logger).Log("msg", "failed to register service discovery metrics", "err", err) + os.Exit(1) + } + + sdAdapter := adapter.NewAdapter(ctx, *outputFile, "exampleSD", disc, logger, metrics, reg) sdAdapter.Run() <-ctx.Done() diff --git a/documentation/examples/custom-sd/adapter/adapter.go b/documentation/examples/custom-sd/adapter/adapter.go index 7fbf94aa9..dcf5a2b78 100644 --- a/documentation/examples/custom-sd/adapter/adapter.go +++ b/documentation/examples/custom-sd/adapter/adapter.go @@ -163,12 +163,12 @@ func (a *Adapter) Run() { } // NewAdapter creates a new instance of Adapter. -func NewAdapter(ctx context.Context, file, name string, d discovery.Discoverer, logger log.Logger) *Adapter { +func NewAdapter(ctx context.Context, file, name string, d discovery.Discoverer, logger log.Logger, sdMetrics map[string]discovery.DiscovererMetrics, registerer prometheus.Registerer) *Adapter { return &Adapter{ ctx: ctx, disc: d, groups: make(map[string]*customSD), - manager: discovery.NewManager(ctx, logger, prometheus.NewRegistry()), + manager: discovery.NewManager(ctx, logger, registerer, sdMetrics), output: file, name: name, logger: logger, diff --git a/documentation/examples/custom-sd/adapter/adapter_test.go b/documentation/examples/custom-sd/adapter/adapter_test.go index 8e5920eb4..329ca8c29 100644 --- a/documentation/examples/custom-sd/adapter/adapter_test.go +++ b/documentation/examples/custom-sd/adapter/adapter_test.go @@ -18,9 +18,11 @@ import ( "os" "testing" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" + "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/targetgroup" ) @@ -227,6 +229,12 @@ func TestWriteOutput(t *testing.T) { defer os.Remove(tmpfile.Name()) tmpfile.Close() require.NoError(t, err) - adapter := NewAdapter(ctx, tmpfile.Name(), "test_sd", nil, nil) + + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + sdMetrics, err := discovery.RegisterSDMetrics(reg, refreshMetrics) + require.NoError(t, err) + + adapter := NewAdapter(ctx, tmpfile.Name(), "test_sd", nil, nil, sdMetrics, reg) require.NoError(t, adapter.writeOutput()) }